diff --git a/kaldb/src/main/java/com/slack/kaldb/logstore/LuceneIndexStoreImpl.java b/kaldb/src/main/java/com/slack/kaldb/logstore/LuceneIndexStoreImpl.java index 06c08813e9..6169c06d26 100644 --- a/kaldb/src/main/java/com/slack/kaldb/logstore/LuceneIndexStoreImpl.java +++ b/kaldb/src/main/java/com/slack/kaldb/logstore/LuceneIndexStoreImpl.java @@ -62,6 +62,8 @@ public class LuceneIndexStoreImpl implements LogStore { private final io.micrometer.core.instrument.Timer commitsTimer; private final io.micrometer.core.instrument.Timer refreshesTimer; + private final String MAX_RAM_BUFFER_SIZE_MB = "maxRamBufferSizeMb"; + // TODO: Set the policy via a lucene config file. public static LuceneIndexStoreImpl makeLogStore( File dataDirectory, KaldbConfigs.LuceneConfig luceneConfig, MeterRegistry metricsRegistry) @@ -148,10 +150,14 @@ private IndexWriterConfig buildIndexWriterConfig( SnapshotDeletionPolicy snapshotDeletionPolicy, LuceneIndexStoreConfig config, MeterRegistry metricsRegistry) { + int ramBufferSizeMb = Integer.getInteger(MAX_RAM_BUFFER_SIZE_MB, 1024); + boolean useCFSFiles = ramBufferSizeMb <= 128; final IndexWriterConfig indexWriterCfg = new IndexWriterConfig(analyzer) .setOpenMode(IndexWriterConfig.OpenMode.CREATE) .setMergeScheduler(new KalDBMergeScheduler(metricsRegistry)) + .setRAMBufferSizeMB(ramBufferSizeMb) + .setUseCompoundFile(useCFSFiles) // we sort by timestamp descending, as that is the order we expect to return results the // majority of the time .setIndexSort( @@ -162,6 +168,12 @@ private IndexWriterConfig buildIndexWriterConfig( true))) .setIndexDeletionPolicy(snapshotDeletionPolicy); + // See + // https://lucene.apache.org/core/9_5_0/core/org/apache/lucene/index/IndexWriterConfig.html#setUseCompoundFile(boolean) + if (ramBufferSizeMb >= 128) { + indexWriterCfg.getMergePolicy().setNoCFSRatio(0.0); + } + if (config.enableTracing) { indexWriterCfg.setInfoStream(System.out); }