Skip to content

Commit

Permalink
813 -> 814
Browse files Browse the repository at this point in the history
  • Loading branch information
jpountz committed Mar 14, 2024
1 parent c1bcebf commit cce0fe3
Show file tree
Hide file tree
Showing 10 changed files with 36 additions and 35 deletions.
3 changes: 2 additions & 1 deletion server/src/main/java/module-info.java
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
* Side Public License, v 1.
*/

import org.elasticsearch.index.codec.Elasticsearch814Codec;
import org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat;
import org.elasticsearch.plugins.internal.RestExtension;

Expand Down Expand Up @@ -434,7 +435,7 @@
with
org.elasticsearch.index.codec.vectors.ES813FlatVectorFormat,
org.elasticsearch.index.codec.vectors.ES813Int8FlatVectorFormat;
provides org.apache.lucene.codecs.Codec with org.elasticsearch.index.codec.Elasticsearch813Codec;
provides org.apache.lucene.codecs.Codec with Elasticsearch814Codec;

exports org.elasticsearch.cluster.routing.allocation.shards
to
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.common.util.FeatureFlag;
import org.elasticsearch.core.Nullable;
import org.elasticsearch.index.codec.zstd.Zstd813StoredFieldsFormat;
import org.elasticsearch.index.codec.zstd.Zstd814StoredFieldsFormat;
import org.elasticsearch.index.mapper.MapperService;

import java.util.HashMap;
Expand Down Expand Up @@ -44,7 +44,7 @@ public CodecService(@Nullable MapperService mapperService, BigArrays bigArrays)

Codec legacyBestSpeedCodec = new LegacyPerFieldMapperCodec(Lucene99Codec.Mode.BEST_SPEED, mapperService, bigArrays);
if (ZSTD_STORED_FIELDS_FEATURE_FLAG.isEnabled()) {
codecs.put(DEFAULT_CODEC, new PerFieldMapperCodec(Zstd813StoredFieldsFormat.Mode.BEST_SPEED, mapperService, bigArrays));
codecs.put(DEFAULT_CODEC, new PerFieldMapperCodec(Zstd814StoredFieldsFormat.Mode.BEST_SPEED, mapperService, bigArrays));
} else {
codecs.put(DEFAULT_CODEC, legacyBestSpeedCodec);
}
Expand All @@ -54,7 +54,7 @@ public CodecService(@Nullable MapperService mapperService, BigArrays bigArrays)
if (ZSTD_STORED_FIELDS_FEATURE_FLAG.isEnabled()) {
codecs.put(
BEST_COMPRESSION_CODEC,
new PerFieldMapperCodec(Zstd813StoredFieldsFormat.Mode.BEST_COMPRESSION, mapperService, bigArrays)
new PerFieldMapperCodec(Zstd814StoredFieldsFormat.Mode.BEST_COMPRESSION, mapperService, bigArrays)
);
} else {
codecs.put(BEST_COMPRESSION_CODEC, legacyBestCompressionCodec);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,51 +20,51 @@
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
import org.elasticsearch.index.codec.zstd.Zstd813StoredFieldsFormat;
import org.elasticsearch.index.codec.zstd.Zstd814StoredFieldsFormat;

/**
* Elasticsearch codec as of 8.13. This extends the Lucene 9.9 codec to compressed stored fields with ZSTD instead of LZ4/DEFLATE. See
* {@link Zstd813StoredFieldsFormat}.
* {@link Zstd814StoredFieldsFormat}.
*/
public class Elasticsearch813Codec extends FilterCodec {
public class Elasticsearch814Codec extends FilterCodec {

private final StoredFieldsFormat storedFieldsFormat;

private final PostingsFormat defaultPostingsFormat;
private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
return Elasticsearch813Codec.this.getPostingsFormatForField(field);
return Elasticsearch814Codec.this.getPostingsFormatForField(field);
}
};

private final DocValuesFormat defaultDVFormat;
private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() {
@Override
public DocValuesFormat getDocValuesFormatForField(String field) {
return Elasticsearch813Codec.this.getDocValuesFormatForField(field);
return Elasticsearch814Codec.this.getDocValuesFormatForField(field);
}
};

private final KnnVectorsFormat defaultKnnVectorsFormat;
private final KnnVectorsFormat knnVectorsFormat = new PerFieldKnnVectorsFormat() {
@Override
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
return Elasticsearch813Codec.this.getKnnVectorsFormatForField(field);
return Elasticsearch814Codec.this.getKnnVectorsFormatForField(field);
}
};

/** Public no-arg constructor, needed for SPI loading at read-time. */
public Elasticsearch813Codec() {
this(Zstd813StoredFieldsFormat.Mode.BEST_SPEED);
public Elasticsearch814Codec() {
this(Zstd814StoredFieldsFormat.Mode.BEST_SPEED);
}

/**
* Constructor. Takes a {@link Zstd813StoredFieldsFormat.Mode} that describes whether to optimize for retrieval speed at the expense of worse space-efficiency or vice-versa.
* Constructor. Takes a {@link Zstd814StoredFieldsFormat.Mode} that describes whether to optimize for retrieval speed at the expense of worse space-efficiency or vice-versa.
*/
public Elasticsearch813Codec(Zstd813StoredFieldsFormat.Mode mode) {
super("Elasticsearch813", new Lucene99Codec());
this.storedFieldsFormat = new Zstd813StoredFieldsFormat(mode);
public Elasticsearch814Codec(Zstd814StoredFieldsFormat.Mode mode) {
super("Elasticsearch814", new Lucene99Codec());
this.storedFieldsFormat = new Zstd814StoredFieldsFormat(mode);
this.defaultPostingsFormat = new Lucene99PostingsFormat();
this.defaultDVFormat = new Lucene90DocValuesFormat();
this.defaultKnnVectorsFormat = new Lucene99HnswVectorsFormat();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
import org.apache.lucene.codecs.PostingsFormat;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.index.codec.zstd.Zstd813StoredFieldsFormat;
import org.elasticsearch.index.codec.zstd.Zstd814StoredFieldsFormat;
import org.elasticsearch.index.mapper.MapperService;

/**
Expand All @@ -25,11 +25,11 @@
* per index in real time via the mapping API. If no specific postings format or vector format is
* configured for a specific field the default postings or vector format is used.
*/
public final class PerFieldMapperCodec extends Elasticsearch813Codec {
public final class PerFieldMapperCodec extends Elasticsearch814Codec {

private final PerFieldFormatSupplier formatSupplier;

public PerFieldMapperCodec(Zstd813StoredFieldsFormat.Mode compressionMode, MapperService mapperService, BigArrays bigArrays) {
public PerFieldMapperCodec(Zstd814StoredFieldsFormat.Mode compressionMode, MapperService mapperService, BigArrays bigArrays) {
super(compressionMode);
this.formatSupplier = new PerFieldFormatSupplier(mapperService, bigArrays);
// If the below assertion fails, it is a sign that Lucene released a new codec. You must create a copy of the current Elasticsearch
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
* (64kB and 32kB respectively). In contrast, ZSTD doesn't have such a limitation and can better take advantage of large compression
* buffers.
*/
public final class Zstd813StoredFieldsFormat extends Lucene90CompressingStoredFieldsFormat {
public final class Zstd814StoredFieldsFormat extends Lucene90CompressingStoredFieldsFormat {

// ZSTD has special optimizations for inputs that are less than 16kB and less than 256kB. So subtract a bit of memory from 16kB and
// 256kB to make our inputs unlikely to grow beyond 16kB for BEST_SPEED and 256kB for BEST_COMPRESSION.
Expand All @@ -52,12 +52,12 @@ private Mode(int level, int blockSizeInBytes, int blockDocCount) {
}
}

public Zstd813StoredFieldsFormat(Mode mode) {
public Zstd814StoredFieldsFormat(Mode mode) {
this(mode.level, mode.blockSizeInBytes, mode.blockDocCount);
}

Zstd813StoredFieldsFormat(int level, int blockSizeInBytes, int blockDocCount) {
super("ZstdStoredFields813", new ZstdCompressionMode(level), blockSizeInBytes, blockDocCount, 10);
Zstd814StoredFieldsFormat(int level, int blockSizeInBytes, int blockDocCount) {
super("ZstdStoredFields814", new ZstdCompressionMode(level), blockSizeInBytes, blockDocCount, 10);
}

private static class ZstdCompressionMode extends CompressionMode {
Expand Down
Original file line number Diff line number Diff line change
@@ -1 +1 @@
org.elasticsearch.index.codec.Elasticsearch813Codec
org.elasticsearch.index.codec.Elasticsearch814Codec
Original file line number Diff line number Diff line change
Expand Up @@ -45,21 +45,21 @@ public class CodecTests extends ESTestCase {
public void testResolveDefaultCodecs() throws Exception {
CodecService codecService = createCodecService();
assertThat(codecService.codec("default"), instanceOf(PerFieldMapperCodec.class));
assertThat(codecService.codec("default"), instanceOf(Elasticsearch813Codec.class));
assertThat(codecService.codec("default"), instanceOf(Elasticsearch814Codec.class));
}

public void testDefault() throws Exception {
Codec codec = createCodecService().codec("default");
assertEquals(
"Zstd813StoredFieldsFormat(compressionMode=ZSTD(level=0), chunkSize=14336, maxDocsPerChunk=128, blockShift=10)",
"Zstd814StoredFieldsFormat(compressionMode=ZSTD(level=0), chunkSize=14336, maxDocsPerChunk=128, blockShift=10)",
codec.storedFieldsFormat().toString()
);
}

public void testBestCompression() throws Exception {
Codec codec = createCodecService().codec("best_compression");
assertEquals(
"Zstd813StoredFieldsFormat(compressionMode=ZSTD(level=3), chunkSize=245760, maxDocsPerChunk=2048, blockShift=10)",
"Zstd814StoredFieldsFormat(compressionMode=ZSTD(level=3), chunkSize=245760, maxDocsPerChunk=2048, blockShift=10)",
codec.storedFieldsFormat().toString()
);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@

import org.apache.lucene.codecs.Codec;
import org.apache.lucene.tests.index.BaseStoredFieldsFormatTestCase;
import org.elasticsearch.index.codec.Elasticsearch813Codec;
import org.elasticsearch.index.codec.Elasticsearch814Codec;

public class Zstd813BestCompressionStoredFieldsFormatTests extends BaseStoredFieldsFormatTestCase {
public class Zstd814BestCompressionStoredFieldsFormatTests extends BaseStoredFieldsFormatTestCase {

private final Codec codec = new Elasticsearch813Codec(Zstd813StoredFieldsFormat.Mode.BEST_COMPRESSION);
private final Codec codec = new Elasticsearch814Codec(Zstd814StoredFieldsFormat.Mode.BEST_COMPRESSION);

@Override
protected Codec getCodec() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@

import org.apache.lucene.codecs.Codec;
import org.apache.lucene.tests.index.BaseStoredFieldsFormatTestCase;
import org.elasticsearch.index.codec.Elasticsearch813Codec;
import org.elasticsearch.index.codec.Elasticsearch814Codec;

public class Zstd813BestSpeedStoredFieldsFormatTests extends BaseStoredFieldsFormatTestCase {
public class Zstd814BestSpeedStoredFieldsFormatTests extends BaseStoredFieldsFormatTestCase {

private final Codec codec = new Elasticsearch813Codec(Zstd813StoredFieldsFormat.Mode.BEST_SPEED);
private final Codec codec = new Elasticsearch814Codec(Zstd814StoredFieldsFormat.Mode.BEST_SPEED);

@Override
protected Codec getCodec() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
import org.elasticsearch.index.codec.PerFieldMapperCodec;
import org.elasticsearch.index.codec.zstd.Zstd813StoredFieldsFormat;
import org.elasticsearch.index.codec.zstd.Zstd814StoredFieldsFormat;
import org.elasticsearch.index.fielddata.FieldDataContext;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
Expand Down Expand Up @@ -243,7 +243,7 @@ protected static void withLuceneIndex(
CheckedConsumer<DirectoryReader, IOException> test
) throws IOException {
IndexWriterConfig iwc = new IndexWriterConfig(IndexShard.buildIndexAnalyzer(mapperService)).setCodec(
new PerFieldMapperCodec(Zstd813StoredFieldsFormat.Mode.BEST_SPEED, mapperService, BigArrays.NON_RECYCLING_INSTANCE)
new PerFieldMapperCodec(Zstd814StoredFieldsFormat.Mode.BEST_SPEED, mapperService, BigArrays.NON_RECYCLING_INSTANCE)
);
try (Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc)) {
builder.accept(iw);
Expand Down

0 comments on commit cce0fe3

Please sign in to comment.