Skip to content
This repository has been archived by the owner on Aug 2, 2022. It is now read-only.

Commit

Permalink
Opendistro Release 1.9.0 (#532)
Browse files Browse the repository at this point in the history 
* prepare odfe 1.9

* Fix all ES 7.8 compile and build errors

* Revert changes as Lombok is working now

* Update CustomExternalTestCluster.java

* Fix license headers check

* Use splitFieldsByMetadata to separate fields when calling SearchHit constructor

* More fixes for ODFE 1.9

* Remove todo statement

* Add ODFE 1.9.0 release notes
  • Loading branch information
@joshuali925
joshuali925 authored Jun 24, 2020
1 parent fb2ed91 commit 254f2e0
Show file tree
Hide file tree
Showing 30 changed files with 192 additions and 128 deletions.
12 changes: 9 additions & 3 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

buildscript {
ext {
es_version = "7.7.0"
es_version = "7.8.0"
}
// This isn't applying from repositories.gradle so repeating it here
repositories {
Expand All @@ -40,7 +40,7 @@ repositories {
}

ext {
opendistroVersion = '1.8.0'
opendistroVersion = '1.9.0'
isSnapshot = "true" == System.getProperty("build.snapshot", "true")
}

Expand Down Expand Up @@ -97,6 +97,10 @@ licenseHeaders {
excludes = ['com/amazon/opendistroforelasticsearch/sql/antlr/parser/**']
}

tasks.withType(licenseHeaders.class) {
additionalLicense 'AL ', 'Apache', 'Licensed under the Apache License, Version 2.0 (the "License")'
}

// TODO: need to fix java doc to enable JavaDoc
javadoc.enabled = false
esplugin {
Expand Down Expand Up @@ -272,14 +276,16 @@ dependencies {
testCompile group: "org.elasticsearch.client", name: 'transport', version: "${es_version}"

// JDBC drivers for comparison test. Somehow Apache Derby throws security permission exception.
testCompile group: 'com.amazon.opendistroforelasticsearch.client', name: 'opendistro-sql-jdbc', version: '1.3.0.0'
testCompile group: 'com.amazon.opendistroforelasticsearch.client', name: 'opendistro-sql-jdbc', version: '1.8.0.0'
testCompile group: 'com.h2database', name: 'h2', version: '1.4.200'
testCompile group: 'org.xerial', name: 'sqlite-jdbc', version: '3.28.0'
//testCompile group: 'org.apache.derby', name: 'derby', version: '10.15.1.3'
}

apply plugin: 'nebula.ospackage'

validateNebulaPom.enabled = false

// This is afterEvaluate because the bundlePlugin ZIP task is updated afterEvaluate and changes the ZIP name to match the plugin name
afterEvaluate {
ospackage {
Expand Down
2 changes: 1 addition & 1 deletion gradle.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@
# permissions and limitations under the License.
#

version=1.8.0
version=1.9.0
2 changes: 1 addition & 1 deletion gradle/wrapper/gradle-wrapper.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
# permissions and limitations under the License.
#

distributionUrl=https\://services.gradle.org/distributions/gradle-6.4-all.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-6.5-all.zip
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
zipStorePath=wrapper/dists
Expand Down
16 changes: 16 additions & 0 deletions release-notes/opendistro-elasticsearch-sql.release-notes-1.9.0.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
## 2020-06-23 Version 1.9.0.0

### Features
#### Elasticsearch Compatibility
* Feature [#532](https://github.com/opendistro-for-elasticsearch/sql/pull/532): Elasticsearch 7.8.0 compatibility (issue: [#531](https://github.com/opendistro-for-elasticsearch/sql/issues/531))

#### Documentation
* Feature [#486](https://github.com/opendistro-for-elasticsearch/sql/pull/486): Add Github badges to README

### Enhancements
#### SQL Features
* Feature [#473](https://github.com/opendistro-for-elasticsearch/sql/pull/473): Support Integration Tests for Security enabled ODFE cluster

### Bugfixes
* Bugfix [#489](https://github.com/opendistro-for-elasticsearch/sql/pull/489): Fix ANTLR grammar for negative integer and floating point number (issue: [#488](https://github.com/opendistro-for-elasticsearch/sql/issues/488))
* Bugfix [#522](https://github.com/opendistro-for-elasticsearch/sql/pull/522): Bug fix, support long type for aggregation (issue: [#521](https://github.com/opendistro-for-elasticsearch/sql/issues/521))
10 changes: 5 additions & 5 deletions src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,11 +109,11 @@ public void setClusterService(ClusterService clusterService) {
this.clusterService = clusterService;

clusterService.addListener(event -> {
if (event.metaDataChanged()) {
if (event.metadataChanged()) {
// State in cluster service is already changed to event.state() before listener fired
if (LOG.isDebugEnabled()) {
LOG.debug("Metadata in cluster state changed: {}",
new IndexMappings(clusterService.state().metaData()));
new IndexMappings(clusterService.state().metadata()));
}
cache.invalidateAll();
}
Expand Down Expand Up @@ -169,8 +169,8 @@ public IndexMappings getFieldMappings(String[] indices, String[] types) {
}

/**
* Get field mappings by index expressions, type and field filter. Because IndexMetaData/MappingMetaData
* is hard to convert to FieldMappingMetaData, custom mapping domain objects are being used here. In future,
* Get field mappings by index expressions, type and field filter. Because IndexMetadata/MappingMetadata
* is hard to convert to FieldMappingMetadata, custom mapping domain objects are being used here. In future,
* it should be moved to domain model layer for all ES specific knowledge.
* <p>
* Note that cluster state may be change inside ES so it's possible to read different state in 2 accesses
Expand Down Expand Up @@ -222,7 +222,7 @@ private IndexMappings findMappings(ClusterState state, String[] indices, String[
Function<String, Predicate<String>> fieldFilter) throws IOException {
LOG.debug("Cache didn't help. Load and parse mapping in cluster state");
return new IndexMappings(
state.metaData().findMappings(indices, types, fieldFilter)
state.metadata().findMappings(indices, types, fieldFilter)
);
}

Expand Down
14 changes: 7 additions & 7 deletions src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMapping.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
import java.util.Map;

import static java.util.Collections.emptyMap;
import static org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
import static org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetadata;

/**
* Field mapping that parses native ES mapping.
Expand All @@ -39,7 +39,7 @@ public class FieldMapping {
/**
* Native mapping information returned from ES
*/
private final Map<String, FieldMappingMetaData> typeMappings;
private final Map<String, FieldMappingMetadata> typeMappings;

/**
* Maps a field name to Field object that specified in query explicitly
Expand All @@ -51,7 +51,7 @@ public FieldMapping(String fieldName) {
}

public FieldMapping(String fieldName,
Map<String, FieldMappingMetaData> typeMappings,
Map<String, FieldMappingMetadata> typeMappings,
Map<String, Field> specifiedFieldByNames) {

this.fieldName = fieldName;
Expand Down Expand Up @@ -119,16 +119,16 @@ public String path() {
}

/**
* Used to retrieve the type of fields from metaData map structures for both regular and nested fields
* Used to retrieve the type of fields from metadata map structures for both regular and nested fields
*/
@SuppressWarnings("unchecked")
public String type() {
FieldMappingMetaData metaData = typeMappings.get(fieldName);
Map<String, Object> source = metaData.sourceAsMap();
FieldMappingMetadata metadata = typeMappings.get(fieldName);
Map<String, Object> source = metadata.sourceAsMap();
String[] fieldPath = fieldName.split("\\.");

/*
* When field is not nested the metaData source is fieldName -> type
* When field is not nested the metadata source is fieldName -> type
* When it is nested or contains "." in general (ex. fieldName.nestedName) the source is nestedName -> type
*/
String root = (fieldPath.length == 1) ? fieldName : fieldPath[1];
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMappings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

package com.amazon.opendistroforelasticsearch.sql.esdomain.mapping;

import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.cluster.metadata.MappingMetadata;
import org.json.JSONObject;

import java.util.HashMap;
Expand Down Expand Up @@ -61,7 +61,7 @@ public class FieldMappings implements Mappings<Map<String, Object>> {
*/
private final Map<String, Object> fieldMappings;

public FieldMappings(MappingMetaData mappings) {
public FieldMappings(MappingMetadata mappings) {
fieldMappings = mappings.sourceAsMap();
}

Expand Down
12 changes: 6 additions & 6 deletions src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/IndexMappings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@

package com.amazon.opendistroforelasticsearch.sql.esdomain.mapping;

import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.metadata.MappingMetadata;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.common.collect.ImmutableOpenMap;

import java.util.Map;
Expand Down Expand Up @@ -55,12 +55,12 @@ public IndexMappings() {
this.indexMappings = emptyMap();
}

public IndexMappings(MetaData metaData) {
this.indexMappings = buildMappings(metaData.indices(),
indexMetaData -> new TypeMappings(indexMetaData.getMappings()));
public IndexMappings(Metadata metadata) {
this.indexMappings = buildMappings(metadata.indices(),
indexMetadata -> new TypeMappings(indexMetadata.getMappings()));
}

public IndexMappings(ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings) {
public IndexMappings(ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetadata>> mappings) {
this.indexMappings = buildMappings(mappings, TypeMappings::new);
}

Expand Down
4 changes: 2 additions & 2 deletions src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/TypeMappings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

package com.amazon.opendistroforelasticsearch.sql.esdomain.mapping;

import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.cluster.metadata.MappingMetadata;
import org.elasticsearch.common.collect.ImmutableOpenMap;

import java.util.Map;
Expand All @@ -36,7 +36,7 @@ public class TypeMappings implements Mappings<FieldMappings> {
*/
private final Map<String, FieldMappings> typeMappings;

public TypeMappings(ImmutableOpenMap<String, MappingMetaData> mappings) {
public TypeMappings(ImmutableOpenMap<String, MappingMetadata> mappings) {
typeMappings = buildMappings(mappings, FieldMappings::new);
}

Expand Down
2 changes: 1 addition & 1 deletion src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/AsyncRestExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ private void async(Client client, Map<String, String> params, QueryAction queryA

// Preserve context of calling thread to ensure headers of requests are forwarded when running blocking actions
threadPool.schedule(
threadPool.preserveContext(LogUtils.withCurrentContext(runnable)),
LogUtils.withCurrentContext(runnable),
new TimeValue(0L),
SQL_WORKER_THREAD_POOL_NAME
);
Expand Down
16 changes: 8 additions & 8 deletions .../java/com/amazon/opendistroforelasticsearch/sql/executor/GetIndexRequestRestListener.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
import org.elasticsearch.cluster.metadata.AliasMetaData;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.cluster.metadata.AliasMetadata;
import org.elasticsearch.cluster.metadata.MappingMetadata;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ToXContent;
Expand Down Expand Up @@ -77,22 +77,22 @@ public RestResponse buildResponse(GetIndexResponse getIndexResponse, XContentBui
return new BytesRestResponse(RestStatus.OK, builder);
}

private void writeAliases(List<AliasMetaData> aliases, XContentBuilder builder, ToXContent.Params params)
private void writeAliases(List<AliasMetadata> aliases, XContentBuilder builder, ToXContent.Params params)
throws IOException {
builder.startObject(Fields.ALIASES);
if (aliases != null) {
for (AliasMetaData alias : aliases) {
AliasMetaData.Builder.toXContent(alias, builder, params);
for (AliasMetadata alias : aliases) {
AliasMetadata.Builder.toXContent(alias, builder, params);
}
}
builder.endObject();
}

private void writeMappings(ImmutableOpenMap<String, MappingMetaData> mappings,
private void writeMappings(ImmutableOpenMap<String, MappingMetadata> mappings,
XContentBuilder builder, ToXContent.Params params) throws IOException {
builder.startObject(Fields.MAPPINGS);
if (mappings != null) {
for (ObjectObjectCursor<String, MappingMetaData> typeEntry : mappings) {
for (ObjectObjectCursor<String, MappingMetadata> typeEntry : mappings) {
builder.field(typeEntry.key);
builder.map(typeEntry.value.sourceAsMap());
}
Expand All @@ -114,4 +114,4 @@ static class Fields {
static final String SETTINGS = "settings";
static final String WARMERS = "warmers";
}
}
}
2 changes: 1 addition & 1 deletion ...va/com/amazon/opendistroforelasticsearch/sql/executor/cursor/CursorAsyncRestExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ private void async(Client client, Map<String, String> params, RestChannel channe

// Preserve context of calling thread to ensure headers of requests are forwarded when running blocking actions
threadPool.schedule(
threadPool.preserveContext(LogUtils.withCurrentContext(runnable)),
LogUtils.withCurrentContext(runnable),
new TimeValue(0L),
SQL_WORKER_THREAD_POOL_NAME
);
Expand Down
36 changes: 18 additions & 18 deletions ...ain/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DescribeResultSet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.cluster.metadata.MappingMetadata;
import org.elasticsearch.common.collect.ImmutableOpenMap;

import java.util.ArrayList;
Expand Down Expand Up @@ -89,33 +89,33 @@ private List<Column> loadColumns() {
private List<Row> loadRows() {
List<Row> rows = new ArrayList<>();
GetIndexResponse indexResponse = (GetIndexResponse) queryResult;
ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> indexMappings = indexResponse.getMappings();
ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetadata>> indexMappings = indexResponse.getMappings();

// Iterate through indices in indexMappings
for (ObjectObjectCursor<String, ImmutableOpenMap<String, MappingMetaData>> indexCursor : indexMappings) {
for (ObjectObjectCursor<String, ImmutableOpenMap<String, MappingMetadata>> indexCursor : indexMappings) {
String index = indexCursor.key;

// Check to see if index matches given pattern
if (matchesPattern(index, statement.getIndexPattern())) {
ImmutableOpenMap<String, MappingMetaData> typeMapping = indexCursor.value;
ImmutableOpenMap<String, MappingMetadata> typeMapping = indexCursor.value;
// Assuming ES 6.x, iterate through the only type of the index to get mapping data
for (ObjectObjectCursor<String, MappingMetaData> typeCursor : typeMapping) {
MappingMetaData mappingMetaData = typeCursor.value;
for (ObjectObjectCursor<String, MappingMetadata> typeCursor : typeMapping) {
MappingMetadata mappingMetadata = typeCursor.value;
// Load rows for each field in the mapping
rows.addAll(loadIndexData(index, mappingMetaData.getSourceAsMap()));
rows.addAll(loadIndexData(index, mappingMetadata.getSourceAsMap()));
}
}
}
return rows;
}

@SuppressWarnings("unchecked")
private List<Row> loadIndexData(String index, Map<String, Object> mappingMetaData) {
private List<Row> loadIndexData(String index, Map<String, Object> mappingMetadata) {
List<Row> rows = new ArrayList<>();

Map<String, String> flattenedMetaData = flattenMappingMetaData(mappingMetaData, "", new HashMap<>());
Map<String, String> flattenedMetadata = flattenMappingMetadata(mappingMetadata, "", new HashMap<>());
int position = 1; // Used as an arbitrary ORDINAL_POSITION value for the time being
for (Entry<String, String> entry : flattenedMetaData.entrySet()) {
for (Entry<String, String> entry : flattenedMetadata.entrySet()) {
String columnPattern = statement.getColumnPattern();

// Check to see if column name matches pattern, if given
Expand Down Expand Up @@ -153,21 +153,21 @@ private Map<String, Object> loadRowData(String index, String column, String type
* 'GetIndexRequestBuilder' that was used in the old ShowQueryAction. Since the format of the resulting meta data
* is different, this method is being used to flatten and retrieve types.
* <p>
* In the future, should look for a way to generalize this since Schema is currently using FieldMappingMetaData
* whereas here we are using MappingMetaData.
* In the future, should look for a way to generalize this since Schema is currently using FieldMappingMetadata
* whereas here we are using MappingMetadata.
*/
@SuppressWarnings("unchecked")
private Map<String, String> flattenMappingMetaData(Map<String, Object> mappingMetaData,
private Map<String, String> flattenMappingMetadata(Map<String, Object> mappingMetadata,
String currPath,
Map<String, String> flattenedMapping) {
Map<String, Object> properties = (Map<String, Object>) mappingMetaData.get("properties");
Map<String, Object> properties = (Map<String, Object>) mappingMetadata.get("properties");
for (Entry<String, Object> entry : properties.entrySet()) {
Map<String, Object> metaData = (Map<String, Object>) entry.getValue();
Map<String, Object> metadata = (Map<String, Object>) entry.getValue();

String fullPath = addToPath(currPath, entry.getKey());
flattenedMapping.put(fullPath, (String) metaData.getOrDefault("type", DEFAULT_OBJECT_DATATYPE));
if (metaData.containsKey("properties")) {
flattenedMapping = flattenMappingMetaData(metaData, fullPath, flattenedMapping);
flattenedMapping.put(fullPath, (String) metadata.getOrDefault("type", DEFAULT_OBJECT_DATATYPE));
if (metadata.containsKey("properties")) {
flattenedMapping = flattenMappingMetadata(metadata, fullPath, flattenedMapping);
}
}

Expand Down
Loading

0 comments on commit 254f2e0

Please sign in to comment.