From 2a1b221bbfa843032f664bf249884d473655e590 Mon Sep 17 00:00:00 2001 From: Robin Duda Date: Fri, 29 Mar 2019 22:05:45 +0100 Subject: [PATCH 1/4] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4e715810..945bc57d 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # chili-core [![Build Status](https://travis-ci.org/codingchili/chili-core.svg?branch=master)](https://travis-ci.org/codingchili/chili-core) [![](https://jitpack.io/v/codingchili/chili-core.svg)](https://jitpack.io/#codingchili/chili-core) The chili core is an opinionated framework for creating microservices with focus on speed of development and time to market. -It's based on the Vert.x toolkit for maximum power and uses Hazelcast for plug-and-play clustering. +It's based on the Vert.x toolkit for maximum power and uses Hazelcast for plug-and-play clustering. This project is small with only 22k LOC. Find the official documentation [here](https://codingchili.github.io/chili-core/). From 6f2bdcd2f7e3e24711d67e003cd7b33055cad8fd Mon Sep 17 00:00:00 2001 From: Robin Duda Date: Mon, 1 Apr 2019 15:32:19 +0200 Subject: [PATCH 2/4] upgrade elasticsearch client from 6.4.1 to 7.0.0-rc1. --- .../configuration/system/RemoteStorage.java | 18 +++ .../codingchili/core/storage/HazelMapIT.java | 2 +- .../core/storage/MapTestCases.java | 7 +- docs/storage.md | 10 +- storage/elastic/build.gradle | 2 +- .../codingchili/core/storage/ElasticMap.java | 124 ++++++++---------- .../core/storage/ElasticMapIT.java | 4 +- 7 files changed, 88 insertions(+), 79 deletions(-) diff --git a/core/main/java/com/codingchili/core/configuration/system/RemoteStorage.java b/core/main/java/com/codingchili/core/configuration/system/RemoteStorage.java index 4cd27903..9f01d577 100644 --- a/core/main/java/com/codingchili/core/configuration/system/RemoteStorage.java +++ b/core/main/java/com/codingchili/core/configuration/system/RemoteStorage.java @@ -11,6 +11,7 @@ public class RemoteStorage { private String database = CoreStrings.DEFAULT_DB; private Integer port = 27017; private int persistInterval = 3000; + private boolean secure = false; /** * @param host the hostname of the remote storage @@ -113,4 +114,21 @@ public RemoteStorage setPersistInterval(int persistInterval) { this.persistInterval = persistInterval; return this; } + + /** + * @return true if the storage plugin should attempt to use a secure connection. + * If the plugin supports security and fails to enable it an error must be thrown + * and storage initialization fail. + */ + public boolean isSecure() { + return secure; + } + + /** + * @param secure indicates that this storage must be loaded with a secure + * connection. + */ + public void setSecure(boolean secure) { + this.secure = secure; + } } diff --git a/core/test/java/com/codingchili/core/storage/HazelMapIT.java b/core/test/java/com/codingchili/core/storage/HazelMapIT.java index 8f3a6b2e..3c1a2e30 100644 --- a/core/test/java/com/codingchili/core/storage/HazelMapIT.java +++ b/core/test/java/com/codingchili/core/storage/HazelMapIT.java @@ -44,6 +44,6 @@ public void tearDown(TestContext test) { @Before public void setUp(TestContext test) { - super.setUp(test.async(), HazelMap.class, context); + super.setUp(test, HazelMap.class, context); } } diff --git a/core/test/java/com/codingchili/core/storage/MapTestCases.java b/core/test/java/com/codingchili/core/storage/MapTestCases.java index 86330015..da19796b 100644 --- a/core/test/java/com/codingchili/core/storage/MapTestCases.java +++ b/core/test/java/com/codingchili/core/storage/MapTestCases.java @@ -55,7 +55,7 @@ public class MapTestCases { protected AsyncStorage store; protected void setUp(TestContext test, Class plugin) { - setUp(test.async(), plugin, new SystemContext()); + setUp(test, plugin, new SystemContext()); } @After @@ -63,9 +63,10 @@ public void tearDown(TestContext test) { context.close(test.asyncAssertSuccess()); } - protected void setUp(Async async, Class plugin, CoreContext context) { + protected void setUp(TestContext test, Class plugin, CoreContext context) { this.context = new StorageContext<>(context); this.plugin = plugin; + Async async = test.async(); new StorageLoader(context) .withDB(plugin.getSimpleName(), COLLECTION) @@ -76,7 +77,7 @@ protected void setUp(Async async, Class plugin, CoreCont store = result.result(); prepareStore(async); } else { - throw new RuntimeException(result.cause()); + test.fail(result.cause()); } }); } diff --git a/docs/storage.md b/docs/storage.md index e3616c7b..c9a39b27 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -1,3 +1,11 @@ +# Storage + query dsl + query api -supported engines \ No newline at end of file + +supported engines + +supported versions + +ElasticSearch 7.0.0 \ No newline at end of file diff --git a/storage/elastic/build.gradle b/storage/elastic/build.gradle index 0f465996..a43d3bb3 100644 --- a/storage/elastic/build.gradle +++ b/storage/elastic/build.gradle @@ -18,7 +18,7 @@ jar { } dependencies { - compile 'org.elasticsearch.client:elasticsearch-rest-high-level-client:6.4.1' + compile 'org.elasticsearch.client:elasticsearch-rest-high-level-client:7.0.0-rc1' compile 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.9.7' compileOnly project(':core') diff --git a/storage/elastic/src/main/java/com/codingchili/core/storage/ElasticMap.java b/storage/elastic/src/main/java/com/codingchili/core/storage/ElasticMap.java index e82fd380..9f71e289 100644 --- a/storage/elastic/src/main/java/com/codingchili/core/storage/ElasticMap.java +++ b/storage/elastic/src/main/java/com/codingchili/core/storage/ElasticMap.java @@ -1,14 +1,17 @@ package com.codingchili.core.storage; -import io.vertx.core.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + import org.apache.http.HttpHost; import org.elasticsearch.ElasticsearchStatusException; -import org.elasticsearch.action.*; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; -import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetRequest; @@ -17,31 +20,42 @@ import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; -import org.elasticsearch.client.*; +import org.elasticsearch.client.IndicesClient; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.indices.CreateIndexRequest; +import org.elasticsearch.client.indices.CreateIndexResponse; +import org.elasticsearch.client.indices.GetIndexRequest; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexNotFoundException; -import org.elasticsearch.index.query.*; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.RegexpFlag; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortOrder; -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; - import com.codingchili.core.context.StorageContext; -import com.codingchili.core.logging.Logger; import com.codingchili.core.protocol.Serializer; import com.codingchili.core.security.Validator; -import com.codingchili.core.storage.exception.*; - -import static com.codingchili.core.context.FutureHelper.*; +import com.codingchili.core.storage.exception.NothingToRemoveException; +import com.codingchili.core.storage.exception.NothingToUpdateException; +import com.codingchili.core.storage.exception.StorageFailureException; +import com.codingchili.core.storage.exception.ValueAlreadyPresentException; +import com.codingchili.core.storage.exception.ValueMissingException; +import static com.codingchili.core.context.FutureHelper.error; +import static com.codingchili.core.context.FutureHelper.result; + +import io.vertx.core.AsyncResult; +import io.vertx.core.Future; +import io.vertx.core.Handler; /** * Map implementation that uses ElasticSearch. @@ -54,22 +68,16 @@ public class ElasticMap implements AsyncStorage { private StorageContext context; private RestHighLevelClient client; private String index; - private String type; - private Logger logger; public ElasticMap(Future> future, StorageContext context) { this.context = context; - this.logger = context.logger(getClass()); - this.index = context.database().toLowerCase() + "abcnnna"; - this.type = context.collection().toLowerCase(); + this.index = constructIndexName(context); try { client = new RestHighLevelClient( RestClient.builder( - new HttpHost(context.host(), context.port(), "http"))); - + new HttpHost(context.host(), context.port(), scheme(context)))); - // multiple requests just because we cannot do an addIfNotExists anymore. createIndexIfNotExists().setHandler(done -> { if (done.succeeded()) { future.complete(ElasticMap.this); @@ -82,16 +90,29 @@ public ElasticMap(Future> future, StorageContext cont } } + private String scheme(StorageContext context) { + return context.storage().isSecure() ? "https" : "http"; + } + + private String constructIndexName(StorageContext context) { + if (context.collection() != null) { + return String.format("%s.%s", + context.database().toLowerCase(), + context.collection().toLowerCase()); + } else { + return context.database(); + } + } + private Future createIndexIfNotExists() { Future future = Future.future(); IndicesClient indices = client.indices(); - indices.existsAsync(new GetIndexRequest().indices(index), RequestOptions.DEFAULT, new ActionListener() { + indices.existsAsync(new GetIndexRequest(index), RequestOptions.DEFAULT, new ActionListener() { @Override public void onResponse(Boolean exists) { if (!exists) { - CreateIndexRequest request = new CreateIndexRequest(index, - Settings.builder().build()); + CreateIndexRequest request = new CreateIndexRequest(index); indices.createAsync(request, RequestOptions.DEFAULT, new ActionListener() { @Override @@ -121,7 +142,6 @@ public void onFailure(Exception e) { public void get(String key, Handler> handler) { GetRequest request = new GetRequest() .index(index) - .type(type) .id(key); client.getAsync(request, RequestOptions.DEFAULT, new ActionListener() { @@ -149,7 +169,6 @@ public void onFailure(Exception e) { public void put(Value value, Handler> handler) { IndexRequest request = new IndexRequest() .index(index) - .type(type) .source(Serializer.buffer(value).getBytes(), XContentType.JSON) .id(value.getId()); @@ -170,7 +189,6 @@ public void onFailure(Exception e) { public void putIfAbsent(Value value, Handler> handler) { IndexRequest request = new IndexRequest() .index(index) - .type(type) .source(Serializer.buffer(value).getBytes(), XContentType.JSON) .id(value.getId()); @@ -197,15 +215,10 @@ public void onFailure(Exception e) { }); } - private Throwable nested(Throwable exception) { - return exception.getCause().getCause(); - } - @Override public void remove(String key, Handler> handler) { DeleteRequest request = new DeleteRequest() .index(index) - .type(type) .id(key); client.deleteAsync(request, RequestOptions.DEFAULT, new ActionListener() { @@ -229,7 +242,6 @@ public void onFailure(Exception e) { public void update(Value value, Handler> handler) { UpdateRequest request = new UpdateRequest() .index(index) - .type(type) .doc(Serializer.buffer(value).getBytes(), XContentType.JSON) .id(value.getId()); @@ -263,7 +275,6 @@ public void onFailure(Exception e) { public void values(Handler>> handler) { SearchRequest request = new SearchRequest() .indices(index) - .types(type) .source(new SearchSourceBuilder() .query(QueryBuilders.matchAllQuery()) .size(MAX_RESULTS) @@ -292,12 +303,11 @@ public void onFailure(Exception e) { @Override public void clear(Handler> handler) { - DeleteIndexRequest request = new DeleteIndexRequest() - .indices(index); + DeleteIndexRequest request = new DeleteIndexRequest(index); - client.indices().deleteAsync(request, RequestOptions.DEFAULT, new ActionListener() { + client.indices().deleteAsync(request, RequestOptions.DEFAULT, new ActionListener() { @Override - public void onResponse(DeleteIndexResponse response) { + public void onResponse(AcknowledgedResponse response) { if (response.isAcknowledged()) { handler.handle(result()); } else { @@ -311,38 +321,11 @@ public void onFailure(Exception e) { //handler.handle(error(e)); } }); - - /*SearchSourceBuilder builder = new SearchSourceBuilder() - .query(QueryBuilders.matchAllQuery()) - .size(Integer.MAX_VALUE); - - SearchRequest search = new SearchRequest() - .indices(index) - .types(type) - .source(builder); - - DeleteByQueryRequest request = new DeleteByQueryRequest(search).; - - DeleteRequest deleter = new DeleteRequest() - . - - client.deleteAsync(request, RequestOptions.DEFAULT, new ActionListener() { - @Override - public void onResponse(DeleteResponse deleteResponse) { - - } - - @Override - public void onFailure(Exception e) { - - } - });*/ } @Override public void size(Handler> handler) { SearchRequest request = new SearchRequest() - .types(type) .indices(index); SearchSourceBuilder source = new SearchSourceBuilder() @@ -356,7 +339,7 @@ public void size(Handler> handler) { @Override public void onResponse(SearchResponse response) { if (response.status().equals(RestStatus.OK)) { - handler.handle(result((int) response.getHits().getTotalHits())); + handler.handle(result((int) response.getHits().getTotalHits().value)); } else { handler.handle(result(0)); } @@ -448,7 +431,6 @@ public void execute(Handler>> handler) { SearchSourceBuilder source = getRequestWithOptions().query(query); SearchRequest request = new SearchRequest() .indices(index) - .types(type) .source(source); client.searchAsync(request, RequestOptions.DEFAULT, new ActionListener() { diff --git a/storage/elastic/src/test/java/com/codingchili/core/storage/ElasticMapIT.java b/storage/elastic/src/test/java/com/codingchili/core/storage/ElasticMapIT.java index e9756f40..77ead9b1 100644 --- a/storage/elastic/src/test/java/com/codingchili/core/storage/ElasticMapIT.java +++ b/storage/elastic/src/test/java/com/codingchili/core/storage/ElasticMapIT.java @@ -13,7 +13,7 @@ * Tests for the storage providers in core. Reuse these tests when new * storage subsystems are implemented using the StorageLoader. */ -@Ignore("Requires running elasticsearch 6.4.1+, travis runs an older version.") +@Ignore("Requires running elasticsearch 7.0.0+, travis runs an older version.") @RunWith(VertxUnitRunner.class) public class ElasticMapIT extends MapTestCases { private static final int ELASTIC_REFRESH = 1200; @@ -57,7 +57,7 @@ public void testClear(TestContext test) { @Ignore("Test case is dependent on the configured analyzer.") @Override public void testQueryWithUppercases(TestContext test) { - super.testQueryWithUppercases(test); + //super.testQueryWithUppercases(test); } @Ignore("Searching with case sensitivity is not supported for ElasticSearch.") From 5d81d59538bbdfd8cdeab13a3e1ffc81a10a653d Mon Sep 17 00:00:00 2001 From: Robin Duda Date: Mon, 1 Apr 2019 17:08:48 +0200 Subject: [PATCH 3/4] build with vertx 3.6.3. (up from 3.6.0) --- core/build.gradle | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/build.gradle b/core/build.gradle index 2d240ac3..edaedcc2 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -27,11 +27,11 @@ artifacts { } dependencies { - compile 'io.vertx:vertx-core:3.6.0' - compile 'io.vertx:vertx-web:3.6.0' - compile 'io.vertx:vertx-hazelcast:3.6.0' - compile 'io.vertx:vertx-mongo-client:3.6.0' - compile 'io.vertx:vertx-dropwizard-metrics:3.6.0' + compile 'io.vertx:vertx-core:3.6.3' + compile 'io.vertx:vertx-web:3.6.3' + compile 'io.vertx:vertx-hazelcast:3.6.3' + compile 'io.vertx:vertx-mongo-client:3.6.3' + compile 'io.vertx:vertx-dropwizard-metrics:3.6.3' compile 'de.neuland-bfi:jade4j:1.2.7' compile 'de.mkammerer:argon2-jvm:2.5' compile 'org.fusesource.jansi:jansi:1.17.1' @@ -39,6 +39,6 @@ dependencies { compile 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.9.7' - testCompile 'io.vertx:vertx-unit:3.6.0' + testCompile 'io.vertx:vertx-unit:3.6.3' testCompile 'junit:junit:4.12' } From fe07cf057fdce212570744f532a6a4048f942146 Mon Sep 17 00:00:00 2001 From: Robin Duda Date: Mon, 1 Apr 2019 17:14:04 +0200 Subject: [PATCH 4/4] implement missing test method in vertx-unit mock. --- build.gradle | 2 +- .../java/com/codingchili/core/listener/RestRequestTest.java | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 32376aac..f0f12982 100644 --- a/build.gradle +++ b/build.gradle @@ -2,7 +2,7 @@ apply plugin: 'java' apply plugin: 'idea' apply plugin: 'maven' -project.version = "1.1.2" +project.version = "1.1.3" project.group = 'com.github.codingchili.chili-core' subprojects { diff --git a/core/test/java/com/codingchili/core/listener/RestRequestTest.java b/core/test/java/com/codingchili/core/listener/RestRequestTest.java index 4bc5998d..679db27f 100644 --- a/core/test/java/com/codingchili/core/listener/RestRequestTest.java +++ b/core/test/java/com/codingchili/core/listener/RestRequestTest.java @@ -355,6 +355,11 @@ public void fail(Throwable throwable) { } + @Override + public void fail(int i, Throwable throwable) { + + } + @Override public RoutingContext put(String s, Object o) { return null;