From deae5a5bc1ccb8b1d3a1b86f5a2daca95315e002 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 26 Apr 2018 21:31:17 +0200 Subject: [PATCH 01/16] Micro-optimization of uniform sampling reservoir - use primitive double arrays instead of lists - merge add and remove operations into a single operation --- .../collector/NumericAggregateCollector.java | 6 +- .../collector/UniformSamplingReservoir.java | 188 +++++++++++------- .../NumericAggregateCollectorTest.java | 6 +- .../UniformSamplingReservoirTest.java | 11 +- 4 files changed, 127 insertions(+), 84 deletions(-) diff --git a/src/main/java/org/radarcns/stream/collector/NumericAggregateCollector.java b/src/main/java/org/radarcns/stream/collector/NumericAggregateCollector.java index a06ff945..2d74b46f 100644 --- a/src/main/java/org/radarcns/stream/collector/NumericAggregateCollector.java +++ b/src/main/java/org/radarcns/stream/collector/NumericAggregateCollector.java @@ -276,9 +276,9 @@ public Builder reservoir(UniformSamplingReservoir reservoir) { */ @Deprecated @JsonSetter - public Builder history(List history) { - min(history.get(0)); - max(history.get(history.size() - 1)); + public Builder history(double[] history) { + min(history[0]); + max(history[history.length - 1]); reservoir(new UniformSamplingReservoir(history)); return this; } diff --git a/src/main/java/org/radarcns/stream/collector/UniformSamplingReservoir.java b/src/main/java/org/radarcns/stream/collector/UniformSamplingReservoir.java index 1d9e10d5..b1cd72b7 100644 --- a/src/main/java/org/radarcns/stream/collector/UniformSamplingReservoir.java +++ b/src/main/java/org/radarcns/stream/collector/UniformSamplingReservoir.java @@ -5,7 +5,6 @@ import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashSet; import java.util.LinkedList; import java.util.List; @@ -22,14 +21,15 @@ * are computed exactly. */ public class UniformSamplingReservoir { - private final List samples; + private final double[] samples; private final int maxSize; private int count; + private transient int currentLength; private static final int MAX_SIZE_DEFAULT = 999; /** Empty reservoir with default maximum size. */ public UniformSamplingReservoir() { - this(Collections.emptyList(), 0, MAX_SIZE_DEFAULT); + this(new double[] {}, 0, MAX_SIZE_DEFAULT); } /** @@ -37,8 +37,8 @@ public UniformSamplingReservoir() { * @param allValues list of values to sample from. * @throws NullPointerException if given allValues are {@code null}. */ - public UniformSamplingReservoir(List allValues) { - this(allValues, allValues.size(), MAX_SIZE_DEFAULT); + public UniformSamplingReservoir(double[] allValues) { + this(allValues, allValues.length, MAX_SIZE_DEFAULT); } /** @@ -50,87 +50,120 @@ public UniformSamplingReservoir(List allValues) { */ @JsonCreator public UniformSamplingReservoir( - @JsonProperty("samples") List samples, + @JsonProperty("samples") double[] samples, @JsonProperty("count") int count, @JsonProperty("maxSize") int maxSize) { if (samples == null) { throw new IllegalArgumentException("Samples may not be null"); } if (maxSize <= 0) { - throw new IllegalArgumentException("Reservoir size must be strictly positive"); + throw new IllegalArgumentException("Reservoir maximum size must be strictly positive"); } - this.maxSize = maxSize; - if (count < 0) { throw new IllegalArgumentException("Reservoir size must be positive"); } + this.samples = new double[maxSize]; + this.maxSize = maxSize; this.count = count; - this.samples = new ArrayList<>(maxSize); initializeReservoir(samples); - Collections.sort(this.samples); } /** Sample from given list of samples to initialize the reservoir. */ - private void initializeReservoir(List samples) { - int numSamples = samples.size(); + private void initializeReservoir(double[] initSamples) { + int length = Math.min(initSamples.length, count); + + if (length == 0) { + currentLength = 0; + return; + } + + ThreadLocalRandom random = ThreadLocalRandom.current(); + // There are much more samples than the size permits. Random sample from the // given list. Duplicate addresses are retried, and for each hit there is at least a // 50% probability of getting a number that has not yet been picked. - if (numSamples > maxSize * 2) { - ThreadLocalRandom random = ThreadLocalRandom.current(); - Set indexes = new HashSet<>(); - while (indexes.size() < maxSize) { - indexes.add(random.nextInt(numSamples)); - } - for (Integer index : indexes) { - this.samples.add(samples.get(index)); + if (length > maxSize * 2) { + Set indexSet = new HashSet<>(); + int sampleIndex = 0; + while (sampleIndex < maxSize) { + int initIndex = random.nextInt(length); + // only add to samples if index set does not yet contain value + if (indexSet.add(initIndex)) { + this.samples[sampleIndex] = initSamples[initIndex]; + sampleIndex++; + } } + currentLength = maxSize; + // There are not much more samples than the size permits. Make a list from all indexes // and at random pick and remove index from that. Put all the samples at given // indexes in the reservoir. Do not do retry sampling as above, as the final entry may // have a probability of 1/maxSize of actually being picked. - } else if (numSamples > maxSize) { - LinkedList indexes = new LinkedList<>(); - for (int i = 0; i < numSamples; i++) { - indexes.add(i); + } else if (length > maxSize) { + LinkedList allInitIndexes = new LinkedList<>(); + for (int i = 0; i < length; i++) { + allInitIndexes.add(i); } - ThreadLocalRandom random = ThreadLocalRandom.current(); - for (int i = 0; i < maxSize; i++) { - int index = indexes.remove(random.nextInt(indexes.size())); - this.samples.add(samples.get(index)); + for (int sampleIndex = 0; sampleIndex < maxSize; sampleIndex++) { + int initIndex = allInitIndexes.remove(random.nextInt(allInitIndexes.size())); + this.samples[sampleIndex] = initSamples[initIndex]; } + currentLength = maxSize; } else { - this.samples.addAll(samples); + System.arraycopy(initSamples, 0, this.samples, 0, length); + currentLength = length; } + Arrays.sort(this.samples, 0, currentLength); } /** Add a sample to the reservoir. */ public void add(double value) { - boolean doAdd; - int removeIndex; - - if (count < maxSize) { - doAdd = true; - } else { - removeIndex = ThreadLocalRandom.current().nextInt(count); + if (currentLength == maxSize) { + int removeIndex = ThreadLocalRandom.current().nextInt(count); if (removeIndex < maxSize) { - samples.remove(removeIndex); - doAdd = true; - } else { - doAdd = false; + removeAndAdd(removeIndex, value); } + } else { + removeAndAdd(currentLength, value); + currentLength++; } - if (doAdd) { - int index = Collections.binarySearch(samples, value); - if (index >= 0) { - samples.add(index, value); - } else { - samples.add(-index - 1, value); - } + count++; + } + + private void removeAndAdd(int removeIndex, double value) { + int addIndex = Arrays.binarySearch(samples, 0, currentLength, value); + if (addIndex < 0) { + addIndex = -addIndex - 1; } - count++; + int copySrc; + int copyDest; + int copyLength; + + // start: [a, b, c, d, e] + if (removeIndex < addIndex) { + // removeIndex = 2, value = d2 -> addIndex = 4 + addIndex--; + // new addIndex -> 3 + // copy(3 -> 2, len 1) + // end: [a, b, d, d2, e] + copySrc = removeIndex + 1; + copyDest = removeIndex; + copyLength = addIndex - removeIndex; + } else { + // removeIndex = 2, value = a2 -> addIndex = 1 + // copy(1 -> 2, len 1) + // end: [a, a2, b, d, e] + copySrc = addIndex; + copyDest = addIndex + 1; + copyLength = removeIndex - addIndex; + } + + if (copyLength > 0) { + System.arraycopy(samples, copySrc, samples, copyDest, copyLength); + } + samples[addIndex] = value; } /** @@ -139,27 +172,34 @@ public void add(double value) { * @return list with size three, of the 25, 50 and 75 percentiles. */ public List getQuartiles() { - int length = samples.size(); - - List quartiles; - if (length == 1) { - Double elem = samples.get(0); - quartiles = Arrays.asList(elem, elem, elem); - } else { - quartiles = new ArrayList<>(3); - for (int i = 1; i <= 3; i++) { - double pos = i * (length + 1) * 0.25d; // 25 percentile steps - int intPos = (int) pos; - if (intPos == 0) { - quartiles.add(samples.get(0)); - } else if (intPos == length) { - quartiles.add(samples.get(length - 1)); - } else { - double diff = pos - intPos; - double base = samples.get(intPos - 1); - quartiles.add(base + diff * (samples.get(intPos) - base)); + List quartiles = new ArrayList<>(3); + + switch (currentLength) { + case 0: + quartiles.add(Double.NaN); + quartiles.add(Double.NaN); + quartiles.add(Double.NaN); + break; + case 1: + quartiles.add(samples[0]); + quartiles.add(samples[0]); + quartiles.add(samples[0]); + break; + default: + for (int i = 1; i <= 3; i++) { + double pos = i * (currentLength + 1) * 0.25d; // 25 percentile steps + int intPos = (int) pos; + if (intPos == 0) { + quartiles.add(samples[0]); + } else if (intPos == currentLength) { + quartiles.add(samples[currentLength - 1]); + } else { + double diff = pos - intPos; + double base = samples[intPos - 1]; + quartiles.add(base + diff * (samples[intPos] - base)); + } } - } + break; } return quartiles; @@ -167,7 +207,11 @@ public List getQuartiles() { /** Get the currently stored samples. */ public List getSamples() { - return Collections.unmodifiableList(samples); + List doubleList = new ArrayList<>(currentLength); + for (int i = 0; i < currentLength; i++) { + doubleList.add(samples[i]); + } + return doubleList; } /** Get the maximum size of this reservoir. */ @@ -191,7 +235,7 @@ public boolean equals(Object o) { UniformSamplingReservoir that = (UniformSamplingReservoir) o; return count == that.count && maxSize == that.maxSize - && Objects.equals(samples, that.samples); + && Arrays.equals(samples, that.samples); } @Override @@ -202,7 +246,7 @@ public int hashCode() { @Override public String toString() { return "UniformSamplingReservoir{" - + "samples=" + samples + + "samples=" + Arrays.toString(samples) + ", maxSize=" + maxSize + ", count=" + count + '}'; diff --git a/src/test/java/org/radarcns/stream/collector/NumericAggregateCollectorTest.java b/src/test/java/org/radarcns/stream/collector/NumericAggregateCollectorTest.java index 8d2bde35..1bc2930a 100644 --- a/src/test/java/org/radarcns/stream/collector/NumericAggregateCollectorTest.java +++ b/src/test/java/org/radarcns/stream/collector/NumericAggregateCollectorTest.java @@ -171,7 +171,7 @@ public void testSerialization() throws IOException { mapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); valueCollector = new NumericAggregateCollector.Builder("test") - .history(Arrays.asList(-1d, 15d)) + .history(new double[] {-1d, 15d}) .sum(BigDecimal.valueOf(14)) .build(); @@ -183,7 +183,7 @@ public void testSerialization() throws IOException { @Test public void testReservoirBuilder() { valueCollector = new NumericAggregateCollector.Builder("test") - .reservoir(new UniformSamplingReservoir(Arrays.asList(-1d, 15d), 2, 1)) + .reservoir(new UniformSamplingReservoir(new double[] {-1d, 15d}, 2, 1)) .build(); assertEquals(2, valueCollector.getCount()); @@ -193,7 +193,7 @@ public void testReservoirBuilder() { @Test public void testReservoirBuilderUnlimited() { valueCollector = new NumericAggregateCollector.Builder("name") - .reservoir(new UniformSamplingReservoir(Arrays.asList(-1d, 15d), 2, 1000)) + .reservoir(new UniformSamplingReservoir(new double[] {-1d, 15d}, 2, 1000)) .build(); assertEquals(2, valueCollector.getCount()); diff --git a/src/test/java/org/radarcns/stream/collector/UniformSamplingReservoirTest.java b/src/test/java/org/radarcns/stream/collector/UniformSamplingReservoirTest.java index 136740eb..2148b956 100644 --- a/src/test/java/org/radarcns/stream/collector/UniformSamplingReservoirTest.java +++ b/src/test/java/org/radarcns/stream/collector/UniformSamplingReservoirTest.java @@ -2,18 +2,17 @@ import org.junit.Test; -import java.util.Arrays; -import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.concurrent.ThreadLocalRandom; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class UniformSamplingReservoirTest { @Test public void add() { - UniformSamplingReservoir reservoir = new UniformSamplingReservoir(Arrays.asList(0.1, 0.3, 0.5), 3, 3); + UniformSamplingReservoir reservoir = new UniformSamplingReservoir(new double[] {0.1, 0.3, 0.5}, 3, 3); reservoir.add(0.7); assertEquals(3, reservoir.getSamples().size()); assertEquals(3, reservoir.getMaxSize()); @@ -25,7 +24,7 @@ public void add() { @Test public void addRandom() { - UniformSamplingReservoir reservoir = new UniformSamplingReservoir(Collections.emptyList(), 0, 50); + UniformSamplingReservoir reservoir = new UniformSamplingReservoir(new double[0], 0, 50); ThreadLocalRandom random = ThreadLocalRandom.current(); for (int i = 0; i < 100; i++) { @@ -38,7 +37,7 @@ public void addRandom() { @Test public void addFromRandom() { - UniformSamplingReservoir reservoir = new UniformSamplingReservoir(Collections.emptyList(), 0, 50); + UniformSamplingReservoir reservoir = new UniformSamplingReservoir(new double[0], 0, 50); ThreadLocalRandom random = ThreadLocalRandom.current(); From 8fc9604e1d31165fc20e24bee365e80dc425f298 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 30 Apr 2018 08:59:48 +0200 Subject: [PATCH 02/16] Only serialize used part of the uniform sampling reservoir --- .../org/radarcns/stream/collector/UniformSamplingReservoir.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/radarcns/stream/collector/UniformSamplingReservoir.java b/src/main/java/org/radarcns/stream/collector/UniformSamplingReservoir.java index b1cd72b7..7464706a 100644 --- a/src/main/java/org/radarcns/stream/collector/UniformSamplingReservoir.java +++ b/src/main/java/org/radarcns/stream/collector/UniformSamplingReservoir.java @@ -1,6 +1,7 @@ package org.radarcns.stream.collector; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonGetter; import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; @@ -206,6 +207,7 @@ public List getQuartiles() { } /** Get the currently stored samples. */ + @JsonGetter public List getSamples() { List doubleList = new ArrayList<>(currentLength); for (int i = 0; i < currentLength; i++) { From 162c93b52678fcbe9a09fa89370cf2dd9ab446f2 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 30 Apr 2018 11:47:04 +0200 Subject: [PATCH 03/16] More compact addAndRemove --- .../collector/UniformSamplingReservoir.java | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/radarcns/stream/collector/UniformSamplingReservoir.java b/src/main/java/org/radarcns/stream/collector/UniformSamplingReservoir.java index 7464706a..83b12cf6 100644 --- a/src/main/java/org/radarcns/stream/collector/UniformSamplingReservoir.java +++ b/src/main/java/org/radarcns/stream/collector/UniformSamplingReservoir.java @@ -138,10 +138,6 @@ private void removeAndAdd(int removeIndex, double value) { addIndex = -addIndex - 1; } - int copySrc; - int copyDest; - int copyLength; - // start: [a, b, c, d, e] if (removeIndex < addIndex) { // removeIndex = 2, value = d2 -> addIndex = 4 @@ -149,21 +145,16 @@ private void removeAndAdd(int removeIndex, double value) { // new addIndex -> 3 // copy(3 -> 2, len 1) // end: [a, b, d, d2, e] - copySrc = removeIndex + 1; - copyDest = removeIndex; - copyLength = addIndex - removeIndex; - } else { + if (removeIndex < addIndex) { + System.arraycopy(samples, removeIndex + 1, samples, removeIndex, addIndex - removeIndex); + } + } else if (removeIndex > addIndex) { // removeIndex = 2, value = a2 -> addIndex = 1 // copy(1 -> 2, len 1) // end: [a, a2, b, d, e] - copySrc = addIndex; - copyDest = addIndex + 1; - copyLength = removeIndex - addIndex; + System.arraycopy(samples, addIndex, samples, addIndex + 1, removeIndex - addIndex); } - if (copyLength > 0) { - System.arraycopy(samples, copySrc, samples, copyDest, copyLength); - } samples[addIndex] = value; } From 5837a612d7dfe178fa4cf2cc7c1b132c0f44d2f0 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 30 Apr 2018 11:51:33 +0200 Subject: [PATCH 04/16] Code style fix --- .../radarcns/stream/collector/UniformSamplingReservoir.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/radarcns/stream/collector/UniformSamplingReservoir.java b/src/main/java/org/radarcns/stream/collector/UniformSamplingReservoir.java index 83b12cf6..b9bdbbe6 100644 --- a/src/main/java/org/radarcns/stream/collector/UniformSamplingReservoir.java +++ b/src/main/java/org/radarcns/stream/collector/UniformSamplingReservoir.java @@ -146,13 +146,15 @@ private void removeAndAdd(int removeIndex, double value) { // copy(3 -> 2, len 1) // end: [a, b, d, d2, e] if (removeIndex < addIndex) { - System.arraycopy(samples, removeIndex + 1, samples, removeIndex, addIndex - removeIndex); + System.arraycopy(samples, removeIndex + 1, samples, removeIndex, + addIndex - removeIndex); } } else if (removeIndex > addIndex) { // removeIndex = 2, value = a2 -> addIndex = 1 // copy(1 -> 2, len 1) // end: [a, a2, b, d, e] - System.arraycopy(samples, addIndex, samples, addIndex + 1, removeIndex - addIndex); + System.arraycopy(samples, addIndex, samples, addIndex + 1, + removeIndex - addIndex); } samples[addIndex] = value; From 88fbf4fd046e80edd558a204ab2ae58c74237807 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 6 Jun 2018 10:59:19 +0200 Subject: [PATCH 05/16] Refresh schema IDs daily. --- .../producer/rest/SchemaRetriever.java | 45 ++++++++++++++++--- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/radarcns/producer/rest/SchemaRetriever.java b/src/main/java/org/radarcns/producer/rest/SchemaRetriever.java index 9212b5ab..6aab116c 100644 --- a/src/main/java/org/radarcns/producer/rest/SchemaRetriever.java +++ b/src/main/java/org/radarcns/producer/rest/SchemaRetriever.java @@ -50,6 +50,7 @@ public class SchemaRetriever implements Closeable { private static final Schema NULL_SCHEMA = Schema.create(Type.NULL); private static final Map, Schema> PRIMITIVE_SCHEMAS = new HashMap<>(); private static final byte[] SCHEMA = utf8("{\"schema\":"); + private static final long MAX_VALIDITY = 86400L; static { PRIMITIVE_SCHEMAS.put(Long.class, Schema.create(Type.LONG)); @@ -61,7 +62,7 @@ public class SchemaRetriever implements Closeable { PRIMITIVE_SCHEMAS.put(byte[].class, Schema.create(Type.BYTES)); } - private final ConcurrentMap cache; + private final ConcurrentMap cache; private RestClient httpClient; public SchemaRetriever(ServerConfig config, long connectionTimeout) { @@ -114,15 +115,15 @@ protected ParsedSchemaMetadata retrieveSchemaMetadata(String subject, int versio public ParsedSchemaMetadata getSchemaMetadata(String topic, boolean ofValue, int version) throws IOException { String subject = subject(topic, ofValue); - ParsedSchemaMetadata value = cache.get(subject); - if (value == null) { - value = retrieveSchemaMetadata(subject, version); - ParsedSchemaMetadata oldValue = cache.putIfAbsent(subject, value); + TimedSchemaMetadata value = cache.get(subject); + if (value == null || value.isExpired()) { + value = new TimedSchemaMetadata(retrieveSchemaMetadata(subject, version)); + TimedSchemaMetadata oldValue = cache.putIfAbsent(subject, value); if (oldValue != null) { value = oldValue; } } - return value; + return value.metadata; } /** Parse a schema from string. */ @@ -152,7 +153,7 @@ public void addSchemaMetadata(String topic, boolean ofValue, ParsedSchemaMetadat int schemaId = node.getInt("id"); metadata.setId(schemaId); } - cache.put(subject, metadata); + cache.put(subject, new TimedSchemaMetadata(metadata)); } /** @@ -223,4 +224,34 @@ public static Schema getSchema(Object object) { + object.getClass() + " can not be schematized. " + "Pass null, a primitive CONTENT_TYPE or a GenericContainer."); } + + private class TimedSchemaMetadata { + private final ParsedSchemaMetadata metadata; + private final long expiry; + + TimedSchemaMetadata(ParsedSchemaMetadata metadata) { + expiry = System.currentTimeMillis() + MAX_VALIDITY; + this.metadata = Objects.requireNonNull(metadata); + } + + boolean isExpired() { + return expiry < System.currentTimeMillis(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + return metadata.equals(((TimedSchemaMetadata)o).metadata); + } + + @Override + public int hashCode() { + return metadata.hashCode(); + } + } } From b728d5b7df25584eaeaa2a2e93fa75881ddddfb0 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 6 Jun 2018 11:08:14 +0200 Subject: [PATCH 06/16] Do not re-use expired metadata --- .../java/org/radarcns/producer/rest/SchemaRetriever.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/java/org/radarcns/producer/rest/SchemaRetriever.java b/src/main/java/org/radarcns/producer/rest/SchemaRetriever.java index 6aab116c..a036931d 100644 --- a/src/main/java/org/radarcns/producer/rest/SchemaRetriever.java +++ b/src/main/java/org/radarcns/producer/rest/SchemaRetriever.java @@ -118,10 +118,7 @@ public ParsedSchemaMetadata getSchemaMetadata(String topic, boolean ofValue, int TimedSchemaMetadata value = cache.get(subject); if (value == null || value.isExpired()) { value = new TimedSchemaMetadata(retrieveSchemaMetadata(subject, version)); - TimedSchemaMetadata oldValue = cache.putIfAbsent(subject, value); - if (oldValue != null) { - value = oldValue; - } + cache.put(subject, value); } return value.metadata; } From 649cc62eb1756983d3025560f764f48457d6062f Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 26 Jun 2018 15:27:04 +0200 Subject: [PATCH 07/16] Revert RestClient to HTTP/1.1 This solves some spurious connection issues. --- .../java/org/radarcns/producer/rest/RestClient.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/radarcns/producer/rest/RestClient.java b/src/main/java/org/radarcns/producer/rest/RestClient.java index b9876408..c15378af 100644 --- a/src/main/java/org/radarcns/producer/rest/RestClient.java +++ b/src/main/java/org/radarcns/producer/rest/RestClient.java @@ -20,6 +20,7 @@ import okhttp3.HttpUrl; import okhttp3.OkHttpClient; import okhttp3.OkHttpClient.Builder; +import okhttp3.Protocol; import okhttp3.Request; import okhttp3.Response; import okhttp3.ResponseBody; @@ -37,6 +38,7 @@ import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; import java.security.cert.CertificateException; +import java.util.Collections; import java.util.Objects; import java.util.concurrent.TimeUnit; @@ -147,10 +149,11 @@ public boolean verify(String hostname, SSLSession session) { private OkHttpClient buildHttpClient(Builder builder, ManagedConnectionPool connectionPool) { builder - .connectTimeout(timeout, TimeUnit.SECONDS) - .writeTimeout(timeout, TimeUnit.SECONDS) - .readTimeout(timeout, TimeUnit.SECONDS) - .proxy(config.getHttpProxy()); + .connectTimeout(timeout, TimeUnit.SECONDS) + .writeTimeout(timeout, TimeUnit.SECONDS) + .readTimeout(timeout, TimeUnit.SECONDS) + .proxy(config.getHttpProxy()) + .protocols(Collections.singletonList(Protocol.HTTP_1_1)); if (connectionPool != null) { builder.connectionPool(connectionPool.acquire()); From 2bf3c8f0ffc537674fd648b833348f66ba6ec0ae Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 26 Jun 2018 15:31:01 +0200 Subject: [PATCH 08/16] Updated gradle --- build.gradle | 15 +++++++-------- gradle/wrapper/gradle-wrapper.jar | Bin 54329 -> 54413 bytes gradle/wrapper/gradle-wrapper.properties | 2 +- settings.gradle | 2 +- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/build.gradle b/build.gradle index c872f9db..98264100 100644 --- a/build.gradle +++ b/build.gradle @@ -16,7 +16,7 @@ plugins { // Get bintray version - id 'com.jfrog.bintray' version '1.8.0' + id 'com.jfrog.bintray' version '1.7.3' id 'com.jfrog.artifactory' version '4.5.4' } @@ -323,10 +323,10 @@ publishing { } bintray { - user = project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER') - key = project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_API_KEY') - override = false - publications = ['RadarCommonsPublication'] + user project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER') + key project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_API_KEY') + override false + publications 'RadarCommonsPublication' pkg { repo = project.group name = rootProject.name @@ -363,7 +363,6 @@ artifactoryPublish { publications('RadarCommonsPublication') } -task wrapper(type: Wrapper) { - gradleVersion = '4.4' - distributionType 'all' +wrapper { + gradleVersion '4.8' } diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 01b8bf6b1f99cad9213fc495b33ad5bbab8efd20..1948b9074f1016d15d505d185bc3f73deb82d8c8 100644 GIT binary patch delta 7399 zcmY+JWmFVEyv3IV=@jV(>F!z@2@w$K7LZtALE5DjX{1>i=?)12X{2LeX}_~3Zr zS8+Z^CuOc*#vOS(9rx~{;vK=PN*oInvlWewm9q?HBbE;o_x;6zen+o_GVn=8O(eH- z0)EfmN&CUxAA=#~myMvmSzmnLDUQEToNE8Pge)u4hI*|3WS3HwwAbF|+Kkrxy8RW1 z??)%`HVm^w1eCuXP2E{E;f<)3hKM`Oseb8MPK9vsjC>Z*q5_X-j(mtvtlZhGDv)~% z#TB1X$q}_U<3&+j9?ZXp+bva%z-n;_E3-ABH+=ow9qj^ycH5G;~+ zWT)EPw2{$k@s&u(VIx%xn+kMNP@I;Yf$3!`5c#HlLKhh?*t7kx{Cz)1hh0&nD zfT^r%5#6%+to!O2&X`C#*_?EB3LDqIMw<9fooHoRymc(M$!ysO@>{4gXBiAr9nL9m zTcUM%7k|_i76U~z3=13;e;IQNZw+4YFt@aty-)wlJF_u0S^IALUh|0DxnBP7){Y*| zGV2TveLN+#GXU{Y)t$wyr-4kC+g|67!HDLwd7(MG*SmoO20!2CXYk`%YPZcFeTb&7SbGsyiRZ+&JzpftORRs>7bdt$pp4tZo;5o-ponrL!*VRI?y{APgqidMw%s?D zq%AxTlU(UV?r|^&hU}2c+_x$}E=Ijjdu)s8(X)GO1Hbl?*+d;@>W%c^=NbWqKY1yT zIMGPShAa%oxyX+Rv5qIgF0z(5`|TonPGiUQc~N7m8!h&AF>eooiL)Rj9wQiBs!P%NBkhXAI~+#_MG(sqJJ!D zAv#?B>&o&@kyBFG>RH67un37ampQ?&Y+(JJ?_2ly{hB&ljO~&~sV{A7+eQ_r`0YFh ziNlp8dV}vTvk~&b)>(_ za+HH<)=~rD4ld@hre7Fj7`YHy{y@F1$1D`9#zm5?BI1R!Fg@d}Ah*bYLZ=bzGN|#d>6Kl=gD#y07Lb2wx+fs1 ztF~who=cp0(U4pR+r-^a<;xeP+_mL_SPqoLY^7a$7KrE#eB!KHeywu%%s?kknex?= zKnM*#2KQoa5V>spdk2cab*d^_6X7s;^sTYzekuQF>@>IObfetv{fdf194^l^H{le7 z5#6h5lJlq&+bRMk62huwA%hUn5Ys!Z`u2u9=BOdtr4oTKnEBbIZ$E`{75~E8rni*3 za!VROb5@60DrBtg{13F#QS!|I4Pq4c;WQV%UrEaQad8t0nxg;;D|^lfS0+TxybqOViPItSXJr;vn?4w*Q{Dt7?-}+wCV!t)B-4^GLVlxA zclVHP$zWoE#V5u}3b_TFfmGRKwfHQalUhEfuw+tw983Ar6J!z{;3#usz`0a;>y}GB zpkOjqCU{^NZ>=U1hQ3FIUr8ZQU@^dS$wg?`?DP3MXngWVz-C~2s@TN7!Pj$!&&%;l z$`0#jw2b2{J-uA2@ICb9(OZleU5s&xz5IdKVt12Q*zjaLgH~eN9%^4 zO*+c%A1GdHpkq7(0I;zE0DuA@J(bKDhZF#);sgK~;cl4Z@K>TFFukotIdVL54%xef?Zw-b{eA9-Tt9m@_$D*scgxN@WYgzM*qBdeJnHfHg@>|8E{rCR zek^T(J(fAvGBX$m6F~#33~P(Z7lTaJ1?e|@EU{k)4I-8#zyny#3G8aZf?Y2_obpYl z@4zJ-wD=RsIb8dtoS@}#VIT-@&q<3rp(o4IRe$q}DuYF{uE+j~19?2*i5VZxYu+@( zxDXgoTu#}aVpjaVF?X92WNE;IJz-|S@iY3q-KbMTd8rsC7Up`^UNjjXaFF3Rs#64j>K)kbolbxvX|nnMKjPju|#qG3!3BlIETM_kl66m6rl0U zLCzI^>t=2LiVb!NNnI|t1-oFlDf?B zvnp33^R{yE&u_N_sj)QFhLDB1=4$BT?Lw?mI&K!s+o7#tuZ;Y-%BAiV%Sy{*H&b%PBJdR6*PhpX^#BFv|zig}INi&)8Qs2Rv+ z_dVFiI-mAx_C{ooXU`x!R1SdlFCG-7u z0FKm9E zNsAwG<`Cmk$}=(BNNJbIG=$38+aWI7hM0#rI30&wiE7wz0d!S?C7dipCbq}G*xl?_ zeVTaZADV@Q;wS@}r#NpNRs804rj)NfX)JzHu-m=Xi<7aUYdg1LHL!LC0Z=ze(jthUu{LF@;9DtP zQdBb44+toJ%CGgZwpjIm3O4icnI2wCFO#8uja{Y(@pIFNMI_Da&-5YTiDwOz*>1>} z8MkT__@%qnNu27-jyq^&Jarqs%P~~LMF6UNkRNT0dI>7cT3hN~;zsLbT|Ex#gF;Y*+jz#c={hPXn> zF@DDlNEl_7yHLvcsCTJjI6W1dpgO!S)9AA{q17p|R_!s%x~xca7Tu6Y%WEtx^2B@g z>UoyvyfN`)Ve6*$bpIQpGT3w7-%vE@=?KG7uyqNcgZA7W^)(4oa_*;N!Mtl8lo7&9 zxs$$Wbft>lS=z=))csA!s9I7m%U4NxAiL5#^ReblNhaGND>=ysBP9^obbZuw+vLt_e(*IUYmxTXZ9KP;4D zQIMv8OBRqfMY5+0+bz8iPQl!vN|9AXm3fkxf0-)0NPl^E<0v5e1y@%rK&v>w^UKTJ z(hgk9!n18?4dU+FBLM_XHen^rnswGKQBv(eZA=P?{i3)VoR8!gNvKH3rErfJpV%vZvK6P z2n+qxMu(BzMy;lrSS7A#k@iYL3bq$xbi@Tc?bPJT#Qj0*3fBx}$UUm@xT_X%b&hUb zi={6+?9VbV&uG@1Db~Xmve9ZJ58Z7G)6gW+Q`0vdn$` z%6eb*DVhuk>8jVI>B{RzS?4dk6b{X-VAK1|6K6kVt%gv@a1XC1Cn|Mnqh8cuB&}$~ zYak)zf|*WSmm%U$f8sJ2+~By~`&_Hj&W3>Ne~2_%3}~AE)at&{up#6M) zwSXt?l{P(_p?lMIUohdTo|zW0lkx?r7Z_I4Pr`4Ij4hFBEY>ja=Em=<^Dp)^;|yA% z=d49!{FNF-1tij*I)2R0h5GHw{AzFUy`GQ4G82lT|GxP<5jCeF5)mj>FJ4YY++i|9 zAR+w=2`W0VE^j9{4MpBCQDC*FgtMb^GI!KFecG@rM9n!3?)UOD)BA4-xitoBmnPkx z<@YRss#`u(ojmTm&U&{Au!0k9{l|>v$L;&S7c&-IffN`T*4Z5}X16!sn~R#Q7^^sJ z^_NyuTOjAkUCK*_Nrq>S@>`bYvs{0|U6nRKZau12SyYK1RKQJZ7d2f}fH|#EH(p#T zPI(P#nl0(8XRdb~IBqc7moPK+-IM*tWUFX*z6nwdvzpO-IC!Ap&@)F|cGgs3U7+sJ zNx`6_TSz#_Yu-MjBe%7L%CS+`V5|c+cLv^#e)(npORtI;Ve5O`U66}DNL)b->=GV_=P3(@@hZ)Bv9t@aS2GjE!rIUq{2j>lE(cJ~C0-$$50; zoM_dvrgz(~qVn(*r1`sk^wV_DWaJ>-u*-ZC9HFc1|Ert*LyuF#(jORUw$^W>+-jwa zF)7yD5n%}rpxT=zkQ~Hu`0?t+Sf9HzmI$jii(A12%}KJQo4{Af9l2Y7ouX>^ zCg1GyB!9AL!$bcA>TyFT<`|=DSsRuA01PxIz_sw-PQKJtmxQDt!H zGXeX5Usat>`w-pkL-(hdIsKS3 zI7yh0{!eW|yxkA^5o5^1vkD!M#{G-OyyF%>@X7}%`U@D@4TS|Xd8~{RoBU$Z`B+AT zt1Ko9rE32TlZ+~Jx)n8!75|+~@3!p21WzRxw4lNGyI3Uk8yfZEx?-%Ijx@4Z0G}+s zzMS-zz>fcd0*wQ^f$}N0N}5ivub?+s#y1@4yu5=>EETRiJO{x5_z~-Oj%?9=*;XPI z#lE+whYJ{>h*G0%M?ng^p8QP`eGehOgYvG`E%|8cQ!@sI@rdJ>Qp&60S?&b)95Shx zqr&MVWuj@_)i=l#?MY+&)mKz`Es!wQ75nlAr0%49a?sJ)wxH=pJ$E19R`=_MZO#fr zp-2!&60i{a4NK?>*pP-UYxtTD2qE4<8f#`j-ok2^{b={kYCQ<(O4#L(r786x+(Scf z%Hi2v?8{;=UvhHdYJ!x(vg)dW+zO}ml;B%v)3hSMI?!ny*iYWkUW|RcD-`bUEo&aWA z={j3NUUvD?&+#e!G0PJmU1OEM3-C_Rf)%+NT}d9~0zb>S7P?UT-vWJan9U5j7aABI z^4L~JMtsMqpify=8#V8kAOBvQvON*1-Nk^PgF$5;gb7zC08JNeezSS)?dq*xXi_!E zE1ubJP+Y@quQ0FowD*RqQ`!Vtfu^zdn5b-6kO9#|<~KA)XZR|`#dO%i%=3)5Fy1PZ zSz#pfN8{{u{N!fdo92~|?h~t1suYdPu4-Mb4zg-++OIT|bX%U{;^+p&LpcaQWAKmu zFi1edi`Svr43NXZ@u7F?tUI*fE-!)*UNhD(*Ah*z03=`Cfg#9?A@l`(PyU&J0Z0*# z4<&zh{+q|$UMVb%RNv5%boTk_QTKPA6^KNv7hAM2{7F(Hum5xwp{pyfQ#Q=MExMt z0-j`Hs-@Le^(yF>7On89cVJ3kvJ`nsyk3C@u|>0yow4AW-*RAvb_&b8bxBW(y{pb` zz1tO{JyNK_&rq~YE=%)~rTo73 zQ1&0(moUYYlRC&GLG2)IYkFYp!_Ay1o=6~8Q(SNUDboYGGoGm}uP-Tw^kA>%g|--= zc;WqzC&aYhOJU*Uunhc#;G6ia)!{txx39Y1fL5Xr`7P~F$&?nK9@B}G6iii9c+-I% zRlBUkt>>vNDju*FEb ztSa}!4PMNl;-KG-$nJg0qa5VdWo_JZI=Ru93Y5oON|J8ob$#-E-sb1|z`1$J`iG0A ze0#BLT6=dd`y6iREN$=YL&+lygLKJpB;|qD9mYT&h&%^+P%Pt(Ii}R(@whzj;zD%e zJ9>S9nusr6Jpk}p0}K0zl;;_|y+`}sw^lt-FD}Oa`O0D*RHglY<+i^Cxc_aRMB&|i zWPht~1n|;9F5slvKUuAY4<8;OhGPs-1K(-U*O-B@fO)meiy_4u)pZO7LAjSKC)}1RpoNkm0NbmhuvcR231%bFe z|M1eA8ou;5$@Tpwxsf1Xa=<@?1Ic~;ICwc6HEyJcmK{_&jGie;sSQJ{Y7T@ ze=U?4J~>4V9PEV0cY^+Hh2%6fS{7Vy+78VH-Zm|S@$cwQr^(U2!?9<$F)4fh^JxDA Da{n=i delta 7286 zcmZ9Rbx;&g+xB4z>F$ygmhO_26p&m(y1RR+rCVz0?rv5Zl$K@z5d=iKq`O}q=lSM) z=RGrL=679l&OImoyY5d_NF9|(NkYhqG^nA;Z}m{KPTkMk5utrnlua-*4)nYlvfFu7 z1O!1S4JSLqpGcQHNbQ<;+&UOf-_^htF2Bf*jaj6Q9)f}HB$FXU`3~8_tNE=X>Knep zKkOORSCMV4_BA?X_C26DpQ;olBB#T;pZ9Su!C&rmZ!1rPT5?t$)6sUo*SR8V0U0*| zYj4*?zz=H;y{M*?&#hPnr|np;Q`cp9$#_*plho)TaiOSuG03X+EN_x!rAg+_Ety=E zcIN1ttTL1PlKVcG6O=JZU~z>sD+VNk2!t8l*g%D&F6hC^8?y?qBzV>*S2NiYnM4m~TsfA`!RX(J$3vCf8JWP{Z; z;(ZYb3G8PTw_Pglk6i(7#wUMXsjc;4^NdyPV|^G%uW0P-J+G~bb?KODx0$9CDKDUg zIEKsD$a0*BGS`_8@yV(c3GMm~Il}ocTC>Ct$;(pqt(g14l^@fts}{H|=Hy=z@+{KG zBaXQ7vG`<;sDTU=}u0-#;4;u-}!-E-xn)m%Oh zl@@gx?`hFi2O^?*bly_OT>dBn^5O?)piMrHQ**R$S9b1hB^#Rfk1OH@_LZAE&A<~% zeHBzvXUhiDx2~Kn>5SGN2Xs3k?w!MGU+0Tj?J&(Y^VIB-1r3$_2#h1n-bm`Zi`~si zD4;ffMpo=vhO&f1o+yr1JV82aHUme_De_SHG>smx@= zoBVhHhx)DCmxLSBj0EX`u-W?O5g$wO zXi~2L(zUs>K=L5k=8RJrY{}U(UQXa(Ybw_Ck24GK6X9d3#^F`z1@^|74Mk8rWRvu^ zEWu>GIa1g2k>U&74W&i2y1m*=bwr@Mdc8AYEw*4SWZqcZ@n<=gsv_%~g{!H#RR>?A z0ZP%JL`mD*k_5^1>51R%7vb8hF`O}teLhvUZ=Q=c8+?^VmN(9W9fMXi*_mtf9jY3G zM$HK<2Q2f&Fnm^oar$<|=T4DgA#8u_m*sy72FsZkGjdRyg5j5mWB@kCq~WVd9*fr5 zHu%9{+dxAd*J^nMJHhu`>4ET*Ncsr>*&{!F2af~T*ud2+it8PCq|QCsgURhUhqcRl zNGOr)nV-;N>%uzA$d~QG^*H^(A;*c|-OGm3=2LIU`)TqUcESG3{_B-xh#&6|X1`KP zh<8q*lgY5?&sJvLDGrh_7$=KNT@<@^2I;QT4InK?P;cFi*&K5mcg!HO~8YpMH(O?QL z0DV3W=N;vSSBpSI@0T^*5^LR?HFt_K^2i>qc`-J6$KQbh>Rol(x);+IXO|HQ#5V^{B z@2EaTv{qzA9It!(P7F_ZsD3>!mNY^9{E{<5D2Xh(4Y?yqTGQ>ANmdIpHFmnlW@8p& z+yG9{K09Wu>Gyp++b$;&vdfOL9n|n~X^)59*NM>1s8I7keA<)2RM9AI79NWehHGz= zV~ubY>AnY2Ifz^{d{8~2FL1YUpz(ua;syeNzIrov!yG2G{TZaW46`inyHe&JIv~LQgLJtceOt#L#q@gt2 zb)8%^@-7n9B)L&2y{h=+oZvFyd*MXax-kCU7u=DsJLF%lyK@F#Wn!E{w1=koZy&hS zm64E%5D?JN5D*Zg5%MrQq!A*?^UcoG9c8ein)p+9o<$GalLvMxT`U6{e{?Wr9PU*E;$~#a?a1K9yzKAOe2YR<9pF_3K&} z7Hk{>k7@!NT=xANT!1GKYiDQY6zQ*987|hn+qEwqy%&MtD;G6!Wm(L-61*!nDGIsl zLVoaU`1uTEOSxohw;eC-+&6;xHhL|HQ8%~^0 zQw$W7*_j6U>Ll4j`QkWW6c7~eiwO-@oQv%P(|7nIcB5=@5^>$oGa@U~$G2JxvoY*g zFFImth>h{KA~3j8$m3-A3o;a6W@)*gyYBcmp5&0X46<^Dgj>YtVtk3y8)Pb4$s2}fJBnEdcP$LZ9 z!6HXqflI`M-fNL>Om@qyV-h*1y*H47CA39UW^sXh{iF56UHD9hwy4kDn)uT&4(aT- z1tTEKGO$m}p;YM2_**#1ZG4Gci$ryt=;x|ndAZEY{xdwUuPL?!cPo<_N*2IeEV!z3 z(r|hVFv178LRjpamGj&9;|z1GG0DdC#r8S4quYp1qQh{VEi3xHQ~U^a4QDkoR6Yf0 zxYpR|tu;4Hef5chQi#9)C#=cM=py0jGbQ8)H!;6a@yTpWiC=e1=TPf}?=D8Un^%nQ z?84@?wRm_xo-8dA>H&Q)zpdUU>#lKnJld@l&q6R;l6A8&0esajPee&b8La;9kMhpU11h&o$_B_)9eVO< za0ue-!fX=x((<{1wnT{j_&`a)Q^xJDq@2-fyWgA+inr4Q zff}YMK)3duLZ6PU_1z8GrYT!6a*5XsUG&L|BE58|cMK+-?;<%u2ulponTVeOE1`YY zAflZ~0$=A*Mb&GAQ=R!pZSBc3Ib%e4K z^0gG+^)~pUUH`1?uu0fz)m(rayIZL#m>6Zid-;I9AN6^4es0=_eFd4>c_=hkC3G-0V39IsCC!KtND#hN=bh!jF8Zy8pku#%ez zV~9!3Lo6Qf^XiY?Z#_~uRY)F?s_4SL6CSTTtP8jNp$+6a(Cp8bhmNQw;aeb2{jdsk zkwZnEiX9f#)30(Slk`{m_5s;}fMG zHX;?q9IuaAq&&ruTWXN;0nN-xMy;tJMDQ80<4LG_I-;%GuO0GRe7e+}SQG1aXCnDc zsYlU<49*6D=Eqz&eUwc>CYv&KzJh72+1=z92(;(C01W2lo0>uM>nik?4*F6fZ73Gc7Sb!X@6 zGR=Yf@{QH^ks$Z}Y4>(gWd(CwAiwNy{PeO)%~e*g=cb| zs%5L5b^Tr1z3E>n&%L@8*11MUGPHa1UDWT`0+3pFE!YHHitrw)$4oXGWFj1~@h3SP z?AbI*SprJNgwBNUzAq6rn-k0Kp+vao+>!KZZ!h0^r$J7BzfIQfc0i3d(Y(uf0w#jSj#?Fln7^*xIOtq5AkzcJu53aBqhN;hJ$y@-Qed{*pNv_A{4 zD|&=^@zw92NvO0mF1=yQ(YC|h&(m!o)PL37ZhdJEO}MthqF!|w)-l-iF zeG}7nT?2V^`ZW%F%6wM}X6~ae8j{FCW|f>+)3D=OEjdSgt{+P~*hrYEp3}v;j);JDUH`JV*7_8|_+220W9;vrhbx zf@>`+`3+tM=FNuJ7k ze_KB)E!wWr+Mud*ah#*DT9${SYj$2;>$~mH=*F$O{(LD_DqY)Ki?eG_C*4h93`>}P zg%l-U;W%ltgqr*qPf-TjwS9{cHxED}XNhX)DQ}QR3 zBnXu~-{w9V(XH60)jK_uDYQlG@4q^ZjGB;R@z&1El(mB*Z_v_gv?cndfscV{5L#1q z3Q7mNSp7INBr}=MxeMZk6L!?-HS*BChob)Jnn%FN>)4+>+Ei1IG!L-1<>zsaDT-Ik z{0;Gq>g8ivciO+g%%vacnl69Nz>>dbk3ky7VM_(ZjXY`JnKa?iga@OWEYWp-f%7xa zP`(eUeFRWfQTnMruH1_yBTg~HL_28Dv-mAl_%TxN8EZBwk3i4(Uy4o>wV%TNRGETug(4qSoZF4?aFrUcPMXACoVn0b@xQ4oLB`1Jrf5^d?EQWK?OOA z_hN``vs(g+_$xl1hP0y?%)H(wjF1G~2?cx_`?9^9Y>?Q)TzE}=kesA&)wZ{#HY$_v zwUV*TH$SFXlFlzU=JAYfj82FKg6awhSH>_D1rO24b>iJbVc7gkL3e$IbI_-bhpb6M-#ty@XO)XAdSJTM*grGC;y)QGHrpA7`IZ2~5`y8L4B~qov7Nqa92WZG;YyT*?^A6yj$jU){>)j@>t;d}APa+nypN zrOtmIC!Umsn!K*EX<&ph?(wfb{*nRmE#V!ux=CPp3fnD-48|ArI@S7%T>&psRF0XT zJ67C9xSW(0@$p?q!)1znqZ**d#nI%S;_tiQCd=t6vRbWlXZ-HaKQdMblJ|{rmIW_A zWU|*={#YhDXOlQqKpzZ$KMBn-MJWoz3au5Mhx4xaz^iqLb2)h@P#kt zH`jbZhGh3Z6(fXhsqPxORaHzhLI|q8$OjxFDNB)5y|>Ritbe1>Ti*&Um&GMu#`t7Lv*aNLW~ro& z|K&wJf>t5F>&(92xpooDVwmC-nta0-k&lE{#VOt)Xhx2$!4Nrt^>u6HlkwXp5x;?n z!CC-c=MTe>Mb#TtL9suOVICoY?@NMYZkS%|-706rOvJAIbn=r&($+$IL>R^MPQAvY)JE%4F7ZCz`7r74jAmssbkL{p&9|fy08@_-aO*(L}aky z{BVyEnDz}!n6(Hdta8s4N*<5;)d2qiXPBib#{>G-v!7I;JHS;OIsHS2H?1^3N&Y#h z2+cMGtJI?xLJWnu6}9xo`$Jrv<-pqr#Mg84yOn!z4fphSBfRSM!L@1pVb!#E0EH<5 z&bIhmwte|dbOOlsEcqpNCpqXHEZjLHFi_7xzHOFz#u4?h8+zPd+rba*jcG#&#H@MJ z0}Czu?mjpe__T*D-dmUul-i;`OEViED0zxa|n@RhX@di|?YlCK_2 zff>E8gsdoU@%{L*Gb?!L)g3s)j68DK3QFh5B(en+FAOl19;emqY8r~Sxe-^l6}a_7 zK+Qtph9Z88H;mfb>J(DF>92tP&qe2Q)oe9ng$ERnvcjOgJQ!K9UVE8o@QW4XKN5n%K;?&32~GziXM z1LW1Ui>o@haGCD34&o^|NP%%mQ22z%tXQEgBCf2^7)0UvoEz3j ze!72imd@61MO6FYfd|cgqbE21W+h){?=HmR=j)9VuwUPYibsj|t)dqT+5`5j*sw6r zyuNXxd+?O;_+*%;s>5&cYNspW+R#IcgWtlp4ZT*u6f2Ld6nEJ5Z*NpZiP2TbOCbW> zIu?E4r%MqxewwHGc{PU{_<^*&Gj*v~9WUewAx~0@l2Hq0+BZxgH_=x0`OPgoaYS0^ zBJQ8u#I9M#on-~SjK0vgQ+K>hQu^W>hQx0d zZd>@Ij-(p+KmARg{LPvkn)bzl$sxFKbJ;-borZT4ZFK$qc)Pj`y$uQ~Cvt;g(JYI& zqPPRU5;vaQ^zQM@E%02$0JOchzn^d#3D&%ka>b*EN%xubp6neZx#rk@w?|pSYl6-3 zR@nVM8AoL}*fKvw{|e4{J-qUIPHo2L8`ys?I>hWuOI?_Lr1%rztdiL0k(_*&`MfOF)pEhJy;7DAXA! zRJ?$t;2Qq?UtayJ3eMwm0W{&Mp_AL&s$!sZv)Ze=Tk1PoNLHkOPNJtP%%oVK%+g!{? zke*2M$=+nYXND(gUYqObH)&fKHD%^Dqt`MxG6}bgvlB-E?p4zI&4=fn=0o@|Gg}E< zQ5~uAmOUBEiA1S=i5AB1t!&tAj-?KA<@ls*52fJFPKjXqI%V5sUJI1io4&}93yA&le{o{ zQg;cvn#a9hAa%q!%3HzN^Y*^EqNS$pXi&rISN*d;Mdcz`a6Hx3ynAv45e8Iwjrj0dd}}swB95L2Q95^7~=(AtI;CvHn)XgN95{ z0LHxjS`5&y6Fh)c?|%rHM1iXM5JR~qSpmv^|Fy&bQi1>Q_oN&kJmeoLO{oEx!vA6K zlsv#B?jHiDzPbsLK={bTkzY>;F9s0)kEIf7Ya=;X@my@u0CY z6adTgf4jKhsL<~-+<^C4|CLgR^iVgr2_hH@hdTn=ivHU4(2F4wsLw0~Agu+u+C>lj zJj)Gm>iUbE|8B-KM}g=8m723bRDf2_StC+Fap!p{|MQi9LsJC&zmx3$pX4t&Kn>^F KPzU?}WBw1fHw(7_ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index b6517bb1..d2c45a4b 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-4.8-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.4-all.zip diff --git a/settings.gradle b/settings.gradle index 2aa63da0..be483c09 100644 --- a/settings.gradle +++ b/settings.gradle @@ -17,4 +17,4 @@ rootProject.name = 'radar-commons' include ':radar-commons-testing' - +enableFeaturePreview('STABLE_PUBLISHING') From 78286f84bb9d6a031b2800c4a3c71125ec5eb8f6 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 26 Jun 2018 15:31:18 +0200 Subject: [PATCH 09/16] Fixed default access warnings --- .../java/org/radarcns/producer/rest/RestSender.java | 10 +++++----- .../org/radarcns/producer/rest/TopicRequestBody.java | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/radarcns/producer/rest/RestSender.java b/src/main/java/org/radarcns/producer/rest/RestSender.java index 53d96ea9..5c47bc9a 100644 --- a/src/main/java/org/radarcns/producer/rest/RestSender.java +++ b/src/main/java/org/radarcns/producer/rest/RestSender.java @@ -276,11 +276,11 @@ public RestSender build() { } } - public static final class RequestProperties { - public final String acceptType; - public final MediaType contentType; - public final boolean useCompression; - public final Headers headers; + static final class RequestProperties { + final String acceptType; + final MediaType contentType; + final boolean useCompression; + final Headers headers; RequestProperties(String acceptType, MediaType contentType, boolean useCompression, Headers headers) { diff --git a/src/main/java/org/radarcns/producer/rest/TopicRequestBody.java b/src/main/java/org/radarcns/producer/rest/TopicRequestBody.java index e5836324..016260f3 100644 --- a/src/main/java/org/radarcns/producer/rest/TopicRequestBody.java +++ b/src/main/java/org/radarcns/producer/rest/TopicRequestBody.java @@ -31,7 +31,7 @@ class TopicRequestBody extends RequestBody { protected final TopicRequestData data; private final MediaType mediaType; - TopicRequestBody(TopicRequestData requestData, MediaType mediaType) throws IOException { + TopicRequestBody(TopicRequestData requestData, MediaType mediaType) { this.data = requestData; this.mediaType = mediaType; } @@ -48,14 +48,14 @@ public void writeTo(BufferedSink sink) throws IOException { } } - String content() throws IOException { + private String content() throws IOException { try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { data.writeToStream(out); return out.toString(); } } - public static String topicRequestContent(Request request) throws IOException { + static String topicRequestContent(Request request) throws IOException { TopicRequestBody body = (TopicRequestBody) request.body(); if (body == null) { return null; From a264e47193d38f39b2415342e8f9f2afebe34273 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 26 Jun 2018 16:38:45 +0200 Subject: [PATCH 10/16] Refactor module setup --- .gitignore | 2 +- .travis.yml | 3 +- build.gradle | 197 +----------------- radar-commons-server/build.gradle | 153 ++++++++++++++ .../org/radarcns/config/AvroTopicConfig.java | 0 .../radarcns/config/ServerConfigMixin.java | 21 ++ .../org/radarcns/config/YamlConfigLoader.java | 1 + .../producer/direct/DirectSender.java | 12 +- .../collector/AggregateListCollector.java | 0 .../collector/NumericAggregateCollector.java | 0 .../stream/collector/RecordCollector.java | 0 .../collector/UniformSamplingReservoir.java | 0 .../org/radarcns/util/RollingTimeAverage.java | 0 .../util/serde/AbstractKafkaAvroSerde.java | 0 .../util/serde/KafkaAvroSerializer.java | 0 .../org/radarcns/config/ServerConfigTest.java | 0 .../collector/AggregateListCollectorTest.java | 0 .../NumericAggregateCollectorTest.java | 3 +- .../UniformSamplingReservoirTest.java | 0 .../util/serde/KafkaAvroSerializerTest.java | 0 radar-commons-testing/build.gradle | 8 +- radar-commons/build.gradle | 175 ++++++++++++++++ .../org/radarcns/config/ServerConfig.java | 5 - .../java/org/radarcns/data/AvroDecoder.java | 0 .../java/org/radarcns/data/AvroEncoder.java | 0 .../org/radarcns/data/AvroRecordData.java | 0 .../org/radarcns/data/AvroRecordWriter.java | 0 .../main/java/org/radarcns/data/Record.java | 0 .../java/org/radarcns/data/RecordData.java | 0 .../radarcns/data/SpecificRecordDecoder.java | 0 .../radarcns/data/SpecificRecordEncoder.java | 0 .../java/org/radarcns/data/StringEncoder.java | 0 .../main/java/org/radarcns/data/TimedInt.java | 0 .../producer/AuthenticationException.java | 0 .../radarcns/producer/BatchedKafkaSender.java | 0 .../org/radarcns/producer/KafkaSender.java | 0 .../radarcns/producer/KafkaTopicSender.java | 0 .../producer/rest/ConnectionState.java | 0 .../producer/rest/GzipTopicRequestBody.java | 0 .../producer/rest/ManagedConnectionPool.java | 0 .../producer/rest/ParsedSchemaMetadata.java | 0 .../radarcns/producer/rest/RestClient.java | 0 .../radarcns/producer/rest/RestException.java | 0 .../radarcns/producer/rest/RestSender.java | 0 .../producer/rest/RestTopicSender.java | 0 .../producer/rest/SchemaRetriever.java | 0 .../producer/rest/TopicRequestBody.java | 0 .../producer/rest/TopicRequestData.java | 0 .../java/org/radarcns/topic/AvroTopic.java | 0 .../java/org/radarcns/topic/KafkaTopic.java | 0 .../java/org/radarcns/topic/SensorTopic.java | 0 .../java/org/radarcns/util/Serialization.java | 0 .../main/java/org/radarcns/util/Strings.java | 0 .../data/SpecificRecordDecorderTest.java | 0 .../data/SpecificRecordEncoderTest.java | 13 +- .../org/radarcns/data/StringEncoderTest.java | 0 .../producer/rest/ConnectionStateTest.java | 0 .../producer/rest/RestClientTest.java | 0 .../producer/rest/RestSenderTest.java | 0 .../producer/rest/SchemaRetrieverTest.java | 0 .../org/radarcns/topic/KafkaTopicTest.java | 0 .../org/radarcns/topic/SensorTopicTest.java | 1 - .../org/radarcns/util/SerializationTest.java | 0 .../src}/test/resources/integration.yml | 0 .../src}/test/resources/integration_test.csv | 0 settings.gradle | 4 +- 66 files changed, 376 insertions(+), 222 deletions(-) create mode 100644 radar-commons-server/build.gradle rename {src => radar-commons-server/src}/main/java/org/radarcns/config/AvroTopicConfig.java (100%) create mode 100644 radar-commons-server/src/main/java/org/radarcns/config/ServerConfigMixin.java rename {src => radar-commons-server/src}/main/java/org/radarcns/config/YamlConfigLoader.java (97%) rename {src => radar-commons-server/src}/main/java/org/radarcns/producer/direct/DirectSender.java (88%) rename {src => radar-commons-server/src}/main/java/org/radarcns/stream/collector/AggregateListCollector.java (100%) rename {src => radar-commons-server/src}/main/java/org/radarcns/stream/collector/NumericAggregateCollector.java (100%) rename {src => radar-commons-server/src}/main/java/org/radarcns/stream/collector/RecordCollector.java (100%) rename {src => radar-commons-server/src}/main/java/org/radarcns/stream/collector/UniformSamplingReservoir.java (100%) rename {src => radar-commons-server/src}/main/java/org/radarcns/util/RollingTimeAverage.java (100%) rename {src => radar-commons-server/src}/main/java/org/radarcns/util/serde/AbstractKafkaAvroSerde.java (100%) rename {src => radar-commons-server/src}/main/java/org/radarcns/util/serde/KafkaAvroSerializer.java (100%) rename {src => radar-commons-server/src}/test/java/org/radarcns/config/ServerConfigTest.java (100%) rename {src => radar-commons-server/src}/test/java/org/radarcns/stream/collector/AggregateListCollectorTest.java (100%) rename {src => radar-commons-server/src}/test/java/org/radarcns/stream/collector/NumericAggregateCollectorTest.java (98%) rename {src => radar-commons-server/src}/test/java/org/radarcns/stream/collector/UniformSamplingReservoirTest.java (100%) rename {src => radar-commons-server/src}/test/java/org/radarcns/util/serde/KafkaAvroSerializerTest.java (100%) create mode 100644 radar-commons/build.gradle rename {src => radar-commons/src}/main/java/org/radarcns/config/ServerConfig.java (97%) rename {src => radar-commons/src}/main/java/org/radarcns/data/AvroDecoder.java (100%) rename {src => radar-commons/src}/main/java/org/radarcns/data/AvroEncoder.java (100%) rename {src => radar-commons/src}/main/java/org/radarcns/data/AvroRecordData.java (100%) rename {src => radar-commons/src}/main/java/org/radarcns/data/AvroRecordWriter.java (100%) rename {src => radar-commons/src}/main/java/org/radarcns/data/Record.java (100%) rename {src => radar-commons/src}/main/java/org/radarcns/data/RecordData.java (100%) rename {src => radar-commons/src}/main/java/org/radarcns/data/SpecificRecordDecoder.java (100%) rename {src => radar-commons/src}/main/java/org/radarcns/data/SpecificRecordEncoder.java (100%) rename {src => radar-commons/src}/main/java/org/radarcns/data/StringEncoder.java (100%) rename {src => radar-commons/src}/main/java/org/radarcns/data/TimedInt.java (100%) rename {src => radar-commons/src}/main/java/org/radarcns/producer/AuthenticationException.java (100%) rename {src => radar-commons/src}/main/java/org/radarcns/producer/BatchedKafkaSender.java (100%) rename {src => radar-commons/src}/main/java/org/radarcns/producer/KafkaSender.java (100%) rename {src => radar-commons/src}/main/java/org/radarcns/producer/KafkaTopicSender.java (100%) rename {src => radar-commons/src}/main/java/org/radarcns/producer/rest/ConnectionState.java (100%) rename {src => radar-commons/src}/main/java/org/radarcns/producer/rest/GzipTopicRequestBody.java (100%) rename {src => radar-commons/src}/main/java/org/radarcns/producer/rest/ManagedConnectionPool.java (100%) rename {src => radar-commons/src}/main/java/org/radarcns/producer/rest/ParsedSchemaMetadata.java (100%) rename {src => radar-commons/src}/main/java/org/radarcns/producer/rest/RestClient.java (100%) rename {src => radar-commons/src}/main/java/org/radarcns/producer/rest/RestException.java (100%) rename {src => radar-commons/src}/main/java/org/radarcns/producer/rest/RestSender.java (100%) rename {src => radar-commons/src}/main/java/org/radarcns/producer/rest/RestTopicSender.java (100%) rename {src => radar-commons/src}/main/java/org/radarcns/producer/rest/SchemaRetriever.java (100%) rename {src => radar-commons/src}/main/java/org/radarcns/producer/rest/TopicRequestBody.java (100%) rename {src => radar-commons/src}/main/java/org/radarcns/producer/rest/TopicRequestData.java (100%) rename {src => radar-commons/src}/main/java/org/radarcns/topic/AvroTopic.java (100%) rename {src => radar-commons/src}/main/java/org/radarcns/topic/KafkaTopic.java (100%) rename {src => radar-commons/src}/main/java/org/radarcns/topic/SensorTopic.java (100%) rename {src => radar-commons/src}/main/java/org/radarcns/util/Serialization.java (100%) rename {src => radar-commons/src}/main/java/org/radarcns/util/Strings.java (100%) rename {src => radar-commons/src}/test/java/org/radarcns/data/SpecificRecordDecorderTest.java (100%) rename {src => radar-commons/src}/test/java/org/radarcns/data/SpecificRecordEncoderTest.java (98%) rename {src => radar-commons/src}/test/java/org/radarcns/data/StringEncoderTest.java (100%) rename {src => radar-commons/src}/test/java/org/radarcns/producer/rest/ConnectionStateTest.java (100%) rename {src => radar-commons/src}/test/java/org/radarcns/producer/rest/RestClientTest.java (100%) rename {src => radar-commons/src}/test/java/org/radarcns/producer/rest/RestSenderTest.java (100%) rename {src => radar-commons/src}/test/java/org/radarcns/producer/rest/SchemaRetrieverTest.java (100%) rename {src => radar-commons/src}/test/java/org/radarcns/topic/KafkaTopicTest.java (100%) rename {src => radar-commons/src}/test/java/org/radarcns/topic/SensorTopicTest.java (98%) rename {src => radar-commons/src}/test/java/org/radarcns/util/SerializationTest.java (100%) rename {src => radar-commons/src}/test/resources/integration.yml (100%) rename {src => radar-commons/src}/test/resources/integration_test.csv (100%) diff --git a/.gitignore b/.gitignore index ba935722..0bcbc7f8 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,6 @@ build/ /src/avroSources/java/ # Generated compile files -/out/ +out/ backend.log local.properties diff --git a/.travis.yml b/.travis.yml index 31c3c1af..8a032264 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,8 +22,7 @@ deploy: api_key: ${GH_TOKEN} file_glob: true file: - - "build/libs/*.jar" - - "testing/build/libs/*.jar" + - "*/build/libs/*.jar" skip_cleanup: true on: tags: true diff --git a/build.gradle b/build.gradle index 98264100..d5874cc5 100644 --- a/build.gradle +++ b/build.gradle @@ -20,7 +20,7 @@ plugins { id 'com.jfrog.artifactory' version '4.5.4' } -allprojects { +subprojects { // Apply the plugins apply plugin: 'java' apply plugin: 'java-library' @@ -41,16 +41,16 @@ allprojects { ext.githubRepoName = 'RADAR-CNS/RADAR-Commons' ext.slf4jVersion = '1.7.25' - ext.kafkaVersion = '0.11.0.1' + ext.kafkaVersion = '1.1.1-cp1' ext.avroVersion = '1.8.2' - ext.confluentVersion = '3.3.1' - ext.jacksonVersion = '2.9.3' - ext.okhttpVersion = '3.9.1' + ext.confluentVersion = '4.1.1' + ext.jacksonVersion = '2.9.6' + ext.okhttpVersion = '3.10.0' ext.junitVersion = '4.12' ext.mockitoVersion = '2.13.0' ext.hamcrestVersion = '1.3' ext.codacyVersion = '2.0.1' - ext.radarSchemasVersion = '0.2.3' + ext.radarSchemasVersion = '0.3.4' ext.orgJsonVersion = '20170516' ext.githubUrl = 'https://github.com/' + githubRepoName + '.git' @@ -178,191 +178,6 @@ allprojects { } } -targetCompatibility = '1.7' -sourceCompatibility = '1.7' - -ext.description = 'RADAR Common utilities library containing streaming features and utils.' - -//---------------------------------------------------------------------------// -// Sources and classpath configurations // -//---------------------------------------------------------------------------// - -configurations { - codacy -} - -configurations.compile { - resolutionStrategy.cacheChangingModulesFor 0, 'SECONDS' -} - -// In this section you declare where to find the dependencies of your project -repositories { - maven { url 'https://jitpack.io' } - maven { url 'https://oss.jfrog.org/artifactory/oss-snapshot-local' } -} - -// In this section you declare the dependencies for your production and test code -dependencies { - api (group: 'org.apache.avro', name: 'avro', version: avroVersion) { - exclude group: 'org.xerial.snappy', module: 'snappy-java' - exclude group: 'com.thoughtworks.paranamer', module: 'paranamer' - exclude group: 'org.apache.commons', module: 'commons-compress' - exclude group: 'org.tukaani', module: 'xz' - } - - // to implement producers and consumers - api group: 'org.apache.kafka', name: 'kafka-clients', version: kafkaVersion - api group: 'com.squareup.okhttp3', name: 'okhttp', version: okhttpVersion - - // For POJO classes and ConfigLoader - implementation group: 'com.fasterxml.jackson.core' , name: 'jackson-databind' , version: jacksonVersion - implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: jacksonVersion - implementation group: 'org.json', name: 'json', version: orgJsonVersion - - // The production code uses the SLF4J logging API at compile time - implementation group: 'org.slf4j', name:'slf4j-api', version: slf4jVersion - - // Direct producer uses KafkaAvroSerializer if initialized - testImplementation (group: 'io.confluent', name: 'kafka-avro-serializer', version: confluentVersion) { - exclude group: 'com.101tec' - exclude group: 'org.slf4j', module: 'slf4j-log4j12' - } - testImplementation group: 'org.radarcns', name: 'radar-schemas-commons', version: radarSchemasVersion - testImplementation group: 'junit', name: 'junit', version: junitVersion - testImplementation group: 'org.mockito', name: 'mockito-core', version: mockitoVersion - testImplementation group: 'org.hamcrest', name: 'hamcrest-all', version: hamcrestVersion - testImplementation group: 'com.squareup.okhttp3', name: 'mockwebserver', version: okhttpVersion - testRuntime group: 'org.slf4j', name: 'slf4j-simple', version: slf4jVersion - - codacy group: 'com.github.codacy', name: 'codacy-coverage-reporter', version: codacyVersion -} - -//---------------------------------------------------------------------------// -// Testing // -//---------------------------------------------------------------------------// -checkstyle { - // ignore tests - sourceSets = [sourceSets.main] -} - -pmd { - // ignore tests - sourceSets = [sourceSets.main] -} - -jacocoTestReport { - reports { - xml.enabled true - csv.enabled false - html.enabled false - } -} - -task downloadDependencies(type: Exec) { - configurations.testRuntime.files - configurations.codacy.files - configurations.jacocoAnt.files - commandLine 'echo', 'Downloaded all dependencies' -} - -task sendCoverageToCodacy(type: JavaExec, dependsOn: jacocoTestReport) { - main = 'com.codacy.CodacyCoverageReporter' - classpath = configurations.codacy - args = ['-l', 'Java', '-r', "${buildDir}/reports/jacoco/test/jacocoTestReport.xml"] -} - -//---------------------------------------------------------------------------// -// Build system metadata // -//---------------------------------------------------------------------------// - -ext.sharedManifest = manifest { - attributes("Implementation-Title": rootProject.name, - "Implementation-Version": version) -} - -jar { - manifest.from sharedManifest -} -// custom tasks for creating source/javadoc jars -task sourcesJar(type: Jar, dependsOn: classes) { - classifier = 'sources' - from sourceSets.main.allSource - manifest.from sharedManifest -} - -task javadocJar(type: Jar, dependsOn: javadoc) { - classifier = 'javadoc' - from javadoc.destinationDir - manifest.from sharedManifest -} - -// add javadoc/source jar tasks as artifacts -artifacts { - archives javadocJar - archives sourcesJar -} - -publishing { - publications { - RadarCommonsPublication(MavenPublication) { - from components.java - artifact sourcesJar - artifact javadocJar - groupId project.group - artifactId rootProject.name - version project.version - pom.withXml { - def root = asNode() - root.appendNode('description', description) - root.appendNode('name', rootProject.name) - root.appendNode('url', githubUrl) - root.children().last() + pomConfig - } - } - } -} - -bintray { - user project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER') - key project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_API_KEY') - override false - publications 'RadarCommonsPublication' - pkg { - repo = project.group - name = rootProject.name - userOrg = 'radar-cns' - desc = description - licenses = ['Apache-2.0'] - websiteUrl = website - issueTrackerUrl = issueUrl - vcsUrl = githubUrl - githubRepo = githubRepoName - githubReleaseNotesFile = 'README.md' - version { - name = project.version - desc = description - vcsTag = System.getenv('TRAVIS_TAG') - released = new Date() - } - } -} - -artifactory { - contextUrl = 'https://oss.jfrog.org/artifactory' - publish { - repository { - repoKey = 'oss-snapshot-local' - username = project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER') - password = project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_API_KEY') - maven = true - } - } -} - -artifactoryPublish { - publications('RadarCommonsPublication') -} - wrapper { gradleVersion '4.8' } diff --git a/radar-commons-server/build.gradle b/radar-commons-server/build.gradle new file mode 100644 index 00000000..c0516fe8 --- /dev/null +++ b/radar-commons-server/build.gradle @@ -0,0 +1,153 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +ext.moduleName = 'radar-commons-server' +ext.description = 'RADAR Common testing library mocking code and utilities.' + +targetCompatibility = '1.8' +sourceCompatibility = '1.8' + +repositories { + maven { url 'https://oss.jfrog.org/artifactory/oss-snapshot-local' } +} + +dependencies { + api project(':radar-commons') + + // For POJO classes and ConfigLoader + implementation group: 'com.fasterxml.jackson.core' , name: 'jackson-databind' , version: jacksonVersion + implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: jacksonVersion + + api group: 'org.apache.avro', name: 'avro', version: avroVersion + + implementation group: 'com.fasterxml.jackson.core' , name: 'jackson-databind' , version: jacksonVersion + implementation group: 'org.apache.kafka', name: 'kafka-clients', version: kafkaVersion + + testImplementation group: 'org.mockito', name: 'mockito-core', version: mockitoVersion + // Direct producer uses KafkaAvroSerializer if initialized + testImplementation (group: 'io.confluent', name: 'kafka-avro-serializer', version: confluentVersion) { + exclude group: 'com.101tec' + exclude group: 'org.slf4j', module: 'slf4j-log4j12' + } + testImplementation group: 'org.radarcns', name: 'radar-schemas-commons', version: radarSchemasVersion + // Direct producer uses KafkaAvroSerializer if initialized + testImplementation group: 'junit', name: 'junit', version: junitVersion + testRuntime group: 'org.slf4j', name: 'slf4j-simple', version: slf4jVersion +} + +checkstyle { + // ignore tests + sourceSets = [sourceSets.main] + ignoreFailures = false +} + +pmd { + // ignore tests + sourceSets = [sourceSets.main] + ignoreFailures = false +} + +ext.sharedManifest = manifest { + attributes("Implementation-Title": moduleName, + "Implementation-Version": version) +} + +jar { + baseName moduleName + manifest.from sharedManifest +} + +// custom tasks for creating source/javadoc jars +task sourcesJar(type: Jar, dependsOn: classes) { + baseName moduleName + classifier = 'sources' + from sourceSets.main.allSource + manifest.from sharedManifest +} + +task javadocJar(type: Jar, dependsOn: javadoc) { + baseName moduleName + classifier = 'javadoc' + from javadoc.destinationDir + manifest.from sharedManifest +} + +// add javadoc/source jar tasks as artifacts +artifacts { + archives sourcesJar, javadocJar +} + +publishing { + publications { + RadarCommonsServerPublication(MavenPublication) { + from components.java + artifact sourcesJar + artifact javadocJar + groupId project.group + artifactId moduleName + version project.version + pom.withXml { + def root = asNode() + root.appendNode('description', description) + root.appendNode('name', moduleName) + root.appendNode('url', githubUrl) + root.children().last() + pomConfig + } + } + } +} + +bintray { + user project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER') + key project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_API_KEY') + override false + publications 'RadarCommonsServerPublication' + pkg { + repo = project.group + name = moduleName + userOrg = 'radar-cns' + desc = description + licenses = ['Apache-2.0'] + websiteUrl = website + issueTrackerUrl = issueUrl + vcsUrl = githubUrl + githubRepo = githubRepoName + githubReleaseNotesFile = 'README.md' + version { + name = project.version + desc = description + vcsTag = System.getenv('TRAVIS_TAG') + released = new Date() + } + } +} + + +artifactory { + contextUrl = 'https://oss.jfrog.org/artifactory' + publish { + repository { + repoKey = 'oss-snapshot-local' + username = project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER') + password = project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_API_KEY') + maven = true + } + } +} + +artifactoryPublish { + publications('RadarCommonsServerPublication') +} diff --git a/src/main/java/org/radarcns/config/AvroTopicConfig.java b/radar-commons-server/src/main/java/org/radarcns/config/AvroTopicConfig.java similarity index 100% rename from src/main/java/org/radarcns/config/AvroTopicConfig.java rename to radar-commons-server/src/main/java/org/radarcns/config/AvroTopicConfig.java diff --git a/radar-commons-server/src/main/java/org/radarcns/config/ServerConfigMixin.java b/radar-commons-server/src/main/java/org/radarcns/config/ServerConfigMixin.java new file mode 100644 index 00000000..baf80340 --- /dev/null +++ b/radar-commons-server/src/main/java/org/radarcns/config/ServerConfigMixin.java @@ -0,0 +1,21 @@ +package org.radarcns.config; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSetter; + +/** Mixin class to load ServerConfig files with. */ +public abstract class ServerConfigMixin { + public ServerConfigMixin() { + // POJO initializer + } + + /** Proxy host name. Null if not set. */ + @JsonProperty("proxy_host") + public abstract String getProxyHost(); + + @JsonProperty("proxy_port") + public abstract String getProxyPort(); + + @JsonSetter("path") + public abstract void setPath(String path); +} diff --git a/src/main/java/org/radarcns/config/YamlConfigLoader.java b/radar-commons-server/src/main/java/org/radarcns/config/YamlConfigLoader.java similarity index 97% rename from src/main/java/org/radarcns/config/YamlConfigLoader.java rename to radar-commons-server/src/main/java/org/radarcns/config/YamlConfigLoader.java index 6fe9c521..7ec4aacc 100644 --- a/src/main/java/org/radarcns/config/YamlConfigLoader.java +++ b/radar-commons-server/src/main/java/org/radarcns/config/YamlConfigLoader.java @@ -42,6 +42,7 @@ public YamlConfigLoader() { .withIsGetterVisibility(JsonAutoDetect.Visibility.NONE) .withSetterVisibility(JsonAutoDetect.Visibility.NONE) .withCreatorVisibility(JsonAutoDetect.Visibility.NONE)); + mapper.addMixIn(ServerConfig.class, ServerConfigMixin.class); } public T load(File file, Class configClass) throws IOException { diff --git a/src/main/java/org/radarcns/producer/direct/DirectSender.java b/radar-commons-server/src/main/java/org/radarcns/producer/direct/DirectSender.java similarity index 88% rename from src/main/java/org/radarcns/producer/direct/DirectSender.java rename to radar-commons-server/src/main/java/org/radarcns/producer/direct/DirectSender.java index 30757d92..77c98293 100644 --- a/src/main/java/org/radarcns/producer/direct/DirectSender.java +++ b/radar-commons-server/src/main/java/org/radarcns/producer/direct/DirectSender.java @@ -24,7 +24,6 @@ import org.radarcns.producer.KafkaTopicSender; import org.radarcns.topic.AvroTopic; -import java.io.IOException; import java.util.Properties; /** @@ -38,8 +37,7 @@ public DirectSender(Properties properties) { } @Override - public KafkaTopicSender sender(final AvroTopic topic) - throws IOException { + public KafkaTopicSender sender(final AvroTopic topic) { return new DirectTopicSender<>(topic); } @@ -68,13 +66,13 @@ private DirectTopicSender(AvroTopic topic) { } @Override - public void send(K key, V value) throws IOException { + public void send(K key, V value) { producer.send(new ProducerRecord<>(name, key, value)); producer.flush(); } @Override - public void send(RecordData records) throws IOException { + public void send(RecordData records) { for (Record record : records) { producer.send(new ProducerRecord<>(name, record.key, record.value)); } @@ -87,12 +85,12 @@ public void clear() { } @Override - public void flush() throws IOException { + public void flush() { // noop } @Override - public void close() throws IOException { + public void close() { // noop } } diff --git a/src/main/java/org/radarcns/stream/collector/AggregateListCollector.java b/radar-commons-server/src/main/java/org/radarcns/stream/collector/AggregateListCollector.java similarity index 100% rename from src/main/java/org/radarcns/stream/collector/AggregateListCollector.java rename to radar-commons-server/src/main/java/org/radarcns/stream/collector/AggregateListCollector.java diff --git a/src/main/java/org/radarcns/stream/collector/NumericAggregateCollector.java b/radar-commons-server/src/main/java/org/radarcns/stream/collector/NumericAggregateCollector.java similarity index 100% rename from src/main/java/org/radarcns/stream/collector/NumericAggregateCollector.java rename to radar-commons-server/src/main/java/org/radarcns/stream/collector/NumericAggregateCollector.java diff --git a/src/main/java/org/radarcns/stream/collector/RecordCollector.java b/radar-commons-server/src/main/java/org/radarcns/stream/collector/RecordCollector.java similarity index 100% rename from src/main/java/org/radarcns/stream/collector/RecordCollector.java rename to radar-commons-server/src/main/java/org/radarcns/stream/collector/RecordCollector.java diff --git a/src/main/java/org/radarcns/stream/collector/UniformSamplingReservoir.java b/radar-commons-server/src/main/java/org/radarcns/stream/collector/UniformSamplingReservoir.java similarity index 100% rename from src/main/java/org/radarcns/stream/collector/UniformSamplingReservoir.java rename to radar-commons-server/src/main/java/org/radarcns/stream/collector/UniformSamplingReservoir.java diff --git a/src/main/java/org/radarcns/util/RollingTimeAverage.java b/radar-commons-server/src/main/java/org/radarcns/util/RollingTimeAverage.java similarity index 100% rename from src/main/java/org/radarcns/util/RollingTimeAverage.java rename to radar-commons-server/src/main/java/org/radarcns/util/RollingTimeAverage.java diff --git a/src/main/java/org/radarcns/util/serde/AbstractKafkaAvroSerde.java b/radar-commons-server/src/main/java/org/radarcns/util/serde/AbstractKafkaAvroSerde.java similarity index 100% rename from src/main/java/org/radarcns/util/serde/AbstractKafkaAvroSerde.java rename to radar-commons-server/src/main/java/org/radarcns/util/serde/AbstractKafkaAvroSerde.java diff --git a/src/main/java/org/radarcns/util/serde/KafkaAvroSerializer.java b/radar-commons-server/src/main/java/org/radarcns/util/serde/KafkaAvroSerializer.java similarity index 100% rename from src/main/java/org/radarcns/util/serde/KafkaAvroSerializer.java rename to radar-commons-server/src/main/java/org/radarcns/util/serde/KafkaAvroSerializer.java diff --git a/src/test/java/org/radarcns/config/ServerConfigTest.java b/radar-commons-server/src/test/java/org/radarcns/config/ServerConfigTest.java similarity index 100% rename from src/test/java/org/radarcns/config/ServerConfigTest.java rename to radar-commons-server/src/test/java/org/radarcns/config/ServerConfigTest.java diff --git a/src/test/java/org/radarcns/stream/collector/AggregateListCollectorTest.java b/radar-commons-server/src/test/java/org/radarcns/stream/collector/AggregateListCollectorTest.java similarity index 100% rename from src/test/java/org/radarcns/stream/collector/AggregateListCollectorTest.java rename to radar-commons-server/src/test/java/org/radarcns/stream/collector/AggregateListCollectorTest.java diff --git a/src/test/java/org/radarcns/stream/collector/NumericAggregateCollectorTest.java b/radar-commons-server/src/test/java/org/radarcns/stream/collector/NumericAggregateCollectorTest.java similarity index 98% rename from src/test/java/org/radarcns/stream/collector/NumericAggregateCollectorTest.java rename to radar-commons-server/src/test/java/org/radarcns/stream/collector/NumericAggregateCollectorTest.java index 1bc2930a..c5d00a48 100644 --- a/src/test/java/org/radarcns/stream/collector/NumericAggregateCollectorTest.java +++ b/radar-commons-server/src/test/java/org/radarcns/stream/collector/NumericAggregateCollectorTest.java @@ -30,7 +30,6 @@ import java.io.IOException; import java.math.BigDecimal; -import java.util.Arrays; /** * Created by nivethika on 20-12-16. @@ -137,7 +136,7 @@ public void testRecordType() { ApplicationRecordCounts.getClassSchema()); this.valueCollector = new NumericAggregateCollector("recordsCached", ApplicationRecordCounts.getClassSchema()); - this.valueCollector = new NumericAggregateCollector("start", + this.valueCollector = new NumericAggregateCollector("timeStart", AggregateKey.getClassSchema()); } diff --git a/src/test/java/org/radarcns/stream/collector/UniformSamplingReservoirTest.java b/radar-commons-server/src/test/java/org/radarcns/stream/collector/UniformSamplingReservoirTest.java similarity index 100% rename from src/test/java/org/radarcns/stream/collector/UniformSamplingReservoirTest.java rename to radar-commons-server/src/test/java/org/radarcns/stream/collector/UniformSamplingReservoirTest.java diff --git a/src/test/java/org/radarcns/util/serde/KafkaAvroSerializerTest.java b/radar-commons-server/src/test/java/org/radarcns/util/serde/KafkaAvroSerializerTest.java similarity index 100% rename from src/test/java/org/radarcns/util/serde/KafkaAvroSerializerTest.java rename to radar-commons-server/src/test/java/org/radarcns/util/serde/KafkaAvroSerializerTest.java diff --git a/radar-commons-testing/build.gradle b/radar-commons-testing/build.gradle index b4395b03..f3634085 100644 --- a/radar-commons-testing/build.gradle +++ b/radar-commons-testing/build.gradle @@ -30,19 +30,21 @@ run { ext.testingName = 'radar-commons-testing' ext.description = 'RADAR Common testing library mocking code and utilities.' -targetCompatibility = '1.7' -sourceCompatibility = '1.7' +targetCompatibility = '1.8' +sourceCompatibility = '1.8' repositories { maven { url 'https://oss.jfrog.org/artifactory/oss-snapshot-local' } } dependencies { - api rootProject + api project(':radar-commons') + api project(':radar-commons-server') api group: 'org.apache.avro', name: 'avro', version: avroVersion api group: 'org.radarcns', name: 'radar-schemas-commons', version: radarSchemasVersion implementation group: 'com.fasterxml.jackson.core' , name: 'jackson-databind' , version: jacksonVersion + implementation group: 'org.apache.kafka', name: 'kafka-clients', version: kafkaVersion runtimeOnly group: 'org.slf4j', name: 'slf4j-simple', version: slf4jVersion diff --git a/radar-commons/build.gradle b/radar-commons/build.gradle new file mode 100644 index 00000000..28721de6 --- /dev/null +++ b/radar-commons/build.gradle @@ -0,0 +1,175 @@ +targetCompatibility = '1.7' +sourceCompatibility = '1.7' + +ext.description = 'RADAR Common utilities library.' + +//---------------------------------------------------------------------------// +// Sources and classpath configurations // +//---------------------------------------------------------------------------// + +configurations { + codacy +} + +configurations.compile { + resolutionStrategy.cacheChangingModulesFor 0, 'SECONDS' +} + +// In this section you declare where to find the dependencies of your project +repositories { + maven { url 'https://jitpack.io' } + maven { url 'https://oss.jfrog.org/artifactory/oss-snapshot-local' } +} + +// In this section you declare the dependencies for your production and test code +dependencies { + api (group: 'org.apache.avro', name: 'avro', version: avroVersion) { + exclude group: 'org.xerial.snappy', module: 'snappy-java' + exclude group: 'com.thoughtworks.paranamer', module: 'paranamer' + exclude group: 'org.apache.commons', module: 'commons-compress' + exclude group: 'org.tukaani', module: 'xz' + } + + // to implement producers and consumers + api group: 'com.squareup.okhttp3', name: 'okhttp', version: okhttpVersion + + implementation group: 'org.json', name: 'json', version: orgJsonVersion + + // The production code uses the SLF4J logging API at compile time + implementation group: 'org.slf4j', name:'slf4j-api', version: slf4jVersion + + testImplementation group: 'com.fasterxml.jackson.core' , name: 'jackson-databind' , version: jacksonVersion + testImplementation group: 'org.radarcns', name: 'radar-schemas-commons', version: radarSchemasVersion + testImplementation group: 'junit', name: 'junit', version: junitVersion + testImplementation group: 'org.mockito', name: 'mockito-core', version: mockitoVersion + testImplementation group: 'com.squareup.okhttp3', name: 'mockwebserver', version: okhttpVersion + testRuntime group: 'org.slf4j', name: 'slf4j-simple', version: slf4jVersion + + codacy group: 'com.github.codacy', name: 'codacy-coverage-reporter', version: codacyVersion +} + +//---------------------------------------------------------------------------// +// Testing // +//---------------------------------------------------------------------------// +checkstyle { + // ignore tests + sourceSets = [sourceSets.main] +} + +pmd { + // ignore tests + sourceSets = [sourceSets.main] +} + +jacocoTestReport { + reports { + xml.enabled true + csv.enabled false + html.enabled false + } +} + +task downloadDependencies(type: Exec) { + configurations.testRuntime.files + configurations.codacy.files + configurations.jacocoAnt.files + commandLine 'echo', 'Downloaded all dependencies' +} + +task sendCoverageToCodacy(type: JavaExec, dependsOn: jacocoTestReport) { + main = 'com.codacy.CodacyCoverageReporter' + classpath = configurations.codacy + args = ['-l', 'Java', '-r', "${buildDir}/reports/jacoco/test/jacocoTestReport.xml"] +} + +//---------------------------------------------------------------------------// +// Build system metadata // +//---------------------------------------------------------------------------// + +ext.sharedManifest = manifest { + attributes("Implementation-Title": rootProject.name, + "Implementation-Version": version) +} + +jar { + manifest.from sharedManifest +} +// custom tasks for creating source/javadoc jars +task sourcesJar(type: Jar, dependsOn: classes) { + classifier = 'sources' + from sourceSets.main.allSource + manifest.from sharedManifest +} + +task javadocJar(type: Jar, dependsOn: javadoc) { + classifier = 'javadoc' + from javadoc.destinationDir + manifest.from sharedManifest +} + +// add javadoc/source jar tasks as artifacts +artifacts { + archives javadocJar + archives sourcesJar +} + +publishing { + publications { + RadarCommonsPublication(MavenPublication) { + from components.java + artifact sourcesJar + artifact javadocJar + groupId project.group + artifactId rootProject.name + version project.version + pom.withXml { + def root = asNode() + root.appendNode('description', description) + root.appendNode('name', rootProject.name) + root.appendNode('url', githubUrl) + root.children().last() + pomConfig + } + } + } +} + +bintray { + user project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER') + key project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_API_KEY') + override false + publications 'RadarCommonsPublication' + pkg { + repo = project.group + name = rootProject.name + userOrg = 'radar-cns' + desc = description + licenses = ['Apache-2.0'] + websiteUrl = website + issueTrackerUrl = issueUrl + vcsUrl = githubUrl + githubRepo = githubRepoName + githubReleaseNotesFile = 'README.md' + version { + name = project.version + desc = description + vcsTag = System.getenv('TRAVIS_TAG') + released = new Date() + } + } +} + +artifactory { + contextUrl = 'https://oss.jfrog.org/artifactory' + publish { + repository { + repoKey = 'oss-snapshot-local' + username = project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER') + password = project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_API_KEY') + maven = true + } + } +} + +artifactoryPublish { + publications('RadarCommonsPublication') +} \ No newline at end of file diff --git a/src/main/java/org/radarcns/config/ServerConfig.java b/radar-commons/src/main/java/org/radarcns/config/ServerConfig.java similarity index 97% rename from src/main/java/org/radarcns/config/ServerConfig.java rename to radar-commons/src/main/java/org/radarcns/config/ServerConfig.java index a23b8ca0..237906d9 100644 --- a/src/main/java/org/radarcns/config/ServerConfig.java +++ b/radar-commons/src/main/java/org/radarcns/config/ServerConfig.java @@ -16,8 +16,6 @@ package org.radarcns.config; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSetter; import okhttp3.HttpUrl; import java.net.InetSocketAddress; @@ -36,9 +34,7 @@ public class ServerConfig { private int port = -1; private String protocol; private String path = null; - @JsonProperty("proxy_host") private String proxyHost; - @JsonProperty("proxy_port") private int proxyPort = -1; private boolean unsafe = false; @@ -203,7 +199,6 @@ public String getPath() { return path; } - @JsonSetter("path") public final void setPath(String path) { if (path == null) { this.path = "/"; diff --git a/src/main/java/org/radarcns/data/AvroDecoder.java b/radar-commons/src/main/java/org/radarcns/data/AvroDecoder.java similarity index 100% rename from src/main/java/org/radarcns/data/AvroDecoder.java rename to radar-commons/src/main/java/org/radarcns/data/AvroDecoder.java diff --git a/src/main/java/org/radarcns/data/AvroEncoder.java b/radar-commons/src/main/java/org/radarcns/data/AvroEncoder.java similarity index 100% rename from src/main/java/org/radarcns/data/AvroEncoder.java rename to radar-commons/src/main/java/org/radarcns/data/AvroEncoder.java diff --git a/src/main/java/org/radarcns/data/AvroRecordData.java b/radar-commons/src/main/java/org/radarcns/data/AvroRecordData.java similarity index 100% rename from src/main/java/org/radarcns/data/AvroRecordData.java rename to radar-commons/src/main/java/org/radarcns/data/AvroRecordData.java diff --git a/src/main/java/org/radarcns/data/AvroRecordWriter.java b/radar-commons/src/main/java/org/radarcns/data/AvroRecordWriter.java similarity index 100% rename from src/main/java/org/radarcns/data/AvroRecordWriter.java rename to radar-commons/src/main/java/org/radarcns/data/AvroRecordWriter.java diff --git a/src/main/java/org/radarcns/data/Record.java b/radar-commons/src/main/java/org/radarcns/data/Record.java similarity index 100% rename from src/main/java/org/radarcns/data/Record.java rename to radar-commons/src/main/java/org/radarcns/data/Record.java diff --git a/src/main/java/org/radarcns/data/RecordData.java b/radar-commons/src/main/java/org/radarcns/data/RecordData.java similarity index 100% rename from src/main/java/org/radarcns/data/RecordData.java rename to radar-commons/src/main/java/org/radarcns/data/RecordData.java diff --git a/src/main/java/org/radarcns/data/SpecificRecordDecoder.java b/radar-commons/src/main/java/org/radarcns/data/SpecificRecordDecoder.java similarity index 100% rename from src/main/java/org/radarcns/data/SpecificRecordDecoder.java rename to radar-commons/src/main/java/org/radarcns/data/SpecificRecordDecoder.java diff --git a/src/main/java/org/radarcns/data/SpecificRecordEncoder.java b/radar-commons/src/main/java/org/radarcns/data/SpecificRecordEncoder.java similarity index 100% rename from src/main/java/org/radarcns/data/SpecificRecordEncoder.java rename to radar-commons/src/main/java/org/radarcns/data/SpecificRecordEncoder.java diff --git a/src/main/java/org/radarcns/data/StringEncoder.java b/radar-commons/src/main/java/org/radarcns/data/StringEncoder.java similarity index 100% rename from src/main/java/org/radarcns/data/StringEncoder.java rename to radar-commons/src/main/java/org/radarcns/data/StringEncoder.java diff --git a/src/main/java/org/radarcns/data/TimedInt.java b/radar-commons/src/main/java/org/radarcns/data/TimedInt.java similarity index 100% rename from src/main/java/org/radarcns/data/TimedInt.java rename to radar-commons/src/main/java/org/radarcns/data/TimedInt.java diff --git a/src/main/java/org/radarcns/producer/AuthenticationException.java b/radar-commons/src/main/java/org/radarcns/producer/AuthenticationException.java similarity index 100% rename from src/main/java/org/radarcns/producer/AuthenticationException.java rename to radar-commons/src/main/java/org/radarcns/producer/AuthenticationException.java diff --git a/src/main/java/org/radarcns/producer/BatchedKafkaSender.java b/radar-commons/src/main/java/org/radarcns/producer/BatchedKafkaSender.java similarity index 100% rename from src/main/java/org/radarcns/producer/BatchedKafkaSender.java rename to radar-commons/src/main/java/org/radarcns/producer/BatchedKafkaSender.java diff --git a/src/main/java/org/radarcns/producer/KafkaSender.java b/radar-commons/src/main/java/org/radarcns/producer/KafkaSender.java similarity index 100% rename from src/main/java/org/radarcns/producer/KafkaSender.java rename to radar-commons/src/main/java/org/radarcns/producer/KafkaSender.java diff --git a/src/main/java/org/radarcns/producer/KafkaTopicSender.java b/radar-commons/src/main/java/org/radarcns/producer/KafkaTopicSender.java similarity index 100% rename from src/main/java/org/radarcns/producer/KafkaTopicSender.java rename to radar-commons/src/main/java/org/radarcns/producer/KafkaTopicSender.java diff --git a/src/main/java/org/radarcns/producer/rest/ConnectionState.java b/radar-commons/src/main/java/org/radarcns/producer/rest/ConnectionState.java similarity index 100% rename from src/main/java/org/radarcns/producer/rest/ConnectionState.java rename to radar-commons/src/main/java/org/radarcns/producer/rest/ConnectionState.java diff --git a/src/main/java/org/radarcns/producer/rest/GzipTopicRequestBody.java b/radar-commons/src/main/java/org/radarcns/producer/rest/GzipTopicRequestBody.java similarity index 100% rename from src/main/java/org/radarcns/producer/rest/GzipTopicRequestBody.java rename to radar-commons/src/main/java/org/radarcns/producer/rest/GzipTopicRequestBody.java diff --git a/src/main/java/org/radarcns/producer/rest/ManagedConnectionPool.java b/radar-commons/src/main/java/org/radarcns/producer/rest/ManagedConnectionPool.java similarity index 100% rename from src/main/java/org/radarcns/producer/rest/ManagedConnectionPool.java rename to radar-commons/src/main/java/org/radarcns/producer/rest/ManagedConnectionPool.java diff --git a/src/main/java/org/radarcns/producer/rest/ParsedSchemaMetadata.java b/radar-commons/src/main/java/org/radarcns/producer/rest/ParsedSchemaMetadata.java similarity index 100% rename from src/main/java/org/radarcns/producer/rest/ParsedSchemaMetadata.java rename to radar-commons/src/main/java/org/radarcns/producer/rest/ParsedSchemaMetadata.java diff --git a/src/main/java/org/radarcns/producer/rest/RestClient.java b/radar-commons/src/main/java/org/radarcns/producer/rest/RestClient.java similarity index 100% rename from src/main/java/org/radarcns/producer/rest/RestClient.java rename to radar-commons/src/main/java/org/radarcns/producer/rest/RestClient.java diff --git a/src/main/java/org/radarcns/producer/rest/RestException.java b/radar-commons/src/main/java/org/radarcns/producer/rest/RestException.java similarity index 100% rename from src/main/java/org/radarcns/producer/rest/RestException.java rename to radar-commons/src/main/java/org/radarcns/producer/rest/RestException.java diff --git a/src/main/java/org/radarcns/producer/rest/RestSender.java b/radar-commons/src/main/java/org/radarcns/producer/rest/RestSender.java similarity index 100% rename from src/main/java/org/radarcns/producer/rest/RestSender.java rename to radar-commons/src/main/java/org/radarcns/producer/rest/RestSender.java diff --git a/src/main/java/org/radarcns/producer/rest/RestTopicSender.java b/radar-commons/src/main/java/org/radarcns/producer/rest/RestTopicSender.java similarity index 100% rename from src/main/java/org/radarcns/producer/rest/RestTopicSender.java rename to radar-commons/src/main/java/org/radarcns/producer/rest/RestTopicSender.java diff --git a/src/main/java/org/radarcns/producer/rest/SchemaRetriever.java b/radar-commons/src/main/java/org/radarcns/producer/rest/SchemaRetriever.java similarity index 100% rename from src/main/java/org/radarcns/producer/rest/SchemaRetriever.java rename to radar-commons/src/main/java/org/radarcns/producer/rest/SchemaRetriever.java diff --git a/src/main/java/org/radarcns/producer/rest/TopicRequestBody.java b/radar-commons/src/main/java/org/radarcns/producer/rest/TopicRequestBody.java similarity index 100% rename from src/main/java/org/radarcns/producer/rest/TopicRequestBody.java rename to radar-commons/src/main/java/org/radarcns/producer/rest/TopicRequestBody.java diff --git a/src/main/java/org/radarcns/producer/rest/TopicRequestData.java b/radar-commons/src/main/java/org/radarcns/producer/rest/TopicRequestData.java similarity index 100% rename from src/main/java/org/radarcns/producer/rest/TopicRequestData.java rename to radar-commons/src/main/java/org/radarcns/producer/rest/TopicRequestData.java diff --git a/src/main/java/org/radarcns/topic/AvroTopic.java b/radar-commons/src/main/java/org/radarcns/topic/AvroTopic.java similarity index 100% rename from src/main/java/org/radarcns/topic/AvroTopic.java rename to radar-commons/src/main/java/org/radarcns/topic/AvroTopic.java diff --git a/src/main/java/org/radarcns/topic/KafkaTopic.java b/radar-commons/src/main/java/org/radarcns/topic/KafkaTopic.java similarity index 100% rename from src/main/java/org/radarcns/topic/KafkaTopic.java rename to radar-commons/src/main/java/org/radarcns/topic/KafkaTopic.java diff --git a/src/main/java/org/radarcns/topic/SensorTopic.java b/radar-commons/src/main/java/org/radarcns/topic/SensorTopic.java similarity index 100% rename from src/main/java/org/radarcns/topic/SensorTopic.java rename to radar-commons/src/main/java/org/radarcns/topic/SensorTopic.java diff --git a/src/main/java/org/radarcns/util/Serialization.java b/radar-commons/src/main/java/org/radarcns/util/Serialization.java similarity index 100% rename from src/main/java/org/radarcns/util/Serialization.java rename to radar-commons/src/main/java/org/radarcns/util/Serialization.java diff --git a/src/main/java/org/radarcns/util/Strings.java b/radar-commons/src/main/java/org/radarcns/util/Strings.java similarity index 100% rename from src/main/java/org/radarcns/util/Strings.java rename to radar-commons/src/main/java/org/radarcns/util/Strings.java diff --git a/src/test/java/org/radarcns/data/SpecificRecordDecorderTest.java b/radar-commons/src/test/java/org/radarcns/data/SpecificRecordDecorderTest.java similarity index 100% rename from src/test/java/org/radarcns/data/SpecificRecordDecorderTest.java rename to radar-commons/src/test/java/org/radarcns/data/SpecificRecordDecorderTest.java diff --git a/src/test/java/org/radarcns/data/SpecificRecordEncoderTest.java b/radar-commons/src/test/java/org/radarcns/data/SpecificRecordEncoderTest.java similarity index 98% rename from src/test/java/org/radarcns/data/SpecificRecordEncoderTest.java rename to radar-commons/src/test/java/org/radarcns/data/SpecificRecordEncoderTest.java index d5dd5db5..9bac2eb8 100644 --- a/src/test/java/org/radarcns/data/SpecificRecordEncoderTest.java +++ b/radar-commons/src/test/java/org/radarcns/data/SpecificRecordEncoderTest.java @@ -17,21 +17,18 @@ package org.radarcns.data; import junit.framework.TestCase; - -import org.radarcns.passive.empatica.EmpaticaE4Acceleration; -import org.radarcns.passive.empatica.EmpaticaE4BloodVolumePulse; import org.radarcns.kafka.ObservationKey; +import org.radarcns.passive.empatica.EmpaticaE4BloodVolumePulse; +import org.radarcns.passive.phone.PhoneAcceleration; +import org.radarcns.topic.AvroTopic; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.Arrays; import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; -import org.radarcns.passive.phone.PhoneAcceleration; -import org.radarcns.topic.AvroTopic; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - public class SpecificRecordEncoderTest extends TestCase { private static final Logger logger = LoggerFactory.getLogger(SpecificRecordEncoderTest.class); diff --git a/src/test/java/org/radarcns/data/StringEncoderTest.java b/radar-commons/src/test/java/org/radarcns/data/StringEncoderTest.java similarity index 100% rename from src/test/java/org/radarcns/data/StringEncoderTest.java rename to radar-commons/src/test/java/org/radarcns/data/StringEncoderTest.java diff --git a/src/test/java/org/radarcns/producer/rest/ConnectionStateTest.java b/radar-commons/src/test/java/org/radarcns/producer/rest/ConnectionStateTest.java similarity index 100% rename from src/test/java/org/radarcns/producer/rest/ConnectionStateTest.java rename to radar-commons/src/test/java/org/radarcns/producer/rest/ConnectionStateTest.java diff --git a/src/test/java/org/radarcns/producer/rest/RestClientTest.java b/radar-commons/src/test/java/org/radarcns/producer/rest/RestClientTest.java similarity index 100% rename from src/test/java/org/radarcns/producer/rest/RestClientTest.java rename to radar-commons/src/test/java/org/radarcns/producer/rest/RestClientTest.java diff --git a/src/test/java/org/radarcns/producer/rest/RestSenderTest.java b/radar-commons/src/test/java/org/radarcns/producer/rest/RestSenderTest.java similarity index 100% rename from src/test/java/org/radarcns/producer/rest/RestSenderTest.java rename to radar-commons/src/test/java/org/radarcns/producer/rest/RestSenderTest.java diff --git a/src/test/java/org/radarcns/producer/rest/SchemaRetrieverTest.java b/radar-commons/src/test/java/org/radarcns/producer/rest/SchemaRetrieverTest.java similarity index 100% rename from src/test/java/org/radarcns/producer/rest/SchemaRetrieverTest.java rename to radar-commons/src/test/java/org/radarcns/producer/rest/SchemaRetrieverTest.java diff --git a/src/test/java/org/radarcns/topic/KafkaTopicTest.java b/radar-commons/src/test/java/org/radarcns/topic/KafkaTopicTest.java similarity index 100% rename from src/test/java/org/radarcns/topic/KafkaTopicTest.java rename to radar-commons/src/test/java/org/radarcns/topic/KafkaTopicTest.java diff --git a/src/test/java/org/radarcns/topic/SensorTopicTest.java b/radar-commons/src/test/java/org/radarcns/topic/SensorTopicTest.java similarity index 98% rename from src/test/java/org/radarcns/topic/SensorTopicTest.java rename to radar-commons/src/test/java/org/radarcns/topic/SensorTopicTest.java index 845fabdb..25993713 100644 --- a/src/test/java/org/radarcns/topic/SensorTopicTest.java +++ b/radar-commons/src/test/java/org/radarcns/topic/SensorTopicTest.java @@ -16,7 +16,6 @@ package org.radarcns.topic; -import com.fasterxml.jackson.databind.annotation.JsonTypeResolver; import org.apache.avro.Schema; import org.apache.avro.Schema.Type; import org.apache.avro.SchemaBuilder; diff --git a/src/test/java/org/radarcns/util/SerializationTest.java b/radar-commons/src/test/java/org/radarcns/util/SerializationTest.java similarity index 100% rename from src/test/java/org/radarcns/util/SerializationTest.java rename to radar-commons/src/test/java/org/radarcns/util/SerializationTest.java diff --git a/src/test/resources/integration.yml b/radar-commons/src/test/resources/integration.yml similarity index 100% rename from src/test/resources/integration.yml rename to radar-commons/src/test/resources/integration.yml diff --git a/src/test/resources/integration_test.csv b/radar-commons/src/test/resources/integration_test.csv similarity index 100% rename from src/test/resources/integration_test.csv rename to radar-commons/src/test/resources/integration_test.csv diff --git a/settings.gradle b/settings.gradle index be483c09..3fe7e71c 100644 --- a/settings.gradle +++ b/settings.gradle @@ -14,7 +14,7 @@ * limitations under the License. */ -rootProject.name = 'radar-commons' - +include ':radar-commons' include ':radar-commons-testing' +include ':radar-commons-server' enableFeaturePreview('STABLE_PUBLISHING') From ef89fbd6a594174df12539b1e2471562e9aa8ff8 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 26 Jun 2018 16:45:37 +0200 Subject: [PATCH 11/16] Fixed line length --- .../stream/collector/UniformSamplingReservoirTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/radar-commons-server/src/test/java/org/radarcns/stream/collector/UniformSamplingReservoirTest.java b/radar-commons-server/src/test/java/org/radarcns/stream/collector/UniformSamplingReservoirTest.java index 2148b956..71dfc99a 100644 --- a/radar-commons-server/src/test/java/org/radarcns/stream/collector/UniformSamplingReservoirTest.java +++ b/radar-commons-server/src/test/java/org/radarcns/stream/collector/UniformSamplingReservoirTest.java @@ -12,7 +12,8 @@ public class UniformSamplingReservoirTest { @Test public void add() { - UniformSamplingReservoir reservoir = new UniformSamplingReservoir(new double[] {0.1, 0.3, 0.5}, 3, 3); + UniformSamplingReservoir reservoir = new UniformSamplingReservoir( + new double[] {0.1, 0.3, 0.5}, 3, 3); reservoir.add(0.7); assertEquals(3, reservoir.getSamples().size()); assertEquals(3, reservoir.getMaxSize()); From a0cae6b0551fd812fd9049435950dd95bd8949ea Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 2 Jul 2018 16:36:03 +0200 Subject: [PATCH 12/16] Checkstyle update --- .../org/radarcns/config/AvroTopicConfig.java | 2 +- .../org/radarcns/config/YamlConfigLoader.java | 17 ++++++++++++++++- .../radarcns/producer/direct/DirectSender.java | 2 +- .../collector/NumericAggregateCollector.java | 8 +++++++- 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/radar-commons-server/src/main/java/org/radarcns/config/AvroTopicConfig.java b/radar-commons-server/src/main/java/org/radarcns/config/AvroTopicConfig.java index 3408b7d8..58535a85 100644 --- a/radar-commons-server/src/main/java/org/radarcns/config/AvroTopicConfig.java +++ b/radar-commons-server/src/main/java/org/radarcns/config/AvroTopicConfig.java @@ -23,7 +23,7 @@ import java.util.List; /** - * Specifies an Avro topic + * Specifies an Avro topic. */ public class AvroTopicConfig { private String topic; diff --git a/radar-commons-server/src/main/java/org/radarcns/config/YamlConfigLoader.java b/radar-commons-server/src/main/java/org/radarcns/config/YamlConfigLoader.java index 7ec4aacc..349dc0bc 100644 --- a/radar-commons-server/src/main/java/org/radarcns/config/YamlConfigLoader.java +++ b/radar-commons-server/src/main/java/org/radarcns/config/YamlConfigLoader.java @@ -26,12 +26,15 @@ import java.io.IOException; /** - * A YAML Config file loader, to load YAML files into equivalent POJO Objects + * A YAML Config file loader, to load YAML files into equivalent POJO Objects. */ public class YamlConfigLoader { private final ObjectMapper mapper; + /** + * Default loader. + */ public YamlConfigLoader() { mapper = new ObjectMapper(new YAMLFactory()); mapper.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE); @@ -45,14 +48,26 @@ public YamlConfigLoader() { mapper.addMixIn(ServerConfig.class, ServerConfigMixin.class); } + /** + * Load a YAML file into given class. + * @param file file to load. + * @param configClass class for the config. + * @param type of the config. + * @return loaded file. + * @throws IOException if the file cannot be opened or parsed. + */ public T load(File file, Class configClass) throws IOException { return mapper.readValue(file, configClass); } + /** Store config into given YAML file. */ public void store(File file, Object config) throws IOException { mapper.writeValue(file, config); } + /** + * Pretty-print the given object as a YAML string. + */ public String prettyString(Object config) { // pretty print mapper.enable(SerializationFeature.INDENT_OUTPUT); diff --git a/radar-commons-server/src/main/java/org/radarcns/producer/direct/DirectSender.java b/radar-commons-server/src/main/java/org/radarcns/producer/direct/DirectSender.java index 77c98293..e1945c5b 100644 --- a/radar-commons-server/src/main/java/org/radarcns/producer/direct/DirectSender.java +++ b/radar-commons-server/src/main/java/org/radarcns/producer/direct/DirectSender.java @@ -27,7 +27,7 @@ import java.util.Properties; /** - * Directly sends a message to Kafka using a KafkaProducer + * Directly sends a message to Kafka using a KafkaProducer. */ public class DirectSender implements KafkaSender { private final KafkaProducer producer; diff --git a/radar-commons-server/src/main/java/org/radarcns/stream/collector/NumericAggregateCollector.java b/radar-commons-server/src/main/java/org/radarcns/stream/collector/NumericAggregateCollector.java index 2d74b46f..ecd74b58 100644 --- a/radar-commons-server/src/main/java/org/radarcns/stream/collector/NumericAggregateCollector.java +++ b/radar-commons-server/src/main/java/org/radarcns/stream/collector/NumericAggregateCollector.java @@ -46,7 +46,8 @@ public class NumericAggregateCollector implements RecordCollector { private BigDecimal sum; private final UniformSamplingReservoir reservoir; - public NumericAggregateCollector(Builder builder) { + /** Aggregate collector from builder. */ + private NumericAggregateCollector(Builder builder) { this.name = builder.nameValue; this.pos = builder.posValue; this.fieldType = builder.fieldTypeValue; @@ -56,10 +57,12 @@ public NumericAggregateCollector(Builder builder) { this.reservoir = builder.reservoirValue; } + /** Aggregate collector with only a field name. */ public NumericAggregateCollector(String fieldName) { this(fieldName, null); } + /** Aggregate collector with a field name and accompanying schema containing the name. */ public NumericAggregateCollector(String fieldName, Schema schema) { sum = BigDecimal.ZERO; min = Double.POSITIVE_INFINITY; @@ -119,6 +122,7 @@ public NumericAggregateCollector add(SpecificRecord record) { } } + /** Add a single sample. */ public NumericAggregateCollector add(float value) { return this.add(floatToDouble(value)); } @@ -176,6 +180,7 @@ public List getQuartile() { return reservoir.getQuartiles(); } + /** Difference between the first quartile and third quartile (IQR). */ public double getInterQuartileRange() { List quartiles = getQuartile(); return BigDecimal.valueOf(quartiles.get(2)) @@ -283,6 +288,7 @@ public Builder history(double[] history) { return this; } + /** Build the aggregator. */ public NumericAggregateCollector build() { return new NumericAggregateCollector(this); } From 9cf43fe5da28447e918d70ce2e8a1a97c04b78ad Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 2 Jul 2018 16:59:42 +0200 Subject: [PATCH 13/16] Added unsafe Confluent code (fixes #52) --- README.md | 18 +- radar-commons-server/build.gradle | 2 +- radar-commons-unsafe/build.gradle | 157 +++++++++ .../AbstractKafkaAvroDeserializer.java | 315 ++++++++++++++++++ settings.gradle | 1 + 5 files changed, 490 insertions(+), 3 deletions(-) create mode 100644 radar-commons-unsafe/build.gradle create mode 100644 radar-commons-unsafe/src/main/java/io/confluent/kafka/serializers/AbstractKafkaAvroDeserializer.java diff --git a/README.md b/README.md index 610dd113..d1ce16c7 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,14 @@ repositories { } dependencies { - compile group: 'org.radarcns', name: 'radar-commons', version: '0.8.2' + implementation group: 'org.radarcns', name: 'radar-commons', version: '0.9.0' +} +``` + +For server utilities, include `radar-commons-server`: +```gradle +dependencies { + implementation group: 'org.radarcns', name: 'radar-commons-server', version: '0.9.0' } ``` @@ -26,7 +33,14 @@ repositories { } dependencies { - testCompile group: 'org.radarcns', name: 'radar-commons-testing', version: '0.8.2' + testImplementation group: 'org.radarcns', name: 'radar-commons-testing', version: '0.9.0' +} +``` + +Finally, if the schema registry is losing old schemas and your code is not recovering, include `radar-commons-unsafe`. Ensure that it comes in the classpath before any Confluent code. This will override the Confluent Avro deserializer to recover from failure when a message with unknown schema ID is passed. +```gradle +dependencies { + runtimeOnly group: 'org.radarcns', name: 'radar-commons-unsafe', version: '0.9.0' } ``` diff --git a/radar-commons-server/build.gradle b/radar-commons-server/build.gradle index c0516fe8..85402bf2 100644 --- a/radar-commons-server/build.gradle +++ b/radar-commons-server/build.gradle @@ -15,7 +15,7 @@ */ ext.moduleName = 'radar-commons-server' -ext.description = 'RADAR Common testing library mocking code and utilities.' +ext.description = 'RADAR Common server library utilities.' targetCompatibility = '1.8' sourceCompatibility = '1.8' diff --git a/radar-commons-unsafe/build.gradle b/radar-commons-unsafe/build.gradle new file mode 100644 index 00000000..b49430ec --- /dev/null +++ b/radar-commons-unsafe/build.gradle @@ -0,0 +1,157 @@ +targetCompatibility = '1.8' +sourceCompatibility = '1.8' + +ext.moduleName = 'radar-commons-unsafe' +ext.description = 'RADAR Common unsafe libraries overriding stock Kafka and Confluent code.' + +//---------------------------------------------------------------------------// +// Sources and classpath configurations // +//---------------------------------------------------------------------------// + +configurations { + codacy +} + +configurations.compile { + resolutionStrategy.cacheChangingModulesFor 0, 'SECONDS' +} + +// In this section you declare where to find the dependencies of your project +repositories { + maven { url 'https://jitpack.io' } + maven { url 'https://oss.jfrog.org/artifactory/oss-snapshot-local' } +} + +// In this section you declare the dependencies for your production and test code +dependencies { + compileOnly group: 'io.confluent', name: 'kafka-avro-serializer', version: confluentVersion + compileOnly group: 'org.apache.kafka', name: 'kafka-clients', version: kafkaVersion + compileOnly group: 'org.apache.kafka', name: 'kafka_2.11', version: kafkaVersion + +} + +//---------------------------------------------------------------------------// +// Testing // +//---------------------------------------------------------------------------// +checkstyle { + // ignore tests + sourceSets = [sourceSets.main] +} + +pmd { + // ignore tests + sourceSets = [sourceSets.main] +} + +jacocoTestReport { + reports { + xml.enabled true + csv.enabled false + html.enabled false + } +} + +task downloadDependencies(type: Exec) { + configurations.testRuntime.files + configurations.codacy.files + configurations.jacocoAnt.files + commandLine 'echo', 'Downloaded all dependencies' +} + +task sendCoverageToCodacy(type: JavaExec, dependsOn: jacocoTestReport) { + main = 'com.codacy.CodacyCoverageReporter' + classpath = configurations.codacy + args = ['-l', 'Java', '-r', "${buildDir}/reports/jacoco/test/jacocoTestReport.xml"] +} + +//---------------------------------------------------------------------------// +// Build system metadata // +//---------------------------------------------------------------------------// + +ext.sharedManifest = manifest { + attributes("Implementation-Title": moduleName, + "Implementation-Version": version) +} + +jar { + manifest.from sharedManifest +} +// custom tasks for creating source/javadoc jars +task sourcesJar(type: Jar, dependsOn: classes) { + classifier = 'sources' + from sourceSets.main.allSource + manifest.from sharedManifest +} + +task javadocJar(type: Jar, dependsOn: javadoc) { + classifier = 'javadoc' + from javadoc.destinationDir + manifest.from sharedManifest +} + +// add javadoc/source jar tasks as artifacts +artifacts { + archives javadocJar + archives sourcesJar +} + +publishing { + publications { + RadarCommonsPublication(MavenPublication) { + from components.java + artifact sourcesJar + artifact javadocJar + groupId project.group + artifactId moduleName + version project.version + pom.withXml { + def root = asNode() + root.appendNode('description', description) + root.appendNode('name', moduleName) + root.appendNode('url', githubUrl) + root.children().last() + pomConfig + } + } + } +} + +bintray { + user project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER') + key project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_API_KEY') + override false + publications 'RadarCommonsPublication' + pkg { + repo = project.group + name = moduleName + userOrg = 'radar-cns' + desc = description + licenses = ['Apache-2.0'] + websiteUrl = website + issueTrackerUrl = issueUrl + vcsUrl = githubUrl + githubRepo = githubRepoName + githubReleaseNotesFile = 'README.md' + version { + name = project.version + desc = description + vcsTag = System.getenv('TRAVIS_TAG') + released = new Date() + } + } +} + +artifactory { + contextUrl = 'https://oss.jfrog.org/artifactory' + publish { + repository { + repoKey = 'oss-snapshot-local' + username = project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER') + password = project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_API_KEY') + maven = true + } + } +} + +artifactoryPublish { + publications('RadarCommonsPublication') +} \ No newline at end of file diff --git a/radar-commons-unsafe/src/main/java/io/confluent/kafka/serializers/AbstractKafkaAvroDeserializer.java b/radar-commons-unsafe/src/main/java/io/confluent/kafka/serializers/AbstractKafkaAvroDeserializer.java new file mode 100644 index 00000000..264c2fdc --- /dev/null +++ b/radar-commons-unsafe/src/main/java/io/confluent/kafka/serializers/AbstractKafkaAvroDeserializer.java @@ -0,0 +1,315 @@ +/* + * Copyright 2014 Confluent Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.confluent.kafka.serializers; + +import io.confluent.kafka.schemaregistry.client.SchemaMetadata; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericContainer; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.io.DatumReader; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.specific.SpecificData; +import org.apache.avro.specific.SpecificDatumReader; +import org.apache.avro.specific.SpecificRecord; +import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.common.errors.SerializationException; +import org.codehaus.jackson.node.JsonNodeFactory; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException; +import kafka.utils.VerifiableProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class AbstractKafkaAvroDeserializer extends AbstractKafkaAvroSerDe { + + public static final String SCHEMA_REGISTRY_SCHEMA_VERSION_PROP = + "schema.registry.schema.version"; + + private final Map> oldToNewIdMap = new IdentityHashMap<>(); + private final Map> oldToNewVersionMap = new IdentityHashMap<>(); + + private static final Logger logger = LoggerFactory.getLogger(AbstractKafkaAvroDeserializer.class); + + private final DecoderFactory decoderFactory = DecoderFactory.get(); + protected boolean useSpecificAvroReader = false; + private final Map readerSchemaCache = new ConcurrentHashMap(); + + + /** + * Sets properties for this deserializer without overriding the schema registry client itself. + * Useful for testing, where a mock client is injected. + */ + protected void configure(KafkaAvroDeserializerConfig config) { + configureClientProperties(config); + useSpecificAvroReader = config + .getBoolean(KafkaAvroDeserializerConfig.SPECIFIC_AVRO_READER_CONFIG); + } + + protected KafkaAvroDeserializerConfig deserializerConfig(Map props) { + try { + return new KafkaAvroDeserializerConfig(props); + } catch (io.confluent.common.config.ConfigException e) { + throw new ConfigException(e.getMessage()); + } + } + + protected KafkaAvroDeserializerConfig deserializerConfig(VerifiableProperties props) { + try { + return new KafkaAvroDeserializerConfig(props.props()); + } catch (io.confluent.common.config.ConfigException e) { + throw new ConfigException(e.getMessage()); + } + } + + private ByteBuffer getByteBuffer(byte[] payload) { + ByteBuffer buffer = ByteBuffer.wrap(payload); + if (buffer.get() != MAGIC_BYTE) { + throw new SerializationException("Unknown magic byte!"); + } + return buffer; + } + + /** + * Deserializes the payload without including schema information for primitive types, maps, and + * arrays. Just the resulting deserialized object is returned. + * + *

This behavior is the norm for Decoders/Deserializers. + * + * @param payload serialized data + * @return the deserialized object + */ + protected Object deserialize(byte[] payload) throws SerializationException { + return deserialize(false, null, null, payload, null); + } + + /** + * Just like single-parameter version but accepts an Avro schema to use for reading + * + * @param payload serialized data + * @param readerSchema schema to use for Avro read (optional, enables Avro projection) + * @return the deserialized object + */ + protected Object deserialize(byte[] payload, Schema readerSchema) throws SerializationException { + return deserialize(false, null, null, payload, readerSchema); + } + + // The Object return type is a bit messy, but this is the simplest way to have + // flexible decoding and not duplicate deserialization code multiple times for different variants. + protected Object deserialize(boolean includeSchemaAndVersion, String topic, Boolean isKey, + byte[] payload, Schema readerSchema) throws SerializationException { + // Even if the caller requests schema & version, if the payload is null we cannot include it. + // The caller must handle this case. + if (payload == null) { + return null; + } + + int id = -1; + try { + ByteBuffer buffer = getByteBuffer(payload); + id = buffer.getInt(); + String subject = includeSchemaAndVersion ? getSubjectName(topic, isKey, null) : null; + Schema schema; + SchemaMetadata schemaMetadata = null; + Map subjectIdMap = oldToNewIdMap.get(subject); + try { + if (subjectIdMap != null) { + Integer newId = subjectIdMap.get(id); + if (newId != null) { + id = newId; + } + } + schema = getBySubjectAndId(subject, id); + } catch (RestClientException ex) { + if (ex.getErrorCode() == 40403) { + logger.debug("Trying to get id from Latest SchemaMetadata from schemaRegistry for subject {}", subject); + int oldId = id; + schemaMetadata = schemaRegistry.getLatestSchemaMetadata(subject); + id = schemaMetadata.getId(); + schema = new Schema.Parser().parse(schemaMetadata.getSchema()); + + if (subjectIdMap == null) { + subjectIdMap = new HashMap<>(); + oldToNewIdMap.put(subject, subjectIdMap); + } + // keep a track of a subject's ids map so that subsequent records don't query the wrong schema id + subjectIdMap.put(oldId, id); + + logger.debug("success -> schemaMetadata.getId({}) for subject {}", id, subject); + + } else { + throw ex; + } + } + int length = buffer.limit() - 1 - idSize; + final Object result; + if (schema.getType().equals(Schema.Type.BYTES)) { + byte[] bytes = new byte[length]; + buffer.get(bytes, 0, length); + result = bytes; + } else { + int start = buffer.position() + buffer.arrayOffset(); + DatumReader reader = getDatumReader(schema, readerSchema); + Object + object = + reader.read(null, decoderFactory.binaryDecoder(buffer.array(), start, length, null)); + + if (schema.getType().equals(Schema.Type.STRING)) { + object = object.toString(); // Utf8 -> String + } + result = object; + } + + if (includeSchemaAndVersion) { + // Annotate the schema with the version. Note that we only do this if the schema + + // version are requested, i.e. in Kafka Connect converters. This is critical because that + // code *will not* rely on exact schema equality. Regular deserializers *must not* include + // this information because it would return schemas which are not equivalent. + // + // Note, however, that we also do not fill in the connect.version field. This allows the + // Converter to let a version provided by a Kafka Connect source take priority over the + // schema registry's ordering (which is implicit by auto-registration time rather than + // explicit from the Connector). + Integer version = null; + if (schemaMetadata != null) { + version = schemaMetadata.getVersion(); + } else { + Map schemaVersionMap = oldToNewVersionMap.get(subject); + if (schemaVersionMap != null) { + version = schemaVersionMap.get(schema); + } + try { + if (version == null) { + version = schemaRegistry.getVersion(subject, schema); + } + } catch (RestClientException e) { + if (e.getErrorCode() == 40403) { + logger.debug("Trying to get version from Latest SchemaMetadata from schemaRegistry for subject {}", subject); + schemaMetadata = schemaRegistry.getLatestSchemaMetadata(subject); + version = schemaMetadata.getVersion(); + + if (schemaVersionMap == null) { + schemaVersionMap = new IdentityHashMap<>(); + oldToNewVersionMap.put(subject, schemaVersionMap); + } + + schemaVersionMap.put(schema, version); + logger.debug("success -> schemaMetadata.getVersion({}) for subject {}", version, subject); + } else { + throw e; + } + } + } + + + if (schema.getType() == Schema.Type.UNION) { + // Can't set additional properties on a union schema since it's just a list, so set it + // on the first non-null entry + for (Schema memberSchema : schema.getTypes()) { + if (memberSchema.getType() != Schema.Type.NULL) { + memberSchema.addProp(SCHEMA_REGISTRY_SCHEMA_VERSION_PROP, + JsonNodeFactory.instance.numberNode(version)); + break; + } + } + } else { + schema.addProp(SCHEMA_REGISTRY_SCHEMA_VERSION_PROP, + JsonNodeFactory.instance.numberNode(version)); + } + if (schema.getType().equals(Schema.Type.RECORD)) { + return result; + } else { + return new NonRecordContainer(schema, result); + } + } else { + return result; + } + } catch (IOException | RuntimeException e) { + // avro deserialization may throw AvroRuntimeException, NullPointerException, etc + throw new SerializationException("Error deserializing Avro message for id " + id, e); + } catch (RestClientException e) { + throw new SerializationException("Error retrieving Avro schema for id " + id, e); + } + } + + /** + * Deserializes the payload and includes schema information, with version information from the + * schema registry embedded in the schema. + * + * @param payload the serialized data + * @return a GenericContainer with the schema and data, either as a {@link NonRecordContainer}, + * {@link org.apache.avro.generic.GenericRecord}, or {@link SpecificRecord} + */ + protected GenericContainer deserializeWithSchemaAndVersion(String topic, boolean isKey, + byte[] payload) + throws SerializationException { + return (GenericContainer) deserialize(true, topic, isKey, payload, null); + } + + private DatumReader getDatumReader(Schema writerSchema, Schema readerSchema) { + boolean writerSchemaIsPrimitive = getPrimitiveSchemas().values().contains(writerSchema); + // do not use SpecificDatumReader if writerSchema is a primitive + if (useSpecificAvroReader && !writerSchemaIsPrimitive) { + if (readerSchema == null) { + readerSchema = getReaderSchema(writerSchema); + } + return new SpecificDatumReader(writerSchema, readerSchema); + } else { + if (readerSchema == null) { + return new GenericDatumReader(writerSchema); + } + return new GenericDatumReader(writerSchema, readerSchema); + } + } + + @SuppressWarnings("unchecked") + private Schema getReaderSchema(Schema writerSchema) { + Schema readerSchema = readerSchemaCache.get(writerSchema.getFullName()); + if (readerSchema == null) { + Class readerClass = SpecificData.get().getClass(writerSchema); + if (readerClass != null) { + try { + readerSchema = readerClass.newInstance().getSchema(); + } catch (InstantiationException e) { + throw new SerializationException(writerSchema.getFullName() + + " specified by the " + + "writers schema could not be instantiated to " + + "find the readers schema."); + } catch (IllegalAccessException e) { + throw new SerializationException(writerSchema.getFullName() + + " specified by the " + + "writers schema is not allowed to be instantiated " + + "to find the readers schema."); + } + readerSchemaCache.put(writerSchema.getFullName(), readerSchema); + } else { + throw new SerializationException("Could not find class " + + writerSchema.getFullName() + + " specified in writer's schema whilst finding reader's " + + "schema for a SpecificRecord."); + } + } + return readerSchema; + } + +} diff --git a/settings.gradle b/settings.gradle index 3fe7e71c..caa8aae6 100644 --- a/settings.gradle +++ b/settings.gradle @@ -17,4 +17,5 @@ include ':radar-commons' include ':radar-commons-testing' include ':radar-commons-server' +include ':radar-commons-unsafe' enableFeaturePreview('STABLE_PUBLISHING') From 9e20ff384c4692f9b4e746687d66c22326d6f059 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 2 Jul 2018 17:01:16 +0200 Subject: [PATCH 14/16] Bump version --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index a4e480b8..7679c414 100644 --- a/build.gradle +++ b/build.gradle @@ -36,7 +36,7 @@ subprojects { // Configuration // //---------------------------------------------------------------------------// - version = '0.8.2' + version = '0.9.0' group = 'org.radarcns' ext.githubRepoName = 'RADAR-CNS/RADAR-Commons' From 9e5eee27187eb8b9718d587be570c108188396bf Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 2 Jul 2018 17:06:40 +0200 Subject: [PATCH 15/16] Remove style checks for unsafe module --- radar-commons-unsafe/build.gradle | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/radar-commons-unsafe/build.gradle b/radar-commons-unsafe/build.gradle index b49430ec..d9fd30f8 100644 --- a/radar-commons-unsafe/build.gradle +++ b/radar-commons-unsafe/build.gradle @@ -34,13 +34,13 @@ dependencies { // Testing // //---------------------------------------------------------------------------// checkstyle { - // ignore tests - sourceSets = [sourceSets.main] + // ignore everything + sourceSets = [] } pmd { - // ignore tests - sourceSets = [sourceSets.main] + // ignore everything + sourceSets = [] } jacocoTestReport { From 3e8659caa211a1c23cc4b27050790b4c518a18f5 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 2 Jul 2018 17:26:43 +0200 Subject: [PATCH 16/16] Fixed radar-base references --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d1ce16c7..12d62c6b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # RADAR-Commons -[![Build Status](https://travis-ci.org/RADAR-CNS/radar-commons.svg?branch=master)](https://travis-ci.org/RADAR-CNS/radar-commons) -[![Codacy Badge](https://api.codacy.com/project/badge/Grade/9fe7a419c83e4798af671e468c7e91cf)](https://www.codacy.com/app/RADAR-CNS/RADAR-Commons?utm_source=github.com&utm_medium=referral&utm_content=RADAR-CNS/RADAR-Commons&utm_campaign=Badge_Grade) +[![Build Status](https://travis-ci.org/RADAR-base/radar-commons.svg?branch=master)](https://travis-ci.org/RADAR-base/radar-commons) +[![Codacy Badge](https://api.codacy.com/project/badge/Grade/9fe7a419c83e4798af671e468c7e91cf)](https://www.codacy.com/app/RADAR-base/radar-commons?utm_source=github.com&utm_medium=referral&utm_content=RADAR-base/radar-commons&utm_campaign=Badge_Grade) Common utilities library containing basic schemas, streaming features, testing bridges and utils.