From a39a3d53e3ee204df62568d9f45f298fbb3b70a6 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 8 Apr 2021 17:20:04 -0400 Subject: [PATCH 01/54] Updated dictionary service to work with spring boot 2.4.4 --- api/pom.xml | 14 ++++++++++++-- pom.xml | 2 +- service/pom.xml | 19 +++++++++++++++++-- .../metadata/DefaultMetadataFieldScanner.java | 7 +++---- .../src/main/resources/config/bootstrap.yml | 6 ++---- 5 files changed, 35 insertions(+), 13 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 49cf95f..4c3e6f7 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 1.7 + 1.8 dictionary-api @@ -17,7 +17,7 @@ https://github.com/NationalSecurityAgency/datawave-dictionary-service - 1.0 + 1.3-SNAPSHOT 25.1-jre @@ -48,6 +48,16 @@ junit test + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.vintage + junit-vintage-engine + test + diff --git a/pom.xml b/pom.xml index 74abdb8..1fb7987 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 1.7 + 1.8 dictionary-service-parent diff --git a/service/pom.xml b/service/pom.xml index 9072f2a..82cd47f 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-service-parent - 1.7.1 + 2.1-SNAPSHOT dictionary-service @@ -20,7 +20,7 @@ datawave.microservice.dictionary.DictionaryService 1.2-SNAPSHOT - 1.6.3 + 1.6.6-SNAPSHOT 1.10.19 3.3.1-2 @@ -93,6 +93,21 @@ datawave-in-memory-accumulo test + + junit + junit + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.vintage + junit-vintage-engine + test + diff --git a/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java b/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java index fb2a86d..0ddf9b3 100644 --- a/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java +++ b/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java @@ -19,14 +19,14 @@ import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.iterators.user.WholeRowIterator; import org.apache.hadoop.io.Text; -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.text.SimpleDateFormat; import java.util.Collection; import java.util.Collections; +import java.util.Date; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; @@ -38,7 +38,6 @@ public class DefaultMetadataFieldScanner { private static final Logger log = LoggerFactory.getLogger(DefaultMetadataFieldScanner.class); private static final String TIMESTAMP_FORMAT = "yyyyMMddHHmmss"; - private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormat.forPattern(TIMESTAMP_FORMAT); private final MarkingFunctions markingFunctions; private final ResponseObjectFactory responseObjectFactory; @@ -272,7 +271,7 @@ private void setType() { // Set the last updated date for the current {@link DefaultMetadataField} based on the timestamp of the current entry. private void setLastUpdated() { - currField.setLastUpdated(DATE_FORMATTER.print(currKey.getTimestamp())); + currField.setLastUpdated(new SimpleDateFormat(TIMESTAMP_FORMAT).format(new Date(currKey.getTimestamp()))); } } } diff --git a/service/src/main/resources/config/bootstrap.yml b/service/src/main/resources/config/bootstrap.yml index d2a1dc7..b50f184 100644 --- a/service/src/main/resources/config/bootstrap.yml +++ b/service/src/main/resources/config/bootstrap.yml @@ -16,16 +16,14 @@ spring: --- # For the dev profile, check localhost for the config server by default -spring: - profiles: 'dev' +spring.config.activate.on-profile: 'dev' cloud: config: uri: '${CONFIG_SERVER_URL:http://localhost:8888/configserver}' --- -spring: - profiles: 'consul' +spring.config.activate.on-profile: 'consul' cloud: config: # Use Consul to locate the configuration server and bootstrap app config. From f7f415173d90ac58ee0268416fa21de14c2d89ca Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Tue, 18 May 2021 16:24:53 -0400 Subject: [PATCH 02/54] Fixed activation profile syntax in bootstrap.yml --- service/src/main/resources/config/bootstrap.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/service/src/main/resources/config/bootstrap.yml b/service/src/main/resources/config/bootstrap.yml index b50f184..2714ce7 100644 --- a/service/src/main/resources/config/bootstrap.yml +++ b/service/src/main/resources/config/bootstrap.yml @@ -16,14 +16,16 @@ spring: --- # For the dev profile, check localhost for the config server by default -spring.config.activate.on-profile: 'dev' +spring: + config.activate.on-profile: 'dev' cloud: config: uri: '${CONFIG_SERVER_URL:http://localhost:8888/configserver}' --- -spring.config.activate.on-profile: 'consul' +spring: + config.activate.on-profile: 'consul' cloud: config: # Use Consul to locate the configuration server and bootstrap app config. From ac82994b41711e6c39f6ee24594101f23668aa11 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 17 Sep 2021 12:58:20 -0400 Subject: [PATCH 03/54] Updated webjars-locator-core version to get it working again. --- service/pom.xml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/service/pom.xml b/service/pom.xml index 82cd47f..bcb822d 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -21,8 +21,9 @@ datawave.microservice.dictionary.DictionaryService 1.2-SNAPSHOT 1.6.6-SNAPSHOT - 1.10.19 - 3.3.1-2 + 1.10.25 + 3.6.0 + 0.48 @@ -46,6 +47,11 @@ jquery ${version.webjars.jquery} + + org.webjars + webjars-locator-core + ${version.webjars.locator} + gov.nsa.datawave.microservice spring-boot-starter-datawave From 68550fd6932bcce5cdc62b24b6abbd8b98445278 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 27 Jan 2022 11:14:22 -0500 Subject: [PATCH 04/54] Updated datawave microservice parent version to 1.9-SNAPSHOT. --- api/pom.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 4c3e6f7..03345a8 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 1.8 + 1.9-SNAPSHOT dictionary-api diff --git a/pom.xml b/pom.xml index 1fb7987..4f79a9d 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 1.8 + 1.9-SNAPSHOT dictionary-service-parent From 0b38338d3a68e852b79e56bae417525b483fc777 Mon Sep 17 00:00:00 2001 From: cmitchell Date: Tue, 19 Apr 2022 11:22:49 -0400 Subject: [PATCH 05/54] Api and service changes for hte model bean (not complete - tests and cleanup to follow) --- api/pom.xml | 17 +- .../data}/DataDictionaryBase.java | 4 +- .../data}/DefaultDataDictionary.java | 4 +- .../data}/DefaultDescription.java | 2 +- .../data}/DefaultDictionaryField.java | 2 +- .../data}/DefaultFields.java | 2 +- .../data}/DescriptionBase.java | 2 +- .../data}/DictionaryFieldBase.java | 2 +- .../data}/FieldsBase.java | 2 +- .../edge}/DefaultEdgeDictionary.java | 2 +- .../edge}/DefaultMetadata.java | 2 +- .../edge}/EdgeDictionaryBase.java | 2 +- .../edge}/EventField.java | 2 +- .../edge}/MetadataBase.java | 2 +- .../metadata/DefaultMetadataField.java | 72 +-- .../metadata/MetadataFieldBase.java | 5 +- .../datawave/webservice/model/Direction.java | 26 ++ .../datawave/webservice/model/Mapping.java | 49 ++ .../java/datawave/webservice/model/Model.java | 110 +++++ .../webservice/model/ModelKeyParser.java | 114 +++++ .../datawave/webservice/model/ModelList.java | 116 +++++ .../DefaultDescriptionTest.java | 9 +- service/pom.xml | 15 +- .../AccumuloConnectionService.java | 76 +++ .../datawave/microservice/Connection.java | 85 ++++ ...ons.java => DataDictionaryController.java} | 40 +- ...ons.java => EdgeDictionaryController.java} | 12 +- .../DictionaryServiceConfiguration.java | 28 +- .../config/ResponseObjectFactory.java | 10 +- .../dictionary/data/DataDictionary.java | 35 ++ ...onaryImpl.java => DataDictionaryImpl.java} | 40 +- .../data/DatawaveDataDictionary.java | 34 -- ...dgeDictionary.java => EdgeDictionary.java} | 6 +- ...onaryImpl.java => EdgeDictionaryImpl.java} | 13 +- .../metadata/DefaultMetadataFieldScanner.java | 16 +- .../metadata/MetadataDescriptionsHelper.java | 2 +- .../MetadataDescriptionsHelperFactory.java | 2 +- .../microservice/model/ModelController.java | 431 ++++++++++++++++++ .../model/config/ModelConfiguration.java | 18 + .../model/config/ModelProperties.java | 18 + ...java => DataDictionaryControllerTest.java} | 22 +- ...lTest.java => DataDictionaryImplTest.java} | 33 +- ...lTest.java => EdgeDictionaryImplTest.java} | 46 +- .../DefaultMetadataFieldScannerTest.java | 23 +- 44 files changed, 1268 insertions(+), 285 deletions(-) rename api/src/main/java/datawave/webservice/{results/datadictionary => dictionary/data}/DataDictionaryBase.java (90%) rename api/src/main/java/datawave/webservice/{results/datadictionary => dictionary/data}/DefaultDataDictionary.java (98%) rename api/src/main/java/datawave/webservice/{results/datadictionary => dictionary/data}/DefaultDescription.java (98%) rename api/src/main/java/datawave/webservice/{results/datadictionary => dictionary/data}/DefaultDictionaryField.java (99%) rename api/src/main/java/datawave/webservice/{results/datadictionary => dictionary/data}/DefaultFields.java (99%) rename api/src/main/java/datawave/webservice/{results/datadictionary => dictionary/data}/DescriptionBase.java (94%) rename api/src/main/java/datawave/webservice/{results/datadictionary => dictionary/data}/DictionaryFieldBase.java (93%) rename api/src/main/java/datawave/webservice/{results/datadictionary => dictionary/data}/FieldsBase.java (94%) rename api/src/main/java/datawave/webservice/{results/edgedictionary => dictionary/edge}/DefaultEdgeDictionary.java (99%) rename api/src/main/java/datawave/webservice/{results/edgedictionary => dictionary/edge}/DefaultMetadata.java (99%) rename api/src/main/java/datawave/webservice/{results/edgedictionary => dictionary/edge}/EdgeDictionaryBase.java (94%) rename api/src/main/java/datawave/webservice/{results/edgedictionary => dictionary/edge}/EventField.java (99%) rename api/src/main/java/datawave/webservice/{results/edgedictionary => dictionary/edge}/MetadataBase.java (97%) rename api/src/main/java/datawave/webservice/{query/result => }/metadata/DefaultMetadataField.java (86%) rename api/src/main/java/datawave/webservice/{query/result => }/metadata/MetadataFieldBase.java (87%) create mode 100644 api/src/main/java/datawave/webservice/model/Direction.java create mode 100644 api/src/main/java/datawave/webservice/model/Mapping.java create mode 100644 api/src/main/java/datawave/webservice/model/Model.java create mode 100644 api/src/main/java/datawave/webservice/model/ModelKeyParser.java create mode 100644 api/src/main/java/datawave/webservice/model/ModelList.java create mode 100644 service/src/main/java/datawave/microservice/AccumuloConnectionService.java create mode 100644 service/src/main/java/datawave/microservice/Connection.java rename service/src/main/java/datawave/microservice/dictionary/{DataDictionaryOperations.java => DataDictionaryController.java} (90%) rename service/src/main/java/datawave/microservice/dictionary/{EdgeDictionaryOperations.java => EdgeDictionaryController.java} (89%) create mode 100644 service/src/main/java/datawave/microservice/dictionary/data/DataDictionary.java rename service/src/main/java/datawave/microservice/dictionary/data/{DatawaveDataDictionaryImpl.java => DataDictionaryImpl.java} (85%) delete mode 100644 service/src/main/java/datawave/microservice/dictionary/data/DatawaveDataDictionary.java rename service/src/main/java/datawave/microservice/dictionary/edge/{DatawaveEdgeDictionary.java => EdgeDictionary.java} (58%) rename service/src/main/java/datawave/microservice/dictionary/edge/{DefaultDatawaveEdgeDictionaryImpl.java => EdgeDictionaryImpl.java} (89%) create mode 100644 service/src/main/java/datawave/microservice/model/ModelController.java create mode 100644 service/src/main/java/datawave/microservice/model/config/ModelConfiguration.java create mode 100644 service/src/main/java/datawave/microservice/model/config/ModelProperties.java rename service/src/test/java/datawave/microservice/dictionary/{DataDictionaryOperationsTest.java => DataDictionaryControllerTest.java} (94%) rename service/src/test/java/datawave/microservice/dictionary/data/{DatawaveDataDictionaryImplTest.java => DataDictionaryImplTest.java} (94%) rename service/src/test/java/datawave/microservice/dictionary/edge/{DefaultDatawaveEdgeDictionaryImplTest.java => EdgeDictionaryImplTest.java} (83%) diff --git a/api/pom.xml b/api/pom.xml index 03345a8..6532456 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -44,18 +44,19 @@ base-rest-responses - junit - junit - test + jakarta.xml.bind + jakarta.xml.bind-api + 2.3.3 + compile - org.junit.jupiter - junit-jupiter-engine - test + org.projectlombok + lombok + provided - org.junit.vintage - junit-vintage-engine + org.junit.jupiter + junit-jupiter-engine test diff --git a/api/src/main/java/datawave/webservice/results/datadictionary/DataDictionaryBase.java b/api/src/main/java/datawave/webservice/dictionary/data/DataDictionaryBase.java similarity index 90% rename from api/src/main/java/datawave/webservice/results/datadictionary/DataDictionaryBase.java rename to api/src/main/java/datawave/webservice/dictionary/data/DataDictionaryBase.java index 6d469e7..2c573ca 100644 --- a/api/src/main/java/datawave/webservice/results/datadictionary/DataDictionaryBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DataDictionaryBase.java @@ -1,8 +1,8 @@ -package datawave.webservice.results.datadictionary; +package datawave.webservice.dictionary.data; import io.protostuff.Message; import datawave.webservice.HtmlProvider; -import datawave.webservice.query.result.metadata.MetadataFieldBase; +import datawave.webservice.metadata.MetadataFieldBase; import datawave.webservice.result.BaseResponse; import datawave.webservice.result.TotalResultsAware; diff --git a/api/src/main/java/datawave/webservice/results/datadictionary/DefaultDataDictionary.java b/api/src/main/java/datawave/webservice/dictionary/data/DefaultDataDictionary.java similarity index 98% rename from api/src/main/java/datawave/webservice/results/datadictionary/DefaultDataDictionary.java rename to api/src/main/java/datawave/webservice/dictionary/data/DefaultDataDictionary.java index 7739b43..98017d0 100644 --- a/api/src/main/java/datawave/webservice/results/datadictionary/DefaultDataDictionary.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DefaultDataDictionary.java @@ -1,4 +1,4 @@ -package datawave.webservice.results.datadictionary; +package datawave.webservice.dictionary.data; import java.io.IOException; import java.text.MessageFormat; @@ -19,7 +19,7 @@ import com.google.common.collect.Lists; import datawave.webservice.HtmlProvider; -import datawave.webservice.query.result.metadata.DefaultMetadataField; +import datawave.webservice.metadata.DefaultMetadataField; import datawave.webservice.result.TotalResultsAware; import io.protostuff.Input; diff --git a/api/src/main/java/datawave/webservice/results/datadictionary/DefaultDescription.java b/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java similarity index 98% rename from api/src/main/java/datawave/webservice/results/datadictionary/DefaultDescription.java rename to api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java index 054a191..bd165e4 100644 --- a/api/src/main/java/datawave/webservice/results/datadictionary/DefaultDescription.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java @@ -1,4 +1,4 @@ -package datawave.webservice.results.datadictionary; +package datawave.webservice.dictionary.data; import datawave.webservice.query.result.event.MapSchema; import io.protostuff.Input; diff --git a/api/src/main/java/datawave/webservice/results/datadictionary/DefaultDictionaryField.java b/api/src/main/java/datawave/webservice/dictionary/data/DefaultDictionaryField.java similarity index 99% rename from api/src/main/java/datawave/webservice/results/datadictionary/DefaultDictionaryField.java rename to api/src/main/java/datawave/webservice/dictionary/data/DefaultDictionaryField.java index 92ddda9..92592ad 100644 --- a/api/src/main/java/datawave/webservice/results/datadictionary/DefaultDictionaryField.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DefaultDictionaryField.java @@ -1,4 +1,4 @@ -package datawave.webservice.results.datadictionary; +package datawave.webservice.dictionary.data; import java.io.IOException; import java.util.Collection; diff --git a/api/src/main/java/datawave/webservice/results/datadictionary/DefaultFields.java b/api/src/main/java/datawave/webservice/dictionary/data/DefaultFields.java similarity index 99% rename from api/src/main/java/datawave/webservice/results/datadictionary/DefaultFields.java rename to api/src/main/java/datawave/webservice/dictionary/data/DefaultFields.java index c4db2b7..f36eef1 100644 --- a/api/src/main/java/datawave/webservice/results/datadictionary/DefaultFields.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DefaultFields.java @@ -1,4 +1,4 @@ -package datawave.webservice.results.datadictionary; +package datawave.webservice.dictionary.data; import com.google.common.collect.Lists; import com.google.common.collect.Multimap; diff --git a/api/src/main/java/datawave/webservice/results/datadictionary/DescriptionBase.java b/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java similarity index 94% rename from api/src/main/java/datawave/webservice/results/datadictionary/DescriptionBase.java rename to api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java index 93a50d4..1b961de 100644 --- a/api/src/main/java/datawave/webservice/results/datadictionary/DescriptionBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java @@ -1,4 +1,4 @@ -package datawave.webservice.results.datadictionary; +package datawave.webservice.dictionary.data; import io.protostuff.Message; import datawave.webservice.query.result.event.HasMarkings; diff --git a/api/src/main/java/datawave/webservice/results/datadictionary/DictionaryFieldBase.java b/api/src/main/java/datawave/webservice/dictionary/data/DictionaryFieldBase.java similarity index 93% rename from api/src/main/java/datawave/webservice/results/datadictionary/DictionaryFieldBase.java rename to api/src/main/java/datawave/webservice/dictionary/data/DictionaryFieldBase.java index de116b1..8522473 100644 --- a/api/src/main/java/datawave/webservice/results/datadictionary/DictionaryFieldBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DictionaryFieldBase.java @@ -1,4 +1,4 @@ -package datawave.webservice.results.datadictionary; +package datawave.webservice.dictionary.data; import io.protostuff.Message; diff --git a/api/src/main/java/datawave/webservice/results/datadictionary/FieldsBase.java b/api/src/main/java/datawave/webservice/dictionary/data/FieldsBase.java similarity index 94% rename from api/src/main/java/datawave/webservice/results/datadictionary/FieldsBase.java rename to api/src/main/java/datawave/webservice/dictionary/data/FieldsBase.java index c63bdeb..8f6672d 100644 --- a/api/src/main/java/datawave/webservice/results/datadictionary/FieldsBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/FieldsBase.java @@ -1,4 +1,4 @@ -package datawave.webservice.results.datadictionary; +package datawave.webservice.dictionary.data; import com.google.common.collect.Multimap; import io.protostuff.Message; diff --git a/api/src/main/java/datawave/webservice/results/edgedictionary/DefaultEdgeDictionary.java b/api/src/main/java/datawave/webservice/dictionary/edge/DefaultEdgeDictionary.java similarity index 99% rename from api/src/main/java/datawave/webservice/results/edgedictionary/DefaultEdgeDictionary.java rename to api/src/main/java/datawave/webservice/dictionary/edge/DefaultEdgeDictionary.java index fb61f9b..745ae69 100644 --- a/api/src/main/java/datawave/webservice/results/edgedictionary/DefaultEdgeDictionary.java +++ b/api/src/main/java/datawave/webservice/dictionary/edge/DefaultEdgeDictionary.java @@ -1,4 +1,4 @@ -package datawave.webservice.results.edgedictionary; +package datawave.webservice.dictionary.edge; import java.io.IOException; import java.util.ArrayList; diff --git a/api/src/main/java/datawave/webservice/results/edgedictionary/DefaultMetadata.java b/api/src/main/java/datawave/webservice/dictionary/edge/DefaultMetadata.java similarity index 99% rename from api/src/main/java/datawave/webservice/results/edgedictionary/DefaultMetadata.java rename to api/src/main/java/datawave/webservice/dictionary/edge/DefaultMetadata.java index a7e2bbf..52be165 100644 --- a/api/src/main/java/datawave/webservice/results/edgedictionary/DefaultMetadata.java +++ b/api/src/main/java/datawave/webservice/dictionary/edge/DefaultMetadata.java @@ -1,4 +1,4 @@ -package datawave.webservice.results.edgedictionary; +package datawave.webservice.dictionary.edge; import java.io.IOException; import java.io.Serializable; diff --git a/api/src/main/java/datawave/webservice/results/edgedictionary/EdgeDictionaryBase.java b/api/src/main/java/datawave/webservice/dictionary/edge/EdgeDictionaryBase.java similarity index 94% rename from api/src/main/java/datawave/webservice/results/edgedictionary/EdgeDictionaryBase.java rename to api/src/main/java/datawave/webservice/dictionary/edge/EdgeDictionaryBase.java index 5ddbab6..759122e 100644 --- a/api/src/main/java/datawave/webservice/results/edgedictionary/EdgeDictionaryBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/edge/EdgeDictionaryBase.java @@ -1,4 +1,4 @@ -package datawave.webservice.results.edgedictionary; +package datawave.webservice.dictionary.edge; import java.util.List; diff --git a/api/src/main/java/datawave/webservice/results/edgedictionary/EventField.java b/api/src/main/java/datawave/webservice/dictionary/edge/EventField.java similarity index 99% rename from api/src/main/java/datawave/webservice/results/edgedictionary/EventField.java rename to api/src/main/java/datawave/webservice/dictionary/edge/EventField.java index 6faf7f5..c69db71 100644 --- a/api/src/main/java/datawave/webservice/results/edgedictionary/EventField.java +++ b/api/src/main/java/datawave/webservice/dictionary/edge/EventField.java @@ -1,4 +1,4 @@ -package datawave.webservice.results.edgedictionary; +package datawave.webservice.dictionary.edge; import java.io.IOException; import java.io.Serializable; diff --git a/api/src/main/java/datawave/webservice/results/edgedictionary/MetadataBase.java b/api/src/main/java/datawave/webservice/dictionary/edge/MetadataBase.java similarity index 97% rename from api/src/main/java/datawave/webservice/results/edgedictionary/MetadataBase.java rename to api/src/main/java/datawave/webservice/dictionary/edge/MetadataBase.java index eb1e100..9067d2f 100644 --- a/api/src/main/java/datawave/webservice/results/edgedictionary/MetadataBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/edge/MetadataBase.java @@ -1,4 +1,4 @@ -package datawave.webservice.results.edgedictionary; +package datawave.webservice.dictionary.edge; import java.util.List; import java.util.Map; diff --git a/api/src/main/java/datawave/webservice/query/result/metadata/DefaultMetadataField.java b/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java similarity index 86% rename from api/src/main/java/datawave/webservice/query/result/metadata/DefaultMetadataField.java rename to api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java index 4653914..9ad382b 100644 --- a/api/src/main/java/datawave/webservice/query/result/metadata/DefaultMetadataField.java +++ b/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java @@ -1,12 +1,13 @@ -package datawave.webservice.query.result.metadata; +package datawave.webservice.metadata; import datawave.webservice.query.result.event.MapSchema; -import datawave.webservice.results.datadictionary.DefaultDescription; -import datawave.webservice.results.datadictionary.DescriptionBase; +import datawave.webservice.dictionary.data.DefaultDescription; +import datawave.webservice.dictionary.data.DescriptionBase; import io.protostuff.Input; import io.protostuff.Message; import io.protostuff.Output; import io.protostuff.Schema; +import lombok.Data; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; @@ -27,6 +28,7 @@ import java.util.Map; import java.util.Set; +@Data @XmlAccessorType(XmlAccessType.NONE) @XmlType(propOrder = {"fieldName", "internalFieldName", "dataType", "descriptions", "indexOnly", "forwardIndexed", "reverseIndexed", "normalized", "tokenized", "types"}) @@ -68,70 +70,26 @@ public class DefaultMetadataField extends MetadataFieldBase(); @@ -139,30 +97,10 @@ public void addType(String type) { types.add(type); } - public List getTypes() { - return types; - } - - public void setTypes(List types) { - this.types = types; - } - - public Set getDescriptions() { - return descriptions; - } - public void setDescription(Collection descriptions) { this.descriptions = new HashSet<>(descriptions); } - public void setLastUpdated(String lastUpdated) { - this.lastUpdated = lastUpdated; - } - - public String getLastUpdated() { - return this.lastUpdated; - } - @Override public int hashCode() { return new HashCodeBuilder(6197, 7993).append(this.fieldName).hashCode(); diff --git a/api/src/main/java/datawave/webservice/query/result/metadata/MetadataFieldBase.java b/api/src/main/java/datawave/webservice/metadata/MetadataFieldBase.java similarity index 87% rename from api/src/main/java/datawave/webservice/query/result/metadata/MetadataFieldBase.java rename to api/src/main/java/datawave/webservice/metadata/MetadataFieldBase.java index 8fbf52f..fc7eaaa 100644 --- a/api/src/main/java/datawave/webservice/query/result/metadata/MetadataFieldBase.java +++ b/api/src/main/java/datawave/webservice/metadata/MetadataFieldBase.java @@ -1,7 +1,7 @@ -package datawave.webservice.query.result.metadata; +package datawave.webservice.metadata; import io.protostuff.Message; -import datawave.webservice.results.datadictionary.DescriptionBase; +import datawave.webservice.dictionary.data.DescriptionBase; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -11,6 +11,7 @@ import java.util.List; import java.util.Set; +// TODO if nothing other than DefaultMetadataField extends this class, remove this class and rename DefaultMetadataField to MetadataField @XmlAccessorType(XmlAccessType.NONE) @XmlSeeAlso(DefaultMetadataField.class) public abstract class MetadataFieldBase implements Serializable, Message { diff --git a/api/src/main/java/datawave/webservice/model/Direction.java b/api/src/main/java/datawave/webservice/model/Direction.java new file mode 100644 index 0000000..3484860 --- /dev/null +++ b/api/src/main/java/datawave/webservice/model/Direction.java @@ -0,0 +1,26 @@ +package datawave.webservice.model; + +import java.io.Serializable; + +import javax.xml.bind.annotation.XmlEnum; + +@XmlEnum +public enum Direction implements Serializable { + + FORWARD("forward"), REVERSE("reverse"); + + private final String value; + + Direction(String value) { + this.value = value; + } + + public String getValue() { + return this.value; + } + + public static Direction getDirection(String value) { + return Direction.valueOf(value.toUpperCase()); + } + +} diff --git a/api/src/main/java/datawave/webservice/model/Mapping.java b/api/src/main/java/datawave/webservice/model/Mapping.java new file mode 100644 index 0000000..ac0d50c --- /dev/null +++ b/api/src/main/java/datawave/webservice/model/Mapping.java @@ -0,0 +1,49 @@ +package datawave.webservice.model; + +import java.io.Serializable; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; + +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.commons.lang.builder.CompareToBuilder; + +@Data +@NoArgsConstructor +@XmlAccessorType(XmlAccessType.NONE) +public class Mapping implements Serializable, Comparable { + + private static final long serialVersionUID = 1L; + + @XmlAttribute(required = true) + private String datatype; + + @XmlAttribute(required = true) + private String fieldName; + + @XmlAttribute(required = true) + private String modelFieldName; + + @XmlAttribute(required = true) + private Direction direction; + + @XmlAttribute(required = true) + private String columnVisibility; + + @Override + public int compareTo(Mapping obj) { + + if (obj == null) { + throw new IllegalArgumentException("can not compare null"); + } + + if (obj == this) + return 0; + + return new CompareToBuilder().append(datatype, obj.datatype).append(fieldName, obj.fieldName).append(modelFieldName, obj.modelFieldName) + .append(direction, obj.direction).append(columnVisibility, obj.columnVisibility).toComparison(); + } + +} diff --git a/api/src/main/java/datawave/webservice/model/Model.java b/api/src/main/java/datawave/webservice/model/Model.java new file mode 100644 index 0000000..46246dd --- /dev/null +++ b/api/src/main/java/datawave/webservice/model/Model.java @@ -0,0 +1,110 @@ +package datawave.webservice.model; + +import java.io.Serializable; +import java.text.MessageFormat; +import java.util.TreeSet; + +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlElementWrapper; +import javax.xml.bind.annotation.XmlRootElement; + +import datawave.webservice.HtmlProvider; +import datawave.webservice.result.BaseResponse; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@EqualsAndHashCode(callSuper = true) +@XmlRootElement +public class Model extends BaseResponse implements Serializable, HtmlProvider { + + private static final long serialVersionUID = 1L; + private String jqueryUri; + private String dataTablesUri; + private static final String TITLE = "Model Description", EMPTY = ""; + private static final String DATA_TABLES_TEMPLATE = "\n" + + "\n" + "\n"; + + public Model(String jqueryUri, String datatablesUri) { + this.jqueryUri = jqueryUri; + this.dataTablesUri = datatablesUri; + } + + @XmlAttribute(required = true) + private String name; + + @XmlElementWrapper(name = "Mappings") + @XmlElement(name = "Mappings") + private TreeSet mappings = new TreeSet(); + + /* + * (non-Javadoc) + * + * @see datawave.webservice.HtmlProvider#getTitle() + */ + @Override + public String getTitle() { + return TITLE; + } + + /* + * (non-Javadoc) + * + * @see datawave.webservice.HtmlProvider#getPageHeader() + */ + @Override + public String getPageHeader() { + return TITLE; + } + + /* + * (non-Javadoc) + * + * @see datawave.webservice.HtmlProvider#getHeadContent() + */ + @Override + public String getHeadContent() { + return MessageFormat.format(DATA_TABLES_TEMPLATE, jqueryUri, dataTablesUri); + } + + /* + * (non-Javadoc) + * + * @see datawave.webservice.HtmlProvider#getMainContent() + */ + @Override + public String getMainContent() { + StringBuilder builder = new StringBuilder(); + + builder.append("
\n"); + builder.append("
\n"); + builder.append("\n"); + + builder.append(""); + builder.append(""); + + for (Mapping f : this.getMappings()) { + builder.append(""); + builder.append(""); + builder.append(""); + builder.append(""); + builder.append(""); + builder.append(""); + } + + builder.append(""); + builder.append("
VisibilityFieldNameDataTypeModelFieldNameDirection
").append(f.getColumnVisibility()).append("").append(f.getFieldName()).append("").append(f.getDatatype()).append("").append(f.getModelFieldName()).append("").append(f.getDirection()).append("
\n"); + builder.append("
\n"); + builder.append("
\n"); + builder.append("
"); + + return builder.toString(); + } + +} diff --git a/api/src/main/java/datawave/webservice/model/ModelKeyParser.java b/api/src/main/java/datawave/webservice/model/ModelKeyParser.java new file mode 100644 index 0000000..8bb106a --- /dev/null +++ b/api/src/main/java/datawave/webservice/model/ModelKeyParser.java @@ -0,0 +1,114 @@ +package datawave.webservice.model; + +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; + +import java.util.Set; + +public class ModelKeyParser { + + public static final String NULL_BYTE = "\0"; + public static final Value NULL_VALUE = new Value(new byte[0]); + private static final Logger log = Logger.getLogger(ModelKeyParser.class); + + public static Mapping parseKey(Key key, Set auths) { + Mapping mapping = new Mapping(); + String row = key.getRow().toString(); + String[] colf = key.getColumnFamily().toString().split(NULL_BYTE); + String[] colq = key.getColumnQualifier().toString().split(NULL_BYTE); + String cv = key.getColumnVisibility().toString(); + + String datatype = null; + Direction direction; + String dataField; + String modelField; + Boolean indexOnlyField = false; + + // colf.length == 1 is the Older style, no datatype and nothing to do. Start at length == 2 + if (colf.length == 2) { + datatype = colf[1]; + } else { + throw new IllegalArgumentException("Key in unknown format, colf parts: " + colf.length); + } + if (2 == colq.length) { + direction = Direction.getDirection(colq[1]); + if (Direction.REVERSE.equals(direction)) { + dataField = row; + modelField = colq[0]; + } else { + dataField = colq[0]; + modelField = row; + } + } else if (3 == colq.length && Direction.FORWARD == Direction.getDirection(colq[2])) { + dataField = colq[0]; + modelField = row; + direction = Direction.FORWARD; // we already checked it in the if condition + } else { + log.error("Error parsing key: " + key); + throw new IllegalArgumentException("Key in unknown format, colq parts: " + colq.length); + } + + mapping.setColumnVisibility(cv); + mapping.setDatatype(datatype); + mapping.setDirection(direction); + mapping.setFieldName(dataField); + mapping.setModelFieldName(modelField); + return mapping; + } + + public static Key createKey(Mapping mapping, String modelName) { + ColumnVisibility cv = new ColumnVisibility(mapping.getColumnVisibility()); + + String inName = Direction.REVERSE.equals(mapping.getDirection()) ? mapping.getFieldName() : mapping.getModelFieldName(); + String outName = Direction.REVERSE.equals(mapping.getDirection()) ? mapping.getModelFieldName() : mapping.getFieldName(); + String dataType = StringUtils.isEmpty(mapping.getDatatype()) ? "" : NULL_BYTE + mapping.getDatatype().trim(); + return new Key(inName, // Row + modelName + dataType, // ColFam + outName + NULL_BYTE + mapping.getDirection().getValue(), // ColQual + cv, // Visibility + System.currentTimeMillis() // Timestamp + ); + } + + public static Mutation createMutation(Mapping mapping, String modelName) { + ColumnVisibility cv = new ColumnVisibility(mapping.getColumnVisibility()); + Mutation m; + String dataType = StringUtils.isEmpty(mapping.getDatatype()) ? "" : NULL_BYTE + mapping.getDatatype().trim(); + + if (Direction.REVERSE.equals(mapping.getDirection())) { + // Reverse mappings should not have indexOnly designators. If they do, scrub it off. + m = new Mutation(mapping.getFieldName()); + m.put(modelName + dataType, mapping.getModelFieldName() + NULL_BYTE + mapping.getDirection().getValue(), cv, System.currentTimeMillis(), + NULL_VALUE); + return m; + } else { + m = new Mutation(mapping.getModelFieldName()); + m.put(modelName + dataType, mapping.getFieldName() + NULL_BYTE + mapping.getDirection().getValue(), cv, System.currentTimeMillis(), NULL_VALUE); + return m; + } + } + + public static Mutation createDeleteMutation(Mapping mapping, String modelName) { + ColumnVisibility cv = new ColumnVisibility(mapping.getColumnVisibility()); + Mutation m; + String dataType = StringUtils.isEmpty(mapping.getDatatype()) ? "" : NULL_BYTE + mapping.getDatatype().trim(); + + if (Direction.REVERSE.equals(mapping.getDirection())) { + m = new Mutation(mapping.getFieldName()); + m.putDelete(modelName + dataType, mapping.getModelFieldName() + NULL_BYTE + mapping.getDirection().getValue(), cv, System.currentTimeMillis()); + return m; + } else { + m = new Mutation(mapping.getModelFieldName()); + m.putDelete(modelName + dataType, mapping.getFieldName() + NULL_BYTE + mapping.getDirection().getValue(), cv, System.currentTimeMillis()); + m.putDelete(modelName + dataType, mapping.getFieldName() + NULL_BYTE + "index_only" + NULL_BYTE + mapping.getDirection().getValue(), cv, + System.currentTimeMillis()); + return m; + } + } + +} diff --git a/api/src/main/java/datawave/webservice/model/ModelList.java b/api/src/main/java/datawave/webservice/model/ModelList.java new file mode 100644 index 0000000..375daed --- /dev/null +++ b/api/src/main/java/datawave/webservice/model/ModelList.java @@ -0,0 +1,116 @@ +package datawave.webservice.model; + +import java.io.Serializable; +import java.text.MessageFormat; +import java.util.HashSet; + +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlElementWrapper; +import javax.xml.bind.annotation.XmlRootElement; + +import datawave.webservice.HtmlProvider; +import datawave.webservice.result.BaseResponse; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@EqualsAndHashCode(callSuper = true) +@XmlRootElement +public class ModelList extends BaseResponse implements Serializable, HtmlProvider { + + private String jqueryUri; + private String dataTablesUri; + private String modelTableName; + + private static final long serialVersionUID = 1L; + private static final String TITLE = "Model Names"; + private static final String DATA_TABLES_TEMPLATE = "\n" + + "\n" + "\n"; + + public ModelList(String jqueryUri, String datatablesUri, String modelTableName) { + this.jqueryUri = jqueryUri; + this.dataTablesUri = datatablesUri; + this.modelTableName = modelTableName; + } + + @XmlElementWrapper(name = "ModelNames") + @XmlElement(name = "ModelName") + private HashSet names = null; + + public HashSet getNames() { + return names; + } + + public void setNames(HashSet names) { + this.names = names; + } + + /* + * (non-Javadoc) + * + * @see datawave.webservice.HtmlProvider#getTitle() + */ + @Override + public String getTitle() { + return TITLE; + } + + /* + * (non-Javadoc) + * + * @see datawave.webservice.HtmlProvider#getPageHeader() + */ + @Override + public String getPageHeader() { + return getTitle(); + } + + /* + * (non-Javadoc) + * + * @see datawave.webservice.HtmlProvider#getHeadContent() + */ + @Override + public String getHeadContent() { + return MessageFormat.format(DATA_TABLES_TEMPLATE, jqueryUri, dataTablesUri); + } + + /* + * (non-Javadoc) + * + * @see datawave.webservice.HtmlProvider#getMainContent() + */ + @Override + public String getMainContent() { + StringBuilder builder = new StringBuilder(); + + if (this.getNames() == null || this.getNames().isEmpty()) { + builder.append("No models available."); + } else { + builder.append("
\n"); + builder.append("
\n"); + builder.append("\n"); + builder.append(""); + builder.append(""); + + for (String name : this.getNames()) { + // highlight alternating rows + builder.append(""); + builder.append(""); + builder.append("\n"); + } + builder.append(""); + builder.append("
Model Name
").append(name).append("
\n"); + builder.append("
\n"); + builder.append("
\n"); + builder.append("
"); + } + return builder.toString(); + } + +} diff --git a/api/src/test/java/datawave/webservice/results/datadictionary/DefaultDescriptionTest.java b/api/src/test/java/datawave/webservice/results/datadictionary/DefaultDescriptionTest.java index e5af511..6745901 100644 --- a/api/src/test/java/datawave/webservice/results/datadictionary/DefaultDescriptionTest.java +++ b/api/src/test/java/datawave/webservice/results/datadictionary/DefaultDescriptionTest.java @@ -1,6 +1,9 @@ package datawave.webservice.results.datadictionary; -import org.junit.Test; +import datawave.webservice.dictionary.data.DefaultDescription; +import datawave.webservice.dictionary.data.DefaultDictionaryField; +import datawave.webservice.dictionary.data.DefaultFields; +import org.junit.jupiter.api.Test; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; @@ -17,8 +20,8 @@ import java.util.Map; import java.util.Set; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; public class DefaultDescriptionTest { diff --git a/service/pom.xml b/service/pom.xml index bcb822d..e2858a0 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -95,13 +95,13 @@ webjars-locator-core - gov.nsa.datawave.contrib - datawave-in-memory-accumulo - test + org.projectlombok + lombok + provided - junit - junit + gov.nsa.datawave.contrib + datawave-in-memory-accumulo test @@ -109,11 +109,6 @@ junit-jupiter-engine test - - org.junit.vintage - junit-vintage-engine - test - diff --git a/service/src/main/java/datawave/microservice/AccumuloConnectionService.java b/service/src/main/java/datawave/microservice/AccumuloConnectionService.java new file mode 100644 index 0000000..cb457a7 --- /dev/null +++ b/service/src/main/java/datawave/microservice/AccumuloConnectionService.java @@ -0,0 +1,76 @@ +package datawave.microservice; + +import datawave.accumulo.util.security.UserAuthFunctions; +import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.dictionary.config.DataDictionaryProperties; +import datawave.security.authorization.DatawaveUser; +import datawave.security.util.ScannerHelper; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; + +import java.util.Set; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +/** + * Provides a connection to accumulo for use by controllers + * + * If other services (other than dictionary/model) are going to need to connect to accumulo, should we move this into the accumulo service and pass the + * individual properties (instead of a properties object and calling getters) + */ +@Service +public class AccumuloConnectionService { + + private final Connector accumuloConnector; + private final DataDictionaryProperties dataDictionaryConfiguration; + private final UserAuthFunctions userAuthFunctions; + + public AccumuloConnectionService(DataDictionaryProperties dataDictionaryConfiguration, UserAuthFunctions userAuthFunctions, + @Qualifier("warehouse") Connector accumuloConnector) { + this.dataDictionaryConfiguration = dataDictionaryConfiguration; + this.userAuthFunctions = userAuthFunctions; + this.accumuloConnector = accumuloConnector; + } + + public Connection getConnection(String modelTable, String modelName, ProxiedUserDetails user) { + return getConnection(dataDictionaryConfiguration.getMetadataTableName(), modelTable, modelName, user); + } + + public Connection getConnection(String metadataTable, String modelTable, String modelName, ProxiedUserDetails user) { + Connection helper = new Connection(); + helper.setMetadataTable(getSupplierValueIfBlank(metadataTable, dataDictionaryConfiguration::getMetadataTableName)); + helper.setModelTable(getSupplierValueIfBlank(modelTable, dataDictionaryConfiguration::getModelTableName)); + helper.setModelName(getSupplierValueIfBlank(modelName, dataDictionaryConfiguration::getModelName)); + helper.setAuths(getAuths(user)); + helper.setConnector(accumuloConnector); + return helper; + } + + private String getSupplierValueIfBlank(final String value, final Supplier supplier) { + return StringUtils.isBlank(value) ? supplier.get() : value; + } + + public Set getAuths(ProxiedUserDetails currentUser) { + //@formatter:off + return currentUser.getProxiedUsers().stream() + .map(DatawaveUser::getAuths) + .map(a -> new Authorizations(a.toArray(new String[0]))) + .collect(Collectors.toSet()); + //@formatter:on + } + + public Set getDowngradedAuthorizations(String requestedAuthorizations, ProxiedUserDetails currentUser) { + DatawaveUser primaryUser = currentUser.getPrimaryUser(); + return userAuthFunctions.mergeAuthorizations(userAuthFunctions.getRequestedAuthorizations(requestedAuthorizations, currentUser.getPrimaryUser()), + currentUser.getProxiedUsers(), u -> u != primaryUser); + } + + public Scanner getScanner(String modelTable, ProxiedUserDetails currentUser) throws TableNotFoundException { + return ScannerHelper.createScanner(this.accumuloConnector, modelTable, this.getAuths(currentUser)); + } +} diff --git a/service/src/main/java/datawave/microservice/Connection.java b/service/src/main/java/datawave/microservice/Connection.java new file mode 100644 index 0000000..f653e1b --- /dev/null +++ b/service/src/main/java/datawave/microservice/Connection.java @@ -0,0 +1,85 @@ +package datawave.microservice; + +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.commons.lang.builder.ToStringBuilder; + +import java.util.Objects; +import java.util.Set; + +public class Connection { + + private Connector connector; + + private Set auths; + + private String metadataTable; + + private String modelTable; + + private String modelName; + + public Connector getConnector() { + return connector; + } + + public void setConnector(Connector connector) { + this.connector = connector; + } + + public Set getAuths() { + return auths; + } + + public void setAuths(Set authorizations) { + this.auths = authorizations; + } + + public String getMetadataTable() { + return metadataTable; + } + + public void setMetadataTable(String metadataTable) { + this.metadataTable = metadataTable; + } + + public String getModelTable() { + return modelTable; + } + + public void setModelTable(String modelTable) { + this.modelTable = modelTable; + } + + public String getModelName() { + return modelName; + } + + public void setModelName(String modelName) { + this.modelName = modelName; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Connection that = (Connection) o; + return Objects.equals(connector, that.connector) && Objects.equals(auths, that.auths) && Objects.equals(metadataTable, that.metadataTable) + && Objects.equals(modelTable, that.modelTable) && Objects.equals(modelName, that.modelName); + } + + @Override + public int hashCode() { + return Objects.hash(connector, auths, metadataTable, modelTable, modelName); + } + + @Override + public String toString() { + return new ToStringBuilder(this).append("connector", connector).append("auths", auths).append("metadataTable", metadataTable) + .append("modelTable", modelTable).append("modelName", modelName).toString(); + } +} diff --git a/service/src/main/java/datawave/microservice/dictionary/DataDictionaryOperations.java b/service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java similarity index 90% rename from service/src/main/java/datawave/microservice/dictionary/DataDictionaryOperations.java rename to service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java index 4a85c23..1dcd106 100644 --- a/service/src/main/java/datawave/microservice/dictionary/DataDictionaryOperations.java +++ b/service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java @@ -8,15 +8,15 @@ import datawave.microservice.authorization.user.ProxiedUserDetails; import datawave.microservice.dictionary.config.DataDictionaryProperties; import datawave.microservice.dictionary.config.ResponseObjectFactory; -import datawave.microservice.dictionary.data.ConnectionConfig; -import datawave.microservice.dictionary.data.DatawaveDataDictionary; +import datawave.microservice.Connection; +import datawave.microservice.dictionary.data.DataDictionary; import datawave.security.authorization.DatawaveUser; -import datawave.webservice.query.result.metadata.MetadataFieldBase; +import datawave.webservice.metadata.MetadataFieldBase; import datawave.webservice.result.VoidResponse; -import datawave.webservice.results.datadictionary.DataDictionaryBase; -import datawave.webservice.results.datadictionary.DescriptionBase; -import datawave.webservice.results.datadictionary.DictionaryFieldBase; -import datawave.webservice.results.datadictionary.FieldsBase; +import datawave.webservice.dictionary.data.DataDictionaryBase; +import datawave.webservice.dictionary.data.DescriptionBase; +import datawave.webservice.dictionary.data.DictionaryFieldBase; +import datawave.webservice.dictionary.data.FieldsBase; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.security.Authorizations; import org.apache.commons.lang.StringUtils; @@ -54,10 +54,10 @@ @RequestMapping(path = "/data/v1", produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.TEXT_XML_VALUE, PROTOSTUFF_VALUE, MediaType.TEXT_HTML_VALUE, "text/x-yaml", "application/x-yaml"}) @EnableConfigurationProperties(DataDictionaryProperties.class) -public class DataDictionaryOperations,DICT extends DataDictionaryBase,META extends MetadataFieldBase,FIELD extends DictionaryFieldBase,FIELDS extends FieldsBase> { +public class DataDictionaryController,DICT extends DataDictionaryBase,META extends MetadataFieldBase,FIELD extends DictionaryFieldBase,FIELDS extends FieldsBase> { private final DataDictionaryProperties dataDictionaryConfiguration; - private final DatawaveDataDictionary dataDictionary; + private final DataDictionary dataDictionary; private final ResponseObjectFactory responseObjectFactory; private final UserAuthFunctions userAuthFunctions; private final Connector accumuloConnector; @@ -68,7 +68,7 @@ public class DataDictionaryOperations,DICT ex } }; - public DataDictionaryOperations(DataDictionaryProperties dataDictionaryConfiguration, DatawaveDataDictionary dataDictionary, + public DataDictionaryController(DataDictionaryProperties dataDictionaryConfiguration, DataDictionary dataDictionary, ResponseObjectFactory responseObjectFactory, UserAuthFunctions userAuthFunctions, @Qualifier("warehouse") Connector accumuloConnector) { this.dataDictionaryConfiguration = dataDictionaryConfiguration; @@ -86,7 +86,7 @@ public DataDictionaryBase get(@RequestParam(required = false) String @RequestParam(required = false) String metadataTableName, @RequestParam(name = "auths", required = false) String queryAuthorizations, @RequestParam(defaultValue = "") String dataTypeFilters, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { - ConnectionConfig connectionConfig = getConnectionConfig(metadataTableName, modelTableName, modelName, currentUser); + Connection connectionConfig = getConnectionConfig(metadataTableName, modelTableName, modelName, currentUser); // If the user provides authorizations, intersect it with their actual authorizations connectionConfig.setAuths(getDowngradedAuthorizations(queryAuthorizations, currentUser)); @@ -120,7 +120,7 @@ public DataDictionaryBase get(@RequestParam(required = false) String @Timed(name = "dw.dictionary.data.uploadDescriptions", absolute = true) public VoidResponse uploadDescriptions(@RequestBody FIELDS fields, @RequestParam(required = false) String modelName, @RequestParam(required = false) String modelTable, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { - ConnectionConfig connectionConfig = getConnectionConfig(modelTable, modelName, currentUser); + Connection connectionConfig = getConnectionConfig(modelTable, modelName, currentUser); List list = fields.getFields(); for (FIELD desc : list) { dataDictionary.setDescription(connectionConfig, desc); @@ -195,7 +195,7 @@ public VoidResponse setDescriptionPost(@RequestParam String fieldName, @RequestP desc.setMarkings(markings); desc.setDescription(description); - ConnectionConfig connectionConfig = getConnectionConfig(modelTable, modelName, currentUser); + Connection connectionConfig = getConnectionConfig(modelTable, modelName, currentUser); dataDictionary.setDescription(connectionConfig, fieldName, datatype, desc); // TODO: reload model table cache? // cache.reloadCache(modelTable); @@ -220,7 +220,7 @@ public VoidResponse setDescriptionPost(@RequestParam String fieldName, @RequestP @Timed(name = "dw.dictionary.data.allDescriptions", absolute = true) public FIELDS allDescriptions(@RequestParam(required = false) String modelName, @RequestParam(required = false) String modelTable, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { - ConnectionConfig connectionConfig = getConnectionConfig(modelTable, modelName, currentUser); + Connection connectionConfig = getConnectionConfig(modelTable, modelName, currentUser); Multimap,DESC> descriptions = dataDictionary.getDescriptions(connectionConfig); FIELDS response = this.responseObjectFactory.getFields(); response.setDescriptions(descriptions); @@ -246,7 +246,7 @@ public FIELDS allDescriptions(@RequestParam(required = false) String modelName, @Timed(name = "dw.dictionary.data.datatypeDescriptions", absolute = true) public FIELDS datatypeDescriptions(@PathVariable("datatype") String datatype, @RequestParam(required = false) String modelName, @RequestParam(required = false) String modelTable, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { - ConnectionConfig connectionConfig = getConnectionConfig(modelTable, modelName, currentUser); + Connection connectionConfig = getConnectionConfig(modelTable, modelName, currentUser); Multimap,DESC> descriptions = dataDictionary.getDescriptions(connectionConfig, datatype); FIELDS response = this.responseObjectFactory.getFields(); response.setDescriptions(descriptions); @@ -274,7 +274,7 @@ public FIELDS datatypeDescriptions(@PathVariable("datatype") String datatype, @R @Timed(name = "dw.dictionary.data.fieldNameDescription", absolute = true) public FIELDS fieldNameDescription(@PathVariable String fieldName, @PathVariable String datatype, @RequestParam(required = false) String modelName, @RequestParam(required = false) String modelTable, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { - ConnectionConfig connectionConfig = getConnectionConfig(modelTable, modelName, currentUser); + Connection connectionConfig = getConnectionConfig(modelTable, modelName, currentUser); Set descriptions = dataDictionary.getDescriptions(connectionConfig, fieldName, datatype); FIELDS response = responseObjectFactory.getFields(); if (!descriptions.isEmpty()) { @@ -319,7 +319,7 @@ public VoidResponse deleteDescription(@PathVariable String fieldName, @PathVaria desc.setDescription(""); desc.setMarkings(markings); - ConnectionConfig connectionConfig = getConnectionConfig(modelTable, modelName, currentUser); + Connection connectionConfig = getConnectionConfig(modelTable, modelName, currentUser); dataDictionary.deleteDescription(connectionConfig, fieldName, datatype, desc); // TODO: reload model table cache? @@ -327,12 +327,12 @@ public VoidResponse deleteDescription(@PathVariable String fieldName, @PathVaria return new VoidResponse(); } - private ConnectionConfig getConnectionConfig(String modelTable, String modelName, ProxiedUserDetails user) { + private Connection getConnectionConfig(String modelTable, String modelName, ProxiedUserDetails user) { return getConnectionConfig(dataDictionaryConfiguration.getMetadataTableName(), modelTable, modelName, user); } - private ConnectionConfig getConnectionConfig(String metadataTable, String modelTable, String modelName, ProxiedUserDetails user) { - ConnectionConfig helper = new ConnectionConfig(); + private Connection getConnectionConfig(String metadataTable, String modelTable, String modelName, ProxiedUserDetails user) { + Connection helper = new Connection(); helper.setMetadataTable(getSupplierValueIfBlank(metadataTable, dataDictionaryConfiguration::getMetadataTableName)); helper.setModelTable(getSupplierValueIfBlank(modelTable, dataDictionaryConfiguration::getModelTableName)); helper.setModelName(getSupplierValueIfBlank(modelName, dataDictionaryConfiguration::getModelName)); diff --git a/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryOperations.java b/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java similarity index 89% rename from service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryOperations.java rename to service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java index 9a20961..d576fba 100644 --- a/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryOperations.java +++ b/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java @@ -4,10 +4,10 @@ import datawave.accumulo.util.security.UserAuthFunctions; import datawave.microservice.authorization.user.ProxiedUserDetails; import datawave.microservice.dictionary.config.EdgeDictionaryProperties; -import datawave.microservice.dictionary.edge.DatawaveEdgeDictionary; +import datawave.microservice.dictionary.edge.EdgeDictionary; import datawave.security.authorization.DatawaveUser; -import datawave.webservice.results.edgedictionary.EdgeDictionaryBase; -import datawave.webservice.results.edgedictionary.MetadataBase; +import datawave.webservice.dictionary.edge.EdgeDictionaryBase; +import datawave.webservice.dictionary.edge.MetadataBase; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.security.Authorizations; import org.apache.commons.lang.StringUtils; @@ -32,15 +32,15 @@ @RequestMapping(path = "/edge/v1", produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.TEXT_XML_VALUE, PROTOSTUFF_VALUE, MediaType.TEXT_HTML_VALUE, "text/x-yaml", "application/x-yaml"}) @EnableConfigurationProperties(EdgeDictionaryProperties.class) -public class EdgeDictionaryOperations,META extends MetadataBase> { +public class EdgeDictionaryController,META extends MetadataBase> { private Logger log = LoggerFactory.getLogger(getClass()); private final EdgeDictionaryProperties edgeDictionaryProperties; - private final DatawaveEdgeDictionary edgeDictionary; + private final EdgeDictionary edgeDictionary; private final UserAuthFunctions userAuthFunctions; private final Connector accumuloConnector; - public EdgeDictionaryOperations(EdgeDictionaryProperties edgeDictionaryProperties, DatawaveEdgeDictionary edgeDictionary, + public EdgeDictionaryController(EdgeDictionaryProperties edgeDictionaryProperties, EdgeDictionary edgeDictionary, UserAuthFunctions userAuthFunctions, @Qualifier("warehouse") Connector accumuloConnector) { this.edgeDictionaryProperties = edgeDictionaryProperties; this.edgeDictionary = edgeDictionary; diff --git a/service/src/main/java/datawave/microservice/dictionary/config/DictionaryServiceConfiguration.java b/service/src/main/java/datawave/microservice/dictionary/config/DictionaryServiceConfiguration.java index 65d446a..7eee282 100644 --- a/service/src/main/java/datawave/microservice/dictionary/config/DictionaryServiceConfiguration.java +++ b/service/src/main/java/datawave/microservice/dictionary/config/DictionaryServiceConfiguration.java @@ -5,19 +5,19 @@ import datawave.marking.MarkingFunctions; import datawave.microservice.config.accumulo.AccumuloProperties; import datawave.microservice.config.web.DatawaveServerProperties; -import datawave.microservice.dictionary.data.DatawaveDataDictionary; -import datawave.microservice.dictionary.data.DatawaveDataDictionaryImpl; -import datawave.microservice.dictionary.edge.DatawaveEdgeDictionary; -import datawave.microservice.dictionary.edge.DefaultDatawaveEdgeDictionaryImpl; +import datawave.microservice.dictionary.data.DataDictionary; +import datawave.microservice.dictionary.data.DataDictionaryImpl; +import datawave.microservice.dictionary.edge.EdgeDictionary; +import datawave.microservice.dictionary.edge.EdgeDictionaryImpl; import datawave.microservice.metadata.MetadataDescriptionsHelper; import datawave.microservice.metadata.MetadataDescriptionsHelperFactory; import datawave.query.util.MetadataHelperFactory; -import datawave.webservice.query.result.metadata.DefaultMetadataField; -import datawave.webservice.results.datadictionary.DefaultDataDictionary; -import datawave.webservice.results.datadictionary.DefaultDescription; -import datawave.webservice.results.datadictionary.DefaultDictionaryField; -import datawave.webservice.results.datadictionary.DefaultFields; -import datawave.webservice.results.datadictionary.FieldsBase; +import datawave.webservice.metadata.DefaultMetadataField; +import datawave.webservice.dictionary.data.DefaultDataDictionary; +import datawave.webservice.dictionary.data.DefaultDescription; +import datawave.webservice.dictionary.data.DefaultDictionaryField; +import datawave.webservice.dictionary.data.DefaultFields; +import datawave.webservice.dictionary.data.FieldsBase; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.jackson.Jackson2ObjectMapperBuilderCustomizer; @@ -51,16 +51,16 @@ public MetadataDescriptionsHelper metadataHelperWithDescript @Bean @ConditionalOnMissingBean - public DatawaveDataDictionary datawaveDataDictionary(MarkingFunctions markingFunctions, + public DataDictionary datawaveDataDictionary(MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, MetadataHelperFactory metadataHelperFactory, MetadataDescriptionsHelperFactory metadataDescriptionsHelperFactory) { - return new DatawaveDataDictionaryImpl(markingFunctions, responseObjectFactory, metadataHelperFactory, metadataDescriptionsHelperFactory); + return new DataDictionaryImpl(markingFunctions, responseObjectFactory, metadataHelperFactory, metadataDescriptionsHelperFactory); } @Bean @ConditionalOnMissingBean - public DatawaveEdgeDictionary datawaveEdgeDictionary(MetadataHelperFactory metadataHelperFactory) { - return new DefaultDatawaveEdgeDictionaryImpl(metadataHelperFactory); + public EdgeDictionary datawaveEdgeDictionary(MetadataHelperFactory metadataHelperFactory) { + return new EdgeDictionaryImpl(metadataHelperFactory); } @Bean diff --git a/service/src/main/java/datawave/microservice/dictionary/config/ResponseObjectFactory.java b/service/src/main/java/datawave/microservice/dictionary/config/ResponseObjectFactory.java index 1128a74..9df009f 100644 --- a/service/src/main/java/datawave/microservice/dictionary/config/ResponseObjectFactory.java +++ b/service/src/main/java/datawave/microservice/dictionary/config/ResponseObjectFactory.java @@ -1,10 +1,10 @@ package datawave.microservice.dictionary.config; -import datawave.webservice.query.result.metadata.MetadataFieldBase; -import datawave.webservice.results.datadictionary.DataDictionaryBase; -import datawave.webservice.results.datadictionary.DescriptionBase; -import datawave.webservice.results.datadictionary.DictionaryFieldBase; -import datawave.webservice.results.datadictionary.FieldsBase; +import datawave.webservice.metadata.MetadataFieldBase; +import datawave.webservice.dictionary.data.DataDictionaryBase; +import datawave.webservice.dictionary.data.DescriptionBase; +import datawave.webservice.dictionary.data.DictionaryFieldBase; +import datawave.webservice.dictionary.data.FieldsBase; public interface ResponseObjectFactory,DICT extends DataDictionaryBase,META extends MetadataFieldBase,FIELD extends DictionaryFieldBase,FIELDS extends FieldsBase> { DICT getDataDictionary(); diff --git a/service/src/main/java/datawave/microservice/dictionary/data/DataDictionary.java b/service/src/main/java/datawave/microservice/dictionary/data/DataDictionary.java new file mode 100644 index 0000000..49fae88 --- /dev/null +++ b/service/src/main/java/datawave/microservice/dictionary/data/DataDictionary.java @@ -0,0 +1,35 @@ +package datawave.microservice.dictionary.data; + +import com.google.common.collect.Multimap; +import datawave.microservice.Connection; +import datawave.webservice.metadata.MetadataFieldBase; +import datawave.webservice.dictionary.data.DescriptionBase; +import datawave.webservice.dictionary.data.DictionaryFieldBase; + +import java.util.Collection; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +public interface DataDictionary,DESC extends DescriptionBase,FIELD extends DictionaryFieldBase> { + + Map getNormalizationMap(); + + void setNormalizationMap(Map normalizationMap); + + Collection getFields(Connection connectionConfig, Collection dataTypeFilters, int numThreads) throws Exception; + + void setDescription(Connection connectionConfig, FIELD description) throws Exception; + + void setDescription(Connection connectionConfig, String fieldName, String datatype, DESC description) throws Exception; + + void setDescriptions(Connection connectionConfig, String fieldName, String datatype, Set descriptions) throws Exception; + + Multimap,DESC> getDescriptions(Connection connectionConfig) throws Exception; + + Multimap,DESC> getDescriptions(Connection connectionConfig, String datatype) throws Exception; + + Set getDescriptions(Connection connectionConfig, String fieldName, String datatype) throws Exception; + + void deleteDescription(Connection connectionConfig, String fieldName, String datatype, DESC description) throws Exception; +} diff --git a/service/src/main/java/datawave/microservice/dictionary/data/DatawaveDataDictionaryImpl.java b/service/src/main/java/datawave/microservice/dictionary/data/DataDictionaryImpl.java similarity index 85% rename from service/src/main/java/datawave/microservice/dictionary/data/DatawaveDataDictionaryImpl.java rename to service/src/main/java/datawave/microservice/dictionary/data/DataDictionaryImpl.java index 14bece1..2c8cc7b 100644 --- a/service/src/main/java/datawave/microservice/dictionary/data/DatawaveDataDictionaryImpl.java +++ b/service/src/main/java/datawave/microservice/dictionary/data/DataDictionaryImpl.java @@ -4,6 +4,7 @@ import com.google.common.collect.Maps; import com.google.common.collect.Multimap; import datawave.marking.MarkingFunctions; +import datawave.microservice.Connection; import datawave.microservice.dictionary.config.ResponseObjectFactory; import datawave.microservice.metadata.DefaultMetadataFieldScanner; import datawave.microservice.metadata.MetadataDescriptionsHelper; @@ -12,11 +13,11 @@ import datawave.query.util.MetadataEntry; import datawave.query.util.MetadataHelper; import datawave.query.util.MetadataHelperFactory; -import datawave.webservice.query.result.metadata.DefaultMetadataField; -import datawave.webservice.results.datadictionary.DefaultDataDictionary; -import datawave.webservice.results.datadictionary.DefaultDescription; -import datawave.webservice.results.datadictionary.DefaultDictionaryField; -import datawave.webservice.results.datadictionary.DefaultFields; +import datawave.webservice.metadata.DefaultMetadataField; +import datawave.webservice.dictionary.data.DefaultDataDictionary; +import datawave.webservice.dictionary.data.DefaultDescription; +import datawave.webservice.dictionary.data.DefaultDictionaryField; +import datawave.webservice.dictionary.data.DefaultFields; import org.apache.accumulo.core.client.TableNotFoundException; import java.util.Collection; @@ -27,7 +28,8 @@ import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; -public class DatawaveDataDictionaryImpl implements DatawaveDataDictionary { +// TODO Check if something else implements the DataDictionary interface, and if not, remove it and rename this class to just DataDictionary. +public class DataDictionaryImpl implements DataDictionary { private final MarkingFunctions markingFunctions; private final ResponseObjectFactory responseObjectFactory; @@ -35,7 +37,7 @@ public class DatawaveDataDictionaryImpl implements DatawaveDataDictionary metadataDescriptionsHelperFactory; private Map normalizationMap = Maps.newHashMap(); - public DatawaveDataDictionaryImpl(MarkingFunctions markingFunctions, + public DataDictionaryImpl(MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, MetadataHelperFactory metadataHelperFactory, MetadataDescriptionsHelperFactory metadataDescriptionsHelperFactory) { this.markingFunctions = markingFunctions; @@ -71,7 +73,7 @@ public void setNormalizationMap(Map normalizationMap) { * @return a collection of metadata fields */ @Override - public Collection getFields(ConnectionConfig connectionConfig, Collection dataTypeFilters, int numThreads) throws Exception { + public Collection getFields(Connection connectionConfig, Collection dataTypeFilters, int numThreads) throws Exception { Map aliases = getAliases(connectionConfig); DefaultMetadataFieldScanner scanner = new DefaultMetadataFieldScanner(markingFunctions, responseObjectFactory, normalizationMap, connectionConfig, numThreads); @@ -91,7 +93,7 @@ public Collection getFields(ConnectionConfig connectionCon * the description to add */ @Override - public void setDescription(ConnectionConfig connectionConfig, DefaultDictionaryField description) throws Exception { + public void setDescription(Connection connectionConfig, DefaultDictionaryField description) throws Exception { this.setDescriptions(connectionConfig, description.getFieldName(), description.getDatatype(), description.getDescriptions()); } @@ -112,7 +114,7 @@ public void setDescription(ConnectionConfig connectionConfig, DefaultDictionaryF * the description to add */ @Override - public void setDescription(ConnectionConfig connectionConfig, String fieldName, String datatype, DefaultDescription description) throws Exception { + public void setDescription(Connection connectionConfig, String fieldName, String datatype, DefaultDescription description) throws Exception { setDescriptions(connectionConfig, fieldName, datatype, Collections.singleton(description)); } @@ -133,7 +135,7 @@ public void setDescription(ConnectionConfig connectionConfig, String fieldName, * the descriptions to add */ @Override - public void setDescriptions(ConnectionConfig connectionConfig, String fieldName, String datatype, Set descriptions) throws Exception { + public void setDescriptions(Connection connectionConfig, String fieldName, String datatype, Set descriptions) throws Exception { // TODO - Consider skipping this. We check for an alias when retrieving descriptions or metadata fields anyway, rendering this unnecessary. String alias = getAlias(fieldName, connectionConfig); if (null != alias) { @@ -158,7 +160,7 @@ public void setDescriptions(ConnectionConfig connectionConfig, String fieldName, * @return a {@link Multimap} with {@literal } entry keys mapped to their associated descriptions */ @Override - public Multimap,DefaultDescription> getDescriptions(ConnectionConfig connectionConfig) throws Exception { + public Multimap,DefaultDescription> getDescriptions(Connection connectionConfig) throws Exception { MetadataDescriptionsHelper descriptionsHelper = getInitializedDescriptionsHelper(connectionConfig); Multimap descriptions = descriptionsHelper.getDescriptions((Set) null); return transformKeys(descriptions, connectionConfig); @@ -178,7 +180,7 @@ public Multimap,DefaultDescription> getDescriptions(Connect * @return a {@link Multimap} with {@literal } entry keys mapped to their associated descriptions */ @Override - public Multimap,DefaultDescription> getDescriptions(ConnectionConfig connectionConfig, String datatype) throws Exception { + public Multimap,DefaultDescription> getDescriptions(Connection connectionConfig, String datatype) throws Exception { MetadataDescriptionsHelper helper = getInitializedDescriptionsHelper(connectionConfig); Multimap descriptions = helper.getDescriptions(datatype); return transformKeys(descriptions, connectionConfig); @@ -200,7 +202,7 @@ public Multimap,DefaultDescription> getDescriptions(Connect * @return the descriptions */ @Override - public Set getDescriptions(ConnectionConfig connectionConfig, String fieldName, String datatype) throws Exception { + public Set getDescriptions(Connection connectionConfig, String fieldName, String datatype) throws Exception { String alias = getAlias(fieldName, connectionConfig); MetadataDescriptionsHelper helper = getInitializedDescriptionsHelper(connectionConfig); return alias == null ? helper.getDescriptions(fieldName, datatype) : helper.getDescriptions(alias, datatype); @@ -223,7 +225,7 @@ public Set getDescriptions(ConnectionConfig connectionConfig * the description to delete */ @Override - public void deleteDescription(ConnectionConfig connectionConfig, String fieldName, String datatype, DefaultDescription description) throws Exception { + public void deleteDescription(Connection connectionConfig, String fieldName, String datatype, DefaultDescription description) throws Exception { String alias = getAlias(fieldName, connectionConfig); if (alias != null) { fieldName = alias; @@ -235,7 +237,7 @@ public void deleteDescription(ConnectionConfig connectionConfig, String fieldNam // Transform the MetadataEntry key of the specified map into entries. // If an alias exists for a field name, that alias will be returned instead of the field name. private Multimap,DefaultDescription> transformKeys(Multimap descriptions, - ConnectionConfig connectionConfig) throws ExecutionException, TableNotFoundException { + Connection connectionConfig) throws ExecutionException, TableNotFoundException { Map aliases = getAliases(connectionConfig); Multimap,DefaultDescription> transformedDescriptions = HashMultimap.create(); for (Entry entry : descriptions.entries()) { @@ -252,14 +254,14 @@ private Multimap,DefaultDescription> transformKeys(Multimap } // Return a new, initialized metadata description helper. - private MetadataDescriptionsHelper getInitializedDescriptionsHelper(ConnectionConfig connectionConfig) { + private MetadataDescriptionsHelper getInitializedDescriptionsHelper(Connection connectionConfig) { MetadataDescriptionsHelper helper = metadataDescriptionsHelperFactory.createMetadataDescriptionsHelper(); helper.initialize(connectionConfig.getConnector(), connectionConfig.getMetadataTable(), connectionConfig.getAuths()); return helper; } // Return the alias map for the query model in the specified connection config. - private Map getAliases(ConnectionConfig connectionConfig) throws ExecutionException, TableNotFoundException { + private Map getAliases(Connection connectionConfig) throws ExecutionException, TableNotFoundException { MetadataHelper helper = metadataHelperFactory.createMetadataHelper(connectionConfig.getConnector(), connectionConfig.getMetadataTable(), connectionConfig.getAuths()); QueryModel model = helper.getQueryModel(connectionConfig.getModelTable(), connectionConfig.getModelName()); @@ -267,7 +269,7 @@ private Map getAliases(ConnectionConfig connectionConfig) throws } // Retrieve the alias for the specified field name from the alias map for the specified field name. - private String getAlias(String fieldName, ConnectionConfig connectionConfig) throws ExecutionException, TableNotFoundException { + private String getAlias(String fieldName, Connection connectionConfig) throws ExecutionException, TableNotFoundException { Map map = getAliases(connectionConfig); Map reversedMap = map.entrySet().stream().collect(Collectors.toMap(Entry::getValue, Entry::getKey)); return reversedMap.get(fieldName); diff --git a/service/src/main/java/datawave/microservice/dictionary/data/DatawaveDataDictionary.java b/service/src/main/java/datawave/microservice/dictionary/data/DatawaveDataDictionary.java deleted file mode 100644 index 6cc5f41..0000000 --- a/service/src/main/java/datawave/microservice/dictionary/data/DatawaveDataDictionary.java +++ /dev/null @@ -1,34 +0,0 @@ -package datawave.microservice.dictionary.data; - -import com.google.common.collect.Multimap; -import datawave.webservice.query.result.metadata.MetadataFieldBase; -import datawave.webservice.results.datadictionary.DescriptionBase; -import datawave.webservice.results.datadictionary.DictionaryFieldBase; - -import java.util.Collection; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; - -public interface DatawaveDataDictionary,DESC extends DescriptionBase,FIELD extends DictionaryFieldBase> { - - Map getNormalizationMap(); - - void setNormalizationMap(Map normalizationMap); - - Collection getFields(ConnectionConfig connectionConfig, Collection dataTypeFilters, int numThreads) throws Exception; - - void setDescription(ConnectionConfig connectionConfig, FIELD description) throws Exception; - - void setDescription(ConnectionConfig connectionConfig, String fieldName, String datatype, DESC description) throws Exception; - - void setDescriptions(ConnectionConfig connectionConfig, String fieldName, String datatype, Set descriptions) throws Exception; - - Multimap,DESC> getDescriptions(ConnectionConfig connectionConfig) throws Exception; - - Multimap,DESC> getDescriptions(ConnectionConfig connectionConfig, String datatype) throws Exception; - - Set getDescriptions(ConnectionConfig connectionConfig, String fieldName, String datatype) throws Exception; - - void deleteDescription(ConnectionConfig connectionConfig, String fieldName, String datatype, DESC description) throws Exception; -} diff --git a/service/src/main/java/datawave/microservice/dictionary/edge/DatawaveEdgeDictionary.java b/service/src/main/java/datawave/microservice/dictionary/edge/EdgeDictionary.java similarity index 58% rename from service/src/main/java/datawave/microservice/dictionary/edge/DatawaveEdgeDictionary.java rename to service/src/main/java/datawave/microservice/dictionary/edge/EdgeDictionary.java index 6c4b03d..a1521f2 100644 --- a/service/src/main/java/datawave/microservice/dictionary/edge/DatawaveEdgeDictionary.java +++ b/service/src/main/java/datawave/microservice/dictionary/edge/EdgeDictionary.java @@ -1,13 +1,13 @@ package datawave.microservice.dictionary.edge; -import datawave.webservice.results.edgedictionary.EdgeDictionaryBase; -import datawave.webservice.results.edgedictionary.MetadataBase; +import datawave.webservice.dictionary.edge.EdgeDictionaryBase; +import datawave.webservice.dictionary.edge.MetadataBase; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.security.Authorizations; import java.util.Set; -public interface DatawaveEdgeDictionary,META extends MetadataBase> { +public interface EdgeDictionary,META extends MetadataBase> { char COL_SEPARATOR = '/'; EDGE getEdgeDictionary(String metadataTableName, Connector connector, Set auths, int numThreads) throws Exception; diff --git a/service/src/main/java/datawave/microservice/dictionary/edge/DefaultDatawaveEdgeDictionaryImpl.java b/service/src/main/java/datawave/microservice/dictionary/edge/EdgeDictionaryImpl.java similarity index 89% rename from service/src/main/java/datawave/microservice/dictionary/edge/DefaultDatawaveEdgeDictionaryImpl.java rename to service/src/main/java/datawave/microservice/dictionary/edge/EdgeDictionaryImpl.java index 1aeb97c..79f348d 100644 --- a/service/src/main/java/datawave/microservice/dictionary/edge/DefaultDatawaveEdgeDictionaryImpl.java +++ b/service/src/main/java/datawave/microservice/dictionary/edge/EdgeDictionaryImpl.java @@ -7,9 +7,9 @@ import datawave.query.util.MetadataHelper; import datawave.query.util.MetadataHelperFactory; import datawave.util.StringUtils; -import datawave.webservice.results.edgedictionary.DefaultEdgeDictionary; -import datawave.webservice.results.edgedictionary.DefaultMetadata; -import datawave.webservice.results.edgedictionary.EventField; +import datawave.webservice.dictionary.edge.DefaultEdgeDictionary; +import datawave.webservice.dictionary.edge.DefaultMetadata; +import datawave.webservice.dictionary.edge.EventField; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; @@ -24,12 +24,13 @@ import java.util.Map.Entry; import java.util.Set; -public class DefaultDatawaveEdgeDictionaryImpl implements DatawaveEdgeDictionary { - private static final Logger log = LoggerFactory.getLogger(DefaultDatawaveEdgeDictionaryImpl.class); +// TODO Check if something else implements the EdgeDictionary interface, and if not, remove it and rename this class to just EdgeDictionary. +public class EdgeDictionaryImpl implements EdgeDictionary { + private static final Logger log = LoggerFactory.getLogger(EdgeDictionaryImpl.class); private final MetadataHelperFactory metadataHelperFactory; - public DefaultDatawaveEdgeDictionaryImpl(MetadataHelperFactory metadataHelperFactory) { + public EdgeDictionaryImpl(MetadataHelperFactory metadataHelperFactory) { this.metadataHelperFactory = metadataHelperFactory; } diff --git a/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java b/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java index 0ddf9b3..fac9bd4 100644 --- a/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java +++ b/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java @@ -4,13 +4,13 @@ import datawave.data.ColumnFamilyConstants; import datawave.marking.MarkingFunctions; import datawave.microservice.dictionary.config.ResponseObjectFactory; -import datawave.microservice.dictionary.data.ConnectionConfig; +import datawave.microservice.Connection; import datawave.security.util.ScannerHelper; -import datawave.webservice.query.result.metadata.DefaultMetadataField; -import datawave.webservice.results.datadictionary.DefaultDataDictionary; -import datawave.webservice.results.datadictionary.DefaultDescription; -import datawave.webservice.results.datadictionary.DefaultDictionaryField; -import datawave.webservice.results.datadictionary.DefaultFields; +import datawave.webservice.metadata.DefaultMetadataField; +import datawave.webservice.dictionary.data.DefaultDataDictionary; +import datawave.webservice.dictionary.data.DefaultDescription; +import datawave.webservice.dictionary.data.DefaultDictionaryField; +import datawave.webservice.dictionary.data.DefaultFields; import org.apache.accumulo.core.client.BatchScanner; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.TableNotFoundException; @@ -42,12 +42,12 @@ public class DefaultMetadataFieldScanner { private final MarkingFunctions markingFunctions; private final ResponseObjectFactory responseObjectFactory; private final Map normalizationMap; - private final ConnectionConfig connectionConfig; + private final Connection connectionConfig; private final int numThreads; public DefaultMetadataFieldScanner(MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory, - Map normalizationMap, ConnectionConfig connectionConfig, int numThreads) { + Map normalizationMap, Connection connectionConfig, int numThreads) { this.markingFunctions = markingFunctions; this.responseObjectFactory = responseObjectFactory; this.normalizationMap = normalizationMap; diff --git a/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java b/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java index b7e0f17..0c88b2f 100644 --- a/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java +++ b/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java @@ -8,7 +8,7 @@ import datawave.microservice.dictionary.config.ResponseObjectFactory; import datawave.query.util.MetadataEntry; import datawave.security.util.ScannerHelper; -import datawave.webservice.results.datadictionary.DescriptionBase; +import datawave.webservice.dictionary.data.DescriptionBase; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.BatchWriterConfig; import org.apache.accumulo.core.client.Connector; diff --git a/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelperFactory.java b/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelperFactory.java index a28e6b3..326aceb 100644 --- a/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelperFactory.java +++ b/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelperFactory.java @@ -1,6 +1,6 @@ package datawave.microservice.metadata; -import datawave.webservice.results.datadictionary.DescriptionBase; +import datawave.webservice.dictionary.data.DescriptionBase; import org.springframework.beans.factory.annotation.Lookup; import org.springframework.stereotype.Component; diff --git a/service/src/main/java/datawave/microservice/model/ModelController.java b/service/src/main/java/datawave/microservice/model/ModelController.java new file mode 100644 index 0000000..f79f759 --- /dev/null +++ b/service/src/main/java/datawave/microservice/model/ModelController.java @@ -0,0 +1,431 @@ +package datawave.microservice.model; + +import com.google.common.collect.Sets; +//import datawave.interceptor.RequiredInterceptor; +//import datawave.interceptor.ResponseInterceptor; +//import datawave.security.authorization.DatawavePrincipal; +//import datawave.services.common.cache.AccumuloTableCache; +//import datawave.services.common.connection.AccumuloConnectionFactory; +//import datawave.webservice.common.exception.DatawaveWebApplicationException; +//import datawave.webservice.common.exception.NotFoundException; +//import datawave.webservice.common.exception.PreConditionFailedException; +import datawave.microservice.AccumuloConnectionService; +import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.http.converter.protostuff.ProtostuffHttpMessageConverter; +import datawave.microservice.model.config.ModelProperties; +import datawave.webservice.model.Model; +import datawave.webservice.model.ModelKeyParser; +import datawave.webservice.model.ModelList; +import datawave.webservice.query.exception.DatawaveErrorCode; +import datawave.webservice.query.exception.QueryException; +import datawave.webservice.result.VoidResponse; +import lombok.extern.slf4j.Slf4j; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Value; +//import org.apache.deltaspike.core.api.config.ConfigProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.http.MediaType; +import org.springframework.security.access.annotation.Secured; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.Mapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.HashSet; +import java.util.Map; + +/** + * Service that supports manipulation of models. The models are contained in the data dictionary table. + */ +@Slf4j +@RestController +@RequestMapping(path = "/model", produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.TEXT_XML_VALUE, + ProtostuffHttpMessageConverter.PROTOSTUFF_VALUE, MediaType.TEXT_HTML_VALUE, "text/x-yaml", "application/x-yaml"}) +@Secured({"AuthorizedUser", "AuthorizedQueryServer", "InternalUser", "Administrator", "JBossAdministrator"}) +@EnableConfigurationProperties(ModelProperties.class) +public class ModelController { + + private final String dataTablesUri; + private final String jqueryUri; + private final AccumuloConnectionService accumloConnectionService; + + public static final String DEFAULT_MODEL_TABLE_NAME = "DatawaveMetadata"; + + private static final long BATCH_WRITER_MAX_LATENCY = 1000L; + private static final long BATCH_WRITER_MAX_MEMORY = 10845760; + private static final int BATCH_WRITER_MAX_THREADS = 2; + + private static final HashSet RESERVED_COLF_VALUES = Sets.newHashSet("e", "i", "ri", "f", "tf", "m", "desc", "edge", "t", "n", "h"); + + public ModelController(ModelProperties modelProperties, AccumuloConnectionService accumloConnectionService) { + // this.defaultModelTableName = modelProperties.getDefaultTableName(); + this.dataTablesUri = modelProperties.getDefaultTableName(); + this.jqueryUri = modelProperties.getJqueryUri(); + this.accumloConnectionService = accumloConnectionService; + } + + /** + * Get the names of the models + * + * @param modelTableName + * name of the table that contains the model + * @return ModelList + * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user + * + * @HTTP 200 success + * @HTTP 500 internal server error + */ + @GetMapping("/list") + public ModelList listModelNames(@RequestParam(defaultValue = DEFAULT_MODEL_TABLE_NAME) String modelTableName, + @AuthenticationPrincipal ProxiedUserDetails currentUser) { + + ModelList response = new ModelList(jqueryUri, dataTablesUri, modelTableName); + HashSet modelNames = new HashSet<>(); + + try (Scanner scanner = accumloConnectionService.getScanner(modelTableName, currentUser)) { + for (Map.Entry entry : scanner) { + String colf = entry.getKey().getColumnFamily().toString(); + if (!RESERVED_COLF_VALUES.contains(colf) && !modelNames.contains(colf)) { + String[] parts = colf.split(ModelKeyParser.NULL_BYTE); + if (parts.length == 1) { + modelNames.add(colf); + } else if (parts.length == 2) { + modelNames.add(parts[0]); + } + } + + } + } catch (TableNotFoundException e) { + QueryException qe = new QueryException(DatawaveErrorCode.MODEL_NAME_LIST_ERROR, e); + // log.error(qe); + response.addException(qe.getBottomQueryException()); + // throw new DatawaveWebApplicationException(qe, response); + } finally { + // if (null != connector) { + // try { + // connectionFactory.returnConnection(connector); + // } catch (Exception e) { + // log.error("Error returning connection to factory", e); + // } + // } + } + + response.setNames(modelNames); + return response; + } + + /** + * Administrator credentials required. Insert a new model + * + * @param model + * @param modelTableName + * name of the table that contains the model + * @return datawave.webservice.result.VoidResponse + * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user + * + * @HTTP 200 success + * @HTTP 412 if model already exists with this name, delete it first + * @HTTP 500 internal server error + */ + + @PostMapping("/import") + @Secured({"Administrator", "JBossAdministrator"}) + public VoidResponse importModel(@RequestParam Model model, @RequestParam(defaultValue = DEFAULT_MODEL_TABLE_NAME) String modelTableName, + @AuthenticationPrincipal ProxiedUserDetails currentUser) { + if (log.isDebugEnabled()) { + log.debug("modelTableName: " + (null == modelTableName ? "" : modelTableName)); + } + VoidResponse response = new VoidResponse(); + + ModelList models = listModelNames(modelTableName, currentUser); + // if (models.getNames().contains(model.getName())) + // throw new PreConditionFailedException(null, response); + + // insertMapping(model, modelTableName); + + return response; + } + + /** + * Administrator credentials required. Delete a model with the supplied name + * + * @param name + * model name to delete + * @param modelTableName + * name of the table that contains the model + * @return datawave.webservice.result.VoidResponse + * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user + * + * @HTTP 200 success + * @HTTP 404 model not found + * @HTTP 500 internal server error + */ + @DeleteMapping("/{name}") + @Secured({"Administrator", "JBossAdministrator"}) + public VoidResponse deleteModel(@RequestParam String name, @RequestParam(defaultValue = DEFAULT_MODEL_TABLE_NAME) String modelTableName, + @AuthenticationPrincipal ProxiedUserDetails currentUser) { + if (log.isDebugEnabled()) { + log.debug("model name: " + name); + log.debug("modelTableName: " + (null == modelTableName ? "" : modelTableName)); + } + VoidResponse response = new VoidResponse(); + + ModelList models = listModelNames(modelTableName, currentUser); + // if (!models.getNames().contains(name)) + // throw new NotFoundException(null, response); + + Model model = getModel(name, modelTableName, currentUser); + // deleteMapping(model, modelTableName, reloadCache); + + return response; + } + + /** + * Administrator credentials required. Copy a model + * + * @param name + * model to copy + * @param newName + * name of copied model + * @param modelTableName + * name of the table that contains the model + * @return datawave.webservice.result.VoidResponse + * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user + * + * @HTTP 200 success + * @HTTP 204 model not found + * @HTTP 500 internal server error + */ + @PostMapping("/clone") + @Secured({"Administrator", "JBossAdministrator"}) + public VoidResponse cloneModel(@RequestParam String name, @RequestParam String newName, + @RequestParam(defaultValue = DEFAULT_MODEL_TABLE_NAME) String modelTableName, @AuthenticationPrincipal ProxiedUserDetails currentUser) { + VoidResponse response = new VoidResponse(); + + Model model = getModel(name, modelTableName, currentUser); + // Set the new name + model.setName(newName); + // importModel(model, modelTableName); + return response; + } + + /** + * Retrieve the model and all of its mappings + * + * @param name + * model name + * @param modelTableName + * name of the table that contains the model + * @return Model + * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user + * + * @HTTP 200 success + * @HTTP 404 model not found + * @HTTP 500 internal server error + */ + @GetMapping("/{name}") + public Model getModel(@RequestParam String name, @RequestParam(defaultValue = DEFAULT_MODEL_TABLE_NAME) String modelTableName, + @AuthenticationPrincipal ProxiedUserDetails currentUser) { + Model response = new Model(jqueryUri, dataTablesUri); + // + // // Find out who/what called this method + // Principal p = ctx.getCallerPrincipal(); + // String user = p.getName(); + // Set cbAuths = new HashSet<>(); + // if (p instanceof DatawavePrincipal) { + // DatawavePrincipal cp = (DatawavePrincipal) p; + // user = cp.getShortName(); + // for (Collection auths : cp.getAuthorizations()) { + // cbAuths.add(new Authorizations(auths.toArray(new String[auths.size()]))); + // } + // } + // log.trace(user + " has authorizations " + cbAuths); + // + // Connector connector = null; + // try { + // Map trackingMap = connectionFactory.getTrackingMap(Thread.currentThread().getStackTrace()); + // connector = connectionFactory.getConnection(getCurrentUserDN(), getCurrentProxyServers(), AccumuloConnectionFactory.Priority.LOW, trackingMap); + // try (Scanner scanner = ScannerHelper.createScanner(connector, modelTableName, cbAuths)) { + + try (Scanner scanner = accumloConnectionService.getScanner(modelTableName, currentUser)) { + // IteratorSetting cfg = new IteratorSetting(21, "colfRegex", RegExFilter.class.getName()); + // cfg.addOption(RegExFilter.COLF_REGEX, "^" + name + "(\\x00.*)?"); + // scanner.addScanIterator(cfg); + // for (Entry entry : scanner) { + // FieldMapping mapping = ModelKeyParser.parseKey(entry.getKey(), cbAuths); + // response.getFields().add(mapping); + // } + // } + } catch (TableNotFoundException e) { + // QueryException qe = new QueryException(DatawaveErrorCode.MODEL_FETCH_ERROR, e); + // log.error(qe); + // response.addException(qe.getBottomQueryException()); + // throw new DatawaveWebApplicationException(qe, response); + } finally { + // if (null != connector) { + // try { + // connectionFactory.returnConnection(connector); + // } catch (Exception e) { + // log.error("Error returning connection to factory", e); + } + // } + // } + // + // return 404 if model not found + if (response.getMappings().isEmpty()) { + // throw new NotFoundException(null, response); + } + + response.setName(name); + return response; + } + + /** + * Administrator credentials required. Insert a new field mapping into an existing model + * + * @param model + * list of new field mappings to insert + * @param modelTableName + * name of the table that contains the model + * @return datawave.webservice.result.VoidResponse + * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user + * + * @HTTP 200 success + * @HTTP 500 internal server error + */ + @PostMapping("/insert") + @Secured({"Administrator", "JBossAdministrator"}) + public VoidResponse insertMapping(@RequestParam Model model, @RequestParam(defaultValue = DEFAULT_MODEL_TABLE_NAME) String modelTableName) { + + VoidResponse response = new VoidResponse(); + // + // Connector connector = null; + // BatchWriter writer = null; + // try { + // Map trackingMap = connectionFactory.getTrackingMap(Thread.currentThread().getStackTrace()); + // connector = connectionFactory.getConnection(getCurrentUserDN(), getCurrentProxyServers(), AccumuloConnectionFactory.Priority.LOW, trackingMap); + // writer = connector.createBatchWriter(modelTableName, new BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY, TimeUnit.MILLISECONDS) + // .setMaxMemory(BATCH_WRITER_MAX_MEMORY).setMaxWriteThreads(BATCH_WRITER_MAX_THREADS)); + // for (FieldMapping mapping : model.getFields()) { + // Mutation m = ModelKeyParser.createMutation(mapping, model.getName()); + // writer.addMutation(m); + // } + // } catch (Exception e) { + // log.error("Could not insert mapping.", e); + // QueryException qe = new QueryException(DatawaveErrorCode.INSERT_MAPPING_ERROR, e); + // response.addException(qe.getBottomQueryException()); + // throw new DatawaveWebApplicationException(qe, response); + // } finally { + // if (null != writer) { + // try { + // writer.close(); + // } catch (MutationsRejectedException e1) { + // QueryException qe = new QueryException(DatawaveErrorCode.WRITER_CLOSE_ERROR, e1); + // log.error(qe); + // response.addException(qe); + // throw new DatawaveWebApplicationException(qe, response); + // } + // } + // if (null != connector) { + // try { + // connectionFactory.returnConnection(connector); + // } catch (Exception e) { + // log.error("Error returning connection to factory", e); + // } + // } + // } + // cache.reloadTableCache(tableName); + return response; + } + + /** + * Administrator credentials required. Delete field mappings from an existing model + * + * @param model + * list of field mappings to delete + * @param modelTableName + * name of the table that contains the model + * @return datawave.webservice.result.VoidResponse + * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user + * + * @HTTP 200 success + * @HTTP 500 internal server error + */ + @DeleteMapping("/delete") + @Secured({"Administrator", "JBossAdministrator"}) + public VoidResponse deleteMapping(@RequestParam Model model, @RequestParam String modelTableName, @AuthenticationPrincipal ProxiedUserDetails currentUser) { + return deleteMapping(model, modelTableName, true, currentUser); + } + + private VoidResponse deleteMapping(Model model, String modelTableName, boolean reloadCache, @AuthenticationPrincipal ProxiedUserDetails currentUser) { + VoidResponse response = new VoidResponse(); + // + // Connector connector = null; + // BatchWriter writer = null; + // try { + // Map trackingMap = connectionFactory.getTrackingMap(Thread.currentThread().getStackTrace()); + // connector = connectionFactory.getConnection(getCurrentUserDN(), getCurrentProxyServers(), AccumuloConnectionFactory.Priority.LOW, trackingMap); + // writer = connector.createBatchWriter(modelTableName, new BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY, TimeUnit.MILLISECONDS) + // .setMaxMemory(BATCH_WRITER_MAX_MEMORY).setMaxWriteThreads(BATCH_WRITER_MAX_THREADS)); + // for (FieldMapping mapping : model.getFields()) { + // Mutation m = ModelKeyParser.createDeleteMutation(mapping, model.getName()); + // writer.addMutation(m); + // } + // } catch (Exception e) { + // log.error("Could not delete mapping.", e); + // QueryException qe = new QueryException(DatawaveErrorCode.MAPPING_DELETION_ERROR, e); + // response.addException(qe.getBottomQueryException()); + // throw new DatawaveWebApplicationException(qe, response); + // } finally { + // if (null != writer) { + // try { + // writer.close(); + // } catch (MutationsRejectedException e1) { + // QueryException qe = new QueryException(DatawaveErrorCode.WRITER_CLOSE_ERROR, e1); + // log.error(qe); + // response.addException(qe); + // throw new DatawaveWebApplicationException(qe, response); + // } + // } + // if (null != connector) { + // try { + // connectionFactory.returnConnection(connector); + // } catch (Exception e) { + // log.error("Error returning connection to factory", e); + // } + // } + // } + // if (reloadCache) + // cache.reloadTableCache(tableName); + return response; + } + + // public String getCurrentUserDN() { + // String currentUserDN = null; + // Principal p = ctx.getCallerPrincipal(); + // + // if (p != null && p instanceof DatawavePrincipal) { + // currentUserDN = ((DatawavePrincipal) p).getUserDN().subjectDN(); + // } + // + // return currentUserDN; + // } + // + // public Collection getCurrentProxyServers() { + // Set currentProxyServers = null; + // Principal p = ctx.getCallerPrincipal(); + // + // if (p != null && p instanceof DatawavePrincipal) { + // currentProxyServers = ((DatawavePrincipal) p).getProxyServers(); + // } + // + // return currentProxyServers; + // } + +} diff --git a/service/src/main/java/datawave/microservice/model/config/ModelConfiguration.java b/service/src/main/java/datawave/microservice/model/config/ModelConfiguration.java new file mode 100644 index 0000000..62b06a4 --- /dev/null +++ b/service/src/main/java/datawave/microservice/model/config/ModelConfiguration.java @@ -0,0 +1,18 @@ +package datawave.microservice.model.config; + +import datawave.microservice.config.accumulo.AccumuloProperties; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +@EnableConfigurationProperties(ModelProperties.class) +public class ModelConfiguration { + @Bean + @Qualifier("warehouse") + public AccumuloProperties warehouseAccumuloProperties() { + return new AccumuloProperties(); + } + +} diff --git a/service/src/main/java/datawave/microservice/model/config/ModelProperties.java b/service/src/main/java/datawave/microservice/model/config/ModelProperties.java new file mode 100644 index 0000000..944cf9d --- /dev/null +++ b/service/src/main/java/datawave/microservice/model/config/ModelProperties.java @@ -0,0 +1,18 @@ +package datawave.microservice.model.config; + +import lombok.Getter; + +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +@Getter +@Setter +@Validated +@ConfigurationProperties(prefix = "datawave.model") +public class ModelProperties { + + private String defaultTableName; + private String jqueryUri; + private String dataTablesUri; +} diff --git a/service/src/test/java/datawave/microservice/dictionary/DataDictionaryOperationsTest.java b/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java similarity index 94% rename from service/src/test/java/datawave/microservice/dictionary/DataDictionaryOperationsTest.java rename to service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java index 3cf0f1a..92cedc1 100644 --- a/service/src/test/java/datawave/microservice/dictionary/DataDictionaryOperationsTest.java +++ b/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java @@ -11,18 +11,18 @@ import datawave.security.authorization.DatawaveUser; import datawave.security.authorization.SubjectIssuerDNPair; import datawave.webservice.result.VoidResponse; -import datawave.webservice.results.datadictionary.DefaultDataDictionary; -import datawave.webservice.results.datadictionary.DefaultDescription; -import datawave.webservice.results.datadictionary.DefaultFields; +import datawave.webservice.dictionary.data.DefaultDataDictionary; +import datawave.webservice.dictionary.data.DefaultDescription; +import datawave.webservice.dictionary.data.DefaultFields; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.Instance; import org.apache.accumulo.core.client.TableExistsException; import org.apache.accumulo.core.client.security.tokens.PasswordToken; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.context.SpringBootTest; @@ -37,7 +37,7 @@ import org.springframework.http.MediaType; import org.springframework.http.RequestEntity; import org.springframework.http.ResponseEntity; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; import org.springframework.web.client.DefaultResponseErrorHandler; @@ -51,11 +51,11 @@ import java.util.HashSet; import java.util.Map; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, properties = "spring.main.allow-bean-definition-overriding=true") -public class DataDictionaryOperationsTest { +public class DataDictionaryControllerTest { private static Instance instance = new InMemoryInstance(); @LocalServerPort @@ -75,7 +75,7 @@ public class DataDictionaryOperationsTest { private ProxiedUserDetails adminUser; private ProxiedUserDetails regularUser; - @Before + @BeforeEach public void setUp() throws Exception { // Allow 403 responses through without throwing an exception so we can assert the response in the test. ResponseErrorHandler errorHandler = new DefaultResponseErrorHandler() { diff --git a/service/src/test/java/datawave/microservice/dictionary/data/DatawaveDataDictionaryImplTest.java b/service/src/test/java/datawave/microservice/dictionary/data/DataDictionaryImplTest.java similarity index 94% rename from service/src/test/java/datawave/microservice/dictionary/data/DatawaveDataDictionaryImplTest.java rename to service/src/test/java/datawave/microservice/dictionary/data/DataDictionaryImplTest.java index f379335..d12b9f7 100644 --- a/service/src/test/java/datawave/microservice/dictionary/data/DatawaveDataDictionaryImplTest.java +++ b/service/src/test/java/datawave/microservice/dictionary/data/DataDictionaryImplTest.java @@ -5,6 +5,7 @@ import com.google.common.collect.Multimap; import com.google.common.collect.SetMultimap; import datawave.marking.MarkingFunctions; +import datawave.microservice.Connection; import datawave.microservice.dictionary.config.ResponseObjectFactory; import datawave.microservice.metadata.MetadataDescriptionsHelper; import datawave.microservice.metadata.MetadataDescriptionsHelperFactory; @@ -12,19 +13,19 @@ import datawave.query.util.MetadataEntry; import datawave.query.util.MetadataHelper; import datawave.query.util.MetadataHelperFactory; -import datawave.webservice.query.result.metadata.DefaultMetadataField; -import datawave.webservice.results.datadictionary.DefaultDataDictionary; -import datawave.webservice.results.datadictionary.DefaultDescription; -import datawave.webservice.results.datadictionary.DefaultDictionaryField; -import datawave.webservice.results.datadictionary.DefaultFields; +import datawave.webservice.metadata.DefaultMetadataField; +import datawave.webservice.dictionary.data.DefaultDataDictionary; +import datawave.webservice.dictionary.data.DefaultDescription; +import datawave.webservice.dictionary.data.DefaultDictionaryField; +import datawave.webservice.dictionary.data.DefaultFields; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.security.Authorizations; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; import java.util.Collection; import java.util.Collections; @@ -40,8 +41,8 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -@RunWith(MockitoJUnitRunner.class) -public class DatawaveDataDictionaryImplTest { +@ExtendWith(MockitoExtension.class) +public class DataDictionaryImplTest { private static final String METADATA_TABLE = "metadataTable"; private static final String MODEL_TABLE = "modelTable"; @@ -50,7 +51,7 @@ public class DatawaveDataDictionaryImplTest { private static final String DATATYPE = "csv"; private static final Set AUTHS = Collections.singleton(new Authorizations("PRIVATE")); - private ConnectionConfig connectionConfig; + private Connection connectionConfig; @Mock private Connector connector; @@ -70,17 +71,17 @@ public class DatawaveDataDictionaryImplTest { @Mock private MetadataDescriptionsHelper metadataDescriptionsHelper; - private DatawaveDataDictionaryImpl dataDictionary; + private DataDictionaryImpl dataDictionary; - @Before + @BeforeEach public void setUp() { - connectionConfig = new ConnectionConfig(); + connectionConfig = new Connection(); connectionConfig.setConnector(connector); connectionConfig.setAuths(AUTHS); connectionConfig.setMetadataTable(METADATA_TABLE); connectionConfig.setModelTable(MODEL_TABLE); connectionConfig.setModelName(MODEL_NAME); - dataDictionary = new DatawaveDataDictionaryImpl(markingFunctions, responseObjectFactory, metadataHelperFactory, metadataDescriptionsHelperFactory); + dataDictionary = new DataDictionaryImpl(markingFunctions, responseObjectFactory, metadataHelperFactory, metadataDescriptionsHelperFactory); } @Test diff --git a/service/src/test/java/datawave/microservice/dictionary/edge/DefaultDatawaveEdgeDictionaryImplTest.java b/service/src/test/java/datawave/microservice/dictionary/edge/EdgeDictionaryImplTest.java similarity index 83% rename from service/src/test/java/datawave/microservice/dictionary/edge/DefaultDatawaveEdgeDictionaryImplTest.java rename to service/src/test/java/datawave/microservice/dictionary/edge/EdgeDictionaryImplTest.java index dbc8f63..3d594ba 100644 --- a/service/src/test/java/datawave/microservice/dictionary/edge/DefaultDatawaveEdgeDictionaryImplTest.java +++ b/service/src/test/java/datawave/microservice/dictionary/edge/EdgeDictionaryImplTest.java @@ -5,10 +5,10 @@ import datawave.accumulo.inmemory.InMemoryInstance; import datawave.data.ColumnFamilyConstants; import datawave.metadata.protobuf.EdgeMetadata.MetadataValue; -import datawave.webservice.results.edgedictionary.DefaultEdgeDictionary; -import datawave.webservice.results.edgedictionary.EventField; -import datawave.webservice.results.edgedictionary.DefaultMetadata; -import datawave.webservice.results.edgedictionary.MetadataBase; +import datawave.webservice.dictionary.edge.DefaultEdgeDictionary; +import datawave.webservice.dictionary.edge.EventField; +import datawave.webservice.dictionary.edge.DefaultMetadata; +import datawave.webservice.dictionary.edge.MetadataBase; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; @@ -17,17 +17,17 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.hadoop.io.Text; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @@ -39,11 +39,11 @@ import java.util.List; import java.util.Map; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.fail; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, properties = "spring.main.allow-bean-definition-overriding=true") -public class DefaultDatawaveEdgeDictionaryImplTest { +public class EdgeDictionaryImplTest { public static final String EDGE_TYPE = "TYPE"; public static final String SOURCE_RELATIONSHIP = "OWNER"; @@ -66,9 +66,9 @@ public class DefaultDatawaveEdgeDictionaryImplTest { Method transformResultsMethod; @Autowired - private DatawaveEdgeDictionary impl; + private EdgeDictionary impl; - @Before + @BeforeEach public void setUp() { edgeMetadataRows = HashMultimap.create(); for (Map.Entry entry : permuteKeysAndEdges(EDGE_KEYS, EDGE_VALUE)) { @@ -92,18 +92,18 @@ public void testNoOp() throws InvocationTargetException, IllegalAccessException if (null == transformResultsMethod) fail(); DefaultEdgeDictionary dictionary = (DefaultEdgeDictionary) transformResultsMethod.invoke(impl, HashMultimap.create()); - Assert.assertEquals("Should be empty", dictionary.getTotalResults(), 0L); + Assertions.assertEquals(0L, dictionary.getTotalResults(), "Should be empty"); } @Test public void testWorked() throws InvocationTargetException, IllegalAccessException { if (null == transformResultsMethod) fail(); - Assert.assertEquals("data to be inserted contains as many rows as keys", edgeMetadataRows.keySet().size(), EDGE_KEYS.size()); + Assertions.assertEquals(EDGE_KEYS.size(), edgeMetadataRows.keySet().size(), "data to be inserted contains as many rows as keys"); DefaultEdgeDictionary dictionary = (DefaultEdgeDictionary) transformResultsMethod.invoke(impl, edgeMetadataRows); - Assert.assertEquals("Dictionary should now have some entries", dictionary.getTotalResults(), EDGE_KEYS.size()); - Assert.assertTrue("METADATA not in list. returned list: " + dictionary.getMetadataList() + " expected: " + METADATA, - dictionary.getMetadataList().containsAll(METADATA)); + Assertions.assertEquals(EDGE_KEYS.size(), dictionary.getTotalResults(), "Dictionary should now have some entries"); + Assertions.assertTrue(dictionary.getMetadataList().containsAll(METADATA), + "METADATA not in list. returned list: " + dictionary.getMetadataList() + " expected: " + METADATA); } @Test @@ -117,14 +117,14 @@ public void earliestDateFound() throws InvocationTargetException, IllegalAccessE // Make sure that all Metadata in EdgeDictionary have the start date set to the EARLY_DATE_FIELD for (MetadataBase meta : metadata) { - Assert.assertEquals("Incorrect start date. Expected: " + EARLY_DATE_FIELD + " Found: " + meta.getStartDate(), EARLY_DATE_FIELD, - meta.getStartDate()); + Assertions.assertEquals(EARLY_DATE_FIELD, meta.getStartDate(), + "Incorrect start date. Expected: " + EARLY_DATE_FIELD + " Found: " + meta.getStartDate()); } } private Method getPrivateMethod(String methodName) { - Class clas = DefaultDatawaveEdgeDictionaryImpl.class; + Class clas = EdgeDictionaryImpl.class; for (Method method : clas.getDeclaredMethods()) { if (method.getName().equals(methodName)) { method.setAccessible(true); @@ -162,7 +162,7 @@ private static ArrayList createEdgeKeys() { } private static Key generateKeyForEdgeMetadata(String source_attribute1) { - Text row = new Text(EDGE_TYPE + DatawaveEdgeDictionary.COL_SEPARATOR + SOURCE_RELATIONSHIP + "-" + SINK_RELATIONSHIP); + Text row = new Text(EDGE_TYPE + EdgeDictionary.COL_SEPARATOR + SOURCE_RELATIONSHIP + "-" + SINK_RELATIONSHIP); Text colf = ColumnFamilyConstants.COLF_EDGE; Text colq = new Text(source_attribute1 + "-" + SINK_ATTRIBUTE1); Text colv = new Text(""); diff --git a/service/src/test/java/datawave/microservice/metadata/DefaultMetadataFieldScannerTest.java b/service/src/test/java/datawave/microservice/metadata/DefaultMetadataFieldScannerTest.java index e7e96cb..a88d617 100644 --- a/service/src/test/java/datawave/microservice/metadata/DefaultMetadataFieldScannerTest.java +++ b/service/src/test/java/datawave/microservice/metadata/DefaultMetadataFieldScannerTest.java @@ -4,12 +4,12 @@ import datawave.data.ColumnFamilyConstants; import datawave.marking.MarkingFunctions; import datawave.microservice.dictionary.config.ResponseObjectFactory; -import datawave.microservice.dictionary.data.ConnectionConfig; -import datawave.webservice.query.result.metadata.DefaultMetadataField; -import datawave.webservice.results.datadictionary.DefaultDataDictionary; -import datawave.webservice.results.datadictionary.DefaultDescription; -import datawave.webservice.results.datadictionary.DefaultDictionaryField; -import datawave.webservice.results.datadictionary.DefaultFields; +import datawave.microservice.Connection; +import datawave.webservice.metadata.DefaultMetadataField; +import datawave.webservice.dictionary.data.DefaultDataDictionary; +import datawave.webservice.dictionary.data.DefaultDescription; +import datawave.webservice.dictionary.data.DefaultDictionaryField; +import datawave.webservice.dictionary.data.DefaultFields; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.MutationsRejectedException; @@ -20,10 +20,8 @@ import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.hadoop.io.Text; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.time.LocalDateTime; import java.time.ZoneId; @@ -37,7 +35,6 @@ import static org.assertj.core.api.Assertions.assertThat; -@RunWith(JUnit4.class) public class DefaultMetadataFieldScannerTest { private static final String DATE = "20200115051230"; @@ -68,7 +65,7 @@ public DefaultFields getFields() { private DefaultMetadataFieldScanner scanner; - @Before + @BeforeEach public void setUp() throws Exception { InMemoryInstance instance = new InMemoryInstance(); connector = instance.getConnector("root", new PasswordToken("")); @@ -81,7 +78,7 @@ public void setUp() throws Exception { normalizerMapping.put("datawave.data.type.LcNoDiacriticsType", "Text"); normalizerMapping.put("datawave.data.type.NumberType", "Number"); - ConnectionConfig connectionConfig = new ConnectionConfig(); + Connection connectionConfig = new Connection(); connectionConfig.setConnector(connector); connectionConfig.setMetadataTable(METADATA_TABLE); connectionConfig.setAuths(AUTHS); From d1b4002a86a6ba2ba716a2f845527badb1c019e8 Mon Sep 17 00:00:00 2001 From: cmitchell Date: Wed, 20 Apr 2022 14:17:49 -0400 Subject: [PATCH 06/54] Added test data and base controller test (commn setup). --- service/pom.xml | 5 + .../datawave/microservice/Connection.java | 72 +--- .../dictionary/EdgeDictionaryController.java | 5 +- .../config/DataDictionaryProperties.java | 43 +- .../config/EdgeDictionaryProperties.java | 20 +- .../dictionary/data/ConnectionConfig.java | 85 ---- .../datawave/microservice/ControllerIT.java | 78 ++++ .../DataDictionaryControllerTest.java | 56 +-- .../model/ModelControllerTest.java | 381 ++++++++++++++++++ service/src/test/resources/TestModel_1.xml | 7 + service/src/test/resources/TestModel_2.xml | 8 + .../src/test/resources/config/application.yml | 3 + 12 files changed, 505 insertions(+), 258 deletions(-) delete mode 100644 service/src/main/java/datawave/microservice/dictionary/data/ConnectionConfig.java create mode 100644 service/src/test/java/datawave/microservice/ControllerIT.java create mode 100644 service/src/test/java/datawave/microservice/model/ModelControllerTest.java create mode 100644 service/src/test/resources/TestModel_1.xml create mode 100644 service/src/test/resources/TestModel_2.xml diff --git a/service/pom.xml b/service/pom.xml index e2858a0..03adca0 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -27,6 +27,11 @@ + + com.fasterxml.jackson.dataformat + jackson-dataformat-xml + ${version.jackson} + gov.nsa.datawave.microservice dictionary-api diff --git a/service/src/main/java/datawave/microservice/Connection.java b/service/src/main/java/datawave/microservice/Connection.java index f653e1b..5eb8499 100644 --- a/service/src/main/java/datawave/microservice/Connection.java +++ b/service/src/main/java/datawave/microservice/Connection.java @@ -1,85 +1,17 @@ package datawave.microservice; +import lombok.Data; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.security.Authorizations; -import org.apache.commons.lang.builder.ToStringBuilder; -import java.util.Objects; import java.util.Set; +@Data public class Connection { private Connector connector; - private Set auths; - private String metadataTable; - private String modelTable; - private String modelName; - - public Connector getConnector() { - return connector; - } - - public void setConnector(Connector connector) { - this.connector = connector; - } - - public Set getAuths() { - return auths; - } - - public void setAuths(Set authorizations) { - this.auths = authorizations; - } - - public String getMetadataTable() { - return metadataTable; - } - - public void setMetadataTable(String metadataTable) { - this.metadataTable = metadataTable; - } - - public String getModelTable() { - return modelTable; - } - - public void setModelTable(String modelTable) { - this.modelTable = modelTable; - } - - public String getModelName() { - return modelName; - } - - public void setModelName(String modelName) { - this.modelName = modelName; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - Connection that = (Connection) o; - return Objects.equals(connector, that.connector) && Objects.equals(auths, that.auths) && Objects.equals(metadataTable, that.metadataTable) - && Objects.equals(modelTable, that.modelTable) && Objects.equals(modelName, that.modelName); - } - - @Override - public int hashCode() { - return Objects.hash(connector, auths, metadataTable, modelTable, modelName); - } - - @Override - public String toString() { - return new ToStringBuilder(this).append("connector", connector).append("auths", auths).append("metadataTable", metadataTable) - .append("modelTable", modelTable).append("modelName", modelName).toString(); - } } diff --git a/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java b/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java index d576fba..f645df5 100644 --- a/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java +++ b/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java @@ -8,11 +8,10 @@ import datawave.security.authorization.DatawaveUser; import datawave.webservice.dictionary.edge.EdgeDictionaryBase; import datawave.webservice.dictionary.edge.MetadataBase; +import lombok.extern.slf4j.Slf4j; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.security.Authorizations; import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.http.MediaType; @@ -27,13 +26,13 @@ import static datawave.microservice.http.converter.protostuff.ProtostuffHttpMessageConverter.PROTOSTUFF_VALUE; +@Slf4j @PermitAll @RestController @RequestMapping(path = "/edge/v1", produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.TEXT_XML_VALUE, PROTOSTUFF_VALUE, MediaType.TEXT_HTML_VALUE, "text/x-yaml", "application/x-yaml"}) @EnableConfigurationProperties(EdgeDictionaryProperties.class) public class EdgeDictionaryController,META extends MetadataBase> { - private Logger log = LoggerFactory.getLogger(getClass()); private final EdgeDictionaryProperties edgeDictionaryProperties; private final EdgeDictionary edgeDictionary; diff --git a/service/src/main/java/datawave/microservice/dictionary/config/DataDictionaryProperties.java b/service/src/main/java/datawave/microservice/dictionary/config/DataDictionaryProperties.java index 0fdbd1f..b2b43b4 100644 --- a/service/src/main/java/datawave/microservice/dictionary/config/DataDictionaryProperties.java +++ b/service/src/main/java/datawave/microservice/dictionary/config/DataDictionaryProperties.java @@ -1,5 +1,7 @@ package datawave.microservice.dictionary.config; +import lombok.Getter; +import lombok.Setter; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.validation.annotation.Validated; @@ -7,6 +9,8 @@ import javax.validation.constraints.Positive; import java.util.Map; +@Getter +@Setter @ConfigurationProperties(prefix = "datawave.dictionary.data") @Validated public class DataDictionaryProperties { @@ -21,43 +25,4 @@ public class DataDictionaryProperties { private int numThreads; private Map normalizerMap; - public String getModelName() { - return modelName; - } - - public void setModelName(String modelName) { - this.modelName = modelName; - } - - public String getModelTableName() { - return modelTableName; - } - - public void setModelTableName(String modelTableName) { - this.modelTableName = modelTableName; - } - - public String getMetadataTableName() { - return metadataTableName; - } - - public void setMetadataTableName(String metadataTableName) { - this.metadataTableName = metadataTableName; - } - - public int getNumThreads() { - return numThreads; - } - - public void setNumThreads(int numThreads) { - this.numThreads = numThreads; - } - - public Map getNormalizerMap() { - return normalizerMap; - } - - public void setNormalizerMap(Map normalizerMap) { - this.normalizerMap = normalizerMap; - } } diff --git a/service/src/main/java/datawave/microservice/dictionary/config/EdgeDictionaryProperties.java b/service/src/main/java/datawave/microservice/dictionary/config/EdgeDictionaryProperties.java index 18c029b..4ccb671 100644 --- a/service/src/main/java/datawave/microservice/dictionary/config/EdgeDictionaryProperties.java +++ b/service/src/main/java/datawave/microservice/dictionary/config/EdgeDictionaryProperties.java @@ -1,11 +1,15 @@ package datawave.microservice.dictionary.config; +import lombok.Getter; +import lombok.Setter; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.validation.annotation.Validated; import javax.validation.constraints.NotBlank; import javax.validation.constraints.Positive; +@Getter +@Setter @ConfigurationProperties(prefix = "datawave.dictionary.edge") @Validated public class EdgeDictionaryProperties { @@ -14,20 +18,4 @@ public class EdgeDictionaryProperties { private String metadataTableName; @Positive private int numThreads; - - public String getMetadataTableName() { - return metadataTableName; - } - - public void setMetadataTableName(String metadataTableName) { - this.metadataTableName = metadataTableName; - } - - public int getNumThreads() { - return numThreads; - } - - public void setNumThreads(int numThreads) { - this.numThreads = numThreads; - } } diff --git a/service/src/main/java/datawave/microservice/dictionary/data/ConnectionConfig.java b/service/src/main/java/datawave/microservice/dictionary/data/ConnectionConfig.java deleted file mode 100644 index 75526b9..0000000 --- a/service/src/main/java/datawave/microservice/dictionary/data/ConnectionConfig.java +++ /dev/null @@ -1,85 +0,0 @@ -package datawave.microservice.dictionary.data; - -import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.security.Authorizations; -import org.apache.commons.lang.builder.ToStringBuilder; - -import java.util.Objects; -import java.util.Set; - -public class ConnectionConfig { - - private Connector connector; - - private Set auths; - - private String metadataTable; - - private String modelTable; - - private String modelName; - - public Connector getConnector() { - return connector; - } - - public void setConnector(Connector connector) { - this.connector = connector; - } - - public Set getAuths() { - return auths; - } - - public void setAuths(Set authorizations) { - this.auths = authorizations; - } - - public String getMetadataTable() { - return metadataTable; - } - - public void setMetadataTable(String metadataTable) { - this.metadataTable = metadataTable; - } - - public String getModelTable() { - return modelTable; - } - - public void setModelTable(String modelTable) { - this.modelTable = modelTable; - } - - public String getModelName() { - return modelName; - } - - public void setModelName(String modelName) { - this.modelName = modelName; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ConnectionConfig that = (ConnectionConfig) o; - return Objects.equals(connector, that.connector) && Objects.equals(auths, that.auths) && Objects.equals(metadataTable, that.metadataTable) - && Objects.equals(modelTable, that.modelTable) && Objects.equals(modelName, that.modelName); - } - - @Override - public int hashCode() { - return Objects.hash(connector, auths, metadataTable, modelTable, modelName); - } - - @Override - public String toString() { - return new ToStringBuilder(this).append("connector", connector).append("auths", auths).append("metadataTable", metadataTable) - .append("modelTable", modelTable).append("modelName", modelName).toString(); - } -} diff --git a/service/src/test/java/datawave/microservice/ControllerIT.java b/service/src/test/java/datawave/microservice/ControllerIT.java new file mode 100644 index 0000000..45f0088 --- /dev/null +++ b/service/src/test/java/datawave/microservice/ControllerIT.java @@ -0,0 +1,78 @@ +package datawave.microservice; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Sets; +import datawave.microservice.authorization.jwt.JWTRestTemplate; +import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.security.authorization.DatawaveUser; +import datawave.security.authorization.SubjectIssuerDNPair; +import org.apache.accumulo.core.client.Connector; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.web.client.RestTemplateBuilder; +import org.springframework.boot.web.server.LocalServerPort; +import org.springframework.http.HttpStatus; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.web.client.DefaultResponseErrorHandler; +import org.springframework.web.client.ResponseErrorHandler; + +import javax.annotation.Nonnull; +import java.util.Collections; +import java.util.HashSet; + +/** + * Provides a default setup for controller integration tests (specifically). Contains properties and setup that are common to controller integration tests. The + * setup and dependency injections in this class are done before any in implementing classes. + */ +@ExtendWith(SpringExtension.class) +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +public class ControllerIT { + + @LocalServerPort + protected int webServicePort; + + @Autowired + @Qualifier("warehouse") + protected Connector connector; + + protected ProxiedUserDetails adminUser; + protected ProxiedUserDetails regularUser; + + @Autowired + private RestTemplateBuilder restTemplateBuilder; + + protected JWTRestTemplate jwtRestTemplate; + + @Autowired + private ObjectMapper objectMapper; + + @BeforeAll + public void oneTimeSetup() { + + // Allow 403 responses through without throwing an exception so tests can assert the response appropriately. + ResponseErrorHandler errorHandler = new DefaultResponseErrorHandler() { + @Override + protected boolean hasError(@Nonnull HttpStatus statusCode) { + return super.hasError(statusCode) && statusCode.value() != 403; + } + }; + + jwtRestTemplate = restTemplateBuilder.errorHandler(errorHandler).build(JWTRestTemplate.class); + + SubjectIssuerDNPair dn = SubjectIssuerDNPair.of("userDn", "issuerDn"); + HashSet auths = Sets.newHashSet("PUBLIC", "PRIVATE"); + HashSet roles = Sets.newHashSet("AuthorizedUser", "Administrator"); + long createTime = System.currentTimeMillis(); + adminUser = new ProxiedUserDetails(Collections.singleton(new DatawaveUser(dn, DatawaveUser.UserType.USER, auths, roles, null, createTime)), createTime); + regularUser = new ProxiedUserDetails( + Collections.singleton( + new DatawaveUser(dn, DatawaveUser.UserType.USER, auths, Collections.singleton("AuthorizedUser"), null, createTime)), + createTime); + } + +} diff --git a/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java b/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java index 92cedc1..2fcd8a2 100644 --- a/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java +++ b/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java @@ -5,6 +5,7 @@ import com.google.common.collect.Sets; import datawave.accumulo.inmemory.InMemoryInstance; import datawave.marking.MarkingFunctions; +import datawave.microservice.ControllerIT; import datawave.microservice.authorization.jwt.JWTRestTemplate; import datawave.microservice.authorization.user.ProxiedUserDetails; import datawave.microservice.dictionary.config.DataDictionaryProperties; @@ -53,54 +54,28 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -@ExtendWith(SpringExtension.class) -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, properties = "spring.main.allow-bean-definition-overriding=true") -public class DataDictionaryControllerTest { - private static Instance instance = new InMemoryInstance(); - - @LocalServerPort - private int webServicePort; - - @Autowired - private RestTemplateBuilder restTemplateBuilder; - - @Autowired - @Qualifier("warehouse") - private Connector connector; +public class DataDictionaryControllerTest extends ControllerIT { @Autowired private DataDictionaryProperties dataDictionaryProperties; - private JWTRestTemplate jwtRestTemplate; - private ProxiedUserDetails adminUser; - private ProxiedUserDetails regularUser; + @ComponentScan(basePackages = "datawave.microservice") + @Configuration + public static class DataDictionaryImplTestConfiguration { + @Bean + @Qualifier("warehouse") + public Connector warehouseConnector() throws AccumuloSecurityException, AccumuloException { + return new InMemoryInstance().getConnector("root", new PasswordToken("")); + } + } @BeforeEach public void setUp() throws Exception { - // Allow 403 responses through without throwing an exception so we can assert the response in the test. - ResponseErrorHandler errorHandler = new DefaultResponseErrorHandler() { - @Override - protected boolean hasError(HttpStatus statusCode) { - return super.hasError(statusCode) && statusCode.value() != 403; - } - }; - jwtRestTemplate = restTemplateBuilder.errorHandler(errorHandler).build(JWTRestTemplate.class); - try { connector.tableOperations().create(dataDictionaryProperties.getMetadataTableName()); } catch (TableExistsException e) { // ignore } - - SubjectIssuerDNPair dn = SubjectIssuerDNPair.of("userDn", "issuerDn"); - HashSet auths = Sets.newHashSet("PUBLIC", "PRIVATE"); - HashSet roles = Sets.newHashSet("AuthorizedUser", "Administrator"); - long createTime = System.currentTimeMillis(); - adminUser = new ProxiedUserDetails(Collections.singleton(new DatawaveUser(dn, DatawaveUser.UserType.USER, auths, roles, null, createTime)), createTime); - regularUser = new ProxiedUserDetails( - Collections.singleton( - new DatawaveUser(dn, DatawaveUser.UserType.USER, auths, Collections.singleton("AuthorizedUser"), null, createTime)), - createTime); } @Test @@ -181,13 +156,4 @@ public void testPostDescriptions() { assertEquals(HttpStatus.OK, response.getStatusCode()); } - @ComponentScan(basePackages = "datawave.microservice") - @Configuration - public static class DataDictionaryImplTestConfiguration { - @Bean - @Qualifier("warehouse") - public Connector warehouseConnector() throws AccumuloSecurityException, AccumuloException { - return instance.getConnector("root", new PasswordToken("")); - } - } } diff --git a/service/src/test/java/datawave/microservice/model/ModelControllerTest.java b/service/src/test/java/datawave/microservice/model/ModelControllerTest.java new file mode 100644 index 0000000..0c2c94e --- /dev/null +++ b/service/src/test/java/datawave/microservice/model/ModelControllerTest.java @@ -0,0 +1,381 @@ +package datawave.microservice.model; + +//import static org.powermock.api.easymock.PowerMock.createMock; +//import static org.powermock.api.easymock.PowerMock.createStrictMock; +// +//import java.net.URL; +//import java.util.Arrays; +//import java.util.Collection; +//import java.util.Collections; +//import java.util.HashMap; +//import java.util.HashSet; +//import java.util.Map.Entry; +//import java.util.Set; +// +//import javax.ejb.EJBContext; +//import javax.xml.bind.JAXBContext; +//import javax.xml.bind.Unmarshaller; +// +//import datawave.microservice.dictionary.AccumuloConnectionService; +//import datawave.microservice.dictionary.model.model.ModelKeyParser; +//import datawave.security.authorization.DatawavePrincipal; +//import datawave.security.authorization.DatawaveUser; +//import datawave.security.authorization.DatawaveUser.UserType; +//import datawave.security.authorization.SubjectIssuerDNPair; +//import datawave.security.util.DnUtils.NpeUtils; +//import datawave.security.util.ScannerHelper; +//import datawave.webservice.common.cache.AccumuloTableCache; +//import datawave.webservice.common.connection.AccumuloConnectionFactory; +//import datawave.webservice.common.exception.DatawaveWebApplicationException; +//import datawave.webservice.model.ModelList; +// +//import org.apache.accumulo.core.client.Connector; +//import org.apache.accumulo.core.client.Scanner; +//import datawave.accumulo.inmemory.InMemoryInstance; +//import org.apache.accumulo.core.client.security.tokens.PasswordToken; +//import org.apache.accumulo.core.data.Key; +//import org.apache.accumulo.core.data.Value; +//import org.apache.accumulo.core.security.Authorizations; +//import org.apache.log4j.Level; +//import org.apache.log4j.Logger; +//import org.easymock.EasyMock; +//import org.junit.jupiter.api.BeforeEach; +import datawave.accumulo.inmemory.InMemoryInstance; +import datawave.microservice.ControllerIT; +import datawave.microservice.dictionary.config.DataDictionaryProperties; +import datawave.microservice.model.config.ModelProperties; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.TableExistsException; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +//import org.powermock.api.easymock.PowerMock; +//import org.powermock.core.classloader.annotations.PowerMockIgnore; +//import org.powermock.core.classloader.annotations.PrepareForTest; +//import org.powermock.modules.junit4.PowerMockRunner; +//import org.powermock.reflect.Whitebox; +//import org.springframework.test.context.junit.jupiter.SpringExtension; + +public class ModelControllerTest extends ControllerIT { + + @Autowired + private ModelProperties modelProperties; + + @ComponentScan(basePackages = "datawave.microservice") + @Configuration + public static class DataDictionaryImplTestConfiguration { + @Bean + @Qualifier("warehouse") + public Connector warehouseConnector() throws AccumuloSecurityException, AccumuloException { + return new InMemoryInstance().getConnector("root", new PasswordToken("")); + } + } + + @BeforeEach + public void setUp() throws Exception { + try { + connector.tableOperations().create(modelProperties.getDefaultTableName()); + } catch (TableExistsException e) { + // ignore + } + }; + + // private static final String userDN = "CN=Guy Some Other soguy, OU=ou1, OU=ou2, OU=ou3, O=o1, C=US"; + // private static final String issuerDN = "CN=CA1, OU=ou3, O=o1, C=US"; + // private static final String[] auths = new String[] {"PRIVATE", "PUBLIC"}; + // + // private ModelController modelController; + // private AccumuloConnectionService connection; + //// private AccumuloTableCache cache; + // + // private InMemoryInstance instance = null; + // private Connector connector = null; + //// private DatawavePrincipal principal = null; + // + // private static long TIMESTAMP = System.currentTimeMillis(); + // + //// private datawave.webservice.model.Model MODEL_ONE = null; + //// private datawave.webservice.model.Model MODEL_TWO = null; + // + // @BeforeEach + // public void setup() throws Exception { + //// System.setProperty(NpeUtils.NPE_OU_PROPERTY, "iamnotaperson"); + // System.setProperty("dw.metadatahelper.all.auths", "A,B,C,D"); + // bean = new ModelBean(); + // connectionFactory = createStrictMock(AccumuloConnectionFactory.class); + // ctx = createMock(EJBContext.class); + // cache = createMock(AccumuloTableCache.class); + // Whitebox.setInternalState(bean, EJBContext.class, ctx); + // Whitebox.setInternalState(bean, AccumuloConnectionFactory.class, connectionFactory); + // Whitebox.setInternalState(bean, AccumuloTableCache.class, cache); + // + // instance = new InMemoryInstance("test"); + // connector = instance.getConnector("root", new PasswordToken("")); + // + // DatawaveUser user = new DatawaveUser(SubjectIssuerDNPair.of(userDN, issuerDN), UserType.USER, Arrays.asList(auths), null, null, 0L); + // principal = new DatawavePrincipal(Collections.singletonList(user)); + // + // URL m1Url = ModelBeanTest.class.getResource("/ModelBeanTest_m1.xml"); + // URL m2Url = ModelBeanTest.class.getResource("/ModelBeanTest_m2.xml"); + // JAXBContext ctx = JAXBContext.newInstance(datawave.webservice.model.Model.class); + // Unmarshaller u = ctx.createUnmarshaller(); + // MODEL_ONE = (datawave.webservice.model.Model) u.unmarshal(m1Url); + // MODEL_TWO = (datawave.webservice.model.Model) u.unmarshal(m2Url); + // + // Logger.getLogger(ModelBean.class).setLevel(Level.OFF); + // PowerMock.mockStatic(System.class, System.class.getMethod("currentTimeMillis")); + // } + // + // public void printTable(String tableName) throws Exception { + // Scanner s = connector.createScanner(tableName, new Authorizations(auths)); + // for (Entry entry : s) { + // System.out.println(entry.getKey()); + // } + // } + // + // @After + // public void tearDown() { + // try { + // connector.tableOperations().delete(ModelBean.DEFAULT_MODEL_TABLE_NAME); + // } catch (Exception e) {} + // } + // + //// @Test(expected = DatawaveWebApplicationException.class) + //// public void testModelImportNoTable() throws Exception { + //// HashMap trackingMap = new HashMap<>(); + //// EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); + //// EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); + //// connectionFactory.returnConnection(connector); + //// EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal); + //// PowerMock.replayAll(); + //// + //// bean.importModel(MODEL_ONE, (String) null); + //// PowerMock.verifyAll(); + //// } + //// + // private void importModels() throws Exception { + // connector.tableOperations().create(ModelBean.DEFAULT_MODEL_TABLE_NAME); + // + // HashMap trackingMap = new HashMap<>(); + // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); + // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); + // connectionFactory.returnConnection(connector); + // EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal); + // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); + // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); + // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); + // connectionFactory.returnConnection(connector); + // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); + // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); + // EasyMock.expect(cache.reloadCache(ModelBean.DEFAULT_MODEL_TABLE_NAME)).andReturn(null); + // PowerMock.replayAll(); + // + // bean.importModel(MODEL_ONE, (String) null); + // PowerMock.verifyAll(); + // PowerMock.resetAll(); + // + // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); + // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); + // connectionFactory.returnConnection(connector); + // EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal); + // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); + // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); + // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); + // connectionFactory.returnConnection(connector); + // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); + // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); + // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); + // EasyMock.expect(cache.reloadCache(ModelBean.DEFAULT_MODEL_TABLE_NAME)).andReturn(null); + // PowerMock.replayAll(); + // + // bean.importModel(MODEL_TWO, (String) null); + // + // PowerMock.verifyAll(); + // } + // + // @Test + // public void testListModels() throws Exception { + // importModels(); + // PowerMock.resetAll(); + // + // EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal); + // HashMap trackingMap = new HashMap<>(); + // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); + // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); + // connectionFactory.returnConnection(connector); + // PowerMock.replayAll(); + // + // ModelList list = bean.listModelNames((String) null); + // PowerMock.verifyAll(); + // + // Assert.assertEquals(2, list.getNames().size()); + // Assert.assertTrue(list.getNames().contains(MODEL_ONE.getName())); + // Assert.assertTrue(list.getNames().contains(MODEL_TWO.getName())); + // } + // + // @Test + // public void testModelGet() throws Exception { + // importModels(); + // PowerMock.resetAll(); + // + // EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal); + // HashMap trackingMap = new HashMap<>(); + // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); + // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); + // connectionFactory.returnConnection(connector); + // PowerMock.replayAll(); + // + // datawave.webservice.model.Model model = bean.getModel(MODEL_ONE.getName(), (String) null); + // PowerMock.verifyAll(); + // + // Assert.assertEquals(MODEL_ONE, model); + // } + // + // @Test + // public void testModelDelete() throws Exception { + // importModels(); + // PowerMock.resetAll(); + // + // EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal); + // HashMap trackingMap = new HashMap<>(); + // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); + // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); + // connectionFactory.returnConnection(connector); + // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); + // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); + // connectionFactory.returnConnection(connector); + // EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal); + // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); + // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); + // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); + // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); + // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); + // connectionFactory.returnConnection(connector); + // EasyMock.expect(cache.reloadCache(ModelBean.DEFAULT_MODEL_TABLE_NAME)).andReturn(null); + // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); + // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); + // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); + // PowerMock.replayAll(); + // + // bean.deleteModel(MODEL_TWO.getName(), (String) null); + // PowerMock.verifyAll(); + // PowerMock.resetAll(); + // + // EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal); + // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); + // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); + // connectionFactory.returnConnection(connector); + // PowerMock.replayAll(); + // try { + // bean.getModel(MODEL_TWO.getName(), (String) null); + // Assert.fail("getModel should have failed"); + // } catch (DatawaveWebApplicationException e) { + // if (e.getResponse().getStatus() == 404) { + // // success + // } else { + // Assert.fail("getModel did not return a 404, returned: " + e.getResponse().getStatus()); + // } + // } catch (Exception ex) { + // Assert.fail("getModel did not throw a DatawaveWebApplicationException"); + // } + // PowerMock.verifyAll(); + // PowerMock.resetAll(); + // // Ensure model one still intact + // EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal); + // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); + // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); + // connectionFactory.returnConnection(connector); + // PowerMock.replayAll(); + // datawave.webservice.model.Model model1 = bean.getModel(MODEL_ONE.getName(), (String) null); + // PowerMock.verifyAll(); + // Assert.assertEquals(MODEL_ONE, model1); + // + // } + // + // @Test(expected = DatawaveWebApplicationException.class) + // public void testModelGetInvalidModelName() throws Exception { + // EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal); + // HashMap trackingMap = new HashMap<>(); + // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); + // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); + // connectionFactory.returnConnection(connector); + // PowerMock.replayAll(); + // + // bean.getModel(MODEL_ONE.getName(), (String) null); + // PowerMock.verifyAll(); + // } + // + // @Test + // public void testCloneModel() throws Exception { + // importModels(); + // PowerMock.resetAll(); + // + // EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal); + // HashMap trackingMap = new HashMap<>(); + // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); + // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); + // connectionFactory.returnConnection(connector); + // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); + // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); + // connectionFactory.returnConnection(connector); + // EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal); + // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); + // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); + // EasyMock.expect(cache.reloadCache(ModelBean.DEFAULT_MODEL_TABLE_NAME)).andReturn(null); + // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); + // connectionFactory.returnConnection(connector); + // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); + // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); + // PowerMock.replayAll(); + // + // bean.cloneModel(MODEL_ONE.getName(), "MODEL2", (String) null); + // PowerMock.verifyAll(); + // PowerMock.resetAll(); + // EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal); + // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); + // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); + // connectionFactory.returnConnection(connector); + // PowerMock.replayAll(); + // + // datawave.webservice.model.Model model = bean.getModel("MODEL2", (String) null); + // PowerMock.verifyAll(); + // + // MODEL_ONE.setName("MODEL2"); + // Assert.assertEquals(MODEL_ONE, model); + // + // } + // + // @Test + // public void testCheckModelName() throws Exception { + // String modelTableName = Whitebox.invokeMethod(bean, "checkModelTableName", (String) null); + // Assert.assertEquals(ModelBean.DEFAULT_MODEL_TABLE_NAME, modelTableName); + // modelTableName = "foo"; + // String response = Whitebox.invokeMethod(bean, "checkModelTableName", modelTableName); + // Assert.assertEquals(modelTableName, response); + // + // } + // + // private void dumpModels() throws Exception { + // System.out.println("******************* Start Dump Models **********************"); + // Set cbAuths = new HashSet<>(); + // for (Collection auths : principal.getAuthorizations()) { + // cbAuths.add(new Authorizations(auths.toArray(new String[auths.size()]))); + // } + // + // Scanner scanner = ScannerHelper.createScanner(connector, ModelBean.DEFAULT_MODEL_TABLE_NAME, cbAuths); + // for (Entry entry : scanner) { + // System.out.println(entry.getKey()); + // } + // + // System.out.println("******************* End Dump Models **********************"); + // } + +} diff --git a/service/src/test/resources/TestModel_1.xml b/service/src/test/resources/TestModel_1.xml new file mode 100644 index 0000000..f7a784e --- /dev/null +++ b/service/src/test/resources/TestModel_1.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/service/src/test/resources/TestModel_2.xml b/service/src/test/resources/TestModel_2.xml new file mode 100644 index 0000000..cdc3e04 --- /dev/null +++ b/service/src/test/resources/TestModel_2.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/service/src/test/resources/config/application.yml b/service/src/test/resources/config/application.yml index bd9afd9..3ccdecb 100644 --- a/service/src/test/resources/config/application.yml +++ b/service/src/test/resources/config/application.yml @@ -2,6 +2,9 @@ spring: autoconfigure: exclude: org.springframework.boot.autoconfigure.web.servlet.error.ErrorMvcAutoConfiguration + main: + allow-bean-definition-overriding: true + security: user.password: passwordNotUsed datawave: From 00b866f622f1ac5522a212a94f7416d8f153edce Mon Sep 17 00:00:00 2001 From: cmitchell Date: Wed, 20 Apr 2022 15:41:44 -0400 Subject: [PATCH 07/54] Use Spring's @Secured instead of javax.annotations @RolesAllowed, continue test updates and progress. --- .../dictionary/DataDictionaryController.java | 89 ++++++------------- .../microservice/model/ModelController.java | 10 +-- .../DataDictionaryControllerTest.java | 19 +--- .../model/ModelControllerTest.java | 49 ---------- .../src/test/resources/config/application.yml | 4 + 5 files changed, 32 insertions(+), 139 deletions(-) diff --git a/service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java b/service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java index 1dcd106..8b5b015 100644 --- a/service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java +++ b/service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java @@ -4,25 +4,22 @@ import com.google.common.collect.HashMultimap; import com.google.common.collect.Maps; import com.google.common.collect.Multimap; -import datawave.accumulo.util.security.UserAuthFunctions; +import datawave.microservice.AccumuloConnectionService; import datawave.microservice.authorization.user.ProxiedUserDetails; import datawave.microservice.dictionary.config.DataDictionaryProperties; import datawave.microservice.dictionary.config.ResponseObjectFactory; import datawave.microservice.Connection; import datawave.microservice.dictionary.data.DataDictionary; -import datawave.security.authorization.DatawaveUser; import datawave.webservice.metadata.MetadataFieldBase; import datawave.webservice.result.VoidResponse; import datawave.webservice.dictionary.data.DataDictionaryBase; import datawave.webservice.dictionary.data.DescriptionBase; import datawave.webservice.dictionary.data.DictionaryFieldBase; import datawave.webservice.dictionary.data.FieldsBase; -import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.security.Authorizations; import org.apache.commons.lang.StringUtils; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.http.MediaType; +import org.springframework.security.access.annotation.Secured; import org.springframework.security.core.annotation.AuthenticationPrincipal; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -35,7 +32,6 @@ import org.springframework.web.bind.annotation.RestController; import javax.annotation.security.PermitAll; -import javax.annotation.security.RolesAllowed; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -44,8 +40,6 @@ import java.util.Map.Entry; import java.util.Set; import java.util.function.Consumer; -import java.util.function.Supplier; -import java.util.stream.Collectors; import static datawave.microservice.http.converter.protostuff.ProtostuffHttpMessageConverter.PROTOSTUFF_VALUE; @@ -59,8 +53,9 @@ public class DataDictionaryController,DICT ex private final DataDictionaryProperties dataDictionaryConfiguration; private final DataDictionary dataDictionary; private final ResponseObjectFactory responseObjectFactory; - private final UserAuthFunctions userAuthFunctions; - private final Connector accumuloConnector; + private final AccumuloConnectionService accumuloConnectionService; + // private final UserAuthFunctions userAuthFunctions; + // private final Connector accumuloConnector; private final Consumer TRANSFORM_EMPTY_INTERNAL_FIELD_NAMES = meta -> { if (meta.getInternalFieldName() == null || meta.getInternalFieldName().isEmpty()) { @@ -69,13 +64,11 @@ public class DataDictionaryController,DICT ex }; public DataDictionaryController(DataDictionaryProperties dataDictionaryConfiguration, DataDictionary dataDictionary, - ResponseObjectFactory responseObjectFactory, UserAuthFunctions userAuthFunctions, - @Qualifier("warehouse") Connector accumuloConnector) { + ResponseObjectFactory responseObjectFactory, AccumuloConnectionService accumloConnectionService) { this.dataDictionaryConfiguration = dataDictionaryConfiguration; this.dataDictionary = dataDictionary; this.responseObjectFactory = responseObjectFactory; - this.userAuthFunctions = userAuthFunctions; - this.accumuloConnector = accumuloConnector; + this.accumuloConnectionService = accumloConnectionService; dataDictionary.setNormalizationMap(dataDictionaryConfiguration.getNormalizerMap()); } @@ -86,13 +79,13 @@ public DataDictionaryBase get(@RequestParam(required = false) String @RequestParam(required = false) String metadataTableName, @RequestParam(name = "auths", required = false) String queryAuthorizations, @RequestParam(defaultValue = "") String dataTypeFilters, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { - Connection connectionConfig = getConnectionConfig(metadataTableName, modelTableName, modelName, currentUser); + Connection connection = accumuloConnectionService.getConnection(metadataTableName, modelTableName, modelName, currentUser); // If the user provides authorizations, intersect it with their actual authorizations - connectionConfig.setAuths(getDowngradedAuthorizations(queryAuthorizations, currentUser)); + connection.setAuths(accumuloConnectionService.getDowngradedAuthorizations(queryAuthorizations, currentUser)); Collection dataTypes = (StringUtils.isBlank(dataTypeFilters) ? Collections.emptyList() : Arrays.asList(dataTypeFilters.split(","))); - Collection fields = dataDictionary.getFields(connectionConfig, dataTypes, dataDictionaryConfiguration.getNumThreads()); + Collection fields = dataDictionary.getFields(connection, dataTypes, dataDictionaryConfiguration.getNumThreads()); DICT dataDictionary = responseObjectFactory.getDataDictionary(); dataDictionary.setFields(fields); // Ensure that empty internal field names will be set to the field name instead. @@ -120,10 +113,10 @@ public DataDictionaryBase get(@RequestParam(required = false) String @Timed(name = "dw.dictionary.data.uploadDescriptions", absolute = true) public VoidResponse uploadDescriptions(@RequestBody FIELDS fields, @RequestParam(required = false) String modelName, @RequestParam(required = false) String modelTable, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { - Connection connectionConfig = getConnectionConfig(modelTable, modelName, currentUser); + Connection connection = accumuloConnectionService.getConnection(modelTable, modelName, currentUser); List list = fields.getFields(); for (FIELD desc : list) { - dataDictionary.setDescription(connectionConfig, desc); + dataDictionary.setDescription(connection, desc); } // TODO: reload model table cache? // cache.reloadCache(modelTable); @@ -151,7 +144,7 @@ public VoidResponse uploadDescriptions(@RequestBody FIELDS fields, @RequestParam * @throws Exception * if there is any problem updating the description */ - @RolesAllowed({"Administrator", "JBossAdministrator"}) + @Secured({"Administrator", "JBossAdministrator"}) @PutMapping(path = "/Descriptions/{datatype}/{fieldName}/{description}", produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.TEXT_XML_VALUE, PROTOSTUFF_VALUE, "text/x-yaml", "application/x-yaml"}) @Timed(name = "dw.dictionary.data.setDescriptionPut", absolute = true) @@ -182,7 +175,7 @@ public VoidResponse setDescriptionPut(@PathVariable String fieldName, @PathVaria * @throws Exception * if there is any problem updating the dictionary item description */ - @RolesAllowed({"Administrator", "JBossAdministrator"}) + @Secured({"Administrator", "JBossAdministrator"}) @PostMapping(path = "/Descriptions", produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.TEXT_XML_VALUE, PROTOSTUFF_VALUE, "text/x-yaml", "application/x-yaml"}) @Timed(name = "dw.dictionary.data.setDescriptionPost", absolute = true) @@ -195,8 +188,8 @@ public VoidResponse setDescriptionPost(@RequestParam String fieldName, @RequestP desc.setMarkings(markings); desc.setDescription(description); - Connection connectionConfig = getConnectionConfig(modelTable, modelName, currentUser); - dataDictionary.setDescription(connectionConfig, fieldName, datatype, desc); + Connection connection = accumuloConnectionService.getConnection(modelTable, modelName, currentUser); + dataDictionary.setDescription(connection, fieldName, datatype, desc); // TODO: reload model table cache? // cache.reloadCache(modelTable); return new VoidResponse(); @@ -220,8 +213,8 @@ public VoidResponse setDescriptionPost(@RequestParam String fieldName, @RequestP @Timed(name = "dw.dictionary.data.allDescriptions", absolute = true) public FIELDS allDescriptions(@RequestParam(required = false) String modelName, @RequestParam(required = false) String modelTable, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { - Connection connectionConfig = getConnectionConfig(modelTable, modelName, currentUser); - Multimap,DESC> descriptions = dataDictionary.getDescriptions(connectionConfig); + Connection connection = accumuloConnectionService.getConnection(modelTable, modelName, currentUser); + Multimap,DESC> descriptions = dataDictionary.getDescriptions(connection); FIELDS response = this.responseObjectFactory.getFields(); response.setDescriptions(descriptions); return response; @@ -246,8 +239,8 @@ public FIELDS allDescriptions(@RequestParam(required = false) String modelName, @Timed(name = "dw.dictionary.data.datatypeDescriptions", absolute = true) public FIELDS datatypeDescriptions(@PathVariable("datatype") String datatype, @RequestParam(required = false) String modelName, @RequestParam(required = false) String modelTable, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { - Connection connectionConfig = getConnectionConfig(modelTable, modelName, currentUser); - Multimap,DESC> descriptions = dataDictionary.getDescriptions(connectionConfig, datatype); + Connection connection = accumuloConnectionService.getConnection(modelTable, modelName, currentUser); + Multimap,DESC> descriptions = dataDictionary.getDescriptions(connection, datatype); FIELDS response = this.responseObjectFactory.getFields(); response.setDescriptions(descriptions); return response; @@ -274,8 +267,8 @@ public FIELDS datatypeDescriptions(@PathVariable("datatype") String datatype, @R @Timed(name = "dw.dictionary.data.fieldNameDescription", absolute = true) public FIELDS fieldNameDescription(@PathVariable String fieldName, @PathVariable String datatype, @RequestParam(required = false) String modelName, @RequestParam(required = false) String modelTable, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { - Connection connectionConfig = getConnectionConfig(modelTable, modelName, currentUser); - Set descriptions = dataDictionary.getDescriptions(connectionConfig, fieldName, datatype); + Connection connection = accumuloConnectionService.getConnection(modelTable, modelName, currentUser); + Set descriptions = dataDictionary.getDescriptions(connection, fieldName, datatype); FIELDS response = responseObjectFactory.getFields(); if (!descriptions.isEmpty()) { Multimap,DESC> mmap = HashMultimap.create(); @@ -306,7 +299,7 @@ public FIELDS fieldNameDescription(@PathVariable String fieldName, @PathVariable * @throws Exception * if there is any problem removing the description from Accumulo */ - @RolesAllowed({"Administrator", "JBossAdministrator"}) + @Secured({"Administrator", "JBossAdministrator"}) @DeleteMapping(path = "/Descriptions/{datatype}/{fieldName}", produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.TEXT_XML_VALUE, PROTOSTUFF_VALUE, "text/x-yaml", "application/x-yaml"}) @Timed(name = "dw.dictionary.data.deleteDescription", absolute = true) @@ -319,44 +312,12 @@ public VoidResponse deleteDescription(@PathVariable String fieldName, @PathVaria desc.setDescription(""); desc.setMarkings(markings); - Connection connectionConfig = getConnectionConfig(modelTable, modelName, currentUser); + Connection connection = accumuloConnectionService.getConnection(modelTable, modelName, currentUser); - dataDictionary.deleteDescription(connectionConfig, fieldName, datatype, desc); + dataDictionary.deleteDescription(connection, fieldName, datatype, desc); // TODO: reload model table cache? // cache.reloadCache(modelTable); return new VoidResponse(); } - private Connection getConnectionConfig(String modelTable, String modelName, ProxiedUserDetails user) { - return getConnectionConfig(dataDictionaryConfiguration.getMetadataTableName(), modelTable, modelName, user); - } - - private Connection getConnectionConfig(String metadataTable, String modelTable, String modelName, ProxiedUserDetails user) { - Connection helper = new Connection(); - helper.setMetadataTable(getSupplierValueIfBlank(metadataTable, dataDictionaryConfiguration::getMetadataTableName)); - helper.setModelTable(getSupplierValueIfBlank(modelTable, dataDictionaryConfiguration::getModelTableName)); - helper.setModelName(getSupplierValueIfBlank(modelName, dataDictionaryConfiguration::getModelName)); - helper.setAuths(getAuths(user)); - helper.setConnector(accumuloConnector); - return helper; - } - - private String getSupplierValueIfBlank(final String value, final Supplier supplier) { - return StringUtils.isBlank(value) ? supplier.get() : value; - } - - private Set getAuths(ProxiedUserDetails currentUser) { - //@formatter:off - return currentUser.getProxiedUsers().stream() - .map(DatawaveUser::getAuths) - .map(a -> new Authorizations(a.toArray(new String[0]))) - .collect(Collectors.toSet()); - //@formatter:on - } - - private Set getDowngradedAuthorizations(String requestedAuthorizations, ProxiedUserDetails currentUser) { - DatawaveUser primaryUser = currentUser.getPrimaryUser(); - return userAuthFunctions.mergeAuthorizations(userAuthFunctions.getRequestedAuthorizations(requestedAuthorizations, currentUser.getPrimaryUser()), - currentUser.getProxiedUsers(), u -> u != primaryUser); - } } diff --git a/service/src/main/java/datawave/microservice/model/ModelController.java b/service/src/main/java/datawave/microservice/model/ModelController.java index f79f759..c28460e 100644 --- a/service/src/main/java/datawave/microservice/model/ModelController.java +++ b/service/src/main/java/datawave/microservice/model/ModelController.java @@ -103,17 +103,9 @@ public ModelList listModelNames(@RequestParam(defaultValue = DEFAULT_MODEL_TABLE } } catch (TableNotFoundException e) { QueryException qe = new QueryException(DatawaveErrorCode.MODEL_NAME_LIST_ERROR, e); - // log.error(qe); + log.error(qe.getMessage()); response.addException(qe.getBottomQueryException()); // throw new DatawaveWebApplicationException(qe, response); - } finally { - // if (null != connector) { - // try { - // connectionFactory.returnConnection(connector); - // } catch (Exception e) { - // log.error("Error returning connection to factory", e); - // } - // } } response.setNames(modelNames); diff --git a/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java b/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java index 2fcd8a2..8343caa 100644 --- a/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java +++ b/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java @@ -2,15 +2,10 @@ import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; -import com.google.common.collect.Sets; import datawave.accumulo.inmemory.InMemoryInstance; import datawave.marking.MarkingFunctions; import datawave.microservice.ControllerIT; -import datawave.microservice.authorization.jwt.JWTRestTemplate; -import datawave.microservice.authorization.user.ProxiedUserDetails; import datawave.microservice.dictionary.config.DataDictionaryProperties; -import datawave.security.authorization.DatawaveUser; -import datawave.security.authorization.SubjectIssuerDNPair; import datawave.webservice.result.VoidResponse; import datawave.webservice.dictionary.data.DefaultDataDictionary; import datawave.webservice.dictionary.data.DefaultDescription; @@ -18,17 +13,12 @@ import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.client.Instance; import org.apache.accumulo.core.client.TableExistsException; import org.apache.accumulo.core.client.security.tokens.PasswordToken; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.web.client.RestTemplateBuilder; -import org.springframework.boot.web.server.LocalServerPort; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; @@ -38,18 +28,13 @@ import org.springframework.http.MediaType; import org.springframework.http.RequestEntity; import org.springframework.http.ResponseEntity; -import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; -import org.springframework.web.client.DefaultResponseErrorHandler; -import org.springframework.web.client.ResponseErrorHandler; import org.springframework.web.util.UriComponents; import org.springframework.web.util.UriComponentsBuilder; import java.util.AbstractMap; -import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -69,7 +54,7 @@ public Connector warehouseConnector() throws AccumuloSecurityException, Accumulo } } - @BeforeEach + @BeforeAll public void setUp() throws Exception { try { connector.tableOperations().create(dataDictionaryProperties.getMetadataTableName()); diff --git a/service/src/test/java/datawave/microservice/model/ModelControllerTest.java b/service/src/test/java/datawave/microservice/model/ModelControllerTest.java index 0c2c94e..3e2a160 100644 --- a/service/src/test/java/datawave/microservice/model/ModelControllerTest.java +++ b/service/src/test/java/datawave/microservice/model/ModelControllerTest.java @@ -1,48 +1,7 @@ package datawave.microservice.model; -//import static org.powermock.api.easymock.PowerMock.createMock; -//import static org.powermock.api.easymock.PowerMock.createStrictMock; -// -//import java.net.URL; -//import java.util.Arrays; -//import java.util.Collection; -//import java.util.Collections; -//import java.util.HashMap; -//import java.util.HashSet; -//import java.util.Map.Entry; -//import java.util.Set; -// -//import javax.ejb.EJBContext; -//import javax.xml.bind.JAXBContext; -//import javax.xml.bind.Unmarshaller; -// -//import datawave.microservice.dictionary.AccumuloConnectionService; -//import datawave.microservice.dictionary.model.model.ModelKeyParser; -//import datawave.security.authorization.DatawavePrincipal; -//import datawave.security.authorization.DatawaveUser; -//import datawave.security.authorization.DatawaveUser.UserType; -//import datawave.security.authorization.SubjectIssuerDNPair; -//import datawave.security.util.DnUtils.NpeUtils; -//import datawave.security.util.ScannerHelper; -//import datawave.webservice.common.cache.AccumuloTableCache; -//import datawave.webservice.common.connection.AccumuloConnectionFactory; -//import datawave.webservice.common.exception.DatawaveWebApplicationException; -//import datawave.webservice.model.ModelList; -// -//import org.apache.accumulo.core.client.Connector; -//import org.apache.accumulo.core.client.Scanner; -//import datawave.accumulo.inmemory.InMemoryInstance; -//import org.apache.accumulo.core.client.security.tokens.PasswordToken; -//import org.apache.accumulo.core.data.Key; -//import org.apache.accumulo.core.data.Value; -//import org.apache.accumulo.core.security.Authorizations; -//import org.apache.log4j.Level; -//import org.apache.log4j.Logger; -//import org.easymock.EasyMock; -//import org.junit.jupiter.api.BeforeEach; import datawave.accumulo.inmemory.InMemoryInstance; import datawave.microservice.ControllerIT; -import datawave.microservice.dictionary.config.DataDictionaryProperties; import datawave.microservice.model.config.ModelProperties; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; @@ -50,19 +9,11 @@ import org.apache.accumulo.core.client.TableExistsException; import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; -//import org.powermock.api.easymock.PowerMock; -//import org.powermock.core.classloader.annotations.PowerMockIgnore; -//import org.powermock.core.classloader.annotations.PrepareForTest; -//import org.powermock.modules.junit4.PowerMockRunner; -//import org.powermock.reflect.Whitebox; -//import org.springframework.test.context.junit.jupiter.SpringExtension; public class ModelControllerTest extends ControllerIT { diff --git a/service/src/test/resources/config/application.yml b/service/src/test/resources/config/application.yml index 3ccdecb..4aa9433 100644 --- a/service/src/test/resources/config/application.yml +++ b/service/src/test/resources/config/application.yml @@ -14,6 +14,10 @@ spring: - "cn=testserver.example.com, ou=servers, o=example corp, c=us" datawave: + model: + default-table-name: "DatawaveMetadata" + jquery-uri: "/jquery.min.js" + data-tables-uri: "/jquery.dataTables.min.js" metadata: all-metadata-auths: - PRIVATE,PUBLIC From 3118a104149b7c29bddcedb7bc0b3ebd456e7f9e Mon Sep 17 00:00:00 2001 From: cmitchell Date: Thu, 21 Apr 2022 10:32:12 -0400 Subject: [PATCH 08/54] Removed javax.annotations.PermitAll since spring does that by default and removed javax.annotations.Nonnull from test. --- .../microservice/dictionary/DataDictionaryController.java | 2 -- .../microservice/dictionary/EdgeDictionaryController.java | 2 -- service/src/test/java/datawave/microservice/ControllerIT.java | 3 +-- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java b/service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java index 8b5b015..6731cf3 100644 --- a/service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java +++ b/service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java @@ -31,7 +31,6 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; -import javax.annotation.security.PermitAll; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -43,7 +42,6 @@ import static datawave.microservice.http.converter.protostuff.ProtostuffHttpMessageConverter.PROTOSTUFF_VALUE; -@PermitAll @RestController @RequestMapping(path = "/data/v1", produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.TEXT_XML_VALUE, PROTOSTUFF_VALUE, MediaType.TEXT_HTML_VALUE, "text/x-yaml", "application/x-yaml"}) diff --git a/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java b/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java index f645df5..0d7c0b4 100644 --- a/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java +++ b/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java @@ -21,13 +21,11 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; -import javax.annotation.security.PermitAll; import java.util.Set; import static datawave.microservice.http.converter.protostuff.ProtostuffHttpMessageConverter.PROTOSTUFF_VALUE; @Slf4j -@PermitAll @RestController @RequestMapping(path = "/edge/v1", produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.TEXT_XML_VALUE, PROTOSTUFF_VALUE, MediaType.TEXT_HTML_VALUE, "text/x-yaml", "application/x-yaml"}) diff --git a/service/src/test/java/datawave/microservice/ControllerIT.java b/service/src/test/java/datawave/microservice/ControllerIT.java index 45f0088..66b0896 100644 --- a/service/src/test/java/datawave/microservice/ControllerIT.java +++ b/service/src/test/java/datawave/microservice/ControllerIT.java @@ -20,7 +20,6 @@ import org.springframework.web.client.DefaultResponseErrorHandler; import org.springframework.web.client.ResponseErrorHandler; -import javax.annotation.Nonnull; import java.util.Collections; import java.util.HashSet; @@ -57,7 +56,7 @@ public void oneTimeSetup() { // Allow 403 responses through without throwing an exception so tests can assert the response appropriately. ResponseErrorHandler errorHandler = new DefaultResponseErrorHandler() { @Override - protected boolean hasError(@Nonnull HttpStatus statusCode) { + protected boolean hasError(HttpStatus statusCode) { return super.hasError(statusCode) && statusCode.value() != 403; } }; From a2d7e3b359364c206b79f2f910baacdbbe008916 Mon Sep 17 00:00:00 2001 From: cmitchell Date: Fri, 22 Apr 2022 10:45:12 -0400 Subject: [PATCH 09/54] Replaced built-in java 8 javax.xml.bind annotations with more up-to-date and future leaning jakarta (for java 11 and spring 6 compatibility). --- api/pom.xml | 17 +++++++---- .../dictionary/data/DataDictionaryBase.java | 9 +++--- .../data/DefaultDataDictionary.java | 28 +++++++++---------- .../dictionary/data/DefaultDescription.java | 10 +++---- .../data/DefaultDictionaryField.java | 14 ++++------ .../dictionary/data/DefaultFields.java | 12 ++++---- .../dictionary/data/DescriptionBase.java | 8 +++--- .../dictionary/data/DictionaryFieldBase.java | 6 ++-- .../dictionary/data/FieldsBase.java | 6 ++-- .../edge/DefaultEdgeDictionary.java | 18 ++++++------ .../dictionary/edge/DefaultMetadata.java | 16 +++++------ .../dictionary/edge/EdgeDictionaryBase.java | 7 ++--- .../dictionary/edge/EventField.java | 10 +++---- .../dictionary/edge/MetadataBase.java | 8 ++---- .../metadata/DefaultMetadataField.java | 15 +++++----- .../metadata/MetadataFieldBase.java | 6 ++-- .../datawave/webservice/model/Direction.java | 4 +-- .../datawave/webservice/model/Mapping.java | 7 ++--- .../java/datawave/webservice/model/Model.java | 10 +++---- .../datawave/webservice/model/ModelList.java | 8 ++---- .../DefaultDescriptionTest.java | 10 +++---- service/pom.xml | 6 ++++ 22 files changed, 114 insertions(+), 121 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 6532456..31814b2 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -32,6 +32,17 @@ base-rest-responses ${version.base-responses} + + jakarta.xml.bind + jakarta.xml.bind-api + ${version.jakarta.xml.bind} + + + com.sun.xml.bind + jaxb-impl + ${version.sun.xml.bind} + runtime + @@ -43,12 +54,6 @@ gov.nsa.datawave.microservice base-rest-responses - - jakarta.xml.bind - jakarta.xml.bind-api - 2.3.3 - compile - org.projectlombok lombok diff --git a/api/src/main/java/datawave/webservice/dictionary/data/DataDictionaryBase.java b/api/src/main/java/datawave/webservice/dictionary/data/DataDictionaryBase.java index 2c573ca..136c7d6 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/DataDictionaryBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DataDictionaryBase.java @@ -1,20 +1,19 @@ package datawave.webservice.dictionary.data; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import io.protostuff.Message; import datawave.webservice.HtmlProvider; import datawave.webservice.metadata.MetadataFieldBase; import datawave.webservice.result.BaseResponse; import datawave.webservice.result.TotalResultsAware; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlSeeAlso; import java.util.Collection; import java.util.List; import java.util.function.Consumer; -@XmlAccessorType(XmlAccessType.NONE) -@XmlSeeAlso(DefaultDataDictionary.class) +//@JsonTypeInfo(use = JsonTypeInfo.Id.NAME) +@JsonSubTypes({@JsonSubTypes.Type(value = DefaultDataDictionary.class, name = "DefaultDataDictionary")}) public abstract class DataDictionaryBase extends BaseResponse implements TotalResultsAware, Message, HtmlProvider { public abstract List getFields(); diff --git a/api/src/main/java/datawave/webservice/dictionary/data/DefaultDataDictionary.java b/api/src/main/java/datawave/webservice/dictionary/data/DefaultDataDictionary.java index 98017d0..edea4dc 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/DefaultDataDictionary.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DefaultDataDictionary.java @@ -8,15 +8,9 @@ import java.util.List; import java.util.function.Consumer; -import javax.xml.bind.annotation.XmlAccessOrder; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorOrder; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlElementWrapper; -import javax.xml.bind.annotation.XmlRootElement; -import javax.xml.bind.annotation.XmlTransient; - +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.Lists; import datawave.webservice.HtmlProvider; import datawave.webservice.metadata.DefaultMetadataField; @@ -26,6 +20,14 @@ import io.protostuff.Message; import io.protostuff.Output; import io.protostuff.Schema; +import jakarta.xml.bind.annotation.XmlAccessOrder; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorOrder; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlElementWrapper; +import jakarta.xml.bind.annotation.XmlRootElement; +import jakarta.xml.bind.annotation.XmlTransient; @XmlRootElement(name = "DefaultDataDictionary") @XmlAccessorType(XmlAccessType.NONE) @@ -51,10 +53,10 @@ public class DefaultDataDictionary extends DataDictionaryBase fields = null; + private List fields; @XmlElement(name = "TotalResults") - private Long totalResults = null; + private long totalResults; public DefaultDataDictionary() { this("/webjars/jquery/", "/webjars/datatables/"); @@ -116,10 +118,6 @@ public boolean isInitialized(DefaultDataDictionary message) { } public void writeTo(Output output, DefaultDataDictionary message) throws IOException { - if (message.totalResults != null) { - output.writeUInt64(1, message.totalResults, false); - } - if (message.fields != null) { for (DefaultMetadataField field : message.fields) { if (field != null) diff --git a/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java b/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java index bd165e4..4041010 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java @@ -5,12 +5,12 @@ import io.protostuff.Message; import io.protostuff.Output; import io.protostuff.Schema; +import jakarta.xml.bind.annotation.XmlAccessOrder; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorOrder; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlTransient; -import javax.xml.bind.annotation.XmlAccessOrder; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorOrder; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlTransient; import java.io.IOException; import java.io.Serializable; import java.util.HashMap; diff --git a/api/src/main/java/datawave/webservice/dictionary/data/DefaultDictionaryField.java b/api/src/main/java/datawave/webservice/dictionary/data/DefaultDictionaryField.java index 92592ad..b7f24a4 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/DefaultDictionaryField.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DefaultDictionaryField.java @@ -4,19 +4,17 @@ import java.util.Collection; import java.util.Collections; import java.util.Set; - -import javax.xml.bind.annotation.XmlAccessOrder; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorOrder; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlTransient; - import io.protostuff.Input; import io.protostuff.Message; import io.protostuff.Output; import io.protostuff.Schema; import com.google.common.collect.Sets; +import jakarta.xml.bind.annotation.XmlAccessOrder; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorOrder; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlTransient; @XmlAccessorType(XmlAccessType.NONE) @XmlAccessorOrder(XmlAccessOrder.ALPHABETICAL) diff --git a/api/src/main/java/datawave/webservice/dictionary/data/DefaultFields.java b/api/src/main/java/datawave/webservice/dictionary/data/DefaultFields.java index f36eef1..24a3cf0 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/DefaultFields.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DefaultFields.java @@ -8,13 +8,13 @@ import io.protostuff.Schema; import datawave.webservice.HtmlProvider; import datawave.webservice.result.TotalResultsAware; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlElementWrapper; +import jakarta.xml.bind.annotation.XmlRootElement; +import jakarta.xml.bind.annotation.XmlTransient; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlElementWrapper; -import javax.xml.bind.annotation.XmlRootElement; -import javax.xml.bind.annotation.XmlTransient; import java.io.IOException; import java.util.List; import java.util.Map.Entry; diff --git a/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java b/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java index 1b961de..bdddd34 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java @@ -3,11 +3,11 @@ import io.protostuff.Message; import datawave.webservice.query.result.event.HasMarkings; import datawave.webservice.query.util.StringMapAdapter; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.adapters.XmlJavaTypeAdapter; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; import java.util.Map; @XmlAccessorType(XmlAccessType.NONE) diff --git a/api/src/main/java/datawave/webservice/dictionary/data/DictionaryFieldBase.java b/api/src/main/java/datawave/webservice/dictionary/data/DictionaryFieldBase.java index 8522473..0e2b202 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/DictionaryFieldBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DictionaryFieldBase.java @@ -1,10 +1,10 @@ package datawave.webservice.dictionary.data; import io.protostuff.Message; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlSeeAlso; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlSeeAlso; import java.util.Set; @XmlAccessorType(XmlAccessType.NONE) diff --git a/api/src/main/java/datawave/webservice/dictionary/data/FieldsBase.java b/api/src/main/java/datawave/webservice/dictionary/data/FieldsBase.java index 8f6672d..68e2f07 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/FieldsBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/FieldsBase.java @@ -3,10 +3,10 @@ import com.google.common.collect.Multimap; import io.protostuff.Message; import datawave.webservice.result.BaseResponse; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlSeeAlso; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlSeeAlso; import java.util.List; import java.util.Map; diff --git a/api/src/main/java/datawave/webservice/dictionary/edge/DefaultEdgeDictionary.java b/api/src/main/java/datawave/webservice/dictionary/edge/DefaultEdgeDictionary.java index 745ae69..a7fc6da 100644 --- a/api/src/main/java/datawave/webservice/dictionary/edge/DefaultEdgeDictionary.java +++ b/api/src/main/java/datawave/webservice/dictionary/edge/DefaultEdgeDictionary.java @@ -6,16 +6,6 @@ import java.util.Collections; import java.util.LinkedList; import java.util.List; - -import javax.xml.bind.annotation.XmlAccessOrder; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorOrder; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlElementWrapper; -import javax.xml.bind.annotation.XmlRootElement; -import javax.xml.bind.annotation.XmlTransient; - import io.protostuff.Input; import io.protostuff.Message; import io.protostuff.Output; @@ -24,6 +14,14 @@ import datawave.webservice.query.result.util.protostuff.FieldAccessor; import datawave.webservice.query.result.util.protostuff.ProtostuffField; import datawave.webservice.result.TotalResultsAware; +import jakarta.xml.bind.annotation.XmlAccessOrder; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorOrder; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlElementWrapper; +import jakarta.xml.bind.annotation.XmlRootElement; +import jakarta.xml.bind.annotation.XmlTransient; @XmlRootElement(name = "EdgeDictionary") @XmlAccessorType(XmlAccessType.NONE) diff --git a/api/src/main/java/datawave/webservice/dictionary/edge/DefaultMetadata.java b/api/src/main/java/datawave/webservice/dictionary/edge/DefaultMetadata.java index 52be165..7e8eaba 100644 --- a/api/src/main/java/datawave/webservice/dictionary/edge/DefaultMetadata.java +++ b/api/src/main/java/datawave/webservice/dictionary/edge/DefaultMetadata.java @@ -4,15 +4,6 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.List; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlAttribute; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlElementWrapper; -import javax.xml.bind.annotation.XmlTransient; -import javax.xml.bind.annotation.XmlType; - import com.google.common.base.Objects; import io.protostuff.Input; import io.protostuff.Message; @@ -20,6 +11,13 @@ import io.protostuff.Schema; import datawave.webservice.query.result.util.protostuff.FieldAccessor; import datawave.webservice.query.result.util.protostuff.ProtostuffField; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlAttribute; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlElementWrapper; +import jakarta.xml.bind.annotation.XmlTransient; +import jakarta.xml.bind.annotation.XmlType; import org.apache.commons.lang.StringUtils; @XmlAccessorType(XmlAccessType.NONE) diff --git a/api/src/main/java/datawave/webservice/dictionary/edge/EdgeDictionaryBase.java b/api/src/main/java/datawave/webservice/dictionary/edge/EdgeDictionaryBase.java index 759122e..acfbbcc 100644 --- a/api/src/main/java/datawave/webservice/dictionary/edge/EdgeDictionaryBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/edge/EdgeDictionaryBase.java @@ -2,12 +2,11 @@ import java.util.List; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlSeeAlso; - import io.protostuff.Message; import datawave.webservice.result.BaseResponse; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlSeeAlso; @XmlAccessorType(XmlAccessType.NONE) @XmlSeeAlso(DefaultEdgeDictionary.class) diff --git a/api/src/main/java/datawave/webservice/dictionary/edge/EventField.java b/api/src/main/java/datawave/webservice/dictionary/edge/EventField.java index c69db71..f749296 100644 --- a/api/src/main/java/datawave/webservice/dictionary/edge/EventField.java +++ b/api/src/main/java/datawave/webservice/dictionary/edge/EventField.java @@ -2,11 +2,6 @@ import java.io.IOException; import java.io.Serializable; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlAttribute; -import javax.xml.bind.annotation.XmlTransient; -import javax.xml.bind.annotation.XmlType; import com.google.common.base.Objects; import io.protostuff.Input; @@ -15,6 +10,11 @@ import io.protostuff.Schema; import datawave.webservice.query.result.util.protostuff.FieldAccessor; import datawave.webservice.query.result.util.protostuff.ProtostuffField; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlAttribute; +import jakarta.xml.bind.annotation.XmlTransient; +import jakarta.xml.bind.annotation.XmlType; import org.apache.commons.lang.StringUtils; @XmlAccessorType(XmlAccessType.NONE) diff --git a/api/src/main/java/datawave/webservice/dictionary/edge/MetadataBase.java b/api/src/main/java/datawave/webservice/dictionary/edge/MetadataBase.java index 9067d2f..8bc6ab0 100644 --- a/api/src/main/java/datawave/webservice/dictionary/edge/MetadataBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/edge/MetadataBase.java @@ -2,14 +2,12 @@ import java.util.List; import java.util.Map; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlSeeAlso; - import com.google.common.collect.Maps; import io.protostuff.Message; import datawave.webservice.query.result.event.HasMarkings; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlSeeAlso; @XmlAccessorType(XmlAccessType.NONE) @XmlSeeAlso(DefaultMetadata.class) diff --git a/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java b/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java index 9ad382b..4397f72 100644 --- a/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java +++ b/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java @@ -7,17 +7,16 @@ import io.protostuff.Message; import io.protostuff.Output; import io.protostuff.Schema; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlAttribute; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlElementWrapper; +import jakarta.xml.bind.annotation.XmlTransient; +import jakarta.xml.bind.annotation.XmlType; import lombok.Data; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlAttribute; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlElementWrapper; -import javax.xml.bind.annotation.XmlTransient; -import javax.xml.bind.annotation.XmlType; import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; diff --git a/api/src/main/java/datawave/webservice/metadata/MetadataFieldBase.java b/api/src/main/java/datawave/webservice/metadata/MetadataFieldBase.java index fc7eaaa..ef63df9 100644 --- a/api/src/main/java/datawave/webservice/metadata/MetadataFieldBase.java +++ b/api/src/main/java/datawave/webservice/metadata/MetadataFieldBase.java @@ -2,10 +2,10 @@ import io.protostuff.Message; import datawave.webservice.dictionary.data.DescriptionBase; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlSeeAlso; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlSeeAlso; import java.io.Serializable; import java.util.Collection; import java.util.List; diff --git a/api/src/main/java/datawave/webservice/model/Direction.java b/api/src/main/java/datawave/webservice/model/Direction.java index 3484860..b22578f 100644 --- a/api/src/main/java/datawave/webservice/model/Direction.java +++ b/api/src/main/java/datawave/webservice/model/Direction.java @@ -1,8 +1,8 @@ package datawave.webservice.model; -import java.io.Serializable; +import jakarta.xml.bind.annotation.XmlEnum; -import javax.xml.bind.annotation.XmlEnum; +import java.io.Serializable; @XmlEnum public enum Direction implements Serializable { diff --git a/api/src/main/java/datawave/webservice/model/Mapping.java b/api/src/main/java/datawave/webservice/model/Mapping.java index ac0d50c..2d429e9 100644 --- a/api/src/main/java/datawave/webservice/model/Mapping.java +++ b/api/src/main/java/datawave/webservice/model/Mapping.java @@ -2,10 +2,9 @@ import java.io.Serializable; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlAttribute; - +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlAttribute; import lombok.Data; import lombok.NoArgsConstructor; import org.apache.commons.lang.builder.CompareToBuilder; diff --git a/api/src/main/java/datawave/webservice/model/Model.java b/api/src/main/java/datawave/webservice/model/Model.java index 46246dd..1e08c4d 100644 --- a/api/src/main/java/datawave/webservice/model/Model.java +++ b/api/src/main/java/datawave/webservice/model/Model.java @@ -3,15 +3,13 @@ import java.io.Serializable; import java.text.MessageFormat; import java.util.TreeSet; - -import javax.xml.bind.annotation.XmlAttribute; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlElementWrapper; -import javax.xml.bind.annotation.XmlRootElement; - import datawave.webservice.HtmlProvider; import datawave.webservice.result.BaseResponse; +import jakarta.xml.bind.annotation.XmlAttribute; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlElementWrapper; +import jakarta.xml.bind.annotation.XmlRootElement; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; diff --git a/api/src/main/java/datawave/webservice/model/ModelList.java b/api/src/main/java/datawave/webservice/model/ModelList.java index 375daed..5763f73 100644 --- a/api/src/main/java/datawave/webservice/model/ModelList.java +++ b/api/src/main/java/datawave/webservice/model/ModelList.java @@ -3,13 +3,11 @@ import java.io.Serializable; import java.text.MessageFormat; import java.util.HashSet; - -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlElementWrapper; -import javax.xml.bind.annotation.XmlRootElement; - import datawave.webservice.HtmlProvider; import datawave.webservice.result.BaseResponse; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlElementWrapper; +import jakarta.xml.bind.annotation.XmlRootElement; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; diff --git a/api/src/test/java/datawave/webservice/results/datadictionary/DefaultDescriptionTest.java b/api/src/test/java/datawave/webservice/results/datadictionary/DefaultDescriptionTest.java index 6745901..109bc5b 100644 --- a/api/src/test/java/datawave/webservice/results/datadictionary/DefaultDescriptionTest.java +++ b/api/src/test/java/datawave/webservice/results/datadictionary/DefaultDescriptionTest.java @@ -3,13 +3,13 @@ import datawave.webservice.dictionary.data.DefaultDescription; import datawave.webservice.dictionary.data.DefaultDictionaryField; import datawave.webservice.dictionary.data.DefaultFields; +import jakarta.xml.bind.JAXBContext; +import jakarta.xml.bind.JAXBException; +import jakarta.xml.bind.Marshaller; +import jakarta.xml.bind.Unmarshaller; +import jakarta.xml.bind.annotation.XmlSchema; import org.junit.jupiter.api.Test; -import javax.xml.bind.JAXBContext; -import javax.xml.bind.JAXBException; -import javax.xml.bind.Marshaller; -import javax.xml.bind.Unmarshaller; -import javax.xml.bind.annotation.XmlSchema; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.lang.annotation.Annotation; diff --git a/service/pom.xml b/service/pom.xml index 03adca0..a7517c9 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -86,6 +86,12 @@ org.apache.accumulo accumulo-core + + + * + javax.xml.bind + + org.webjars From 313865b5411d0fc10a2aba67202d8340602d0952 Mon Sep 17 00:00:00 2001 From: cmitchell Date: Fri, 22 Apr 2022 14:58:44 -0400 Subject: [PATCH 10/54] test WIP --- .../dictionary/data/DataDictionaryBase.java | 2 - .../java/datawave/webservice/model/Model.java | 2 + .../model/ModelControllerTest.java | 56 +++++++++++++++++++ 3 files changed, 58 insertions(+), 2 deletions(-) diff --git a/api/src/main/java/datawave/webservice/dictionary/data/DataDictionaryBase.java b/api/src/main/java/datawave/webservice/dictionary/data/DataDictionaryBase.java index 136c7d6..12daef7 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/DataDictionaryBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DataDictionaryBase.java @@ -1,7 +1,6 @@ package datawave.webservice.dictionary.data; import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; import io.protostuff.Message; import datawave.webservice.HtmlProvider; import datawave.webservice.metadata.MetadataFieldBase; @@ -12,7 +11,6 @@ import java.util.List; import java.util.function.Consumer; -//@JsonTypeInfo(use = JsonTypeInfo.Id.NAME) @JsonSubTypes({@JsonSubTypes.Type(value = DefaultDataDictionary.class, name = "DefaultDataDictionary")}) public abstract class DataDictionaryBase extends BaseResponse implements TotalResultsAware, Message, HtmlProvider { diff --git a/api/src/main/java/datawave/webservice/model/Model.java b/api/src/main/java/datawave/webservice/model/Model.java index 1e08c4d..a3c564f 100644 --- a/api/src/main/java/datawave/webservice/model/Model.java +++ b/api/src/main/java/datawave/webservice/model/Model.java @@ -13,11 +13,13 @@ import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; +import org.codehaus.jackson.annotate.JsonTypeInfo; @Data @NoArgsConstructor @EqualsAndHashCode(callSuper = true) @XmlRootElement +// @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class") public class Model extends BaseResponse implements Serializable, HtmlProvider { private static final long serialVersionUID = 1L; diff --git a/service/src/test/java/datawave/microservice/model/ModelControllerTest.java b/service/src/test/java/datawave/microservice/model/ModelControllerTest.java index 3e2a160..6501235 100644 --- a/service/src/test/java/datawave/microservice/model/ModelControllerTest.java +++ b/service/src/test/java/datawave/microservice/model/ModelControllerTest.java @@ -1,19 +1,29 @@ package datawave.microservice.model; +import com.fasterxml.jackson.dataformat.xml.XmlMapper; import datawave.accumulo.inmemory.InMemoryInstance; import datawave.microservice.ControllerIT; import datawave.microservice.model.config.ModelProperties; +import datawave.webservice.model.Model; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.TableExistsException; import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; +import org.springframework.core.io.Resource; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; public class ModelControllerTest extends ControllerIT { @@ -30,6 +40,15 @@ public Connector warehouseConnector() throws AccumuloSecurityException, Accumulo } } + @Value("classpath:TestModel_1.xml") + private Resource model1; + + @Value("classpath:TestModel_2.xml") + private Resource model2; + + private Model MODEL_ONE; + private Model MODEL_TWO; + @BeforeEach public void setUp() throws Exception { try { @@ -37,8 +56,45 @@ public void setUp() throws Exception { } catch (TableExistsException e) { // ignore } + + // URL m1Url = ModelBeanTest.class.getResource("/TestModel_1.xml"); + // URL m2Url = ModelBeanTest.class.getResource("/TestModel_2.xml"); + // JAXBContext ctx = JAXBContext.newInstance(Model.class); + // Unmarshaller u = ctx.createUnmarshaller(); + // MODEL_ONE = (datawave.webservice.model.Model) u.unmarshal(testModel1.getInputStream()); + // MODEL_TWO = (datawave.webservice.model.Model) u.unmarshal(testModel2.getInputStream()); + + // XmlMapper mapper = new XmlMapper(); + // MODEL_ONE = mapper.readValue(inputStreamToString(model1.getInputStream()), Model.class); + // MODEL_TWO = mapper.readValue(inputStreamToString(model2.getInputStream()), Model.class); + // + System.out.println(MODEL_ONE); }; + private String inputStreamToString(InputStream fs) throws IOException { + StringBuilder sb = new StringBuilder(); + String line; + BufferedReader br = new BufferedReader(new InputStreamReader(fs)); + while ((line = br.readLine()) != null) { + sb.append(line); + } + br.close(); + return sb.toString(); + } + + @Test + public void testList() { + // @formatter:off +// UriComponents uri = UriComponentsBuilder.newInstance() +// .scheme("https").host("localhost").port(webServicePort) +// .path("/dictionary/model/list") +// .build(); + // @formatter:on + + // ResponseEntity response = jwtRestTemplate.exchange(adminUser, HttpMethod.GET, uri, ModelList.class); + // assertEquals(HttpStatus.OK, response.getStatusCode()); + } + // private static final String userDN = "CN=Guy Some Other soguy, OU=ou1, OU=ou2, OU=ou3, O=o1, C=US"; // private static final String issuerDN = "CN=CA1, OU=ou3, O=o1, C=US"; // private static final String[] auths = new String[] {"PRIVATE", "PUBLIC"}; From 6bba870fdf90d10293b43a059783bb71eaa6ff3c Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Tue, 26 Apr 2022 18:51:36 +0000 Subject: [PATCH 11/54] Revert "Replaced built-in java 8 javax.xml.bind annotations with more up-to-date and future leaning jakarta (for java 11 and spring 6 compatibility)." This reverts commit a2d7e3b359364c206b79f2f910baacdbbe008916. --- api/pom.xml | 17 ++++------- .../dictionary/data/DataDictionaryBase.java | 7 +++-- .../data/DefaultDataDictionary.java | 28 ++++++++++--------- .../dictionary/data/DefaultDescription.java | 10 +++---- .../data/DefaultDictionaryField.java | 14 ++++++---- .../dictionary/data/DefaultFields.java | 12 ++++---- .../dictionary/data/DescriptionBase.java | 8 +++--- .../dictionary/data/DictionaryFieldBase.java | 6 ++-- .../dictionary/data/FieldsBase.java | 6 ++-- .../edge/DefaultEdgeDictionary.java | 18 ++++++------ .../dictionary/edge/DefaultMetadata.java | 16 ++++++----- .../dictionary/edge/EdgeDictionaryBase.java | 7 +++-- .../dictionary/edge/EventField.java | 10 +++---- .../dictionary/edge/MetadataBase.java | 8 ++++-- .../metadata/DefaultMetadataField.java | 15 +++++----- .../metadata/MetadataFieldBase.java | 6 ++-- .../datawave/webservice/model/Direction.java | 4 +-- .../datawave/webservice/model/Mapping.java | 7 +++-- .../java/datawave/webservice/model/Model.java | 10 ++++--- .../datawave/webservice/model/ModelList.java | 8 ++++-- .../DefaultDescriptionTest.java | 10 +++---- service/pom.xml | 6 ---- 22 files changed, 121 insertions(+), 112 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 02392c3..aba2c44 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -32,17 +32,6 @@ base-rest-responses ${version.base-responses} - - jakarta.xml.bind - jakarta.xml.bind-api - ${version.jakarta.xml.bind} - - - com.sun.xml.bind - jaxb-impl - ${version.sun.xml.bind} - runtime - @@ -54,6 +43,12 @@ gov.nsa.datawave.microservice base-rest-responses + + jakarta.xml.bind + jakarta.xml.bind-api + 2.3.3 + compile + org.projectlombok lombok diff --git a/api/src/main/java/datawave/webservice/dictionary/data/DataDictionaryBase.java b/api/src/main/java/datawave/webservice/dictionary/data/DataDictionaryBase.java index 12daef7..2c573ca 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/DataDictionaryBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DataDictionaryBase.java @@ -1,17 +1,20 @@ package datawave.webservice.dictionary.data; -import com.fasterxml.jackson.annotation.JsonSubTypes; import io.protostuff.Message; import datawave.webservice.HtmlProvider; import datawave.webservice.metadata.MetadataFieldBase; import datawave.webservice.result.BaseResponse; import datawave.webservice.result.TotalResultsAware; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlSeeAlso; import java.util.Collection; import java.util.List; import java.util.function.Consumer; -@JsonSubTypes({@JsonSubTypes.Type(value = DefaultDataDictionary.class, name = "DefaultDataDictionary")}) +@XmlAccessorType(XmlAccessType.NONE) +@XmlSeeAlso(DefaultDataDictionary.class) public abstract class DataDictionaryBase extends BaseResponse implements TotalResultsAware, Message, HtmlProvider { public abstract List getFields(); diff --git a/api/src/main/java/datawave/webservice/dictionary/data/DefaultDataDictionary.java b/api/src/main/java/datawave/webservice/dictionary/data/DefaultDataDictionary.java index edea4dc..98017d0 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/DefaultDataDictionary.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DefaultDataDictionary.java @@ -8,9 +8,15 @@ import java.util.List; import java.util.function.Consumer; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; +import javax.xml.bind.annotation.XmlAccessOrder; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorOrder; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlElementWrapper; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlTransient; + import com.google.common.collect.Lists; import datawave.webservice.HtmlProvider; import datawave.webservice.metadata.DefaultMetadataField; @@ -20,14 +26,6 @@ import io.protostuff.Message; import io.protostuff.Output; import io.protostuff.Schema; -import jakarta.xml.bind.annotation.XmlAccessOrder; -import jakarta.xml.bind.annotation.XmlAccessType; -import jakarta.xml.bind.annotation.XmlAccessorOrder; -import jakarta.xml.bind.annotation.XmlAccessorType; -import jakarta.xml.bind.annotation.XmlElement; -import jakarta.xml.bind.annotation.XmlElementWrapper; -import jakarta.xml.bind.annotation.XmlRootElement; -import jakarta.xml.bind.annotation.XmlTransient; @XmlRootElement(name = "DefaultDataDictionary") @XmlAccessorType(XmlAccessType.NONE) @@ -53,10 +51,10 @@ public class DefaultDataDictionary extends DataDictionaryBase fields; + private List fields = null; @XmlElement(name = "TotalResults") - private long totalResults; + private Long totalResults = null; public DefaultDataDictionary() { this("/webjars/jquery/", "/webjars/datatables/"); @@ -118,6 +116,10 @@ public boolean isInitialized(DefaultDataDictionary message) { } public void writeTo(Output output, DefaultDataDictionary message) throws IOException { + if (message.totalResults != null) { + output.writeUInt64(1, message.totalResults, false); + } + if (message.fields != null) { for (DefaultMetadataField field : message.fields) { if (field != null) diff --git a/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java b/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java index 4041010..bd165e4 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java @@ -5,12 +5,12 @@ import io.protostuff.Message; import io.protostuff.Output; import io.protostuff.Schema; -import jakarta.xml.bind.annotation.XmlAccessOrder; -import jakarta.xml.bind.annotation.XmlAccessType; -import jakarta.xml.bind.annotation.XmlAccessorOrder; -import jakarta.xml.bind.annotation.XmlAccessorType; -import jakarta.xml.bind.annotation.XmlTransient; +import javax.xml.bind.annotation.XmlAccessOrder; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorOrder; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlTransient; import java.io.IOException; import java.io.Serializable; import java.util.HashMap; diff --git a/api/src/main/java/datawave/webservice/dictionary/data/DefaultDictionaryField.java b/api/src/main/java/datawave/webservice/dictionary/data/DefaultDictionaryField.java index b7f24a4..92592ad 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/DefaultDictionaryField.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DefaultDictionaryField.java @@ -4,17 +4,19 @@ import java.util.Collection; import java.util.Collections; import java.util.Set; + +import javax.xml.bind.annotation.XmlAccessOrder; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorOrder; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlTransient; + import io.protostuff.Input; import io.protostuff.Message; import io.protostuff.Output; import io.protostuff.Schema; import com.google.common.collect.Sets; -import jakarta.xml.bind.annotation.XmlAccessOrder; -import jakarta.xml.bind.annotation.XmlAccessType; -import jakarta.xml.bind.annotation.XmlAccessorOrder; -import jakarta.xml.bind.annotation.XmlAccessorType; -import jakarta.xml.bind.annotation.XmlElement; -import jakarta.xml.bind.annotation.XmlTransient; @XmlAccessorType(XmlAccessType.NONE) @XmlAccessorOrder(XmlAccessOrder.ALPHABETICAL) diff --git a/api/src/main/java/datawave/webservice/dictionary/data/DefaultFields.java b/api/src/main/java/datawave/webservice/dictionary/data/DefaultFields.java index 24a3cf0..f36eef1 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/DefaultFields.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DefaultFields.java @@ -8,13 +8,13 @@ import io.protostuff.Schema; import datawave.webservice.HtmlProvider; import datawave.webservice.result.TotalResultsAware; -import jakarta.xml.bind.annotation.XmlAccessType; -import jakarta.xml.bind.annotation.XmlAccessorType; -import jakarta.xml.bind.annotation.XmlElement; -import jakarta.xml.bind.annotation.XmlElementWrapper; -import jakarta.xml.bind.annotation.XmlRootElement; -import jakarta.xml.bind.annotation.XmlTransient; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlElementWrapper; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlTransient; import java.io.IOException; import java.util.List; import java.util.Map.Entry; diff --git a/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java b/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java index bdddd34..1b961de 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java @@ -3,11 +3,11 @@ import io.protostuff.Message; import datawave.webservice.query.result.event.HasMarkings; import datawave.webservice.query.util.StringMapAdapter; -import jakarta.xml.bind.annotation.XmlAccessType; -import jakarta.xml.bind.annotation.XmlAccessorType; -import jakarta.xml.bind.annotation.XmlElement; -import jakarta.xml.bind.annotation.adapters.XmlJavaTypeAdapter; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; import java.util.Map; @XmlAccessorType(XmlAccessType.NONE) diff --git a/api/src/main/java/datawave/webservice/dictionary/data/DictionaryFieldBase.java b/api/src/main/java/datawave/webservice/dictionary/data/DictionaryFieldBase.java index 0e2b202..8522473 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/DictionaryFieldBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DictionaryFieldBase.java @@ -1,10 +1,10 @@ package datawave.webservice.dictionary.data; import io.protostuff.Message; -import jakarta.xml.bind.annotation.XmlAccessType; -import jakarta.xml.bind.annotation.XmlAccessorType; -import jakarta.xml.bind.annotation.XmlSeeAlso; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlSeeAlso; import java.util.Set; @XmlAccessorType(XmlAccessType.NONE) diff --git a/api/src/main/java/datawave/webservice/dictionary/data/FieldsBase.java b/api/src/main/java/datawave/webservice/dictionary/data/FieldsBase.java index 68e2f07..8f6672d 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/FieldsBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/FieldsBase.java @@ -3,10 +3,10 @@ import com.google.common.collect.Multimap; import io.protostuff.Message; import datawave.webservice.result.BaseResponse; -import jakarta.xml.bind.annotation.XmlAccessType; -import jakarta.xml.bind.annotation.XmlAccessorType; -import jakarta.xml.bind.annotation.XmlSeeAlso; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlSeeAlso; import java.util.List; import java.util.Map; diff --git a/api/src/main/java/datawave/webservice/dictionary/edge/DefaultEdgeDictionary.java b/api/src/main/java/datawave/webservice/dictionary/edge/DefaultEdgeDictionary.java index a7fc6da..745ae69 100644 --- a/api/src/main/java/datawave/webservice/dictionary/edge/DefaultEdgeDictionary.java +++ b/api/src/main/java/datawave/webservice/dictionary/edge/DefaultEdgeDictionary.java @@ -6,6 +6,16 @@ import java.util.Collections; import java.util.LinkedList; import java.util.List; + +import javax.xml.bind.annotation.XmlAccessOrder; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorOrder; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlElementWrapper; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlTransient; + import io.protostuff.Input; import io.protostuff.Message; import io.protostuff.Output; @@ -14,14 +24,6 @@ import datawave.webservice.query.result.util.protostuff.FieldAccessor; import datawave.webservice.query.result.util.protostuff.ProtostuffField; import datawave.webservice.result.TotalResultsAware; -import jakarta.xml.bind.annotation.XmlAccessOrder; -import jakarta.xml.bind.annotation.XmlAccessType; -import jakarta.xml.bind.annotation.XmlAccessorOrder; -import jakarta.xml.bind.annotation.XmlAccessorType; -import jakarta.xml.bind.annotation.XmlElement; -import jakarta.xml.bind.annotation.XmlElementWrapper; -import jakarta.xml.bind.annotation.XmlRootElement; -import jakarta.xml.bind.annotation.XmlTransient; @XmlRootElement(name = "EdgeDictionary") @XmlAccessorType(XmlAccessType.NONE) diff --git a/api/src/main/java/datawave/webservice/dictionary/edge/DefaultMetadata.java b/api/src/main/java/datawave/webservice/dictionary/edge/DefaultMetadata.java index 7e8eaba..52be165 100644 --- a/api/src/main/java/datawave/webservice/dictionary/edge/DefaultMetadata.java +++ b/api/src/main/java/datawave/webservice/dictionary/edge/DefaultMetadata.java @@ -4,6 +4,15 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.List; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlElementWrapper; +import javax.xml.bind.annotation.XmlTransient; +import javax.xml.bind.annotation.XmlType; + import com.google.common.base.Objects; import io.protostuff.Input; import io.protostuff.Message; @@ -11,13 +20,6 @@ import io.protostuff.Schema; import datawave.webservice.query.result.util.protostuff.FieldAccessor; import datawave.webservice.query.result.util.protostuff.ProtostuffField; -import jakarta.xml.bind.annotation.XmlAccessType; -import jakarta.xml.bind.annotation.XmlAccessorType; -import jakarta.xml.bind.annotation.XmlAttribute; -import jakarta.xml.bind.annotation.XmlElement; -import jakarta.xml.bind.annotation.XmlElementWrapper; -import jakarta.xml.bind.annotation.XmlTransient; -import jakarta.xml.bind.annotation.XmlType; import org.apache.commons.lang.StringUtils; @XmlAccessorType(XmlAccessType.NONE) diff --git a/api/src/main/java/datawave/webservice/dictionary/edge/EdgeDictionaryBase.java b/api/src/main/java/datawave/webservice/dictionary/edge/EdgeDictionaryBase.java index acfbbcc..759122e 100644 --- a/api/src/main/java/datawave/webservice/dictionary/edge/EdgeDictionaryBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/edge/EdgeDictionaryBase.java @@ -2,11 +2,12 @@ import java.util.List; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlSeeAlso; + import io.protostuff.Message; import datawave.webservice.result.BaseResponse; -import jakarta.xml.bind.annotation.XmlAccessType; -import jakarta.xml.bind.annotation.XmlAccessorType; -import jakarta.xml.bind.annotation.XmlSeeAlso; @XmlAccessorType(XmlAccessType.NONE) @XmlSeeAlso(DefaultEdgeDictionary.class) diff --git a/api/src/main/java/datawave/webservice/dictionary/edge/EventField.java b/api/src/main/java/datawave/webservice/dictionary/edge/EventField.java index f749296..c69db71 100644 --- a/api/src/main/java/datawave/webservice/dictionary/edge/EventField.java +++ b/api/src/main/java/datawave/webservice/dictionary/edge/EventField.java @@ -2,6 +2,11 @@ import java.io.IOException; import java.io.Serializable; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlTransient; +import javax.xml.bind.annotation.XmlType; import com.google.common.base.Objects; import io.protostuff.Input; @@ -10,11 +15,6 @@ import io.protostuff.Schema; import datawave.webservice.query.result.util.protostuff.FieldAccessor; import datawave.webservice.query.result.util.protostuff.ProtostuffField; -import jakarta.xml.bind.annotation.XmlAccessType; -import jakarta.xml.bind.annotation.XmlAccessorType; -import jakarta.xml.bind.annotation.XmlAttribute; -import jakarta.xml.bind.annotation.XmlTransient; -import jakarta.xml.bind.annotation.XmlType; import org.apache.commons.lang.StringUtils; @XmlAccessorType(XmlAccessType.NONE) diff --git a/api/src/main/java/datawave/webservice/dictionary/edge/MetadataBase.java b/api/src/main/java/datawave/webservice/dictionary/edge/MetadataBase.java index 8bc6ab0..9067d2f 100644 --- a/api/src/main/java/datawave/webservice/dictionary/edge/MetadataBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/edge/MetadataBase.java @@ -2,12 +2,14 @@ import java.util.List; import java.util.Map; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlSeeAlso; + import com.google.common.collect.Maps; import io.protostuff.Message; import datawave.webservice.query.result.event.HasMarkings; -import jakarta.xml.bind.annotation.XmlAccessType; -import jakarta.xml.bind.annotation.XmlAccessorType; -import jakarta.xml.bind.annotation.XmlSeeAlso; @XmlAccessorType(XmlAccessType.NONE) @XmlSeeAlso(DefaultMetadata.class) diff --git a/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java b/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java index 4397f72..9ad382b 100644 --- a/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java +++ b/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java @@ -7,16 +7,17 @@ import io.protostuff.Message; import io.protostuff.Output; import io.protostuff.Schema; -import jakarta.xml.bind.annotation.XmlAccessType; -import jakarta.xml.bind.annotation.XmlAccessorType; -import jakarta.xml.bind.annotation.XmlAttribute; -import jakarta.xml.bind.annotation.XmlElement; -import jakarta.xml.bind.annotation.XmlElementWrapper; -import jakarta.xml.bind.annotation.XmlTransient; -import jakarta.xml.bind.annotation.XmlType; import lombok.Data; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlElementWrapper; +import javax.xml.bind.annotation.XmlTransient; +import javax.xml.bind.annotation.XmlType; import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; diff --git a/api/src/main/java/datawave/webservice/metadata/MetadataFieldBase.java b/api/src/main/java/datawave/webservice/metadata/MetadataFieldBase.java index ef63df9..fc7eaaa 100644 --- a/api/src/main/java/datawave/webservice/metadata/MetadataFieldBase.java +++ b/api/src/main/java/datawave/webservice/metadata/MetadataFieldBase.java @@ -2,10 +2,10 @@ import io.protostuff.Message; import datawave.webservice.dictionary.data.DescriptionBase; -import jakarta.xml.bind.annotation.XmlAccessType; -import jakarta.xml.bind.annotation.XmlAccessorType; -import jakarta.xml.bind.annotation.XmlSeeAlso; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlSeeAlso; import java.io.Serializable; import java.util.Collection; import java.util.List; diff --git a/api/src/main/java/datawave/webservice/model/Direction.java b/api/src/main/java/datawave/webservice/model/Direction.java index b22578f..3484860 100644 --- a/api/src/main/java/datawave/webservice/model/Direction.java +++ b/api/src/main/java/datawave/webservice/model/Direction.java @@ -1,9 +1,9 @@ package datawave.webservice.model; -import jakarta.xml.bind.annotation.XmlEnum; - import java.io.Serializable; +import javax.xml.bind.annotation.XmlEnum; + @XmlEnum public enum Direction implements Serializable { diff --git a/api/src/main/java/datawave/webservice/model/Mapping.java b/api/src/main/java/datawave/webservice/model/Mapping.java index 2d429e9..ac0d50c 100644 --- a/api/src/main/java/datawave/webservice/model/Mapping.java +++ b/api/src/main/java/datawave/webservice/model/Mapping.java @@ -2,9 +2,10 @@ import java.io.Serializable; -import jakarta.xml.bind.annotation.XmlAccessType; -import jakarta.xml.bind.annotation.XmlAccessorType; -import jakarta.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; + import lombok.Data; import lombok.NoArgsConstructor; import org.apache.commons.lang.builder.CompareToBuilder; diff --git a/api/src/main/java/datawave/webservice/model/Model.java b/api/src/main/java/datawave/webservice/model/Model.java index a3c564f..e7184b4 100644 --- a/api/src/main/java/datawave/webservice/model/Model.java +++ b/api/src/main/java/datawave/webservice/model/Model.java @@ -3,13 +3,15 @@ import java.io.Serializable; import java.text.MessageFormat; import java.util.TreeSet; + +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlElementWrapper; +import javax.xml.bind.annotation.XmlRootElement; + import datawave.webservice.HtmlProvider; import datawave.webservice.result.BaseResponse; -import jakarta.xml.bind.annotation.XmlAttribute; -import jakarta.xml.bind.annotation.XmlElement; -import jakarta.xml.bind.annotation.XmlElementWrapper; -import jakarta.xml.bind.annotation.XmlRootElement; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; diff --git a/api/src/main/java/datawave/webservice/model/ModelList.java b/api/src/main/java/datawave/webservice/model/ModelList.java index 5763f73..375daed 100644 --- a/api/src/main/java/datawave/webservice/model/ModelList.java +++ b/api/src/main/java/datawave/webservice/model/ModelList.java @@ -3,11 +3,13 @@ import java.io.Serializable; import java.text.MessageFormat; import java.util.HashSet; + +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlElementWrapper; +import javax.xml.bind.annotation.XmlRootElement; + import datawave.webservice.HtmlProvider; import datawave.webservice.result.BaseResponse; -import jakarta.xml.bind.annotation.XmlElement; -import jakarta.xml.bind.annotation.XmlElementWrapper; -import jakarta.xml.bind.annotation.XmlRootElement; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; diff --git a/api/src/test/java/datawave/webservice/results/datadictionary/DefaultDescriptionTest.java b/api/src/test/java/datawave/webservice/results/datadictionary/DefaultDescriptionTest.java index 109bc5b..6745901 100644 --- a/api/src/test/java/datawave/webservice/results/datadictionary/DefaultDescriptionTest.java +++ b/api/src/test/java/datawave/webservice/results/datadictionary/DefaultDescriptionTest.java @@ -3,13 +3,13 @@ import datawave.webservice.dictionary.data.DefaultDescription; import datawave.webservice.dictionary.data.DefaultDictionaryField; import datawave.webservice.dictionary.data.DefaultFields; -import jakarta.xml.bind.JAXBContext; -import jakarta.xml.bind.JAXBException; -import jakarta.xml.bind.Marshaller; -import jakarta.xml.bind.Unmarshaller; -import jakarta.xml.bind.annotation.XmlSchema; import org.junit.jupiter.api.Test; +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; +import javax.xml.bind.Unmarshaller; +import javax.xml.bind.annotation.XmlSchema; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.lang.annotation.Annotation; diff --git a/service/pom.xml b/service/pom.xml index 9ec0df0..28ec429 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -116,12 +116,6 @@ org.apache.accumulo accumulo-core - - - * - javax.xml.bind - - org.webjars From e9d66c01ef14f95e324d13f660d43acaaf8397ea Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 4 May 2022 14:09:07 -0400 Subject: [PATCH 12/54] Removed unused slf4j bindings, and log4j 1.x. --- service/pom.xml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/service/pom.xml b/service/pom.xml index 28ec429..6e7684a 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -49,6 +49,14 @@ servlet-api javax.servlet + + slf4j-reload4j + org.slf4j + + + log4j + log4j + @@ -65,6 +73,14 @@ servlet-api javax.servlet + + slf4j-reload4j + org.slf4j + + + log4j + log4j + @@ -138,6 +154,12 @@ gov.nsa.datawave.contrib datawave-in-memory-accumulo test + + + slf4j-reload4j + org.slf4j + + org.junit.jupiter From d1bcaebb3a056de99bb488525b07734f013b160f Mon Sep 17 00:00:00 2001 From: cmitchell Date: Thu, 5 May 2022 10:58:25 -0400 Subject: [PATCH 13/54] Removing commented out code and finishing test. --- .../datawave/webservice/model/Mapping.java | 48 -- .../java/datawave/webservice/model/Model.java | 81 ++- .../webservice/model/ModelKeyParser.java | 11 +- .../datawave/webservice/model/ModelList.java | 24 +- .../webservice/model/package-info.java | 6 + service/pom.xml | 9 + .../AccumuloConnectionService.java | 11 + .../microservice/model/ModelController.java | 222 +++------ .../model/ModelControllerTest.java | 468 ++++++------------ 9 files changed, 334 insertions(+), 546 deletions(-) delete mode 100644 api/src/main/java/datawave/webservice/model/Mapping.java create mode 100644 api/src/main/java/datawave/webservice/model/package-info.java diff --git a/api/src/main/java/datawave/webservice/model/Mapping.java b/api/src/main/java/datawave/webservice/model/Mapping.java deleted file mode 100644 index 2d429e9..0000000 --- a/api/src/main/java/datawave/webservice/model/Mapping.java +++ /dev/null @@ -1,48 +0,0 @@ -package datawave.webservice.model; - -import java.io.Serializable; - -import jakarta.xml.bind.annotation.XmlAccessType; -import jakarta.xml.bind.annotation.XmlAccessorType; -import jakarta.xml.bind.annotation.XmlAttribute; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.apache.commons.lang.builder.CompareToBuilder; - -@Data -@NoArgsConstructor -@XmlAccessorType(XmlAccessType.NONE) -public class Mapping implements Serializable, Comparable { - - private static final long serialVersionUID = 1L; - - @XmlAttribute(required = true) - private String datatype; - - @XmlAttribute(required = true) - private String fieldName; - - @XmlAttribute(required = true) - private String modelFieldName; - - @XmlAttribute(required = true) - private Direction direction; - - @XmlAttribute(required = true) - private String columnVisibility; - - @Override - public int compareTo(Mapping obj) { - - if (obj == null) { - throw new IllegalArgumentException("can not compare null"); - } - - if (obj == this) - return 0; - - return new CompareToBuilder().append(datatype, obj.datatype).append(fieldName, obj.fieldName).append(modelFieldName, obj.modelFieldName) - .append(direction, obj.direction).append(columnVisibility, obj.columnVisibility).toComparison(); - } - -} diff --git a/api/src/main/java/datawave/webservice/model/Model.java b/api/src/main/java/datawave/webservice/model/Model.java index a3c564f..30f7df1 100644 --- a/api/src/main/java/datawave/webservice/model/Model.java +++ b/api/src/main/java/datawave/webservice/model/Model.java @@ -3,23 +3,23 @@ import java.io.Serializable; import java.text.MessageFormat; import java.util.TreeSet; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlElementWrapper; +import javax.xml.bind.annotation.XmlRootElement; + import datawave.webservice.HtmlProvider; import datawave.webservice.result.BaseResponse; -import jakarta.xml.bind.annotation.XmlAttribute; -import jakarta.xml.bind.annotation.XmlElement; -import jakarta.xml.bind.annotation.XmlElementWrapper; -import jakarta.xml.bind.annotation.XmlRootElement; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; -import org.codehaus.jackson.annotate.JsonTypeInfo; +import org.apache.commons.lang.builder.EqualsBuilder; +import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.commons.lang.builder.ToStringBuilder; -@Data -@NoArgsConstructor -@EqualsAndHashCode(callSuper = true) -@XmlRootElement -// @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class") +@XmlRootElement(name = "Model") +@XmlAccessorType(XmlAccessType.NONE) public class Model extends BaseResponse implements Serializable, HtmlProvider { private static final long serialVersionUID = 1L; @@ -27,21 +27,64 @@ public class Model extends BaseResponse implements Serializable, HtmlProvider { private String dataTablesUri; private static final String TITLE = "Model Description", EMPTY = ""; private static final String DATA_TABLES_TEMPLATE = "\n" - + "\n" + "\n" + + "\n"; public Model(String jqueryUri, String datatablesUri) { this.jqueryUri = jqueryUri; this.dataTablesUri = datatablesUri; + } - @XmlAttribute(required = true) - private String name; + // Only used in ModelBeanTest now + public Model() {}; + + @XmlAttribute(name = "name", required = true) + private String name = null; @XmlElementWrapper(name = "Mappings") - @XmlElement(name = "Mappings") - private TreeSet mappings = new TreeSet(); + @XmlElement(name = "Mapping") + private TreeSet fields = new TreeSet(); + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public TreeSet getFields() { + return fields; + } + + public void setFields(TreeSet fields) { + this.fields = fields; + } + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 37).append(name).append(fields).toHashCode(); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) + return false; + if (obj == this) + return true; + if (obj.getClass() != this.getClass()) + return false; + Model other = (Model) obj; + return new EqualsBuilder().append(name, other.name).append(fields, other.fields).isEquals(); + } + + @Override + public String toString() { + return new ToStringBuilder(this).append("name", name).append("fields", fields).toString(); + } /* * (non-Javadoc) @@ -89,7 +132,7 @@ public String getMainContent() { builder.append("VisibilityFieldNameDataTypeModelFieldNameDirection"); builder.append(""); - for (Mapping f : this.getMappings()) { + for (FieldMapping f : this.getFields()) { builder.append("").append(f.getColumnVisibility()).append(""); builder.append("").append(f.getFieldName()).append(""); builder.append("").append(f.getDatatype()).append(""); diff --git a/api/src/main/java/datawave/webservice/model/ModelKeyParser.java b/api/src/main/java/datawave/webservice/model/ModelKeyParser.java index 8bb106a..6efd8f8 100644 --- a/api/src/main/java/datawave/webservice/model/ModelKeyParser.java +++ b/api/src/main/java/datawave/webservice/model/ModelKeyParser.java @@ -1,5 +1,6 @@ package datawave.webservice.model; +import io.netty.util.Mapping; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; @@ -16,8 +17,8 @@ public class ModelKeyParser { public static final Value NULL_VALUE = new Value(new byte[0]); private static final Logger log = Logger.getLogger(ModelKeyParser.class); - public static Mapping parseKey(Key key, Set auths) { - Mapping mapping = new Mapping(); + public static FieldMapping parseKey(Key key, Set auths) { + FieldMapping mapping = new FieldMapping(); String row = key.getRow().toString(); String[] colf = key.getColumnFamily().toString().split(NULL_BYTE); String[] colq = key.getColumnQualifier().toString().split(NULL_BYTE); @@ -61,7 +62,7 @@ public static Mapping parseKey(Key key, Set auths) { return mapping; } - public static Key createKey(Mapping mapping, String modelName) { + public static Key createKey(FieldMapping mapping, String modelName) { ColumnVisibility cv = new ColumnVisibility(mapping.getColumnVisibility()); String inName = Direction.REVERSE.equals(mapping.getDirection()) ? mapping.getFieldName() : mapping.getModelFieldName(); @@ -75,7 +76,7 @@ public static Key createKey(Mapping mapping, String modelName) { ); } - public static Mutation createMutation(Mapping mapping, String modelName) { + public static Mutation createMutation(FieldMapping mapping, String modelName) { ColumnVisibility cv = new ColumnVisibility(mapping.getColumnVisibility()); Mutation m; String dataType = StringUtils.isEmpty(mapping.getDatatype()) ? "" : NULL_BYTE + mapping.getDatatype().trim(); @@ -93,7 +94,7 @@ public static Mutation createMutation(Mapping mapping, String modelName) { } } - public static Mutation createDeleteMutation(Mapping mapping, String modelName) { + public static Mutation createDeleteMutation(FieldMapping mapping, String modelName) { ColumnVisibility cv = new ColumnVisibility(mapping.getColumnVisibility()); Mutation m; String dataType = StringUtils.isEmpty(mapping.getDatatype()) ? "" : NULL_BYTE + mapping.getDatatype().trim(); diff --git a/api/src/main/java/datawave/webservice/model/ModelList.java b/api/src/main/java/datawave/webservice/model/ModelList.java index 5763f73..fe1d93f 100644 --- a/api/src/main/java/datawave/webservice/model/ModelList.java +++ b/api/src/main/java/datawave/webservice/model/ModelList.java @@ -3,19 +3,15 @@ import java.io.Serializable; import java.text.MessageFormat; import java.util.HashSet; + +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlElementWrapper; +import javax.xml.bind.annotation.XmlRootElement; + import datawave.webservice.HtmlProvider; import datawave.webservice.result.BaseResponse; -import jakarta.xml.bind.annotation.XmlElement; -import jakarta.xml.bind.annotation.XmlElementWrapper; -import jakarta.xml.bind.annotation.XmlRootElement; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; -@Data -@NoArgsConstructor -@EqualsAndHashCode(callSuper = true) -@XmlRootElement +@XmlRootElement(name = "ModelList") public class ModelList extends BaseResponse implements Serializable, HtmlProvider { private String jqueryUri; @@ -25,10 +21,16 @@ public class ModelList extends BaseResponse implements Serializable, HtmlProvide private static final long serialVersionUID = 1L; private static final String TITLE = "Model Names"; private static final String DATA_TABLES_TEMPLATE = "\n" - + "\n" + "\n" + + "\n"; + /** + * Need empty constructor for serialization + */ + public ModelList() {} + public ModelList(String jqueryUri, String datatablesUri, String modelTableName) { this.jqueryUri = jqueryUri; this.dataTablesUri = datatablesUri; diff --git a/api/src/main/java/datawave/webservice/model/package-info.java b/api/src/main/java/datawave/webservice/model/package-info.java new file mode 100644 index 0000000..7ea8543 --- /dev/null +++ b/api/src/main/java/datawave/webservice/model/package-info.java @@ -0,0 +1,6 @@ +@XmlSchema(namespace="http://webservice.datawave/v1", elementFormDefault=XmlNsForm.QUALIFIED, xmlns={@XmlNs(prefix = "http://webservice.datawave/v1", namespaceURI = "${datawave.webservice.namespace}")}) +package datawave.webservice.model; + +import javax.xml.bind.annotation.XmlNs; +import javax.xml.bind.annotation.XmlNsForm; +import javax.xml.bind.annotation.XmlSchema; diff --git a/service/pom.xml b/service/pom.xml index 9ec0df0..5b525fc 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -94,9 +94,18 @@ pom import + + com.sun.xml.bind + jaxb-impl + 2.3.3 + + + com.sun.xml.bind + jaxb-impl + com.fasterxml.jackson.dataformat jackson-dataformat-xml diff --git a/service/src/main/java/datawave/microservice/AccumuloConnectionService.java b/service/src/main/java/datawave/microservice/AccumuloConnectionService.java index cb457a7..262c985 100644 --- a/service/src/main/java/datawave/microservice/AccumuloConnectionService.java +++ b/service/src/main/java/datawave/microservice/AccumuloConnectionService.java @@ -6,8 +6,10 @@ import datawave.security.authorization.DatawaveUser; import datawave.security.util.ScannerHelper; import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.iterators.user.RegExFilter; import org.apache.accumulo.core.security.Authorizations; import org.apache.commons.lang.StringUtils; import org.springframework.beans.factory.annotation.Qualifier; @@ -73,4 +75,13 @@ public Set getDowngradedAuthorizations(String requestedAuthoriza public Scanner getScanner(String modelTable, ProxiedUserDetails currentUser) throws TableNotFoundException { return ScannerHelper.createScanner(this.accumuloConnector, modelTable, this.getAuths(currentUser)); } + + public Scanner getScannerWithRegexIteratorSetting(String name, String modelTable, ProxiedUserDetails currentUser) throws TableNotFoundException { + IteratorSetting cfg = new IteratorSetting(21, "colfRegex", RegExFilter.class.getName()); + cfg.addOption(RegExFilter.COLF_REGEX, "^" + name + "(\\x00.*)?"); + + Scanner scanner = getScanner(modelTable, currentUser); + scanner.addScanIterator(cfg); + return scanner; + } } diff --git a/service/src/main/java/datawave/microservice/model/ModelController.java b/service/src/main/java/datawave/microservice/model/ModelController.java index c28460e..c70cf2b 100644 --- a/service/src/main/java/datawave/microservice/model/ModelController.java +++ b/service/src/main/java/datawave/microservice/model/ModelController.java @@ -10,9 +10,11 @@ //import datawave.webservice.common.exception.NotFoundException; //import datawave.webservice.common.exception.PreConditionFailedException; import datawave.microservice.AccumuloConnectionService; +import datawave.microservice.Connection; import datawave.microservice.authorization.user.ProxiedUserDetails; import datawave.microservice.http.converter.protostuff.ProtostuffHttpMessageConverter; import datawave.microservice.model.config.ModelProperties; +import datawave.webservice.model.FieldMapping; import datawave.webservice.model.Model; import datawave.webservice.model.ModelKeyParser; import datawave.webservice.model.ModelList; @@ -20,11 +22,18 @@ import datawave.webservice.query.exception.QueryException; import datawave.webservice.result.VoidResponse; import lombok.extern.slf4j.Slf4j; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.BatchWriterConfig; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; //import org.apache.deltaspike.core.api.config.ConfigProperty; +import org.apache.accumulo.core.iterators.user.RegExFilter; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.http.MediaType; import org.springframework.security.access.annotation.Secured; @@ -33,20 +42,22 @@ import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.Mapping; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import java.util.HashSet; import java.util.Map; +import java.util.concurrent.TimeUnit; /** * Service that supports manipulation of models. The models are contained in the data dictionary table. */ @Slf4j @RestController -@RequestMapping(path = "/model", produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.TEXT_XML_VALUE, - ProtostuffHttpMessageConverter.PROTOSTUFF_VALUE, MediaType.TEXT_HTML_VALUE, "text/x-yaml", "application/x-yaml"}) +@RequestMapping(path = "/model", + produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.TEXT_XML_VALUE, ProtostuffHttpMessageConverter.PROTOSTUFF_VALUE, MediaType.TEXT_HTML_VALUE, "text/x-yaml", "application/x-yaml"}) @Secured({"AuthorizedUser", "AuthorizedQueryServer", "InternalUser", "Administrator", "JBossAdministrator"}) @EnableConfigurationProperties(ModelProperties.class) public class ModelController { @@ -64,7 +75,6 @@ public class ModelController { private static final HashSet RESERVED_COLF_VALUES = Sets.newHashSet("e", "i", "ri", "f", "tf", "m", "desc", "edge", "t", "n", "h"); public ModelController(ModelProperties modelProperties, AccumuloConnectionService accumloConnectionService) { - // this.defaultModelTableName = modelProperties.getDefaultTableName(); this.dataTablesUri = modelProperties.getDefaultTableName(); this.jqueryUri = modelProperties.getJqueryUri(); this.accumloConnectionService = accumloConnectionService; @@ -81,7 +91,7 @@ public ModelController(ModelProperties modelProperties, AccumuloConnectionServic * @HTTP 200 success * @HTTP 500 internal server error */ - @GetMapping("/list") + @GetMapping("/list") // If we get to change to follow true REST standard, this would just be / and remain a GET public ModelList listModelNames(@RequestParam(defaultValue = DEFAULT_MODEL_TABLE_NAME) String modelTableName, @AuthenticationPrincipal ProxiedUserDetails currentUser) { @@ -105,45 +115,12 @@ public ModelList listModelNames(@RequestParam(defaultValue = DEFAULT_MODEL_TABLE QueryException qe = new QueryException(DatawaveErrorCode.MODEL_NAME_LIST_ERROR, e); log.error(qe.getMessage()); response.addException(qe.getBottomQueryException()); - // throw new DatawaveWebApplicationException(qe, response); } response.setNames(modelNames); return response; } - /** - * Administrator credentials required. Insert a new model - * - * @param model - * @param modelTableName - * name of the table that contains the model - * @return datawave.webservice.result.VoidResponse - * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user - * - * @HTTP 200 success - * @HTTP 412 if model already exists with this name, delete it first - * @HTTP 500 internal server error - */ - - @PostMapping("/import") - @Secured({"Administrator", "JBossAdministrator"}) - public VoidResponse importModel(@RequestParam Model model, @RequestParam(defaultValue = DEFAULT_MODEL_TABLE_NAME) String modelTableName, - @AuthenticationPrincipal ProxiedUserDetails currentUser) { - if (log.isDebugEnabled()) { - log.debug("modelTableName: " + (null == modelTableName ? "" : modelTableName)); - } - VoidResponse response = new VoidResponse(); - - ModelList models = listModelNames(modelTableName, currentUser); - // if (models.getNames().contains(model.getName())) - // throw new PreConditionFailedException(null, response); - - // insertMapping(model, modelTableName); - - return response; - } - /** * Administrator credentials required. Delete a model with the supplied name * @@ -162,18 +139,13 @@ public VoidResponse importModel(@RequestParam Model model, @RequestParam(default @Secured({"Administrator", "JBossAdministrator"}) public VoidResponse deleteModel(@RequestParam String name, @RequestParam(defaultValue = DEFAULT_MODEL_TABLE_NAME) String modelTableName, @AuthenticationPrincipal ProxiedUserDetails currentUser) { - if (log.isDebugEnabled()) { - log.debug("model name: " + name); - log.debug("modelTableName: " + (null == modelTableName ? "" : modelTableName)); - } VoidResponse response = new VoidResponse(); - ModelList models = listModelNames(modelTableName, currentUser); - // if (!models.getNames().contains(name)) - // throw new NotFoundException(null, response); - - Model model = getModel(name, modelTableName, currentUser); - // deleteMapping(model, modelTableName, reloadCache); + if (models.getNames().contains(name)) { + // the specified model exists, so we can proceed with deleting it + Model model = getModel(name, modelTableName, currentUser); + deleteMapping(model, modelTableName, currentUser); + } return response; } @@ -203,7 +175,7 @@ public VoidResponse cloneModel(@RequestParam String name, @RequestParam String n Model model = getModel(name, modelTableName, currentUser); // Set the new name model.setName(newName); - // importModel(model, modelTableName); + insertMapping(model, modelTableName, currentUser); return response; } @@ -225,59 +197,20 @@ public VoidResponse cloneModel(@RequestParam String name, @RequestParam String n public Model getModel(@RequestParam String name, @RequestParam(defaultValue = DEFAULT_MODEL_TABLE_NAME) String modelTableName, @AuthenticationPrincipal ProxiedUserDetails currentUser) { Model response = new Model(jqueryUri, dataTablesUri); - // - // // Find out who/what called this method - // Principal p = ctx.getCallerPrincipal(); - // String user = p.getName(); - // Set cbAuths = new HashSet<>(); - // if (p instanceof DatawavePrincipal) { - // DatawavePrincipal cp = (DatawavePrincipal) p; - // user = cp.getShortName(); - // for (Collection auths : cp.getAuthorizations()) { - // cbAuths.add(new Authorizations(auths.toArray(new String[auths.size()]))); - // } - // } - // log.trace(user + " has authorizations " + cbAuths); - // - // Connector connector = null; - // try { - // Map trackingMap = connectionFactory.getTrackingMap(Thread.currentThread().getStackTrace()); - // connector = connectionFactory.getConnection(getCurrentUserDN(), getCurrentProxyServers(), AccumuloConnectionFactory.Priority.LOW, trackingMap); - // try (Scanner scanner = ScannerHelper.createScanner(connector, modelTableName, cbAuths)) { - - try (Scanner scanner = accumloConnectionService.getScanner(modelTableName, currentUser)) { - // IteratorSetting cfg = new IteratorSetting(21, "colfRegex", RegExFilter.class.getName()); - // cfg.addOption(RegExFilter.COLF_REGEX, "^" + name + "(\\x00.*)?"); - // scanner.addScanIterator(cfg); - // for (Entry entry : scanner) { - // FieldMapping mapping = ModelKeyParser.parseKey(entry.getKey(), cbAuths); - // response.getFields().add(mapping); - // } - // } + try (Scanner scanner = accumloConnectionService.getScannerWithRegexIteratorSetting(name, modelTableName, currentUser)) { + for (Map.Entry entry : scanner) { +// FieldMapping mapping = ModelKeyParser.parseKey(entry.getKey(), cbAuths); +// response.getFields().add(mapping); + } } catch (TableNotFoundException e) { - // QueryException qe = new QueryException(DatawaveErrorCode.MODEL_FETCH_ERROR, e); - // log.error(qe); - // response.addException(qe.getBottomQueryException()); - // throw new DatawaveWebApplicationException(qe, response); - } finally { - // if (null != connector) { - // try { - // connectionFactory.returnConnection(connector); - // } catch (Exception e) { - // log.error("Error returning connection to factory", e); - } - // } - // } - // - // return 404 if model not found - if (response.getMappings().isEmpty()) { - // throw new NotFoundException(null, response); + QueryException qe = new QueryException(DatawaveErrorCode.MODEL_NAME_LIST_ERROR, e); + log.error(qe.getMessage()); + response.addException(qe.getBottomQueryException()); } - response.setName(name); return response; } - + /** * Administrator credentials required. Insert a new field mapping into an existing model * @@ -291,48 +224,58 @@ public Model getModel(@RequestParam String name, @RequestParam(defaultValue = DE * @HTTP 200 success * @HTTP 500 internal server error */ - @PostMapping("/insert") + @PostMapping(value = {"/insert", "/import"}) // If we get to change to standard REST, this would just be / and rely on the @Secured({"Administrator", "JBossAdministrator"}) - public VoidResponse insertMapping(@RequestParam Model model, @RequestParam(defaultValue = DEFAULT_MODEL_TABLE_NAME) String modelTableName) { - + public VoidResponse insertMapping(@RequestBody Model model, @RequestParam(defaultValue = DEFAULT_MODEL_TABLE_NAME) String modelTableName, @AuthenticationPrincipal ProxiedUserDetails currentUser) { + if (log.isDebugEnabled()) { + log.debug("modelTableName: " + (null == modelTableName ? "" : modelTableName)); + } + VoidResponse response = new VoidResponse(); - // - // Connector connector = null; - // BatchWriter writer = null; - // try { - // Map trackingMap = connectionFactory.getTrackingMap(Thread.currentThread().getStackTrace()); - // connector = connectionFactory.getConnection(getCurrentUserDN(), getCurrentProxyServers(), AccumuloConnectionFactory.Priority.LOW, trackingMap); - // writer = connector.createBatchWriter(modelTableName, new BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY, TimeUnit.MILLISECONDS) - // .setMaxMemory(BATCH_WRITER_MAX_MEMORY).setMaxWriteThreads(BATCH_WRITER_MAX_THREADS)); - // for (FieldMapping mapping : model.getFields()) { - // Mutation m = ModelKeyParser.createMutation(mapping, model.getName()); - // writer.addMutation(m); - // } - // } catch (Exception e) { - // log.error("Could not insert mapping.", e); - // QueryException qe = new QueryException(DatawaveErrorCode.INSERT_MAPPING_ERROR, e); - // response.addException(qe.getBottomQueryException()); - // throw new DatawaveWebApplicationException(qe, response); - // } finally { - // if (null != writer) { - // try { - // writer.close(); - // } catch (MutationsRejectedException e1) { - // QueryException qe = new QueryException(DatawaveErrorCode.WRITER_CLOSE_ERROR, e1); - // log.error(qe); - // response.addException(qe); - // throw new DatawaveWebApplicationException(qe, response); - // } - // } - // if (null != connector) { - // try { - // connectionFactory.returnConnection(connector); - // } catch (Exception e) { - // log.error("Error returning connection to factory", e); - // } - // } - // } - // cache.reloadTableCache(tableName); + ModelList models = listModelNames(modelTableName, currentUser); + if (models.getNames().contains(model.getName())) { + // the model already exists -- nothing to do + return response; + } + + BatchWriter writer = null; + + Connection connection = accumloConnectionService.getConnection(modelTableName, model.getName(), currentUser); + try { + // Is the BatchWriterConfig already a spring bean? + writer = connection.getConnector().createBatchWriter(modelTableName, new BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY, TimeUnit.MILLISECONDS) + .setMaxMemory(BATCH_WRITER_MAX_MEMORY).setMaxWriteThreads(BATCH_WRITER_MAX_THREADS)); + } catch (TableNotFoundException e) { + log.error("The " + modelTableName + " could not be found to write to ", e ); + QueryException qe = new QueryException(DatawaveErrorCode.TABLE_NOT_FOUND, e); + response.addException(qe.getBottomQueryException()); + } + + if (writer != null) { + for (FieldMapping mapping : model.getFields()) { + Mutation m = ModelKeyParser.createMutation(mapping, model.getName()); + try { + writer.addMutation(m); + } catch (MutationsRejectedException e) { + // So that we know exactly which mapping causes the error, catch here, but break out of loop + // Unfortunately, this does mean if there are multiple mappings that cause this error, only the first will be logged. + log.error("Could not insert mapping -- " + mapping, e); + QueryException qe = new QueryException(DatawaveErrorCode.INSERT_MAPPING_ERROR, e); + response.addException(qe.getBottomQueryException()); + } finally { + // we know the writer isn't null since we've already checked. + try { + writer.close(); + } catch (MutationsRejectedException e) { + log.error("Error closing the BatchWriter; ", e); + } + } + + } + } + + // Do we already have an AccumuloTableCache bean somewhere? +// cache.reloadTableCache(tableName); return response; } @@ -352,10 +295,9 @@ public VoidResponse insertMapping(@RequestParam Model model, @RequestParam(defau @DeleteMapping("/delete") @Secured({"Administrator", "JBossAdministrator"}) public VoidResponse deleteMapping(@RequestParam Model model, @RequestParam String modelTableName, @AuthenticationPrincipal ProxiedUserDetails currentUser) { - return deleteMapping(model, modelTableName, true, currentUser); - } - - private VoidResponse deleteMapping(Model model, String modelTableName, boolean reloadCache, @AuthenticationPrincipal ProxiedUserDetails currentUser) { + if (log.isDebugEnabled()) { + log.debug("Deleting model name: " + model.getName() + "from modelTableName " + modelTableName); + } VoidResponse response = new VoidResponse(); // // Connector connector = null; diff --git a/service/src/test/java/datawave/microservice/model/ModelControllerTest.java b/service/src/test/java/datawave/microservice/model/ModelControllerTest.java index 6501235..2910f9d 100644 --- a/service/src/test/java/datawave/microservice/model/ModelControllerTest.java +++ b/service/src/test/java/datawave/microservice/model/ModelControllerTest.java @@ -1,15 +1,30 @@ package datawave.microservice.model; import com.fasterxml.jackson.dataformat.xml.XmlMapper; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; import datawave.accumulo.inmemory.InMemoryInstance; +import datawave.marking.MarkingFunctions; import datawave.microservice.ControllerIT; import datawave.microservice.model.config.ModelProperties; +import datawave.webservice.dictionary.data.DefaultDescription; +import datawave.webservice.dictionary.data.DefaultFields; +import datawave.webservice.model.FieldMapping; import datawave.webservice.model.Model; +import datawave.webservice.model.ModelKeyParser; +import datawave.webservice.model.ModelList; +import datawave.webservice.result.VoidResponse; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.BatchWriterConfig; import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.client.TableExistsException; import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.core.data.Mutation; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -19,14 +34,39 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.core.io.Resource; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.util.UriComponents; +import org.springframework.web.util.UriComponentsBuilder; +import javax.xml.bind.JAXBContext; +import javax.xml.bind.Unmarshaller; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.net.URL; +import java.util.AbstractMap; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; + public class ModelControllerTest extends ControllerIT { - + + private static final long BATCH_WRITER_MAX_LATENCY = 1000L; + private static final long BATCH_WRITER_MAX_MEMORY = 10845760; + private static final int BATCH_WRITER_MAX_THREADS = 2; + @Autowired private ModelProperties modelProperties; @@ -56,333 +96,115 @@ public void setUp() throws Exception { } catch (TableExistsException e) { // ignore } - - // URL m1Url = ModelBeanTest.class.getResource("/TestModel_1.xml"); - // URL m2Url = ModelBeanTest.class.getResource("/TestModel_2.xml"); - // JAXBContext ctx = JAXBContext.newInstance(Model.class); - // Unmarshaller u = ctx.createUnmarshaller(); - // MODEL_ONE = (datawave.webservice.model.Model) u.unmarshal(testModel1.getInputStream()); - // MODEL_TWO = (datawave.webservice.model.Model) u.unmarshal(testModel2.getInputStream()); - - // XmlMapper mapper = new XmlMapper(); - // MODEL_ONE = mapper.readValue(inputStreamToString(model1.getInputStream()), Model.class); - // MODEL_TWO = mapper.readValue(inputStreamToString(model2.getInputStream()), Model.class); - // - System.out.println(MODEL_ONE); - }; - - private String inputStreamToString(InputStream fs) throws IOException { - StringBuilder sb = new StringBuilder(); - String line; - BufferedReader br = new BufferedReader(new InputStreamReader(fs)); - while ((line = br.readLine()) != null) { - sb.append(line); + + JAXBContext ctx = JAXBContext.newInstance(Model.class); + Unmarshaller u = ctx.createUnmarshaller(); + MODEL_ONE = (datawave.webservice.model.Model) u.unmarshal(model1.getInputStream()); + MODEL_TWO = (datawave.webservice.model.Model) u.unmarshal(model2.getInputStream()); + + BatchWriter writer = connector.createBatchWriter(modelProperties.getDefaultTableName(), new BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY, TimeUnit.MILLISECONDS) + .setMaxMemory(BATCH_WRITER_MAX_MEMORY).setMaxWriteThreads(BATCH_WRITER_MAX_THREADS)); + + // Seed MODEL_ONE, and leave MODEL_TWO for testing import and other "addition" capabilities + for (FieldMapping mapping : MODEL_ONE.getFields()) { + Mutation m = ModelKeyParser.createMutation(mapping, MODEL_ONE.getName()); + try { + writer.addMutation(m); + } catch (MutationsRejectedException e) { + // ignore + } } - br.close(); - return sb.toString(); } - + + @AfterEach + public void teardown() { + try { + connector.tableOperations().delete(modelProperties.getDefaultTableName()); + } catch (Exception e) {} + } + @Test public void testList() { + // There is 1 model that is guaranteed to be there because it was seeded by the @BeforeEach // @formatter:off -// UriComponents uri = UriComponentsBuilder.newInstance() -// .scheme("https").host("localhost").port(webServicePort) -// .path("/dictionary/model/list") -// .build(); - // @formatter:on + UriComponents uri = UriComponentsBuilder.newInstance() + .scheme("https").host("localhost").port(webServicePort) + .path("/dictionary/model/list") + .build(); + // @formatter:on - // ResponseEntity response = jwtRestTemplate.exchange(adminUser, HttpMethod.GET, uri, ModelList.class); - // assertEquals(HttpStatus.OK, response.getStatusCode()); + ResponseEntity response = jwtRestTemplate.exchange(adminUser, HttpMethod.GET, uri, ModelList.class); + assertEquals(HttpStatus.OK, response.getStatusCode()); + assertEquals(1, response.getBody().getNames().size()); + } + + @Test + public void testImport() { + // Verify that there is only one model so far (the model that is seeded before the test) + // @formatter:off + UriComponents uri = UriComponentsBuilder.newInstance() + .scheme("https").host("localhost").port(webServicePort) + .path("/dictionary/model/list") + .build(); + // @formatter:on + + ResponseEntity modelListResponse = jwtRestTemplate.exchange(adminUser, HttpMethod.GET, uri, ModelList.class); + assertEquals(HttpStatus.OK, modelListResponse.getStatusCode()); + assertEquals(1, modelListResponse.getBody().getNames().size()); + + // Now import a model + // @formatter:off + uri = UriComponentsBuilder.newInstance() + .scheme("https").host("localhost").port(webServicePort) + .path("/dictionary/model/import") + .build(); + // @formatter:on + + HttpHeaders additionalHeaders = new HttpHeaders(); + additionalHeaders.set(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE); + RequestEntity postEntity = jwtRestTemplate.createRequestEntity(adminUser, MODEL_TWO, additionalHeaders, HttpMethod.POST, uri); + ResponseEntity imprtResponse = jwtRestTemplate.exchange(postEntity, VoidResponse.class); + + assertEquals(HttpStatus.OK, imprtResponse.getStatusCode()); + + // Now call list again, and there should be 2 models + // @formatter:off + uri = UriComponentsBuilder.newInstance() + .scheme("https").host("localhost").port(webServicePort) + .path("/dictionary/model/list") + .build(); + // @formatter:on + + ResponseEntity modelListResponse2 = jwtRestTemplate.exchange(adminUser, HttpMethod.GET, uri, ModelList.class); + assertEquals(HttpStatus.OK, modelListResponse2.getStatusCode()); + assertEquals(2, modelListResponse2.getBody().getNames().size()); + } + +// @Test + public void testGet() { + + } + +// @Test + public void testDelete() { + // Verify that there is only one model so far (the model that is seeded before the test) + // @formatter:off + UriComponents uri = UriComponentsBuilder.newInstance() + .scheme("https").host("localhost").port(webServicePort) + .path("/dictionary/model/list") + .build(); + // @formatter:on + + ResponseEntity modelListResponse = jwtRestTemplate.exchange(adminUser, HttpMethod.GET, uri, ModelList.class); + assertEquals(HttpStatus.OK, modelListResponse.getStatusCode()); + assertEquals(1, modelListResponse.getBody().getNames().size()); + + // Call delete + // @formatter:off + uri = UriComponentsBuilder.newInstance() + .scheme("https").host("localhost").port(webServicePort) + .path("/dictionary/model/delete") + .build(); + // @formatter:on } - - // private static final String userDN = "CN=Guy Some Other soguy, OU=ou1, OU=ou2, OU=ou3, O=o1, C=US"; - // private static final String issuerDN = "CN=CA1, OU=ou3, O=o1, C=US"; - // private static final String[] auths = new String[] {"PRIVATE", "PUBLIC"}; - // - // private ModelController modelController; - // private AccumuloConnectionService connection; - //// private AccumuloTableCache cache; - // - // private InMemoryInstance instance = null; - // private Connector connector = null; - //// private DatawavePrincipal principal = null; - // - // private static long TIMESTAMP = System.currentTimeMillis(); - // - //// private datawave.webservice.model.Model MODEL_ONE = null; - //// private datawave.webservice.model.Model MODEL_TWO = null; - // - // @BeforeEach - // public void setup() throws Exception { - //// System.setProperty(NpeUtils.NPE_OU_PROPERTY, "iamnotaperson"); - // System.setProperty("dw.metadatahelper.all.auths", "A,B,C,D"); - // bean = new ModelBean(); - // connectionFactory = createStrictMock(AccumuloConnectionFactory.class); - // ctx = createMock(EJBContext.class); - // cache = createMock(AccumuloTableCache.class); - // Whitebox.setInternalState(bean, EJBContext.class, ctx); - // Whitebox.setInternalState(bean, AccumuloConnectionFactory.class, connectionFactory); - // Whitebox.setInternalState(bean, AccumuloTableCache.class, cache); - // - // instance = new InMemoryInstance("test"); - // connector = instance.getConnector("root", new PasswordToken("")); - // - // DatawaveUser user = new DatawaveUser(SubjectIssuerDNPair.of(userDN, issuerDN), UserType.USER, Arrays.asList(auths), null, null, 0L); - // principal = new DatawavePrincipal(Collections.singletonList(user)); - // - // URL m1Url = ModelBeanTest.class.getResource("/ModelBeanTest_m1.xml"); - // URL m2Url = ModelBeanTest.class.getResource("/ModelBeanTest_m2.xml"); - // JAXBContext ctx = JAXBContext.newInstance(datawave.webservice.model.Model.class); - // Unmarshaller u = ctx.createUnmarshaller(); - // MODEL_ONE = (datawave.webservice.model.Model) u.unmarshal(m1Url); - // MODEL_TWO = (datawave.webservice.model.Model) u.unmarshal(m2Url); - // - // Logger.getLogger(ModelBean.class).setLevel(Level.OFF); - // PowerMock.mockStatic(System.class, System.class.getMethod("currentTimeMillis")); - // } - // - // public void printTable(String tableName) throws Exception { - // Scanner s = connector.createScanner(tableName, new Authorizations(auths)); - // for (Entry entry : s) { - // System.out.println(entry.getKey()); - // } - // } - // - // @After - // public void tearDown() { - // try { - // connector.tableOperations().delete(ModelBean.DEFAULT_MODEL_TABLE_NAME); - // } catch (Exception e) {} - // } - // - //// @Test(expected = DatawaveWebApplicationException.class) - //// public void testModelImportNoTable() throws Exception { - //// HashMap trackingMap = new HashMap<>(); - //// EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); - //// EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); - //// connectionFactory.returnConnection(connector); - //// EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal); - //// PowerMock.replayAll(); - //// - //// bean.importModel(MODEL_ONE, (String) null); - //// PowerMock.verifyAll(); - //// } - //// - // private void importModels() throws Exception { - // connector.tableOperations().create(ModelBean.DEFAULT_MODEL_TABLE_NAME); - // - // HashMap trackingMap = new HashMap<>(); - // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); - // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); - // connectionFactory.returnConnection(connector); - // EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal); - // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); - // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); - // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); - // connectionFactory.returnConnection(connector); - // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); - // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); - // EasyMock.expect(cache.reloadCache(ModelBean.DEFAULT_MODEL_TABLE_NAME)).andReturn(null); - // PowerMock.replayAll(); - // - // bean.importModel(MODEL_ONE, (String) null); - // PowerMock.verifyAll(); - // PowerMock.resetAll(); - // - // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); - // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); - // connectionFactory.returnConnection(connector); - // EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal); - // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); - // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); - // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); - // connectionFactory.returnConnection(connector); - // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); - // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); - // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); - // EasyMock.expect(cache.reloadCache(ModelBean.DEFAULT_MODEL_TABLE_NAME)).andReturn(null); - // PowerMock.replayAll(); - // - // bean.importModel(MODEL_TWO, (String) null); - // - // PowerMock.verifyAll(); - // } - // - // @Test - // public void testListModels() throws Exception { - // importModels(); - // PowerMock.resetAll(); - // - // EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal); - // HashMap trackingMap = new HashMap<>(); - // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); - // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); - // connectionFactory.returnConnection(connector); - // PowerMock.replayAll(); - // - // ModelList list = bean.listModelNames((String) null); - // PowerMock.verifyAll(); - // - // Assert.assertEquals(2, list.getNames().size()); - // Assert.assertTrue(list.getNames().contains(MODEL_ONE.getName())); - // Assert.assertTrue(list.getNames().contains(MODEL_TWO.getName())); - // } - // - // @Test - // public void testModelGet() throws Exception { - // importModels(); - // PowerMock.resetAll(); - // - // EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal); - // HashMap trackingMap = new HashMap<>(); - // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); - // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); - // connectionFactory.returnConnection(connector); - // PowerMock.replayAll(); - // - // datawave.webservice.model.Model model = bean.getModel(MODEL_ONE.getName(), (String) null); - // PowerMock.verifyAll(); - // - // Assert.assertEquals(MODEL_ONE, model); - // } - // - // @Test - // public void testModelDelete() throws Exception { - // importModels(); - // PowerMock.resetAll(); - // - // EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal); - // HashMap trackingMap = new HashMap<>(); - // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); - // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); - // connectionFactory.returnConnection(connector); - // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); - // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); - // connectionFactory.returnConnection(connector); - // EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal); - // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); - // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); - // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); - // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); - // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); - // connectionFactory.returnConnection(connector); - // EasyMock.expect(cache.reloadCache(ModelBean.DEFAULT_MODEL_TABLE_NAME)).andReturn(null); - // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); - // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); - // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); - // PowerMock.replayAll(); - // - // bean.deleteModel(MODEL_TWO.getName(), (String) null); - // PowerMock.verifyAll(); - // PowerMock.resetAll(); - // - // EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal); - // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); - // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); - // connectionFactory.returnConnection(connector); - // PowerMock.replayAll(); - // try { - // bean.getModel(MODEL_TWO.getName(), (String) null); - // Assert.fail("getModel should have failed"); - // } catch (DatawaveWebApplicationException e) { - // if (e.getResponse().getStatus() == 404) { - // // success - // } else { - // Assert.fail("getModel did not return a 404, returned: " + e.getResponse().getStatus()); - // } - // } catch (Exception ex) { - // Assert.fail("getModel did not throw a DatawaveWebApplicationException"); - // } - // PowerMock.verifyAll(); - // PowerMock.resetAll(); - // // Ensure model one still intact - // EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal); - // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); - // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); - // connectionFactory.returnConnection(connector); - // PowerMock.replayAll(); - // datawave.webservice.model.Model model1 = bean.getModel(MODEL_ONE.getName(), (String) null); - // PowerMock.verifyAll(); - // Assert.assertEquals(MODEL_ONE, model1); - // - // } - // - // @Test(expected = DatawaveWebApplicationException.class) - // public void testModelGetInvalidModelName() throws Exception { - // EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal); - // HashMap trackingMap = new HashMap<>(); - // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); - // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); - // connectionFactory.returnConnection(connector); - // PowerMock.replayAll(); - // - // bean.getModel(MODEL_ONE.getName(), (String) null); - // PowerMock.verifyAll(); - // } - // - // @Test - // public void testCloneModel() throws Exception { - // importModels(); - // PowerMock.resetAll(); - // - // EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal); - // HashMap trackingMap = new HashMap<>(); - // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); - // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); - // connectionFactory.returnConnection(connector); - // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); - // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); - // connectionFactory.returnConnection(connector); - // EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal); - // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); - // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); - // EasyMock.expect(cache.reloadCache(ModelBean.DEFAULT_MODEL_TABLE_NAME)).andReturn(null); - // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); - // connectionFactory.returnConnection(connector); - // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); - // EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); - // PowerMock.replayAll(); - // - // bean.cloneModel(MODEL_ONE.getName(), "MODEL2", (String) null); - // PowerMock.verifyAll(); - // PowerMock.resetAll(); - // EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal); - // EasyMock.expect(connectionFactory.getTrackingMap((StackTraceElement[]) EasyMock.anyObject())).andReturn(trackingMap); - // EasyMock.expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(connector); - // connectionFactory.returnConnection(connector); - // PowerMock.replayAll(); - // - // datawave.webservice.model.Model model = bean.getModel("MODEL2", (String) null); - // PowerMock.verifyAll(); - // - // MODEL_ONE.setName("MODEL2"); - // Assert.assertEquals(MODEL_ONE, model); - // - // } - // - // @Test - // public void testCheckModelName() throws Exception { - // String modelTableName = Whitebox.invokeMethod(bean, "checkModelTableName", (String) null); - // Assert.assertEquals(ModelBean.DEFAULT_MODEL_TABLE_NAME, modelTableName); - // modelTableName = "foo"; - // String response = Whitebox.invokeMethod(bean, "checkModelTableName", modelTableName); - // Assert.assertEquals(modelTableName, response); - // - // } - // - // private void dumpModels() throws Exception { - // System.out.println("******************* Start Dump Models **********************"); - // Set cbAuths = new HashSet<>(); - // for (Collection auths : principal.getAuthorizations()) { - // cbAuths.add(new Authorizations(auths.toArray(new String[auths.size()]))); - // } - // - // Scanner scanner = ScannerHelper.createScanner(connector, ModelBean.DEFAULT_MODEL_TABLE_NAME, cbAuths); - // for (Entry entry : scanner) { - // System.out.println(entry.getKey()); - // } - // - // System.out.println("******************* End Dump Models **********************"); - // } - } From f51a7643e52330dde161666994f05193022b2f56 Mon Sep 17 00:00:00 2001 From: cmitchell Date: Thu, 5 May 2022 10:58:53 -0400 Subject: [PATCH 14/54] Removing commented out code and finishing test. --- .../webservice/model/FieldMapping.java | 127 ++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 api/src/main/java/datawave/webservice/model/FieldMapping.java diff --git a/api/src/main/java/datawave/webservice/model/FieldMapping.java b/api/src/main/java/datawave/webservice/model/FieldMapping.java new file mode 100644 index 0000000..59a3427 --- /dev/null +++ b/api/src/main/java/datawave/webservice/model/FieldMapping.java @@ -0,0 +1,127 @@ +package datawave.webservice.model; + +import java.io.Serializable; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; + +import org.apache.commons.lang.builder.CompareToBuilder; +import org.apache.commons.lang.builder.EqualsBuilder; +import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.commons.lang.builder.ToStringBuilder; + +@XmlAccessorType(XmlAccessType.NONE) +public class FieldMapping implements Serializable, Comparable { + + private static final long serialVersionUID = 1L; + + @XmlAttribute(name = "datatype", required = true) + private String datatype = null; + + @XmlAttribute(name = "fieldName", required = true) + private String fieldName = null; + + @XmlAttribute(name = "modelFieldName", required = true) + private String modelFieldName = null; + + @XmlAttribute(name = "direction", required = true) + private Direction direction = null; + + @XmlAttribute(name = "columnVisibility", required = true) + private String columnVisibility = null; + + public FieldMapping() { + super(); + } + + public FieldMapping(String datatype, String fieldName, String modelFieldName, Direction direction, String columnVisibility) { + super(); + this.datatype = datatype; + this.fieldName = fieldName; + this.modelFieldName = modelFieldName; + this.direction = direction; + this.columnVisibility = columnVisibility; + + } + + public String getDatatype() { + return datatype; + } + + public void setDatatype(String datatype) { + this.datatype = datatype; + } + + public String getFieldName() { + return fieldName; + } + + public void setFieldName(String fieldName) { + this.fieldName = fieldName; + } + + public String getModelFieldName() { + return modelFieldName; + } + + public void setModelFieldName(String modelFieldName) { + this.modelFieldName = modelFieldName; + } + + public Direction getDirection() { + return direction; + } + + public void setDirection(Direction direction) { + this.direction = direction; + } + + public String getColumnVisibility() { + return columnVisibility; + } + + public void setColumnVisibility(String columnVisibility) { + this.columnVisibility = columnVisibility; + } + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 37).append(columnVisibility).append(datatype).append(direction).append(fieldName).append(modelFieldName).toHashCode(); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) + return false; + if (obj == this) + return true; + if (obj.getClass() != this.getClass()) + return false; + FieldMapping other = (FieldMapping) obj; + return new EqualsBuilder().append(columnVisibility, other.columnVisibility).append(datatype, other.datatype).append(direction, other.direction) + .append(fieldName, other.fieldName).append(modelFieldName, other.modelFieldName).isEquals(); + } + + @Override + public String toString() { + return new ToStringBuilder(this).append("columnVisibility", columnVisibility).append("datatype", datatype).append("direction", direction) + .append("fieldName", fieldName).append("modelFieldName", modelFieldName).toString(); + } + + @Override + public int compareTo(FieldMapping obj) { + + if (obj == null) { + throw new IllegalArgumentException("can not compare null"); + } + + if (obj == this) + return 0; + + return new CompareToBuilder().append(datatype, ((FieldMapping) obj).datatype).append(fieldName, ((FieldMapping) obj).fieldName) + .append(modelFieldName, ((FieldMapping) obj).modelFieldName).append(direction, ((FieldMapping) obj).direction) + .append(columnVisibility, ((FieldMapping) obj).columnVisibility).toComparison(); + } + +} From f5389715cfd21b72e46cb024becca1638472548c Mon Sep 17 00:00:00 2001 From: cmitchell Date: Thu, 5 May 2022 11:12:08 -0400 Subject: [PATCH 15/54] Removing commented out code and finishing test. --- .../java/datawave/webservice/model/Model.java | 3 +- .../datawave/webservice/model/ModelList.java | 3 +- .../webservice/model/package-info.java | 3 +- service/pom.xml | 18 ++-- .../AccumuloConnectionService.java | 4 +- .../microservice/model/ModelController.java | 86 ++++++++++--------- .../model/ModelControllerTest.java | 68 +++++++-------- 7 files changed, 93 insertions(+), 92 deletions(-) diff --git a/api/src/main/java/datawave/webservice/model/Model.java b/api/src/main/java/datawave/webservice/model/Model.java index 30f7df1..a7dfd3e 100644 --- a/api/src/main/java/datawave/webservice/model/Model.java +++ b/api/src/main/java/datawave/webservice/model/Model.java @@ -27,8 +27,7 @@ public class Model extends BaseResponse implements Serializable, HtmlProvider { private String dataTablesUri; private static final String TITLE = "Model Description", EMPTY = ""; private static final String DATA_TABLES_TEMPLATE = "\n" - + "\n" - + "\n" + "\n"; diff --git a/api/src/main/java/datawave/webservice/model/ModelList.java b/api/src/main/java/datawave/webservice/model/ModelList.java index fe1d93f..d9cef45 100644 --- a/api/src/main/java/datawave/webservice/model/ModelList.java +++ b/api/src/main/java/datawave/webservice/model/ModelList.java @@ -21,8 +21,7 @@ public class ModelList extends BaseResponse implements Serializable, HtmlProvide private static final long serialVersionUID = 1L; private static final String TITLE = "Model Names"; private static final String DATA_TABLES_TEMPLATE = "\n" - + "\n" - + "\n" + "\n"; diff --git a/api/src/main/java/datawave/webservice/model/package-info.java b/api/src/main/java/datawave/webservice/model/package-info.java index 7ea8543..0b4e0d0 100644 --- a/api/src/main/java/datawave/webservice/model/package-info.java +++ b/api/src/main/java/datawave/webservice/model/package-info.java @@ -1,4 +1,5 @@ -@XmlSchema(namespace="http://webservice.datawave/v1", elementFormDefault=XmlNsForm.QUALIFIED, xmlns={@XmlNs(prefix = "http://webservice.datawave/v1", namespaceURI = "${datawave.webservice.namespace}")}) +@XmlSchema(namespace = "http://webservice.datawave/v1", elementFormDefault = XmlNsForm.QUALIFIED, + xmlns = {@XmlNs(prefix = "http://webservice.datawave/v1", namespaceURI = "${datawave.webservice.namespace}")}) package datawave.webservice.model; import javax.xml.bind.annotation.XmlNs; diff --git a/service/pom.xml b/service/pom.xml index bd17929..ffb4d45 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -35,6 +35,11 @@ jackson-dataformat-xml ${version.jackson} + + com.sun.xml.bind + jaxb-impl + 2.3.3 + gov.nsa.datawave.microservice dictionary-api @@ -110,22 +115,17 @@ pom import - - com.sun.xml.bind - jaxb-impl - 2.3.3 - - - com.sun.xml.bind - jaxb-impl - com.fasterxml.jackson.dataformat jackson-dataformat-xml + + com.sun.xml.bind + jaxb-impl + gov.nsa.datawave.microservice dictionary-api diff --git a/service/src/main/java/datawave/microservice/AccumuloConnectionService.java b/service/src/main/java/datawave/microservice/AccumuloConnectionService.java index 262c985..6109fa5 100644 --- a/service/src/main/java/datawave/microservice/AccumuloConnectionService.java +++ b/service/src/main/java/datawave/microservice/AccumuloConnectionService.java @@ -75,11 +75,11 @@ public Set getDowngradedAuthorizations(String requestedAuthoriza public Scanner getScanner(String modelTable, ProxiedUserDetails currentUser) throws TableNotFoundException { return ScannerHelper.createScanner(this.accumuloConnector, modelTable, this.getAuths(currentUser)); } - + public Scanner getScannerWithRegexIteratorSetting(String name, String modelTable, ProxiedUserDetails currentUser) throws TableNotFoundException { IteratorSetting cfg = new IteratorSetting(21, "colfRegex", RegExFilter.class.getName()); cfg.addOption(RegExFilter.COLF_REGEX, "^" + name + "(\\x00.*)?"); - + Scanner scanner = getScanner(modelTable, currentUser); scanner.addScanIterator(cfg); return scanner; diff --git a/service/src/main/java/datawave/microservice/model/ModelController.java b/service/src/main/java/datawave/microservice/model/ModelController.java index c70cf2b..0a194d6 100644 --- a/service/src/main/java/datawave/microservice/model/ModelController.java +++ b/service/src/main/java/datawave/microservice/model/ModelController.java @@ -56,8 +56,8 @@ */ @Slf4j @RestController -@RequestMapping(path = "/model", - produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.TEXT_XML_VALUE, ProtostuffHttpMessageConverter.PROTOSTUFF_VALUE, MediaType.TEXT_HTML_VALUE, "text/x-yaml", "application/x-yaml"}) +@RequestMapping(path = "/model", produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.TEXT_XML_VALUE, + ProtostuffHttpMessageConverter.PROTOSTUFF_VALUE, MediaType.TEXT_HTML_VALUE, "text/x-yaml", "application/x-yaml"}) @Secured({"AuthorizedUser", "AuthorizedQueryServer", "InternalUser", "Administrator", "JBossAdministrator"}) @EnableConfigurationProperties(ModelProperties.class) public class ModelController { @@ -91,7 +91,7 @@ public ModelController(ModelProperties modelProperties, AccumuloConnectionServic * @HTTP 200 success * @HTTP 500 internal server error */ - @GetMapping("/list") // If we get to change to follow true REST standard, this would just be / and remain a GET + @GetMapping("/list") // If we get to change to follow true REST standard, this would just be / and remain a GET public ModelList listModelNames(@RequestParam(defaultValue = DEFAULT_MODEL_TABLE_NAME) String modelTableName, @AuthenticationPrincipal ProxiedUserDetails currentUser) { @@ -141,11 +141,11 @@ public VoidResponse deleteModel(@RequestParam String name, @RequestParam(default @AuthenticationPrincipal ProxiedUserDetails currentUser) { VoidResponse response = new VoidResponse(); ModelList models = listModelNames(modelTableName, currentUser); - if (models.getNames().contains(name)) { - // the specified model exists, so we can proceed with deleting it - Model model = getModel(name, modelTableName, currentUser); - deleteMapping(model, modelTableName, currentUser); - } + if (models.getNames().contains(name)) { + // the specified model exists, so we can proceed with deleting it + Model model = getModel(name, modelTableName, currentUser); + deleteMapping(model, modelTableName, currentUser); + } return response; } @@ -198,10 +198,10 @@ public Model getModel(@RequestParam String name, @RequestParam(defaultValue = DE @AuthenticationPrincipal ProxiedUserDetails currentUser) { Model response = new Model(jqueryUri, dataTablesUri); try (Scanner scanner = accumloConnectionService.getScannerWithRegexIteratorSetting(name, modelTableName, currentUser)) { - for (Map.Entry entry : scanner) { -// FieldMapping mapping = ModelKeyParser.parseKey(entry.getKey(), cbAuths); -// response.getFields().add(mapping); - } + for (Map.Entry entry : scanner) { + // FieldMapping mapping = ModelKeyParser.parseKey(entry.getKey(), cbAuths); + // response.getFields().add(mapping); + } } catch (TableNotFoundException e) { QueryException qe = new QueryException(DatawaveErrorCode.MODEL_NAME_LIST_ERROR, e); log.error(qe.getMessage()); @@ -210,7 +210,7 @@ public Model getModel(@RequestParam String name, @RequestParam(defaultValue = DE response.setName(name); return response; } - + /** * Administrator credentials required. Insert a new field mapping into an existing model * @@ -224,58 +224,60 @@ public Model getModel(@RequestParam String name, @RequestParam(defaultValue = DE * @HTTP 200 success * @HTTP 500 internal server error */ - @PostMapping(value = {"/insert", "/import"}) // If we get to change to standard REST, this would just be / and rely on the + @PostMapping(value = {"/insert", "/import"}) // If we get to change to standard REST, this would just be / and rely on the @Secured({"Administrator", "JBossAdministrator"}) - public VoidResponse insertMapping(@RequestBody Model model, @RequestParam(defaultValue = DEFAULT_MODEL_TABLE_NAME) String modelTableName, @AuthenticationPrincipal ProxiedUserDetails currentUser) { + public VoidResponse insertMapping(@RequestBody Model model, @RequestParam(defaultValue = DEFAULT_MODEL_TABLE_NAME) String modelTableName, + @AuthenticationPrincipal ProxiedUserDetails currentUser) { if (log.isDebugEnabled()) { log.debug("modelTableName: " + (null == modelTableName ? "" : modelTableName)); } - + VoidResponse response = new VoidResponse(); ModelList models = listModelNames(modelTableName, currentUser); if (models.getNames().contains(model.getName())) { - // the model already exists -- nothing to do + // the model already exists -- nothing to do return response; } - - BatchWriter writer = null; - + + BatchWriter writer = null; + Connection connection = accumloConnectionService.getConnection(modelTableName, model.getName(), currentUser); try { // Is the BatchWriterConfig already a spring bean? - writer = connection.getConnector().createBatchWriter(modelTableName, new BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY, TimeUnit.MILLISECONDS) - .setMaxMemory(BATCH_WRITER_MAX_MEMORY).setMaxWriteThreads(BATCH_WRITER_MAX_THREADS)); + writer = connection.getConnector().createBatchWriter(modelTableName, + new BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY, TimeUnit.MILLISECONDS).setMaxMemory(BATCH_WRITER_MAX_MEMORY) + .setMaxWriteThreads(BATCH_WRITER_MAX_THREADS)); } catch (TableNotFoundException e) { - log.error("The " + modelTableName + " could not be found to write to ", e ); + log.error("The " + modelTableName + " could not be found to write to ", e); QueryException qe = new QueryException(DatawaveErrorCode.TABLE_NOT_FOUND, e); response.addException(qe.getBottomQueryException()); } - + if (writer != null) { - for (FieldMapping mapping : model.getFields()) { - Mutation m = ModelKeyParser.createMutation(mapping, model.getName()); + for (FieldMapping mapping : model.getFields()) { + Mutation m = ModelKeyParser.createMutation(mapping, model.getName()); + try { + writer.addMutation(m); + } catch (MutationsRejectedException e) { + // So that we know exactly which mapping causes the error, catch here, but break out of loop + // Unfortunately, this does mean if there are multiple mappings that cause this error, only the first will be logged. + log.error("Could not insert mapping -- " + mapping, e); + QueryException qe = new QueryException(DatawaveErrorCode.INSERT_MAPPING_ERROR, e); + response.addException(qe.getBottomQueryException()); + } finally { + // we know the writer isn't null since we've already checked. try { - writer.addMutation(m); + writer.close(); } catch (MutationsRejectedException e) { - // So that we know exactly which mapping causes the error, catch here, but break out of loop - // Unfortunately, this does mean if there are multiple mappings that cause this error, only the first will be logged. - log.error("Could not insert mapping -- " + mapping, e); - QueryException qe = new QueryException(DatawaveErrorCode.INSERT_MAPPING_ERROR, e); - response.addException(qe.getBottomQueryException()); - } finally { - // we know the writer isn't null since we've already checked. - try { - writer.close(); - } catch (MutationsRejectedException e) { - log.error("Error closing the BatchWriter; ", e); - } + log.error("Error closing the BatchWriter; ", e); } - } + + } } - + // Do we already have an AccumuloTableCache bean somewhere? -// cache.reloadTableCache(tableName); + // cache.reloadTableCache(tableName); return response; } diff --git a/service/src/test/java/datawave/microservice/model/ModelControllerTest.java b/service/src/test/java/datawave/microservice/model/ModelControllerTest.java index 2910f9d..c28f4d0 100644 --- a/service/src/test/java/datawave/microservice/model/ModelControllerTest.java +++ b/service/src/test/java/datawave/microservice/model/ModelControllerTest.java @@ -60,13 +60,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals; - public class ModelControllerTest extends ControllerIT { - + private static final long BATCH_WRITER_MAX_LATENCY = 1000L; private static final long BATCH_WRITER_MAX_MEMORY = 10845760; private static final int BATCH_WRITER_MAX_THREADS = 2; - + @Autowired private ModelProperties modelProperties; @@ -96,15 +95,16 @@ public void setUp() throws Exception { } catch (TableExistsException e) { // ignore } - - JAXBContext ctx = JAXBContext.newInstance(Model.class); - Unmarshaller u = ctx.createUnmarshaller(); - MODEL_ONE = (datawave.webservice.model.Model) u.unmarshal(model1.getInputStream()); - MODEL_TWO = (datawave.webservice.model.Model) u.unmarshal(model2.getInputStream()); - - BatchWriter writer = connector.createBatchWriter(modelProperties.getDefaultTableName(), new BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY, TimeUnit.MILLISECONDS) - .setMaxMemory(BATCH_WRITER_MAX_MEMORY).setMaxWriteThreads(BATCH_WRITER_MAX_THREADS)); - + + JAXBContext ctx = JAXBContext.newInstance(Model.class); + Unmarshaller u = ctx.createUnmarshaller(); + MODEL_ONE = (datawave.webservice.model.Model) u.unmarshal(model1.getInputStream()); + MODEL_TWO = (datawave.webservice.model.Model) u.unmarshal(model2.getInputStream()); + + BatchWriter writer = connector.createBatchWriter(modelProperties.getDefaultTableName(), + new BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY, TimeUnit.MILLISECONDS).setMaxMemory(BATCH_WRITER_MAX_MEMORY) + .setMaxWriteThreads(BATCH_WRITER_MAX_THREADS)); + // Seed MODEL_ONE, and leave MODEL_TWO for testing import and other "addition" capabilities for (FieldMapping mapping : MODEL_ONE.getFields()) { Mutation m = ModelKeyParser.createMutation(mapping, MODEL_ONE.getName()); @@ -115,14 +115,14 @@ public void setUp() throws Exception { } } } - + @AfterEach public void teardown() { - try { - connector.tableOperations().delete(modelProperties.getDefaultTableName()); - } catch (Exception e) {} + try { + connector.tableOperations().delete(modelProperties.getDefaultTableName()); + } catch (Exception e) {} } - + @Test public void testList() { // There is 1 model that is guaranteed to be there because it was seeded by the @BeforeEach @@ -133,11 +133,11 @@ public void testList() { .build(); // @formatter:on - ResponseEntity response = jwtRestTemplate.exchange(adminUser, HttpMethod.GET, uri, ModelList.class); - assertEquals(HttpStatus.OK, response.getStatusCode()); - assertEquals(1, response.getBody().getNames().size()); + ResponseEntity response = jwtRestTemplate.exchange(adminUser, HttpMethod.GET, uri, ModelList.class); + assertEquals(HttpStatus.OK, response.getStatusCode()); + assertEquals(1, response.getBody().getNames().size()); } - + @Test public void testImport() { // Verify that there is only one model so far (the model that is seeded before the test) @@ -147,11 +147,11 @@ public void testImport() { .path("/dictionary/model/list") .build(); // @formatter:on - + ResponseEntity modelListResponse = jwtRestTemplate.exchange(adminUser, HttpMethod.GET, uri, ModelList.class); assertEquals(HttpStatus.OK, modelListResponse.getStatusCode()); assertEquals(1, modelListResponse.getBody().getNames().size()); - + // Now import a model // @formatter:off uri = UriComponentsBuilder.newInstance() @@ -159,14 +159,14 @@ public void testImport() { .path("/dictionary/model/import") .build(); // @formatter:on - + HttpHeaders additionalHeaders = new HttpHeaders(); additionalHeaders.set(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE); RequestEntity postEntity = jwtRestTemplate.createRequestEntity(adminUser, MODEL_TWO, additionalHeaders, HttpMethod.POST, uri); ResponseEntity imprtResponse = jwtRestTemplate.exchange(postEntity, VoidResponse.class); - + assertEquals(HttpStatus.OK, imprtResponse.getStatusCode()); - + // Now call list again, and there should be 2 models // @formatter:off uri = UriComponentsBuilder.newInstance() @@ -174,18 +174,18 @@ public void testImport() { .path("/dictionary/model/list") .build(); // @formatter:on - + ResponseEntity modelListResponse2 = jwtRestTemplate.exchange(adminUser, HttpMethod.GET, uri, ModelList.class); assertEquals(HttpStatus.OK, modelListResponse2.getStatusCode()); assertEquals(2, modelListResponse2.getBody().getNames().size()); } - -// @Test + + // @Test public void testGet() { - + } - -// @Test + + // @Test public void testDelete() { // Verify that there is only one model so far (the model that is seeded before the test) // @formatter:off @@ -194,11 +194,11 @@ public void testDelete() { .path("/dictionary/model/list") .build(); // @formatter:on - + ResponseEntity modelListResponse = jwtRestTemplate.exchange(adminUser, HttpMethod.GET, uri, ModelList.class); assertEquals(HttpStatus.OK, modelListResponse.getStatusCode()); assertEquals(1, modelListResponse.getBody().getNames().size()); - + // Call delete // @formatter:off uri = UriComponentsBuilder.newInstance() From 9f7113e1e3766ee7a564f5c110c4645e858f6f57 Mon Sep 17 00:00:00 2001 From: cmitchell Date: Thu, 5 May 2022 12:15:04 -0400 Subject: [PATCH 16/54] Almost done with testing and removing commented out code. --- .../AccumuloConnectionService.java | 15 +++ .../microservice/model/ModelController.java | 94 ++++++++----------- .../model/ModelControllerTest.java | 28 +++++- 3 files changed, 80 insertions(+), 57 deletions(-) diff --git a/service/src/main/java/datawave/microservice/AccumuloConnectionService.java b/service/src/main/java/datawave/microservice/AccumuloConnectionService.java index 6109fa5..c415c71 100644 --- a/service/src/main/java/datawave/microservice/AccumuloConnectionService.java +++ b/service/src/main/java/datawave/microservice/AccumuloConnectionService.java @@ -5,6 +5,8 @@ import datawave.microservice.dictionary.config.DataDictionaryProperties; import datawave.security.authorization.DatawaveUser; import datawave.security.util.ScannerHelper; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.BatchWriterConfig; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.Scanner; @@ -16,6 +18,7 @@ import org.springframework.stereotype.Service; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -31,6 +34,11 @@ public class AccumuloConnectionService { private final Connector accumuloConnector; private final DataDictionaryProperties dataDictionaryConfiguration; private final UserAuthFunctions userAuthFunctions; + + // These were already hardcodeded values in ModelBean -> ModelController so therefore they weren't made configurable + private static final long BATCH_WRITER_MAX_LATENCY = 1000L; + private static final long BATCH_WRITER_MAX_MEMORY = 10845760; + private static final int BATCH_WRITER_MAX_THREADS = 2; public AccumuloConnectionService(DataDictionaryProperties dataDictionaryConfiguration, UserAuthFunctions userAuthFunctions, @Qualifier("warehouse") Connector accumuloConnector) { @@ -84,4 +92,11 @@ public Scanner getScannerWithRegexIteratorSetting(String name, String modelTable scanner.addScanIterator(cfg); return scanner; } + + public BatchWriter getDefaultBatchWriter(String modelTable, String modelName, ProxiedUserDetails user) throws TableNotFoundException { + Connector connector = getConnection(modelTable, modelName, user).getConnector(); + // TODO Do we need a new instance of BatchWriterConfig each time, or can this be a static or bean object? + return connector.createBatchWriter(modelTable, new BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY, TimeUnit.MILLISECONDS).setMaxMemory(BATCH_WRITER_MAX_MEMORY) + .setMaxWriteThreads(BATCH_WRITER_MAX_THREADS)); + } } diff --git a/service/src/main/java/datawave/microservice/model/ModelController.java b/service/src/main/java/datawave/microservice/model/ModelController.java index 0a194d6..fd4f893 100644 --- a/service/src/main/java/datawave/microservice/model/ModelController.java +++ b/service/src/main/java/datawave/microservice/model/ModelController.java @@ -24,23 +24,19 @@ import lombok.extern.slf4j.Slf4j; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.BatchWriterConfig; -import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; -//import org.apache.deltaspike.core.api.config.ConfigProperty; -import org.apache.accumulo.core.iterators.user.RegExFilter; +import org.apache.accumulo.core.security.Authorizations; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.http.MediaType; import org.springframework.security.access.annotation.Secured; import org.springframework.security.core.annotation.AuthenticationPrincipal; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.Mapping; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; @@ -49,7 +45,7 @@ import java.util.HashSet; import java.util.Map; -import java.util.concurrent.TimeUnit; +import java.util.Set; /** * Service that supports manipulation of models. The models are contained in the data dictionary table. @@ -67,7 +63,7 @@ public class ModelController { private final AccumuloConnectionService accumloConnectionService; public static final String DEFAULT_MODEL_TABLE_NAME = "DatawaveMetadata"; - + private static final long BATCH_WRITER_MAX_LATENCY = 1000L; private static final long BATCH_WRITER_MAX_MEMORY = 10845760; private static final int BATCH_WRITER_MAX_THREADS = 2; @@ -197,10 +193,11 @@ public VoidResponse cloneModel(@RequestParam String name, @RequestParam String n public Model getModel(@RequestParam String name, @RequestParam(defaultValue = DEFAULT_MODEL_TABLE_NAME) String modelTableName, @AuthenticationPrincipal ProxiedUserDetails currentUser) { Model response = new Model(jqueryUri, dataTablesUri); + Set auths = accumloConnectionService.getConnection(modelTableName, name, currentUser).getAuths(); try (Scanner scanner = accumloConnectionService.getScannerWithRegexIteratorSetting(name, modelTableName, currentUser)) { for (Map.Entry entry : scanner) { - // FieldMapping mapping = ModelKeyParser.parseKey(entry.getKey(), cbAuths); - // response.getFields().add(mapping); + FieldMapping mapping = ModelKeyParser.parseKey(entry.getKey(), auths); + response.getFields().add(mapping); } } catch (TableNotFoundException e) { QueryException qe = new QueryException(DatawaveErrorCode.MODEL_NAME_LIST_ERROR, e); @@ -240,13 +237,9 @@ public VoidResponse insertMapping(@RequestBody Model model, @RequestParam(defaul } BatchWriter writer = null; - - Connection connection = accumloConnectionService.getConnection(modelTableName, model.getName(), currentUser); + try { - // Is the BatchWriterConfig already a spring bean? - writer = connection.getConnector().createBatchWriter(modelTableName, - new BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY, TimeUnit.MILLISECONDS).setMaxMemory(BATCH_WRITER_MAX_MEMORY) - .setMaxWriteThreads(BATCH_WRITER_MAX_THREADS)); + writer = accumloConnectionService.getDefaultBatchWriter(modelTableName, model.getName(), currentUser); } catch (TableNotFoundException e) { log.error("The " + modelTableName + " could not be found to write to ", e); QueryException qe = new QueryException(DatawaveErrorCode.TABLE_NOT_FOUND, e); @@ -272,7 +265,6 @@ public VoidResponse insertMapping(@RequestBody Model model, @RequestParam(defaul log.error("Error closing the BatchWriter; ", e); } } - } } @@ -301,44 +293,38 @@ public VoidResponse deleteMapping(@RequestParam Model model, @RequestParam Strin log.debug("Deleting model name: " + model.getName() + "from modelTableName " + modelTableName); } VoidResponse response = new VoidResponse(); - // - // Connector connector = null; - // BatchWriter writer = null; - // try { - // Map trackingMap = connectionFactory.getTrackingMap(Thread.currentThread().getStackTrace()); - // connector = connectionFactory.getConnection(getCurrentUserDN(), getCurrentProxyServers(), AccumuloConnectionFactory.Priority.LOW, trackingMap); - // writer = connector.createBatchWriter(modelTableName, new BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY, TimeUnit.MILLISECONDS) - // .setMaxMemory(BATCH_WRITER_MAX_MEMORY).setMaxWriteThreads(BATCH_WRITER_MAX_THREADS)); - // for (FieldMapping mapping : model.getFields()) { - // Mutation m = ModelKeyParser.createDeleteMutation(mapping, model.getName()); - // writer.addMutation(m); - // } - // } catch (Exception e) { - // log.error("Could not delete mapping.", e); - // QueryException qe = new QueryException(DatawaveErrorCode.MAPPING_DELETION_ERROR, e); - // response.addException(qe.getBottomQueryException()); - // throw new DatawaveWebApplicationException(qe, response); - // } finally { - // if (null != writer) { - // try { - // writer.close(); - // } catch (MutationsRejectedException e1) { - // QueryException qe = new QueryException(DatawaveErrorCode.WRITER_CLOSE_ERROR, e1); - // log.error(qe); - // response.addException(qe); - // throw new DatawaveWebApplicationException(qe, response); - // } - // } - // if (null != connector) { - // try { - // connectionFactory.returnConnection(connector); - // } catch (Exception e) { - // log.error("Error returning connection to factory", e); - // } - // } - // } - // if (reloadCache) - // cache.reloadTableCache(tableName); + BatchWriter writer = null; + + try { + writer = accumloConnectionService.getDefaultBatchWriter(modelTableName, model.getName(), currentUser); + } catch (TableNotFoundException e) { + log.error("The " + modelTableName + " could not be found to write to ", e); + QueryException qe = new QueryException(DatawaveErrorCode.TABLE_NOT_FOUND, e); + response.addException(qe.getBottomQueryException()); + } + + if (writer != null) { + for (FieldMapping mapping : model.getFields()) { + Mutation m = ModelKeyParser.createDeleteMutation(mapping, model.getName()); + try { + writer.addMutation(m); + } catch (MutationsRejectedException e) { + // So that we know exactly which mapping causes the error, catch here, but break out of loop + // Unfortunately, this does mean if there are multiple mappings that cause this error, only the first will be logged. + log.error("Could not delete mapping -- " + mapping, e); + QueryException qe = new QueryException(DatawaveErrorCode.INSERT_MAPPING_ERROR, e); + response.addException(qe.getBottomQueryException()); + } finally { + // we know the writer isn't null since we've already checked. + try { + writer.close(); + } catch (MutationsRejectedException e) { + log.error("Error closing the BatchWriter; ", e); + } + } + } + } + return response; } diff --git a/service/src/test/java/datawave/microservice/model/ModelControllerTest.java b/service/src/test/java/datawave/microservice/model/ModelControllerTest.java index c28f4d0..d664465 100644 --- a/service/src/test/java/datawave/microservice/model/ModelControllerTest.java +++ b/service/src/test/java/datawave/microservice/model/ModelControllerTest.java @@ -180,12 +180,34 @@ public void testImport() { assertEquals(2, modelListResponse2.getBody().getNames().size()); } - // @Test + @Test public void testGet() { - + // Verify that there is only one model so far (the model that is seeded before the test) + // @formatter:off + UriComponents uri = UriComponentsBuilder.newInstance() + .scheme("https").host("localhost").port(webServicePort) + .path("/dictionary/model/list") + .build(); + // @formatter:on + + ResponseEntity modelListResponse = jwtRestTemplate.exchange(adminUser, HttpMethod.GET, uri, ModelList.class); + assertEquals(HttpStatus.OK, modelListResponse.getStatusCode()); + assertEquals(1, modelListResponse.getBody().getNames().size()); + + // Verify that we can get that model by name + // @formatter:off + uri = UriComponentsBuilder.newInstance() + .scheme("https").host("localhost").port(webServicePort) + .path("/dictionary/model/{name}") + .queryParam("name", MODEL_ONE.getName()) + .build(); + // @formatter:on + ResponseEntity modelResponse = jwtRestTemplate.exchange(adminUser, HttpMethod.GET, uri, Model.class); + assertEquals(HttpStatus.OK, modelResponse.getStatusCode()); + assertEquals(MODEL_ONE.getName(), modelResponse.getBody().getName()); } - // @Test +// @Test public void testDelete() { // Verify that there is only one model so far (the model that is seeded before the test) // @formatter:off From 7a37defc86986d631df436dace8db0a2a20fe7f1 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 5 May 2022 13:13:44 -0400 Subject: [PATCH 17/54] Updated the dictionary service to use the metadata starter --- service/pom.xml | 10 ++ .../AccumuloConnectionService.java | 8 +- .../config/AuthorizationsConverter.java | 19 ---- .../config/MetadataHelperConfiguration.java | 101 ------------------ .../config/MetadataHelperProperties.java | 43 -------- .../microservice/model/ModelController.java | 14 +-- .../model/ModelControllerTest.java | 8 +- 7 files changed, 25 insertions(+), 178 deletions(-) delete mode 100644 service/src/main/java/datawave/microservice/metadata/config/AuthorizationsConverter.java delete mode 100644 service/src/main/java/datawave/microservice/metadata/config/MetadataHelperConfiguration.java delete mode 100644 service/src/main/java/datawave/microservice/metadata/config/MetadataHelperProperties.java diff --git a/service/pom.xml b/service/pom.xml index ffb4d45..d919d35 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -23,6 +23,7 @@ 1.3-SNAPSHOT 1.10-SNAPSHOT 1.6.8-SNAPSHOT + 1.0-SNAPSHOT 1.10.25 3.6.0 0.50 @@ -69,6 +70,11 @@ spring-boot-starter-datawave ${version.microservice.starter} + + gov.nsa.datawave.microservice + spring-boot-starter-datawave-metadata + ${version.microservice.starter-metadata} + org.apache.accumulo accumulo-core @@ -138,6 +144,10 @@ gov.nsa.datawave.microservice spring-boot-starter-datawave + + gov.nsa.datawave.microservice + spring-boot-starter-datawave-metadata + org.apache.accumulo accumulo-core diff --git a/service/src/main/java/datawave/microservice/AccumuloConnectionService.java b/service/src/main/java/datawave/microservice/AccumuloConnectionService.java index c415c71..1bf90fa 100644 --- a/service/src/main/java/datawave/microservice/AccumuloConnectionService.java +++ b/service/src/main/java/datawave/microservice/AccumuloConnectionService.java @@ -34,7 +34,7 @@ public class AccumuloConnectionService { private final Connector accumuloConnector; private final DataDictionaryProperties dataDictionaryConfiguration; private final UserAuthFunctions userAuthFunctions; - + // These were already hardcodeded values in ModelBean -> ModelController so therefore they weren't made configurable private static final long BATCH_WRITER_MAX_LATENCY = 1000L; private static final long BATCH_WRITER_MAX_MEMORY = 10845760; @@ -92,11 +92,11 @@ public Scanner getScannerWithRegexIteratorSetting(String name, String modelTable scanner.addScanIterator(cfg); return scanner; } - + public BatchWriter getDefaultBatchWriter(String modelTable, String modelName, ProxiedUserDetails user) throws TableNotFoundException { Connector connector = getConnection(modelTable, modelName, user).getConnector(); // TODO Do we need a new instance of BatchWriterConfig each time, or can this be a static or bean object? - return connector.createBatchWriter(modelTable, new BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY, TimeUnit.MILLISECONDS).setMaxMemory(BATCH_WRITER_MAX_MEMORY) - .setMaxWriteThreads(BATCH_WRITER_MAX_THREADS)); + return connector.createBatchWriter(modelTable, new BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY, TimeUnit.MILLISECONDS) + .setMaxMemory(BATCH_WRITER_MAX_MEMORY).setMaxWriteThreads(BATCH_WRITER_MAX_THREADS)); } } diff --git a/service/src/main/java/datawave/microservice/metadata/config/AuthorizationsConverter.java b/service/src/main/java/datawave/microservice/metadata/config/AuthorizationsConverter.java deleted file mode 100644 index 0972445..0000000 --- a/service/src/main/java/datawave/microservice/metadata/config/AuthorizationsConverter.java +++ /dev/null @@ -1,19 +0,0 @@ -package datawave.microservice.metadata.config; - -import org.apache.accumulo.core.security.Authorizations; -import org.springframework.boot.context.properties.ConfigurationPropertiesBinding; -import org.springframework.core.convert.converter.Converter; -import org.springframework.stereotype.Component; -import org.springframework.util.StringUtils; - -/** - * A spring boot property {@link Converter} that converts a comma-separated list of strings into an Accumulo {@link Authorizations} object. - */ -@Component -@ConfigurationPropertiesBinding -public class AuthorizationsConverter implements Converter { - @Override - public Authorizations convert(String auths) { - return new Authorizations(StringUtils.commaDelimitedListToStringArray(auths)); - } -} diff --git a/service/src/main/java/datawave/microservice/metadata/config/MetadataHelperConfiguration.java b/service/src/main/java/datawave/microservice/metadata/config/MetadataHelperConfiguration.java deleted file mode 100644 index 4e08c10..0000000 --- a/service/src/main/java/datawave/microservice/metadata/config/MetadataHelperConfiguration.java +++ /dev/null @@ -1,101 +0,0 @@ -package datawave.microservice.metadata.config; - -import com.github.benmanes.caffeine.cache.CaffeineSpec; -import datawave.query.composite.CompositeMetadataHelper; -import datawave.query.util.AllFieldMetadataHelper; -import datawave.query.util.MetadataCacheManager; -import datawave.query.util.MetadataHelper; -import datawave.query.util.MetadataHelperFactory; -import datawave.query.util.TypeMetadataHelper; -import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.security.Authorizations; -import org.springframework.beans.factory.BeanFactory; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.cache.CacheManager; -import org.springframework.cache.caffeine.CaffeineCacheManager; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Scope; - -import java.util.Map; -import java.util.Set; - -@Configuration -public class MetadataHelperConfiguration { - @Bean - @ConditionalOnMissingBean - public MetadataHelperProperties metadataHelperProperties() { - return new MetadataHelperProperties(); - } - - @Bean - @ConditionalOnMissingBean - public MetadataHelperFactory metadataHelperFactory(BeanFactory beanFactory, TypeMetadataHelper.Factory typeMetadataHelperFactory) { - return new MetadataHelperFactory(beanFactory, typeMetadataHelperFactory); - } - - @Bean - @ConditionalOnMissingBean - public TypeMetadataHelper.Factory typeMetadataHelperFactory(BeanFactory beanFactory) { - return new TypeMetadataHelper.Factory(beanFactory); - } - - @Bean - @ConditionalOnMissingBean - public MetadataCacheManager metadataCacheManager(@Qualifier("metadataHelperCacheManager") CacheManager cacheManager) { - return new MetadataCacheManager(cacheManager); - } - - @Bean - @Scope("prototype") - @ConditionalOnMissingBean - public MetadataHelper metadataHelper(AllFieldMetadataHelper allFieldMetadataHelper, @Qualifier("allMetadataAuths") Set allMetadataAuths, - Connector connector, String metadataTableName, Set auths, Set fullUserAuths) { - return new MetadataHelper(allFieldMetadataHelper, allMetadataAuths, connector, metadataTableName, auths, fullUserAuths); - } - - @Bean - @Scope("prototype") - @ConditionalOnMissingBean - public AllFieldMetadataHelper allFieldMetadataHelper(TypeMetadataHelper typeMetadataHelper, CompositeMetadataHelper compositeMetadataHelper, - Connector connector, String metadataTableName, Set auths, Set fullUserAuths) { - return new AllFieldMetadataHelper(typeMetadataHelper, compositeMetadataHelper, connector, metadataTableName, auths, fullUserAuths); - } - - @Bean - @Scope("prototype") - @ConditionalOnMissingBean - public CompositeMetadataHelper compositeMetadataHelper(Connector connector, String metadataTableName, Set auths) { - return new CompositeMetadataHelper(connector, metadataTableName, auths); - } - - @Bean - @Scope("prototype") - @ConditionalOnMissingBean - public TypeMetadataHelper typeMetadataHelper(@Qualifier("typeSubstitutions") Map typeSubstitutions, - @Qualifier("allMetadataAuths") Set allMetadataAuths, Connector connector, String metadataTableName, - Set auths, boolean useTypeSubstitution) { - return new TypeMetadataHelper(typeSubstitutions, allMetadataAuths, connector, metadataTableName, auths, useTypeSubstitution); - } - - @Bean(name = "typeSubstitutions") - @ConditionalOnMissingBean(name = "typeSubstitutions") - public Map typeSubstitutions(MetadataHelperProperties metadataHelperProperties) { - return metadataHelperProperties.getTypeSubstitutions(); - } - - @Bean(name = "allMetadataAuths") - @ConditionalOnMissingBean(name = "allMetadataAuths") - public Set allMetadataAuths(MetadataHelperProperties metadataHelperProperties) { - return metadataHelperProperties.getAllMetadataAuths(); - } - - @Bean(name = "metadataHelperCacheManager") - @ConditionalOnMissingBean(name = "metadataHelperCacheManager") - public CacheManager metadataHelperCacheManager() { - CaffeineCacheManager caffeineCacheManager = new CaffeineCacheManager(); - caffeineCacheManager.setCaffeineSpec(CaffeineSpec.parse("maximumSize=100, expireAfterAccess=24h, expireAfterWrite=24h")); - return caffeineCacheManager; - } -} diff --git a/service/src/main/java/datawave/microservice/metadata/config/MetadataHelperProperties.java b/service/src/main/java/datawave/microservice/metadata/config/MetadataHelperProperties.java deleted file mode 100644 index da8a54a..0000000 --- a/service/src/main/java/datawave/microservice/metadata/config/MetadataHelperProperties.java +++ /dev/null @@ -1,43 +0,0 @@ -package datawave.microservice.metadata.config; - -import org.apache.accumulo.core.security.Authorizations; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.validation.annotation.Validated; - -import javax.validation.constraints.NotEmpty; -import javax.validation.constraints.NotNull; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -@Validated -@ConfigurationProperties(prefix = "datawave.metadata") -@EnableConfigurationProperties(MetadataHelperProperties.class) -public class MetadataHelperProperties { - @NotEmpty - private Set allMetadataAuths; - @NotNull - private Map typeSubstitutions; - - public MetadataHelperProperties() { - typeSubstitutions = new HashMap<>(); - typeSubstitutions.put("datawave.data.type.DateType", "datawave.data.type.RawDateType"); - } - - public Set getAllMetadataAuths() { - return allMetadataAuths; - } - - public void setAllMetadataAuths(Set allMetadataAuths) { - this.allMetadataAuths = allMetadataAuths; - } - - public Map getTypeSubstitutions() { - return typeSubstitutions; - } - - public void setTypeSubstitutions(Map typeSubstitutions) { - this.typeSubstitutions = typeSubstitutions; - } -} diff --git a/service/src/main/java/datawave/microservice/model/ModelController.java b/service/src/main/java/datawave/microservice/model/ModelController.java index fd4f893..af064d6 100644 --- a/service/src/main/java/datawave/microservice/model/ModelController.java +++ b/service/src/main/java/datawave/microservice/model/ModelController.java @@ -63,7 +63,7 @@ public class ModelController { private final AccumuloConnectionService accumloConnectionService; public static final String DEFAULT_MODEL_TABLE_NAME = "DatawaveMetadata"; - + private static final long BATCH_WRITER_MAX_LATENCY = 1000L; private static final long BATCH_WRITER_MAX_MEMORY = 10845760; private static final int BATCH_WRITER_MAX_THREADS = 2; @@ -196,8 +196,8 @@ public Model getModel(@RequestParam String name, @RequestParam(defaultValue = DE Set auths = accumloConnectionService.getConnection(modelTableName, name, currentUser).getAuths(); try (Scanner scanner = accumloConnectionService.getScannerWithRegexIteratorSetting(name, modelTableName, currentUser)) { for (Map.Entry entry : scanner) { - FieldMapping mapping = ModelKeyParser.parseKey(entry.getKey(), auths); - response.getFields().add(mapping); + FieldMapping mapping = ModelKeyParser.parseKey(entry.getKey(), auths); + response.getFields().add(mapping); } } catch (TableNotFoundException e) { QueryException qe = new QueryException(DatawaveErrorCode.MODEL_NAME_LIST_ERROR, e); @@ -237,7 +237,7 @@ public VoidResponse insertMapping(@RequestBody Model model, @RequestParam(defaul } BatchWriter writer = null; - + try { writer = accumloConnectionService.getDefaultBatchWriter(modelTableName, model.getName(), currentUser); } catch (TableNotFoundException e) { @@ -294,7 +294,7 @@ public VoidResponse deleteMapping(@RequestParam Model model, @RequestParam Strin } VoidResponse response = new VoidResponse(); BatchWriter writer = null; - + try { writer = accumloConnectionService.getDefaultBatchWriter(modelTableName, model.getName(), currentUser); } catch (TableNotFoundException e) { @@ -302,7 +302,7 @@ public VoidResponse deleteMapping(@RequestParam Model model, @RequestParam Strin QueryException qe = new QueryException(DatawaveErrorCode.TABLE_NOT_FOUND, e); response.addException(qe.getBottomQueryException()); } - + if (writer != null) { for (FieldMapping mapping : model.getFields()) { Mutation m = ModelKeyParser.createDeleteMutation(mapping, model.getName()); @@ -324,7 +324,7 @@ public VoidResponse deleteMapping(@RequestParam Model model, @RequestParam Strin } } } - + return response; } diff --git a/service/src/test/java/datawave/microservice/model/ModelControllerTest.java b/service/src/test/java/datawave/microservice/model/ModelControllerTest.java index d664465..2a6466d 100644 --- a/service/src/test/java/datawave/microservice/model/ModelControllerTest.java +++ b/service/src/test/java/datawave/microservice/model/ModelControllerTest.java @@ -180,7 +180,7 @@ public void testImport() { assertEquals(2, modelListResponse2.getBody().getNames().size()); } - @Test + @Test public void testGet() { // Verify that there is only one model so far (the model that is seeded before the test) // @formatter:off @@ -189,11 +189,11 @@ public void testGet() { .path("/dictionary/model/list") .build(); // @formatter:on - + ResponseEntity modelListResponse = jwtRestTemplate.exchange(adminUser, HttpMethod.GET, uri, ModelList.class); assertEquals(HttpStatus.OK, modelListResponse.getStatusCode()); assertEquals(1, modelListResponse.getBody().getNames().size()); - + // Verify that we can get that model by name // @formatter:off uri = UriComponentsBuilder.newInstance() @@ -207,7 +207,7 @@ public void testGet() { assertEquals(MODEL_ONE.getName(), modelResponse.getBody().getName()); } -// @Test + // @Test public void testDelete() { // Verify that there is only one model so far (the model that is seeded before the test) // @formatter:off From f26fdc266caedef3dc02ae3a99212b4c66669a9a Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Thu, 5 May 2022 18:11:26 +0000 Subject: [PATCH 18/54] Removed management of metadata-utils since the starter is doing that for us --- service/pom.xml | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/service/pom.xml b/service/pom.xml index d919d35..7b69f70 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -21,7 +21,6 @@ datawave.microservice.dictionary.DictionaryService 1.9.3 1.3-SNAPSHOT - 1.10-SNAPSHOT 1.6.8-SNAPSHOT 1.0-SNAPSHOT 1.10.25 @@ -46,25 +45,6 @@ dictionary-api ${version.microservice.dictionary-api} - - gov.nsa.datawave.microservice - metadata-utils - ${version.microservice.metadata-utils} - - - servlet-api - javax.servlet - - - slf4j-reload4j - org.slf4j - - - log4j - log4j - - - gov.nsa.datawave.microservice spring-boot-starter-datawave @@ -136,10 +116,6 @@ gov.nsa.datawave.microservice dictionary-api - - gov.nsa.datawave.microservice - metadata-utils - gov.nsa.datawave.microservice spring-boot-starter-datawave From 7011f6f2239f4f2ec66ea3a91074d73b04c24aec Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Thu, 5 May 2022 20:38:00 +0000 Subject: [PATCH 19/54] Need the bus active to be able to use the table cache --- service/src/test/resources/config/bootstrap.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service/src/test/resources/config/bootstrap.yml b/service/src/test/resources/config/bootstrap.yml index 2c2907d..dd676bb 100644 --- a/service/src/test/resources/config/bootstrap.yml +++ b/service/src/test/resources/config/bootstrap.yml @@ -3,7 +3,7 @@ spring: name: dictionary-test cloud: bus: - enabled: false + enabled: true consul: enabled: false config: @@ -11,4 +11,4 @@ spring: discovery: enabled: false main: - banner-mode: "OFF" \ No newline at end of file + banner-mode: "OFF" From ecd1bdfa63044c16df8fced9fa351004a204f681 Mon Sep 17 00:00:00 2001 From: cmitchell Date: Fri, 6 May 2022 11:19:39 -0400 Subject: [PATCH 20/54] Fully tested the ModelController. --- .../webservice/model/FieldMapping.java | 112 ++---------- .../java/datawave/webservice/model/Model.java | 57 +----- .../datawave/webservice/model/ModelList.java | 30 ++-- .../AccumuloConnectionService.java | 143 +++++++++++++-- .../microservice/model/ModelController.java | 168 +++++------------- .../model/ModelControllerTest.java | 82 ++++++--- 6 files changed, 263 insertions(+), 329 deletions(-) diff --git a/api/src/main/java/datawave/webservice/model/FieldMapping.java b/api/src/main/java/datawave/webservice/model/FieldMapping.java index 59a3427..6db03a6 100644 --- a/api/src/main/java/datawave/webservice/model/FieldMapping.java +++ b/api/src/main/java/datawave/webservice/model/FieldMapping.java @@ -1,113 +1,37 @@ package datawave.webservice.model; -import java.io.Serializable; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.commons.lang.builder.CompareToBuilder; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAttribute; +import java.io.Serializable; -import org.apache.commons.lang.builder.CompareToBuilder; -import org.apache.commons.lang.builder.EqualsBuilder; -import org.apache.commons.lang.builder.HashCodeBuilder; -import org.apache.commons.lang.builder.ToStringBuilder; - +@Data +@NoArgsConstructor +@AllArgsConstructor @XmlAccessorType(XmlAccessType.NONE) public class FieldMapping implements Serializable, Comparable { private static final long serialVersionUID = 1L; - @XmlAttribute(name = "datatype", required = true) - private String datatype = null; - - @XmlAttribute(name = "fieldName", required = true) - private String fieldName = null; - - @XmlAttribute(name = "modelFieldName", required = true) - private String modelFieldName = null; - - @XmlAttribute(name = "direction", required = true) - private Direction direction = null; - - @XmlAttribute(name = "columnVisibility", required = true) - private String columnVisibility = null; - - public FieldMapping() { - super(); - } - - public FieldMapping(String datatype, String fieldName, String modelFieldName, Direction direction, String columnVisibility) { - super(); - this.datatype = datatype; - this.fieldName = fieldName; - this.modelFieldName = modelFieldName; - this.direction = direction; - this.columnVisibility = columnVisibility; - - } - - public String getDatatype() { - return datatype; - } - - public void setDatatype(String datatype) { - this.datatype = datatype; - } - - public String getFieldName() { - return fieldName; - } - - public void setFieldName(String fieldName) { - this.fieldName = fieldName; - } - - public String getModelFieldName() { - return modelFieldName; - } - - public void setModelFieldName(String modelFieldName) { - this.modelFieldName = modelFieldName; - } - - public Direction getDirection() { - return direction; - } - - public void setDirection(Direction direction) { - this.direction = direction; - } - - public String getColumnVisibility() { - return columnVisibility; - } + @XmlAttribute(required = true) + private String datatype; - public void setColumnVisibility(String columnVisibility) { - this.columnVisibility = columnVisibility; - } + @XmlAttribute(required = true) + private String fieldName; - @Override - public int hashCode() { - return new HashCodeBuilder(17, 37).append(columnVisibility).append(datatype).append(direction).append(fieldName).append(modelFieldName).toHashCode(); - } + @XmlAttribute(required = true) + private String modelFieldName; - @Override - public boolean equals(Object obj) { - if (obj == null) - return false; - if (obj == this) - return true; - if (obj.getClass() != this.getClass()) - return false; - FieldMapping other = (FieldMapping) obj; - return new EqualsBuilder().append(columnVisibility, other.columnVisibility).append(datatype, other.datatype).append(direction, other.direction) - .append(fieldName, other.fieldName).append(modelFieldName, other.modelFieldName).isEquals(); - } + @XmlAttribute(required = true) + private Direction direction; - @Override - public String toString() { - return new ToStringBuilder(this).append("columnVisibility", columnVisibility).append("datatype", datatype).append("direction", direction) - .append("fieldName", fieldName).append("modelFieldName", modelFieldName).toString(); - } + @XmlAttribute(required = true) + private String columnVisibility; @Override public int compareTo(FieldMapping obj) { diff --git a/api/src/main/java/datawave/webservice/model/Model.java b/api/src/main/java/datawave/webservice/model/Model.java index a7dfd3e..bde13bc 100644 --- a/api/src/main/java/datawave/webservice/model/Model.java +++ b/api/src/main/java/datawave/webservice/model/Model.java @@ -1,8 +1,8 @@ package datawave.webservice.model; -import java.io.Serializable; -import java.text.MessageFormat; -import java.util.TreeSet; +import datawave.webservice.HtmlProvider; +import datawave.webservice.result.BaseResponse; +import lombok.Data; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -10,14 +10,11 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlElementWrapper; import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; +import java.text.MessageFormat; +import java.util.TreeSet; -import datawave.webservice.HtmlProvider; -import datawave.webservice.result.BaseResponse; - -import org.apache.commons.lang.builder.EqualsBuilder; -import org.apache.commons.lang.builder.HashCodeBuilder; -import org.apache.commons.lang.builder.ToStringBuilder; - +@Data @XmlRootElement(name = "Model") @XmlAccessorType(XmlAccessType.NONE) public class Model extends BaseResponse implements Serializable, HtmlProvider { @@ -41,50 +38,12 @@ public Model(String jqueryUri, String datatablesUri) { public Model() {}; @XmlAttribute(name = "name", required = true) - private String name = null; + private String name; @XmlElementWrapper(name = "Mappings") @XmlElement(name = "Mapping") private TreeSet fields = new TreeSet(); - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public TreeSet getFields() { - return fields; - } - - public void setFields(TreeSet fields) { - this.fields = fields; - } - - @Override - public int hashCode() { - return new HashCodeBuilder(17, 37).append(name).append(fields).toHashCode(); - } - - @Override - public boolean equals(Object obj) { - if (obj == null) - return false; - if (obj == this) - return true; - if (obj.getClass() != this.getClass()) - return false; - Model other = (Model) obj; - return new EqualsBuilder().append(name, other.name).append(fields, other.fields).isEquals(); - } - - @Override - public String toString() { - return new ToStringBuilder(this).append("name", name).append("fields", fields).toString(); - } - /* * (non-Javadoc) * diff --git a/api/src/main/java/datawave/webservice/model/ModelList.java b/api/src/main/java/datawave/webservice/model/ModelList.java index d9cef45..ec793ec 100644 --- a/api/src/main/java/datawave/webservice/model/ModelList.java +++ b/api/src/main/java/datawave/webservice/model/ModelList.java @@ -1,16 +1,19 @@ package datawave.webservice.model; -import java.io.Serializable; -import java.text.MessageFormat; -import java.util.HashSet; +import datawave.webservice.HtmlProvider; +import datawave.webservice.result.BaseResponse; +import lombok.Data; +import lombok.NoArgsConstructor; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlElementWrapper; import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; +import java.text.MessageFormat; +import java.util.HashSet; -import datawave.webservice.HtmlProvider; -import datawave.webservice.result.BaseResponse; - +@Data +@NoArgsConstructor @XmlRootElement(name = "ModelList") public class ModelList extends BaseResponse implements Serializable, HtmlProvider { @@ -25,11 +28,6 @@ public class ModelList extends BaseResponse implements Serializable, HtmlProvide + "$(document).ready(function() '{' $(''#myTable'').dataTable('{'\"bPaginate\": false, \"aaSorting\": [[0, \"asc\"]], \"bStateSave\": true'}') '}')\n" + "\n"; - /** - * Need empty constructor for serialization - */ - public ModelList() {} - public ModelList(String jqueryUri, String datatablesUri, String modelTableName) { this.jqueryUri = jqueryUri; this.dataTablesUri = datatablesUri; @@ -38,15 +36,7 @@ public ModelList(String jqueryUri, String datatablesUri, String modelTableName) @XmlElementWrapper(name = "ModelNames") @XmlElement(name = "ModelName") - private HashSet names = null; - - public HashSet getNames() { - return names; - } - - public void setNames(HashSet names) { - this.names = names; - } + private HashSet names; /* * (non-Javadoc) diff --git a/service/src/main/java/datawave/microservice/AccumuloConnectionService.java b/service/src/main/java/datawave/microservice/AccumuloConnectionService.java index 1bf90fa..365b444 100644 --- a/service/src/main/java/datawave/microservice/AccumuloConnectionService.java +++ b/service/src/main/java/datawave/microservice/AccumuloConnectionService.java @@ -5,29 +5,41 @@ import datawave.microservice.dictionary.config.DataDictionaryProperties; import datawave.security.authorization.DatawaveUser; import datawave.security.util.ScannerHelper; +import datawave.webservice.query.exception.DatawaveErrorCode; +import datawave.webservice.query.exception.QueryException; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.BatchWriterConfig; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.iterators.user.RegExFilter; import org.apache.accumulo.core.security.Authorizations; import org.apache.commons.lang.StringUtils; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; +import java.util.List; import java.util.Set; +import java.util.Spliterator; +import java.util.Spliterators; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import java.util.stream.Collectors; +import java.util.stream.StreamSupport; /** - * Provides a connection to accumulo for use by controllers + * Provides accumlo functionality for use by the dictionary controllers. Keeps the accumulo interface separate from the controllers. * * If other services (other than dictionary/model) are going to need to connect to accumulo, should we move this into the accumulo service and pass the * individual properties (instead of a properties object and calling getters) */ +@Slf4j @Service public class AccumuloConnectionService { @@ -35,7 +47,7 @@ public class AccumuloConnectionService { private final DataDictionaryProperties dataDictionaryConfiguration; private final UserAuthFunctions userAuthFunctions; - // These were already hardcodeded values in ModelBean -> ModelController so therefore they weren't made configurable + // These were already hardcodeded values in ModelBean -> ModelController therefore they weren't made configurable private static final long BATCH_WRITER_MAX_LATENCY = 1000L; private static final long BATCH_WRITER_MAX_MEMORY = 10845760; private static final int BATCH_WRITER_MAX_THREADS = 2; @@ -47,24 +59,58 @@ public AccumuloConnectionService(DataDictionaryProperties dataDictionaryConfigur this.accumuloConnector = accumuloConnector; } + /** + * Returns a Connection representing the connection to accumulo + * + * @param modelTable + * the name of the model table to use in the connection + * @param modelName + * the name of the model to use in the connection + * @param user + * the user (and the authorities and other security aspects related to the user) + * + * @return a Connection representing the connection to accumulo + */ public Connection getConnection(String modelTable, String modelName, ProxiedUserDetails user) { return getConnection(dataDictionaryConfiguration.getMetadataTableName(), modelTable, modelName, user); } + /** + * Returns a Connection representing the connection to accumulo + * + * @param metadataTable + * the name of the metadata table to use in the connection + * @param modelTable + * the name of the model table to use in the connection + * @param modelName + * the name of the model to use in the connection + * @param user + * the user (and the authorities and other security aspects related to the user) + * + * @return a Connection representing the connection to accumulo + */ public Connection getConnection(String metadataTable, String modelTable, String modelName, ProxiedUserDetails user) { - Connection helper = new Connection(); - helper.setMetadataTable(getSupplierValueIfBlank(metadataTable, dataDictionaryConfiguration::getMetadataTableName)); - helper.setModelTable(getSupplierValueIfBlank(modelTable, dataDictionaryConfiguration::getModelTableName)); - helper.setModelName(getSupplierValueIfBlank(modelName, dataDictionaryConfiguration::getModelName)); - helper.setAuths(getAuths(user)); - helper.setConnector(accumuloConnector); - return helper; + Connection connection = new Connection(); + connection.setMetadataTable(getSupplierValueIfBlank(metadataTable, dataDictionaryConfiguration::getMetadataTableName)); + connection.setModelTable(getSupplierValueIfBlank(modelTable, dataDictionaryConfiguration::getModelTableName)); + connection.setModelName(getSupplierValueIfBlank(modelName, dataDictionaryConfiguration::getModelName)); + connection.setAuths(getAuths(user)); + connection.setConnector(accumuloConnector); + return connection; } private String getSupplierValueIfBlank(final String value, final Supplier supplier) { return StringUtils.isBlank(value) ? supplier.get() : value; } + /** + * Return the accumulo authorizations for the user + * + * @param currentUser + * the user for whom to get the authorizations + * + * @return the accumulo authorizations for the user + */ public Set getAuths(ProxiedUserDetails currentUser) { //@formatter:off return currentUser.getProxiedUsers().stream() @@ -74,26 +120,87 @@ public Set getAuths(ProxiedUserDetails currentUser) { //@formatter:on } + /** + * Return the authorizations for the specified user downgraded to the specified level + * + * @param requestedAuthorizations + * the level of authorizations to downgrade to + * @param currentUser + * the user for whom to downgrade the authorizations + * + * @return the authorizations for the specified user downgraded to the specified level + */ public Set getDowngradedAuthorizations(String requestedAuthorizations, ProxiedUserDetails currentUser) { DatawaveUser primaryUser = currentUser.getPrimaryUser(); return userAuthFunctions.mergeAuthorizations(userAuthFunctions.getRequestedAuthorizations(requestedAuthorizations, currentUser.getPrimaryUser()), currentUser.getProxiedUsers(), u -> u != primaryUser); } - public Scanner getScanner(String modelTable, ProxiedUserDetails currentUser) throws TableNotFoundException { - return ScannerHelper.createScanner(this.accumuloConnector, modelTable, this.getAuths(currentUser)); + /** + * Return the keys. If a regexTerm is specified, the keys will be filtered according to that term, otherwise all the keys in the specified table that the + * user has authorizations for will be returned. + * + * @param modelTable + * the model table to get the keys of + * @param currentUser + * the user requesting the keys. Used for authorizations purposes + * @param regexTerm + * the regex (can be just a string) to apply to the IteratorSetting + * + * @return the keys of hte specified table that the specified user has the authorzations to access. + * @throws TableNotFoundException + * Thrown if the table is not found + */ + public List getKeys(String modelTable, ProxiedUserDetails currentUser, String regexTerm) throws TableNotFoundException { + Scanner scanner = ScannerHelper.createScanner(this.accumuloConnector, modelTable, this.getAuths(currentUser)); + if (!regexTerm.isEmpty()) { + IteratorSetting cfg = new IteratorSetting(21, "colfRegex", RegExFilter.class.getName()); + cfg.addOption(RegExFilter.COLF_REGEX, "^" + regexTerm + "(\\x00.*)?"); + scanner.addScanIterator(cfg); + } + + return StreamSupport.stream(Spliterators.spliteratorUnknownSize(scanner.iterator(), Spliterator.ORDERED), false).map(entry -> entry.getKey()) + .collect(Collectors.toList()); } - public Scanner getScannerWithRegexIteratorSetting(String name, String modelTable, ProxiedUserDetails currentUser) throws TableNotFoundException { - IteratorSetting cfg = new IteratorSetting(21, "colfRegex", RegExFilter.class.getName()); - cfg.addOption(RegExFilter.COLF_REGEX, "^" + name + "(\\x00.*)?"); + /** + * Modify (insert, delete, etc) the mappings. Throws an exception if not successful. + * + * @param mutations + * the mutations (insertions, deletions, etc) to perform + * @param modelTable + * the model table + * @param modelName + * the model name + * @param user + * the user requesting the changes + * + * @return A QueryException if anything is unsuccessful, and null if everything is successful. + */ + public QueryException modifyMappings(List mutations, String modelTable, String modelName, ProxiedUserDetails user) { + QueryException exception = null; + Mutation mute = null; + + try { + @Cleanup + BatchWriter writer = getDefaultBatchWriter(modelTable, modelName, user); + + for (Mutation m : mutations) { + mute = m; // make the specific mapping available outside this scope for logging purposes. + writer.addMutation(m); + } + } catch (TableNotFoundException e) { + log.error("The " + modelTable + " could not be found to write to ", e); + exception = new QueryException(DatawaveErrorCode.TABLE_NOT_FOUND, e); + } catch (MutationsRejectedException e) { + log.error("Could not modify mapping -- " + mute, e); + exception = new QueryException(DatawaveErrorCode.INSERT_MAPPING_ERROR, e); + } - Scanner scanner = getScanner(modelTable, currentUser); - scanner.addScanIterator(cfg); - return scanner; + return exception; } - public BatchWriter getDefaultBatchWriter(String modelTable, String modelName, ProxiedUserDetails user) throws TableNotFoundException { + private BatchWriter getDefaultBatchWriter(String modelTable, String modelName, ProxiedUserDetails user) throws TableNotFoundException { Connector connector = getConnection(modelTable, modelName, user).getConnector(); // TODO Do we need a new instance of BatchWriterConfig each time, or can this be a static or bean object? return connector.createBatchWriter(modelTable, new BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY, TimeUnit.MILLISECONDS) diff --git a/service/src/main/java/datawave/microservice/model/ModelController.java b/service/src/main/java/datawave/microservice/model/ModelController.java index af064d6..44290a1 100644 --- a/service/src/main/java/datawave/microservice/model/ModelController.java +++ b/service/src/main/java/datawave/microservice/model/ModelController.java @@ -1,16 +1,7 @@ package datawave.microservice.model; import com.google.common.collect.Sets; -//import datawave.interceptor.RequiredInterceptor; -//import datawave.interceptor.ResponseInterceptor; -//import datawave.security.authorization.DatawavePrincipal; -//import datawave.services.common.cache.AccumuloTableCache; -//import datawave.services.common.connection.AccumuloConnectionFactory; -//import datawave.webservice.common.exception.DatawaveWebApplicationException; -//import datawave.webservice.common.exception.NotFoundException; -//import datawave.webservice.common.exception.PreConditionFailedException; import datawave.microservice.AccumuloConnectionService; -import datawave.microservice.Connection; import datawave.microservice.authorization.user.ProxiedUserDetails; import datawave.microservice.http.converter.protostuff.ProtostuffHttpMessageConverter; import datawave.microservice.model.config.ModelProperties; @@ -22,14 +13,9 @@ import datawave.webservice.query.exception.QueryException; import datawave.webservice.result.VoidResponse; import lombok.extern.slf4j.Slf4j; -import org.apache.accumulo.core.client.BatchWriter; -import org.apache.accumulo.core.client.BatchWriterConfig; -import org.apache.accumulo.core.client.MutationsRejectedException; -import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; -import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.http.MediaType; @@ -44,8 +30,10 @@ import org.springframework.web.bind.annotation.RestController; import java.util.HashSet; -import java.util.Map; +import java.util.List; import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; /** * Service that supports manipulation of models. The models are contained in the data dictionary table. @@ -63,11 +51,6 @@ public class ModelController { private final AccumuloConnectionService accumloConnectionService; public static final String DEFAULT_MODEL_TABLE_NAME = "DatawaveMetadata"; - - private static final long BATCH_WRITER_MAX_LATENCY = 1000L; - private static final long BATCH_WRITER_MAX_MEMORY = 10845760; - private static final int BATCH_WRITER_MAX_THREADS = 2; - private static final HashSet RESERVED_COLF_VALUES = Sets.newHashSet("e", "i", "ri", "f", "tf", "m", "desc", "edge", "t", "n", "h"); public ModelController(ModelProperties modelProperties, AccumuloConnectionService accumloConnectionService) { @@ -83,7 +66,7 @@ public ModelController(ModelProperties modelProperties, AccumuloConnectionServic * name of the table that contains the model * @return ModelList * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user - * + * * @HTTP 200 success * @HTTP 500 internal server error */ @@ -93,24 +76,26 @@ public ModelList listModelNames(@RequestParam(defaultValue = DEFAULT_MODEL_TABLE ModelList response = new ModelList(jqueryUri, dataTablesUri, modelTableName); HashSet modelNames = new HashSet<>(); - - try (Scanner scanner = accumloConnectionService.getScanner(modelTableName, currentUser)) { - for (Map.Entry entry : scanner) { - String colf = entry.getKey().getColumnFamily().toString(); - if (!RESERVED_COLF_VALUES.contains(colf) && !modelNames.contains(colf)) { - String[] parts = colf.split(ModelKeyParser.NULL_BYTE); - if (parts.length == 1) { - modelNames.add(colf); - } else if (parts.length == 2) { - modelNames.add(parts[0]); - } - } - - } + List keys; + try { + keys = accumloConnectionService.getKeys(modelTableName, currentUser, ""); } catch (TableNotFoundException e) { QueryException qe = new QueryException(DatawaveErrorCode.MODEL_NAME_LIST_ERROR, e); log.error(qe.getMessage()); response.addException(qe.getBottomQueryException()); + return response; + } + + Set colFamilies = keys.stream().map(k -> k.getColumnFamily().toString()).filter(colFamily -> !RESERVED_COLF_VALUES.contains(colFamily)) + .collect(Collectors.toSet()); + + for (String colf : colFamilies) { + String[] parts = colf.split(ModelKeyParser.NULL_BYTE); + if (parts.length == 1) { + modelNames.add(colf); + } else if (parts.length == 2) { + modelNames.add(parts[0]); + } } response.setNames(modelNames); @@ -193,17 +178,19 @@ public VoidResponse cloneModel(@RequestParam String name, @RequestParam String n public Model getModel(@RequestParam String name, @RequestParam(defaultValue = DEFAULT_MODEL_TABLE_NAME) String modelTableName, @AuthenticationPrincipal ProxiedUserDetails currentUser) { Model response = new Model(jqueryUri, dataTablesUri); - Set auths = accumloConnectionService.getConnection(modelTableName, name, currentUser).getAuths(); - try (Scanner scanner = accumloConnectionService.getScannerWithRegexIteratorSetting(name, modelTableName, currentUser)) { - for (Map.Entry entry : scanner) { - FieldMapping mapping = ModelKeyParser.parseKey(entry.getKey(), auths); - response.getFields().add(mapping); - } + List keys; + try { + keys = accumloConnectionService.getKeys(modelTableName, currentUser, name); } catch (TableNotFoundException e) { QueryException qe = new QueryException(DatawaveErrorCode.MODEL_NAME_LIST_ERROR, e); log.error(qe.getMessage()); response.addException(qe.getBottomQueryException()); + return response; } + + Set auths = accumloConnectionService.getConnection(modelTableName, name, currentUser).getAuths(); + TreeSet fields = response.getFields(); + keys.forEach(k -> fields.add(ModelKeyParser.parseKey(k, auths))); response.setName(name); return response; } @@ -230,42 +217,12 @@ public VoidResponse insertMapping(@RequestBody Model model, @RequestParam(defaul } VoidResponse response = new VoidResponse(); - ModelList models = listModelNames(modelTableName, currentUser); - if (models.getNames().contains(model.getName())) { - // the model already exists -- nothing to do - return response; - } - - BatchWriter writer = null; + List mutations = model.getFields().stream().map(mapping -> ModelKeyParser.createMutation(mapping, model.getName())) + .collect(Collectors.toList()); - try { - writer = accumloConnectionService.getDefaultBatchWriter(modelTableName, model.getName(), currentUser); - } catch (TableNotFoundException e) { - log.error("The " + modelTableName + " could not be found to write to ", e); - QueryException qe = new QueryException(DatawaveErrorCode.TABLE_NOT_FOUND, e); - response.addException(qe.getBottomQueryException()); - } - - if (writer != null) { - for (FieldMapping mapping : model.getFields()) { - Mutation m = ModelKeyParser.createMutation(mapping, model.getName()); - try { - writer.addMutation(m); - } catch (MutationsRejectedException e) { - // So that we know exactly which mapping causes the error, catch here, but break out of loop - // Unfortunately, this does mean if there are multiple mappings that cause this error, only the first will be logged. - log.error("Could not insert mapping -- " + mapping, e); - QueryException qe = new QueryException(DatawaveErrorCode.INSERT_MAPPING_ERROR, e); - response.addException(qe.getBottomQueryException()); - } finally { - // we know the writer isn't null since we've already checked. - try { - writer.close(); - } catch (MutationsRejectedException e) { - log.error("Error closing the BatchWriter; ", e); - } - } - } + QueryException exception = accumloConnectionService.modifyMappings(mutations, modelTableName, model.getName(), currentUser); + if (exception != null) { + response.addException(exception.getBottomQueryException()); } // Do we already have an AccumuloTableCache bean somewhere? @@ -288,66 +245,21 @@ public VoidResponse insertMapping(@RequestBody Model model, @RequestParam(defaul */ @DeleteMapping("/delete") @Secured({"Administrator", "JBossAdministrator"}) - public VoidResponse deleteMapping(@RequestParam Model model, @RequestParam String modelTableName, @AuthenticationPrincipal ProxiedUserDetails currentUser) { + public VoidResponse deleteMapping(@RequestBody Model model, @RequestParam(defaultValue = DEFAULT_MODEL_TABLE_NAME) String modelTableName, + @AuthenticationPrincipal ProxiedUserDetails currentUser) { if (log.isDebugEnabled()) { log.debug("Deleting model name: " + model.getName() + "from modelTableName " + modelTableName); } VoidResponse response = new VoidResponse(); - BatchWriter writer = null; - try { - writer = accumloConnectionService.getDefaultBatchWriter(modelTableName, model.getName(), currentUser); - } catch (TableNotFoundException e) { - log.error("The " + modelTableName + " could not be found to write to ", e); - QueryException qe = new QueryException(DatawaveErrorCode.TABLE_NOT_FOUND, e); - response.addException(qe.getBottomQueryException()); - } + List mutations = model.getFields().stream().map(mapping -> ModelKeyParser.createDeleteMutation(mapping, model.getName())) + .collect(Collectors.toList()); - if (writer != null) { - for (FieldMapping mapping : model.getFields()) { - Mutation m = ModelKeyParser.createDeleteMutation(mapping, model.getName()); - try { - writer.addMutation(m); - } catch (MutationsRejectedException e) { - // So that we know exactly which mapping causes the error, catch here, but break out of loop - // Unfortunately, this does mean if there are multiple mappings that cause this error, only the first will be logged. - log.error("Could not delete mapping -- " + mapping, e); - QueryException qe = new QueryException(DatawaveErrorCode.INSERT_MAPPING_ERROR, e); - response.addException(qe.getBottomQueryException()); - } finally { - // we know the writer isn't null since we've already checked. - try { - writer.close(); - } catch (MutationsRejectedException e) { - log.error("Error closing the BatchWriter; ", e); - } - } - } + QueryException exception = accumloConnectionService.modifyMappings(mutations, modelTableName, model.getName(), currentUser); + if (exception != null) { + response.addException(exception.getBottomQueryException()); } return response; } - - // public String getCurrentUserDN() { - // String currentUserDN = null; - // Principal p = ctx.getCallerPrincipal(); - // - // if (p != null && p instanceof DatawavePrincipal) { - // currentUserDN = ((DatawavePrincipal) p).getUserDN().subjectDN(); - // } - // - // return currentUserDN; - // } - // - // public Collection getCurrentProxyServers() { - // Set currentProxyServers = null; - // Principal p = ctx.getCallerPrincipal(); - // - // if (p != null && p instanceof DatawavePrincipal) { - // currentProxyServers = ((DatawavePrincipal) p).getProxyServers(); - // } - // - // return currentProxyServers; - // } - } diff --git a/service/src/test/java/datawave/microservice/model/ModelControllerTest.java b/service/src/test/java/datawave/microservice/model/ModelControllerTest.java index 2a6466d..bb1ceca 100644 --- a/service/src/test/java/datawave/microservice/model/ModelControllerTest.java +++ b/service/src/test/java/datawave/microservice/model/ModelControllerTest.java @@ -1,14 +1,8 @@ package datawave.microservice.model; -import com.fasterxml.jackson.dataformat.xml.XmlMapper; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.Multimap; import datawave.accumulo.inmemory.InMemoryInstance; -import datawave.marking.MarkingFunctions; import datawave.microservice.ControllerIT; import datawave.microservice.model.config.ModelProperties; -import datawave.webservice.dictionary.data.DefaultDescription; -import datawave.webservice.dictionary.data.DefaultFields; import datawave.webservice.model.FieldMapping; import datawave.webservice.model.Model; import datawave.webservice.model.ModelKeyParser; @@ -24,7 +18,6 @@ import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.data.Mutation; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -34,31 +27,22 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.core.io.Resource; -import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.RequestEntity; import org.springframework.http.ResponseEntity; -import org.springframework.util.LinkedMultiValueMap; -import org.springframework.util.MultiValueMap; import org.springframework.web.util.UriComponents; import org.springframework.web.util.UriComponentsBuilder; import javax.xml.bind.JAXBContext; import javax.xml.bind.Unmarshaller; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.net.URL; -import java.util.AbstractMap; -import java.util.HashMap; -import java.util.Map; +import java.util.HashSet; import java.util.concurrent.TimeUnit; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; public class ModelControllerTest extends ControllerIT { @@ -139,7 +123,8 @@ public void testList() { } @Test - public void testImport() { + public void testImportAKAInsert() { + // the "/insert" and "/import" paths are the same endpoint, so this test effectively tests both // Verify that there is only one model so far (the model that is seeded before the test) // @formatter:off UriComponents uri = UriComponentsBuilder.newInstance() @@ -180,6 +165,50 @@ public void testImport() { assertEquals(2, modelListResponse2.getBody().getNames().size()); } + @Test + public void testClone() { + // Verify that there is only one model so far (the model that is seeded before the test) + // @formatter:off + UriComponents uri = UriComponentsBuilder.newInstance() + .scheme("https").host("localhost").port(webServicePort) + .path("/dictionary/model/list") + .build(); + // @formatter:on + + ResponseEntity modelListResponse = jwtRestTemplate.exchange(adminUser, HttpMethod.GET, uri, ModelList.class); + assertEquals(HttpStatus.OK, modelListResponse.getStatusCode()); + assertEquals(1, modelListResponse.getBody().getNames().size()); + + // clone it and name the clone "TEST_MODEL" + String newName = "TEST_MODEL"; + // @formatter:off + uri = UriComponentsBuilder.newInstance() + .scheme("https").host("localhost").port(webServicePort) + .path("/dictionary/model/clone") + .queryParam("name", MODEL_ONE.getName()) + .queryParam("newName", newName) + .build(); + // @formatter:on + + HttpHeaders additionalHeaders = new HttpHeaders(); + additionalHeaders.set(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE); + RequestEntity postEntity = jwtRestTemplate.createRequestEntity(adminUser, MODEL_TWO, additionalHeaders, HttpMethod.POST, uri); + ResponseEntity imprtResponse = jwtRestTemplate.exchange(postEntity, VoidResponse.class); + + assertEquals(HttpStatus.OK, imprtResponse.getStatusCode()); + + // There should now be 2 models.... + uri = UriComponentsBuilder.newInstance().scheme("https").host("localhost").port(webServicePort).path("/dictionary/model/list").build(); + // @formatter:on + + ResponseEntity modelListResponse2 = jwtRestTemplate.exchange(adminUser, HttpMethod.GET, uri, ModelList.class); + assertEquals(HttpStatus.OK, modelListResponse2.getStatusCode()); + HashSet models = modelListResponse2.getBody().getNames(); + assertEquals(2, models.size()); + assertTrue(models.contains(MODEL_ONE.getName())); + assertTrue(models.contains(newName)); + } + @Test public void testGet() { // Verify that there is only one model so far (the model that is seeded before the test) @@ -207,7 +236,7 @@ public void testGet() { assertEquals(MODEL_ONE.getName(), modelResponse.getBody().getName()); } - // @Test + @Test public void testDelete() { // Verify that there is only one model so far (the model that is seeded before the test) // @formatter:off @@ -228,5 +257,18 @@ public void testDelete() { .path("/dictionary/model/delete") .build(); // @formatter:on + HttpHeaders additionalHeaders = new HttpHeaders(); + additionalHeaders.set(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE); + RequestEntity deleteEntity = jwtRestTemplate.createRequestEntity(adminUser, MODEL_ONE, additionalHeaders, HttpMethod.DELETE, uri); + ResponseEntity imprtResponse = jwtRestTemplate.exchange(deleteEntity, VoidResponse.class); + assertEquals(HttpStatus.OK, imprtResponse.getStatusCode()); + + // Verify that it was deleted + uri = UriComponentsBuilder.newInstance().scheme("https").host("localhost").port(webServicePort).path("/dictionary/model/list").build(); + // @formatter:on + + ResponseEntity modelListResponse2 = jwtRestTemplate.exchange(adminUser, HttpMethod.GET, uri, ModelList.class); + assertEquals(HttpStatus.OK, modelListResponse.getStatusCode()); + assertTrue(modelListResponse2.getBody().getNames().isEmpty()); } } From 97ad6f8f36574094ceda28371c9ddaab183d78b7 Mon Sep 17 00:00:00 2001 From: cmitchell Date: Fri, 6 May 2022 13:06:15 -0400 Subject: [PATCH 21/54] Final cleanup. --- .../AccumuloConnectionService.java | 15 ++++++++-- .../dictionary/DataDictionaryController.java | 30 +++++++------------ .../dictionary/EdgeDictionaryController.java | 29 +++++------------- 3 files changed, 32 insertions(+), 42 deletions(-) diff --git a/service/src/main/java/datawave/microservice/AccumuloConnectionService.java b/service/src/main/java/datawave/microservice/AccumuloConnectionService.java index 365b444..c97a1cd 100644 --- a/service/src/main/java/datawave/microservice/AccumuloConnectionService.java +++ b/service/src/main/java/datawave/microservice/AccumuloConnectionService.java @@ -60,7 +60,18 @@ public AccumuloConnectionService(DataDictionaryProperties dataDictionaryConfigur } /** - * Returns a Connection representing the connection to accumulo + * Return a basic connection to accumulo (as configured by spring) + * + * @return a basic connection to accumulo + */ + public Connection getConnection() { + Connection connection = new Connection(); + connection.setConnector(accumuloConnector); + return connection; + } + + /** + * Returns a Connection representing the connection to accumulo configured with the specified parameters * * @param modelTable * the name of the model table to use in the connection @@ -76,7 +87,7 @@ public Connection getConnection(String modelTable, String modelName, ProxiedUser } /** - * Returns a Connection representing the connection to accumulo + * Returns a Connection representing the connection to accumulo configured with the specified parameters * * @param metadataTable * the name of the metadata table to use in the connection diff --git a/service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java b/service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java index 6731cf3..2a6a63f 100644 --- a/service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java +++ b/service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java @@ -5,30 +5,30 @@ import com.google.common.collect.Maps; import com.google.common.collect.Multimap; import datawave.microservice.AccumuloConnectionService; +import datawave.microservice.Connection; import datawave.microservice.authorization.user.ProxiedUserDetails; import datawave.microservice.dictionary.config.DataDictionaryProperties; import datawave.microservice.dictionary.config.ResponseObjectFactory; -import datawave.microservice.Connection; import datawave.microservice.dictionary.data.DataDictionary; -import datawave.webservice.metadata.MetadataFieldBase; -import datawave.webservice.result.VoidResponse; import datawave.webservice.dictionary.data.DataDictionaryBase; import datawave.webservice.dictionary.data.DescriptionBase; import datawave.webservice.dictionary.data.DictionaryFieldBase; import datawave.webservice.dictionary.data.FieldsBase; +import datawave.webservice.metadata.MetadataFieldBase; +import datawave.webservice.result.VoidResponse; import org.apache.commons.lang.StringUtils; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.http.MediaType; import org.springframework.security.access.annotation.Secured; import org.springframework.security.core.annotation.AuthenticationPrincipal; import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; import java.util.Arrays; @@ -52,8 +52,6 @@ public class DataDictionaryController,DICT ex private final DataDictionary dataDictionary; private final ResponseObjectFactory responseObjectFactory; private final AccumuloConnectionService accumuloConnectionService; - // private final UserAuthFunctions userAuthFunctions; - // private final Connector accumuloConnector; private final Consumer TRANSFORM_EMPTY_INTERNAL_FIELD_NAMES = meta -> { if (meta.getInternalFieldName() == null || meta.getInternalFieldName().isEmpty()) { @@ -70,8 +68,7 @@ public DataDictionaryController(DataDictionaryProperties dataDictionaryConfigura dataDictionary.setNormalizationMap(dataDictionaryConfiguration.getNormalizerMap()); } - @ResponseBody - @RequestMapping(path = "/") + @GetMapping("/") @Timed(name = "dw.dictionary.data.get", absolute = true) public DataDictionaryBase get(@RequestParam(required = false) String modelName, @RequestParam(required = false) String modelTableName, @RequestParam(required = false) String metadataTableName, @RequestParam(name = "auths", required = false) String queryAuthorizations, @@ -106,7 +103,6 @@ public DataDictionaryBase get(@RequestParam(required = false) String * @throws Exception * if there is any problem uploading the descriptions */ - @ResponseBody @PostMapping(path = "/Descriptions", consumes = {MediaType.APPLICATION_XML_VALUE, MediaType.TEXT_XML_VALUE, MediaType.APPLICATION_JSON_VALUE}) @Timed(name = "dw.dictionary.data.uploadDescriptions", absolute = true) public VoidResponse uploadDescriptions(@RequestBody FIELDS fields, @RequestParam(required = false) String modelName, @@ -143,8 +139,7 @@ public VoidResponse uploadDescriptions(@RequestBody FIELDS fields, @RequestParam * if there is any problem updating the description */ @Secured({"Administrator", "JBossAdministrator"}) - @PutMapping(path = "/Descriptions/{datatype}/{fieldName}/{description}", produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE, - MediaType.TEXT_XML_VALUE, PROTOSTUFF_VALUE, "text/x-yaml", "application/x-yaml"}) + @PutMapping("/Descriptions/{datatype}/{fieldName}/{description}") @Timed(name = "dw.dictionary.data.setDescriptionPut", absolute = true) public VoidResponse setDescriptionPut(@PathVariable String fieldName, @PathVariable String datatype, @PathVariable String description, @RequestParam(required = false) String modelName, @RequestParam(required = false) String modelTable, @RequestParam String columnVisibility, @@ -174,8 +169,7 @@ public VoidResponse setDescriptionPut(@PathVariable String fieldName, @PathVaria * if there is any problem updating the dictionary item description */ @Secured({"Administrator", "JBossAdministrator"}) - @PostMapping(path = "/Descriptions", produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.TEXT_XML_VALUE, - PROTOSTUFF_VALUE, "text/x-yaml", "application/x-yaml"}) + @PostMapping("/Descriptions") @Timed(name = "dw.dictionary.data.setDescriptionPost", absolute = true) public VoidResponse setDescriptionPost(@RequestParam String fieldName, @RequestParam String datatype, @RequestParam String description, @RequestParam(required = false) String modelName, @RequestParam(required = false) String modelTable, @RequestParam String columnVisibility, @@ -206,8 +200,7 @@ public VoidResponse setDescriptionPost(@RequestParam String fieldName, @RequestP * @throws Exception * if there is any problem retrieving the descriptions from Accumulo */ - @RequestMapping(path = "/Descriptions") - @ResponseBody + @GetMapping("/Descriptions") @Timed(name = "dw.dictionary.data.allDescriptions", absolute = true) public FIELDS allDescriptions(@RequestParam(required = false) String modelName, @RequestParam(required = false) String modelTable, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { @@ -233,7 +226,7 @@ public FIELDS allDescriptions(@RequestParam(required = false) String modelName, * @throws Exception * if there is any problem retrieving the descriptions from Accumulo */ - @RequestMapping(path = "/Descriptions/{datatype}") + @GetMapping("/Descriptions/{datatype}") @Timed(name = "dw.dictionary.data.datatypeDescriptions", absolute = true) public FIELDS datatypeDescriptions(@PathVariable("datatype") String datatype, @RequestParam(required = false) String modelName, @RequestParam(required = false) String modelTable, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { @@ -261,7 +254,7 @@ public FIELDS datatypeDescriptions(@PathVariable("datatype") String datatype, @R * @throws Exception * if there is any problem retrieving the descriptions from Accumulo */ - @RequestMapping(path = "/Descriptions/{datatype}/{fieldName}") + @GetMapping("/Descriptions/{datatype}/{fieldName}") @Timed(name = "dw.dictionary.data.fieldNameDescription", absolute = true) public FIELDS fieldNameDescription(@PathVariable String fieldName, @PathVariable String datatype, @RequestParam(required = false) String modelName, @RequestParam(required = false) String modelTable, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { @@ -298,8 +291,7 @@ public FIELDS fieldNameDescription(@PathVariable String fieldName, @PathVariable * if there is any problem removing the description from Accumulo */ @Secured({"Administrator", "JBossAdministrator"}) - @DeleteMapping(path = "/Descriptions/{datatype}/{fieldName}", produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE, - MediaType.TEXT_XML_VALUE, PROTOSTUFF_VALUE, "text/x-yaml", "application/x-yaml"}) + @DeleteMapping("/Descriptions/{datatype}/{fieldName}") @Timed(name = "dw.dictionary.data.deleteDescription", absolute = true) public VoidResponse deleteDescription(@PathVariable String fieldName, @PathVariable String datatype, @RequestParam(required = false) String modelName, @RequestParam(required = false) String modelTable, @RequestParam String columnVisibility, diff --git a/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java b/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java index 0d7c0b4..d62abbb 100644 --- a/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java +++ b/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java @@ -2,27 +2,22 @@ import com.codahale.metrics.annotation.Timed; import datawave.accumulo.util.security.UserAuthFunctions; +import datawave.microservice.AccumuloConnectionService; import datawave.microservice.authorization.user.ProxiedUserDetails; import datawave.microservice.dictionary.config.EdgeDictionaryProperties; import datawave.microservice.dictionary.edge.EdgeDictionary; -import datawave.security.authorization.DatawaveUser; import datawave.webservice.dictionary.edge.EdgeDictionaryBase; import datawave.webservice.dictionary.edge.MetadataBase; import lombok.extern.slf4j.Slf4j; -import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.security.Authorizations; import org.apache.commons.lang.StringUtils; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.http.MediaType; import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; -import java.util.Set; - import static datawave.microservice.http.converter.protostuff.ProtostuffHttpMessageConverter.PROTOSTUFF_VALUE; @Slf4j @@ -35,14 +30,14 @@ public class EdgeDictionaryController private final EdgeDictionaryProperties edgeDictionaryProperties; private final EdgeDictionary edgeDictionary; private final UserAuthFunctions userAuthFunctions; - private final Connector accumuloConnector; + private final AccumuloConnectionService accumuloConnectionService; public EdgeDictionaryController(EdgeDictionaryProperties edgeDictionaryProperties, EdgeDictionary edgeDictionary, - UserAuthFunctions userAuthFunctions, @Qualifier("warehouse") Connector accumuloConnector) { + UserAuthFunctions userAuthFunctions, AccumuloConnectionService accumloConnectionService) { this.edgeDictionaryProperties = edgeDictionaryProperties; this.edgeDictionary = edgeDictionary; this.userAuthFunctions = userAuthFunctions; - this.accumuloConnector = accumuloConnector; + this.accumuloConnectionService = accumloConnectionService; } /** @@ -56,8 +51,7 @@ public EdgeDictionaryController(EdgeDictionaryProperties edgeDictionaryPropertie * @throws Exception * if there is any problem retrieving the edge dictionary from Accumulo */ - @ResponseBody - @RequestMapping(path = "/") + @GetMapping("/") @Timed(name = "dw.dictionary.edge.get", absolute = true) public EdgeDictionaryBase get(@RequestParam(required = false) String metadataTableName, @RequestParam(name = "auths", required = false) String queryAuthorizations, @AuthenticationPrincipal ProxiedUserDetails currentUser) @@ -67,18 +61,11 @@ public EdgeDictionaryBase get(@RequestParam(required = false) String metadataTableName = edgeDictionaryProperties.getMetadataTableName(); } - // If the user provides authorizations, intersect it with their actual authorizations - Set auths = getDowngradedAuthorizations(queryAuthorizations, currentUser); - - EDGE edgeDict = edgeDictionary.getEdgeDictionary(metadataTableName, accumuloConnector, auths, edgeDictionaryProperties.getNumThreads()); + EDGE edgeDict = edgeDictionary.getEdgeDictionary(metadataTableName, accumuloConnectionService.getConnection().getConnector(), + accumuloConnectionService.getDowngradedAuthorizations(queryAuthorizations, currentUser), edgeDictionaryProperties.getNumThreads()); log.info("EDGEDICTIONARY: returning edge dictionary"); return edgeDict; } - private Set getDowngradedAuthorizations(String requestedAuthorizations, ProxiedUserDetails currentUser) { - DatawaveUser primaryUser = currentUser.getPrimaryUser(); - return userAuthFunctions.mergeAuthorizations(userAuthFunctions.getRequestedAuthorizations(requestedAuthorizations, currentUser.getPrimaryUser()), - currentUser.getProxiedUsers(), u -> u != primaryUser); - } } From 7053d54a55e71c1e49b3c76d3efa8434453e1f82 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Mon, 9 May 2022 16:12:25 +0000 Subject: [PATCH 22/54] Removed duplicate bean configuration Updated versions of curator, zookeeper, and ensure in-memory accumulo is included --- service/pom.xml | 48 ++++++++++++++----- .../model/config/ModelConfiguration.java | 18 ------- 2 files changed, 36 insertions(+), 30 deletions(-) delete mode 100644 service/src/main/java/datawave/microservice/model/config/ModelConfiguration.java diff --git a/service/pom.xml b/service/pom.xml index 7b69f70..d875886 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -20,13 +20,15 @@ datawave.microservice.dictionary.DictionaryService 1.9.3 + 5.2.0 + 1.9.3-SNAPSHOT 1.3-SNAPSHOT 1.6.8-SNAPSHOT 1.0-SNAPSHOT 1.10.25 3.6.0 0.50 - 3.4.14 + 3.6.3 @@ -40,6 +42,20 @@ jaxb-impl 2.3.3 + + gov.nsa.datawave.contrib + datawave-in-memory-accumulo + ${version.in-memory-accumulo} + compile + + + + + * + * + + + gov.nsa.datawave.microservice dictionary-api @@ -74,6 +90,21 @@ + + org.apache.curator + curator-client + ${version.curator} + + + org.apache.curator + curator-framework + ${version.curator} + + + org.apache.curator + curator-recipes + ${version.curator} + org.apache.zookeeper zookeeper @@ -112,6 +143,10 @@ com.sun.xml.bind jaxb-impl + + gov.nsa.datawave.contrib + datawave-in-memory-accumulo + gov.nsa.datawave.microservice dictionary-api @@ -145,17 +180,6 @@ lombok provided - - gov.nsa.datawave.contrib - datawave-in-memory-accumulo - test - - - slf4j-reload4j - org.slf4j - - - org.junit.jupiter junit-jupiter-engine diff --git a/service/src/main/java/datawave/microservice/model/config/ModelConfiguration.java b/service/src/main/java/datawave/microservice/model/config/ModelConfiguration.java deleted file mode 100644 index 62b06a4..0000000 --- a/service/src/main/java/datawave/microservice/model/config/ModelConfiguration.java +++ /dev/null @@ -1,18 +0,0 @@ -package datawave.microservice.model.config; - -import datawave.microservice.config.accumulo.AccumuloProperties; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -@EnableConfigurationProperties(ModelProperties.class) -public class ModelConfiguration { - @Bean - @Qualifier("warehouse") - public AccumuloProperties warehouseAccumuloProperties() { - return new AccumuloProperties(); - } - -} From cff8081ba8900f1c236fa61fc9424af65646b2de Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Wed, 11 May 2022 14:08:31 +0000 Subject: [PATCH 23/54] Disable unused table cache for the dictionary service --- service/src/main/resources/config/bootstrap.yml | 5 +++++ service/src/test/resources/config/bootstrap.yml | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/service/src/main/resources/config/bootstrap.yml b/service/src/main/resources/config/bootstrap.yml index 2714ce7..ac159d7 100644 --- a/service/src/main/resources/config/bootstrap.yml +++ b/service/src/main/resources/config/bootstrap.yml @@ -13,6 +13,11 @@ spring: max-attempts: 60 uri: '${CONFIG_SERVER_URL:http://configuration:8888/configserver}' +datawave: + table: + cache: + enabled: false + --- # For the dev profile, check localhost for the config server by default diff --git a/service/src/test/resources/config/bootstrap.yml b/service/src/test/resources/config/bootstrap.yml index dd676bb..3bcd48c 100644 --- a/service/src/test/resources/config/bootstrap.yml +++ b/service/src/test/resources/config/bootstrap.yml @@ -12,3 +12,9 @@ spring: enabled: false main: banner-mode: "OFF" + +datawave: + table: + cache: + enabled: false + From 5909a586916c5f5b064605f59be23e25fcb0a177 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 12 May 2022 13:27:05 -0400 Subject: [PATCH 24/54] Set relative path for full build --- api/pom.xml | 2 +- service/pom.xml | 6 +++--- .../microservice/AccumuloConnectionService.java | 12 ++++++------ .../dictionary/DataDictionaryController.java | 5 +++-- .../dictionary/EdgeDictionaryController.java | 5 +++-- .../microservice/model/ModelController.java | 17 +++++++++-------- 6 files changed, 25 insertions(+), 22 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index aba2c44..69c879a 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -5,7 +5,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent 1.9-SNAPSHOT - + ../../../microservice-parent/pom.xml dictionary-api 1.3-SNAPSHOT diff --git a/service/pom.xml b/service/pom.xml index d875886..82dc82c 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -5,7 +5,7 @@ gov.nsa.datawave.microservice datawave-microservice-service-parent 2.1-SNAPSHOT - + ../../../microservice-service-parent/pom.xml dictionary-service 1.3-SNAPSHOT @@ -43,7 +43,7 @@ 2.3.3 - gov.nsa.datawave.contrib + gov.nsa.datawave datawave-in-memory-accumulo ${version.in-memory-accumulo} compile @@ -144,7 +144,7 @@ jaxb-impl - gov.nsa.datawave.contrib + gov.nsa.datawave datawave-in-memory-accumulo diff --git a/service/src/main/java/datawave/microservice/AccumuloConnectionService.java b/service/src/main/java/datawave/microservice/AccumuloConnectionService.java index c97a1cd..512ea5a 100644 --- a/service/src/main/java/datawave/microservice/AccumuloConnectionService.java +++ b/service/src/main/java/datawave/microservice/AccumuloConnectionService.java @@ -79,7 +79,7 @@ public Connection getConnection() { * the name of the model to use in the connection * @param user * the user (and the authorities and other security aspects related to the user) - * + * * @return a Connection representing the connection to accumulo */ public Connection getConnection(String modelTable, String modelName, ProxiedUserDetails user) { @@ -97,7 +97,7 @@ public Connection getConnection(String modelTable, String modelName, ProxiedUser * the name of the model to use in the connection * @param user * the user (and the authorities and other security aspects related to the user) - * + * * @return a Connection representing the connection to accumulo */ public Connection getConnection(String metadataTable, String modelTable, String modelName, ProxiedUserDetails user) { @@ -119,7 +119,7 @@ private String getSupplierValueIfBlank(final String value, final Supplier getAuths(ProxiedUserDetails currentUser) { @@ -138,7 +138,7 @@ public Set getAuths(ProxiedUserDetails currentUser) { * the level of authorizations to downgrade to * @param currentUser * the user for whom to downgrade the authorizations - * + * * @return the authorizations for the specified user downgraded to the specified level */ public Set getDowngradedAuthorizations(String requestedAuthorizations, ProxiedUserDetails currentUser) { @@ -157,7 +157,7 @@ public Set getDowngradedAuthorizations(String requestedAuthoriza * the user requesting the keys. Used for authorizations purposes * @param regexTerm * the regex (can be just a string) to apply to the IteratorSetting - * + * * @return the keys of hte specified table that the specified user has the authorzations to access. * @throws TableNotFoundException * Thrown if the table is not found @@ -185,7 +185,7 @@ public List getKeys(String modelTable, ProxiedUserDetails currentUser, Stri * the model name * @param user * the user requesting the changes - * + * * @return A QueryException if anything is unsuccessful, and null if everything is successful. */ public QueryException modifyMappings(List mutations, String modelTable, String modelName, ProxiedUserDetails user) { diff --git a/service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java b/service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java index 2a6a63f..46f8c7a 100644 --- a/service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java +++ b/service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java @@ -43,8 +43,9 @@ import static datawave.microservice.http.converter.protostuff.ProtostuffHttpMessageConverter.PROTOSTUFF_VALUE; @RestController -@RequestMapping(path = "/data/v1", produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.TEXT_XML_VALUE, PROTOSTUFF_VALUE, - MediaType.TEXT_HTML_VALUE, "text/x-yaml", "application/x-yaml"}) +@RequestMapping(path = "/data/v1", + produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.TEXT_XML_VALUE, PROTOSTUFF_VALUE, + MediaType.TEXT_HTML_VALUE, "text/x-yaml", "application/x-yaml"}) @EnableConfigurationProperties(DataDictionaryProperties.class) public class DataDictionaryController,DICT extends DataDictionaryBase,META extends MetadataFieldBase,FIELD extends DictionaryFieldBase,FIELDS extends FieldsBase> { diff --git a/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java b/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java index d62abbb..301ab37 100644 --- a/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java +++ b/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java @@ -22,8 +22,9 @@ @Slf4j @RestController -@RequestMapping(path = "/edge/v1", produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.TEXT_XML_VALUE, PROTOSTUFF_VALUE, - MediaType.TEXT_HTML_VALUE, "text/x-yaml", "application/x-yaml"}) +@RequestMapping(path = "/edge/v1", + produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.TEXT_XML_VALUE, PROTOSTUFF_VALUE, + MediaType.TEXT_HTML_VALUE, "text/x-yaml", "application/x-yaml"}) @EnableConfigurationProperties(EdgeDictionaryProperties.class) public class EdgeDictionaryController,META extends MetadataBase> { diff --git a/service/src/main/java/datawave/microservice/model/ModelController.java b/service/src/main/java/datawave/microservice/model/ModelController.java index 44290a1..241a84a 100644 --- a/service/src/main/java/datawave/microservice/model/ModelController.java +++ b/service/src/main/java/datawave/microservice/model/ModelController.java @@ -40,8 +40,9 @@ */ @Slf4j @RestController -@RequestMapping(path = "/model", produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.TEXT_XML_VALUE, - ProtostuffHttpMessageConverter.PROTOSTUFF_VALUE, MediaType.TEXT_HTML_VALUE, "text/x-yaml", "application/x-yaml"}) +@RequestMapping(path = "/model", + produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.TEXT_XML_VALUE, + ProtostuffHttpMessageConverter.PROTOSTUFF_VALUE, MediaType.TEXT_HTML_VALUE, "text/x-yaml", "application/x-yaml"}) @Secured({"AuthorizedUser", "AuthorizedQueryServer", "InternalUser", "Administrator", "JBossAdministrator"}) @EnableConfigurationProperties(ModelProperties.class) public class ModelController { @@ -66,7 +67,7 @@ public ModelController(ModelProperties modelProperties, AccumuloConnectionServic * name of the table that contains the model * @return ModelList * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user - * + * * @HTTP 200 success * @HTTP 500 internal server error */ @@ -111,7 +112,7 @@ public ModelList listModelNames(@RequestParam(defaultValue = DEFAULT_MODEL_TABLE * name of the table that contains the model * @return datawave.webservice.result.VoidResponse * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user - * + * * @HTTP 200 success * @HTTP 404 model not found * @HTTP 500 internal server error @@ -142,7 +143,7 @@ public VoidResponse deleteModel(@RequestParam String name, @RequestParam(default * name of the table that contains the model * @return datawave.webservice.result.VoidResponse * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user - * + * * @HTTP 200 success * @HTTP 204 model not found * @HTTP 500 internal server error @@ -169,7 +170,7 @@ public VoidResponse cloneModel(@RequestParam String name, @RequestParam String n * name of the table that contains the model * @return Model * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user - * + * * @HTTP 200 success * @HTTP 404 model not found * @HTTP 500 internal server error @@ -204,7 +205,7 @@ public Model getModel(@RequestParam String name, @RequestParam(defaultValue = DE * name of the table that contains the model * @return datawave.webservice.result.VoidResponse * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user - * + * * @HTTP 200 success * @HTTP 500 internal server error */ @@ -239,7 +240,7 @@ public VoidResponse insertMapping(@RequestBody Model model, @RequestParam(defaul * name of the table that contains the model * @return datawave.webservice.result.VoidResponse * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user - * + * * @HTTP 200 success * @HTTP 500 internal server error */ From 23f325ef554909832bd25ceed3f1bf5cc6cc3cb8 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 20 May 2022 11:05:05 -0400 Subject: [PATCH 25/54] Added an activation property for the docker profile --- service/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/service/pom.xml b/service/pom.xml index 82dc82c..1d094a8 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -220,6 +220,11 @@ docker + + + microservice-docker + + From 44f5b1536730b1330626607c680c21dc22602c7b Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Fri, 10 Jun 2022 14:00:38 +0000 Subject: [PATCH 26/54] Avoid transient log4j dependencies via zookeeper --- service/pom.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/service/pom.xml b/service/pom.xml index 1d094a8..0998a5f 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -109,6 +109,16 @@ org.apache.zookeeper zookeeper ${version.zookeeper} + + + slf4j-log4j12 + org.slf4j + + + log4j + log4j + + org.webjars From 1f9bf051febd73028c85fa98e89856d32bccb310 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 10 Jun 2022 15:20:38 -0400 Subject: [PATCH 27/54] Added a relative parent pom path for the parent pom. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index be8d15c..502a7c8 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent 1.9-SNAPSHOT - + ../../microservice-parent/pom.xml dictionary-service-parent 1.3-SNAPSHOT From ff435d8ed323a6e3183b0ffbb3af2a1d832fff7b Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 16 Jun 2022 15:32:18 -0400 Subject: [PATCH 28/54] cleaned up imports --- .../dictionary/data/DataDictionaryBase.java | 2 +- .../data/DefaultDataDictionary.java | 32 +++++++++---------- .../data/DefaultDictionaryField.java | 19 ++++++----- .../dictionary/data/DefaultFields.java | 4 +-- .../dictionary/data/DescriptionBase.java | 2 +- .../dictionary/data/FieldsBase.java | 2 +- .../edge/DefaultEdgeDictionary.java | 29 ++++++++--------- .../dictionary/edge/DefaultMetadata.java | 25 +++++++-------- .../dictionary/edge/EdgeDictionaryBase.java | 7 ++-- .../dictionary/edge/EventField.java | 20 ++++++------ .../dictionary/edge/MetadataBase.java | 11 +++---- .../metadata/DefaultMetadataField.java | 2 +- .../metadata/MetadataFieldBase.java | 2 +- .../datawave/webservice/model/Direction.java | 3 +- .../datawave/webservice/model/Mapping.java | 9 +++--- .../webservice/model/ModelKeyParser.java | 1 - .../DictionaryServiceConfiguration.java | 2 +- .../config/ResponseObjectFactory.java | 2 +- .../dictionary/data/DataDictionary.java | 2 +- .../dictionary/data/DataDictionaryImpl.java | 2 +- .../metadata/DefaultMetadataFieldScanner.java | 4 +-- .../model/config/ModelProperties.java | 1 - .../DataDictionaryControllerTest.java | 2 +- .../data/DataDictionaryImplTest.java | 2 +- .../edge/EdgeDictionaryImplTest.java | 3 +- .../DefaultMetadataFieldScannerTest.java | 4 +-- 26 files changed, 91 insertions(+), 103 deletions(-) diff --git a/api/src/main/java/datawave/webservice/dictionary/data/DataDictionaryBase.java b/api/src/main/java/datawave/webservice/dictionary/data/DataDictionaryBase.java index 2c573ca..4b207f2 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/DataDictionaryBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DataDictionaryBase.java @@ -1,10 +1,10 @@ package datawave.webservice.dictionary.data; -import io.protostuff.Message; import datawave.webservice.HtmlProvider; import datawave.webservice.metadata.MetadataFieldBase; import datawave.webservice.result.BaseResponse; import datawave.webservice.result.TotalResultsAware; +import io.protostuff.Message; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; diff --git a/api/src/main/java/datawave/webservice/dictionary/data/DefaultDataDictionary.java b/api/src/main/java/datawave/webservice/dictionary/data/DefaultDataDictionary.java index 98017d0..31ed9d0 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/DefaultDataDictionary.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DefaultDataDictionary.java @@ -1,12 +1,13 @@ package datawave.webservice.dictionary.data; -import java.io.IOException; -import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.function.Consumer; +import com.google.common.collect.Lists; +import datawave.webservice.HtmlProvider; +import datawave.webservice.metadata.DefaultMetadataField; +import datawave.webservice.result.TotalResultsAware; +import io.protostuff.Input; +import io.protostuff.Message; +import io.protostuff.Output; +import io.protostuff.Schema; import javax.xml.bind.annotation.XmlAccessOrder; import javax.xml.bind.annotation.XmlAccessType; @@ -16,16 +17,13 @@ import javax.xml.bind.annotation.XmlElementWrapper; import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlTransient; - -import com.google.common.collect.Lists; -import datawave.webservice.HtmlProvider; -import datawave.webservice.metadata.DefaultMetadataField; -import datawave.webservice.result.TotalResultsAware; - -import io.protostuff.Input; -import io.protostuff.Message; -import io.protostuff.Output; -import io.protostuff.Schema; +import java.io.IOException; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.function.Consumer; @XmlRootElement(name = "DefaultDataDictionary") @XmlAccessorType(XmlAccessType.NONE) diff --git a/api/src/main/java/datawave/webservice/dictionary/data/DefaultDictionaryField.java b/api/src/main/java/datawave/webservice/dictionary/data/DefaultDictionaryField.java index 92592ad..8934a10 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/DefaultDictionaryField.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DefaultDictionaryField.java @@ -1,9 +1,10 @@ package datawave.webservice.dictionary.data; -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; -import java.util.Set; +import com.google.common.collect.Sets; +import io.protostuff.Input; +import io.protostuff.Message; +import io.protostuff.Output; +import io.protostuff.Schema; import javax.xml.bind.annotation.XmlAccessOrder; import javax.xml.bind.annotation.XmlAccessType; @@ -11,12 +12,10 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlTransient; - -import io.protostuff.Input; -import io.protostuff.Message; -import io.protostuff.Output; -import io.protostuff.Schema; -import com.google.common.collect.Sets; +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.Set; @XmlAccessorType(XmlAccessType.NONE) @XmlAccessorOrder(XmlAccessOrder.ALPHABETICAL) diff --git a/api/src/main/java/datawave/webservice/dictionary/data/DefaultFields.java b/api/src/main/java/datawave/webservice/dictionary/data/DefaultFields.java index f36eef1..ec5de52 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/DefaultFields.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DefaultFields.java @@ -2,12 +2,12 @@ import com.google.common.collect.Lists; import com.google.common.collect.Multimap; +import datawave.webservice.HtmlProvider; +import datawave.webservice.result.TotalResultsAware; import io.protostuff.Input; import io.protostuff.Message; import io.protostuff.Output; import io.protostuff.Schema; -import datawave.webservice.HtmlProvider; -import datawave.webservice.result.TotalResultsAware; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; diff --git a/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java b/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java index 1b961de..e8545e4 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java @@ -1,8 +1,8 @@ package datawave.webservice.dictionary.data; -import io.protostuff.Message; import datawave.webservice.query.result.event.HasMarkings; import datawave.webservice.query.util.StringMapAdapter; +import io.protostuff.Message; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; diff --git a/api/src/main/java/datawave/webservice/dictionary/data/FieldsBase.java b/api/src/main/java/datawave/webservice/dictionary/data/FieldsBase.java index 8f6672d..6e751de 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/FieldsBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/FieldsBase.java @@ -1,8 +1,8 @@ package datawave.webservice.dictionary.data; import com.google.common.collect.Multimap; -import io.protostuff.Message; import datawave.webservice.result.BaseResponse; +import io.protostuff.Message; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; diff --git a/api/src/main/java/datawave/webservice/dictionary/edge/DefaultEdgeDictionary.java b/api/src/main/java/datawave/webservice/dictionary/edge/DefaultEdgeDictionary.java index 745ae69..a0d90fa 100644 --- a/api/src/main/java/datawave/webservice/dictionary/edge/DefaultEdgeDictionary.java +++ b/api/src/main/java/datawave/webservice/dictionary/edge/DefaultEdgeDictionary.java @@ -1,11 +1,13 @@ package datawave.webservice.dictionary.edge; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; +import datawave.webservice.HtmlProvider; +import datawave.webservice.query.result.util.protostuff.FieldAccessor; +import datawave.webservice.query.result.util.protostuff.ProtostuffField; +import datawave.webservice.result.TotalResultsAware; +import io.protostuff.Input; +import io.protostuff.Message; +import io.protostuff.Output; +import io.protostuff.Schema; import javax.xml.bind.annotation.XmlAccessOrder; import javax.xml.bind.annotation.XmlAccessType; @@ -15,15 +17,12 @@ import javax.xml.bind.annotation.XmlElementWrapper; import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlTransient; - -import io.protostuff.Input; -import io.protostuff.Message; -import io.protostuff.Output; -import io.protostuff.Schema; -import datawave.webservice.HtmlProvider; -import datawave.webservice.query.result.util.protostuff.FieldAccessor; -import datawave.webservice.query.result.util.protostuff.ProtostuffField; -import datawave.webservice.result.TotalResultsAware; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; @XmlRootElement(name = "EdgeDictionary") @XmlAccessorType(XmlAccessType.NONE) diff --git a/api/src/main/java/datawave/webservice/dictionary/edge/DefaultMetadata.java b/api/src/main/java/datawave/webservice/dictionary/edge/DefaultMetadata.java index 52be165..c6759e9 100644 --- a/api/src/main/java/datawave/webservice/dictionary/edge/DefaultMetadata.java +++ b/api/src/main/java/datawave/webservice/dictionary/edge/DefaultMetadata.java @@ -1,9 +1,13 @@ package datawave.webservice.dictionary.edge; -import java.io.IOException; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; +import com.google.common.base.Objects; +import datawave.webservice.query.result.util.protostuff.FieldAccessor; +import datawave.webservice.query.result.util.protostuff.ProtostuffField; +import io.protostuff.Input; +import io.protostuff.Message; +import io.protostuff.Output; +import io.protostuff.Schema; +import org.apache.commons.lang.StringUtils; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -12,15 +16,10 @@ import javax.xml.bind.annotation.XmlElementWrapper; import javax.xml.bind.annotation.XmlTransient; import javax.xml.bind.annotation.XmlType; - -import com.google.common.base.Objects; -import io.protostuff.Input; -import io.protostuff.Message; -import io.protostuff.Output; -import io.protostuff.Schema; -import datawave.webservice.query.result.util.protostuff.FieldAccessor; -import datawave.webservice.query.result.util.protostuff.ProtostuffField; -import org.apache.commons.lang.StringUtils; +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; @XmlAccessorType(XmlAccessType.NONE) @XmlType(propOrder = {"edgeType", "edgeRelationship", "edgeAttribute1Source", "eventFields"}) diff --git a/api/src/main/java/datawave/webservice/dictionary/edge/EdgeDictionaryBase.java b/api/src/main/java/datawave/webservice/dictionary/edge/EdgeDictionaryBase.java index 759122e..886a1d8 100644 --- a/api/src/main/java/datawave/webservice/dictionary/edge/EdgeDictionaryBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/edge/EdgeDictionaryBase.java @@ -1,13 +1,12 @@ package datawave.webservice.dictionary.edge; -import java.util.List; +import datawave.webservice.result.BaseResponse; +import io.protostuff.Message; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlSeeAlso; - -import io.protostuff.Message; -import datawave.webservice.result.BaseResponse; +import java.util.List; @XmlAccessorType(XmlAccessType.NONE) @XmlSeeAlso(DefaultEdgeDictionary.class) diff --git a/api/src/main/java/datawave/webservice/dictionary/edge/EventField.java b/api/src/main/java/datawave/webservice/dictionary/edge/EventField.java index c69db71..11246da 100644 --- a/api/src/main/java/datawave/webservice/dictionary/edge/EventField.java +++ b/api/src/main/java/datawave/webservice/dictionary/edge/EventField.java @@ -1,22 +1,22 @@ package datawave.webservice.dictionary.edge; -import java.io.IOException; -import java.io.Serializable; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlAttribute; -import javax.xml.bind.annotation.XmlTransient; -import javax.xml.bind.annotation.XmlType; - import com.google.common.base.Objects; +import datawave.webservice.query.result.util.protostuff.FieldAccessor; +import datawave.webservice.query.result.util.protostuff.ProtostuffField; import io.protostuff.Input; import io.protostuff.Message; import io.protostuff.Output; import io.protostuff.Schema; -import datawave.webservice.query.result.util.protostuff.FieldAccessor; -import datawave.webservice.query.result.util.protostuff.ProtostuffField; import org.apache.commons.lang.StringUtils; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlTransient; +import javax.xml.bind.annotation.XmlType; +import java.io.IOException; +import java.io.Serializable; + @XmlAccessorType(XmlAccessType.NONE) @XmlType(propOrder = {"sourceField", "sinkField", "enrichmentField", "enrichmentIndex", "jexlPrecondition"}) public class EventField implements Serializable, Message { diff --git a/api/src/main/java/datawave/webservice/dictionary/edge/MetadataBase.java b/api/src/main/java/datawave/webservice/dictionary/edge/MetadataBase.java index 9067d2f..4e5c240 100644 --- a/api/src/main/java/datawave/webservice/dictionary/edge/MetadataBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/edge/MetadataBase.java @@ -1,15 +1,14 @@ package datawave.webservice.dictionary.edge; -import java.util.List; -import java.util.Map; +import com.google.common.collect.Maps; +import datawave.webservice.query.result.event.HasMarkings; +import io.protostuff.Message; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlSeeAlso; - -import com.google.common.collect.Maps; -import io.protostuff.Message; -import datawave.webservice.query.result.event.HasMarkings; +import java.util.List; +import java.util.Map; @XmlAccessorType(XmlAccessType.NONE) @XmlSeeAlso(DefaultMetadata.class) diff --git a/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java b/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java index 9ad382b..396d999 100644 --- a/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java +++ b/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java @@ -1,8 +1,8 @@ package datawave.webservice.metadata; -import datawave.webservice.query.result.event.MapSchema; import datawave.webservice.dictionary.data.DefaultDescription; import datawave.webservice.dictionary.data.DescriptionBase; +import datawave.webservice.query.result.event.MapSchema; import io.protostuff.Input; import io.protostuff.Message; import io.protostuff.Output; diff --git a/api/src/main/java/datawave/webservice/metadata/MetadataFieldBase.java b/api/src/main/java/datawave/webservice/metadata/MetadataFieldBase.java index fc7eaaa..79e3a93 100644 --- a/api/src/main/java/datawave/webservice/metadata/MetadataFieldBase.java +++ b/api/src/main/java/datawave/webservice/metadata/MetadataFieldBase.java @@ -1,7 +1,7 @@ package datawave.webservice.metadata; -import io.protostuff.Message; import datawave.webservice.dictionary.data.DescriptionBase; +import io.protostuff.Message; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; diff --git a/api/src/main/java/datawave/webservice/model/Direction.java b/api/src/main/java/datawave/webservice/model/Direction.java index 3484860..5a425b2 100644 --- a/api/src/main/java/datawave/webservice/model/Direction.java +++ b/api/src/main/java/datawave/webservice/model/Direction.java @@ -1,8 +1,7 @@ package datawave.webservice.model; -import java.io.Serializable; - import javax.xml.bind.annotation.XmlEnum; +import java.io.Serializable; @XmlEnum public enum Direction implements Serializable { diff --git a/api/src/main/java/datawave/webservice/model/Mapping.java b/api/src/main/java/datawave/webservice/model/Mapping.java index ac0d50c..8445c22 100644 --- a/api/src/main/java/datawave/webservice/model/Mapping.java +++ b/api/src/main/java/datawave/webservice/model/Mapping.java @@ -1,14 +1,13 @@ package datawave.webservice.model; -import java.io.Serializable; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.commons.lang.builder.CompareToBuilder; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAttribute; - -import lombok.Data; -import lombok.NoArgsConstructor; -import org.apache.commons.lang.builder.CompareToBuilder; +import java.io.Serializable; @Data @NoArgsConstructor diff --git a/api/src/main/java/datawave/webservice/model/ModelKeyParser.java b/api/src/main/java/datawave/webservice/model/ModelKeyParser.java index 6efd8f8..bd1b322 100644 --- a/api/src/main/java/datawave/webservice/model/ModelKeyParser.java +++ b/api/src/main/java/datawave/webservice/model/ModelKeyParser.java @@ -1,6 +1,5 @@ package datawave.webservice.model; -import io.netty.util.Mapping; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; diff --git a/service/src/main/java/datawave/microservice/dictionary/config/DictionaryServiceConfiguration.java b/service/src/main/java/datawave/microservice/dictionary/config/DictionaryServiceConfiguration.java index 7eee282..6175a5f 100644 --- a/service/src/main/java/datawave/microservice/dictionary/config/DictionaryServiceConfiguration.java +++ b/service/src/main/java/datawave/microservice/dictionary/config/DictionaryServiceConfiguration.java @@ -12,12 +12,12 @@ import datawave.microservice.metadata.MetadataDescriptionsHelper; import datawave.microservice.metadata.MetadataDescriptionsHelperFactory; import datawave.query.util.MetadataHelperFactory; -import datawave.webservice.metadata.DefaultMetadataField; import datawave.webservice.dictionary.data.DefaultDataDictionary; import datawave.webservice.dictionary.data.DefaultDescription; import datawave.webservice.dictionary.data.DefaultDictionaryField; import datawave.webservice.dictionary.data.DefaultFields; import datawave.webservice.dictionary.data.FieldsBase; +import datawave.webservice.metadata.DefaultMetadataField; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.jackson.Jackson2ObjectMapperBuilderCustomizer; diff --git a/service/src/main/java/datawave/microservice/dictionary/config/ResponseObjectFactory.java b/service/src/main/java/datawave/microservice/dictionary/config/ResponseObjectFactory.java index 9df009f..0505624 100644 --- a/service/src/main/java/datawave/microservice/dictionary/config/ResponseObjectFactory.java +++ b/service/src/main/java/datawave/microservice/dictionary/config/ResponseObjectFactory.java @@ -1,10 +1,10 @@ package datawave.microservice.dictionary.config; -import datawave.webservice.metadata.MetadataFieldBase; import datawave.webservice.dictionary.data.DataDictionaryBase; import datawave.webservice.dictionary.data.DescriptionBase; import datawave.webservice.dictionary.data.DictionaryFieldBase; import datawave.webservice.dictionary.data.FieldsBase; +import datawave.webservice.metadata.MetadataFieldBase; public interface ResponseObjectFactory,DICT extends DataDictionaryBase,META extends MetadataFieldBase,FIELD extends DictionaryFieldBase,FIELDS extends FieldsBase> { DICT getDataDictionary(); diff --git a/service/src/main/java/datawave/microservice/dictionary/data/DataDictionary.java b/service/src/main/java/datawave/microservice/dictionary/data/DataDictionary.java index 49fae88..bfe3e97 100644 --- a/service/src/main/java/datawave/microservice/dictionary/data/DataDictionary.java +++ b/service/src/main/java/datawave/microservice/dictionary/data/DataDictionary.java @@ -2,9 +2,9 @@ import com.google.common.collect.Multimap; import datawave.microservice.Connection; -import datawave.webservice.metadata.MetadataFieldBase; import datawave.webservice.dictionary.data.DescriptionBase; import datawave.webservice.dictionary.data.DictionaryFieldBase; +import datawave.webservice.metadata.MetadataFieldBase; import java.util.Collection; import java.util.Map; diff --git a/service/src/main/java/datawave/microservice/dictionary/data/DataDictionaryImpl.java b/service/src/main/java/datawave/microservice/dictionary/data/DataDictionaryImpl.java index 2c8cc7b..f8a79b1 100644 --- a/service/src/main/java/datawave/microservice/dictionary/data/DataDictionaryImpl.java +++ b/service/src/main/java/datawave/microservice/dictionary/data/DataDictionaryImpl.java @@ -13,11 +13,11 @@ import datawave.query.util.MetadataEntry; import datawave.query.util.MetadataHelper; import datawave.query.util.MetadataHelperFactory; -import datawave.webservice.metadata.DefaultMetadataField; import datawave.webservice.dictionary.data.DefaultDataDictionary; import datawave.webservice.dictionary.data.DefaultDescription; import datawave.webservice.dictionary.data.DefaultDictionaryField; import datawave.webservice.dictionary.data.DefaultFields; +import datawave.webservice.metadata.DefaultMetadataField; import org.apache.accumulo.core.client.TableNotFoundException; import java.util.Collection; diff --git a/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java b/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java index fac9bd4..0be7985 100644 --- a/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java +++ b/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java @@ -3,14 +3,14 @@ import com.google.common.collect.Maps; import datawave.data.ColumnFamilyConstants; import datawave.marking.MarkingFunctions; -import datawave.microservice.dictionary.config.ResponseObjectFactory; import datawave.microservice.Connection; +import datawave.microservice.dictionary.config.ResponseObjectFactory; import datawave.security.util.ScannerHelper; -import datawave.webservice.metadata.DefaultMetadataField; import datawave.webservice.dictionary.data.DefaultDataDictionary; import datawave.webservice.dictionary.data.DefaultDescription; import datawave.webservice.dictionary.data.DefaultDictionaryField; import datawave.webservice.dictionary.data.DefaultFields; +import datawave.webservice.metadata.DefaultMetadataField; import org.apache.accumulo.core.client.BatchScanner; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.TableNotFoundException; diff --git a/service/src/main/java/datawave/microservice/model/config/ModelProperties.java b/service/src/main/java/datawave/microservice/model/config/ModelProperties.java index 944cf9d..3b9f433 100644 --- a/service/src/main/java/datawave/microservice/model/config/ModelProperties.java +++ b/service/src/main/java/datawave/microservice/model/config/ModelProperties.java @@ -1,7 +1,6 @@ package datawave.microservice.model.config; import lombok.Getter; - import lombok.Setter; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.validation.annotation.Validated; diff --git a/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java b/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java index 8343caa..509d280 100644 --- a/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java +++ b/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java @@ -6,10 +6,10 @@ import datawave.marking.MarkingFunctions; import datawave.microservice.ControllerIT; import datawave.microservice.dictionary.config.DataDictionaryProperties; -import datawave.webservice.result.VoidResponse; import datawave.webservice.dictionary.data.DefaultDataDictionary; import datawave.webservice.dictionary.data.DefaultDescription; import datawave.webservice.dictionary.data.DefaultFields; +import datawave.webservice.result.VoidResponse; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.Connector; diff --git a/service/src/test/java/datawave/microservice/dictionary/data/DataDictionaryImplTest.java b/service/src/test/java/datawave/microservice/dictionary/data/DataDictionaryImplTest.java index d12b9f7..8b9df4b 100644 --- a/service/src/test/java/datawave/microservice/dictionary/data/DataDictionaryImplTest.java +++ b/service/src/test/java/datawave/microservice/dictionary/data/DataDictionaryImplTest.java @@ -13,11 +13,11 @@ import datawave.query.util.MetadataEntry; import datawave.query.util.MetadataHelper; import datawave.query.util.MetadataHelperFactory; -import datawave.webservice.metadata.DefaultMetadataField; import datawave.webservice.dictionary.data.DefaultDataDictionary; import datawave.webservice.dictionary.data.DefaultDescription; import datawave.webservice.dictionary.data.DefaultDictionaryField; import datawave.webservice.dictionary.data.DefaultFields; +import datawave.webservice.metadata.DefaultMetadataField; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.security.Authorizations; diff --git a/service/src/test/java/datawave/microservice/dictionary/edge/EdgeDictionaryImplTest.java b/service/src/test/java/datawave/microservice/dictionary/edge/EdgeDictionaryImplTest.java index 3d594ba..d5d657c 100644 --- a/service/src/test/java/datawave/microservice/dictionary/edge/EdgeDictionaryImplTest.java +++ b/service/src/test/java/datawave/microservice/dictionary/edge/EdgeDictionaryImplTest.java @@ -6,10 +6,9 @@ import datawave.data.ColumnFamilyConstants; import datawave.metadata.protobuf.EdgeMetadata.MetadataValue; import datawave.webservice.dictionary.edge.DefaultEdgeDictionary; -import datawave.webservice.dictionary.edge.EventField; import datawave.webservice.dictionary.edge.DefaultMetadata; +import datawave.webservice.dictionary.edge.EventField; import datawave.webservice.dictionary.edge.MetadataBase; - import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.Connector; diff --git a/service/src/test/java/datawave/microservice/metadata/DefaultMetadataFieldScannerTest.java b/service/src/test/java/datawave/microservice/metadata/DefaultMetadataFieldScannerTest.java index a88d617..59a48b9 100644 --- a/service/src/test/java/datawave/microservice/metadata/DefaultMetadataFieldScannerTest.java +++ b/service/src/test/java/datawave/microservice/metadata/DefaultMetadataFieldScannerTest.java @@ -3,13 +3,13 @@ import datawave.accumulo.inmemory.InMemoryInstance; import datawave.data.ColumnFamilyConstants; import datawave.marking.MarkingFunctions; -import datawave.microservice.dictionary.config.ResponseObjectFactory; import datawave.microservice.Connection; -import datawave.webservice.metadata.DefaultMetadataField; +import datawave.microservice.dictionary.config.ResponseObjectFactory; import datawave.webservice.dictionary.data.DefaultDataDictionary; import datawave.webservice.dictionary.data.DefaultDescription; import datawave.webservice.dictionary.data.DefaultDictionaryField; import datawave.webservice.dictionary.data.DefaultFields; +import datawave.webservice.metadata.DefaultMetadataField; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.MutationsRejectedException; From 98edf5f2836ed8ded52b24c601c131bdf52a017c Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 17 Jun 2022 12:47:44 -0400 Subject: [PATCH 29/54] Updated swagger docs --- .../dictionary/DataDictionaryController.java | 28 +++++++++++++++++-- .../dictionary/EdgeDictionaryController.java | 7 ++++- .../microservice/model/ModelController.java | 23 +++++++++++---- 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java b/service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java index 46f8c7a..39746ba 100644 --- a/service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java +++ b/service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java @@ -16,6 +16,8 @@ import datawave.webservice.dictionary.data.FieldsBase; import datawave.webservice.metadata.MetadataFieldBase; import datawave.webservice.result.VoidResponse; +import io.swagger.v3.oas.annotations.ExternalDocumentation; +import io.swagger.v3.oas.annotations.tags.Tag; import org.apache.commons.lang.StringUtils; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.http.MediaType; @@ -42,6 +44,9 @@ import static datawave.microservice.http.converter.protostuff.ProtostuffHttpMessageConverter.PROTOSTUFF_VALUE; +@Tag(name = "Data Dictionary Controller /v1", description = "DataWave Dictionary Operations", + externalDocs = @ExternalDocumentation(description = "Dictionary Service Documentation", + url = "https://github.com/NationalSecurityAgency/datawave-dictionary-service")) @RestController @RequestMapping(path = "/data/v1", produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.TEXT_XML_VALUE, PROTOSTUFF_VALUE, @@ -69,6 +74,25 @@ public DataDictionaryController(DataDictionaryProperties dataDictionaryConfigura dataDictionary.setNormalizationMap(dataDictionaryConfiguration.getNormalizerMap()); } + /** + * Returns the DataDictionary for the given parameters. + * + * @param modelName + * Optional model name + * @param modelTableName + * Optional model table name + * @param metadataTableName + * Optional metadata table name + * @param queryAuthorizations + * Optional query authorizations + * @param dataTypeFilters + * Optional data type filters + * @param currentUser + * the current user + * @return the DataDictionaryBase class (extended) that contains the data dictionary fields + * @throws Exception + * if there is any problem fetching the entries + */ @GetMapping("/") @Timed(name = "dw.dictionary.data.get", absolute = true) public DataDictionaryBase get(@RequestParam(required = false) String modelName, @RequestParam(required = false) String modelTableName, @@ -135,7 +159,7 @@ public VoidResponse uploadDescriptions(@RequestBody FIELDS fields, @RequestParam * ColumnVisibility of the description * @param currentUser * The user sending the request - * @return a {@link VoidResponse} + * @return a VoidResponse * @throws Exception * if there is any problem updating the description */ @@ -287,7 +311,7 @@ public FIELDS fieldNameDescription(@PathVariable String fieldName, @PathVariable * the column visibility * @param currentUser * The user sending the request - * @return a {@link VoidResponse} with operation time and error information + * @return a VoidResponse with operation time and error information * @throws Exception * if there is any problem removing the description from Accumulo */ diff --git a/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java b/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java index 301ab37..94a8c8e 100644 --- a/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java +++ b/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java @@ -8,6 +8,8 @@ import datawave.microservice.dictionary.edge.EdgeDictionary; import datawave.webservice.dictionary.edge.EdgeDictionaryBase; import datawave.webservice.dictionary.edge.MetadataBase; +import io.swagger.v3.oas.annotations.ExternalDocumentation; +import io.swagger.v3.oas.annotations.tags.Tag; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang.StringUtils; import org.springframework.boot.context.properties.EnableConfigurationProperties; @@ -20,6 +22,9 @@ import static datawave.microservice.http.converter.protostuff.ProtostuffHttpMessageConverter.PROTOSTUFF_VALUE; +@Tag(name = "Edge Dictionary Controller /v1", description = "DataWave Edge Dictionary Operations", + externalDocs = @ExternalDocumentation(description = "Dictionary Service Documentation", + url = "https://github.com/NationalSecurityAgency/datawave-dictionary-service")) @Slf4j @RestController @RequestMapping(path = "/edge/v1", @@ -48,7 +53,7 @@ public EdgeDictionaryController(EdgeDictionaryProperties edgeDictionaryPropertie * Name of metadata table (Optional) * @param queryAuthorizations * Authorizations to use - * @return the {@link EdgeDictionaryBase} class (extended) that contains the edge dictionary fields + * @return the EdgeDictionaryBase class (extended) that contains the edge dictionary fields * @throws Exception * if there is any problem retrieving the edge dictionary from Accumulo */ diff --git a/service/src/main/java/datawave/microservice/model/ModelController.java b/service/src/main/java/datawave/microservice/model/ModelController.java index 241a84a..2a0d41c 100644 --- a/service/src/main/java/datawave/microservice/model/ModelController.java +++ b/service/src/main/java/datawave/microservice/model/ModelController.java @@ -12,6 +12,9 @@ import datawave.webservice.query.exception.DatawaveErrorCode; import datawave.webservice.query.exception.QueryException; import datawave.webservice.result.VoidResponse; +import io.swagger.v3.oas.annotations.ExternalDocumentation; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; import lombok.extern.slf4j.Slf4j; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.data.Key; @@ -38,6 +41,9 @@ /** * Service that supports manipulation of models. The models are contained in the data dictionary table. */ +@Tag(name = "Model Controller /v1", description = "DataWave Model Operations", + externalDocs = @ExternalDocumentation(description = "Dictionary Service Documentation", + url = "https://github.com/NationalSecurityAgency/datawave-dictionary-service")) @Slf4j @RestController @RequestMapping(path = "/model", @@ -65,7 +71,7 @@ public ModelController(ModelProperties modelProperties, AccumuloConnectionServic * * @param modelTableName * name of the table that contains the model - * @return ModelList + * @return the ModelList * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user * * @HTTP 200 success @@ -110,13 +116,14 @@ public ModelList listModelNames(@RequestParam(defaultValue = DEFAULT_MODEL_TABLE * model name to delete * @param modelTableName * name of the table that contains the model - * @return datawave.webservice.result.VoidResponse + * @return a VoidResponse * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user * * @HTTP 200 success * @HTTP 404 model not found * @HTTP 500 internal server error */ + @Operation(summary = "Delete a model with the supplied name.") @DeleteMapping("/{name}") @Secured({"Administrator", "JBossAdministrator"}) public VoidResponse deleteModel(@RequestParam String name, @RequestParam(defaultValue = DEFAULT_MODEL_TABLE_NAME) String modelTableName, @@ -141,13 +148,15 @@ public VoidResponse deleteModel(@RequestParam String name, @RequestParam(default * name of copied model * @param modelTableName * name of the table that contains the model - * @return datawave.webservice.result.VoidResponse + * @return a VoidResponse * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user * * @HTTP 200 success * @HTTP 204 model not found * @HTTP 500 internal server error */ + @Operation(summary = "Copy a model.") + @PostMapping("/clone") @Secured({"Administrator", "JBossAdministrator"}) public VoidResponse cloneModel(@RequestParam String name, @RequestParam String newName, @@ -168,7 +177,7 @@ public VoidResponse cloneModel(@RequestParam String name, @RequestParam String n * model name * @param modelTableName * name of the table that contains the model - * @return Model + * @return the Model * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user * * @HTTP 200 success @@ -203,12 +212,13 @@ public Model getModel(@RequestParam String name, @RequestParam(defaultValue = DE * list of new field mappings to insert * @param modelTableName * name of the table that contains the model - * @return datawave.webservice.result.VoidResponse + * @return a VoidResponse * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user * * @HTTP 200 success * @HTTP 500 internal server error */ + @Operation(summary = "Insert a new field mapping into an existing model.") @PostMapping(value = {"/insert", "/import"}) // If we get to change to standard REST, this would just be / and rely on the @Secured({"Administrator", "JBossAdministrator"}) public VoidResponse insertMapping(@RequestBody Model model, @RequestParam(defaultValue = DEFAULT_MODEL_TABLE_NAME) String modelTableName, @@ -238,12 +248,13 @@ public VoidResponse insertMapping(@RequestBody Model model, @RequestParam(defaul * list of field mappings to delete * @param modelTableName * name of the table that contains the model - * @return datawave.webservice.result.VoidResponse + * @return a VoidResponse * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user * * @HTTP 200 success * @HTTP 500 internal server error */ + @Operation(summary = "Delete field mappings from an existing model.") @DeleteMapping("/delete") @Secured({"Administrator", "JBossAdministrator"}) public VoidResponse deleteMapping(@RequestBody Model model, @RequestParam(defaultValue = DEFAULT_MODEL_TABLE_NAME) String modelTableName, From 382669e9a8740f8c50ccb87e9f7cc48fec323004 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 7 Jul 2022 12:20:35 -0400 Subject: [PATCH 30/54] Updated in-memory-accumulo version --- service/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/pom.xml b/service/pom.xml index 0998a5f..39cc93a 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -21,7 +21,7 @@ datawave.microservice.dictionary.DictionaryService 1.9.3 5.2.0 - 1.9.3-SNAPSHOT + 1.10.3-SNAPSHOT 1.3-SNAPSHOT 1.6.8-SNAPSHOT 1.0-SNAPSHOT From ff1df1caae86f4301fbb7d93878621d23b0d7a0d Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 8 Jul 2022 11:08:51 -0400 Subject: [PATCH 31/54] Updated spring boot to 2.7.1 --- service/src/test/java/datawave/microservice/ControllerIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/src/test/java/datawave/microservice/ControllerIT.java b/service/src/test/java/datawave/microservice/ControllerIT.java index 66b0896..c22f089 100644 --- a/service/src/test/java/datawave/microservice/ControllerIT.java +++ b/service/src/test/java/datawave/microservice/ControllerIT.java @@ -13,8 +13,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.web.server.LocalServerPort; import org.springframework.boot.web.client.RestTemplateBuilder; -import org.springframework.boot.web.server.LocalServerPort; import org.springframework.http.HttpStatus; import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.web.client.DefaultResponseErrorHandler; From a155c9316468a6ba5d0ec925edac3623f87acc9f Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Mon, 11 Jul 2022 19:43:16 -0400 Subject: [PATCH 32/54] Updated accumulo to 1.10.2 --- service/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/pom.xml b/service/pom.xml index 39cc93a..514fc60 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -19,7 +19,7 @@ datawave.microservice.dictionary.DictionaryService - 1.9.3 + 1.10.2 5.2.0 1.10.3-SNAPSHOT 1.3-SNAPSHOT From a8ba05d2d3a392b3fe61f9ea3c70ce88cb07267a Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 11 Aug 2022 11:58:14 -0400 Subject: [PATCH 33/54] Updated package-info.java generation --- api/pom.xml | 53 +++++++++++++++++++ .../dictionary/data/DescriptionBase.java | 2 +- .../webservice/model/package-info.java | 7 --- .../dictionary/data/package-info.java | 6 +++ .../dictionary/edge/package-info.java | 6 +++ .../webservice/metadata/package-info.java | 6 +++ .../webservice/model/package-info.java | 6 +++ 7 files changed, 78 insertions(+), 8 deletions(-) delete mode 100644 api/src/main/java/datawave/webservice/model/package-info.java create mode 100644 api/src/main/resources/source-templates/datawave/webservice/dictionary/data/package-info.java create mode 100644 api/src/main/resources/source-templates/datawave/webservice/dictionary/edge/package-info.java create mode 100644 api/src/main/resources/source-templates/datawave/webservice/metadata/package-info.java create mode 100644 api/src/main/resources/source-templates/datawave/webservice/model/package-info.java diff --git a/api/pom.xml b/api/pom.xml index 69c879a..b406832 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -76,4 +76,57 @@ https://raw.githubusercontent.com/NationalSecurityAgency/datawave/mvn-repo + + + + true + src/main/resources + + source-templates/** + + + + + + maven-resources-plugin + + + copy-templated-sources + validate + + copy-resources + + + ${project.build.directory}/generated-sources/templated-sources + + + src/main/resources/source-templates + true + + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.3.0 + + + add-source + generate-sources + + add-source + + + + target/generated-sources/templated-sources + + + + + + + diff --git a/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java b/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java index e8545e4..d83af58 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java @@ -1,7 +1,7 @@ package datawave.webservice.dictionary.data; import datawave.webservice.query.result.event.HasMarkings; -import datawave.webservice.query.util.StringMapAdapter; +import datawave.webservice.xml.util.StringMapAdapter; import io.protostuff.Message; import javax.xml.bind.annotation.XmlAccessType; diff --git a/api/src/main/java/datawave/webservice/model/package-info.java b/api/src/main/java/datawave/webservice/model/package-info.java deleted file mode 100644 index 0b4e0d0..0000000 --- a/api/src/main/java/datawave/webservice/model/package-info.java +++ /dev/null @@ -1,7 +0,0 @@ -@XmlSchema(namespace = "http://webservice.datawave/v1", elementFormDefault = XmlNsForm.QUALIFIED, - xmlns = {@XmlNs(prefix = "http://webservice.datawave/v1", namespaceURI = "${datawave.webservice.namespace}")}) -package datawave.webservice.model; - -import javax.xml.bind.annotation.XmlNs; -import javax.xml.bind.annotation.XmlNsForm; -import javax.xml.bind.annotation.XmlSchema; diff --git a/api/src/main/resources/source-templates/datawave/webservice/dictionary/data/package-info.java b/api/src/main/resources/source-templates/datawave/webservice/dictionary/data/package-info.java new file mode 100644 index 0000000..3970739 --- /dev/null +++ b/api/src/main/resources/source-templates/datawave/webservice/dictionary/data/package-info.java @@ -0,0 +1,6 @@ +@XmlSchema(namespace="${datawave.webservice.namespace}", elementFormDefault=XmlNsForm.QUALIFIED, xmlns={@XmlNs(prefix = "", namespaceURI = "${datawave.webservice.namespace}")}) +package datawave.webservice.dictionary.data; + +import javax.xml.bind.annotation.XmlNs; +import javax.xml.bind.annotation.XmlNsForm; +import javax.xml.bind.annotation.XmlSchema; diff --git a/api/src/main/resources/source-templates/datawave/webservice/dictionary/edge/package-info.java b/api/src/main/resources/source-templates/datawave/webservice/dictionary/edge/package-info.java new file mode 100644 index 0000000..005b670 --- /dev/null +++ b/api/src/main/resources/source-templates/datawave/webservice/dictionary/edge/package-info.java @@ -0,0 +1,6 @@ +@XmlSchema(namespace="${datawave.webservice.namespace}", elementFormDefault=XmlNsForm.QUALIFIED, xmlns={@XmlNs(prefix = "", namespaceURI = "${datawave.webservice.namespace}")}) +package datawave.webservice.dictionary.edge; + +import javax.xml.bind.annotation.XmlNs; +import javax.xml.bind.annotation.XmlNsForm; +import javax.xml.bind.annotation.XmlSchema; diff --git a/api/src/main/resources/source-templates/datawave/webservice/metadata/package-info.java b/api/src/main/resources/source-templates/datawave/webservice/metadata/package-info.java new file mode 100644 index 0000000..5c75f67 --- /dev/null +++ b/api/src/main/resources/source-templates/datawave/webservice/metadata/package-info.java @@ -0,0 +1,6 @@ +@XmlSchema(namespace="${datawave.webservice.namespace}", elementFormDefault=XmlNsForm.QUALIFIED, xmlns={@XmlNs(prefix = "", namespaceURI = "${datawave.webservice.namespace}")}) +package datawave.webservice.metadata; + +import javax.xml.bind.annotation.XmlNs; +import javax.xml.bind.annotation.XmlNsForm; +import javax.xml.bind.annotation.XmlSchema; diff --git a/api/src/main/resources/source-templates/datawave/webservice/model/package-info.java b/api/src/main/resources/source-templates/datawave/webservice/model/package-info.java new file mode 100644 index 0000000..0de9f33 --- /dev/null +++ b/api/src/main/resources/source-templates/datawave/webservice/model/package-info.java @@ -0,0 +1,6 @@ +@XmlSchema(namespace="${datawave.webservice.namespace}", elementFormDefault=XmlNsForm.QUALIFIED, xmlns={@XmlNs(prefix = "", namespaceURI = "${datawave.webservice.namespace}")}) +package datawave.webservice.model; + +import javax.xml.bind.annotation.XmlNs; +import javax.xml.bind.annotation.XmlNsForm; +import javax.xml.bind.annotation.XmlSchema; From f1cae80ec850ed014338ae8d258533284e415246 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 11 Aug 2022 14:22:19 -0400 Subject: [PATCH 34/54] added datawave webservice namespace and fixed jaxb marshalling test --- api/pom.xml | 1 + .../data}/DefaultDescriptionTest.java | 27 ++++++++----------- 2 files changed, 12 insertions(+), 16 deletions(-) rename api/src/test/java/datawave/webservice/{results/datadictionary => dictionary/data}/DefaultDescriptionTest.java (82%) diff --git a/api/pom.xml b/api/pom.xml index b406832..93902bd 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -17,6 +17,7 @@ https://github.com/NationalSecurityAgency/datawave-dictionary-service + http://webservice.datawave/v1 1.3-SNAPSHOT 31.1-jre diff --git a/api/src/test/java/datawave/webservice/results/datadictionary/DefaultDescriptionTest.java b/api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java similarity index 82% rename from api/src/test/java/datawave/webservice/results/datadictionary/DefaultDescriptionTest.java rename to api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java index 6745901..2681ab2 100644 --- a/api/src/test/java/datawave/webservice/results/datadictionary/DefaultDescriptionTest.java +++ b/api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java @@ -1,8 +1,6 @@ -package datawave.webservice.results.datadictionary; +package datawave.webservice.dictionary.data; -import datawave.webservice.dictionary.data.DefaultDescription; -import datawave.webservice.dictionary.data.DefaultDictionaryField; -import datawave.webservice.dictionary.data.DefaultFields; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import javax.xml.bind.JAXBContext; @@ -20,9 +18,6 @@ import java.util.Map; import java.util.Set; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - public class DefaultDescriptionTest { @Test @@ -58,9 +53,9 @@ public void testMarshall() throws JAXBException { String result = out.toString(); - assertTrue(result.contains("columnVisibility")); - assertTrue(result.contains("PRIVATE")); - assertTrue(result.contains("my description")); + Assertions.assertTrue(result.contains("columnVisibility")); + Assertions.assertTrue(result.contains("PRIVATE")); + Assertions.assertTrue(result.contains("my description")); } @Test @@ -106,19 +101,19 @@ public void testUnmarshall() throws JAXBException { int markingCount = 0; for (DefaultDictionaryField f : resp.getFields()) { - assertEquals("myType", f.getDatatype()); + Assertions.assertEquals("myType", f.getDatatype()); for (DefaultDescription d : f.getDescriptions()) { - assertEquals("my description", d.getDescription()); + Assertions.assertEquals("my description", d.getDescription()); Map m = d.getMarkings(); - assertEquals("PRIVATE", m.get("columnVisibility")); + Assertions.assertEquals("PRIVATE", m.get("columnVisibility")); markingCount = m.size(); descriptionCount++; } fieldCount++; } - assertEquals(1, fieldCount); - assertEquals(1, descriptionCount); - assertEquals(1, markingCount); + Assertions.assertEquals(1, fieldCount); + Assertions.assertEquals(1, descriptionCount); + Assertions.assertEquals(1, markingCount); } } From 49b4ca638985c165d0617be1f92168219f30af3b Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Mon, 29 Aug 2022 12:59:38 -0400 Subject: [PATCH 35/54] Removed jakarta --- api/pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 93902bd..0130405 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -44,12 +44,6 @@ gov.nsa.datawave.microservice base-rest-responses - - jakarta.xml.bind - jakarta.xml.bind-api - 2.3.3 - compile - org.projectlombok lombok From 36549e09fa4c3dbf9f434686d8dbec31fdf0eaa0 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Mon, 17 Oct 2022 15:59:10 -0400 Subject: [PATCH 36/54] Updated to use microservice parent 2.0 and microservice service parent 3.0 --- api/pom.xml | 2 +- pom.xml | 2 +- service/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 0130405..12f9811 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 1.9-SNAPSHOT + 2.0 ../../../microservice-parent/pom.xml dictionary-api diff --git a/pom.xml b/pom.xml index 502a7c8..5589001 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 1.9-SNAPSHOT + 2.0 ../../microservice-parent/pom.xml dictionary-service-parent diff --git a/service/pom.xml b/service/pom.xml index 514fc60..04d18c2 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-service-parent - 2.1-SNAPSHOT + 3.1-SNAPSHOT ../../../microservice-service-parent/pom.xml dictionary-service From a8f56406469a194ae3424c19402a1947b69555a4 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 22 Dec 2022 22:22:16 -0500 Subject: [PATCH 37/54] Renamed ProxiedUserDetails as DatawaveUserDetails --- .../AccumuloConnectionService.java | 16 ++++++++-------- .../dictionary/DataDictionaryController.java | 18 +++++++++--------- .../dictionary/EdgeDictionaryController.java | 4 ++-- .../microservice/model/ModelController.java | 14 +++++++------- .../datawave/microservice/ControllerIT.java | 11 ++++++----- 5 files changed, 32 insertions(+), 31 deletions(-) diff --git a/service/src/main/java/datawave/microservice/AccumuloConnectionService.java b/service/src/main/java/datawave/microservice/AccumuloConnectionService.java index 512ea5a..552b6e1 100644 --- a/service/src/main/java/datawave/microservice/AccumuloConnectionService.java +++ b/service/src/main/java/datawave/microservice/AccumuloConnectionService.java @@ -1,7 +1,7 @@ package datawave.microservice; import datawave.accumulo.util.security.UserAuthFunctions; -import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.authorization.user.DatawaveUserDetails; import datawave.microservice.dictionary.config.DataDictionaryProperties; import datawave.security.authorization.DatawaveUser; import datawave.security.util.ScannerHelper; @@ -82,7 +82,7 @@ public Connection getConnection() { * * @return a Connection representing the connection to accumulo */ - public Connection getConnection(String modelTable, String modelName, ProxiedUserDetails user) { + public Connection getConnection(String modelTable, String modelName, DatawaveUserDetails user) { return getConnection(dataDictionaryConfiguration.getMetadataTableName(), modelTable, modelName, user); } @@ -100,7 +100,7 @@ public Connection getConnection(String modelTable, String modelName, ProxiedUser * * @return a Connection representing the connection to accumulo */ - public Connection getConnection(String metadataTable, String modelTable, String modelName, ProxiedUserDetails user) { + public Connection getConnection(String metadataTable, String modelTable, String modelName, DatawaveUserDetails user) { Connection connection = new Connection(); connection.setMetadataTable(getSupplierValueIfBlank(metadataTable, dataDictionaryConfiguration::getMetadataTableName)); connection.setModelTable(getSupplierValueIfBlank(modelTable, dataDictionaryConfiguration::getModelTableName)); @@ -122,7 +122,7 @@ private String getSupplierValueIfBlank(final String value, final Supplier getAuths(ProxiedUserDetails currentUser) { + public Set getAuths(DatawaveUserDetails currentUser) { //@formatter:off return currentUser.getProxiedUsers().stream() .map(DatawaveUser::getAuths) @@ -141,7 +141,7 @@ public Set getAuths(ProxiedUserDetails currentUser) { * * @return the authorizations for the specified user downgraded to the specified level */ - public Set getDowngradedAuthorizations(String requestedAuthorizations, ProxiedUserDetails currentUser) { + public Set getDowngradedAuthorizations(String requestedAuthorizations, DatawaveUserDetails currentUser) { DatawaveUser primaryUser = currentUser.getPrimaryUser(); return userAuthFunctions.mergeAuthorizations(userAuthFunctions.getRequestedAuthorizations(requestedAuthorizations, currentUser.getPrimaryUser()), currentUser.getProxiedUsers(), u -> u != primaryUser); @@ -162,7 +162,7 @@ public Set getDowngradedAuthorizations(String requestedAuthoriza * @throws TableNotFoundException * Thrown if the table is not found */ - public List getKeys(String modelTable, ProxiedUserDetails currentUser, String regexTerm) throws TableNotFoundException { + public List getKeys(String modelTable, DatawaveUserDetails currentUser, String regexTerm) throws TableNotFoundException { Scanner scanner = ScannerHelper.createScanner(this.accumuloConnector, modelTable, this.getAuths(currentUser)); if (!regexTerm.isEmpty()) { IteratorSetting cfg = new IteratorSetting(21, "colfRegex", RegExFilter.class.getName()); @@ -188,7 +188,7 @@ public List getKeys(String modelTable, ProxiedUserDetails currentUser, Stri * * @return A QueryException if anything is unsuccessful, and null if everything is successful. */ - public QueryException modifyMappings(List mutations, String modelTable, String modelName, ProxiedUserDetails user) { + public QueryException modifyMappings(List mutations, String modelTable, String modelName, DatawaveUserDetails user) { QueryException exception = null; Mutation mute = null; @@ -211,7 +211,7 @@ public QueryException modifyMappings(List mutations, String modelTable return exception; } - private BatchWriter getDefaultBatchWriter(String modelTable, String modelName, ProxiedUserDetails user) throws TableNotFoundException { + private BatchWriter getDefaultBatchWriter(String modelTable, String modelName, DatawaveUserDetails user) throws TableNotFoundException { Connector connector = getConnection(modelTable, modelName, user).getConnector(); // TODO Do we need a new instance of BatchWriterConfig each time, or can this be a static or bean object? return connector.createBatchWriter(modelTable, new BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY, TimeUnit.MILLISECONDS) diff --git a/service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java b/service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java index 39746ba..d43b0e2 100644 --- a/service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java +++ b/service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java @@ -6,7 +6,7 @@ import com.google.common.collect.Multimap; import datawave.microservice.AccumuloConnectionService; import datawave.microservice.Connection; -import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.authorization.user.DatawaveUserDetails; import datawave.microservice.dictionary.config.DataDictionaryProperties; import datawave.microservice.dictionary.config.ResponseObjectFactory; import datawave.microservice.dictionary.data.DataDictionary; @@ -97,7 +97,7 @@ public DataDictionaryController(DataDictionaryProperties dataDictionaryConfigura @Timed(name = "dw.dictionary.data.get", absolute = true) public DataDictionaryBase get(@RequestParam(required = false) String modelName, @RequestParam(required = false) String modelTableName, @RequestParam(required = false) String metadataTableName, @RequestParam(name = "auths", required = false) String queryAuthorizations, - @RequestParam(defaultValue = "") String dataTypeFilters, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { + @RequestParam(defaultValue = "") String dataTypeFilters, @AuthenticationPrincipal DatawaveUserDetails currentUser) throws Exception { Connection connection = accumuloConnectionService.getConnection(metadataTableName, modelTableName, modelName, currentUser); // If the user provides authorizations, intersect it with their actual authorizations @@ -131,7 +131,7 @@ public DataDictionaryBase get(@RequestParam(required = false) String @PostMapping(path = "/Descriptions", consumes = {MediaType.APPLICATION_XML_VALUE, MediaType.TEXT_XML_VALUE, MediaType.APPLICATION_JSON_VALUE}) @Timed(name = "dw.dictionary.data.uploadDescriptions", absolute = true) public VoidResponse uploadDescriptions(@RequestBody FIELDS fields, @RequestParam(required = false) String modelName, - @RequestParam(required = false) String modelTable, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { + @RequestParam(required = false) String modelTable, @AuthenticationPrincipal DatawaveUserDetails currentUser) throws Exception { Connection connection = accumuloConnectionService.getConnection(modelTable, modelName, currentUser); List list = fields.getFields(); for (FIELD desc : list) { @@ -168,7 +168,7 @@ public VoidResponse uploadDescriptions(@RequestBody FIELDS fields, @RequestParam @Timed(name = "dw.dictionary.data.setDescriptionPut", absolute = true) public VoidResponse setDescriptionPut(@PathVariable String fieldName, @PathVariable String datatype, @PathVariable String description, @RequestParam(required = false) String modelName, @RequestParam(required = false) String modelTable, @RequestParam String columnVisibility, - @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws Exception { return setDescriptionPost(fieldName, datatype, description, modelName, modelTable, columnVisibility, currentUser); } @@ -198,7 +198,7 @@ public VoidResponse setDescriptionPut(@PathVariable String fieldName, @PathVaria @Timed(name = "dw.dictionary.data.setDescriptionPost", absolute = true) public VoidResponse setDescriptionPost(@RequestParam String fieldName, @RequestParam String datatype, @RequestParam String description, @RequestParam(required = false) String modelName, @RequestParam(required = false) String modelTable, @RequestParam String columnVisibility, - @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws Exception { DESC desc = this.responseObjectFactory.getDescription(); Map markings = Maps.newHashMap(); markings.put("columnVisibility", columnVisibility); @@ -228,7 +228,7 @@ public VoidResponse setDescriptionPost(@RequestParam String fieldName, @RequestP @GetMapping("/Descriptions") @Timed(name = "dw.dictionary.data.allDescriptions", absolute = true) public FIELDS allDescriptions(@RequestParam(required = false) String modelName, @RequestParam(required = false) String modelTable, - @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws Exception { Connection connection = accumuloConnectionService.getConnection(modelTable, modelName, currentUser); Multimap,DESC> descriptions = dataDictionary.getDescriptions(connection); FIELDS response = this.responseObjectFactory.getFields(); @@ -254,7 +254,7 @@ public FIELDS allDescriptions(@RequestParam(required = false) String modelName, @GetMapping("/Descriptions/{datatype}") @Timed(name = "dw.dictionary.data.datatypeDescriptions", absolute = true) public FIELDS datatypeDescriptions(@PathVariable("datatype") String datatype, @RequestParam(required = false) String modelName, - @RequestParam(required = false) String modelTable, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { + @RequestParam(required = false) String modelTable, @AuthenticationPrincipal DatawaveUserDetails currentUser) throws Exception { Connection connection = accumuloConnectionService.getConnection(modelTable, modelName, currentUser); Multimap,DESC> descriptions = dataDictionary.getDescriptions(connection, datatype); FIELDS response = this.responseObjectFactory.getFields(); @@ -282,7 +282,7 @@ public FIELDS datatypeDescriptions(@PathVariable("datatype") String datatype, @R @GetMapping("/Descriptions/{datatype}/{fieldName}") @Timed(name = "dw.dictionary.data.fieldNameDescription", absolute = true) public FIELDS fieldNameDescription(@PathVariable String fieldName, @PathVariable String datatype, @RequestParam(required = false) String modelName, - @RequestParam(required = false) String modelTable, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { + @RequestParam(required = false) String modelTable, @AuthenticationPrincipal DatawaveUserDetails currentUser) throws Exception { Connection connection = accumuloConnectionService.getConnection(modelTable, modelName, currentUser); Set descriptions = dataDictionary.getDescriptions(connection, fieldName, datatype); FIELDS response = responseObjectFactory.getFields(); @@ -320,7 +320,7 @@ public FIELDS fieldNameDescription(@PathVariable String fieldName, @PathVariable @Timed(name = "dw.dictionary.data.deleteDescription", absolute = true) public VoidResponse deleteDescription(@PathVariable String fieldName, @PathVariable String datatype, @RequestParam(required = false) String modelName, @RequestParam(required = false) String modelTable, @RequestParam String columnVisibility, - @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws Exception { Map markings = Maps.newHashMap(); markings.put("columnVisibility", columnVisibility); DESC desc = this.responseObjectFactory.getDescription(); diff --git a/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java b/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java index 94a8c8e..8e55452 100644 --- a/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java +++ b/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java @@ -3,7 +3,7 @@ import com.codahale.metrics.annotation.Timed; import datawave.accumulo.util.security.UserAuthFunctions; import datawave.microservice.AccumuloConnectionService; -import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.authorization.user.DatawaveUserDetails; import datawave.microservice.dictionary.config.EdgeDictionaryProperties; import datawave.microservice.dictionary.edge.EdgeDictionary; import datawave.webservice.dictionary.edge.EdgeDictionaryBase; @@ -60,7 +60,7 @@ public EdgeDictionaryController(EdgeDictionaryProperties edgeDictionaryPropertie @GetMapping("/") @Timed(name = "dw.dictionary.edge.get", absolute = true) public EdgeDictionaryBase get(@RequestParam(required = false) String metadataTableName, - @RequestParam(name = "auths", required = false) String queryAuthorizations, @AuthenticationPrincipal ProxiedUserDetails currentUser) + @RequestParam(name = "auths", required = false) String queryAuthorizations, @AuthenticationPrincipal DatawaveUserDetails currentUser) throws Exception { log.info("EDGEDICTIONARY: entered rest endpoint"); if (null == metadataTableName || StringUtils.isBlank(metadataTableName)) { diff --git a/service/src/main/java/datawave/microservice/model/ModelController.java b/service/src/main/java/datawave/microservice/model/ModelController.java index 2a0d41c..fb0f135 100644 --- a/service/src/main/java/datawave/microservice/model/ModelController.java +++ b/service/src/main/java/datawave/microservice/model/ModelController.java @@ -2,7 +2,7 @@ import com.google.common.collect.Sets; import datawave.microservice.AccumuloConnectionService; -import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.authorization.user.DatawaveUserDetails; import datawave.microservice.http.converter.protostuff.ProtostuffHttpMessageConverter; import datawave.microservice.model.config.ModelProperties; import datawave.webservice.model.FieldMapping; @@ -79,7 +79,7 @@ public ModelController(ModelProperties modelProperties, AccumuloConnectionServic */ @GetMapping("/list") // If we get to change to follow true REST standard, this would just be / and remain a GET public ModelList listModelNames(@RequestParam(defaultValue = DEFAULT_MODEL_TABLE_NAME) String modelTableName, - @AuthenticationPrincipal ProxiedUserDetails currentUser) { + @AuthenticationPrincipal DatawaveUserDetails currentUser) { ModelList response = new ModelList(jqueryUri, dataTablesUri, modelTableName); HashSet modelNames = new HashSet<>(); @@ -127,7 +127,7 @@ public ModelList listModelNames(@RequestParam(defaultValue = DEFAULT_MODEL_TABLE @DeleteMapping("/{name}") @Secured({"Administrator", "JBossAdministrator"}) public VoidResponse deleteModel(@RequestParam String name, @RequestParam(defaultValue = DEFAULT_MODEL_TABLE_NAME) String modelTableName, - @AuthenticationPrincipal ProxiedUserDetails currentUser) { + @AuthenticationPrincipal DatawaveUserDetails currentUser) { VoidResponse response = new VoidResponse(); ModelList models = listModelNames(modelTableName, currentUser); if (models.getNames().contains(name)) { @@ -160,7 +160,7 @@ public VoidResponse deleteModel(@RequestParam String name, @RequestParam(default @PostMapping("/clone") @Secured({"Administrator", "JBossAdministrator"}) public VoidResponse cloneModel(@RequestParam String name, @RequestParam String newName, - @RequestParam(defaultValue = DEFAULT_MODEL_TABLE_NAME) String modelTableName, @AuthenticationPrincipal ProxiedUserDetails currentUser) { + @RequestParam(defaultValue = DEFAULT_MODEL_TABLE_NAME) String modelTableName, @AuthenticationPrincipal DatawaveUserDetails currentUser) { VoidResponse response = new VoidResponse(); Model model = getModel(name, modelTableName, currentUser); @@ -186,7 +186,7 @@ public VoidResponse cloneModel(@RequestParam String name, @RequestParam String n */ @GetMapping("/{name}") public Model getModel(@RequestParam String name, @RequestParam(defaultValue = DEFAULT_MODEL_TABLE_NAME) String modelTableName, - @AuthenticationPrincipal ProxiedUserDetails currentUser) { + @AuthenticationPrincipal DatawaveUserDetails currentUser) { Model response = new Model(jqueryUri, dataTablesUri); List keys; try { @@ -222,7 +222,7 @@ public Model getModel(@RequestParam String name, @RequestParam(defaultValue = DE @PostMapping(value = {"/insert", "/import"}) // If we get to change to standard REST, this would just be / and rely on the @Secured({"Administrator", "JBossAdministrator"}) public VoidResponse insertMapping(@RequestBody Model model, @RequestParam(defaultValue = DEFAULT_MODEL_TABLE_NAME) String modelTableName, - @AuthenticationPrincipal ProxiedUserDetails currentUser) { + @AuthenticationPrincipal DatawaveUserDetails currentUser) { if (log.isDebugEnabled()) { log.debug("modelTableName: " + (null == modelTableName ? "" : modelTableName)); } @@ -258,7 +258,7 @@ public VoidResponse insertMapping(@RequestBody Model model, @RequestParam(defaul @DeleteMapping("/delete") @Secured({"Administrator", "JBossAdministrator"}) public VoidResponse deleteMapping(@RequestBody Model model, @RequestParam(defaultValue = DEFAULT_MODEL_TABLE_NAME) String modelTableName, - @AuthenticationPrincipal ProxiedUserDetails currentUser) { + @AuthenticationPrincipal DatawaveUserDetails currentUser) { if (log.isDebugEnabled()) { log.debug("Deleting model name: " + model.getName() + "from modelTableName " + modelTableName); } diff --git a/service/src/test/java/datawave/microservice/ControllerIT.java b/service/src/test/java/datawave/microservice/ControllerIT.java index c22f089..f95702f 100644 --- a/service/src/test/java/datawave/microservice/ControllerIT.java +++ b/service/src/test/java/datawave/microservice/ControllerIT.java @@ -3,7 +3,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; import datawave.microservice.authorization.jwt.JWTRestTemplate; -import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.authorization.user.DatawaveUserDetails; import datawave.security.authorization.DatawaveUser; import datawave.security.authorization.SubjectIssuerDNPair; import org.apache.accumulo.core.client.Connector; @@ -39,8 +39,8 @@ public class ControllerIT { @Qualifier("warehouse") protected Connector connector; - protected ProxiedUserDetails adminUser; - protected ProxiedUserDetails regularUser; + protected DatawaveUserDetails adminUser; + protected DatawaveUserDetails regularUser; @Autowired private RestTemplateBuilder restTemplateBuilder; @@ -67,8 +67,9 @@ protected boolean hasError(HttpStatus statusCode) { HashSet auths = Sets.newHashSet("PUBLIC", "PRIVATE"); HashSet roles = Sets.newHashSet("AuthorizedUser", "Administrator"); long createTime = System.currentTimeMillis(); - adminUser = new ProxiedUserDetails(Collections.singleton(new DatawaveUser(dn, DatawaveUser.UserType.USER, auths, roles, null, createTime)), createTime); - regularUser = new ProxiedUserDetails( + adminUser = new DatawaveUserDetails(Collections.singleton(new DatawaveUser(dn, DatawaveUser.UserType.USER, auths, roles, null, createTime)), + createTime); + regularUser = new DatawaveUserDetails( Collections.singleton( new DatawaveUser(dn, DatawaveUser.UserType.USER, auths, Collections.singleton("AuthorizedUser"), null, createTime)), createTime); From b1a0c791e9da464d4d36feae1ce631a944d26ff6 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 10 Feb 2023 11:42:47 -0500 Subject: [PATCH 38/54] Updated accumulo utils and datawave starter versions. --- service/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/pom.xml b/service/pom.xml index 04d18c2..df986b6 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -23,7 +23,7 @@ 5.2.0 1.10.3-SNAPSHOT 1.3-SNAPSHOT - 1.6.8-SNAPSHOT + 1.6.9-SNAPSHOT 1.0-SNAPSHOT 1.10.25 3.6.0 From 3d6a3b36e69b591cfff45227269635a3e8d9325a Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Tue, 7 Mar 2023 11:25:28 -0500 Subject: [PATCH 39/54] Updated dictionary project version to 2.0-SNAPSHOT --- api/pom.xml | 2 +- pom.xml | 2 +- service/pom.xml | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 537947a..e880ba8 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-parent/pom.xml dictionary-api - 1.4-SNAPSHOT + 2.0-SNAPSHOT https://code.nsa.gov/datawave-dictionary-service scm:git:https://github.com/NationalSecurityAgency/datawave-dictionary-service.git diff --git a/pom.xml b/pom.xml index 51fd33a..5f8aa60 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ ../../microservice-parent/pom.xml dictionary-service-parent - 1.4-SNAPSHOT + 2.0-SNAPSHOT pom https://code.nsa.gov/datawave-dictionary-service diff --git a/service/pom.xml b/service/pom.xml index 6550bb4..bada8d6 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -4,11 +4,11 @@ gov.nsa.datawave.microservice datawave-microservice-service-parent - 3.1-SNAPSHOT + 3.0 ../../../microservice-service-parent/pom.xml dictionary-service - 1.4-SNAPSHOT + 2.0-SNAPSHOT DATAWAVE Dictionary Service https://code.nsa.gov/datawave-dictionary-service @@ -22,8 +22,8 @@ 1.10.2 5.2.0 1.10.3-SNAPSHOT - 1.4-SNAPSHOT - 1.6.9-SNAPSHOT + 2.0-SNAPSHOT + 2.0-SNAPSHOT 1.0-SNAPSHOT 1.11.4 3.6.0 From abd7471425ea8e56e5e60abf8d2d2257a5c43d1e Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 10 Mar 2023 19:26:44 -0500 Subject: [PATCH 40/54] pr feedback - static assertion imports --- .../data/DefaultDescriptionTest.java | 21 +++++++++++-------- .../edge/EdgeDictionaryImplTest.java | 13 ++++++------ 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java b/api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java index 2681ab2..198a4c5 100644 --- a/api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java +++ b/api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java @@ -18,6 +18,9 @@ import java.util.Map; import java.util.Set; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + public class DefaultDescriptionTest { @Test @@ -53,9 +56,9 @@ public void testMarshall() throws JAXBException { String result = out.toString(); - Assertions.assertTrue(result.contains("columnVisibility")); - Assertions.assertTrue(result.contains("PRIVATE")); - Assertions.assertTrue(result.contains("my description")); + assertTrue(result.contains("columnVisibility")); + assertTrue(result.contains("PRIVATE")); + assertTrue(result.contains("my description")); } @Test @@ -101,19 +104,19 @@ public void testUnmarshall() throws JAXBException { int markingCount = 0; for (DefaultDictionaryField f : resp.getFields()) { - Assertions.assertEquals("myType", f.getDatatype()); + assertEquals("myType", f.getDatatype()); for (DefaultDescription d : f.getDescriptions()) { - Assertions.assertEquals("my description", d.getDescription()); + assertEquals("my description", d.getDescription()); Map m = d.getMarkings(); - Assertions.assertEquals("PRIVATE", m.get("columnVisibility")); + assertEquals("PRIVATE", m.get("columnVisibility")); markingCount = m.size(); descriptionCount++; } fieldCount++; } - Assertions.assertEquals(1, fieldCount); - Assertions.assertEquals(1, descriptionCount); - Assertions.assertEquals(1, markingCount); + assertEquals(1, fieldCount); + assertEquals(1, descriptionCount); + assertEquals(1, markingCount); } } diff --git a/service/src/test/java/datawave/microservice/dictionary/edge/EdgeDictionaryImplTest.java b/service/src/test/java/datawave/microservice/dictionary/edge/EdgeDictionaryImplTest.java index d5d657c..4b3cf84 100644 --- a/service/src/test/java/datawave/microservice/dictionary/edge/EdgeDictionaryImplTest.java +++ b/service/src/test/java/datawave/microservice/dictionary/edge/EdgeDictionaryImplTest.java @@ -38,6 +38,8 @@ import java.util.List; import java.util.Map; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @ExtendWith(SpringExtension.class) @@ -91,17 +93,17 @@ public void testNoOp() throws InvocationTargetException, IllegalAccessException if (null == transformResultsMethod) fail(); DefaultEdgeDictionary dictionary = (DefaultEdgeDictionary) transformResultsMethod.invoke(impl, HashMultimap.create()); - Assertions.assertEquals(0L, dictionary.getTotalResults(), "Should be empty"); + assertEquals(0L, dictionary.getTotalResults(), "Should be empty"); } @Test public void testWorked() throws InvocationTargetException, IllegalAccessException { if (null == transformResultsMethod) fail(); - Assertions.assertEquals(EDGE_KEYS.size(), edgeMetadataRows.keySet().size(), "data to be inserted contains as many rows as keys"); + assertEquals(EDGE_KEYS.size(), edgeMetadataRows.keySet().size(), "data to be inserted contains as many rows as keys"); DefaultEdgeDictionary dictionary = (DefaultEdgeDictionary) transformResultsMethod.invoke(impl, edgeMetadataRows); - Assertions.assertEquals(EDGE_KEYS.size(), dictionary.getTotalResults(), "Dictionary should now have some entries"); - Assertions.assertTrue(dictionary.getMetadataList().containsAll(METADATA), + assertEquals(EDGE_KEYS.size(), dictionary.getTotalResults(), "Dictionary should now have some entries"); + assertTrue(dictionary.getMetadataList().containsAll(METADATA), "METADATA not in list. returned list: " + dictionary.getMetadataList() + " expected: " + METADATA); } @@ -116,8 +118,7 @@ public void earliestDateFound() throws InvocationTargetException, IllegalAccessE // Make sure that all Metadata in EdgeDictionary have the start date set to the EARLY_DATE_FIELD for (MetadataBase meta : metadata) { - Assertions.assertEquals(EARLY_DATE_FIELD, meta.getStartDate(), - "Incorrect start date. Expected: " + EARLY_DATE_FIELD + " Found: " + meta.getStartDate()); + assertEquals(EARLY_DATE_FIELD, meta.getStartDate(), "Incorrect start date. Expected: " + EARLY_DATE_FIELD + " Found: " + meta.getStartDate()); } } From 7ca55b5a7491cb3eaf3269a4e43500afb79bed7c Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 17 Mar 2023 10:24:41 -0400 Subject: [PATCH 41/54] Moved Direction, FieldMapping, and ModelKeyParser to metadata-utils. --- api/pom.xml | 20 +++ .../metadata/MetadataFieldBase.java | 1 - .../datawave/webservice/model/Direction.java | 25 ---- .../webservice/model/FieldMapping.java | 51 -------- .../datawave/webservice/model/Mapping.java | 1 + .../java/datawave/webservice/model/Model.java | 1 + .../webservice/model/ModelKeyParser.java | 114 ------------------ .../AccumuloConnectionService.java | 2 +- .../metadata/DefaultMetadataFieldScanner.java | 7 +- .../microservice/model/ModelController.java | 11 +- .../model/ModelControllerTest.java | 4 +- 11 files changed, 36 insertions(+), 201 deletions(-) delete mode 100644 api/src/main/java/datawave/webservice/model/Direction.java delete mode 100644 api/src/main/java/datawave/webservice/model/FieldMapping.java delete mode 100644 api/src/main/java/datawave/webservice/model/ModelKeyParser.java diff --git a/api/pom.xml b/api/pom.xml index 38a0650..5a4a83a 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -20,6 +20,7 @@ http://webservice.datawave/v1 3.0-SNAPSHOT 31.1-jre + 3.0-SNAPSHOT @@ -43,6 +44,21 @@ + + gov.nsa.datawave.microservice + metadata-utils + ${version.microservice.metadata-utils} + + + log4j + log4j + + + avro + org.apache.avro + + + @@ -54,6 +70,10 @@ gov.nsa.datawave.microservice base-rest-responses + + gov.nsa.datawave.microservice + metadata-utils + org.projectlombok lombok diff --git a/api/src/main/java/datawave/webservice/metadata/MetadataFieldBase.java b/api/src/main/java/datawave/webservice/metadata/MetadataFieldBase.java index 79e3a93..b13ea8d 100644 --- a/api/src/main/java/datawave/webservice/metadata/MetadataFieldBase.java +++ b/api/src/main/java/datawave/webservice/metadata/MetadataFieldBase.java @@ -11,7 +11,6 @@ import java.util.List; import java.util.Set; -// TODO if nothing other than DefaultMetadataField extends this class, remove this class and rename DefaultMetadataField to MetadataField @XmlAccessorType(XmlAccessType.NONE) @XmlSeeAlso(DefaultMetadataField.class) public abstract class MetadataFieldBase implements Serializable, Message { diff --git a/api/src/main/java/datawave/webservice/model/Direction.java b/api/src/main/java/datawave/webservice/model/Direction.java deleted file mode 100644 index 5a425b2..0000000 --- a/api/src/main/java/datawave/webservice/model/Direction.java +++ /dev/null @@ -1,25 +0,0 @@ -package datawave.webservice.model; - -import javax.xml.bind.annotation.XmlEnum; -import java.io.Serializable; - -@XmlEnum -public enum Direction implements Serializable { - - FORWARD("forward"), REVERSE("reverse"); - - private final String value; - - Direction(String value) { - this.value = value; - } - - public String getValue() { - return this.value; - } - - public static Direction getDirection(String value) { - return Direction.valueOf(value.toUpperCase()); - } - -} diff --git a/api/src/main/java/datawave/webservice/model/FieldMapping.java b/api/src/main/java/datawave/webservice/model/FieldMapping.java deleted file mode 100644 index 4b23c6e..0000000 --- a/api/src/main/java/datawave/webservice/model/FieldMapping.java +++ /dev/null @@ -1,51 +0,0 @@ -package datawave.webservice.model; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.apache.commons.lang3.builder.CompareToBuilder; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlAttribute; -import java.io.Serializable; - -@Data -@NoArgsConstructor -@AllArgsConstructor -@XmlAccessorType(XmlAccessType.NONE) -public class FieldMapping implements Serializable, Comparable { - - private static final long serialVersionUID = 1L; - - @XmlAttribute(required = true) - private String datatype; - - @XmlAttribute(required = true) - private String fieldName; - - @XmlAttribute(required = true) - private String modelFieldName; - - @XmlAttribute(required = true) - private Direction direction; - - @XmlAttribute(required = true) - private String columnVisibility; - - @Override - public int compareTo(FieldMapping obj) { - - if (obj == null) { - throw new IllegalArgumentException("can not compare null"); - } - - if (obj == this) - return 0; - - return new CompareToBuilder().append(datatype, ((FieldMapping) obj).datatype).append(fieldName, ((FieldMapping) obj).fieldName) - .append(modelFieldName, ((FieldMapping) obj).modelFieldName).append(direction, ((FieldMapping) obj).direction) - .append(columnVisibility, ((FieldMapping) obj).columnVisibility).toComparison(); - } - -} diff --git a/api/src/main/java/datawave/webservice/model/Mapping.java b/api/src/main/java/datawave/webservice/model/Mapping.java index f375a72..985842c 100644 --- a/api/src/main/java/datawave/webservice/model/Mapping.java +++ b/api/src/main/java/datawave/webservice/model/Mapping.java @@ -1,5 +1,6 @@ package datawave.webservice.model; +import datawave.query.model.Direction; import lombok.Data; import lombok.NoArgsConstructor; import org.apache.commons.lang3.builder.CompareToBuilder; diff --git a/api/src/main/java/datawave/webservice/model/Model.java b/api/src/main/java/datawave/webservice/model/Model.java index bde13bc..6aab3d5 100644 --- a/api/src/main/java/datawave/webservice/model/Model.java +++ b/api/src/main/java/datawave/webservice/model/Model.java @@ -1,5 +1,6 @@ package datawave.webservice.model; +import datawave.query.model.FieldMapping; import datawave.webservice.HtmlProvider; import datawave.webservice.result.BaseResponse; import lombok.Data; diff --git a/api/src/main/java/datawave/webservice/model/ModelKeyParser.java b/api/src/main/java/datawave/webservice/model/ModelKeyParser.java deleted file mode 100644 index 9011bc4..0000000 --- a/api/src/main/java/datawave/webservice/model/ModelKeyParser.java +++ /dev/null @@ -1,114 +0,0 @@ -package datawave.webservice.model; - -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Mutation; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.core.security.ColumnVisibility; -import org.apache.commons.lang3.StringUtils; -import org.apache.log4j.Logger; - -import java.util.Set; - -public class ModelKeyParser { - - public static final String NULL_BYTE = "\0"; - public static final Value NULL_VALUE = new Value(new byte[0]); - private static final Logger log = Logger.getLogger(ModelKeyParser.class); - - public static FieldMapping parseKey(Key key, Set auths) { - FieldMapping mapping = new FieldMapping(); - String row = key.getRow().toString(); - String[] colf = key.getColumnFamily().toString().split(NULL_BYTE); - String[] colq = key.getColumnQualifier().toString().split(NULL_BYTE); - String cv = key.getColumnVisibility().toString(); - - String datatype = null; - Direction direction; - String dataField; - String modelField; - Boolean indexOnlyField = false; - - // colf.length == 1 is the Older style, no datatype and nothing to do. Start at length == 2 - if (colf.length == 2) { - datatype = colf[1]; - } else { - throw new IllegalArgumentException("Key in unknown format, colf parts: " + colf.length); - } - if (2 == colq.length) { - direction = Direction.getDirection(colq[1]); - if (Direction.REVERSE.equals(direction)) { - dataField = row; - modelField = colq[0]; - } else { - dataField = colq[0]; - modelField = row; - } - } else if (3 == colq.length && Direction.FORWARD == Direction.getDirection(colq[2])) { - dataField = colq[0]; - modelField = row; - direction = Direction.FORWARD; // we already checked it in the if condition - } else { - log.error("Error parsing key: " + key); - throw new IllegalArgumentException("Key in unknown format, colq parts: " + colq.length); - } - - mapping.setColumnVisibility(cv); - mapping.setDatatype(datatype); - mapping.setDirection(direction); - mapping.setFieldName(dataField); - mapping.setModelFieldName(modelField); - return mapping; - } - - public static Key createKey(FieldMapping mapping, String modelName) { - ColumnVisibility cv = new ColumnVisibility(mapping.getColumnVisibility()); - - String inName = Direction.REVERSE.equals(mapping.getDirection()) ? mapping.getFieldName() : mapping.getModelFieldName(); - String outName = Direction.REVERSE.equals(mapping.getDirection()) ? mapping.getModelFieldName() : mapping.getFieldName(); - String dataType = StringUtils.isEmpty(mapping.getDatatype()) ? "" : NULL_BYTE + mapping.getDatatype().trim(); - return new Key(inName, // Row - modelName + dataType, // ColFam - outName + NULL_BYTE + mapping.getDirection().getValue(), // ColQual - cv, // Visibility - System.currentTimeMillis() // Timestamp - ); - } - - public static Mutation createMutation(FieldMapping mapping, String modelName) { - ColumnVisibility cv = new ColumnVisibility(mapping.getColumnVisibility()); - Mutation m; - String dataType = StringUtils.isEmpty(mapping.getDatatype()) ? "" : NULL_BYTE + mapping.getDatatype().trim(); - - if (Direction.REVERSE.equals(mapping.getDirection())) { - // Reverse mappings should not have indexOnly designators. If they do, scrub it off. - m = new Mutation(mapping.getFieldName()); - m.put(modelName + dataType, mapping.getModelFieldName() + NULL_BYTE + mapping.getDirection().getValue(), cv, System.currentTimeMillis(), - NULL_VALUE); - return m; - } else { - m = new Mutation(mapping.getModelFieldName()); - m.put(modelName + dataType, mapping.getFieldName() + NULL_BYTE + mapping.getDirection().getValue(), cv, System.currentTimeMillis(), NULL_VALUE); - return m; - } - } - - public static Mutation createDeleteMutation(FieldMapping mapping, String modelName) { - ColumnVisibility cv = new ColumnVisibility(mapping.getColumnVisibility()); - Mutation m; - String dataType = StringUtils.isEmpty(mapping.getDatatype()) ? "" : NULL_BYTE + mapping.getDatatype().trim(); - - if (Direction.REVERSE.equals(mapping.getDirection())) { - m = new Mutation(mapping.getFieldName()); - m.putDelete(modelName + dataType, mapping.getModelFieldName() + NULL_BYTE + mapping.getDirection().getValue(), cv, System.currentTimeMillis()); - return m; - } else { - m = new Mutation(mapping.getModelFieldName()); - m.putDelete(modelName + dataType, mapping.getFieldName() + NULL_BYTE + mapping.getDirection().getValue(), cv, System.currentTimeMillis()); - m.putDelete(modelName + dataType, mapping.getFieldName() + NULL_BYTE + "index_only" + NULL_BYTE + mapping.getDirection().getValue(), cv, - System.currentTimeMillis()); - return m; - } - } - -} diff --git a/service/src/main/java/datawave/microservice/AccumuloConnectionService.java b/service/src/main/java/datawave/microservice/AccumuloConnectionService.java index c17adf9..deb24ae 100644 --- a/service/src/main/java/datawave/microservice/AccumuloConnectionService.java +++ b/service/src/main/java/datawave/microservice/AccumuloConnectionService.java @@ -143,7 +143,7 @@ public Set getAuths(DatawaveUserDetails currentUser) { */ public Set getDowngradedAuthorizations(String requestedAuthorizations, DatawaveUserDetails currentUser) { DatawaveUser primaryUser = currentUser.getPrimaryUser(); - return userAuthFunctions.mergeAuthorizations(userAuthFunctions.getRequestedAuthorizations(requestedAuthorizations, currentUser.getPrimaryUser()), + return userAuthFunctions.mergeAuthorizations(userAuthFunctions.getRequestedAuthorizations(requestedAuthorizations, primaryUser), currentUser.getProxiedUsers(), u -> u != primaryUser); } diff --git a/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java b/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java index 034456c..d76b02e 100644 --- a/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java +++ b/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java @@ -24,6 +24,10 @@ import java.io.IOException; import java.text.SimpleDateFormat; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; import java.util.Collection; import java.util.Collections; import java.util.Date; @@ -271,7 +275,8 @@ private void setType() { // Set the last updated date for the current {@link DefaultMetadataField} based on the timestamp of the current entry. private void setLastUpdated() { - currField.setLastUpdated(new SimpleDateFormat(TIMESTAMP_FORMAT).format(new Date(currKey.getTimestamp()))); + currField.setLastUpdated(Instant.ofEpochMilli(currKey.getTimestamp()).atZone(ZoneId.systemDefault()).toLocalDateTime() + .format(DateTimeFormatter.ofPattern(TIMESTAMP_FORMAT))); } } } diff --git a/service/src/main/java/datawave/microservice/model/ModelController.java b/service/src/main/java/datawave/microservice/model/ModelController.java index fb0f135..4dcadc0 100644 --- a/service/src/main/java/datawave/microservice/model/ModelController.java +++ b/service/src/main/java/datawave/microservice/model/ModelController.java @@ -5,9 +5,9 @@ import datawave.microservice.authorization.user.DatawaveUserDetails; import datawave.microservice.http.converter.protostuff.ProtostuffHttpMessageConverter; import datawave.microservice.model.config.ModelProperties; -import datawave.webservice.model.FieldMapping; +import datawave.query.model.FieldMapping; +import datawave.query.model.ModelKeyParser; import datawave.webservice.model.Model; -import datawave.webservice.model.ModelKeyParser; import datawave.webservice.model.ModelList; import datawave.webservice.query.exception.DatawaveErrorCode; import datawave.webservice.query.exception.QueryException; @@ -61,7 +61,7 @@ public class ModelController { private static final HashSet RESERVED_COLF_VALUES = Sets.newHashSet("e", "i", "ri", "f", "tf", "m", "desc", "edge", "t", "n", "h"); public ModelController(ModelProperties modelProperties, AccumuloConnectionService accumloConnectionService) { - this.dataTablesUri = modelProperties.getDefaultTableName(); + this.dataTablesUri = modelProperties.getDataTablesUri(); this.jqueryUri = modelProperties.getJqueryUri(); this.accumloConnectionService = accumloConnectionService; } @@ -198,9 +198,8 @@ public Model getModel(@RequestParam String name, @RequestParam(defaultValue = DE return response; } - Set auths = accumloConnectionService.getConnection(modelTableName, name, currentUser).getAuths(); TreeSet fields = response.getFields(); - keys.forEach(k -> fields.add(ModelKeyParser.parseKey(k, auths))); + keys.forEach(k -> fields.add(ModelKeyParser.parseKey(k))); response.setName(name); return response; } @@ -219,7 +218,7 @@ public Model getModel(@RequestParam String name, @RequestParam(defaultValue = DE * @HTTP 500 internal server error */ @Operation(summary = "Insert a new field mapping into an existing model.") - @PostMapping(value = {"/insert", "/import"}) // If we get to change to standard REST, this would just be / and rely on the + @PostMapping(value = {"/insert", "/import"}) @Secured({"Administrator", "JBossAdministrator"}) public VoidResponse insertMapping(@RequestBody Model model, @RequestParam(defaultValue = DEFAULT_MODEL_TABLE_NAME) String modelTableName, @AuthenticationPrincipal DatawaveUserDetails currentUser) { diff --git a/service/src/test/java/datawave/microservice/model/ModelControllerTest.java b/service/src/test/java/datawave/microservice/model/ModelControllerTest.java index c01c60b..3d452b5 100644 --- a/service/src/test/java/datawave/microservice/model/ModelControllerTest.java +++ b/service/src/test/java/datawave/microservice/model/ModelControllerTest.java @@ -4,9 +4,9 @@ import datawave.accumulo.inmemory.InMemoryInstance; import datawave.microservice.ControllerIT; import datawave.microservice.model.config.ModelProperties; -import datawave.webservice.model.FieldMapping; +import datawave.query.model.FieldMapping; +import datawave.query.model.ModelKeyParser; import datawave.webservice.model.Model; -import datawave.webservice.model.ModelKeyParser; import datawave.webservice.model.ModelList; import datawave.webservice.result.VoidResponse; import org.apache.accumulo.core.client.AccumuloClient; From f02f4dfa887b1fbe23ef2b90ec4767879bace102 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 22 Jun 2023 12:23:49 -0400 Subject: [PATCH 42/54] Updated with latest changes from integration --- api/pom.xml | 2 +- service/src/test/resources/TestModel_1.xml | 2 +- service/src/test/resources/TestModel_2.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 5a4a83a..1105296 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -17,7 +17,7 @@ https://github.com/NationalSecurityAgency/datawave-dictionary-service - http://webservice.datawave/v1 + http://webservice.datawave.nsa/v1 3.0-SNAPSHOT 31.1-jre 3.0-SNAPSHOT diff --git a/service/src/test/resources/TestModel_1.xml b/service/src/test/resources/TestModel_1.xml index f7a784e..c4ff8e8 100644 --- a/service/src/test/resources/TestModel_1.xml +++ b/service/src/test/resources/TestModel_1.xml @@ -1,4 +1,4 @@ - + diff --git a/service/src/test/resources/TestModel_2.xml b/service/src/test/resources/TestModel_2.xml index cdc3e04..68019f2 100644 --- a/service/src/test/resources/TestModel_2.xml +++ b/service/src/test/resources/TestModel_2.xml @@ -1,4 +1,4 @@ - + From ed1a68e49024e6a94dcdf3821749f8da4511b461 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Tue, 27 Jun 2023 13:27:45 -0400 Subject: [PATCH 43/54] formatted code with impsort plugin. --- .../dictionary/data/DataDictionaryBase.java | 15 +++--- .../data/DefaultDataDictionary.java | 32 ++++++------ .../dictionary/data/DefaultDescription.java | 21 ++++---- .../data/DefaultDictionaryField.java | 20 +++---- .../dictionary/data/DefaultFields.java | 22 ++++---- .../dictionary/data/DescriptionBase.java | 9 ++-- .../dictionary/data/DictionaryFieldBase.java | 5 +- .../dictionary/data/FieldsBase.java | 12 +++-- .../edge/DefaultEdgeDictionary.java | 29 ++++++----- .../dictionary/edge/DefaultMetadata.java | 27 +++++----- .../dictionary/edge/EdgeDictionaryBase.java | 7 +-- .../dictionary/edge/EventField.java | 21 ++++---- .../dictionary/edge/MetadataBase.java | 12 +++-- .../metadata/DefaultMetadataField.java | 38 +++++++------- .../metadata/MetadataFieldBase.java | 13 ++--- .../datawave/webservice/model/Mapping.java | 12 +++-- .../java/datawave/webservice/model/Model.java | 15 +++--- .../datawave/webservice/model/ModelList.java | 15 +++--- .../data/DefaultDescriptionTest.java | 17 +++--- .../AccumuloConnectionService.java | 35 +++++++------ .../datawave/microservice/Connection.java | 5 +- .../dictionary/DataDictionaryController.java | 52 ++++++++++--------- .../dictionary/EdgeDictionaryController.java | 22 ++++---- .../config/DataDictionaryProperties.java | 12 +++-- .../DictionaryServiceConfiguration.java | 16 +++--- .../config/DictionaryServiceProperties.java | 5 +- .../config/EdgeDictionaryProperties.java | 9 ++-- .../dictionary/data/DataDictionary.java | 11 ++-- .../dictionary/data/DataDictionaryImpl.java | 20 +++---- .../dictionary/edge/EdgeDictionary.java | 7 +-- .../dictionary/edge/EdgeDictionaryImpl.java | 28 +++++----- .../metadata/DefaultMetadataFieldScanner.java | 46 ++++++++-------- .../metadata/MetadataDescriptionsHelper.java | 34 ++++++------ .../MetadataDescriptionsHelperFactory.java | 3 +- .../microservice/model/ModelController.java | 44 ++++++++-------- .../model/config/ModelProperties.java | 5 +- .../datawave/microservice/ControllerIT.java | 18 ++++--- .../DataDictionaryControllerTest.java | 32 ++++++------ .../data/DataDictionaryImplTest.java | 46 ++++++++-------- .../edge/EdgeDictionaryImplTest.java | 46 ++++++++-------- .../DefaultMetadataFieldScannerTest.java | 47 +++++++++-------- .../model/ModelControllerTest.java | 34 ++++++------ 42 files changed, 493 insertions(+), 426 deletions(-) diff --git a/api/src/main/java/datawave/webservice/dictionary/data/DataDictionaryBase.java b/api/src/main/java/datawave/webservice/dictionary/data/DataDictionaryBase.java index 4b207f2..0a4462f 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/DataDictionaryBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DataDictionaryBase.java @@ -1,18 +1,19 @@ package datawave.webservice.dictionary.data; +import java.util.Collection; +import java.util.List; +import java.util.function.Consumer; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlSeeAlso; + import datawave.webservice.HtmlProvider; import datawave.webservice.metadata.MetadataFieldBase; import datawave.webservice.result.BaseResponse; import datawave.webservice.result.TotalResultsAware; import io.protostuff.Message; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlSeeAlso; -import java.util.Collection; -import java.util.List; -import java.util.function.Consumer; - @XmlAccessorType(XmlAccessType.NONE) @XmlSeeAlso(DefaultDataDictionary.class) public abstract class DataDictionaryBase extends BaseResponse implements TotalResultsAware, Message, HtmlProvider { diff --git a/api/src/main/java/datawave/webservice/dictionary/data/DefaultDataDictionary.java b/api/src/main/java/datawave/webservice/dictionary/data/DefaultDataDictionary.java index 31ed9d0..e51fb71 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/DefaultDataDictionary.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DefaultDataDictionary.java @@ -1,13 +1,12 @@ package datawave.webservice.dictionary.data; -import com.google.common.collect.Lists; -import datawave.webservice.HtmlProvider; -import datawave.webservice.metadata.DefaultMetadataField; -import datawave.webservice.result.TotalResultsAware; -import io.protostuff.Input; -import io.protostuff.Message; -import io.protostuff.Output; -import io.protostuff.Schema; +import java.io.IOException; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.function.Consumer; import javax.xml.bind.annotation.XmlAccessOrder; import javax.xml.bind.annotation.XmlAccessType; @@ -17,13 +16,16 @@ import javax.xml.bind.annotation.XmlElementWrapper; import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlTransient; -import java.io.IOException; -import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.function.Consumer; + +import com.google.common.collect.Lists; + +import datawave.webservice.HtmlProvider; +import datawave.webservice.metadata.DefaultMetadataField; +import datawave.webservice.result.TotalResultsAware; +import io.protostuff.Input; +import io.protostuff.Message; +import io.protostuff.Output; +import io.protostuff.Schema; @XmlRootElement(name = "DefaultDataDictionary") @XmlAccessorType(XmlAccessType.NONE) diff --git a/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java b/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java index bd165e4..5dcb4e4 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DefaultDescription.java @@ -1,21 +1,22 @@ package datawave.webservice.dictionary.data; -import datawave.webservice.query.result.event.MapSchema; -import io.protostuff.Input; -import io.protostuff.Message; -import io.protostuff.Output; -import io.protostuff.Schema; +import java.io.IOException; +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; import javax.xml.bind.annotation.XmlAccessOrder; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorOrder; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlTransient; -import java.io.IOException; -import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; + +import datawave.webservice.query.result.event.MapSchema; +import io.protostuff.Input; +import io.protostuff.Message; +import io.protostuff.Output; +import io.protostuff.Schema; @XmlAccessorType(XmlAccessType.NONE) @XmlAccessorOrder(XmlAccessOrder.ALPHABETICAL) diff --git a/api/src/main/java/datawave/webservice/dictionary/data/DefaultDictionaryField.java b/api/src/main/java/datawave/webservice/dictionary/data/DefaultDictionaryField.java index 8934a10..f7567f0 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/DefaultDictionaryField.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DefaultDictionaryField.java @@ -1,10 +1,9 @@ package datawave.webservice.dictionary.data; -import com.google.common.collect.Sets; -import io.protostuff.Input; -import io.protostuff.Message; -import io.protostuff.Output; -import io.protostuff.Schema; +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.Set; import javax.xml.bind.annotation.XmlAccessOrder; import javax.xml.bind.annotation.XmlAccessType; @@ -12,10 +11,13 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlTransient; -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; -import java.util.Set; + +import com.google.common.collect.Sets; + +import io.protostuff.Input; +import io.protostuff.Message; +import io.protostuff.Output; +import io.protostuff.Schema; @XmlAccessorType(XmlAccessType.NONE) @XmlAccessorOrder(XmlAccessOrder.ALPHABETICAL) diff --git a/api/src/main/java/datawave/webservice/dictionary/data/DefaultFields.java b/api/src/main/java/datawave/webservice/dictionary/data/DefaultFields.java index ec5de52..7695115 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/DefaultFields.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DefaultFields.java @@ -1,7 +1,19 @@ package datawave.webservice.dictionary.data; +import java.io.IOException; +import java.util.List; +import java.util.Map.Entry; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlElementWrapper; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlTransient; + import com.google.common.collect.Lists; import com.google.common.collect.Multimap; + import datawave.webservice.HtmlProvider; import datawave.webservice.result.TotalResultsAware; import io.protostuff.Input; @@ -9,16 +21,6 @@ import io.protostuff.Output; import io.protostuff.Schema; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlElementWrapper; -import javax.xml.bind.annotation.XmlRootElement; -import javax.xml.bind.annotation.XmlTransient; -import java.io.IOException; -import java.util.List; -import java.util.Map.Entry; - @XmlRootElement(name = "DefaultFieldsResponse") @XmlAccessorType(XmlAccessType.NONE) public class DefaultFields extends FieldsBase diff --git a/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java b/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java index d83af58..851fdbb 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DescriptionBase.java @@ -1,14 +1,15 @@ package datawave.webservice.dictionary.data; -import datawave.webservice.query.result.event.HasMarkings; -import datawave.webservice.xml.util.StringMapAdapter; -import io.protostuff.Message; +import java.util.Map; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; -import java.util.Map; + +import datawave.webservice.query.result.event.HasMarkings; +import datawave.webservice.xml.util.StringMapAdapter; +import io.protostuff.Message; @XmlAccessorType(XmlAccessType.NONE) public abstract class DescriptionBase implements HasMarkings, Message { diff --git a/api/src/main/java/datawave/webservice/dictionary/data/DictionaryFieldBase.java b/api/src/main/java/datawave/webservice/dictionary/data/DictionaryFieldBase.java index 8522473..ba52d97 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/DictionaryFieldBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/DictionaryFieldBase.java @@ -1,11 +1,12 @@ package datawave.webservice.dictionary.data; -import io.protostuff.Message; +import java.util.Set; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlSeeAlso; -import java.util.Set; + +import io.protostuff.Message; @XmlAccessorType(XmlAccessType.NONE) @XmlSeeAlso(DefaultDictionaryField.class) diff --git a/api/src/main/java/datawave/webservice/dictionary/data/FieldsBase.java b/api/src/main/java/datawave/webservice/dictionary/data/FieldsBase.java index 6e751de..df1ae61 100644 --- a/api/src/main/java/datawave/webservice/dictionary/data/FieldsBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/data/FieldsBase.java @@ -1,14 +1,16 @@ package datawave.webservice.dictionary.data; -import com.google.common.collect.Multimap; -import datawave.webservice.result.BaseResponse; -import io.protostuff.Message; +import java.util.List; +import java.util.Map; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlSeeAlso; -import java.util.List; -import java.util.Map; + +import com.google.common.collect.Multimap; + +import datawave.webservice.result.BaseResponse; +import io.protostuff.Message; @XmlAccessorType(XmlAccessType.NONE) @XmlSeeAlso(DefaultFields.class) diff --git a/api/src/main/java/datawave/webservice/dictionary/edge/DefaultEdgeDictionary.java b/api/src/main/java/datawave/webservice/dictionary/edge/DefaultEdgeDictionary.java index a0d90fa..4f4dc92 100644 --- a/api/src/main/java/datawave/webservice/dictionary/edge/DefaultEdgeDictionary.java +++ b/api/src/main/java/datawave/webservice/dictionary/edge/DefaultEdgeDictionary.java @@ -1,13 +1,11 @@ package datawave.webservice.dictionary.edge; -import datawave.webservice.HtmlProvider; -import datawave.webservice.query.result.util.protostuff.FieldAccessor; -import datawave.webservice.query.result.util.protostuff.ProtostuffField; -import datawave.webservice.result.TotalResultsAware; -import io.protostuff.Input; -import io.protostuff.Message; -import io.protostuff.Output; -import io.protostuff.Schema; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; import javax.xml.bind.annotation.XmlAccessOrder; import javax.xml.bind.annotation.XmlAccessType; @@ -17,12 +15,15 @@ import javax.xml.bind.annotation.XmlElementWrapper; import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlTransient; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; + +import datawave.webservice.HtmlProvider; +import datawave.webservice.query.result.util.protostuff.FieldAccessor; +import datawave.webservice.query.result.util.protostuff.ProtostuffField; +import datawave.webservice.result.TotalResultsAware; +import io.protostuff.Input; +import io.protostuff.Message; +import io.protostuff.Output; +import io.protostuff.Schema; @XmlRootElement(name = "EdgeDictionary") @XmlAccessorType(XmlAccessType.NONE) diff --git a/api/src/main/java/datawave/webservice/dictionary/edge/DefaultMetadata.java b/api/src/main/java/datawave/webservice/dictionary/edge/DefaultMetadata.java index 9620e33..0f84c6b 100644 --- a/api/src/main/java/datawave/webservice/dictionary/edge/DefaultMetadata.java +++ b/api/src/main/java/datawave/webservice/dictionary/edge/DefaultMetadata.java @@ -1,13 +1,9 @@ package datawave.webservice.dictionary.edge; -import com.google.common.base.Objects; -import datawave.webservice.query.result.util.protostuff.FieldAccessor; -import datawave.webservice.query.result.util.protostuff.ProtostuffField; -import io.protostuff.Input; -import io.protostuff.Message; -import io.protostuff.Output; -import io.protostuff.Schema; -import org.apache.commons.lang3.StringUtils; +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -16,10 +12,17 @@ import javax.xml.bind.annotation.XmlElementWrapper; import javax.xml.bind.annotation.XmlTransient; import javax.xml.bind.annotation.XmlType; -import java.io.IOException; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; + +import org.apache.commons.lang3.StringUtils; + +import com.google.common.base.Objects; + +import datawave.webservice.query.result.util.protostuff.FieldAccessor; +import datawave.webservice.query.result.util.protostuff.ProtostuffField; +import io.protostuff.Input; +import io.protostuff.Message; +import io.protostuff.Output; +import io.protostuff.Schema; @XmlAccessorType(XmlAccessType.NONE) @XmlType(propOrder = {"edgeType", "edgeRelationship", "edgeAttribute1Source", "eventFields"}) diff --git a/api/src/main/java/datawave/webservice/dictionary/edge/EdgeDictionaryBase.java b/api/src/main/java/datawave/webservice/dictionary/edge/EdgeDictionaryBase.java index 886a1d8..09440ac 100644 --- a/api/src/main/java/datawave/webservice/dictionary/edge/EdgeDictionaryBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/edge/EdgeDictionaryBase.java @@ -1,12 +1,13 @@ package datawave.webservice.dictionary.edge; -import datawave.webservice.result.BaseResponse; -import io.protostuff.Message; +import java.util.List; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlSeeAlso; -import java.util.List; + +import datawave.webservice.result.BaseResponse; +import io.protostuff.Message; @XmlAccessorType(XmlAccessType.NONE) @XmlSeeAlso(DefaultEdgeDictionary.class) diff --git a/api/src/main/java/datawave/webservice/dictionary/edge/EventField.java b/api/src/main/java/datawave/webservice/dictionary/edge/EventField.java index 041b94d..12f1f7a 100644 --- a/api/src/main/java/datawave/webservice/dictionary/edge/EventField.java +++ b/api/src/main/java/datawave/webservice/dictionary/edge/EventField.java @@ -1,21 +1,24 @@ package datawave.webservice.dictionary.edge; +import java.io.IOException; +import java.io.Serializable; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlTransient; +import javax.xml.bind.annotation.XmlType; + +import org.apache.commons.lang3.StringUtils; + import com.google.common.base.Objects; + import datawave.webservice.query.result.util.protostuff.FieldAccessor; import datawave.webservice.query.result.util.protostuff.ProtostuffField; import io.protostuff.Input; import io.protostuff.Message; import io.protostuff.Output; import io.protostuff.Schema; -import org.apache.commons.lang3.StringUtils; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlAttribute; -import javax.xml.bind.annotation.XmlTransient; -import javax.xml.bind.annotation.XmlType; -import java.io.IOException; -import java.io.Serializable; @XmlAccessorType(XmlAccessType.NONE) @XmlType(propOrder = {"sourceField", "sinkField", "enrichmentField", "enrichmentIndex", "jexlPrecondition"}) diff --git a/api/src/main/java/datawave/webservice/dictionary/edge/MetadataBase.java b/api/src/main/java/datawave/webservice/dictionary/edge/MetadataBase.java index 4e5c240..ed18822 100644 --- a/api/src/main/java/datawave/webservice/dictionary/edge/MetadataBase.java +++ b/api/src/main/java/datawave/webservice/dictionary/edge/MetadataBase.java @@ -1,14 +1,16 @@ package datawave.webservice.dictionary.edge; -import com.google.common.collect.Maps; -import datawave.webservice.query.result.event.HasMarkings; -import io.protostuff.Message; +import java.util.List; +import java.util.Map; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlSeeAlso; -import java.util.List; -import java.util.Map; + +import com.google.common.collect.Maps; + +import datawave.webservice.query.result.event.HasMarkings; +import io.protostuff.Message; @XmlAccessorType(XmlAccessType.NONE) @XmlSeeAlso(DefaultMetadata.class) diff --git a/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java b/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java index 294c6ac..9f00e1b 100644 --- a/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java +++ b/api/src/main/java/datawave/webservice/metadata/DefaultMetadataField.java @@ -1,23 +1,5 @@ package datawave.webservice.metadata; -import datawave.webservice.dictionary.data.DefaultDescription; -import datawave.webservice.dictionary.data.DescriptionBase; -import datawave.webservice.query.result.event.MapSchema; -import io.protostuff.Input; -import io.protostuff.Message; -import io.protostuff.Output; -import io.protostuff.Schema; -import lombok.Data; -import org.apache.commons.lang3.builder.EqualsBuilder; -import org.apache.commons.lang3.builder.HashCodeBuilder; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlAttribute; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlElementWrapper; -import javax.xml.bind.annotation.XmlTransient; -import javax.xml.bind.annotation.XmlType; import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; @@ -28,6 +10,26 @@ import java.util.Map; import java.util.Set; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlElementWrapper; +import javax.xml.bind.annotation.XmlTransient; +import javax.xml.bind.annotation.XmlType; + +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; + +import datawave.webservice.dictionary.data.DefaultDescription; +import datawave.webservice.dictionary.data.DescriptionBase; +import datawave.webservice.query.result.event.MapSchema; +import io.protostuff.Input; +import io.protostuff.Message; +import io.protostuff.Output; +import io.protostuff.Schema; +import lombok.Data; + @Data @XmlAccessorType(XmlAccessType.NONE) @XmlType(propOrder = {"fieldName", "internalFieldName", "dataType", "descriptions", "indexOnly", "forwardIndexed", "reverseIndexed", "normalized", "tokenized", diff --git a/api/src/main/java/datawave/webservice/metadata/MetadataFieldBase.java b/api/src/main/java/datawave/webservice/metadata/MetadataFieldBase.java index b13ea8d..ecbd4bc 100644 --- a/api/src/main/java/datawave/webservice/metadata/MetadataFieldBase.java +++ b/api/src/main/java/datawave/webservice/metadata/MetadataFieldBase.java @@ -1,16 +1,17 @@ package datawave.webservice.metadata; -import datawave.webservice.dictionary.data.DescriptionBase; -import io.protostuff.Message; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlSeeAlso; import java.io.Serializable; import java.util.Collection; import java.util.List; import java.util.Set; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlSeeAlso; + +import datawave.webservice.dictionary.data.DescriptionBase; +import io.protostuff.Message; + @XmlAccessorType(XmlAccessType.NONE) @XmlSeeAlso(DefaultMetadataField.class) public abstract class MetadataFieldBase implements Serializable, Message { diff --git a/api/src/main/java/datawave/webservice/model/Mapping.java b/api/src/main/java/datawave/webservice/model/Mapping.java index 985842c..4c61591 100644 --- a/api/src/main/java/datawave/webservice/model/Mapping.java +++ b/api/src/main/java/datawave/webservice/model/Mapping.java @@ -1,14 +1,16 @@ package datawave.webservice.model; -import datawave.query.model.Direction; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.apache.commons.lang3.builder.CompareToBuilder; +import java.io.Serializable; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAttribute; -import java.io.Serializable; + +import org.apache.commons.lang3.builder.CompareToBuilder; + +import datawave.query.model.Direction; +import lombok.Data; +import lombok.NoArgsConstructor; @Data @NoArgsConstructor diff --git a/api/src/main/java/datawave/webservice/model/Model.java b/api/src/main/java/datawave/webservice/model/Model.java index 6aab3d5..48cbfda 100644 --- a/api/src/main/java/datawave/webservice/model/Model.java +++ b/api/src/main/java/datawave/webservice/model/Model.java @@ -1,9 +1,8 @@ package datawave.webservice.model; -import datawave.query.model.FieldMapping; -import datawave.webservice.HtmlProvider; -import datawave.webservice.result.BaseResponse; -import lombok.Data; +import java.io.Serializable; +import java.text.MessageFormat; +import java.util.TreeSet; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -11,9 +10,11 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlElementWrapper; import javax.xml.bind.annotation.XmlRootElement; -import java.io.Serializable; -import java.text.MessageFormat; -import java.util.TreeSet; + +import datawave.query.model.FieldMapping; +import datawave.webservice.HtmlProvider; +import datawave.webservice.result.BaseResponse; +import lombok.Data; @Data @XmlRootElement(name = "Model") diff --git a/api/src/main/java/datawave/webservice/model/ModelList.java b/api/src/main/java/datawave/webservice/model/ModelList.java index ec793ec..0ee6e2e 100644 --- a/api/src/main/java/datawave/webservice/model/ModelList.java +++ b/api/src/main/java/datawave/webservice/model/ModelList.java @@ -1,16 +1,17 @@ package datawave.webservice.model; -import datawave.webservice.HtmlProvider; -import datawave.webservice.result.BaseResponse; -import lombok.Data; -import lombok.NoArgsConstructor; +import java.io.Serializable; +import java.text.MessageFormat; +import java.util.HashSet; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlElementWrapper; import javax.xml.bind.annotation.XmlRootElement; -import java.io.Serializable; -import java.text.MessageFormat; -import java.util.HashSet; + +import datawave.webservice.HtmlProvider; +import datawave.webservice.result.BaseResponse; +import lombok.Data; +import lombok.NoArgsConstructor; @Data @NoArgsConstructor diff --git a/api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java b/api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java index fdaa73d..4834a6e 100644 --- a/api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java +++ b/api/src/test/java/datawave/webservice/dictionary/data/DefaultDescriptionTest.java @@ -1,12 +1,8 @@ package datawave.webservice.dictionary.data; -import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; -import javax.xml.bind.JAXBContext; -import javax.xml.bind.JAXBException; -import javax.xml.bind.Marshaller; -import javax.xml.bind.Unmarshaller; -import javax.xml.bind.annotation.XmlSchema; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.lang.annotation.Annotation; @@ -17,8 +13,13 @@ import java.util.Map; import java.util.Set; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; +import javax.xml.bind.Unmarshaller; +import javax.xml.bind.annotation.XmlSchema; + +import org.junit.jupiter.api.Test; public class DefaultDescriptionTest { diff --git a/service/src/main/java/datawave/microservice/AccumuloConnectionService.java b/service/src/main/java/datawave/microservice/AccumuloConnectionService.java index deb24ae..b1d6701 100644 --- a/service/src/main/java/datawave/microservice/AccumuloConnectionService.java +++ b/service/src/main/java/datawave/microservice/AccumuloConnectionService.java @@ -1,14 +1,14 @@ package datawave.microservice; -import datawave.accumulo.util.security.UserAuthFunctions; -import datawave.microservice.authorization.user.DatawaveUserDetails; -import datawave.microservice.dictionary.config.DataDictionaryProperties; -import datawave.security.authorization.DatawaveUser; -import datawave.security.util.ScannerHelper; -import datawave.webservice.query.exception.DatawaveErrorCode; -import datawave.webservice.query.exception.QueryException; -import lombok.Cleanup; -import lombok.extern.slf4j.Slf4j; +import java.util.List; +import java.util.Set; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.BatchWriterConfig; @@ -24,14 +24,15 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; -import java.util.List; -import java.util.Set; -import java.util.Spliterator; -import java.util.Spliterators; -import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; +import datawave.accumulo.util.security.UserAuthFunctions; +import datawave.microservice.authorization.user.DatawaveUserDetails; +import datawave.microservice.dictionary.config.DataDictionaryProperties; +import datawave.security.authorization.DatawaveUser; +import datawave.security.util.ScannerHelper; +import datawave.webservice.query.exception.DatawaveErrorCode; +import datawave.webservice.query.exception.QueryException; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; /** * Provides accumlo functionality for use by the dictionary controllers. Keeps the accumulo interface separate from the controllers. diff --git a/service/src/main/java/datawave/microservice/Connection.java b/service/src/main/java/datawave/microservice/Connection.java index fb841c9..5fb9abc 100644 --- a/service/src/main/java/datawave/microservice/Connection.java +++ b/service/src/main/java/datawave/microservice/Connection.java @@ -1,10 +1,11 @@ package datawave.microservice; -import lombok.Data; +import java.util.Set; + import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.security.Authorizations; -import java.util.Set; +import lombok.Data; @Data public class Connection { diff --git a/service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java b/service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java index d43b0e2..66830e8 100644 --- a/service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java +++ b/service/src/main/java/datawave/microservice/dictionary/DataDictionaryController.java @@ -1,9 +1,36 @@ package datawave.microservice.dictionary; +import static datawave.microservice.http.converter.protostuff.ProtostuffHttpMessageConverter.PROTOSTUFF_VALUE; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.function.Consumer; + +import org.apache.commons.lang.StringUtils; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.http.MediaType; +import org.springframework.security.access.annotation.Secured; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + import com.codahale.metrics.annotation.Timed; import com.google.common.collect.HashMultimap; import com.google.common.collect.Maps; import com.google.common.collect.Multimap; + import datawave.microservice.AccumuloConnectionService; import datawave.microservice.Connection; import datawave.microservice.authorization.user.DatawaveUserDetails; @@ -18,31 +45,6 @@ import datawave.webservice.result.VoidResponse; import io.swagger.v3.oas.annotations.ExternalDocumentation; import io.swagger.v3.oas.annotations.tags.Tag; -import org.apache.commons.lang.StringUtils; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.http.MediaType; -import org.springframework.security.access.annotation.Secured; -import org.springframework.security.core.annotation.AuthenticationPrincipal; -import org.springframework.web.bind.annotation.DeleteMapping; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.PutMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; - -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.function.Consumer; - -import static datawave.microservice.http.converter.protostuff.ProtostuffHttpMessageConverter.PROTOSTUFF_VALUE; @Tag(name = "Data Dictionary Controller /v1", description = "DataWave Dictionary Operations", externalDocs = @ExternalDocumentation(description = "Dictionary Service Documentation", diff --git a/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java b/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java index ffa1401..4edf5d1 100644 --- a/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java +++ b/service/src/main/java/datawave/microservice/dictionary/EdgeDictionaryController.java @@ -1,6 +1,18 @@ package datawave.microservice.dictionary; +import static datawave.microservice.http.converter.protostuff.ProtostuffHttpMessageConverter.PROTOSTUFF_VALUE; + +import org.apache.commons.lang.StringUtils; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.http.MediaType; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + import com.codahale.metrics.annotation.Timed; + import datawave.accumulo.util.security.UserAuthFunctions; import datawave.microservice.AccumuloConnectionService; import datawave.microservice.authorization.user.DatawaveUserDetails; @@ -11,16 +23,6 @@ import io.swagger.v3.oas.annotations.ExternalDocumentation; import io.swagger.v3.oas.annotations.tags.Tag; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang.StringUtils; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.http.MediaType; -import org.springframework.security.core.annotation.AuthenticationPrincipal; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; - -import static datawave.microservice.http.converter.protostuff.ProtostuffHttpMessageConverter.PROTOSTUFF_VALUE; @Tag(name = "Edge Dictionary Controller /v1", description = "DataWave Edge Dictionary Operations", externalDocs = @ExternalDocumentation(description = "Dictionary Service Documentation", diff --git a/service/src/main/java/datawave/microservice/dictionary/config/DataDictionaryProperties.java b/service/src/main/java/datawave/microservice/dictionary/config/DataDictionaryProperties.java index b2b43b4..459e9f7 100644 --- a/service/src/main/java/datawave/microservice/dictionary/config/DataDictionaryProperties.java +++ b/service/src/main/java/datawave/microservice/dictionary/config/DataDictionaryProperties.java @@ -1,13 +1,15 @@ package datawave.microservice.dictionary.config; -import lombok.Getter; -import lombok.Setter; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.validation.annotation.Validated; +import java.util.Map; import javax.validation.constraints.NotBlank; import javax.validation.constraints.Positive; -import java.util.Map; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +import lombok.Getter; +import lombok.Setter; @Getter @Setter diff --git a/service/src/main/java/datawave/microservice/dictionary/config/DictionaryServiceConfiguration.java b/service/src/main/java/datawave/microservice/dictionary/config/DictionaryServiceConfiguration.java index 6175a5f..d263cfc 100644 --- a/service/src/main/java/datawave/microservice/dictionary/config/DictionaryServiceConfiguration.java +++ b/service/src/main/java/datawave/microservice/dictionary/config/DictionaryServiceConfiguration.java @@ -1,6 +1,15 @@ package datawave.microservice.dictionary.config; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.jackson.Jackson2ObjectMapperBuilderCustomizer; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Scope; + import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + import datawave.accumulo.util.security.UserAuthFunctions; import datawave.marking.MarkingFunctions; import datawave.microservice.config.accumulo.AccumuloProperties; @@ -18,13 +27,6 @@ import datawave.webservice.dictionary.data.DefaultFields; import datawave.webservice.dictionary.data.FieldsBase; import datawave.webservice.metadata.DefaultMetadataField; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.boot.autoconfigure.jackson.Jackson2ObjectMapperBuilderCustomizer; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Scope; @Configuration @EnableConfigurationProperties(DictionaryServiceProperties.class) diff --git a/service/src/main/java/datawave/microservice/dictionary/config/DictionaryServiceProperties.java b/service/src/main/java/datawave/microservice/dictionary/config/DictionaryServiceProperties.java index 3f85b26..2adea91 100644 --- a/service/src/main/java/datawave/microservice/dictionary/config/DictionaryServiceProperties.java +++ b/service/src/main/java/datawave/microservice/dictionary/config/DictionaryServiceProperties.java @@ -1,10 +1,11 @@ package datawave.microservice.dictionary.config; -import datawave.microservice.config.accumulo.AccumuloProperties; +import javax.validation.Valid; + import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.validation.annotation.Validated; -import javax.validation.Valid; +import datawave.microservice.config.accumulo.AccumuloProperties; @ConfigurationProperties(prefix = "datawave.dictionary") @Validated diff --git a/service/src/main/java/datawave/microservice/dictionary/config/EdgeDictionaryProperties.java b/service/src/main/java/datawave/microservice/dictionary/config/EdgeDictionaryProperties.java index 4ccb671..4023a02 100644 --- a/service/src/main/java/datawave/microservice/dictionary/config/EdgeDictionaryProperties.java +++ b/service/src/main/java/datawave/microservice/dictionary/config/EdgeDictionaryProperties.java @@ -1,12 +1,13 @@ package datawave.microservice.dictionary.config; -import lombok.Getter; -import lombok.Setter; +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.Positive; + import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.validation.annotation.Validated; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.Positive; +import lombok.Getter; +import lombok.Setter; @Getter @Setter diff --git a/service/src/main/java/datawave/microservice/dictionary/data/DataDictionary.java b/service/src/main/java/datawave/microservice/dictionary/data/DataDictionary.java index bfe3e97..c2b64a2 100644 --- a/service/src/main/java/datawave/microservice/dictionary/data/DataDictionary.java +++ b/service/src/main/java/datawave/microservice/dictionary/data/DataDictionary.java @@ -1,16 +1,17 @@ package datawave.microservice.dictionary.data; +import java.util.Collection; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + import com.google.common.collect.Multimap; + import datawave.microservice.Connection; import datawave.webservice.dictionary.data.DescriptionBase; import datawave.webservice.dictionary.data.DictionaryFieldBase; import datawave.webservice.metadata.MetadataFieldBase; -import java.util.Collection; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; - public interface DataDictionary,DESC extends DescriptionBase,FIELD extends DictionaryFieldBase> { Map getNormalizationMap(); diff --git a/service/src/main/java/datawave/microservice/dictionary/data/DataDictionaryImpl.java b/service/src/main/java/datawave/microservice/dictionary/data/DataDictionaryImpl.java index 7c43479..85dd4b8 100644 --- a/service/src/main/java/datawave/microservice/dictionary/data/DataDictionaryImpl.java +++ b/service/src/main/java/datawave/microservice/dictionary/data/DataDictionaryImpl.java @@ -1,8 +1,19 @@ package datawave.microservice.dictionary.data; +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; + +import org.apache.accumulo.core.client.TableNotFoundException; + import com.google.common.collect.HashMultimap; import com.google.common.collect.Maps; import com.google.common.collect.Multimap; + import datawave.marking.MarkingFunctions; import datawave.microservice.Connection; import datawave.microservice.dictionary.config.ResponseObjectFactory; @@ -18,15 +29,6 @@ import datawave.webservice.dictionary.data.DefaultDictionaryField; import datawave.webservice.dictionary.data.DefaultFields; import datawave.webservice.metadata.DefaultMetadataField; -import org.apache.accumulo.core.client.TableNotFoundException; - -import java.util.Collection; -import java.util.Collections; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; public class DataDictionaryImpl implements DataDictionary { diff --git a/service/src/main/java/datawave/microservice/dictionary/edge/EdgeDictionary.java b/service/src/main/java/datawave/microservice/dictionary/edge/EdgeDictionary.java index ea6d9ea..a7ba058 100644 --- a/service/src/main/java/datawave/microservice/dictionary/edge/EdgeDictionary.java +++ b/service/src/main/java/datawave/microservice/dictionary/edge/EdgeDictionary.java @@ -1,11 +1,12 @@ package datawave.microservice.dictionary.edge; -import datawave.webservice.dictionary.edge.EdgeDictionaryBase; -import datawave.webservice.dictionary.edge.MetadataBase; +import java.util.Set; + import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.security.Authorizations; -import java.util.Set; +import datawave.webservice.dictionary.edge.EdgeDictionaryBase; +import datawave.webservice.dictionary.edge.MetadataBase; public interface EdgeDictionary,META extends MetadataBase> { char COL_SEPARATOR = '/'; diff --git a/service/src/main/java/datawave/microservice/dictionary/edge/EdgeDictionaryImpl.java b/service/src/main/java/datawave/microservice/dictionary/edge/EdgeDictionaryImpl.java index d8813a8..ff691b0 100644 --- a/service/src/main/java/datawave/microservice/dictionary/edge/EdgeDictionaryImpl.java +++ b/service/src/main/java/datawave/microservice/dictionary/edge/EdgeDictionaryImpl.java @@ -1,7 +1,22 @@ package datawave.microservice.dictionary.edge; +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; +import java.util.Map.Entry; +import java.util.Set; + +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.hadoop.io.Text; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.google.common.collect.SetMultimap; import com.google.protobuf.InvalidProtocolBufferException; + import datawave.metadata.protobuf.EdgeMetadata; import datawave.metadata.protobuf.EdgeMetadata.MetadataValue; import datawave.query.util.MetadataHelper; @@ -10,19 +25,6 @@ import datawave.webservice.dictionary.edge.DefaultEdgeDictionary; import datawave.webservice.dictionary.edge.DefaultMetadata; import datawave.webservice.dictionary.edge.EventField; -import org.apache.accumulo.core.client.AccumuloClient; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.security.Authorizations; -import org.apache.hadoop.io.Text; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Collection; -import java.util.LinkedList; -import java.util.List; -import java.util.Map.Entry; -import java.util.Set; public class EdgeDictionaryImpl implements EdgeDictionary { private static final Logger log = LoggerFactory.getLogger(EdgeDictionaryImpl.class); diff --git a/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java b/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java index d76b02e..bd5a845 100644 --- a/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java +++ b/service/src/main/java/datawave/microservice/metadata/DefaultMetadataFieldScanner.java @@ -1,27 +1,5 @@ package datawave.microservice.metadata; -import com.google.common.collect.Maps; -import datawave.data.ColumnFamilyConstants; -import datawave.marking.MarkingFunctions; -import datawave.microservice.Connection; -import datawave.microservice.dictionary.config.ResponseObjectFactory; -import datawave.security.util.ScannerHelper; -import datawave.webservice.dictionary.data.DefaultDataDictionary; -import datawave.webservice.dictionary.data.DefaultDescription; -import datawave.webservice.dictionary.data.DefaultDictionaryField; -import datawave.webservice.dictionary.data.DefaultFields; -import datawave.webservice.metadata.DefaultMetadataField; -import org.apache.accumulo.core.client.BatchScanner; -import org.apache.accumulo.core.client.IteratorSetting; -import org.apache.accumulo.core.client.TableNotFoundException; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Range; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.user.WholeRowIterator; -import org.apache.hadoop.io.Text; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.text.SimpleDateFormat; import java.time.Instant; @@ -38,6 +16,30 @@ import java.util.SortedMap; import java.util.stream.Collectors; +import org.apache.accumulo.core.client.BatchScanner; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.user.WholeRowIterator; +import org.apache.hadoop.io.Text; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Maps; + +import datawave.data.ColumnFamilyConstants; +import datawave.marking.MarkingFunctions; +import datawave.microservice.Connection; +import datawave.microservice.dictionary.config.ResponseObjectFactory; +import datawave.security.util.ScannerHelper; +import datawave.webservice.dictionary.data.DefaultDataDictionary; +import datawave.webservice.dictionary.data.DefaultDescription; +import datawave.webservice.dictionary.data.DefaultDictionaryField; +import datawave.webservice.dictionary.data.DefaultFields; +import datawave.webservice.metadata.DefaultMetadataField; + public class DefaultMetadataFieldScanner { private static final Logger log = LoggerFactory.getLogger(DefaultMetadataFieldScanner.class); diff --git a/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java b/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java index c7dc51b..8bba8c9 100644 --- a/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java +++ b/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelper.java @@ -1,14 +1,14 @@ package datawave.microservice.metadata; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.Multimaps; -import com.google.common.collect.SetMultimap; -import datawave.data.ColumnFamilyConstants; -import datawave.marking.MarkingFunctions; -import datawave.microservice.dictionary.config.ResponseObjectFactory; -import datawave.query.util.MetadataEntry; -import datawave.security.util.ScannerHelper; -import datawave.webservice.dictionary.data.DescriptionBase; +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.util.Collections; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.BatchWriterConfig; @@ -28,14 +28,16 @@ import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; -import java.util.Collections; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimaps; +import com.google.common.collect.SetMultimap; -import static java.nio.charset.StandardCharsets.UTF_8; +import datawave.data.ColumnFamilyConstants; +import datawave.marking.MarkingFunctions; +import datawave.microservice.dictionary.config.ResponseObjectFactory; +import datawave.query.util.MetadataEntry; +import datawave.security.util.ScannerHelper; +import datawave.webservice.dictionary.data.DescriptionBase; /** * Helper class to handle get/set of descriptions on the metadata table. diff --git a/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelperFactory.java b/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelperFactory.java index 326aceb..6d9e57d 100644 --- a/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelperFactory.java +++ b/service/src/main/java/datawave/microservice/metadata/MetadataDescriptionsHelperFactory.java @@ -1,9 +1,10 @@ package datawave.microservice.metadata; -import datawave.webservice.dictionary.data.DescriptionBase; import org.springframework.beans.factory.annotation.Lookup; import org.springframework.stereotype.Component; +import datawave.webservice.dictionary.data.DescriptionBase; + @Component public class MetadataDescriptionsHelperFactory> { @Lookup diff --git a/service/src/main/java/datawave/microservice/model/ModelController.java b/service/src/main/java/datawave/microservice/model/ModelController.java index 4dcadc0..5396dc8 100644 --- a/service/src/main/java/datawave/microservice/model/ModelController.java +++ b/service/src/main/java/datawave/microservice/model/ModelController.java @@ -1,21 +1,11 @@ package datawave.microservice.model; -import com.google.common.collect.Sets; -import datawave.microservice.AccumuloConnectionService; -import datawave.microservice.authorization.user.DatawaveUserDetails; -import datawave.microservice.http.converter.protostuff.ProtostuffHttpMessageConverter; -import datawave.microservice.model.config.ModelProperties; -import datawave.query.model.FieldMapping; -import datawave.query.model.ModelKeyParser; -import datawave.webservice.model.Model; -import datawave.webservice.model.ModelList; -import datawave.webservice.query.exception.DatawaveErrorCode; -import datawave.webservice.query.exception.QueryException; -import datawave.webservice.result.VoidResponse; -import io.swagger.v3.oas.annotations.ExternalDocumentation; -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.tags.Tag; -import lombok.extern.slf4j.Slf4j; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; + import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; @@ -32,11 +22,23 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.TreeSet; -import java.util.stream.Collectors; +import com.google.common.collect.Sets; + +import datawave.microservice.AccumuloConnectionService; +import datawave.microservice.authorization.user.DatawaveUserDetails; +import datawave.microservice.http.converter.protostuff.ProtostuffHttpMessageConverter; +import datawave.microservice.model.config.ModelProperties; +import datawave.query.model.FieldMapping; +import datawave.query.model.ModelKeyParser; +import datawave.webservice.model.Model; +import datawave.webservice.model.ModelList; +import datawave.webservice.query.exception.DatawaveErrorCode; +import datawave.webservice.query.exception.QueryException; +import datawave.webservice.result.VoidResponse; +import io.swagger.v3.oas.annotations.ExternalDocumentation; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.extern.slf4j.Slf4j; /** * Service that supports manipulation of models. The models are contained in the data dictionary table. diff --git a/service/src/main/java/datawave/microservice/model/config/ModelProperties.java b/service/src/main/java/datawave/microservice/model/config/ModelProperties.java index 3b9f433..560fb32 100644 --- a/service/src/main/java/datawave/microservice/model/config/ModelProperties.java +++ b/service/src/main/java/datawave/microservice/model/config/ModelProperties.java @@ -1,10 +1,11 @@ package datawave.microservice.model.config; -import lombok.Getter; -import lombok.Setter; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.validation.annotation.Validated; +import lombok.Getter; +import lombok.Setter; + @Getter @Setter @Validated diff --git a/service/src/test/java/datawave/microservice/ControllerIT.java b/service/src/test/java/datawave/microservice/ControllerIT.java index c12ef57..01e4c2b 100644 --- a/service/src/test/java/datawave/microservice/ControllerIT.java +++ b/service/src/test/java/datawave/microservice/ControllerIT.java @@ -1,11 +1,8 @@ package datawave.microservice; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Sets; -import datawave.microservice.authorization.jwt.JWTRestTemplate; -import datawave.microservice.authorization.user.DatawaveUserDetails; -import datawave.security.authorization.DatawaveUser; -import datawave.security.authorization.SubjectIssuerDNPair; +import java.util.Collections; +import java.util.HashSet; + import org.apache.accumulo.core.client.AccumuloClient; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.TestInstance; @@ -20,8 +17,13 @@ import org.springframework.web.client.DefaultResponseErrorHandler; import org.springframework.web.client.ResponseErrorHandler; -import java.util.Collections; -import java.util.HashSet; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Sets; + +import datawave.microservice.authorization.jwt.JWTRestTemplate; +import datawave.microservice.authorization.user.DatawaveUserDetails; +import datawave.security.authorization.DatawaveUser; +import datawave.security.authorization.SubjectIssuerDNPair; /** * Provides a default setup for controller integration tests (specifically). Contains properties and setup that are common to controller integration tests. The diff --git a/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java b/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java index 5a0999a..445e1e9 100644 --- a/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java +++ b/service/src/test/java/datawave/microservice/dictionary/DataDictionaryControllerTest.java @@ -1,16 +1,11 @@ package datawave.microservice.dictionary; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.Multimap; -import datawave.accumulo.inmemory.InMemoryAccumuloClient; -import datawave.accumulo.inmemory.InMemoryInstance; -import datawave.marking.MarkingFunctions; -import datawave.microservice.ControllerIT; -import datawave.microservice.dictionary.config.DataDictionaryProperties; -import datawave.webservice.dictionary.data.DefaultDataDictionary; -import datawave.webservice.dictionary.data.DefaultDescription; -import datawave.webservice.dictionary.data.DefaultFields; -import datawave.webservice.result.VoidResponse; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.AbstractMap; +import java.util.HashMap; +import java.util.Map; + import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.TableExistsException; @@ -32,11 +27,18 @@ import org.springframework.web.util.UriComponents; import org.springframework.web.util.UriComponentsBuilder; -import java.util.AbstractMap; -import java.util.HashMap; -import java.util.Map; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; -import static org.junit.jupiter.api.Assertions.assertEquals; +import datawave.accumulo.inmemory.InMemoryAccumuloClient; +import datawave.accumulo.inmemory.InMemoryInstance; +import datawave.marking.MarkingFunctions; +import datawave.microservice.ControllerIT; +import datawave.microservice.dictionary.config.DataDictionaryProperties; +import datawave.webservice.dictionary.data.DefaultDataDictionary; +import datawave.webservice.dictionary.data.DefaultDescription; +import datawave.webservice.dictionary.data.DefaultFields; +import datawave.webservice.result.VoidResponse; public class DataDictionaryControllerTest extends ControllerIT { diff --git a/service/src/test/java/datawave/microservice/dictionary/data/DataDictionaryImplTest.java b/service/src/test/java/datawave/microservice/dictionary/data/DataDictionaryImplTest.java index 3e493e5..ee899cb 100644 --- a/service/src/test/java/datawave/microservice/dictionary/data/DataDictionaryImplTest.java +++ b/service/src/test/java/datawave/microservice/dictionary/data/DataDictionaryImplTest.java @@ -1,9 +1,33 @@ package datawave.microservice.dictionary.data; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; + +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.security.Authorizations; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + import com.google.common.collect.HashMultimap; import com.google.common.collect.Maps; import com.google.common.collect.Multimap; import com.google.common.collect.SetMultimap; + import datawave.marking.MarkingFunctions; import datawave.microservice.Connection; import datawave.microservice.dictionary.config.ResponseObjectFactory; @@ -18,28 +42,6 @@ import datawave.webservice.dictionary.data.DefaultDictionaryField; import datawave.webservice.dictionary.data.DefaultFields; import datawave.webservice.metadata.DefaultMetadataField; -import org.apache.accumulo.core.client.AccumuloClient; -import org.apache.accumulo.core.client.TableNotFoundException; -import org.apache.accumulo.core.security.Authorizations; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ExecutionException; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) public class DataDictionaryImplTest { diff --git a/service/src/test/java/datawave/microservice/dictionary/edge/EdgeDictionaryImplTest.java b/service/src/test/java/datawave/microservice/dictionary/edge/EdgeDictionaryImplTest.java index d02b8cd..e6fe25c 100644 --- a/service/src/test/java/datawave/microservice/dictionary/edge/EdgeDictionaryImplTest.java +++ b/service/src/test/java/datawave/microservice/dictionary/edge/EdgeDictionaryImplTest.java @@ -1,15 +1,19 @@ package datawave.microservice.dictionary.edge; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.SetMultimap; -import datawave.accumulo.inmemory.InMemoryAccumuloClient; -import datawave.accumulo.inmemory.InMemoryInstance; -import datawave.data.ColumnFamilyConstants; -import datawave.metadata.protobuf.EdgeMetadata.MetadataValue; -import datawave.webservice.dictionary.edge.DefaultEdgeDictionary; -import datawave.webservice.dictionary.edge.DefaultMetadata; -import datawave.webservice.dictionary.edge.EventField; -import datawave.webservice.dictionary.edge.MetadataBase; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; @@ -27,19 +31,17 @@ import org.springframework.context.annotation.Configuration; import org.springframework.test.context.junit.jupiter.SpringExtension; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.SetMultimap; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; +import datawave.accumulo.inmemory.InMemoryAccumuloClient; +import datawave.accumulo.inmemory.InMemoryInstance; +import datawave.data.ColumnFamilyConstants; +import datawave.metadata.protobuf.EdgeMetadata.MetadataValue; +import datawave.webservice.dictionary.edge.DefaultEdgeDictionary; +import datawave.webservice.dictionary.edge.DefaultMetadata; +import datawave.webservice.dictionary.edge.EventField; +import datawave.webservice.dictionary.edge.MetadataBase; @ExtendWith(SpringExtension.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, properties = "spring.main.allow-bean-definition-overriding=true") diff --git a/service/src/test/java/datawave/microservice/metadata/DefaultMetadataFieldScannerTest.java b/service/src/test/java/datawave/microservice/metadata/DefaultMetadataFieldScannerTest.java index 1faa916..502959c 100644 --- a/service/src/test/java/datawave/microservice/metadata/DefaultMetadataFieldScannerTest.java +++ b/service/src/test/java/datawave/microservice/metadata/DefaultMetadataFieldScannerTest.java @@ -1,16 +1,18 @@ package datawave.microservice.metadata; -import datawave.accumulo.inmemory.InMemoryAccumuloClient; -import datawave.accumulo.inmemory.InMemoryInstance; -import datawave.data.ColumnFamilyConstants; -import datawave.marking.MarkingFunctions; -import datawave.microservice.Connection; -import datawave.microservice.dictionary.config.ResponseObjectFactory; -import datawave.webservice.dictionary.data.DefaultDataDictionary; -import datawave.webservice.dictionary.data.DefaultDescription; -import datawave.webservice.dictionary.data.DefaultDictionaryField; -import datawave.webservice.dictionary.data.DefaultFields; -import datawave.webservice.metadata.DefaultMetadataField; +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.BatchWriterConfig; @@ -24,18 +26,17 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import static org.assertj.core.api.Assertions.assertThat; +import datawave.accumulo.inmemory.InMemoryAccumuloClient; +import datawave.accumulo.inmemory.InMemoryInstance; +import datawave.data.ColumnFamilyConstants; +import datawave.marking.MarkingFunctions; +import datawave.microservice.Connection; +import datawave.microservice.dictionary.config.ResponseObjectFactory; +import datawave.webservice.dictionary.data.DefaultDataDictionary; +import datawave.webservice.dictionary.data.DefaultDescription; +import datawave.webservice.dictionary.data.DefaultDictionaryField; +import datawave.webservice.dictionary.data.DefaultFields; +import datawave.webservice.metadata.DefaultMetadataField; public class DefaultMetadataFieldScannerTest { diff --git a/service/src/test/java/datawave/microservice/model/ModelControllerTest.java b/service/src/test/java/datawave/microservice/model/ModelControllerTest.java index 3d452b5..ab1533c 100644 --- a/service/src/test/java/datawave/microservice/model/ModelControllerTest.java +++ b/service/src/test/java/datawave/microservice/model/ModelControllerTest.java @@ -1,14 +1,14 @@ package datawave.microservice.model; -import datawave.accumulo.inmemory.InMemoryAccumuloClient; -import datawave.accumulo.inmemory.InMemoryInstance; -import datawave.microservice.ControllerIT; -import datawave.microservice.model.config.ModelProperties; -import datawave.query.model.FieldMapping; -import datawave.query.model.ModelKeyParser; -import datawave.webservice.model.Model; -import datawave.webservice.model.ModelList; -import datawave.webservice.result.VoidResponse; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.HashSet; +import java.util.concurrent.TimeUnit; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.Unmarshaller; + import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.BatchWriter; @@ -35,13 +35,15 @@ import org.springframework.web.util.UriComponents; import org.springframework.web.util.UriComponentsBuilder; -import javax.xml.bind.JAXBContext; -import javax.xml.bind.Unmarshaller; -import java.util.HashSet; -import java.util.concurrent.TimeUnit; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import datawave.accumulo.inmemory.InMemoryAccumuloClient; +import datawave.accumulo.inmemory.InMemoryInstance; +import datawave.microservice.ControllerIT; +import datawave.microservice.model.config.ModelProperties; +import datawave.query.model.FieldMapping; +import datawave.query.model.ModelKeyParser; +import datawave.webservice.model.Model; +import datawave.webservice.model.ModelList; +import datawave.webservice.result.VoidResponse; public class ModelControllerTest extends ControllerIT { From bb35cbe27ab483adba209c6ee992e486b0b6fcc2 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 28 Jun 2023 15:10:15 -0400 Subject: [PATCH 44/54] added skipServices profile --- pom.xml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c352db2..a4c28b1 100644 --- a/pom.xml +++ b/pom.xml @@ -13,7 +13,6 @@ https://code.nsa.gov/datawave-dictionary-service api - service scm:git:https://github.com/NationalSecurityAgency/datawave-dictionary-service.git @@ -37,4 +36,17 @@ https://raw.githubusercontent.com/NationalSecurityAgency/datawave/mvn-repo + + + services + + + !skipServices + + + + service + + + From 1e2776cfec606f4e2be0a8c5ea18a429e87f8c3a Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 20 Sep 2023 15:09:27 -0400 Subject: [PATCH 45/54] Updated accumulo --- service/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/pom.xml b/service/pom.xml index 24a1b69..cdd8edc 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -19,7 +19,7 @@ datawave.microservice.dictionary.DictionaryService - 2.1.0 + 2.1.1 5.2.0 3.0-SNAPSHOT 3.0-SNAPSHOT From dc88e2979d2ea070b5be2cbb4919987c31b59403 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 5 Oct 2023 13:42:37 +0000 Subject: [PATCH 46/54] bumped versions for some modules --- api/pom.xml | 2 +- pom.xml | 2 +- service/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 1105296..f51b703 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 2.1-SNAPSHOT + 3.0-SNAPSHOT ../../../microservice-parent/pom.xml dictionary-api diff --git a/pom.xml b/pom.xml index a4c28b1..a47bb1f 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 2.1-SNAPSHOT + 3.0-SNAPSHOT ../../microservice-parent/pom.xml dictionary-service-parent diff --git a/service/pom.xml b/service/pom.xml index cdd8edc..273bec1 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-service-parent - 3.1-SNAPSHOT + 4.0-SNAPSHOT ../../../microservice-service-parent/pom.xml dictionary-service From 4b1a9d52952726fefe22dba1901eda51e09970b1 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 12 Oct 2023 14:58:09 +0000 Subject: [PATCH 47/54] updated to use semver --- api/pom.xml | 8 ++++---- pom.xml | 4 ++-- service/pom.xml | 12 ++++++------ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index f51b703..55d4cf0 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -4,11 +4,11 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 3.0-SNAPSHOT + 3.0.0-SNAPSHOT ../../../microservice-parent/pom.xml dictionary-api - 3.0-SNAPSHOT + 3.0.0-SNAPSHOT https://code.nsa.gov/datawave-dictionary-service scm:git:https://github.com/NationalSecurityAgency/datawave-dictionary-service.git @@ -18,9 +18,9 @@ http://webservice.datawave.nsa/v1 - 3.0-SNAPSHOT + 3.0.0-SNAPSHOT 31.1-jre - 3.0-SNAPSHOT + 3.0.0-SNAPSHOT diff --git a/pom.xml b/pom.xml index a47bb1f..dd7a776 100644 --- a/pom.xml +++ b/pom.xml @@ -4,11 +4,11 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 3.0-SNAPSHOT + 3.0.0-SNAPSHOT ../../microservice-parent/pom.xml dictionary-service-parent - 3.0-SNAPSHOT + 3.0.0-SNAPSHOT pom https://code.nsa.gov/datawave-dictionary-service diff --git a/service/pom.xml b/service/pom.xml index 273bec1..20e3d68 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -4,11 +4,11 @@ gov.nsa.datawave.microservice datawave-microservice-service-parent - 4.0-SNAPSHOT + 4.0.0-SNAPSHOT ../../../microservice-service-parent/pom.xml dictionary-service - 3.0-SNAPSHOT + 3.0.0-SNAPSHOT DATAWAVE Dictionary Service https://code.nsa.gov/datawave-dictionary-service @@ -21,10 +21,10 @@ datawave.microservice.dictionary.DictionaryService 2.1.1 5.2.0 - 3.0-SNAPSHOT - 3.0-SNAPSHOT - 3.0-SNAPSHOT - 2.0-SNAPSHOT + 3.0.0-SNAPSHOT + 3.0.0-SNAPSHOT + 3.0.0-SNAPSHOT + 2.0.0-SNAPSHOT 1.11.4 3.6.0 0.50 From e86e13e5239548a7aea3a5893d86ad45e48b07ff Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 12 Oct 2023 16:18:57 +0000 Subject: [PATCH 48/54] bumped dependencies to tagged releases --- api/pom.xml | 6 +++--- pom.xml | 2 +- service/pom.xml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 55d4cf0..5ccb47c 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 3.0.0-SNAPSHOT + 3.0.0 ../../../microservice-parent/pom.xml dictionary-api @@ -18,9 +18,9 @@ http://webservice.datawave.nsa/v1 - 3.0.0-SNAPSHOT + 3.0.0 31.1-jre - 3.0.0-SNAPSHOT + 3.0.0 diff --git a/pom.xml b/pom.xml index dd7a776..5d798aa 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 3.0.0-SNAPSHOT + 3.0.0 ../../microservice-parent/pom.xml dictionary-service-parent diff --git a/service/pom.xml b/service/pom.xml index 20e3d68..79e3648 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -21,7 +21,7 @@ datawave.microservice.dictionary.DictionaryService 2.1.1 5.2.0 - 3.0.0-SNAPSHOT + 3.0.0 3.0.0-SNAPSHOT 3.0.0-SNAPSHOT 2.0.0-SNAPSHOT From 930902626bd40d9f4a68bb361e9cb0c60fbaec87 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 12 Oct 2023 16:19:38 +0000 Subject: [PATCH 49/54] [maven-release-plugin] prepare release api-3.0.0 --- api/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 5ccb47c..b5b196d 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -8,12 +8,12 @@ ../../../microservice-parent/pom.xml dictionary-api - 3.0.0-SNAPSHOT + 3.0.0 https://code.nsa.gov/datawave-dictionary-service scm:git:https://github.com/NationalSecurityAgency/datawave-dictionary-service.git scm:git:git@github.com:NationalSecurityAgency/datawave-dictionary-service.git - HEAD + api-3.0.0 https://github.com/NationalSecurityAgency/datawave-dictionary-service From 898c15d398f8bb29f3fac9e3fb2ccb1bf623468f Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 12 Oct 2023 16:19:40 +0000 Subject: [PATCH 50/54] [maven-release-plugin] prepare for next development iteration --- api/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index b5b196d..5b2a9d9 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -8,12 +8,12 @@ ../../../microservice-parent/pom.xml dictionary-api - 3.0.0 + 3.0.1-SNAPSHOT https://code.nsa.gov/datawave-dictionary-service scm:git:https://github.com/NationalSecurityAgency/datawave-dictionary-service.git scm:git:git@github.com:NationalSecurityAgency/datawave-dictionary-service.git - api-3.0.0 + HEAD https://github.com/NationalSecurityAgency/datawave-dictionary-service From 1138102e3f9d4ebfac2789d182a52b081e4f2ecf Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 12 Oct 2023 16:58:28 +0000 Subject: [PATCH 51/54] bumped dependencies to tagged releases --- service/pom.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/service/pom.xml b/service/pom.xml index 79e3648..d10d7d5 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-service-parent - 4.0.0-SNAPSHOT + 4.0.0 ../../../microservice-service-parent/pom.xml dictionary-service @@ -22,9 +22,9 @@ 2.1.1 5.2.0 3.0.0 - 3.0.0-SNAPSHOT - 3.0.0-SNAPSHOT - 2.0.0-SNAPSHOT + 3.0.0 + 3.0.0 + 2.0.0 1.11.4 3.6.0 0.50 From 3e8a3e9014557e29548ff80341faf793666deacc Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 12 Oct 2023 17:00:52 +0000 Subject: [PATCH 52/54] [maven-release-plugin] prepare release service-3.0.0 --- service/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service/pom.xml b/service/pom.xml index d10d7d5..9288ef0 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -8,13 +8,13 @@ ../../../microservice-service-parent/pom.xml dictionary-service - 3.0.0-SNAPSHOT + 3.0.0 DATAWAVE Dictionary Service https://code.nsa.gov/datawave-dictionary-service scm:git:https://github.com/NationalSecurityAgency/datawave-dictionary-service.git scm:git:git@github.com:NationalSecurityAgency/datawave-dictionary-service.git - HEAD + service-3.0.0 https://github.com/NationalSecurityAgency/datawave-dictionary-service From 422f67aa7e27aa2b17422298aeaf86a270ecb490 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 12 Oct 2023 17:00:53 +0000 Subject: [PATCH 53/54] [maven-release-plugin] prepare for next development iteration --- service/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service/pom.xml b/service/pom.xml index 9288ef0..600b264 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -8,13 +8,13 @@ ../../../microservice-service-parent/pom.xml dictionary-service - 3.0.0 + 3.0.1-SNAPSHOT DATAWAVE Dictionary Service https://code.nsa.gov/datawave-dictionary-service scm:git:https://github.com/NationalSecurityAgency/datawave-dictionary-service.git scm:git:git@github.com:NationalSecurityAgency/datawave-dictionary-service.git - service-3.0.0 + HEAD https://github.com/NationalSecurityAgency/datawave-dictionary-service From 86be0dd8321e9cd38356591a0ff7d86c4e3ef0e7 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 12 Oct 2023 18:15:47 +0000 Subject: [PATCH 54/54] bumped parent pom version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5d798aa..e21dfbf 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ ../../microservice-parent/pom.xml dictionary-service-parent - 3.0.0-SNAPSHOT + 3.0.1-SNAPSHOT pom https://code.nsa.gov/datawave-dictionary-service