From 2b7b4c3fcbccec1160781f7e1fd2786061532f7a Mon Sep 17 00:00:00 2001 From: Paul Latzelsperger <43503240+beardyinc@users.noreply.github.com> Date: Tue, 20 Jul 2021 08:07:11 +0200 Subject: [PATCH] Feature/create maven artifacts (#108) * added publications to many modules * added workflow to release packages to maven repo * improved module artifact generation * minor fixes * do not publish an "all" lilb * trasnfertype has default contenttype * use snapshot version to enable replacing * added better logging, and a CollectionUtils.isAnyOf() method * extracted blobstore api into a separate common lib * fixed custom processor dockerfile * bumped snapshot version Co-authored-by: beardyinc --- .github/workflows/publish.yaml | 27 ++++++++++++++++ build.gradle.kts | 19 +++++++++++- common/azure/build.gradle.kts | 31 +++++++++++++++++++ .../dagx/common/azure}/BlobStoreApi.java | 2 +- .../dagx/common/azure}/BlobStoreApiImpl.java | 2 +- common/{ => util}/build.gradle.kts | 9 ++++++ .../common/collection/CollectionUtil.java | 8 +++++ .../microsoft/dagx/common/http/HttpUtil.java | 0 .../dagx/common/settings/SettingsHelper.java | 14 +++++++++ .../dagx/common/string/StringUtils.java | 0 .../common/collection/CollectionUtilTest.java | 9 ++++++ .../dagx/common/http/HttpUtilTest.java | 0 .../dagx/common/string/StringUtilsTest.java | 0 .../testfixtures/AbstractAzureBlobTest.java | 0 .../common/testfixtures/AbstractS3Test.java | 0 .../dagx/common/testfixtures/TestUtils.java | 0 core/build.gradle.kts | 9 ++++++ .../catalog/catalog-atlas/build.gradle.kts | 2 +- .../configuration-fs/build.gradle.kts | 10 +++++- extensions/control-http/build.gradle.kts | 9 ++++++ .../dataseed/dataseed-aws/build.gradle.kts | 2 +- .../dataseed/dataseed-azure/build.gradle.kts | 2 +- .../azure/AzureBlobDataseedExtension.java | 2 +- .../dataseed/dataseed-nifi/build.gradle.kts | 2 +- .../dataseed/dataseed-policy/build.gradle.kts | 2 +- .../events/events-azure/build.gradle.kts | 10 +++++- extensions/iam/iam-mock/build.gradle.kts | 8 +++++ extensions/iam/oauth2/build.gradle.kts | 9 +++++- extensions/ids/build.gradle.kts | 10 +++++- .../ids/ids-api-catalog/build.gradle.kts | 9 +++++- .../ids/ids-api-transfer/build.gradle.kts | 9 +++++- .../transfer/ArtifactRequestController.java | 7 +++-- extensions/ids/ids-core/build.gradle.kts | 10 +++++- .../ids/ids-policy-mock/build.gradle.kts | 8 +++++ extensions/ids/ids-spi/build.gradle.kts | 8 +++++ .../metadata/metadata-memory/build.gradle.kts | 8 +++++ .../policy-registry-memory/build.gradle.kts | 11 +++++-- .../protocol-loopback/build.gradle.kts | 9 +++++- extensions/protocol/web/build.gradle.kts | 9 +++++- extensions/schema/build.gradle.kts | 9 +++++- .../security/security-azure/build.gradle.kts | 10 +++++- .../security/security-fs/build.gradle.kts | 9 +++++- .../transfer/transfer-core/build.gradle.kts | 8 +++++ .../transfer/TransferProcessManagerImpl.java | 2 +- .../transfer-demo-protocols/build.gradle.kts | 8 +++++ .../transfer/transfer-nifi/build.gradle.kts | 2 +- .../transfer-provision-aws/build.gradle.kts | 13 +++++--- .../transfer-provision-azure/build.gradle.kts | 11 ++++++- .../azure/AzureProvisionExtension.java | 4 +-- .../blob/ObjectContainerStatusChecker.java | 2 +- .../azure/blob/ObjectStorageProvisioner.java | 2 +- .../ObjectContainerStatusCheckerTest.java | 2 +- .../blob/ObjectStorageProvisionerTest.java | 2 +- .../transfer-store-cosmos/build.gradle.kts | 10 +++++- .../cosmos/CosmosTransferProcessStore.java | 2 +- .../CosmosTransferProcessStoreExtension.java | 7 ++++- .../transfer-store-memory/build.gradle.kts | 10 ++++-- ...InMemoryTransferProcessStoreExtension.java | 7 +++++ external/nifi/Dockerfile | 2 +- external/nifi/processors/build.gradle | 2 +- gradle/wrapper/gradle-wrapper.properties | 2 +- policy/policy-engine/build.gradle.kts | 8 +++++ policy/policy-model/build.gradle.kts | 9 +++++- runtime/build.gradle.kts | 8 +++++ settings.gradle.kts | 5 +-- spi/build.gradle.kts | 9 ++++++ .../domain/transfer/TransferProcess.java | 4 +-- .../types/domain/transfer/TransferType.java | 13 ++++++-- 68 files changed, 413 insertions(+), 55 deletions(-) create mode 100644 .github/workflows/publish.yaml create mode 100644 common/azure/build.gradle.kts rename {extensions/transfer/transfer-provision-azure/src/main/java/com/microsoft/dagx/transfer/provision/azure/provider => common/azure/src/main/java/com/microsoft/dagx/common/azure}/BlobStoreApi.java (92%) rename {extensions/transfer/transfer-provision-azure/src/main/java/com/microsoft/dagx/transfer/provision/azure/provider => common/azure/src/main/java/com/microsoft/dagx/common/azure}/BlobStoreApiImpl.java (98%) rename common/{ => util}/build.gradle.kts (74%) rename common/{ => util}/src/main/java/com/microsoft/dagx/common/collection/CollectionUtil.java (75%) rename common/{ => util}/src/main/java/com/microsoft/dagx/common/http/HttpUtil.java (100%) rename common/{ => util}/src/main/java/com/microsoft/dagx/common/settings/SettingsHelper.java (63%) rename common/{ => util}/src/main/java/com/microsoft/dagx/common/string/StringUtils.java (100%) rename common/{ => util}/src/test/java/com/microsoft/dagx/common/collection/CollectionUtilTest.java (80%) rename common/{ => util}/src/test/java/com/microsoft/dagx/common/http/HttpUtilTest.java (100%) rename common/{ => util}/src/test/java/com/microsoft/dagx/common/string/StringUtilsTest.java (100%) rename common/{ => util}/src/testFixtures/java/com/microsoft/dagx/common/testfixtures/AbstractAzureBlobTest.java (100%) rename common/{ => util}/src/testFixtures/java/com/microsoft/dagx/common/testfixtures/AbstractS3Test.java (100%) rename common/{ => util}/src/testFixtures/java/com/microsoft/dagx/common/testfixtures/TestUtils.java (100%) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml new file mode 100644 index 00000000..5e34c256 --- /dev/null +++ b/.github/workflows/publish.yaml @@ -0,0 +1,27 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Publish package to GitHub Packages +on: + release: + types: [created] +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v2 + with: + java-version: '11' + distribution: 'adopt' + - name: Validate Gradle wrapper + uses: gradle/wrapper-validation-action@e6e38bacfdf1a337459f332974bb2327a31aaf4b + - name: Publish package + run: gradle publishAllPublicationToGitHubPackagesRepository + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/build.gradle.kts b/build.gradle.kts index 1aa8740e..7ae24273 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -5,6 +5,7 @@ plugins { `java-library` + `maven-publish` } repositories { @@ -30,7 +31,10 @@ subprojects { } allprojects { + apply(plugin = "maven-publish") pluginManager.withPlugin("java-library") { + group = "com.microsoft" + version = "0.0.1-SNAPSHOT.1" dependencies { api("org.jetbrains:annotations:${jetBrainsAnnotationsVersion}") api("com.fasterxml.jackson.core:jackson-core:${jacksonVersion}") @@ -45,6 +49,19 @@ allprojects { } + publishing { + repositories { + maven { + name = "GitHubPackages" + url = uri("https://maven.pkg.github.com/microsoft/data-appliance-gx") + credentials { + username = System.getenv("GITHUB_ACTOR") + password = System.getenv("GITHUB_TOKEN") + } + } + } + } + } tasks.withType { @@ -61,4 +78,4 @@ allprojects { val test by tasks.getting(Test::class) { useJUnitPlatform() -} +} \ No newline at end of file diff --git a/common/azure/build.gradle.kts b/common/azure/build.gradle.kts new file mode 100644 index 00000000..44226971 --- /dev/null +++ b/common/azure/build.gradle.kts @@ -0,0 +1,31 @@ +/* + * Copyright (c) Microsoft Corporation. + * All rights reserved. + */ + +plugins { + `java-library` + `java-test-fixtures` + `maven-publish` +} + +val storageBlobVersion: String by project; + +dependencies { + api(project(":spi")) + api(project(":common:util")) + api("com.azure:azure-storage-blob:${storageBlobVersion}") + + + testFixturesImplementation("org.junit.jupiter:junit-jupiter-api:5.5.2") + testFixturesRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.5.2") +} + +publishing { + publications { + create("common.azure") { + artifactId = "edc.common.azure" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/extensions/transfer/transfer-provision-azure/src/main/java/com/microsoft/dagx/transfer/provision/azure/provider/BlobStoreApi.java b/common/azure/src/main/java/com/microsoft/dagx/common/azure/BlobStoreApi.java similarity index 92% rename from extensions/transfer/transfer-provision-azure/src/main/java/com/microsoft/dagx/transfer/provision/azure/provider/BlobStoreApi.java rename to common/azure/src/main/java/com/microsoft/dagx/common/azure/BlobStoreApi.java index ec6a2823..9f9bb8fa 100644 --- a/extensions/transfer/transfer-provision-azure/src/main/java/com/microsoft/dagx/transfer/provision/azure/provider/BlobStoreApi.java +++ b/common/azure/src/main/java/com/microsoft/dagx/common/azure/BlobStoreApi.java @@ -4,7 +4,7 @@ * */ -package com.microsoft.dagx.transfer.provision.azure.provider; +package com.microsoft.dagx.common.azure; import com.azure.storage.blob.models.BlobItem; diff --git a/extensions/transfer/transfer-provision-azure/src/main/java/com/microsoft/dagx/transfer/provision/azure/provider/BlobStoreApiImpl.java b/common/azure/src/main/java/com/microsoft/dagx/common/azure/BlobStoreApiImpl.java similarity index 98% rename from extensions/transfer/transfer-provision-azure/src/main/java/com/microsoft/dagx/transfer/provision/azure/provider/BlobStoreApiImpl.java rename to common/azure/src/main/java/com/microsoft/dagx/common/azure/BlobStoreApiImpl.java index 65d5cf10..6b4ae8cd 100644 --- a/extensions/transfer/transfer-provision-azure/src/main/java/com/microsoft/dagx/transfer/provision/azure/provider/BlobStoreApiImpl.java +++ b/common/azure/src/main/java/com/microsoft/dagx/common/azure/BlobStoreApiImpl.java @@ -4,7 +4,7 @@ * */ -package com.microsoft.dagx.transfer.provision.azure.provider; +package com.microsoft.dagx.common.azure; import com.azure.core.util.BinaryData; import com.azure.storage.blob.BlobServiceClient; diff --git a/common/build.gradle.kts b/common/util/build.gradle.kts similarity index 74% rename from common/build.gradle.kts rename to common/util/build.gradle.kts index 2a9c7584..bb537b49 100644 --- a/common/build.gradle.kts +++ b/common/util/build.gradle.kts @@ -6,6 +6,7 @@ plugins { `java-library` `java-test-fixtures` + `maven-publish` } val storageBlobVersion: String by project; @@ -21,3 +22,11 @@ dependencies { testFixturesRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.5.2") } +publishing { + publications { + create("common.util") { + artifactId = "edc.common.util" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/common/src/main/java/com/microsoft/dagx/common/collection/CollectionUtil.java b/common/util/src/main/java/com/microsoft/dagx/common/collection/CollectionUtil.java similarity index 75% rename from common/src/main/java/com/microsoft/dagx/common/collection/CollectionUtil.java rename to common/util/src/main/java/com/microsoft/dagx/common/collection/CollectionUtil.java index 6f6e3b53..fa77c347 100644 --- a/common/src/main/java/com/microsoft/dagx/common/collection/CollectionUtil.java +++ b/common/util/src/main/java/com/microsoft/dagx/common/collection/CollectionUtil.java @@ -6,6 +6,7 @@ package com.microsoft.dagx.common.collection; +import java.util.Arrays; import java.util.Collection; import java.util.Map; @@ -27,4 +28,11 @@ private static boolean isEmpty(Map map) { return map == null || map.isEmpty(); } + public static boolean isAnyOf(T t, T... collection) { + if (collection == null) { + return false; + } + + return Arrays.asList(collection).contains(t); + } } diff --git a/common/src/main/java/com/microsoft/dagx/common/http/HttpUtil.java b/common/util/src/main/java/com/microsoft/dagx/common/http/HttpUtil.java similarity index 100% rename from common/src/main/java/com/microsoft/dagx/common/http/HttpUtil.java rename to common/util/src/main/java/com/microsoft/dagx/common/http/HttpUtil.java diff --git a/common/src/main/java/com/microsoft/dagx/common/settings/SettingsHelper.java b/common/util/src/main/java/com/microsoft/dagx/common/settings/SettingsHelper.java similarity index 63% rename from common/src/main/java/com/microsoft/dagx/common/settings/SettingsHelper.java rename to common/util/src/main/java/com/microsoft/dagx/common/settings/SettingsHelper.java index c8d64930..738760f8 100644 --- a/common/src/main/java/com/microsoft/dagx/common/settings/SettingsHelper.java +++ b/common/util/src/main/java/com/microsoft/dagx/common/settings/SettingsHelper.java @@ -7,7 +7,9 @@ package com.microsoft.dagx.common.settings; import com.microsoft.dagx.common.string.StringUtils; +import com.microsoft.dagx.spi.DagxException; import com.microsoft.dagx.spi.system.ServiceExtensionContext; +import org.jetbrains.annotations.NotNull; import java.util.UUID; @@ -25,4 +27,16 @@ public static synchronized String getConnectorId(ServiceExtensionContext context } return connectorId; } + + /** + * Convenience method to either get a particular setting or throw a {@link DagxException} + */ + @NotNull + public static String getSettingOrThrow(ServiceExtensionContext serviceContext, String setting) { + var value = serviceContext.getSetting(setting, null); + if (value == null) { + throw new DagxException("could not find setting " + setting + " or it was null"); + } + return value; + } } diff --git a/common/src/main/java/com/microsoft/dagx/common/string/StringUtils.java b/common/util/src/main/java/com/microsoft/dagx/common/string/StringUtils.java similarity index 100% rename from common/src/main/java/com/microsoft/dagx/common/string/StringUtils.java rename to common/util/src/main/java/com/microsoft/dagx/common/string/StringUtils.java diff --git a/common/src/test/java/com/microsoft/dagx/common/collection/CollectionUtilTest.java b/common/util/src/test/java/com/microsoft/dagx/common/collection/CollectionUtilTest.java similarity index 80% rename from common/src/test/java/com/microsoft/dagx/common/collection/CollectionUtilTest.java rename to common/util/src/test/java/com/microsoft/dagx/common/collection/CollectionUtilTest.java index 8db62fe9..b39a0c68 100644 --- a/common/src/test/java/com/microsoft/dagx/common/collection/CollectionUtilTest.java +++ b/common/util/src/test/java/com/microsoft/dagx/common/collection/CollectionUtilTest.java @@ -46,4 +46,13 @@ void isNotEmpty_map() { assertThat(CollectionUtil.isNotEmpty(Collections.singletonMap("testkey", "testValue"))).isTrue(); assertThat(CollectionUtil.isNotEmpty(Map.of("foo", "bar", "foo2", "baz"))).isTrue(); } + + @Test + void isAnyOf() { + int[] nullArray = null; + assertThat(CollectionUtil.isAnyOf(15, nullArray)).isFalse(); + assertThat(CollectionUtil.isAnyOf(15, 15, 16, 17)).isTrue(); + assertThat(CollectionUtil.isAnyOf("foobar", "foo", "bar", "baz", "foobar")).isTrue(); + assertThat(CollectionUtil.isAnyOf("foobar", "bar", "baz")).isFalse(); + } } \ No newline at end of file diff --git a/common/src/test/java/com/microsoft/dagx/common/http/HttpUtilTest.java b/common/util/src/test/java/com/microsoft/dagx/common/http/HttpUtilTest.java similarity index 100% rename from common/src/test/java/com/microsoft/dagx/common/http/HttpUtilTest.java rename to common/util/src/test/java/com/microsoft/dagx/common/http/HttpUtilTest.java diff --git a/common/src/test/java/com/microsoft/dagx/common/string/StringUtilsTest.java b/common/util/src/test/java/com/microsoft/dagx/common/string/StringUtilsTest.java similarity index 100% rename from common/src/test/java/com/microsoft/dagx/common/string/StringUtilsTest.java rename to common/util/src/test/java/com/microsoft/dagx/common/string/StringUtilsTest.java diff --git a/common/src/testFixtures/java/com/microsoft/dagx/common/testfixtures/AbstractAzureBlobTest.java b/common/util/src/testFixtures/java/com/microsoft/dagx/common/testfixtures/AbstractAzureBlobTest.java similarity index 100% rename from common/src/testFixtures/java/com/microsoft/dagx/common/testfixtures/AbstractAzureBlobTest.java rename to common/util/src/testFixtures/java/com/microsoft/dagx/common/testfixtures/AbstractAzureBlobTest.java diff --git a/common/src/testFixtures/java/com/microsoft/dagx/common/testfixtures/AbstractS3Test.java b/common/util/src/testFixtures/java/com/microsoft/dagx/common/testfixtures/AbstractS3Test.java similarity index 100% rename from common/src/testFixtures/java/com/microsoft/dagx/common/testfixtures/AbstractS3Test.java rename to common/util/src/testFixtures/java/com/microsoft/dagx/common/testfixtures/AbstractS3Test.java diff --git a/common/src/testFixtures/java/com/microsoft/dagx/common/testfixtures/TestUtils.java b/common/util/src/testFixtures/java/com/microsoft/dagx/common/testfixtures/TestUtils.java similarity index 100% rename from common/src/testFixtures/java/com/microsoft/dagx/common/testfixtures/TestUtils.java rename to common/util/src/testFixtures/java/com/microsoft/dagx/common/testfixtures/TestUtils.java diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 25ead46b..9a584a7f 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -7,6 +7,7 @@ val slf4jVersion: String by project plugins { `java-library` + `maven-publish` } dependencies { @@ -14,3 +15,11 @@ dependencies { api("org.slf4j:slf4j-api:${slf4jVersion}") } +publishing { + publications { + create("core") { + artifactId = "edc.core" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/extensions/catalog/catalog-atlas/build.gradle.kts b/extensions/catalog/catalog-atlas/build.gradle.kts index 6ef19a9c..016aaf23 100644 --- a/extensions/catalog/catalog-atlas/build.gradle.kts +++ b/extensions/catalog/catalog-atlas/build.gradle.kts @@ -10,7 +10,7 @@ plugins { dependencies { api(project(":spi")) - api(project(":common")) + api(project(":common:util")) api(project(":extensions:schema")) testImplementation(project(":distributions:junit")) diff --git a/extensions/configuration/configuration-fs/build.gradle.kts b/extensions/configuration/configuration-fs/build.gradle.kts index 8defb91c..bdd9c1b5 100644 --- a/extensions/configuration/configuration-fs/build.gradle.kts +++ b/extensions/configuration/configuration-fs/build.gradle.kts @@ -5,10 +5,18 @@ plugins { `java-library` + `maven-publish` } dependencies { api(project(":spi")) } - +publishing { + publications { + create("configuration-fs") { + artifactId = "edc.configuration-fs" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/extensions/control-http/build.gradle.kts b/extensions/control-http/build.gradle.kts index 9292cdc9..92192384 100644 --- a/extensions/control-http/build.gradle.kts +++ b/extensions/control-http/build.gradle.kts @@ -7,6 +7,7 @@ val rsApi: String by project plugins { `java-library` + `maven-publish` } dependencies { @@ -17,3 +18,11 @@ dependencies { } +publishing { + publications { + create("control-http") { + artifactId = "edc.control-http" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/extensions/dataseed/dataseed-aws/build.gradle.kts b/extensions/dataseed/dataseed-aws/build.gradle.kts index bfc48f3f..737d9889 100644 --- a/extensions/dataseed/dataseed-aws/build.gradle.kts +++ b/extensions/dataseed/dataseed-aws/build.gradle.kts @@ -11,7 +11,7 @@ val awsVersion: String by project dependencies { api(project(":spi")) - implementation(project(":common")) + implementation(project(":common:util")) implementation(project(":extensions:transfer:transfer-provision-aws")) diff --git a/extensions/dataseed/dataseed-azure/build.gradle.kts b/extensions/dataseed/dataseed-azure/build.gradle.kts index 85042693..6b3e8c3f 100644 --- a/extensions/dataseed/dataseed-azure/build.gradle.kts +++ b/extensions/dataseed/dataseed-azure/build.gradle.kts @@ -11,7 +11,7 @@ val storageBlobVersion: String by project dependencies { api(project(":spi")) - implementation(project(":common")) + implementation(project(":common:util")) implementation(project(":extensions:transfer:transfer-provision-azure")) } diff --git a/extensions/dataseed/dataseed-azure/src/main/java/com/microsoft/dagx/dataseed/azure/AzureBlobDataseedExtension.java b/extensions/dataseed/dataseed-azure/src/main/java/com/microsoft/dagx/dataseed/azure/AzureBlobDataseedExtension.java index 7d7206ee..54a1123e 100644 --- a/extensions/dataseed/dataseed-azure/src/main/java/com/microsoft/dagx/dataseed/azure/AzureBlobDataseedExtension.java +++ b/extensions/dataseed/dataseed-azure/src/main/java/com/microsoft/dagx/dataseed/azure/AzureBlobDataseedExtension.java @@ -6,13 +6,13 @@ package com.microsoft.dagx.dataseed.azure; +import com.microsoft.dagx.common.azure.BlobStoreApi; import com.microsoft.dagx.spi.monitor.Monitor; import com.microsoft.dagx.spi.security.Vault; import com.microsoft.dagx.spi.security.VaultResponse; import com.microsoft.dagx.spi.system.ServiceExtension; import com.microsoft.dagx.spi.system.ServiceExtensionContext; import com.microsoft.dagx.transfer.provision.azure.AzureSasToken; -import com.microsoft.dagx.transfer.provision.azure.provider.BlobStoreApi; import java.io.IOException; import java.io.InputStream; diff --git a/extensions/dataseed/dataseed-nifi/build.gradle.kts b/extensions/dataseed/dataseed-nifi/build.gradle.kts index ca91485d..1ff3c967 100644 --- a/extensions/dataseed/dataseed-nifi/build.gradle.kts +++ b/extensions/dataseed/dataseed-nifi/build.gradle.kts @@ -10,5 +10,5 @@ plugins { dependencies { api(project(":spi")) - implementation(project(":common")) + implementation(project(":common:util")) } diff --git a/extensions/dataseed/dataseed-policy/build.gradle.kts b/extensions/dataseed/dataseed-policy/build.gradle.kts index 94a5bb91..b163e52f 100644 --- a/extensions/dataseed/dataseed-policy/build.gradle.kts +++ b/extensions/dataseed/dataseed-policy/build.gradle.kts @@ -10,5 +10,5 @@ plugins { dependencies { api(project(":spi")) - api(project(":common")) + api(project(":common:util")) } diff --git a/extensions/events/events-azure/build.gradle.kts b/extensions/events/events-azure/build.gradle.kts index e17911b8..58d5e8b0 100644 --- a/extensions/events/events-azure/build.gradle.kts +++ b/extensions/events/events-azure/build.gradle.kts @@ -12,8 +12,16 @@ val eventGridSdkVersion: String by project dependencies { api(project(":spi")) implementation(project(":extensions:schema")) - implementation(project(":common")) + implementation(project(":common:util")) implementation("com.azure:azure-messaging-eventgrid:${eventGridSdkVersion}") } +publishing { + publications { + create("events-azure") { + artifactId = "edc.events-azure" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/extensions/iam/iam-mock/build.gradle.kts b/extensions/iam/iam-mock/build.gradle.kts index bfedcdf4..2507d999 100644 --- a/extensions/iam/iam-mock/build.gradle.kts +++ b/extensions/iam/iam-mock/build.gradle.kts @@ -10,3 +10,11 @@ dependencies { } +publishing { + publications { + create("iam-mock") { + artifactId = "edc.iam-mock" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/extensions/iam/oauth2/build.gradle.kts b/extensions/iam/oauth2/build.gradle.kts index 0e724dcc..93dd3a59 100644 --- a/extensions/iam/oauth2/build.gradle.kts +++ b/extensions/iam/oauth2/build.gradle.kts @@ -14,4 +14,11 @@ dependencies { implementation("com.auth0:java-jwt:${jwtVersion}") } - +publishing { + publications { + create("oauth2") { + artifactId = "edc.iam.oauth2" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/extensions/ids/build.gradle.kts b/extensions/ids/build.gradle.kts index 27d201dc..211931c6 100644 --- a/extensions/ids/build.gradle.kts +++ b/extensions/ids/build.gradle.kts @@ -14,6 +14,14 @@ dependencies { api(project(":extensions:ids:ids-core")) api(project(":extensions:ids:ids-api-catalog")) api(project(":extensions:ids:ids-api-transfer")) + api(project(":extensions:ids:ids-policy-mock")) } - +publishing { + publications { + create("ids") { + artifactId = "edc.ids" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/extensions/ids/ids-api-catalog/build.gradle.kts b/extensions/ids/ids-api-catalog/build.gradle.kts index 76ed56e7..28105b75 100644 --- a/extensions/ids/ids-api-catalog/build.gradle.kts +++ b/extensions/ids/ids-api-catalog/build.gradle.kts @@ -21,4 +21,11 @@ dependencies { } - +publishing { + publications { + create("ids-api-catalog") { + artifactId = "edc.ids-api-catalog" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/extensions/ids/ids-api-transfer/build.gradle.kts b/extensions/ids/ids-api-transfer/build.gradle.kts index 76ed56e7..09c6ab52 100644 --- a/extensions/ids/ids-api-transfer/build.gradle.kts +++ b/extensions/ids/ids-api-transfer/build.gradle.kts @@ -21,4 +21,11 @@ dependencies { } - +publishing { + publications { + create("ids-api-transfer") { + artifactId = "edc.ids-api-transfer" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/extensions/ids/ids-api-transfer/src/main/java/com/microsoft/dagx/ids/api/transfer/ArtifactRequestController.java b/extensions/ids/ids-api-transfer/src/main/java/com/microsoft/dagx/ids/api/transfer/ArtifactRequestController.java index 2d8a6711..4a2d5dce 100644 --- a/extensions/ids/ids-api-transfer/src/main/java/com/microsoft/dagx/ids/api/transfer/ArtifactRequestController.java +++ b/extensions/ids/ids-api-transfer/src/main/java/com/microsoft/dagx/ids/api/transfer/ArtifactRequestController.java @@ -70,6 +70,7 @@ public ArtifactRequestController(DapsService dapsService, public Response request(ArtifactRequestMessage message) { var verificationResult = dapsService.verifyAndConvertToken(message.getSecurityToken().getTokenValue()); if (!verificationResult.valid()) { + monitor.info(() -> "verification failed for request " + message.getId()); return Response.status(Response.Status.FORBIDDEN).entity(new RejectionMessageBuilder()._rejectionReason_(NOT_AUTHENTICATED).build()).build(); } @@ -93,6 +94,7 @@ public Response request(ArtifactRequestMessage message) { var policyResult = policyService.evaluateRequest(clientConnectorId, correlationId, verificationResult.token(), policy); if (!policyResult.valid()) { + monitor.info("Policy evaluation failed"); return Response.status(Response.Status.FORBIDDEN).entity(new RejectionMessageBuilder()._rejectionReason_(NOT_AUTHORIZED).build()).build(); } @@ -101,10 +103,11 @@ public Response request(ArtifactRequestMessage message) { @SuppressWarnings("unchecked") var destinationMap = (Map) message.getProperties().get(DESTINATION_KEY); var type = (String) destinationMap.get("type"); - @SuppressWarnings("unchecked") var properties = (Map) destinationMap.get("properties"); + @SuppressWarnings("unchecked") Map properties = (Map) destinationMap.get("properties"); var secretName = (String) destinationMap.get("keyName"); - var dataDestination = DataAddress.Builder.newInstance().type(type).properties(cast(properties)).keyName(secretName).build(); + final Map cast = cast(properties); + var dataDestination = DataAddress.Builder.newInstance().type(type).properties(cast).keyName(secretName).build(); var dataRequest = DataRequest.Builder.newInstance().id(randomUUID().toString()).dataEntry(entry).dataDestination(dataDestination).protocol(IDS_REST).build(); diff --git a/extensions/ids/ids-core/build.gradle.kts b/extensions/ids/ids-core/build.gradle.kts index b9dccec6..cb3cc0c0 100644 --- a/extensions/ids/ids-core/build.gradle.kts +++ b/extensions/ids/ids-core/build.gradle.kts @@ -13,7 +13,7 @@ plugins { dependencies { api(project(":spi")) - api(project(":common")) + api(project(":common:util")) api(project(":extensions:ids:ids-spi")) api("de.fraunhofer.iais.eis.ids.infomodel:java:${infoModelVersion}") @@ -23,3 +23,11 @@ dependencies { } +publishing { + publications { + create("ids-core") { + artifactId = "edc.ids-core" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/extensions/ids/ids-policy-mock/build.gradle.kts b/extensions/ids/ids-policy-mock/build.gradle.kts index 0b78d851..68596b73 100644 --- a/extensions/ids/ids-policy-mock/build.gradle.kts +++ b/extensions/ids/ids-policy-mock/build.gradle.kts @@ -12,3 +12,11 @@ dependencies { } +publishing { + publications { + create("ids-policy-mock") { + artifactId = "edc.ids-policy-mock" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/extensions/ids/ids-spi/build.gradle.kts b/extensions/ids/ids-spi/build.gradle.kts index 4e744a21..88adb37d 100644 --- a/extensions/ids/ids-spi/build.gradle.kts +++ b/extensions/ids/ids-spi/build.gradle.kts @@ -22,3 +22,11 @@ dependencies { } +publishing { + publications { + create("ids-spi") { + artifactId = "edc.ids-spi" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/extensions/metadata/metadata-memory/build.gradle.kts b/extensions/metadata/metadata-memory/build.gradle.kts index cde66eb5..eaa54354 100644 --- a/extensions/metadata/metadata-memory/build.gradle.kts +++ b/extensions/metadata/metadata-memory/build.gradle.kts @@ -11,3 +11,11 @@ plugins { dependencies { api(project(":spi")) } +publishing { + publications { + create("metadata-mem") { + artifactId = "edc.metadata-memory" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/extensions/policy/policy-registry-memory/build.gradle.kts b/extensions/policy/policy-registry-memory/build.gradle.kts index ec249c82..c92dcc5f 100644 --- a/extensions/policy/policy-registry-memory/build.gradle.kts +++ b/extensions/policy/policy-registry-memory/build.gradle.kts @@ -5,8 +5,13 @@ plugins { dependencies { api(project(":spi")) - - } - +publishing { + publications { + create("policy-mem") { + artifactId = "edc.policy-registry-memory" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/extensions/protocol/protocol-loopback/build.gradle.kts b/extensions/protocol/protocol-loopback/build.gradle.kts index 25ff71ff..c89d2443 100644 --- a/extensions/protocol/protocol-loopback/build.gradle.kts +++ b/extensions/protocol/protocol-loopback/build.gradle.kts @@ -12,4 +12,11 @@ dependencies { api(project(":spi")) } - +publishing { + publications { + create("proto-loopback") { + artifactId = "edc.protocol-loopback" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/extensions/protocol/web/build.gradle.kts b/extensions/protocol/web/build.gradle.kts index 36f9be2f..d33b7575 100644 --- a/extensions/protocol/web/build.gradle.kts +++ b/extensions/protocol/web/build.gradle.kts @@ -32,4 +32,11 @@ dependencies { } - +publishing { + publications { + create("web") { + artifactId = "edc.protocol-web" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/extensions/schema/build.gradle.kts b/extensions/schema/build.gradle.kts index 6896baad..c9a5a1d4 100644 --- a/extensions/schema/build.gradle.kts +++ b/extensions/schema/build.gradle.kts @@ -13,4 +13,11 @@ dependencies { testImplementation(project(":distributions:junit")) } - +publishing { + publications { + create("schema") { + artifactId = "edc.schema" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/extensions/security/security-azure/build.gradle.kts b/extensions/security/security-azure/build.gradle.kts index 72300dda..29e46009 100644 --- a/extensions/security/security-azure/build.gradle.kts +++ b/extensions/security/security-azure/build.gradle.kts @@ -11,7 +11,7 @@ plugins { dependencies { api(project(":spi")) - implementation(project(":common")) + implementation(project(":common:util")) implementation("com.azure:azure-security-keyvault-secrets:4.2.3") implementation("com.azure:azure-identity:1.2.0") implementation("jakarta.ws.rs:jakarta.ws.rs-api:${rsApi}") @@ -23,3 +23,11 @@ dependencies { } +publishing { + publications { + create("security-azure") { + artifactId = "edc.security-azure" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/extensions/security/security-fs/build.gradle.kts b/extensions/security/security-fs/build.gradle.kts index 8defb91c..fe57dcd8 100644 --- a/extensions/security/security-fs/build.gradle.kts +++ b/extensions/security/security-fs/build.gradle.kts @@ -11,4 +11,11 @@ dependencies { api(project(":spi")) } - +publishing { + publications { + create("security-fs") { + artifactId = "edc.security-fs" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/extensions/transfer/transfer-core/build.gradle.kts b/extensions/transfer/transfer-core/build.gradle.kts index bc0234b6..e2ee24cb 100644 --- a/extensions/transfer/transfer-core/build.gradle.kts +++ b/extensions/transfer/transfer-core/build.gradle.kts @@ -16,3 +16,11 @@ dependencies { } +publishing { + publications { + create("transfer-core") { + artifactId = "edc.transfer-core" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/extensions/transfer/transfer-core/src/main/java/com/microsoft/dagx/transfer/core/transfer/TransferProcessManagerImpl.java b/extensions/transfer/transfer-core/src/main/java/com/microsoft/dagx/transfer/core/transfer/TransferProcessManagerImpl.java index 6bedfb63..8bf93fe3 100644 --- a/extensions/transfer/transfer-core/src/main/java/com/microsoft/dagx/transfer/core/transfer/TransferProcessManagerImpl.java +++ b/extensions/transfer/transfer-core/src/main/java/com/microsoft/dagx/transfer/core/transfer/TransferProcessManagerImpl.java @@ -173,7 +173,7 @@ private int checkProvisioned() { List requestAcked = transferProcessStore.nextForState(TransferProcessStates.REQUESTED_ACK.code(), batchSize); for (var process : requestAcked) { - if (!process.getProvisionedResourceSet().empty()) { + if (process.getProvisionedResourceSet() != null && !process.getProvisionedResourceSet().empty()) { if (process.getDataRequest().getTransferType().isFinite()) { process.transitionInProgress(); diff --git a/extensions/transfer/transfer-demo-protocols/build.gradle.kts b/extensions/transfer/transfer-demo-protocols/build.gradle.kts index 0d1e21c7..f8191b37 100644 --- a/extensions/transfer/transfer-demo-protocols/build.gradle.kts +++ b/extensions/transfer/transfer-demo-protocols/build.gradle.kts @@ -27,3 +27,11 @@ dependencies { } +publishing { + publications { + create("transfer-demo-protocols") { + artifactId = "edc.transfer-demo-protocols" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/extensions/transfer/transfer-nifi/build.gradle.kts b/extensions/transfer/transfer-nifi/build.gradle.kts index 1b0b35d1..a78dec1c 100644 --- a/extensions/transfer/transfer-nifi/build.gradle.kts +++ b/extensions/transfer/transfer-nifi/build.gradle.kts @@ -12,7 +12,7 @@ plugins { dependencies { api(project(":spi")) - api(project(":common")) + api(project(":common:util")) api(project(":core")) implementation(project(":extensions:schema")) diff --git a/extensions/transfer/transfer-provision-aws/build.gradle.kts b/extensions/transfer/transfer-provision-aws/build.gradle.kts index 2fd5e30e..5258fedf 100644 --- a/extensions/transfer/transfer-provision-aws/build.gradle.kts +++ b/extensions/transfer/transfer-provision-aws/build.gradle.kts @@ -18,10 +18,15 @@ dependencies { implementation("software.amazon.awssdk:sts:${awsVersion}") implementation("software.amazon.awssdk:iam:${awsVersion}") - testImplementation(testFixtures(project(":common"))) + testImplementation(testFixtures(project(":common:util"))) } - - - +publishing { + publications { + create("transfer-provision-aws") { + artifactId = "edc.transfer-provision-aws" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/extensions/transfer/transfer-provision-azure/build.gradle.kts b/extensions/transfer/transfer-provision-azure/build.gradle.kts index f47d3503..f8eb32c5 100644 --- a/extensions/transfer/transfer-provision-azure/build.gradle.kts +++ b/extensions/transfer/transfer-provision-azure/build.gradle.kts @@ -11,11 +11,20 @@ val storageBlobVersion: String by project dependencies { api(project(":spi")) + api(project(":common:azure")) api(project(":extensions:schema")) implementation("com.azure:azure-storage-blob:${storageBlobVersion}") - testImplementation(testFixtures(project(":common"))) + testImplementation(testFixtures(project(":common:util"))) } +publishing { + publications { + create("transfer-provision-azure") { + artifactId = "edc.transfer-provision-azure" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/extensions/transfer/transfer-provision-azure/src/main/java/com/microsoft/dagx/transfer/provision/azure/AzureProvisionExtension.java b/extensions/transfer/transfer-provision-azure/src/main/java/com/microsoft/dagx/transfer/provision/azure/AzureProvisionExtension.java index a3c1d110..7252f5d1 100644 --- a/extensions/transfer/transfer-provision-azure/src/main/java/com/microsoft/dagx/transfer/provision/azure/AzureProvisionExtension.java +++ b/extensions/transfer/transfer-provision-azure/src/main/java/com/microsoft/dagx/transfer/provision/azure/AzureProvisionExtension.java @@ -5,6 +5,8 @@ package com.microsoft.dagx.transfer.provision.azure; +import com.microsoft.dagx.common.azure.BlobStoreApi; +import com.microsoft.dagx.common.azure.BlobStoreApiImpl; import com.microsoft.dagx.spi.monitor.Monitor; import com.microsoft.dagx.spi.security.Vault; import com.microsoft.dagx.spi.system.ServiceExtension; @@ -14,8 +16,6 @@ import com.microsoft.dagx.spi.types.TypeManager; import com.microsoft.dagx.spi.types.domain.transfer.StatusCheckerRegistry; import com.microsoft.dagx.transfer.provision.azure.blob.*; -import com.microsoft.dagx.transfer.provision.azure.provider.BlobStoreApi; -import com.microsoft.dagx.transfer.provision.azure.provider.BlobStoreApiImpl; import net.jodah.failsafe.RetryPolicy; import java.util.Set; diff --git a/extensions/transfer/transfer-provision-azure/src/main/java/com/microsoft/dagx/transfer/provision/azure/blob/ObjectContainerStatusChecker.java b/extensions/transfer/transfer-provision-azure/src/main/java/com/microsoft/dagx/transfer/provision/azure/blob/ObjectContainerStatusChecker.java index 100b743e..d4d5b8ea 100644 --- a/extensions/transfer/transfer-provision-azure/src/main/java/com/microsoft/dagx/transfer/provision/azure/blob/ObjectContainerStatusChecker.java +++ b/extensions/transfer/transfer-provision-azure/src/main/java/com/microsoft/dagx/transfer/provision/azure/blob/ObjectContainerStatusChecker.java @@ -6,8 +6,8 @@ package com.microsoft.dagx.transfer.provision.azure.blob; +import com.microsoft.dagx.common.azure.BlobStoreApi; import com.microsoft.dagx.spi.types.domain.transfer.StatusChecker; -import com.microsoft.dagx.transfer.provision.azure.provider.BlobStoreApi; import net.jodah.failsafe.Failsafe; import net.jodah.failsafe.RetryPolicy; diff --git a/extensions/transfer/transfer-provision-azure/src/main/java/com/microsoft/dagx/transfer/provision/azure/blob/ObjectStorageProvisioner.java b/extensions/transfer/transfer-provision-azure/src/main/java/com/microsoft/dagx/transfer/provision/azure/blob/ObjectStorageProvisioner.java index 40d0ed4c..58268b0b 100644 --- a/extensions/transfer/transfer-provision-azure/src/main/java/com/microsoft/dagx/transfer/provision/azure/blob/ObjectStorageProvisioner.java +++ b/extensions/transfer/transfer-provision-azure/src/main/java/com/microsoft/dagx/transfer/provision/azure/blob/ObjectStorageProvisioner.java @@ -6,6 +6,7 @@ package com.microsoft.dagx.transfer.provision.azure.blob; +import com.microsoft.dagx.common.azure.BlobStoreApi; import com.microsoft.dagx.spi.monitor.Monitor; import com.microsoft.dagx.spi.transfer.provision.ProvisionContext; import com.microsoft.dagx.spi.transfer.provision.Provisioner; @@ -13,7 +14,6 @@ import com.microsoft.dagx.spi.types.domain.transfer.ProvisionedResource; import com.microsoft.dagx.spi.types.domain.transfer.ResourceDefinition; import com.microsoft.dagx.transfer.provision.azure.AzureSasToken; -import com.microsoft.dagx.transfer.provision.azure.provider.BlobStoreApi; import net.jodah.failsafe.RetryPolicy; import java.time.OffsetDateTime; diff --git a/extensions/transfer/transfer-provision-azure/src/test/java/com/microsoft/dagx/transfer/provision/azure/blob/ObjectContainerStatusCheckerTest.java b/extensions/transfer/transfer-provision-azure/src/test/java/com/microsoft/dagx/transfer/provision/azure/blob/ObjectContainerStatusCheckerTest.java index e5eddf66..706fbaca 100644 --- a/extensions/transfer/transfer-provision-azure/src/test/java/com/microsoft/dagx/transfer/provision/azure/blob/ObjectContainerStatusCheckerTest.java +++ b/extensions/transfer/transfer-provision-azure/src/test/java/com/microsoft/dagx/transfer/provision/azure/blob/ObjectContainerStatusCheckerTest.java @@ -6,10 +6,10 @@ package com.microsoft.dagx.transfer.provision.azure.blob; +import com.microsoft.dagx.common.azure.BlobStoreApiImpl; import com.microsoft.dagx.common.testfixtures.AbstractAzureBlobTest; import com.microsoft.dagx.common.testfixtures.TestUtils; import com.microsoft.dagx.spi.security.Vault; -import com.microsoft.dagx.transfer.provision.azure.provider.BlobStoreApiImpl; import net.jodah.failsafe.RetryPolicy; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/extensions/transfer/transfer-provision-azure/src/test/java/com/microsoft/dagx/transfer/provision/azure/blob/ObjectStorageProvisionerTest.java b/extensions/transfer/transfer-provision-azure/src/test/java/com/microsoft/dagx/transfer/provision/azure/blob/ObjectStorageProvisionerTest.java index 55bc8f32..c7db99e0 100644 --- a/extensions/transfer/transfer-provision-azure/src/test/java/com/microsoft/dagx/transfer/provision/azure/blob/ObjectStorageProvisionerTest.java +++ b/extensions/transfer/transfer-provision-azure/src/test/java/com/microsoft/dagx/transfer/provision/azure/blob/ObjectStorageProvisionerTest.java @@ -7,6 +7,7 @@ package com.microsoft.dagx.transfer.provision.azure.blob; import com.azure.storage.blob.models.BlobStorageException; +import com.microsoft.dagx.common.azure.BlobStoreApi; import com.microsoft.dagx.spi.monitor.Monitor; import com.microsoft.dagx.spi.transfer.provision.ProvisionContext; import com.microsoft.dagx.spi.transfer.response.ResponseStatus; @@ -14,7 +15,6 @@ import com.microsoft.dagx.spi.types.domain.transfer.ProvisionedResource; import com.microsoft.dagx.spi.types.domain.transfer.ResourceDefinition; import com.microsoft.dagx.transfer.provision.azure.AzureSasToken; -import com.microsoft.dagx.transfer.provision.azure.provider.BlobStoreApi; import net.jodah.failsafe.RetryPolicy; import org.easymock.Capture; import org.easymock.MockType; diff --git a/extensions/transfer/transfer-store-cosmos/build.gradle.kts b/extensions/transfer/transfer-store-cosmos/build.gradle.kts index b23a83b7..1c1f9897 100644 --- a/extensions/transfer/transfer-store-cosmos/build.gradle.kts +++ b/extensions/transfer/transfer-store-cosmos/build.gradle.kts @@ -11,9 +11,17 @@ val cosmosSdkVersion: String by project dependencies { api(project(":spi")) - api(project(":common")) + api(project(":common:util")) implementation("com.azure:azure-cosmos:${cosmosSdkVersion}") } +publishing { + publications { + create("transfer-store-cosmos") { + artifactId = "edc.transfer-store-cosmos" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/extensions/transfer/transfer-store-cosmos/src/main/java/com/microsoft/dagx/transfer/store/cosmos/CosmosTransferProcessStore.java b/extensions/transfer/transfer-store-cosmos/src/main/java/com/microsoft/dagx/transfer/store/cosmos/CosmosTransferProcessStore.java index 12af55a6..b798c0cc 100644 --- a/extensions/transfer/transfer-store-cosmos/src/main/java/com/microsoft/dagx/transfer/store/cosmos/CosmosTransferProcessStore.java +++ b/extensions/transfer/transfer-store-cosmos/src/main/java/com/microsoft/dagx/transfer/store/cosmos/CosmosTransferProcessStore.java @@ -204,7 +204,7 @@ private void handleResponse(CosmosItemResponse response) { } private TransferProcessDocument convertObject(Object databaseDocument) { - return typeManager.readValue(typeManager.writeValueAsBytes(databaseDocument), TransferProcessDocument.class); + return typeManager.readValue(typeManager.writeValueAsString(databaseDocument), TransferProcessDocument.class); } private void release(String processId, Object connectorId) { diff --git a/extensions/transfer/transfer-store-cosmos/src/main/java/com/microsoft/dagx/transfer/store/cosmos/CosmosTransferProcessStoreExtension.java b/extensions/transfer/transfer-store-cosmos/src/main/java/com/microsoft/dagx/transfer/store/cosmos/CosmosTransferProcessStoreExtension.java index 7521d085..342d2492 100644 --- a/extensions/transfer/transfer-store-cosmos/src/main/java/com/microsoft/dagx/transfer/store/cosmos/CosmosTransferProcessStoreExtension.java +++ b/extensions/transfer/transfer-store-cosmos/src/main/java/com/microsoft/dagx/transfer/store/cosmos/CosmosTransferProcessStoreExtension.java @@ -43,7 +43,7 @@ public class CosmosTransferProcessStoreExtension implements ServiceExtension { @DagxSetting private final static String COSMOS_PARTITION_KEY_SETTING = "dagx.cosmos.partitionkey"; private final static String DEFAULT_PARTITION_KEY = "dagx"; - private final static String CONTAINER_NAME = "dagx-transferprocess"; + private final static String CONTAINER_NAME = "transferprocess"; private Monitor monitor; @@ -107,6 +107,11 @@ public Set requires() { return Set.of("dagx:blobstoreapi"); } + @Override + public Set provides() { + return Set.of("dagx:transferprocessstore"); + } + @Override public void start() { monitor.info("Started Initialized Cosmos Transfer Process Store extension"); diff --git a/extensions/transfer/transfer-store-memory/build.gradle.kts b/extensions/transfer/transfer-store-memory/build.gradle.kts index 829c819f..0cca72a1 100644 --- a/extensions/transfer/transfer-store-memory/build.gradle.kts +++ b/extensions/transfer/transfer-store-memory/build.gradle.kts @@ -13,5 +13,11 @@ dependencies { } - - +publishing { + publications { + create("transfer-store-memory") { + artifactId = "edc.transfer-store-memory" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/extensions/transfer/transfer-store-memory/src/main/java/com/microsoft/dagx/transfer/store/memory/InMemoryTransferProcessStoreExtension.java b/extensions/transfer/transfer-store-memory/src/main/java/com/microsoft/dagx/transfer/store/memory/InMemoryTransferProcessStoreExtension.java index e5891bb3..df723b59 100644 --- a/extensions/transfer/transfer-store-memory/src/main/java/com/microsoft/dagx/transfer/store/memory/InMemoryTransferProcessStoreExtension.java +++ b/extensions/transfer/transfer-store-memory/src/main/java/com/microsoft/dagx/transfer/store/memory/InMemoryTransferProcessStoreExtension.java @@ -10,6 +10,8 @@ import com.microsoft.dagx.spi.system.ServiceExtensionContext; import com.microsoft.dagx.spi.transfer.store.TransferProcessStore; +import java.util.Set; + /** * Provides an in-memory implementation of the {@link com.microsoft.dagx.spi.transfer.store.TransferProcessStore} for testing. */ @@ -23,6 +25,11 @@ public void initialize(ServiceExtensionContext context) { monitor.info("Initialized In-Memory Transfer Process Store extension"); } + @Override + public Set provides() { + return Set.of("dagx:transferprocessstore"); + } + @Override public void start() { monitor.info("Started Initialized In-Memory Transfer Process Store extension"); diff --git a/external/nifi/Dockerfile b/external/nifi/Dockerfile index 7f0f0872..6f7ab241 100644 --- a/external/nifi/Dockerfile +++ b/external/nifi/Dockerfile @@ -1,5 +1,5 @@ FROM apache/nifi:latest as nifi -COPY ./processors/build/libs/processors.nar /opt/nifi/nifi-current/lib/dagx-processors.nar +COPY ./processors/build/libs/processors*.nar /opt/nifi/nifi-current/lib/dagx-processors.nar ENTRYPOINT ["/opt/nifi/scripts/start.sh"] diff --git a/external/nifi/processors/build.gradle b/external/nifi/processors/build.gradle index dce3fad8..2cbfd54d 100644 --- a/external/nifi/processors/build.gradle +++ b/external/nifi/processors/build.gradle @@ -141,7 +141,7 @@ dependencies { testImplementation("org.assertj:assertj-core:3.19.0") testImplementation(project(":spi")) - testImplementation(testFixtures(project(":common"))) + testImplementation(testFixtures(project(":common:util"))) } diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index f371643e..05679dc3 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.0-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.1.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/policy/policy-engine/build.gradle.kts b/policy/policy-engine/build.gradle.kts index 2d92a63c..6680268a 100644 --- a/policy/policy-engine/build.gradle.kts +++ b/policy/policy-engine/build.gradle.kts @@ -12,3 +12,11 @@ dependencies { api(project(":policy:policy-model")) } +publishing { + publications { + create("policy-engine") { + artifactId = "edc.policy-engine" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/policy/policy-model/build.gradle.kts b/policy/policy-model/build.gradle.kts index 509a27b4..630f0459 100644 --- a/policy/policy-model/build.gradle.kts +++ b/policy/policy-model/build.gradle.kts @@ -10,4 +10,11 @@ plugins { dependencies { } - +publishing { + publications { + create("policy-model") { + artifactId = "edc.policy-model" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/runtime/build.gradle.kts b/runtime/build.gradle.kts index ee0e19ef..0f85f7b6 100644 --- a/runtime/build.gradle.kts +++ b/runtime/build.gradle.kts @@ -14,3 +14,11 @@ dependencies { testImplementation("org.junit.jupiter:junit-jupiter-api:5.5.2") testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.5.2") } +publishing { + publications { + create("runtime") { + artifactId = "edc.runtime" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index 5194463a..f33cd834 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -7,7 +7,8 @@ rootProject.name = "dagx" include(":spi") include(":core") -include(":common") +include(":common:util") +include(":common:azure") include(":distributions:demo") include(":distributions:demo-e2e") @@ -68,4 +69,4 @@ include(":runtime") include(":client-runtime") include(":client") -include(":external:nifi:processors") \ No newline at end of file +include(":external:nifi:processors") diff --git a/spi/build.gradle.kts b/spi/build.gradle.kts index 7b5f9b91..0bd052b4 100644 --- a/spi/build.gradle.kts +++ b/spi/build.gradle.kts @@ -10,6 +10,7 @@ val jodahFailsafeVersion: String by project plugins { `java-library` + `maven-publish` } dependencies { @@ -24,3 +25,11 @@ dependencies { api(project(":policy:policy-model")) } +publishing { + publications { + create("spi") { + artifactId = "edc.spi" + from(components["java"]) + } + } +} \ No newline at end of file diff --git a/spi/src/main/java/com/microsoft/dagx/spi/types/domain/transfer/TransferProcess.java b/spi/src/main/java/com/microsoft/dagx/spi/types/domain/transfer/TransferProcess.java index 238ea18c..00e44112 100644 --- a/spi/src/main/java/com/microsoft/dagx/spi/types/domain/transfer/TransferProcess.java +++ b/spi/src/main/java/com/microsoft/dagx/spi/types/domain/transfer/TransferProcess.java @@ -178,7 +178,7 @@ public void transitionStreaming() { public void transitionCompleted() { // clients are in REQUESTED_ACK state after sending a request to the provider, they can directly transition to COMPLETED when the transfer is complete - transition(TransferProcessStates.COMPLETED, TransferProcessStates.IN_PROGRESS, TransferProcessStates.REQUESTED_ACK); + transition(TransferProcessStates.COMPLETED, TransferProcessStates.IN_PROGRESS, TransferProcessStates.REQUESTED_ACK, TransferProcessStates.STREAMING); } public void transitionDeprovisioning() { @@ -226,7 +226,7 @@ public Builder toBuilder() { private void transition(TransferProcessStates end, TransferProcessStates... starts) { if (Arrays.stream(starts).noneMatch(s -> s.code() == state)) { - throw new IllegalStateException(format("Cannot transition from state %s to %s", state, end.code())); + throw new IllegalStateException(format("Cannot transition from state %s to %s", TransferProcessStates.from(state), TransferProcessStates.from(end.code()))); } stateCount = state == end.code() ? stateCount + 1 : 1; state = end.code(); diff --git a/spi/src/main/java/com/microsoft/dagx/spi/types/domain/transfer/TransferType.java b/spi/src/main/java/com/microsoft/dagx/spi/types/domain/transfer/TransferType.java index 3d84cbec..a52fdd78 100644 --- a/spi/src/main/java/com/microsoft/dagx/spi/types/domain/transfer/TransferType.java +++ b/spi/src/main/java/com/microsoft/dagx/spi/types/domain/transfer/TransferType.java @@ -6,10 +6,16 @@ package com.microsoft.dagx.spi.types.domain.transfer; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; + import java.util.Objects; +@JsonDeserialize(builder = TransferType.Builder.class) public class TransferType { - private String contentType; + private static final String DEFAULT_CONTENT_TYPE = "application/octet-stream"; + private String contentType = DEFAULT_CONTENT_TYPE; private boolean isFinite = true; public String getContentType() { @@ -20,14 +26,15 @@ public boolean isFinite() { return isFinite; } - + @JsonPOJOBuilder(withPrefix = "") public static final class Builder { - private String contentType = "application/octet-stream"; + private String contentType = DEFAULT_CONTENT_TYPE; private boolean isFinite = true; private Builder() { } + @JsonCreator public static Builder transferType() { return new Builder(); }