Skip to content

Commit

Permalink
add e2e data plane test (wip)
Browse files Browse the repository at this point in the history
  • Loading branch information
paullatzelsperger committed Feb 27, 2024
1 parent b120852 commit 293845d
Show file tree
Hide file tree
Showing 15 changed files with 331 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -63,5 +63,4 @@ public AccessTokenDataStore defaultAccessTokenDataStore() {
public PipelineService pipelineService(ServiceExtensionContext context) {
return new PipelineServiceImpl(context.getMonitor());
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -15,23 +15,32 @@
package org.eclipse.edc.connector.dataplane.framework;

import org.eclipse.edc.connector.api.client.spi.transferprocess.TransferProcessApiClient;
import org.eclipse.edc.connector.dataplane.framework.iam.DataPlaneAuthorizationServiceImpl;
import org.eclipse.edc.connector.dataplane.framework.manager.DataPlaneManagerImpl;
import org.eclipse.edc.connector.dataplane.framework.registry.TransferServiceRegistryImpl;
import org.eclipse.edc.connector.dataplane.framework.registry.TransferServiceSelectionStrategy;
import org.eclipse.edc.connector.dataplane.spi.Endpoint;
import org.eclipse.edc.connector.dataplane.spi.iam.DataPlaneAccessControlService;
import org.eclipse.edc.connector.dataplane.spi.iam.DataPlaneAccessTokenService;
import org.eclipse.edc.connector.dataplane.spi.iam.DataPlaneAuthorizationService;
import org.eclipse.edc.connector.dataplane.spi.iam.PublicEndpointGenerator;
import org.eclipse.edc.connector.dataplane.spi.manager.DataPlaneManager;
import org.eclipse.edc.connector.dataplane.spi.pipeline.DataTransferExecutorServiceContainer;
import org.eclipse.edc.connector.dataplane.spi.pipeline.PipelineService;
import org.eclipse.edc.connector.dataplane.spi.registry.TransferServiceRegistry;
import org.eclipse.edc.connector.dataplane.spi.store.DataPlaneStore;
import org.eclipse.edc.runtime.metamodel.annotation.Extension;
import org.eclipse.edc.runtime.metamodel.annotation.Inject;
import org.eclipse.edc.runtime.metamodel.annotation.Provider;
import org.eclipse.edc.runtime.metamodel.annotation.Provides;
import org.eclipse.edc.runtime.metamodel.annotation.Setting;
import org.eclipse.edc.spi.result.Result;
import org.eclipse.edc.spi.retry.ExponentialWaitStrategy;
import org.eclipse.edc.spi.system.ExecutorInstrumentation;
import org.eclipse.edc.spi.system.ServiceExtension;
import org.eclipse.edc.spi.system.ServiceExtensionContext;
import org.eclipse.edc.spi.telemetry.Telemetry;
import org.eclipse.edc.spi.types.domain.DataAddress;
import org.eclipse.edc.statemachine.retry.EntityRetryProcessConfiguration;
import org.jetbrains.annotations.NotNull;

Expand Down Expand Up @@ -88,6 +97,12 @@ public class DataPlaneFrameworkExtension implements ServiceExtension {

@Inject
private PipelineService pipelineService;
@Inject
private DataPlaneAccessTokenService accessTokenService;
@Inject
private DataPlaneAccessControlService accessControlService;
@Inject
private DataPlaneAuthorizationService dataPlaneAuthService;

@Override
public String name() {
Expand Down Expand Up @@ -120,6 +135,7 @@ public void initialize(ServiceExtensionContext context) {
.transferServiceRegistry(transferServiceRegistry)
.store(store)
.transferProcessClient(transferProcessApiClient)
.authorizationService(dataPlaneAuthService)
.monitor(monitor)
.telemetry(telemetry)
.build();
Expand All @@ -139,6 +155,17 @@ public void shutdown() {
}
}

@Provider
public DataPlaneAuthorizationService authorizationService(ServiceExtensionContext context) {
var endpointGenerator = new PublicEndpointGenerator() {
@Override
public Result<Endpoint> generateFor(DataAddress sourceDataAddress) {
return Result.success(null);
}
};
return new DataPlaneAuthorizationServiceImpl(accessTokenService, endpointGenerator, accessControlService, context.getParticipantId(), clock);
}

@NotNull
private EntityRetryProcessConfiguration getEntityRetryProcessConfiguration(ServiceExtensionContext context) {
var retryLimit = context.getSetting(DATAPLANE_SEND_RETRY_LIMIT, DEFAULT_SEND_RETRY_LIMIT);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ public DataPlaneAuthorizationServiceImpl(DataPlaneAccessTokenService accessToken
@Override
public Result<DataAddress> createEndpointDataReference(DataFlowStartMessage message) {
var endpoint = endpointGenerator.generateFor(message.getSourceDataAddress());

return endpoint.compose(e -> accessTokenService.obtainToken(createTokenParams(message), message.getSourceDataAddress()))
.compose(tokenRepresentation -> createDataAddress(tokenRepresentation, endpoint.getContent()));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import org.eclipse.edc.connector.core.entity.AbstractStateEntityManager;
import org.eclipse.edc.connector.dataplane.spi.DataFlow;
import org.eclipse.edc.connector.dataplane.spi.DataFlowStates;
import org.eclipse.edc.connector.dataplane.spi.iam.DataPlaneAuthorizationService;
import org.eclipse.edc.connector.dataplane.spi.manager.DataPlaneManager;
import org.eclipse.edc.connector.dataplane.spi.registry.TransferServiceRegistry;
import org.eclipse.edc.connector.dataplane.spi.store.DataPlaneStore;
Expand Down Expand Up @@ -49,6 +50,7 @@ public class DataPlaneManagerImpl extends AbstractStateEntityManager<DataFlow, D

private TransferServiceRegistry transferServiceRegistry;
private TransferProcessApiClient transferProcessClient;
private DataPlaneAuthorizationService authorizationService;

private DataPlaneManagerImpl() {

Expand All @@ -65,6 +67,7 @@ public Result<Boolean> validate(DataFlowStartMessage dataRequest) {

@Override
public void initiate(DataFlowStartMessage dataRequest) {
var result = authorizationService.createEndpointDataReference(dataRequest);
var dataFlow = DataFlow.Builder.newInstance()
.id(dataRequest.getProcessId())
.source(dataRequest.getSourceDataAddress())
Expand Down Expand Up @@ -214,6 +217,11 @@ public Builder transferProcessClient(TransferProcessApiClient transferProcessCli
manager.transferProcessClient = transferProcessClient;
return this;
}

public Builder authorizationService(DataPlaneAuthorizationService service) {
manager.authorizationService = service;
return this;
}
}

}
2 changes: 2 additions & 0 deletions settings.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,8 @@ include(":system-tests:e2e-transfer-test:backend-service")
include(":system-tests:e2e-transfer-test:control-plane")
include(":system-tests:e2e-transfer-test:data-plane")
include(":system-tests:e2e-transfer-test:runner")
include(":system-tests:e2e-dataplane-tests:runtimes:data-plane")
include(":system-tests:e2e-dataplane-tests:tests")
include(":system-tests:management-api:management-api-test-runner")
include(":system-tests:management-api:management-api-test-runtime")
include(":system-tests:protocol-test")
Expand Down
5 changes: 5 additions & 0 deletions system-tests/e2e-dataplane-tests/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# Data Plane E2E Tests

this module is intended to run test against a fully-fledged DataPlane, for example using the Signaling API.

At the time of this writing this does **not** involve a control plane!
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
* Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
*
* This program and the accompanying materials are made available under the
* terms of the Apache License, Version 2.0 which is available at
* https://www.apache.org/licenses/LICENSE-2.0
*
* SPDX-License-Identifier: Apache-2.0
*
* Contributors:
* Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation
*/

plugins {
`java-library`
}

dependencies {
implementation(project(":core:data-plane:data-plane-core"))
implementation(project(":extensions:control-plane:api:control-plane-api-client"))
implementation(project(":extensions:data-plane:data-plane-http"))
implementation(project(":extensions:data-plane:data-plane-control-api"))
implementation(project(":extensions:data-plane:data-plane-public-api"))
implementation(project(":extensions:common:vault:vault-filesystem"))
}

edcBuild {
publish.set(false)
}
47 changes: 47 additions & 0 deletions system-tests/e2e-dataplane-tests/tests/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/*
* Copyright (c) 2022 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
*
* This program and the accompanying materials are made available under the
* terms of the Apache License, Version 2.0 which is available at
* https://www.apache.org/licenses/LICENSE-2.0
*
* SPDX-License-Identifier: Apache-2.0
*
* Contributors:
* Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation
* Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
*
*/

plugins {
java
}

dependencies {
testImplementation(project(":spi:control-plane:transfer-spi"))
testImplementation(project(":extensions:common:sql:sql-core"))

testImplementation(project(":core:common:junit"))
testImplementation(testFixtures(project(":extensions:common:sql:sql-core")))
testImplementation(testFixtures(project(":extensions:control-plane:api:management-api:management-api-test-fixtures")))
testImplementation(project(":core:common:junit"))
testImplementation(project(":extensions:common:json-ld"))
testImplementation(libs.jakartaJson)

testImplementation(libs.postgres)
testImplementation(libs.restAssured)
testImplementation(libs.assertj)
testImplementation(libs.awaitility)
testImplementation(libs.junit.jupiter.api)
testImplementation(libs.mockserver.netty)
testImplementation(libs.mockserver.client)
testImplementation(libs.kafkaClients)

testCompileOnly(project(":system-tests:e2e-transfer-test:backend-service"))
testCompileOnly(project(":system-tests:e2e-transfer-test:control-plane"))
testCompileOnly(project(":system-tests:e2e-transfer-test:data-plane"))
}

edcBuild {
publish.set(false)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
/*
* Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
*
* This program and the accompanying materials are made available under the
* terms of the Apache License, Version 2.0 which is available at
* https://www.apache.org/licenses/LICENSE-2.0
*
* SPDX-License-Identifier: Apache-2.0
*
* Contributors:
* Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation
*
*/

package org.eclipse.edc.test.e2e;

import org.eclipse.edc.junit.annotations.EndToEndTest;
import org.eclipse.edc.junit.extensions.EdcClassRuntimesExtension;
import org.eclipse.edc.junit.extensions.EdcRuntimeExtension;
import org.eclipse.edc.spi.types.domain.DataAddress;
import org.eclipse.edc.spi.types.domain.transfer.DataFlowStartMessage;
import org.eclipse.edc.test.e2e.participant.DataPlaneParticipant;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;

import java.time.Duration;

@EndToEndTest
public class DataplaneEndToEndTest {

protected static final DataPlaneParticipant DATAPLANE = DataPlaneParticipant.Builder.newInstance()
.name("provider")
.id("urn:connector:provider")
.build();
@RegisterExtension
static EdcClassRuntimesExtension runtimes = new EdcClassRuntimesExtension(
new EdcRuntimeExtension(
":system-tests:e2e-dataplane-tests:runtimes:data-plane",
"provider-data-plane",
DATAPLANE.dataPlaneConfiguration()
)
);
protected final Duration timeout = Duration.ofSeconds(60);

@Test
void foobar() {
// no impl yet, we don't have a DataPlaneSignalingApiController yet

var flowMessage = DataFlowStartMessage.Builder.newInstance()
.processId("test-processId")
.sourceDataAddress(DataAddress.Builder.newInstance().type("Http").build())
.destinationDataAddress(DataAddress.Builder.newInstance().type("Http").build())
.assetId("test-asset")
.agreementId("test-agreement")
.build();
var result = DATAPLANE.initiateTransfer(flowMessage);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
/*
* Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
*
* This program and the accompanying materials are made available under the
* terms of the Apache License, Version 2.0 which is available at
* https://www.apache.org/licenses/LICENSE-2.0
*
* SPDX-License-Identifier: Apache-2.0
*
* Contributors:
* Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation
*
*/

package org.eclipse.edc.test.e2e.participant;

import com.fasterxml.jackson.annotation.JsonCreator;
import io.restassured.http.ContentType;
import jakarta.json.JsonObject;
import org.eclipse.edc.spi.types.domain.transfer.DataFlowStartMessage;
import org.eclipse.edc.test.system.utils.Participant;
import org.hamcrest.Matchers;
import org.jetbrains.annotations.NotNull;

import java.net.URI;
import java.util.HashMap;
import java.util.Map;

import static java.io.File.separator;
import static org.eclipse.edc.junit.testfixtures.TestUtils.getFreePort;

public class DataPlaneParticipant extends Participant {

private final URI controlPlaneControl = URI.create("http://localhost:" + getFreePort() + "/control");
private final URI dataPlaneDefault = URI.create("http://localhost:" + getFreePort());
private final URI dataPlaneControl = URI.create("http://localhost:" + getFreePort() + "/control");
private final URI dataPlanePublic = URI.create("http://localhost:" + getFreePort() + "/public");
private final Endpoint dataplaneControlApi = new Endpoint(dataPlaneControl);

private DataPlaneParticipant() {
super();
}

public Map<String, String> dataPlaneConfiguration() {
return new HashMap<>() {
{
put("web.http.port", String.valueOf(dataPlaneDefault.getPort()));
put("web.http.path", "/api");
put("web.http.public.port", String.valueOf(dataPlanePublic.getPort()));
put("web.http.public.path", "/public");
put("web.http.control.port", String.valueOf(dataPlaneControl.getPort()));
put("web.http.control.path", dataPlaneControl.getPath());
put("edc.vault", resourceAbsolutePath(getName() + "-vault.properties"));
put("edc.keystore", resourceAbsolutePath("certs/cert.pfx"));
put("edc.keystore.password", "123456");
put("edc.dataplane.token.validation.endpoint", controlPlaneControl + "/token");
put("edc.dataplane.http.sink.partition.size", "1");
}
};
}

/**
* Uses the data plane's control API to initiate a transfer
*/
public JsonObject initiateTransfer(DataFlowStartMessage startMessage) {
return dataplaneControlApi.baseRequest()
.contentType(ContentType.JSON)
.body(startMessage)
.post("/transfer")
.then()
.body(Matchers.notNullValue())
.extract().body().as(JsonObject.class);
}

@NotNull
private String resourceAbsolutePath(String filename) {
return System.getProperty("user.dir") + separator + "build" + separator + "resources" + separator + "test" + separator + filename;
}

public static final class Builder extends Participant.Builder<DataPlaneParticipant, Builder> {

private Builder() {
super(new DataPlaneParticipant());
}

@JsonCreator
public static Builder newInstance() {
return new Builder();
}

@Override
public DataPlaneParticipant build() {
super.managementEndpoint(new Endpoint(URI.create("http://localhost:" + getFreePort() + "/api/management")));
super.protocolEndpoint(new Endpoint(URI.create("http://localhost:" + getFreePort() + "/protocol")));
super.build();
return participant;
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
-----BEGIN CERTIFICATE-----
MIIDazCCAlOgAwIBAgIUZ3/sZXYzW4PjmOXKrZn6WBmUJ+4wDQYJKoZIhvcNAQEL
BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMjAyMjMxNTA2MDNaFw0zMjAy
MjExNTA2MDNaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw
HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB
AQUAA4IBDwAwggEKAoIBAQDBl6XaJnXTL+6DWip3aBhU+MzmY4d1V9hbTm1tiZ3g
E0VbUrvGO3LoYaxpPv6zFmsg3uJv6JxVAde7EddidN0ITHB9cQNdAfdUJ5njmsGS
PbdQuOQTHw0aG7/QvTI/nsvfEE6e0lbV/0e7DHacZT/+OztBH1RwkG2ymM94Hf8H
I6x7q6yfRTAZOqeOMrPCYTcluAgE9NskoPvjX5qASakBtXISKIsOU84N0/2HDN3W
EGMXvoHUQu6vrij6BwiwxKaw1AKwWENKoga775bPXN3M+JTSaIKE7dZbKzvx0Zi0
h5X+bxc3BJi3Z/CsUBCzE+Y0SFetOiYmyl/2YmnneYoVAgMBAAGjUzBRMB0GA1Ud
DgQWBBTvK1wVERwjni4B2vdH7KtEJeVWFzAfBgNVHSMEGDAWgBTvK1wVERwjni4B
2vdH7KtEJeVWFzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBn
QHiPA7OBYukHd9gS7c0HXE+fsWcS3GZeLqcHfQQnV3pte1vTmu9//IVW71wNCJ1/
rySRyODPQoPehxEcyHwupNZSzXK//nPlTdSgjMfFxscvt1YndyQLQYCfyOJMixAe
Aqrb14GTFHUUrdor0PyElhkULjkOXUrSIsdBrfWrwLTkelE8NK3tb5ZG8KPzD9Jy
+NwEPPr9d+iHkUkM7EFWw/cl56wka9ryBb97RI7DqbO6/j6OXHMk4GByxKv7DSIR
IvF9/Dw20qytajtaHV0pluFcOBuFc0NfiDvCaQlbTsfjzbc6UmZWbOi9YOJl3VQ/
g3h+15GuzbsSzOCOEYOT
-----END CERTIFICATE-----
Binary file not shown.
Loading

0 comments on commit 293845d

Please sign in to comment.