diff --git a/README.md b/README.md index 61df0e2..4e58191 100644 --- a/README.md +++ b/README.md @@ -267,11 +267,20 @@ to select the ## Transfer Data +Now that both participants are set up, we can transfer data from the Provider to the Consumer. +There are two use case supported here: + +- Http proxy +- Certificates sharing via HTTP + +### Http proxy + EDC-V offers a one-stop-shop API to transfer data. This is achieved by two endpoints, one that fetches the catalog ( -`Data Transfer/Get Catalog`) and another endpoint (`Data Transfer/Get Data`) that initiates the contract negotiation, +`Data Transfer/Http Todo/Get Catalog`) and another endpoint (`Data Transfer/Http Todo/Get Data`) that initiates the +contract negotiation, waits for its successful completion, then starts the data transfer. -Perform the entire sequence by running both requests in the `Data Transfer` folder in Bruno: +Perform the entire sequence by running both requests in the `Data Transfer/Http Todo` folder in Bruno: ![img.png](docs/images/bruno_transfer.png) @@ -297,6 +306,30 @@ from https://jsonplaceholder.typicode.com/todos, something like: ] ``` +### Certificates sharing via HTTP + +The second use case demonstrates how certificates can be shared between participants using EDC-V's HTTP data +transfer capabilities. + +First we need to upload a certificate to the Provider. This is done by running the +`Data Transfer/Http Certs/Provider/Upload Certificate` request in Bruno: + +![img.png](docs/images/bruno_upload_certificate.png) + +by selecting a file to upload (e.g. a `.pdf` file). Additional metadata can be provided in the request body using +the `metadata` field. + +Then perform the entire sequence by running both requests in the `Data Transfer/Http Certs/Consumer` folder in Bruno: + +![img.png](docs/images/bruno_certificate_consumer.png) + +which: + +- Fetches the catalog from the Provider storing the offer id for the certificate asset +- Setup the transfer request using the offer id (contract negotiation + transfer initiation) storing the access token +- Query the provider for listing the available certificates storing the first certificate id +- Finally, download the certificate using the certificate id + ## Automated tests JAD comes with a set of automated tests that can be run against the deployed services. These tests are located in the diff --git a/docs/images/bruno_certificate_consumer.png b/docs/images/bruno_certificate_consumer.png new file mode 100644 index 0000000..e660e2a Binary files /dev/null and b/docs/images/bruno_certificate_consumer.png differ diff --git a/docs/images/bruno_transfer.png b/docs/images/bruno_transfer.png index 518cd59..f023626 100644 Binary files a/docs/images/bruno_transfer.png and b/docs/images/bruno_transfer.png differ diff --git a/docs/images/bruno_upload_certificate.png b/docs/images/bruno_upload_certificate.png new file mode 100644 index 0000000..08ae228 Binary files /dev/null and b/docs/images/bruno_upload_certificate.png differ diff --git a/extensions/api/mgmt/src/main/java/org/eclipse/edc/virtualized/api/data/DataApiController.java b/extensions/api/mgmt/src/main/java/org/eclipse/edc/virtualized/api/data/DataApiController.java index fe5b64b..ac524a4 100644 --- a/extensions/api/mgmt/src/main/java/org/eclipse/edc/virtualized/api/data/DataApiController.java +++ b/extensions/api/mgmt/src/main/java/org/eclipse/edc/virtualized/api/data/DataApiController.java @@ -115,6 +115,32 @@ public void getData(@PathParam("participantContextId") String participantContext }); + } + + @POST + @Path("/transfer") + public void setupTransfer(@PathParam("participantContextId") String participantContextId, DataRequest dataRequest, @Suspended AsyncResponse response) { + var participantContext = participantContextService.getParticipantContext(participantContextId); + if (participantContext.failed()) { + response.resume(Response.status(404).entity("Participant context '%s' not found".formatted(participantContextId)).build()); + } + dataRequestService.setupTransfer(participantContext.getContent(), dataRequest) + .whenComplete((result, throwable) -> { + try { + if (throwable != null) { + response.resume(Response.status(500).entity(throwable.getMessage()).build()); + + } else if (result.succeeded()) { + response.resume(result.getContent()); + } else { + response.resume(Response.status(500).entity(result.getFailureDetail()).build()); + } + } catch (Throwable mapped) { + response.resume(Response.status(500).entity(mapped.getMessage()).build()); + } + }); + + } private T toResponse(StatusResult result, Throwable throwable) throws Throwable { diff --git a/extensions/api/mgmt/src/main/java/org/eclipse/edc/virtualized/service/DataRequestService.java b/extensions/api/mgmt/src/main/java/org/eclipse/edc/virtualized/service/DataRequestService.java index 647c654..0fa8e80 100644 --- a/extensions/api/mgmt/src/main/java/org/eclipse/edc/virtualized/service/DataRequestService.java +++ b/extensions/api/mgmt/src/main/java/org/eclipse/edc/virtualized/service/DataRequestService.java @@ -38,6 +38,7 @@ import java.net.URI; import java.net.http.HttpRequest; import java.net.http.HttpResponse; +import java.util.Map; import java.util.concurrent.CompletableFuture; import static java.net.http.HttpClient.newHttpClient; @@ -71,6 +72,16 @@ public CompletableFuture> getData(ParticipantContext parti .thenApply(ServiceResult::success); } + public CompletableFuture>> setupTransfer(ParticipantContext participantContext, DataRequest dataRequest) { + return initiateContractNegotiation(participantContext, dataRequest) + .thenCompose(this::waitForContractNegotiation) + .thenCompose(contractNegotiation -> startTransferProcess(participantContext, contractNegotiation)) + .thenCompose(this::waitForTransferProcess) + .thenCompose(transferProcess -> getEdr(transferProcess.getId())) + .thenCompose(edr -> CompletableFuture.completedFuture(edr.getProperties())) + .thenApply(ServiceResult::success); + } + private CompletableFuture initiateContractNegotiation(ParticipantContext participantContext, DataRequest dataRequest) { var addressForDid = getAddressForDid(dataRequest.providerId()); if (addressForDid.failed()) { diff --git a/extensions/data-plane-certs/build.gradle.kts b/extensions/data-plane-certs/build.gradle.kts new file mode 100644 index 0000000..876018a --- /dev/null +++ b/extensions/data-plane-certs/build.gradle.kts @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2025 Metaform Systems, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Metaform Systems, Inc. - initial API and implementation + * + */ + +plugins { + `java-library` + id(libs.plugins.swagger.get().pluginId) +} + +dependencies { + api(libs.edc.spi.http) + api(libs.edc.spi.transaction) + api(libs.edc.spi.web) + api(libs.edc.spi.dataplane) + implementation(libs.jersey.multipart) + implementation(libs.edc.lib.util) + implementation(libs.edc.lib.sql) + implementation(libs.edc.core.sql.bootstrapper) + implementation(libs.edc.lib.util.dataplane) + implementation(libs.edc.dataplane.iam) + implementation(libs.jakarta.rsApi) + + testImplementation(libs.edc.lib.http) + testImplementation(libs.edc.junit) + testImplementation(libs.restAssured) + testImplementation(testFixtures(libs.edc.core.jersey)) + +} +edcBuild { + swagger { + apiGroup.set("public-api") + } +} + + diff --git a/extensions/data-plane-certs/src/main/java/org/eclipse/edc/virtualized/dataplane/cert/CertExchangeExtension.java b/extensions/data-plane-certs/src/main/java/org/eclipse/edc/virtualized/dataplane/cert/CertExchangeExtension.java new file mode 100644 index 0000000..c9ab5f5 --- /dev/null +++ b/extensions/data-plane-certs/src/main/java/org/eclipse/edc/virtualized/dataplane/cert/CertExchangeExtension.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2025 Metaform Systems, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Metaform Systems, Inc. - initial API and implementation + * + */ + +package org.eclipse.edc.virtualized.dataplane.cert; + +import org.eclipse.edc.connector.dataplane.iam.service.DataPlaneAuthorizationServiceImpl; +import org.eclipse.edc.connector.dataplane.spi.Endpoint; +import org.eclipse.edc.connector.dataplane.spi.edr.EndpointDataReferenceServiceRegistry; +import org.eclipse.edc.connector.dataplane.spi.iam.DataPlaneAuthorizationService; +import org.eclipse.edc.connector.dataplane.spi.iam.PublicEndpointGeneratorService; +import org.eclipse.edc.runtime.metamodel.annotation.Configuration; +import org.eclipse.edc.runtime.metamodel.annotation.Extension; +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.runtime.metamodel.annotation.Setting; +import org.eclipse.edc.runtime.metamodel.annotation.Settings; +import org.eclipse.edc.spi.system.Hostname; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; +import org.eclipse.edc.transaction.spi.TransactionContext; +import org.eclipse.edc.virtualized.dataplane.cert.api.CertExchangePublicController; +import org.eclipse.edc.virtualized.dataplane.cert.api.CertInternalExchangeController; +import org.eclipse.edc.virtualized.dataplane.cert.store.CertStore; +import org.eclipse.edc.web.spi.WebService; +import org.eclipse.edc.web.spi.configuration.PortMapping; +import org.eclipse.edc.web.spi.configuration.PortMappingRegistry; + +import static org.eclipse.edc.virtualized.dataplane.cert.CertExchangeExtension.NAME; + +@Extension(NAME) +public class CertExchangeExtension implements ServiceExtension { + public static final String NAME = "Cert Exchange Extension"; + public static final String API_CONTEXT = "certs"; + private static final int DEFAULT_CERTS_PORT = 8186; + private static final String DEFAULT_CERTS_PATH = "/api/data"; + + + @Setting(description = "Base url of the public public API endpoint without the trailing slash. This should point to the public certs endpoint configured.", + required = false, + key = "edc.dataplane.api.certs.baseurl", warnOnMissingConfig = true) + private String publicBaseUrl; + + @Configuration + private CertApiConfiguration apiConfiguration; + + @Inject + private Hostname hostname; + + @Inject + private PortMappingRegistry portMappingRegistry; + + @Inject + private DataPlaneAuthorizationService authorizationService; + @Inject + private PublicEndpointGeneratorService generatorService; + + @Inject + private EndpointDataReferenceServiceRegistry endpointDataReferenceServiceRegistry; + + @Inject + private WebService webService; + + @Inject + private CertStore certStore; + + @Inject + private TransactionContext transactionContext; + + @Override + public void initialize(ServiceExtensionContext context) { + var portMapping = new PortMapping(API_CONTEXT, apiConfiguration.port(), apiConfiguration.path()); + portMappingRegistry.register(portMapping); + + if (publicBaseUrl == null) { + publicBaseUrl = "http://%s:%d%s".formatted(hostname.get(), portMapping.port(), portMapping.path()); + context.getMonitor().warning("The public API endpoint was not explicitly configured, the default '%s' will be used.".formatted(publicBaseUrl)); + } + var endpoint = Endpoint.url(publicBaseUrl); + generatorService.addGeneratorFunction("HttpCertData", dataAddress -> endpoint); + webService.registerResource(API_CONTEXT, new CertExchangePublicController(authorizationService, certStore, transactionContext)); + webService.registerResource("control", new CertInternalExchangeController(certStore, transactionContext)); + + if (authorizationService instanceof DataPlaneAuthorizationServiceImpl dpAuthService) { + endpointDataReferenceServiceRegistry.register("HttpCertData", dpAuthService); + } + } + + @Settings + record CertApiConfiguration( + @Setting(key = "web.http." + API_CONTEXT + ".port", description = "Port for " + API_CONTEXT + " api context", defaultValue = DEFAULT_CERTS_PORT + "") + int port, + @Setting(key = "web.http." + API_CONTEXT + ".path", description = "Path for " + API_CONTEXT + " api context", defaultValue = DEFAULT_CERTS_PATH) + String path + ) { + + } +} diff --git a/extensions/data-plane-certs/src/main/java/org/eclipse/edc/virtualized/dataplane/cert/CertExchangeSqlExtension.java b/extensions/data-plane-certs/src/main/java/org/eclipse/edc/virtualized/dataplane/cert/CertExchangeSqlExtension.java new file mode 100644 index 0000000..90a2183 --- /dev/null +++ b/extensions/data-plane-certs/src/main/java/org/eclipse/edc/virtualized/dataplane/cert/CertExchangeSqlExtension.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2025 Metaform Systems, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Metaform Systems, Inc. - initial API and implementation + * + */ + +package org.eclipse.edc.virtualized.dataplane.cert; + +import org.eclipse.edc.runtime.metamodel.annotation.Extension; +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.runtime.metamodel.annotation.Provider; +import org.eclipse.edc.runtime.metamodel.annotation.Setting; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; +import org.eclipse.edc.spi.types.TypeManager; +import org.eclipse.edc.sql.QueryExecutor; +import org.eclipse.edc.sql.bootstrapper.SqlSchemaBootstrapper; +import org.eclipse.edc.transaction.datasource.spi.DataSourceRegistry; +import org.eclipse.edc.transaction.spi.TransactionContext; +import org.eclipse.edc.virtualized.dataplane.cert.store.CertStore; +import org.eclipse.edc.virtualized.dataplane.cert.store.sql.SqlCertStore; + +import static org.eclipse.edc.virtualized.dataplane.cert.CertExchangeSqlExtension.NAME; + + +@Extension(NAME) +public class CertExchangeSqlExtension implements ServiceExtension { + public static final String NAME = "Cert Exchange Sql Store Extension"; + + @Setting(description = "The datasource to be used", defaultValue = DataSourceRegistry.DEFAULT_DATASOURCE, key = "edc.sql.store.certs.datasource") + private String dataSourceName; + + @Inject + private DataSourceRegistry dataSourceRegistry; + @Inject + private TransactionContext transactionContext; + @Inject + private TypeManager typeManager; + @Inject + private QueryExecutor queryExecutor; + + @Inject + private SqlSchemaBootstrapper sqlSchemaBootstrapper; + + @Override + public void initialize(ServiceExtensionContext context) { + sqlSchemaBootstrapper.addStatementFromResource(dataSourceName, "certs-schema.sql"); + } + + @Provider + public CertStore certStore() { + return new SqlCertStore(dataSourceRegistry, dataSourceName, transactionContext, typeManager.getMapper(), queryExecutor); + } +} diff --git a/extensions/data-plane-certs/src/main/java/org/eclipse/edc/virtualized/dataplane/cert/api/CertExchangePublicController.java b/extensions/data-plane-certs/src/main/java/org/eclipse/edc/virtualized/dataplane/cert/api/CertExchangePublicController.java new file mode 100644 index 0000000..5dca370 --- /dev/null +++ b/extensions/data-plane-certs/src/main/java/org/eclipse/edc/virtualized/dataplane/cert/api/CertExchangePublicController.java @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2025 Metaform Systems, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Metaform Systems, Inc. - initial API and implementation + * + */ + +package org.eclipse.edc.virtualized.dataplane.cert.api; + +import com.fasterxml.jackson.core.type.TypeReference; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.HeaderParam; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.WebApplicationException; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.StreamingOutput; +import org.eclipse.edc.connector.dataplane.spi.iam.DataPlaneAuthorizationService; +import org.eclipse.edc.spi.query.QuerySpec; +import org.eclipse.edc.transaction.spi.TransactionContext; +import org.eclipse.edc.virtualized.dataplane.cert.model.CertMetadata; +import org.eclipse.edc.virtualized.dataplane.cert.store.CertStore; +import org.jetbrains.annotations.NotNull; + +import java.io.InputStream; +import java.util.List; +import java.util.Map; + +import static jakarta.ws.rs.core.HttpHeaders.AUTHORIZATION; +import static jakarta.ws.rs.core.Response.Status.FORBIDDEN; +import static jakarta.ws.rs.core.Response.Status.UNAUTHORIZED; + +@Path("certs") +public class CertExchangePublicController { + + private final DataPlaneAuthorizationService authorizationService; + private final CertStore certStore; + private final TransactionContext transactionContext; + + public CertExchangePublicController(DataPlaneAuthorizationService authorizationService, CertStore certStore, TransactionContext transactionContext) { + this.authorizationService = authorizationService; + this.certStore = certStore; + this.transactionContext = transactionContext; + } + + @POST + @Path("/request") + public List queryCertificates(@HeaderParam(AUTHORIZATION) String token, QuerySpec querySpec) { + return transactionContext.execute(() -> { + checkAuth(token); + return certStore.queryMetadata(querySpec); + }); + } + + @GET + @Path("/{id}") + public Response certificateDownload(@HeaderParam(AUTHORIZATION) String token, @PathParam("id") String id) { + return transactionContext.execute(() -> { + checkAuth(token); + var metadata = certStore.getMetadata(id); + if (metadata == null) { + return Response.status(Response.Status.NOT_FOUND).build(); + } + StreamingOutput stream = output -> { + try (InputStream is = certStore.retrieve(id)) { + is.transferTo(output); + } + }; + + return Response.ok(stream) + .header("Content-Type", metadata.contentType()) + .build(); + }); + } + + private void checkAuth(String token) { + if (token == null) { + throw new WebApplicationException(UNAUTHORIZED); + } + + var sourceDataAddress = authorizationService.authorize(token, Map.of()); + if (sourceDataAddress.failed()) { + throw new WebApplicationException(FORBIDDEN); + + } + } + + @NotNull + protected TypeReference getTypeRef() { + return new TypeReference<>() { + }; + } +} diff --git a/extensions/data-plane-certs/src/main/java/org/eclipse/edc/virtualized/dataplane/cert/api/CertInternalExchangeController.java b/extensions/data-plane-certs/src/main/java/org/eclipse/edc/virtualized/dataplane/cert/api/CertInternalExchangeController.java new file mode 100644 index 0000000..79596d0 --- /dev/null +++ b/extensions/data-plane-certs/src/main/java/org/eclipse/edc/virtualized/dataplane/cert/api/CertInternalExchangeController.java @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2025 Metaform Systems, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Metaform Systems, Inc. - initial API and implementation + * + */ + +package org.eclipse.edc.virtualized.dataplane.cert.api; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.ws.rs.BadRequestException; +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.StreamingOutput; +import org.eclipse.edc.spi.query.QuerySpec; +import org.eclipse.edc.transaction.spi.TransactionContext; +import org.eclipse.edc.virtualized.dataplane.cert.model.CertMetadata; +import org.eclipse.edc.virtualized.dataplane.cert.store.CertStore; +import org.glassfish.jersey.media.multipart.FormDataBodyPart; +import org.glassfish.jersey.media.multipart.FormDataParam; +import org.jetbrains.annotations.NotNull; + +import java.io.InputStream; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +@Path("certs") +public class CertInternalExchangeController { + + private final CertStore certStore; + private final TransactionContext transactionContext; + private final ObjectMapper objectMapper = new ObjectMapper(); + + public CertInternalExchangeController(CertStore certStore, TransactionContext transactionContext) { + this.certStore = certStore; + this.transactionContext = transactionContext; + } + + @POST + @Consumes(MediaType.MULTIPART_FORM_DATA) + public Response certificateUpload( + @FormDataParam("metadata") String metadata, + @FormDataParam("file") InputStream stream, + @FormDataParam("file") FormDataBodyPart bodyPart + ) { + var mediaType = bodyPart.getMediaType(); + var contentType = mediaType != null ? mediaType.toString() : "unknown"; + try { + Map certMetadataInput = objectMapper.convertValue(objectMapper.readTree(metadata), getTypeRef()); + byte[] bytes = stream.readAllBytes(); + var certMetadata = new CertMetadata( + java.util.UUID.randomUUID().toString(), + contentType, + certMetadataInput + ); + transactionContext.execute(() -> certStore.store(certMetadata.id(), certMetadata, bytes)); + return Response.ok().entity(certMetadata).build(); + } catch (Exception e) { + throw new BadRequestException(e); + } + + } + + @POST + @Path("/request") + public List queryCertificates(QuerySpec querySpec) { + + var query = Optional.ofNullable(querySpec) + .orElseGet(() -> QuerySpec.Builder.newInstance().build()); + + return transactionContext.execute(() -> certStore.queryMetadata(query)); + } + + @DELETE + @Path("/{id}") + public Response certificateDelete(@PathParam("id") String id) { + return transactionContext.execute(() -> { + certStore.delete(id); + return Response.ok().build(); + }); + } + + @GET + @Path("/{id}") + public Response certificateDownload(@PathParam("id") String id) { + return transactionContext.execute(() -> { + var metadata = certStore.getMetadata(id); + if (metadata == null) { + return Response.status(Response.Status.NOT_FOUND).build(); + } + StreamingOutput stream = output -> { + try (InputStream is = certStore.retrieve(id)) { + is.transferTo(output); + } + }; + + return Response.ok(stream) + .header("Content-Type", metadata.contentType()) + .build(); + }); + } + + @NotNull + protected TypeReference getTypeRef() { + return new TypeReference<>() { + }; + } +} diff --git a/extensions/data-plane-certs/src/main/java/org/eclipse/edc/virtualized/dataplane/cert/model/CertMetadata.java b/extensions/data-plane-certs/src/main/java/org/eclipse/edc/virtualized/dataplane/cert/model/CertMetadata.java new file mode 100644 index 0000000..eaf8446 --- /dev/null +++ b/extensions/data-plane-certs/src/main/java/org/eclipse/edc/virtualized/dataplane/cert/model/CertMetadata.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2025 Metaform Systems, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Metaform Systems, Inc. - initial API and implementation + * + */ + +package org.eclipse.edc.virtualized.dataplane.cert.model; + +import java.util.Map; + +public record CertMetadata(String id, String contentType, Map properties) { + +} diff --git a/extensions/data-plane-certs/src/main/java/org/eclipse/edc/virtualized/dataplane/cert/store/CertStore.java b/extensions/data-plane-certs/src/main/java/org/eclipse/edc/virtualized/dataplane/cert/store/CertStore.java new file mode 100644 index 0000000..a27a474 --- /dev/null +++ b/extensions/data-plane-certs/src/main/java/org/eclipse/edc/virtualized/dataplane/cert/store/CertStore.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2025 Metaform Systems, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Metaform Systems, Inc. - initial API and implementation + * + */ + +package org.eclipse.edc.virtualized.dataplane.cert.store; + +import org.eclipse.edc.spi.query.QuerySpec; +import org.eclipse.edc.virtualized.dataplane.cert.model.CertMetadata; + +import java.io.InputStream; +import java.util.List; + +/** + * Store for data plane certificates. + */ +public interface CertStore { + + /** + * Stores a certificate along with its metadata. + * + * @param id the unique identifier for the certificate + * @param metadata the metadata associated with the certificate + * @param content the certificate content as a byte array + */ + void store(String id, CertMetadata metadata, byte[] content); + + /** + * Deletes a certificate by its unique identifier. + * + * @param id the unique identifier of the certificate to delete + */ + void delete(String id); + + /** + * Queries certificate metadata based on the provided query specification. + * + * @param querySpec the query specification + * @return a list of certificate metadata matching the query + */ + List queryMetadata(QuerySpec querySpec); + + /** + * Retrieves the metadata of a certificate by its unique identifier. + * + * @param id the unique identifier of the certificate + * @return the certificate metadata, or null if not found + */ + CertMetadata getMetadata(String id); + + /** + * Retrieves the certificate data as an input stream by its unique identifier. + * + * @param id the unique identifier of the certificate + * @return the input stream of the certificate data + */ + InputStream retrieve(String id); +} diff --git a/extensions/data-plane-certs/src/main/java/org/eclipse/edc/virtualized/dataplane/cert/store/sql/SqlCertStore.java b/extensions/data-plane-certs/src/main/java/org/eclipse/edc/virtualized/dataplane/cert/store/sql/SqlCertStore.java new file mode 100644 index 0000000..a4faeb2 --- /dev/null +++ b/extensions/data-plane-certs/src/main/java/org/eclipse/edc/virtualized/dataplane/cert/store/sql/SqlCertStore.java @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2025 Metaform Systems, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Metaform Systems, Inc. - initial API and implementation + * + */ + +package org.eclipse.edc.virtualized.dataplane.cert.store.sql; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.eclipse.edc.spi.EdcException; +import org.eclipse.edc.spi.query.QuerySpec; +import org.eclipse.edc.sql.QueryExecutor; +import org.eclipse.edc.sql.store.AbstractSqlStore; +import org.eclipse.edc.sql.translation.SqlQueryStatement; +import org.eclipse.edc.transaction.datasource.spi.DataSourceRegistry; +import org.eclipse.edc.transaction.spi.TransactionContext; +import org.eclipse.edc.virtualized.dataplane.cert.model.CertMetadata; +import org.eclipse.edc.virtualized.dataplane.cert.store.CertStore; + +import java.io.InputStream; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; + +public class SqlCertStore extends AbstractSqlStore implements CertStore { + + public SqlCertStore(DataSourceRegistry dataSourceRegistry, String dataSourceName, TransactionContext transactionContext, ObjectMapper objectMapper, QueryExecutor queryExecutor) { + super(dataSourceRegistry, dataSourceName, transactionContext, objectMapper, queryExecutor); + } + + @Override + public void store(String id, CertMetadata metadata, byte[] content) { + transactionContext.execute(() -> { + try (var connection = getConnection()) { + var stmt = "INSERT INTO edc_certs (id, metadata, data) VALUES (?, ?::jsonb, ?)"; + var ps = connection.prepareStatement(stmt); + ps.setString(1, id); + ps.setString(2, toJson(metadata)); + ps.setBytes(3, content); + ps.executeUpdate(); + } catch (SQLException e) { + throw new EdcException(e); + } + }); + + } + + @Override + public void delete(String id) { + transactionContext.execute(() -> { + try (var connection = getConnection()) { + var stmt = "DELETE FROM edc_certs WHERE id = ?"; + var ps = connection.prepareStatement(stmt); + ps.setString(1, id); + ps.executeUpdate(); + } catch (SQLException e) { + throw new EdcException(e); + } + }); + } + + @Override + public List queryMetadata(QuerySpec querySpec) { + return transactionContext.execute(() -> { + try (var connection = getConnection()) { + var stmt = new SqlQueryStatement("SELECT metadata FROM edc_certs", querySpec.getLimit(), querySpec.getOffset()); + return queryExecutor.query(connection, true, this::mapMetadata, stmt.getQueryAsString(), stmt.getParameters()).toList(); + } catch (SQLException e) { + throw new EdcException(e); + } + }); + } + + private CertMetadata mapMetadata(ResultSet resultSet) throws SQLException { + var metadataJson = resultSet.getString("metadata"); + return fromJson(metadataJson, CertMetadata.class); + } + + @Override + public CertMetadata getMetadata(String id) { + return transactionContext.execute(() -> { + try (var connection = getConnection()) { + var stmt = "SELECT metadata FROM edc_certs WHERE id = ?"; + return queryExecutor.query(connection, true, this::mapMetadata, stmt, id) + .findFirst() + .orElse(null); + } catch (SQLException e) { + throw new EdcException(e); + } + }); + } + + @Override + public InputStream retrieve(String id) { + return transactionContext.execute(() -> { + try (var connection = getConnection()) { + var stmt = "SELECT data FROM edc_certs WHERE id = ?"; + return queryExecutor.query(connection, true, rs -> rs.getBinaryStream("data"), stmt, id) + .findFirst() + .orElse(null); + } catch (SQLException e) { + throw new EdcException(e); + } + }); + } +} diff --git a/extensions/data-plane-certs/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/data-plane-certs/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 0000000..e8c6508 --- /dev/null +++ b/extensions/data-plane-certs/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1,15 @@ +# +# Copyright (c) 2025 Metaform Systems, Inc. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# +# Contributors: +# Metaform Systems, Inc. - initial API and implementation +# +# +org.eclipse.edc.virtualized.dataplane.cert.CertExchangeExtension +org.eclipse.edc.virtualized.dataplane.cert.CertExchangeSqlExtension \ No newline at end of file diff --git a/extensions/data-plane-certs/src/main/resources/certs-schema.sql b/extensions/data-plane-certs/src/main/resources/certs-schema.sql new file mode 100644 index 0000000..89b6b5f --- /dev/null +++ b/extensions/data-plane-certs/src/main/resources/certs-schema.sql @@ -0,0 +1,22 @@ +-- +-- Copyright (c) 2025 Metaform Systems, Inc. +-- +-- This program and the accompanying materials are made available under the +-- terms of the Apache License, Version 2.0 which is available at +-- https://www.apache.org/licenses/LICENSE-2.0 +-- +-- SPDX-License-Identifier: Apache-2.0 +-- +-- Contributors: +-- Metaform Systems, Inc. - initial API and implementation +-- + +-- THIS SCHEMA HAS BEEN WRITTEN AND TESTED ONLY FOR POSTGRES + +-- table: edc_certs +CREATE TABLE IF NOT EXISTS edc_certs +( + id VARCHAR PRIMARY KEY, + metadata JSONB default '{}', + data BYTEA +); diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 3e7b384..c78311b 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -11,6 +11,7 @@ jackson-annotations = "2.20" restAssured = "6.0.0" rsApi = "4.0.0" swagger = "2.2.40" +jersey = "3.1.11" [libraries] @@ -24,10 +25,12 @@ edc-vault-hashicorp = { module = "org.eclipse.edc:vault-hashicorp", version.ref edc-store-participantcontext-config-sql = { module = "org.eclipse.edc:participantcontext-config-store-sql", version.ref = "edc" } edc-lib-http = { module = "org.eclipse.edc:http-lib", version.ref = "edc" } edc-lib-util = { module = "org.eclipse.edc:util-lib", version.ref = "edc" } +edc-lib-sql = { module = "org.eclipse.edc:sql-lib", version.ref = "edc" } edc-lib-util-dataplane = { module = "org.eclipse.edc:data-plane-util", version.ref = "edc" } edc-core-edrstore = { module = "org.eclipse.edc:edr-store-core", version.ref = "edc" } edc-edrstore-receiver = { module = "org.eclipse.edc:edr-store-receiver", version.ref = "edc" } - +edc-core-sql-bootstrapper = { module = "org.eclipse.edc:sql-bootstrapper", version.ref = "edc" } +edc-dataplane-iam = { module = "org.eclipse.edc:data-plane-iam", version.ref = "edc" } # EDC spi dependencies edc-spi-participantcontext = { module = "org.eclipse.edc:connector-participant-context-spi", version.ref = "edc" } edc-spi-participantcontext-config = { module = "org.eclipse.edc:participant-context-config-spi", version.ref = "edc" } @@ -73,6 +76,7 @@ jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind", ver tink = { module = "com.google.crypto.tink:tink", version = "1.20.0" } restAssured = { module = "io.rest-assured:rest-assured", version.ref = "restAssured" } jakarta-rsApi = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version.ref = "rsApi" } +jersey-multipart = { module = "org.glassfish.jersey.media:jersey-media-multipart", version.ref = "jersey" } [plugins] shadow = { id = "com.gradleup.shadow", version = "8.3.6" } diff --git a/k8s/apps/dataplane.yaml b/k8s/apps/dataplane.yaml index b651fd9..0844a06 100644 --- a/k8s/apps/dataplane.yaml +++ b/k8s/apps/dataplane.yaml @@ -46,6 +46,8 @@ spec: name: debug-port - containerPort: 8083 name: control-port + - containerPort: 8186 + name: certs-port - containerPort: 8080 name: health-port livenessProbe: @@ -87,6 +89,9 @@ spec: - name: control port: 8083 targetPort: 8083 + - name: certs + port: 8186 + targetPort: 8186 - name: public port: 11002 targetPort: 11002 @@ -111,4 +116,47 @@ spec: service: name: dataplane port: - number: 11002 \ No newline at end of file + number: 11002 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress-dataplane-control + namespace: edc-v + annotations: + nginx.ingress.kubernetes.io/rewrite-target: "/$2" + nginx.ingress.kubernetes.io/use-regex: "true" +spec: + ingressClassName: nginx + rules: + - http: + paths: + - path: /app/internal(/|$)(.*) + pathType: ImplementationSpecific + backend: + service: + name: dataplane + port: + number: 8083 + +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress-dataplane-certs-control + namespace: edc-v + annotations: + nginx.ingress.kubernetes.io/rewrite-target: "/$2" + nginx.ingress.kubernetes.io/use-regex: "true" +spec: + ingressClassName: nginx + rules: + - http: + paths: + - path: /app/public(/|$)(.*) + pathType: ImplementationSpecific + backend: + service: + name: dataplane + port: + number: 8186 \ No newline at end of file diff --git a/launchers/dataplane/build.gradle.kts b/launchers/dataplane/build.gradle.kts index 074f660..10a7bd5 100644 --- a/launchers/dataplane/build.gradle.kts +++ b/launchers/dataplane/build.gradle.kts @@ -24,6 +24,7 @@ dependencies { exclude("org.eclipse.edc", "data-plane-self-registration") } runtimeOnly(project(":extensions:data-plane-public-api-v2")) + runtimeOnly(project(":extensions:data-plane-certs")) runtimeOnly(libs.edc.core.participantcontext.config) runtimeOnly(libs.edc.vault.hashicorp) diff --git a/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Get Catalog.bru b/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Certs/Consumer/Get Catalog.bru similarity index 90% rename from requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Get Catalog.bru rename to requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Certs/Consumer/Get Catalog.bru index 81ed5e0..6beee10 100644 --- a/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Get Catalog.bru +++ b/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Certs/Consumer/Get Catalog.bru @@ -21,8 +21,10 @@ script:post-response { const json = res.getBody(); // Navigate the expected structure: Catalog -> dataset[0] -> hasPolicy[0] -> @id + const assetId = "cert_asset"; const datasets = json.dataset || []; - const firstDataset = datasets[0] || {}; + const firstDataset = datasets.filter(dataset => dataset["@id"] === assetId)[0] || {}; + const policies = firstDataset.hasPolicy || []; const firstPolicy = policies[0] || {}; const policyId = firstPolicy['@id']; diff --git a/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Certs/Consumer/Get certificate.bru b/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Certs/Consumer/Get certificate.bru new file mode 100644 index 0000000..03c0c3d --- /dev/null +++ b/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Certs/Consumer/Get certificate.bru @@ -0,0 +1,20 @@ +meta { + name: Get certificate + type: http + seq: 4 +} + +get { + url: {{baseURL}}/app/public/api/data/certs/{{cert_id}} + body: json + auth: none +} + +headers { + Authorization: {{ACCESS_TOKEN}} +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Certs/Consumer/List certificates.bru b/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Certs/Consumer/List certificates.bru new file mode 100644 index 0000000..421d751 --- /dev/null +++ b/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Certs/Consumer/List certificates.bru @@ -0,0 +1,39 @@ +meta { + name: List certificates + type: http + seq: 3 +} + +post { + url: {{baseURL}}/app/public/api/data/certs/request + body: json + auth: none +} + +headers { + Authorization: {{ACCESS_TOKEN}} +} + +body:json { + { + + } +} + +script:post-response { + try { + const json = res.getBody(); + const id = json[0] ? json[0].id : null; + + if (id) { + bru.setEnvVar("cert_id", id); + } + }catch (e) { + console.error("Failed to fetch cert id", e); + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Certs/Consumer/Setup Transfer.bru b/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Certs/Consumer/Setup Transfer.bru new file mode 100644 index 0000000..7dd12a0 --- /dev/null +++ b/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Certs/Consumer/Setup Transfer.bru @@ -0,0 +1,42 @@ +meta { + name: Setup Transfer + type: http + seq: 2 +} + +post { + url: {{baseURL}}/cp/api/mgmt/v1alpha/participants/{{consumer_id}}/transfer + body: json + auth: inherit +} + +body:json { + { + "providerId":"did:web:identityhub.edc-v.svc.cluster.local%3A7083:provider", + "policyId": "{{POLICY_ID}}" + } +} + +script:post-response { + try { + const json = res.getBody(); + const accessToken = json['https://w3id.org/edc/v0.0.1/ns/authorization']; + + if (accessToken && typeof accessToken === 'string') { + bru.setVar('ACCESS_TOKEN', accessToken); + } else { + console.warn('Access token not found'); + } + + } catch (e) { + console.error('Failed to parse response or set Access token:', e); + test('Response is valid JSON', function () { + throw e; + }); + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Certs/Consumer/folder.bru b/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Certs/Consumer/folder.bru new file mode 100644 index 0000000..e1bb3d3 --- /dev/null +++ b/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Certs/Consumer/folder.bru @@ -0,0 +1,8 @@ +meta { + name: Consumer + seq: 2 +} + +auth { + mode: inherit +} diff --git a/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Certs/Provider/Query Certificates.bru b/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Certs/Provider/Query Certificates.bru new file mode 100644 index 0000000..df9cf2d --- /dev/null +++ b/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Certs/Provider/Query Certificates.bru @@ -0,0 +1,16 @@ +meta { + name: Query Certificates + type: http + seq: 2 +} + +post { + url: {{baseURL}}/app/internal/api/control/certs/request + body: json + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Certs/Provider/Upload Certificate.bru b/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Certs/Provider/Upload Certificate.bru new file mode 100644 index 0000000..79b0c6f --- /dev/null +++ b/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Certs/Provider/Upload Certificate.bru @@ -0,0 +1,28 @@ +meta { + name: Upload Certificate + type: http + seq: 1 +} + +post { + url: {{baseURL}}/app/internal/api/control/certs + body: multipartForm + auth: inherit +} + +body:json { + { + "providerId":"did:web:identityhub.edc-v.svc.cluster.local%3A7083:provider", + "policyId": "{{POLICY_ID}}" + } +} + +body:multipart-form { + metadata: {} + file: @file(/home/wolf4ood/Downloads/ticket_19402661.pdf) +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Certs/Provider/folder.bru b/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Certs/Provider/folder.bru new file mode 100644 index 0000000..d886242 --- /dev/null +++ b/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Certs/Provider/folder.bru @@ -0,0 +1,8 @@ +meta { + name: Provider + seq: 1 +} + +auth { + mode: inherit +} diff --git a/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Certs/folder.bru b/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Certs/folder.bru new file mode 100644 index 0000000..7735250 --- /dev/null +++ b/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Certs/folder.bru @@ -0,0 +1,8 @@ +meta { + name: Http Certs + seq: 2 +} + +auth { + mode: inherit +} diff --git a/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Todo/Get Catalog.bru b/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Todo/Get Catalog.bru new file mode 100644 index 0000000..1846990 --- /dev/null +++ b/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Todo/Get Catalog.bru @@ -0,0 +1,55 @@ +meta { + name: Get Catalog + type: http + seq: 1 +} + +post { + url: {{baseURL}}/cp/api/mgmt/v1alpha/participants/{{consumer_id}}/catalog + body: json + auth: inherit +} + +body:json { + { + "counterPartyDid": "did:web:identityhub.edc-v.svc.cluster.local%3A7083:provider" + } +} + +script:post-response { + try { + const json = res.getBody(); + + // Navigate the expected structure: Catalog -> dataset[0] -> hasPolicy[0] -> @id + const assetId = "todo_asset"; + const datasets = json.dataset || []; + const firstDataset = datasets.filter(dataset => dataset["@id"] === assetId)[0] || {}; + + const policies = firstDataset.hasPolicy || []; + const firstPolicy = policies[0] || {}; + const policyId = firstPolicy['@id']; + + if (policyId && typeof policyId === 'string') { + bru.setVar('POLICY_ID', policyId); + console.log('Offer ID (POLICY_ID) is:', policyId); + } else { + console.warn('Policy ID not found in response at dataset[0].hasPolicy[0]["@id"]'); + } + + // Optional: assertion to ensure it exists + test('Policy ID is present and stored', function () { + expect(policyId, 'Policy ID should exist').to.be.a('string').and.not.empty; + }); + + } catch (e) { + console.error('Failed to parse response or set POLICY_ID:', e); + test('Response is valid JSON', function () { + throw e; + }); + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Get Data.bru b/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Todo/Get Data.bru similarity index 100% rename from requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Get Data.bru rename to requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Todo/Get Data.bru diff --git a/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Todo/folder.bru b/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Todo/folder.bru new file mode 100644 index 0000000..c4e0559 --- /dev/null +++ b/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/Http Todo/folder.bru @@ -0,0 +1,8 @@ +meta { + name: Http Todo + seq: 1 +} + +auth { + mode: inherit +} diff --git a/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/folder.bru b/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/folder.bru index 7346460..0caf013 100644 --- a/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/folder.bru +++ b/requests/EDC-V Onboarding/EDC-V Management/Data Transfer/folder.bru @@ -1,6 +1,6 @@ meta { name: Data Transfer - seq: 6 + seq: 3 } auth { diff --git a/requests/EDC-V Onboarding/EDC-V Management/Prepare Provider Participant/Create Cert Asset.bru b/requests/EDC-V Onboarding/EDC-V Management/Prepare Provider Participant/Create Cert Asset.bru new file mode 100644 index 0000000..6da26de --- /dev/null +++ b/requests/EDC-V Onboarding/EDC-V Management/Prepare Provider Participant/Create Cert Asset.bru @@ -0,0 +1,36 @@ +meta { + name: Create Cert Asset + type: http + seq: 2 +} + +post { + url: {{baseURL}}/cp/api/mgmt/v4alpha/participants/{{provider_id}}/assets + body: json + auth: inherit +} + +body:json { + { + "@context": [ + "https://w3id.org/edc/connector/management/v2" + ], + "@type": "Asset", + "@id": "{{ASSET_CERT_ID}}", + "properties": { + "description": "This asset requires the Membership credential to access" + }, + "privateProperties": { + "permission": "{{ASSET_PERMISSION}}" + }, + "dataAddress": { + "@type": "DataAddress", + "type": "HttpCertData" + } + } +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/requests/EDC-V Onboarding/EDC-V Management/Prepare Provider Participant/Create Contract Definition.bru b/requests/EDC-V Onboarding/EDC-V Management/Prepare Provider Participant/Create Contract Definition.bru index 9218530..302065f 100644 --- a/requests/EDC-V Onboarding/EDC-V Management/Prepare Provider Participant/Create Contract Definition.bru +++ b/requests/EDC-V Onboarding/EDC-V Management/Prepare Provider Participant/Create Contract Definition.bru @@ -1,7 +1,7 @@ meta { name: Create Contract Definition type: http - seq: 8 + seq: 3 } post { @@ -22,9 +22,9 @@ body:json { "assetsSelector": [ { "@type": "Criterion", - "operandLeft": "https://w3id.org/edc/v0.0.1/ns/id", + "operandLeft": "privateProperties.'https://w3id.org/edc/v0.0.1/ns/permission'", "operator" : "=", - "operandRight" : "{{ASSET_ID}}" + "operandRight" : "{{ASSET_PERMISSION}}" } ] } diff --git a/requests/EDC-V Onboarding/EDC-V Management/Prepare Provider Participant/Create Policy.bru b/requests/EDC-V Onboarding/EDC-V Management/Prepare Provider Participant/Create Policy.bru index de1f366..10119fd 100644 --- a/requests/EDC-V Onboarding/EDC-V Management/Prepare Provider Participant/Create Policy.bru +++ b/requests/EDC-V Onboarding/EDC-V Management/Prepare Provider Participant/Create Policy.bru @@ -1,7 +1,7 @@ meta { name: Create Policy type: http - seq: 8 + seq: 4 } post { diff --git a/requests/EDC-V Onboarding/EDC-V Management/Prepare Provider Participant/Create Asset.bru b/requests/EDC-V Onboarding/EDC-V Management/Prepare Provider Participant/Create Todo Asset.bru similarity index 81% rename from requests/EDC-V Onboarding/EDC-V Management/Prepare Provider Participant/Create Asset.bru rename to requests/EDC-V Onboarding/EDC-V Management/Prepare Provider Participant/Create Todo Asset.bru index bc85d66..13a3f84 100644 --- a/requests/EDC-V Onboarding/EDC-V Management/Prepare Provider Participant/Create Asset.bru +++ b/requests/EDC-V Onboarding/EDC-V Management/Prepare Provider Participant/Create Todo Asset.bru @@ -1,7 +1,7 @@ meta { - name: Create Asset + name: Create Todo Asset type: http - seq: 8 + seq: 1 } post { @@ -16,10 +16,13 @@ body:json { "https://w3id.org/edc/connector/management/v2" ], "@type": "Asset", - "@id": "{{ASSET_ID}}", + "@id": "{{ASSET_TODO_ID}}", "properties": { "description": "This asset requires the Membership credential to access" }, + "privateProperties": { + "permission": "{{ASSET_PERMISSION}}" + }, "dataAddress": { "@type": "DataAddress", "type": "HttpData", diff --git a/requests/EDC-V Onboarding/EDC-V Management/Prepare Provider Participant/Prepare Dataplane.bru b/requests/EDC-V Onboarding/EDC-V Management/Prepare Provider Participant/Prepare Dataplane.bru index 3747575..9682b8f 100644 --- a/requests/EDC-V Onboarding/EDC-V Management/Prepare Provider Participant/Prepare Dataplane.bru +++ b/requests/EDC-V Onboarding/EDC-V Management/Prepare Provider Participant/Prepare Dataplane.bru @@ -1,7 +1,7 @@ meta { name: Prepare Dataplane type: http - seq: 8 + seq: 5 } post { @@ -13,7 +13,8 @@ post { body:json { { "allowedSourceTypes": [ - "HttpData" + "HttpData", + "HttpCertData" ], "allowedTransferTypes": [ "HttpData-PULL" diff --git a/requests/EDC-V Onboarding/EDC-V Management/Prepare Provider Participant/folder.bru b/requests/EDC-V Onboarding/EDC-V Management/Prepare Provider Participant/folder.bru index 43cea18..e998c7a 100644 --- a/requests/EDC-V Onboarding/EDC-V Management/Prepare Provider Participant/folder.bru +++ b/requests/EDC-V Onboarding/EDC-V Management/Prepare Provider Participant/folder.bru @@ -7,7 +7,9 @@ auth { } vars:pre-request { - ASSET_ID: membership_asset + ASSET_PERMISSION: membership_asset POLICY_DEF_ID: membership_policy CONTRACT_DEF_ID: membership_contract_def + ASSET_TODO_ID: todo_asset + ASSET_CERT_ID: cert_asset } diff --git a/settings.gradle.kts b/settings.gradle.kts index e34dc0a..801adc7 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -27,6 +27,7 @@ pluginManagement { rootProject.name = "jad" include(":extensions:api:mgmt") include(":extensions:data-plane-public-api-v2") +include(":extensions:data-plane-certs") include(":tests:end2end") // launcher modules diff --git a/tests/end2end/src/test/java/org/eclipse/edc/jad/tests/DataTransferEndToEndTest.java b/tests/end2end/src/test/java/org/eclipse/edc/jad/tests/DataTransferEndToEndTest.java index 7cb0639..ceabe2c 100644 --- a/tests/end2end/src/test/java/org/eclipse/edc/jad/tests/DataTransferEndToEndTest.java +++ b/tests/end2end/src/test/java/org/eclipse/edc/jad/tests/DataTransferEndToEndTest.java @@ -27,6 +27,8 @@ import org.junit.jupiter.api.Test; import java.io.IOException; +import java.util.List; +import java.util.Map; import static io.restassured.RestAssured.given; import static org.assertj.core.api.Assertions.assertThat; @@ -49,6 +51,11 @@ public class DataTransferEndToEndTest { private static final String VAULT_TOKEN = "root"; + private static final ConsoleMonitor MONITOR = new ConsoleMonitor(ConsoleMonitor.Level.DEBUG, true); + private static ClientCredentials providerCredentials; + private static ClientCredentials consumerCredentials; + + static String loadResourceFile(String resourceName) { try (var is = Thread.currentThread().getContextClassLoader().getResourceAsStream(resourceName)) { if (is == null) { @@ -70,31 +77,61 @@ static void prepare() { return om; } )); - } - - @Test - void testDataTransfer() { - var monitor = new ConsoleMonitor(ConsoleMonitor.Level.DEBUG, true); - var adminToken = createKeycloakToken("admin", "edc-v-admin-secret", "issuer-admin-api:write", "identity-api:write", "management-api:write", "identity-api:read"); createCelExpression(adminToken); - monitor.info("Create cell and dataspace profile"); + MONITOR.info("Create cell and dataspace profile"); var cellId = getCellId(); // onboard consumer - monitor.info("Onboarding consumer"); - var po = new ParticipantOnboarding("consumer", "did:web:identityhub.edc-v.svc.cluster.local%3A7083:consumer", VAULT_TOKEN, monitor.withPrefix("Consumer")); - var consumerCredentials = po.execute(cellId); + MONITOR.info("Onboarding consumer"); + var po = new ParticipantOnboarding("consumer", "did:web:identityhub.edc-v.svc.cluster.local%3A7083:consumer", VAULT_TOKEN, MONITOR.withPrefix("Consumer")); + consumerCredentials = po.execute(cellId); // onboard provider - monitor.info("Onboarding provider"); - var providerPo = new ParticipantOnboarding("provider", "did:web:identityhub.edc-v.svc.cluster.local%3A7083:provider", VAULT_TOKEN, monitor.withPrefix("Provider")); - var providerCredentials = providerPo.execute(cellId); + MONITOR.info("Onboarding provider"); + var providerPo = new ParticipantOnboarding("provider", "did:web:identityhub.edc-v.svc.cluster.local%3A7083:provider", VAULT_TOKEN, MONITOR.withPrefix("Provider")); + providerCredentials = providerPo.execute(cellId); + } + + /** + * Creates a Common Expression Language (CEL) entry in the control plane + * + * @param accessToken OAuth2 token + */ + private static void createCelExpression(String accessToken) { + var template = loadResourceFile("create_cel_expression.json"); + + given() + .baseUri(BASE_URL) + .auth().oauth2(accessToken) + .contentType("application/json") + .body(template) + .post("/cp/api/mgmt/v4alpha/celexpressions") + .then() + .statusCode(200); + } + + /** + * Creates a cell in CFM. + * + * @return the Cell ID + */ + private static String getCellId() { + return given() + .contentType(APPLICATION_JSON) + .get(TM_BASE_URL + "/api/v1alpha1/cells") + .then() + .statusCode(200) + .extract().jsonPath().getString("[0].id"); + } + + @Test + void testTodoDataTransfer() { // seed provider - monitor.info("Seeding provider"); + MONITOR.info("Seeding provider"); var providerAccessToken = getAccessToken(providerCredentials.clientId(), providerCredentials.clientSecret(), "management-api:write").accessToken(); var assetId = createAsset(providerCredentials.clientId(), providerAccessToken); @@ -103,11 +140,11 @@ void testDataTransfer() { registerDataplane(providerCredentials.clientId(), providerAccessToken); // perform data transfer - monitor.info("Starting data transfer"); + MONITOR.info("Starting data transfer"); var catalog = fetchCatalog(consumerCredentials); - monitor.info("Catalog received, starting data transfer"); - var offerId = catalog.datasets().get(0).offers().get(0).id(); + MONITOR.info("Catalog received, starting data transfer"); + var offerId = catalog.datasets().stream().filter(dataSet -> dataSet.id().equals(assetId)).findFirst().get().offers().get(0).id(); assertThat(offerId).isNotNull(); //download dummy data @@ -128,74 +165,74 @@ void testDataTransfer() { assertThat(jsonResponse).isNotNull(); } + @Test + void testCertDataTransfer() { - private CatalogResponse fetchCatalog(ClientCredentials consumerCredentials) { - var accessToken = getAccessToken(consumerCredentials.clientId(), consumerCredentials.clientSecret(), "management-api:read"); + // seed provider + MONITOR.info("Seeding provider"); + var providerAccessToken = getAccessToken(providerCredentials.clientId(), providerCredentials.clientSecret(), "management-api:write").accessToken(); - return given() + var assetId = createCertAsset(providerCredentials.clientId(), providerAccessToken); + var policyDefId = createPolicyDef(providerCredentials.clientId(), providerAccessToken); + createContractDef(providerCredentials.clientId(), providerAccessToken, policyDefId, assetId); + registerDataplane(providerCredentials.clientId(), providerAccessToken); + + // perform data transfer + MONITOR.info("Starting data transfer"); + var catalog = fetchCatalog(consumerCredentials); + + MONITOR.info("Catalog received, starting data transfer"); + var offerId = catalog.datasets().stream().filter(dataSet -> dataSet.id().equals(assetId)).findFirst().get().offers().get(0).id(); + assertThat(offerId).isNotNull(); + + // trigger transfer + var transferResponse = given() .baseUri(BASE_URL) - .auth().oauth2(accessToken.accessToken()) - .contentType("application/json") + .auth().oauth2(getAccessToken(consumerCredentials.clientId(), consumerCredentials.clientSecret(), "management-api:write").accessToken()) .body(""" { - "counterPartyDid": "did:web:identityhub.edc-v.svc.cluster.local%3A7083:provider" + "providerId":"did:web:identityhub.edc-v.svc.cluster.local%%3A7083:provider", + "policyId": "%s" } - """) - .post("/cp/api/mgmt/v1alpha/participants/%s/catalog".formatted(consumerCredentials.clientId())) + """.formatted(offerId)) + .contentType("application/json") + .post("/cp/api/mgmt/v1alpha/participants/%s/transfer".formatted(consumerCredentials.clientId())) .then() .statusCode(200) - .extract().body() - .as(CatalogResponse.class); - } + .extract().body().as(Map.class); - private String createDataspaceProfile() { - return given() - .baseUri(TM_BASE_URL) - .contentType(APPLICATION_JSON) - .body(""" - { - "artifacts": [], - "properties": {} - } - """) - .post("/api/v1alpha1/dataspace-profiles") - .then() - .statusCode(201) - .log().ifValidationFails() - .extract().body().jsonPath().getString("id"); - } + var accessToken = transferResponse.get("https://w3id.org/edc/v0.0.1/ns/authorization"); - /** - * Creates a cell in CFM. - * - * @return the Cell ID - */ - private String getCellId() { - return given() - .contentType(APPLICATION_JSON) - .get(TM_BASE_URL + "/api/v1alpha1/cells") + var list = given() + .baseUri(BASE_URL) + .header("Authorization", accessToken) + .body("{}") + .contentType("application/json") + .post("app/public/api/data/certs/request") .then() .statusCode(200) - .extract().jsonPath().getString("[0].id"); - } + .extract().body().as(List.class); + assertThat(list).isEmpty(); + } - /** - * Creates a Common Expression Language (CEL) entry in the control plane - * - * @param accessToken OAuth2 token - */ - private void createCelExpression(String accessToken) { - var template = loadResourceFile("create_cel_expression.json"); + private CatalogResponse fetchCatalog(ClientCredentials consumerCredentials) { + var accessToken = getAccessToken(consumerCredentials.clientId(), consumerCredentials.clientSecret(), "management-api:read"); - given() + return given() .baseUri(BASE_URL) - .auth().oauth2(accessToken) + .auth().oauth2(accessToken.accessToken()) .contentType("application/json") - .body(template) - .post("/cp/api/mgmt/v4alpha/celexpressions") + .body(""" + { + "counterPartyDid": "did:web:identityhub.edc-v.svc.cluster.local%3A7083:provider" + } + """) + .post("/cp/api/mgmt/v1alpha/participants/%s/catalog".formatted(consumerCredentials.clientId())) .then() - .statusCode(200); + .statusCode(200) + .extract().body() + .as(CatalogResponse.class); } /** @@ -212,7 +249,7 @@ private void registerDataplane(String participantContextId, String accessToken) .auth().oauth2(accessToken) .body(""" { - "allowedSourceTypes": [ "HttpData" ], + "allowedSourceTypes": [ "HttpData", "HttpCertData" ], "allowedTransferTypes": [ "HttpData-PULL" ], "url": "http://dataplane.edc-v.svc.cluster.local:8083/api/control/v1/dataflows" } @@ -236,6 +273,19 @@ private String createAsset(String participantContextId, String accessToken) { .extract().jsonPath().getString(ID); } + private String createCertAsset(String participantContextId, String accessToken) { + var template = loadResourceFile("asset-cert.json"); + return given() + .baseUri(BASE_URL) + .auth().oauth2(accessToken) + .contentType("application/json") + .body(template) + .post("/cp/api/mgmt/v4alpha/participants/%s/assets".formatted(participantContextId)) + .then() + .statusCode(200) + .extract().jsonPath().getString(ID); + } + private String createPolicyDef(String participantContextId, String accessToken) { var template = loadResourceFile("policy-def.json"); return given() diff --git a/tests/end2end/src/test/java/org/eclipse/edc/jad/tests/model/DataSet.java b/tests/end2end/src/test/java/org/eclipse/edc/jad/tests/model/DataSet.java index 0bbe4b7..69ae8bc 100644 --- a/tests/end2end/src/test/java/org/eclipse/edc/jad/tests/model/DataSet.java +++ b/tests/end2end/src/test/java/org/eclipse/edc/jad/tests/model/DataSet.java @@ -20,7 +20,7 @@ import java.util.List; @JsonIgnoreProperties(ignoreUnknown = true) -public record DataSet(@JsonProperty("hasPolicy") List offers) { +public record DataSet(@JsonProperty("@id") String id, @JsonProperty("hasPolicy") List offers) { } diff --git a/tests/end2end/src/test/resources/asset-cert.json b/tests/end2end/src/test/resources/asset-cert.json new file mode 100644 index 0000000..7b8635c --- /dev/null +++ b/tests/end2end/src/test/resources/asset-cert.json @@ -0,0 +1,13 @@ +{ + "@context": [ + "https://w3id.org/edc/connector/management/v2" + ], + "@type": "Asset", + "properties": { + "description": "This asset requires the Membership credential to access" + }, + "dataAddress": { + "@type": "DataAddress", + "type": "HttpCertData" + } +} \ No newline at end of file