Merge branch 'develop' into feature/cmp-747/bpn-block

.gitignore

@@ -61,4 +61,8 @@ target
 
 *.tgz
 .tgz
-tgz
+tgz
+
+## Backup Files
+*.bck
+*.bak

CHANGELOG.md

@@ -24,6 +24,28 @@
 
 The changelog format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [released]
+## [v1.4.0] - 14-12-2023
+
+## Added
+- Added script to automate the uploading of various passport types
+- Added script to delete data from the data provider
+- Added check for empty or null contractIds with retry attempts
+- Added descriptive logs to search and create methods
+
+## Updated
+- Updated ingress settings and backend configuration in the helm chart
+- Refactored helm values to show only user relevant settings
+
+## Issued Fixed
+- Fixed the timeout time for each negotiation
+- Fixed the long waiting time by implementing timeout when doing the negotiation
+- Fixed the null contract ids creation
+
+## Deleted
+- Remove the legacy style to register/delete the testdata from the data provider
+
+
 ## [released]
 ## [v1.3.1] - 08-11-2023
 

INSTALL.md

@@ -31,10 +31,18 @@ You must have [Helm](https://helm.sh/),  [Minikube](https://minikube.sigs.k8s.io
 
 ## Install
 
+### First Step: Configuration
+
+First configure the [`values.yaml`](./charts/digital-product-pass/values.yaml) file with the secrets and the necessary configuration for starting the application correctly.
+
+> **TIP**: For a correct Catena-X integration get the correct credentials from the Portal! You can also place this secrets in a Vault so that the credentials are safe!
+
+### Second Step: Deployment
+
 To install the application using the configured helm charts use the following command from the project root directory:
 
 ```bash
-helm install digital-product-pass ./charts/digital-product-pass -f charts/digital-product-pass/values.yaml -f charts/digital-product-pass/values-int.yaml 
+helm install digital-product-pass ./charts/digital-product-pass -f charts/digital-product-pass/values.yaml 
 ``` 
 
 > **NOTE**: This command will deploy the complete application.

README.md

@@ -44,7 +44,7 @@ In particular, the appliction is used to access the battery passport data provid
 
 Here is a preview from the DPP App UI, where we visualize a test battery passport in this case.
 
-![General Info View](./docs/arc42/GraphicBatteryPassportViewGeneralInfo.png)
+![General Info View](./docs/arc42/media/GraphicBatteryPassportViewGeneralInfo.png)
 
 > **Note**: For more information check the [documentation section](./docs/README.md)
 
@@ -56,6 +56,7 @@ To get started you can have a look into our documentation:
 | ----------------------------------------------------------------          |-------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | [Arc42](./docs/arc42/Arc42.md)                                             | Main Architecture Document (Arc42) of Digital Product Pass Application                                                                                      |
 | [Administration Guide](./docs/admin%20guide/Admin_Guide.md)                  | Administration Guide explaining the infrastructure and how to configure the application                                                                     |
+| [Data Retrieval Guide](./docs/data%20retrieval%20guide/DataRetrievalGuide.md)                  | Guide on how to retrieve data from the Catena-X Network as the Digital Product Pass                                                                    |
 | [Backend Documentation](./consumer-backend/productpass/readme.md)          | Backend documentation Product Passport App                                                                                                                  |
 | [Deployment in Hotel Budapest](./deployment/README.md)                     | Technical Guide - Deployment in ArgoCD Hotel Budapest (integration environment)                                                                             |
 | [Docker Overview](./docker/README.md)                                      | Overview on general docker commands                                                                                                                         |

charts/digital-product-pass/templates/deployment-backend.yaml

@@ -84,8 +84,8 @@ spec:
             - name: backend-config
               mountPath: /app/config
             - name: pvc-backend
-              mountPath: /app/data
-              subPath: data
+              mountPath: /app/data/process
+              subPath: data/process
             - name: pvc-backend
               mountPath: /app/log
               subPath: log

charts/digital-product-pass/values-dev.yaml

@@ -104,4 +104,5 @@ oauth:
     enabled: true
     bpn: *bpn
   roleCheck:
-    enabled: false  
+    enabled: false  
+

consumer-backend/productpass/src/main/java/org/eclipse/tractusx/productpass/config/DtrConfig.java

@@ -152,6 +152,7 @@ public static class Timeouts{
         Integer negotiation;
         Integer transfer;
         Integer digitalTwin;
+        Integer dtrRequestProcess;
 
         /** GETTERS AND SETTERS **/
         public Integer getSearch() {
@@ -179,6 +180,14 @@ public Integer getDigitalTwin() {
         public void setDigitalTwin(Integer digitalTwin) {
             this.digitalTwin = digitalTwin;
         }
+
+        public Integer getDtrRequestProcess() {
+            return dtrRequestProcess;
+        }
+
+        public void setDtrRequestProcess(Integer dtrRequestProcess) {
+            this.dtrRequestProcess = dtrRequestProcess;
+        }
     }
 
     /**

consumer-backend/productpass/src/main/java/org/eclipse/tractusx/productpass/http/controllers/api/ContractController.java

@@ -155,6 +155,7 @@ public Response create(@Valid @RequestBody DiscoverySearch searchBody) {
                 return httpUtil.buildResponse(response, httpResponse);
             }
             String processId = processManager.initProcess();
+            LogUtil.printMessage("Creating process [" + processId + "] for "+searchBody.getType() + ": "+ searchBody.getId());
             ConcurrentHashMap<String, List<Dtr>> dataModel = null;
             if(dtrConfig.getTemporaryStorage().getEnabled()) {
                 try {
@@ -187,7 +188,7 @@ public Response create(@Valid @RequestBody DiscoverySearch searchBody) {
                     for(Dtr dtr: dtrs){
 
                         Long validUntil  = dtr.getValidUntil();
-                        if(validUntil == null || validUntil < currentTimestamp){
+                        if(dtr.getContractId() == null || dtr.getContractId().isEmpty() || validUntil == null || validUntil < currentTimestamp){
                             requestDtrs = true; // If the cache invalidation time has come request Dtrs
                             break;
                         }
@@ -270,7 +271,7 @@ public Response search(@Valid @RequestBody Search searchBody) {
                 response = httpUtil.getBadRequest("No processId was found on the request body!");
                 return httpUtil.buildResponse(response, httpResponse);
             }
-
+            
             String processId = searchBody.getProcessId();
             if(processId.isEmpty()){
                 response = httpUtil.getBadRequest("Process id is required for decentral digital twin registry searches!");
@@ -286,9 +287,12 @@ public Response search(@Valid @RequestBody Search searchBody) {
                 return httpUtil.buildResponse(response, httpResponse);
             }
             Boolean childrenCondition = searchBody.getChildren();
+            String logPrint = "[" + processId + "] Creating search for "+searchBody.getIdType() + ": "+ searchBody.getId();
             if(childrenCondition != null){
+                LogUtil.printMessage(logPrint + " with drilldown enabled");
                 process = processManager.createProcess(processId, childrenCondition, httpRequest); // Store the children condition
             }else {
+                LogUtil.printMessage(logPrint + " with drilldown disabled");
                 process = processManager.createProcess(processId, httpRequest);
             }
             Status status = processManager.getStatus(processId);

consumer-backend/productpass/src/main/java/org/eclipse/tractusx/productpass/managers/DtrSearchManager.java

@@ -69,7 +69,7 @@ public class DtrSearchManager {
     private ConcurrentHashMap<String, List<Dtr>> dtrDataModel;
     private ConcurrentHashMap<String, Catalog> catalogsCache;
     private final long searchTimeoutSeconds;
-    private final long negotiationTimeoutSeconds;
+    private final long dtrRequestProcessTimeout;
     private final String fileName = "dtrDataModel.json";
     private String dtrDataModelFilePath;
     private State state;
@@ -93,8 +93,7 @@ public DtrSearchManager(FileUtil fileUtil, JsonUtil jsonUtil, DataTransferServic
         this.dtrDataModelFilePath = this.createDataModelFile();
         this.dtrDataModel = this.loadDtrDataModel();
         this.searchTimeoutSeconds = this.dtrConfig.getTimeouts().getSearch();
-        this.negotiationTimeoutSeconds = this.dtrConfig.getTimeouts().getNegotiation();
-
+        this.dtrRequestProcessTimeout = this.dtrConfig.getTimeouts().getDtrRequestProcess();
     }
 
     /** GETTERS AND SETTERS **/
@@ -205,7 +204,7 @@ public void run() {
     }
     public void searchEndpoint(String processId, String bpn, String endpoint){
         //Search Digital Twin Catalog for each connectionURL with a timeout time
-        Thread asyncThread = ThreadUtil.runThread(searchDigitalTwinCatalogExecutor(endpoint), "ProcessDtrDataModel");
+        Thread asyncThread = ThreadUtil.runThread(searchDigitalTwinCatalogExecutor(endpoint), "SearchEndpoint"+processId+"-"+bpn+"-"+endpoint);
         try {
             if (!asyncThread.join(Duration.ofSeconds(searchTimeoutSeconds))) {
                 asyncThread.interrupt();
@@ -228,11 +227,11 @@ public void searchEndpoint(String processId, String bpn, String endpoint){
         if (contractOffers instanceof LinkedHashMap) {
             Dataset dataset = (Dataset) jsonUtil.bindObject(contractOffers, Dataset.class);
             if (dataset != null) {
-                Thread singleOfferThread = ThreadUtil.runThread(createAndSaveDtr(dataset, bpn, endpoint, processId), "CreateAndSaveDtr");
+                Thread singleOfferThread = ThreadUtil.runThread(createAndSaveDtr(dataset, bpn, endpoint, processId), "CreateAndSaveDtr-"+processId+"-"+bpn+"-"+endpoint);
                 try {
-                    if (!singleOfferThread.join(Duration.ofSeconds(negotiationTimeoutSeconds))) {
+                    if (!singleOfferThread.join(Duration.ofSeconds(this.dtrRequestProcessTimeout))) {
                         singleOfferThread.interrupt();
-                        LogUtil.printWarning("Failed to retrieve the Catalog due a timeout for the URL: " + endpoint);
+                        LogUtil.printWarning("Failed to retrieve do contract negotiations due a timeout for the URL: " + endpoint);
                         return;
                     }
                 } catch (InterruptedException e) {
@@ -246,11 +245,11 @@ public void searchEndpoint(String processId, String bpn, String endpoint){
             return;
         }
         contractOfferList.parallelStream().forEach(dataset -> {
-            Thread multipleOffersThread = ThreadUtil.runThread(createAndSaveDtr(dataset, bpn, endpoint, processId), "CreateAndSaveDtr");
+            Thread multipleOffersThread = ThreadUtil.runThread(createAndSaveDtr(dataset, bpn, endpoint, processId), "CreateAndSaveDtr-"+processId+"-"+bpn+"-"+endpoint);
             try {
-                if (!multipleOffersThread.join(Duration.ofSeconds(negotiationTimeoutSeconds))) {
+                if (!multipleOffersThread.join(Duration.ofSeconds(this.dtrRequestProcessTimeout))) {
                     multipleOffersThread.interrupt();
-                    LogUtil.printWarning("Failed to retrieve the Catalog due a timeout for the URL: " + endpoint);
+                    LogUtil.printWarning("Failed to retrieve the contract negotiations due a timeout for the URL: " + endpoint);
                 }
             } catch (InterruptedException e) {
                 throw new RuntimeException(e);
@@ -426,16 +425,22 @@ private Runnable createAndSaveDtr(Dataset dataset, String bpn, String connection
             public void run() {
                 try {
                     Offer offer = dataTransferService.buildOffer(dataset, 0);
-                    String builtDataEndpoint =CatenaXUtil.buildDataEndpoint(connectionUrl);
+                    String builtDataEndpoint = CatenaXUtil.buildDataEndpoint(connectionUrl);
                     IdResponse negotiationResponse = dataTransferService.doContractNegotiation(offer, bpn, builtDataEndpoint);
                     if (negotiationResponse == null) {
                         return;
                     }
-                    Negotiation negotiation = dataTransferService.seeNegotiation(negotiationResponse.getId());
+                    Integer millis =  dtrConfig.getTimeouts().getNegotiation() * 1000; // Set max timeout from seconds to milliseconds
+                    // If negotiation takes way too much time give timeout
+                    Negotiation negotiation = ThreadUtil.timeout(millis, ()->dataTransferService.seeNegotiation(negotiationResponse.getId()), null);
                     if (negotiation == null) {
                         LogUtil.printWarning("It was not possible to do ContractNegotiation for URL: " + connectionUrl);
                         return;
                     }
+                    if(negotiation.getContractAgreementId() == null || negotiation.getContractAgreementId().isEmpty()){
+                        LogUtil.printError("It was not possible to get an Contract Agreemment Id for the URL: " + connectionUrl);
+                        return;
+                    }
                     Dtr dtr = new Dtr(negotiation.getContractAgreementId(), connectionUrl, offer.getAssetId(), bpn, DateTimeUtil.addHoursToCurrentTimestamp(dtrConfig.getTemporaryStorage().getLifetime()));
                     if (dtrConfig.getTemporaryStorage().getEnabled()) {
                         addConnectionToBpnEntry(bpn, dtr);

consumer-backend/productpass/src/main/java/utils/ThreadUtil.java

@@ -134,12 +134,12 @@ public static <V> V timeout(Integer milliseconds, Callable<V> function, V timeou
     {
         try {
             ExecutorService executor = Executors.newSingleThreadExecutor();
-            Future<V> future = executor.submit(function);
             boolean timeout = false;
             V returnObject = null;
             try {
+                Future<V> future = executor.submit(function);
                 returnObject = future.get(milliseconds, TimeUnit.MILLISECONDS);
-            } catch (TimeoutException e) {
+            } catch (Exception e) {
                 timeout = true;
             }
             executor.shutdownNow();

consumer-backend/productpass/src/main/resources/application.yml

@@ -83,7 +83,8 @@ configuration:
       subModel: "/submodel-descriptors"
     timeouts:
       search: 10
-      negotiation: 40
+      negotiation: 10
+      dtrRequestProcess: 40
       transfer: 10
       digitalTwin: 20
     temporaryStorage:

deployment/infrastructure/testing/testdata/testdata-payload.json

@@ -15,11 +15,19 @@
             "odrl:or": [
               {
                 "@type": "Constraint",
-                "odrl:leftOperand": "PURPOSE",
+                "odrl:leftOperand": "Membership",
                 "odrl:operator": {
                   "@id": "odrl:eq"
                 },
-                "odrl:rightOperand": "DPP"
+                "odrl:rightOperand": "active"
+              },
+              {
+                "@type": "Constraint",
+                "odrl:leftOperand": "FrameworkAgreement.sustainability",
+                "odrl:operator": {
+                  "@id": "odrl:eq"
+                },
+                "odrl:rightOperand": "active"
               }
             ]
           }
@@ -2306,6 +2314,45 @@
           "semanticId": "urn:bamm:io.catenax.transmission.transmission_pass:1.0.0#TransmissionPass"
         }
       ]
+    },
+    {
+      "catenaXId": "urn:uuid:b2c4c99d-f187-45ad-905b-e2c4e1ededba",
+      "specificAssetIds": [
+        {
+          "name": "manufacturerPartId",
+          "value": "XYZ78901",
+          "allowedBpns": ["BPNL00000003CRHL", "BPNL00000000CBA5"]
+        },
+        {
+          "name": "partInstanceId",
+          "value": "ABC123",
+          "allowedBpns": ["BPNL00000003CRHL", "BPNL00000000CBA5"]
+        }
+      ],
+      "type": "physicalDimensions",
+      "description": [
+        {
+          "language": "en",
+          "text": "Physical Dimensions shell descriptor"
+        }
+      ],
+      "submodels": [
+        {
+          "name": "physicalDeminsions",
+          "data": {
+            "partInstanceId": "ABC123",
+            "physicalDimensionsProperty": {
+              "width": 1000.0,
+              "length": 20000.1,
+              "weight": 100.7,
+              "diameter": 0.03,
+              "height": 0.1
+            }
+          },
+          "description": "Physical Dimensions Submodel",
+          "semanticId": "urn:bamm:io.catenax.shared.physical_dimension:2.0.0#PhysicalDimensions"
+        }
+      ]
     }
   ]
 }

docs/RELEASE_USER.md

@@ -23,6 +23,16 @@
 # Release Notes Digital Product Pass Application
 User friendly relase notes without especific technical details.
 
+**November 14 2023 (Version 1.4.0)**
+*14.12.2023*
+
+### Added
+#### DPP test data uploader
+A script is refactored to upload/remove testdata set from the data provider setup. This speeds up the automatic uploading of various passes types into the provider's digital twin registry, data service and EDC connector.
+
+### Updated
+#### Optimize contract negotiation time
+There was a long waiting time during the contract negotiation. This time is now reduced and the negotiation is perfomred faster.
 
 **November 08 2023 (Version 1.3.1)**
 *08.11.2023*