[Feat]: encryption/decryption for private fields

.github/workflows/maven.yml

@@ -28,8 +28,8 @@ jobs:
     #   with:
     #     ## limits ssh access and adds the ssh public key for the user which triggered the workflow ie Sreejith-K
     #     limit-access-to-actor: true
-    # - name: Build and test
-    #   run: make test
+    - name: Build and test
+      run: make test
 #  test:
 #    runs-on: ubuntu-latest
 #    steps:

Makefile

@@ -20,6 +20,7 @@ build: java/registry/target/registry.jar
 	make -C services/metrics docker
 	make -C services/digilocker-certificate-api docker
 	make -C services/bulk_issuance docker
+	make -C services/encryption-service docker
 	docker build -t ghcr.io/sunbird-rc/sunbird-rc-nginx .
 
 java/registry/target/registry.jar: $(SOURCES)

docker-compose.yml

@@ -352,3 +352,13 @@ services:
       test:
         wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit
         1
+  encryption-service:
+    image: ghcr.io/sunbird-rc/encryption-service:v0.0.14
+    ports:
+      - '8013:8013'
+    environment:
+      spring.datasource.url: jdbc:postgresql://db:5432/registry
+      spring.flyway.url: jdbc:postgresql://db:5432/registry
+    depends_on:
+      db:
+        condition: service_healthy

java/registry/src/main/java/dev/sunbirdrc/registry/helper/RegistryHelper.java

@@ -309,6 +309,8 @@ public JsonNode readEntity(String userId, String entityType, String label, boole
                 resultNode = includePrivateFields ? decryptionHelper.getDecryptedJson(resultNode) : resultNode;
             }
             resultNode = vTransformer.transform(viewTemplate, resultNode);
+        } else if (encryptionEnabled) {
+            resultNode = decryptionHelper.getDecryptedJson(resultNode);
         }
         logger.debug("readEntity ends");
         if(isEventsEnabled) {

java/registry/src/main/java/dev/sunbirdrc/registry/service/DecryptionHelper.java

@@ -1,6 +1,8 @@
 package dev.sunbirdrc.registry.service;
 
 import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import dev.sunbirdrc.registry.exception.EncryptionException;
 import dev.sunbirdrc.registry.util.PrivateField;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
@@ -14,8 +16,10 @@ public class DecryptionHelper extends PrivateField {
 
     public JsonNode getDecryptedJson(JsonNode rootNode) throws EncryptionException {
         String rootFieldName = rootNode.fieldNames().next();
-        process(rootNode.get(rootFieldName), rootFieldName, null);
-        return rootNode;
+        JsonNode updatedNode = process(rootNode.get(rootFieldName), rootFieldName, null);
+        ObjectNode objectNode = JsonNodeFactory.instance.objectNode();
+        objectNode.set(rootFieldName, updatedNode);
+        return objectNode;
     }
 
     protected Map<String, Object> performOperation(Map<String, Object> plainMap) throws EncryptionException {

java/registry/src/main/java/dev/sunbirdrc/registry/service/EncryptionHelper.java

@@ -1,6 +1,8 @@
 package dev.sunbirdrc.registry.service;
 
 import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import dev.sunbirdrc.registry.exception.EncryptionException;
 import dev.sunbirdrc.registry.util.PrivateField;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
@@ -17,8 +19,9 @@ protected Map<String, Object> performOperation(Map<String, Object> plainMap) thr
 
     public JsonNode getEncryptedJson(JsonNode rootNode) throws EncryptionException {
         String rootFieldName = rootNode.fieldNames().next();
-        process(rootNode.get(rootFieldName), rootFieldName, null);
-
-        return rootNode;
+        JsonNode updatedNode = process(rootNode.get(rootFieldName), rootFieldName, null);
+        ObjectNode objectNode = JsonNodeFactory.instance.objectNode();
+        objectNode.set(rootFieldName, updatedNode);
+        return objectNode;
     }
 }

java/registry/src/main/java/dev/sunbirdrc/registry/service/impl/EncryptionServiceImpl.java

@@ -23,8 +23,7 @@
 import org.springframework.web.client.ResourceAccessException;
 import org.springframework.web.client.RestClientException;
 
-import java.util.HashMap;
-import java.util.Map;
+import java.util.*;
 
 import static dev.sunbirdrc.registry.middleware.util.Constants.CONNECTION_FAILURE;
 import static dev.sunbirdrc.registry.middleware.util.Constants.SUNBIRD_ENCRYPTION_SERVICE_NAME;
@@ -36,6 +35,10 @@ public class EncryptionServiceImpl implements EncryptionService {
 	private static Logger logger = LoggerFactory.getLogger(EncryptionServiceImpl.class);
 	@Value("${encryption.enabled}")
 	private boolean encryptionEnabled;
+	@Value("${encryption.tenant.id}")
+	private String encryptionTenantId;
+	@Value("${encryption.method}")
+	private String encryptionMethod;
 	@Value("${encryption.uri}")
 	private String encryptionUri;
 	@Value("${decryption.uri}")
@@ -62,20 +65,7 @@ public class EncryptionServiceImpl implements EncryptionService {
 	 */
 	@Override
 	public String encrypt(Object propertyValue) throws EncryptionException {
-		logger.debug("encrypt starts with value");
-		MultiValueMap<String, Object> map = new LinkedMultiValueMap<>();
-		map.add("value", propertyValue);
-		HttpEntity<MultiValueMap<String, Object>> request = new HttpEntity<>(map);
-		try {
-			ResponseEntity<String> response = retryRestTemplate.postForEntity(encryptionUri, request);
-			return response.getBody();
-		} catch (ResourceAccessException e) {
-			logger.error("ResourceAccessException while connecting encryption service : {}", ExceptionUtils.getStackTrace(e));
-			throw new EncryptionException("Exception while connecting encryption service! ");
-		} catch (Exception e) {
-			logger.error("Exception in encryption service !: {}", ExceptionUtils.getStackTrace(e));
-			throw new EncryptionException("Exception in encryption service");
-		}
+		return this.doEncrypt(propertyValue, encryptionUri).toString();
 	}
 
 	/** decrypts the input
@@ -85,21 +75,7 @@ public String encrypt(Object propertyValue) throws EncryptionException {
 	 */
 	@Override
 	public String decrypt(Object propertyValue) throws EncryptionException {
-		logger.debug("decrypt starts with value {}", propertyValue);
-		MultiValueMap<String, Object> map = new LinkedMultiValueMap<>();
-		map.add("value", propertyValue);
-		HttpEntity<MultiValueMap<String, Object>> request = new HttpEntity<>(map);
-		try {
-			ResponseEntity<String> response = retryRestTemplate.postForEntity(decryptionUri,request);
-			logger.info("Property decrypted successfully !");
-			return response.getBody();
-		} catch (ResourceAccessException e) {
-			logger.error("ResourceAccessException while connecting decryption service : {}", ExceptionUtils.getStackTrace(e));
-			throw new EncryptionException("Exception while connecting encryption service ! ");
-		} catch (Exception e) {
-			logger.error("Exception in decryption service !: {}", ExceptionUtils.getStackTrace(e));
-			throw new EncryptionException("Exception in encryption service ! ");
-		}
+		return this.doDecrypt(propertyValue, decryptionUri).toString();
 	}
 
 	/** encrypts the input which is in Map format
@@ -109,19 +85,39 @@ public String decrypt(Object propertyValue) throws EncryptionException {
 	 */
 	@Override
 	public Map<String, Object> encrypt(Map<String, Object> propertyValue) throws EncryptionException {
+		return this.doEncrypt(propertyValue, encryptionBatchUri);
+	}
+
+	/** decrypts the input which is in Map format
+	 * @param propertyValue - input is in format Map<String, Object>
+	 * @return Map<String, Object>
+	 * @throws EncryptionException
+	 */
+	@Override
+	public Map<String, Object> decrypt(Map<String, Object> propertyValue) throws EncryptionException {
+		return this.doDecrypt(propertyValue, decryptionBatchUri);
+	}
+
+	private <T> T doEncrypt(T propertyValue, String uri) throws EncryptionException {
 		logger.debug("encrypt starts with value {}", propertyValue);
 		Map<String, Object> map = new HashMap<>();
-		map.put("value", propertyValue);
+		Map<String, Object> encReqObj = new HashMap<>();
+		encReqObj.put("tenantId", encryptionTenantId);
+		encReqObj.put("value", propertyValue);
+		encReqObj.put("type", encryptionMethod);
+		map.put("encryptionRequests", Collections.singletonList(encReqObj));
 
 		HttpHeaders headers = new HttpHeaders();
 		headers.setContentType(MediaType.APPLICATION_JSON);
 		HttpEntity<String> entity = new HttpEntity<>(gson.toJson(map), headers);
 		try {
 			watch.start("EncryptionServiceImpl.encryptBatch");
-			ResponseEntity<String> response = retryRestTemplate.postForEntity(encryptionBatchUri,entity);
+			ResponseEntity<String> response = retryRestTemplate.postForEntity(uri, entity);
 			watch.stop("EncryptionServiceImpl.encryptBatch");
-			return gson.fromJson(response.getBody(), new TypeToken<HashMap<String, Object>>() {
+			List<T> results = gson.fromJson(response.getBody(), new TypeToken<List<T>>() {
 			}.getType());
+			assert results != null;
+			return results.get(0);
 		} catch (ResourceAccessException e) {
 			logger.error("Exception while connecting encryption service : {}", ExceptionUtils.getStackTrace(e));
 			throw new EncryptionException("Exception while connecting encryption service! ");
@@ -131,26 +127,18 @@ public Map<String, Object> encrypt(Map<String, Object> propertyValue) throws Enc
 		}
 	}
 
-	/** decrypts the input which is in Map format
-	 * @param propertyValue - input is in format Map<String, Object>
-	 * @return Map<String, Object>
-	 * @throws EncryptionException
-	 */
-	@Override
-	public Map<String, Object> decrypt(Map<String, Object> propertyValue) throws EncryptionException {
+	private  <T> T  doDecrypt(T propertyValue, String uri) throws EncryptionException {
 		logger.debug("decrypt starts with value {}", propertyValue);
-		Map<String, Object> map = new HashMap<>();
-		map.put("value", propertyValue);
 
 		HttpHeaders headers = new HttpHeaders();
 		headers.setContentType(MediaType.APPLICATION_JSON);
-		HttpEntity<String> entity = new HttpEntity<>(gson.toJson(map), headers);
+		HttpEntity<String> entity = new HttpEntity<>(gson.toJson(propertyValue), headers);
 
 		try {
 			watch.start("EncryptionServiceImpl.decryptBatch");
-			ResponseEntity<String> response = retryRestTemplate.postForEntity(decryptionBatchUri,entity);
+			ResponseEntity<String> response = retryRestTemplate.postForEntity(uri, entity);
 			watch.stop("EncryptionServiceImpl.decryptBatch");
-			return gson.fromJson(response.getBody(), new TypeToken<HashMap<String, Object>>() {
+			return gson.fromJson(response.getBody(), new TypeToken<T>() {
 			}.getType());
 		} catch (ResourceAccessException e) {
 			logger.error("Exception while connecting decryption service : {}", ExceptionUtils.getStackTrace(e));

java/registry/src/main/java/dev/sunbirdrc/registry/service/impl/RegistryServiceImpl.java

@@ -212,12 +212,12 @@ public String addEntity(Shard shard, String userId, JsonNode rootNode, boolean s
         String vertexLabel = rootNode.fieldNames().next();
         Definition definition = null;
 
-        systemFieldsHelper.ensureCreateAuditFields(vertexLabel, rootNode.get(vertexLabel), userId);
-
         if (encryptionEnabled) {
             rootNode = encryptionHelper.getEncryptedJson(rootNode);
         }
 
+        systemFieldsHelper.ensureCreateAuditFields(vertexLabel, rootNode.get(vertexLabel), userId);
+
         if (!skipSignature) {
             generateCredentials(rootNode, vertexLabel);
         }
@@ -287,10 +287,10 @@ private void generateCredentials(JsonNode rootNode, String vertexLabel) throws S
     public void updateEntity(Shard shard, String userId, String id, String jsonString, boolean skipSignature) throws Exception {
         JsonNode inputNode = objectMapper.readTree(jsonString);
         String entityType = inputNode.fields().next().getKey();
-        systemFieldsHelper.ensureUpdateAuditFields(entityType, inputNode.get(entityType), userId);
         if (encryptionEnabled) {
             inputNode = encryptionHelper.getEncryptedJson(inputNode);
         }
+        systemFieldsHelper.ensureUpdateAuditFields(entityType, inputNode.get(entityType), userId);
 
         DatabaseProvider databaseProvider = shard.getDatabaseProvider();
         IRegistryDao registryDao = new RegistryDaoImpl(databaseProvider, definitionsManager, uuidPropertyName);

java/registry/src/main/java/dev/sunbirdrc/registry/util/PrivateField.java

@@ -3,18 +3,23 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.jayway.jsonpath.DocumentContext;
+import com.jayway.jsonpath.JsonPath;
 import dev.sunbirdrc.registry.exception.EncryptionException;
 import dev.sunbirdrc.registry.middleware.util.Constants;
+import dev.sunbirdrc.registry.middleware.util.JSONUtil;
 import dev.sunbirdrc.registry.service.EncryptionService;
 import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 
+import java.io.IOException;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 public class PrivateField {
     @Autowired
@@ -31,13 +36,13 @@ public class PrivateField {
      */
     public Map<String, Object> getPrivateFields(JsonNode rootNode, List<String> privatePropertyLst) {
         Map<String, Object> plainKeyValues = new HashMap<>();
-        rootNode.fields().forEachRemaining(entry -> {
-            JsonNode entryValue = entry.getValue();
-            if (entryValue.isValueNode()) {
-                if (privatePropertyLst.contains(entry.getKey()))
-                    plainKeyValues.put(entry.getKey(), entryValue.asText());
-            }
-        });
+        if(privatePropertyLst != null) {
+            DocumentContext documentContext = JsonPath.parse(rootNode.toString());
+            privatePropertyLst.forEach(path -> {
+                Object read = documentContext.read(path);
+                plainKeyValues.put(path, read);
+            });
+        }
         return plainKeyValues;
     }
 
@@ -49,17 +54,12 @@ public Map<String, Object> getPrivateFields(JsonNode rootNode, List<String> priv
      * @param privatePropertyLst
      * @param privateFieldMap Contains the values encrypted/decrypted based on base call
      */
-    public JsonNode replacePrivateFields(JsonNode rootNode, List<String> privatePropertyLst, Map<String, Object> privateFieldMap) {
-
-        rootNode.fields().forEachRemaining(entry -> {
-            JsonNode entryValue = entry.getValue();
-
-            if (entryValue.isValueNode() && privatePropertyLst.contains(entry.getKey())) {
-                String privateFieldValue = privateFieldMap.get(entry.getKey()).toString();
-                JsonNode encryptedValNode = JsonNodeFactory.instance.textNode(privateFieldValue);
-                entry.setValue(encryptedValNode);
-            }
-        });
+    public JsonNode replacePrivateFields(JsonNode rootNode, List<String> privatePropertyLst, Map<String, Object> privateFieldMap) throws IOException {
+        if (privatePropertyLst != null) {
+            DocumentContext documentContext = JsonPath.parse(rootNode.toString());
+            privateFieldMap.forEach(documentContext::set);
+            return JSONUtil.convertStringJsonNode(documentContext.jsonString());
+        }
         return rootNode;
     }
 
@@ -70,35 +70,50 @@ protected Map<String, Object> performOperation(Map<String, Object> plainMap) thr
     protected JsonNode processPrivateFields(JsonNode element, String rootDefinitionName, String childFieldName) throws EncryptionException {
         JsonNode tempElement = element;
         Definition definition = definitionsManager.getDefinition(rootDefinitionName);;
-        if (null != childFieldName) {
+        if (null != childFieldName && definition != null) {
             String defnName = definition.getDefinitionNameForField(childFieldName);
             Definition childDefinition = definitionsManager.getDefinition(defnName);
             if (null == childDefinition) {
-                logger.error("Cannot get child name definition {}", childFieldName);
+                logger.info("Cannot get child name definition {}", childFieldName);
                 return element;
             }
             definition = childDefinition;
+        } else if (definition == null) {
+            return element;
         }
 
-        List<String> privatePropertyLst = definition.getOsSchemaConfiguration().getPrivateFields();
+        List<String> privatePropertyLst = definition.getOsSchemaConfiguration().getPrivateFields()
+                .stream().map(d -> {
+                    if (!d.startsWith("$."))
+                        return String.format("$.%s", d.replaceAll("/", "."));
+                    return d;
+                }).collect(Collectors.toList());
         Map<String, Object> plainMap = getPrivateFields(element, privatePropertyLst);
         if (null != plainMap && !plainMap.isEmpty()) {
             Map<String, Object> encodedMap = performOperation(plainMap);
-            tempElement = replacePrivateFields(element, privatePropertyLst, encodedMap);
+            try {
+                tempElement = replacePrivateFields(element, privatePropertyLst, encodedMap);
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
         }
         return tempElement;
     }
 
-    private void processArray(ArrayNode arrayNode, String rootFieldName, String fieldName) throws EncryptionException {
+    private ArrayNode processArray(ArrayNode arrayNode, String rootFieldName, String fieldName) throws EncryptionException {
+        ArrayNode updatedArrayNode = JsonNodeFactory.instance.arrayNode();
         for (JsonNode jsonNode : arrayNode) {
+            JsonNode updatedNode = jsonNode;
             if (jsonNode.isObject()) {
-                process(jsonNode, rootFieldName, fieldName);
+                updatedNode = process(jsonNode, rootFieldName, fieldName);
             }
+            updatedArrayNode.add(updatedNode);
         }
+        return updatedArrayNode;
     }
 
     protected JsonNode process(JsonNode jsonNode, String rootFieldName, String fieldName) throws EncryptionException {
-        processPrivateFields(jsonNode, rootFieldName, fieldName);
+        jsonNode = processPrivateFields(jsonNode, rootFieldName, fieldName);
 
         String tempFieldName = fieldName;
         if (null == tempFieldName) {
@@ -121,9 +136,9 @@ protected JsonNode process(JsonNode jsonNode, String rootFieldName, String field
 
                 if (isNotSignatures && entryValue.isObject()) {
                     // Recursive calls
-                    process(entryValue, tempFieldName, entry.getKey());
+                    entry.setValue(process(entryValue, tempFieldName, entry.getKey()));
                 } else if (isNotSignatures && entryValue.isArray()) {
-                    processArray((ArrayNode) entryValue, tempFieldName, entry.getKey());
+                    entry.setValue(processArray((ArrayNode) entryValue, tempFieldName, entry.getKey()));
                 }
             } catch (EncryptionException e) {
                 logger.error("Exception occurred in PrivateField: {}", ExceptionUtils.getStackTrace(e));