fix: suggester field synonyms (#93)

* fix(synonyms): add pre processing class

* test(synonyms): unit test for pre processing class

* refactor: clean up

* fix(synonyms): apply pre-processing on json questionnaire inputs

* test(synonyms): json to xml unit test

* chore: bump version

pom.xml

@@ -4,7 +4,7 @@
 	<groupId>fr.insee.pogues</groupId>
 	<artifactId>pogues-model</artifactId>
 	<packaging>jar</packaging>
-	<version>1.3.0</version>
+	<version>1.3.1</version>
 	<name>Pogues Model</name>
 	<description>Classes and converters for the Pogues model</description>
 	<url>https://inseefr.github.io/Pogues-Model/</url>

src/main/java/fr/insee/pogues/PreProcessingException.java

@@ -0,0 +1,9 @@
+package fr.insee.pogues;
+
+public class PreProcessingException extends RuntimeException {
+
+    public PreProcessingException(String message, Exception e) {
+        super(message, e);
+    }
+
+}

src/main/java/fr/insee/pogues/conversion/ConversionConstants.java

@@ -0,0 +1,8 @@
+package fr.insee.pogues.conversion;
+
+public class ConversionConstants {
+
+    public static final String JSON_CONTENT_TYPE = "application/json";
+    public static final String UTF_8_ENCODING = "UTF-8";
+    public static final String TRANSLATION_COMPLETE_MESSAGE = "Translation complete";
+}

src/main/java/fr/insee/pogues/conversion/JSONDeserializer.java

@@ -10,13 +10,17 @@
 import javax.xml.bind.JAXBException;
 import javax.xml.bind.Unmarshaller;
 import javax.xml.transform.stream.StreamSource;
+import java.io.IOException;
 import java.io.InputStream;
+import java.io.StringReader;
+import java.nio.file.Files;
+import java.nio.file.Path;
 
 public class JSONDeserializer {
 
 	private static final Logger logger = LoggerFactory.getLogger(JSONDeserializer.class);
 
-	public Questionnaire deserialize(String fileName) throws JAXBException {
+	public Questionnaire deserialize(String fileName) throws JAXBException, IOException {
 
 		if (fileName == null || fileName.isEmpty()) {
 			// TODO: throwing an exception instead of silent failing would be better here
@@ -27,8 +31,12 @@ public Questionnaire deserialize(String fileName) throws JAXBException {
 		}
 
 		logger.debug("Deserializing questionnaire from file {}", fileName);
-		StreamSource json = new StreamSource(fileName);
-		return deserializeStreamSource(json);
+
+		String fileContent = Files.readString(Path.of(fileName));
+		String preProcessedString = new JSONSynonymsPreProcessor().transform(fileContent);
+		StreamSource preProcessedStream = new StreamSource(new StringReader(preProcessedString));
+
+		return deserializeStreamSource(preProcessedStream);
 	}
 
 	/**
@@ -39,11 +47,13 @@ public Questionnaire deserialize(String fileName) throws JAXBException {
 	 */
 	public Questionnaire deserialize(InputStream jsonQuestionnaireInputStream) throws JAXBException {
 		logger.debug("Deserializing json questionnaire from input stream.");
-		StreamSource json = new StreamSource(jsonQuestionnaireInputStream);
-		return deserializeStreamSource(json);
+		String preProcessedString = new JSONSynonymsPreProcessor().transform(jsonQuestionnaireInputStream);
+		StreamSource preProcessedStream = new StreamSource(new StringReader(preProcessedString));
+		return deserializeStreamSource(preProcessedStream);
 	}
 
 	private static Questionnaire deserializeStreamSource(StreamSource json) throws JAXBException {
+
 		JAXBContext context = JAXBContext.newInstance(Questionnaire.class);
 		Unmarshaller unmarshaller = context.createUnmarshaller();
 		unmarshaller.setProperty(UnmarshallerProperties.MEDIA_TYPE, "application/json");

src/main/java/fr/insee/pogues/conversion/JSONSynonymsPreProcessor.java

@@ -0,0 +1,143 @@
+package fr.insee.pogues.conversion;
+
+import fr.insee.pogues.PreProcessingException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.json.*;
+import java.io.*;
+
+/** Pogues application generates suggester "synonyms" parameter as key/value pairs.
+ * This modeling is not adapted for XSD generation (at least without a hack).
+ * This class is intended to transform key/value pairs generated by Pogues into source/target fields that are defined
+ * in the xml schema. */
+public class JSONSynonymsPreProcessor {
+
+    private static final Logger logger  = LoggerFactory.getLogger(JSONSynonymsPreProcessor.class);
+
+    public String transform(String jsonQuestionnaireString) {
+        // Should throw an exception, yet it may have impacts in Pogues-Back-Office
+        if (jsonQuestionnaireString == null) {
+            logger.warn("null string given in JSON synonyms pre-processing method.");
+            return null;
+        }
+        return transform(new ByteArrayInputStream(jsonQuestionnaireString.getBytes()));
+    }
+
+    public String transform(InputStream jsonQuestionnaireInputStream) {
+        // Should throw an exception, yet it may have impacts in Pogues-Back-Office
+        if (jsonQuestionnaireInputStream == null) {
+            logger.warn("null input stream given in JSON synonyms pre-processing method.");
+            return null;
+        }
+
+        logger.info("Pre-processing json questionnaire stream source...");
+
+        OutputStream outputStream = new ByteArrayOutputStream();
+
+        try (JsonReader jsonReader = Json.createReader(jsonQuestionnaireInputStream);
+             JsonWriter jsonWriter = Json.createWriter(outputStream)) {
+
+            JsonObject jsonQuestionnaire = jsonReader.readObject();
+
+            // We will copy the entire input json object, except the "symLinks" attribute in PairwiseLinks components
+            JsonObjectBuilder jsonQuestionnaireBuilder = Json.createObjectBuilder();
+            editQuestionnaire(jsonQuestionnaire, jsonQuestionnaireBuilder);
+
+            jsonWriter.writeObject(jsonQuestionnaireBuilder.build());
+        }
+
+        String result = outputStream.toString();
+        try {
+            outputStream.close();
+        } catch (IOException e) {
+            throw new PreProcessingException("IO exception occurred when trying to close pre processing output.", e);
+        }
+
+        return result;
+    }
+
+
+    private static void editQuestionnaire(JsonObject jsonQuestionnaire, JsonObjectBuilder jsonQuestionnaireBuilder) {
+        jsonQuestionnaire.forEach((key, jsonValue) -> {
+            if (! "CodeLists".equals(key)) {
+                jsonQuestionnaireBuilder.add(key, jsonValue);
+            } else {
+                editCodeLists(jsonQuestionnaireBuilder, (JsonObject) jsonValue);
+            }
+        });
+    }
+
+    private static void editCodeLists(JsonObjectBuilder jsonQuestionnaireBuilder, JsonObject jsonCodeLists) {
+        JsonObjectBuilder jsonCodeListsBuilder = Json.createObjectBuilder();
+        JsonArray jsonCodeList = jsonCodeLists.getJsonArray("CodeList");
+        editCodeListArray(jsonCodeListsBuilder, jsonCodeList);
+        jsonQuestionnaireBuilder.add("CodeLists", jsonCodeListsBuilder.build());
+    }
+
+    private static void editCodeListArray(JsonObjectBuilder jsonCodeLists, JsonArray jsonCodeListArray) {
+        JsonArrayBuilder jsonCodeListArrayBuilder = Json.createArrayBuilder();
+        for (JsonValue jsonValue : jsonCodeListArray) {
+            JsonObject jsonCodeList = (JsonObject) jsonValue;
+            editCodeList(jsonCodeListArrayBuilder, jsonCodeList);
+        }
+        jsonCodeLists.add("CodeList", jsonCodeListArrayBuilder.build());
+    }
+
+    private static void editCodeList(JsonArrayBuilder jsonCodeListArrayBuilder, JsonObject jsonCodeList) {
+        JsonObjectBuilder jsonCodeListBuilder = Json.createObjectBuilder();
+        jsonCodeList.forEach((key, jsonValue) -> {
+            if (! "SuggesterParameters".equals(key)) {
+                jsonCodeListBuilder.add(key, jsonValue);
+            } else {
+                editSuggesterParameters(jsonCodeListBuilder, (JsonObject) jsonValue);
+            }
+        });
+        jsonCodeListArrayBuilder.add(jsonCodeListBuilder.build());
+    }
+
+    private static void editSuggesterParameters(JsonObjectBuilder jsonCodeListBuilder, JsonObject jsonSuggesterParameters) {
+        JsonObjectBuilder jsonSuggesterParametersBuilder = Json.createObjectBuilder();
+        jsonSuggesterParameters.forEach((key, jsonValue) -> {
+            if (! "fields".equals(key)) {
+                jsonSuggesterParametersBuilder.add(key, jsonValue);
+            } else {
+                editFieldsArray(jsonSuggesterParametersBuilder, (JsonArray) jsonValue);
+            }
+        });
+        jsonCodeListBuilder.add("SuggesterParameters", jsonSuggesterParametersBuilder.build());
+    }
+
+    private static void editFieldsArray(JsonObjectBuilder jsonSuggesterParametersBuilder, JsonArray jsonFieldsArray) {
+        JsonArrayBuilder jsonFieldsArrayBuilder = Json.createArrayBuilder();
+        for (JsonValue jsonValue : jsonFieldsArray) {
+            JsonObject jsonFields = (JsonObject) jsonValue;
+            editFields(jsonFieldsArrayBuilder, jsonFields);
+        }
+        jsonSuggesterParametersBuilder.add("fields", jsonFieldsArrayBuilder.build());
+    }
+
+    private static void editFields(JsonArrayBuilder jsonFieldsArrayBuilder, JsonObject jsonFields) {
+        JsonObjectBuilder jsonFieldsBuilder = Json.createObjectBuilder();
+        jsonFields.forEach((key, jsonValue) -> {
+            if (! "synonyms".equals(key)) {
+                jsonFieldsBuilder.add(key, jsonValue);
+            } else {
+                editSynonyms(jsonFieldsBuilder, (JsonObject) jsonValue);
+            }
+        });
+        jsonFieldsArrayBuilder.add(jsonFieldsBuilder.build());
+    }
+
+    private static void editSynonyms(JsonObjectBuilder jsonFieldsBuilder, JsonObject jsonSynonyms) {
+        JsonArrayBuilder jsonSynonymsArrayBuilder = Json.createArrayBuilder();
+        jsonSynonyms.forEach((name, jsonValue) -> {
+            JsonObjectBuilder jsonSynonymBuilder = Json.createObjectBuilder();
+            jsonSynonymBuilder.add("source", name);
+            jsonSynonymBuilder.add("target", jsonValue);
+            jsonSynonymsArrayBuilder.add(jsonSynonymBuilder.build());
+        });
+        jsonFieldsBuilder.add("synonyms", jsonSynonymsArrayBuilder.build());
+    }
+
+}