add specialised request for parsing material names in parallel

lfoppiano · Oct 13, 2023 · 5b4389a · 5b4389a
1 parent 7486701
commit 5b4389a
Show file tree

Hide file tree

Showing 2 changed files with 72 additions and 0 deletions.
diff --git a/src/main/java/org/grobid/core/engines/MaterialParser.java b/src/main/java/org/grobid/core/engines/MaterialParser.java
@@ -7,11 +7,13 @@
 import org.apache.commons.text.StringEscapeUtils;
 import org.grobid.core.GrobidModel;
 import org.grobid.core.analyzers.DeepAnalyzer;
+import org.grobid.core.data.document.Span;
 import org.grobid.core.data.material.ChemicalComposition;
 import org.grobid.core.data.material.Formula;
 import org.grobid.core.data.material.Material;
 import org.grobid.core.engines.label.TaggingLabel;
 import org.grobid.core.exceptions.GrobidException;
+import org.grobid.core.exceptions.GrobidExceptionStatus;
 import org.grobid.core.features.FeaturesVectorMaterial;
 import org.grobid.core.layout.BoundingBox;
 import org.grobid.core.layout.LayoutToken;
@@ -87,6 +89,53 @@ public List<Material> process(String text) {
         return process(SuperconductorsParser.textToLayoutTokens(text));
     }
 
+    public List<List<Material>> processParallel(List<String> texts) {
+        List<List<LayoutToken>> asLayoutTokens = texts.stream().map(SuperconductorsParser::textToLayoutTokens).collect(Collectors.toList());
+        return processParallelLT(asLayoutTokens);
+    }
+
+    public List<List<Material>> processParallelLT(List<List<LayoutToken>> layoutTokensBatch) {
+
+        List<List<Material>> entities = new ArrayList<>();
+
+        //Normalisation
+        List<List<LayoutToken>> normalisedTokens = layoutTokensBatch.stream()
+            .map(SuperconductorsParser::normalizeAndRetokenizeLayoutTokens)
+            .toList();
+
+        try {
+            List<String> tokensWithFeatures = normalisedTokens.stream().map(nt -> addFeatures(nt) + "\n").toList();
+
+            String labellingResult = null;
+            try {
+                labellingResult = label(tokensWithFeatures);
+            } catch (Exception e) {
+                throw new GrobidException("CRF labeling for superconductors parsing failed.", e);
+            }
+
+            List<String> resultingBlocks = Arrays.asList(labellingResult.split("\n\n"));
+            List<List<Material>> localEntities = extractParallelResults(normalisedTokens, resultingBlocks);
+
+            entities.addAll(localEntities);
+        } catch (Exception e) {
+            throw new GrobidException("An exception occurred while running Grobid.", e);
+        }
+
+        return entities;
+    }
+
+    public List<List<Material>> extractParallelResults(List<List<LayoutToken>> tokens, List<String> results) {
+        List<List<Material>> spans = new ArrayList<>();
+        if (tokens.size() != results.size()) {
+            throw new GrobidException("One of the text provided is invalid or empty and cannot be tagged. Please provide a clean input.", GrobidExceptionStatus.BAD_INPUT_DATA);
+        }
+        for (int i = 0; i < tokens.size(); i++) {
+            spans.add(extractResults(tokens.get(i), results.get(i)));
+        }
+
+        return spans;
+    }
+
 
     public List<Material> process(List<LayoutToken> tokens) {
 

diff --git a/src/main/java/org/grobid/service/controller/MaterialController.java b/src/main/java/org/grobid/service/controller/MaterialController.java
@@ -12,6 +12,7 @@
 import javax.ws.rs.Path;
 import javax.ws.rs.Produces;
 import javax.ws.rs.core.MediaType;
+import java.util.Arrays;
 import java.util.List;
 
 @Singleton
@@ -42,10 +43,32 @@ public List<Material> processTextSuperconductorsGet(@FormDataParam("text") Strin
         return parseMaterial(text);
     }
 
+
+    @Path("parse2")
+    @Produces(MediaType.APPLICATION_JSON)
+    @POST
+    public List<List<Material>> processTextSuperconductorsPost2(@FormDataParam("texts") String texts) {
+        return parseMaterials(texts);
+    }
+
+    @Path("parse2")
+    @Produces(MediaType.APPLICATION_JSON)
+    @GET
+    public List<List<Material>> processTextSuperconductorsGet2(@FormDataParam("texts") String texts) {
+        return parseMaterials(texts);
+    }
+
     private List<Material> parseMaterial(@FormDataParam("text") String text) {
         String textPreprocessed = text.replace("\r\n", "\n");
 
         return materialParser.process(textPreprocessed);
     }
 
+    private List<List<Material>> parseMaterials(@FormDataParam("text") String text) {
+        String textPreprocessed = text.replace("\r\n", "\n");
+
+        List<String> list = Arrays.asList(textPreprocessed.split("\n"));
+        return materialParser.processParallel(list);
+    }
+
 }