Moved the filtering of markings for the D2KB and Entity Typing class …

…into an EvaluationDecorator to handle the problem not only for experiment tasks but for experiment sub tasks in the same way.

src/main/java/org/aksw/gerbil/evaluate/EvaluatorFactory.java

@@ -32,13 +32,15 @@
 import org.aksw.gerbil.evaluate.impl.SpanMergingEvaluatorDecorator;
 import org.aksw.gerbil.evaluate.impl.SubTaskAverageCalculator;
 import org.aksw.gerbil.evaluate.impl.filter.MarkingFilteringEvaluatorDecorator;
+import org.aksw.gerbil.evaluate.impl.filter.SearcherBasedNotMatchingMarkingFilter;
 import org.aksw.gerbil.matching.Matching;
 import org.aksw.gerbil.matching.MatchingsSearcher;
 import org.aksw.gerbil.matching.MatchingsSearcherFactory;
 import org.aksw.gerbil.matching.impl.CompoundMatchingsCounter;
 import org.aksw.gerbil.matching.impl.HierarchicalMatchingsCounter;
 import org.aksw.gerbil.matching.impl.MatchingsCounterImpl;
 import org.aksw.gerbil.matching.impl.MeaningMatchingsSearcher;
+import org.aksw.gerbil.matching.impl.StrongSpanMatchingsSearcher;
 import org.aksw.gerbil.semantic.kb.ExactWhiteListBasedUriKBClassifier;
 import org.aksw.gerbil.semantic.kb.SimpleWhiteListBasedUriKBClassifier;
 import org.aksw.gerbil.semantic.kb.UriKBClassifier;
@@ -142,20 +144,25 @@ protected Evaluator createEvaluator(ExperimentType type, ExperimentTaskConfigura
                     FMeasureCalculator.MICRO_F1_SCORE_NAME, new DoubleResultComparator());
         }
         case D2KB: {
-            return new ConfidenceScoreEvaluatorDecorator<NamedEntity>(
-                    new InKBClassBasedFMeasureCalculator<NamedEntity>(new CompoundMatchingsCounter<NamedEntity>(
-                            (MatchingsSearcher<NamedEntity>) MatchingsSearcherFactory
-                                    .createSpanMatchingsSearcher(configuration.matching),
-                            new MeaningMatchingsSearcher<NamedEntity>(globalClassifier)), globalClassifier),
-                    FMeasureCalculator.MICRO_F1_SCORE_NAME, new DoubleResultComparator());
+            return new SearcherBasedNotMatchingMarkingFilter<NamedEntity>(
+                    new StrongSpanMatchingsSearcher<NamedEntity>(),
+                    new ConfidenceScoreEvaluatorDecorator<NamedEntity>(
+                            new InKBClassBasedFMeasureCalculator<NamedEntity>(
+                                    new CompoundMatchingsCounter<NamedEntity>(
+                                            (MatchingsSearcher<NamedEntity>) MatchingsSearcherFactory
+                                                    .createSpanMatchingsSearcher(configuration.matching),
+                                            new MeaningMatchingsSearcher<NamedEntity>(globalClassifier)),
+                                    globalClassifier),
+                            FMeasureCalculator.MICRO_F1_SCORE_NAME, new DoubleResultComparator()));
         }
         case ETyping: {
-            return new ConfidenceScoreEvaluatorDecorator<TypedSpan>(
-                    new HierarchicalFMeasureCalculator<TypedSpan>(new HierarchicalMatchingsCounter<TypedSpan>(
-                            (MatchingsSearcher<TypedSpan>) MatchingsSearcherFactory
-                                    .createSpanMatchingsSearcher(configuration.matching),
-                            globalClassifier, inferencer)),
-                    FMeasureCalculator.MICRO_F1_SCORE_NAME, new DoubleResultComparator());
+            return new SearcherBasedNotMatchingMarkingFilter<TypedSpan>(new StrongSpanMatchingsSearcher<TypedSpan>(),
+                    new ConfidenceScoreEvaluatorDecorator<TypedSpan>(
+                            new HierarchicalFMeasureCalculator<TypedSpan>(new HierarchicalMatchingsCounter<TypedSpan>(
+                                    (MatchingsSearcher<TypedSpan>) MatchingsSearcherFactory
+                                            .createSpanMatchingsSearcher(configuration.matching),
+                                    globalClassifier, inferencer)),
+                            FMeasureCalculator.MICRO_F1_SCORE_NAME, new DoubleResultComparator()));
         }
         case OKE_Task1: {
             ExperimentTaskConfiguration subTaskConfig;

src/main/java/org/aksw/gerbil/evaluate/impl/filter/SearcherBasedNotMatchingMarkingFilter.java

@@ -0,0 +1,59 @@
+package org.aksw.gerbil.evaluate.impl.filter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.aksw.gerbil.evaluate.AbstractEvaluatorDecorator;
+import org.aksw.gerbil.evaluate.EvaluationResultContainer;
+import org.aksw.gerbil.evaluate.Evaluator;
+import org.aksw.gerbil.matching.MatchingsSearcher;
+import org.aksw.gerbil.transfer.nif.Marking;
+
+import com.carrotsearch.hppc.BitSet;
+
+/**
+ * This evaluator decorator removes every marking from the given list that does
+ * not match the given gold standard list based on a given
+ * {@link MatchingsSearcher} instance.
+ * 
+ * @author Michael Röder ([email protected])
+ *
+ */
+public class SearcherBasedNotMatchingMarkingFilter<T extends Marking> extends AbstractEvaluatorDecorator<T> {
+
+    protected MatchingsSearcher<T> searcher;
+
+    public SearcherBasedNotMatchingMarkingFilter(MatchingsSearcher<T> searcher, Evaluator<T> evaluator) {
+        super(evaluator);
+        this.searcher = searcher;
+    }
+
+    protected List<List<T>> filterListOfMarkings(List<List<T>> markings, List<List<T>> goldStandard) {
+        List<List<T>> filteredMarkings = new ArrayList<List<T>>(markings.size());
+        for (int i = 0; i < markings.size(); ++i) {
+            filteredMarkings.add(filterMarkings(markings.get(i), goldStandard.get(i)));
+        }
+        return filteredMarkings;
+    }
+
+    protected List<T> filterMarkings(List<T> markings, List<T> goldStandard) {
+        BitSet matchingElements;
+        BitSet alreadyUsedResults = new BitSet(goldStandard.size());
+        List<T> filteredMarkings = new ArrayList<T>(markings.size());
+        for (T marking : markings) {
+            matchingElements = searcher.findMatchings(marking, goldStandard, alreadyUsedResults);
+            if (!matchingElements.isEmpty()) {
+                filteredMarkings.add(marking);
+                alreadyUsedResults.set(matchingElements.nextSetBit(0));
+            }
+        }
+        return filteredMarkings;
+    }
+
+    @Override
+    public void evaluate(List<List<T>> annotatorResults, List<List<T>> goldStandard,
+            EvaluationResultContainer results) {
+        evaluator.evaluate(filterListOfMarkings(annotatorResults, goldStandard), goldStandard, results);
+    }
+
+}

src/main/java/org/aksw/gerbil/execute/ExperimentTask.java

@@ -45,8 +45,6 @@
 import org.aksw.gerbil.evaluate.SubTaskResult;
 import org.aksw.gerbil.evaluate.impl.FMeasureCalculator;
 import org.aksw.gerbil.exceptions.GerbilException;
-import org.aksw.gerbil.matching.filter.SearcherBasedNotMatchingMarkingFilter;
-import org.aksw.gerbil.matching.impl.StrongSpanMatchingsSearcher;
 import org.aksw.gerbil.semantic.sameas.DatasetBasedSameAsRetriever;
 import org.aksw.gerbil.semantic.sameas.MultipleSameAsRetriever;
 import org.aksw.gerbil.semantic.sameas.SameAsRetriever;
@@ -307,19 +305,11 @@ protected EvaluationResult runExperiment(Dataset dataset, Annotator annotator,
                 List<List<MeaningSpan>> results = new ArrayList<List<MeaningSpan>>(dataset.size());
                 List<List<MeaningSpan>> goldStandard = new ArrayList<List<MeaningSpan>>(dataset.size());
                 D2KBAnnotator linker = ((D2KBAnnotator) annotator);
-                // For D2KB we have to filter the results to get those results
-                // that are matching the positions
-                SearcherBasedNotMatchingMarkingFilter<MeaningSpan> filter = new SearcherBasedNotMatchingMarkingFilter<MeaningSpan>(
-                        new StrongSpanMatchingsSearcher<MeaningSpan>());
-                List<MeaningSpan> documentGS;
 
                 for (Document document : dataset.getInstances()) {
-                    documentGS = document.getMarkings(MeaningSpan.class);
                     // reduce the document to a text and a list of Spans
-                    results.add(filter.filterMarkings(
-                            linker.performD2KBTask(DocumentInformationReducer.reduceToTextAndSpans(document)),
-                            documentGS));
-                    goldStandard.add(documentGS);
+                    results.add(linker.performD2KBTask(DocumentInformationReducer.reduceToTextAndSpans(document)));
+                    goldStandard.add(document.getMarkings(MeaningSpan.class));
                     taskState.increaseExperimentStepCount();
                 }
                 if (annotatorOutputWriter != null) {
@@ -413,20 +403,11 @@ protected EvaluationResult runExperiment(Dataset dataset, Annotator annotator,
                 List<List<TypedSpan>> results = new ArrayList<List<TypedSpan>>(dataset.size());
                 List<List<TypedSpan>> goldStandard = new ArrayList<List<TypedSpan>>(dataset.size());
                 EntityTyper typer = ((EntityTyper) annotator);
-                // For ETyping we have to filter the results to get those
-                // results
-                // that are matching the positions
-                SearcherBasedNotMatchingMarkingFilter<TypedSpan> filter = new SearcherBasedNotMatchingMarkingFilter<TypedSpan>(
-                        new StrongSpanMatchingsSearcher<TypedSpan>());
-                List<TypedSpan> documentGS;
 
                 for (Document document : dataset.getInstances()) {
-                    documentGS = document.getMarkings(TypedSpan.class);
                     // reduce the document to a text and a list of Spans
-                    results.add(filter.filterMarkings(
-                            typer.performTyping(DocumentInformationReducer.reduceToTextAndSpans(document)),
-                            documentGS));
-                    goldStandard.add(documentGS);
+                    results.add(typer.performTyping(DocumentInformationReducer.reduceToTextAndSpans(document)));
+                    goldStandard.add(document.getMarkings(TypedSpan.class));
                     taskState.increaseExperimentStepCount();
                 }
                 if (annotatorOutputWriter != null) {

src/main/java/org/aksw/gerbil/utils/filter/MarkingFilter.java

@@ -22,8 +22,20 @@
 
 public interface MarkingFilter<T extends Marking> {
 
+    /**
+     * Returns true if the marking is good and does not have to be filtered out.
+     * 
+     * @param marking
+     * @return
+     */
     public boolean isMarkingGood(T marking);
 
+    /**
+     * Returns a filtered list based on the given list.
+     * 
+     * @param markings
+     * @return
+     */
     public List<T> filterList(List<T> markings);
 
     public List<List<T>> filterListOfLists(List<List<T>> markings);

src/test/java/org/aksw/gerbil/SingleRunTest.java

@@ -43,10 +43,10 @@ public class SingleRunTest implements TaskObserver {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(SingleRunTest.class);
 
-    private static final String ANNOTATOR_NAME = "FOX";
-    private static final String DATASET_NAME = "N3-Reuters-128";
+    private static final String ANNOTATOR_NAME = "TagMe 2";
+    private static final String DATASET_NAME = "ACE2004";
     private static final ExperimentType EXPERIMENT_TYPE = ExperimentType.D2KB;
-    private static final Matching MATCHING = Matching.WEAK_ANNOTATION_MATCH;
+    private static final Matching MATCHING = Matching.STRONG_ENTITY_MATCH;
 
     public static void main(String[] args) throws Exception {
         SingleRunTest test = new SingleRunTest();