Skip to content

Commit

Permalink
Fix configuration env substitution
Browse files Browse the repository at this point in the history 
When a single subject and predicate pair have multiple
values, all but the last one were lost.
  • Loading branch information
@skodapetr
skodapetr committed Sep 19, 2024
1 parent 9a5477a commit eb266fc
Show file tree
Hide file tree
Showing 2 changed files with 124 additions and 55 deletions.
122 changes: 67 additions & 55 deletions ...main/java/com/linkedpipes/etl/executor/component/configuration/SubstituteEnvironment.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,27 +11,20 @@
import org.eclipse.rdf4j.model.ValueFactory;
import org.eclipse.rdf4j.model.impl.SimpleValueFactory;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;

class SubstituteEnvironment {

static class StatementPair {
/**
* Represents a value for substitution.
* We can have more than one value or substitution value.
*/
static class Substitution {

public Resource resource;
public List<Value> values = new ArrayList<>();

public IRI predicate;

public Value value;

public Value substitute;

public StatementPair(IRI predicate) {
this.predicate = predicate;
}
public List<Value> substitutions = new ArrayList<>();

}

Expand All @@ -45,59 +38,72 @@ public static EntityReference substitute(
List<Statement> statements = collectStatements(
referenceSource, reference.getGraph());

// Create map of predicates.
Map<IRI, StatementPair> predicateMap =
collectPredicates(referenceSource, configurationType)
.stream().collect(Collectors.toMap(
iri -> iri,
StatementPair::new));

// Create map for substitution.
Map<IRI, StatementPair> substitutionMap =
predicateMap.values().stream().collect(Collectors.toMap(
item -> valueFactory.createIRI(
item.predicate.stringValue() + "Substitution"),
item -> item));

// Collect information.
// Create map of predicates and substitute predicates for given resource.
// We use convention that the substitution predicate
// is the predicate + "Substitution";
// The idea is to share the same internal, so we can
// access the same data using predicate as well as
// substitution predicate.
Map<IRI, Map<Resource, Substitution>> predicateMap = new HashMap<>();
Map<IRI, Map<Resource, Substitution>> substitutionMap = new HashMap<>();
for (var iri : collectPredicates(referenceSource, configurationType)) {
Map<Resource, Substitution> value = new HashMap<>();
predicateMap.put(iri, value);
var substitutionIri = valueFactory.createIRI(iri + "Substitution");
substitutionMap.put(substitutionIri, value);
}

// Next we iterate the statements searching for values or
// predicates and substitute predicates.
// We need to pair them together for substitution.
// Other statements we just pass along.
List<Statement> nextStatements = new ArrayList<>();
for (Statement statement : statements) {
Resource subject = statement.getSubject();
IRI predicate = statement.getPredicate();

if (predicateMap.containsKey(predicate)) {
StatementPair pair = predicateMap.get(predicate);
if (pair.resource != null
&& pair.resource != statement.getSubject()) {
throw new RdfUtilsException("Not supported!");
}
pair.resource = statement.getSubject();
pair.value = statement.getObject();
Objects.requireNonNull(predicateMap.get(predicate))
.computeIfAbsent(subject, key -> new Substitution())
.values.add(statement.getObject());
} else if (substitutionMap.containsKey(predicate)) {
StatementPair pair = substitutionMap.get(predicate);
if (pair.resource != null
&& pair.resource != statement.getSubject()) {
throw new RdfUtilsException("Not supported!");
}
pair.resource = statement.getSubject();
pair.substitute = statement.getObject();
Objects.requireNonNull(substitutionMap.get(predicate))
.computeIfAbsent(subject, key -> new Substitution())
.values.add(statement.getObject());
} else {
nextStatements.add(statement);
}
}

// Generate back statements.
for (StatementPair pair : predicateMap.values()) {
Value value = pair.value;
if (pair.substitute != null) {
value = valueFactory.createLiteral(
substitute(env, pair.substitute.stringValue()));
}
if (pair.resource == null || value == null) {
continue;
// Next we need to assemble back statements in
// predicateMap or substitute them by the substitutions.
for (var predicateEntry : predicateMap.entrySet()) {
IRI predicate = predicateEntry.getKey();
for (var resourceEntry : predicateEntry.getValue().entrySet()) {
Resource subject = resourceEntry.getKey();
var value = resourceEntry.getValue();
if (value.substitutions.isEmpty()) {
// We keep the original values.
for (var object : value.values) {
nextStatements.add(valueFactory.createStatement(
subject, predicate, object));
}
} else {
// We use the values from substitution.
for (var object : value.substitutions) {
// We support substitution only for strings.
var nextObject = valueFactory.createLiteral(
substitute(env, object.stringValue()));
nextStatements.add(valueFactory.createStatement(
subject, predicate, nextObject));
}
}

}
nextStatements.add(valueFactory.createStatement(
pair.resource, pair.predicate, value));
}

// At the last step we just store the statements in a new store
// using a new graph.
IRI nextGraph = valueFactory.createIRI(
reference.getGraph() + "/substituted");
Rdf4jSource nextSource = Rdf4jSource.createInMemory();
Expand All @@ -107,6 +113,9 @@ public static EntityReference substitute(
reference.getResource(), nextGraph.stringValue(), nextSource);
}

/**
* @return All controlled predicates for given configuration type.
*/
private static Set<IRI> collectPredicates(
Rdf4jSource referenceSource, String configurationType)
throws RdfUtilsException {
Expand All @@ -119,6 +128,9 @@ private static Set<IRI> collectPredicates(
.collect(Collectors.toSet());
}

/**
* @return All statements in given graph.
*/
private static List<Statement> collectStatements
(Rdf4jSource source, String graph) {
List<Statement> result = new ArrayList<>();
Expand Down
57 changes: 57 additions & 0 deletions .../java/com/linkedpipes/etl/executor/component/configuration/SubstituteEnvironmentTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,21 @@
package com.linkedpipes.etl.executor.component.configuration;

import com.linkedpipes.etl.executor.ExecutorException;
import com.linkedpipes.etl.executor.api.v1.vocabulary.LP_OBJECTS;
import com.linkedpipes.etl.executor.rdf.entity.EntityReference;
import com.linkedpipes.etl.rdf.rdf4j.Rdf4jSource;
import com.linkedpipes.etl.rdf.utils.RdfBuilder;
import com.linkedpipes.etl.rdf.utils.RdfUtilsException;
import com.linkedpipes.etl.rdf.utils.model.RdfTriple;
import com.linkedpipes.etl.rdf.utils.vocabulary.RDF;
import org.eclipse.rdf4j.model.Statement;
import org.eclipse.rdf4j.model.impl.SimpleValueFactory;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class SubstituteEnvironmentTest {
Expand All @@ -21,4 +32,50 @@ public void simpleSubstitution() throws ExecutorException {
env, "x-{LP_ETL_HOST}:{LP_ETL_PORT}"));
}

@Test
public void preserveStatements() throws ExecutorException, RdfUtilsException {
Map<String, String> env = new HashMap<>();
env.put("LP_ETL_HOST", "lp");
env.put("LP_ETL_PORT", "8080");

final var configurationClass = "http://localhost/Configuration";
final var property = "http://localhost/1";
final var graph = "http://localhost/graph";

var source = Rdf4jSource.createInMemory();
var builder = RdfBuilder.create(source, "http://localhost/graph");

// Configuration entity.
builder.entity("http://localhost/entity")
.iri(RDF.TYPE, LP_OBJECTS.DESCRIPTION)
.iri(LP_OBJECTS.HAS_DESCRIBE, configurationClass)
.iri(LP_OBJECTS.HAS_MEMBER, property);
builder.entity(property)
.iri(LP_OBJECTS.HAS_PROPERTY, "http://localhost/predicate")
.iri(LP_OBJECTS.HAS_CONTROL, "http://localhost/predicateControl");

// Data entity.
builder.entity("http://localhost/entity")
.iri(RDF.TYPE, configurationClass)
.iri("http://localhost/predicate", "http://localhost/1")
.iri("http://localhost/predicate", "http://localhost/2");

builder.commit();

var actual = SubstituteEnvironment.substitute(
env,
source,
new EntityReference("http://localhost/entity",graph,null),
configurationClass);

List<Statement> sourceList = new ArrayList<>();
source.statements(null, graph, sourceList::add);

List<RdfTriple> targetList = new ArrayList<>();
actual.getSource().triples(null, targetList::add);

// We should get the same number of triples.
Assertions.assertEquals(sourceList.size(), targetList.size());
}

}

0 comments on commit eb266fc

Please sign in to comment.