From 58ff21395482bf458478be6faeeb8406ebea16a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=B8iseth-Gilje=2C=20Eivind?= Date: Thu, 9 Feb 2017 23:16:25 +0100 Subject: [PATCH 1/3] Boolean expression with equal to for join --- .../src/main/antlr4/no/ssb/vtl/parser/VTL.g4 | 13 ++- .../visitors/BooleanExpressionVisitor.java | 80 +++++++++++++++++++ .../visitors/join/JoinExpressionVisitor.java | 48 +++++------ .../join/JoinFilterClauseVisitor.java | 20 ++--- .../ssb/vtl/script/VTLScriptEngineTest.java | 4 +- .../join/JoinFilterClauseVisitorTest.java | 71 ++++++++++++++++ 6 files changed, 188 insertions(+), 48 deletions(-) create mode 100644 java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/BooleanExpressionVisitor.java create mode 100644 java-vtl-script/src/test/java/no/ssb/vtl/script/visitors/join/JoinFilterClauseVisitorTest.java diff --git a/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 b/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 index 714f69ed..f0c023b8 100644 --- a/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 +++ b/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 @@ -93,16 +93,13 @@ aggregate : 'aggregate' ; //WS : [ \t\n\t] -> skip ; booleanExpression - : booleanExpression AND booleanExpression - | booleanExpression ( OR booleanExpression | XOR booleanExpression ) - | booleanEquallity + : booleanExpression op=AND booleanExpression + | booleanExpression op=(OR|XOR) booleanExpression + | booleanEquality | BOOLEAN_CONSTANT ; -booleanEquallity - : booleanEquallity ( ( EQ | NE | LE | GE ) booleanEquallity ) - | datasetExpression - | constant - // typed constant? +booleanEquality + : componentRef op=( EQ | NE | LE | GE ) constant ; //datasetExpression diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/BooleanExpressionVisitor.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/BooleanExpressionVisitor.java new file mode 100644 index 00000000..4d7ba4d2 --- /dev/null +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/BooleanExpressionVisitor.java @@ -0,0 +1,80 @@ +package no.ssb.vtl.script.visitors; + +import no.ssb.vtl.model.Component; +import no.ssb.vtl.model.DataPoint; +import no.ssb.vtl.model.Dataset; +import no.ssb.vtl.parser.VTLBaseVisitor; +import no.ssb.vtl.parser.VTLParser; +import org.antlr.v4.runtime.misc.ParseCancellationException; + +import java.util.function.Predicate; + +public class BooleanExpressionVisitor extends VTLBaseVisitor> { + + private final ReferenceVisitor referenceVisitor; + + public BooleanExpressionVisitor(ReferenceVisitor referenceVisitor) { + this.referenceVisitor = referenceVisitor; + } + + @Override + public Predicate visitBooleanExpression(VTLParser.BooleanExpressionContext ctx) { + if (ctx.BOOLEAN_CONSTANT() != null) { + Boolean booleanConstant = Boolean.valueOf(ctx.BOOLEAN_CONSTANT().getText()); + return dataPoints -> booleanConstant; + }else if (ctx.op != null) { + Predicate left = visit(ctx.booleanExpression(0)); + Predicate right = visit(ctx.booleanExpression(1)); + switch (ctx.op.getType()) { + case VTLParser.AND: + return left.and(right); + case VTLParser.OR: + return left.or(right); + case VTLParser.XOR: + return left.or(right).and(left.and(right).negate()); + default: + throw new ParseCancellationException("Unsupported boolean operation: " + ctx.op.getText()); + } + } else if (ctx.booleanEquality() != null) { + return visit(ctx.booleanEquality()); + } else { + return super.visit(ctx); + } + } + + @Override + public Predicate visitBooleanEquality(VTLParser.BooleanEqualityContext ctx) { + Component component = (Component) referenceVisitor.visit(ctx.componentRef()); + Object scalar = getScalar(ctx.constant()); + + Predicate predicate; + + switch (ctx.op.getType()) { + case VTLParser.EQ: + predicate = dataPoint -> dataPoint.get().equals(scalar); + break; + default: + throw new ParseCancellationException("Unsupported boolean equality operator"); + } + + return tuple -> tuple.stream() + .filter(dataPoint -> component.equals(dataPoint.getComponent())) + .anyMatch(predicate); + } + + private Object getScalar(VTLParser.ConstantContext ctx) { + String constant = ctx.getText(); + if (ctx.BOOLEAN_CONSTANT() != null) { + return Boolean.valueOf(constant); + } else if (ctx.FLOAT_CONSTANT() != null) { + return Float.valueOf(constant); + } else if (ctx.INTEGER_CONSTANT() != null) { + return Integer.valueOf(constant); + } else if (ctx.NULL_CONSTANT() != null) { + return null; + } else { //String + return constant.replace("\"", ""); + } + } + +} diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinExpressionVisitor.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinExpressionVisitor.java index 3c82200e..9800a3dc 100644 --- a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinExpressionVisitor.java +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinExpressionVisitor.java @@ -72,6 +72,30 @@ public Dataset visitJoinExpression(VTLParser.JoinExpressionContext ctx) { // // return joinClause.apply(workingDataset); } + + @Override + protected Dataset aggregateResult(Dataset aggregate, Dataset nextResult) { + // Compute the new scope. + Dataset currentDataset = firstNonNull(nextResult, aggregate); + + Set previous = Optional.ofNullable(aggregate) + .map(Dataset::getDataStructure) + .map(ForwardingMap::keySet) + .orElse(Collections.emptySet()); + Set current = currentDataset.getDataStructure().keySet(); + + Set referencesToRemove = Sets.difference(previous, current); + Set referencesToAdd = Sets.difference(current, previous); + + for (String key : referencesToRemove) { + joinScope.remove(key); + } + for (String key : referencesToAdd) { + joinScope.put(key, currentDataset.getDataStructure().get(key)); + } + + return workingDataset = currentDataset; + } @Override public Dataset visitJoinCalcClause(VTLParser.JoinCalcClauseContext ctx) { @@ -115,30 +139,6 @@ public String toString() { }; } - @Override - protected Dataset aggregateResult(Dataset aggregate, Dataset nextResult) { - // Compute the new scope. - Dataset currentDataset = firstNonNull(nextResult, aggregate); - - Set previous = Optional.ofNullable(aggregate) - .map(Dataset::getDataStructure) - .map(ForwardingMap::keySet) - .orElse(Collections.emptySet()); - Set current = currentDataset.getDataStructure().keySet(); - - Set referencesToRemove = Sets.difference(previous, current); - Set referencesToAdd = Sets.difference(current, previous); - - for (String key : referencesToRemove) { - joinScope.remove(key); - } - for (String key : referencesToAdd) { - joinScope.put(key, currentDataset.getDataStructure().get(key)); - } - - return workingDataset = currentDataset; - } - @Override public Dataset visitJoinFoldClause(VTLParser.JoinFoldClauseContext ctx) { JoinFoldClauseVisitor visitor = new JoinFoldClauseVisitor(workingDataset, referenceVisitor); diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinFilterClauseVisitor.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinFilterClauseVisitor.java index 52f3c6da..50316714 100644 --- a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinFilterClauseVisitor.java +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinFilterClauseVisitor.java @@ -4,35 +4,27 @@ import no.ssb.vtl.parser.VTLBaseVisitor; import no.ssb.vtl.parser.VTLParser; import no.ssb.vtl.script.operations.FilterOperator; +import no.ssb.vtl.script.visitors.BooleanExpressionVisitor; import no.ssb.vtl.script.visitors.ReferenceVisitor; -import java.util.Set; import java.util.function.Predicate; -import java.util.stream.Collectors; -import java.util.stream.Stream; import static com.google.common.base.Preconditions.*; public class JoinFilterClauseVisitor extends VTLBaseVisitor { private final Dataset dataset; + private final ReferenceVisitor referenceVisitor; - JoinFilterClauseVisitor(Dataset dataset) { - this.dataset = checkNotNull(dataset, "dataset was null"); - } - public JoinFilterClauseVisitor(Dataset dataset, ReferenceVisitor referenceVisitor) { this.dataset = checkNotNull(dataset, "dataset was null"); + this.referenceVisitor = referenceVisitor; } @Override public FilterOperator visitJoinFilterClause(VTLParser.JoinFilterClauseContext ctx) { - Set components = Stream.of("id1").collect(Collectors.toSet()); - Predicate predicate = tuple -> tuple.ids().stream() - .filter(dataPoint -> components.contains(dataPoint.getComponent().getName())) - .anyMatch(dataPoint -> dataPoint.get().equals("1")); //TODO do not hardcode filter criteria - FilterOperator filterOperator = new FilterOperator(dataset, predicate); - System.out.println("Created new filterOperator: " + filterOperator); - return filterOperator; //TODO: Do not hardcode component to filter on + BooleanExpressionVisitor booleanExpressionVisitor = new BooleanExpressionVisitor(referenceVisitor); + Predicate predicate = booleanExpressionVisitor.visit(ctx.joinFilterExpression().booleanExpression()); + return new FilterOperator(dataset, predicate); } } diff --git a/java-vtl-script/src/test/java/no/ssb/vtl/script/VTLScriptEngineTest.java b/java-vtl-script/src/test/java/no/ssb/vtl/script/VTLScriptEngineTest.java index ac7ca09e..f4d17459 100644 --- a/java-vtl-script/src/test/java/no/ssb/vtl/script/VTLScriptEngineTest.java +++ b/java-vtl-script/src/test/java/no/ssb/vtl/script/VTLScriptEngineTest.java @@ -151,7 +151,7 @@ public void testJoin() throws Exception { engine.eval("" + "ds3 := [ds1, ds2]{" + // id1, id2, ds1.m1, ds1.m2, d2.m1, d2.m2, at1, at2 - " filter id1 = 1," + // id1, id2, ds1.m1, ds1.m2, d2.m1, d2.m2, at1, at2 + " filter id1 = \"1\" and m1 = 30 or m1 = 10," + //TODO: precedence " ident = ds1.m1 + ds2.m2 - ds1.m2 - ds2.m1," + // id1, id2, ds1.m1, ds1.m2, d2.m1, d2.m2, at1, at2, ident " keep ident, ds1.m1, ds2.m1, ds2.m2," + // id1, id2, ds1.m1, ds2.m1, ds2.m2, ident " drop ds2.m1," + // id1, id2, ds1.m1, ds2.m2, ident @@ -172,7 +172,7 @@ public void testJoin() throws Exception { assertThat(ds3.getDataStructure().values()) .haveAtLeastOne(componentWith("renamedId1", Role.IDENTIFIER)) .haveAtLeastOne(componentWith("id2", Role.IDENTIFIER)) - .haveAtLeastOne(componentWith("ds2.m2", Role.MEASURE)) + .haveAtLeastOne(componentWith("m2", Role.MEASURE)) .haveAtLeastOne(componentWith("m1", Role.MEASURE)) .haveAtLeastOne(componentWith("ident", Role.MEASURE)); diff --git a/java-vtl-script/src/test/java/no/ssb/vtl/script/visitors/join/JoinFilterClauseVisitorTest.java b/java-vtl-script/src/test/java/no/ssb/vtl/script/visitors/join/JoinFilterClauseVisitorTest.java new file mode 100644 index 00000000..e5168209 --- /dev/null +++ b/java-vtl-script/src/test/java/no/ssb/vtl/script/visitors/join/JoinFilterClauseVisitorTest.java @@ -0,0 +1,71 @@ +package no.ssb.vtl.script.visitors.join; + +import com.google.common.collect.ImmutableMap; +import no.ssb.vtl.connector.Connector; +import no.ssb.vtl.model.Component; +import no.ssb.vtl.model.DataPoint; +import no.ssb.vtl.model.DataStructure; +import no.ssb.vtl.model.Dataset; +import no.ssb.vtl.script.VTLScriptEngine; +import org.junit.Test; + +import javax.script.Bindings; +import javax.script.ScriptContext; +import javax.script.ScriptEngine; + +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.*; +import static org.mockito.Mockito.*; + +public class JoinFilterClauseVisitorTest { + + private Dataset dataset = mock(Dataset.class); + private Connector connector = mock(Connector.class); + private ScriptEngine engine = new VTLScriptEngine(connector); + private Bindings bindings = engine.getBindings(ScriptContext.ENGINE_SCOPE); + + @Test + public void testSimpleBooleanFilter() throws Exception { + + Dataset ds1 = mock(Dataset.class); + DataStructure structure1 = DataStructure.of( + (o, aClass) -> o, + "id1", Component.Role.IDENTIFIER, String.class, + "m1", Component.Role.MEASURE, Integer.class + ); + when(ds1.getDataStructure()).thenReturn(structure1); + + when(ds1.get()).then(invocation -> Stream.of( + structure1.wrap(ImmutableMap.of( + "id1", "1", + "m1", 10 + )), + structure1.wrap(ImmutableMap.of( + "id1", "2", + "m1", 100 + )) + )); + + bindings.put("ds1", ds1); + + + engine.eval("" + + "ds3 := [ds1]{" + + " filter id1 = \"1\" and m1 = 10" + + "}" + + ""); + + + assertThat(bindings).containsKey("ds3"); + assertThat(bindings.get("ds3")).isInstanceOf(Dataset.class); + Dataset ds3 = (Dataset) bindings.get("ds3"); + + assertThat(ds3.get()) + .flatExtracting(input -> input) + .extracting(DataPoint::get) + .containsExactly( + "1", 10 + ); + } +} \ No newline at end of file From b2a9a0bf2978155891dc0c7d0894a3cec68d6583 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=B8iseth-Gilje=2C=20Eivind?= Date: Fri, 10 Feb 2017 14:17:07 +0100 Subject: [PATCH 2/3] Impement boolean comparison operators --- .../src/main/antlr4/no/ssb/vtl/parser/VTL.g4 | 4 ++- .../visitors/BooleanExpressionVisitor.java | 33 +++++++++++++++++++ .../join/JoinFilterClauseVisitorTest.java | 2 +- 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 b/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 index f0c023b8..2c1fd657 100644 --- a/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 +++ b/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 @@ -99,7 +99,7 @@ booleanExpression | BOOLEAN_CONSTANT ; booleanEquality - : componentRef op=( EQ | NE | LE | GE ) constant + : componentRef op=( EQ | NE | LE | LT | GE | GT ) constant ; //datasetExpression @@ -109,7 +109,9 @@ booleanEquality EQ : '=' ; NE : '<>' ; LE : '<=' ; +LT : '<' ; GE : '>=' ; +GT : '>' ; AND : 'and' ; OR : 'or' ; diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/BooleanExpressionVisitor.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/BooleanExpressionVisitor.java index 4d7ba4d2..a3bd2c93 100644 --- a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/BooleanExpressionVisitor.java +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/BooleanExpressionVisitor.java @@ -9,6 +9,8 @@ import java.util.function.Predicate; +import static java.lang.String.*; + public class BooleanExpressionVisitor extends VTLBaseVisitor> { private final ReferenceVisitor referenceVisitor; @@ -53,6 +55,21 @@ public Predicate visitBooleanEquality(VTLParser.BooleanEqualityCo case VTLParser.EQ: predicate = dataPoint -> dataPoint.get().equals(scalar); break; + case VTLParser.NE: + predicate = dataPoint -> !dataPoint.get().equals(scalar); + break; + case VTLParser.LE: + predicate = dataPoint -> compare(dataPoint.get(), scalar) <= 0; + break; + case VTLParser.LT: + predicate = dataPoint -> compare(dataPoint.get(), scalar) < 0; + break; + case VTLParser.GE: + predicate = dataPoint -> compare(dataPoint.get(), scalar) >= 0; + break; + case VTLParser.GT: + predicate = dataPoint -> compare(dataPoint.get(), scalar) > 0; + break; default: throw new ParseCancellationException("Unsupported boolean equality operator"); } @@ -62,6 +79,22 @@ public Predicate visitBooleanEquality(VTLParser.BooleanEqualityCo .anyMatch(predicate); } + private int compare(Object value, Object scalar) { + if (value instanceof Integer && scalar instanceof Integer) { + return ((Integer) value).compareTo((Integer) scalar); + } else if (value instanceof Float && scalar instanceof Float) { + return ((Float) value).compareTo((Float) scalar); + } else if (value instanceof Boolean && scalar instanceof Boolean) { + return ((Boolean) value).compareTo((Boolean) scalar); + } else if (value instanceof String && scalar instanceof String) { + return ((String) value).compareTo((String) scalar); + } + throw new ParseCancellationException( + format("Cannot compare %s of type %s with %s of type %s", + value, value.getClass(), scalar, scalar.getClass()) + ); + } + private Object getScalar(VTLParser.ConstantContext ctx) { String constant = ctx.getText(); if (ctx.BOOLEAN_CONSTANT() != null) { diff --git a/java-vtl-script/src/test/java/no/ssb/vtl/script/visitors/join/JoinFilterClauseVisitorTest.java b/java-vtl-script/src/test/java/no/ssb/vtl/script/visitors/join/JoinFilterClauseVisitorTest.java index e5168209..a92360c3 100644 --- a/java-vtl-script/src/test/java/no/ssb/vtl/script/visitors/join/JoinFilterClauseVisitorTest.java +++ b/java-vtl-script/src/test/java/no/ssb/vtl/script/visitors/join/JoinFilterClauseVisitorTest.java @@ -52,7 +52,7 @@ public void testSimpleBooleanFilter() throws Exception { engine.eval("" + "ds3 := [ds1]{" + - " filter id1 = \"1\" and m1 = 10" + + " filter id1 = \"1\" and m1 > 9" + "}" + ""); From 5407e4d27523f79cdfaa6d8515d3d15a66507b0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=B8iseth-Gilje=2C=20Eivind?= Date: Fri, 10 Feb 2017 19:40:15 +0100 Subject: [PATCH 3/3] Generalize boolean comparision operation to handle a mix of component and scalar operands --- .../src/main/antlr4/no/ssb/vtl/parser/VTL.g4 | 6 +- .../visitors/BooleanExpressionVisitor.java | 81 +++++++++++-------- .../ssb/vtl/script/visitors/ParamVisitor.java | 35 ++++++++ .../join/JoinFilterClauseVisitorTest.java | 65 ++++++++++++++- 4 files changed, 147 insertions(+), 40 deletions(-) create mode 100644 java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/ParamVisitor.java diff --git a/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 b/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 index 2c1fd657..f7e17fb4 100644 --- a/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 +++ b/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 @@ -99,7 +99,11 @@ booleanExpression | BOOLEAN_CONSTANT ; booleanEquality - : componentRef op=( EQ | NE | LE | LT | GE | GT ) constant + : left=booleanParam op=( EQ | NE | LE | LT | GE | GT ) right=booleanParam + ; +booleanParam + : componentRef + | constant ; //datasetExpression diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/BooleanExpressionVisitor.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/BooleanExpressionVisitor.java index a3bd2c93..f9ab759f 100644 --- a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/BooleanExpressionVisitor.java +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/BooleanExpressionVisitor.java @@ -1,13 +1,18 @@ package no.ssb.vtl.script.visitors; +import com.google.common.collect.Iterables; import no.ssb.vtl.model.Component; import no.ssb.vtl.model.DataPoint; import no.ssb.vtl.model.Dataset; import no.ssb.vtl.parser.VTLBaseVisitor; import no.ssb.vtl.parser.VTLParser; +import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.misc.ParseCancellationException; +import java.lang.String; +import java.util.function.BiPredicate; import java.util.function.Predicate; +import java.util.stream.Collectors; import static java.lang.String.*; @@ -46,37 +51,58 @@ public Predicate visitBooleanExpression(VTLParser.BooleanExpressi @Override public Predicate visitBooleanEquality(VTLParser.BooleanEqualityContext ctx) { - Component component = (Component) referenceVisitor.visit(ctx.componentRef()); - Object scalar = getScalar(ctx.constant()); + ParamVisitor paramVisitor = new ParamVisitor(referenceVisitor); + Object left = paramVisitor.visit(ctx.left); + Object right = paramVisitor.visit(ctx.right); - Predicate predicate; + BiPredicate booleanOperation = getBooleanOperation(ctx.op); - switch (ctx.op.getType()) { + if (isComp(left) && !isComp(right)) { + return tuple -> tuple.stream() + .filter(dataPoint -> left.equals(dataPoint.getComponent())) + .anyMatch(dataPoint -> booleanOperation.test(dataPoint.get(), right)); + } else if (!isComp(left) && isComp(right)){ + return tuple -> tuple.stream() + .filter(dataPoint -> right.equals(dataPoint.getComponent())) + .anyMatch(dataPoint -> booleanOperation.test(left, dataPoint.get())); + } else if (isComp(left) && isComp(right)) { + return tuple -> { + DataPoint rightDataPoint = getOnlyElement(right, tuple); + DataPoint leftDataPoint = getOnlyElement(left, tuple); + return booleanOperation.test(leftDataPoint.get(), rightDataPoint.get()); + }; + } else { + return tuple -> booleanOperation.test(left, right); + } + } + + private BiPredicate getBooleanOperation(Token op) { + switch (op.getType()) { case VTLParser.EQ: - predicate = dataPoint -> dataPoint.get().equals(scalar); - break; + return Object::equals; case VTLParser.NE: - predicate = dataPoint -> !dataPoint.get().equals(scalar); - break; + return (l, r) -> !l.equals(r); case VTLParser.LE: - predicate = dataPoint -> compare(dataPoint.get(), scalar) <= 0; - break; + return (l, r) -> compare(l, r) <= 0; case VTLParser.LT: - predicate = dataPoint -> compare(dataPoint.get(), scalar) < 0; - break; + return (l, r) -> compare(l, r) < 0; case VTLParser.GE: - predicate = dataPoint -> compare(dataPoint.get(), scalar) >= 0; - break; + return (l, r) -> compare(l, r) >= 0; case VTLParser.GT: - predicate = dataPoint -> compare(dataPoint.get(), scalar) > 0; - break; + return (l, r) -> compare(l, r) > 0; default: - throw new ParseCancellationException("Unsupported boolean equality operator"); + throw new ParseCancellationException("Unsupported boolean equality operator " + op); } - - return tuple -> tuple.stream() + } + + private DataPoint getOnlyElement(Object component, Dataset.Tuple tuple) { + return Iterables.getOnlyElement(tuple.stream() .filter(dataPoint -> component.equals(dataPoint.getComponent())) - .anyMatch(predicate); + .collect(Collectors.toList())); + } + + private boolean isComp(Object o) { + return o instanceof Component; } private int compare(Object value, Object scalar) { @@ -95,19 +121,4 @@ private int compare(Object value, Object scalar) { ); } - private Object getScalar(VTLParser.ConstantContext ctx) { - String constant = ctx.getText(); - if (ctx.BOOLEAN_CONSTANT() != null) { - return Boolean.valueOf(constant); - } else if (ctx.FLOAT_CONSTANT() != null) { - return Float.valueOf(constant); - } else if (ctx.INTEGER_CONSTANT() != null) { - return Integer.valueOf(constant); - } else if (ctx.NULL_CONSTANT() != null) { - return null; - } else { //String - return constant.replace("\"", ""); - } - } - } diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/ParamVisitor.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/ParamVisitor.java new file mode 100644 index 00000000..aaea7e73 --- /dev/null +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/ParamVisitor.java @@ -0,0 +1,35 @@ +package no.ssb.vtl.script.visitors; + +import no.ssb.vtl.parser.VTLBaseVisitor; +import no.ssb.vtl.parser.VTLParser; + +public class ParamVisitor extends VTLBaseVisitor { + + private final ReferenceVisitor referenceVisitor; + + public ParamVisitor(ReferenceVisitor referenceVisitor) { + this.referenceVisitor = referenceVisitor; + } + + @Override + public Object visitComponentRef(VTLParser.ComponentRefContext ctx) { + return referenceVisitor.visit(ctx); + } + + @Override + public Object visitConstant(VTLParser.ConstantContext ctx) { + String constant = ctx.getText(); + if (ctx.BOOLEAN_CONSTANT() != null) { + return Boolean.valueOf(constant); + } else if (ctx.FLOAT_CONSTANT() != null) { + return Float.valueOf(constant); + } else if (ctx.INTEGER_CONSTANT() != null) { + return Integer.valueOf(constant); + } else if (ctx.NULL_CONSTANT() != null) { + return null; + } else { //String + return constant.replace("\"", ""); + } + } + +} diff --git a/java-vtl-script/src/test/java/no/ssb/vtl/script/visitors/join/JoinFilterClauseVisitorTest.java b/java-vtl-script/src/test/java/no/ssb/vtl/script/visitors/join/JoinFilterClauseVisitorTest.java index a92360c3..ed2b9833 100644 --- a/java-vtl-script/src/test/java/no/ssb/vtl/script/visitors/join/JoinFilterClauseVisitorTest.java +++ b/java-vtl-script/src/test/java/no/ssb/vtl/script/visitors/join/JoinFilterClauseVisitorTest.java @@ -7,6 +7,7 @@ import no.ssb.vtl.model.DataStructure; import no.ssb.vtl.model.Dataset; import no.ssb.vtl.script.VTLScriptEngine; +import org.junit.Before; import org.junit.Test; import javax.script.Bindings; @@ -24,11 +25,12 @@ public class JoinFilterClauseVisitorTest { private Connector connector = mock(Connector.class); private ScriptEngine engine = new VTLScriptEngine(connector); private Bindings bindings = engine.getBindings(ScriptContext.ENGINE_SCOPE); + private Dataset ds1; + private Dataset ds2; - @Test - public void testSimpleBooleanFilter() throws Exception { - - Dataset ds1 = mock(Dataset.class); + @Before + public void setUp() throws Exception { + ds1 = mock(Dataset.class); DataStructure structure1 = DataStructure.of( (o, aClass) -> o, "id1", Component.Role.IDENTIFIER, String.class, @@ -46,7 +48,36 @@ public void testSimpleBooleanFilter() throws Exception { "m1", 100 )) )); + + + ds2 = mock(Dataset.class); + DataStructure structure2 = DataStructure.of( + (o, aClass) -> o, + "id1", Component.Role.IDENTIFIER, String.class, + "m1", Component.Role.MEASURE, Integer.class, + "m2", Component.Role.MEASURE, Integer.class, + "a1", Component.Role.ATTRIBUTE, String.class + ); + when(ds2.getDataStructure()).thenReturn(structure2); + when(ds2.get()).then(invocation -> Stream.of( + structure2.wrap(ImmutableMap.of( + "id1", "1", + "m1", 10, + "m2", 10, + "a1", "test" + )), + structure2.wrap(ImmutableMap.of( + "id1", "2", + "m1", 100, + "m2", 10, + "a1", "2" + )) + )); + } + + @Test + public void testSimpleBooleanFilter() throws Exception { bindings.put("ds1", ds1); @@ -68,4 +99,30 @@ public void testSimpleBooleanFilter() throws Exception { "1", 10 ); } + + + @Test + public void testBooleanComponents() throws Exception { + bindings.put("ds2", ds2); + + + engine.eval("" + + "ds3 := [ds2]{" + + " filter id1 = a1 or m1 > m2" + + "}" + + ""); + + + assertThat(bindings).containsKey("ds3"); + assertThat(bindings.get("ds3")).isInstanceOf(Dataset.class); + Dataset ds3 = (Dataset) bindings.get("ds3"); + + assertThat(ds3.get()) + .flatExtracting(input -> input) + .extracting(DataPoint::get) + .containsExactly( + "2", 100, 10, "2" + ); + + } } \ No newline at end of file