Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added Array, Set and operator 'IN' #110

Closed
wants to merge 18 commits into from
Closed
Show file tree
Hide file tree
Changes from 10 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion thunx-bom/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ dependencies {
api project(':thunx-encoding-json')
api project(':thunx-predicates-querydsl')

api 'com.contentgrid.opa-java-client:opa-async-java-client:0.4.0'
api 'com.contentgrid.opa-java-client:opa-async-java-client:0.4.1-SNAPSHOT'
}
}

Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package com.contentgrid.thunx.predicates.model;

import lombok.EqualsAndHashCode;

import java.util.Collection;

@EqualsAndHashCode
public class CollectionValue implements Scalar<Collection<Scalar<?>>> {

private final Collection<Scalar<?>> value;

@Override
public Collection<Scalar<?>> getValue() {
return value;
}

protected CollectionValue(Collection<Scalar<?>> value) {
this.value = value;
}

@Override
public Class<? extends Collection<Scalar<?>>> getResultType() {
return null;
}

@Override
public <R, C> R accept(ThunkExpressionVisitor<R, C> visitor, C context) {
return visitor.visit(this, context);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,18 @@ public static Comparison lessOrEquals(ThunkExpression<?> left, ThunkExpression<?
return new Comparison(Operator.LESS_THEN_OR_EQUAL_TO, left, right);
}

public static Comparison in (@NonNull List<ThunkExpression<?>> terms) {
assertTermSizeIsTwo(terms);
return in(terms.get(0), terms.get(1));
}

public static Comparison in (ThunkExpression<?> left, ThunkExpression<?> right) {
if (right == null || ((CollectionValue) right).getValue().isEmpty()) {
return null;
}
return new Comparison(Operator.IN, left, right);
}

private static void assertTermSizeIsTwo(List<ThunkExpression<?>> terms) {
if (terms.size() == 2) {
return;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ enum Operator {
GREATER_THAN_OR_EQUAL_TO("gte", Boolean.class, (FunctionExpressionFactory<Boolean>) Comparison::greaterOrEquals),
LESS_THAN("lt", Boolean.class, (FunctionExpressionFactory<Boolean>) Comparison::less),
LESS_THEN_OR_EQUAL_TO("lte", Boolean.class, (FunctionExpressionFactory<Boolean>) Comparison::lessOrEquals),
IN("internal.member_2", Boolean.class, (FunctionExpressionFactory<Boolean>) Comparison::in),
ystrict marked this conversation as resolved.
Show resolved Hide resolved

// Logical operator
AND("and", Boolean.class, (FunctionExpressionFactory<Boolean>) LogicalOperation::uncheckedConjunction),
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.contentgrid.thunx.predicates.model;

import java.math.BigDecimal;
import java.util.Collection;

public interface Scalar<T> extends ThunkExpression<T> {

Expand Down Expand Up @@ -30,6 +31,10 @@ static BooleanValue of(boolean value) {
return new BooleanValue(value);
}

static CollectionValue of(Collection<Scalar<?>> value) {
ystrict marked this conversation as resolved.
Show resolved Hide resolved
return new CollectionValue(value);
}

static NullValue nullValue() {
return NullValue.INSTANCE;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,8 @@ public interface ThunkExpressionVisitor<T, C> {
T visit(FunctionExpression<?> functionExpression, C context);
T visit(SymbolicReference symbolicReference, C context);
T visit(Variable variable, C context);
default T visit(CollectionValue collection, C context) {
throw new UnsupportedOperationException("Visit for CollectionValue is not yet implemented.");
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

During development/experimenting this is fine, but we should remove the default implementation.

At some, all visitors need to implement this method, so we can be sure that we can handle collection values.

One of the important ones that is still missing an implementation is the ExpressionJsonConverter.JsonEncoderVisitor, without which we can't send the partial expression to the application.

}

}
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.contentgrid.thunx.pdp.opa;


import com.contentgrid.opa.rego.ast.Expression;
import com.contentgrid.opa.rego.ast.Query;
import com.contentgrid.opa.rego.ast.QuerySet;
import com.contentgrid.opa.rego.ast.RegoVisitor;
Expand All @@ -11,6 +12,7 @@
import com.contentgrid.opa.rego.ast.Term.Null;
import com.contentgrid.opa.rego.ast.Term.Numeric;
import com.contentgrid.opa.rego.ast.Term.Ref;
import com.contentgrid.opa.rego.ast.Term.SetTerm;
import com.contentgrid.opa.rego.ast.Term.Text;
import com.contentgrid.opa.rego.ast.Term.Var;
import com.contentgrid.thunx.predicates.model.Comparison;
Expand All @@ -21,8 +23,12 @@
import com.contentgrid.thunx.predicates.model.SymbolicReference.StringPathElement;
import com.contentgrid.thunx.predicates.model.ThunkExpression;
import com.contentgrid.thunx.predicates.model.Variable;
import lombok.Getter;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.Function;
import java.util.stream.Collectors;

Expand Down Expand Up @@ -57,6 +63,7 @@ public ThunkExpression<Boolean> convert(Query query) {

var expressions = query.stream()
.map(this::convert)
.filter(Objects::nonNull)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are there ever null values returned from this::convert?

I think that would indicate a bug, so filtering them out might be counterproductive, and actually result in an expression that can't be converted being dropped, thereby widening the access policies, which is an undesired effect.

We should prefer erroring out (even if it's a null pointer exception) over accidentally allowing access to data that should have been protected.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This filtering is done because there is an ability to return empty Term for in operation.
Example:
"index": 4, "terms": [ { "type": "ref", "value": [ { "type": "var", "value": "internal" }, { "type": "string", "value": "member_2" } ] }, { "type": "ref", "value": [ { "type": "var", "value": "input" }, { "type": "string", "value": "entity" }, { "type": "string", "value": "accountId" } ] }, { "type": "array", "value": [] # will return null in conversion, cause we need to skip empty in clause } ] }

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's why in this place we are filtering empty in cases

.peek(expr -> {
if (!Boolean.class.isAssignableFrom(expr.getResultType())) {
// there are non-boolean expressions in here ?
Expand Down Expand Up @@ -92,7 +99,8 @@ static class PredicatesVisitor implements RegoVisitor<ThunkExpression<?>> {
Map.entry("gt", Comparison::greater),
Map.entry("gte", Comparison::greaterOrEquals),
Map.entry("lt", Comparison::less),
Map.entry("lte", Comparison::lessOrEquals)
Map.entry("lte", Comparison::lessOrEquals),
Map.entry("internal.member_2", Comparison::in)
vierbergenlars marked this conversation as resolved.
Show resolved Hide resolved
);

@Override
Expand Down Expand Up @@ -279,6 +287,89 @@ public ThunkExpression<?> visit(Null nullValue) {

@Override
public ThunkExpression<?> visit(ArrayTerm arrayTerm) {
ScalarCollectingRegoVisitor collectingRegoVisitor = new ScalarCollectingRegoVisitor();
arrayTerm.getValue().forEach(scalarTerm -> scalarTerm.accept(collectingRegoVisitor));
return Scalar.of(collectingRegoVisitor.getScalars());
}

@Override
public ThunkExpression<?> visit(SetTerm setTerm) {
ScalarCollectingRegoVisitor collectingRegoVisitor = new ScalarCollectingRegoVisitor();
setTerm.getValue().forEach(scalarTerm -> scalarTerm.accept(collectingRegoVisitor));
return Scalar.of(collectingRegoVisitor.getScalars());
}
}

static class ScalarCollectingRegoVisitor implements RegoVisitor<ThunkExpression<?>> {

@Getter
List<Scalar<?>> scalars = new ArrayList<>();
ystrict marked this conversation as resolved.
Show resolved Hide resolved

@Override
public ThunkExpression<?> visit(QuerySet querySet) {
return null;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

All these rego types are unexpected in a scalar collection, so they should throw an exception instead of silently dropping the term.

For example, in rego, you can have a Ref in an array, for example, when constructing input.entity.something in [input.entity.optionA, input.entity.optionB]

That is probably (I didn't check) a valid rego expression, but is not supported as entry in the current CollectionValue.

It's fine to limit ourselves to only scalar values for now, but we should not drop those terms silently.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've added support of collection in collection for opa-client (xenit-eu/opa-java-client#28) and for thunx

}

@Override
public ThunkExpression<?> visit(Query query) {
return null;
}

@Override
public ThunkExpression<?> visit(Expression expression) {
return null;
}

@Override
public ThunkExpression<?> visit(Ref ref) {
return null;
}

@Override
public ThunkExpression<?> visit(Call call) {
return null;
}

@Override
public ThunkExpression<?> visit(Var var) {
return null;
}

@Override
public ThunkExpression<?> visit(Numeric numeric) {
Scalar<?> scalar = Scalar.of(numeric.getValue());
scalars.add(scalar);
return scalar;
}

@Override
public ThunkExpression<?> visit(Text text) {
Scalar<?> scalar = Scalar.of(text.getValue());
scalars.add(scalar);
return scalar;
}

@Override
public ThunkExpression<?> visit(Bool bool) {
return null;
ystrict marked this conversation as resolved.
Show resolved Hide resolved
}

@Override
public ThunkExpression<?> visit(Null aNull) {
Scalar<?> scalar = Scalar.nullValue();
scalars.add(scalar);
return scalar;
}

@Override
public ThunkExpression<?> visit(ArrayTerm arrayTerm) {
arrayTerm.getValue().forEach(scalarTerm -> scalarTerm.accept(this));
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Doing this would unpack a rego array ['a', 'b', ['c', 'd']] into ['a', 'b', 'c', 'd'], which is not a valid conversion.

Instead, this would probably need to use the same code as used above to collect scalars, and add the whole resulting CollectionValue as a scalar value.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Now SetTerm and ArrayTerm extends Term, that means that SetTerm cannot have SetTerm as a value. If you want to have such case when SetTerm contains SetTerm, then we need to change opa-client, to make SetTerm and ArrayTerm extend ScalarTerm

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've added support of collection in collection for opa-client (xenit-eu/opa-java-client#28) and for thunx

return null;
}

@Override
public ThunkExpression<?> visit(SetTerm setTerm) {
setTerm.getValue().forEach(scalarTerm -> scalarTerm.accept(this));
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same remark as for the array case, nested sets shouldn't be flattened.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fixed

return null;
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
import com.contentgrid.thunx.predicates.model.Scalar;
import com.contentgrid.thunx.predicates.model.SymbolicReference;
import java.util.List;
import java.util.Set;

import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;

Expand Down Expand Up @@ -135,6 +137,25 @@ void less_or_equals() {
));
}

@Test
void in() {
// input.entity.security in {4, 5}
var opaExpr = new Expression(0, List.of(
new Term.Ref(List.of(new Term.Var("internal"), new Term.Text("member_2"))),
new Term.Ref(List.of(
new Term.Var("input"),
new Term.Text("entity"),
new Term.Text("security")
)),
new Term.SetTerm(Set.of(new Term.Numeric(4), new Term.Numeric(5)))
));

assertThat(converter.convert(opaExpr)).isEqualTo(Comparison.in(
SymbolicReference.of("entity", path -> path.string("security")),
Scalar.of(Set.of(new Term.Numeric(4), new Term.Numeric(5)))
));
}


}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package com.contentgrid.thunx.predicates.querydsl;

import com.contentgrid.opa.rego.ast.Term;
ystrict marked this conversation as resolved.
Show resolved Hide resolved
import com.contentgrid.thunx.predicates.model.CollectionValue;
import com.contentgrid.thunx.predicates.model.FunctionExpression;
import com.contentgrid.thunx.predicates.model.Scalar;
import com.contentgrid.thunx.predicates.model.SymbolicReference;
Expand All @@ -13,18 +15,19 @@
import com.querydsl.core.types.Predicate;
import com.querydsl.core.types.dsl.Expressions;
import com.querydsl.core.types.dsl.PathBuilder;
import java.lang.reflect.Field;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.AccessLevel;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;

import javax.persistence.Embedded;
import javax.persistence.ManyToMany;
import javax.persistence.ManyToOne;
import javax.persistence.OneToMany;
import javax.persistence.OneToOne;
import lombok.AccessLevel;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import java.lang.reflect.Field;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;

@RequiredArgsConstructor(access = AccessLevel.PACKAGE)
class QueryDslConvertingVisitor implements ThunkExpressionVisitor<Expression<?>, QueryDslConversionContext> {
Expand Down Expand Up @@ -70,6 +73,9 @@ public Expression<?> visit(FunctionExpression<?> function, QueryDslConversionCon
case LESS_THAN:
assertTwoTerms(terms);
return Expressions.booleanOperation(Ops.LT, terms.toArray(new Expression[0]));
case IN:
assertTwoTerms(terms);
return ExpressionUtils.predicate(Ops.IN, terms.get(0), terms.get(1));
case OR:
return ExpressionUtils.anyOf(terms.stream().map(term -> (Predicate) term).collect(Collectors.toList()));
case AND:
Expand Down Expand Up @@ -121,21 +127,21 @@ public Expression<?> visit(SymbolicReference symbolicReference, QueryDslConversi
* annotations
*/
private static void assertNotReferencingEntity(@NonNull SymbolicReference symbolicReference,
@NonNull PathBuilder<?> builder) {
@NonNull PathBuilder<?> builder) {

var blacklist = Set.of(OneToOne.class, OneToMany.class, ManyToOne.class, ManyToMany.class, Embedded.class);
var element = builder.getAnnotatedElement();
if (blacklist.stream().anyMatch(element::isAnnotationPresent)) {
var msg = String.format("Cannot use `%s` as an expression, because it refers to a relation, not an attribute.",
symbolicReference,
element instanceof Field ? ((Field)element).getType().getName() + " " + ((Field)element).getName() : element);
element instanceof Field ? ((Field) element).getType().getName() + " " + ((Field) element).getName() : element);

throw new IllegalArgumentException(msg);
}
}

private PathBuilder<?> traversePath(SymbolicReference symbolicReference, PathBuilder<?> builder,
String pathElement) {
String pathElement) {

// we want to build a typed path, making sure the segments in the path are valid
var property = this.accessStrategy.getProperty(builder.getType(), pathElement).orElseThrow(() -> {
Expand Down Expand Up @@ -173,4 +179,9 @@ public Expression<?> visit(Variable variable, QueryDslConversionContext context)
// TODO could there be more variables available, than just the subject-path-builder ?
throw new UnsupportedOperationException("converting variable to querydsl is not yet implemented");
}

@Override
public Expression<?> visit(CollectionValue collection, QueryDslConversionContext context) {
return Expressions.constant(collection.getValue().stream().map(Scalar::getValue).collect(Collectors.toList()));
}
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.contentgrid.thunx.visitor.reducer;

import com.contentgrid.thunx.predicates.model.CollectionValue;
import com.contentgrid.thunx.predicates.model.FunctionExpression;
import com.contentgrid.thunx.predicates.model.FunctionExpression.Operator;
import com.contentgrid.thunx.predicates.model.LogicalOperation;
Expand Down Expand Up @@ -54,4 +55,9 @@ public ThunkExpression<?> visit(SymbolicReference symbolicReference) {
public ThunkExpression<?> visit(Variable variable) {
return variable;
}

@Override
public ThunkExpression<?> visit(CollectionValue collection, Void context) {
return collection;
}
}