From 4d52a425e445dce583dcc658b6801bd3b0694626 Mon Sep 17 00:00:00 2001 From: Luthien-in-edhil Date: Mon, 4 Jan 2016 18:26:04 +0100 Subject: [PATCH] #777 Unordered children in hierarchical display: replaced slow cypher queries with fast path traversal --- neo4j-startup-plugin/pom.xml | 37 +++- .../eu/europeana/neo4j/fetch/Children.java | 169 +++++++++++++----- .../neo4j/fetch/FollowingSiblings.java | 74 ++++---- .../neo4j/fetch/PrecedingSiblings.java | 79 ++++---- .../eu/europeana/neo4j/initial/Startup.java | 5 +- 5 files changed, 244 insertions(+), 120 deletions(-) diff --git a/neo4j-startup-plugin/pom.xml b/neo4j-startup-plugin/pom.xml index cbfef89..0332637 100644 --- a/neo4j-startup-plugin/pom.xml +++ b/neo4j-startup-plugin/pom.xml @@ -12,6 +12,7 @@ ${project.build.directory}/endorsed UTF-8 + 2.1.8 @@ -29,13 +30,45 @@ org.neo4j server-api - 2.0.3 + ${neo4j.version} org.neo4j neo4j-cypher - 2.1.2 + ${neo4j.version} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + org.codehaus.jackson jackson-jaxrs diff --git a/neo4j-startup-plugin/src/main/java/eu/europeana/neo4j/fetch/Children.java b/neo4j-startup-plugin/src/main/java/eu/europeana/neo4j/fetch/Children.java index f85d882..ffe98bc 100644 --- a/neo4j-startup-plugin/src/main/java/eu/europeana/neo4j/fetch/Children.java +++ b/neo4j-startup-plugin/src/main/java/eu/europeana/neo4j/fetch/Children.java @@ -6,26 +6,22 @@ package eu.europeana.neo4j.fetch; import eu.europeana.neo4j.mapper.ObjectMapper; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; -import javax.ws.rs.DefaultValue; -import javax.ws.rs.GET; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; +import org.neo4j.graphdb.*; +import org.neo4j.graphdb.index.Index; +import org.neo4j.graphdb.index.IndexHits; +import org.neo4j.graphdb.index.IndexManager; +import org.neo4j.graphdb.traversal.Evaluators; +import org.neo4j.graphdb.traversal.TraversalDescription; +import org.neo4j.graphdb.traversal.Uniqueness; +import org.neo4j.tooling.GlobalGraphOperations; + +import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import org.neo4j.cypher.javacompat.ExecutionEngine; -import org.neo4j.cypher.javacompat.ExecutionResult; -import org.neo4j.graphdb.GraphDatabaseService; -import org.neo4j.graphdb.Node; -import org.neo4j.graphdb.Transaction; -import org.neo4j.kernel.impl.util.StringLogger; +import java.io.IOException; +import java.util.*; /** * @@ -34,48 +30,131 @@ @javax.ws.rs.Path("/children") public class Children { - + private static final RelationshipType HAS_PART = DynamicRelationshipType.withName("dcterms:hasPart"); + private static final RelationshipType IS_FAKE = DynamicRelationshipType.withName("isFakeOrder"); + private static final RelationshipType IS_NEXT = DynamicRelationshipType.withName("edm:isNextInSequence"); - private GraphDatabaseService db; - private ExecutionEngine engine; +// private GraphDatabaseService db; +// private ExecutionEngine engine; + +// public Children(@Context GraphDatabaseService db) { +// this.db = db; +// this.engine = new ExecutionEngine(db, StringLogger.SYSTEM); +// } - public Children(@Context GraphDatabaseService db) { - this.db = db; - this.engine = new ExecutionEngine(db, StringLogger.SYSTEM); - } @GET @javax.ws.rs.Path("/nodeId/{nodeId}") @Produces(MediaType.APPLICATION_JSON) - - public Response getchildren(@PathParam("nodeId") String nodeId, @QueryParam("offset") @DefaultValue("0") int offset, - @QueryParam("limit") @DefaultValue("10") int limit) { + public Response getChildren(@PathParam("nodeId") String nodeId, + @QueryParam("offset") @DefaultValue("0") int offset, + @QueryParam("limit") @DefaultValue("10") int limit, + @Context GraphDatabaseService db) throws IOException { List children = new ArrayList<>(); - Transaction tx = db.beginTx(); - try { - ExecutionResult result = engine.execute( - "start parent = node:edmsearch2(rdf_about=\"" + nodeId + "\") " - + "MATCH (parent)-[:`dcterms:hasPart`]->(child) " - + "WHERE NOT ()-[:isFakeOrder]->(child) " - + "AND NOT ()-[:`edm:isNextInSequence`]->(child) " - + "WITH child AS first " - + "MATCH (first)-[:isFakeOrder|`edm:isNextInSequence`*]->(next) " - + "WITH DISTINCT first + COLLECT(next) AS spool " - + "UNWIND spool as children RETURN children " - + "SKIP " + offset + " LIMIT " + limit); - Iterator childIterator = result.columnAs("children"); - while (childIterator.hasNext()) { - children.add(childIterator.next()); + try ( Transaction tx = db.beginTx() ) { + IndexManager index = db.index(); + Index edmsearch2 = index.forNodes("edmsearch2"); + IndexHits hits = edmsearch2.get("rdf_about", nodeId); + Node parent = hits.getSingle(); + if (parent==null) { + throw new IllegalArgumentException("no node found in index for rdf_about = " + nodeId); + } + Node first = null; + + // Get all children + for (Relationship r1 : parent.getRelationships(Direction.OUTGOING, HAS_PART)) { + Node child = r1.getEndNode(); + if ((child.getDegree(IS_FAKE, Direction.INCOMING) == 0) && + (child.getDegree(IS_NEXT, Direction.INCOMING) == 0)) { + first = child; + } } - } catch (Exception e) { - Logger.getLogger(this.getClass().getCanonicalName()).log(Level.SEVERE, e.getMessage()); - } finally { + if (first==null) { + throw new IllegalArgumentException("no first child for node " + parent); + } + + // Go up to limit hops away + TraversalDescription td = db.traversalDescription() + .depthFirst() + .relationships(IS_FAKE, Direction.OUTGOING) + .relationships(IS_NEXT, Direction.OUTGOING) + .uniqueness(Uniqueness.RELATIONSHIP_GLOBAL) + .evaluator(Evaluators.toDepth(limit - 1)); + + // Add to the results + for (org.neo4j.graphdb.Path path : td.traverse(first)) { + Node child = path.endNode(); + children.add(child); + } String obj = new ObjectMapper().siblingsToJson(children, "siblings"); tx.success(); - tx.finish(); return Response.ok().entity(obj).header(HttpHeaders.CONTENT_TYPE, "application/json").build(); } } + + @GET + @javax.ws.rs.Path("degree") + public String getDegreeHistogram(@Context GraphDatabaseService gds) throws IOException { + SortedMap histogram = new TreeMap<>(); + try (Transaction tx = gds.beginTx()) { + for (Node n: GlobalGraphOperations.at(gds).getAllNodes()) { + int degree = n.getDegree(); + + Integer val = histogram.get(degree); + if (val==null) { + histogram.put(degree, 1); + } else { + histogram.put(degree, val+1); + } + + } + tx.success(); + } + StringBuilder sb = new StringBuilder(); + for (Map.Entry entry: histogram.entrySet()) { + sb.append(entry.getKey()).append(",").append(entry.getValue()).append("\n"); + } + return sb.toString(); + } + + + + +// @GET +// @javax.ws.rs.Path("/nodeId/{nodeId}") +// @Produces(MediaType.APPLICATION_JSON) +// +// public Response getchildren(@PathParam("nodeId") String nodeId, +// @QueryParam("offset") @DefaultValue("0") int offset, +// @QueryParam("limit") @DefaultValue("10") int limit) { +// List children = new ArrayList<>(); +// Transaction tx = db.beginTx(); +// try { +// ExecutionResult result = engine.execute( +// "start parent = node:edmsearch2(rdf_about=\"" + nodeId + "\") " +// + "MATCH (parent)-[:`dcterms:hasPart`]->(child) " +// + "WHERE NOT ()-[:isFakeOrder]->(child) " +// + "AND NOT ()-[:`edm:isNextInSequence`]->(child) " +// + "WITH child AS first " +// + "MATCH (first)-[:isFakeOrder|`edm:isNextInSequence`*]->(next) " +// + "WITH DISTINCT first + COLLECT(next) AS spool " +// + "UNWIND spool as children RETURN children " +// + "SKIP " + offset + " LIMIT " + limit); +// Iterator childIterator = result.columnAs("children"); +// while (childIterator.hasNext()) { +// children.add(childIterator.next()); +// } +// } catch (Exception e) { +// Logger.getLogger(this.getClass().getCanonicalName()).log(Level.SEVERE, e.getMessage()); +// } finally { +// +// String obj = new ObjectMapper().siblingsToJson(children, "siblings"); +// tx.success(); +// tx.finish(); +// return Response.ok().entity(obj).header(HttpHeaders.CONTENT_TYPE, +// "application/json").build(); +// } +// } } \ No newline at end of file diff --git a/neo4j-startup-plugin/src/main/java/eu/europeana/neo4j/fetch/FollowingSiblings.java b/neo4j-startup-plugin/src/main/java/eu/europeana/neo4j/fetch/FollowingSiblings.java index 17176c1..9343a31 100644 --- a/neo4j-startup-plugin/src/main/java/eu/europeana/neo4j/fetch/FollowingSiblings.java +++ b/neo4j-startup-plugin/src/main/java/eu/europeana/neo4j/fetch/FollowingSiblings.java @@ -6,26 +6,21 @@ package eu.europeana.neo4j.fetch; import eu.europeana.neo4j.mapper.ObjectMapper; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; -import javax.ws.rs.DefaultValue; -import javax.ws.rs.GET; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; +import org.neo4j.graphdb.*; +import org.neo4j.graphdb.index.Index; +import org.neo4j.graphdb.index.IndexHits; +import org.neo4j.graphdb.index.IndexManager; +import org.neo4j.graphdb.traversal.Evaluators; +import org.neo4j.graphdb.traversal.TraversalDescription; +import org.neo4j.graphdb.traversal.Uniqueness; + +import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import org.neo4j.cypher.javacompat.ExecutionEngine; -import org.neo4j.cypher.javacompat.ExecutionResult; -import org.neo4j.graphdb.GraphDatabaseService; -import org.neo4j.graphdb.Node; -import org.neo4j.graphdb.Transaction; -import org.neo4j.kernel.impl.util.StringLogger; +import java.util.ArrayList; +import java.util.List; /** * @@ -34,14 +29,14 @@ @javax.ws.rs.Path("/following") public class FollowingSiblings { - + + private static final RelationshipType IS_FAKE = DynamicRelationshipType.withName("isFakeOrder"); + private static final RelationshipType IS_NEXT = DynamicRelationshipType.withName("edm:isNextInSequence"); private GraphDatabaseService db; - private ExecutionEngine engine; public FollowingSiblings(@Context GraphDatabaseService db) { this.db = db; - this.engine = new ExecutionEngine(db, StringLogger.SYSTEM); } @GET @@ -50,26 +45,37 @@ public FollowingSiblings(@Context GraphDatabaseService db) { public Response getfollowing(@PathParam("nodeId") String nodeId, @QueryParam("limit") @DefaultValue("10") int limit) { + List followingSiblings = new ArrayList<>(); - Transaction tx = db.beginTx(); - - try { - ExecutionResult result = engine.execute( - "start self = node:edmsearch2(rdf_about=\"" + nodeId + "\") " - + " MATCH (self)-[:isFakeOrder|`edm:isNextInSequence`*]->(following) " - + "RETURN following LIMIT " + limit); - Iterator followingIterator = result.columnAs("following"); - int i = 0; - while (followingIterator.hasNext()) { - followingSiblings.add(followingIterator.next()); + boolean first = false; + try ( Transaction tx = db.beginTx() ) { + IndexManager index = db.index(); + Index edmsearch2 = index.forNodes("edmsearch2"); + IndexHits hits = edmsearch2.get("rdf_about", nodeId); + Node sibling = hits.getSingle(); + if (sibling==null) { + throw new IllegalArgumentException("no node found in index for rdf_about = " + nodeId); } - } catch (Exception e) { - Logger.getLogger(this.getClass().getCanonicalName()).log(Level.SEVERE, e.getMessage()); - } finally { + // Gather all ye following brothers and sisters but take heed! No more than in 'limit' number shall ye come! + TraversalDescription td = db.traversalDescription() + .breadthFirst() + .relationships(IS_FAKE, Direction.INCOMING) + .relationships(IS_NEXT, Direction.INCOMING) + .uniqueness(Uniqueness.RELATIONSHIP_GLOBAL) + .evaluator(Evaluators.toDepth(limit )); + + // Add to the results + for (org.neo4j.graphdb.Path path : td.traverse(sibling)) { + Node child = path.endNode(); + if (first) { + followingSiblings.add(child); + } else { + first = true; + } + } String obj = new ObjectMapper().siblingsToJson(followingSiblings, "siblings"); tx.success(); - tx.finish(); return Response.ok().entity(obj).header(HttpHeaders.CONTENT_TYPE, "application/json").build(); } diff --git a/neo4j-startup-plugin/src/main/java/eu/europeana/neo4j/fetch/PrecedingSiblings.java b/neo4j-startup-plugin/src/main/java/eu/europeana/neo4j/fetch/PrecedingSiblings.java index f904bff..eb56ce5 100644 --- a/neo4j-startup-plugin/src/main/java/eu/europeana/neo4j/fetch/PrecedingSiblings.java +++ b/neo4j-startup-plugin/src/main/java/eu/europeana/neo4j/fetch/PrecedingSiblings.java @@ -6,26 +6,21 @@ package eu.europeana.neo4j.fetch; import eu.europeana.neo4j.mapper.ObjectMapper; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; -import javax.ws.rs.DefaultValue; -import javax.ws.rs.GET; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; +import org.neo4j.graphdb.*; +import org.neo4j.graphdb.index.Index; +import org.neo4j.graphdb.index.IndexHits; +import org.neo4j.graphdb.index.IndexManager; +import org.neo4j.graphdb.traversal.Evaluators; +import org.neo4j.graphdb.traversal.TraversalDescription; +import org.neo4j.graphdb.traversal.Uniqueness; + +import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import org.neo4j.cypher.javacompat.ExecutionEngine; -import org.neo4j.cypher.javacompat.ExecutionResult; -import org.neo4j.graphdb.GraphDatabaseService; -import org.neo4j.graphdb.Node; -import org.neo4j.graphdb.Transaction; -import org.neo4j.kernel.impl.util.StringLogger; +import java.util.ArrayList; +import java.util.List; /** * @@ -34,42 +29,52 @@ @javax.ws.rs.Path("/preceding") public class PrecedingSiblings { - + + private static final RelationshipType IS_FAKE = DynamicRelationshipType.withName("isFakeOrder"); + private static final RelationshipType IS_NEXT = DynamicRelationshipType.withName("edm:isNextInSequence"); private GraphDatabaseService db; - private ExecutionEngine engine; public PrecedingSiblings(@Context GraphDatabaseService db) { this.db = db; - this.engine = new ExecutionEngine(db, StringLogger.SYSTEM); } @GET @javax.ws.rs.Path("/nodeId/{nodeId}") @Produces(MediaType.APPLICATION_JSON) - - public Response getpreceding(@PathParam("nodeId") String nodeId, - @QueryParam("limit") @DefaultValue("10") int limit) { + public Response getpreceding(@PathParam("nodeId") String nodeId, + @QueryParam("limit") @DefaultValue("10") int limit) { List precedingSiblings = new ArrayList<>(); - Transaction tx = db.beginTx(); - - try { - ExecutionResult result = engine.execute( - "start self = node:edmsearch2(rdf_about=\"" + nodeId + "\") " - + " MATCH (preceding)-[:isFakeOrder|`edm:isNextInSequence`*]->(self) " - + "RETURN preceding LIMIT " + limit); - Iterator precedingIterator = result.columnAs("preceding"); - int i = 0; - while (precedingIterator.hasNext()) { - precedingSiblings.add(precedingIterator.next()); + boolean first = false; + try ( Transaction tx = db.beginTx() ) { + IndexManager index = db.index(); + Index edmsearch2 = index.forNodes("edmsearch2"); + IndexHits hits = edmsearch2.get("rdf_about", nodeId); + Node sibling = hits.getSingle(); + if (sibling==null) { + throw new IllegalArgumentException("no node found in index for rdf_about = " + nodeId); + } + + // Gather all ye preceding brothers and sisters but take heed! No more than in 'limit' number shall ye come! + TraversalDescription td = db.traversalDescription() + .breadthFirst() + .relationships(IS_FAKE, Direction.OUTGOING) + .relationships(IS_NEXT, Direction.OUTGOING) + .uniqueness(Uniqueness.RELATIONSHIP_GLOBAL) + .evaluator(Evaluators.toDepth(limit)); + + // Add to the results + for (org.neo4j.graphdb.Path path : td.traverse(sibling)) { + Node child = path.endNode(); + if (first) { + precedingSiblings.add(child); + } else { + first = true; + } } - } catch (Exception e) { - Logger.getLogger(this.getClass().getCanonicalName()).log(Level.SEVERE, e.getMessage()); - } finally { String obj = new ObjectMapper().siblingsToJson(precedingSiblings, "siblings"); tx.success(); - tx.finish(); return Response.ok().entity(obj).header(HttpHeaders.CONTENT_TYPE, "application/json").build(); } diff --git a/neo4j-startup-plugin/src/main/java/eu/europeana/neo4j/initial/Startup.java b/neo4j-startup-plugin/src/main/java/eu/europeana/neo4j/initial/Startup.java index 57527d0..3d8ef5c 100644 --- a/neo4j-startup-plugin/src/main/java/eu/europeana/neo4j/initial/Startup.java +++ b/neo4j-startup-plugin/src/main/java/eu/europeana/neo4j/initial/Startup.java @@ -56,8 +56,9 @@ public Startup(@Context GraphDatabaseService db) { @javax.ws.rs.Path("/nodeId/{nodeId}") @Produces(MediaType.APPLICATION_JSON) - public Response hierarchy(@PathParam("nodeId") String nodeId, @QueryParam("length") @DefaultValue("32") int length, - @QueryParam("lengthBefore") @DefaultValue("8") int lengthBefore) { + public Response hierarchy(@PathParam("nodeId") String nodeId, + @QueryParam("length") @DefaultValue("32") int length, + @QueryParam("lengthBefore") @DefaultValue("8") int lengthBefore) { Hierarchy hierarchy = new Hierarchy(); List parents = new ArrayList<>(); Transaction tx = db.beginTx();