diff --git a/.gitignore b/.gitignore index f19a265..3973d8e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ target/ !.mvn/wrapper/maven-wrapper.jar !**/src/main/**/target/ !**/src/test/**/target/ +goblinWeaver_data ### STS ### .apt_generated @@ -31,4 +32,4 @@ build/ ### VS Code ### .vscode/ -osvData/ \ No newline at end of file +osvData/ diff --git a/pom.xml b/pom.xml index b2d4c14..256c908 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.cifre.sap.su goblinWeaver - 1.0.0 + 2.0.0 goblinWeaver Weaver to query Neo4j ecosystem dependency graph diff --git a/readme.md b/readme.md index 8d9894d..b1554e6 100644 --- a/readme.md +++ b/readme.md @@ -1,21 +1,31 @@ -# Gobelin Weaver +# Goblin Weaver [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](LICENSE.txt) -The Weaver is a REST API that takes a Cypher query and desired additional information as input and returns the result of this query on a Neo4j ecosystem dependency graph, enriching it on the fly according to the user’s needs. +The Weaver is a REST API for querying the Maven Central's dependency graph and enriching it by adding information to it according to the user's needs. -This weaver comes in addition to the ecosystem dependency graph miner available here: https://github.com/Goblin-Ecosystem/goblinWeaver. -An example of usage of this tool is available here: https://github.com/Goblin-Ecosystem/mavenDatasetExperiences -A Zenodo archive that contains the associated dataset dump and the Weaver jar is available here: https://zenodo.org/records/10291589. +This weaver use the Maven ecosystem dependency graph available here: https://zenodo.org/records/10605656 -If you use the dataset dump present in Zenodo, please use a Neo4j version 4.x. +This graph was created by the Goblin Miner, and can be updated by it: https://github.com/Goblin-Ecosystem/goblinMiner + +An example of usage of this tool is available here: https://github.com/Goblin-Ecosystem/mavenDatasetExperiences (on Weaver version 1.0.0). + +If you use the dataset dump present in Zenodo, please use a Neo4j database version 4.x. ## Added values - CVE: We use the osv.dev dataset to get CVEs information (https://osv.dev/). - CVE_AGGREGATED: Aggregate release and dependencies (with transitivity) CVE. - FRESHNESS: Corresponds, for a specific release, to the number of more recent releases available and to the time elapsed in milliseconds between the specific release and the most recent release. - FRESHNESS_AGGREGATED: Aggregate release and dependencies (with transitivity) freshness. +- POPULARITY_1_YEAR: Corresponds, for a specific release, the number of version released within a maximum of one year after the current graph date using the specified release. +- POPULARITY_1_YEAR_AGGREGATED: Aggregate release and dependencies (with transitivity) POPULARITY_1_YEAR. - SPEED: Corresponds to the average number of releases per day of an artifact. More information here: https://benevol2022.github.io/papers/DamienJaime.pdf +## Memoization +To avoid having to calculate the same metrics several times, added values are stored in the database graph once calculated. +These new "AddedValue" type nodes are linked with "Release" or "Artifact" nodes. +Due to their changing nature, routes are available in the API to remove them. +However, the Weaver itself can delete them on change: when updating the CVE database, the "CVE" added values are deleted, when updating the database "FRESHNESS", "POPULARITY" and "SPEED" added values are deleted. + ## Requirements - Java 17 - Maven, with MAVEN_HOME defines @@ -31,10 +41,10 @@ The program will first download the osv.dev dataset and create a folder called " If you already have downloaded this dataset and you don't want to update it, you can add the "noUpdate" argument on the java -jar command. Example: -> java -Dneo4jUri="bolt://localhost:7687/" -Dneo4jUser="neo4j" -Dneo4jPassword="Password1" -jar .\target\goblinWeaver-1.0.0.jar +> java -Dneo4jUri="bolt://localhost:7687/" -Dneo4jUser="neo4j" -Dneo4jPassword="Password1" -jar .\target\goblinWeaver-2.0.0.jar -> java -Dneo4jUri="bolt://localhost:7687/" -Dneo4jUser="neo4j" -Dneo4jPassword="Password1" -jar .\target\goblinWeaver-1.0.0.jar +> java -Dneo4jUri="bolt://localhost:7687/" -Dneo4jUser="neo4j" -Dneo4jPassword="Password1" -jar .\target\goblinWeaver-2.0.0.jar noUpdate ## Use the API Pre-designed requests are available, but you can also send your own Cypher requests directly to the API. @@ -47,9 +57,10 @@ A swagger documentation of the API is available here: The Weaver is designed to be extensible, allowing a user to easily add information their research need. Here's how to add an added value: 1. Go to weaver/addedValue/AddedValueEnum and add the name of your new value. -2. Create a new class that implements weaver/addedValue/AddedValue. -3. Write your internal logic in this new class. -4. Go to weaver/Weaver and modify the "getValuesToAdd" method to define which type of element your new value can be applied to (Release, Artifact, dependency edge or relationship_AR edges). +2. Fill the three methods of this enumeration with your new added value +3. Create a new class that extends weaver/addedValue/AbstractAddedValue. +4. (optional) If you also want to create an aggregated value of your new added value, create a new class that extends your previous new class and implements the "AggregateValue" interface. +5. Write your internal logic in this new class. ## Licensing -Copyright 2023 SAP SE or an SAP affiliate company and Neo4j Ecosystem Weaver. Please see our [LICENSE](LICENSE) for copyright and license information. +Copyright 2024 SAP SE or an SAP affiliate company and Neo4j Ecosystem Weaver. Please see our [LICENSE](LICENSE) for copyright and license information. diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/Neo4jEcosystemWeaverApplication.java b/src/main/java/com/cifre/sap/su/goblinWeaver/Neo4jEcosystemWeaverApplication.java new file mode 100644 index 0000000..eab5bf4 --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/Neo4jEcosystemWeaverApplication.java @@ -0,0 +1,19 @@ +package com.cifre.sap.su.goblinWeaver; + +import com.cifre.sap.su.goblinWeaver.graphDatabase.GraphDatabaseSingleton; +import com.cifre.sap.su.goblinWeaver.utils.GraphUpdatedChecker; +import com.cifre.sap.su.goblinWeaver.utils.OsvProceeding; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class Neo4jEcosystemWeaverApplication { + + public static void main(String[] args) { + GraphDatabaseSingleton.getInstance(); // Init database connection + GraphUpdatedChecker.deleteAddedValuesIfUpdated(); // Check if database was updated + OsvProceeding.initOsvData(args); // Download CVE dataset + SpringApplication.run(Neo4jEcosystemWeaverApplication.class, args); // Run API + } + +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/api/controllers/ArtifactController.java b/src/main/java/com/cifre/sap/su/goblinWeaver/api/controllers/ArtifactController.java new file mode 100644 index 0000000..a38734c --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/api/controllers/ArtifactController.java @@ -0,0 +1,42 @@ +package com.cifre.sap.su.goblinWeaver.api.controllers; + +import com.cifre.sap.su.goblinWeaver.api.entities.ArtifactQuery; +import com.cifre.sap.su.goblinWeaver.graphDatabase.GraphDatabaseInterface; +import com.cifre.sap.su.goblinWeaver.graphDatabase.GraphDatabaseSingleton; +import com.cifre.sap.su.goblinWeaver.graphEntities.InternGraph; +import com.cifre.sap.su.goblinWeaver.weaver.Weaver; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.json.simple.JSONObject; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@Tag(name = "Artifacts") +public class ArtifactController { + + @Operation( + description = "Get a specific artifact from groupId:ArtifactId with added values", + summary = "Get a specific artifact from GA" + ) + @PostMapping("/artifact") + public JSONObject getSpecificArtifact(@RequestBody ArtifactQuery artifactQuery) { + GraphDatabaseInterface gdb = GraphDatabaseSingleton.getInstance(); + InternGraph graph = gdb.executeQuery(gdb.getQueryDictionary().getSpecificArtifactQuery(artifactQuery.toString())); + Weaver.weaveGraph(graph, artifactQuery.getAddedValues()); + return graph.getJsonGraph(); + } + + @Operation( + description = "Get all releases of an artifact from groupId:ArtifactId with added values", + summary = "Get all releases of an artifact from GA" + ) + @PostMapping("/artifact/releases") + public JSONObject getArtifactReleases(@RequestBody ArtifactQuery artifactQuery) { + GraphDatabaseInterface gdb = GraphDatabaseSingleton.getInstance(); + InternGraph graph = gdb.executeQuery(gdb.getQueryDictionary().getArtifactReleasesQuery(artifactQuery.toString())); + Weaver.weaveGraph(graph, artifactQuery.getAddedValues()); + return graph.getJsonGraph(); + } +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/api/controllers/CypherController.java b/src/main/java/com/cifre/sap/su/goblinWeaver/api/controllers/CypherController.java new file mode 100644 index 0000000..0517c2b --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/api/controllers/CypherController.java @@ -0,0 +1,26 @@ +package com.cifre.sap.su.goblinWeaver.api.controllers; + +import com.cifre.sap.su.goblinWeaver.api.entities.CypherQuery; +import com.cifre.sap.su.goblinWeaver.graphDatabase.GraphDatabaseSingleton; +import com.cifre.sap.su.goblinWeaver.graphEntities.InternGraph; +import com.cifre.sap.su.goblinWeaver.weaver.Weaver; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.json.simple.JSONObject; +import org.springframework.web.bind.annotation.*; + +@RestController +@Tag(name = "Cypher query") +public class CypherController { + + @Operation( + description = "Execute a cypher query to the dependency graph with added values", + summary = "Execute a cypher query" + ) + @PostMapping("/cypher") + public JSONObject executeCypherQuery(@RequestBody CypherQuery queryRequest) { + InternGraph graph = GraphDatabaseSingleton.getInstance().executeQuery(queryRequest.getQuery()); + Weaver.weaveGraph(graph, queryRequest.getAddedValues()); + return graph.getJsonGraph(); + } +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/api/controllers/GraphController.java b/src/main/java/com/cifre/sap/su/goblinWeaver/api/controllers/GraphController.java new file mode 100644 index 0000000..e5203e6 --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/api/controllers/GraphController.java @@ -0,0 +1,161 @@ +package com.cifre.sap.su.goblinWeaver.api.controllers; + +import com.cifre.sap.su.goblinWeaver.api.entities.ReleaseQueryList; +import com.cifre.sap.su.goblinWeaver.graphDatabase.GraphDatabaseSingleton; +import com.cifre.sap.su.goblinWeaver.graphEntities.InternGraph; +import com.cifre.sap.su.goblinWeaver.graphEntities.edges.DependencyEdge; +import com.cifre.sap.su.goblinWeaver.graphEntities.nodes.NodeObject; +import com.cifre.sap.su.goblinWeaver.graphEntities.nodes.NodeType; +import com.cifre.sap.su.goblinWeaver.graphEntities.nodes.ReleaseNode; +import com.cifre.sap.su.goblinWeaver.weaver.Weaver; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.json.simple.JSONObject; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +@RestController +@Tag(name = "Graph") +public class GraphController { + + @Operation( + description = "Get the project rooted graph", + summary = "Get the project rooted all graph from releases dependencies list" + ) + @PostMapping("/graph/rootedGraph") + public JSONObject getRootedGraph(@RequestBody ReleaseQueryList releaseQueryList) { + InternGraph resultGraph = new InternGraph(); + resultGraph.addNode(new ReleaseNode("ROOT", "ROOT", 0, "")); + for (ReleaseQueryList.Release release : releaseQueryList.getReleases()) { + resultGraph.addEdge(new DependencyEdge("ROOT", release.getGa(), release.getVersion(), "compile")); + } + resultGraph.mergeGraph( + GraphDatabaseSingleton.getInstance() + .getRootedGraph( + releaseQueryList.getReleases().stream().map(ReleaseQueryList.Release::getGav).collect(Collectors.toSet() + ) + ) + ); + Weaver.weaveGraph(resultGraph, releaseQueryList.getAddedValues()); + return resultGraph.getJsonGraph(); + } + + @Operation( + description = "Get the project rooted all possibilities graph", + summary = "Get the project rooted all possibilities graph from releases dependencies list" + ) + @PostMapping("/graph/allPossibilitiesRooted") + public JSONObject getAllPossibilitiesRootedGraph(@RequestBody ReleaseQueryList releaseQueryList) { + InternGraph resultGraph = new InternGraph(); + resultGraph.addNode(new ReleaseNode("ROOT", "ROOT", 0, "")); + for (ReleaseQueryList.Release release : releaseQueryList.getReleases()) { + resultGraph.addEdge(new DependencyEdge("ROOT", release.getGa(), release.getVersion(), "compile")); + } + resultGraph.mergeGraph( + GraphDatabaseSingleton.getInstance() + .getAllPossibilitiesGraph( + releaseQueryList.getReleases().stream().map(ReleaseQueryList.Release::getGa).collect(Collectors.toSet() + ) + ) + ); + Weaver.weaveGraph(resultGraph, releaseQueryList.getAddedValues()); + return resultGraph.getJsonGraph(); + } + + @Operation( + description = "Get the project rooted direct dependencies possibilities graph", + summary = "Get the project rooted direct dependencies possibilities graph from releases dependencies list" + ) + @PostMapping("/graph/directPossibilitiesRooted") + public JSONObject getDirectPossibilitiesRootedGraph(@RequestBody ReleaseQueryList releaseQueryList) { + InternGraph resultGraph = new InternGraph(); + resultGraph.addNode(new ReleaseNode("ROOT", "ROOT", 0, "")); + for (ReleaseQueryList.Release release : releaseQueryList.getReleases()) { + resultGraph.addEdge(new DependencyEdge("ROOT", release.getGa(), release.getVersion(), "compile")); + } + resultGraph.mergeGraph( + GraphDatabaseSingleton.getInstance() + .getDirectPossibilitiesGraph( + releaseQueryList.getReleases().stream().map(ReleaseQueryList.Release::getGa).collect(Collectors.toSet() + ) + ) + ); + Weaver.weaveGraph(resultGraph, releaseQueryList.getAddedValues()); + return resultGraph.getJsonGraph(); + } + + @Operation( + description = "Get the project rooted direct dependencies possibilities graph with transitive", + summary = "Get the project rooted direct dependencies possibilities graph with transitive version from releases dependencies list" + ) + @PostMapping("/graph/directPossibilitiesWithTransitiveRooted") + public JSONObject getDirectPossibilitiesWithTransitiveRootedGraph(@RequestBody ReleaseQueryList releaseQueryList) { + InternGraph resultGraph = new InternGraph(); + resultGraph.addNode(new ReleaseNode("ROOT", "ROOT", 0, "")); + for (ReleaseQueryList.Release release : releaseQueryList.getReleases()) { + resultGraph.addEdge(new DependencyEdge("ROOT", release.getGa(), release.getVersion(), "compile")); + } + // Get direct all possibilities + InternGraph directAllPossibilities = GraphDatabaseSingleton.getInstance() + .getDirectPossibilitiesGraph(releaseQueryList.getReleases().stream().map(ReleaseQueryList.Release::getGa).collect(Collectors.toSet())); + resultGraph.mergeGraph(directAllPossibilities); + // Get Releases dependencies + Map parameters = new HashMap<>(); + Set visitedReleases = new HashSet<>(); + Set releasesToTreat = directAllPossibilities.getGraphNodes().stream().filter(n -> n.getType().equals(NodeType.RELEASE)).map(NodeObject::getId).collect(Collectors.toSet()); + while (!releasesToTreat.isEmpty()){ + parameters.put("releaseIdList",releasesToTreat); + InternGraph queryResult = GraphDatabaseSingleton.getInstance().executeQueryWithParameters(GraphDatabaseSingleton.getInstance().getQueryDictionary().getDependencyGraphFromReleaseIdListParameter(), parameters); + resultGraph.mergeGraph(queryResult); + visitedReleases.addAll(releasesToTreat); + Set newReleaseToTreat = resultGraph.getGraphNodes().stream().filter(node -> node instanceof ReleaseNode).map(NodeObject::getId).collect(Collectors.toSet()); + newReleaseToTreat.removeAll(visitedReleases); + releasesToTreat.clear(); + releasesToTreat.addAll(newReleaseToTreat); + } + + Weaver.weaveGraph(resultGraph, releaseQueryList.getAddedValues()); + return resultGraph.getJsonGraph(); + } + + @Operation( + description = "Get the project rooted direct dependencies possibilities graph with transitive", + summary = "Get the project rooted direct dependencies possibilities graph with transitive version from releases dependencies list" + ) + @PostMapping("/graph/directNewPossibilitiesWithTransitiveRooted") + public JSONObject getDirectNewPossibilitiesWithTransitiveRootedGraph(@RequestBody ReleaseQueryList releaseQueryList) { + InternGraph resultGraph = new InternGraph(); + resultGraph.addNode(new ReleaseNode("ROOT", "ROOT", 0, "")); + for (ReleaseQueryList.Release release : releaseQueryList.getReleases()) { + resultGraph.addEdge(new DependencyEdge("ROOT", release.getGa(), release.getVersion(), "compile")); + } + // Get direct all possibilities + InternGraph directAllPossibilities = GraphDatabaseSingleton.getInstance() + .getDirectNewPossibilitiesGraph(releaseQueryList.getReleases()); + resultGraph.mergeGraph(directAllPossibilities); + // Get Releases dependencies + Map parameters = new HashMap<>(); + Set visitedReleases = new HashSet<>(); + Set releasesToTreat = directAllPossibilities.getGraphNodes().stream().filter(n -> n.getType().equals(NodeType.RELEASE)).map(NodeObject::getId).collect(Collectors.toSet()); + while (!releasesToTreat.isEmpty()){ + parameters.put("releaseIdList",releasesToTreat); + InternGraph queryResult = GraphDatabaseSingleton.getInstance().executeQueryWithParameters(GraphDatabaseSingleton.getInstance().getQueryDictionary().getDependencyGraphFromReleaseIdListParameter(), parameters); + resultGraph.mergeGraph(queryResult); + visitedReleases.addAll(releasesToTreat); + Set newReleaseToTreat = resultGraph.getGraphNodes().stream().filter(node -> node instanceof ReleaseNode).map(NodeObject::getId).collect(Collectors.toSet()); + newReleaseToTreat.removeAll(visitedReleases); + releasesToTreat.clear(); + releasesToTreat.addAll(newReleaseToTreat); + } + + Weaver.weaveGraph(resultGraph, releaseQueryList.getAddedValues()); + return resultGraph.getJsonGraph(); + } +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/api/controllers/ReleaseController.java b/src/main/java/com/cifre/sap/su/goblinWeaver/api/controllers/ReleaseController.java new file mode 100644 index 0000000..0d09e2b --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/api/controllers/ReleaseController.java @@ -0,0 +1,54 @@ +package com.cifre.sap.su.goblinWeaver.api.controllers; + +import com.cifre.sap.su.goblinWeaver.api.entities.ReleaseQuery; +import com.cifre.sap.su.goblinWeaver.graphDatabase.GraphDatabaseInterface; +import com.cifre.sap.su.goblinWeaver.graphDatabase.GraphDatabaseSingleton; +import com.cifre.sap.su.goblinWeaver.graphEntities.InternGraph; +import com.cifre.sap.su.goblinWeaver.weaver.Weaver; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.json.simple.JSONObject; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@Tag(name = "Releases") +public class ReleaseController { + + @Operation( + description = "Get a specific release from groupId:ArtifactId:Version with added values", + summary = "Get a specific release from GAV" + ) + @PostMapping("/release") + public JSONObject getSpecificRelease(@RequestBody ReleaseQuery releaseQuery) { + GraphDatabaseInterface gdb = GraphDatabaseSingleton.getInstance(); + InternGraph graph = gdb.executeQuery(gdb.getQueryDictionary().getSpecificRelease(releaseQuery.toString())); + Weaver.weaveGraph(graph, releaseQuery.getAddedValues()); + return graph.getJsonGraph(); + } + + @Operation( + description = "Get newer versions of a release from groupId:ArtifactId:Version with added values", + summary = "Get newer versions of a release from GAV" + ) + @PostMapping("/release/newVersions") + public JSONObject getNewerReleases(@RequestBody ReleaseQuery releaseQuery) { + GraphDatabaseInterface gdb = GraphDatabaseSingleton.getInstance(); + InternGraph graph = gdb.executeQuery(gdb.getQueryDictionary().getNewerReleases(releaseQuery.toString(), releaseQuery.getGa())); + Weaver.weaveGraph(graph, releaseQuery.getAddedValues()); + return graph.getJsonGraph(); + } + + @Operation( + description = "Get dependents of a release from groupId:ArtifactId:Version with added values", + summary = "Get release dependents from GAV" + ) + @PostMapping("/release/dependents") + public JSONObject getReleaseDependent(@RequestBody ReleaseQuery releaseQuery) { + GraphDatabaseInterface gdb = GraphDatabaseSingleton.getInstance(); + InternGraph graph = gdb.executeQuery(gdb.getQueryDictionary().getReleaseDependent(releaseQuery.getGa(), releaseQuery.getVersion())); + Weaver.weaveGraph(graph, releaseQuery.getAddedValues()); + return graph.getJsonGraph(); + } +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/api/controllers/UtilsController.java b/src/main/java/com/cifre/sap/su/goblinWeaver/api/controllers/UtilsController.java new file mode 100644 index 0000000..a306460 --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/api/controllers/UtilsController.java @@ -0,0 +1,46 @@ +package com.cifre.sap.su.goblinWeaver.api.controllers; + +import com.cifre.sap.su.goblinWeaver.api.entities.AddedValueQuery; +import com.cifre.sap.su.goblinWeaver.graphDatabase.GraphDatabaseSingleton; +import com.cifre.sap.su.goblinWeaver.weaver.addedValue.AddedValueEnum; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; + +import java.util.EnumSet; + + +@RestController +@Tag(name = "Utils") +public class UtilsController { + + @Operation( + description = "Get the list of existing added values on the Weaver", + summary = "Get existing added values" + ) + @GetMapping("/addedValues") + public AddedValueEnum[] getAddedValue() { + return AddedValueEnum.values(); + } + + @Operation( + description = "Delete all added values nodes on database", + summary = "Delete all added values" + ) + @DeleteMapping("/addedValues") + public void deleteAddedValues(){ + GraphDatabaseSingleton.getInstance().removeAddedValuesOnGraph(EnumSet.allOf(AddedValueEnum.class)); + } + + @Operation( + description = "Delete specific added values nodes on database", + summary = "Delete specific added values" + ) + @DeleteMapping("/addedValue") + public void deleteAddedValue(@RequestBody AddedValueQuery addedValueQuery){ + GraphDatabaseSingleton.getInstance().removeAddedValuesOnGraph(addedValueQuery.getAddedValues()); + } +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/api/entities/AddedValueQuery.java b/src/main/java/com/cifre/sap/su/goblinWeaver/api/entities/AddedValueQuery.java new file mode 100644 index 0000000..9d27b5b --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/api/entities/AddedValueQuery.java @@ -0,0 +1,17 @@ +package com.cifre.sap.su.goblinWeaver.api.entities; + +import com.cifre.sap.su.goblinWeaver.weaver.addedValue.AddedValueEnum; + +import java.util.Set; + +public class AddedValueQuery { + private Set addedValues; + + public Set getAddedValues() { + return addedValues; + } + + public void setAddedValues(Set addedValues) { + this.addedValues = addedValues; + } +} diff --git a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/api/entities/ArtifactQuery.java b/src/main/java/com/cifre/sap/su/goblinWeaver/api/entities/ArtifactQuery.java similarity index 64% rename from src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/api/entities/ArtifactQuery.java rename to src/main/java/com/cifre/sap/su/goblinWeaver/api/entities/ArtifactQuery.java index 31496eb..cc1f45e 100644 --- a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/api/entities/ArtifactQuery.java +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/api/entities/ArtifactQuery.java @@ -1,13 +1,13 @@ -package com.cifre.sap.su.neo4jEcosystemWeaver.api.entities; +package com.cifre.sap.su.goblinWeaver.api.entities; -import com.cifre.sap.su.neo4jEcosystemWeaver.weaver.addedValue.AddedValueEnum; +import com.cifre.sap.su.goblinWeaver.weaver.addedValue.AddedValueEnum; -import java.util.List; +import java.util.Set; public class ArtifactQuery { private String groupId; private String artifactId; - private List addedValues; + private Set addedValues; public String getGroupId() { return groupId; @@ -25,11 +25,11 @@ public void setArtifactId(String artifactId) { this.artifactId = artifactId; } - public List getAddedValues() { + public Set getAddedValues() { return addedValues; } - public void setAddedValues(List addedValues) { + public void setAddedValues(Set addedValues) { this.addedValues = addedValues; } diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/api/entities/CypherQuery.java b/src/main/java/com/cifre/sap/su/goblinWeaver/api/entities/CypherQuery.java new file mode 100644 index 0000000..62be95d --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/api/entities/CypherQuery.java @@ -0,0 +1,27 @@ +package com.cifre.sap.su.goblinWeaver.api.entities; + +import com.cifre.sap.su.goblinWeaver.weaver.addedValue.AddedValueEnum; + +import java.util.Set; + +public class CypherQuery { + private String query; + private Set addedValues; + + public String getQuery() { + return query; + } + + public void setQuery(String query) { + this.query = query; + } + + public Set getAddedValues() { + return addedValues; + } + + public void setAddedValues(Set addedValues) { + this.addedValues = addedValues; + } +} + diff --git a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/api/entities/ReleaseQuery.java b/src/main/java/com/cifre/sap/su/goblinWeaver/api/entities/ReleaseQuery.java similarity index 69% rename from src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/api/entities/ReleaseQuery.java rename to src/main/java/com/cifre/sap/su/goblinWeaver/api/entities/ReleaseQuery.java index 4fccfd8..f5e33dc 100644 --- a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/api/entities/ReleaseQuery.java +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/api/entities/ReleaseQuery.java @@ -1,14 +1,15 @@ -package com.cifre.sap.su.neo4jEcosystemWeaver.api.entities; +package com.cifre.sap.su.goblinWeaver.api.entities; -import com.cifre.sap.su.neo4jEcosystemWeaver.weaver.addedValue.AddedValueEnum; +import com.cifre.sap.su.goblinWeaver.weaver.addedValue.AddedValueEnum; +import io.swagger.v3.oas.annotations.Hidden; -import java.util.List; +import java.util.Set; public class ReleaseQuery { private String groupId; private String artifactId; private String version; - private List addedValues; + private Set addedValues; public String getGroupId() { return groupId; @@ -34,14 +35,15 @@ public void setVersion(String version) { this.version = version; } - public List getAddedValues() { + public Set getAddedValues() { return addedValues; } - public void setAddedValues(List addedValues) { + public void setAddedValues(Set addedValues) { this.addedValues = addedValues; } + @Hidden public String getGa(){ return groupId+":"+artifactId; } diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/api/entities/ReleaseQueryList.java b/src/main/java/com/cifre/sap/su/goblinWeaver/api/entities/ReleaseQueryList.java new file mode 100644 index 0000000..416dc8c --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/api/entities/ReleaseQueryList.java @@ -0,0 +1,67 @@ +package com.cifre.sap.su.goblinWeaver.api.entities; + +import com.cifre.sap.su.goblinWeaver.weaver.addedValue.AddedValueEnum; +import io.swagger.v3.oas.annotations.Hidden; + +import java.util.Set; + +public class ReleaseQueryList { + Set releases; + private Set addedValues; + + public Set getReleases() { + return releases; + } + + public void setReleases(Set releases) { + this.releases = releases; + } + + public Set getAddedValues() { + return addedValues; + } + + public void setAddedValues(Set addedValues) { + this.addedValues = addedValues; + } + + public static class Release{ + private String groupId; + private String artifactId; + private String version; + + public String getGroupId() { + return groupId; + } + + public void setGroupId(String groupId) { + this.groupId = groupId; + } + + public String getArtifactId() { + return artifactId; + } + + public void setArtifactId(String artifactId) { + this.artifactId = artifactId; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + @Hidden + public String getGa(){ + return groupId+":"+artifactId; + } + + @Hidden + public String getGav(){ + return groupId+":"+artifactId+":"+version; + } + } +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/graphDatabase/GraphDatabaseInterface.java b/src/main/java/com/cifre/sap/su/goblinWeaver/graphDatabase/GraphDatabaseInterface.java new file mode 100644 index 0000000..ba57a9b --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/graphDatabase/GraphDatabaseInterface.java @@ -0,0 +1,25 @@ +package com.cifre.sap.su.goblinWeaver.graphDatabase; + +import com.cifre.sap.su.goblinWeaver.api.entities.ReleaseQueryList; +import com.cifre.sap.su.goblinWeaver.graphEntities.InternGraph; +import com.cifre.sap.su.goblinWeaver.graphEntities.nodes.NodeType; +import com.cifre.sap.su.goblinWeaver.weaver.addedValue.AddedValue; +import com.cifre.sap.su.goblinWeaver.weaver.addedValue.AddedValueEnum; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +public interface GraphDatabaseInterface { + QueryDictionary getQueryDictionary(); + InternGraph executeQuery(String query); + InternGraph executeQueryWithParameters(String query, Map parameters); + Map> getNodeAddedValues(List nodeIds, Set addedValues, NodeType nodeType); + void addAddedValues(List> computedAddedValues); + void putOneAddedValueOnGraph(String nodeId, AddedValueEnum addedValueType, String value); + void removeAddedValuesOnGraph(Set addedValuesType); + InternGraph getRootedGraph(Set artifactIdList); + InternGraph getAllPossibilitiesGraph(Set artifactIdList); + InternGraph getDirectPossibilitiesGraph(Set artifactIdList); + InternGraph getDirectNewPossibilitiesGraph(Set artifactIdList); +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/graphDatabase/GraphDatabaseSingleton.java b/src/main/java/com/cifre/sap/su/goblinWeaver/graphDatabase/GraphDatabaseSingleton.java new file mode 100644 index 0000000..17c6d7b --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/graphDatabase/GraphDatabaseSingleton.java @@ -0,0 +1,27 @@ +package com.cifre.sap.su.goblinWeaver.graphDatabase; + +import com.cifre.sap.su.goblinWeaver.graphDatabase.neo4j.Neo4jGraphDatabase; + +public class GraphDatabaseSingleton { + + private static GraphDatabaseInterface graphDatabase; + + private GraphDatabaseSingleton() { + // private constructor to prevent instantiation + } + + public static GraphDatabaseInterface getInstance() { + if (graphDatabase == null) { + synchronized (GraphDatabaseInterface.class) { + //TODO: To be changed if you want to add a new graph database + if (graphDatabase == null) { + String uri = System.getProperty("neo4jUri"); + String user = System.getProperty("neo4jUser"); + String password = System.getProperty("neo4jPassword"); + graphDatabase = new Neo4jGraphDatabase(uri, user, password); + } + } + } + return graphDatabase; + } +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/graphDatabase/QueryDictionary.java b/src/main/java/com/cifre/sap/su/goblinWeaver/graphDatabase/QueryDictionary.java new file mode 100644 index 0000000..d2cb7ca --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/graphDatabase/QueryDictionary.java @@ -0,0 +1,17 @@ +package com.cifre.sap.su.goblinWeaver.graphDatabase; + +public interface QueryDictionary { + String getSpecificArtifactQuery(String artifactId); + String getArtifactReleasesQuery(String artifactId); + String getSpecificRelease(String releaseId); + String getReleaseDependent(String artifactId, String releaseVersion); + String getNewerReleases(String releaseId, String artifactId); + String getReleaseFreshness(String releaseId); + String getReleasePopularity1Year(String artifactGa, String releaseVersion); + String getArtifactRhythm(String artifactId); + String getReleaseDirectCompileDependencies(String artifactId); + String getLinkedArtifactReleasesAndEdgesQuery(String artifactId); + String getReleaseDirectCompileDependenciesEdgeAndArtifact(String artifactId); + String getLastReleaseTimestamp(); + String getDependencyGraphFromReleaseIdListParameter(); +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/graphDatabase/neo4j/Neo4jGraphDatabase.java b/src/main/java/com/cifre/sap/su/goblinWeaver/graphDatabase/neo4j/Neo4jGraphDatabase.java new file mode 100644 index 0000000..61f17fc --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/graphDatabase/neo4j/Neo4jGraphDatabase.java @@ -0,0 +1,314 @@ +package com.cifre.sap.su.goblinWeaver.graphDatabase.neo4j; + +import com.cifre.sap.su.goblinWeaver.api.entities.ReleaseQueryList; +import com.cifre.sap.su.goblinWeaver.graphDatabase.GraphDatabaseInterface; +import com.cifre.sap.su.goblinWeaver.graphDatabase.QueryDictionary; +import com.cifre.sap.su.goblinWeaver.graphEntities.GraphObject; +import com.cifre.sap.su.goblinWeaver.graphEntities.InternGraph; +import com.cifre.sap.su.goblinWeaver.graphEntities.ValueObject; +import com.cifre.sap.su.goblinWeaver.graphEntities.edges.DependencyEdge; +import com.cifre.sap.su.goblinWeaver.graphEntities.edges.EdgeObject; +import com.cifre.sap.su.goblinWeaver.graphEntities.edges.EdgeType; +import com.cifre.sap.su.goblinWeaver.graphEntities.edges.RelationshipArEdge; +import com.cifre.sap.su.goblinWeaver.graphEntities.nodes.ArtifactNode; +import com.cifre.sap.su.goblinWeaver.graphEntities.nodes.NodeObject; +import com.cifre.sap.su.goblinWeaver.graphEntities.nodes.NodeType; +import com.cifre.sap.su.goblinWeaver.graphEntities.nodes.ReleaseNode; +import com.cifre.sap.su.goblinWeaver.weaver.addedValue.AddedValue; +import com.cifre.sap.su.goblinWeaver.weaver.addedValue.AddedValueEnum; +import org.neo4j.driver.*; +import org.neo4j.driver.Record; +import org.neo4j.driver.types.Node; +import org.neo4j.driver.types.Path; +import org.neo4j.driver.types.Relationship; +import org.neo4j.driver.types.TypeSystem; +import org.neo4j.driver.util.Pair; + +import java.util.*; +import java.util.stream.Collectors; + +public class Neo4jGraphDatabase implements GraphDatabaseInterface { + private final Driver driver; + private final QueryDictionary queryDictionary = new Neo4jQueryDictionary(); + + public Neo4jGraphDatabase(String uri, String user, String password) { + driver = GraphDatabase.driver(uri, AuthTokens.basic(user, password)); + //Init index for added values + try (Session session = driver.session()) { + session.run("CREATE CONSTRAINT addedValueConstraint IF NOT EXISTS FOR (n:AddedValue) REQUIRE n.id IS UNIQUE"); + } + } + + public QueryDictionary getQueryDictionary() { + return queryDictionary; + } + + @Override + public InternGraph executeQuery(String query) { + try (Session session = driver.session()) { + return treatNeo4jResult(session.run(query)); + } + } + + @Override + public InternGraph executeQueryWithParameters(String query, Map parameters) { + try (Session session = driver.session()) { + return treatNeo4jResult(session.run(query, parameters)); + } + } + + private InternGraph treatNeo4jResult(Result result){ + InternGraph graph = new InternGraph(); + while (result.hasNext()) { + Record record = result.next(); + for (Pair pair : record.fields()) { + if (pair.value().hasType(TypeSystem.getDefault().NODE())){ + NodeObject nodeObject = generateNode(pair.value().asNode()); + if(nodeObject != null){ + graph.addNode(nodeObject); + } + } + else if (pair.value().hasType(TypeSystem.getDefault().RELATIONSHIP())){ + EdgeObject edgeObject = generateRelationship(pair.value().asRelationship()); + if(edgeObject != null){ + graph.addEdge(edgeObject); + } + } + else if (pair.value().hasType(TypeSystem.getDefault().PATH())) { + for(GraphObject graphObject : generatePath(pair.value().asPath())){ + if (graphObject instanceof NodeObject) { + graph.addNode((NodeObject) graphObject); + } else { + graph.addEdge((EdgeObject) graphObject); + } + } + } + else if (pair.value().hasType(TypeSystem.getDefault().LIST())){ + List list = pair.value().asList(); + for (Object item : list) { + if (item instanceof Node) { + Node node = (Node) item; + NodeObject nodeObject = generateNode(node); + if(nodeObject != null) { + graph.addNode(nodeObject); + } + } + else if (item instanceof Relationship) { + Relationship relationship = (Relationship) item; + EdgeObject edgeObject = generateRelationship(relationship); + if(edgeObject != null) { + graph.addEdge(edgeObject); + } + } + } + } + else{ + graph.addValue(new ValueObject(pair.key(), pair.value().toString().replaceAll("[\"]",""))); + } + } + } + return graph; + } + + @Override + public Map> getNodeAddedValues(List nodeIds, Set addedValues, NodeType nodeType) { + Map> IdAndAddedValuesMap = new HashMap<>(); + String query = "MATCH (a:AddedValue) WHERE a.id IN $addedValuesIds RETURN a"; + Map parameters = new HashMap<>(); + parameters.put("addedValuesIds", nodeIds.stream() + .flatMap(nodeId -> addedValues.stream().map(addedValue -> nodeId + ":" + addedValue.toString())) + .collect(Collectors.toList())); + try (Session session = driver.session()) { + Result result = session.run(query, parameters); + while (result.hasNext()) { + Record record = result.next(); + Map innerMap = new HashMap<>(); + Pair pair = record.fields().get(0); + if (pair.value().hasType(TypeSystem.getDefault().NODE())) { + innerMap.put(AddedValueEnum.valueOf(pair.value().asNode().get("type").asString()), pair.value().asNode().get("value").asString().replace("\\", "")); + String nodeId = pair.value().asNode().get("id").asString(); + int lastIndex = nodeId.lastIndexOf(':'); + nodeId = nodeId.substring(0,lastIndex); + IdAndAddedValuesMap.computeIfAbsent(nodeId, k -> new HashMap<>()).putAll(innerMap); + } + } + } + return IdAndAddedValuesMap; + } + + @Override + public void addAddedValues(List> computedAddedValues){ + try (Session session = driver.session()) { + Transaction tx = session.beginTransaction(); + int batch = 0; + Map parameters = new HashMap<>(); + for(AddedValue addedValue : computedAddedValues){ + batch++; + parameters.clear(); + parameters.put("sourceId",addedValue.getNodeId()); + parameters.put("addedValueId",addedValue.getNodeId()+":"+addedValue.getAddedValueEnum().toString()); + parameters.put("addedValueType", addedValue.getAddedValueEnum().toString()); + parameters.put("value", addedValue.valueToString(addedValue.getValue())); + tx.run("MATCH (r:"+addedValue.getAddedValueEnum().getTargetNodeType().enumToLabel()+" {id: $sourceId}) CREATE (r)-[l:addedValues]->(v:AddedValue {id: $addedValueId, type: $addedValueType, value: $value})", parameters); + if(batch % 10000 == 0){ + tx.commit(); + tx.close(); + tx = session.beginTransaction(); + } + } + tx.commit(); + tx.close(); + } catch (Exception e) { + System.out.println("Fail to add added values:\n"+e.getMessage()); + } + } + + @Override + public void putOneAddedValueOnGraph(String nodeId, AddedValueEnum addedValueType, String value){ + Map parameters = new HashMap<>(); + parameters.put("nodeId", nodeId); + parameters.put("addedValueId", nodeId+":"+addedValueType.toString()); + parameters.put("addedValueType", addedValueType.toString()); + parameters.put("value", value); + + String query = "MATCH (r:"+addedValueType.getTargetNodeType().enumToLabel()+" {id:$nodeId}) " + + "CREATE (r)-[l:addedValues]->(v:AddedValue {id: $addedValueId, type: $addedValueType, value: $value})"; + try (Session session = driver.session()) { + session.run(query, parameters); + } + } + + @Override + public void removeAddedValuesOnGraph(Set addedValuesType){ + StringBuilder cypherQuery = new StringBuilder(); + cypherQuery.append("MATCH (n:AddedValue) ") + .append("WHERE n.type IN ["); + int i = 0; + for (AddedValueEnum type : addedValuesType) { + cypherQuery.append("'").append(type).append("'"); + if (++i < addedValuesType.size()) { + cypherQuery.append(", "); + } + } + cypherQuery.append("] ") + .append("CALL { WITH n DETACH DELETE n } IN TRANSACTIONS OF 10000 ROWS;"); + try (Session session = driver.session()) { + session.run(cypherQuery.toString()); + } + } + + @Override + public InternGraph getRootedGraph(Set releaseIdList){ + InternGraph rootedGraph = new InternGraph(); + Set releaseToTreat = new HashSet<>(releaseIdList); + Set visitedRelease = new HashSet<>(); + Map parameters = new HashMap<>(); + String query = "MATCH (a:Artifact)-[re:relationship_AR]->(r:Release)-[d:dependency]->(a2:Artifact)-[re2:relationship_AR]->(target:Release) " + + "WHERE a.id = $artifactId AND r.id = $releaseId AND d.scope = 'compile' AND target.version=d.targetVersion " + + "RETURN a,re,r,d,a2,re2,target"; + while (!releaseToTreat.isEmpty()){ + String releaseId = releaseToTreat.iterator().next(); + String[] splitedReleaseId = releaseId.split(":"); + String artifactId = splitedReleaseId[0]+":"+splitedReleaseId[1]; + parameters.put("releaseId",releaseId); + parameters.put("artifactId",artifactId); + InternGraph resultGraph = executeQueryWithParameters(query, parameters); + rootedGraph.mergeGraph(resultGraph); + visitedRelease.add(releaseId); + releaseToTreat.remove(releaseId); + Set newReleaseToTreat = resultGraph.getGraphNodes().stream().filter(node -> node instanceof ReleaseNode).map(NodeObject::getId).collect(Collectors.toSet()); + newReleaseToTreat.removeAll(visitedRelease); + releaseToTreat.addAll(newReleaseToTreat); + } + return rootedGraph; + } + + @Override + public InternGraph getAllPossibilitiesGraph(Set artifactIdList){ + InternGraph graphAllPossibilities = new InternGraph(); + Set artifactToTreat = new HashSet<>(artifactIdList); + Set visitedArtifact = new HashSet<>(); + Map parameters = new HashMap<>(); + String query = "MATCH (a:Artifact)-[e:relationship_AR*]->(r:Release) " + + "WHERE a.id IN $artifactIdList " + + "WITH a, r, e MATCH (r)-[d:dependency]->(a2:Artifact) " + + "WHERE d.scope = 'compile' " + + "RETURN a,e,r,d, a2"; + while (!artifactToTreat.isEmpty()){ + parameters.put("artifactIdList",artifactToTreat); + InternGraph resultGraph = executeQueryWithParameters(query, parameters); + graphAllPossibilities.mergeGraph(resultGraph); + visitedArtifact.addAll(artifactToTreat); + Set newArtifactToTreat = resultGraph.getGraphNodes().stream().filter(node -> node instanceof ArtifactNode).map(NodeObject::getId).collect(Collectors.toSet()); + newArtifactToTreat.removeAll(visitedArtifact); + artifactToTreat.clear(); + artifactToTreat.addAll(newArtifactToTreat); + } + return graphAllPossibilities; + } + + @Override + public InternGraph getDirectPossibilitiesGraph(Set artifactIdList){ + Map parameters = new HashMap<>(); + String query = "MATCH (a:Artifact)-[e:relationship_AR]->(r:Release) " + + "WHERE a.id IN $artifactIdList " + + "RETURN a,e,r"; + parameters.put("artifactIdList", artifactIdList); + return executeQueryWithParameters(query, parameters); + } + + @Override + public InternGraph getDirectNewPossibilitiesGraph(Set releaseIdList){ + InternGraph resultGraph = new InternGraph(); + String query = "MATCH (r:Release) WHERE r.id = $releaseId WITH r.timestamp as currentTimestamp "+ + "MATCH (a:Artifact)-[e:relationship_AR]->(r:Release) " + + "WHERE a.id = $artifactId AND r.timestamp >= currentTimestamp " + + "RETURN a,e,r"; + Map parameters = new HashMap<>(); + for(ReleaseQueryList.Release release : releaseIdList){ + parameters.put("releaseId", release.getGav()); + parameters.put("artifactId", release.getGa()); + InternGraph graph = executeQueryWithParameters(query, parameters); + resultGraph.mergeGraph(graph); + } + return resultGraph; + } + + private static NodeObject generateNode(Node neo4jNode){ + NodeType nodeType = NodeType.neo4jLabelToEnum(neo4jNode.labels().iterator().next()); + if(nodeType != null){ + if (nodeType.equals(NodeType.ARTIFACT)){ + return new ArtifactNode(neo4jNode.elementId(), neo4jNode.get("id").asString(), neo4jNode.get("found").asBoolean()); + } + else if (nodeType.equals(NodeType.RELEASE)){ + return new ReleaseNode(neo4jNode.elementId(), neo4jNode.get("id").asString(), neo4jNode.get("timestamp").asLong(), neo4jNode.get("version").asString()); + } + } + return null; + } + + private static EdgeObject generateRelationship(Relationship neo4jRelationship){ + EdgeType edgeType = EdgeType.neo4jTypeToEnum(neo4jRelationship.type()); + if(edgeType != null) { + if (edgeType.equals(EdgeType.DEPENDENCY)) { + return new DependencyEdge(neo4jRelationship.startNodeElementId(), neo4jRelationship.endNodeElementId(), + neo4jRelationship.get("targetVersion").asString(), neo4jRelationship.get("scope").asString()); + } + else if (edgeType.equals(EdgeType.RELATIONSHIP_AR)){ + return new RelationshipArEdge(neo4jRelationship.startNodeElementId(), neo4jRelationship.endNodeElementId()); + } + } + return null; + } + + private static Set generatePath(Path path){ + Set resultSet = new HashSet<>(); + for (Node node : path.nodes()){ + resultSet.add(generateNode(node) != null ? generateNode(node) : null); + } + for (Relationship relationship : path.relationships()){ + resultSet.add(generateRelationship(relationship) != null ? generateRelationship(relationship) : null); + } + return resultSet; + } +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/graphDatabase/neo4j/Neo4jQueryDictionary.java b/src/main/java/com/cifre/sap/su/goblinWeaver/graphDatabase/neo4j/Neo4jQueryDictionary.java new file mode 100644 index 0000000..ea4275a --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/graphDatabase/neo4j/Neo4jQueryDictionary.java @@ -0,0 +1,115 @@ +package com.cifre.sap.su.goblinWeaver.graphDatabase.neo4j; + +import com.cifre.sap.su.goblinWeaver.graphDatabase.QueryDictionary; +import com.cifre.sap.su.goblinWeaver.utils.GraphUpdatedChecker; + +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; + +public class Neo4jQueryDictionary implements QueryDictionary { + + @Override + public String getSpecificArtifactQuery(String artifactId){ + return "MATCH (a:Artifact) " + + "WHERE a.id = '" + artifactId + "' " + + "RETURN a"; + } + + @Override + public String getArtifactReleasesQuery(String artifactId) { + return "MATCH (a:Artifact)-[e:relationship_AR]->(r:Release) " + + "WHERE a.id = '" + artifactId + "' " + + "RETURN r"; + } + + @Override + public String getLinkedArtifactReleasesAndEdgesQuery(String artifactId) { + return "MATCH (a:Artifact)-[e:relationship_AR]->(r:Release) " + + "WHERE a.id = '" + artifactId + "' " + + "RETURN a,e,r"; + } + + @Override + public String getSpecificRelease(String releaseId) { + return "MATCH (r:Release) " + + "WHERE r.id = '" + releaseId + "' " + + "RETURN r"; + } + + @Override + public String getReleaseDependent(String artifactId, String releaseVersion) { + return "MATCH (r:Release)-[d:dependency]->(a:Artifact) " + + "WHERE a.id = '"+artifactId+"' AND d.targetVersion = '"+releaseVersion+"' " + + "RETURN r"; + } + + @Override + public String getNewerReleases(String releaseId, String artifactId) { + return "MATCH (r:Release) " + + "WHERE r.id = '"+releaseId+"' " + + "WITH r.timestamp as timestamp " + + "MATCH (a:Artifact)-[e:relationship_AR]->(r:Release) " + + "WHERE a.id = '"+artifactId+"' AND r.timestamp > timestamp " + + "RETURN r"; + } + + @Override + public String getReleaseFreshness(String releaseId) { + return "MATCH (r1:Release)<-[:relationship_AR]-(:Artifact)-[:relationship_AR]->(r2:Release) " + + "WHERE r1.id = '"+releaseId+"' AND r2.timestamp > r1.timestamp " + + "WITH r2, r2.timestamp - r1.timestamp AS difference " + + "RETURN count(r2) AS numberMissedRelease, max(difference) AS outdatedTimeInMs"; + } + + @Override + public String getReleasePopularity1Year(String artifactGa, String releaseVersion) { + LocalDate startDate = Instant.ofEpochMilli(GraphUpdatedChecker.getDatabaseLastReleaseTimestamp()) + .atZone(ZoneId.systemDefault()) + .toLocalDate(); + LocalDate oneYearAgo = startDate.minus(1, ChronoUnit.YEARS); + ZonedDateTime zonedDateTime = oneYearAgo.atStartOfDay(ZoneId.systemDefault()); + long oneYearAgoTimestampMillis = zonedDateTime.toInstant().toEpochMilli(); + return "MATCH (r:Release)-[d:dependency]->(a:Artifact) " + + "WHERE a.id = '"+artifactGa+"' AND d.targetVersion = '"+releaseVersion+"' AND r.timestamp > "+oneYearAgoTimestampMillis+" " + + "RETURN count(d)"; + } + + @Override + public String getArtifactRhythm(String artifactId) { + return "MATCH (a:Artifact) -[e:relationship_AR]-> (r:Release) " + + "WHERE a.id = '"+artifactId+"' " + + "RETURN r.timestamp AS timestamp"; + } + + @Override + public String getReleaseDirectCompileDependencies(String artifactId) { + return "MATCH (r:Release)-[d:dependency]->(a:Artifact) " + + "WHERE r.id = '"+artifactId+"' AND (d.scope = 'compile') " + + "WITH a,d " + + "MATCH (dep:Release {id: a.id+':'+d.targetVersion}) " + + "RETURN dep"; + } + + @Override + public String getReleaseDirectCompileDependenciesEdgeAndArtifact(String artifactId) { + return "MATCH (r:Release)-[d:dependency]->(a:Artifact) " + + "WHERE r.id = '"+artifactId+"' AND (d.scope = 'compile') " + + "RETURN a,d"; + } + + @Override + public String getLastReleaseTimestamp(){ + return "MATCH (r:Release) " + + "RETURN MAX(r.timestamp) AS maxTimestamp"; + } + + @Override + public String getDependencyGraphFromReleaseIdListParameter(){ + return "MATCH (r:Release)-[d:dependency]->(a:Artifact)-[e:relationship_AR]->(r2:Release) " + + "WHERE r.id IN $releaseIdList AND d.scope = 'compile' AND r2.version = d.targetVersion " + + "RETURN d,a,e,r2"; + } +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/GraphObject.java b/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/GraphObject.java new file mode 100644 index 0000000..8aef79e --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/GraphObject.java @@ -0,0 +1,7 @@ +package com.cifre.sap.su.goblinWeaver.graphEntities; + +import org.json.simple.JSONObject; + +public interface GraphObject { + JSONObject getJsonObject(); +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/InternGraph.java b/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/InternGraph.java new file mode 100644 index 0000000..92c08f1 --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/InternGraph.java @@ -0,0 +1,89 @@ +package com.cifre.sap.su.goblinWeaver.graphEntities; + +import com.cifre.sap.su.goblinWeaver.graphEntities.edges.EdgeObject; +import com.cifre.sap.su.goblinWeaver.graphEntities.nodes.NodeObject; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; + +import java.util.*; + +public class InternGraph { + private final Set graphNodes = new HashSet<>(); + private final Set graphEdges = new HashSet<>(); + private final Set graphValues = new HashSet<>(); + + public InternGraph() { + } + + public void addNode(NodeObject node){ + this.graphNodes.add(node); + } + + public void addEdge(EdgeObject edge){ + this.graphEdges.add(edge); + } + + public void addValue(ValueObject value){ + this.graphValues.add(value); + } + + public Set getGraphNodes() { + return graphNodes; + } + + public Set getGraphValues() { + return graphValues; + } + + public Set getGraphEdges() { + return graphEdges; + } + + public JSONObject getJsonGraph(){ + resolveEdgesId(); + JSONObject graphJSON = new JSONObject(); + addObjectsToJSONArray(graphNodes, "nodes", graphJSON); + addObjectsToJSONArray(graphEdges, "edges", graphJSON); + addObjectsToJSONArray(graphValues, "values", graphJSON); + return graphJSON; + } + + public void mergeGraph(InternGraph graph){ + this.graphNodes.addAll(graph.getGraphNodes()); + this.graphEdges.addAll(graph.getGraphEdges()); + this.graphValues.addAll(graph.getGraphValues()); + } + + private void addObjectsToJSONArray(Set objects, String key, JSONObject graphJSON) { + if (!objects.isEmpty()) { + JSONArray array = new JSONArray(); + Iterator iterator = objects.iterator(); + while (iterator.hasNext()) { + GraphObject obj = iterator.next(); + array.add(obj.getJsonObject()); + iterator.remove(); + } + graphJSON.put(key, array); + } + } + + private void resolveEdgesId(){ + Map nodeMap = new HashMap<>(); + for (NodeObject node : graphNodes) { + nodeMap.put(node.getNeo4jId(), node.getId()); + } + + for (EdgeObject edge : graphEdges) { + String sourceId = nodeMap.get(edge.getSourceId()); + String targetId = nodeMap.get(edge.getTargetId()); + + // Mettre à jour les identifiants des arêtes en conséquence + if (sourceId != null) { + edge.setSourceId(sourceId); + } + if (targetId != null) { + edge.setTargetId(targetId); + } + } + } +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/ValueObject.java b/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/ValueObject.java new file mode 100644 index 0000000..aaccc4e --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/ValueObject.java @@ -0,0 +1,28 @@ +package com.cifre.sap.su.goblinWeaver.graphEntities; + +import org.json.simple.JSONObject; + +import java.util.Collections; + +public class ValueObject implements GraphObject { + private final String key; + private final String value; + + public ValueObject(String key, String value) { + this.key = key; + this.value = value; + } + + public String getKey() { + return key; + } + + public String getValue() { + return value; + } + + @Override + public JSONObject getJsonObject() { + return new JSONObject(Collections.singletonMap(key, value)); + } +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/edges/DependencyEdge.java b/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/edges/DependencyEdge.java new file mode 100644 index 0000000..1fb5243 --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/edges/DependencyEdge.java @@ -0,0 +1,22 @@ +package com.cifre.sap.su.goblinWeaver.graphEntities.edges; + +import org.json.simple.JSONObject; + +public class DependencyEdge extends EdgeObject { + private final String targetVersion; + private final String scope; + + public DependencyEdge(String sourceId, String targetId, String targetVersion, String scope) { + super(sourceId, targetId, EdgeType.DEPENDENCY); + this.targetVersion = targetVersion; + this.scope = scope; + } + + @Override + public JSONObject getJsonObject() { + JSONObject jsonObject = super.getJsonObject(); + jsonObject.put("targetVersion", targetVersion); + jsonObject.put("scope", scope); + return jsonObject; + } +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/edges/EdgeObject.java b/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/edges/EdgeObject.java new file mode 100644 index 0000000..0d7677d --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/edges/EdgeObject.java @@ -0,0 +1,59 @@ +package com.cifre.sap.su.goblinWeaver.graphEntities.edges; + +import com.cifre.sap.su.goblinWeaver.graphEntities.GraphObject; +import org.json.simple.JSONObject; + +import java.util.Objects; + +public abstract class EdgeObject implements GraphObject { + private String sourceId; + private String targetId; + private final EdgeType type; + + public EdgeObject(String sourceId, String targetId, EdgeType type) { + this.sourceId = sourceId; + this.targetId = targetId; + this.type = type; + } + + public String getSourceId() { + return sourceId; + } + + public String getTargetId() { + return targetId; + } + + public EdgeType getType() { + return type; + } + + public void setSourceId(String sourceId) { + this.sourceId = sourceId; + } + + public void setTargetId(String targetId) { + this.targetId = targetId; + } + + public JSONObject getJsonObject() { + JSONObject jsonObject = new JSONObject(); + jsonObject.put("sourceId",sourceId); + jsonObject.put("targetId",targetId); + jsonObject.put("type",type); + return jsonObject; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + EdgeObject that = (EdgeObject) o; + return sourceId.equals(that.sourceId) && targetId.equals(that.targetId) && type == that.type; + } + + @Override + public int hashCode() { + return Objects.hash(sourceId, targetId, type); + } +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/edges/EdgeType.java b/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/edges/EdgeType.java new file mode 100644 index 0000000..0b48883 --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/edges/EdgeType.java @@ -0,0 +1,15 @@ +package com.cifre.sap.su.goblinWeaver.graphEntities.edges; + +public enum EdgeType { + DEPENDENCY, + RELATIONSHIP_AR; + + public static EdgeType neo4jTypeToEnum(String type) { + for (EdgeType edgeType : EdgeType.values()) { + if (edgeType.name().equalsIgnoreCase(type)) { + return edgeType; + } + } + return null; + } +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/edges/RelationshipArEdge.java b/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/edges/RelationshipArEdge.java new file mode 100644 index 0000000..1c57130 --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/edges/RelationshipArEdge.java @@ -0,0 +1,15 @@ +package com.cifre.sap.su.goblinWeaver.graphEntities.edges; + +import org.json.simple.JSONObject; + +public class RelationshipArEdge extends EdgeObject{ + + public RelationshipArEdge(String sourceId, String targetId) { + super(sourceId, targetId, EdgeType.RELATIONSHIP_AR); + } + + @Override + public JSONObject getJsonObject() { + return super.getJsonObject(); + } +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/nodes/ArtifactNode.java b/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/nodes/ArtifactNode.java new file mode 100644 index 0000000..6f08c67 --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/nodes/ArtifactNode.java @@ -0,0 +1,19 @@ +package com.cifre.sap.su.goblinWeaver.graphEntities.nodes; + +import org.json.simple.JSONObject; + +public class ArtifactNode extends NodeObject { + private final boolean found; + + public ArtifactNode(String neo4jId, String id, boolean found) { + super(neo4jId, id, NodeType.ARTIFACT); + this.found = found; + } + + @Override + public JSONObject getJsonObject() { + JSONObject jsonObject = super.getJsonObject(); + jsonObject.put("found", found); + return jsonObject; + } +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/nodes/NodeObject.java b/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/nodes/NodeObject.java new file mode 100644 index 0000000..a2296f3 --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/nodes/NodeObject.java @@ -0,0 +1,61 @@ +package com.cifre.sap.su.goblinWeaver.graphEntities.nodes; + +import com.cifre.sap.su.goblinWeaver.graphEntities.GraphObject; +import com.cifre.sap.su.goblinWeaver.weaver.addedValue.AddedValue; +import org.json.simple.JSONObject; + +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; + +public abstract class NodeObject implements GraphObject { + private final String neo4jId; + private final String id; + private final NodeType type; + private final Set addedValues = new HashSet<>(); + + public NodeObject(String neo4jId, String id, NodeType type) { + this.neo4jId = neo4jId; + this.id = id; + this.type = type; + } + + public String getId() { + return id; + } + + public NodeType getType() { + return type; + } + + public String getNeo4jId() { + return neo4jId; + } + + public void addAddedValue(AddedValue addedValue){ + this.addedValues.add(addedValue); + } + + public JSONObject getJsonObject() { + JSONObject jsonObject = new JSONObject(); + jsonObject.put("id",id); + jsonObject.put("nodeType",type); + for(AddedValue addedValue : addedValues){ + jsonObject.putAll(addedValue.getValueMap()); + } + return jsonObject; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NodeObject that = (NodeObject) o; + return neo4jId.equals(that.neo4jId); + } + + @Override + public int hashCode() { + return Objects.hash(neo4jId); + } +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/nodes/NodeType.java b/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/nodes/NodeType.java new file mode 100644 index 0000000..db4736c --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/nodes/NodeType.java @@ -0,0 +1,20 @@ +package com.cifre.sap.su.goblinWeaver.graphEntities.nodes; + +public enum NodeType { + RELEASE, + ARTIFACT; + + public static NodeType neo4jLabelToEnum(String label) { + for (NodeType nodeType : NodeType.values()) { + if (nodeType.name().equalsIgnoreCase(label)) { + return nodeType; + } + } + return null; + } + + public String enumToLabel(){ + String enumString = this.name(); + return enumString.substring(0, 1).toUpperCase() + enumString.substring(1).toLowerCase(); + } +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/nodes/ReleaseNode.java b/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/nodes/ReleaseNode.java new file mode 100644 index 0000000..4f38d02 --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/nodes/ReleaseNode.java @@ -0,0 +1,22 @@ +package com.cifre.sap.su.goblinWeaver.graphEntities.nodes; + +import org.json.simple.JSONObject; + +public class ReleaseNode extends NodeObject { + private final long timestamp; + private final String version; + + public ReleaseNode(String neo4jId, String id, long timestamp, String version) { + super(neo4jId, id, NodeType.RELEASE); + this.timestamp = timestamp; + this.version = version; + } + + @Override + public JSONObject getJsonObject() { + JSONObject jsonObject = super.getJsonObject(); + jsonObject.put("version", version); + jsonObject.put("timestamp", timestamp); + return jsonObject; + } +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/utils/ConstantProperties.java b/src/main/java/com/cifre/sap/su/goblinWeaver/utils/ConstantProperties.java new file mode 100644 index 0000000..49c8f55 --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/utils/ConstantProperties.java @@ -0,0 +1,10 @@ +package com.cifre.sap.su.goblinWeaver.utils; + +public class ConstantProperties { + + // Paths + public static final String dataFolderPath = "goblinWeaver_data"; + public static final String osvDataFolderPath = dataFolderPath+"/osvData"; + public static final String databaseStatusFile = dataFolderPath+"/databaseStatus.txt"; + +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/utils/GraphUpdatedChecker.java b/src/main/java/com/cifre/sap/su/goblinWeaver/utils/GraphUpdatedChecker.java new file mode 100644 index 0000000..19fdf8f --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/utils/GraphUpdatedChecker.java @@ -0,0 +1,67 @@ +package com.cifre.sap.su.goblinWeaver.utils; + +import com.cifre.sap.su.goblinWeaver.graphDatabase.GraphDatabaseSingleton; +import com.cifre.sap.su.goblinWeaver.graphEntities.InternGraph; +import com.cifre.sap.su.goblinWeaver.weaver.addedValue.AddedValueEnum; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Set; + +public class GraphUpdatedChecker { + private static final File rootFolder = new File(ConstantProperties.dataFolderPath); + private static final File saveFile = new File(ConstantProperties.databaseStatusFile); + + public static void deleteAddedValuesIfUpdated(){ + System.out.println("Check if Database was updated"); + if (!rootFolder.exists()) { + rootFolder.mkdirs(); + generateCountFile(); + return; + } + if (!saveFile.exists()) { + generateCountFile(); + return; + } + if(checkIfUpdated()){ + System.out.println("Database was updated, remove added values"); + GraphDatabaseSingleton.getInstance().removeAddedValuesOnGraph(Set.of(AddedValueEnum.FRESHNESS, AddedValueEnum.FRESHNESS_AGGREGATED, AddedValueEnum.SPEED)); + generateCountFile(); + } + } + + private static void generateCountFile() { + InternGraph result = GraphDatabaseSingleton.getInstance().executeQuery(GraphDatabaseSingleton.getInstance().getQueryDictionary().getLastReleaseTimestamp()); + try (BufferedWriter writer = Files.newBufferedWriter(saveFile.toPath())) { + writer.write(String.valueOf(result.getGraphValues().iterator().next().getValue())); + } catch (IOException e) { + e.printStackTrace(); + } + } + + private static boolean checkIfUpdated(){ + try (BufferedReader reader = Files.newBufferedReader(saveFile.toPath())) { + String fileCount = reader.readLine(); + InternGraph result = GraphDatabaseSingleton.getInstance().executeQuery(GraphDatabaseSingleton.getInstance().getQueryDictionary().getLastReleaseTimestamp()); + String graphCount = result.getGraphValues().iterator().next().getValue(); + if(fileCount.equals(graphCount)){ + return false; + } + } catch (IOException e) { + e.printStackTrace(); + } + return true; + } + + public static long getDatabaseLastReleaseTimestamp(){ + try (BufferedReader reader = Files.newBufferedReader(saveFile.toPath())) { + return Long.parseLong(reader.readLine()); + } catch (IOException e) { + InternGraph result = GraphDatabaseSingleton.getInstance().executeQuery(GraphDatabaseSingleton.getInstance().getQueryDictionary().getLastReleaseTimestamp()); + return Long.parseLong(result.getGraphValues().iterator().next().getValue()); + } + } +} diff --git a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/utils/OsvDataSingleton.java b/src/main/java/com/cifre/sap/su/goblinWeaver/utils/OsvDataSingleton.java similarity index 94% rename from src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/utils/OsvDataSingleton.java rename to src/main/java/com/cifre/sap/su/goblinWeaver/utils/OsvDataSingleton.java index 6ac9e15..af0f55d 100644 --- a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/utils/OsvDataSingleton.java +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/utils/OsvDataSingleton.java @@ -1,4 +1,4 @@ -package com.cifre.sap.su.neo4jEcosystemWeaver.utils; +package com.cifre.sap.su.goblinWeaver.utils; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; diff --git a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/utils/OsvProceeding.java b/src/main/java/com/cifre/sap/su/goblinWeaver/utils/OsvProceeding.java similarity index 93% rename from src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/utils/OsvProceeding.java rename to src/main/java/com/cifre/sap/su/goblinWeaver/utils/OsvProceeding.java index 51d6454..83fc62b 100644 --- a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/utils/OsvProceeding.java +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/utils/OsvProceeding.java @@ -1,5 +1,7 @@ -package com.cifre.sap.su.neo4jEcosystemWeaver.utils; +package com.cifre.sap.su.goblinWeaver.utils; +import com.cifre.sap.su.goblinWeaver.graphDatabase.GraphDatabaseSingleton; +import com.cifre.sap.su.goblinWeaver.weaver.addedValue.AddedValueEnum; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; @@ -10,11 +12,12 @@ import java.net.URL; import java.util.HashMap; import java.util.Map; +import java.util.Set; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; public class OsvProceeding { - private static final String ROOT_PATH = "osvData"; + private static final String ROOT_PATH = ConstantProperties.osvDataFolderPath; private static final String DATA_PATH = ROOT_PATH+"/maven"; public static final String AGGREGATED_DATA_FILE = ROOT_PATH+"/aggregated_data.json"; private static final String OSV_DATA_URL = "https://storage.googleapis.com/osv-vulnerabilities/Maven/all.zip"; @@ -37,6 +40,7 @@ public static void initOsvData(String[] args){ createAggregateDataFile(); try { OsvDataSingleton.getDataJsonObject(); + GraphDatabaseSingleton.getInstance().removeAddedValuesOnGraph(Set.of(AddedValueEnum.CVE, AddedValueEnum.CVE_AGGREGATED)); } catch (IOException | ParseException e) { e.printStackTrace(); } diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/Weaver.java b/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/Weaver.java new file mode 100644 index 0000000..abe4909 --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/Weaver.java @@ -0,0 +1,100 @@ +package com.cifre.sap.su.goblinWeaver.weaver; + +import com.cifre.sap.su.goblinWeaver.graphDatabase.GraphDatabaseSingleton; +import com.cifre.sap.su.goblinWeaver.graphEntities.InternGraph; +import com.cifre.sap.su.goblinWeaver.graphEntities.nodes.NodeObject; +import com.cifre.sap.su.goblinWeaver.graphEntities.nodes.NodeType; +import com.cifre.sap.su.goblinWeaver.weaver.addedValue.*; + +import java.lang.reflect.InvocationTargetException; +import java.util.*; +import java.util.stream.Collectors; + +public class Weaver { + + public static void weaveGraph (InternGraph graph, Set addedValues){ + if(addedValues.isEmpty()){ + return; + } + List> computedAddedValues = new ArrayList<>(); + for(NodeType nodeType : NodeType.values()){ + Set nodeTypeAddedValues = addedValues.stream().filter(a -> a.getTargetNodeType().equals(nodeType)).collect(Collectors.toSet()); + if(!nodeTypeAddedValues.isEmpty()){ + for (List nodeBatch : nodeIdToBatch(graph, nodeType)) { + Map> resolvedNodeAddedValues = GraphDatabaseSingleton.getInstance().getNodeAddedValues(nodeBatch.stream().map(NodeObject::getId).toList(), addedValues, nodeType); + computedAddedValues.addAll(fillNodeAddedValues(nodeBatch, nodeTypeAddedValues, resolvedNodeAddedValues)); + GraphDatabaseSingleton.getInstance().addAddedValues(computedAddedValues); + computedAddedValues.clear(); + } + } + } + } + + private static List> nodeIdToBatch(InternGraph graph, NodeType type) { + int batchSize = 100000; + List> batches = new ArrayList<>(); + int i = 0; + List currentBatch = new ArrayList<>(); + List nodeObjectsTyped = graph.getGraphNodes().stream().filter(node -> node.getType().equals(type)).toList(); + for (NodeObject node : nodeObjectsTyped) { + currentBatch.add(node); + if (++i % batchSize == 0 || i == nodeObjectsTyped.size()) { + batches.add(new ArrayList<>(currentBatch)); + currentBatch.clear(); + } + } + return batches; + } + + private static List> fillNodeAddedValues(List nodes, Set nodeTypeAddedValues, Map> resolvedNodeAddedValues) { + List> computedAddedValues = new ArrayList<>(); + for (NodeObject node : nodes){ + for (AddedValueEnum addedValueEnum : nodeTypeAddedValues) { + try { + AddedValue addedValue = addedValueEnum.getAddedValueClass().getDeclaredConstructor(String.class).newInstance(node.getId()); + // If addedValue is present on graph + if (resolvedNodeAddedValues.containsKey(node.getId()) && resolvedNodeAddedValues.get(node.getId()).containsKey(addedValueEnum)) { + addedValue.setValue(resolvedNodeAddedValues.get(node.getId()).get(addedValueEnum)); + } else { + addedValue.computeValue(); + // Aggregated values put on graph when compute + if (!addedValueEnum.isAggregatedValue()) { + computedAddedValues.add(addedValue); + } + } + node.addAddedValue(addedValue); + } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + e.printStackTrace(); + } + } + resolvedNodeAddedValues.remove(node.getId()); + } + return computedAddedValues; + } + + private static List> fillNodeAddedValuesOld(InternGraph graph, Set nodeTypeAddedValues, Map> resolvedNodeAddedValues, NodeType nodeType) { + List> computedAddedValues = new ArrayList<>(); + for (NodeObject node : graph.getGraphNodes().stream().filter(node -> node.getType().equals(nodeType)).collect(Collectors.toSet())){ + String nodeId = node.getId(); + for(AddedValueEnum addedValueEnum : nodeTypeAddedValues){ + try { + AddedValue addedValue = addedValueEnum.getAddedValueClass().getDeclaredConstructor(String.class).newInstance(nodeId); + // If addedValue is present on graph + if(resolvedNodeAddedValues.containsKey(nodeId) && resolvedNodeAddedValues.get(nodeId).containsKey(addedValueEnum)){ + addedValue.setValue(resolvedNodeAddedValues.get(nodeId).get(addedValueEnum)); + } + else{ + addedValue.computeValue(); + if(!addedValueEnum.isAggregatedValue()){ + computedAddedValues.add(addedValue); + } + } + node.addAddedValue(addedValue); + } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + e.printStackTrace(); + } + } + } + return computedAddedValues; + } +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/AbstractAddedValue.java b/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/AbstractAddedValue.java new file mode 100644 index 0000000..b7ac3ab --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/AbstractAddedValue.java @@ -0,0 +1,33 @@ +package com.cifre.sap.su.goblinWeaver.weaver.addedValue; + +import java.util.Collections; +import java.util.Map; + +public abstract class AbstractAddedValue implements AddedValue{ + protected final String nodeId; + protected T value; + + public AbstractAddedValue(String nodeId) { + this.nodeId = nodeId; + } + + @Override + public String getNodeId(){ + return nodeId; + } + + @Override + public T getValue(){ + return value; + } + + @Override + public Map getValueMap() { + return Collections.singletonMap(getAddedValueEnum().getJsonKey(), value); + } + + @Override + public void setValue(String value) { + this.value = this.stringToValue(value); + } +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/AddedValue.java b/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/AddedValue.java new file mode 100644 index 0000000..20bf6c6 --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/AddedValue.java @@ -0,0 +1,14 @@ +package com.cifre.sap.su.goblinWeaver.weaver.addedValue; + +import java.util.Map; + +public interface AddedValue { + AddedValueEnum getAddedValueEnum(); + String getNodeId(); + void setValue(String value); + void computeValue(); + Map getValueMap(); + T stringToValue(String jsonString); + String valueToString(T value); + T getValue(); +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/AddedValueEnum.java b/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/AddedValueEnum.java new file mode 100644 index 0000000..5c6e21a --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/AddedValueEnum.java @@ -0,0 +1,45 @@ +package com.cifre.sap.su.goblinWeaver.weaver.addedValue; + +import com.cifre.sap.su.goblinWeaver.graphEntities.nodes.NodeType; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +@JsonDeserialize(using = AddedValueEnumDeserializer.class) +public enum AddedValueEnum { + CVE, + CVE_AGGREGATED, + FRESHNESS, + FRESHNESS_AGGREGATED, + POPULARITY_1_YEAR, + POPULARITY_1_YEAR_AGGREGATED, + SPEED; + + public NodeType getTargetNodeType(){ + return switch (this) { + case CVE, CVE_AGGREGATED, FRESHNESS, FRESHNESS_AGGREGATED, POPULARITY_1_YEAR, POPULARITY_1_YEAR_AGGREGATED -> NodeType.RELEASE; + case SPEED -> NodeType.ARTIFACT; + }; + } + + public Class> getAddedValueClass(){ + return switch (this) { + case CVE -> Cve.class; + case CVE_AGGREGATED -> CveAggregated.class; + case FRESHNESS -> Freshness.class; + case FRESHNESS_AGGREGATED -> FreshnessAggregated.class; + case POPULARITY_1_YEAR -> Popularity1Year.class; + case POPULARITY_1_YEAR_AGGREGATED -> Popularity1YearAggregated.class; + case SPEED -> Speed.class; + }; + } + + public boolean isAggregatedValue(){ + return switch (this) { + case CVE_AGGREGATED, FRESHNESS_AGGREGATED, POPULARITY_1_YEAR_AGGREGATED -> true; + default -> false; + }; + } + + public String getJsonKey(){ + return this.name().toLowerCase(); + } +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/AddedValueEnumDeserializer.java b/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/AddedValueEnumDeserializer.java new file mode 100644 index 0000000..42f2fcb --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/AddedValueEnumDeserializer.java @@ -0,0 +1,19 @@ +package com.cifre.sap.su.goblinWeaver.weaver.addedValue; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; + +import java.io.IOException; + +public class AddedValueEnumDeserializer extends StdDeserializer { + + protected AddedValueEnumDeserializer() { + super(AddedValueEnum.class); + } + + @Override + public AddedValueEnum deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { + return AddedValueEnum.valueOf(p.getText().toUpperCase()); + } +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/AggregateValue.java b/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/AggregateValue.java new file mode 100644 index 0000000..ebbea18 --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/AggregateValue.java @@ -0,0 +1,45 @@ +package com.cifre.sap.su.goblinWeaver.weaver.addedValue; + +import com.cifre.sap.su.goblinWeaver.graphDatabase.GraphDatabaseInterface; +import com.cifre.sap.su.goblinWeaver.graphDatabase.GraphDatabaseSingleton; +import com.cifre.sap.su.goblinWeaver.graphEntities.InternGraph; +import com.cifre.sap.su.goblinWeaver.graphEntities.nodes.NodeObject; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +public interface AggregateValue extends AddedValue{ + + default T computeAggregatedValue(String nodeId, Set visiting){ + // For cycles + if(visiting.contains(nodeId)){ + return getZeroValue(); + } + visiting.add(nodeId); + GraphDatabaseInterface gdb = GraphDatabaseSingleton.getInstance(); + // Check if value exist + Map> alreadyCalculatedAddedValue = gdb.getNodeAddedValues(List.of(nodeId), Set.of(getAddedValueEnum()), getAddedValueEnum().getTargetNodeType()); + // Value exist + if(alreadyCalculatedAddedValue.containsKey(nodeId) && alreadyCalculatedAddedValue.get(nodeId).containsKey(getAddedValueEnum())){ + return this.stringToValue(alreadyCalculatedAddedValue.get(nodeId).get(getAddedValueEnum())); + } + else{ + T computedValue = computeMetric(nodeId); + InternGraph graph = gdb.executeQuery(gdb.getQueryDictionary().getReleaseDirectCompileDependencies(nodeId)); + for(NodeObject dep : graph.getGraphNodes()){ + computedValue = mergeValue(computedValue, computeAggregatedValue(dep.getId(), visiting)); + } + visiting.remove(nodeId); + //Add calculated value on graph and return + gdb.putOneAddedValueOnGraph(nodeId, getAddedValueEnum(), valueToString(computedValue)); + return computedValue; + } + } + + T mergeValue(T computedValue, T computeAggregatedValue); + + T computeMetric(String nodeId); + + T getZeroValue(); +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/Cve.java b/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/Cve.java new file mode 100644 index 0000000..dad0902 --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/Cve.java @@ -0,0 +1,91 @@ +package com.cifre.sap.su.goblinWeaver.weaver.addedValue; + +import com.cifre.sap.su.goblinWeaver.utils.OsvDataSingleton; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; + +import java.io.IOException; +import java.util.*; + +public class Cve extends AbstractAddedValue>>{ + + public Cve(String nodeId) { + super(nodeId); + } + + @Override + public AddedValueEnum getAddedValueEnum(){ + return AddedValueEnum.CVE; + } + + @Override + public void computeValue(){ + value = getCveFromGav(nodeId); + } + + @Override + public Set> stringToValue(String jsonString){ + Set> resultSet = new HashSet<>(); + try { + JSONParser parser = new JSONParser(); + JSONObject jsonObject = (JSONObject) parser.parse(jsonString); + + JSONArray cveArray = (JSONArray) jsonObject.get(getAddedValueEnum().getJsonKey()); + if(cveArray != null) { + for (Object obj : cveArray) { + JSONObject cveJson = (JSONObject) obj; + + Map cveMap = Map.of( + "cwe", (String) cveJson.get("cwe"), + "severity", (String) cveJson.get("severity"), + "name", (String) cveJson.get("name") + ); + + resultSet.add(cveMap); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + return resultSet; + } + + @Override + public String valueToString(Set> value){ + JSONArray jsonArray = new JSONArray(); + + for (Map map : value) { + JSONObject jsonObject = new JSONObject(); + jsonObject.putAll(map); + jsonArray.add(jsonObject); + } + + JSONObject finalObject = new JSONObject(); + finalObject.put(getAddedValueEnum().getJsonKey(), jsonArray); + + return finalObject.toJSONString().replace("\"", "\\\""); + } + + protected static Set> getCveFromGav(String gav){ + Set> resultSet = new HashSet<>(); + try { + JSONObject jsonObject = OsvDataSingleton.getDataJsonObject(); + JSONArray cveArray = (JSONArray) jsonObject.get(gav); + if (cveArray != null) { + for (Object vulnerability : cveArray) { + JSONObject vulnerabilityJsonObject = (JSONObject) vulnerability; + Map cveMap = new HashMap(); + cveMap.put("name", (String) vulnerabilityJsonObject.get("name")); + cveMap.put("cwe", (String) vulnerabilityJsonObject.get("cwe_ids")); + cveMap.put("severity", (String) vulnerabilityJsonObject.get("severity")); + resultSet.add(cveMap); + } + } + } catch (IOException | ParseException e) { + e.printStackTrace(); + } + return resultSet; + } +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/CveAggregated.java b/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/CveAggregated.java new file mode 100644 index 0000000..78c3f42 --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/CveAggregated.java @@ -0,0 +1,37 @@ +package com.cifre.sap.su.goblinWeaver.weaver.addedValue; + +import java.util.*; + +public class CveAggregated extends Cve implements AggregateValue>>{ + + public CveAggregated(String nodeId) { + super(nodeId); + } + + @Override + public AddedValueEnum getAddedValueEnum() { + return AddedValueEnum.CVE_AGGREGATED; + } + + @Override + public void computeValue() { + super.value = computeAggregatedValue(nodeId, new HashSet<>()); + } + + @Override + public Set> mergeValue(Set> computedValue, Set> computeAggregatedValue) { + Set> mergedSet = new HashSet<>(computedValue); + mergedSet.addAll(computeAggregatedValue); + return mergedSet; + } + + @Override + public Set> computeMetric(String nodeId) { + return getCveFromGav(nodeId); + } + + @Override + public Set> getZeroValue() { + return new HashSet<>(); + } +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/Freshness.java b/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/Freshness.java new file mode 100644 index 0000000..05ac1ec --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/Freshness.java @@ -0,0 +1,65 @@ +package com.cifre.sap.su.goblinWeaver.weaver.addedValue; + +import com.cifre.sap.su.goblinWeaver.graphDatabase.GraphDatabaseInterface; +import com.cifre.sap.su.goblinWeaver.graphDatabase.GraphDatabaseSingleton; +import com.cifre.sap.su.goblinWeaver.graphEntities.InternGraph; +import com.cifre.sap.su.goblinWeaver.graphEntities.ValueObject; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; + +import java.util.HashMap; +import java.util.Map; + +public class Freshness extends AbstractAddedValue>{ + + + public Freshness(String nodeId) { + super(nodeId); + } + + @Override + public AddedValueEnum getAddedValueEnum(){ + return AddedValueEnum.FRESHNESS; + } + + @Override + public void computeValue() { + this.value = getFreshnessMapFromGav(nodeId); + } + + @Override + public Map stringToValue(String jsonString){ + Map resultMap = new HashMap<>(); + try { + JSONParser parser = new JSONParser(); + JSONObject jsonObject = (JSONObject) parser.parse(jsonString); + + JSONObject freshnessJson = (JSONObject) jsonObject.get(getAddedValueEnum().getJsonKey()); + resultMap.put("numberMissedRelease", (String) freshnessJson.get("numberMissedRelease")); + resultMap.put("outdatedTimeInMs", (String) freshnessJson.get("outdatedTimeInMs")); + } catch (Exception e) { + e.printStackTrace(); + } + return resultMap; + } + + @Override + public String valueToString(Map value){ + JSONObject jsonObject = new JSONObject(); + jsonObject.putAll(value); + JSONObject finalObject = new JSONObject(); + finalObject.put(getAddedValueEnum().getJsonKey(), value); + return finalObject.toJSONString().replace("\"", "\\\""); + } + + protected static Map getFreshnessMapFromGav(String nodeId){ + Map freshnessMap = new HashMap<>(); + GraphDatabaseInterface gdb = GraphDatabaseSingleton.getInstance(); + InternGraph graph = gdb.executeQuery(gdb.getQueryDictionary().getReleaseFreshness(nodeId)); + for(ValueObject value : graph.getGraphValues()){ + String valueNotNull = value.getValue().equals("NULL") ? "0" : value.getValue(); + freshnessMap.put(value.getKey(),valueNotNull); + } + return freshnessMap; + } +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/FreshnessAggregated.java b/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/FreshnessAggregated.java new file mode 100644 index 0000000..6cc0612 --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/FreshnessAggregated.java @@ -0,0 +1,46 @@ +package com.cifre.sap.su.goblinWeaver.weaver.addedValue; + +import java.util.*; + +public class FreshnessAggregated extends Freshness implements AggregateValue>{ + + public FreshnessAggregated(String nodeId) { + super(nodeId); + } + + @Override + public AddedValueEnum getAddedValueEnum() { + return AddedValueEnum.FRESHNESS_AGGREGATED; + } + + @Override + public void computeValue() { + super.value = computeAggregatedValue(nodeId, new HashSet<>()); + } + + + @Override + public Map mergeValue(Map computedValue, Map computeAggregatedValue) { + int totalNumberMissedRelease = Integer.parseInt(computedValue.get("numberMissedRelease")); + long totalOutdatedTimeInMs = Long.parseLong(computedValue.get("outdatedTimeInMs")); + totalNumberMissedRelease += Integer.parseInt(computeAggregatedValue.get("numberMissedRelease")); + totalOutdatedTimeInMs += Long.parseLong(computeAggregatedValue.get("outdatedTimeInMs")); + Map aggregatedFreshnessMap = new HashMap<>(); + aggregatedFreshnessMap.put("numberMissedRelease", Integer.toString(totalNumberMissedRelease)); + aggregatedFreshnessMap.put("outdatedTimeInMs", Long.toString(totalOutdatedTimeInMs)); + return aggregatedFreshnessMap; + } + + @Override + public Map computeMetric(String nodeId) { + return getFreshnessMapFromGav(nodeId); + } + + @Override + public Map getZeroValue() { + Map emptyFreshness = new HashMap<>(); + emptyFreshness.put("numberMissedRelease", "0"); + emptyFreshness.put("outdatedTimeInMs", "0"); + return emptyFreshness; + } +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/Popularity1Year.java b/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/Popularity1Year.java new file mode 100644 index 0000000..82c0b70 --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/Popularity1Year.java @@ -0,0 +1,52 @@ +package com.cifre.sap.su.goblinWeaver.weaver.addedValue; + +import com.cifre.sap.su.goblinWeaver.graphDatabase.GraphDatabaseInterface; +import com.cifre.sap.su.goblinWeaver.graphDatabase.GraphDatabaseSingleton; +import com.cifre.sap.su.goblinWeaver.graphEntities.InternGraph; +import com.cifre.sap.su.goblinWeaver.graphEntities.ValueObject; + +import java.util.Iterator; + +public class Popularity1Year extends AbstractAddedValue{ + + public Popularity1Year(String nodeId) { + super(nodeId); + } + + @Override + public AddedValueEnum getAddedValueEnum() { + return AddedValueEnum.POPULARITY_1_YEAR; + } + + @Override + public Integer stringToValue(String jsonString) { + return Integer.valueOf(jsonString); + } + + @Override + public String valueToString(Integer value) { + return String.valueOf(value); + } + + @Override + public void computeValue() { + super.value = fillPopularity1Year(nodeId); + + } + + protected int fillPopularity1Year(String gav){ + int popularity = 0; + String[] splitedGav = gav.split(":"); + if (splitedGav.length == 3) { + String artifactGa = splitedGav[0]+":"+splitedGav[1]; + String releaseVersion = splitedGav[2]; + GraphDatabaseInterface gdb = GraphDatabaseSingleton.getInstance(); + InternGraph graph = gdb.executeQuery(gdb.getQueryDictionary().getReleasePopularity1Year(artifactGa, releaseVersion)); + Iterator valueIterator = graph.getGraphValues().iterator(); + if(valueIterator.hasNext()) { + popularity = Integer.parseInt(graph.getGraphValues().iterator().next().getValue()); + } + } + return popularity; + } +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/Popularity1YearAggregated.java b/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/Popularity1YearAggregated.java new file mode 100644 index 0000000..90b4239 --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/Popularity1YearAggregated.java @@ -0,0 +1,35 @@ +package com.cifre.sap.su.goblinWeaver.weaver.addedValue; + +import java.util.HashSet; + +public class Popularity1YearAggregated extends Popularity1Year implements AggregateValue{ + + public Popularity1YearAggregated(String nodeId) { + super(nodeId); + } + + @Override + public AddedValueEnum getAddedValueEnum() { + return AddedValueEnum.POPULARITY_1_YEAR_AGGREGATED; + } + + @Override + public void computeValue() { + super.value = computeAggregatedValue(nodeId, new HashSet<>()); + } + + @Override + public Integer mergeValue(Integer computedValue, Integer computeAggregatedValue) { + return computedValue + computeAggregatedValue; + } + + @Override + public Integer computeMetric(String nodeId) { + return super.fillPopularity1Year(nodeId); + } + + @Override + public Integer getZeroValue() { + return 0; + } +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/Speed.java b/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/Speed.java new file mode 100644 index 0000000..9a2a0d7 --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/weaver/addedValue/Speed.java @@ -0,0 +1,59 @@ +package com.cifre.sap.su.goblinWeaver.weaver.addedValue; + +import com.cifre.sap.su.goblinWeaver.graphDatabase.GraphDatabaseInterface; +import com.cifre.sap.su.goblinWeaver.graphDatabase.GraphDatabaseSingleton; +import com.cifre.sap.su.goblinWeaver.graphEntities.InternGraph; +import com.cifre.sap.su.goblinWeaver.graphEntities.ValueObject; + +import java.time.Duration; +import java.time.Instant; +import java.util.*; + +public class Speed extends AbstractAddedValue { + + public Speed(String nodeId) { + super(nodeId); + } + + @Override + public AddedValueEnum getAddedValueEnum(){ + return AddedValueEnum.SPEED; + } + + + @Override + public void computeValue() { + this.value = fillSpeed(); + } + + @Override + public Double stringToValue(String jsonString) { + return Double.valueOf(jsonString); + } + + @Override + public String valueToString(Double value) { + return String.valueOf(value); + } + + private double fillSpeed(){ + TreeSet releasesTimeStampSet = new TreeSet<>(); + GraphDatabaseInterface gdb = GraphDatabaseSingleton.getInstance(); + InternGraph graph = gdb.executeQuery(gdb.getQueryDictionary().getArtifactRhythm(nodeId)); + for(ValueObject value : graph.getGraphValues()){ + releasesTimeStampSet.add(Long.parseLong(value.getValue())); + } + if(releasesTimeStampSet.size() < 2){ + return 0; + } + // Calculate average speed by days + Instant minTimestamp = Instant.ofEpochMilli(releasesTimeStampSet.first()); + Instant maxTimestamp = Instant.ofEpochMilli(releasesTimeStampSet.last()); + long durationDays = Duration.between(minTimestamp, maxTimestamp).toDays(); + if (durationDays == 0) { + return 0; + } else { + return (double) releasesTimeStampSet.size() / durationDays; + } + } +} diff --git a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/Neo4jEcosystemWeaverApplication.java b/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/Neo4jEcosystemWeaverApplication.java deleted file mode 100644 index 30c7e00..0000000 --- a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/Neo4jEcosystemWeaverApplication.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.cifre.sap.su.neo4jEcosystemWeaver; - -import com.cifre.sap.su.neo4jEcosystemWeaver.utils.Neo4jDriverSingleton; -import com.cifre.sap.su.neo4jEcosystemWeaver.utils.OsvProceeding; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; - -@SpringBootApplication -public class Neo4jEcosystemWeaverApplication { - - public static void main(String[] args) { - OsvProceeding.initOsvData(args); // Download CVE dataset - Neo4jDriverSingleton.getDriverInstance(); // Init database connection - SpringApplication.run(Neo4jEcosystemWeaverApplication.class, args); // Run API - } - -} diff --git a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/api/controllers/ArtifactController.java b/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/api/controllers/ArtifactController.java deleted file mode 100644 index 2dc20b2..0000000 --- a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/api/controllers/ArtifactController.java +++ /dev/null @@ -1,80 +0,0 @@ -package com.cifre.sap.su.neo4jEcosystemWeaver.api.controllers; - -import com.cifre.sap.su.neo4jEcosystemWeaver.api.entities.ArtifactQuery; -import com.cifre.sap.su.neo4jEcosystemWeaver.utils.Neo4jDriverSingleton; -import com.cifre.sap.su.neo4jEcosystemWeaver.utils.Neo4jUtils; -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.tags.Tag; -import org.neo4j.driver.*; -import org.neo4j.driver.Record; -import org.neo4j.driver.types.TypeSystem; -import org.neo4j.driver.util.Pair; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RestController; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -@RestController -@Tag(name = "Artifacts") -public class ArtifactController { - - @Operation( - description = "Get a specific artifact from groupId:ArtifactId with added values", - summary = "Get a specific artifact from GA" - ) - @PostMapping("/artifact") - public List> getSpecificArtifact(@RequestBody ArtifactQuery artifactQuery) { - String query = "MATCH (a:Artifact) " + - "WHERE a.id = '" + artifactQuery.toString() + "' " + - "RETURN a"; - Driver driver = Neo4jDriverSingleton.getDriverInstance(); - try (Session session = driver.session()) { - Result result = session.run(query); - List> rows = new ArrayList<>(); - while (result.hasNext()) { - Record record = result.next(); - Map row = new HashMap<>(); - for (Pair pair : record.fields()) { - if (pair.value().hasType(TypeSystem.getDefault().NODE())){ - Map nodeMap = Neo4jUtils.generateNodeRow(pair, artifactQuery.getAddedValues()); - row.put("node", nodeMap); - } - } - rows.add(row); - } - return rows; - } - } - - @Operation( - description = "Get all releases of an artifact from groupId:ArtifactId with added values", - summary = "Get all releases of an artifact from GA" - ) - @PostMapping("/artifact/releases") - public List> getArtifactReleases(@RequestBody ArtifactQuery artifactQuery) { - String query = "MATCH (a:Artifact)-[e:relationship_AR]->(r:Release) " + - "WHERE a.id = '" + artifactQuery.toString() + "' " + - "RETURN r"; - Driver driver = Neo4jDriverSingleton.getDriverInstance(); - try (Session session = driver.session()) { - Result result = session.run(query); - List> rows = new ArrayList<>(); - while (result.hasNext()) { - Record record = result.next(); - Map row = new HashMap<>(); - for (Pair pair : record.fields()) { - if (pair.value().hasType(TypeSystem.getDefault().NODE())){ - Map nodeMap = Neo4jUtils.generateNodeRow(pair, artifactQuery.getAddedValues()); - row.put("node", nodeMap); - } - } - rows.add(row); - } - return rows; - } - } -} diff --git a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/api/controllers/CypherController.java b/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/api/controllers/CypherController.java deleted file mode 100644 index cf078be..0000000 --- a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/api/controllers/CypherController.java +++ /dev/null @@ -1,52 +0,0 @@ -package com.cifre.sap.su.neo4jEcosystemWeaver.api.controllers; - -import com.cifre.sap.su.neo4jEcosystemWeaver.api.entities.CypherQuery; -import com.cifre.sap.su.neo4jEcosystemWeaver.utils.Neo4jDriverSingleton; -import com.cifre.sap.su.neo4jEcosystemWeaver.utils.Neo4jUtils; -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.tags.Tag; -import org.neo4j.driver.*; -import org.neo4j.driver.Record; -import org.neo4j.driver.types.TypeSystem; -import org.neo4j.driver.util.Pair; -import org.springframework.web.bind.annotation.*; - -import java.util.*; - -@RestController -@Tag(name = "Cypher query") -public class CypherController { - - @Operation( - description = "Execute a cypher query to the dependency graph with added values", - summary = "Execute a cypher query" - ) - @PostMapping("/cypher") - public List> executeCypherQuery(@RequestBody CypherQuery queryRequest) { - Driver driver = Neo4jDriverSingleton.getDriverInstance(); - try (Session session = driver.session()) { - Result result = session.run(queryRequest.getQuery()); - List> rows = new ArrayList<>(); - while (result.hasNext()) { - Record record = result.next(); - Map row = new HashMap<>(); - for (Pair pair : record.fields()) { - if (pair.value().hasType(TypeSystem.getDefault().NODE())){ - row.put("node", Neo4jUtils.generateNodeRow(pair, queryRequest.getAddedValues())); - } - else if (pair.value().hasType(TypeSystem.getDefault().RELATIONSHIP())){ - row.put("relationship", Neo4jUtils.generateRelationshipRow(pair, queryRequest.getAddedValues())); - } - else if (pair.value().hasType(TypeSystem.getDefault().PATH())) { - row.putAll(Neo4jUtils.generatePathRow(pair, queryRequest.getAddedValues())); - } - else{ - row.put(pair.key(), pair.value().toString().replaceAll("[\"]","")); - } - } - rows.add(row); - } - return rows; - } - } -} diff --git a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/api/controllers/ReleaseController.java b/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/api/controllers/ReleaseController.java deleted file mode 100644 index 77be832..0000000 --- a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/api/controllers/ReleaseController.java +++ /dev/null @@ -1,112 +0,0 @@ -package com.cifre.sap.su.neo4jEcosystemWeaver.api.controllers; - -import com.cifre.sap.su.neo4jEcosystemWeaver.api.entities.ReleaseQuery; -import com.cifre.sap.su.neo4jEcosystemWeaver.utils.GraphUtils; -import com.cifre.sap.su.neo4jEcosystemWeaver.utils.Neo4jDriverSingleton; -import com.cifre.sap.su.neo4jEcosystemWeaver.utils.Neo4jUtils; -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.tags.Tag; -import org.neo4j.driver.*; -import org.neo4j.driver.Record; -import org.neo4j.driver.types.TypeSystem; -import org.neo4j.driver.util.Pair; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RestController; - -import java.util.*; -import java.util.stream.Collectors; - -@RestController -@Tag(name = "Releases") -public class ReleaseController { - - @Operation( - description = "Get a specific release from groupId:ArtifactId:Version with added values", - summary = "Get a specific release from GAV" - ) - @PostMapping("/release") - public List> getSpecificRelease(@RequestBody ReleaseQuery releaseQuery) { - String query = "MATCH (r:Release) " + - "WHERE r.id = '" + releaseQuery.toString() + "' " + - "RETURN r"; - Driver driver = Neo4jDriverSingleton.getDriverInstance(); - try (Session session = driver.session()) { - Result result = session.run(query); - List> rows = new ArrayList<>(); - while (result.hasNext()) { - Record record = result.next(); - Map row = new HashMap<>(); - for (Pair pair : record.fields()) { - if (pair.value().hasType(TypeSystem.getDefault().NODE())){ - Map nodeMap = Neo4jUtils.generateNodeRow(pair, releaseQuery.getAddedValues()); - row.put("node", nodeMap); - } - } - rows.add(row); - } - return rows; - } - } - - @Operation( - description = "Get newer versions of a release from groupId:ArtifactId:Version with added values", - summary = "Get newer versions of a release from GAV" - ) - @PostMapping("/release/newVersions") - public List> getNewerReleases(@RequestBody ReleaseQuery releaseQuery) { - String queryTimestamp = "MATCH (r:Release) " + - "WHERE r.id = '" + releaseQuery.toString() + "' " + - "RETURN r.timestamp as timestamp"; - Driver driver = Neo4jDriverSingleton.getDriverInstance(); - try (Session session = driver.session()) { - Result resultTimestamp = session.run(queryTimestamp); - long timestamp = resultTimestamp.next().fields().get(0).value().asLong(); - String queryNewReleases = "MATCH (a:Artifact)-[e:relationship_AR]->(r:Release) " + - "WHERE a.id = '" + releaseQuery.getGa() + "' AND r.timestamp > "+timestamp+" " + - "RETURN r"; - Result resultNewReleases = session.run(queryNewReleases); - List> rows = new ArrayList<>(); - while (resultNewReleases.hasNext()) { - Record record = resultNewReleases.next(); - Map row = new HashMap<>(); - for (Pair pair : record.fields()) { - if (pair.value().hasType(TypeSystem.getDefault().NODE())){ - Map nodeMap = Neo4jUtils.generateNodeRow(pair, releaseQuery.getAddedValues()); - row.put("node", nodeMap); - } - } - rows.add(row); - } - return rows; - } - } - - @Operation( - description = "Get dependents of a release from groupId:ArtifactId:Version with added values", - summary = "Get release dependents from GAV" - ) - @PostMapping("/release/dependents") - public List> getReleaseDependent(@RequestBody ReleaseQuery releaseQuery) { - String query = "MATCH (r:Release)-[d:dependency]->(a:Artifact) " + - "WHERE a.id = '"+releaseQuery.getGa()+"' AND d.targetVersion = '"+releaseQuery.getVersion()+"' " + - "RETURN r"; - Driver driver = Neo4jDriverSingleton.getDriverInstance(); - try (Session session = driver.session()) { - Result result = session.run(query); - List> rows = new ArrayList<>(); - while (result.hasNext()) { - Record record = result.next(); - Map row = new HashMap<>(); - for (Pair pair : record.fields()) { - if (pair.value().hasType(TypeSystem.getDefault().NODE())){ - Map nodeMap = Neo4jUtils.generateNodeRow(pair, releaseQuery.getAddedValues()); - row.put("node", nodeMap); - } - } - rows.add(row); - } - return rows; - } - } -} diff --git a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/api/controllers/UtilsController.java b/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/api/controllers/UtilsController.java deleted file mode 100644 index c3c3331..0000000 --- a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/api/controllers/UtilsController.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.cifre.sap.su.neo4jEcosystemWeaver.api.controllers; - -import com.cifre.sap.su.neo4jEcosystemWeaver.weaver.addedValue.AddedValueEnum; -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.tags.Tag; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RestController; - - -@RestController -@Tag(name = "Utils") -public class UtilsController { - - @Operation( - description = "Get the list of existing added values on the Weaver", - summary = "Get existing added values" - ) - @GetMapping("/addedValues") - public AddedValueEnum[] getAddedValue() { - return AddedValueEnum.values(); - } -} diff --git a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/api/entities/CypherQuery.java b/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/api/entities/CypherQuery.java deleted file mode 100644 index 6b180e6..0000000 --- a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/api/entities/CypherQuery.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.cifre.sap.su.neo4jEcosystemWeaver.api.entities; - -import com.cifre.sap.su.neo4jEcosystemWeaver.weaver.addedValue.AddedValueEnum; - -import java.util.List; - -public class CypherQuery { - private String query; - private List addedValues; - - public String getQuery() { - return query; - } - - public void setQuery(String query) { - this.query = query; - } - - public List getAddedValues() { - return addedValues; - } - - public void setAddedValues(List addedValues) { - this.addedValues = addedValues; - } -} - diff --git a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/utils/ApiUtils.java b/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/utils/ApiUtils.java deleted file mode 100644 index 3b86fbd..0000000 --- a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/utils/ApiUtils.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.cifre.sap.su.neo4jEcosystemWeaver.utils; - -import org.json.simple.JSONObject; -import org.json.simple.parser.JSONParser; - -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.net.HttpURLConnection; -import java.net.URL; -import java.nio.charset.StandardCharsets; - -public class ApiUtils { - - public static JSONObject callWithData(String urlString, String method, String data){ - try { - URL url = new URL(urlString); - HttpURLConnection http = (HttpURLConnection) url.openConnection(); - http.setRequestMethod(method); - http.setDoOutput(true); - http.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); - - byte[] out = data.getBytes(StandardCharsets.UTF_8); - - OutputStream stream = http.getOutputStream(); - stream.write(out); - - if(http.getResponseCode() == 200){ - JSONParser jsonParser = new JSONParser(); - return (JSONObject)jsonParser.parse( - new InputStreamReader(http.getInputStream(), StandardCharsets.UTF_8)); - } - http.disconnect(); - } catch (IOException | org.json.simple.parser.ParseException e) { - System.out.println("Unable to connect to API:\n" + e); - } - return null; - } -} diff --git a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/utils/GraphUtils.java b/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/utils/GraphUtils.java deleted file mode 100644 index d711403..0000000 --- a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/utils/GraphUtils.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.cifre.sap.su.neo4jEcosystemWeaver.utils; - -import org.neo4j.driver.*; -import org.neo4j.driver.Record; - -import java.util.*; - -public class GraphUtils { - - public static Set getGavAndDependenciesSetWithTransitive(String gav, Set visitedGav){ - if (visitedGav.contains(gav)) { - return visitedGav; - } - visitedGav.add(gav); - // Query to get the dependencies of the given release without test dependencies - String query = "MATCH (r:Release)-[d:dependency]->(a:Artifact) " + - "WHERE r.id = '" + gav + "' AND NOT (d.scope = 'test') " + - "RETURN a.id AS artifactId, d.targetVersion AS targetVersion"; - Driver driver = Neo4jDriverSingleton.getDriverInstance(); - try (Session session = driver.session()) { - Result result = session.run(query); - while (result.hasNext()) { - Record record = result.next(); - String newGav = record.get("artifactId").toString().replaceAll("[\"]","") - +":" - +record.get("targetVersion").toString().replaceAll("[\"]",""); - getGavAndDependenciesSetWithTransitive(newGav, visitedGav); - } - } - return visitedGav; - } -} diff --git a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/utils/Neo4jDriverSingleton.java b/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/utils/Neo4jDriverSingleton.java deleted file mode 100644 index 6ef41c0..0000000 --- a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/utils/Neo4jDriverSingleton.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.cifre.sap.su.neo4jEcosystemWeaver.utils; - -import org.neo4j.driver.AuthTokens; -import org.neo4j.driver.Driver; -import org.neo4j.driver.GraphDatabase; - -public class Neo4jDriverSingleton { - - private static Driver driver; - - private Neo4jDriverSingleton() { - // private constructor to prevent instantiation - } - - public static Driver getDriverInstance() { - if (driver == null) { - synchronized (Neo4jDriverSingleton.class) { - if (driver == null) { - String uri = System.getProperty("neo4jUri"); - String user = System.getProperty("neo4jUser"); - String password = System.getProperty("neo4jPassword"); - driver = GraphDatabase.driver(uri, AuthTokens.basic(user, password)); - } - } - } - return driver; - } -} diff --git a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/utils/Neo4jUtils.java b/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/utils/Neo4jUtils.java deleted file mode 100644 index 6dbe1b3..0000000 --- a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/utils/Neo4jUtils.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.cifre.sap.su.neo4jEcosystemWeaver.utils; - -import com.cifre.sap.su.neo4jEcosystemWeaver.weaver.Weaver; -import com.cifre.sap.su.neo4jEcosystemWeaver.weaver.addedValue.AddedValueEnum; -import org.neo4j.driver.Value; -import org.neo4j.driver.types.Node; -import org.neo4j.driver.types.Path; -import org.neo4j.driver.types.Relationship; -import org.neo4j.driver.util.Pair; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class Neo4jUtils { - - public static Map generateNodeRow(Pair pair, List addedValues){ - Node node = pair.value().asNode(); - Map nodeMap = new HashMap<>(node.asMap()); - nodeMap.put("labels", node.labels()); - nodeMap.put("elementId", node.elementId()); - Weaver.fillNode(nodeMap, addedValues); - return nodeMap; - } - - public static Map generateRelationshipRow(Pair pair, List addedValues){ - Relationship relationship = pair.value().asRelationship(); - Map relationshipMap = new HashMap<>(relationship.asMap()); - relationshipMap.put("type", relationship.type()); - relationshipMap.put("sourceElementId", relationship.startNodeElementId()); - relationshipMap.put("targetElementId", relationship.endNodeElementId()); - Weaver.fillRelationship(relationshipMap, addedValues); - return relationshipMap; - } - - public static Map generatePathRow(Pair pair, List addedValues){ - Map row = new HashMap<>(); - Path path = pair.value().asPath(); - for (Node node : path.nodes()){ - Map nodeMap = new HashMap<>(node.asMap()); - nodeMap.put("labels", node.labels()); - nodeMap.put("elementId", node.elementId()); - Weaver.fillNode(nodeMap, addedValues); - row.put("node", nodeMap); - } - for (Relationship relationship : path.relationships()){ - Map relationshipMap = new HashMap<>(relationship.asMap()); - relationshipMap.put("type", relationship.type()); - relationshipMap.put("sourceElementId", relationship.startNodeElementId()); - relationshipMap.put("targetElementId", relationship.endNodeElementId()); - Weaver.fillRelationship(relationshipMap, addedValues); - row.put("relationship", relationshipMap); - } - return row; - } -} diff --git a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/weaver/Weaver.java b/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/weaver/Weaver.java deleted file mode 100644 index 632c8d7..0000000 --- a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/weaver/Weaver.java +++ /dev/null @@ -1,68 +0,0 @@ -package com.cifre.sap.su.neo4jEcosystemWeaver.weaver; - -import com.cifre.sap.su.neo4jEcosystemWeaver.weaver.addedValue.*; -import org.neo4j.driver.util.Pair; - -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -public class Weaver { - - public static void fillNode(Map nodeMap, List addedValues){ - String nodeId = (String) nodeMap.get("id"); - String nodeType = ( (List) nodeMap.get("labels")).get(0); - for(AddedValue addedValue : getNodeValuesToAdd(nodeId, nodeType, addedValues)){ - Pair value = addedValue.getValue(); - nodeMap.put(value.key(), value.value()); - } - } - - public static void fillRelationship(Map relationshipMap, List addedValues){ - String relationshipType = (String) relationshipMap.get("type"); - for(AddedValue addedValue : getRelationshipValuesToAdd(relationshipType, addedValues)){ - Pair value = addedValue.getValue(); - relationshipMap.put(value.key(), value.value()); - } - } - - private static Set getNodeValuesToAdd(String nodeId, String nodeType, List addedValues){ - Set set = new HashSet<>(); - // Release vertices - if(nodeType.equals("Release")){ - //Put release added values here - if(addedValues.contains(AddedValueEnum.CVE)){ - set.add(new Cve(nodeId)); - } - if(addedValues.contains(AddedValueEnum.CVE_AGGREGATED)){ - set.add(new CveAggregated(nodeId)); - } - if(addedValues.contains(AddedValueEnum.FRESHNESS)){ - set.add(new Freshness(nodeId)); - } - if(addedValues.contains(AddedValueEnum.FRESHNESS_AGGREGATED)){ - set.add(new FreshnessAggregated(nodeId)); - } - } - // Artifact vertices - if(nodeType.equals("Artifact")){ - //Put artifact added values here - if(addedValues.contains(AddedValueEnum.SPEED)){ - set.add(new Speed(nodeId)); - } - } - return set; - } - - private static Set getRelationshipValuesToAdd(String relationshipType, List addedValues){ - Set set = new HashSet<>(); - if(relationshipType.equals("dependency")){ - //Put dependency added values here - } - if(relationshipType.equals("relationship_AR")){ - //Put relationship_AR added values here - } - return set; - } -} diff --git a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/weaver/addedValue/AddedValue.java b/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/weaver/addedValue/AddedValue.java deleted file mode 100644 index 33c075a..0000000 --- a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/weaver/addedValue/AddedValue.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.cifre.sap.su.neo4jEcosystemWeaver.weaver.addedValue; - -import org.neo4j.driver.util.Pair; - -public interface AddedValue { - Pair getValue(); -} diff --git a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/weaver/addedValue/AddedValueEnum.java b/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/weaver/addedValue/AddedValueEnum.java deleted file mode 100644 index 3258017..0000000 --- a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/weaver/addedValue/AddedValueEnum.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.cifre.sap.su.neo4jEcosystemWeaver.weaver.addedValue; - -public enum AddedValueEnum { - CVE, - CVE_AGGREGATED, - FRESHNESS, - FRESHNESS_AGGREGATED, - SPEED -} diff --git a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/weaver/addedValue/Cve.java b/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/weaver/addedValue/Cve.java deleted file mode 100644 index d97f931..0000000 --- a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/weaver/addedValue/Cve.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.cifre.sap.su.neo4jEcosystemWeaver.weaver.addedValue; - -import com.cifre.sap.su.neo4jEcosystemWeaver.weaver.common.ReusedCalculation; -import org.neo4j.driver.util.Pair; - -import java.util.Map; -import java.util.Set; - -public class Cve implements AddedValue{ - private final String gav; - - public Cve(String gav) { - this.gav = gav; - } - - @Override - public Pair getValue() { - return new Pair<>() { - @Override - public String key() { - return "CVE"; - } - - @Override - public Set> value() { - return fillCve(); - } - }; - } - - private Set> fillCve(){ - return ReusedCalculation.getCveFromGav(gav); - } -} diff --git a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/weaver/addedValue/CveAggregated.java b/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/weaver/addedValue/CveAggregated.java deleted file mode 100644 index 2aaf83b..0000000 --- a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/weaver/addedValue/CveAggregated.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.cifre.sap.su.neo4jEcosystemWeaver.weaver.addedValue; - -import com.cifre.sap.su.neo4jEcosystemWeaver.utils.GraphUtils; -import com.cifre.sap.su.neo4jEcosystemWeaver.weaver.common.ReusedCalculation; -import org.neo4j.driver.util.Pair; - -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -public class CveAggregated implements AddedValue{ - private final String gav; - - public CveAggregated(String gav) { - this.gav = gav; - } - - @Override - public Pair getValue() { - return new Pair<>() { - @Override - public String key() { - return "CVE_aggregated"; - } - - @Override - public Set> value() { - return fillAggregatedCve(); - } - }; - } - - private Set> fillAggregatedCve(){ - Set> allCVE = new HashSet<>(); - Set gavToTreat = GraphUtils.getGavAndDependenciesSetWithTransitive(gav, new HashSet<>()); - for(String toTreat : gavToTreat){ - allCVE.addAll(ReusedCalculation.getCveFromGav(toTreat)); - } - return allCVE; - } -} diff --git a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/weaver/addedValue/Freshness.java b/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/weaver/addedValue/Freshness.java deleted file mode 100644 index 802d7d4..0000000 --- a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/weaver/addedValue/Freshness.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.cifre.sap.su.neo4jEcosystemWeaver.weaver.addedValue; - -import com.cifre.sap.su.neo4jEcosystemWeaver.weaver.common.ReusedCalculation; -import org.neo4j.driver.util.Pair; - -import java.util.Map; - -public class Freshness implements AddedValue{ - private final String gav; - - public Freshness(String gav) { - this.gav = gav; - } - - @Override - public Pair getValue() { - return new Pair<>() { - @Override - public String key() { - return "freshness"; - } - - @Override - public Map value() { - return fillFreshness(); - } - }; - } - - private Map fillFreshness() { - return ReusedCalculation.getFreshnessMapFromGav(gav); - } -} diff --git a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/weaver/addedValue/FreshnessAggregated.java b/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/weaver/addedValue/FreshnessAggregated.java deleted file mode 100644 index ca8e25e..0000000 --- a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/weaver/addedValue/FreshnessAggregated.java +++ /dev/null @@ -1,48 +0,0 @@ -package com.cifre.sap.su.neo4jEcosystemWeaver.weaver.addedValue; - -import com.cifre.sap.su.neo4jEcosystemWeaver.utils.GraphUtils; -import com.cifre.sap.su.neo4jEcosystemWeaver.weaver.common.ReusedCalculation; -import org.neo4j.driver.util.Pair; - -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -public class FreshnessAggregated implements AddedValue{ - private final String gav; - - public FreshnessAggregated(String gav) { - this.gav = gav; - } - - @Override - public Pair getValue() { - return new Pair<>() { - @Override - public String key() { - return "freshness_aggregated"; - } - - @Override - public Map value() { - return fillAggregatedFreshness(); - } - }; - } - - private Map fillAggregatedFreshness() { - Map aggregatedFreshnessMap = new HashMap<>(); - Set gavToTreat = GraphUtils.getGavAndDependenciesSetWithTransitive(gav, new HashSet<>()); - int totalNumberMissedRelease = 0; - long totalOutdatedTimeInMs = 0; - for(String toTreat : gavToTreat){ - Map freshnessToAdd = ReusedCalculation.getFreshnessMapFromGav(toTreat); - totalNumberMissedRelease += Integer.parseInt(freshnessToAdd.get("numberMissedRelease")); - totalOutdatedTimeInMs += Long.parseLong(freshnessToAdd.get("outdatedTimeInMs")); - } - aggregatedFreshnessMap.put("numberMissedRelease", Integer.toString(totalNumberMissedRelease)); - aggregatedFreshnessMap.put("outdatedTimeInMs", Long.toString(totalOutdatedTimeInMs)); - return aggregatedFreshnessMap; - } -} diff --git a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/weaver/addedValue/Speed.java b/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/weaver/addedValue/Speed.java deleted file mode 100644 index 4485e8d..0000000 --- a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/weaver/addedValue/Speed.java +++ /dev/null @@ -1,67 +0,0 @@ -package com.cifre.sap.su.neo4jEcosystemWeaver.weaver.addedValue; - -import com.cifre.sap.su.neo4jEcosystemWeaver.utils.Neo4jDriverSingleton; -import org.neo4j.driver.Driver; -import org.neo4j.driver.Record; -import org.neo4j.driver.Result; -import org.neo4j.driver.Session; -import org.neo4j.driver.Value; -import org.neo4j.driver.util.Pair; - -import java.time.Duration; -import java.time.Instant; -import java.util.*; - -public class Speed implements AddedValue { - private final String ga; - - public Speed(String ga) { - this.ga = ga; - } - - @Override - public Pair getValue() { - return new Pair<>() { - @Override - public String key() { - return "speed"; - } - - @Override - public Double value() { - return fillSpeed(); - } - }; - } - - private double fillSpeed(){ - String query = "MATCH (a:Artifact) -[e:relationship_AR]-> (r:Release) " + - "WHERE a.id = '"+ga+"' " + - "RETURN r.timestamp AS timestamp"; - // Get all artifact Releases timestamp - Driver driver = Neo4jDriverSingleton.getDriverInstance(); - TreeSet releasesTimeStampSet = new TreeSet<>(); - try (Session session = driver.session()) { - Result result = session.run(query); - while (result.hasNext()) { - Record record = result.next(); - for (Pair pair : record.fields()) { - releasesTimeStampSet.add(Long.parseLong(pair.value().toString())); - } - } - } - if(releasesTimeStampSet.size() < 2){ - return 0; - } - // Calculate average speed by days - Instant minTimestamp = Instant.ofEpochMilli(releasesTimeStampSet.first()); - Instant maxTimestamp = Instant.ofEpochMilli(releasesTimeStampSet.last()); - long durationDays = Duration.between(minTimestamp, maxTimestamp).toDays(); - if (durationDays == 0) { - return 0; - } else { - return (double) releasesTimeStampSet.size() / durationDays; - } - } - -} diff --git a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/weaver/common/ReusedCalculation.java b/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/weaver/common/ReusedCalculation.java deleted file mode 100644 index 7700bb5..0000000 --- a/src/main/java/com/cifre/sap/su/neo4jEcosystemWeaver/weaver/common/ReusedCalculation.java +++ /dev/null @@ -1,90 +0,0 @@ -package com.cifre.sap.su.neo4jEcosystemWeaver.weaver.common; - -import com.cifre.sap.su.neo4jEcosystemWeaver.utils.ApiUtils; -import com.cifre.sap.su.neo4jEcosystemWeaver.utils.Neo4jDriverSingleton; -import com.cifre.sap.su.neo4jEcosystemWeaver.utils.OsvDataSingleton; -import org.json.simple.JSONArray; -import org.json.simple.JSONObject; -import org.json.simple.parser.ParseException; -import org.neo4j.driver.*; -import org.neo4j.driver.Record; -import org.neo4j.driver.util.Pair; - -import java.io.IOException; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -public class ReusedCalculation { - public static Set> getCveFromGav(String gav){ - Set> resultSet = new HashSet<>(); - try { - JSONObject jsonObject = OsvDataSingleton.getDataJsonObject(); - JSONArray cveArray = (JSONArray) jsonObject.get(gav); - if (cveArray != null) { - for (Object vulnerability : cveArray) { - JSONObject vulnerabilityJsonObject = (JSONObject) vulnerability; - Map cveMap = new HashMap(); - cveMap.put("name", (String) vulnerabilityJsonObject.get("name")); - cveMap.put("cwe", (String) vulnerabilityJsonObject.get("cwe_ids")); - cveMap.put("severity", (String) vulnerabilityJsonObject.get("severity")); - resultSet.add(cveMap); - } - } - } catch (IOException | ParseException e) { - e.printStackTrace(); - } - return resultSet; - } - - public static Set> getCveFromApiWithGav(String gav){ - Set> resultSet = new HashSet<>(); - String[] splitedGav = gav.split(":"); - String packageName = splitedGav[0]+":"+splitedGav[1]; - String version = splitedGav[2]; - JSONObject jsonObject = ApiUtils.callWithData("https://api.osv.dev/v1/query", "POST", "{\"version\": \""+version+"\", \"package\": {\"name\": \""+packageName+"\", \"ecosystem\": \"Maven\"}}"); - if (jsonObject != null) { - JSONArray vulnerabilities = (JSONArray) jsonObject.get("vulns"); - if (vulnerabilities != null) { - for (Object vulnerability : vulnerabilities) { - JSONObject vulnerabilityJsonObject = (JSONObject) vulnerability; - String vulnerabilityName = vulnerabilityJsonObject.get("aliases") != null ? (vulnerabilityJsonObject.get("aliases").toString().split("\"]")[0].substring(2)) : ""; - JSONObject detailJsonObject = (JSONObject) vulnerabilityJsonObject.get("database_specific"); - String vulnerabilitySeverity = ""; - String vulnerabilityCwe = ""; - if(detailJsonObject != null){ - vulnerabilitySeverity = detailJsonObject.get("severity") != null ? detailJsonObject.get("severity").toString() : ""; - vulnerabilityCwe = detailJsonObject.get("cwe_ids") != null ? detailJsonObject.get("cwe_ids").toString() : ""; - } - Map cveMap = new HashMap(); - cveMap.put("name", vulnerabilityName.replaceAll("[\"]","")); - cveMap.put("cwe", vulnerabilityCwe.replaceAll("[\"]","")); - cveMap.put("severity", vulnerabilitySeverity.replaceAll("[\"]","")); - resultSet.add(cveMap); - } - } - } - return resultSet; - } - - public static Map getFreshnessMapFromGav(String gav){ - Map freshnessMap = new HashMap<>(); - String query = "MATCH (r1:Release)<-[:relationship_AR]-(:Artifact)-[:relationship_AR]->(r2:Release) " + - "WHERE r1.id = '"+gav+"' AND r2.timestamp > r1.timestamp " + - "WITH r2, r2.timestamp - r1.timestamp AS difference " + - "RETURN count(r2) AS numberMissedRelease, max(difference) AS outdatedTimeInMs"; - Driver driver = Neo4jDriverSingleton.getDriverInstance(); - try (Session session = driver.session()) { - Result result = session.run(query); - while (result.hasNext()) { - Record record = result.next(); - for (Pair pair : record.fields()) { - String value = pair.value().toString().equals("NULL") ? "0" : pair.value().toString(); - freshnessMap.put(pair.key(),value); - } - } - } - return freshnessMap; - } -} diff --git a/src/test/java/com/cifre/sap/su/neo4jEcosystemWeaver/Neo4jEcosystemWeaverApplicationTests.java b/src/test/java/com/cifre/sap/su/goblinWeaver/Neo4jEcosystemWeaverApplicationTests.java similarity index 80% rename from src/test/java/com/cifre/sap/su/neo4jEcosystemWeaver/Neo4jEcosystemWeaverApplicationTests.java rename to src/test/java/com/cifre/sap/su/goblinWeaver/Neo4jEcosystemWeaverApplicationTests.java index 3f771fe..50eb552 100644 --- a/src/test/java/com/cifre/sap/su/neo4jEcosystemWeaver/Neo4jEcosystemWeaverApplicationTests.java +++ b/src/test/java/com/cifre/sap/su/goblinWeaver/Neo4jEcosystemWeaverApplicationTests.java @@ -1,4 +1,4 @@ -package com.cifre.sap.su.neo4jEcosystemWeaver; +package com.cifre.sap.su.goblinWeaver; import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.SpringBootTest;