Template-based Query Generation: Generalization and User Configuration

tools/template_based_query_generation/src/main/java/QueryGenerator.java

@@ -1,22 +1,17 @@
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.gson.Gson;
+import data.Table;
 import graph.MarkovChain;
 import graph.Node;
-import parser.*;
+import parser.FeatureType;
+import parser.User;
+import parser.Utils;
 import query.Query;
 import query.Skeleton;
 import token.Tokenizer;
 
-import java.io.BufferedReader;
 import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.*;
 
-import static java.nio.charset.StandardCharsets.UTF_8;
-
 /**
  * Class that parses config file and creates queries from markov chain
  */
@@ -29,40 +24,39 @@ public class QueryGenerator {
   private final String filePathDependenciesDDL = "./src/main/resources/dialect_config/ddl_dependencies.json";
   private final String filePathDependenciesDML = "./src/main/resources/dialect_config/dml_dependencies.json";
   private final String filePathDependenciesDQL = "./src/main/resources/dialect_config/dql_dependencies.json";
+  private final String filePathUser = "./src/main/resources/user_config/config.json";
 
   private final MarkovChain<Query> markovChain;
   private Random r = new Random();
-  private Node<Query> source = new Node<>(new Query(FeatureType.FEATURE_ROOT), r);
+  private final User user = Utils.getUser(Paths.get(filePathUser));
+  private Node<Query> source = new Node<>(new Query(user.getStartFeature()), r);
+  private Node<Query> sink = new Node<>(new Query(user.getEndFeature()), r);
 
   /**
-   *
-   * @throws Exception
+   * Query generator that converts query skeletons to real query strings ready for output
+   * @throws IOException if the IO for user parsing fails
    */
-  public QueryGenerator() throws Exception {
-    // TODO (Victor):
-    //  1. Use parser.Utils to parse user json and create graph.MarkovChain and nodes
-    //  2. Generate number of queries given in config
-    //  3. pass to them to Keyword or query.Skeleton
-
-    // create nodes
-    Map<String, Node<Query>> nodeMap = new HashMap<>();
-    addNodeMap(nodeMap, Paths.get(filePathConfigDDL), r);
-    addNodeMap(nodeMap, Paths.get(filePathConfigDML), r);
-    addNodeMap(nodeMap, Paths.get(filePathConfigDQL), r);
-
-    // TODO (Victor): Parse these two helper nodes from user config
-    nodeMap.put("FEATURE_ROOT", source);
-    nodeMap.put("FEATURE_SINK", new Node<>(new Query(FeatureType.FEATURE_SINK), r));
-
-    Map<String, List<String>> neighborMap = new HashMap<>();
-    addNeighborMap(neighborMap, nodeMap.keySet(), Paths.get(filePathDependenciesDDL));
-    addNeighborMap(neighborMap, nodeMap.keySet(), Paths.get(filePathDependenciesDML));
-    addNeighborMap(neighborMap, nodeMap.keySet(), Paths.get(filePathDependenciesDQL));
-    addNeighborMap(neighborMap, nodeMap.keySet(), Paths.get(filePathDependenciesRoot));
-
-    for (String nodeKey : nodeMap.keySet()) {
+  public QueryGenerator() throws IOException {
+
+    // create map of references to nodes
+    Map<FeatureType, Node<Query>> nodeMap = new HashMap<>();
+    Utils.addNodeMap(nodeMap, Paths.get(filePathConfigDDL), r);
+    Utils.addNodeMap(nodeMap, Paths.get(filePathConfigDML), r);
+    Utils.addNodeMap(nodeMap, Paths.get(filePathConfigDQL), r);
+    nodeMap.put(user.getStartFeature(), source);
+    nodeMap.put(user.getEndFeature(), sink);
+
+    // create map of nodes to their neighbors
+    Map<FeatureType, List<FeatureType>> neighborMap = new HashMap<>();
+    Utils.addNeighborMap(neighborMap, nodeMap.keySet(), Paths.get(filePathDependenciesDDL));
+    Utils.addNeighborMap(neighborMap, nodeMap.keySet(), Paths.get(filePathDependenciesDML));
+    Utils.addNeighborMap(neighborMap, nodeMap.keySet(), Paths.get(filePathDependenciesDQL));
+    Utils.addNeighborMap(neighborMap, nodeMap.keySet(), Paths.get(filePathDependenciesRoot));
+
+    // set neighbors for each node
+    for (FeatureType nodeKey : nodeMap.keySet()) {
       HashSet<Node<Query>> nodeNeighbors = new HashSet<>();
-      for (String neighbor : neighborMap.get(nodeKey)) {
+      for (FeatureType neighbor : neighborMap.get(nodeKey)) {
         if (nodeMap.keySet().contains(neighbor)) {
           nodeNeighbors.add(nodeMap.get(neighbor));
         }
@@ -74,76 +68,40 @@ public QueryGenerator() throws Exception {
   }
 
   /**
-   * generates queries from markov chain starting from root
+   * @return real queries from markov chain starting from root
    */
-  public void generateQueries(int numberQueries) {
-    ImmutableList.Builder<String> postgreBuilder = ImmutableList.builder();
-    ImmutableList.Builder<String> bigQueryBuilder = ImmutableList.builder();
+  public void generateQueries() throws IOException {
+    Map<String, List<String>> dialectQueries = new HashMap<>();
+
+    for (String dialect : user.getDialectIndicators().keySet()) {
+      if (user.getDialectIndicators().get(dialect)) {
+        dialectQueries.put(dialect, new ArrayList<>());
+      }
+    }
+
     Tokenizer tokenizer = new Tokenizer(r);
 
     int i = 0;
-    while (i < numberQueries) {
+    while (i < user.getNumQueries()) {
       List<Query> rawQueries = markovChain.randomWalk(source);
-
       if (rawQueries.get(rawQueries.size()-1).getType() == FeatureType.FEATURE_SINK) {
-        List<Query> actualQueries = rawQueries.subList(2, rawQueries.size()-1);
+        List<Query> actualQueries = rawQueries.subList(2, rawQueries.size() - 1);
         Skeleton skeleton = new Skeleton(actualQueries, tokenizer);
-        postgreBuilder.add(String.join(" ", skeleton.getPostgreSkeleton()));
-        bigQueryBuilder.add(String.join(" ", skeleton.getBigQuerySkeleton()));
-        bigQueryBuilder.add(";");
-        i++;
-      }
-    }
-
-    ImmutableList<String> postgreSyntax = postgreBuilder.build();
-    ImmutableList<String> bigQuerySyntax = bigQueryBuilder.build();
-
-    ImmutableMap.Builder<String, ImmutableList<String>> builder = ImmutableMap.builder();
-    builder.put("PostgreSQL", postgreSyntax);
-    builder.put("BigQuery", bigQuerySyntax);
-    ImmutableMap<String, ImmutableList<String>> outputs = builder.build();
-
-    try {
-      Utils.writeDirectory(outputs);
-    } catch (IOException exception){
-      exception.printStackTrace();
-    }
-  }
-
-  private Map<String, Node<Query>> addNodeMap(Map<String, Node<Query>> nodeMap, Path input, Random r) {
-    try {
-      BufferedReader reader = Files.newBufferedReader(input, UTF_8);
-      Gson gson = new Gson();
-      FeatureIndicators featureIndicators = gson.fromJson(reader, FeatureIndicators.class);
-
-      for (FeatureIndicator featureIndicator : featureIndicators.getFeatureIndicators()) {
-        if (featureIndicator.getIsIncluded()) {
-          nodeMap.put(featureIndicator.getFeature().name(), new Node<>(new Query(featureIndicator.getFeature()), r));
+        for (String dialect : user.getDialectIndicators().keySet()) {
+          if (user.getDialectIndicators().get(dialect)) {
+            dialectQueries.get(dialect).add(String.join(" ", skeleton.getDialectSkeletons().get(dialect)) + ";");
+          }
         }
       }
-    } catch (IOException exception) {
-      exception.printStackTrace();
+      i++;
     }
 
-    return nodeMap;
-  }
+    Table dataTable = tokenizer.getTable();
 
-  private Map<String, List<String>> addNeighborMap(Map<String, List<String>> neighborMap, Set<String> nodes, Path input) {
     try {
-      BufferedReader reader = Files.newBufferedReader(input, UTF_8);
-      Gson gson = new Gson();
-      Dependencies dependencies = gson.fromJson(reader, Dependencies.class);
-
-      for (Dependency dependency : dependencies.getDependencies()) {
-        if (nodes.contains(dependency.getNode())) {
-          neighborMap.put(dependency.getNode(), dependency.getNeighbors());
-        }
-      }
-    } catch (IOException exception) {
+      Utils.writeDirectory(dialectQueries, dataTable);
+    } catch (IOException exception){
       exception.printStackTrace();
     }
-
-    return neighborMap;
   }
-
 }

tools/template_based_query_generation/src/main/java/data/Table.java

@@ -5,6 +5,7 @@
 
 import java.math.BigDecimal;
 import java.util.ArrayList;
+import java.util.List;
 
 
 /**
@@ -24,7 +25,7 @@ public class Table {
   public Table(String name) {
     this.name = name;
     this.numRows = 0;
-    this.schema = new ArrayList<MutablePair<String, DataType>>();
+    this.schema = new ArrayList<>();
   }
 
   /**
@@ -36,7 +37,7 @@ public void addColumn(String columnName, DataType type) {
     this.schema.add(new MutablePair(columnName, type));
   }
 
-  public ArrayList<MutablePair<String, DataType>> getSchema() {
+  public List<MutablePair<String, DataType>> getSchema() {
     return this.schema;
   }
 
@@ -71,7 +72,7 @@ public String getRandomColumn() {
    * @return name of random column of given type
    */
   public String getRandomColumn(DataType type) {
-    ArrayList<MutablePair<String, DataType>> columns = new ArrayList<MutablePair<String, DataType>>();
+    List<MutablePair<String, DataType>> columns = new ArrayList<>();
     for (MutablePair<String, DataType> col: this.schema) {
       if (col.getRight() == type) columns.add(col);
     }
@@ -86,39 +87,39 @@ public String getRandomColumn(DataType type) {
    * @return column of data with type dataType and numRows rows
    * @throws IllegalArgumentException
    */
-  public ArrayList<?> generateColumn(int numRows, DataType dataType) throws IllegalArgumentException {
+  public List<?> generateColumn(int numRows, DataType dataType) throws IllegalArgumentException {
     if (dataType.isIntegerType()) {
-      ArrayList<Integer> data = new ArrayList<Integer>();
+      List<Integer> data = new ArrayList<>();
       for (int i = 0; i < numRows; i++) {
         data.add(Utils.generateRandomIntegerData(dataType));
       }
       return data;
     } else if (dataType.isLongType()) {
-      ArrayList<Long> data = new ArrayList<Long>();
+      List<Long> data = new ArrayList<>();
       for (int i = 0; i < numRows; i++) {
         data.add(Utils.generateRandomLongData(dataType));
       }
       return data;
     } else if (dataType.isDoubleType()) {
-      ArrayList<Double> data = new ArrayList<Double>();
+      List<Double> data = new ArrayList<>();
       for (int i = 0; i < numRows; i++) {
         data.add(Utils.generateRandomDoubleData(dataType));
       }
       return data;
     } else if (dataType.isBigDecimalType()) {
-      ArrayList<BigDecimal> data = new ArrayList<BigDecimal>();
+      List<BigDecimal> data = new ArrayList<>();
       for (int i = 0; i < numRows; i++) {
         data.add(Utils.generateRandomBigDecimalData(dataType));
       }
       return data;
     } else if (dataType.isStringType()) {
-      ArrayList<String> data = new ArrayList<String>();
+      List<String> data = new ArrayList<>();
       for (int i = 0; i < numRows; i++) {
         data.add(Utils.generateRandomStringData(dataType));
       }
       return data;
     } else if (dataType.isBooleanType()) {
-      ArrayList<Boolean> data = new ArrayList<Boolean>();
+      List<Boolean> data = new ArrayList<>();
       for (int i = 0; i < numRows; i++) {
         data.add(Utils.generateRandomBooleanData(dataType));
       }
@@ -132,7 +133,7 @@ public ArrayList<?> generateColumn(int numRows, DataType dataType) throws Illega
    *
    * @return sample data with number of rows being number of rows in table
    */
-  public ArrayList<ArrayList<?>> generateData() {
+  public List<List<?>> generateData() {
     return generateData(this.numRows);
   }
 
@@ -142,10 +143,10 @@ public ArrayList<ArrayList<?>> generateData() {
    * @param numRows number of rows to generate
    * @return sample data with number of rows being numRows
    */
-  public ArrayList<ArrayList<?>> generateData(int numRows) {
-    ArrayList<ArrayList<?>> data = new ArrayList<ArrayList<?>>();
+  public List<List<?>> generateData(int numRows) {
+    List<List<?>> data = new ArrayList<>();
     for (int i = 0; i < this.schema.size(); i++) {
-      ArrayList<?> column = this.generateColumn(numRows, this.schema.get(i).getRight());
+      List<?> column = this.generateColumn(numRows, this.schema.get(i).getRight());
       data.add(column);
     }
     return data;

tools/template_based_query_generation/src/main/java/graph/MarkovChain.java

@@ -17,7 +17,7 @@ public class MarkovChain<E> {
    * @param nodes
    */
   public MarkovChain(HashMap<Node<E>, HashMap<Node<E>, Double>> nodes) {
-    this.nodes = new HashSet<Node<E>>();
+    this.nodes = new HashSet<>();
     this.nodes.addAll(nodes.keySet());
     for (Node<E> n : nodes.keySet()) {
       n.setNeighbors(nodes.get(n));
@@ -38,7 +38,7 @@ public MarkovChain(HashSet<Node<E>> nodes) {
    * @return list of nodes for a random walk from start node
    */
   public ArrayList<E> randomWalk(Node<E> start) {
-    ArrayList<E> walk = new ArrayList<E>();
+    ArrayList<E> walk = new ArrayList<>();
     Node<E> current = start;
     while (current.hasNextNode()) {
       walk.add(current.getObj());

tools/template_based_query_generation/src/main/java/graph/Node.java

@@ -21,7 +21,7 @@ public class Node<E> {
 	public Node(E obj, int seed) {
 		this.obj = obj;
 		this.r = new Random(seed);
-		this.setNeighbors(new HashMap<Node<E>, Double>());
+		this.setNeighbors(new HashMap<>());
 	}
 
 	/**
@@ -31,14 +31,14 @@ public Node(E obj, int seed) {
 	public Node(E obj, Random r) {
 		this.obj = obj;
 		this.r = r;
-		this.setNeighbors(new HashMap<Node<E>, Double>());
+		this.setNeighbors(new HashMap<>());
 	}
 
 	/**
 	 * updates neighborList and cProbabilities when neighbors is changed
 	 */
 	private void updateProbabilities(HashMap<Node<E>, Double> neighbors) {
-		TreeMap<Double, Node<E>> newCumulativeProbabilities = new TreeMap<Double, Node<E>>();
+		TreeMap<Double, Node<E>> newCumulativeProbabilities = new TreeMap<>();
 		if (neighbors.size() != 0) {
 			double total = 0;
 			for (Node<E> n: neighbors.keySet()) {
@@ -90,7 +90,7 @@ public void setNeighbors(HashMap<Node<E>, Double> neighbors) {
 	 * @param neighbors
 	 */
 	public void setNeighbors(HashSet<Node<E>> neighbors) {
-		HashMap<Node<E>, Double> edges = new HashMap<Node<E>, Double>();
+		HashMap<Node<E>, Double> edges = new HashMap<>();
 		double c = (neighbors.size() == 0) ? 0 : 1.0/neighbors.size();
 		for (Node<E> n: neighbors) {
 			edges.put(n, c);
@@ -102,4 +102,4 @@ public TreeMap<Double, Node<E>> getCumulativeProbabilities() {
 		return this.cumulativeProbabilities;
 	}
 
-}
+}

tools/template_based_query_generation/src/main/java/parser/Dependency.java

@@ -8,24 +8,24 @@
 public class Dependency {
 
   /* the current node */
-  private String node;
+  private FeatureType node;
 
   /* the possible neighbors to the current node */
-  private List<String> neighbors;
+  private List<FeatureType> neighbors;
 
-  public String getNode() {
+  public FeatureType getNode() {
     return this.node;
   }
 
-  public void setNode(String node) {
+  public void setNode(FeatureType node) {
     this.node = node;
   }
 
-  public List<String> getNeighbors() {
+  public List<FeatureType> getNeighbors() {
     return this.neighbors;
   }
 
-  public void setNeighbors(List<String> neighbors) {
+  public void setNeighbors(List<FeatureType> neighbors) {
     this.neighbors = neighbors;
   }
 }