Sync parser to latest Emulator DDL

Add handling for If Not Exists on tables and indexes

src/main/java/com/google/cloud/solutions/spannerddl/diff/DdlDiff.java

@@ -255,7 +255,7 @@ public List<String> generateDifferenceStatements(Map<String, Boolean> options)
         newTablesCreationOrder.entrySet()) {
       if (tableDifferences.entriesOnlyOnRight().containsKey(newTableEntry.getKey())) {
         LOG.info("Creating new table: {}", newTableEntry.getKey());
-        output.add(newTableEntry.getValue().toString());
+        output.add(newTableEntry.getValue().toStringOptionalExistClause(false));
       }
     }
 
@@ -278,14 +278,14 @@ public List<String> generateDifferenceStatements(Map<String, Boolean> options)
     // Create new indexes
     for (ASTcreate_index_statement index : indexDifferences.entriesOnlyOnRight().values()) {
       LOG.info("Creating new index: {}", index.getIndexName());
-      output.add(index.toString());
+      output.add(index.toStringOptionalExistClause(false));
     }
 
     // Re-create modified indexes...
     for (ValueDifference<ASTcreate_index_statement> difference :
         indexDifferences.entriesDiffering().values()) {
       LOG.info("Re-creating changed index: {}", difference.leftValue().getIndexName());
-      output.add(difference.rightValue().toString());
+      output.add(difference.rightValue().toStringOptionalExistClause(false));
     }
 
     // Create new constraints.
@@ -901,11 +901,11 @@ private static void printHelpAndExit(int exitStatus) {
                   + "By default, changes to indexes will also cause a failure. The"
                   + " --"
                   + ALLOW_RECREATE_INDEXES_OPT
-                  + " command line option enables index changes by"
-                  + " generating statements to drop and recreate the index which will trigger a"
-                  + " long running operation to rebuild the index.\n\n"
-                  + "By default, changes to Check and Foreign key constraints will also cause a failure. The"
-                  + " --"
+                  + " command line option enables index changes by generating statements to drop"
+                  + " and recreate the index which will trigger a long running operation to rebuild"
+                  + " the index.\n\n"
+                  + "By default, changes to Check and Foreign key constraints will also cause a"
+                  + " failure. The --"
                   + ALLOW_RECREATE_CONSTRAINTS_OPT
                   + " command line option enables check and foreign key constraint changes by"
                   + " generating statements to drop and recreate the constraint. These are disabled"

src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTcolumns.java

@@ -32,10 +32,10 @@ public ASTcolumns(DdlParser p, int id) {
 
   @Override
   public String toString() {
-    return "("
+    return "( "
         + Joiner.on(", ")
             .skipNulls()
             .join(ASTTreeUtils.getChildrenAssertType(children, ASTkey_part.class))
-        + ")";
+        + " )";
   }
 }

src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTcreate_index_statement.java

@@ -39,6 +39,15 @@ public String getIndexName() {
 
   @Override
   public String toString() {
+    return toStringOptionalExistClause(true);
+  }
+
+  /**
+   * Create string version, optionally including the IF NOT EXISTS clause
+   *
+   * @param includeExists
+   */
+  public String toStringOptionalExistClause(boolean includeExists) {
     validateChildren();
     ASTindex_interleave_clause interleave =
         getOptionalChildByType(children, ASTindex_interleave_clause.class);
@@ -50,7 +59,7 @@ public String toString() {
             getOptionalChildByType(children, ASTunique_index.class),
             getOptionalChildByType(children, ASTnull_filtered.class),
             "INDEX",
-            getOptionalChildByType(children, ASTif_not_exists.class),
+            (includeExists ? getOptionalChildByType(children, ASTif_not_exists.class) : null),
             getIndexName(),
             "ON",
             getChildByType(children, ASTtable.class),
@@ -86,8 +95,9 @@ public int compareTo(ASTcreate_index_statement other) {
   @Override
   public boolean equals(Object other) {
     if (other instanceof ASTcreate_index_statement) {
-      // lazy: compare normalised text rendering.
-      return this.toString().equals(other.toString());
+      // lazy: compare text rendering, but don't take into account IF NOT EXISTS statements
+      return this.toStringOptionalExistClause(false)
+          .equals(((ASTcreate_index_statement) other).toStringOptionalExistClause(false));
     }
     return false;
   }

src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTcreate_table_statement.java

@@ -91,6 +91,15 @@ public ASTcreate_table_statement clearConstraints() {
 
   @Override
   public String toString() {
+    return toStringOptionalExistClause(true);
+  }
+
+  /**
+   * Create string version, optionally including the IF NOT EXISTS clause
+   *
+   * @param includeExists
+   */
+  public String toStringOptionalExistClause(boolean includeExists) {
     verifyTableElements();
 
     List<String> tabledef = new ArrayList<>();
@@ -105,10 +114,12 @@ public String toString() {
         .skipNulls()
         .join(
             "CREATE TABLE",
-            Objects.toString(getOptionalChildByType(children, ASTif_not_exists.class), null),
+            (includeExists
+                ? Objects.toString(getOptionalChildByType(children, ASTif_not_exists.class), null)
+                : null),
             getTableName(),
             // add cols and constraints
-            "(" + Joiner.on(", ").skipNulls().join(tabledef) + ")",
+            "( " + Joiner.on(", ").skipNulls().join(tabledef) + " )",
             // add table suffixes, separated by ","
             Joiner.on(", ")
                 .skipNulls()
@@ -139,8 +150,9 @@ private void verifyTableElements() {
   @Override
   public boolean equals(Object other) {
     if (other instanceof ASTcreate_table_statement) {
-      // lazy: compare text rendering.
-      return this.toString().equals(other.toString());
+      // lazy: compare text rendering, but don't take into account IF NOT EXISTS statements
+      return this.toStringOptionalExistClause(false)
+          .equals(((ASTcreate_table_statement) other).toStringOptionalExistClause(false));
     }
     return false;
   }

src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTforeign_key.java

@@ -17,6 +17,10 @@
 package com.google.cloud.solutions.spannerddl.parser;
 
 import com.google.cloud.solutions.spannerddl.diff.ASTTreeUtils;
+import com.google.common.base.Joiner;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
 
 public class ASTforeign_key extends SimpleNode {
 
@@ -51,7 +55,8 @@ private List<String> identifierListToStringList(ASTidentifier_list idList) {
   public String getReferencedTableName() {
     int child = 1;
     if (children[0] instanceof ASTconstraint_name) {
-      token = token.next;
+      // skip constraint name when specified.
+      child++;
     }
     return ASTTreeUtils.tokensToString((ASTreferenced_table) children[child], false);
   }
@@ -70,20 +75,20 @@ public String getDeleteOption() {
     if (deleteOption != null) {
       return " " + deleteOption;
     } else {
-      return "";
+      return " ON DELETE NO ACTION";
     }
   }
 
   public String toString() {
     return "CONSTRAINT "
         + getName()
-        + " FOREIGN KEY ("
+        + " FOREIGN KEY ( "
         + Joiner.on(", ").join(getConstrainedColumnNames())
-        + ") REFERENCES "
+        + " ) REFERENCES "
         + getReferencedTableName()
-        + " ("
+        + " ( "
         + Joiner.on(", ").join(getReferencedColumnNames())
-        + ")"
+        + " )"
         + getDeleteOption();
   }
 

src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTstored_column_list.java

@@ -32,8 +32,8 @@ public ASTstored_column_list(DdlParser p, int id) {
 
   @Override
   public String toString() {
-    return "STORING ("
+    return "STORING ( "
         + Joiner.on(", ").join(ASTTreeUtils.getChildrenAssertType(children, ASTstored_column.class))
-        + ")";
+        + " )";
   }
 }

src/main/jjtree-sources/ddl_expression.jjt

@@ -36,6 +36,7 @@ void googlesql_operator() :
   | ">="
   | "~"
   | "|"
+  | "||"
   | "^"
   | "&"
 }

src/test/java/com/google/cloud/solutions/spannerddl/diff/DdlDiffFromFilesTest.java

@@ -40,7 +40,7 @@ public void compareDddTextFiles() throws IOException {
 
     String segmentName = null;
     try {
-      while (originalSegmentIt.hasNext()) {
+      while (originalSegmentIt.hasNext() && newSegmentIt.hasNext() && expectedOutputIt.hasNext()) {
         Map.Entry<String, String> originalSegment = originalSegmentIt.next();
         segmentName = originalSegment.getKey();
         Map.Entry<String, String> newSegment = newSegmentIt.next();
@@ -108,9 +108,26 @@ public void compareDddTextFiles() throws IOException {
             .that(diff)
             .isEqualTo(expectedDiffNoDrops);
       }
-    } catch (Throwable e) {
-      e.printStackTrace(System.err);
-      fail("Unexpected exception when processing segment " + segmentName + ": " + e);
+    } catch (DdlDiffException e) {
+      fail("DdlDiffException when processing segment " + segmentName + ": " + e);
+    } catch (Exception e) {
+      throw new Error("Unexpected exception when processing segment " + segmentName + ": " + e, e);
+    }
+
+    if (originalSegmentIt.hasNext()) {
+      throw new Error(
+          "Mismatched number of segments: Others have finished, but Original still has "
+              + originalSegmentIt.next().getKey());
+    }
+    if (newSegmentIt.hasNext()) {
+      throw new Error(
+          "Mismatched number of segments: Others have finished, but New still has "
+              + newSegmentIt.next().getKey());
+    }
+    if (expectedOutputIt.hasNext()) {
+      throw new Error(
+          "Mismatched number of segments: Others have finished, but Expected still has "
+              + expectedOutputIt.next().getKey());
     }
   }
 }

src/test/java/com/google/cloud/solutions/spannerddl/diff/DdlDiffTest.java

@@ -50,14 +50,14 @@ public void parseMultiDdlStatements() throws DdlDiffException {
     assertThat(result).hasSize(4);
 
     assertThat(result.get(0).toString())
-        .isEqualTo("CREATE TABLE test1 (col1 INT64) PRIMARY KEY (col1 ASC)");
+        .isEqualTo("CREATE TABLE test1 ( col1 INT64 ) PRIMARY KEY (col1 ASC)");
     assertThat(result.get(1).toString())
-        .isEqualTo("CREATE TABLE test2 (col2 FLOAT64) PRIMARY KEY (col2 ASC)");
+        .isEqualTo("CREATE TABLE test2 ( col2 FLOAT64 ) PRIMARY KEY (col2 ASC)");
     assertThat(result.get(2).toString())
         .isEqualTo(
-            "CREATE TABLE test3 (col3 STRING(MAX)) PRIMARY KEY (col3 ASC), "
+            "CREATE TABLE test3 ( col3 STRING(MAX) ) PRIMARY KEY (col3 ASC), "
                 + "INTERLEAVE IN PARENT testparent ON DELETE NO ACTION");
-    assertThat(result.get(3).toString()).isEqualTo("CREATE INDEX index1 ON table1 (col1 ASC)");
+    assertThat(result.get(3).toString()).isEqualTo("CREATE INDEX index1 ON table1 ( col1 ASC )");
   }
 
   @Test
@@ -420,6 +420,26 @@ public void parseAlterDatabaseDifferentDbNames() throws ParseException {
         "Multiple database IDs defined");
   }
 
+  @Test
+  public void diffCreateTableDifferentExists() throws DdlDiffException {
+    assertThat(
+            getDiff(
+                "create table test1 ( col1 int64 ) primary key (col1);",
+                "create table if not exists test1 ( col1 int64 ) primary key (col1);",
+                true))
+        .isEmpty();
+  }
+
+  @Test
+  public void diffCreateIndexDifferentExists() throws DdlDiffException {
+    assertThat(
+            getDiff(
+                "CREATE INDEX myindex ON mytable ( col1 );",
+                "CREATE INDEX IF NOT EXISTS myindex ON mytable ( col1 );",
+                true))
+        .isEmpty();
+  }
+
   private static void getDiffCheckDdlDiffException(
       String originalDdl, String newDdl, boolean allowDropStatements, String exceptionContains) {
     try {
@@ -514,21 +534,22 @@ public void differentIndexesWithRecreate() throws DdlDiffException {
         .isEqualTo(
             Arrays.asList(
                 "DROP INDEX index1",
-                "CREATE UNIQUE NULL_FILTERED INDEX index1 ON table1 (col1 ASC)"));
+                "CREATE UNIQUE NULL_FILTERED INDEX index1 ON table1 ( col1 ASC )"));
     assertThat(
             DdlDiff.build(
-                    "Create unique null_filtered index index1 on table1 (col1 desc)",
-                    "Create unique null_filtered index index1 on table1 (col2 desc)")
+                    "Create unique null_filtered index index1 on table1 ( col1 desc )",
+                    "Create unique null_filtered index index1 on table1 ( col2 desc )")
                 .generateDifferenceStatements(options))
         .isEqualTo(
             Arrays.asList(
                 "DROP INDEX index1",
-                "CREATE UNIQUE NULL_FILTERED INDEX index1 ON table1 (col2 DESC)"));
+                "CREATE UNIQUE NULL_FILTERED INDEX index1 ON table1 ( col2 DESC )"));
     assertThat(
             DdlDiff.build(
-                    "Create unique null_filtered index index1 on table1 (col1 desc)",
-                    "Create index index1 on table1 (col1 desc)")
+                    "Create unique null_filtered index index1 on table1 ( col1 desc )",
+                    "Create index index1 on table1 ( col1 desc )")
                 .generateDifferenceStatements(options))
-        .isEqualTo(Arrays.asList("DROP INDEX index1", "CREATE INDEX index1 ON table1 (col1 DESC)"));
+        .isEqualTo(
+            Arrays.asList("DROP INDEX index1", "CREATE INDEX index1 ON table1 ( col1 DESC )"));
   }
 }

src/test/java/com/google/cloud/solutions/spannerddl/parser/DDLParserFromFileTest.java

@@ -34,7 +34,7 @@ public void validateDDLfromFile() throws IOException {
         parser.ddl_statement();
         ASTddl_statement parsedStatement = (ASTddl_statement) parser.jjtree.rootNode();
 
-        assertWithMessage("Mismatch for section " + segmentName)
+        assertWithMessage("Mismatch for section " + segmentName + ":")
             .that(parsedStatement.toString())
             .isEqualTo(ddlStatement);
       } catch (ParseException e) {