Sync ddl parser code to latest Emulator DDL, add IF NOT EXISTS handli…

…ng (#61)

* Work in progress - adding new features

* Sync parser to latest Emulator DDL

Add handling for If Not Exists on tables and indexes

src/main/java/com/google/cloud/solutions/spannerddl/diff/DdlDiff.java

@@ -255,7 +255,7 @@ public List<String> generateDifferenceStatements(Map<String, Boolean> options)
         newTablesCreationOrder.entrySet()) {
       if (tableDifferences.entriesOnlyOnRight().containsKey(newTableEntry.getKey())) {
         LOG.info("Creating new table: {}", newTableEntry.getKey());
-        output.add(newTableEntry.getValue().toString());
+        output.add(newTableEntry.getValue().toStringOptionalExistClause(false));
       }
     }
 
@@ -278,14 +278,14 @@ public List<String> generateDifferenceStatements(Map<String, Boolean> options)
     // Create new indexes
     for (ASTcreate_index_statement index : indexDifferences.entriesOnlyOnRight().values()) {
       LOG.info("Creating new index: {}", index.getIndexName());
-      output.add(index.toString());
+      output.add(index.toStringOptionalExistClause(false));
     }
 
     // Re-create modified indexes...
     for (ValueDifference<ASTcreate_index_statement> difference :
         indexDifferences.entriesDiffering().values()) {
       LOG.info("Re-creating changed index: {}", difference.leftValue().getIndexName());
-      output.add(difference.rightValue().toString());
+      output.add(difference.rightValue().toStringOptionalExistClause(false));
     }
 
     // Create new constraints.
@@ -901,11 +901,11 @@ private static void printHelpAndExit(int exitStatus) {
                   + "By default, changes to indexes will also cause a failure. The"
                   + " --"
                   + ALLOW_RECREATE_INDEXES_OPT
-                  + " command line option enables index changes by"
-                  + " generating statements to drop and recreate the index which will trigger a"
-                  + " long running operation to rebuild the index.\n\n"
-                  + "By default, changes to Check and Foreign key constraints will also cause a failure. The"
-                  + " --"
+                  + " command line option enables index changes by generating statements to drop"
+                  + " and recreate the index which will trigger a long running operation to rebuild"
+                  + " the index.\n\n"
+                  + "By default, changes to Check and Foreign key constraints will also cause a"
+                  + " failure. The --"
                   + ALLOW_RECREATE_CONSTRAINTS_OPT
                   + " command line option enables check and foreign key constraint changes by"
                   + " generating statements to drop and recreate the constraint. These are disabled"

src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTalter_database_statement.java

@@ -31,7 +31,7 @@ public ASToptions_clause getOptionsClause() {
   }
 
   public String getDbName() {
-    return ASTTreeUtils.tokensToString((ASTdatabase_name) jjtGetChild(0));
+    return ASTTreeUtils.tokensToString((ASTdb_name) jjtGetChild(0));
   }
 
   @Override

src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTalter_index_statement.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.solutions.spannerddl.parser;
+
+public class ASTalter_index_statement extends SimpleNode {
+  public ASTalter_index_statement(int id) {
+    super(id);
+    throw new UnsupportedOperationException("Not Implemented");
+  }
+
+  public ASTalter_index_statement(DdlParser p, int id) {
+    super(p, id);
+    throw new UnsupportedOperationException("Not Implemented");
+  }
+}

src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTalter_model_statement.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.solutions.spannerddl.parser;
+
+public class ASTalter_model_statement extends SimpleNode {
+  public ASTalter_model_statement(int id) {
+    super(id);
+    throw new UnsupportedOperationException("Not Implemented");
+  }
+
+  public ASTalter_model_statement(DdlParser p, int id) {
+    super(p, id);
+    throw new UnsupportedOperationException("Not Implemented");
+  }
+}

src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTalter_schema_statement.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.solutions.spannerddl.parser;
+
+public class ASTalter_schema_statement extends SimpleNode {
+  public ASTalter_schema_statement(int id) {
+    super(id);
+    throw new UnsupportedOperationException("Not Implemented");
+  }
+
+  public ASTalter_schema_statement(DdlParser p, int id) {
+    super(p, id);
+    throw new UnsupportedOperationException("Not Implemented");
+  }
+}

src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTalter_sequence_statement.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.solutions.spannerddl.parser;
+
+public class ASTalter_sequence_statement extends SimpleNode {
+  public ASTalter_sequence_statement(int id) {
+    super(id);
+    throw new UnsupportedOperationException("Not Implemented");
+  }
+
+  public ASTalter_sequence_statement(DdlParser p, int id) {
+    super(p, id);
+    throw new UnsupportedOperationException("Not Implemented");
+  }
+}

src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTcolumn_def.java

@@ -36,7 +36,7 @@ public String getColumnName() {
   }
 
   public String getColumnTypeString() {
-    return getColumnType().toString().toUpperCase();
+    return getColumnType().toString();
   }
 
   public ASTcolumn_type getColumnType() {
@@ -52,13 +52,17 @@ public ASTcolumn_default_clause getColumnDefaultClause() {
   }
 
   public boolean isNotNull() {
-    return (children.length > 2 && children[2] instanceof ASTnot_null);
+    return ASTTreeUtils.getOptionalChildByType(children, ASTnot_null.class) != null;
   }
 
   public @Nullable ASToptions_clause getOptionsClause() {
     return ASTTreeUtils.getOptionalChildByType(children, ASToptions_clause.class);
   }
 
+  public boolean isHidden() {
+    return ASTTreeUtils.getOptionalChildByType(children, ASThidden.class) != null;
+  }
+
   @Override
   public String toString() {
     // check for unknown/unsupported children
@@ -72,6 +76,7 @@ public String toString() {
             (isNotNull() ? "NOT NULL" : null),
             getGenerationClause(),
             getColumnDefaultClause(),
+            (isHidden() ? "HIDDEN" : null),
             getOptionsClause());
   }
 
@@ -102,6 +107,10 @@ private void validate() {
       // default value
       index++;
     }
+    if (index < children.length && children[index] instanceof ASThidden) {
+      // default value
+      index++;
+    }
     if (index < children.length && children[index] instanceof ASToptions_clause) {
       // options
       index++;

src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTcolumn_type.java

@@ -45,21 +45,31 @@ public ASTcolumn_type getArraySubType() {
   public String toString() {
 
     String typeName = getTypeName();
-    if (children == null) {
-      return typeName;
-    } else {
-      switch (typeName) {
-        case "STRING":
-        case "BYTES":
-          // length.
-          return typeName + "(" + ((ASTlength) children[0]) + ")";
-        case "ARRAY":
-          return "ARRAY<" + ((ASTcolumn_type) children[0]) + ">";
-        case "PG": // PG.pgtype
+    switch (typeName) {
+      case "FLOAT64":
+      case "INT64":
+      case "BOOL":
+      case "TIMESTAMP":
+      case "DATE":
+      case "NUMERIC":
+      case "JSON":
+        return typeName;
+      case "STRING":
+      case "BYTES":
+        // length.
+        return typeName + "(" + ((ASTlength) children[0]) + ")";
+      case "ARRAY":
+        return "ARRAY<" + ((ASTcolumn_type) children[0]) + ">";
+      case "PG": // PG.pgtype
+        return ASTTreeUtils.tokensToString(this).toUpperCase();
+      case "STRUCT":
+        if (jjtGetNumChildren() > 0) {
           return ASTTreeUtils.tokensToString(this);
-        default:
-          throw new IllegalArgumentException("Unknown column type " + typeName);
-      }
+        } else {
+          return "STRUCT <>";
+        }
+      default:
+        throw new IllegalArgumentException("Unknown column type " + typeName);
     }
   }
 

src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTcolumns.java

@@ -32,8 +32,10 @@ public ASTcolumns(DdlParser p, int id) {
 
   @Override
   public String toString() {
-    return "("
-        + Joiner.on(", ").join(ASTTreeUtils.getChildrenAssertType(children, ASTkey_part.class))
-        + ")";
+    return "( "
+        + Joiner.on(", ")
+            .skipNulls()
+            .join(ASTTreeUtils.getChildrenAssertType(children, ASTkey_part.class))
+        + " )";
   }
 }

src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTcreate_index_statement.java

@@ -16,8 +16,11 @@
 
 package com.google.cloud.solutions.spannerddl.parser;
 
-/** Abstract Syntax Tree parser object for "create_index_statement" token */
+import static com.google.cloud.solutions.spannerddl.diff.ASTTreeUtils.getChildByType;
+import static com.google.cloud.solutions.spannerddl.diff.ASTTreeUtils.getOptionalChildByType;
+
 import com.google.cloud.solutions.spannerddl.diff.ASTTreeUtils;
+import com.google.common.base.Joiner;
 
 public class ASTcreate_index_statement extends SimpleNode
     implements Comparable<ASTcreate_index_statement> {
@@ -36,33 +39,52 @@ public String getIndexName() {
 
   @Override
   public String toString() {
-    StringBuilder ret = new StringBuilder("CREATE");
+    return toStringOptionalExistClause(true);
+  }
+
+  /**
+   * Create string version, optionally including the IF NOT EXISTS clause
+   *
+   * @param includeExists
+   */
+  public String toStringOptionalExistClause(boolean includeExists) {
+    validateChildren();
+    ASTindex_interleave_clause interleave =
+        getOptionalChildByType(children, ASTindex_interleave_clause.class);
+
+    return Joiner.on(" ")
+        .skipNulls()
+        .join(
+            "CREATE",
+            getOptionalChildByType(children, ASTunique_index.class),
+            getOptionalChildByType(children, ASTnull_filtered.class),
+            "INDEX",
+            (includeExists ? getOptionalChildByType(children, ASTif_not_exists.class) : null),
+            getIndexName(),
+            "ON",
+            getChildByType(children, ASTtable.class),
+            getChildByType(children, ASTcolumns.class),
+            getOptionalChildByType(children, ASTstored_column_list.class),
+            (interleave != null ? "," : null),
+            interleave);
+  }
+
+  private void validateChildren() {
     for (Node child : children) {
       switch (child.getId()) {
         case DdlParserTreeConstants.JJTUNIQUE_INDEX:
         case DdlParserTreeConstants.JJTNULL_FILTERED:
         case DdlParserTreeConstants.JJTCOLUMNS:
         case DdlParserTreeConstants.JJTSTORED_COLUMN_LIST:
-          ret.append(" ");
-          ret.append(child);
-          break;
+        case DdlParserTreeConstants.JJTIF_NOT_EXISTS:
         case DdlParserTreeConstants.JJTNAME:
-          ret.append(" INDEX ");
-          ret.append(child);
-          break;
         case DdlParserTreeConstants.JJTTABLE:
-          ret.append(" ON ");
-          ret.append(child);
-          break;
         case DdlParserTreeConstants.JJTINDEX_INTERLEAVE_CLAUSE:
-          ret.append(", ");
-          ret.append(child);
           break;
         default:
-          throw new IllegalArgumentException("Unknown node ID " + child.getClass());
+          throw new IllegalArgumentException("Unknown child node " + child.getClass());
       }
     }
-    return ret.toString();
   }
 
   @Override
@@ -73,8 +95,9 @@ public int compareTo(ASTcreate_index_statement other) {
   @Override
   public boolean equals(Object other) {
     if (other instanceof ASTcreate_index_statement) {
-      // lazy: compare text rendering.
-      return this.toString().equals(other.toString());
+      // lazy: compare text rendering, but don't take into account IF NOT EXISTS statements
+      return this.toStringOptionalExistClause(false)
+          .equals(((ASTcreate_index_statement) other).toStringOptionalExistClause(false));
     }
     return false;
   }

src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTcreate_model_statement.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.solutions.spannerddl.parser;
+
+// TODO
+public class ASTcreate_model_statement extends SimpleNode {
+  public ASTcreate_model_statement(int id) {
+    super(id);
+    throw new UnsupportedOperationException("Not Implemented");
+  }
+
+  public ASTcreate_model_statement(DdlParser p, int id) {
+    super(p, id);
+    throw new UnsupportedOperationException("Not Implemented");
+  }
+}