feat: add temp project and dataset id config properties

.github/workflows/java-pr.yml

@@ -79,7 +79,7 @@ jobs:
   java_build:
     name: Build
     timeout-minutes: 60
-    runs-on: [self-hosted, it]
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout Code
         uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
@@ -94,7 +94,7 @@ jobs:
     name: Unit Tests
     needs: [java_build]
     timeout-minutes: 60
-    runs-on: [self-hosted, it]
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout Code
         uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
@@ -126,7 +126,7 @@ jobs:
     needs: [spotless_check, checkstyle_check, java_build, java_unit_tests]
     timeout-minutes: 60
     # Run on any runner that matches all the specified runs-on values.
-    runs-on: [self-hosted, it]
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout Code
         uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
@@ -156,7 +156,7 @@ jobs:
     needs: [java_integration_smoke_tests_templates]
     timeout-minutes: 240
     # Run on any runner that matches all the specified runs-on values.
-    runs-on: [self-hosted, it]
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout Code
         uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
@@ -187,7 +187,7 @@ jobs:
     needs: [spotless_check, checkstyle_check, java_build, java_unit_tests, java_integration_tests_templates]
     timeout-minutes: 600
     # Run on any runner that matches all the specified runs-on values.
-    runs-on: [self-hosted, perf]
+    runs-on: ubuntu-latest
     steps:
     - name: Checkout Code
       uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0

.github/workflows/neo4j-pr.yml

@@ -0,0 +1,66 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Checks that are intended to run on PRs containing Java code.
+
+name: Neo4j PR
+
+on:
+  pull_request:
+    branches:
+      - 'main'
+    paths:
+      - 'v2/googlecloud-to-neo4j/**'
+  workflow_dispatch:
+
+env:
+  MAVEN_OPTS: -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=error
+
+permissions: read-all
+
+jobs:
+  neo4j_integration_test_templates:
+    name: Neo4j Template Custom Integration Tests
+    timeout-minutes: 60
+    # Run on any runner that matches all the specified runs-on values.
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+      - name: Setup Environment
+        id: setup-env
+        uses: ./.github/actions/setup-env
+      - name: Login
+        run: gcloud auth application-default login --no-browser
+        env:
+          CLOUDSDK_AUTH_ACCESS_TOKEN: ${{ secrets.ACCOUNTTOKEN }}
+      - name: Run Neo4j Custom Integration Tests
+        env:
+          DT_IT_ACCESS_TOKEN: ${{ secrets.ACCOUNTTOKEN }}
+        run: |
+          mvn --also-make --projects v2/googlecloud-to-neo4j verify \
+          -Dexcluded.neo4j.tests="" \
+          -Dproject=team-connectors-dev \
+          -DstageBucket=emre-dataflow-example \
+          -Pneo4jCustomIntegrationTests
+      - name: Upload Tests Report
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        if: always() # always run even if the previous step fails
+        with:
+          name: surefire-test-results
+          path: '**/surefire-reports/TEST-*.xml'
+          retention-days: 1
+      - name: Cleanup Java Environment
+        uses: ./.github/actions/cleanup-java-env
+        if: always()

pom.xml

@@ -106,6 +106,7 @@
     <load.tests>com.google.cloud.teleport.metadata.TemplateLoadTest</load.tests>
     <direct-runner.tests>com.google.cloud.teleport.metadata.DirectRunnerTest</direct-runner.tests>
     <excluded.spanner.tests></excluded.spanner.tests>
+    <excluded.neo4j.tests>com.google.cloud.teleport.v2.neo4j.templates.Neo4jCustomTestCategory</excluded.neo4j.tests>
 
     <licenseHeaderFile>JAVA_LICENSE_HEADER</licenseHeaderFile>
   </properties>
@@ -271,6 +272,7 @@
           They can be run manually using 'mvn -Dexcluded.spanner.tests="" -Dtest=<testname> test' -->
           <excludedGroups>${integration.tests}</excludedGroups>
           <excludedGroups>${excluded.spanner.tests}</excludedGroups>
+          <excludedGroups>${excluded.neo4j.tests}</excludedGroups>
           <trimStackTrace>false</trimStackTrace>
           <systemPropertyVariables>
             <!-- Allow this project to be tested with JDK 21 -->
@@ -458,6 +460,71 @@
         </plugins>
       </build>
     </profile>
+    <profile>
+      <id>neo4jCustomIntegrationTests</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <properties>
+        <!-- Skip coverage checks, unit tests are skipped -->
+        <jacoco.skip>true</jacoco.skip>
+        <jib.skip>true</jib.skip>
+        <skipShade>true</skipShade>
+        <spotless.check.skip>true</spotless.check.skip>
+        <checkstyle.skip>true</checkstyle.skip>
+        <mdep.analyze.skip>true</mdep.analyze.skip>
+        <!-- Some modules may yield no integration tests -->
+        <failIfNoTests>false</failIfNoTests>
+        <!-- Parallelism settings. Default is 2, set to consider methods -->
+        <itParallelismType>classesAndMethods</itParallelismType>
+        <itParallelism>2</itParallelism>
+      </properties>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <version>${surefire.version}</version>
+            <configuration combine.self="override">
+              <systemProperties>
+                <property>
+                  <name>beamPythonVersion</name>
+                  <value>${beam-python.version}</value>
+                </property>
+                <property>
+                  <name>beamJavaVersion</name>
+                  <value>${beam.version}</value>
+                </property>
+                <property>
+                  <name>beamMavenRepo</name>
+                  <value>${beam-maven-repo}</value>
+                </property>
+                <property>
+                  <name>directRunnerTest</name>
+                  <value>true</value>
+                </property>
+              </systemProperties>
+              <includes>
+                <include>**/*IT.java</include>
+              </includes>
+<!--              <excludes>-->
+<!--                <exclude>**/KafkaToKafkaIT.java</exclude>-->
+<!--                <exclude>**/*Test.java</exclude>-->
+<!--              </excludes>-->
+<!--              <excludedGroups>${direct-runner.tests}</excludedGroups>-->
+<!--              <excludedGroups>${integration.tests}</excludedGroups>-->
+              <groups>
+                com.google.cloud.teleport.v2.neo4j.templates.Neo4jCustomTestCategory
+              </groups>
+              <reuseForks>true</reuseForks>
+              <parallel>${itParallelismType}</parallel>
+              <threadCount>${itParallelism}</threadCount>
+              <trimStackTrace>false</trimStackTrace>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
     <profile>
       <id>templatesLoadTests</id>
       <activation>

v2/googlecloud-to-neo4j/src/main/java/com/google/cloud/teleport/v2/neo4j/model/helpers/SqlQuerySpec.java → v2/googlecloud-to-neo4j/src/main/java/com/google/cloud/teleport/v2/neo4j/model/helpers/BigQuerySpec.java

@@ -19,16 +19,25 @@
  * Convenience object for invoking SQL query as well as providing descriptions for read and cast
  * phase of transform.
  */
-public class SqlQuerySpec {
+public class BigQuerySpec {
 
   private final String readDescription;
   private final String castDescription;
   private final String sql;
+  private final String queryTempProject;
+  private final String queryTempDataset;
 
-  public SqlQuerySpec(String readDescription, String castDescription, String sql) {
+  public BigQuerySpec(
+      String readDescription,
+      String castDescription,
+      String sql,
+      String queryTempProject,
+      String queryTempDataset) {
     this.readDescription = readDescription;
     this.castDescription = castDescription;
     this.sql = sql;
+    this.queryTempProject = queryTempProject;
+    this.queryTempDataset = queryTempDataset;
   }
 
   public String getReadDescription() {
@@ -43,29 +52,50 @@ public String getSql() {
     return sql;
   }
 
-  public static class SqlQuerySpecBuilder {
+  public String getQueryTempProject() {
+    return queryTempProject;
+  }
+
+  public String getQueryTempDataset() {
+    return queryTempDataset;
+  }
+
+  public static class BigQuerySpecBuilder {
 
     private String readDescription;
     private String castDescription;
     private String sql;
+    private String queryTempProject;
+    private String queryTempDataset;
 
-    public SqlQuerySpecBuilder readDescription(String readDescription) {
+    public BigQuerySpecBuilder readDescription(String readDescription) {
       this.readDescription = readDescription;
       return this;
     }
 
-    public SqlQuerySpecBuilder castDescription(String castDescription) {
+    public BigQuerySpecBuilder castDescription(String castDescription) {
       this.castDescription = castDescription;
       return this;
     }
 
-    public SqlQuerySpecBuilder sql(String sql) {
+    public BigQuerySpecBuilder sql(String sql) {
       this.sql = sql;
       return this;
     }
 
-    public SqlQuerySpec build() {
-      return new SqlQuerySpec(readDescription, castDescription, sql);
+    public BigQuerySpecBuilder queryTempProject(String queryTempProject) {
+      this.queryTempProject = queryTempProject;
+      return this;
+    }
+
+    public BigQuerySpecBuilder queryTempDataset(String queryTempDataset) {
+      this.queryTempDataset = queryTempDataset;
+      return this;
+    }
+
+    public BigQuerySpec build() {
+      return new BigQuerySpec(
+          readDescription, castDescription, sql, queryTempProject, queryTempDataset);
     }
   }
 }

v2/googlecloud-to-neo4j/src/main/java/com/google/cloud/teleport/v2/neo4j/model/sources/BigQuerySource.java

@@ -22,11 +22,19 @@ public class BigQuerySource implements Source {
 
   private final String name;
   private final String query;
+  private final String queryTempProject;
+  private final String queryTempDataset;
 
   public BigQuerySource(String name, String query) {
+    this(name, query, null, null);
+  }
 
+  public BigQuerySource(
+      String name, String query, String queryTempProject, String queryTempDataset) {
     this.name = name;
     this.query = query;
+    this.queryTempProject = queryTempProject;
+    this.queryTempDataset = queryTempDataset;
   }
 
   @Override
@@ -43,6 +51,14 @@ public String getQuery() {
     return query;
   }
 
+  public String getQueryTempProject() {
+    return queryTempProject;
+  }
+
+  public String getQueryTempDataset() {
+    return queryTempDataset;
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) {
@@ -62,6 +78,19 @@ public int hashCode() {
 
   @Override
   public String toString() {
-    return "BigQuerySource{" + "name='" + name + '\'' + ", query='" + query + '\'' + '}';
+    return "BigQuerySource{"
+        + "name='"
+        + name
+        + '\''
+        + ", query='"
+        + query
+        + '\''
+        + ", queryTempProject='"
+        + queryTempProject
+        + '\''
+        + ", queryTempDataset='"
+        + queryTempDataset
+        + '\''
+        + '}';
   }
 }

v2/googlecloud-to-neo4j/src/main/java/com/google/cloud/teleport/v2/neo4j/model/sources/BigQuerySourceProvider.java

@@ -15,7 +15,9 @@
  */
 package com.google.cloud.teleport.v2.neo4j.model.sources;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import java.util.Optional;
 import org.neo4j.importer.v1.sources.SourceProvider;
 
 public class BigQuerySourceProvider implements SourceProvider<BigQuerySource> {
@@ -27,6 +29,10 @@ public String supportedType() {
 
   @Override
   public BigQuerySource provide(ObjectNode node) {
-    return new BigQuerySource(node.get("name").textValue(), node.get("query").textValue());
+    return new BigQuerySource(
+        node.get("name").textValue(),
+        node.get("query").textValue(),
+        Optional.ofNullable(node.get("query_temp_project")).map(JsonNode::textValue).orElse(null),
+        Optional.ofNullable(node.get("query_temp_dataset")).map(JsonNode::textValue).orElse(null));
   }
 }