Merge pull request #17 from invitae/update-upstream

Update upstream
invitae · Nov 24, 2021 · 0044d86 · 0044d86
2 parents fc1b7d0 + 726992a
commit 0044d86
Show file tree

Hide file tree

Showing 133 changed files with 16,874 additions and 12,529 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -0,0 +1 @@
+* @sfc-gh-zli @sfc-gh-japatel @sfc-gh-rramachandran
diff --git a/.github/scripts/decrypt_secret.sh b/.github/scripts/decrypt_secret.sh
@@ -8,6 +8,9 @@ snowflake_deployment=$1
 if [ $snowflake_deployment = 'aws' ]; then
   gpg --quiet --batch --yes --decrypt --passphrase="$SNOWFLAKE_TEST_PROFILE_SECRET" \
   --output profile.json .github/scripts/profile.json.gpg
+elif [ $snowflake_deployment = 'gcs' ]; then
+  gpg --quiet --batch --yes --decrypt --passphrase="$SNOWFLAKE_TEST_PROFILE_SECRET" \
+  --output profile.json .github/scripts/profile_gcs.json.gpg
 else
   gpg --quiet --batch --yes --decrypt --passphrase="$SNOWFLAKE_TEST_PROFILE_SECRET" \
   --output profile.json .github/scripts/profile_azure.json.gpg

diff --git a/.github/scripts/perf_test.json.gpg b/.github/scripts/perf_test.json.gpg
diff --git a/.github/scripts/profile.json.gpg b/.github/scripts/profile.json.gpg
diff --git a/.github/scripts/profile_azure.json.gpg b/.github/scripts/profile_azure.json.gpg
diff --git a/.github/scripts/profile_gcs.json.gpg b/.github/scripts/profile_gcs.json.gpg
diff --git a/.github/scripts/squid.conf b/.github/scripts/squid.conf
@@ -0,0 +1,34 @@
+acl SSL_ports port 443
+acl Safe_ports port 80          # http
+acl Safe_ports port 21          # ftp
+acl Safe_ports port 443         # https
+acl Safe_ports port 70          # gopher
+acl Safe_ports port 210         # wais
+acl Safe_ports port 1025-65535  # unregistered ports
+acl Safe_ports port 280         # http-mgmt
+acl Safe_ports port 488         # gss-http
+acl Safe_ports port 591         # filemaker
+acl Safe_ports port 777         # multiling http
+acl CONNECT method CONNECT
+
+http_access deny !Safe_ports
+http_access deny CONNECT !SSL_ports
+
+http_port 3128
+coredump_dir /var/spool/squid
+
+refresh_pattern ^ftp:           1440    20%     10080
+refresh_pattern ^gopher:        1440    0%      1440
+refresh_pattern -i (/cgi-bin/|\?) 0     0%      0
+refresh_pattern (Release|Packages(.gz)*)$      0       20%     2880
+refresh_pattern .               0       20%     4320
+
+auth_param basic program /usr/lib/squid/basic_ncsa_auth /etc/squid/passwords
+auth_param basic realm proxy
+acl authenticated proxy_auth REQUIRED
+http_access allow authenticated
+
+http_access allow localhost
+
+ident_lookup_access deny all
+http_access deny all
diff --git a/.github/workflows/End2EndFullTest.yml b/.github/workflows/End2EndFullTest.yml
@@ -38,6 +38,20 @@ jobs:
         sudo apt-get -y install jq
         sudo apt-get -y install protobuf-compiler
 
+    - name: Install Squid as Proxy Server and Apache Utils for Password Authentication
+      run: |
+        sudo apt-get update
+        sudo apt-get -y install squid3
+        sudo apt-get install apache2-utils
+
+    - name: Change squid config and run Proxy Server
+      run: |
+        sudo touch /etc/squid/passwords
+        sudo chmod 777 /etc/squid/passwords
+        sudo htpasswd -db -c /etc/squid/passwords admin test
+        sudo mv .github/scripts/squid.conf /etc/squid/squid.conf
+        sudo service squid start
+
     - name: Build with Unit and Integration Test
       env:
         JACOCO_COVERAGE: true

diff --git a/.github/workflows/End2EndFullTestAzure.yml b/.github/workflows/End2EndFullTestAzure.yml
@@ -38,6 +38,20 @@ jobs:
         sudo apt-get -y install jq
         sudo apt-get -y install protobuf-compiler
 
+    - name: Install Squid as Proxy Server and Apache Utils for Password Authentication
+      run: |
+        sudo apt-get update
+        sudo apt-get -y install squid3
+        sudo apt-get install apache2-utils
+
+    - name: Change squid config and run Proxy Server
+      run: |
+        sudo touch /etc/squid/passwords
+        sudo chmod 777 /etc/squid/passwords
+        sudo htpasswd -db -c /etc/squid/passwords admin test
+        sudo mv .github/scripts/squid.conf /etc/squid/squid.conf
+        sudo service squid start
+
     - name: Build with Unit and Integration Test
       env:
         JACOCO_COVERAGE: true

diff --git a/.github/workflows/End2EndFullTestGCS.yml b/.github/workflows/End2EndFullTestGCS.yml
@@ -0,0 +1,109 @@
+name: Kafka Connector End2End Full Version Test GCS
+
+on:
+  push:
+    branches-ignore:
+      - '**'
+#on:
+#  push:
+#    branches: [ master ]
+#  pull_request:
+#    branches: '**'
+
+jobs:
+  build:
+    runs-on: ubuntu-18.04
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v2
+      - name: "Install Java 8"
+        uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+      - name: Install Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: '3.6'
+          architecture: 'x64'
+      - name: Decrypt profile.json
+        run: ./.github/scripts/decrypt_secret.sh 'gcs'
+        env:
+          SNOWFLAKE_TEST_PROFILE_SECRET: ${{ secrets.SNOWFLAKE_TEST_PROFILE_SECRET }}
+      - name: Install Dependency
+        run: |
+          pip3 install --upgrade setuptools
+          pip3 install requests certifi "confluent-kafka[avro,json,protobuf]>=1.5.0"
+          pip3 install avro-python3 kafka-python
+          pip3 install protobuf
+          pip3 install --upgrade snowflake-connector-python
+          curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
+          sudo apt-get -y install jq
+          sudo apt-get -y install protobuf-compiler
+
+      - name: Install Squid as Proxy Server and Apache Utils for Password Authentication
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install squid3
+          sudo apt-get install apache2-utils
+
+      - name: Change squid config and run Proxy Server
+        run: |
+          sudo touch /etc/squid/passwords
+          sudo chmod 777 /etc/squid/passwords
+          sudo htpasswd -db -c /etc/squid/passwords admin test
+          sudo mv .github/scripts/squid.conf /etc/squid/squid.conf
+          sudo service squid start
+
+      - name: Build with Unit and Integration Test
+        env:
+          JACOCO_COVERAGE: true
+          SNOWFLAKE_CREDENTIAL_FILE: "../profile.json"
+          SHELL: "/bin/bash"
+        run: |
+          cd test
+          ./build_apache.sh ../../snowflake-kafka-connector
+
+      - name: End to End Test of Confluent Platform 5.2.0
+        env:
+          SNOWFLAKE_CREDENTIAL_FILE: "../profile.json"
+        run: |
+          cd test
+          ./run_test_confluent.sh 5.2.0 ./apache_properties
+
+      - name: End to End Test of Apache Plarform 2.2.0
+        env:
+          SNOWFLAKE_CREDENTIAL_FILE: "../profile.json"
+        run: |
+          cd test
+          ./run_test_apache.sh 2.2.0 ./apache_properties
+
+      - name: End to End Test of Confluent Platform 5.4.0
+        env:
+          SNOWFLAKE_CREDENTIAL_FILE: "../profile.json"
+        run: |
+          cd test
+          ./run_test_confluent.sh 5.4.0 ./apache_properties
+
+      - name: End to End Test of Apache Plarform 2.4.0
+        env:
+          SNOWFLAKE_CREDENTIAL_FILE: "../profile.json"
+        run: |
+          cd test
+          ./run_test_apache.sh 2.4.0 ./apache_properties
+
+      - name: End to End Test of Confluent Platform 5.5.0
+        env:
+          SNOWFLAKE_CREDENTIAL_FILE: "../profile.json"
+        run: |
+          cd test
+          ./run_test_confluent.sh 5.5.0 ./apache_properties
+
+      - name: End to End Test of Apache Plarform 2.5.0
+        env:
+          SNOWFLAKE_CREDENTIAL_FILE: "../profile.json"
+        run: |
+          cd test
+          ./run_test_apache.sh 2.5.0 ./apache_properties
+
+      - name: Code Coverage
+        uses: codecov/codecov-action@v1
diff --git a/.github/workflows/End2EndTestApacheAws.yml b/.github/workflows/End2EndTestApacheAws.yml
@@ -37,8 +37,22 @@ jobs:
         pip3 install protobuf
         pip3 install --upgrade snowflake-connector-python
         curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
-        sudo apt-get -y install jq
+        sudo apt-get -y install jq vim
         sudo apt-get -y install protobuf-compiler
+
+    - name: Install Squid as Proxy Server and Apache Utils for Password Authentication
+      run: |
+        sudo apt-get update
+        sudo apt-get -y install squid3
+        sudo apt-get install apache2-utils
+
+    - name: Change squid config and run Proxy Server
+      run: |
+        sudo touch /etc/squid/passwords
+        sudo chmod 777 /etc/squid/passwords
+        sudo htpasswd -db -c /etc/squid/passwords admin test
+        sudo mv .github/scripts/squid.conf /etc/squid/squid.conf
+        sudo service squid start
     
     - name: Build with Unit Test
       env:

diff --git a/.github/workflows/End2EndTestApacheAzure.yml b/.github/workflows/End2EndTestApacheAzure.yml
@@ -39,6 +39,20 @@ jobs:
         curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
         sudo apt-get -y install jq
         sudo apt-get -y install protobuf-compiler
+
+    - name: Install Squid as Proxy Server and Apache Utils for Password Authentication
+      run: |
+        sudo apt-get update
+        sudo apt-get -y install squid3
+        sudo apt-get install apache2-utils
+
+    - name: Change squid config and run Proxy Server
+      run: |
+        sudo touch /etc/squid/passwords
+        sudo chmod 777 /etc/squid/passwords
+        sudo htpasswd -db -c /etc/squid/passwords admin test
+        sudo mv .github/scripts/squid.conf /etc/squid/squid.conf
+        sudo service squid start
     
     - name: Build with Unit Test
       env:

diff --git a/.github/workflows/End2EndTestApacheGCS.yml b/.github/workflows/End2EndTestApacheGCS.yml
@@ -0,0 +1,71 @@
+name: Kafka Connector Apache End2End Test GCS
+
+on:
+  push:
+    branches-ignore:
+      - '**'
+#on:
+#  push:
+#    branches: [ master ]
+#  pull_request:
+#    branches: '**'
+
+jobs:
+  build:
+    runs-on: ubuntu-18.04
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v2
+      - name: "Install Java 8"
+        uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+      - name: Install Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: '3.6'
+          architecture: 'x64'
+      - name: Decrypt profile.json
+        run: ./.github/scripts/decrypt_secret.sh 'gcs'
+        env:
+          SNOWFLAKE_TEST_PROFILE_SECRET: ${{ secrets.SNOWFLAKE_TEST_PROFILE_SECRET }}
+      - name: Install Dependency
+        run: |
+          pip3 install --upgrade setuptools
+          pip3 install requests certifi "confluent-kafka[avro,json,protobuf]>=1.5.0"
+          pip3 install avro-python3 kafka-python
+          pip3 install protobuf
+          pip3 install --upgrade snowflake-connector-python
+          curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
+          sudo apt-get -y install jq vim
+          sudo apt-get -y install protobuf-compiler
+
+      - name: Install Squid as Proxy Server and Apache Utils for Password Authentication
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install squid3
+          sudo apt-get install apache2-utils
+
+      - name: Change squid config and run Proxy Server
+        run: |
+          sudo touch /etc/squid/passwords
+          sudo chmod 777 /etc/squid/passwords
+          sudo htpasswd -db -c /etc/squid/passwords admin test
+          sudo mv .github/scripts/squid.conf /etc/squid/squid.conf
+          sudo service squid start
+
+      - name: Build with Unit Test
+        env:
+          JACOCO_COVERAGE: true
+          SNOWFLAKE_CREDENTIAL_FILE: "../profile.json"
+          SHELL: "/bin/bash"
+        run: |
+          cd test
+          ./build_apache.sh ../../snowflake-kafka-connector package
+
+      - name: End to End Test of Apache Plarform 2.5
+        env:
+          SNOWFLAKE_CREDENTIAL_FILE: "../profile.json"
+        run: |
+          cd test
+          ./run_test_apache.sh 2.5.0 ./apache_properties
diff --git a/.github/workflows/End2EndTestConfluentAws.yml b/.github/workflows/End2EndTestConfluentAws.yml
@@ -37,9 +37,23 @@ jobs:
         pip3 install protobuf
         pip3 install --upgrade snowflake-connector-python
         curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
-        sudo apt-get -y install jq
+        sudo apt-get -y install jq vim
         sudo apt-get -y install protobuf-compiler
 
+    - name: Install Squid as Proxy Server and Apache Utils for Password Authentication
+      run: |
+        sudo apt-get update
+        sudo apt-get -y install squid3
+        sudo apt-get install apache2-utils
+
+    - name: Change squid config and run Proxy Server
+      run: |
+        sudo touch /etc/squid/passwords
+        sudo chmod 777 /etc/squid/passwords
+        sudo htpasswd -db -c /etc/squid/passwords admin test
+        sudo mv .github/scripts/squid.conf /etc/squid/squid.conf
+        sudo service squid start
+
     - name: Build with Unit Test
       env:
         JACOCO_COVERAGE: true

diff --git a/.github/workflows/End2EndTestConfluentAzure.yml b/.github/workflows/End2EndTestConfluentAzure.yml
@@ -37,8 +37,22 @@ jobs:
         pip3 install protobuf
         pip3 install --upgrade snowflake-connector-python
         curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
-        sudo apt-get -y install jq
+        sudo apt-get -y install jq vim
         sudo apt-get -y install protobuf-compiler
+
+    - name: Install Squid as Proxy Server and Apache Utils for Password Authentication
+      run: |
+        sudo apt-get update
+        sudo apt-get -y install squid3
+        sudo apt-get install apache2-utils
+
+    - name: Change squid config and run Proxy Server
+      run: |
+        sudo touch /etc/squid/passwords
+        sudo chmod 777 /etc/squid/passwords
+        sudo htpasswd -db -c /etc/squid/passwords admin test
+        sudo mv .github/scripts/squid.conf /etc/squid/squid.conf
+        sudo service squid start
     
     - name: Build with Unit Test
       env:
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		* @sfc-gh-zli @sfc-gh-japatel @sfc-gh-rramachandran