Add CI for VTop example (#16007)

Signed-off-by: Manan Gupta <[email protected]> Signed-off-by: Florent Poinsard <[email protected]> Signed-off-by: Rohit Nayak <[email protected]> Co-authored-by: Florent Poinsard <[email protected]> Co-authored-by: Rohit Nayak <[email protected]>
vitessio · Nov 28, 2024 · aad2e46 · aad2e46
1 parent 2f281de
commit aad2e46
Show file tree

Hide file tree

Showing 5 changed files with 121 additions and 24 deletions.
diff --git a/.github/workflows/vtop_example.yml b/.github/workflows/vtop_example.yml
@@ -0,0 +1,97 @@
+name: vtop_example
+on: [push, pull_request]
+concurrency:
+  group: format('{0}-{1}', ${{ github.ref }}, 'vtop_example')
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: VTop Example
+    runs-on: self-hosted
+
+    steps:
+      - name: Skip CI
+        run: |
+          if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then
+            echo "skipping CI due to the 'Skip CI' label"
+            exit 1
+          fi
+
+      - name: Check if workflow needs to be skipped
+        id: skip-workflow
+        run: |
+          skip='false'
+          if [[ "${{github.event.pull_request}}" ==  "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then
+            skip='true'
+          fi
+          echo Skip ${skip}
+          echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT
+          
+          PR_DATA=$(curl -s\
+            -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+            -H "Accept: application/vnd.github.v3+json" \
+            "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}")
+          draft=$(echo "$PR_DATA" | jq .draft -r)
+          echo "is_draft=${draft}" >> $GITHUB_OUTPUT
+
+      - name: Check out code
+        if: steps.skip-workflow.outputs.skip-workflow == 'false'
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - name: Check for changes in relevant files
+        if: steps.skip-workflow.outputs.skip-workflow == 'false'
+        uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1
+        id: changes
+        with:
+          token: ''
+          filters: |
+            end_to_end:
+              - 'go/**/*.go'
+              - 'go/vt/sidecardb/**/*.sql'
+              - 'test.go'
+              - 'Makefile'
+              - 'build.env'
+              - 'go.[sumod]'
+              - 'proto/*.proto'
+              - 'tools/**'
+              - 'config/**'
+              - 'bootstrap.sh'
+              - 'examples/**'
+              - 'test/**'
+              - '.github/workflows/vtop_example.yml'
+
+      - name: Set up Go
+        if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
+        uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+        with:
+          go-version-file: go.mod
+
+      - name: Tune the OS
+        if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
+        run: |
+          echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
+
+      - name: Get dependencies
+        if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
+        run: |
+          # Install everything we need, and configure
+          sudo apt-get install -y eatmydata make
+          go mod download
+
+      # needed for vtctldclient
+      - name: Build vitess
+        if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
+        run: |
+          make build
+
+      - name: Install kubectl & kind
+        if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
+        run: |
+          make install_kubectl_kind
+
+      - name: vtop_example
+        if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
+        timeout-minutes: 60
+        run: |
+          source build.env
+          eatmydata -- go run test.go -docker=false -skip-build -print-log -follow -retry=1 -timeout=60m vtop_example
diff --git a/examples/operator/101_initial_cluster.yaml b/examples/operator/101_initial_cluster.yaml
@@ -15,7 +15,7 @@ spec:
     vtbackup: vitess/lite:latest
     vtorc: vitess/lite:latest
     mysqld:
-      mysql80Compatible: vitess/lite:latest
+      mysql80Compatible: mysql:8.0.30
     mysqldExporter: prom/mysqld-exporter:v0.11.0
   cells:
   - name: zone1
@@ -155,23 +155,6 @@ stringData:
     # Vitess defaults
     ###############################################################################
 
-    # Vitess-internal database.
-    CREATE DATABASE IF NOT EXISTS _vt;
-    # Note that definitions of local_metadata and shard_metadata should be the same
-    # as in production which is defined in go/vt/mysqlctl/metadata_tables.go.
-    CREATE TABLE IF NOT EXISTS _vt.local_metadata (
-      name VARCHAR(255) NOT NULL,
-      value VARCHAR(255) NOT NULL,
-      db_name VARBINARY(255) NOT NULL,
-      PRIMARY KEY (db_name, name)
-      ) ENGINE=InnoDB;
-    CREATE TABLE IF NOT EXISTS _vt.shard_metadata (
-      name VARCHAR(255) NOT NULL,
-      value MEDIUMBLOB NOT NULL,
-      db_name VARBINARY(255) NOT NULL,
-      PRIMARY KEY (db_name, name)
-      ) ENGINE=InnoDB;
-
     # Admin user with all privileges.
     CREATE USER 'vt_dba'@'localhost';
     GRANT ALL ON *.* TO 'vt_dba'@'localhost';
@@ -200,19 +183,24 @@ stringData:
       ON *.* TO 'vt_allprivs'@'localhost';
 
     # User for slave replication connections.
-    # TODO: Should we set a password on this since it allows remote connections?
     CREATE USER 'vt_repl'@'%';
     GRANT REPLICATION SLAVE ON *.* TO 'vt_repl'@'%';
 
-    # User for Vitess filtered replication (binlog player).
-    # Same permissions as vt_app.
+    # User for Vitess VReplication (base vstreamers and vplayer).
     CREATE USER 'vt_filtered'@'localhost';
     GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,
       REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,
       LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW,
       SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER
       ON *.* TO 'vt_filtered'@'localhost';
 
+    # User for general MySQL monitoring.
+    CREATE USER 'vt_monitoring'@'localhost';
+    GRANT SELECT, PROCESS, SUPER, REPLICATION CLIENT, RELOAD
+    ON *.* TO 'vt_monitoring'@'localhost';
+    GRANT SELECT, UPDATE, DELETE, DROP
+    ON performance_schema.* TO 'vt_monitoring'@'localhost';
+
     # custom sql is used to add custom scripts like creating users/passwords. We use it in our tests
     # {{custom_sql}}
 

diff --git a/test/config.json b/test/config.json
@@ -1238,6 +1238,17 @@
 			"RetryMax": 1,
 			"Tags": []
 		},
+		"vtop_example": {
+			"File": "",
+			"Args": [],
+			"Command": [
+				"test/vtop_example.sh"
+			],
+			"Manual": false,
+			"Shard": "",
+			"RetryMax": 1,
+			"Tags": []
+		},
 		"vtorc_primary_failure": {
 			"File": "unused.go",
 			"Args": ["vitess.io/vitess/go/test/endtoend/vtorc/primaryfailure"],

diff --git a/test/vtop_example.sh b/test/vtop_example.sh
@@ -482,11 +482,12 @@ EOF
   waitForKeyspaceToBeServing customer 80- 1
 }
 
+kind delete cluster --name kind || true
 
 # Build the docker image for vitess/lite using the local code
 docker build -f docker/lite/Dockerfile -t vitess/lite:pr .
 # Build the docker image for vitess/vtadmin using the local code
-docker build -f docker/binaries/vtadmin/Dockerfile --build-arg VT_BASE_VER=pr -t vitess/vtadmin:pr .
+docker build -f docker/binaries/vtadmin/Dockerfile --build-arg VT_BASE_VER=pr -t vitess/vtadmin:pr ./docker/binaries/vtadmin
 
 # Print the docker images available
 docker image ls

diff --git a/tools/get_kubectl_kind.sh b/tools/get_kubectl_kind.sh
@@ -12,7 +12,7 @@ source build.env
 mkdir -p "$VTROOT/bin"
 cd "$VTROOT/bin"
 
-KUBE_VERSION="${KUBE_VERSION:-v1.21.1}"
+KUBE_VERSION="${KUBE_VERSION:-v1.31.0}"
 KUBERNETES_RELEASE_URL="${KUBERNETES_RELEASE_URL:-https://dl.k8s.io}"
 
 # Download kubectl if needed.
@@ -28,7 +28,7 @@ ln -sf "kubectl-${KUBE_VERSION}" kubectl
 if ! command -v kind &> /dev/null
 then
     echo "Downloading kind..."
-    curl -L https://kind.sigs.k8s.io/dl/v0.12.0/kind-linux-amd64 > "kind"
+    curl -L https://kind.sigs.k8s.io/dl/v0.22.0/kind-linux-amd64 > "kind"
     chmod +x "kind"
     echo "Installed kind"
 else