Add CI for VTop example

.github/workflows/vtop_example.yml

@@ -0,0 +1,97 @@
+name: vtop_example
+on: [push, pull_request]
+concurrency:
+  group: format('{0}-{1}', ${{ github.ref }}, 'vtop_example')
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: VTop Example
+    runs-on: self-hosted
+
+    steps:
+      - name: Skip CI
+        run: |
+          if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then
+            echo "skipping CI due to the 'Skip CI' label"
+            exit 1
+          fi
+
+      - name: Check if workflow needs to be skipped
+        id: skip-workflow
+        run: |
+          skip='false'
+          if [[ "${{github.event.pull_request}}" ==  "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then
+            skip='true'
+          fi
+          echo Skip ${skip}
+          echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT
+
+          PR_DATA=$(curl -s\
+            -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+            -H "Accept: application/vnd.github.v3+json" \
+            "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}")
+          draft=$(echo "$PR_DATA" | jq .draft -r)
+          echo "is_draft=${draft}" >> $GITHUB_OUTPUT
+
+      - name: Check out code
+        if: steps.skip-workflow.outputs.skip-workflow == 'false'
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - name: Check for changes in relevant files
+        if: steps.skip-workflow.outputs.skip-workflow == 'false'
+        uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1
+        id: changes
+        with:
+          token: ''
+          filters: |
+            end_to_end:
+              - 'go/**/*.go'
+              - 'go/vt/sidecardb/**/*.sql'
+              - 'test.go'
+              - 'Makefile'
+              - 'build.env'
+              - 'go.[sumod]'
+              - 'proto/*.proto'
+              - 'tools/**'
+              - 'config/**'
+              - 'bootstrap.sh'
+              - 'examples/**'
+              - 'test/**'
+              - '.github/workflows/vtop_example.yml'
+
+      - name: Set up Go
+        if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
+        uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+        with:
+          go-version-file: go.mod
+
+      - name: Tune the OS
+        if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
+        run: |
+          echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
+
+      - name: Get dependencies
+        if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
+        run: |
+          # Install everything we need, and configure
+          sudo apt-get install -y eatmydata make
+          go mod download
+
+      # needed for vtctldclient
+      - name: Build vitess
+        if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
+        run: |
+          make build
+
+      - name: Install kubectl & kind
+        if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
+        run: |
+          make install_kubectl_kind
+
+      - name: vtop_example
+        if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
+        timeout-minutes: 60
+        run: |
+          source build.env
+          eatmydata -- go run test.go -docker=false -skip-build -print-log -follow -retry=1 -timeout=60m vtop_example

examples/operator/101_initial_cluster.yaml

@@ -15,7 +15,7 @@ spec:
     vtbackup: vitess/lite:latest
     vtorc: vitess/lite:latest
     mysqld:
-      mysql80Compatible: vitess/lite:latest
+      mysql80Compatible: mysql:8.0.30
     mysqldExporter: prom/mysqld-exporter:v0.11.0
   cells:
   - name: zone1
@@ -155,23 +155,6 @@ stringData:
     # Vitess defaults
     ###############################################################################
 
-    # Vitess-internal database.
-    CREATE DATABASE IF NOT EXISTS _vt;
-    # Note that definitions of local_metadata and shard_metadata should be the same
-    # as in production which is defined in go/vt/mysqlctl/metadata_tables.go.
-    CREATE TABLE IF NOT EXISTS _vt.local_metadata (
-      name VARCHAR(255) NOT NULL,
-      value VARCHAR(255) NOT NULL,
-      db_name VARBINARY(255) NOT NULL,
-      PRIMARY KEY (db_name, name)
-      ) ENGINE=InnoDB;
-    CREATE TABLE IF NOT EXISTS _vt.shard_metadata (
-      name VARCHAR(255) NOT NULL,
-      value MEDIUMBLOB NOT NULL,
-      db_name VARBINARY(255) NOT NULL,
-      PRIMARY KEY (db_name, name)
-      ) ENGINE=InnoDB;
-
     # Admin user with all privileges.
     CREATE USER 'vt_dba'@'localhost';
     GRANT ALL ON *.* TO 'vt_dba'@'localhost';
@@ -200,19 +183,24 @@ stringData:
       ON *.* TO 'vt_allprivs'@'localhost';
 
     # User for slave replication connections.
-    # TODO: Should we set a password on this since it allows remote connections?
     CREATE USER 'vt_repl'@'%';
     GRANT REPLICATION SLAVE ON *.* TO 'vt_repl'@'%';
 
-    # User for Vitess filtered replication (binlog player).
-    # Same permissions as vt_app.
+    # User for Vitess VReplication (base vstreamers and vplayer).
     CREATE USER 'vt_filtered'@'localhost';
     GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,
       REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,
       LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW,
       SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER
       ON *.* TO 'vt_filtered'@'localhost';
 
+    # User for general MySQL monitoring.
+    CREATE USER 'vt_monitoring'@'localhost';
+    GRANT SELECT, PROCESS, SUPER, REPLICATION CLIENT, RELOAD
+    ON *.* TO 'vt_monitoring'@'localhost';
+    GRANT SELECT, UPDATE, DELETE, DROP
+    ON performance_schema.* TO 'vt_monitoring'@'localhost';
+
     # custom sql is used to add custom scripts like creating users/passwords. We use it in our tests
     # {{custom_sql}}
 

test/config.json

@@ -1238,6 +1238,17 @@
 			"RetryMax": 1,
 			"Tags": []
 		},
+		"vtop_example": {
+			"File": "",
+			"Args": [],
+			"Command": [
+				"test/vtop_example.sh"
+			],
+			"Manual": false,
+			"Shard": "",
+			"RetryMax": 1,
+			"Tags": []
+		},
 		"vtorc_primary_failure": {
 			"File": "unused.go",
 			"Args": ["vitess.io/vitess/go/test/endtoend/vtorc/primaryfailure"],

test/vtop_example.sh

@@ -68,6 +68,10 @@ function checkPodStatusWithTimeout() {
     sleep 1
   done
   echo -e "ERROR: checkPodStatusWithTimeout timeout to find pod matching:\ngot:\n$out\nfor regex: $regex"
+  vttabletPod=$(kubectl get pods | grep -E "vttablet" | head -n 1 | awk '{print $1}')
+  kubectl describe pod $vttabletPod
+  kubectl logs $vttabletPod -c vttablet
+  kubectl logs $vttabletPod -c mysqld
   exit 1
 }
 
@@ -482,11 +486,12 @@ EOF
   waitForKeyspaceToBeServing customer 80- 1
 }
 
+kind delete cluster --name kind || true
 
 # Build the docker image for vitess/lite using the local code
 docker build -f docker/lite/Dockerfile -t vitess/lite:pr .
 # Build the docker image for vitess/vtadmin using the local code
-docker build -f docker/binaries/vtadmin/Dockerfile --build-arg VT_BASE_VER=pr -t vitess/vtadmin:pr .
+docker build -f docker/binaries/vtadmin/Dockerfile --build-arg VT_BASE_VER=pr -t vitess/vtadmin:pr ./docker/binaries/vtadmin
 
 # Print the docker images available
 docker image ls

tools/get_kubectl_kind.sh

@@ -12,7 +12,7 @@ source build.env
 mkdir -p "$VTROOT/bin"
 cd "$VTROOT/bin"
 
-KUBE_VERSION="${KUBE_VERSION:-v1.21.1}"
+KUBE_VERSION="${KUBE_VERSION:-v1.31.0}"
 KUBERNETES_RELEASE_URL="${KUBERNETES_RELEASE_URL:-https://dl.k8s.io}"
 
 # Download kubectl if needed.
@@ -28,7 +28,7 @@ ln -sf "kubectl-${KUBE_VERSION}" kubectl
 if ! command -v kind &> /dev/null
 then
     echo "Downloading kind..."
-    curl -L https://kind.sigs.k8s.io/dl/v0.12.0/kind-linux-amd64 > "kind"
+    curl -L https://kind.sigs.k8s.io/dl/v0.22.0/kind-linux-amd64 > "kind"
     chmod +x "kind"
     echo "Installed kind"
 else