From aad2e469aee458443a884ad7f200cac6e215ed92 Mon Sep 17 00:00:00 2001 From: Manan Gupta <35839558+GuptaManan100@users.noreply.github.com> Date: Fri, 29 Nov 2024 00:45:43 +0530 Subject: [PATCH] Add CI for VTop example (#16007) Signed-off-by: Manan Gupta Signed-off-by: Florent Poinsard Signed-off-by: Rohit Nayak Co-authored-by: Florent Poinsard Co-authored-by: Rohit Nayak --- .github/workflows/vtop_example.yml | 97 ++++++++++++++++++++++ examples/operator/101_initial_cluster.yaml | 30 ++----- test/config.json | 11 +++ test/vtop_example.sh | 3 +- tools/get_kubectl_kind.sh | 4 +- 5 files changed, 121 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/vtop_example.yml diff --git a/.github/workflows/vtop_example.yml b/.github/workflows/vtop_example.yml new file mode 100644 index 00000000000..fb5ae87c101 --- /dev/null +++ b/.github/workflows/vtop_example.yml @@ -0,0 +1,97 @@ +name: vtop_example +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'vtop_example') + cancel-in-progress: true + +jobs: + build: + name: VTop Example + runs-on: self-hosted + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + PR_DATA=$(curl -s\ + -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") + draft=$(echo "$PR_DATA" | jq .draft -r) + echo "is_draft=${draft}" >> $GITHUB_OUTPUT + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'go/vt/sidecardb/**/*.sql' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.[sumod]' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - 'examples/**' + - 'test/**' + - '.github/workflows/vtop_example.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + # Install everything we need, and configure + sudo apt-get install -y eatmydata make + go mod download + + # needed for vtctldclient + - name: Build vitess + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + make build + + - name: Install kubectl & kind + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + make install_kubectl_kind + + - name: vtop_example + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 60 + run: | + source build.env + eatmydata -- go run test.go -docker=false -skip-build -print-log -follow -retry=1 -timeout=60m vtop_example \ No newline at end of file diff --git a/examples/operator/101_initial_cluster.yaml b/examples/operator/101_initial_cluster.yaml index c26219254f1..a4f9f447e21 100644 --- a/examples/operator/101_initial_cluster.yaml +++ b/examples/operator/101_initial_cluster.yaml @@ -15,7 +15,7 @@ spec: vtbackup: vitess/lite:latest vtorc: vitess/lite:latest mysqld: - mysql80Compatible: vitess/lite:latest + mysql80Compatible: mysql:8.0.30 mysqldExporter: prom/mysqld-exporter:v0.11.0 cells: - name: zone1 @@ -155,23 +155,6 @@ stringData: # Vitess defaults ############################################################################### - # Vitess-internal database. - CREATE DATABASE IF NOT EXISTS _vt; - # Note that definitions of local_metadata and shard_metadata should be the same - # as in production which is defined in go/vt/mysqlctl/metadata_tables.go. - CREATE TABLE IF NOT EXISTS _vt.local_metadata ( - name VARCHAR(255) NOT NULL, - value VARCHAR(255) NOT NULL, - db_name VARBINARY(255) NOT NULL, - PRIMARY KEY (db_name, name) - ) ENGINE=InnoDB; - CREATE TABLE IF NOT EXISTS _vt.shard_metadata ( - name VARCHAR(255) NOT NULL, - value MEDIUMBLOB NOT NULL, - db_name VARBINARY(255) NOT NULL, - PRIMARY KEY (db_name, name) - ) ENGINE=InnoDB; - # Admin user with all privileges. CREATE USER 'vt_dba'@'localhost'; GRANT ALL ON *.* TO 'vt_dba'@'localhost'; @@ -200,12 +183,10 @@ stringData: ON *.* TO 'vt_allprivs'@'localhost'; # User for slave replication connections. - # TODO: Should we set a password on this since it allows remote connections? CREATE USER 'vt_repl'@'%'; GRANT REPLICATION SLAVE ON *.* TO 'vt_repl'@'%'; - # User for Vitess filtered replication (binlog player). - # Same permissions as vt_app. + # User for Vitess VReplication (base vstreamers and vplayer). CREATE USER 'vt_filtered'@'localhost'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE, REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES, @@ -213,6 +194,13 @@ stringData: SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER ON *.* TO 'vt_filtered'@'localhost'; + # User for general MySQL monitoring. + CREATE USER 'vt_monitoring'@'localhost'; + GRANT SELECT, PROCESS, SUPER, REPLICATION CLIENT, RELOAD + ON *.* TO 'vt_monitoring'@'localhost'; + GRANT SELECT, UPDATE, DELETE, DROP + ON performance_schema.* TO 'vt_monitoring'@'localhost'; + # custom sql is used to add custom scripts like creating users/passwords. We use it in our tests # {{custom_sql}} diff --git a/test/config.json b/test/config.json index c911232ce74..1e278546c7a 100644 --- a/test/config.json +++ b/test/config.json @@ -1238,6 +1238,17 @@ "RetryMax": 1, "Tags": [] }, + "vtop_example": { + "File": "", + "Args": [], + "Command": [ + "test/vtop_example.sh" + ], + "Manual": false, + "Shard": "", + "RetryMax": 1, + "Tags": [] + }, "vtorc_primary_failure": { "File": "unused.go", "Args": ["vitess.io/vitess/go/test/endtoend/vtorc/primaryfailure"], diff --git a/test/vtop_example.sh b/test/vtop_example.sh index 5ff90a2be7e..c537c0f844c 100755 --- a/test/vtop_example.sh +++ b/test/vtop_example.sh @@ -482,11 +482,12 @@ EOF waitForKeyspaceToBeServing customer 80- 1 } +kind delete cluster --name kind || true # Build the docker image for vitess/lite using the local code docker build -f docker/lite/Dockerfile -t vitess/lite:pr . # Build the docker image for vitess/vtadmin using the local code -docker build -f docker/binaries/vtadmin/Dockerfile --build-arg VT_BASE_VER=pr -t vitess/vtadmin:pr . +docker build -f docker/binaries/vtadmin/Dockerfile --build-arg VT_BASE_VER=pr -t vitess/vtadmin:pr ./docker/binaries/vtadmin # Print the docker images available docker image ls diff --git a/tools/get_kubectl_kind.sh b/tools/get_kubectl_kind.sh index 57df414fdd8..169b120aaa0 100755 --- a/tools/get_kubectl_kind.sh +++ b/tools/get_kubectl_kind.sh @@ -12,7 +12,7 @@ source build.env mkdir -p "$VTROOT/bin" cd "$VTROOT/bin" -KUBE_VERSION="${KUBE_VERSION:-v1.21.1}" +KUBE_VERSION="${KUBE_VERSION:-v1.31.0}" KUBERNETES_RELEASE_URL="${KUBERNETES_RELEASE_URL:-https://dl.k8s.io}" # Download kubectl if needed. @@ -28,7 +28,7 @@ ln -sf "kubectl-${KUBE_VERSION}" kubectl if ! command -v kind &> /dev/null then echo "Downloading kind..." - curl -L https://kind.sigs.k8s.io/dl/v0.12.0/kind-linux-amd64 > "kind" + curl -L https://kind.sigs.k8s.io/dl/v0.22.0/kind-linux-amd64 > "kind" chmod +x "kind" echo "Installed kind" else