diff --git a/jobs/tikv/migration/latest/pull_integration_kafka_test.groovy b/jobs/tikv/migration/latest/pull_integration_kafka_test.groovy new file mode 100644 index 0000000000..2e2c9ee0e1 --- /dev/null +++ b/jobs/tikv/migration/latest/pull_integration_kafka_test.groovy @@ -0,0 +1,39 @@ +// REF: https:///plugin/job-dsl/api-viewer/index.html +// For trunk and latest release branches. +pipelineJob('tikv/migration/pull_integration_kafka_test') { + logRotator { + daysToKeep(30) + } + parameters { + // Ref: https://docs.prow.k8s.io/docs/jobs/#job-environment-variables + stringParam("BUILD_ID") + stringParam("PROW_JOB_ID") + stringParam("JOB_SPEC") + } + properties { + // priority(0) // 0 fast than 1 + githubProjectUrl("https://github.com/tikv/migration") + } + + definition { + cpsScm { + lightweight(true) + scriptPath("pipelines/tikv/migration/latest/pull_integration_kafka_test.groovy") + scm { + git{ + remote { + url('https://github.com/PingCAP-QE/ci.git') + } + branch('main') + extensions { + cloneOptions { + depth(1) + shallow(true) + timeout(5) + } + } + } + } + } + } +} diff --git a/pipelines/tikv/migration/latest/pod-pull_integration_kafka_test.yaml b/pipelines/tikv/migration/latest/pod-pull_integration_kafka_test.yaml new file mode 100644 index 0000000000..7bd376f8aa --- /dev/null +++ b/pipelines/tikv/migration/latest/pod-pull_integration_kafka_test.yaml @@ -0,0 +1,84 @@ +apiVersion: v1 +kind: Pod +spec: + securityContext: + fsGroup: 1000 + containers: + - name: zookeeper + image: wurstmeister/zookeeper + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 200m + memory: 4Gi + limits: + cpu: 2000m + memory: 4Gi + tty: true + volumeMounts: + - mountPath: /tmp + name: volume-0 + - name: kafka + image: wurstmeister/kafka:2.12-2.4.1 + imagePullPolicy: IfNotPresent + env: + - name: KAFKA_CREATE_TOPICS + value: big-message-test:1:1 + - name: KAFKA_BROKER_ID + value: "1" + - name: KAFKA_ZOOKEEPER_CONNECT + value: localhost:2181 + - name: KAFKA_MESSAGE_MAX_BYTES + value: "11534336" + - name: KAFKA_REPLICA_FETCH_MAX_BYTES + value: "11534336" + - name: KAFKA_ADVERTISED_LISTENERS + value: PLAINTEXT://127.0.0.1:9092 + - name: KAFKA_LISTENERS + value: PLAINTEXT://127.0.0.1:9092 + - name: ZK + value: zk + resources: + requests: + cpu: 200m + memory: 4Gi + limits: + cpu: 2000m + memory: 4Gi + tty: true + volumeMounts: + - mountPath: /tmp + name: volume-0 + - name: golang + image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" + tty: true + resources: + requests: + memory: 8Gi + cpu: "4" + limits: + memory: 8Gi + cpu: "4" + - name: net-tool + image: hub.pingcap.net/jenkins/network-multitool + tty: true + resources: + limits: + memory: 128Mi + cpu: 100m + - name: report + image: hub.pingcap.net/jenkins/python3-requests:latest + tty: true + resources: + limits: + memory: 256Mi + cpu: 100m + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 diff --git a/pipelines/tikv/migration/latest/pull_integration_kafka_test.groovy b/pipelines/tikv/migration/latest/pull_integration_kafka_test.groovy new file mode 100644 index 0000000000..f64aab99c7 --- /dev/null +++ b/pipelines/tikv/migration/latest/pull_integration_kafka_test.groovy @@ -0,0 +1,111 @@ +// REF: https://www.jenkins.io/doc/book/pipeline/syntax/#declarative-pipeline +// Keep small than 400 lines: https://issues.jenkins.io/browse/JENKINS-37984 +// should triggerd for master and latest release branches +@Library('tipipeline') _ + +final K8S_NAMESPACE = "jenkins-tidb" +final GIT_FULL_REPO_NAME = 'tikv/migration' +final GIT_CREDENTIALS_ID = 'github-sre-bot-ssh' +final POD_TEMPLATE_FILE = 'pipelines/tikv/migration/latest/pod-pull_integration_kafka_test.yaml' +final REFS = readJSON(text: params.JOB_SPEC).refs + +pipeline { + agent { + kubernetes { + namespace K8S_NAMESPACE + yamlFile POD_TEMPLATE_FILE + defaultContainer 'golang' + } + } + environment { + FILE_SERVER_URL = 'http://fileserver.pingcap.net' + } + options { + timeout(time: 40, unit: 'MINUTES') + parallelsAlwaysFailFast() + skipDefaultCheckout() + } + stages { + stage('Debug info') { + steps { + sh label: 'Debug info', script: """ + printenv + echo "-------------------------" + go env + echo "-------------------------" + echo "debug command: kubectl -n ${K8S_NAMESPACE} exec -ti ${NODE_NAME} bash" + """ + container(name: 'net-tool') { + sh 'dig github.com' + } + } + } + stage('Checkout') { + options { timeout(time: 5, unit: 'MINUTES') } + steps { + dir("migration") { + cache(path: "./", includes: '**/*', key: prow.getCacheKey('git', REFS), restoreKeys: prow.getRestoreKeys('git', REFS)) { + retry(2) { + script { + prow.checkoutRefs(REFS) + } + } + sh """ + git rev-parse --show-toplevel + git status + git status -s . + """ + } + } + } + } + stage('Prepare') { + steps { + dir('migration') { + cache(path: "./cdc", includes: '**/*', key: "ws/${BUILD_TAG}/tikvcdc") { + container("golang") { + sh label: 'integration test prepare', script: """#!/usr/bin/env bash + cd cdc/ + make prepare_test_binaries + make check_third_party_binary + make integration_test_build + """ + } + } + } + } + } + stage('Tests') { + matrix { + axes { + axis { + name 'TEST_GROUP' + values 'G00', 'G01', 'G02', 'G03', 'G04', 'G05', 'G06', 'G07', 'G08', 'G09', 'G10', 'G11', 'G12', 'others' + } + } + agent { + kubernetes { + namespace K8S_NAMESPACE + yamlFile POD_TEMPLATE_FILE + defaultContainer 'golang' + } + } + stages { + stage("Test") { + options { timeout(time: 25, unit: 'MINUTES') } + steps { + dir('migration') { + cache(path: "./cdc", includes: '**/*', key: "ws/${BUILD_TAG}/tikvcdc") { + sh label: "TEST_GROUP ${TEST_GROUP}",script: """#!/usr/bin/env bash + cd cdc/ + ./tests/integration_tests/run_group.sh kafka ${TEST_GROUP} + """ + } + } + } + } + } + } + } + } +} diff --git a/pipelines/tikv/migration/latest/pull_integration_test.groovy b/pipelines/tikv/migration/latest/pull_integration_test.groovy index 017c6ab6fa..5b5b85687a 100644 --- a/pipelines/tikv/migration/latest/pull_integration_test.groovy +++ b/pipelines/tikv/migration/latest/pull_integration_test.groovy @@ -98,7 +98,7 @@ pipeline { cache(path: "./cdc", includes: '**/*', key: "ws/${BUILD_TAG}/tikvcdc") { sh label: "TEST_GROUP ${TEST_GROUP}",script: """#!/usr/bin/env bash cd cdc/ - ./tests/integration_tests/run_group.sh ${TEST_GROUP} + ./tests/integration_tests/run_group.sh tikv ${TEST_GROUP} """ } } diff --git a/prow-jobs/tikv-migration-latest-presubmits.yaml b/prow-jobs/tikv-migration-latest-presubmits.yaml index a5a6c115c4..5e85325dd9 100644 --- a/prow-jobs/tikv-migration-latest-presubmits.yaml +++ b/prow-jobs/tikv-migration-latest-presubmits.yaml @@ -10,6 +10,19 @@ presubmits: rerun_command: "/test pull-integration-test" branches: - ^main$ + - ^cdc-release-.*$ + - ^br-release-.*$ + - name: tikv/migration/pull_integration_kafka_test + agent: jenkins + decorate: false # need add this. + always_run: true + context: pull-integration-kafka-test + trigger: "(?m)^/test (?:.*? )?pull-integration-kafka-test(?: .*?)?$" + rerun_command: "/test pull-integration-kafka-test" + branches: + - ^main$ + - ^cdc-release-.*$ + - ^br-release-.*$ - name: tikv/migration/pull_unit_test agent: jenkins decorate: false # need add this. @@ -19,3 +32,5 @@ presubmits: rerun_command: "/test pull-unit-test" branches: - ^main$ + - ^cdc-release-.*$ + - ^br-release-.*$