Jenkinsfile2

import hudson.model.Result
import hudson.model.*;
import jenkins.model.CauseOfInterruption
node {
}
file_zh_changed = ''
file_en_changed = ''
file_no_doc_changed = '1'
file_only_tdgpt_change_except = '1'
tdgpt_file = "forecastoperator.c|anomalywindowoperator.c|tanalytics.h|tanalytics.c|tdgpt_cases.task|analytics"


def abortPreviousBuilds() {
    def currentJobName = env.JOB_NAME
    def currentBuildNumber = env.BUILD_NUMBER.toInteger()
    def jobs = Jenkins.instance.getItemByFullName(currentJobName)
    def builds = jobs.getBuilds()

    for (build in builds) {
        if (!build.isBuilding()) {
            continue;
        }

        if (currentBuildNumber == build.getNumber().toInteger()) {
            continue;
        }

        build.doKill()    //doTerm(),doKill(),doTerm()
    }
}
//  abort previous build
abortPreviousBuilds()
def abort_previous(){
    def buildNumber = env.BUILD_NUMBER as int
    if (buildNumber > 1) milestone(buildNumber - 1)
    milestone(buildNumber)
}
def check_docs(){
    if (env.CHANGE_URL =~ /\/TDengine\//) {
        sh '''
            hostname
            date
            env
        '''
        sh '''
            cd ${WKC}
            git reset --hard
            git clean -f
            git remote prune origin
            git fetch
            git checkout ''' + env.CHANGE_TARGET + '''
            git pull >/dev/null
            git fetch origin +refs/pull/${CHANGE_ID}/merge
            git checkout -qf FETCH_HEAD 
        '''

        file_zh_changed = sh (
            script: '''
                cd ${WKC}
                git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${CHANGE_TARGET}`|grep  "^docs/zh/" || :
            ''',
            returnStdout: true
        )

        file_en_changed = sh (
            script: '''
                cd ${WKC}
                git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${CHANGE_TARGET}`|grep  "^docs/en/" || :
            ''',
            returnStdout: true
        )
        
        file_no_doc_changed = sh (
            script: '''
                cd ${WKC}
                git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${CHANGE_TARGET}`|grep -v "^docs/en/"|grep -v "^docs/zh/"|grep -v ".md$" || :
            ''',
            returnStdout: true
        ).trim()

        file_only_tdgpt_change_except = sh (
            script: '''
                cd ${WKC}
                git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${CHANGE_TARGET}`|grep -v "^docs/en/"|grep -v "^docs/zh/"|grep -v ".md$" | grep -Ev "forecastoperator.c|anomalywindowoperator.c|tanalytics.h|tanalytics.c|tdgpt_cases.task|analytics" ||:
            ''',
            returnStdout: true
        ).trim()

        echo "file_zh_changed: ${file_zh_changed}"
        echo "file_en_changed: ${file_en_changed}"
        echo "file_no_doc_changed: ${file_no_doc_changed}"
        echo "file_only_tdgpt_change_except: ${file_only_tdgpt_change_except}"
    }
}

def build_pre_docs(){
    if (env.CHANGE_URL =~ /\/TDengine\//) {
        sh '''
            hostname
            date
            env
        '''
        
        sh '''
            cd ${DOC_WKC}/${td_repo}
            git reset --hard
            git clean -f
            git remote prune origin
            git fetch
            git checkout ''' + env.CHANGE_TARGET + '''
            git pull >/dev/null
            git fetch origin +refs/pull/${CHANGE_ID}/merge
            git checkout -qf FETCH_HEAD 
        '''
    }
}

def build_zh_docs(){
    sh '''
        cd ${DOC_WKC}/${zh_doc_repo}
        # git pull 
        yarn ass local
        yarn build
    '''
}

def build_en_docs(){
    sh '''
        cd ${DOC_WKC}/${en_doc_repo}
        # git pull 
        yarn ass local
        yarn build
    '''
}


def pre_test(){
    sh '''
        hostname
        date
        env
    '''
    sh '''
        cd ${WK}
        git reset --hard
        git remote prune origin
        git fetch
        cd ${WKC}
        git reset --hard
        git clean -f
        rm -rf examples/rust/
        git remote prune origin
        git fetch
    '''
    script {
        sh '''
            cd ${WK}
            git checkout ''' + env.CHANGE_TARGET + '''
            cd ${WKC}
            git checkout ''' + env.CHANGE_TARGET + '''
        '''
    }
    if (env.CHANGE_URL =~ /\/TDengine\//) {
        sh '''
            cd ${WKC}
            git remote prune origin
            git pull >/dev/null
            git log -5
            echo "`date "+%Y%m%d-%H%M%S"` ${JOB_NAME}:${BRANCH_NAME}:${BUILD_ID}:${CHANGE_TARGET}" >>${WKDIR}/jenkins.log
            echo "CHANGE_BRANCH:${CHANGE_BRANCH}" >>${WKDIR}/jenkins.log
            echo "community log: `git log -5`" >>${WKDIR}/jenkins.log
            git fetch origin +refs/pull/${CHANGE_ID}/merge
            git checkout -qf FETCH_HEAD
            git log -5
            echo "community log merged: `git log -5`" >>${WKDIR}/jenkins.log
            cd ${WK}
            git pull >/dev/null
            git log -5
            echo "tdinternal log: `git log -5`" >>${WKDIR}/jenkins.log
        '''
    } else if (env.CHANGE_URL =~ /\/TDinternal\//) {
        sh '''
            cd ${WK}
            git pull >/dev/null
            git log -5
            echo "`date "+%Y%m%d-%H%M%S"` ${JOB_NAME}:${BRANCH_NAME}:${BUILD_ID}:${CHANGE_TARGET}" >>${WKDIR}/jenkins.log
            echo "CHANGE_BRANCH:${CHANGE_BRANCH}" >>${WKDIR}/jenkins.log
            echo "tdinternal log: `git log -5`" >>${WKDIR}/jenkins.log
            git fetch origin +refs/pull/${CHANGE_ID}/merge
            git checkout -qf FETCH_HEAD
            git log -5
            echo "tdinternal log merged: `git log -5`" >>${WKDIR}/jenkins.log
            cd ${WKC}
            git remote prune origin
            git pull >/dev/null
            git log -5
            echo "community log: `git log -5`" >>${WKDIR}/jenkins.log
        '''
    } else {
        sh '''
            echo "unmatched repository ${CHANGE_URL}"
        '''
    }
    sh '''
        cd ${WKC}
        git rm --cached tools/taos-tools 2>/dev/null || :
        git rm --cached tools/taosadapter 2>/dev/null || :
        git rm --cached tools/taosws-rs 2>/dev/null || :
        git rm --cached examples/rust 2>/dev/null || :
    '''
    sh '''
        cd ${WKC}
        git submodule update --init --recursive
    '''
    return 1
}

def pre_test_build_mac() {
    sh '''
        hostname
        date
    '''
    sh '''
        cd ${WK}
        rm -rf debug
        mkdir debug
    '''
    sh '''
        cd ${WK}/debug
        cmake .. -DBUILD_TEST=true -DBUILD_HTTPS=false  -DCMAKE_BUILD_TYPE=Release
        make -j10
        ctest -j10 || exit 7
    '''
    sh '''
        date
    '''
}

def pre_test_win(){
    bat '''
        hostname
        taskkill /f /t /im python.exe
        taskkill /f /t /im bash.exe
        taskkill /f /t /im taosd.exe
        ipconfig
        set
        date /t
        time /t
        rd /s /Q %WIN_INTERNAL_ROOT%\\debug || exit 0
    '''
    bat '''
        cd %WIN_INTERNAL_ROOT%
        git reset --hard
        git remote prune origin
        git fetch
    '''
    bat '''
        cd %WIN_COMMUNITY_ROOT%
        git clean -f
        git reset --hard
        git remote prune origin
        git fetch
    '''
    script {
        bat '''
            cd %WIN_INTERNAL_ROOT%
            git checkout ''' + env.CHANGE_TARGET + '''
        '''
        bat '''
            cd %WIN_COMMUNITY_ROOT%
            git checkout ''' + env.CHANGE_TARGET + '''
        '''
    }
    script {
        if (env.CHANGE_URL =~ /\/TDengine\//) {
            bat '''
                cd %WIN_INTERNAL_ROOT%
                git pull origin ''' + env.CHANGE_TARGET + '''
            '''
            bat '''
                cd %WIN_COMMUNITY_ROOT%
                git remote prune origin
                git pull  origin ''' + env.CHANGE_TARGET + '''
            '''
            bat '''
                cd %WIN_COMMUNITY_ROOT%
                git fetch origin +refs/pull/%CHANGE_ID%/merge
            '''
            bat '''
                cd %WIN_COMMUNITY_ROOT%
                git checkout -qf FETCH_HEAD
            '''
        } else if (env.CHANGE_URL =~ /\/TDinternal\//) {
            bat '''
                cd %WIN_INTERNAL_ROOT%
                git pull origin ''' + env.CHANGE_TARGET + '''
            '''
            bat '''
                cd %WIN_INTERNAL_ROOT%
                git fetch origin +refs/pull/%CHANGE_ID%/merge
            '''
            bat '''
                cd %WIN_INTERNAL_ROOT%
                git checkout -qf FETCH_HEAD
            '''
            bat '''
                cd %WIN_COMMUNITY_ROOT%
                git remote prune origin
                git pull
            '''
        } else {
            bat '''
                echo "unmatched repository %CHANGE_URL%"
            '''
        }
    }
    bat '''
        cd %WIN_INTERNAL_ROOT%
        git branch
        git log -5
    '''
    bat '''
        cd %WIN_COMMUNITY_ROOT%
        git branch
        git log -5
    '''
    bat '''
        cd %WIN_COMMUNITY_ROOT%
        git rm --cached tools/taos-tools 2>nul
        git rm --cached tools/taosadapter 2>nul
        git rm --cached tools/taosws-rs 2>nul
        git rm --cached examples/rust 2>nul
        exit 0
    '''
    bat '''
        cd %WIN_COMMUNITY_ROOT%
        git submodule update --init --recursive
    '''
}

def pre_test_build_win() {
    bat '''
        echo "building ..."
        time /t
        cd %WIN_INTERNAL_ROOT%
        mkdir debug
        cd debug
        time /t
        call "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" x64
        set CL=/MP8
        echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> cmake"
        time /t
        cmake .. -G "NMake Makefiles JOM" -DBUILD_TEST=true -DBUILD_TOOLS=true || exit 7
        echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> jom -j 6"
        time /t
        jom -j 6 || exit 8
        time /t
    '''
    bat '''
        cd %WIN_COMMUNITY_ROOT%/tests/ci
        pip3 install taospy==2.7.21 
        pip3 install taos-ws-py==0.3.8
        xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32
    '''
    return 1
}

def run_win_ctest() {
    bat '''
        echo "windows ctest ..."
        time /t
        cd %WIN_INTERNAL_ROOT%\\debug
        ctest -j 1 || exit 7
        time /t
    '''
}

def run_win_test() {
    bat '''
        echo "windows test ..."
        xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32
        ls -l C:\\Windows\\System32\\taos.dll
        time /t
        cd %WIN_SYSTEM_TEST_ROOT%
        echo "testing ..."
        test-all.bat ci
        time /t
    '''
}

pipeline {
    agent any
    options { skipDefaultCheckout() } 
    environment{
        WKDIR = '/var/lib/jenkins/workspace'
        WK = '/var/lib/jenkins/workspace/TDinternal'
        WKC = '/var/lib/jenkins/workspace/TDinternal/community'
        WKPY = '/var/lib/jenkins/workspace/taos-connector-python'
        DOC_WKC = '/root/doc_ci_work'
        td_repo = 'TDengine'
        zh_doc_repo = 'docs.taosdata.com'
        en_doc_repo = 'docs.tdengine.com'
        tools_repo = 'taos-tools'
    }
    stages {
        stage ('check doc file changed') {
            agent{label " slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52  || slave1_59 || slave1_63 ||  worker03 || slave215 || slave217 || slave219 || Mac_catalina "}            
            steps {
                check_docs()
            }
        }
        
        stage ('pre for build docs') {
            when {
                beforeAgent true
                expression { env.CHANGE_BRANCH =~ /(?i)doc.*/ || file_zh_changed != '' || file_en_changed != '' }
            }
            agent{label "doc_build_0_30"}
            steps {
                build_pre_docs()
            }
        }

        stage('build Docs') {
            when {
                beforeAgent true
                expression { env.CHANGE_BRANCH =~ /(?i)doc.*/ || file_zh_changed != '' || file_en_changed != '' }
            }
            parallel {
                stage('build zh docs') {
                    agent{label "doc_build_0_30"}
                    when {
                        expression { file_zh_changed != '' }
                    }
                    steps {
                        build_zh_docs()
                    }
                }
                stage('build en docs') {
                    agent{label "doc_build_0_30"}
                    when {
                        expression { file_en_changed != '' }
                    }
                    steps {
                        build_en_docs()
                    }
                }
            }
            post {
                unsuccessful {
                    error('build docs stage failed, terminating pipeline.')
                }
            }
        }

        stage('run test') {
            when {
                expression {
                    file_no_doc_changed != '' && env.CHANGE_TARGET != 'docs-cloud'
                }
            }
            parallel {
                stage('windows test') {
                    agent{label " windows10_01 || windows10_02 || windows10_03 || windows10_04 "}
                    environment{
                        WIN_INTERNAL_ROOT="C:\\workspace\\${env.EXECUTOR_NUMBER}\\TDinternal"
                        WIN_COMMUNITY_ROOT="C:\\workspace\\${env.EXECUTOR_NUMBER}\\TDinternal\\community"
                        WIN_SYSTEM_TEST_ROOT="C:\\workspace\\${env.EXECUTOR_NUMBER}\\TDinternal\\community\\tests\\system-test"
                    }
                    when {
                        beforeAgent true
                        expression { file_only_tdgpt_change_except != '' }
                    }
                    steps {
                        catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
                            timeout(time: 126, unit: 'MINUTES'){
                                    pre_test_win()
                                    pre_test_build_win()
                                    run_win_ctest()
                                    run_win_test()
                            }
                        }
                    }
                }
                stage('mac test') {
                    agent{label " Mac_catalina "}
                    when {
                        beforeAgent true
                        expression { file_only_tdgpt_change_except != '' }
                    }
                    steps {
                        catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
                            timeout(time: 60, unit: 'MINUTES'){
                                pre_test()
                                pre_test_build_mac()
                            }
                        }
                    }
                }
                stage('linux test') {
                    agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || slave1_59 || slave1_63 || worker03 || slave215 || slave217 || slave219 "}
                    options { skipDefaultCheckout() } 
                    when {
                        changeRequest()
                    }
                    steps {
                        script {
                            def linux_node_ip = sh (
                                script: 'ip addr|grep 192|grep -v virbr|awk "{print \\\$2}"|sed "s/\\/.*//"',
                                returnStdout: true
                            ).trim()
                            echo "${linux_node_ip}"
                            echo "${WKDIR}/restore.sh -p ${BRANCH_NAME} -n ${BUILD_ID} -c {container name}"
                        }
                        catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
                            timeout(time: 200, unit: 'MINUTES'){
                                pre_test()
                                script {
                                    sh '''
                                    mkdir -p ${WKDIR}/tmp/${BRANCH_NAME}_${BUILD_ID}
                                    echo "''' + file_no_doc_changed + '''" > ${WKDIR}/tmp/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt
                                    '''
                                    sh '''
                                       cd ${WKC}/tests/parallel_test
                                       ./run_check_assert_container.sh -d ${WKDIR}
                                    '''
                                    sh '''
                                       cd ${WKC}/tests/parallel_test
                                       ./run_check_void_container.sh -d ${WKDIR}
                                    '''
                                    sh '''
                                        date
                                        rm -rf ${WKC}/debug
                                        cd ${WKC}/tests/parallel_test
                                        time ./container_build.sh -w ${WKDIR} -e
                                    '''
                                    def extra_param = ""
                                    def log_server_file = "/home/log_server.json"
                                    def timeout_cmd = ""
                                    if (fileExists(log_server_file)) {
                                        def log_server_enabled = sh (
                                            script: 'jq .enabled ' + log_server_file,
                                            returnStdout: true
                                        ).trim()
                                        def timeout_param = sh (
                                            script: 'jq .timeout ' + log_server_file,
                                            returnStdout: true
                                        ).trim()
                                        if (timeout_param != "null" &&  timeout_param != "0") {
                                            timeout_cmd = "timeout " + timeout_param
                                        }
                                        if (log_server_enabled == "1") {
                                            def log_server = sh (
                                                script: 'jq .server ' + log_server_file + ' | sed "s/\\\"//g"',
                                                returnStdout: true
                                            ).trim()
                                            if (log_server != "null" && log_server != "") { 
                                                extra_param = "-w " + log_server
                                            }
                                        }
                                    }
                                    sh '''
                                       cd ${WKC}/tests/parallel_test
                                       ./run_scan_container.sh -d ${WKDIR} -b ${BRANCH_NAME}_${BUILD_ID}  -f ${WKDIR}/tmp/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt ''' + extra_param + '''
                                    '''
                                     if ( file_no_doc_changed =~ /forecastoperator.c|anomalywindowoperator.c|tanalytics.h|tanalytics.c|tdgpt_cases.task|analytics/ ) {
                                        sh '''
                                            cd ${WKC}/tests/parallel_test
                                            export DEFAULT_RETRY_TIME=2
                                            date
                                            timeout 600 time ./run.sh -e -m /home/m.json -t tdgpt_cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 300 ''' + extra_param + '''
                                        '''
                                    }
                                    if ( file_only_tdgpt_change_except != '' ) {
                                        sh '''
                                            cd ${WKC}/tests/parallel_test
                                            export DEFAULT_RETRY_TIME=2
                                            date
                                            ''' + timeout_cmd + ''' time ./run.sh -e -m /home/m.json -t cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 1200 ''' + extra_param + '''
                                        '''
                                    }
                                }
                            }
                        }
                    }
                }
            }
        }
    }
    post {  
        success {
            emailext (
                subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' SUCCESS",
                body: """<!DOCTYPE html>
                <html>
                <head>
                <meta charset="UTF-8">
                </head>
                <body leftmargin="8" marginwidth="0" topmargin="8" marginheight="4" offset="0">
                    <table width="95%" cellpadding="0" cellspacing="0" style="font-size: 16pt; font-family: Tahoma, Arial, Helvetica, sans-serif">
                        <tr>
                            <td><br />
                                <b><font color="#0B610B"><font size="6">构建信息</font></font></b>
                                <hr size="2" width="100%" align="center" /></td>
                        </tr>
                        <tr>
                            <td>
                                <ul>
                                <div style="font-size:18px">
                                    <li>构建名称>>分支：${env.BRANCH_NAME}</li>
                                    <li>构建结果：<span style="color:green"> Successful </span></li>
                                    <li>构建编号：${BUILD_NUMBER}</li>
                                    <li>触发用户：${env.CHANGE_AUTHOR}</li>
                                    <li>提交信息：${env.CHANGE_TITLE}</li>
                                    <li>构建地址：<a href=${BUILD_URL}>${BUILD_URL}</a></li>
                                    <li>构建日志：<a href=${BUILD_URL}console>${BUILD_URL}console</a></li>
                                    
                                </div>
                                </ul>
                            </td>
                        </tr>
                    </table></font>
                </body>
                </html>""",
                to: "${env.CHANGE_AUTHOR_EMAIL}",
                from: "support@taosdata.com"
            )
        }
        failure {
            emailext (
                subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' FAIL",
                body: """<!DOCTYPE html>
                <html>
                <head>
                <meta charset="UTF-8">
                </head>
                <body leftmargin="8" marginwidth="0" topmargin="8" marginheight="4" offset="0">
                    <table width="95%" cellpadding="0" cellspacing="0" style="font-size: 16pt; font-family: Tahoma, Arial, Helvetica, sans-serif">
                        <tr>
                            <td><br />
                                <b><font color="#0B610B"><font size="6">构建信息</font></font></b>
                                <hr size="2" width="100%" align="center" /></td>
                        </tr>
                        <tr>
                            <td>
                                <ul>
                                <div style="font-size:18px">
                                    <li>构建名称>>分支：${env.BRANCH_NAME}</li>
                                    <li>构建结果：<span style="color:red"> Failure </span></li>
                                    <li>构建编号：${BUILD_NUMBER}</li>
                                    <li>触发用户：${env.CHANGE_AUTHOR}</li>
                                    <li>提交信息：${env.CHANGE_TITLE}</li>
                                    <li>构建地址：<a href=${BUILD_URL}>${BUILD_URL}</a></li>
                                    <li>构建日志：<a href=${BUILD_URL}console>${BUILD_URL}console</a></li>
                                    
                                </div>
                                </ul>
                            </td>
                        </tr>
                    </table></font>
                </body>
                </html>""",
                to: "${env.CHANGE_AUTHOR_EMAIL}",
                from: "support@taosdata.com"
            )
        }
    } 
}