Merge pull request #25 from rht-labs/feature/multibranch-github

🕺 ADD - multibranch for github to dsl 🕺
rht-labs · Jul 15, 2020 · d2b96cc · d2b96cc
2 parents 32143ab + 4ef0fef
commit d2b96cc
Show file tree

Hide file tree

Showing 3 changed files with 236 additions and 53 deletions.
diff --git a/configuration/jobs/seed-multibranch-pipelines/config.xml b/configuration/jobs/seed-multibranch-pipelines/config.xml
@@ -17,19 +17,15 @@
         <artifactNumToKeep>-1</artifactNumToKeep>
       </strategy>
     </jenkins.model.BuildDiscarderProperty>
-    <com.sonyericsson.jenkins.plugins.bfa.model.ScannerJobProperty plugin="build-failure-analyzer@1.20.0">
+    <com.sonyericsson.jenkins.plugins.bfa.model.ScannerJobProperty plugin="build-failure-analyzer@1.26.0">
       <doNotScan>false</doNotScan>
     </com.sonyericsson.jenkins.plugins.bfa.model.ScannerJobProperty>
-    <com.dabsquared.gitlabjenkins.connection.GitLabConnectionProperty plugin="[email protected].9">
+    <com.dabsquared.gitlabjenkins.connection.GitLabConnectionProperty plugin="[email protected].13">
       <gitLabConnection></gitLabConnection>
     </com.dabsquared.gitlabjenkins.connection.GitLabConnectionProperty>
-    <org.jenkinsci.plugins.gitlablogo.GitlabLogoProperty plugin="[email protected].3">
+    <org.jenkinsci.plugins.gitlablogo.GitlabLogoProperty plugin="[email protected].5">
       <repositoryName></repositoryName>
     </org.jenkinsci.plugins.gitlablogo.GitlabLogoProperty>
-    <org.jenkinsci.plugins.gogs.GogsProjectProperty plugin="[email protected]">
-      <gogsSecret></gogsSecret>
-      <gogsUsePayload>false</gogsUsePayload>
-    </org.jenkinsci.plugins.gogs.GogsProjectProperty>
     <com.sonyericsson.rebuild.RebuildSettings plugin="[email protected]">
       <autoRebuild>false</autoRebuild>
       <rebuildDisabled>false</rebuildDisabled>
@@ -48,70 +44,131 @@
   </triggers>
   <concurrentBuild>false</concurrentBuild>
   <builders>
-    <javaposse.jobdsl.plugin.ExecuteDslScripts plugin="job-dsl@1.70">
+    <javaposse.jobdsl.plugin.ExecuteDslScripts plugin="job-dsl@1.77">
       <scriptText>// Groovy script used to seed Jenkins with multi-branch pipeline jobs:
 // 1. Call GitLab API to get each git project in a given group
 // 2. Check if project is archived, if so skip it.
 // 3. Check if there is a Jenkinsfile (on master) in each of the found projects
 // 4. Generate a pipeline using the Jenkinsfile and add it to the queue on first creation
 // 5. Every 10 mins run again
 
-def gitlabHost = System.getenv(&quot;GITLAB_HOST&quot;) ?: &quot;https://gitlab.apps.proj.example.com&quot;
-def gitlabToken = System.getenv(&quot;GITLAB_TOKEN&quot;) ?: &quot;mytoken123&quot;
-def projectName = System.getenv(&quot;GITLAB_GROUP_NAME&quot;) ?: &quot;rht-labs&quot;
-def projectsApi = new URL(&quot;${gitlabHost}/api/v4/groups/${projectName}/projects?per_page=100&quot;)
 
+// GITLAB
+def gitlabHost = System.getenv("GITLAB_HOST") ?: "https://gitlab.apps.proj.example.com"
+def gitlabToken = System.getenv("GITLAB_TOKEN")
+def projectName = System.getenv("GITLAB_GROUP_NAME") ?: "rht-labs"
+def gitlabProjectsApi = new URL("${gitlabHost}/api/v4/groups/${projectName}/projects?per_page=100")
 
-try {
-    def projects = new groovy.json.JsonSlurper().parse(projectsApi.newReader(requestProperties: [&apos;PRIVATE-TOKEN&apos;: gitlabToken]))
 
-    projects.each {
-        def project = &quot;${it.path}&quot;
-        def gitPath = it.http_url_to_repo
+// GITHUB
+def githubHost = "https://api.github.com"
+// Token needed for rate limiting issues....
+def githubToken = System.getenv("GITHUB_TOKEN")
+def githubAccount = System.getenv("GITHUB_ACCOUNT")
+def githubOrg = System.getenv("GITHUB_ORG") ?: false
+//eg  https://api.github.com/users/springdo/repos or 
 
-        if (it.archived) {
-            print &quot;skipping project ${project} because it has been archived\n\n&quot;
-            return
-        }
+def githubProjects = githubOrg ? new URL("${githubHost}/org/${githubAccount}/repos?per_page=100") : new URL("${githubHost}/users/${githubAccount}/repos?per_page=100")
 
-        try {
-            def filesApi = new URL(&quot;${gitlabHost}/api/v4/projects/${it.id}/repository/files/Jenkinsfile?ref=master&quot;)
-            def files = new groovy.json.JsonSlurper().parse(filesApi.newReader(requestProperties: [&apos;PRIVATE-TOKEN&apos;: gitlabToken]))
 
-            if (!jenkins.model.Jenkins.instance.getItemByFullName(project)) {
-                print &quot;About to create ${project} for the first time, this will result in a triggering the build after this run to prepare the ${project} pipeline\n\n&quot;
-                queue(project)
-            }
+def createMultibranchPipelineJob(project, gitPath) {
+    def buildNamespace = System.getenv("BUILD_NAMESPACE") ?: "labs-ci-cd"
+    def buildGitAuthSecret = System.getenv("BUILD_GIT_AUTH_SECRET") ?: "git-auth"
 
-            // Build Jenkins multibranc jobs
-            multibranchPipelineJob(project) {
-                branchSources {
-                    git {
-                        remote(gitPath)
-                        credentialsId(&apos;labs-ci-cd-jenkins-git-password&apos;)
-                    }
-                }
-                triggers {
-                    computedFolderWebHookTrigger {
-                      // The token to match with webhook token.
-                    	token(project)
-                    }
-                }
-                orphanedItemStrategy {
-                    discardOldItems {
-                        numToKeep(10)
-                    }
-                }
+    // Build Jenkins multibranc jobs
+    multibranchPipelineJob(project) {
+        branchSources {
+            git {
+                id("${project}")
+                remote(gitPath)
+                credentialsId("${buildNamespace}-${buildGitAuthSecret}")
+            }
+        }
+        triggers {
+            computedFolderWebHookTrigger {
+              // The token to match with webhook token.
+              token(project)
             }
         }
-        catch(Exception e) {
-                println e
-                print &quot;skipping project ${project} because it has no Jenkinsfile\n\n&quot;
+        orphanedItemStrategy {
+            discardOldItems {
+              // Set to 0 to autoprune jobs once branch is deleted 
+              numToKeep(0)
+            }
         }
     }
-} catch(Exception e) {
-    print &quot;\n\n Please make sure you have set  GITLAB_HOST, GITLAB_TOKEN and GITLAB_GROUP_NAME in your deploy config for Jenkins \n\n\n&quot;
-    throw e
+}
+
+def addJobToQueue(project){
+  if (!jenkins.model.Jenkins.instance.getItemByFullName(project)) {
+    print "About to create ${project} for the first time, this will result in a triggering the build after this run to prepare the ${project} pipeline\n\n"
+    queue(project)
+  }
+}
+// if GITLAB* set ....
+if (gitlabToken) {
+  try {
+      def projects = new groovy.json.JsonSlurper().parse(gitlabProjectsApi.newReader(requestProperties: ['PRIVATE-TOKEN': gitlabToken]))
+
+      projects.each {
+          def project = "${it.path}"
+          def gitPath = it.http_url_to_repo
+
+          if (it.archived) {
+              print "skipping project ${project} because it has been archived\n\n"
+              return
+          }
+
+          try {
+              def filesApi = new URL("${gitlabHost}/api/v4/projects/${it.id}/repository/files/Jenkinsfile?ref=master")
+              def files = new groovy.json.JsonSlurper().parse(filesApi.newReader(requestProperties: ['PRIVATE-TOKEN': gitlabToken]))
+
+              createMultibranchPipelineJob(project, gitPath)
+              addJobToQueue(project)
+
+          }
+          catch(Exception e) {
+                  println e
+                  print "skipping project ${project} because it has no Jenkinsfile\n\n"
+          }
+      }
+  } catch(Exception e) {
+      print "\n\n Please make sure you have set  GITLAB_HOST, GITLAB_TOKEN and GITLAB_GROUP_NAME in your deploy config for Jenkins \n\n\n"
+      throw e
+  }
+} else if (githubAccount) {
+  try {
+      def projects = new groovy.json.JsonSlurper().parse(githubProjects.newReader(requestProperties: ['Authorization': "token ${githubToken}"]))
+
+      projects.each {
+          def project = it.name
+          def gitPath = it.clone_url
+
+          if (it.archived) {
+              print "skipping project ${project} because it has been archived\n\n"
+              return
+          }
+
+          try {
+            // https://api.github.com/repos/$ORG or $USER/name/contents/Jenkinsfile
+              def filesApi = new URL("${githubHost}/repos/${githubAccount}/${project}/contents/Jenkinsfile")
+              def files = new groovy.json.JsonSlurper().parse(filesApi.newReader(requestProperties: ['Authorization': "token ${githubToken}"]))
+
+
+              createMultibranchPipelineJob(project, gitPath)
+              addJobToQueue(project)
+          }
+          catch(Exception e) {
+                  println e
+                  print "skipping project ${project} because it has no Jenkinsfile\n\n"
+          }
+      }
+  } catch(Exception e) {
+      print "\n\n Oops! something went wrong..... Try setting the GITHUB_* Env Vars \n\n\n"
+      throw e
+  }
+} else {
+    print "\n\n No tokens set in the Environment eg GITHUB* or GITLAB* so not sure what to do ..... &#129335;‍♂️ \n\n\n"
 }</scriptText>
       <usingScriptText>true</usingScriptText>
       <sandbox>false</sandbox>

diff --git a/configuration/jobs/seed-multibranch-pipelines/seed-multibranch-pipeline.groovy b/configuration/jobs/seed-multibranch-pipelines/seed-multibranch-pipeline.groovy
@@ -0,0 +1,125 @@
+// Groovy script used to seed Jenkins with multi-branch pipeline jobs:
+// 1. Call GitLab API to get each git project in a given group
+// 2. Check if project is archived, if so skip it.
+// 3. Check if there is a Jenkinsfile (on master) in each of the found projects
+// 4. Generate a pipeline using the Jenkinsfile and add it to the queue on first creation
+// 5. Every 10 mins run again
+
+
+// GITLAB
+def gitlabHost = System.getenv("GITLAB_HOST") ?: "https://gitlab.apps.proj.example.com"
+def gitlabToken = System.getenv("GITLAB_TOKEN")
+def projectName = System.getenv("GITLAB_GROUP_NAME") ?: "rht-labs"
+def gitlabProjectsApi = new URL("${gitlabHost}/api/v4/groups/${projectName}/projects?per_page=100")
+
+
+// GITHUB
+def githubHost = "https://api.github.com"
+// Token needed for rate limiting issues....
+def githubToken = System.getenv("GITHUB_TOKEN")
+def githubAccount = System.getenv("GITHUB_ACCOUNT")
+def githubOrg = System.getenv("GITHUB_ORG") ?: false
+//eg  https://api.github.com/users/springdo/repos or 
+
+def githubProjects = githubOrg ? new URL("${githubHost}/org/${githubAccount}/repos?per_page=100") : new URL("${githubHost}/users/${githubAccount}/repos?per_page=100")
+
+
+def createMultibranchPipelineJob(project, gitPath) {
+    def buildNamespace = System.getenv("BUILD_NAMESPACE") ?: "labs-ci-cd"
+    def buildGitAuthSecret = System.getenv("BUILD_GIT_AUTH_SECRET") ?: "git-auth"
+
+    // Build Jenkins multibranc jobs
+    multibranchPipelineJob(project) {
+        branchSources {
+            git {
+                id("${project}")
+                remote(gitPath)
+                credentialsId("${buildNamespace}-${buildGitAuthSecret}")
+            }
+        }
+        triggers {
+            computedFolderWebHookTrigger {
+              // The token to match with webhook token.
+              token(project)
+            }
+        }
+        orphanedItemStrategy {
+            discardOldItems {
+              // Set to 0 to autoprune jobs once branch is deleted 
+              numToKeep(0)
+            }
+        }
+    }
+}
+
+def addJobToQueue(project){
+  if (!jenkins.model.Jenkins.instance.getItemByFullName(project)) {
+    print "About to create ${project} for the first time, this will result in a triggering the build after this run to prepare the ${project} pipeline\n\n"
+    queue(project)
+  }
+}
+// if GITLAB* set ....
+if (gitlabToken) {
+  try {
+      def projects = new groovy.json.JsonSlurper().parse(gitlabProjectsApi.newReader(requestProperties: ['PRIVATE-TOKEN': gitlabToken]))
+
+      projects.each {
+          def project = "${it.path}"
+          def gitPath = it.http_url_to_repo
+
+          if (it.archived) {
+              print "skipping project ${project} because it has been archived\n\n"
+              return
+          }
+
+          try {
+              def filesApi = new URL("${gitlabHost}/api/v4/projects/${it.id}/repository/files/Jenkinsfile?ref=master")
+              def files = new groovy.json.JsonSlurper().parse(filesApi.newReader(requestProperties: ['PRIVATE-TOKEN': gitlabToken]))
+
+              createMultibranchPipelineJob(project, gitPath)
+              addJobToQueue(project)
+
+          }
+          catch(Exception e) {
+                  println e
+                  print "skipping project ${project} because it has no Jenkinsfile\n\n"
+          }
+      }
+  } catch(Exception e) {
+      print "\n\n Please make sure you have set  GITLAB_HOST, GITLAB_TOKEN and GITLAB_GROUP_NAME in your deploy config for Jenkins \n\n\n"
+      throw e
+  }
+} else if (githubAccount) {
+  try {
+      def projects = new groovy.json.JsonSlurper().parse(githubProjects.newReader(requestProperties: ['Authorization': "token ${githubToken}"]))
+
+      projects.each {
+          def project = it.name
+          def gitPath = it.clone_url
+
+          if (it.archived) {
+              print "skipping project ${project} because it has been archived\n\n"
+              return
+          }
+
+          try {
+            // https://api.github.com/repos/$ORG or $USER/name/contents/Jenkinsfile
+              def filesApi = new URL("${githubHost}/repos/${githubAccount}/${project}/contents/Jenkinsfile")
+              def files = new groovy.json.JsonSlurper().parse(filesApi.newReader(requestProperties: ['Authorization': "token ${githubToken}"]))
+
+
+              createMultibranchPipelineJob(project, gitPath)
+              addJobToQueue(project)
+          }
+          catch(Exception e) {
+                  println e
+                  print "skipping project ${project} because it has no Jenkinsfile\n\n"
+          }
+      }
+  } catch(Exception e) {
+      print "\n\n Oops! something went wrong..... Try setting the GITHUB_* Env Vars \n\n\n"
+      throw e
+  }
+} else {
+    print "\n\n No tokens set in the Environment eg GITHUB* or GITLAB* so not sure what to do ..... 🤷‍♂️ \n\n\n"
+}
diff --git a/plugins.txt b/plugins.txt
@@ -50,6 +50,7 @@ config-file-provider:3.6.3
 copy-to-slave:1.4.4
 credentials-binding:1.23
 credentials:2.3.7
+cucumber-reports:5.3.0
 cvs:2.16
 dashboard-view:2.12
 display-url-api:2.3.2