From 44839e3f8431e237266db431bfc450e916f75a2d Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Mon, 17 Oct 2022 10:14:30 +0530 Subject: [PATCH 1/3] quality pipeline in gitlab --- .../setup-quality-pipeline.asciidoc | 1 + .../setup-quality-pipeline.asciidoc | 16 +- .../github/setup-quality-pipeline.asciidoc | 2 +- .../gitlab/setup-quality-pipeline.asciidoc | 11 + .../pipelines/common/pipeline_generator.lib | 109 +++------- .../pipelines/gitlab/pipeline_generator.sh | 188 ++++++++++++++++++ .../gitlab/templates/common/.gitlab-ci.yml | 19 ++ .../templates/quality/quality-pipeline.cfg | 37 ++++ .../quality/quality-pipeline.yml.template | 25 +++ 9 files changed, 318 insertions(+), 90 deletions(-) create mode 100644 documentation/src/gitlab/setup-quality-pipeline.asciidoc create mode 100644 scripts/pipelines/gitlab/pipeline_generator.sh create mode 100644 scripts/pipelines/gitlab/templates/common/.gitlab-ci.yml create mode 100644 scripts/pipelines/gitlab/templates/quality/quality-pipeline.cfg create mode 100644 scripts/pipelines/gitlab/templates/quality/quality-pipeline.yml.template diff --git a/documentation/src/azure-devops/setup-quality-pipeline.asciidoc b/documentation/src/azure-devops/setup-quality-pipeline.asciidoc index 17239e63d..75447f2a7 100644 --- a/documentation/src/azure-devops/setup-quality-pipeline.asciidoc +++ b/documentation/src/azure-devops/setup-quality-pipeline.asciidoc @@ -1,5 +1,6 @@ :provider: Azure DevOps :pipeline_type: pipeline +:trigger_sentence_azure: :trigger_sentence: This pipeline will be configured in order to be triggered every time the test pipeline is executed successfully on a commit :pipeline_type2: pipeline :path_provider: azure-devops diff --git a/documentation/src/common_templates/setup-quality-pipeline.asciidoc b/documentation/src/common_templates/setup-quality-pipeline.asciidoc index 025c7e48f..e7a0c40db 100644 --- a/documentation/src/common_templates/setup-quality-pipeline.asciidoc +++ b/documentation/src/common_templates/setup-quality-pipeline.asciidoc @@ -24,7 +24,7 @@ endif::[] ``` pipeline_generator.sh \ -c \ - -n <{pipeline_type} name> \ +ifdef::trigger_sentence_azure,trigger_sentence_github[ -n <{pipeline_type} name> \] -l \ --sonar-url \ --sonar-token \ @@ -41,7 +41,7 @@ NOTE: The config file for the quality {pipeline_type} is located at `/scripts/pi [subs=attributes+] ``` -c, --config-file [Required] Configuration file containing {pipeline_type} definition. --n, --pipeline-name [Required] Name that will be set to the {pipeline_type}. +ifdef::trigger_sentence_azure,trigger_sentence_github[-n, --pipeline-name [Required] Name that will be set to the {pipeline_type}.] -l, --language [Required] Language or framework of the project. --sonar-url [Required] SonarQube URL. --sonar-token [Required] SonarQube token. @@ -58,19 +58,25 @@ ifdef::test-pipeline[ --test-pipeline-name [Required] Test {pipeline_type [subs=attributes+] ``` -./pipeline_generator.sh -c ./templates/quality/quality-pipeline.cfg -n quarkus-project-quality -l quarkus --sonar-url http://1.2.3.4:9000 --sonar-token 6ce6663b63fc02881c6ea4c7cBa6563b8247a04e -d C:/Users/$USERNAME/Desktop/quarkus-project {extra_args_quarkus} -b develop -w +ifdef::trigger_sentence_azure,trigger_sentence_github[./pipeline_generator.sh -c ./templates/quality/quality-pipeline.cfg -n quarkus-project-quality -l quarkus --sonar-url http://1.2.3.4:9000 --sonar-token 6ce6663b63fc02881c6ea4c7cBa6563b8247a04e -d C:/Users/$USERNAME/Desktop/quarkus-project {extra_args_quarkus} -b develop -w] + +ifdef::trigger_sentence_gitlab[./pipeline_generator.sh -c ./templates/quality/quality-pipeline.cfg -l quarkus --sonar-url http://1.2.3.4:9000 --sonar-token 6ce6663b63fc02881c6ea4c7cBa6563b8247a04e -d C:/Users/$USERNAME/Desktop/quarkus-project {extra_args_quarkus} -b develop -w] ``` ==== Node.js project [subs=attributes+] ``` -./pipeline_generator.sh -c ./templates/quality/quality-pipeline.cfg -n node-project-quality -l node --sonar-url http://1.2.3.4:9000 --sonar-token 6ce6663b63fc02881c6ea4c7cBa6563b8247a04e -d C:/Users/$USERNAME/Desktop/node-project {extra_args_quarkus} -b develop -w +ifdef::trigger_sentence_azure,trigger_sentence_github[./pipeline_generator.sh -c ./templates/quality/quality-pipeline.cfg -n node-project-quality -l node --sonar-url http://1.2.3.4:9000 --sonar-token 6ce6663b63fc02881c6ea4c7cBa6563b8247a04e -d C:/Users/$USERNAME/Desktop/node-project {extra_args_quarkus} -b develop -w] + +ifdef::trigger_sentence_gitlab[./pipeline_generator.sh -c ./templates/quality/quality-pipeline.cfg -l node --sonar-url http://1.2.3.4:9000 --sonar-token 6ce6663b63fc02881c6ea4c7cBa6563b8247a04e -d C:/Users/$USERNAME/Desktop/node-project {extra_args_quarkus} -b develop -w] ``` ==== Angular project [subs=attributes+] ``` -./pipeline_generator.sh -c ./templates/quality/quality-pipeline.cfg -n angular-project-quality -l angular --sonar-url http://1.2.3.4:9000 --sonar-token 6ce6663b63fc02881c6ea4c7cBa6563b8247a04e -d C:/Users/$USERNAME/Desktop/angular-project {extra_args_angular} -b develop -w +ifdef::trigger_sentence_azure,trigger_sentence_github[./pipeline_generator.sh -c ./templates/quality/quality-pipeline.cfg -n angular-project-quality -l angular --sonar-url http://1.2.3.4:9000 --sonar-token 6ce6663b63fc02881c6ea4c7cBa6563b8247a04e -d C:/Users/$USERNAME/Desktop/angular-project {extra_args_angular} -b develop -w] + +ifdef::trigger_sentence_gitlab[./pipeline_generator.sh -c ./templates/quality/quality-pipeline.cfg -l angular --sonar-url http://1.2.3.4:9000 --sonar-token 6ce6663b63fc02881c6ea4c7cBa6563b8247a04e -d C:/Users/$USERNAME/Desktop/angular-project {extra_args_angular} -b develop -w] ``` \ No newline at end of file diff --git a/documentation/src/github/setup-quality-pipeline.asciidoc b/documentation/src/github/setup-quality-pipeline.asciidoc index 6e79574f8..7377c5808 100644 --- a/documentation/src/github/setup-quality-pipeline.asciidoc +++ b/documentation/src/github/setup-quality-pipeline.asciidoc @@ -1,5 +1,6 @@ :provider: GitHub :pipeline_type: workflow +:trigger_sentence_github: :trigger_sentence: This workflow will be configured to be executed as a job inside a CI workflow after the test (or build, if no test) job :pipeline_type2: GitHub action :path_provider: github @@ -8,4 +9,3 @@ :extra_args_angular: :extra_sentence_ci: Please note that this workflow, although manually triggerable, is designed to be executed as part of a CI workflow, which you can create following this xref:setup-ci-pipeline.asciidoc[guide]. include::../common_templates/setup-quality-pipeline.asciidoc[] - diff --git a/documentation/src/gitlab/setup-quality-pipeline.asciidoc b/documentation/src/gitlab/setup-quality-pipeline.asciidoc new file mode 100644 index 000000000..884d5588b --- /dev/null +++ b/documentation/src/gitlab/setup-quality-pipeline.asciidoc @@ -0,0 +1,11 @@ +:provider: Gitlab +:pipeline_type: pipeline +:trigger_sentence_gitlab: +:trigger_sentence: This pipeline will be configured in order to be triggered every time the test pipeline is executed successfully on a commit +:pipeline_type2: Gitlab +:path_provider: gitlab +:extra_args_quarkus: +:extra_args_node: +:extra_args_angular: +:extra_sentence_ci: Please note that this pipeline, although manually triggerable, is designed to be executed as part of a gitlab CI pipeline. +include::../common_templates/setup-quality-pipeline.asciidoc[] diff --git a/scripts/pipelines/common/pipeline_generator.lib b/scripts/pipelines/common/pipeline_generator.lib index 698f09346..f79d2b24a 100644 --- a/scripts/pipelines/common/pipeline_generator.lib +++ b/scripts/pipelines/common/pipeline_generator.lib @@ -1,43 +1,36 @@ #!/bin/bash function help { echo "" - echo "Generates a $pipeline_type on $provider based on the given definition." + echo "Generates a pipeline on $provider based on the given definition." echo "" echo "Common flags:" - echo " -c, --config-file [Required] Configuration file containing $pipeline_type definition." - echo " -n, --pipeline-name [Required] Name that will be set to the $pipeline_type." + echo " -c, --config-file [Required] Configuration file containing pipeline definition." + echo " -n, --pipeline-name [Required] Name that will be set to the pipeline." echo " -d, --local-directory [Required] Local directory of your project." - echo " -a, --artifact-path Path to be persisted as an artifact after $pipeline_type execution, e.g. where the application stores logs or any other blob on runtime." + echo " -a, --artifact-path Path to be persisted as an artifact after pipeline execution, e.g. where the application stores logs or any other blob on runtime." echo " -b, --target-branch Name of the branch to which the Pull Request will target. PR is not created if the flag is not provided." echo " -w Open the Pull Request on the web browser if it cannot be automatically merged. Requires -b flag." echo "" - echo "Build $pipeline_type flags:" + echo "Build pipeline flags:" echo " -l, --language [Required] Language or framework of the project." echo " -t, --target-directory Target directory of build process. Takes precedence over the language/framework default one." echo "" - echo "Test $pipeline_type flags:" + echo "Test pipeline flags:" echo " -l, --language [Required] Language or framework of the project." - [ "$provider" == "azure-devops" ] && echo " --build-pipeline-name [Required] Build $pipeline_type name." + [ "$provider" == "azure-devops" ] && echo " --build-pipeline-name [Required] Build pipeline name." echo "" - echo "Quality $pipeline_type flags:" + echo "Quality pipeline flags:" echo " -l, --language [Required] Language or framework of the project." echo " --sonar-url [Required] Sonarqube URL." echo " --sonar-token [Required] Sonarqube token." - [ "$provider" == "azure-devops" ] && echo " --build-pipeline-name [Required] Build $pipeline_type name." - [ "$provider" == "azure-devops" ] && echo " --test-pipeline-name [Required] Test $pipeline_type name." + [ "$provider" == "azure-devops" ] && echo " --build-pipeline-name [Required] Build pipeline name." + [ "$provider" == "azure-devops" ] && echo " --test-pipeline-name [Required] Test pipeline name." echo "" - [ "$provider" == "github" ] && echo "" - [ "$provider" == "github" ] && echo "CI pipeline flags" - [ "$provider" == "github" ] && echo " --build-pipeline-name [Required] Name of the job calling the build $pipeline_type." - [ "$provider" == "github" ] && echo " --test-pipeline-name Name of the job calling the test $pipeline_type." - [ "$provider" == "github" ] && echo " --quality-pipeline-name Name of the job calling the quality $pipeline_type." - [ "$provider" == "github" ] && echo "" echo "Package pipeline flags:" echo " -l, --language [Required, if dockerfile not set] Language or framework of the project." echo " --dockerfile [Required, if language not set] Path from the root of the project to its Dockerfile. Takes precedence over the language/framework default one." - [ "$provider" == "azure-devops" ] && echo " --build-pipeline-name [Required] Build $pipeline_type name." - [ "$provider" == "azure-devops" ] && echo " --quality-pipeline-name [Required] Quality $pipeline_type name." - [ "$provider" == "github" ] && echo " --ci-pipeline-name [Required] CI $pipeline_type name." + [ "$provider" == "azure-devops" ] && echo " --build-pipeline-name [Required] Build pipeline name." + echo " --quality-pipeline-name [Required] Quality pipeline name." echo " -i, --image-name [Required] Name (excluding tag) for the generated container image." echo " -u, --registry-user [Required, unless AWS] Container registry login user." echo " -p, --registry-password [Required, unless AWS] Container registry login password." @@ -45,52 +38,24 @@ function help { echo " --aws-secret-access-key [Required, if AWS] AWS account secret access key." echo " --aws-region [Required, if AWS] AWS region for ECR." echo "" - echo "Library package $pipeline_type flags:" + echo "Library package pipeline flags:" echo " -l, --language [Required] Language or framework of the project." echo "" - echo "Azure AKS provisioning $pipeline_type flags:" + echo "Deploy pipeline flags:" + echo "" + echo "Azure AKS provisioning pipeline flags:" echo " --resource-group [Required] Name of the resource group for the cluster." echo " --storage-account [Required] Name of the storage account for the cluster." echo " --storage-container [Required] Name of the storage container where the Terraform state of the cluster will be stored." - echo " --rancher Install Rancher to manage the cluster." echo "" - echo "AWS EKS provisioning $pipeline_type flags:" + echo "AWS EKS provisioning pipeline flags:" echo " --cluster-name [Required] Name for the cluster." echo " --s3-bucket [Required] Name of the S3 bucket where the Terraform state of the cluster will be stored." echo " --s3-key-path [Required] Path within the S3 bucket where the Terraform state of the cluster will be stored." - echo " --aws-access-key [Required, on first run] AWS account access key ID." - echo " --aws-secret-access-key [Required, on first run] AWS account secret access key." - echo " --aws-region [Required, on first run] AWS region for provisioning resources." - - echo " --rancher Install Rancher to manage the cluster." - echo "" - echo "Deploy $pipeline_type flags:" - echo "" - echo " --package-pipeline-name [Required] Package $pipeline_type name." - echo " --env-provision-pipeline-name [Required] Environment provisioning $pipeline_type name." - echo " --k8s-provider [Required] Kubernetes cluster provider name. Accepted values: EKS, AKS." - echo " --k8s-namespace [Required] Kubernetes namespace where the application will be deployed." - echo " --k8s-deploy-files-path [Required] Path from the root of the project to the YAML manifests directory." - echo " --k8s-image-pull-secret-name Name for the generated secret containing registry credentials. Required when using a private registry to host images." - echo "" exit } -function validateRegistryLoginCredentials { - # if the user chose to push to a registry and the user has not already given a password - # then prompt the user - if [ -v dockerUser ] && [ ! -v dockerPassword ] - then - read -rsp "Please enter Docker registry password..." dockerPassword - fi - - if [ -v awsRegion ] && [ -v awsAccessKey ] && [ ! -v awsSecretAccessKey ] - then - read -rsp "Please enter AWS secret access key..." awsSecretAccessKey - fi -} - function ensurePathFormat { currentDirectory=$(pwd) @@ -112,7 +77,7 @@ function importConfigFile { # Check if the config file was supplied. if test -z "$configFile" then - echo -e "${red}Error: $pipeline_type definition configuration file not specified." >&2 + echo -e "${red}Error: Pipeline definition configuration file not specified." >&2 exit 2 fi @@ -142,6 +107,9 @@ function checkInstallations { elif ([ "$provider" == "azure-devops" ] && ! [ -x "$(command -v az)" ]); then echo -e "${red}Error: Azure CLI is not installed." >&2 exit 127 + # elif ([ "$provider" == "gitlab" ] && ! [ -x "$(command -v glab)" ]); then + # echo -e "${red}Error: Azure CLI is not installed." >&2 + # exit 127 fi # Check if Python is installed @@ -169,31 +137,12 @@ function copyYAMLFile { # Generate pipeline YAML from template and put it in the repository. # We cannot use a variable in the definition of resource in the pipeline so we have to use a placeholder to replace it with the value we need - commonEnvSubstList='${buildPipelineName} ${testPipelineName} ${qualityPipelineName} ${pipelineName} ${ciPipelineName} ${packagePipelineName}' + commonEnvSubstList='${buildPipelineName} ${testPipelineName} ${qualityPipelineName} ${pipelineName}' envsubst "${commonEnvSubstList} ${specificEnvSubstList}" < "${hangarPath}/${templatesPath}/${yamlFile}.template" > "${localDirectory}/${pipelinePath}/${yamlFile}" - - # Check if an extra artifact to store is supplied. - if test ! -z "$artifactPath" - then - # Add the extra step to the YAML. - if [ "$provider" == "azure-devops" ] - then - cat "${hangarPath}/${commonTemplatesPath}/store-extra-path.yml" >> "${localDirectory}/${pipelinePath}/${yamlFile}" - elif [ "$provider" == "github" ] - then - storeExtraPathContent="\n - name: Publish Additional Output Artifact\n uses: actions\/upload-artifact@v3\n with:\n name: additional-pipeline-output\n path: \"\${{ env.artifactPath }}\"" - sed -i "s/# mark to insert step for additonal artifact #/$storeExtraPathContent\n/" "${localDirectory}/${pipelinePath}/${yamlFile}" - fi - else - if [ "$provider" == "github" ] - then - sed -i '/# mark to insert step for additonal artifact #/d' "${localDirectory}/${pipelinePath}/${yamlFile}" - fi - fi } function copyCommonScript { - echo -e "${green}Copying the script(s) common to any $pipeline_type files into your directory..." + echo -e "${green}Copying the script(s) common to any pipeline files into your directory..." echo -ne ${white} ! (ls "${hangarPath}/${commonTemplatesPath}"/*.sh) &> /dev/null || cp "${hangarPath}/${commonTemplatesPath}"/*.sh "${localDirectory}/${scriptFilePath}" @@ -204,7 +153,7 @@ function commitCommonFiles { echo -ne ${white} # Move into the project's directory and pushing the template into the Azure DevOps repository. - cd "${localDirectory}" + cd ${localDirectory} # Add the YAML files. git add "$pipelinePath" -f @@ -215,12 +164,4 @@ function commitCommonFiles { git commit -m "Adding the source YAML" git push -u origin ${sourceBranch} -} - -function setTargetDirectory { - case $language in - node | angular) targetDirectory="./" ;; - quarkus*) targetDirectory="./target/" ;; - *) echo -e "${red}Error: Specified language '${language}' is not supported." >&2; exit 1 - esac -} +} \ No newline at end of file diff --git a/scripts/pipelines/gitlab/pipeline_generator.sh b/scripts/pipelines/gitlab/pipeline_generator.sh new file mode 100644 index 000000000..f06dae09b --- /dev/null +++ b/scripts/pipelines/gitlab/pipeline_generator.sh @@ -0,0 +1,188 @@ +#!/bin/bash +set -e +FLAGS=$(getopt -a --options c:n:d:a:b:l:i:u:p:hw --long "config-file:,pipeline-name:,local-directory:,artifact-path:,target-branch:,language:,build-pipeline-name:,sonar-url:,sonar-token:,image-name:,registry-user:,registry-password:,storage-container:,cluster-name:,s3-bucket:,s3-key-path:,quality-pipeline-name:,dockerfile:,test-pipeline-name:,aws-access-key:,aws-secret-access-key:,aws-region:,package-pipeline-name:,env-provision-pipeline-name:,k8s-provider:,k8s-namespace:,k8s-deploy-files-path:,k8s-image-pull-secret-name:,help,rancher" -- "$@") + +eval set -- "$FLAGS" +while true; do + case "$1" in + -c | --config-file) configFile=$2; shift 2;; + -n | --pipeline-name) export pipelineName=$2; shift 2;; + -d | --local-directory) localDirectory=$2; shift 2;; + -a | --artifact-path) artifactPath=$2; shift 2;; + -b | --target-branch) targetBranch=$2; shift 2;; + -l | --language) language=$2; shift 2;; + --build-pipeline-name) export buildPipelineName=$2; shift 2;; + --sonar-url) sonarUrl=$2; shift 2;; + --sonar-token) sonarToken=$2; shift 2;; + -i | --image-name) imageName=$2; shift 2;; + -u | --registry-user) dockerUser=$2; shift 2;; + -p | --registry-password) dockerPassword=$2; shift 2;; + --storage-container) storageContainerName=$2; shift 2;; + --rancher) installRancher="true"; shift 1;; + --cluster-name) clusterName=$2; shift 2;; + --s3-bucket) s3Bucket=$2; shift 2;; + --s3-key-path) s3KeyPath=$2; shift 2;; + --quality-pipeline-name) export qualityPipelineName=$2; shift 2;; + --test-pipeline-name) export testPipelineName=$2; shift 2;; + --dockerfile) dockerFile=$2; shift 2;; + --aws-access-key) awsAccessKey="$2"; shift 2;; + --aws-secret-access-key) awsSecretAccessKey="$2"; shift 2;; + --aws-region) awsRegion="$2"; shift 2;; + --package-pipeline-name) export packagePipelineName=$2; shift 2;; + --env-provision-pipeline-name) envProvisionPipelineName="$2"; shift 2;; + --k8s-provider) k8sProvider=$2; shift 2;; + --k8s-namespace) k8sNamespace="$2"; shift 2;; + --k8s-deploy-files-path) k8sDeployFiles=$2; shift 2;; + --k8s-image-pull-secret-name) k8sImagePullSecret=$2; shift 2;; + -h | --help) help="true"; shift 1;; + -w) webBrowser="true"; shift 1;; + --) shift; break;; + esac +done + +# Colours for the messages. +white='\e[1;37m' +green='\e[1;32m' +red='\e[0;31m' + +# Common var +commonTemplatesPath="scripts/pipelines/gitlab/templates/common" # Path for common files of the pipelines +pipelinePath=".pipelines" # Path to the pipelines. +scriptFilePath=".pipelines/scripts" # Path to the scripts. +gitlabCiFile=".gitlab-ci.yml" +export provider="gitlab" + +function obtainHangarPath { + + # This line goes to the script directory independent of wherever the user is and then jumps 3 directories back to get the path + hangarPath=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && cd ../../.. && pwd ) +} + +function addAdditionalArtifact { + # Check if an extra artifact to store is supplied. + if test ! -z "$artifactPath" + then + # Add the extra step to the YAML. + grep " artifacts:" "${localDirectory}/${pipelinePath}/${yamlFile}" > /dev/null && storeExtraPathContent=" - \"$artifactPath\"" + grep " artifacts:" "${localDirectory}/${pipelinePath}/${yamlFile}" > /dev/null || storeExtraPathContent="\n artifacts:\n paths:\n - \"$artifactPath\"" + sed -i "s/# mark to insert step for additonal artifact #/$storeExtraPathContent\n/" "${localDirectory}/${pipelinePath}/${yamlFile}" + else + echo "The '-a' flag has not been set, skipping the step to add additional artifact." + sed -i '/# mark to insert step for additonal artifact #/d' "${localDirectory}/${pipelinePath}/${yamlFile}" + fi +} + +# Function that adds the variables to be used in the pipeline. +function addCommonPipelineVariables { + if test -z "${artifactPath}" + then + echo "Skipping creation of the variable artifactPath as the flag has not been used." + # Delete the commentary to set the artifactPath input/var + sed -i '/# mark to insert additional artifact env var #/d' "${localDirectory}/${pipelinePath}/${yamlFile}" + else + # add the input for the additional artifact + grep "variables:" "${localDirectory}/${pipelinePath}/${yamlFile}" > /dev/null && textArtifactPathVar=" artifactPath: ${artifactPath//\//\\/}" + grep "variables:" "${localDirectory}/${pipelinePath}/${yamlFile}" > /dev/null || textArtifactPathVar="variables:\n artifactPath: \"${artifactPath//\//\\/}\"" + sed -i "s/# mark to insert additional artifact env var #/$textArtifactPathVar/" "${localDirectory}/${pipelinePath}/${yamlFile}" + fi +} + +function addCiFile { + echo -e "${green}Copying and commiting the gitlab ci file." + echo -ne ${white} + + cp "${hangarPath}/${commonTemplatesPath}/${gitlabCiFile}" "${localDirectory}/${gitlabCiFile}" + testCommit=$(git status) + if echo "$testCommit" | grep "nothing to commit, working tree clean" > /dev/null + then + echo "gilab-ci file already present with same content, nothing to commit." + else + git add "${gitlabCiFile}" -f + git commit -m "adding gitlab-ci.yml" + git push + fi +} + +function createPR { + # Check if a target branch is supplied. + if test -z "$targetBranch" + then + # No branch specified in the parameters, no Pull Request is created, the code will be stored in the current branch. + echo -e "${green}No branch specified to do the Pull Request, changes left in the ${sourceBranch} branch." + exit + else + echo -e "${green}Creating a Pull Request..." + echo -ne "${white}" + repoURL=$(git config --get remote.origin.url) + repoNameWithGit="${repoURL/https:\/\/gitlab.com\/}" + repoName="${repoNameWithGit/.git}" + # Create the Pull Request to merge into the specified branch. + #debug + echo "glab mr create -b \"$targetBranch\" -d \"merge request $sourceBranch\" -s \"$sourceBranch\" -H \"${repoName}\" -t \"merge $sourceBranch\"" + pr=$(glab mr create -b "$targetBranch" -d "merge request $sourceBranch" -s "$sourceBranch" -H "${repoName}" -t "merge $sourceBranch") + + # Create merge approval to the pull request. + glab mr approvers "$sourceBranch" -R "${repoName}" + + #trying to merge + if glab mr merge -s $(basename "$pr") -y + then + # Pull Request merged successfully. + echo -e "${green}Pull Request merged into $targetBranch branch successfully." + exit + else + # Check if the -w flag is activated. + if [[ "$webBrowser" == "true" ]] + then + # -w flag is activated and a page with the corresponding Pull Request is opened in the web browser. + echo -e "${green}Pull Request successfully created." + echo -e "${green}Opening the Pull Request on the web browser..." + python -m webbrowser "$pr" + exit + else + # -w flag is not activated and the URL to the Pull Request is shown in the console. + echo -e "${green}Pull Request successfully created." + echo -e "${green}To review the Pull Request and accept it, click on the following link:" + echo "${pr}" + exit + fi + fi + fi +} + + +obtainHangarPath + +# Load common functions +. "$hangarPath/scripts/pipelines/common/pipeline_generator.lib" + +if [[ "$help" == "true" ]]; then help; fi + +ensurePathFormat + +importConfigFile + +checkInstallations + +createNewBranch + +type addPipelineVariables &> /dev/null && addPipelineVariables + +copyYAMLFile + +addAdditionalArtifact + +copyCommonScript + +type copyScript &> /dev/null && copyScript + +# This function does not exists for the github pipeline generator at this moment, but I let the line with 'type' to keep the same structure as the others pipeline generator +type addCommonPipelineVariables &> /dev/null && addCommonPipelineVariables + +commitCommonFiles + +type commitFiles &> /dev/null && commitFiles + +addCiFile + +createPR \ No newline at end of file diff --git a/scripts/pipelines/gitlab/templates/common/.gitlab-ci.yml b/scripts/pipelines/gitlab/templates/common/.gitlab-ci.yml new file mode 100644 index 000000000..4f5b09c04 --- /dev/null +++ b/scripts/pipelines/gitlab/templates/common/.gitlab-ci.yml @@ -0,0 +1,19 @@ +include: + - '.pipelines/*.yml' + +workflow: + rules: + - if: '$CI_PIPELINE_SOURCE == "web"' + when: always + - when: never + +# stages: +# - build +# - test +# - quality +# - package +# - deploy + +# default: +# image: maven:3-jdk-11 +# tags: ['docker_ruby'] diff --git a/scripts/pipelines/gitlab/templates/quality/quality-pipeline.cfg b/scripts/pipelines/gitlab/templates/quality/quality-pipeline.cfg new file mode 100644 index 000000000..56c99983f --- /dev/null +++ b/scripts/pipelines/gitlab/templates/quality/quality-pipeline.cfg @@ -0,0 +1,37 @@ +# Mandatory flags. +mandatoryFlags="$localDirectory,$language,$sonarUrl,$sonarToken," +# Path to the templates. +templatesPath="scripts/pipelines/github/templates/quality" +# Path to the common templates folder +commonTemplatesPipelinePath="scripts/pipelines/common/templates/quality" +# YAML file name. +yamlFile="quality-pipeline.yml" +# Script name. +scriptFile="quality.sh" +# Source branch. +sourceBranch="feature/quality-pipeline" + +# Function that copies the script that passes the quality tests to the application. +function copyScript { + + cp "${hangarPath}/${commonTemplatesPipelinePath}/${language}-quality.sh.template" "${localDirectory}/${scriptFilePath}/${scriptFile}" +} + +function addPipelineVariables { + + repoURL=$(git config --get remote.origin.url) + repoNameWithGit="${repoURL/https:\/\/github.com\/}" + repoName="${repoNameWithGit/.git}" + #gh secret set SONARQUBE_TOKEN -a actions -b "$sonarToken" -R "$repoName" + #glab auth login --token $GITLAB_PAT + glab variable set SONARQUBE_TOKEN1 -v "$sonarToken" -t "env_var" + + #if test -z $targetDirectory + #then + # setTargetDirectory + #fi + + export sonarUrl + export targetDirectory + specificEnvSubstList='${sonarUrl} ${targetDirectory}' +} \ No newline at end of file diff --git a/scripts/pipelines/gitlab/templates/quality/quality-pipeline.yml.template b/scripts/pipelines/gitlab/templates/quality/quality-pipeline.yml.template new file mode 100644 index 000000000..ea2813058 --- /dev/null +++ b/scripts/pipelines/gitlab/templates/quality/quality-pipeline.yml.template @@ -0,0 +1,25 @@ +default: + image: mcerverc/maven-podman:v2 + +workflow: + rules: + - if: '$CI_PIPELINE_SOURCE == "web"' + when: always + +variables: + SONAR_URL: + value: "$sonarUrl" + description: "Sonar URL" + SONAR_TOKEN: + value: "$sonarToken" + description: "Sonar access token" + PROJECT_PATH: + value: "$targetDirectory" + description: "Build artifacts target directory" + +SonarQube analysis: + script: + - "curl --header 'JOB-TOKEN: $CI_JOB_TOKEN' '$CI_JOB_URL/-/jobs/$CI_JOB_ID/artifacts/${CI_COMMIT_REF_NAME}/raw/target?job=build'" + - chmod +x .pipelines/scripts/quality.sh + - .pipelines/scripts/quality.sh + \ No newline at end of file From faed8d915c45be78da61e397e1cab7ee8a7459b1 Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Mon, 17 Oct 2022 04:45:12 +0000 Subject: [PATCH 2/3] Automatic generation of documentation --- .../setup-quality-pipeline.asciidoc | 4 + .../github/setup-quality-pipeline.asciidoc | 4 + .../gitlab/setup-quality-pipeline.asciidoc | 81 +++++++++++++++++++ 3 files changed, 89 insertions(+) create mode 100644 documentation/gitlab/setup-quality-pipeline.asciidoc diff --git a/documentation/azure-devops/setup-quality-pipeline.asciidoc b/documentation/azure-devops/setup-quality-pipeline.asciidoc index 66f6aee24..0fbb77b08 100644 --- a/documentation/azure-devops/setup-quality-pipeline.asciidoc +++ b/documentation/azure-devops/setup-quality-pipeline.asciidoc @@ -1,5 +1,6 @@ :provider: Azure DevOps :pipeline_type: pipeline +:trigger_sentence_azure: :trigger_sentence: This pipeline will be configured in order to be triggered every time the test pipeline is executed successfully on a commit :pipeline_type2: pipeline :path_provider: azure-devops @@ -65,6 +66,7 @@ NOTE: The config file for the quality {pipeline_type} is located at `/scripts/pi [subs=attributes+] ``` ./pipeline_generator.sh -c ./templates/quality/quality-pipeline.cfg -n quarkus-project-quality -l quarkus --sonar-url http://1.2.3.4:9000 --sonar-token 6ce6663b63fc02881c6ea4c7cBa6563b8247a04e -d C:/Users/$USERNAME/Desktop/quarkus-project {extra_args_quarkus} -b develop -w + ``` ==== Node.js project @@ -72,6 +74,7 @@ NOTE: The config file for the quality {pipeline_type} is located at `/scripts/pi [subs=attributes+] ``` ./pipeline_generator.sh -c ./templates/quality/quality-pipeline.cfg -n node-project-quality -l node --sonar-url http://1.2.3.4:9000 --sonar-token 6ce6663b63fc02881c6ea4c7cBa6563b8247a04e -d C:/Users/$USERNAME/Desktop/node-project {extra_args_quarkus} -b develop -w + ``` ==== Angular project @@ -79,4 +82,5 @@ NOTE: The config file for the quality {pipeline_type} is located at `/scripts/pi [subs=attributes+] ``` ./pipeline_generator.sh -c ./templates/quality/quality-pipeline.cfg -n angular-project-quality -l angular --sonar-url http://1.2.3.4:9000 --sonar-token 6ce6663b63fc02881c6ea4c7cBa6563b8247a04e -d C:/Users/$USERNAME/Desktop/angular-project {extra_args_angular} -b develop -w + ``` diff --git a/documentation/github/setup-quality-pipeline.asciidoc b/documentation/github/setup-quality-pipeline.asciidoc index 04d6fc246..a1130f62e 100644 --- a/documentation/github/setup-quality-pipeline.asciidoc +++ b/documentation/github/setup-quality-pipeline.asciidoc @@ -1,5 +1,6 @@ :provider: GitHub :pipeline_type: workflow +:trigger_sentence_github: :trigger_sentence: This workflow will be configured to be executed as a job inside a CI workflow after the test (or build, if no test) job :pipeline_type2: GitHub action :path_provider: github @@ -62,6 +63,7 @@ NOTE: The config file for the quality {pipeline_type} is located at `/scripts/pi [subs=attributes+] ``` ./pipeline_generator.sh -c ./templates/quality/quality-pipeline.cfg -n quarkus-project-quality -l quarkus --sonar-url http://1.2.3.4:9000 --sonar-token 6ce6663b63fc02881c6ea4c7cBa6563b8247a04e -d C:/Users/$USERNAME/Desktop/quarkus-project {extra_args_quarkus} -b develop -w + ``` ==== Node.js project @@ -69,6 +71,7 @@ NOTE: The config file for the quality {pipeline_type} is located at `/scripts/pi [subs=attributes+] ``` ./pipeline_generator.sh -c ./templates/quality/quality-pipeline.cfg -n node-project-quality -l node --sonar-url http://1.2.3.4:9000 --sonar-token 6ce6663b63fc02881c6ea4c7cBa6563b8247a04e -d C:/Users/$USERNAME/Desktop/node-project {extra_args_quarkus} -b develop -w + ``` ==== Angular project @@ -76,4 +79,5 @@ NOTE: The config file for the quality {pipeline_type} is located at `/scripts/pi [subs=attributes+] ``` ./pipeline_generator.sh -c ./templates/quality/quality-pipeline.cfg -n angular-project-quality -l angular --sonar-url http://1.2.3.4:9000 --sonar-token 6ce6663b63fc02881c6ea4c7cBa6563b8247a04e -d C:/Users/$USERNAME/Desktop/angular-project {extra_args_angular} -b develop -w + ``` diff --git a/documentation/gitlab/setup-quality-pipeline.asciidoc b/documentation/gitlab/setup-quality-pipeline.asciidoc new file mode 100644 index 000000000..b923a411f --- /dev/null +++ b/documentation/gitlab/setup-quality-pipeline.asciidoc @@ -0,0 +1,81 @@ +:provider: Gitlab +:pipeline_type: pipeline +:trigger_sentence_gitlab: +:trigger_sentence: This pipeline will be configured in order to be triggered every time the test pipeline is executed successfully on a commit +:pipeline_type2: Gitlab +:path_provider: gitlab +:extra_args_quarkus: +:extra_args_node: +:extra_args_angular: +:extra_sentence_ci: Please note that this pipeline, although manually triggerable, is designed to be executed as part of a gitlab CI pipeline. += Setting up a Quality {pipeline_type} on {provider} + +In this section we will create a quality {pipeline_type} for analyzing project code with SonarQube. {trigger_sentence}, and consumes the artifact produced by the build {pipeline_type}. + +The creation of this {pipeline_type2} will follow the project workflow, so a new branch named `feature/quality-pipeline` will be created and the YAML file for the {pipeline_type} will be pushed to it. + +Then, a Pull Request (PR) will be created in order to merge the new branch into the appropriate branch (provided in `-b` flag). The PR will be automatically merged if the repository policies are met. If the merge is not possible, either the PR URL will be shown as output, or it will be opened in your web browser if using `-w` flag. + +The script located at `/scripts/pipelines/{path_provider}/pipeline_generator.sh` will automatically create this new branch, create a quality {pipeline_type} based on a YAML template appropriate for the project programming language or framework, create the Pull Request, and if it is possible, merge this new branch into the specified branch. + +{extra_sentence_ci} + +== Prerequisites + +* This script will commit and push the corresponding YAML template into your repository, so please be sure your local repository is up-to-date (i.e you have pulled the latest changes with `git pull`). +* Generate a SonarQube https://docs.sonarqube.org/latest/user-guide/user-token/[token] (just follow the section "Generating a token"). + +== Creating the {pipeline_type} using provided script + +=== Usage +[subs=attributes+] +``` +pipeline_generator.sh \ + -c \ + -l \ + --sonar-url \ + --sonar-token \ + -d \ + [-b ] \ + [-w] +``` + +NOTE: The config file for the quality {pipeline_type} is located at `/scripts/pipelines/{path_provider}/templates/quality/quality-pipeline.cfg`. + +=== Flags +[subs=attributes+] +``` +-c, --config-file [Required] Configuration file containing {pipeline_type} definition. +-l, --language [Required] Language or framework of the project. + --sonar-url [Required] SonarQube URL. + --sonar-token [Required] SonarQube token. +-d, --local-directory [Required] Local directory of your project. +-b, --target-branch Name of the branch to which the Pull Request will target. PR is not created if the flag is not provided. +-w Open the Pull Request on the web browser if it cannot be automatically merged. Requires -b flag. +``` + +=== Examples + +==== Quarkus project + +[subs=attributes+] +``` + +./pipeline_generator.sh -c ./templates/quality/quality-pipeline.cfg -l quarkus --sonar-url http://1.2.3.4:9000 --sonar-token 6ce6663b63fc02881c6ea4c7cBa6563b8247a04e -d C:/Users/$USERNAME/Desktop/quarkus-project {extra_args_quarkus} -b develop -w +``` + +==== Node.js project + +[subs=attributes+] +``` + +./pipeline_generator.sh -c ./templates/quality/quality-pipeline.cfg -l node --sonar-url http://1.2.3.4:9000 --sonar-token 6ce6663b63fc02881c6ea4c7cBa6563b8247a04e -d C:/Users/$USERNAME/Desktop/node-project {extra_args_quarkus} -b develop -w +``` + +==== Angular project + +[subs=attributes+] +``` + +./pipeline_generator.sh -c ./templates/quality/quality-pipeline.cfg -l angular --sonar-url http://1.2.3.4:9000 --sonar-token 6ce6663b63fc02881c6ea4c7cBa6563b8247a04e -d C:/Users/$USERNAME/Desktop/angular-project {extra_args_angular} -b develop -w +``` From c3d361387913576c161ebcb7eec9ce952b90a59f Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Mon, 17 Oct 2022 11:44:03 +0530 Subject: [PATCH 3/3] Update quality-pipeline.cfg --- .../gitlab/templates/quality/quality-pipeline.cfg | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/scripts/pipelines/gitlab/templates/quality/quality-pipeline.cfg b/scripts/pipelines/gitlab/templates/quality/quality-pipeline.cfg index 56c99983f..1f35c9a14 100644 --- a/scripts/pipelines/gitlab/templates/quality/quality-pipeline.cfg +++ b/scripts/pipelines/gitlab/templates/quality/quality-pipeline.cfg @@ -1,7 +1,7 @@ # Mandatory flags. mandatoryFlags="$localDirectory,$language,$sonarUrl,$sonarToken," # Path to the templates. -templatesPath="scripts/pipelines/github/templates/quality" +templatesPath="scripts/pipelines/gitlab/templates/quality" # Path to the common templates folder commonTemplatesPipelinePath="scripts/pipelines/common/templates/quality" # YAML file name. @@ -22,16 +22,9 @@ function addPipelineVariables { repoURL=$(git config --get remote.origin.url) repoNameWithGit="${repoURL/https:\/\/github.com\/}" repoName="${repoNameWithGit/.git}" - #gh secret set SONARQUBE_TOKEN -a actions -b "$sonarToken" -R "$repoName" - #glab auth login --token $GITLAB_PAT - glab variable set SONARQUBE_TOKEN1 -v "$sonarToken" -t "env_var" - - #if test -z $targetDirectory - #then - # setTargetDirectory - #fi + glab variable set SONARQUBE_TOKEN -v "$sonarToken" -t "env_var" export sonarUrl export targetDirectory specificEnvSubstList='${sonarUrl} ${targetDirectory}' -} \ No newline at end of file +}