From d28c6ab4deaaf942064df7f67225d6d2e9ba3534 Mon Sep 17 00:00:00 2001 From: Darpan Lalwani Date: Thu, 4 Jan 2024 20:53:27 +0530 Subject: [PATCH 1/6] Quality Dashboard Integrated --- pom.xml | 12 +- .../ci/common/constants/Constants.java | 19 ++ .../BrowserStackBuildWrapperDescriptor.java | 6 +- .../QualityDashboardAPIUtil.java | 71 +++++ .../QualityDashboardInit.java | 162 ++++++++++ .../QualityDashboardInitItemListener.java | 123 ++++++++ .../QualityDashboardPipelineTracker.java | 292 ++++++++++++++++++ .../QualityDashboardUtil.java | 18 ++ 8 files changed, 696 insertions(+), 7 deletions(-) create mode 100644 src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardAPIUtil.java create mode 100644 src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardInit.java create mode 100644 src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardInitItemListener.java create mode 100644 src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardPipelineTracker.java create mode 100644 src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardUtil.java diff --git a/pom.xml b/pom.xml index 2754938..2cbbf70 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ browserstack-integration - 1.2.12-SNAPSHOT + 1.2.13-SNAPSHOT hpi BrowserStack @@ -37,8 +37,8 @@ HEAD - @@ -53,7 +53,7 @@ - 2.334 8 @@ -62,8 +62,8 @@ 0.8 UTF-8 - UA-79358556-2 diff --git a/src/main/java/com/browserstack/automate/ci/common/constants/Constants.java b/src/main/java/com/browserstack/automate/ci/common/constants/Constants.java index 92623a8..43e2f75 100644 --- a/src/main/java/com/browserstack/automate/ci/common/constants/Constants.java +++ b/src/main/java/com/browserstack/automate/ci/common/constants/Constants.java @@ -61,4 +61,23 @@ public static final class SessionStatus { public static final String UNMARKED = "unmarked"; public static final String PASSED = "passed"; } + + public static final class QualityDashboardAPI { + + public static final String URL_BASE = "https://quality-dashboard-api.browserstack.com/api/v1/jenkins"; + public static final String IS_INIT_SETUP_REQUIRED = URL_BASE + "/init-setup-required"; + + public static final String HISTORY_FOR_DAYS = URL_BASE + "/history-for-days"; + + public static final String ADD_ALL_PIPELINES = URL_BASE + "/add-all-pipelines"; + + public static final String ITEM_CRUD = URL_BASE + "/item"; + public static final String IS_QD_ENABLED = URL_BASE + "/qd-enabled"; + public static final String IS_PIPELINE_ENABLED = URL_BASE + "/pipeline-enabled"; + public static final String GET_RESULT_DIRECTORY = URL_BASE + "/get-result-directory"; + + public static final String UPLOAD_RESULT_ZIP = URL_BASE + "/upload-result"; + public static final String STORE_PIPELINE_RESULTS = URL_BASE + "/save-results"; + + } } diff --git a/src/main/java/com/browserstack/automate/ci/jenkins/BrowserStackBuildWrapperDescriptor.java b/src/main/java/com/browserstack/automate/ci/jenkins/BrowserStackBuildWrapperDescriptor.java index aae934f..ad96d32 100644 --- a/src/main/java/com/browserstack/automate/ci/jenkins/BrowserStackBuildWrapperDescriptor.java +++ b/src/main/java/com/browserstack/automate/ci/jenkins/BrowserStackBuildWrapperDescriptor.java @@ -3,6 +3,7 @@ import com.browserstack.automate.ci.common.BrowserStackBuildWrapperOperations; import com.browserstack.automate.ci.common.analytics.Analytics; import com.browserstack.automate.ci.jenkins.local.LocalConfig; +import com.browserstack.automate.ci.jenkins.qualityDashboard.QualityDashboardInit; import hudson.Extension; import hudson.model.AbstractProject; import hudson.model.Item; @@ -51,8 +52,11 @@ public boolean configure(StaplerRequest req, JSONObject formData) throws FormExc if (config.has("usageStatsEnabled")) { setEnableUsageStats(config.getBoolean("usageStatsEnabled")); } + if (config.has("credentialsId")){ + QualityDashboardInit qdInit = new QualityDashboardInit(); + qdInit.pluginConfiguredNotif(); + } } - return true; } diff --git a/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardAPIUtil.java b/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardAPIUtil.java new file mode 100644 index 0000000..4ca6200 --- /dev/null +++ b/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardAPIUtil.java @@ -0,0 +1,71 @@ +package com.browserstack.automate.ci.jenkins.qualityDashboard; + +import com.browserstack.automate.ci.jenkins.BrowserStackCredentials; +import okhttp3.*; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +public class QualityDashboardAPIUtil { + + OkHttpClient client = new OkHttpClient(); + + public Response makeGetRequestToQd(String getUrl, BrowserStackCredentials browserStackCredentials) { + try { + Request request = new Request.Builder() + .url(getUrl) + .header("Authorization", Credentials.basic(browserStackCredentials.getUsername(), browserStackCredentials.getDecryptedAccesskey())) + .build(); + Response response = client.newCall(request).execute(); + return response; + } catch(IOException e) { + e.printStackTrace(); + } + return null; + } + + public Response makePostRequestToQd(String postUrl, BrowserStackCredentials browserStackCredentials, RequestBody requestBody) { + try { + Request request = new Request.Builder() + .url(postUrl) + .header("Authorization", Credentials.basic(browserStackCredentials.getUsername(), browserStackCredentials.getDecryptedAccesskey())) + .post(requestBody) + .build(); + Response response = client.newCall(request).execute(); + return response; + } catch(IOException e) { + e.printStackTrace(); + } + return null; + } + + public Response makePutRequestToQd(String postUrl, BrowserStackCredentials browserStackCredentials, RequestBody requestBody) { + try { + Request request = new Request.Builder() + .url(postUrl) + .header("Authorization", Credentials.basic(browserStackCredentials.getUsername(), browserStackCredentials.getDecryptedAccesskey())) + .put(requestBody) + .build(); + Response response = client.newCall(request).execute(); + return response; + } catch(IOException e) { + e.printStackTrace(); + } + return null; + } + + public Response makeDeleteRequestToQd(String postUrl, BrowserStackCredentials browserStackCredentials, RequestBody requestBody) { + try { + Request request = new Request.Builder() + .url(postUrl) + .header("Authorization", Credentials.basic(browserStackCredentials.getUsername(), browserStackCredentials.getDecryptedAccesskey())) + .delete(requestBody) + .build(); + Response response = client.newCall(request).execute(); + return response; + } catch(IOException e) { + e.printStackTrace(); + } + return null; + } +} diff --git a/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardInit.java b/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardInit.java new file mode 100644 index 0000000..4a53fb4 --- /dev/null +++ b/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardInit.java @@ -0,0 +1,162 @@ +package com.browserstack.automate.ci.jenkins.qualityDashboard; + +import com.browserstack.automate.ci.common.constants.Constants; +import com.browserstack.automate.ci.jenkins.BrowserStackCredentials; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; +import hudson.Extension; +import hudson.init.InitMilestone; +import hudson.init.Initializer; +import hudson.model.Result; +import java.sql.Timestamp; +import java.time.Instant; +import jenkins.model.Jenkins; +import okhttp3.*; +import org.jenkinsci.plugins.workflow.job.WorkflowJob; +import org.jenkinsci.plugins.workflow.job.WorkflowRun; +import java.time.temporal.ChronoUnit; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.util.ArrayList; +import java.util.List; + +@Extension +public class QualityDashboardInit { + + static QualityDashboardAPIUtil apiUtil = new QualityDashboardAPIUtil(); + + @Initializer(after = InitMilestone.PLUGINS_PREPARED) + public static void postInstall() { + initQDSetupIfRequired(); + } + + public void pluginConfiguredNotif() { + initQDSetupIfRequired(); + } + + private static void initQDSetupIfRequired() { + BrowserStackCredentials browserStackCredentials = QualityDashboardUtil.getBrowserStackCreds(); + if(browserStackCredentials!=null) { + checkQDIntegrationAndDumpMetaData(browserStackCredentials); + } + } + + private static void checkQDIntegrationAndDumpMetaData(BrowserStackCredentials browserStackCredentials) { + if(initialQDSetupRequired(browserStackCredentials)) { + List pipelineDetailsMapList = getExistingPipelineDump(browserStackCredentials); + if(!pipelineDetailsMapList.isEmpty()){ + syncInitialDataWithQD(pipelineDetailsMapList, browserStackCredentials); + } + } + } + + private static boolean initialQDSetupRequired(BrowserStackCredentials browserStackCredentials) { + try { + Response response = apiUtil.makeGetRequestToQd(Constants.QualityDashboardAPI.IS_INIT_SETUP_REQUIRED, browserStackCredentials); + if (response != null && response.code() == HttpURLConnection.HTTP_OK) { + ResponseBody responseBody = response.body(); + if(responseBody != null && responseBody.string().equals("REQUIRED")) { + return true; + } + } + } catch(IOException e) { + e.printStackTrace(); + } + return false; + } + + private static List getExistingPipelineDump(BrowserStackCredentials browserStackCredentials) { + List pipelineDetailsMapList = new ArrayList<>(); + Jenkins jenkins = Jenkins.getInstanceOrNull(); + Instant thresholdInstant = Instant.now().minus(getHistoryForDays(browserStackCredentials), ChronoUnit.DAYS); + if (jenkins != null) { + jenkins.getAllItems().forEach(job -> { + if(job instanceof WorkflowJob) { + String pipelineName = job.getFullName(); + List pipelineDetailsList = new ArrayList<>(); + List allBuilds = ((WorkflowJob) job).getBuilds(); + if(!allBuilds.isEmpty()) { + allBuilds.stream().filter(build -> Instant.ofEpochMilli(build.getTimeInMillis()).isAfter(thresholdInstant) ).forEach( + build -> { + int buildNumber = build.getNumber(); + long duration = build.getDuration(); + Result overallResult = build.getResult(); + long endTimeInMillis = build.getTimeInMillis(); + Timestamp endTime = new Timestamp(endTimeInMillis); + PipelineDetails pipelineDetail = new PipelineDetails(buildNumber, duration, overallResult.toString(), endTime ); + pipelineDetailsList.add(pipelineDetail); + } + ); + } + PipelineDetailsMap pipelineDetailsMap = new PipelineDetailsMap(pipelineName, pipelineDetailsList); + pipelineDetailsMapList.add(pipelineDetailsMap); + } + }); + } + return pipelineDetailsMapList; + } + + private static int getHistoryForDays(BrowserStackCredentials browserStackCredentials) { + int no_of_days = 90; + try { + Response response = apiUtil.makeGetRequestToQd(Constants.QualityDashboardAPI.HISTORY_FOR_DAYS, browserStackCredentials); + if (response != null && response.code() == HttpURLConnection.HTTP_OK) { + ResponseBody responseBody = response.body(); + if(responseBody != null) { + String responseBodyStr = responseBody.string(); + if(responseBodyStr!=null) + no_of_days = Integer.parseInt(responseBodyStr); + } + } + } catch(IOException e) { + e.printStackTrace(); + } finally { + return no_of_days; + } + } + + private static void syncInitialDataWithQD(List pipelineDetailsMapList, BrowserStackCredentials browserStackCredentials) { + try { + ObjectMapper objectMapper = new ObjectMapper(); + String jsonBody = objectMapper.writeValueAsString(pipelineDetailsMapList); + + RequestBody requestBody = RequestBody.create(MediaType.parse("application/json"), jsonBody); + Response response = apiUtil.makePostRequestToQd(Constants.QualityDashboardAPI.ADD_ALL_PIPELINES, browserStackCredentials, requestBody); + } catch(IOException e) { + e.printStackTrace(); + } + } +} + +class PipelineDetails { + + @JsonProperty("buildNumber") + private Integer buildNumber; + @JsonProperty("buildDuration") + private Long buildDuration; + @JsonProperty("buildStatus") + private String buildStatus; + + @JsonProperty("endTime") + private Timestamp endTime; + + public PipelineDetails(Integer buildNumber, Long buildDuration, String buildStatus, Timestamp endTime) { + this.buildNumber = buildNumber; + this.buildDuration = buildDuration; + this.buildStatus = buildStatus; + this.endTime = endTime; + } +} + +class PipelineDetailsMap { + @JsonProperty("pipelineName") + private String pipelineName; + + @JsonProperty("pipelineDetails") + private List pipelineDetails; + + public PipelineDetailsMap(String pipelineName, List pipelineDetails) { + this.pipelineName = pipelineName; + this.pipelineDetails = pipelineDetails; + } +} diff --git a/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardInitItemListener.java b/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardInitItemListener.java new file mode 100644 index 0000000..6dcbe8f --- /dev/null +++ b/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardInitItemListener.java @@ -0,0 +1,123 @@ +package com.browserstack.automate.ci.jenkins.qualityDashboard; + +import com.browserstack.automate.ci.common.constants.Constants; +import com.browserstack.automate.ci.jenkins.BrowserStackCredentials; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import hudson.Extension; +import hudson.model.Item; +import hudson.model.listeners.ItemListener; +import okhttp3.MediaType; +import okhttp3.RequestBody; +import okhttp3.Response; +import org.jenkinsci.plugins.workflow.job.WorkflowJob; +import java.io.IOException; +import java.io.Serializable; +import java.net.HttpURLConnection; + +@Extension +public class QualityDashboardInitItemListener extends ItemListener { + + @Override + public void onCreated(Item job) { + String itemName = job.getFullName(); + String itemType = getItemTypeModified(job); + if(itemType != null && itemType.equals("PIPELINE")) { + try { + String jsonBody = getJsonReqBody(new ItemUpdate(itemName, itemType)); + syncItemListToQD(jsonBody, Constants.QualityDashboardAPI.ITEM_CRUD, "POST"); + } catch(IOException e) { + e.printStackTrace(); + } + } + } + + @Override + public void onDeleted(Item job) { + String itemName = job.getFullName(); + String itemType = getItemTypeModified(job); + if(itemType != null) { + try { + String jsonBody = getJsonReqBody(new ItemUpdate(itemName, itemType)); + syncItemListToQD(jsonBody, Constants.QualityDashboardAPI.ITEM_CRUD, "DELETE"); + } catch(IOException e) { + e.printStackTrace(); + } + } + } + + @Override + public void onRenamed(Item job, String oldName, String newName) { + String itemType = getItemTypeModified(job); + if(itemType != null) { + try { + oldName = job.getParent().getFullName() + "/" + oldName; + newName = job.getParent().getFullName() + "/" + newName; + String jsonBody = getJsonReqBody(new ItemRename(oldName, newName, itemType)); + syncItemListToQD(jsonBody, Constants.QualityDashboardAPI.ITEM_CRUD, "PUT"); + } catch(IOException e) { + e.printStackTrace(); + } + } + } + + private String getItemTypeModified(Item job) { + String itemType = null; + boolean isFolderRenamed = job.getClass().getName().contains("Folder"); + boolean isPipelineRenamed = job instanceof WorkflowJob; + if(isFolderRenamed || isPipelineRenamed) { + itemType = isPipelineRenamed ? "PIPELINE" : "FOLDER"; + } + return itemType; + } + + private String getJsonReqBody( T item) throws JsonProcessingException { + ObjectMapper objectMapper = new ObjectMapper(); + String jsonBody = objectMapper.writeValueAsString(item); + return jsonBody; + } + + private Response syncItemListToQD(String jsonBody, String url, String typeOfRequest) { + RequestBody requestBody = RequestBody.create(MediaType.parse("application/json"), jsonBody); + QualityDashboardAPIUtil apiUtil = new QualityDashboardAPIUtil(); + BrowserStackCredentials browserStackCredentials = QualityDashboardUtil.getBrowserStackCreds(); + if(typeOfRequest.equals("PUT")) { + return apiUtil.makePutRequestToQd(url, browserStackCredentials, requestBody); + } else if(typeOfRequest.equals("DELETE")) { + return apiUtil.makeDeleteRequestToQd(url, browserStackCredentials, requestBody); + } else { + return apiUtil.makePostRequestToQd(url, browserStackCredentials, requestBody); + } + } +} + +class ItemUpdate implements Serializable { + @JsonProperty("item") + private String itemName; + + @JsonProperty("itemType") + private String itemType; + + public ItemUpdate(String itemName, String itemType) { + this.itemName = itemName; + this.itemType = itemType; + } +} + +class ItemRename implements Serializable { + @JsonProperty("fromName") + private String fromItemName; + + @JsonProperty("toName") + private String toItemName; + + @JsonProperty("itemType") + private String itemType; + + public ItemRename(String fromItemName, String toItemName, String itemType) { + this.fromItemName = fromItemName; + this.toItemName = toItemName; + this.itemType = itemType; + } +} diff --git a/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardPipelineTracker.java b/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardPipelineTracker.java new file mode 100644 index 0000000..449e6fe --- /dev/null +++ b/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardPipelineTracker.java @@ -0,0 +1,292 @@ +package com.browserstack.automate.ci.jenkins.qualityDashboard; + +import com.browserstack.automate.ci.common.constants.Constants; +import com.browserstack.automate.ci.jenkins.BrowserStackCredentials; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; +import hudson.Extension; +import hudson.model.*; +import hudson.model.listeners.RunListener; +import io.jenkins.cli.shaded.org.apache.commons.lang.StringUtils; +import jenkins.model.Jenkins; +import okhttp3.*; +import org.apache.commons.io.FileUtils; +import org.jenkinsci.plugins.workflow.job.WorkflowJob; +import org.jenkinsci.plugins.workflow.job.WorkflowRun; + +import java.io.File; +import java.io.IOException; +import java.io.Serializable; +import java.net.HttpURLConnection; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Timestamp; +import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +@Extension +public class QualityDashboardPipelineTracker extends RunListener { + + QualityDashboardAPIUtil apiUtil = new QualityDashboardAPIUtil(); + + //check if this is the final step of the build + // if yes, get the result dir for the pipeline + // compose an object which contains, status, duration and zip fo test directory (if configured) + // check if this was called by upstream proj. if so, get result dir for the parent pipeline + // if parent pipeline configured then, copy the test results in that dir. If result dir not configured for parent, create a browserstack-artifacts dir if not present and copy the result there + @Override + public void onCompleted(Run run, TaskListener listener) { + super.onCompleted(run, listener); + BrowserStackCredentials browserStackCredentials = QualityDashboardUtil.getBrowserStackCreds(); + if(browserStackCredentials!=null) { + try { + WorkflowRun workflowRun = (WorkflowRun) run; + WorkflowJob workflowJob = workflowRun.getParent(); + String jobName = workflowJob.getFullName(); + if(isQDEnabled(browserStackCredentials) && isPipelineEnabledForQD(browserStackCredentials, jobName)) { + Result overallResult = run.getResult(); + if(overallResult != null) { + String finalZipFilePath = null; String qdS3Url = null; + String finalPathToZip = getFinalZipPath(run, browserStackCredentials); + if(StringUtils.isNotEmpty(finalPathToZip)) { + int buildNumber = run.getNumber(); + copyDirectoryToParentIfRequired(run, finalPathToZip, browserStackCredentials); + finalZipFilePath = packZip(finalPathToZip, run, jobName); + qdS3Url = uploadZipToQd(finalZipFilePath, browserStackCredentials, jobName, buildNumber); + } + sendBuildDataToQD(run, overallResult, qdS3Url, browserStackCredentials); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + private void sendBuildDataToQD(Run run, Result overallResult, String finalZipPath, BrowserStackCredentials browserStackCredentials) { + Long pipelineDuration = getPipelineDuration(run); + try { + String jobName = run.getParent().getFullName(); + int buildNumber = run.getNumber(); + long endTimeInMillis = run.getTimeInMillis(); + Timestamp endTime = new Timestamp(endTimeInMillis); + PipelineResults pipelineResultsReqObj = new PipelineResults(buildNumber, pipelineDuration, overallResult.toString(), finalZipPath, jobName, endTime); + ObjectMapper objectMapper = new ObjectMapper(); + String jsonBody = objectMapper.writeValueAsString(pipelineResultsReqObj); + + RequestBody requestBody = RequestBody.create(MediaType.parse("application/json"), jsonBody); + apiUtil.makePostRequestToQd(Constants.QualityDashboardAPI.STORE_PIPELINE_RESULTS, browserStackCredentials, requestBody); + } catch(IOException e) { + e.printStackTrace(); + } + } + + private Long getPipelineDuration(Run build) { + long startTime = build.getStartTimeInMillis(); + long endTime = System.currentTimeMillis(); + long duration = (endTime - startTime) / 1000; + return duration; + } + + private boolean checkIfPathIsFound(String filePath) { + Path path = Paths.get(filePath); + return Files.exists(path) ? true : false; + } + private String getFinalZipPath(Run run, BrowserStackCredentials browserStackCredentials) { + String finalZipPath = null; + String currentResultDir = getResultDirForPipeline(getUrlForPipeline(run), browserStackCredentials); + if(StringUtils.isNotEmpty(currentResultDir) && checkIfPathIsFound(currentResultDir)) { + finalZipPath = currentResultDir; + } else { + String defaultWorkspaceDir = getDefaultWorkspaceDirectory(run); + if(StringUtils.isNotEmpty(defaultWorkspaceDir)) { + String jobName = run.getParent().getName(); + defaultWorkspaceDir = defaultWorkspaceDir + "/workspace/" + jobName + "/browserstack-artifacts"; + finalZipPath = checkIfPathIsFound(defaultWorkspaceDir) ? defaultWorkspaceDir : null; + } + } + return finalZipPath; + } + + private String getDefaultWorkspaceDirectory(Run run) { + String pipelineDirectory = null; + Jenkins jenkins = Jenkins.getInstanceOrNull(); + String workspacePath = jenkins.getRootDir().getAbsolutePath(); + if (StringUtils.isNotEmpty(workspacePath)) { + pipelineDirectory = workspacePath; + } + return pipelineDirectory; + } + + private String getUrlForPipeline(Run build) { + return build.getParent().getFullName(); + } + + private boolean isQDEnabled(BrowserStackCredentials browserStackCredentials) throws IOException { + Response response = apiUtil.makeGetRequestToQd(Constants.QualityDashboardAPI.IS_QD_ENABLED, browserStackCredentials); + if (response != null && response.code() == HttpURLConnection.HTTP_OK) { + ResponseBody responseBody = response.body(); + if(responseBody != null && Boolean.parseBoolean(response.body().string())) { + return true; + } + } + return false; + } + + private boolean isPipelineEnabledForQD(BrowserStackCredentials browserStackCredentials, String pipelineName) throws IOException { + QualityDashboardGetDetailsForPipeline getPipelineEnabledObj = new QualityDashboardGetDetailsForPipeline(pipelineName); + ObjectMapper objectMapper = new ObjectMapper(); + String jsonBody = objectMapper.writeValueAsString(getPipelineEnabledObj); + RequestBody requestBody = RequestBody.create(MediaType.parse("application/json"), jsonBody); + Response response = apiUtil.makePostRequestToQd(Constants.QualityDashboardAPI.IS_PIPELINE_ENABLED, browserStackCredentials, requestBody); + if (response != null && response.code() == HttpURLConnection.HTTP_OK) { + ResponseBody responseBody = response.body(); + if(responseBody != null && Boolean.parseBoolean(response.body().string())) { + return true; + } + } + return false; + } + + private String getResultDirForPipeline(String pipelineUrl, BrowserStackCredentials browserStackCredentials) { + String resultDir = null; + try { + QualityDashboardGetDetailsForPipeline getResultDirReqObj = new QualityDashboardGetDetailsForPipeline(pipelineUrl); + ObjectMapper objectMapper = new ObjectMapper(); + String jsonBody = objectMapper.writeValueAsString(getResultDirReqObj); + + RequestBody requestBody = RequestBody.create(MediaType.parse("application/json"), jsonBody); + Response response = apiUtil.makePostRequestToQd(Constants.QualityDashboardAPI.GET_RESULT_DIRECTORY, browserStackCredentials, requestBody); + if (response != null && response.code() == HttpURLConnection.HTTP_OK) { + String responseBody = response.body() !=null ? response.body().string() : null; + resultDir = responseBody; + } + } catch(IOException e) { + e.printStackTrace(); + } + return resultDir; + } + + private String packZip(String sourceDirPath, Run run, String jobName) { + String defaultWorkspaceDir = getDefaultWorkspaceDirectory(run); + String zipFilePath = defaultWorkspaceDir + "/workspace/" + jobName + "/browserstack-artifacts.zip"; + try { + Files.deleteIfExists(Paths.get(zipFilePath)); + Path p = Files.createFile(Paths.get(zipFilePath)); + try (ZipOutputStream zs = new ZipOutputStream(Files.newOutputStream(p))) { + Path pp = Paths.get(sourceDirPath); + Files.walk(pp) + .filter(path -> !Files.isDirectory(path)) + .forEach(path -> { + ZipEntry zipEntry = new ZipEntry(pp.relativize(path).toString()); + try { + zs.putNextEntry(zipEntry); + Files.copy(path, zs); + zs.closeEntry(); + } catch (IOException e) { + System.err.println(e); + } + }); + } + } catch (IOException e) { + e.printStackTrace(); + } + return zipFilePath; + } + + private String uploadZipToQd(String pathToZip, BrowserStackCredentials browserStackCredentials, String jobName, int buildNumber) throws IOException { + String qdS3Url = null; + File fileToUpload = new File(pathToZip); + RequestBody requestBody = new MultipartBody.Builder() + .setType(MultipartBody.FORM) + .addFormDataPart("file", fileToUpload.getName(), + RequestBody.create(MediaType.parse("application/octet-stream"), fileToUpload)) + .addFormDataPart("jobName", jobName) + .addFormDataPart("buildNumber", String.valueOf(buildNumber)) + .build(); + + Response response = apiUtil.makePostRequestToQd(Constants.QualityDashboardAPI.UPLOAD_RESULT_ZIP, browserStackCredentials, requestBody); + if (response != null && response.code() == HttpURLConnection.HTTP_OK) { + qdS3Url = response.body() !=null ? response.body().string() : null; + } + return qdS3Url; + } + + private void copyDirectoryToParentIfRequired(Run run, String finalParentPathFrom, BrowserStackCredentials browserStackCredentials) throws IOException { + String finalParentPathTo = null; + String upStreamProj = upStreamPipelineUrl(run); + if(StringUtils.isNotEmpty(upStreamProj)) { + String parentResultDir = getResultDirForPipeline(upStreamProj, browserStackCredentials); + if(StringUtils.isNotEmpty(parentResultDir) && checkIfPathIsFound(parentResultDir)) { + finalParentPathTo = parentResultDir; + } else { + String defaultWorkspaceDir = getDefaultWorkspaceDirectory(run); + if(StringUtils.isNotEmpty(defaultWorkspaceDir) && checkIfPathIsFound(defaultWorkspaceDir)) { + defaultWorkspaceDir = defaultWorkspaceDir + "/workspace/" + upStreamProj + "/browserstack-artifacts"; + boolean pathAlreadyExists = checkIfPathIsFound(defaultWorkspaceDir); + if(!pathAlreadyExists) { + Files.createDirectory(Paths.get(defaultWorkspaceDir)); + } + finalParentPathTo = defaultWorkspaceDir; + } + } + if(StringUtils.isNotEmpty(finalParentPathTo)) { + FileUtils.copyDirectoryToDirectory(new File(finalParentPathFrom), new File(finalParentPathTo)); + int buildNum = run.getNumber(); + File finalParentFromFile = new File(finalParentPathFrom); + File newZipDir = new File(finalParentPathTo + "/" + finalParentFromFile.getName() + "_" + buildNum); + FileUtils.moveDirectory(new File(finalParentPathTo + "/" + finalParentFromFile.getName()), newZipDir); + } + } + } + + private String upStreamPipelineUrl(Run run) { + String upstreamProjectName = null; + List causes = run.getCauses(); + for (Cause cause : causes) { + if (cause instanceof Cause.UpstreamCause) { + Cause.UpstreamCause upstreamCause = (Cause.UpstreamCause) cause; + upstreamProjectName = upstreamCause.getUpstreamProject(); + } + } + return upstreamProjectName; + } +} + +class QualityDashboardGetDetailsForPipeline implements Serializable { + @JsonProperty("url") + private String pipeline; + public QualityDashboardGetDetailsForPipeline(String pipeline) { + this.pipeline = pipeline; + } +} + +class PipelineResults implements Serializable { + + @JsonProperty("buildNumber") + private Integer buildNumber; + + @JsonProperty("pipelineName") + private String pipelineName; + @JsonProperty("buildDuration") + private Long buildDuration; + + @JsonProperty("endTime") + private Timestamp endTime; + @JsonProperty("buildStatus") + private String buildStatus; + + @JsonProperty("zipFile") + private String zipFile; + + public PipelineResults(Integer buildNumber, Long buildDuration, String buildStatus, String zipFile, String pipelineName, Timestamp endTime) { + this.buildNumber = buildNumber; + this.buildDuration = buildDuration; + this.buildStatus = buildStatus; + this.zipFile = zipFile; + this.pipelineName = pipelineName; + this.endTime = endTime; + } +} diff --git a/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardUtil.java b/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardUtil.java new file mode 100644 index 0000000..e2dff06 --- /dev/null +++ b/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardUtil.java @@ -0,0 +1,18 @@ +package com.browserstack.automate.ci.jenkins.qualityDashboard; + +import com.browserstack.automate.ci.jenkins.BrowserStackCredentials; +import com.cloudbees.plugins.credentials.CredentialsProvider; +import com.cloudbees.plugins.credentials.common.StandardCredentials; +import jenkins.model.Jenkins; + +import java.util.ArrayList; +import java.util.List; + +public class QualityDashboardUtil { + public static BrowserStackCredentials getBrowserStackCreds() { + Jenkins jenkins = Jenkins.getInstanceOrNull(); + List creds = CredentialsProvider.lookupCredentials(StandardCredentials.class,jenkins,null,new ArrayList<>()); + BrowserStackCredentials browserStackCredentials = (BrowserStackCredentials) creds.stream().filter(c -> c instanceof BrowserStackCredentials).findFirst().orElse(null); + return browserStackCredentials; + } +} From ef74517dd92370389018c551b197a08e86418e94 Mon Sep 17 00:00:00 2001 From: Darpan Lalwani Date: Fri, 5 Jan 2024 14:06:37 +0530 Subject: [PATCH 2/6] reverting version bump --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2cbbf70..aee9173 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ browserstack-integration - 1.2.13-SNAPSHOT + 1.2.12-SNAPSHOT hpi BrowserStack From 3a4a238ad4f58a961520febaf02dca22d7081a51 Mon Sep 17 00:00:00 2001 From: Darpan Lalwani Date: Wed, 17 Jan 2024 08:25:06 +0530 Subject: [PATCH 3/6] Quality Dashboard Enhancements --- pom.xml | 16 +- .../ci/common/constants/Constants.java | 9 +- .../QualityDashboardAPIUtil.java | 25 +- .../QualityDashboardInit.java | 238 ++++++++++++++---- .../QualityDashboardInitItemListener.java | 6 +- .../QualityDashboardPipelineTracker.java | 107 ++++---- 6 files changed, 306 insertions(+), 95 deletions(-) diff --git a/pom.xml b/pom.xml index aee9173..2154151 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ browserstack-integration - 1.2.12-SNAPSHOT + 1.2.13-SNAPSHOT hpi BrowserStack @@ -177,6 +177,19 @@ 1.24 + + net.lingala.zip4j + zip4j + 2.11.5 + + + + org.zeroturnaround + zt-zip + 1.16 + jar + + com.google.code.gson @@ -286,6 +299,7 @@ spring-beans 5.3.18 + diff --git a/src/main/java/com/browserstack/automate/ci/common/constants/Constants.java b/src/main/java/com/browserstack/automate/ci/common/constants/Constants.java index 43e2f75..97240c0 100644 --- a/src/main/java/com/browserstack/automate/ci/common/constants/Constants.java +++ b/src/main/java/com/browserstack/automate/ci/common/constants/Constants.java @@ -63,13 +63,16 @@ public static final class SessionStatus { } public static final class QualityDashboardAPI { + public static final String URL_BASE = "https://quality-dashboard.browserstack.com/api/v1/jenkins"; - public static final String URL_BASE = "https://quality-dashboard-api.browserstack.com/api/v1/jenkins"; + public static final String LOG_MESSAGE = URL_BASE + "/log-message"; public static final String IS_INIT_SETUP_REQUIRED = URL_BASE + "/init-setup-required"; public static final String HISTORY_FOR_DAYS = URL_BASE + "/history-for-days"; - public static final String ADD_ALL_PIPELINES = URL_BASE + "/add-all-pipelines"; + public static final String SAVE_PIPELINES = URL_BASE + "/save-pipelines"; + + public static final String SAVE_PIPELINE_RESULTS = URL_BASE + "/save-pipeline-results"; public static final String ITEM_CRUD = URL_BASE + "/item"; public static final String IS_QD_ENABLED = URL_BASE + "/qd-enabled"; @@ -79,5 +82,7 @@ public static final class QualityDashboardAPI { public static final String UPLOAD_RESULT_ZIP = URL_BASE + "/upload-result"; public static final String STORE_PIPELINE_RESULTS = URL_BASE + "/save-results"; + public static final String PROJECTS_PAGE_SIZE = URL_BASE + "/projects-page-size"; + public static final String RESULTS_PAGE_SIZE = URL_BASE + "/results-page-size"; } } diff --git a/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardAPIUtil.java b/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardAPIUtil.java index 4ca6200..963416e 100644 --- a/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardAPIUtil.java +++ b/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardAPIUtil.java @@ -1,10 +1,13 @@ package com.browserstack.automate.ci.jenkins.qualityDashboard; +import com.browserstack.automate.ci.common.constants.Constants; import com.browserstack.automate.ci.jenkins.BrowserStackCredentials; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import okhttp3.*; - import java.io.IOException; -import java.util.concurrent.TimeUnit; +import java.io.Serializable; public class QualityDashboardAPIUtil { @@ -68,4 +71,22 @@ public Response makeDeleteRequestToQd(String postUrl, BrowserStackCredentials br } return null; } + + public void logToQD(BrowserStackCredentials browserStackCredentials, String logMessage) throws JsonProcessingException { + LogMessage logMessageObj = new LogMessage(logMessage); + ObjectMapper objectMapper = new ObjectMapper(); + String jsonBody = objectMapper.writeValueAsString(logMessageObj); + RequestBody requestBody = RequestBody.create(MediaType.parse("application/json"), jsonBody); + makePostRequestToQd(Constants.QualityDashboardAPI.LOG_MESSAGE, browserStackCredentials, requestBody); + } +} + +class LogMessage implements Serializable { + + @JsonProperty("message") + private String message; + + public LogMessage(String message) { + this.message = message; + } } diff --git a/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardInit.java b/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardInit.java index 4a53fb4..52da1fe 100644 --- a/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardInit.java +++ b/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardInit.java @@ -3,7 +3,9 @@ import com.browserstack.automate.ci.common.constants.Constants; import com.browserstack.automate.ci.jenkins.BrowserStackCredentials; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Lists; import hudson.Extension; import hudson.init.InitMilestone; import hudson.init.Initializer; @@ -27,73 +29,174 @@ public class QualityDashboardInit { @Initializer(after = InitMilestone.PLUGINS_PREPARED) public static void postInstall() { - initQDSetupIfRequired(); + try { + initQDSetupIfRequired(); + } catch(Exception e) { + e.printStackTrace(); + } } public void pluginConfiguredNotif() { - initQDSetupIfRequired(); + try { + initQDSetupIfRequired(); + } catch(Exception e) { + e.printStackTrace(); + } + } + + private static String exceptionToString(Throwable throwable) { + return throwable.toString(); } - private static void initQDSetupIfRequired() { + private static void initQDSetupIfRequired() throws JsonProcessingException { BrowserStackCredentials browserStackCredentials = QualityDashboardUtil.getBrowserStackCreds(); - if(browserStackCredentials!=null) { - checkQDIntegrationAndDumpMetaData(browserStackCredentials); + try { + if(browserStackCredentials!=null) { + apiUtil.logToQD(browserStackCredentials,"Starting plugin data export to QD"); + checkQDIntegrationAndDumpMetaData(browserStackCredentials); + } + } catch (Exception e) { + try { + apiUtil.logToQD(browserStackCredentials, "Global exception in data export is:"); + } catch (Exception ex) { + String exceptionString = exceptionToString(ex); + apiUtil.logToQD(browserStackCredentials, "Global exception in exception data export is:" + exceptionString); + } + } + } - private static void checkQDIntegrationAndDumpMetaData(BrowserStackCredentials browserStackCredentials) { + private static void checkQDIntegrationAndDumpMetaData(BrowserStackCredentials browserStackCredentials) throws JsonProcessingException { if(initialQDSetupRequired(browserStackCredentials)) { - List pipelineDetailsMapList = getExistingPipelineDump(browserStackCredentials); - if(!pipelineDetailsMapList.isEmpty()){ - syncInitialDataWithQD(pipelineDetailsMapList, browserStackCredentials); + List allPipelines = getAllPipelines(browserStackCredentials); + if(!allPipelines.isEmpty()){ + boolean projectsSavedSuccessfully = sendPipelinesPaginated(browserStackCredentials, allPipelines); + if(projectsSavedSuccessfully) { + List allBuilds = getAllBuilds(browserStackCredentials); + if(!allBuilds.isEmpty()){ + sendBuildsPaginated(browserStackCredentials, allBuilds); + } else { + apiUtil.logToQD(browserStackCredentials,"No Build Results data found"); + } + } else { + apiUtil.logToQD(browserStackCredentials,"Projects import failed, so not importing build results"); + } + } else { + apiUtil.logToQD(browserStackCredentials,"No pipelines detected"); } - } + } } - private static boolean initialQDSetupRequired(BrowserStackCredentials browserStackCredentials) { + private static boolean initialQDSetupRequired(BrowserStackCredentials browserStackCredentials) throws JsonProcessingException { try { Response response = apiUtil.makeGetRequestToQd(Constants.QualityDashboardAPI.IS_INIT_SETUP_REQUIRED, browserStackCredentials); if (response != null && response.code() == HttpURLConnection.HTTP_OK) { ResponseBody responseBody = response.body(); if(responseBody != null && responseBody.string().equals("REQUIRED")) { + apiUtil.logToQD(browserStackCredentials,"Initial QD setup is required"); return true; } } } catch(IOException e) { e.printStackTrace(); } + apiUtil.logToQD(browserStackCredentials,"Initial QD setup is not required"); return false; } - private static List getExistingPipelineDump(BrowserStackCredentials browserStackCredentials) { - List pipelineDetailsMapList = new ArrayList<>(); + private static List getAllPipelines(BrowserStackCredentials browserStackCredentials) throws JsonProcessingException { + List allPipelines = new ArrayList<>(); Jenkins jenkins = Jenkins.getInstanceOrNull(); - Instant thresholdInstant = Instant.now().minus(getHistoryForDays(browserStackCredentials), ChronoUnit.DAYS); if (jenkins != null) { jenkins.getAllItems().forEach(job -> { if(job instanceof WorkflowJob) { String pipelineName = job.getFullName(); - List pipelineDetailsList = new ArrayList<>(); + allPipelines.add(pipelineName); + } + }); + } else { + apiUtil.logToQD(browserStackCredentials,"Issue getting Jenkins Instance"); + } + return allPipelines; + } + + private static boolean sendPipelinesPaginated(BrowserStackCredentials browserStackCredentials, List allPipelines) { + boolean isSuccess = true; + int pageSize = getProjectPageSize(browserStackCredentials); + List> pipelinesInSmallerBatches = Lists.partition(allPipelines, pageSize); + int totalPages = !pipelinesInSmallerBatches.isEmpty() ? pipelinesInSmallerBatches.size() : 0; + int page = 0; + for(List singlePagePipelineList : pipelinesInSmallerBatches) { + try { + page++; + ObjectMapper objectMapper = new ObjectMapper(); + PipelinesPaginated pipelinesPaginated = new PipelinesPaginated(page, totalPages, singlePagePipelineList); + String jsonBody = objectMapper.writeValueAsString(pipelinesPaginated); + RequestBody requestBody = RequestBody.create(MediaType.parse("application/json"), jsonBody); + Response response = apiUtil.makePostRequestToQd(Constants.QualityDashboardAPI.SAVE_PIPELINES, browserStackCredentials, requestBody); + if (response == null || response.code() != HttpURLConnection.HTTP_OK) { + apiUtil.logToQD(browserStackCredentials,"Got Non 200 response while saving projects"); + isSuccess = false; + break; + } + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + return isSuccess; + } + + private static List getAllBuilds(BrowserStackCredentials browserStackCredentials) { + List allBuildResults = new ArrayList<>(); + Jenkins jenkins = Jenkins.getInstanceOrNull(); + Instant thresholdInstant = Instant.now().minus(getHistoryForDays(browserStackCredentials), ChronoUnit.DAYS); + if (jenkins != null) { + jenkins.getAllItems().forEach(job -> { + if (job instanceof WorkflowJob) { + String pipelineName = job.getFullName(); List allBuilds = ((WorkflowJob) job).getBuilds(); if(!allBuilds.isEmpty()) { - allBuilds.stream().filter(build -> Instant.ofEpochMilli(build.getTimeInMillis()).isAfter(thresholdInstant) ).forEach( - build -> { - int buildNumber = build.getNumber(); - long duration = build.getDuration(); - Result overallResult = build.getResult(); - long endTimeInMillis = build.getTimeInMillis(); - Timestamp endTime = new Timestamp(endTimeInMillis); - PipelineDetails pipelineDetail = new PipelineDetails(buildNumber, duration, overallResult.toString(), endTime ); - pipelineDetailsList.add(pipelineDetail); - } + allBuilds.stream().filter(build -> Instant.ofEpochMilli(build.getTimeInMillis()).isAfter(thresholdInstant)).forEach( + build -> { + int buildNumber = build.getNumber(); + long duration = build.getDuration(); + Result overallResult = build.getResult(); + long endTimeInMillis = build.getTimeInMillis(); + Timestamp endTime = new Timestamp(endTimeInMillis); + String result = overallResult != null ? overallResult.toString() : null; + PipelineDetails pipelineDetail = new PipelineDetails(pipelineName, buildNumber, duration, result, endTime); + allBuildResults.add(pipelineDetail); + } ); } - PipelineDetailsMap pipelineDetailsMap = new PipelineDetailsMap(pipelineName, pipelineDetailsList); - pipelineDetailsMapList.add(pipelineDetailsMap); } }); } - return pipelineDetailsMapList; + return allBuildResults; + } + + private static void sendBuildsPaginated(BrowserStackCredentials browserStackCredentials, List allBuilds) { + int pageSize = getResultPageSize(browserStackCredentials); + List> buildResultsInSmallerBatches = Lists.partition(allBuilds, pageSize); + int totalPages = !buildResultsInSmallerBatches.isEmpty() ? buildResultsInSmallerBatches.size() : 0; + int page = 0; + for(List buildResultList : buildResultsInSmallerBatches) { + try { + page++; + ObjectMapper objectMapper = new ObjectMapper(); + BuildResultsPaginated buildResultsPaginated = new BuildResultsPaginated(page, totalPages, buildResultList); + String jsonBody = objectMapper.writeValueAsString(buildResultsPaginated); + RequestBody requestBody = RequestBody.create(MediaType.parse("application/json"), jsonBody); + Response response = apiUtil.makePostRequestToQd(Constants.QualityDashboardAPI.SAVE_PIPELINE_RESULTS, browserStackCredentials, requestBody); + if (response == null || response.code() != HttpURLConnection.HTTP_OK) { + apiUtil.logToQD(browserStackCredentials,"Got Non 200 response while saving projects"); + break; + } + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } } private static int getHistoryForDays(BrowserStackCredentials browserStackCredentials) { @@ -115,21 +218,50 @@ private static int getHistoryForDays(BrowserStackCredentials browserStackCredent } } - private static void syncInitialDataWithQD(List pipelineDetailsMapList, BrowserStackCredentials browserStackCredentials) { + private static int getProjectPageSize(BrowserStackCredentials browserStackCredentials) { + int projectPageSize = 2000; try { - ObjectMapper objectMapper = new ObjectMapper(); - String jsonBody = objectMapper.writeValueAsString(pipelineDetailsMapList); + Response response = apiUtil.makeGetRequestToQd(Constants.QualityDashboardAPI.PROJECTS_PAGE_SIZE, browserStackCredentials); + if (response != null && response.code() == HttpURLConnection.HTTP_OK) { + ResponseBody responseBody = response.body(); + if(responseBody != null) { + String responseBodyStr = responseBody.string(); + if(responseBodyStr!=null) + projectPageSize = Integer.parseInt(responseBodyStr); + } + } + } catch(IOException e) { + e.printStackTrace(); + } finally { + return projectPageSize; + } + } - RequestBody requestBody = RequestBody.create(MediaType.parse("application/json"), jsonBody); - Response response = apiUtil.makePostRequestToQd(Constants.QualityDashboardAPI.ADD_ALL_PIPELINES, browserStackCredentials, requestBody); + private static int getResultPageSize(BrowserStackCredentials browserStackCredentials) { + int resultPageSize = 1000; + try { + Response response = apiUtil.makeGetRequestToQd(Constants.QualityDashboardAPI.RESULTS_PAGE_SIZE, browserStackCredentials); + if (response != null && response.code() == HttpURLConnection.HTTP_OK) { + ResponseBody responseBody = response.body(); + if(responseBody != null) { + String responseBodyStr = responseBody.string(); + if(responseBodyStr!=null) + resultPageSize = Integer.parseInt(responseBodyStr); + } + } } catch(IOException e) { e.printStackTrace(); + } finally { + return resultPageSize; } } } class PipelineDetails { + @JsonProperty("pipelineName") + private String pipelineName; + @JsonProperty("buildNumber") private Integer buildNumber; @JsonProperty("buildDuration") @@ -140,7 +272,8 @@ class PipelineDetails { @JsonProperty("endTime") private Timestamp endTime; - public PipelineDetails(Integer buildNumber, Long buildDuration, String buildStatus, Timestamp endTime) { + public PipelineDetails(String pipelineName, Integer buildNumber, Long buildDuration, String buildStatus, Timestamp endTime) { + this.pipelineName = pipelineName; this.buildNumber = buildNumber; this.buildDuration = buildDuration; this.buildStatus = buildStatus; @@ -148,15 +281,36 @@ public PipelineDetails(Integer buildNumber, Long buildDuration, String buildStat } } -class PipelineDetailsMap { - @JsonProperty("pipelineName") - private String pipelineName; +class PipelinesPaginated { + @JsonProperty("page") + private int page; - @JsonProperty("pipelineDetails") - private List pipelineDetails; + @JsonProperty("totalPages") + private int totalPages; - public PipelineDetailsMap(String pipelineName, List pipelineDetails) { - this.pipelineName = pipelineName; - this.pipelineDetails = pipelineDetails; + @JsonProperty("pipelines") + private List pipelines; + + public PipelinesPaginated(int page, int totalPages, List pipelines) { + this.page = page; + this.totalPages = totalPages; + this.pipelines = pipelines; + } +} + +class BuildResultsPaginated { + @JsonProperty("page") + private int page; + + @JsonProperty("totalPages") + private int totalPages; + + @JsonProperty("builds") + private List builds; + + public BuildResultsPaginated(int page, int totalPages, List builds) { + this.page = page; + this.totalPages = totalPages; + this.builds = builds; } } diff --git a/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardInitItemListener.java b/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardInitItemListener.java index 6dcbe8f..86bec77 100644 --- a/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardInitItemListener.java +++ b/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardInitItemListener.java @@ -14,7 +14,6 @@ import org.jenkinsci.plugins.workflow.job.WorkflowJob; import java.io.IOException; import java.io.Serializable; -import java.net.HttpURLConnection; @Extension public class QualityDashboardInitItemListener extends ItemListener { @@ -78,15 +77,18 @@ private String getJsonReqBody( T item) throws JsonProcessingException { return jsonBody; } - private Response syncItemListToQD(String jsonBody, String url, String typeOfRequest) { + private Response syncItemListToQD(String jsonBody, String url, String typeOfRequest) throws JsonProcessingException { RequestBody requestBody = RequestBody.create(MediaType.parse("application/json"), jsonBody); QualityDashboardAPIUtil apiUtil = new QualityDashboardAPIUtil(); BrowserStackCredentials browserStackCredentials = QualityDashboardUtil.getBrowserStackCreds(); if(typeOfRequest.equals("PUT")) { + apiUtil.logToQD(browserStackCredentials, "Syncing Item Update - PUT"); return apiUtil.makePutRequestToQd(url, browserStackCredentials, requestBody); } else if(typeOfRequest.equals("DELETE")) { + apiUtil.logToQD(browserStackCredentials, "Syncing Item Deleted - DELETE"); return apiUtil.makeDeleteRequestToQd(url, browserStackCredentials, requestBody); } else { + apiUtil.logToQD(browserStackCredentials, "Syncing Item Added - POST"); return apiUtil.makePostRequestToQd(url, browserStackCredentials, requestBody); } } diff --git a/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardPipelineTracker.java b/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardPipelineTracker.java index 449e6fe..32f2763 100644 --- a/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardPipelineTracker.java +++ b/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardPipelineTracker.java @@ -3,6 +3,7 @@ import com.browserstack.automate.ci.common.constants.Constants; import com.browserstack.automate.ci.jenkins.BrowserStackCredentials; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import hudson.Extension; import hudson.model.*; @@ -13,6 +14,7 @@ import org.apache.commons.io.FileUtils; import org.jenkinsci.plugins.workflow.job.WorkflowJob; import org.jenkinsci.plugins.workflow.job.WorkflowRun; +import org.zeroturnaround.zip.ZipUtil; import java.io.File; import java.io.IOException; @@ -23,48 +25,65 @@ import java.nio.file.Paths; import java.sql.Timestamp; import java.util.List; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; @Extension public class QualityDashboardPipelineTracker extends RunListener { QualityDashboardAPIUtil apiUtil = new QualityDashboardAPIUtil(); - //check if this is the final step of the build - // if yes, get the result dir for the pipeline - // compose an object which contains, status, duration and zip fo test directory (if configured) - // check if this was called by upstream proj. if so, get result dir for the parent pipeline - // if parent pipeline configured then, copy the test results in that dir. If result dir not configured for parent, create a browserstack-artifacts dir if not present and copy the result there @Override public void onCompleted(Run run, TaskListener listener) { super.onCompleted(run, listener); BrowserStackCredentials browserStackCredentials = QualityDashboardUtil.getBrowserStackCreds(); if(browserStackCredentials!=null) { + WorkflowRun workflowRun = (WorkflowRun) run; + WorkflowJob workflowJob = workflowRun.getParent(); + String jobName = workflowJob.getFullName(); + int buildNumber = run.getNumber(); try { - WorkflowRun workflowRun = (WorkflowRun) run; - WorkflowJob workflowJob = workflowRun.getParent(); - String jobName = workflowJob.getFullName(); if(isQDEnabled(browserStackCredentials) && isPipelineEnabledForQD(browserStackCredentials, jobName)) { Result overallResult = run.getResult(); if(overallResult != null) { - String finalZipFilePath = null; String qdS3Url = null; + String qdS3Url = null; String finalPathToZip = getFinalZipPath(run, browserStackCredentials); + apiUtil.logToQD(browserStackCredentials, "Final Computed Zip Path for jobName: " + jobName + " and buildNumber: " + buildNumber + " is: " + finalPathToZip); if(StringUtils.isNotEmpty(finalPathToZip)) { - int buildNumber = run.getNumber(); + apiUtil.logToQD(browserStackCredentials, "Found artifacts in configured path for jobName: " + jobName + " and buildNumber: " + buildNumber); copyDirectoryToParentIfRequired(run, finalPathToZip, browserStackCredentials); - finalZipFilePath = packZip(finalPathToZip, run, jobName); - qdS3Url = uploadZipToQd(finalZipFilePath, browserStackCredentials, jobName, buildNumber); + qdS3Url = zipArtifactsAndUploadToQD(finalPathToZip, browserStackCredentials, jobName, buildNumber); + } else if(run.getHasArtifacts()) { + apiUtil.logToQD(browserStackCredentials, "No artifacts in configured path but found archive artifacts for jobName: " + jobName + " and buildNumber: " + buildNumber); + finalPathToZip = run.getArtifactsDir().getAbsolutePath(); + apiUtil.logToQD(browserStackCredentials, "Got artifact path for jobName: " + jobName + " and buildNumber: " + buildNumber + " as: " + finalPathToZip); + qdS3Url = zipArtifactsAndUploadToQD(finalPathToZip, browserStackCredentials, jobName, buildNumber); + } else { + apiUtil.logToQD(browserStackCredentials, "Finally no artifacts found for jobName: " + jobName + " and buildNumber: " + buildNumber); } sendBuildDataToQD(run, overallResult, qdS3Url, browserStackCredentials); + } else { + apiUtil.logToQD(browserStackCredentials, "Null Result Captured for jobName: " + jobName + " and buildNumber: " + buildNumber); } } } catch (IOException e) { + try { + apiUtil.logToQD(browserStackCredentials, "Global Exception for jobName: " + jobName + " and buildNumber: " + buildNumber + " is: " + e.toString()); + } catch (JsonProcessingException ex) { + throw new RuntimeException(ex); + } throw new RuntimeException(e); } } } + private String zipArtifactsAndUploadToQD (String finalPathToZip, BrowserStackCredentials browserStackCredentials, String jobName, int buildNumber) throws IOException { + String finalZipFilePath = packZip(finalPathToZip, jobName, browserStackCredentials); + apiUtil.logToQD(browserStackCredentials, "Final zip file's path for jobName: " + jobName + " and buildNumber: " + buildNumber + " is:" + finalZipFilePath); + String qdS3Url = uploadZipToQd(finalZipFilePath, browserStackCredentials, jobName, buildNumber); + Files.deleteIfExists(Paths.get(finalZipFilePath)); + apiUtil.logToQD(browserStackCredentials, "Deleted file from server after upload for jobName: " + jobName + " and buildNumber: " + buildNumber); + return qdS3Url; + } + private void sendBuildDataToQD(Run run, Result overallResult, String finalZipPath, BrowserStackCredentials browserStackCredentials) { Long pipelineDuration = getPipelineDuration(run); try { @@ -77,6 +96,7 @@ private void sendBuildDataToQD(Run run, Result overallResult, String finalZipPat String jsonBody = objectMapper.writeValueAsString(pipelineResultsReqObj); RequestBody requestBody = RequestBody.create(MediaType.parse("application/json"), jsonBody); + apiUtil.logToQD(browserStackCredentials, "Sending Final Results for jobName: " + jobName + " and buildNumber: " + buildNumber); apiUtil.makePostRequestToQd(Constants.QualityDashboardAPI.STORE_PIPELINE_RESULTS, browserStackCredentials, requestBody); } catch(IOException e) { e.printStackTrace(); @@ -94,9 +114,9 @@ private boolean checkIfPathIsFound(String filePath) { Path path = Paths.get(filePath); return Files.exists(path) ? true : false; } - private String getFinalZipPath(Run run, BrowserStackCredentials browserStackCredentials) { + private String getFinalZipPath(Run run, BrowserStackCredentials browserStackCredentials) throws JsonProcessingException { String finalZipPath = null; - String currentResultDir = getResultDirForPipeline(getUrlForPipeline(run), browserStackCredentials); + String currentResultDir = getResultDirForPipeline(getUrlForPipeline(run), browserStackCredentials, run.getNumber()); if(StringUtils.isNotEmpty(currentResultDir) && checkIfPathIsFound(currentResultDir)) { finalZipPath = currentResultDir; } else { @@ -111,13 +131,9 @@ private String getFinalZipPath(Run run, BrowserStackCredentials browserStackCred } private String getDefaultWorkspaceDirectory(Run run) { - String pipelineDirectory = null; Jenkins jenkins = Jenkins.getInstanceOrNull(); - String workspacePath = jenkins.getRootDir().getAbsolutePath(); - if (StringUtils.isNotEmpty(workspacePath)) { - pipelineDirectory = workspacePath; - } - return pipelineDirectory; + String workspacePath = jenkins != null && jenkins.getRootDir() != null ? jenkins.getRootDir().getAbsolutePath() : null; + return StringUtils.isNotEmpty(workspacePath) ? workspacePath : null; } private String getUrlForPipeline(Run build) { @@ -129,9 +145,11 @@ private boolean isQDEnabled(BrowserStackCredentials browserStackCredentials) thr if (response != null && response.code() == HttpURLConnection.HTTP_OK) { ResponseBody responseBody = response.body(); if(responseBody != null && Boolean.parseBoolean(response.body().string())) { + apiUtil.logToQD(browserStackCredentials, "QD enabled check passed"); return true; } } + apiUtil.logToQD(browserStackCredentials, "QD enabled check failed"); return false; } @@ -144,13 +162,15 @@ private boolean isPipelineEnabledForQD(BrowserStackCredentials browserStackCrede if (response != null && response.code() == HttpURLConnection.HTTP_OK) { ResponseBody responseBody = response.body(); if(responseBody != null && Boolean.parseBoolean(response.body().string())) { + apiUtil.logToQD(browserStackCredentials, "Pipeline enabled - pipelineName: " + pipelineName); return true; } } + apiUtil.logToQD(browserStackCredentials, "Pipeline disabled - pipelineName: " + pipelineName); return false; } - private String getResultDirForPipeline(String pipelineUrl, BrowserStackCredentials browserStackCredentials) { + private String getResultDirForPipeline(String pipelineUrl, BrowserStackCredentials browserStackCredentials, int buildNumber) throws JsonProcessingException { String resultDir = null; try { QualityDashboardGetDetailsForPipeline getResultDirReqObj = new QualityDashboardGetDetailsForPipeline(pipelineUrl); @@ -166,34 +186,29 @@ private String getResultDirForPipeline(String pipelineUrl, BrowserStackCredentia } catch(IOException e) { e.printStackTrace(); } + resultDir = resultDir !=null && resultDir.contains("%build_number%") ? resultDir.replace("%build_number%", String.valueOf(buildNumber)) : resultDir; + apiUtil.logToQD(browserStackCredentials, "Result Directory for jobName: " + pipelineUrl + " and buildNumber: " + buildNumber + " is resultDir: " + resultDir); return resultDir; } - private String packZip(String sourceDirPath, Run run, String jobName) { - String defaultWorkspaceDir = getDefaultWorkspaceDirectory(run); - String zipFilePath = defaultWorkspaceDir + "/workspace/" + jobName + "/browserstack-artifacts.zip"; + private String packZip(String sourceDirPath, String jobName, BrowserStackCredentials browserStackCredentials) throws JsonProcessingException { + Path zipPath = Paths.get(sourceDirPath).getParent(); + String zipFile = zipPath.toString() + "/browserstack-artifacts.zip"; + Path zipFilePath = Paths.get(zipFile); + apiUtil.logToQD(browserStackCredentials, "zipFilePath for jobName: " + jobName + " is:" + zipFilePath); try { - Files.deleteIfExists(Paths.get(zipFilePath)); - Path p = Files.createFile(Paths.get(zipFilePath)); - try (ZipOutputStream zs = new ZipOutputStream(Files.newOutputStream(p))) { - Path pp = Paths.get(sourceDirPath); - Files.walk(pp) - .filter(path -> !Files.isDirectory(path)) - .forEach(path -> { - ZipEntry zipEntry = new ZipEntry(pp.relativize(path).toString()); - try { - zs.putNextEntry(zipEntry); - Files.copy(path, zs); - zs.closeEntry(); - } catch (IOException e) { - System.err.println(e); - } - }); - } + Files.deleteIfExists(zipFilePath); + ZipUtil.pack(new File(sourceDirPath), new File(zipFile)); + apiUtil.logToQD(browserStackCredentials, "zipFile size for jobName: " + jobName + " is:" + Files.size(zipFilePath)); } catch (IOException e) { - e.printStackTrace(); + String exceptionString = exceptionToString(e); + apiUtil.logToQD(browserStackCredentials, "Error creating zip for jobName: " + jobName + " is:" + exceptionString); } - return zipFilePath; + return zipFile; + } + + private String exceptionToString(Throwable throwable) { + return throwable.toString(); } private String uploadZipToQd(String pathToZip, BrowserStackCredentials browserStackCredentials, String jobName, int buildNumber) throws IOException { @@ -218,7 +233,7 @@ private void copyDirectoryToParentIfRequired(Run run, String finalParentPathFrom String finalParentPathTo = null; String upStreamProj = upStreamPipelineUrl(run); if(StringUtils.isNotEmpty(upStreamProj)) { - String parentResultDir = getResultDirForPipeline(upStreamProj, browserStackCredentials); + String parentResultDir = getResultDirForPipeline(upStreamProj, browserStackCredentials, run.getNumber()); if(StringUtils.isNotEmpty(parentResultDir) && checkIfPathIsFound(parentResultDir)) { finalParentPathTo = parentResultDir; } else { From f51fbf0f4269a64a4f86c6e2661c5241dcc70b4a Mon Sep 17 00:00:00 2001 From: Darpan Lalwani Date: Wed, 17 Jan 2024 08:27:13 +0530 Subject: [PATCH 4/6] Quality Dashboard --- .../qualityDashboard/QualityDashboardPipelineTracker.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardPipelineTracker.java b/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardPipelineTracker.java index 32f2763..fb7d80e 100644 --- a/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardPipelineTracker.java +++ b/src/main/java/com/browserstack/automate/ci/jenkins/qualityDashboard/QualityDashboardPipelineTracker.java @@ -79,8 +79,12 @@ private String zipArtifactsAndUploadToQD (String finalPathToZip, BrowserStackCre String finalZipFilePath = packZip(finalPathToZip, jobName, browserStackCredentials); apiUtil.logToQD(browserStackCredentials, "Final zip file's path for jobName: " + jobName + " and buildNumber: " + buildNumber + " is:" + finalZipFilePath); String qdS3Url = uploadZipToQd(finalZipFilePath, browserStackCredentials, jobName, buildNumber); - Files.deleteIfExists(Paths.get(finalZipFilePath)); - apiUtil.logToQD(browserStackCredentials, "Deleted file from server after upload for jobName: " + jobName + " and buildNumber: " + buildNumber); + if(StringUtils.isNotEmpty(finalZipFilePath)) { + Files.deleteIfExists(Paths.get(finalZipFilePath)); + apiUtil.logToQD(browserStackCredentials, "Deleted file from server after upload for jobName: " + jobName + " and buildNumber: " + buildNumber); + } else { + apiUtil.logToQD(browserStackCredentials, "No zip file to delete for jobName: " + jobName + " and buildNumber: " + buildNumber); + } return qdS3Url; } From c0cb1df44488de3424a3c2eb5c9a46bb4052717a Mon Sep 17 00:00:00 2001 From: Darpan Lalwani Date: Wed, 17 Jan 2024 08:29:11 +0530 Subject: [PATCH 5/6] Quality Dashboard --- pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pom.xml b/pom.xml index 2154151..93a243b 100644 --- a/pom.xml +++ b/pom.xml @@ -177,12 +177,6 @@ 1.24 - - net.lingala.zip4j - zip4j - 2.11.5 - - org.zeroturnaround zt-zip From 76a6388612fcc32f9d7a827da1dbcdc7a3cd8b22 Mon Sep 17 00:00:00 2001 From: Darpan Lalwani Date: Thu, 18 Jan 2024 14:05:47 +0530 Subject: [PATCH 6/6] reverting version bump --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 93a243b..719d9b2 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ browserstack-integration - 1.2.13-SNAPSHOT + 1.2.12-SNAPSHOT hpi BrowserStack