#126: Monitoring for ide-urls

.github/workflows/update-urls.yml

@@ -1,8 +1,6 @@
 name: Update URLS
 on:
-  workflow_dispatch:
-  schedule:
-    - cron: '0 3 * * *'
+  push
 jobs:
   updateURLS:
     runs-on: ubuntu-latest
@@ -14,7 +12,7 @@ jobs:
       - name: Checkout ide-urls
         uses: actions/checkout@v3
         with:
-          repository: devonfw/ide-urls
+          repository: alfeilex/ide-urls
           path: ide-urls
           token: ${{ secrets.ACTION_PUSH_TOKEN }}
       - name: Set up Java
@@ -27,7 +25,9 @@ jobs:
         run: |
           cd cli
           mvn -B -ntp -Dstyle.color=always install
-          mvn -B -ntp -Dstyle.color=always exec:java -Dexec.mainClass="com.devonfw.tools.ide.url.UpdateInitiator" -Dexec.args="../ide-urls PT5H30M"
+          mvn -B -ntp -Dstyle.color=always exec:java -Dexec.mainClass="com.devonfw.tools.ide.url.UpdateInitiator" -Dexec.args="../ide-urls PT5H30M" | tee log.txt
+          sed -n '/ERROR REPORT FROM/,/ERROR REPORT ENDS/p' log.txt > error_report.txt
+          cat error_report.txt
       - name: Commit and push to ide-urls
         run: |
           cd ide-urls

cli/src/main/java/com/devonfw/tools/ide/url/UpdateInitiator.java

@@ -8,6 +8,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.devonfw.tools.ide.url.model.UrlErrorReport;
 import com.devonfw.tools.ide.url.updater.UpdateManager;
 
 /**
@@ -54,5 +55,7 @@ public static void main(String[] args) {
 
     UpdateManager updateManager = new UpdateManager(repoPath, expirationTime);
     updateManager.updateAll();
+
+    logger.warn(UrlErrorReport.getReport());
   }
 }

cli/src/main/java/com/devonfw/tools/ide/url/model/UrlErrorReport.java

@@ -0,0 +1,45 @@
+package com.devonfw.tools.ide.url.model;
+
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.List;
+
+public class UrlErrorReport {
+
+  private static final UrlErrorReport instance = new UrlErrorReport();
+  private static UrlErrorReport getInstance() {
+    return instance;
+  }
+  public static List<UrlErrorState> urlErrorStates = new ArrayList<>();
+
+  private static UrlErrorState addErrorState(String toolWithEdition) {
+    UrlErrorState state = new UrlErrorState(toolWithEdition);
+    urlErrorStates.add(state);
+    return state;
+  }
+
+  public static UrlErrorState getErrorState(String toolWithEdition){
+    if (!urlErrorStates.isEmpty()) {
+      for (UrlErrorState state : urlErrorStates) {
+        if (state.getToolWithEdition().equals(toolWithEdition)) {
+          return state;
+        } else {
+          return addErrorState(toolWithEdition);
+        }
+      }
+    } else {
+      return addErrorState(toolWithEdition);
+    }
+    return null;
+  }
+
+  public static String getReport() {
+    StringBuilder report = new StringBuilder();
+    report.append("\nERROR REPORT FROM: ").append(LocalDateTime.now()).append("\n");
+    for (UrlErrorState state : urlErrorStates) {
+      report.append(state.toString()).append("\n");
+    }
+    report.append("ERROR REPORT ENDS").append("\n");
+    return report.toString();
+  }
+}

cli/src/main/java/com/devonfw/tools/ide/url/model/UrlErrorState.java

@@ -0,0 +1,107 @@
+package com.devonfw.tools.ide.url.model;
+
+public final class UrlErrorState {
+
+  private final String toolWithEdition;
+  private int additionSuccesses;
+  private int additionFailures;
+  private int verificationSuccesses;
+  private int verificationFailures;
+
+  public UrlErrorState(String toolWithEdition) {
+
+    this.toolWithEdition = toolWithEdition;
+    this.additionSuccesses = 0;
+    this.additionFailures = 0;
+    this.verificationSuccesses = 0;
+    this.verificationFailures = 0;
+  }
+
+  public String getToolWithEdition() {
+
+    return toolWithEdition;
+  }
+
+  public int getAdditionSuccesses() {
+
+    return additionSuccesses;
+  }
+
+  public void setAdditionSuccesses(int additionSuccesses) {
+
+    this.additionSuccesses = additionSuccesses;
+  }
+
+  public int getAdditionFailures() {
+
+    return additionFailures;
+  }
+
+  public void setAdditionFailures(int additionFailures) {
+
+    this.additionFailures = additionFailures;
+  }
+
+  public int getVerificationSuccesses() {
+
+    return verificationSuccesses;
+  }
+
+  public void setVerificationSuccesses(int verificationSuccesses) {
+
+    this.verificationSuccesses = verificationSuccesses;
+  }
+
+  public int getVerificationFailures() {
+
+    return verificationFailures;
+  }
+
+  public void setVerificationFailures(int verificationFailures) {
+
+    this.verificationFailures = verificationFailures;
+  }
+
+  private int getTotalAdditions() {
+    return this.additionFailures + this.additionSuccesses;
+  }
+  private int getTotalVerification() {
+    return this.verificationFailures + this.verificationSuccesses;
+  }
+
+  private String getErrorRate(int failures, int totals) {
+    if (failures == 0) {
+      return "0.00";
+    } else {
+      double errorRate = (double) failures / totals * 100;
+      return String.format("%.2f", errorRate);
+    }
+  }
+  public void updateAdditions(boolean success) {
+    if (success) {
+      this.additionSuccesses += 1;
+    } else {
+      this.additionFailures += 1;
+    }
+  }
+
+  public void updateVerifications(boolean success) {
+    if (success) {
+      this.verificationSuccesses += 1;
+    } else {
+      this.verificationFailures += 1;
+    }
+  }
+
+  @Override
+  public String toString() {
+
+    String additionState = "versions added: " + getAdditionFailures() + " failed, " + getAdditionSuccesses() + " succeeded, "
+        + getTotalAdditions() + " total, " + getErrorRate(getAdditionFailures(), getTotalAdditions()) + "% error";
+
+    String verificationState = " - versions verified: " + getVerificationFailures() + " failed, " + getVerificationSuccesses() +
+        " succeeded, " + getTotalVerification() + " total, " + getErrorRate(getVerificationFailures(), getTotalVerification()) + "% error";
+
+    return getToolWithEdition() + ": " + additionState + verificationState;
+  }
+}

cli/src/main/java/com/devonfw/tools/ide/url/updater/AbstractUrlUpdater.java

@@ -21,6 +21,8 @@
 
 import com.devonfw.tools.ide.os.OperatingSystem;
 import com.devonfw.tools.ide.os.SystemArchitecture;
+import com.devonfw.tools.ide.url.model.UrlErrorReport;
+import com.devonfw.tools.ide.url.model.UrlErrorState;
 import com.devonfw.tools.ide.url.model.file.UrlChecksum;
 import com.devonfw.tools.ide.url.model.file.UrlDownloadFile;
 import com.devonfw.tools.ide.url.model.file.UrlFile;
@@ -70,6 +72,9 @@ public abstract class AbstractUrlUpdater extends AbstractProcessorWithTimeout im
 
   private static final Logger logger = LoggerFactory.getLogger(AbstractUrlUpdater.class);
 
+  protected UrlErrorState urlErrorState = UrlErrorReport.getErrorState(getToolWithEdition());
+
+
   /**
    * @return the name of the {@link UrlTool tool} handled by this updater.
    */
@@ -440,6 +445,9 @@ private void doUpdateStatusJson(boolean success, int statusCode, UrlVersion urlV
 
       logger.info("For tool {} and version {} the download verification succeeded with status code {} for URL {}.", tool,
           version, code, url);
+
+      urlErrorState.updateVerifications(true);
+
     } else {
       if (status != null) {
         if (errorStatus == null) {
@@ -465,6 +473,8 @@ private void doUpdateStatusJson(boolean success, int statusCode, UrlVersion urlV
       }
       logger.warn("For tool {} and version {} the download verification failed with status code {} for URL {}.", tool,
           version, code, url);
+
+      urlErrorState.updateVerifications(false);
     }
     if (modified) {
       urlStatusFile.setStatusJson(statusJson); // hack to set modified (better solution welcome)
@@ -517,7 +527,7 @@ public void update(UrlRepository urlRepository) {
 
     UrlTool tool = urlRepository.getOrCreateChild(getTool());
     UrlEdition edition = tool.getOrCreateChild(getEdition());
-    updateExistingVersions(edition);
+    //updateExistingVersions(edition);
     Set<String> versions = getVersions();
     String toolWithEdition = getToolWithEdition();
     logger.info("For tool {} we found the following versions : {}", toolWithEdition, versions);
@@ -534,8 +544,10 @@ public void update(UrlRepository urlRepository) {
           urlVersion = edition.getOrCreateChild(version);
           addVersion(urlVersion);
           urlVersion.save();
+          urlErrorState.updateAdditions(true);
         } catch (Exception e) {
           logger.error("For tool {} we failed to add version {}.", toolWithEdition, version, e);
+          urlErrorState.updateAdditions(false);
         }
       }
     }