MODHAADM-71 log pruning

NEWS.MD

@@ -1,3 +1,7 @@
+## 1.3.0 IN PROGRESS
+
+* [MODHAADM-71](https://issues.folio.org/browse/MODHAADM-71) Provides timer process for automatic purge of past job runs and logs.
+
 ## 1.2.1 2024-09-06
 
 * [MODHAADM-94](https://issues.folio.org/browse/MODHAADM-94) Observes Harvester's timezone when fetching logs for latest harvest run.

README.MD

@@ -759,6 +759,32 @@ to push
 the correct, up-to-date [status information in the POST body](src/main/resources/openapi/schemas/harvestJobStatus.json)
 to the history.
 
+#### Automatic clean up of past job runs and job logs
+
+The module has a scheduled process for cleaning up old job runs together with their logs and any failed records saved for the jobs. By default,
+the job is set to run each night at 2 AM in the Central European time zone (CET), and it will then by default remove jobs that 
+are more than three months old.
+
+The timer process can be disabled with
+
+```
+curl -XPATCH -d'{"id":"mod-harvester-admin_0","routingEntry":{"delay":"0"}}' \
+    http://localhost:9130/_/proxy/tenants/<tenant>/timers
+```
+
+The age at which old jobs should be deleted can be changed by posting a configuration like this to `configurations/entries` :
+
+```
+{
+  "module": "HARVESTER_ADMIN",
+  "configName": "PURGE_LOGS_AFTER",
+  "value": "2 MONTHS"
+}
+```
+
+The format for `value` is an integer followed by a time unit that can be any of "DAY[S]", "TAG[E]", "WEEK[S]", 
+"WOCHE[N], "MONTH[S]", or "MONAT[E]". It can be uppercase or lowercase.
+
 #### View current and historic harvest job logs and error reports
 
 If the logs are saved to history, there are thus two sets of APIs for retrieving configurations and logs; one for
@@ -808,6 +834,7 @@ updates happens several times a day in which case the current logs will frequent
 | Content          | [Error report for a single incoming record](src/main/resources/openapi/schemas/failedRecordCurrentJob.json)                                                                                                                                                                                                                        | [Error report for a single incoming record](src/main/resources/openapi/schemas/failedRecordPreviousJob.json)                                                                                                                                                                                                                                                                                                                                                       |
 | Identifier       | `harvestableId` and `recordNumber`                                                                                                                                                                                                                                                                                                 | `id` (uuid)                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
 | Mutating?        | Yes, the error report can disappear with next job run.                                                                                                                                                                                                                                                                             | No. Unless the administrator decides do delete old logs and error reports.                                                                                                                                                                                                                                                                                                                                                                                         |
+
 
 ### Running harvest jobs
 

descriptors/ModuleDescriptor-template.json

@@ -83,6 +83,10 @@
           "pathPattern": "/harvester-admin/previous-jobs",
           "permissionsRequired": ["harvester-admin.previous-jobs.collection.get"]
         }, {
+          "methods": ["POST"],
+          "pathPattern": "/harvester-admin/previous-jobs",
+          "permissionsRequired": ["harvester-admin.previous-jobs.item.post"]
+        },{
           "methods": ["GET"],
           "pathPattern": "/harvester-admin/previous-jobs/failed-records",
           "permissionsRequired": ["harvester-admin.previous-jobs.failed-records.collection.get"]
@@ -102,10 +106,18 @@
           "methods": ["GET"],
           "pathPattern": "/harvester-admin/previous-jobs/{id}/log",
           "permissionsRequired": ["harvester-admin.previous-jobs.log.get"]
+        },{
+          "methods": ["POST"],
+          "pathPattern": "/harvester-admin/previous-jobs/{id}/log",
+          "permissionsRequired": ["harvester-admin.previous-jobs.log.post"]
         },{
           "methods": ["GET"],
           "pathPattern": "/harvester-admin/previous-jobs/{id}/failed-records",
           "permissionsRequired": ["harvester-admin.previous-jobs.failed-records.collection.get"]
+        },{
+          "methods": ["POST"],
+          "pathPattern": "/harvester-admin/previous-jobs/{id}/failed-records",
+          "permissionsRequired": ["harvester-admin.previous-jobs.failed-records.collection.post"]
         },{
           "methods": ["GET"],
           "pathPattern": "/harvester-admin/storages",
@@ -224,6 +236,26 @@
           "permissionsRequired": []
         }
       ]
+    },
+    {
+      "id": "_timer",
+      "version": "1.0",
+      "interfaceType": "system",
+      "handlers": [
+        {
+          "methods": [
+            "POST"
+          ],
+          "pathPattern": "/harvester-admin/purge-aged-logs",
+          "modulePermissions": [
+            "configuration.entries.collection.get"
+          ],
+          "schedule": {
+            "cron": "0 2 * * *",
+            "zone": "CET"
+          }
+        }
+      ]
     }
   ],
   "requires": [],
@@ -288,6 +320,11 @@
       "displayName": "harvester admin - view info about a finished harvest job ",
       "description": "view info about a finished harvest job"
     },
+    {
+      "permissionName": "harvester-admin.previous-jobs.item.post",
+      "displayName": "harvester admin - backdoor for adding job logs",
+      "description": "add previous job information directly to the database independently of a job run, i.e. by import from a different FOLIO instance"
+    },
     {
       "permissionName": "harvester-admin.previous-jobs.item.delete",
       "displayName": "harvester admin - delete a previous job run with all its logs",
@@ -298,11 +335,21 @@
       "displayName": "harvester admin - view past harvest job logs",
       "description": "get log statements for past harvest jobs"
     },
+    {
+      "permissionName": "harvester-admin.previous-jobs.log.post",
+      "displayName": "harvester-admin - backdoor for creating logs for a job",
+      "description": "creating logs for a job without running a job, for example to import logs from another FOLIO instance"
+    },
     {
       "permissionName": "harvester-admin.previous-jobs.failed-records.collection.get",
       "displayName": "harvester admin - view failed records for a past harvest job",
       "description": "get failed records for past harvest jobs"
     },
+    {
+      "permissionName": "harvester-admin.previous-jobs.failed-records.collection.post",
+      "displayName": "harvester admin - backdoor for adding failed record entries",
+      "description": "add failed record entries without running a job, for example to import failure records from another FOLIO instance"
+    },
     {
       "permissionName": "harvester-admin.storages.collection.get",
       "displayName": "harvester admin - get storage collection",
@@ -483,10 +530,13 @@
         "harvester-admin.harvestables.failed-records.item.get",
         "harvester-admin.previous-jobs.collection.get",
         "harvester-admin.previous-jobs.item.get",
+        "harvester-admin.previous-jobs.item.post",
         "harvester-admin.previous-jobs.item.delete",
         "harvester-admin.previous-jobs.log.get",
+        "harvester-admin.previous-jobs.log.post",
         "harvester-admin.previous-jobs.failed-records.collection.get",
-        "harvester-admin.previous-jobs.failed-records.item.get"
+        "harvester-admin.previous-jobs.failed-records.item.get",
+        "harvester-admin.previous-jobs.failed-records.collection.post"
       ]
     }
   ],

pom.xml

@@ -3,14 +3,13 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>org.folio</groupId>
     <artifactId>mod-harvester-admin</artifactId>
-    <version>1.2.2-SNAPSHOT</version>
+    <version>1.3.0-SNAPSHOT</version>
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <okapi.version>5.3.0</okapi.version>
         <vertx.version>4.5.3</vertx.version>
         <maven.compiler.source>18</maven.compiler.source>
         <maven.compiler.target>18</maven.compiler.target>
-        <maven.test.skip>true</maven.test.skip>
     </properties>
     <dependencyManagement>
         <dependencies>
@@ -73,7 +72,7 @@
             <dependency>
                 <groupId>org.testcontainers</groupId>
                 <artifactId>testcontainers-bom</artifactId>
-                <version>1.19.6</version>
+                <version>1.20.1</version>
                 <type>pom</type>
                 <scope>import</scope>
             </dependency>
@@ -143,11 +142,17 @@
         </dependency>
         <dependency>
             <groupId>org.apache.logging.log4j</groupId>
-            <artifactId>log4j-api</artifactId>
+            <artifactId>log4j-core</artifactId>
         </dependency>
         <dependency>
             <groupId>org.apache.logging.log4j</groupId>
-            <artifactId>log4j-core</artifactId>
+            <artifactId>log4j-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-jdk14</artifactId>
+            <version>2.1.0-alpha1</version>
+            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.folio.okapi</groupId>
@@ -179,17 +184,6 @@
             <artifactId>rest-assured</artifactId>
             <scope>test</scope>
         </dependency>
-        <!-- logging see http://www.slf4j.org/legacy.html -->
-        <dependency>
-            <groupId>org.apache.logging.log4j</groupId>
-            <artifactId>log4j-slf4j-impl</artifactId>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.logging.log4j</groupId>
-            <artifactId>log4j-jul</artifactId>
-            <scope>test</scope>
-        </dependency>
         <dependency>
             <groupId>org.folio.okapi</groupId>
             <artifactId>okapi-testing</artifactId>
@@ -256,15 +250,10 @@
                         <harvester_port>8080</harvester_port>
                         <acl_filter_by_tenant>false</acl_filter_by_tenant>
                     </environmentVariables>
-                    <excludes>
-                        <exclude>**/package/**XYZ.class</exclude>
-                    </excludes>
                     <includes>
-                        <include>**/test/HarvesterAdminTestSuite.class</include>
-                        <include>**/UnitTest.class</include>
+                        <include>**/test/NoHarvesterTestSuite.class</include>
                     </includes>
                 </configuration>
-
             </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
@@ -444,6 +433,33 @@
 
         </plugins>
     </build>
+    <profiles>
+        <profile>
+            <id>harvesterTests</id>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-surefire-plugin</artifactId>
+                        <version>2.22.2</version>
+                        <configuration>
+                            <useSystemClassLoader>false</useSystemClassLoader>
+                            <environmentVariables>
+                                <harvester_protocal>http</harvester_protocal>
+                                <harvester_host>localhost</harvester_host>
+                                <harvester_port>8080</harvester_port>
+                                <acl_filter_by_tenant>false</acl_filter_by_tenant>
+                            </environmentVariables>
+                            <includes>
+                                <include>**/test/HarvesterIntegrationTestSuite.class</include>
+                                <include>**/test/NoHarvesterTestSuite.class</include>
+                            </includes>
+                        </configuration>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
     <repositories>
         <repository>
             <id>folio-nexus</id>

src/main/java/org/folio/harvesteradmin/MainVerticle.java

@@ -3,7 +3,7 @@
 import io.vertx.core.AbstractVerticle;
 import io.vertx.core.Promise;
 import io.vertx.core.http.HttpServerOptions;
-import org.folio.harvesteradmin.dataaccess.statics.LegacyServiceConfig;
+import org.folio.harvesteradmin.legacydata.statics.LegacyServiceConfig;
 import org.folio.harvesteradmin.service.HarvestAdminService;
 import org.folio.okapi.common.Config;
 import org.folio.tlib.RouterCreator;

src/main/java/org/folio/harvesteradmin/foliodata/ConfigurationsClient.java

@@ -0,0 +1,37 @@
+package org.folio.harvesteradmin.foliodata;
+
+import io.vertx.core.Future;
+import io.vertx.core.json.JsonArray;
+import io.vertx.core.json.JsonObject;
+import io.vertx.ext.web.RoutingContext;
+import io.vertx.reactivex.core.Promise;
+
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+
+public class ConfigurationsClient {
+    private static final String CONFIGURATIONS_PATH = "/configurations/entries";
+    private static final String RECORDS = "configs";
+    public static final String MODULE_HARVESTER_ADMIN = "HARVESTER_ADMIN";
+    public static final String CONFIG_NAME_PURGE_LOGS_AFTER = "PURGE_LOGS_AFTER";
+
+    public static Future<String> getStringValue (RoutingContext routingContext, String moduleName, String configName) {
+        String query = "module==" + moduleName + " and configName==" + configName + " and enabled=true";
+        Promise<String> promise = Promise.promise();
+        Folio.okapiClient(routingContext).get(CONFIGURATIONS_PATH +
+                        "?query=(" + URLEncoder.encode(query, StandardCharsets.UTF_8) +")")
+                .onComplete(response -> {
+                    JsonObject json = new JsonObject(response.result());
+                    JsonArray entries = json.getJsonArray(RECORDS);
+                    if (entries.isEmpty()) {
+                        promise.complete(null);
+
+                    } else {
+                        JsonObject entry = entries.getJsonObject(0);
+                        promise.complete(entry.getString("value"));
+                    }
+                });
+        return promise.future();
+    }
+
+}

src/main/java/org/folio/harvesteradmin/foliodata/Folio.java

@@ -0,0 +1,22 @@
+package org.folio.harvesteradmin.foliodata;
+
+import io.vertx.ext.web.RoutingContext;
+import org.folio.okapi.common.OkapiClient;
+import org.folio.okapi.common.WebClientFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class Folio {
+    public static OkapiClient okapiClient(RoutingContext ctx) {
+        OkapiClient client = new OkapiClient(WebClientFactory.getWebClient(ctx.vertx()), ctx);
+        Map<String, String> headers = new HashMap<>();
+        headers.put("Content-type", "application/json");
+        if (ctx.request().getHeader("X-Okapi-Tenant") != null) headers.put("X-Okapi-Tenant", ctx.request().getHeader("X-Okapi-Tenant"));
+        if (ctx.request().getHeader("X-Okapi-Token") != null) headers.put("X-Okapi-Token", ctx.request().getHeader("X-Okapi-Token"));
+        headers.put("Accept", "application/json, text/plain");
+        client.setHeaders(headers);
+        return client;
+    }
+
+}

src/main/java/org/folio/harvesteradmin/dataaccess/JobLauncher.java → src/main/java/org/folio/harvesteradmin/legacydata/JobLauncher.java

@@ -1,6 +1,6 @@
-package org.folio.harvesteradmin.dataaccess;
+package org.folio.harvesteradmin.legacydata;
 
-import static org.folio.harvesteradmin.dataaccess.statics.ApiPaths.HARVESTER_HARVESTABLES_PATH;
+import static org.folio.harvesteradmin.legacydata.statics.ApiPaths.HARVESTER_HARVESTABLES_PATH;
 import static org.folio.okapi.common.HttpResponse.responseJson;
 import static org.folio.okapi.common.HttpResponse.responseText;
 
@@ -22,7 +22,7 @@ public class JobLauncher extends LegacyHarvesterStorage {
   private static final int BAD_REQUEST = 400;
   private static final int OK = 200;
 
-  private static SimpleDateFormat dateFormat;
+  private final SimpleDateFormat dateFormat;
 
   /**
    * Constructor.