Skip to content

Commit

Permalink
MODHAADM-71 wip
Browse files Browse the repository at this point in the history
  Reorganize Java classes (rename packages, classes, move classes)
  Update logging dependencies
  Add Okapi client for configuration look-up
  Preparations for unit tests (settable clock, APIs for populating jobs and logs with sample data)
  • Loading branch information
nielserik committed Sep 11, 2024
1 parent c24714d commit 15db5e0
Show file tree
Hide file tree
Showing 36 changed files with 429 additions and 179 deletions.
1 change: 1 addition & 0 deletions descriptors/ModuleDescriptor-template.json
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,7 @@
],
"pathPattern": "/harvester-admin/purge-aged-logs",
"modulePermissions": [
"configuration.entries.collection.get"
],
"unit": "minute",
"delay": "2"
Expand Down
23 changes: 9 additions & 14 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers-bom</artifactId>
<version>1.19.6</version>
<version>1.20.1</version>
<type>pom</type>
<scope>import</scope>
</dependency>
Expand Down Expand Up @@ -143,11 +143,17 @@
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<artifactId>log4j-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<artifactId>log4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-jdk14</artifactId>
<version>2.1.0-alpha1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.folio.okapi</groupId>
Expand Down Expand Up @@ -179,17 +185,6 @@
<artifactId>rest-assured</artifactId>
<scope>test</scope>
</dependency>
<!-- logging see http://www.slf4j.org/legacy.html -->
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-jul</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.folio.okapi</groupId>
<artifactId>okapi-testing</artifactId>
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/folio/harvesteradmin/MainVerticle.java
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import io.vertx.core.AbstractVerticle;
import io.vertx.core.Promise;
import io.vertx.core.http.HttpServerOptions;
import org.folio.harvesteradmin.dataaccess.statics.LegacyServiceConfig;
import org.folio.harvesteradmin.legacydata.statics.LegacyServiceConfig;
import org.folio.harvesteradmin.service.HarvestAdminService;
import org.folio.okapi.common.Config;
import org.folio.tlib.RouterCreator;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package org.folio.harvesteradmin.foliodata;

import io.vertx.core.Future;
import io.vertx.core.json.JsonArray;
import io.vertx.core.json.JsonObject;
import io.vertx.ext.web.RoutingContext;
import io.vertx.reactivex.core.Promise;

import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;

public class ConfigurationsClient {
private static final String CONFIGURATIONS_PATH = "/configurations/entries";
private static final String RECORDS = "configs";
public static final String MODULE_HARVESTER_ADMIN = "HARVESTER_ADMIN";
public static final String CONFIG_NAME_PURGE_LOGS_AFTER = "PURGE_LOGS_AFTER";
public static Future<JsonArray> getEntries(RoutingContext routingContext, String moduleName, String configName) {

String query = "module==" + moduleName + " and configName==" + configName + " and enabled==true";
return Folio.okapiClient(routingContext).get(CONFIGURATIONS_PATH +
"?query=(" + URLEncoder.encode(query, StandardCharsets.UTF_8) +")")
.map(result -> new JsonObject(result).getJsonArray(RECORDS))
.recover(e -> Future.failedFuture(e.getMessage()));
}

public static Future<String> getStringValue (RoutingContext routingContext, String moduleName, String configName) {
String query = "module=" + moduleName + " and configName=" + configName + " and enabled=true";
Promise<String> promise = Promise.promise();
Folio.okapiClient(routingContext).get(CONFIGURATIONS_PATH +
"?query=(" + URLEncoder.encode(query, StandardCharsets.UTF_8) +")")
.onComplete(response -> {
JsonObject json = new JsonObject(response.result());
JsonArray entries = json.getJsonArray(RECORDS);
if (entries.isEmpty()) {
promise.complete(null);

} else {
JsonObject entry = entries.getJsonObject(0);
promise.complete(entry.getString("value"));
}
});
return promise.future();
}

}
22 changes: 22 additions & 0 deletions src/main/java/org/folio/harvesteradmin/foliodata/Folio.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
package org.folio.harvesteradmin.foliodata;

import io.vertx.ext.web.RoutingContext;
import org.folio.okapi.common.OkapiClient;
import org.folio.okapi.common.WebClientFactory;

import java.util.HashMap;
import java.util.Map;

public class Folio {
public static OkapiClient okapiClient(RoutingContext ctx) {
OkapiClient client = new OkapiClient(WebClientFactory.getWebClient(ctx.vertx()), ctx);
Map<String, String> headers = new HashMap<>();
headers.put("Content-type", "application/json");
if (ctx.request().getHeader("X-Okapi-Tenant") != null) headers.put("X-Okapi-Tenant", ctx.request().getHeader("X-Okapi-Tenant"));
if (ctx.request().getHeader("X-Okapi-Token") != null) headers.put("X-Okapi-Token", ctx.request().getHeader("X-Okapi-Token"));
headers.put("Accept", "application/json, text/plain");
client.setHeaders(headers);
return client;
}

}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package org.folio.harvesteradmin.dataaccess;
package org.folio.harvesteradmin.legacydata;

import static org.folio.harvesteradmin.dataaccess.statics.ApiPaths.HARVESTER_HARVESTABLES_PATH;
import static org.folio.harvesteradmin.legacydata.statics.ApiPaths.HARVESTER_HARVESTABLES_PATH;
import static org.folio.okapi.common.HttpResponse.responseJson;
import static org.folio.okapi.common.HttpResponse.responseText;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,17 +1,17 @@
package org.folio.harvesteradmin.dataaccess;
package org.folio.harvesteradmin.legacydata;

import static org.folio.harvesteradmin.dataaccess.statics.ApiPaths.HARVESTER_HARVESTABLES_PATH;
import static org.folio.harvesteradmin.dataaccess.statics.ApiPaths.HARVESTER_STEPS_PATH;
import static org.folio.harvesteradmin.dataaccess.statics.ApiPaths.HARVESTER_STORAGES_PATH;
import static org.folio.harvesteradmin.dataaccess.statics.ApiPaths.HARVESTER_TRANSFORMATIONS_PATH;
import static org.folio.harvesteradmin.dataaccess.statics.ApiPaths.HARVESTER_TSAS_PATH;
import static org.folio.harvesteradmin.dataaccess.statics.ApiPaths.harvesterPathByRequestPath;
import static org.folio.harvesteradmin.dataaccess.statics.EntityRootNames.mapToNameOfArrayOfEntities;
import static org.folio.harvesteradmin.dataaccess.statics.EntityRootNames.mapToNameOfRootOfEntity;
import static org.folio.harvesteradmin.dataaccess.statics.EntityRootNames.typeToEmbeddedTypeMap;
import static org.folio.harvesteradmin.dataaccess.statics.RequestParameters.crosswalkCqlFieldNames;
import static org.folio.harvesteradmin.dataaccess.statics.RequestParameters.crosswalkRequestParameterNames;
import static org.folio.harvesteradmin.dataaccess.statics.RequestParameters.supportedGetRequestParameters;
import static org.folio.harvesteradmin.legacydata.statics.ApiPaths.HARVESTER_HARVESTABLES_PATH;
import static org.folio.harvesteradmin.legacydata.statics.ApiPaths.HARVESTER_STEPS_PATH;
import static org.folio.harvesteradmin.legacydata.statics.ApiPaths.HARVESTER_STORAGES_PATH;
import static org.folio.harvesteradmin.legacydata.statics.ApiPaths.HARVESTER_TRANSFORMATIONS_PATH;
import static org.folio.harvesteradmin.legacydata.statics.ApiPaths.HARVESTER_TSAS_PATH;
import static org.folio.harvesteradmin.legacydata.statics.ApiPaths.harvesterPathByRequestPath;
import static org.folio.harvesteradmin.legacydata.statics.EntityRootNames.mapToNameOfArrayOfEntities;
import static org.folio.harvesteradmin.legacydata.statics.EntityRootNames.mapToNameOfRootOfEntity;
import static org.folio.harvesteradmin.legacydata.statics.EntityRootNames.typeToEmbeddedTypeMap;
import static org.folio.harvesteradmin.legacydata.statics.RequestParameters.crosswalkCqlFieldNames;
import static org.folio.harvesteradmin.legacydata.statics.RequestParameters.crosswalkRequestParameterNames;
import static org.folio.harvesteradmin.legacydata.statics.RequestParameters.supportedGetRequestParameters;
import static org.folio.okapi.common.HttpResponse.responseText;

import io.vertx.core.AsyncResult;
Expand All @@ -33,7 +33,6 @@
import java.io.StringReader;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.HashMap;
Expand All @@ -47,17 +46,18 @@
import javax.xml.transform.TransformerException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.folio.harvesteradmin.dataaccess.dataconverters.JsonToHarvesterXml;
import org.folio.harvesteradmin.dataaccess.responsehandlers.ProcessedHarvesterResponse;
import org.folio.harvesteradmin.dataaccess.responsehandlers.ProcessedHarvesterResponseDelete;
import org.folio.harvesteradmin.dataaccess.responsehandlers.ProcessedHarvesterResponseGet;
import org.folio.harvesteradmin.dataaccess.responsehandlers.ProcessedHarvesterResponseGetById;
import org.folio.harvesteradmin.dataaccess.responsehandlers.ProcessedHarvesterResponseGetUniqueByName;
import org.folio.harvesteradmin.dataaccess.responsehandlers.ProcessedHarvesterResponsePost;
import org.folio.harvesteradmin.dataaccess.responsehandlers.ProcessedHarvesterResponsePut;
import org.folio.harvesteradmin.dataaccess.statics.ApiPaths;
import org.folio.harvesteradmin.dataaccess.statics.EntityRootNames;
import org.folio.harvesteradmin.dataaccess.statics.LegacyServiceConfig;
import org.folio.harvesteradmin.legacydata.dataconverters.JsonToHarvesterXml;
import org.folio.harvesteradmin.legacydata.responsehandlers.ProcessedHarvesterResponse;
import org.folio.harvesteradmin.legacydata.responsehandlers.ProcessedHarvesterResponseDelete;
import org.folio.harvesteradmin.legacydata.responsehandlers.ProcessedHarvesterResponseGet;
import org.folio.harvesteradmin.legacydata.responsehandlers.ProcessedHarvesterResponseGetById;
import org.folio.harvesteradmin.legacydata.responsehandlers.ProcessedHarvesterResponseGetUniqueByName;
import org.folio.harvesteradmin.legacydata.responsehandlers.ProcessedHarvesterResponsePost;
import org.folio.harvesteradmin.legacydata.responsehandlers.ProcessedHarvesterResponsePut;
import org.folio.harvesteradmin.legacydata.statics.ApiPaths;
import org.folio.harvesteradmin.legacydata.statics.EntityRootNames;
import org.folio.harvesteradmin.legacydata.statics.LegacyServiceConfig;
import org.folio.harvesteradmin.utils.SettableClock;
import org.folio.okapi.common.GenericCompositeFuture;
import org.xml.sax.SAXException;

Expand Down Expand Up @@ -338,7 +338,7 @@ public Future<ProcessedHarvesterResponsePost> doPostConfigRecord(RoutingContext
} else {
JsonObject jsonToPost = routingContext.body().asJsonObject();
if (harvesterPath.equals(HARVESTER_HARVESTABLES_PATH)) {
jsonToPost.put("lastUpdated", iso_instant.format(Instant.now()));
jsonToPost.put("lastUpdated", iso_instant.format(SettableClock.getInstant()));
JsonObject transformationReference = jsonToPost.getJsonObject("transformation");
if (!transformationReference.containsKey("entityType")) {
transformationReference.put("entityType", "basicTransformation");
Expand Down Expand Up @@ -414,7 +414,7 @@ public Future<ProcessedHarvesterResponsePut> putConfigRecord(RoutingContext rout
if (harvesterPath.equals(HARVESTER_TRANSFORMATIONS_PATH)) {
return putTransformation(routingContext);
} else if (harvesterPath.equals(HARVESTER_HARVESTABLES_PATH)) {
jsonToPut.put("lastUpdated", iso_instant.format(Instant.now()));
jsonToPut.put("lastUpdated", iso_instant.format(SettableClock.getInstant()));
}
}
return putConfigRecord(routingContext, harvesterPath, jsonToPut, id);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.folio.harvesteradmin.dataaccess.dataconverters;
package org.folio.harvesteradmin.legacydata.dataconverters;

import io.vertx.core.json.DecodeException;
import io.vertx.core.json.JsonArray;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
package org.folio.harvesteradmin.dataaccess.dataconverters;
package org.folio.harvesteradmin.legacydata.dataconverters;

import static org.folio.harvesteradmin.dataaccess.dataconverters.JsonToHarvesterXml.writeXmlNodeToString;
import static org.folio.harvesteradmin.legacydata.dataconverters.JsonToHarvesterXml.writeXmlNodeToString;

import io.vertx.core.json.DecodeException;
import io.vertx.core.json.JsonArray;
import io.vertx.core.json.JsonObject;
import java.util.Map;
import javax.xml.transform.TransformerException;
import org.w3c.dom.Document;

import org.w3c.dom.Element;
import org.w3c.dom.Node;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.folio.harvesteradmin.dataaccess.dataconverters;
package org.folio.harvesteradmin.legacydata.dataconverters;

import io.vertx.core.json.JsonArray;
import io.vertx.core.json.JsonObject;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.folio.harvesteradmin.dataaccess.responsehandlers;
package org.folio.harvesteradmin.legacydata.responsehandlers;

import io.vertx.core.json.JsonObject;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package org.folio.harvesteradmin.dataaccess.responsehandlers;
package org.folio.harvesteradmin.legacydata.responsehandlers;

import static org.folio.harvesteradmin.dataaccess.LegacyHarvesterStorage.NO_CONTENT;
import static org.folio.harvesteradmin.legacydata.LegacyHarvesterStorage.NO_CONTENT;

import io.vertx.core.AsyncResult;
import io.vertx.core.buffer.Buffer;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
package org.folio.harvesteradmin.dataaccess.responsehandlers;
package org.folio.harvesteradmin.legacydata.responsehandlers;

import io.vertx.core.AsyncResult;
import io.vertx.core.buffer.Buffer;
import io.vertx.core.json.JsonObject;
import io.vertx.ext.web.client.HttpResponse;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.folio.harvesteradmin.dataaccess.dataconverters.HarvesterXml2Json;
import org.folio.harvesteradmin.legacydata.dataconverters.HarvesterXml2Json;

public class ProcessedHarvesterResponseGet extends ProcessedHarvesterResponse {
private static final Pattern exceptionDescriptionPattern =
Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
package org.folio.harvesteradmin.dataaccess.responsehandlers;
package org.folio.harvesteradmin.legacydata.responsehandlers;

import io.vertx.core.AsyncResult;
import io.vertx.core.buffer.Buffer;
import io.vertx.core.json.JsonObject;
import io.vertx.ext.web.client.HttpResponse;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.folio.harvesteradmin.dataaccess.dataconverters.HarvesterXml2Json;
import org.folio.harvesteradmin.dataaccess.statics.LegacyServiceConfig;
import org.folio.harvesteradmin.legacydata.dataconverters.HarvesterXml2Json;
import org.folio.harvesteradmin.legacydata.statics.LegacyServiceConfig;


public class ProcessedHarvesterResponseGetById extends ProcessedHarvesterResponse {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.folio.harvesteradmin.dataaccess.responsehandlers;
package org.folio.harvesteradmin.legacydata.responsehandlers;

import io.vertx.core.json.JsonObject;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.folio.harvesteradmin.dataaccess.responsehandlers;
package org.folio.harvesteradmin.legacydata.responsehandlers;

import io.vertx.core.AsyncResult;
import io.vertx.core.buffer.Buffer;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.folio.harvesteradmin.dataaccess.responsehandlers;
package org.folio.harvesteradmin.legacydata.responsehandlers;

import io.vertx.core.AsyncResult;
import io.vertx.core.buffer.Buffer;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.folio.harvesteradmin.dataaccess.statics;
package org.folio.harvesteradmin.legacydata.statics;

import java.util.HashMap;
import java.util.Map;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
package org.folio.harvesteradmin.dataaccess.statics;
package org.folio.harvesteradmin.legacydata.statics;

import static org.folio.harvesteradmin.dataaccess.statics.ApiPaths.HARVESTER_HARVESTABLES_PATH;
import static org.folio.harvesteradmin.dataaccess.statics.ApiPaths.HARVESTER_STEPS_PATH;
import static org.folio.harvesteradmin.dataaccess.statics.ApiPaths.HARVESTER_STORAGES_PATH;
import static org.folio.harvesteradmin.dataaccess.statics.ApiPaths.HARVESTER_TRANSFORMATIONS_PATH;
import static org.folio.harvesteradmin.dataaccess.statics.ApiPaths.HARVESTER_TSAS_PATH;
import static org.folio.harvesteradmin.legacydata.statics.ApiPaths.HARVESTER_HARVESTABLES_PATH;
import static org.folio.harvesteradmin.legacydata.statics.ApiPaths.HARVESTER_STEPS_PATH;
import static org.folio.harvesteradmin.legacydata.statics.ApiPaths.HARVESTER_STORAGES_PATH;
import static org.folio.harvesteradmin.legacydata.statics.ApiPaths.HARVESTER_TRANSFORMATIONS_PATH;
import static org.folio.harvesteradmin.legacydata.statics.ApiPaths.HARVESTER_TSAS_PATH;

import java.util.HashMap;
import java.util.Map;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.folio.harvesteradmin.dataaccess.statics;
package org.folio.harvesteradmin.legacydata.statics;

import java.lang.management.ManagementFactory;
import org.apache.logging.log4j.Level;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.folio.harvesteradmin.dataaccess.statics;
package org.folio.harvesteradmin.legacydata.statics;

import java.util.HashMap;
import java.util.HashSet;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@
import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.folio.harvesteradmin.modulestorage.Storage;

import org.folio.harvesteradmin.moduledata.database.Tables;
import org.folio.tlib.postgres.PgCqlDefinition;
import org.folio.tlib.postgres.cqlfield.PgCqlFieldAlwaysMatches;

Expand Down Expand Up @@ -98,7 +99,7 @@ public String makeCreateTableSql(String schema) {
.forEach(field -> columnsDdl.append(field.pgColumn().getColumnDdl()).append(","));
columnsDdl.deleteCharAt(columnsDdl.length() - 1); // remove ending comma

return "CREATE TABLE IF NOT EXISTS " + schema + "." + Storage.Table.harvest_job
return "CREATE TABLE IF NOT EXISTS " + schema + "." + Tables.harvest_job
+ "("
+ columnsDdl
+ ")";
Expand All @@ -108,7 +109,7 @@ public String makeCreateTableSql(String schema) {
* INSERT INTO statement.
*/
public String makeInsertTemplate(String schema) {
return "INSERT INTO " + schema + "." + Storage.Table.harvest_job
return "INSERT INTO " + schema + "." + Tables.harvest_job
+ " ("
+ HarvestJobField.ID + ", "
+ HarvestJobField.HARVESTABLE_ID.columnName() + ", "
Expand Down
Loading

0 comments on commit 15db5e0

Please sign in to comment.