diff --git a/pom.xml b/pom.xml index 480d401b..62a3bc62 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-shared - 1.8.0-SNAPSHOT + 1.9.0 ${project.groupId}:${project.artifactId} Library for all the shared uid2 operations @@ -153,6 +153,11 @@ jackson-databind 2.10.2 + + software.amazon.qldb + amazon-qldb-driver-java + 2.3.1 + diff --git a/src/main/java/com/uid2/shared/audit/Actions.java b/src/main/java/com/uid2/shared/audit/Actions.java new file mode 100644 index 00000000..c99a603e --- /dev/null +++ b/src/main/java/com/uid2/shared/audit/Actions.java @@ -0,0 +1,12 @@ +package com.uid2.shared.audit; + +public enum Actions { + LIST, + GET, + UPDATE, + CREATE, + DELETE, + DISABLE, + ENABLE, + REVEAL +} diff --git a/src/main/java/com/uid2/shared/audit/AuditFactory.java b/src/main/java/com/uid2/shared/audit/AuditFactory.java new file mode 100644 index 00000000..673e0026 --- /dev/null +++ b/src/main/java/com/uid2/shared/audit/AuditFactory.java @@ -0,0 +1,31 @@ +package com.uid2.shared.audit; + +import io.vertx.core.json.JsonObject; + +import java.util.HashMap; +import java.util.Map; + +/** + * AuditFactory controls the instantiation/creation of AuditMiddleware objects. + * Depending on the needs of the specific implementation, the AuditFactory + * can be implemented to always return the same AuditMiddleware object, create a new + * AuditMiddleware object for every class that calls getAuditMiddleware(Class), or + * exhibit some other behavior. + */ +public class AuditFactory { + private static final Map middlewareMap = new HashMap<>(); + + /** + * Returns an AuditMiddleware object with the designated configuration. If one does + * not already exist, creates a new AuditMiddleware object using the configuration. + * + * @return the designated AuditMiddleware object for the passed class. + */ + public static IAuditMiddleware getAuditMiddleware(JsonObject config){ + if(!middlewareMap.containsKey(config)){ + middlewareMap.put(config, new AuditMiddlewareImpl(new QLDBAuditWriter(config))); + } + return middlewareMap.get(config); + } + +} diff --git a/src/main/java/com/uid2/shared/audit/AuditMiddlewareImpl.java b/src/main/java/com/uid2/shared/audit/AuditMiddlewareImpl.java new file mode 100644 index 00000000..cc6bb129 --- /dev/null +++ b/src/main/java/com/uid2/shared/audit/AuditMiddlewareImpl.java @@ -0,0 +1,71 @@ +package com.uid2.shared.audit; + +import com.uid2.shared.auth.IAuthorizable; +import io.vertx.ext.web.RoutingContext; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; + +public class AuditMiddlewareImpl implements IAuditMiddleware{ + private final IAuditWriter auditWriter; + + public AuditMiddlewareImpl(IAuditWriter writer){ + this.auditWriter = writer; + } + + @Override + public Function, Boolean> handle(RoutingContext rc) { + InnerAuditHandler auditHandler = new InnerAuditHandler(rc, auditWriter); + return auditHandler::writeLogs; + } + + private static class InnerAuditHandler{ + private final RoutingContext rc; + private final IAuditWriter auditWriter; + private InnerAuditHandler(RoutingContext rc, IAuditWriter auditWriter) { + this.rc = rc; + this.auditWriter = auditWriter; + } + + public boolean writeLogs(List modelList){ + String ipAddress = getIPAddress(rc); + List auditModelList = new ArrayList<>(); + for(OperationModel model : modelList) { + auditModelList.add(new QLDBAuditModel(model.itemType, model.itemKey, model.actionTaken, ipAddress, + rc != null ? ((IAuthorizable) rc.data().get("api-client")).getContact() : null, + System.getenv("HOSTNAME"), Instant.now().getEpochSecond(), model.itemHash, model.summary)); + } + return auditWriter.writeLogs(auditModelList); + } + + private static String getIPAddress(RoutingContext rc) { + if(rc == null){ + return null; + } + List listIP = rc.request().headers().getAll("X-Forwarded-For"); + List publicIPs = new ArrayList<>(); + for(String str : listIP){ + try { + InetAddress address = InetAddress.getByName(str); + if(!address.isSiteLocalAddress()){ + publicIPs.add(address); + } + } + catch(UnknownHostException ignored){ + + } + + } + if(publicIPs.isEmpty()){ + return rc.request().remoteAddress().toString(); + } + else{ + return publicIPs.get(0).getHostAddress(); //arbitrary if multiple + } + } + } +} diff --git a/src/main/java/com/uid2/shared/audit/IAuditInit.java b/src/main/java/com/uid2/shared/audit/IAuditInit.java new file mode 100644 index 00000000..c5ca4489 --- /dev/null +++ b/src/main/java/com/uid2/shared/audit/IAuditInit.java @@ -0,0 +1,16 @@ +package com.uid2.shared.audit; + +import java.util.Collection; + +/** + * Responsible for the initialization of the QLDB table and any initial entries it must contain. + */ +public interface IAuditInit { + + /** + * Creates a table with the name specified in the config, with all entries as specified by modelList, and sets up + * any necessary configuration. + * @param modelList the models to add to the audit database. + */ + void init(Collection modelList); +} diff --git a/src/main/java/com/uid2/shared/audit/IAuditMiddleware.java b/src/main/java/com/uid2/shared/audit/IAuditMiddleware.java new file mode 100644 index 00000000..1a92500f --- /dev/null +++ b/src/main/java/com/uid2/shared/audit/IAuditMiddleware.java @@ -0,0 +1,22 @@ +package com.uid2.shared.audit; + +import io.vertx.ext.web.RoutingContext; + +import java.util.List; +import java.util.function.Function; + +/** + * AuditMiddleware objects are intended to be attached to any endpoint that the system + * wants to keep track of via logging to an external source, and pass logging data to an AuditWriter object. + */ +public interface IAuditMiddleware { + + /** + * Handle to attach to any route whose actions require logging. + * + * @param rc the RoutingContext of the endpoint access that initiated the request. + * @return a function that takes a List of OperationModels and returns whether the audit + * writing was successful. + */ + Function, Boolean> handle(RoutingContext rc); +} diff --git a/src/main/java/com/uid2/shared/audit/IAuditModel.java b/src/main/java/com/uid2/shared/audit/IAuditModel.java new file mode 100644 index 00000000..6f3fe583 --- /dev/null +++ b/src/main/java/com/uid2/shared/audit/IAuditModel.java @@ -0,0 +1,34 @@ +package com.uid2.shared.audit; + +import io.vertx.core.json.JsonObject; + +/** + * An AuditModel contains fields that collectively logs all necessary details of an action + * that reads or writes sensitive information in the uid2-admin server. AuditModel objects should + * be unmodifiable and answer the following questions: + * + * • what happened? + * • when did it happen? + * • who initiated it? + * • on what did it happen? + * • where was it observed? + * • from where was it initiated? + * • to where was it going? + */ +public interface IAuditModel { + + /** + * Converts the AuditModel to JSON format to be used in document-store databases. + * Every field should be a key in the resulting JSON object. + * + * @return a JSON representation of this AuditModel. + */ + JsonObject writeToJson(); + + /** + * Converts the AuditModel into a readable String format to be used in text logs. + * + * @return a String representation of this AuditModel. + */ + String writeToString(); +} diff --git a/src/main/java/com/uid2/shared/audit/IAuditWriter.java b/src/main/java/com/uid2/shared/audit/IAuditWriter.java new file mode 100644 index 00000000..d3ba646e --- /dev/null +++ b/src/main/java/com/uid2/shared/audit/IAuditWriter.java @@ -0,0 +1,16 @@ +package com.uid2.shared.audit; + +import java.util.Collection; + +/** + * AuditWriter is responsible for the logic to write out to designated logging databases. + */ +public interface IAuditWriter { + /** + * Logs the information in the AuditModel to an external database(s). + * Does not log any information if model == null. + * + * @param model the AuditModel to write out. + */ + boolean writeLogs(Collection model); +} diff --git a/src/main/java/com/uid2/shared/audit/OperationModel.java b/src/main/java/com/uid2/shared/audit/OperationModel.java new file mode 100644 index 00000000..66bb8601 --- /dev/null +++ b/src/main/java/com/uid2/shared/audit/OperationModel.java @@ -0,0 +1,25 @@ +package com.uid2.shared.audit; + +/** + * Store the type of data, action that occurred, and any extra information necessary to know about the operation. + * Also stores the itemKey representing the operation. itemKey should be designed such that all read/writes + * affecting the same row(s) in the same table share the same value. It should be hashed in case the row identifier + * itself is sensitive information. + */ +public class OperationModel { + + public final Type itemType; + public final String itemKey; + public final Actions actionTaken; + public final String itemHash; + public final String summary; + + public OperationModel(Type itemType, String itemKey, Actions actionTaken, + String itemHash, String summary){ + this.itemType = itemType; + this.itemKey = itemKey; + this.actionTaken = actionTaken; + this.itemHash = itemHash; + this.summary = summary; + } +} diff --git a/src/main/java/com/uid2/shared/audit/QLDBAuditModel.java b/src/main/java/com/uid2/shared/audit/QLDBAuditModel.java new file mode 100644 index 00000000..9d962946 --- /dev/null +++ b/src/main/java/com/uid2/shared/audit/QLDBAuditModel.java @@ -0,0 +1,88 @@ +package com.uid2.shared.audit; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.vertx.core.json.JsonObject; + +import java.io.IOException; + +public class QLDBAuditModel implements IAuditModel{ + + /** + * The table that the user accesses. + */ + @JsonIgnore + public final Type itemType; + /** + * An identifier for the row in the table that is accessed. Is null if more than one row is accessed at the same + * time, e.g. listing all values in a table. If itemKey should be secret, hash before putting it into the model. + */ + @JsonIgnore + public final String itemKey; + /** + * Describes the action the user performed on the table (e.g. read ("GET"), write ("CREATE"/"DELETE")...) + */ + public final Actions actionTaken; + /** + * The IP of the user making the HTTP request. + */ + public final String clientIP; + /** + * The email of the user making the HTTP request. + */ + public final String userEmail; + /** + * The server that processed the HTTP request. + */ + public final String hostNode; + /** + * The time that the HTTP request was received by the server. + */ + public final long timeEpochSecond; + /** + * The hash of the entire item being accessed/modified by the user. Is null if more than one + * row is accessed at the same time (which should only be get/list queries; otherwise make multiple + * queries). + */ + public final String itemHash; + /** + * Names the exact operation done to the item (e.g. rekeyed, revealed, disabled, etc.) + */ + public final String summary; + + public QLDBAuditModel(Type itemType, String itemKey, Actions actionTaken, String clientIP, + String userEmail, String hostNode, long timeEpochSecond, String itemHash, String summary){ + this.itemType = itemType; + this.itemKey = itemKey; + this.actionTaken = actionTaken; + this.clientIP = clientIP; + this.userEmail = userEmail; + this.hostNode = hostNode; + this.timeEpochSecond = timeEpochSecond; + this.itemHash = itemHash; + this.summary = summary; + } + + @Override + public JsonObject writeToJson() { + ObjectMapper mapper = new ObjectMapper(); + JsonObject jo; + try { + jo = new JsonObject(mapper.writeValueAsString(this)); + } + catch(IOException e){ + e.printStackTrace(); + jo = new JsonObject(); + } + JsonObject outerJo = new JsonObject(); + outerJo.put("itemType", itemType); + outerJo.put("itemKey", itemKey); + outerJo.put("data", jo); + return outerJo; + } + + @Override + public String writeToString() { + return writeToJson().toString(); + } +} diff --git a/src/main/java/com/uid2/shared/audit/QLDBAuditWriter.java b/src/main/java/com/uid2/shared/audit/QLDBAuditWriter.java new file mode 100644 index 00000000..1274e10e --- /dev/null +++ b/src/main/java/com/uid2/shared/audit/QLDBAuditWriter.java @@ -0,0 +1,89 @@ +package com.uid2.shared.audit; + +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonValue; +import com.amazon.ion.system.IonSystemBuilder; +import io.vertx.core.json.JsonObject; +import io.vertx.core.logging.Logger; +import io.vertx.core.logging.LoggerFactory; +import software.amazon.awssdk.services.qldbsession.QldbSessionClient; +import software.amazon.qldb.QldbDriver; +import software.amazon.qldb.Result; +import software.amazon.qldb.RetryPolicy; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +public class QLDBAuditWriter implements IAuditWriter{ + private static final IonSystem ionSys = IonSystemBuilder.standard().build(); + private static final Logger logger = LoggerFactory.getLogger(QLDBAuditWriter.class); + private static final Logger auditLogger = LoggerFactory.getLogger("com.uid2.admin.audit"); + private QldbDriver qldbDriver; + private final String logTable; + private final boolean qldbLogging; + public QLDBAuditWriter(JsonObject config){ + try { + qldbDriver = QldbDriver.builder() + .ledger(config.getString("qldb_ledger_name")) + .transactionRetryPolicy(RetryPolicy.builder().maxRetries(3).build()) + .sessionClientBuilder(QldbSessionClient.builder()) + .build(); + } + catch (Exception e){ + logger.error("cannot establish connection with QLDB"); + } + logTable = config.getString("qldb_table_name"); + qldbLogging = config.getBoolean("enable_qldb_admin_logging"); + } + @Override + public boolean writeLogs(Collection models) { + AtomicBoolean successfulLog = new AtomicBoolean(true); + if (qldbLogging) { + try { + qldbDriver.execute(txn -> { + for (IAuditModel model : models) { + if (!(model instanceof QLDBAuditModel)) { //should never be true, but check in case + successfulLog.set(false); + logger.error("Only QLDBAuditModel should be passed into QLDBAuditWriter"); + txn.abort(); + break; + } + QLDBAuditModel qldbModel = (QLDBAuditModel) model; + JsonObject jsonObject = qldbModel.writeToJson(); + String query; + List sanitizedInputs = new ArrayList<>(); + if (qldbModel.actionTaken == Actions.CREATE) { + query = "INSERT INTO " + logTable + " VALUE ?"; + sanitizedInputs.add(ionSys.newLoader().load(jsonObject.toString()).get(0)); + } else { + query = "UPDATE " + logTable + " AS t SET data = ? WHERE t.itemType = ? AND t.itemKey = ?"; + sanitizedInputs.add(ionSys.newLoader().load(jsonObject.getJsonObject("data").toString()).get(0)); + sanitizedInputs.add(ionSys.newString(qldbModel.itemType.toString())); + sanitizedInputs.add(ionSys.newString(qldbModel.itemKey)); + } + Result r = txn.execute(query, sanitizedInputs); + if (!r.iterator().hasNext()) { + logger.error("Malformed audit log input: no log written to QLDB"); + successfulLog.set(false); + txn.abort(); + break; + } + } + }); + + } catch (Exception e) { + logger.error("QLDB log failed: " + e.getClass().getSimpleName()); + auditLogger.error("QLDB log failed" + e.getClass().getSimpleName()); + return false; + } + } + if (successfulLog.get()) { + for (IAuditModel model : models) { + auditLogger.info(model.writeToString()); + } + } + return successfulLog.get(); + } +} diff --git a/src/main/java/com/uid2/shared/audit/QLDBInit.java b/src/main/java/com/uid2/shared/audit/QLDBInit.java new file mode 100644 index 00000000..a8e2ee16 --- /dev/null +++ b/src/main/java/com/uid2/shared/audit/QLDBInit.java @@ -0,0 +1,103 @@ +package com.uid2.shared.audit; + +import com.amazon.ion.IonList; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonSystem; +import com.amazon.ion.system.IonSystemBuilder; +import io.vertx.core.logging.Logger; +import io.vertx.core.logging.LoggerFactory; +import software.amazon.qldb.QldbDriver; +import software.amazon.qldb.Result; + +import java.util.Collection; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Handles creating a table, indices, and inserts documents passed to it. Note that all classes that extend + * QLDBInit should initialize qldbDriver and qldbTableName in their constructor. + */ +public abstract class QLDBInit implements IAuditInit{ + + protected final IonSystem ionSys = IonSystemBuilder.standard().build(); + protected QldbDriver qldbDriver; + protected String qldbTableName; + private final Logger LOGGER = LoggerFactory.getLogger(QLDBInit.class); + + @Override + public void init(Collection modelList){ + try { + if (!hasTableBeenCreated()) { + createTable(); + } + if(!haveIndicesBeenCreated()){ + createIndices(); + } + for(OperationModel model : modelList){ + insertIntoQLDB(model); + } + LOGGER.info("initialized qldb"); + } + catch(Exception e){ + LOGGER.warn("qldb not initialized"); + } + } + + protected boolean hasTableBeenCreated() { + try { + AtomicBoolean hasTableBeenCreated = new AtomicBoolean(false); + qldbDriver.execute(txn -> { + Result result = txn.execute("SELECT * FROM information_schema.user_tables WHERE name = ?", + ionSys.newString(qldbTableName)); + hasTableBeenCreated.set(!result.isEmpty()); + }); + return hasTableBeenCreated.get(); + } catch (Exception e) { + throw new RuntimeException("AWS configuration not set up"); + } + } + + protected boolean haveIndicesBeenCreated(){ //Assumes table exists + try { + AtomicBoolean hasTableBeenCreated = new AtomicBoolean(false); + qldbDriver.execute(txn -> { + Result result = txn.execute("SELECT indexes FROM information_schema.user_tables WHERE name = ?", + ionSys.newString(qldbTableName)); + hasTableBeenCreated.set(((IonList)(((IonStruct)(result.iterator().next())).iterator().next())).size() != 0); + }); + return hasTableBeenCreated.get(); + } + catch (Exception e) { + throw new RuntimeException("AWS configuration not set up"); + } + } + + private void createTable() { //creates the logs table. Assumes it doesn't already exist + try { + qldbDriver.execute(txn -> { + txn.execute("CREATE TABLE " + qldbTableName); + }); + } catch (Exception e) { + e.printStackTrace(); + } + } + + private void createIndices() { //creates indices on itemType and itemKey. Assumes they don't already exist. + try { + qldbDriver.execute(txn -> { + txn.execute("CREATE INDEX ON " + qldbTableName + "(itemType)"); + txn.execute("CREATE INDEX ON " + qldbTableName + "(itemKey)"); + }); + } catch (Exception e) { + e.printStackTrace(); + } + } + + private void insertIntoQLDB(OperationModel model) { //populates qldb + QLDBAuditModel auditModel = new QLDBAuditModel(model.itemType, model.itemKey, model.actionTaken, null, + null, null, -1, model.itemHash, model.summary); + qldbDriver.execute(txn -> { + txn.execute("INSERT INTO " + qldbTableName + " VALUE ?", + ionSys.newLoader().load(auditModel.writeToJson().toString()).get(0)); + }); + } +} diff --git a/src/main/java/com/uid2/shared/audit/Type.java b/src/main/java/com/uid2/shared/audit/Type.java new file mode 100644 index 00000000..324dc8c1 --- /dev/null +++ b/src/main/java/com/uid2/shared/audit/Type.java @@ -0,0 +1,17 @@ +package com.uid2.shared.audit; + +/** + * An enum of all table entities that uid2-admin handles. + */ + +public enum Type { + SITE, + CLIENT, + KEYACL, + KEY, + SALT, + OPERATOR, + ENCLAVE, + PARTNER, + ADMIN +}