[IA-2860] Add CRL code to create/delete Azure VMs (#100)

* First pass on Compute cow, integration tests pass * WIP logging framework * Flesh out logging with Context object * Minor * Add Janitor integration * Cleanup * Fix bug * Add Javadoc, refactor class names slightly * Improve comments and cleanup * Disable Azure test * [IA-2860] Azure models/tests for Network / Disk / VM resources CREATE / GET / DELETE (#99) * tests passing for vm creation with resources * refactor model * cleanup Co-authored-by: jdcanas <[email protected]> * Clean up, refactor some packages * Final clean-up, tests pass * Update documentation * Apply code review feedback: 1. Generalize OperationAnnotator to work with GCP and Azure 2. Use Builder pattern for request data objects 3. Remove unnecessary integration tests 4. Various other fixes * Add comment * Add back lost comment * Make directories * Put back .gitignore, CI chokes without it * Final PR feedback * Fix OPerationAnnotatorSpec * Also need to bump the platform version * Really fix OperationAnnotatorTest * OperationAnnotatorTest fix again * Fix azure resource manager integration test Co-authored-by: Justin Canas <[email protected]> Co-authored-by: jdcanas <[email protected]>
DataBiosphere · Sep 30, 2021 · 1ad78a1 · 1ad78a1
1 parent a5f9379
commit 1ad78a1
Show file tree

Hide file tree

Showing 34 changed files with 1,624 additions and 57 deletions.
diff --git a/README.md b/README.md
@@ -146,7 +146,9 @@ Artifact Repository can be found [here](https://broadinstitute.jfrog.io/broadins
 #### google-storage
 Wraps [Google Cloud Storage API](https://cloud.google.com/storage/docs/apis).  
 Artifact Repository can be found [here](https://broadinstitute.jfrog.io/broadinstitute/webapp/#/artifacts/browse/tree/General/libs-snapshot-local/bio/terra/cloud-resource-lib/google-storage).
-
+#### azure-resourcemanager-compute
+Wraps [Azure Compute API](https://docs.microsoft.com/en-us/rest/api/compute/).
+Artifact Repository can be found [here](https://broadinstitute.jfrog.io/broadinstitute/webapp/#/artifacts/browse/tree/General/libs-snapshot-local/bio/terra/cloud-resource-lib/azure-resourcemanager-compute).
 #### cloud-resource-schema
 The general schema for how cloud resources are presented in CRL.   
 Artifact Repository can be found [here](https://broadinstitute.jfrog.io/broadinstitute/webapp/#/artifacts/browse/tree/General/libs-snapshot-local/bio/terra/cloud-resource-lib/cloud-resource-schema).
diff --git a/azure-resourcemanager-common/build.gradle b/azure-resourcemanager-common/build.gradle
@@ -0,0 +1,16 @@
+plugins {
+    // Use common Gradle settings for Java libraries defined in conventions plugin:
+    // /buildSrc/src/main/groovy/terra-cloud-resource-lib.library-conventions.gradle
+    id 'terra-cloud-resource-lib.library-conventions'
+}
+
+dependencies {
+    api project(':common')
+
+    api group: 'com.azure', name: 'azure-identity', version: '1.3.5'
+    implementation group: 'com.azure', name: 'azure-core', version: '1.19.0'
+    implementation group: 'com.azure.resourcemanager', name: 'azure-resourcemanager-resources', version: '2.7.0'
+
+    testFixturesImplementation group: 'com.azure', name: 'azure-core', version: '1.19.0'
+    testFixturesImplementation group: 'com.azure.resourcemanager', name: 'azure-resourcemanager-resources', version: '2.7.0'
+}
diff --git a/azure-resourcemanager-common/gradle.lockfile b/azure-resourcemanager-common/gradle.lockfile
diff --git a/azure-resourcemanager-common/gradle.properties b/azure-resourcemanager-common/gradle.properties
@@ -0,0 +1,3 @@
+# Please update platform to consume version updates, see here for more:
+# https://github.com/DataBiosphere/terra-cloud-resource-lib#publishing-an-update
+version = 0.1.0
diff --git a/...in/java/bio/terra/cloudres/azure/resourcemanager/common/AzureResourceCleanupRecorder.java b/...in/java/bio/terra/cloudres/azure/resourcemanager/common/AzureResourceCleanupRecorder.java
@@ -0,0 +1,47 @@
+package bio.terra.cloudres.azure.resourcemanager.common;
+
+import static bio.terra.cloudres.azure.resourcemanager.common.Defaults.CLOUD_RESOURCE_REQUEST_DATA_KEY;
+
+import bio.terra.cloudres.common.ClientConfig;
+import bio.terra.cloudres.common.cleanup.CleanupRecorder;
+import com.azure.core.http.policy.HttpRequestLogger;
+import com.azure.core.http.policy.HttpRequestLoggingContext;
+import com.azure.core.util.Context;
+import com.azure.core.util.logging.ClientLogger;
+import java.util.Optional;
+import reactor.core.publisher.Mono;
+
+/**
+ * Intercepts Azure cloud resource creations to record them for cleanup.
+ *
+ * <p>Implemented as a {@link HttpRequestLogger} to record created resources as soon as the request
+ * is made. This is a no-op if the HTTP request is not a cloud resource creation.
+ */
+public class AzureResourceCleanupRecorder implements HttpRequestLogger {
+  private final ClientConfig clientConfig;
+
+  AzureResourceCleanupRecorder(ClientConfig clientConfig) {
+    this.clientConfig = clientConfig;
+  }
+
+  @Override
+  public Mono<Void> logRequest(ClientLogger logger, HttpRequestLoggingContext loggingOptions) {
+    final Context context = loggingOptions.getContext();
+
+    Optional.ofNullable(context)
+        .flatMap(c -> c.getData(CLOUD_RESOURCE_REQUEST_DATA_KEY))
+        .ifPresent(
+            data -> {
+              ResourceManagerRequestData requestData = (ResourceManagerRequestData) data;
+              requestData
+                  .resourceUidCreation()
+                  .ifPresent(
+                      resourceUid ->
+                          CleanupRecorder.record(
+                              resourceUid,
+                              requestData.resourceCreationMetadata().orElse(null),
+                              clientConfig));
+            });
+    return Mono.empty();
+  }
+}
diff --git a/...on/src/main/java/bio/terra/cloudres/azure/resourcemanager/common/AzureResponseLogger.java b/...on/src/main/java/bio/terra/cloudres/azure/resourcemanager/common/AzureResponseLogger.java
@@ -0,0 +1,139 @@
+package bio.terra.cloudres.azure.resourcemanager.common;
+
+import static bio.terra.cloudres.azure.resourcemanager.common.Defaults.CLOUD_RESOURCE_REQUEST_DATA_KEY;
+
+import bio.terra.cloudres.common.ClientConfig;
+import bio.terra.cloudres.common.OperationAnnotator;
+import bio.terra.cloudres.common.OperationData;
+import com.azure.core.http.ContentType;
+import com.azure.core.http.HttpHeaders;
+import com.azure.core.http.HttpRequest;
+import com.azure.core.http.HttpResponse;
+import com.azure.core.http.policy.HttpResponseLogger;
+import com.azure.core.http.policy.HttpResponseLoggingContext;
+import com.azure.core.util.CoreUtils;
+import com.azure.core.util.logging.ClientLogger;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Charsets;
+import com.google.gson.JsonObject;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.WritableByteChannel;
+import java.time.Duration;
+import java.util.Optional;
+import java.util.OptionalInt;
+import java.util.function.Consumer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import reactor.core.publisher.Flux;
+import reactor.core.publisher.Mono;
+
+/**
+ * Intercepts Azure HTTP responses and invokes {@link OperationAnnotator} to annotate cloud resource
+ * operations with logs, traces, and metrics.
+ */
+public class AzureResponseLogger implements HttpResponseLogger {
+  private static final int MAX_BODY_LOG_SIZE = 1024 * 16;
+  private static final Logger logger = LoggerFactory.getLogger(AzureResponseLogger.class);
+
+  private final OperationAnnotator operationAnnotator;
+
+  AzureResponseLogger(ClientConfig clientConfig) {
+    // Note we use our own Logger instead of the ClientLogger wrapper that Azure provides.
+    this.operationAnnotator = new OperationAnnotator(clientConfig, logger);
+  }
+
+  @VisibleForTesting
+  AzureResponseLogger(OperationAnnotator operationAnnotator) {
+    this.operationAnnotator = operationAnnotator;
+  }
+
+  @Override
+  public Mono<HttpResponse> logResponse(
+      ClientLogger clientLogger, HttpResponseLoggingContext loggingOptions) {
+    final HttpResponse response = loggingOptions.getHttpResponse();
+    final HttpRequest request = response.getRequest();
+
+    // Always add request method and request URL
+    JsonObject requestDataJson = new JsonObject();
+    requestDataJson.addProperty("requestMethod", request.getHttpMethod().toString());
+    requestDataJson.addProperty("requestUrl", request.getUrl().toString());
+
+    // Optionally add rich request data if provided in the logging context
+    Optional<ResourceManagerRequestData> requestData =
+        Optional.ofNullable(loggingOptions.getContext())
+            .flatMap(c -> c.getData(CLOUD_RESOURCE_REQUEST_DATA_KEY))
+            .map(o -> (ResourceManagerRequestData) o);
+    requestData.ifPresent(d -> requestDataJson.add("requestBody", d.serialize()));
+
+    // Add the raw request/response body only if debug logging is enabled, as it may be very
+    // verbose.
+    if (logger.isDebugEnabled()) {
+      logBody(
+          request.getHeaders(),
+          request.getBody(),
+          s -> requestDataJson.addProperty("rawRequestBody", s));
+      logBody(
+          response.getHeaders(),
+          response.buffer().getBody(),
+          s -> requestDataJson.addProperty("rawResponseBody", s));
+    }
+
+    // Build OperationData object.
+    OperationData operationData =
+        OperationData.builder()
+            .setDuration(
+                Optional.ofNullable(loggingOptions.getResponseDuration()).orElse(Duration.ZERO))
+            .setTryCount(OptionalInt.of(loggingOptions.getTryCount()))
+            .setExecutionException(Optional.empty())
+            .setHttpStatusCode(OptionalInt.of(response.getStatusCode()))
+            .setCloudOperation(
+                requestData
+                    .map(ResourceManagerRequestData::cloudOperation)
+                    .orElse(ResourceManagerOperation.AZURE_RESOURCE_MANAGER_UNKNOWN_OPERATION))
+            .setRequestData(requestDataJson)
+            .build();
+
+    // Invoke OperationAnnotator to record the operation.
+    operationAnnotator.recordOperation(operationData);
+
+    return Mono.justOrEmpty(response);
+  }
+
+  private static void logBody(
+      HttpHeaders headers, Flux<ByteBuffer> body, Consumer<String> consumer) {
+    // Ensure we have a valid content length
+    String contentLengthString = headers.getValue("Content-Length");
+    if (CoreUtils.isNullOrEmpty(contentLengthString)) {
+      return;
+    }
+    final long contentLength;
+    try {
+      contentLength = Long.parseLong(contentLengthString);
+    } catch (NumberFormatException | NullPointerException e) {
+      return;
+    }
+
+    // The body is logged if the Content-Type is not "application/octet-stream" and the
+    // body isn't empty and is less than 16KB in size.
+    String contentTypeHeader = headers.getValue("Content-Type");
+    if (!ContentType.APPLICATION_OCTET_STREAM.equalsIgnoreCase(contentTypeHeader)
+        && contentLength != 0
+        && contentLength < MAX_BODY_LOG_SIZE) {
+      ByteArrayOutputStream outputStream = new ByteArrayOutputStream((int) contentLength);
+      WritableByteChannel bodyContentChannel = Channels.newChannel(outputStream);
+      body.flatMap(
+              byteBuffer -> {
+                try {
+                  bodyContentChannel.write(byteBuffer.duplicate());
+                  return Mono.just(byteBuffer);
+                } catch (IOException e) {
+                  return Mono.error(e);
+                }
+              })
+          .doFinally(ignored -> consumer.accept(outputStream.toString(Charsets.UTF_8)));
+    }
+  }
+}
diff --git a/...anager-common/src/main/java/bio/terra/cloudres/azure/resourcemanager/common/Defaults.java b/...anager-common/src/main/java/bio/terra/cloudres/azure/resourcemanager/common/Defaults.java
@@ -0,0 +1,68 @@
+package bio.terra.cloudres.azure.resourcemanager.common;
+
+import bio.terra.cloudres.common.ClientConfig;
+import com.azure.core.http.policy.HttpLogDetailLevel;
+import com.azure.core.http.policy.HttpLogOptions;
+import com.azure.core.util.Context;
+import com.azure.resourcemanager.resources.fluentcore.arm.AzureConfigurable;
+
+/** Provides defaults for working with the Azure Resource Manager API. */
+public class Defaults {
+  private Defaults() {}
+
+  static final String CLOUD_RESOURCE_REQUEST_DATA_KEY = "crlRequestData";
+
+  /**
+   * Builds a standard {@link Context} object for passing {@link ResourceManagerRequestData} to
+   * Azure Resource Manager APIs. This should be used to enrich structured logging, and track
+   * resource creations for clean-up.
+   *
+   * <p>Example usage:
+   *
+   * <pre>
+   *     computeManager
+   *         .networkManager()
+   *         .publicIpAddresses()
+   *         .define(name)
+   *         .withRegion(region)
+   *         .withExistingResourceGroup(resourceGroupName)
+   *         .withDynamicIP()
+   *         .create(
+   *             Defaults.buildContext(
+   *                 CreatePublicIpRequestData.builder()
+   *                     .setResourceGroupName(resourceGroupName)
+   *                     .setName(name)
+   *                     .setRegion(region)
+   *                     .setIpAllocationMethod(IpAllocationMethod.DYNAMIC)
+   *                     .build()));
+   * </pre>
+   */
+  public static Context buildContext(ResourceManagerRequestData requestData) {
+    return new Context(CLOUD_RESOURCE_REQUEST_DATA_KEY, requestData);
+  }
+
+  /**
+   * Configures a client for CRL usage.
+   *
+   * <p>Example usage:
+   *
+   * <pre>
+   *    crlConfigure(clientConfig, ComputeManager.configure())
+   *        .authenticate(tokenCredential, azureProfile);
+   * </pre>
+   *
+   * @param clientConfig client configuration object to manage CRL behavior.
+   * @param configurable Azure client to configure.
+   * @return a configured Azure client.
+   */
+  public static <T extends AzureConfigurable<T>> T crlConfigure(
+      ClientConfig clientConfig, AzureConfigurable<T> configurable) {
+    return configurable.withLogOptions(
+        new HttpLogOptions()
+            .setRequestLogger(new AzureResourceCleanupRecorder(clientConfig))
+            .setResponseLogger(new AzureResponseLogger(clientConfig))
+            // Since we are providing our own loggers this value isn't actually used; however it
+            // does need to be set to a value other than NONE for the loggers to fire.
+            .setLogLevel(HttpLogDetailLevel.BASIC));
+  }
+}
diff --git a/...c/main/java/bio/terra/cloudres/azure/resourcemanager/common/ResourceManagerOperation.java b/...c/main/java/bio/terra/cloudres/azure/resourcemanager/common/ResourceManagerOperation.java
@@ -0,0 +1,12 @@
+package bio.terra.cloudres.azure.resourcemanager.common;
+
+import bio.terra.cloudres.common.CloudOperation;
+
+/**
+ * {@link CloudOperation} for using Azure ResourceManager API.
+ *
+ * <p>This is used as a generic fallback; in practice, a more specific value should be used.
+ */
+public enum ResourceManagerOperation implements CloudOperation {
+  AZURE_RESOURCE_MANAGER_UNKNOWN_OPERATION
+}
diff --git a/...main/java/bio/terra/cloudres/azure/resourcemanager/common/ResourceManagerRequestData.java b/...main/java/bio/terra/cloudres/azure/resourcemanager/common/ResourceManagerRequestData.java
@@ -0,0 +1,42 @@
+package bio.terra.cloudres.azure.resourcemanager.common;
+
+import bio.terra.cloudres.common.CloudOperation;
+import bio.terra.janitor.model.CloudResourceUid;
+import bio.terra.janitor.model.ResourceMetadata;
+import com.google.gson.JsonObject;
+import java.util.Optional;
+
+/**
+ * An abstract representation of data passed to Azure Resource Manager requests.
+ *
+ * <p>Contains functionality for serializing request data for structured logging; and tracking
+ * resource creations for clean-up.
+ */
+public interface ResourceManagerRequestData {
+
+  /** The {@link CloudOperation} value for this request. */
+  CloudOperation cloudOperation();
+
+  /**
+   * The {@link CloudResourceUid} of the resource that will be created by this request, if this
+   * request creates a resource.
+   *
+   * <p>Should be overridden by subclasses that create resources.
+   */
+  default Optional<CloudResourceUid> resourceUidCreation() {
+    return Optional.empty();
+  }
+
+  /**
+   * The {@link ResourceMetadata} of the resource that will be created by this request, if this
+   * request creates a resource that should have metadata.
+   *
+   * <p>Should only be non-empty when {@link #resourceUidCreation()} is present.
+   */
+  default Optional<ResourceMetadata> resourceCreationMetadata() {
+    return Optional.empty();
+  }
+
+  /** How to serialize the request for logging. */
+  JsonObject serialize();
+}