ALS-7014: Presigned url initial commit with some test code

data/pom.xml

@@ -49,6 +49,17 @@
 			<groupId>org.projectlombok</groupId>
 			<artifactId>lombok</artifactId>
 		</dependency>
+		<dependency>
+			<groupId>software.amazon.awssdk</groupId>
+			<artifactId>s3</artifactId>
+			<exclusions>
+				<!--Spring boot will complain about this dep on startup. It's not needed (we use SLF4J + Logback)-->
+				<exclusion>
+					<groupId>commons-logging</groupId>
+					<artifactId>commons-logging</artifactId>
+				</exclusion>
+			</exclusions>
+		</dependency>
 	</dependencies>
 
 </project>

data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/upload/SignUrlService.java

@@ -0,0 +1,74 @@
+package edu.harvard.hms.dbmi.avillach.hpds.data.upload;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.presigner.S3Presigner;
+import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest;
+import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest;
+
+import java.io.File;
+import java.time.Duration;
+import java.util.HashMap;
+import java.util.Map;
+
+@Component
+public class SignUrlService {
+
+    private static Logger log = LoggerFactory.getLogger(SignUrlService.class);
+
+    public static void main(String[] args) {
+        SignUrlService signUrlService = new SignUrlService();
+
+        signUrlService.uploadFile(new File("/Users/ryan/Downloads/test.txt"), "ramari-testing", "test2.txt");
+        String presignedGetUrl = signUrlService.createPresignedGetUrl("ramari-testing", "test2.txt");
+        log.info(presignedGetUrl);
+    }
+
+    public void uploadFile(File file, String bucketName, String objectKey) {
+        Region region = Region.US_EAST_1;
+        S3Client s3 = S3Client.builder()
+                .region(region)
+                .build();
+
+        putS3Object(s3, bucketName, objectKey, file);
+        s3.close();
+    }
+
+    // This example uses RequestBody.fromFile to avoid loading the whole file into
+    // memory.
+    public void putS3Object(S3Client s3, String bucketName, String objectKey, File file) {
+        Map<String, String> metadata = new HashMap<>();
+        PutObjectRequest putOb = PutObjectRequest.builder()
+                .bucket(bucketName)
+                .key(objectKey)
+                .metadata(metadata)
+                .build();
+
+        s3.putObject(putOb, RequestBody.fromFile(file));
+        log.info("Successfully placed " + objectKey + " into bucket " + bucketName);
+    }
+
+    public String createPresignedGetUrl(String bucketName, String keyName) {
+        S3Presigner presigner = S3Presigner.builder().region(Region.US_EAST_1).build();
+        GetObjectRequest objectRequest = GetObjectRequest.builder()
+                .bucket(bucketName)
+                .key(keyName)
+                .build();
+
+        GetObjectPresignRequest presignRequest = GetObjectPresignRequest.builder()
+                .signatureDuration(Duration.ofMinutes(10))  // The URL will expire in 10 minutes.
+                .getObjectRequest(objectRequest)
+                .build();
+
+        PresignedGetObjectRequest presignedRequest = presigner.presignGetObject(presignRequest);
+        log.info("Presigned URL: [{}]", presignedRequest.url().toString());
+
+        return presignedRequest.url().toExternalForm();
+    }
+}

pom.xml

@@ -28,6 +28,7 @@
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<dockerfile-maven-version>1.4.10</dockerfile-maven-version>
+		<aws.version>2.20.153</aws.version>
 	</properties>
 	<repositories>
 		<repository>
@@ -320,7 +321,18 @@
 				<artifactId>snappy-java</artifactId>
 				<version>1.1.10.5</version>
 			</dependency>
-
+			<dependency>
+				<groupId>software.amazon.awssdk</groupId>
+				<artifactId>s3</artifactId>
+				<version>${aws.version}</version>
+				<exclusions>
+					<!--Spring boot will complain about this dep on startup. It's not needed (we use SLF4J + Logback)-->
+					<exclusion>
+						<groupId>commons-logging</groupId>
+						<artifactId>commons-logging</artifactId>
+					</exclusion>
+				</exclusions>
+			</dependency>
 
 		</dependencies>
 	</dependencyManagement>

processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/AsyncResult.java

@@ -43,6 +43,10 @@ public MediaType getResponseType() {
 		return responseType;
 	}
 
+	public File getFile() {
+		return stream.getFile();
+	}
+
 	public static enum Status{
 		SUCCESS {
 			@Override

processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/ResultStoreStream.java

@@ -117,4 +117,8 @@ public long estimatedSize() {
 	public void closeWriter() {
 		writer.close();
 	}
+
+	public File getFile() {
+		return writer.getFile();
+	}
 }

service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/PicSureService.java

@@ -1,6 +1,7 @@
 package edu.harvard.hms.dbmi.avillach.hpds.service;
 
 import java.io.ByteArrayInputStream;
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
@@ -9,6 +10,7 @@
 import java.util.stream.Collectors;
 
 import edu.harvard.hms.dbmi.avillach.hpds.data.genotype.InfoColumnMeta;
+import edu.harvard.hms.dbmi.avillach.hpds.data.upload.SignUrlService;
 import edu.harvard.hms.dbmi.avillach.hpds.service.util.Paginator;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -41,13 +43,15 @@ public class PicSureService {
 
 	@Autowired
 	public PicSureService(QueryService queryService, TimelineProcessor timelineProcessor, CountProcessor countProcessor,
-						  VariantListProcessor variantListProcessor, AbstractProcessor abstractProcessor, Paginator paginator) {
+						  VariantListProcessor variantListProcessor, AbstractProcessor abstractProcessor, Paginator paginator,
+						  SignUrlService signUrlService) {
 		this.queryService = queryService;
 		this.timelineProcessor = timelineProcessor;
 		this.countProcessor = countProcessor;
 		this.variantListProcessor = variantListProcessor;
 		this.abstractProcessor = abstractProcessor;
 		this.paginator = paginator;
+		this.signUrlService = signUrlService;
 		Crypto.loadDefaultKey();
 	}
 
@@ -67,6 +71,8 @@ public PicSureService(QueryService queryService, TimelineProcessor timelineProce
 
 	private final Paginator paginator;
 
+	private final SignUrlService signUrlService;
+
 	private static final String QUERY_METADATA_FIELD = "queryMetadata";
 	private static final int RESPONSE_CACHE_SIZE = 50;
 
@@ -237,6 +243,36 @@ public ResponseEntity queryResult(@PathVariable("resourceQueryId") UUID queryId,
 		}
 	}
 
+	@PostMapping(value = "/query/{resourceQueryId}/signed-url")
+	public ResponseEntity querySignedURL(@PathVariable("resourceQueryId") UUID queryId, @RequestBody QueryRequest resultRequest) throws IOException {
+		AsyncResult result = queryService.getResultFor(queryId.toString());
+		if (result == null) {
+			// This happens sometimes when users immediately request the status for a query
+			// before it can be initialized. We wait a bit and try again before throwing an
+			// error.
+			try {
+				Thread.sleep(100);
+			} catch (InterruptedException e) {
+				return ResponseEntity.status(500).build();
+			}
+
+			result = queryService.getResultFor(queryId.toString());
+			if (result == null) {
+				return ResponseEntity.status(404).build();
+			}
+		}
+		if (result.getStatus() == AsyncResult.Status.SUCCESS) {
+			File file = result.getFile();
+			signUrlService.uploadFile(file, "ramari-testing", file.getName());
+			String presignedGetUrl = signUrlService.createPresignedGetUrl("ramari-testing", file.getName());
+			return ResponseEntity.ok()
+					.contentType(result.getResponseType())
+					.body(presignedGetUrl);
+		} else {
+			return ResponseEntity.status(400).body("Status : " + result.getStatus().name());
+		}
+	}
+
 	@PostMapping("/query/{resourceQueryId}/status")
 	public QueryStatus queryStatus(@PathVariable("resourceQueryId") UUID queryId, @RequestBody QueryRequest request) {
 		return convertToQueryStatus(queryService.getStatusFor(queryId.toString()));