fix: Store the resources in S3 buckets (data file and revisions offse…

…t files) sirixdb#582

bundles/sirix-core/src/main/java/org/sirix/io/cloud/CloudPlatform.java

@@ -0,0 +1,7 @@
+package org.sirix.io.cloud;
+
+public enum CloudPlatform {
+
+	AWS, GCP, AZURE
+
+}

bundles/sirix-core/src/main/java/org/sirix/io/cloud/CloudStorageConnectionFactory.java

@@ -0,0 +1,7 @@
+package org.sirix.io.cloud;
+
+public interface CloudStorageConnectionFactory {
+
+
+
+}

bundles/sirix-core/src/main/java/org/sirix/io/cloud/ICloudStorage.java

@@ -0,0 +1,9 @@
+package org.sirix.io.cloud;
+
+import org.sirix.io.IOStorage;
+
+public interface ICloudStorage extends IOStorage {
+
+
+
+}

bundles/sirix-core/src/main/java/org/sirix/io/cloud/amazon/AmazonS3Storage.java

@@ -0,0 +1,197 @@
+package org.sirix.io.cloud.amazon;
+
+import java.nio.file.Path;
+
+import org.sirix.access.ResourceConfiguration;
+import org.sirix.io.Reader;
+import org.sirix.io.RevisionFileData;
+import org.sirix.io.Writer;
+import org.sirix.io.bytepipe.ByteHandler;
+import org.sirix.io.bytepipe.ByteHandlerPipeline;
+import org.sirix.io.cloud.ICloudStorage;
+import org.sirix.utils.LogWrapper;
+import org.slf4j.LoggerFactory;
+
+import com.github.benmanes.caffeine.cache.AsyncCache;
+
+import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
+import software.amazon.awssdk.core.waiters.WaiterResponse;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
+import software.amazon.awssdk.services.s3.model.HeadBucketRequest;
+import software.amazon.awssdk.services.s3.model.HeadBucketResponse;
+import software.amazon.awssdk.services.s3.model.NoSuchBucketException;
+import software.amazon.awssdk.services.s3.model.S3Exception;
+import software.amazon.awssdk.services.s3.waiters.S3Waiter;
+
+/**
+ * Factory to provide Amazon S3 as storage backend
+ * 
+ * @Auther Sanket Band (@sband)
+ **/
+
+public class AmazonS3Storage implements ICloudStorage {
+
+	/**
+	 * Data file name.
+	 */
+	private static final String FILENAME = "sirix.data";
+
+	/**
+	 * Revisions file name.
+	 */
+	private static final String REVISIONS_FILENAME = "sirix.revisions";
+
+    /**
+     * Instance to local storage.
+    */
+    private final Path file;
+
+	/**
+	 * S3 storage bucket name
+	 * 
+	*/
+	private String bucketName;
+
+	private S3Client s3Client;
+
+	/** Logger. */
+	private static final LogWrapper LOGGER = new LogWrapper(LoggerFactory.getLogger(AmazonS3Storage.class));
+
+	/**
+	 * Byte handler pipeline.
+	*/
+    private final ByteHandlerPipeline byteHandlerPipeline;
+
+	/**
+	 * Revision file data cache.
+	*/
+    private final AsyncCache<Integer, RevisionFileData> cache;
+
+	/**
+	 * Support AWS authentication only with .aws credentials file with the required
+	 * profile name from the creds file
+	 */
+	public AmazonS3Storage(String bucketName, String awsProfile,
+			String region,
+			boolean shouldCreateBucketIfNotExists, final ResourceConfiguration resourceConfig,
+			AsyncCache<Integer, RevisionFileData> cache,
+			ByteHandlerPipeline byteHandlerPipeline) {
+		this.bucketName = bucketName;
+		this.s3Client = this.getS3Client(awsProfile,region);
+		/*
+		 * If the bucket does not exist, should create a new bucket based on the boolean
+		 */
+		/*
+		 * Exit the system if the cloud storage bucket cannot be created Alternatively,
+		 * we could just set a flag that could be checked before creating a reader or
+		 * writer. Return null if the bucket is not created OR does not exist But that
+		 * would keep the user under false impression that the bucket is created OR
+		 * exists already even if it does not exists
+		 */
+		if (!isBucketExists(bucketName, s3Client)) {
+			if (shouldCreateBucketIfNotExists) {
+				createBucket(bucketName, s3Client);
+			} else {
+				LOGGER.error(String.format("Bucket: %s, does not exists on Amazon S3 storage, exiting the system",
+						bucketName));
+				System.exit(1);
+			}
+		}
+		this.cache = cache;
+		this.byteHandlerPipeline = byteHandlerPipeline; 
+		this.file = resourceConfig.resourcePath;
+	}
+
+	private void createBucket(String bucketName, S3Client s3Client) {
+		try {
+			S3Waiter s3Waiter = s3Client.waiter();
+			CreateBucketRequest bucketRequest = CreateBucketRequest.builder().bucket(bucketName).build();
+
+			s3Client.createBucket(bucketRequest);
+			HeadBucketRequest bucketRequestWait = HeadBucketRequest.builder().bucket(bucketName).build();
+
+			WaiterResponse<HeadBucketResponse> waiterResponse = s3Waiter.waitUntilBucketExists(bucketRequestWait);
+			if (waiterResponse.matched().response().isPresent()) {
+				LOGGER.info(String.format("S3 bucket: %s has been created.", bucketName));
+			}
+		} catch (S3Exception e) {
+			LOGGER.error(e.awsErrorDetails().errorMessage());
+			LOGGER.error(String.format("Bucket: %s could not be created. Will not consume S3 storage", bucketName));
+			System.exit(1);
+		}
+	}
+
+	private boolean isBucketExists(String bucketName, S3Client s3Client) {
+		HeadBucketRequest headBucketRequest = HeadBucketRequest.builder().bucket(bucketName).build();
+
+		try {
+			s3Client.headBucket(headBucketRequest);
+			return true;
+		} catch (NoSuchBucketException e) {
+			return false;
+		} 
+	}
+
+	private S3Client getS3Client(String awsProfile, String region) {
+		S3Client s3Client = null;
+		s3Client = S3Client.builder()
+	            .region(Region.of(region))
+	            .credentialsProvider(ProfileCredentialsProvider.create(awsProfile))
+	            .build();
+		return s3Client;
+	}
+
+	@Override
+	public Writer createWriter() {
+		/*
+		 * This would create a writer that connects to the
+		 * remote storagee*/
+		return null;
+	}
+
+	@Override
+	public Reader createReader() {
+		/*
+		 * This would create a reader that connects to the 
+		 * remote storage on cloud
+		 * */
+		return null;
+	}
+
+	@Override
+	public void close() {
+		// TODO Auto-generated method stub
+
+	}
+
+	@Override
+	public boolean exists() {
+		// TODO Auto-generated method stub
+		return false;
+	}
+
+	@Override
+	public ByteHandler getByteHandler() {
+		return this.byteHandlerPipeline;
+	}
+
+	/**
+	 * Getting path for data file.
+	 * This path would be used on the local storage
+	 * @return the path for this data file
+	 */
+	private Path getDataFilePath() {
+	  return file.resolve(ResourceConfiguration.ResourcePaths.DATA.getPath()).resolve(FILENAME);
+	}
+
+	/**
+	 * Getting concrete storage for this file.
+	 *
+	 * @return the concrete storage for this database
+	 */
+	private Path getRevisionFilePath() {
+	  return file.resolve(ResourceConfiguration.ResourcePaths.DATA.getPath()).resolve(REVISIONS_FILENAME);
+	}
+}

bundles/sirix-core/src/main/java/org/sirix/io/cloud/amazon/AmazonS3StorageReader.java

@@ -0,0 +1,122 @@
+package org.sirix.io.cloud.amazon;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.RandomAccessFile;
+import java.nio.file.FileSystems;
+import java.time.Instant;
+
+import org.checkerframework.checker.nullness.qual.Nullable;
+import org.sirix.api.PageReadOnlyTrx;
+import org.sirix.io.Reader;
+import org.sirix.io.RevisionFileData;
+import org.sirix.io.bytepipe.ByteHandler;
+import org.sirix.io.file.FileReader;
+import org.sirix.page.PagePersister;
+import org.sirix.page.PageReference;
+import org.sirix.page.RevisionRootPage;
+import org.sirix.page.SerializationType;
+import org.sirix.page.interfaces.Page;
+import org.sirix.utils.LogWrapper;
+import org.slf4j.LoggerFactory;
+
+import com.github.benmanes.caffeine.cache.Cache;
+
+import software.amazon.awssdk.core.ResponseBytes;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectResponse;
+import software.amazon.awssdk.services.s3.model.S3Exception;
+
+public class AmazonS3StorageReader implements Reader {
+
+	/**
+	 * S3 storage bucket name
+	 * 
+	*/
+	private String bucketName;
+
+	private S3Client s3Client;
+
+	/** Logger. */
+	private static final LogWrapper LOGGER = new LogWrapper(LoggerFactory.getLogger(AmazonS3StorageReader.class));
+
+
+	private FileReader reader;
+
+	public AmazonS3StorageReader(String bucketName, S3Client s3Client, 
+			final RandomAccessFile dataFile, 
+			final RandomAccessFile revisionsOffsetFile,
+			final ByteHandler byteHandler, 
+			final SerializationType serializationType, 
+			final PagePersister pagePersister,
+		    final Cache<Integer, RevisionFileData> cache) {
+		this.bucketName = bucketName;
+		this.s3Client = s3Client;
+		this.reader = new FileReader(dataFile,
+				revisionsOffsetFile,
+				byteHandler,
+				serializationType,
+				pagePersister,
+                cache);
+	}
+
+	private void readObjectDataFromS3(String keyName) {
+
+		try {
+            GetObjectRequest objectRequest = GetObjectRequest
+                .builder()
+                .key(keyName)
+                .bucket(bucketName)
+                .build();
+
+            ResponseBytes<GetObjectResponse> objectBytes = s3Client.getObjectAsBytes(objectRequest);
+            byte[] data = objectBytes.asByteArray();
+            String path = System.getProperty("java.io.tmpdir")	+ FileSystems.getDefault().getSeparator() + keyName;
+            // Write the data to a local file.
+            File myFile = new File(path);
+            OutputStream os = new FileOutputStream(myFile);
+            os.write(data);
+            os.close();
+        } catch (IOException ex) {
+            ex.printStackTrace();
+        } catch (S3Exception e) {
+            System.err.println(e.awsErrorDetails().errorMessage());
+            System.exit(1);
+        }
+    }
+
+	@Override
+	public PageReference readUberPageReference() {
+		return reader.readUberPageReference();
+	}
+
+	@Override
+	public Page read(PageReference key, @Nullable PageReadOnlyTrx pageReadTrx) {
+		return reader.read(key, pageReadTrx);
+	}
+
+	@Override
+	public void close() {
+		s3Client.close();
+		reader.close();
+	}
+
+	@Override
+	public RevisionRootPage readRevisionRootPage(int revision, PageReadOnlyTrx pageReadTrx) {
+		return reader.readRevisionRootPage(revision, pageReadTrx);
+	}
+
+	@Override
+	public Instant readRevisionRootPageCommitTimestamp(int revision) {
+		return reader.readRevisionRootPageCommitTimestamp(revision);
+	}
+
+	@Override
+	public RevisionFileData getRevisionFileData(int revision) {
+		return reader.getRevisionFileData(revision);
+	}
+
+}

bundles/sirix-core/src/main/java/org/sirix/io/cloud/amazon/AmazonS3StorageWriter.java

@@ -0,0 +1,5 @@
+package org.sirix.io.cloud.amazon;
+
+public class AmazonS3StorageWriter {
+
+}