From 08a5794ca10210c31fa68482aa01d3745a433418 Mon Sep 17 00:00:00 2001 From: Nathan Buckingham Date: Tue, 6 Aug 2024 16:51:48 -0400 Subject: [PATCH 1/3] 116959: First pass of jCloud port --- dspace-api/pom.xml | 28 ++ .../storage/bitstore/BitstreamByteSource.java | 50 +++ .../bitstore/JCloudBitStoreService.java | 304 ++++++++++++++++++ dspace/config/modules/assetstore.cfg | 20 +- dspace/config/spring/api/bitstore.xml | 18 ++ 5 files changed, 416 insertions(+), 4 deletions(-) create mode 100644 dspace-api/src/main/java/org/dspace/storage/bitstore/BitstreamByteSource.java create mode 100644 dspace-api/src/main/java/org/dspace/storage/bitstore/JCloudBitStoreService.java diff --git a/dspace-api/pom.xml b/dspace-api/pom.xml index 79094ddbb8e1..2d21b0bd596d 100644 --- a/dspace-api/pom.xml +++ b/dspace-api/pom.xml @@ -839,6 +839,34 @@ + + + org.apache.jclouds + jclouds-core + 2.5.0 + + + com.google.code.gson + gson + + + com.sun.xml.bind + jaxb-impl + + + + + + org.apache.jclouds + jclouds-blobstore + 2.5.0 + + + + org.apache.jclouds.provider + aws-s3 + 2.5.0 + diff --git a/dspace-api/src/main/java/org/dspace/storage/bitstore/BitstreamByteSource.java b/dspace-api/src/main/java/org/dspace/storage/bitstore/BitstreamByteSource.java new file mode 100644 index 000000000000..36b970fb5ff8 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/storage/bitstore/BitstreamByteSource.java @@ -0,0 +1,50 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.storage.bitstore; + +import java.io.IOException; +import java.io.InputStream; +import java.sql.SQLException; + +import com.google.common.io.ByteSource; +import org.dspace.authorize.AuthorizeException; +import org.dspace.content.Bitstream; +import org.dspace.content.factory.ContentServiceFactory; +import org.dspace.content.service.BitstreamService; +import org.dspace.core.Context; + +public class BitstreamByteSource extends ByteSource { + + private static final BitstreamService bitstreamService = ContentServiceFactory.getInstance().getBitstreamService(); + + public Bitstream getBitstream() { + return bitstream; + } + + private final Bitstream bitstream; + + public BitstreamByteSource(Bitstream bitstream) { + this.bitstream = bitstream; + } + + @Override + public InputStream openStream() throws IOException { + try { + return bitstreamService.retrieve(new Context(), bitstream); + } catch (SQLException | AuthorizeException e) { + throw new IOException(e.getMessage(), e); + } + } + + @Override + public long size() throws IOException { + return bitstream.getSizeBytes(); + } + + +} diff --git a/dspace-api/src/main/java/org/dspace/storage/bitstore/JCloudBitStoreService.java b/dspace-api/src/main/java/org/dspace/storage/bitstore/JCloudBitStoreService.java new file mode 100644 index 000000000000..82116bd7ece8 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/storage/bitstore/JCloudBitStoreService.java @@ -0,0 +1,304 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.storage.bitstore; + +import com.google.common.hash.HashCode; +import com.google.common.io.ByteSource; +import com.google.common.io.Files; +import com.google.common.net.MediaType; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.dspace.content.Bitstream; +import org.dspace.content.BitstreamFormat; +import org.dspace.core.Context; +import org.dspace.core.Utils; +import org.dspace.utils.DSpace; +import org.jclouds.ContextBuilder; +import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.BlobStoreContext; +import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.domain.BlobMetadata; +import org.jclouds.blobstore.options.ListContainerOptions; +import org.jclouds.blobstore.options.PutOptions.Builder; +import org.jclouds.io.ContentMetadata; +import org.jclouds.javax.annotation.Nullable; +import org.springframework.beans.factory.annotation.Autowired; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.sql.SQLException; +import java.util.Map; + +/** + * JCloudBitstream asset store service + * + * @author Mark Diggory + */ +public class JCloudBitStoreService extends BaseBitStoreService { + + private static final Logger log = LogManager.getLogger(JCloudBitStoreService.class); + + private String proivderOrApi; + private ContextBuilder builder; + private BlobStoreContext blobStoreContext; + private String container; + private String subFolder; + private String identity; + private String credential; + private String endpoint; + + private boolean useRelativePath; + private boolean enabled = false; + private int counter = 0; + private int maxCounter= 100; + + private static final String CSA = "MD5"; + + @SuppressWarnings("WeakerAccess") + public JCloudBitStoreService() { + } + + @Autowired + public void setUseRelativePath(boolean useRelativePath) { + this.useRelativePath = useRelativePath; + } + + @Autowired + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + @Autowired + public void setContainer(String container) { + this.container = container; + } + + public void setSubFolder(String subFolder) { + this.subFolder = subFolder; + } + + @Autowired + public void setIdentity(String identity) { + this.identity = identity; + } + + @Autowired + public void setCredentials(@Nullable String credential) { + this.credential = credential; + } + + @Autowired + public void setProivderOrApi(String proivderOrApi) { + this.proivderOrApi = proivderOrApi; + } + + public void setEndpoint(String endpoint) { + this.endpoint = endpoint; + } + + public void setMaxCounter(int maxCounter) { + this.maxCounter = maxCounter; + } + + @Override + public void init() throws IOException { + if (this.isInitialized()) { + return; + } + this.builder = ContextBuilder.newBuilder(proivderOrApi); + if (endpoint != null) { + this.builder = this.builder.endpoint(endpoint); + } + blobStoreContext = this.builder.credentials(identity, credential).build(BlobStoreContext.class); + this.initialized = true; + } + + private synchronized void refreshContextIfNeeded() { + counter++; + // Destroying and recreating the connection between JClouds and CloudFiles + if (counter == maxCounter) { + counter = 0; + blobStoreContext.close(); + blobStoreContext = this.builder.credentials(identity, credential).buildView(BlobStoreContext.class); + } + } + + @Override + public String generateId() { + return Utils.generateKey(); + } + + @Override + public InputStream get(final Bitstream bitstream) throws IOException { + final File file = getFile(bitstream); + return get(file); + } + + private InputStream get(File file) throws IOException { + BlobStore blobStore = blobStoreContext.getBlobStore(); + if (blobStore.blobExists(getContainer(), file.toString())) { + Blob blob = blobStore.getBlob(getContainer(), file.toString()); + refreshContextIfNeeded(); + return blob.getPayload().openStream(); + } + throw new IOException("File not found: " + file); + } + + @Override + public void remove(Bitstream bitstream) throws IOException { + File file = getFile(bitstream); + BlobStore blobStore = blobStoreContext.getBlobStore(); + blobStore.removeBlob(getContainer(), file.toString()); + deleteParents(file); + } + + private void deleteParents(File file) { + if (file == null) { + return; + } + final BlobStore blobStore = blobStoreContext.getBlobStore(); + for (int i = 0; i < directoryLevels; i++) { + final File directory = file.getParentFile(); + final ListContainerOptions options = new ListContainerOptions(); + options.inDirectory(directory.getPath()); + long blobs = blobStore.countBlobs(getContainer(), options); + if (blobs != 0){ + break; + } + blobStore.deleteDirectory(getContainer(), directory.getPath()); + file = directory; + } + } + + public void put(ByteSource byteSource, Bitstream bitstream) throws IOException { + + final File file = getFile(bitstream); + + /* set type to sane default */ + String type = MediaType.OCTET_STREAM.toString(); + + /* attempt to get type if the source is a Bitstream */ + if (byteSource instanceof BitstreamByteSource) { + type = getMIMEType(((BitstreamByteSource) byteSource).getBitstream()); + } + + BlobStore blobStore = blobStoreContext.getBlobStore(); + String container = getContainer(); + + if (!blobStore.containerExists(container)) { + blobStore.createContainerInLocation(null, container); + } + + Blob blob = blobStore.blobBuilder(file.toString()) + .payload(byteSource) + .contentDisposition(file.toString()) + .contentLength(byteSource.size()) + .contentType(type) + .build(); + + /* Utilize large file transfer to S3 via multipart post */ + blobStore.putBlob(container, blob, Builder.multipart()); + } + + @Override + public void put(Bitstream bitstream, InputStream in) throws IOException { + File tmp = File.createTempFile("jclouds", "cache"); + try { + // Inefficient caching strategy, however allows for use of JClouds store directly without CachingStore. + // Make sure there is sufficient storage in temp directory. + Files.asByteSink(tmp).writeFrom(in); + in.close(); + put(Files.asByteSource(tmp), bitstream); + } finally { + if (!tmp.delete()) { + tmp.deleteOnExit(); + } + } + } + + public static String getMIMEType(final Bitstream bitstream) { + try { + BitstreamFormat format = bitstream.getFormat(new Context()); + return format == null ? null : format.getMIMEType(); + } catch (SQLException ignored) { + throw new RuntimeException(ignored); + } + } + + @Override + @SuppressWarnings("unchecked") + public Map about(Bitstream bitstream, Map attrs) throws IOException { + File file = getFile(bitstream); + BlobStore blobStore = blobStoreContext.getBlobStore(); + BlobMetadata blobMetadata = blobStore.blobMetadata(getContainer(), file.toString()); + if(blobMetadata != null){ + ContentMetadata contentMetadata = blobMetadata.getContentMetadata(); + + if (contentMetadata != null) { + attrs.put("size_bytes", String.valueOf(contentMetadata.getContentLength())); + final HashCode hashCode = contentMetadata.getContentMD5AsHashCode(); + if (hashCode != null) { + attrs.put("checksum", Utils.toHex(contentMetadata.getContentMD5AsHashCode().asBytes())); + attrs.put("checksum_algorithm", CSA); + } + attrs.put("modified", String.valueOf(blobMetadata.getLastModified().getTime())); + + attrs.put("ContentDisposition", contentMetadata.getContentDisposition()); + attrs.put("ContentEncoding", contentMetadata.getContentEncoding()); + attrs.put("ContentLanguage", contentMetadata.getContentLanguage()); + attrs.put("ContentType", contentMetadata.getContentType()); + + if (contentMetadata.getExpires() != null) { + attrs.put("Expires", contentMetadata.getExpires().getTime()); + } + } + + return attrs; + } + return null; + } + + public File getFile(Bitstream bitstream) throws IOException { + StringBuilder sb = new StringBuilder(); + String id = bitstream.getInternalId(); + sb.append(getIntermediatePath(id)); + sb.append(id); + if (log.isDebugEnabled()) + { + log.debug("Local filename for " + id + " is " + sb.toString()); + } + return new File(sb.toString()); + } + + /** + * Gets the URI of the content within the store. + * + * @param id the bitstream internal id. + * @return the URI, which is a relative path to the content. + */ + @SuppressWarnings("unused") // used by AVS2 + public URI getStoredURI(String id) { + StringBuilder sb = new StringBuilder(); + sb.append(getIntermediatePath(id)); + sb.append(id); + if (log.isDebugEnabled()) + { + log.debug("Local URI for " + id + " is " + sb.toString()); + } + return URI.create(sb.toString()); + } + + private String getContainer(){ + if(container == null){ + container = new DSpace().getConfigurationService().getProperty("dspace.hostname"); + } + return container; + } +} diff --git a/dspace/config/modules/assetstore.cfg b/dspace/config/modules/assetstore.cfg index cbee6bd2c3a4..2ceb618beeb0 100644 --- a/dspace/config/modules/assetstore.cfg +++ b/dspace/config/modules/assetstore.cfg @@ -12,10 +12,10 @@ assetstore.dir = ${dspace.dir}/assetstore # This value will be used as `incoming` default store inside the `bitstore.xml` # Possible values are: # - 0: to use the `localStore`; -# - 1: to use the `s3Store`. +# - 1: to use the `s3Store`. # If you want to add additional assetstores, they must be added to that bitstore.xml # and new values should be provided as key-value pairs in the `stores` map of the -# `bitstore.xml` configuration. +# `bitstore.xml` configuration. assetstore.index.primary = 0 #---------------------------------------------------------------# @@ -33,7 +33,7 @@ assetstore.s3.enabled = false # When true: it splits the path into subfolders, each of these # are 2-chars (2-bytes) length, the last is the filename and could have # at max 3-chars (3-bytes). -# When false: is used the absolute path using full filename. +# When false: is used the absolute path using full filename. assetstore.s3.useRelativePath = false # S3 bucket name to store assets in. If unspecified, by default DSpace will @@ -54,4 +54,16 @@ assetstore.s3.awsSecretKey = # If the credentials are left empty, # then this setting is ignored and the default AWS region will be used. -assetstore.s3.awsRegionName = \ No newline at end of file +assetstore.s3.awsRegionName = + + +### JCloudSettings + +# Enabled the JCloudStore +assetstore.s3.generic.enabled = false +assetstore.s3.generic.useRelativePath = false +assetstore.s3.generic.subfolder = assetstore +assetstore.s3.endpoint = +assetstore.s3.generic.container = +assetstore.s3.endpoint = +assetstore.s3.maxCounter = 100 diff --git a/dspace/config/spring/api/bitstore.xml b/dspace/config/spring/api/bitstore.xml index 15bb3ef1580b..a7ab8ddaa669 100644 --- a/dspace/config/spring/api/bitstore.xml +++ b/dspace/config/spring/api/bitstore.xml @@ -9,6 +9,7 @@ + @@ -36,6 +37,23 @@ + + + + + + + + + + + + + + + + + From 91d0debcbeaeaec58c32bcfc960a5ffe26e843f6 Mon Sep 17 00:00:00 2001 From: Nathan Buckingham Date: Wed, 7 Aug 2024 17:08:20 -0400 Subject: [PATCH 2/3] 116959: Fix dependancy issue with guava, fix lint, Add tests from s3, Allow for overrides to propeties --- dspace-api/pom.xml | 17 +- .../storage/bitstore/BaseBitStoreService.java | 14 +- .../bitstore/JCloudBitStoreService.java | 152 ++++++++++++----- .../bitstore/JCloudBitStoreServiceTest.java | 156 ++++++++++++++++++ dspace/config/modules/assetstore.cfg | 1 - dspace/config/spring/api/bitstore.xml | 16 +- pom.xml | 6 + 7 files changed, 301 insertions(+), 61 deletions(-) create mode 100644 dspace-api/src/test/java/org/dspace/storage/bitstore/JCloudBitStoreServiceTest.java diff --git a/dspace-api/pom.xml b/dspace-api/pom.xml index 2d21b0bd596d..4efdcfd5cbc2 100644 --- a/dspace-api/pom.xml +++ b/dspace-api/pom.xml @@ -648,11 +648,6 @@ 1.1.1 - - com.google.guava - guava - - org.postgresql postgresql @@ -686,6 +681,12 @@ com.google.api-client google-api-client + + + com.google.guava + guava-jdk5 + + com.google.http-client @@ -862,6 +863,12 @@ 2.5.0 + + org.apache.jclouds.api + filesystem + 2.5.0 + + org.apache.jclouds.provider aws-s3 diff --git a/dspace-api/src/main/java/org/dspace/storage/bitstore/BaseBitStoreService.java b/dspace-api/src/main/java/org/dspace/storage/bitstore/BaseBitStoreService.java index 209c1e21e74d..6925878d400c 100644 --- a/dspace-api/src/main/java/org/dspace/storage/bitstore/BaseBitStoreService.java +++ b/dspace-api/src/main/java/org/dspace/storage/bitstore/BaseBitStoreService.java @@ -73,7 +73,7 @@ protected String getIntermediatePath(String internalId) { * an attempt to make a path traversal attack, so ignore the path prefix. The * internal-ID is supposed to be just a filename, so this will not affect normal * operation. - * + * * @param sInternalId * @return Sanitized id */ @@ -86,7 +86,7 @@ protected String sanitizeIdentifier(String sInternalId) { /** * Append separator to target {@code StringBuilder} - * + * * @param path */ protected void appendSeparator(StringBuilder path) { @@ -97,7 +97,7 @@ protected void appendSeparator(StringBuilder path) { /** * Utility that checks string ending with separator - * + * * @param bufFilename * @return */ @@ -109,7 +109,7 @@ protected boolean endsWithSeparator(StringBuilder bufFilename) { * Splits internalId into several subpaths using {@code digitsPerLevel} that * indicates the folder name length, and {@code direcoryLevels} that indicates * the maximum number of subfolders. - * + * * @param internalId bitStream identifier * @param path */ @@ -125,7 +125,7 @@ protected void populatePathSplittingId(String internalId, StringBuilder path) { /** * Extract substring if is in range, otherwise will truncate to length - * + * * @param internalId * @param startIndex * @param endIndex @@ -140,7 +140,7 @@ protected String extractSubstringFrom(String internalId, int startIndex, int end /** * Checks if the {@code String} is longer than {@code endIndex} - * + * * @param internalId * @param endIndex * @return @@ -151,7 +151,7 @@ protected boolean isLonger(String internalId, int endIndex) { /** * Retrieves a map of useful metadata about the File (size, checksum, modified) - * + * * @param file The File to analyze * @param attrs The map where we are storing values * @return Map of updated metadatas / attrs diff --git a/dspace-api/src/main/java/org/dspace/storage/bitstore/JCloudBitStoreService.java b/dspace-api/src/main/java/org/dspace/storage/bitstore/JCloudBitStoreService.java index 82116bd7ece8..71185cf5c32b 100644 --- a/dspace-api/src/main/java/org/dspace/storage/bitstore/JCloudBitStoreService.java +++ b/dspace-api/src/main/java/org/dspace/storage/bitstore/JCloudBitStoreService.java @@ -7,16 +7,27 @@ */ package org.dspace.storage.bitstore; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.sql.SQLException; +import java.util.Map; +import java.util.Properties; + import com.google.common.hash.HashCode; import com.google.common.io.ByteSource; import com.google.common.io.Files; import com.google.common.net.MediaType; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.dspace.content.Bitstream; import org.dspace.content.BitstreamFormat; import org.dspace.core.Context; import org.dspace.core.Utils; +import org.dspace.storage.bitstore.factory.StorageServiceFactory; +import org.dspace.storage.bitstore.service.BitstreamStorageService; import org.dspace.utils.DSpace; import org.jclouds.ContextBuilder; import org.jclouds.blobstore.BlobStore; @@ -27,25 +38,18 @@ import org.jclouds.blobstore.options.PutOptions.Builder; import org.jclouds.io.ContentMetadata; import org.jclouds.javax.annotation.Nullable; -import org.springframework.beans.factory.annotation.Autowired; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.sql.SQLException; -import java.util.Map; /** * JCloudBitstream asset store service * - * @author Mark Diggory + * @author Mark Diggory, Nathan Buckingham */ public class JCloudBitStoreService extends BaseBitStoreService { private static final Logger log = LogManager.getLogger(JCloudBitStoreService.class); - private String proivderOrApi; + private Properties properties; + private String providerOrApi; private ContextBuilder builder; private BlobStoreContext blobStoreContext; private String container; @@ -57,25 +61,25 @@ public class JCloudBitStoreService extends BaseBitStoreService { private boolean useRelativePath; private boolean enabled = false; private int counter = 0; - private int maxCounter= 100; + private int maxCounter = 100; private static final String CSA = "MD5"; - @SuppressWarnings("WeakerAccess") public JCloudBitStoreService() { } - @Autowired + public JCloudBitStoreService(String providerOrApi) { + this.providerOrApi = providerOrApi; + } + public void setUseRelativePath(boolean useRelativePath) { this.useRelativePath = useRelativePath; } - @Autowired public void setEnabled(boolean enabled) { this.enabled = enabled; } - @Autowired public void setContainer(String container) { this.container = container; } @@ -84,19 +88,16 @@ public void setSubFolder(String subFolder) { this.subFolder = subFolder; } - @Autowired public void setIdentity(String identity) { this.identity = identity; } - @Autowired public void setCredentials(@Nullable String credential) { this.credential = credential; } - @Autowired - public void setProivderOrApi(String proivderOrApi) { - this.proivderOrApi = proivderOrApi; + public void setProviderOrApi(String providerOrApi) { + this.providerOrApi = providerOrApi; } public void setEndpoint(String endpoint) { @@ -107,17 +108,31 @@ public void setMaxCounter(int maxCounter) { this.maxCounter = maxCounter; } + public void setOverrides(Properties overrides) { + this.properties = overrides; + } + + @Override + public boolean isEnabled() { + return this.enabled; + } + @Override public void init() throws IOException { if (this.isInitialized()) { return; } - this.builder = ContextBuilder.newBuilder(proivderOrApi); - if (endpoint != null) { - this.builder = this.builder.endpoint(endpoint); + try { + this.builder = ContextBuilder.newBuilder(providerOrApi); + if (endpoint != null) { + this.builder = this.builder.endpoint(endpoint); + } + blobStoreContext = this.builder.overrides(properties) + .credentials(identity, credential).buildView(BlobStoreContext.class); + this.initialized = true; + } catch (Exception e) { + this.initialized = false; } - blobStoreContext = this.builder.credentials(identity, credential).build(BlobStoreContext.class); - this.initialized = true; } private synchronized void refreshContextIfNeeded() { @@ -126,7 +141,8 @@ private synchronized void refreshContextIfNeeded() { if (counter == maxCounter) { counter = 0; blobStoreContext.close(); - blobStoreContext = this.builder.credentials(identity, credential).buildView(BlobStoreContext.class); + blobStoreContext = this.builder.overrides(properties) + .credentials(identity, credential).buildView(BlobStoreContext.class); } } @@ -159,6 +175,59 @@ public void remove(Bitstream bitstream) throws IOException { deleteParents(file); } + /** + * Utility Method: Prefix the key with a subfolder, if this instance assets are stored within subfolder + * + * @param id DSpace bitstream internal ID + * @return full key prefixed with a subfolder, if applicable + */ + public String getFullKey(String id) { + StringBuilder bufFilename = new StringBuilder(); + if (StringUtils.isNotEmpty(this.subFolder)) { + bufFilename.append(this.subFolder); + appendSeparator(bufFilename); + } + + if (this.useRelativePath) { + bufFilename.append(getRelativePath(id)); + } else { + bufFilename.append(id); + } + + if (log.isDebugEnabled()) { + log.debug("S3 filepath for " + id + " is " + + bufFilename.toString()); + } + + return bufFilename.toString(); + } + + /** + * there are 2 cases: + * - conventional bitstream, conventional storage + * - registered bitstream, conventional storage + * conventional bitstream: dspace ingested, dspace random name/path + * registered bitstream: registered to dspace, any name/path + * + * @param sInternalId + * @return Computed Relative path + */ + private String getRelativePath(String sInternalId) { + BitstreamStorageService bitstreamStorageService = StorageServiceFactory.getInstance() + .getBitstreamStorageService(); + + String sIntermediatePath = StringUtils.EMPTY; + if (bitstreamStorageService.isRegisteredBitstream(sInternalId)) { + sInternalId = sInternalId.substring(2); + } else { + sInternalId = sanitizeIdentifier(sInternalId); + sIntermediatePath = getIntermediatePath(sInternalId); + } + + return sIntermediatePath + sInternalId; + } + + private void deleteParents(File file) { if (file == null) { return; @@ -169,7 +238,7 @@ private void deleteParents(File file) { final ListContainerOptions options = new ListContainerOptions(); options.inDirectory(directory.getPath()); long blobs = blobStore.countBlobs(getContainer(), options); - if (blobs != 0){ + if (blobs != 0) { break; } blobStore.deleteDirectory(getContainer(), directory.getPath()); @@ -238,7 +307,7 @@ public Map about(Bitstream bitstream, Map attrs) throws IOException { File file = getFile(bitstream); BlobStore blobStore = blobStoreContext.getBlobStore(); BlobMetadata blobMetadata = blobStore.blobMetadata(getContainer(), file.toString()); - if(blobMetadata != null){ + if (blobMetadata != null) { ContentMetadata contentMetadata = blobMetadata.getContentMetadata(); if (contentMetadata != null) { @@ -259,22 +328,18 @@ public Map about(Bitstream bitstream, Map attrs) throws IOException { attrs.put("Expires", contentMetadata.getExpires().getTime()); } } - return attrs; } return null; } public File getFile(Bitstream bitstream) throws IOException { - StringBuilder sb = new StringBuilder(); String id = bitstream.getInternalId(); - sb.append(getIntermediatePath(id)); - sb.append(id); - if (log.isDebugEnabled()) - { - log.debug("Local filename for " + id + " is " + sb.toString()); + id = getFullKey(id); + if (log.isDebugEnabled()) { + log.debug("Local filename for " + bitstream.getInternalId() + " is " + id); } - return new File(sb.toString()); + return new File(id); } /** @@ -285,18 +350,15 @@ public File getFile(Bitstream bitstream) throws IOException { */ @SuppressWarnings("unused") // used by AVS2 public URI getStoredURI(String id) { - StringBuilder sb = new StringBuilder(); - sb.append(getIntermediatePath(id)); - sb.append(id); - if (log.isDebugEnabled()) - { - log.debug("Local URI for " + id + " is " + sb.toString()); + String tempID = getFullKey(id); + if (log.isDebugEnabled()) { + log.debug("Local URI for " + id + " is " + tempID); } - return URI.create(sb.toString()); + return URI.create(id); } - private String getContainer(){ - if(container == null){ + private String getContainer() { + if (container == null) { container = new DSpace().getConfigurationService().getProperty("dspace.hostname"); } return container; diff --git a/dspace-api/src/test/java/org/dspace/storage/bitstore/JCloudBitStoreServiceTest.java b/dspace-api/src/test/java/org/dspace/storage/bitstore/JCloudBitStoreServiceTest.java new file mode 100644 index 000000000000..097aa71432c4 --- /dev/null +++ b/dspace-api/src/test/java/org/dspace/storage/bitstore/JCloudBitStoreServiceTest.java @@ -0,0 +1,156 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.storage.bitstore; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +import java.io.File; +import java.io.IOException; + +import org.dspace.AbstractUnitTest; +import org.dspace.content.Bitstream; +import org.hamcrest.Matchers; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; + +/** + * @author Nathan Buckingham + * + */ +public class JCloudBitStoreServiceTest extends AbstractUnitTest { + + + private JCloudBitStoreService jCloudBitStoreService; + + @Mock + private Bitstream bitstream; + + @Before + public void setUp() throws Exception { + this.jCloudBitStoreService = new JCloudBitStoreService("filesystem"); + } + + + @Test + public void givenBitStreamIdentifierLongerThanPossibleWhenIntermediatePathIsComputedThenIsSplittedAndTruncated() { + String path = "01234567890123456789"; + String computedPath = this.jCloudBitStoreService.getIntermediatePath(path); + String expectedPath = "01" + File.separator + "23" + File.separator + "45" + File.separator; + assertThat(computedPath, equalTo(expectedPath)); + } + + + @Test + public void givenBitStreamIdentifierShorterThanAFolderLengthWhenIntermediatePathIsComputedThenIsSingleFolder() { + String path = "0"; + String computedPath = this.jCloudBitStoreService.getIntermediatePath(path); + String expectedPath = "0" + File.separator; + assertThat(computedPath, equalTo(expectedPath)); + } + + @Test + public void givenPartialBitStreamIdentifierWhenIntermediatePathIsComputedThenIsCompletlySplitted() { + String path = "01234"; + String computedPath = this.jCloudBitStoreService.getIntermediatePath(path); + String expectedPath = "01" + File.separator + "23" + File.separator + "4" + File.separator; + assertThat(computedPath, equalTo(expectedPath)); + } + + @Test + public void givenMaxLengthBitStreamIdentifierWhenIntermediatePathIsComputedThenIsSplittedAllAsSubfolder() { + String path = "012345"; + String computedPath = this.jCloudBitStoreService.getIntermediatePath(path); + String expectedPath = "01" + File.separator + "23" + File.separator + "45" + File.separator; + assertThat(computedPath, equalTo(expectedPath)); + } + + @Test + public void givenBitStreamIdentifierWhenIntermediatePathIsComputedThenNotEndingDoubleSlash() throws IOException { + StringBuilder path = new StringBuilder("01"); + String computedPath = this.jCloudBitStoreService.getIntermediatePath(path.toString()); + int slashes = computeSlashes(path.toString()); + assertThat(computedPath, Matchers.endsWith(File.separator)); + assertThat(computedPath.split(File.separator).length, Matchers.equalTo(slashes)); + + path.append("2"); + computedPath = this.jCloudBitStoreService.getIntermediatePath(path.toString()); + assertThat(computedPath, Matchers.not(Matchers.endsWith(File.separator + File.separator))); + + path.append("3"); + computedPath = this.jCloudBitStoreService.getIntermediatePath(path.toString()); + assertThat(computedPath, Matchers.not(Matchers.endsWith(File.separator + File.separator))); + + path.append("4"); + computedPath = this.jCloudBitStoreService.getIntermediatePath(path.toString()); + assertThat(computedPath, Matchers.not(Matchers.endsWith(File.separator + File.separator))); + + path.append("56789"); + computedPath = this.jCloudBitStoreService.getIntermediatePath(path.toString()); + assertThat(computedPath, Matchers.not(Matchers.endsWith(File.separator + File.separator))); + } + + @Test + public void givenBitStreamIdentidierWhenIntermediatePathIsComputedThenMustBeSplitted() throws IOException { + StringBuilder path = new StringBuilder("01"); + String computedPath = this.jCloudBitStoreService.getIntermediatePath(path.toString()); + int slashes = computeSlashes(path.toString()); + assertThat(computedPath, Matchers.endsWith(File.separator)); + assertThat(computedPath.split(File.separator).length, Matchers.equalTo(slashes)); + + path.append("2"); + computedPath = this.jCloudBitStoreService.getIntermediatePath(path.toString()); + slashes = computeSlashes(path.toString()); + assertThat(computedPath, Matchers.endsWith(File.separator)); + assertThat(computedPath.split(File.separator).length, Matchers.equalTo(slashes)); + + path.append("3"); + computedPath = this.jCloudBitStoreService.getIntermediatePath(path.toString()); + slashes = computeSlashes(path.toString()); + assertThat(computedPath, Matchers.endsWith(File.separator)); + assertThat(computedPath.split(File.separator).length, Matchers.equalTo(slashes)); + + path.append("4"); + computedPath = this.jCloudBitStoreService.getIntermediatePath(path.toString()); + slashes = computeSlashes(path.toString()); + assertThat(computedPath, Matchers.endsWith(File.separator)); + assertThat(computedPath.split(File.separator).length, Matchers.equalTo(slashes)); + + path.append("56789"); + computedPath = this.jCloudBitStoreService.getIntermediatePath(path.toString()); + slashes = computeSlashes(path.toString()); + assertThat(computedPath, Matchers.endsWith(File.separator)); + assertThat(computedPath.split(File.separator).length, Matchers.equalTo(slashes)); + } + + @Test + public void givenBitStreamIdentifierWithSlashesWhenSanitizedThenSlashesMustBeRemoved() { + String sInternalId = new StringBuilder("01") + .append(File.separator) + .append("22") + .append(File.separator) + .append("33") + .append(File.separator) + .append("4455") + .toString(); + String computedPath = this.jCloudBitStoreService.sanitizeIdentifier(sInternalId); + assertThat(computedPath, Matchers.not(Matchers.startsWith(File.separator))); + assertThat(computedPath, Matchers.not(Matchers.endsWith(File.separator))); + assertThat(computedPath, Matchers.not(Matchers.containsString(File.separator))); + } + + private int computeSlashes(String internalId) { + int minimum = internalId.length(); + int slashesPerLevel = minimum / S3BitStoreService.digitsPerLevel; + int odd = Math.min(1, minimum % S3BitStoreService.digitsPerLevel); + int slashes = slashesPerLevel + odd; + return Math.min(slashes, S3BitStoreService.directoryLevels); + } + +} diff --git a/dspace/config/modules/assetstore.cfg b/dspace/config/modules/assetstore.cfg index 2ceb618beeb0..f918a4354f9a 100644 --- a/dspace/config/modules/assetstore.cfg +++ b/dspace/config/modules/assetstore.cfg @@ -65,5 +65,4 @@ assetstore.s3.generic.useRelativePath = false assetstore.s3.generic.subfolder = assetstore assetstore.s3.endpoint = assetstore.s3.generic.container = -assetstore.s3.endpoint = assetstore.s3.maxCounter = 100 diff --git a/dspace/config/spring/api/bitstore.xml b/dspace/config/spring/api/bitstore.xml index a7ab8ddaa669..967ea1f2ee15 100644 --- a/dspace/config/spring/api/bitstore.xml +++ b/dspace/config/spring/api/bitstore.xml @@ -1,7 +1,8 @@ + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:util="http://www.springframework.org/schema/util" + xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd http://www.springframework.org/schema/util https://www.springframework.org/schema/util/spring-util.xsd" + default-lazy-init="true"> @@ -37,6 +38,9 @@ + + @@ -44,11 +48,17 @@ - + + + + ./local/filesystemstorage + + + diff --git a/pom.xml b/pom.xml index da2ada0e4d17..69422eb11f21 100644 --- a/pom.xml +++ b/pom.xml @@ -1707,6 +1707,12 @@ com.google.api-client google-api-client 1.23.0 + + + com.google.guava + guava-jdk5 + + com.google.http-client From fc9a21c8e9dd7760e6e8031718ab464a2e6a3658 Mon Sep 17 00:00:00 2001 From: Nathan Buckingham Date: Mon, 12 Aug 2024 16:41:00 -0400 Subject: [PATCH 3/3] 116959: JCloudBitStoreTests, replace, remove and get bitstream --- .../bitstore/JCloudBitStoreService.java | 7 +- .../data/dspaceFolder/config/bitstore.xml | 69 ++++++++++++++++++ .../config/modules/assetstore.cfg | 70 +++++++++++++++++++ .../bitstore/JCloudBitStoreServiceTest.java | 68 +++++++++++++++++- dspace/config/modules/assetstore.cfg | 1 + dspace/config/spring/api/bitstore.xml | 10 +-- 6 files changed, 219 insertions(+), 6 deletions(-) create mode 100644 dspace-api/src/test/data/dspaceFolder/config/bitstore.xml create mode 100644 dspace-api/src/test/data/dspaceFolder/config/modules/assetstore.cfg diff --git a/dspace-api/src/main/java/org/dspace/storage/bitstore/JCloudBitStoreService.java b/dspace-api/src/main/java/org/dspace/storage/bitstore/JCloudBitStoreService.java index 71185cf5c32b..098328e64fd3 100644 --- a/dspace-api/src/main/java/org/dspace/storage/bitstore/JCloudBitStoreService.java +++ b/dspace-api/src/main/java/org/dspace/storage/bitstore/JCloudBitStoreService.java @@ -72,6 +72,11 @@ public JCloudBitStoreService(String providerOrApi) { this.providerOrApi = providerOrApi; } + protected JCloudBitStoreService(BlobStoreContext blobStoreContext, String providerOrApi) { + this.blobStoreContext = blobStoreContext; + this.providerOrApi = providerOrApi; + } + public void setUseRelativePath(boolean useRelativePath) { this.useRelativePath = useRelativePath; } @@ -359,7 +364,7 @@ public URI getStoredURI(String id) { private String getContainer() { if (container == null) { - container = new DSpace().getConfigurationService().getProperty("dspace.hostname"); + container = new DSpace().getConfigurationService().getProperty("dspace.name"); } return container; } diff --git a/dspace-api/src/test/data/dspaceFolder/config/bitstore.xml b/dspace-api/src/test/data/dspaceFolder/config/bitstore.xml new file mode 100644 index 000000000000..58e19e6ecd12 --- /dev/null +++ b/dspace-api/src/test/data/dspaceFolder/config/bitstore.xml @@ -0,0 +1,69 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ./local/filesystemstorage + + + + + + + + + + + diff --git a/dspace-api/src/test/data/dspaceFolder/config/modules/assetstore.cfg b/dspace-api/src/test/data/dspaceFolder/config/modules/assetstore.cfg new file mode 100644 index 000000000000..5743b225b91f --- /dev/null +++ b/dspace-api/src/test/data/dspaceFolder/config/modules/assetstore.cfg @@ -0,0 +1,70 @@ +#---------------------------------------------------------------# +#-----------------STORAGE CONFIGURATIONS------------------------# +#---------------------------------------------------------------# +# Configuration properties used by the bitstore.xml config file # +# # +#---------------------------------------------------------------# + +# assetstore.dir, look at DSPACE/config/spring/api/bitstore.xml for more options +assetstore.dir = ${dspace.dir}/assetstore + +# Configures the primary store to be local or S3. +# This value will be used as `incoming` default store inside the `bitstore.xml` +# Possible values are: +# - 0: to use the `localStore`; +# - 1: to use the `s3Store`. +# If you want to add additional assetstores, they must be added to that bitstore.xml +# and new values should be provided as key-value pairs in the `stores` map of the +# `bitstore.xml` configuration. +assetstore.index.primary = 0 + +#---------------------------------------------------------------# +#-------------- Amazon S3 Specific Configurations --------------# +#---------------------------------------------------------------# +# The below configurations are only used if the primary storename +# is set to 's3Store' or the 's3Store' is configured as a secondary store +# in your bitstore.xml + +# Enables or disables the store initialization during startup, without initialization the store won't work. +# if changed to true, a lazy initialization will be tried on next store usage, be careful an excecption could be thrown +assetstore.s3.enabled = false + +# For using a relative path (xx/xx/xx/xxx...) set to true, default it false +# When true: it splits the path into subfolders, each of these +# are 2-chars (2-bytes) length, the last is the filename and could have +# at max 3-chars (3-bytes). +# When false: is used the absolute path using full filename. +assetstore.s3.useRelativePath = false + +# S3 bucket name to store assets in. If unspecified, by default DSpace will +# create a bucket based on the hostname of `dspace.ui.url` setting. +assetstore.s3.bucketName = + +# Subfolder to organize assets within the bucket, in case this bucket +# is shared. Optional, default is root level of bucket +assetstore.s3.subfolder = + +# please don't use root credentials in production but rely on the aws credentials default +# discovery mechanism to configure them (ENV VAR, EC2 Iam role, etc.) +# The preferred approach for security reason is to use the IAM user credentials, but isn't always possible. +# More information about credentials here: https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html +# More information about IAM usage here: https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/java-dg-roles.html +assetstore.s3.awsAccessKey = +assetstore.s3.awsSecretKey = + +# If the credentials are left empty, +# then this setting is ignored and the default AWS region will be used. +assetstore.s3.awsRegionName = + + +### JCloudSettings + +# Enabled the JCloudStore +assetstore.s3.generic.enabled = false +assetstore.s3.generic.useRelativePath = true +assetstore.s3.generic.subfolder = assetstore +assetstore.s3.endpoint = +assetstore.s3.generic.provider = filesystem + +assetstore.s3.generic.awsAccessKey = +assetstore.s3.generic.awsSecretKey = diff --git a/dspace-api/src/test/java/org/dspace/storage/bitstore/JCloudBitStoreServiceTest.java b/dspace-api/src/test/java/org/dspace/storage/bitstore/JCloudBitStoreServiceTest.java index 097aa71432c4..1ada1e20d3e1 100644 --- a/dspace-api/src/test/java/org/dspace/storage/bitstore/JCloudBitStoreServiceTest.java +++ b/dspace-api/src/test/java/org/dspace/storage/bitstore/JCloudBitStoreServiceTest.java @@ -9,16 +9,29 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.io.File; import java.io.IOException; +import java.io.InputStream; +import com.google.common.io.ByteSource; import org.dspace.AbstractUnitTest; import org.dspace.content.Bitstream; import org.hamcrest.Matchers; +import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.BlobStoreContext; +import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.domain.BlobBuilder; +import org.jclouds.blobstore.domain.BlobBuilder.PayloadBlobBuilder; +import org.jclouds.io.Payload; import org.junit.Before; import org.junit.Test; +import org.mockito.ArgumentMatchers; import org.mockito.Mock; +import org.mockito.Mockito; /** * @author Nathan Buckingham @@ -29,12 +42,65 @@ public class JCloudBitStoreServiceTest extends AbstractUnitTest { private JCloudBitStoreService jCloudBitStoreService; + @Mock + private BlobStoreContext blobStoreContext; + + @Mock + private BlobStore blobStore; + @Mock private Bitstream bitstream; @Before public void setUp() throws Exception { - this.jCloudBitStoreService = new JCloudBitStoreService("filesystem"); + this.jCloudBitStoreService = new JCloudBitStoreService(blobStoreContext, "filesystem"); + } + + @Test + public void getBitstreamTest() throws Exception { + Blob blob = Mockito.mock(Blob.class); + Payload payload = Mockito.mock(Payload.class); + InputStream inputStream = Mockito.mock(InputStream.class); + when(blob.getPayload()).thenReturn(payload); + when(payload.openStream()).thenReturn(inputStream); + when(blobStoreContext.getBlobStore()).thenReturn(blobStore); + when(blobStore.getBlob(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(blob); + when(blobStore.blobExists(ArgumentMatchers.any(), any())).thenReturn(true); + assertThat(this.jCloudBitStoreService.get(bitstream), Matchers.equalTo(inputStream)); + } + + @Test + public void removeBitstreamTest() throws Exception { + String bitStreamId = "BitStreamId"; + when(bitstream.getInternalId()).thenReturn(bitStreamId); + when(blobStoreContext.getBlobStore()).thenReturn(blobStore); + try { + this.jCloudBitStoreService.remove(bitstream); + } catch (Exception e) { + // will fail due to trying to remove files + } + verify(this.blobStore, Mockito.times(1)).removeBlob(ArgumentMatchers.any(), ArgumentMatchers.any()); + } + + @Test + public void replaceBitStreamTest() throws Exception { + Blob blob = Mockito.mock(Blob.class); + File file = Mockito.mock(File.class); + BlobBuilder blobBuilder = Mockito.mock(BlobBuilder.class); + PayloadBlobBuilder payloadBlobBuilder = Mockito.mock(PayloadBlobBuilder.class); + + when(blobStoreContext.getBlobStore()).thenReturn(blobStore); + when(blobStore.blobBuilder(ArgumentMatchers.any())).thenReturn(blobBuilder); + when(blobBuilder.payload(ArgumentMatchers.any(ByteSource.class))).thenReturn(payloadBlobBuilder); + when(payloadBlobBuilder.contentDisposition(ArgumentMatchers.any())).thenReturn(payloadBlobBuilder); + when(payloadBlobBuilder.contentLength(ArgumentMatchers.any(long.class))).thenReturn(payloadBlobBuilder); + when(payloadBlobBuilder.contentType(ArgumentMatchers.any(String.class))).thenReturn(payloadBlobBuilder); + when(payloadBlobBuilder.build()).thenReturn(blob); + ByteSource byteSource = Mockito.mock(ByteSource.class); + String mockedTag = "1a7771d5fdd7bfdfc84033c70b1ba555"; + this.jCloudBitStoreService.put(byteSource, bitstream); + verify(blobStore, Mockito.times(1)).putBlob(ArgumentMatchers.any(), + ArgumentMatchers.any(), ArgumentMatchers.any()); } diff --git a/dspace/config/modules/assetstore.cfg b/dspace/config/modules/assetstore.cfg index f918a4354f9a..2c91788e82eb 100644 --- a/dspace/config/modules/assetstore.cfg +++ b/dspace/config/modules/assetstore.cfg @@ -66,3 +66,4 @@ assetstore.s3.generic.subfolder = assetstore assetstore.s3.endpoint = assetstore.s3.generic.container = assetstore.s3.maxCounter = 100 +assetstore.s3.generic.provider = aws-s3 diff --git a/dspace/config/spring/api/bitstore.xml b/dspace/config/spring/api/bitstore.xml index 967ea1f2ee15..43247772375a 100644 --- a/dspace/config/spring/api/bitstore.xml +++ b/dspace/config/spring/api/bitstore.xml @@ -10,7 +10,7 @@ - + @@ -42,13 +42,14 @@ id="propertyBaseDir"/> + - + Subfolder to organize assets within the bucket, in case this bucket is shared + Optional, default is root level of bucket + -->