Skip to content

Commit

Permalink
Merge branch 'feature-jcloud-store-7.4' into feature-jcloud-store-9.0
Browse files Browse the repository at this point in the history
  • Loading branch information
Nathan Buckingham committed Sep 5, 2024
2 parents 8156c8d + ab32f4d commit bf4718d
Show file tree
Hide file tree
Showing 9 changed files with 192 additions and 132 deletions.
16 changes: 15 additions & 1 deletion dspace-api/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -857,15 +857,29 @@
</exclusions>
</dependency>

<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
</dependency>

<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</dependency>

<dependency>
<groupId>org.apache.jclouds</groupId>
<artifactId>jclouds-core</artifactId>
<version>2.5.0</version>
<exclusions>
<exclusion>
<exclusion>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.xml.bind</groupId>
<artifactId>jaxb-impl</artifactId>
Expand Down
105 changes: 64 additions & 41 deletions dspace-api/src/main/java/org/dspace/storage/bitstore/JCloudBitStoreService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.sql.SQLException;
import java.util.Map;
import java.util.Properties;
Expand All @@ -19,6 +18,7 @@
import com.google.common.io.ByteSource;
import com.google.common.io.Files;
import com.google.common.net.MediaType;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
Expand All @@ -28,7 +28,6 @@
import org.dspace.core.Utils;
import org.dspace.storage.bitstore.factory.StorageServiceFactory;
import org.dspace.storage.bitstore.service.BitstreamStorageService;
import org.dspace.utils.DSpace;
import org.jclouds.ContextBuilder;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.BlobStoreContext;
Expand All @@ -52,8 +51,17 @@ public class JCloudBitStoreService extends BaseBitStoreService {
private String providerOrApi;
private ContextBuilder builder;
private BlobStoreContext blobStoreContext;

/**
* container for all the assets
*/
private String container;
private String subFolder;

/**
* (Optional) subfolder within bucket where objects are stored
*/
private String subfolder = null;

private String identity;
private String credential;
private String endpoint;
Expand Down Expand Up @@ -89,8 +97,12 @@ public void setContainer(String container) {
this.container = container;
}

public void setSubFolder(String subFolder) {
this.subFolder = subFolder;
public String getSubfolder() {
return subfolder;
}

public void setSubfolder(String subfolder) {
this.subfolder = subfolder;
}

public void setIdentity(String identity) {
Expand Down Expand Up @@ -132,10 +144,16 @@ public void init() throws IOException {
if (endpoint != null) {
this.builder = this.builder.endpoint(endpoint);
}
blobStoreContext = this.builder.overrides(properties)
.credentials(identity, credential).buildView(BlobStoreContext.class);
if (properties != null && !properties.isEmpty()) {
this.builder = this.builder.overrides(properties);
}
if (identity != null && credential != null) {
this.builder = this.builder.credentials(identity, credential);
}
blobStoreContext = this.builder.buildView(BlobStoreContext.class);
this.initialized = true;
} catch (Exception e) {
log.error(e.getMessage(),e);
this.initialized = false;
}
}
Expand All @@ -146,8 +164,7 @@ private synchronized void refreshContextIfNeeded() {
if (counter == maxCounter) {
counter = 0;
blobStoreContext.close();
blobStoreContext = this.builder.overrides(properties)
.credentials(identity, credential).buildView(BlobStoreContext.class);
blobStoreContext = this.builder.buildView(BlobStoreContext.class);
}
}

Expand Down Expand Up @@ -188,8 +205,8 @@ public void remove(Bitstream bitstream) throws IOException {
*/
public String getFullKey(String id) {
StringBuilder bufFilename = new StringBuilder();
if (StringUtils.isNotEmpty(this.subFolder)) {
bufFilename.append(this.subFolder);
if (StringUtils.isNotEmpty(this.subfolder)) {
bufFilename.append(this.subfolder);
appendSeparator(bufFilename);
}

Expand All @@ -200,7 +217,7 @@ public String getFullKey(String id) {
}

if (log.isDebugEnabled()) {
log.debug("S3 filepath for " + id + " is "
log.debug("Container filepath for " + id + " is "
+ bufFilename.toString());
}

Expand Down Expand Up @@ -253,7 +270,7 @@ private void deleteParents(File file) {

public void put(ByteSource byteSource, Bitstream bitstream) throws IOException {

final File file = getFile(bitstream);
String key = getFullKey(bitstream.getInternalId());

/* set type to sane default */
String type = MediaType.OCTET_STREAM.toString();
Expand All @@ -270,9 +287,9 @@ public void put(ByteSource byteSource, Bitstream bitstream) throws IOException
blobStore.createContainerInLocation(null, container);
}

Blob blob = blobStore.blobBuilder(file.toString())
Blob blob = blobStore.blobBuilder(key)
.payload(byteSource)
.contentDisposition(file.toString())
.contentDisposition(key)
.contentLength(byteSource.size())
.contentType(type)
.build();
Expand All @@ -281,18 +298,42 @@ public void put(ByteSource byteSource, Bitstream bitstream) throws IOException
blobStore.putBlob(container, blob, Builder.multipart());
}

/**
* Store a stream of bits.
*
* <p>
* If this method returns successfully, the bits have been stored.
* If an exception is thrown, the bits have not been stored.
* </p>
*
* @param in The stream of bits to store
* @throws java.io.IOException If a problem occurs while storing the bits
*/
@Override
public void put(Bitstream bitstream, InputStream in) throws IOException {
File tmp = File.createTempFile("jclouds", "cache");
String key = getFullKey(bitstream.getInternalId());
//Copy istream to temp file, and send the file, with some metadata
File scratchFile = File.createTempFile(bitstream.getInternalId(), "s3bs");
try {
// Inefficient caching strategy, however allows for use of JClouds store directly without CachingStore.
// Make sure there is sufficient storage in temp directory.
Files.asByteSink(tmp).writeFrom(in);
in.close();
put(Files.asByteSource(tmp), bitstream);

FileUtils.copyInputStreamToFile(in, scratchFile);
long contentLength = scratchFile.length();
// The ETag may or may not be and MD5 digest of the object data.
// Therefore, we precalculate before uploading
String localChecksum = org.dspace.curate.Utils.checksum(scratchFile, CSA);

put(Files.asByteSource(scratchFile), bitstream);

bitstream.setSizeBytes(contentLength);
bitstream.setChecksum(localChecksum);
bitstream.setChecksumAlgorithm(CSA);

} catch (Exception e) {
log.error("put(" + bitstream.getInternalId() + ", is)", e);
throw new IOException(e);
} finally {
if (!tmp.delete()) {
tmp.deleteOnExit();
if (!scratchFile.delete()) {
scratchFile.deleteOnExit();
}
}
}
Expand Down Expand Up @@ -347,25 +388,7 @@ public File getFile(Bitstream bitstream) throws IOException {
return new File(id);
}

/**
* Gets the URI of the content within the store.
*
* @param id the bitstream internal id.
* @return the URI, which is a relative path to the content.
*/
@SuppressWarnings("unused") // used by AVS2
public URI getStoredURI(String id) {
String tempID = getFullKey(id);
if (log.isDebugEnabled()) {
log.debug("Local URI for " + id + " is " + tempID);
}
return URI.create(id);
}

private String getContainer() {
if (container == null) {
container = new DSpace().getConfigurationService().getProperty("dspace.name");
}
return container;
}
}
14 changes: 13 additions & 1 deletion dspace-api/src/test/data/dspaceFolder/config/local.cfg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,4 +191,16 @@ ldn.notify.inbox.block-untrusted-ip = true
# ERROR LOGGING #
###########################################
# Log full stacktrace of other common 4xx errors (for easier debugging of these errors in tests)
logging.server.include-stacktrace-for-httpcode = 422, 400
logging.server.include-stacktrace-for-httpcode = 422, 400

### JCloudSettings

# Enabled the JCloudStore
assetstore.s3.generic.enabled = true
assetstore.s3.generic.useRelativePath = false
assetstore.s3.generic.subfolder = assetstore
assetstore.s3.generic.provider = filesystem
assetstore.s3.generic.container = assetstore-jclouds-container
assetstore.s3.generic.awsAccessKey =
assetstore.s3.generic.awsSecretKey =

70 changes: 0 additions & 70 deletions dspace-api/src/test/data/dspaceFolder/config/modules/assetstore.cfg
View file

This file was deleted.

16 changes: 6 additions & 10 deletions ...est/data/dspaceFolder/config/bitstore.xml → ...paceFolder/config/spring/api/bitstore.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,23 +45,19 @@
<bean name="jcloudStore" class="org.dspace.storage.bitstore.JCloudBitStoreService" scope="singleton" lazy-init="true">
<property name="enabled" value="${assetstore.s3.generic.enabled:false}"/>
<property name="useRelativePath" value="${assetstore.s3.generic.useRelativePath:false}"/>
<property name="providerOrApi" value="${assetstore.s3.generic.provider}"/>
<property name="container" value="${assetstore.s3.generic.container}"/>
<property name="endpoint" value="${assetstore.s3.endpoint}"/>

<property name="providerOrApi" value="aws-s3"/>
<property name="identity" value="${assetstore.s3.generic.awsAccessKey}"/>
<property name="credentials" value="${assetstore.s3.generic.awsSecretKey}"/>
<property name="maxCounter" value="${assetstore.s3.maxCounter:100}"/>

<property name="overrides">
<props>
<prop key="#{propertyBaseDir}">./local/filesystemstorage</prop>
<prop key="#{propertyBaseDir}">target/testing/dspace</prop>
</props>
</property>

<!-- Subfolder to organize assets within the bucket, in case this bucket is shared -->
<!--
Subfolder to organize assets within the bucket, in case this bucket is shared -->
<!-- Optional, default is root level of bucket -->
<property name="subFolder" value="${assetstore.s3.generic.subfolder}"/>
<property name="subfolder" value="${assetstore.s3.generic.subfolder}"/>

</bean>

<!-- <bean name="localStore2 ... -->
Expand Down
Loading

0 comments on commit bf4718d

Please sign in to comment.