com.sun.xml.bind
jaxb-impl
diff --git a/dspace-api/src/main/java/org/dspace/storage/bitstore/JCloudBitStoreService.java b/dspace-api/src/main/java/org/dspace/storage/bitstore/JCloudBitStoreService.java
index 098328e64fd3..773649e75cb0 100644
--- a/dspace-api/src/main/java/org/dspace/storage/bitstore/JCloudBitStoreService.java
+++ b/dspace-api/src/main/java/org/dspace/storage/bitstore/JCloudBitStoreService.java
@@ -10,7 +10,6 @@
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
-import java.net.URI;
import java.sql.SQLException;
import java.util.Map;
import java.util.Properties;
@@ -19,6 +18,7 @@
import com.google.common.io.ByteSource;
import com.google.common.io.Files;
import com.google.common.net.MediaType;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -28,7 +28,6 @@
import org.dspace.core.Utils;
import org.dspace.storage.bitstore.factory.StorageServiceFactory;
import org.dspace.storage.bitstore.service.BitstreamStorageService;
-import org.dspace.utils.DSpace;
import org.jclouds.ContextBuilder;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.BlobStoreContext;
@@ -52,8 +51,17 @@ public class JCloudBitStoreService extends BaseBitStoreService {
private String providerOrApi;
private ContextBuilder builder;
private BlobStoreContext blobStoreContext;
+
+ /**
+ * container for all the assets
+ */
private String container;
- private String subFolder;
+
+ /**
+ * (Optional) subfolder within bucket where objects are stored
+ */
+ private String subfolder = null;
+
private String identity;
private String credential;
private String endpoint;
@@ -89,8 +97,12 @@ public void setContainer(String container) {
this.container = container;
}
- public void setSubFolder(String subFolder) {
- this.subFolder = subFolder;
+ public String getSubfolder() {
+ return subfolder;
+ }
+
+ public void setSubfolder(String subfolder) {
+ this.subfolder = subfolder;
}
public void setIdentity(String identity) {
@@ -132,10 +144,16 @@ public void init() throws IOException {
if (endpoint != null) {
this.builder = this.builder.endpoint(endpoint);
}
- blobStoreContext = this.builder.overrides(properties)
- .credentials(identity, credential).buildView(BlobStoreContext.class);
+ if (properties != null && !properties.isEmpty()) {
+ this.builder = this.builder.overrides(properties);
+ }
+ if (identity != null && credential != null) {
+ this.builder = this.builder.credentials(identity, credential);
+ }
+ blobStoreContext = this.builder.buildView(BlobStoreContext.class);
this.initialized = true;
} catch (Exception e) {
+ log.error(e.getMessage(),e);
this.initialized = false;
}
}
@@ -146,8 +164,7 @@ private synchronized void refreshContextIfNeeded() {
if (counter == maxCounter) {
counter = 0;
blobStoreContext.close();
- blobStoreContext = this.builder.overrides(properties)
- .credentials(identity, credential).buildView(BlobStoreContext.class);
+ blobStoreContext = this.builder.buildView(BlobStoreContext.class);
}
}
@@ -188,8 +205,8 @@ public void remove(Bitstream bitstream) throws IOException {
*/
public String getFullKey(String id) {
StringBuilder bufFilename = new StringBuilder();
- if (StringUtils.isNotEmpty(this.subFolder)) {
- bufFilename.append(this.subFolder);
+ if (StringUtils.isNotEmpty(this.subfolder)) {
+ bufFilename.append(this.subfolder);
appendSeparator(bufFilename);
}
@@ -200,7 +217,7 @@ public String getFullKey(String id) {
}
if (log.isDebugEnabled()) {
- log.debug("S3 filepath for " + id + " is "
+ log.debug("Container filepath for " + id + " is "
+ bufFilename.toString());
}
@@ -253,7 +270,7 @@ private void deleteParents(File file) {
public void put(ByteSource byteSource, Bitstream bitstream) throws IOException {
- final File file = getFile(bitstream);
+ String key = getFullKey(bitstream.getInternalId());
/* set type to sane default */
String type = MediaType.OCTET_STREAM.toString();
@@ -270,9 +287,9 @@ public void put(ByteSource byteSource, Bitstream bitstream) throws IOException
blobStore.createContainerInLocation(null, container);
}
- Blob blob = blobStore.blobBuilder(file.toString())
+ Blob blob = blobStore.blobBuilder(key)
.payload(byteSource)
- .contentDisposition(file.toString())
+ .contentDisposition(key)
.contentLength(byteSource.size())
.contentType(type)
.build();
@@ -281,18 +298,42 @@ public void put(ByteSource byteSource, Bitstream bitstream) throws IOException
blobStore.putBlob(container, blob, Builder.multipart());
}
+ /**
+ * Store a stream of bits.
+ *
+ *
+ * If this method returns successfully, the bits have been stored.
+ * If an exception is thrown, the bits have not been stored.
+ *
+ *
+ * @param in The stream of bits to store
+ * @throws java.io.IOException If a problem occurs while storing the bits
+ */
@Override
public void put(Bitstream bitstream, InputStream in) throws IOException {
- File tmp = File.createTempFile("jclouds", "cache");
+ String key = getFullKey(bitstream.getInternalId());
+ //Copy istream to temp file, and send the file, with some metadata
+ File scratchFile = File.createTempFile(bitstream.getInternalId(), "s3bs");
try {
- // Inefficient caching strategy, however allows for use of JClouds store directly without CachingStore.
- // Make sure there is sufficient storage in temp directory.
- Files.asByteSink(tmp).writeFrom(in);
- in.close();
- put(Files.asByteSource(tmp), bitstream);
+
+ FileUtils.copyInputStreamToFile(in, scratchFile);
+ long contentLength = scratchFile.length();
+ // The ETag may or may not be and MD5 digest of the object data.
+ // Therefore, we precalculate before uploading
+ String localChecksum = org.dspace.curate.Utils.checksum(scratchFile, CSA);
+
+ put(Files.asByteSource(scratchFile), bitstream);
+
+ bitstream.setSizeBytes(contentLength);
+ bitstream.setChecksum(localChecksum);
+ bitstream.setChecksumAlgorithm(CSA);
+
+ } catch (Exception e) {
+ log.error("put(" + bitstream.getInternalId() + ", is)", e);
+ throw new IOException(e);
} finally {
- if (!tmp.delete()) {
- tmp.deleteOnExit();
+ if (!scratchFile.delete()) {
+ scratchFile.deleteOnExit();
}
}
}
@@ -347,25 +388,7 @@ public File getFile(Bitstream bitstream) throws IOException {
return new File(id);
}
- /**
- * Gets the URI of the content within the store.
- *
- * @param id the bitstream internal id.
- * @return the URI, which is a relative path to the content.
- */
- @SuppressWarnings("unused") // used by AVS2
- public URI getStoredURI(String id) {
- String tempID = getFullKey(id);
- if (log.isDebugEnabled()) {
- log.debug("Local URI for " + id + " is " + tempID);
- }
- return URI.create(id);
- }
-
private String getContainer() {
- if (container == null) {
- container = new DSpace().getConfigurationService().getProperty("dspace.name");
- }
return container;
}
}
diff --git a/dspace-api/src/test/data/dspaceFolder/config/local.cfg b/dspace-api/src/test/data/dspaceFolder/config/local.cfg
index b44f319a35f6..5dc967b75dea 100644
--- a/dspace-api/src/test/data/dspaceFolder/config/local.cfg
+++ b/dspace-api/src/test/data/dspaceFolder/config/local.cfg
@@ -191,4 +191,16 @@ ldn.notify.inbox.block-untrusted-ip = true
# ERROR LOGGING #
###########################################
# Log full stacktrace of other common 4xx errors (for easier debugging of these errors in tests)
-logging.server.include-stacktrace-for-httpcode = 422, 400
\ No newline at end of file
+logging.server.include-stacktrace-for-httpcode = 422, 400
+
+### JCloudSettings
+
+# Enabled the JCloudStore
+assetstore.s3.generic.enabled = true
+assetstore.s3.generic.useRelativePath = false
+assetstore.s3.generic.subfolder = assetstore
+assetstore.s3.generic.provider = filesystem
+assetstore.s3.generic.container = assetstore-jclouds-container
+assetstore.s3.generic.awsAccessKey =
+assetstore.s3.generic.awsSecretKey =
+
diff --git a/dspace-api/src/test/data/dspaceFolder/config/modules/assetstore.cfg b/dspace-api/src/test/data/dspaceFolder/config/modules/assetstore.cfg
deleted file mode 100644
index 5743b225b91f..000000000000
--- a/dspace-api/src/test/data/dspaceFolder/config/modules/assetstore.cfg
+++ /dev/null
@@ -1,70 +0,0 @@
-#---------------------------------------------------------------#
-#-----------------STORAGE CONFIGURATIONS------------------------#
-#---------------------------------------------------------------#
-# Configuration properties used by the bitstore.xml config file #
-# #
-#---------------------------------------------------------------#
-
-# assetstore.dir, look at DSPACE/config/spring/api/bitstore.xml for more options
-assetstore.dir = ${dspace.dir}/assetstore
-
-# Configures the primary store to be local or S3.
-# This value will be used as `incoming` default store inside the `bitstore.xml`
-# Possible values are:
-# - 0: to use the `localStore`;
-# - 1: to use the `s3Store`.
-# If you want to add additional assetstores, they must be added to that bitstore.xml
-# and new values should be provided as key-value pairs in the `stores` map of the
-# `bitstore.xml` configuration.
-assetstore.index.primary = 0
-
-#---------------------------------------------------------------#
-#-------------- Amazon S3 Specific Configurations --------------#
-#---------------------------------------------------------------#
-# The below configurations are only used if the primary storename
-# is set to 's3Store' or the 's3Store' is configured as a secondary store
-# in your bitstore.xml
-
-# Enables or disables the store initialization during startup, without initialization the store won't work.
-# if changed to true, a lazy initialization will be tried on next store usage, be careful an excecption could be thrown
-assetstore.s3.enabled = false
-
-# For using a relative path (xx/xx/xx/xxx...) set to true, default it false
-# When true: it splits the path into subfolders, each of these
-# are 2-chars (2-bytes) length, the last is the filename and could have
-# at max 3-chars (3-bytes).
-# When false: is used the absolute path using full filename.
-assetstore.s3.useRelativePath = false
-
-# S3 bucket name to store assets in. If unspecified, by default DSpace will
-# create a bucket based on the hostname of `dspace.ui.url` setting.
-assetstore.s3.bucketName =
-
-# Subfolder to organize assets within the bucket, in case this bucket
-# is shared. Optional, default is root level of bucket
-assetstore.s3.subfolder =
-
-# please don't use root credentials in production but rely on the aws credentials default
-# discovery mechanism to configure them (ENV VAR, EC2 Iam role, etc.)
-# The preferred approach for security reason is to use the IAM user credentials, but isn't always possible.
-# More information about credentials here: https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html
-# More information about IAM usage here: https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/java-dg-roles.html
-assetstore.s3.awsAccessKey =
-assetstore.s3.awsSecretKey =
-
-# If the credentials are left empty,
-# then this setting is ignored and the default AWS region will be used.
-assetstore.s3.awsRegionName =
-
-
-### JCloudSettings
-
-# Enabled the JCloudStore
-assetstore.s3.generic.enabled = false
-assetstore.s3.generic.useRelativePath = true
-assetstore.s3.generic.subfolder = assetstore
-assetstore.s3.endpoint =
-assetstore.s3.generic.provider = filesystem
-
-assetstore.s3.generic.awsAccessKey =
-assetstore.s3.generic.awsSecretKey =
diff --git a/dspace-api/src/test/data/dspaceFolder/config/bitstore.xml b/dspace-api/src/test/data/dspaceFolder/config/spring/api/bitstore.xml
similarity index 85%
rename from dspace-api/src/test/data/dspaceFolder/config/bitstore.xml
rename to dspace-api/src/test/data/dspaceFolder/config/spring/api/bitstore.xml
index 58e19e6ecd12..d5ba46b592b4 100644
--- a/dspace-api/src/test/data/dspaceFolder/config/bitstore.xml
+++ b/dspace-api/src/test/data/dspaceFolder/config/spring/api/bitstore.xml
@@ -45,23 +45,19 @@
+
-
-
-
-
-
-
- ./local/filesystemstorage
+ target/testing/dspace
-
-
+
-
+
+
diff --git a/dspace-api/src/test/java/org/dspace/content/BitstreamJCloudBitstoreTest.java b/dspace-api/src/test/java/org/dspace/content/BitstreamJCloudBitstoreTest.java
new file mode 100644
index 000000000000..ec8688b3bea2
--- /dev/null
+++ b/dspace-api/src/test/java/org/dspace/content/BitstreamJCloudBitstoreTest.java
@@ -0,0 +1,73 @@
+/**
+ * The contents of this file are subject to the license and copyright
+ * detailed in the LICENSE and NOTICE files at the root of the source
+ * tree and available online at
+ *
+ * http://www.dspace.org/license/
+ */
+package org.dspace.content;
+
+import static org.junit.Assert.assertTrue;
+
+import org.dspace.content.factory.ContentServiceFactory;
+import org.dspace.content.service.BitstreamFormatService;
+import org.dspace.services.ConfigurationService;
+import org.dspace.services.factory.DSpaceServicesFactory;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Unit Tests for class Bitstream
+ *
+ * @author Mark Diggory
+ */
+public class BitstreamJCloudBitstoreTest extends BitstreamTest {
+
+ protected BitstreamFormatService bitstreamFormatService = ContentServiceFactory.getInstance()
+ .getBitstreamFormatService();
+
+ private final ConfigurationService configurationService
+ = DSpaceServicesFactory.getInstance().getConfigurationService();
+
+
+ /**
+ * This method will be run before every test as per @Before. It will
+ * initialize resources required for the tests.
+ *
+ * Other methods can be annotated with @Before here or in subclasses
+ * but no execution order is guaranteed
+ */
+ @Before
+ @Override
+ public void init() {
+ configurationService.setProperty("assetstore.index.primary", "2");
+ super.init();
+ }
+
+ /**
+ * Test of getStoreNumber method, of class Bitstream.
+ */
+ @Test
+ @Override
+ public void testGetStoreNumber() {
+ //stored in store 2 by default
+ assertTrue("testGetStoreNumber 2", bs.getStoreNumber() == 2);
+ }
+
+ /**
+ * This method will be run after every test as per @After. It will
+ * clean resources initialized by the @Before methods.
+ *
+ * Other methods can be annotated with @After here or in subclasses
+ * but no execution order is guaranteed
+ */
+ @After
+ @Override
+ public void destroy() {
+ configurationService.setProperty("assetstore.index.primary", "0");
+ super.destroy();
+ }
+
+
+}
diff --git a/dspace-api/src/test/java/org/dspace/content/BitstreamTest.java b/dspace-api/src/test/java/org/dspace/content/BitstreamTest.java
index e85a0fc7b78d..865af32ec014 100644
--- a/dspace-api/src/test/java/org/dspace/content/BitstreamTest.java
+++ b/dspace-api/src/test/java/org/dspace/content/BitstreamTest.java
@@ -58,7 +58,7 @@ public class BitstreamTest extends AbstractDSpaceObjectTest {
/**
* BitStream instance for the tests
*/
- private Bitstream bs;
+ protected Bitstream bs;
/**
* Spy of AuthorizeService to use for tests
diff --git a/dspace/config/spring/api/bitstore.xml b/dspace/config/spring/api/bitstore.xml
index 43247772375a..5e05174129f9 100644
--- a/dspace/config/spring/api/bitstore.xml
+++ b/dspace/config/spring/api/bitstore.xml
@@ -38,11 +38,11 @@
+
+
-
+