MCR-3126 add s3 support

-use own RepositoryBuilder -add S3RepositoryProvider -improve directory history method by using inventory API
MyCoRe-Org · Nov 20, 2024 · 8fe9e47 · 8fe9e47
1 parent 323e38f
commit 8fe9e47
Show file tree

Hide file tree

Showing 10 changed files with 850 additions and 44 deletions.
diff --git a/mycore-ocfl/pom.xml b/mycore-ocfl/pom.xml
@@ -47,10 +47,20 @@
       <groupId>io.ocfl</groupId>
       <artifactId>ocfl-java-api</artifactId>
     </dependency>
+    <dependency>
+      <groupId>io.ocfl</groupId>
+      <artifactId>ocfl-java-aws</artifactId>
+    </dependency>
     <dependency>
       <groupId>io.ocfl</groupId>
       <artifactId>ocfl-java-core</artifactId>
     </dependency>
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>netty-nio-client</artifactId>
+      <!-- use same version as ocfl-java -->
+      <version>2.26.21</version>
+    </dependency>
     <dependency>
       <groupId>jakarta.inject</groupId>
       <artifactId>jakarta.inject-api</artifactId>

diff --git a/mycore-ocfl/src/main/java/org/mycore/ocfl/metadata/MCROCFLXMLMetadataManager.java b/mycore-ocfl/src/main/java/org/mycore/ocfl/metadata/MCROCFLXMLMetadataManager.java
@@ -233,7 +233,7 @@ public MCRContent retrieveContent(MCRObjectID mcrid) throws IOException {
     protected InputStream getStoredContentStream(MCRObjectID mcrid, OcflObjectVersion storeObject) throws IOException {
         String path = buildFilePath(mcrid);
         OcflObjectVersionFile file = storeObject.getFile(path);
-        if(file == null) {
+        if (file == null) {
             throw new IOException("Couldn't find path '" + path + "' in '" + storeObject.getObjectId() + "'.");
         }
         return file.getStream();
@@ -335,9 +335,10 @@ public int getHighestStoredID(String project, String type) {
         int maxDepth = Integer.MAX_VALUE;
 
         MCROCFLRepositoryProvider ocflRepoProvider = MCROCFLRepositoryProvider.getProvider(repositoryKey);
-        if(!(ocflRepoProvider instanceof MCROCFLLocalRepositoryProvider localRepositoryProvider)) {
-            throw new MCROCFLException("Cannot call getHighestStoredID() on non local repository."
-                + " Use MCRFileBaseCacheObjectIDGenerator instead!");
+        if (!(ocflRepoProvider instanceof MCROCFLLocalRepositoryProvider localRepositoryProvider)) {
+            throw new MCROCFLException("Cannot call getHighestStoredID() on non local repository. Set "
+                + "MCR.Metadata.ObjectID.Generator.Class=org.mycore.datamodel.common.MCRFileBaseCacheObjectIDGenerator "
+                + "instead!");
         }
 
         OcflExtensionConfig config = localRepositoryProvider.getExtensionConfig();

diff --git a/mycore-ocfl/src/main/java/org/mycore/ocfl/repository/MCROCFLLocalRepositoryProvider.java b/mycore-ocfl/src/main/java/org/mycore/ocfl/repository/MCROCFLLocalRepositoryProvider.java
@@ -26,7 +26,6 @@
 import org.mycore.common.config.annotation.MCRPostConstruction;
 import org.mycore.common.config.annotation.MCRProperty;
 
-import io.ocfl.core.OcflRepositoryBuilder;
 import io.ocfl.core.extension.OcflExtensionConfig;
 import io.ocfl.core.storage.OcflStorageBuilder;
 
@@ -67,23 +66,18 @@ public abstract class MCROCFLLocalRepositoryProvider implements MCROCFLRepositor
     public void init(String prop) throws IOException {
         Files.createDirectories(workDir);
         Files.createDirectories(repositoryRoot);
-        OcflRepositoryBuilder builder = new OcflRepositoryBuilder()
+        String id = prop.substring(REPOSITORY_PROPERTY_PREFIX.length());
+        repository = new MCROCFLRepositoryBuilder()
+            .id(id)
+            .remote(isRemote())
             .defaultLayoutConfig(getExtensionConfig())
             .storage(this::configureStorage)
-            .workDir(workDir);
-        String id = prop.substring(REPOSITORY_PROPERTY_PREFIX.length());
-        this.repository = createRepository(id, builder);
+            .workDir(workDir)
+            .buildMCR();
     }
 
-    /**
-     * Creates and initializes an OCFL repository with the specified repository ID and builder configuration.
-     *
-     * @param id the unique identifier for the OCFL repository.
-     * @param builder the {@link OcflRepositoryBuilder} to build and configure the repository.
-     * @return a new {@link MCROCFLRepository} instance.
-     */
-    protected MCROCFLRepository createRepository(String id, OcflRepositoryBuilder builder) {
-        return new MCROCFLRepository(id, builder.build(), false);
+    public boolean isRemote() {
+        return false;
     }
 
     /**

diff --git a/mycore-ocfl/src/main/java/org/mycore/ocfl/repository/MCROCFLRepository.java b/mycore-ocfl/src/main/java/org/mycore/ocfl/repository/MCROCFLRepository.java
@@ -26,7 +26,6 @@
 import java.util.stream.Stream;
 
 import io.ocfl.api.OcflConfig;
-import io.ocfl.api.OcflConstants;
 import io.ocfl.api.OcflObjectUpdater;
 import io.ocfl.api.OcflOption;
 import io.ocfl.api.OcflRepository;
@@ -42,7 +41,8 @@
 import io.ocfl.api.model.VersionDetails;
 import io.ocfl.api.model.VersionInfo;
 import io.ocfl.api.model.VersionNum;
-import io.ocfl.core.util.PercentEscaper;
+import io.ocfl.core.model.Inventory;
+import io.ocfl.core.storage.OcflStorage;
 
 /**
  * A wrapper class for an OCFL repository that adds custom functionality specific to MyCoRe.
@@ -59,7 +59,7 @@ public class MCROCFLRepository implements OcflRepository {
 
     private final boolean remote;
 
-    private final PercentEscaper percentEscaper;
+    private final OcflStorage storage;
 
     /**
      * Constructs a new {@code MCROCFLRepository}.
@@ -68,15 +68,11 @@ public class MCROCFLRepository implements OcflRepository {
      * @param repository the base OCFL repository to delegate operations to.
      * @param remote if this is a remote or local repository
      */
-    public MCROCFLRepository(String id, OcflRepository repository, boolean remote) {
+    public MCROCFLRepository(String id, OcflRepository repository, boolean remote, OcflStorage storage) {
         this.id = id;
         this.base = repository;
         this.remote = remote;
-        this.percentEscaper = PercentEscaper.builderWithSafeAlphaNumeric()
-            .addSafeChars("-_")
-            .plusForSpace(false)
-            .useUppercase(false)
-            .build();
+        this.storage = storage;
     }
 
     /**
@@ -204,6 +200,9 @@ public FileChangeHistory directoryChangeHistory(String objectId, String logicalP
      */
     public FileChangeHistory directoryChangeHistory(String objectId, String logicalPath, VersionNum targetVersion)
         throws NotFoundException {
+
+        var inventory = requireInventory(ObjectVersionId.version(objectId, targetVersion));
+
         String relativeDirectoryPath = logicalPath.startsWith("/") ? logicalPath.substring(1) : logicalPath;
 
         VersionNum tailVersionNumber = VersionNum.V1;
@@ -219,7 +218,8 @@ public FileChangeHistory directoryChangeHistory(String objectId, String logicalP
             VersionDetails details = describeVersion(ObjectVersionId.version(objectId, versionNum));
             List<String> directoryContent = details.getFiles().stream()
                 .map(FileDetails::getPath)
-                .map(filePath -> toDirectoryContent(relativeDirectoryPath, filePath))
+                .map(
+                    filePath -> toDirectoryContent(relativeDirectoryPath, filePath))
                 .filter(Objects::nonNull)
                 .toList();
             if ((empty && !directoryContent.isEmpty()) || (!empty && directoryContent.isEmpty())) {
@@ -232,8 +232,7 @@ public FileChangeHistory directoryChangeHistory(String objectId, String logicalP
                 directoryChange.setObjectVersionId(details.getObjectVersionId());
                 String finalVersion = versionNum.toString();
                 details.getFiles().stream().findAny().ifPresent(fileDetails -> {
-                    String fileRelativePath = fileDetails.getStorageRelativePath();
-                    String relativeStoragePath = getStorageRelativePath(fileRelativePath, objectId, finalVersion);
+                    String relativeStoragePath = getStorageRelativePath(inventory, finalVersion);
                     String directoryStoragePath = relativeDirectoryPath.isEmpty() ? relativeStoragePath
                         : relativeStoragePath + "/" + relativeDirectoryPath;
                     directoryChange.setStorageRelativePath(directoryStoragePath);
@@ -255,10 +254,20 @@ private String toDirectoryContent(String directory, String file) {
         return nextPathSegmentIndex != -1 ? relativePath.substring(0, nextPathSegmentIndex) : relativePath;
     }
 
-    private String getStorageRelativePath(String fileRelativePath, String objectId, String version) {
-        String encodedObjectId = this.percentEscaper.escape(objectId);
-        String layoutConfigPath = fileRelativePath.substring(0, fileRelativePath.indexOf("/" + encodedObjectId));
-        return layoutConfigPath + "/" + encodedObjectId + "/" + version + "/" + OcflConstants.DEFAULT_CONTENT_DIRECTORY;
+    private String getStorageRelativePath(Inventory inventory, String version) {
+        return inventory.getObjectRootPath() + "/" + version + "/" + inventory.getContentDirectory();
+    }
+
+    protected Inventory requireInventory(ObjectVersionId objectId) {
+        var inventory = loadInventory(objectId);
+        if (inventory == null) {
+            throw new NotFoundException(String.format("Object %s was not found.", objectId));
+        }
+        return inventory;
+    }
+
+    protected Inventory loadInventory(ObjectVersionId objectId) {
+        return storage.loadInventory(objectId.getObjectId());
     }
 
     /**