Fix minio bogues (#834)

* fix(no information about minio objects reported in error) WIP - Modify exception message to log more information - Adapt exception handler to process these exceptions - Start writing test * tests(log errors with minio) green tests * refactor (always use FileOperations to process documents) WIP * refactor (always use FileOperations to process documents) WIP * refactor (always use FileOperations to process documents) WIP * test(fix test about minio errors) - Object Document with id attribute to be compliant with existing code * pom(pre-release) * fix: solve unit test --------- Co-authored-by: Fabrice B <[email protected]> Co-authored-by: Emmanuel DEMEY <[email protected]>
InseeFr · Dec 13, 2024 · 3eefa1d · 3eefa1d
1 parent 3aa58f6
commit 3eefa1d
Show file tree

Hide file tree

Showing 55 changed files with 907 additions and 840 deletions.
diff --git a/pom.xml b/pom.xml
@@ -13,7 +13,7 @@
     <groupId>fr.insee.rmes</groupId>
     <artifactId>Bauhaus-BO</artifactId>
     <packaging>jar</packaging>
-    <version>4.1.6</version>
+    <version>4.1.7-beta</version>
     <name>Bauhaus-Back-Office</name>
     <description>Back-office services for Bauhaus</description>
     <url>https://github.com/InseeFr/Bauhaus-Back-Office</url>

diff --git a/src/main/java/fr/insee/rmes/bauhaus_services/FileSystemOperation.java b/src/main/java/fr/insee/rmes/bauhaus_services/FileSystemOperation.java
@@ -22,16 +22,16 @@ public void delete(String path) {
         try {
             Files.delete(Paths.get(path));
         } catch (IOException e) {
-            throw new RmesFileException("Failed to delete file: " + path, e);
+            throw new RmesFileException(path, "Failed to delete file: " + path, e);
         }
     }
 
     @Override
-    public InputStream read(String fileName) {
+    public byte[] read(String fileName) {
         try {
-            return Files.newInputStream(Paths.get(config.getDocumentsStorageGestion()).resolve(fileName));
+            return Files.readAllBytes(Paths.get(config.getDocumentsStorageGestion()).resolve(fileName));
         } catch (IOException e) {
-            throw new RmesFileException("Failed to read file: " + fileName, e);
+            throw new RmesFileException(fileName, "Failed to read file: " + fileName, e);
         }
     }
 
@@ -40,7 +40,7 @@ public void write(InputStream content, Path destPath) {
         try {
             Files.copy(content, destPath, StandardCopyOption.REPLACE_EXISTING);
         } catch (IOException e) {
-            throw new RmesFileException("Failed to write file: " + destPath, e);
+            throw new RmesFileException(destPath.toString(),"Failed to write file: " + destPath, e);
         }
     }
 
@@ -51,12 +51,17 @@ public void copy(String srcPath, String destPath)  {
         try {
             Files.copy(file, targetPath.resolve(file.getFileName()), StandardCopyOption.REPLACE_EXISTING);
         } catch (IOException e) {
-            throw new RmesFileException("Failed to copy file : " + srcPath + " to " + destPath, e);
+            throw new RmesFileException(srcPath, "Failed to copy file : " + srcPath + " to " + destPath, e);
         }
     }
 
     @Override
     public boolean dirExists(Path gestionStorageFolder) {
         return Files.isDirectory(requireNonNull(gestionStorageFolder));
     }
+
+    @Override
+    public boolean exists(Path path) {
+        return false;
+    }
 }
diff --git a/src/main/java/fr/insee/rmes/bauhaus_services/FilesOperations.java b/src/main/java/fr/insee/rmes/bauhaus_services/FilesOperations.java
@@ -5,9 +5,11 @@
 
 public interface FilesOperations {
     void delete(String path);
-    InputStream read(String path);
+    byte[] read(String path);
     void write(InputStream content, Path destPath);
     void copy(String srcPath, String destPath);
 
     boolean dirExists(Path gestionStorageFolder);
+
+    boolean exists(Path path);
 }
diff --git a/src/main/java/fr/insee/rmes/bauhaus_services/MinioFilesOperation.java b/src/main/java/fr/insee/rmes/bauhaus_services/MinioFilesOperation.java
@@ -3,71 +3,72 @@
 import fr.insee.rmes.exceptions.RmesFileException;
 import io.minio.*;
 import io.minio.errors.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Path;
 import java.security.InvalidKeyException;
 import java.security.NoSuchAlgorithmException;
 
-public record MinioFilesOperation(MinioClient minioClient, String bucketName, String directoryGestion, String directoryPublication) implements FilesOperations {
+import static java.util.Objects.requireNonNull;
 
-    private static final Logger logger = LoggerFactory.getLogger(MinioFilesOperation.class);
+public record MinioFilesOperation(MinioClient minioClient, String bucketName, String directoryGestion, String directoryPublication) implements FilesOperations {
 
     @Override
-    public InputStream read(String pathFile){
-        String objectName= extractFileName(pathFile);
-
-        try {
-            return minioClient.getObject(GetObjectArgs.builder()
-                    .bucket(bucketName)
-                    .object(directoryGestion +"/"+ objectName)
-                    .build());
+    public byte[] read(String pathFile){
+        String objectName = getObjectName(requireNonNull(pathFile));
+        try(InputStream inputStream = minioClient.getObject(GetObjectArgs.builder()
+                .bucket(bucketName)
+                .object(objectName)
+                .build())) {
+            return inputStream.readAllBytes();
         } catch (MinioException | InvalidKeyException | IOException | NoSuchAlgorithmException e) {
-            throw new RmesFileException("Error reading file: " + objectName, e);
+            throw new RmesFileException(pathFile, "Error reading file: " + pathFile+" as object `"+objectName+"` in bucket "+bucketName, e);
         }
     }
-    private static String extractFileName(String filePath) {
-        if (filePath == null || filePath.isEmpty()) {
-            return "";
-        }
-        return Path.of(filePath).getFileName().toString();
+
+    private String getObjectName(String pathFile) {
+        return getObjectName(Path.of(pathFile));
+    }
+
+    private String getObjectName(Path pathFile) {
+        return directoryGestion + "/" + pathFile.getFileName().toString();
     }
 
 
     @Override
-    public void write(InputStream content, Path objectName) {
+    public void write(InputStream content, Path filePath) {
+        String objectName = getObjectName(requireNonNull(filePath));
         try {
             minioClient.putObject(PutObjectArgs.builder()
                     .bucket(bucketName)
-                    .object(directoryGestion +"/"+ objectName.getFileName().toString())
+                    .object(objectName)
                     .stream(content, content.available(), -1)
                     .build());
         } catch (IOException | ErrorResponseException | InsufficientDataException | InternalException |
                  InvalidKeyException | InvalidResponseException | NoSuchAlgorithmException | ServerException |
                  XmlParserException e) {
-            throw new RmesFileException("Error writing file: " + objectName, e);
+            throw new RmesFileException(filePath.toString(), "Error writing file: " + filePath+ "as object `"+objectName+"` in bucket "+bucketName, e);
         }
     }
 
     @Override
     public void copy(String srcObjectName, String destObjectName)  {
 
+        String srcObject = getObjectName(requireNonNull(srcObjectName));
+        String destObject = getObjectName(requireNonNull(srcObjectName));
         try {
             CopySource source = CopySource.builder()
                     .bucket(bucketName)
-                    .object(directoryGestion +"/"+ extractFileName(srcObjectName))
+                    .object(srcObject)
                     .build();
-
             minioClient.copyObject(CopyObjectArgs.builder()
                     .bucket(bucketName)
-                    .object(directoryPublication +"/"+ extractFileName(srcObjectName))
+                    .object(destObject)
                     .source(source)
                     .build());
         } catch (MinioException | InvalidKeyException | IOException | NoSuchAlgorithmException e) {
-            throw new RmesFileException("Error copying file from " + srcObjectName + " to " + destObjectName, e);
+            throw new RmesFileException(srcObjectName,"Error copying file from `" + srcObject + "` to `" + destObject+"` in bucket "+bucketName, e);
         }
     }
 
@@ -77,15 +78,28 @@ public boolean dirExists(Path gestionStorageFolder) {
         return true;
     }
 
-     @Override
+    @Override
+    public boolean exists(Path path) {
+        var objectName = getObjectName(requireNonNull(path));
+        try {
+            return minioClient.statObject(StatObjectArgs.builder()
+                    .bucket(bucketName)
+                    .object(objectName)
+                    .build()).size() > 0;
+        } catch (MinioException | InvalidKeyException | IOException | NoSuchAlgorithmException e) {
+            return false;
+        }
+    }
+
+    @Override
     public void delete(String objectName) {
         try {
             minioClient.removeObject(RemoveObjectArgs.builder()
                     .bucket(bucketName)
                     .object(objectName)
                     .build());
         } catch (MinioException | InvalidKeyException | IOException | NoSuchAlgorithmException e) {
-            throw new RmesFileException("Error deleting file: " + objectName, e);
+            throw new RmesFileException(objectName,"Error deleting file: " + objectName+" in bucket "+bucketName, e);
         }
     }
 

diff --git a/src/main/java/fr/insee/rmes/bauhaus_services/StampAuthorizationChecker.java b/src/main/java/fr/insee/rmes/bauhaus_services/StampAuthorizationChecker.java
@@ -136,9 +136,9 @@ private IRI findDistributionIRI(String distributionId) {
     private IRI findComponentIRI(String componentId) throws RmesException {
         JSONObject type = repoGestion.getResponseAsObject(StructureQueries.getComponentType(componentId));
         String componentType = type.getString("type");
-        if (componentType.equals(RdfUtils.toString(QB.ATTRIBUTE_PROPERTY))) {
+        if (componentType.equals(QB.ATTRIBUTE_PROPERTY.toString())) {
             return RdfUtils.structureComponentAttributeIRI(componentId);
-        } else if (componentType.equals(RdfUtils.toString(QB.DIMENSION_PROPERTY))) {
+        } else if (componentType.equals(QB.DIMENSION_PROPERTY.toString())) {
             return RdfUtils.structureComponentDimensionIRI(componentId);
         } else {
             return RdfUtils.structureComponentMeasureIRI(componentId);

diff --git a/src/main/java/fr/insee/rmes/bauhaus_services/classifications/ClassificationPublication.java b/src/main/java/fr/insee/rmes/bauhaus_services/classifications/ClassificationPublication.java
@@ -3,7 +3,6 @@
 import fr.insee.rmes.bauhaus_services.Constants;
 import fr.insee.rmes.bauhaus_services.rdf_utils.PublicationUtils;
 import fr.insee.rmes.bauhaus_services.rdf_utils.RdfService;
-import fr.insee.rmes.bauhaus_services.rdf_utils.RdfUtils;
 import fr.insee.rmes.exceptions.ErrorCodes;
 import fr.insee.rmes.exceptions.RmesException;
 import fr.insee.rmes.exceptions.RmesNotFoundException;
@@ -53,7 +52,7 @@ public void transformStatementToPublish(Model model, RepositoryResult<Statement>
 		while (classifStatements.hasNext()) {
 			Statement st = classifStatements.next();
 			// Triplets that don't get published
-			String predicate = RdfUtils.toString(st.getPredicate());
+            String predicate = st.getPredicate().toString();
 			if (!isTripletForPublication(predicate)) {
 				// nothing, wouldn't copy this attr
 			} else {

diff --git a/src/main/java/fr/insee/rmes/bauhaus_services/code_list/CodeListPublication.java b/src/main/java/fr/insee/rmes/bauhaus_services/code_list/CodeListPublication.java
@@ -30,7 +30,7 @@ private void checkIfResourceExists(RepositoryResult<Statement> statements, Resou
 	}
 
 	private boolean shouldExcludeTriplet(Statement statement){
-		String pred = RdfUtils.toString(statement.getPredicate());
+        String pred = statement.getPredicate().toString();
 		return pred.endsWith("validationState")
 				|| pred.endsWith(Constants.CREATOR)
 				|| pred.endsWith(Constants.CONTRIBUTOR)
@@ -54,7 +54,7 @@ private void publishCodeListAndCodesWithConnection(Resource codeListOrCode, Repo
 				}
 
 
-				String predicate = RdfUtils.toString(st.getPredicate());
+                String predicate = st.getPredicate().toString();
 				String object = st.getObject().stringValue();
 				if (RDFS.SEEALSO.toString().equalsIgnoreCase(predicate)) {
 					publishSeeAlsoTriplets(object, connection);

diff --git a/src/main/java/fr/insee/rmes/bauhaus_services/code_list/CodeListServiceImpl.java b/src/main/java/fr/insee/rmes/bauhaus_services/code_list/CodeListServiceImpl.java
@@ -267,7 +267,7 @@ private boolean checkCodeListUnicity(boolean partial, JSONObject codeList, Strin
 		String id = codeList.getString(Constants.ID);
 		if(!partial) {
 			IRI seeAlso = RdfUtils.codeListIRI(CONCEPT + codeList.getString(LAST_CLASS_URI_SEGMENT));
-			return repoGestion.getResponseAsBoolean(CodeListQueries.checkCodeListUnicity(id, iri, RdfUtils.toString(seeAlso), false));
+            return repoGestion.getResponseAsBoolean(CodeListQueries.checkCodeListUnicity(id, iri, seeAlso.toString(), false));
 		}
 		return repoGestion.getResponseAsBoolean(CodeListQueries.checkCodeListUnicity(id, iri, "", true));
 	}
@@ -280,7 +280,7 @@ public String setCodesList(String body, boolean partial) throws RmesException {
 
 		IRI codeListIri = this.generateIri(codesList, partial);
 
-		if(this.checkCodeListUnicity(partial, codesList, RdfUtils.toString(codeListIri))){
+        if(this.checkCodeListUnicity(partial, codesList, codeListIri.toString())){
 			throw new RmesBadRequestException(CodesListErrorCodes.CODE_LIST_UNICITY,
 					"The identifier, IRI and OWL class should be unique", "");
 		}

diff --git a/src/main/java/fr/insee/rmes/bauhaus_services/concepts/ConceptsImpl.java b/src/main/java/fr/insee/rmes/bauhaus_services/concepts/ConceptsImpl.java
@@ -89,7 +89,7 @@ public String getConceptByID(String id)  throws RmesException{
 
 	@Override
 	public String getRelatedConcepts(String id)  throws RmesException{
-		String uriConcept = RdfUtils.toString(RdfUtils.objectIRI(ObjectType.CONCEPT,id));
+        String uriConcept = RdfUtils.objectIRI(ObjectType.CONCEPT, id).toString();
 		JSONArray resQuery = conceptsUtils.getRelatedConcepts(uriConcept);
 		return QueryUtils.correctEmptyGroupConcat(resQuery.toString());
 	}
@@ -102,7 +102,7 @@ public String getRelatedConcepts(String id)  throws RmesException{
 	 */	
 	@Override
 	public String deleteConcept(String id) throws RmesException {
-		String uriConcept = RdfUtils.toString(RdfUtils.objectIRI(ObjectType.CONCEPT,id));
+        String uriConcept = RdfUtils.objectIRI(ObjectType.CONCEPT, id).toString();
 		JSONArray graphArray = conceptsUtils.getGraphsWithConcept(uriConcept);
 
 		/* check concept isn't used in several graphs */

diff --git a/src/main/java/fr/insee/rmes/bauhaus_services/concepts/concepts/ConceptsUtils.java b/src/main/java/fr/insee/rmes/bauhaus_services/concepts/concepts/ConceptsUtils.java
@@ -216,9 +216,9 @@ public JSONArray getRelatedConcepts(String id)  throws RmesException{
 	}
 
 	public HttpStatus deleteConcept(String id) throws RmesException{
-		HttpStatus result =  repoGestion.executeUpdate(ConceptsQueries.deleteConcept(RdfUtils.toString(RdfUtils.objectIRI(ObjectType.CONCEPT,id)),RdfUtils.conceptGraph().toString()));
+        HttpStatus result =  repoGestion.executeUpdate(ConceptsQueries.deleteConcept(RdfUtils.objectIRI(ObjectType.CONCEPT, id).toString(),RdfUtils.conceptGraph().toString()));
 		if (result.equals(HttpStatus.OK)) {
-			result = repositoryPublication.executeUpdate(ConceptsQueries.deleteConcept(RdfUtils.toString(RdfUtils.objectIRIPublication(ObjectType.CONCEPT,id)),RdfUtils.conceptGraph().toString()));
+            result = repositoryPublication.executeUpdate(ConceptsQueries.deleteConcept(RdfUtils.objectIRIPublication(ObjectType.CONCEPT, id).toString(),RdfUtils.conceptGraph().toString()));
 		}
 		return result;
 	}

diff --git a/src/main/java/fr/insee/rmes/bauhaus_services/concepts/publication/ConceptsPublication.java b/src/main/java/fr/insee/rmes/bauhaus_services/concepts/publication/ConceptsPublication.java
@@ -8,10 +8,7 @@
 import fr.insee.rmes.persistance.ontologies.XKOS;
 import fr.insee.rmes.persistance.sparql_queries.concepts.ConceptsQueries;
 import org.apache.http.HttpStatus;
-import org.eclipse.rdf4j.model.Literal;
-import org.eclipse.rdf4j.model.Model;
-import org.eclipse.rdf4j.model.Resource;
-import org.eclipse.rdf4j.model.Statement;
+import org.eclipse.rdf4j.model.*;
 import org.eclipse.rdf4j.model.impl.LinkedHashModel;
 import org.eclipse.rdf4j.model.vocabulary.DCTERMS;
 import org.eclipse.rdf4j.model.vocabulary.SKOS;
@@ -80,7 +77,7 @@ private Boolean prepareOneTripleToPublicationAndCheckIfHasBroader(Model model, L
 
 		Resource subject =  publicationUtils.tranformBaseURIToPublish(st.getSubject());
 		Resource graph = st.getContext();
-		String predicat = RdfUtils.toString(st.getPredicate());
+        String predicat = st.getPredicate().toString();
 
 		if (PublicationUtils.stringEndsWithItemFromList(predicat,notes)) {
 			model.add(subject, st.getPredicate(), publicationUtils.tranformBaseURIToPublish((Resource) st.getObject()),
@@ -147,7 +144,7 @@ private void publishExplanatoryNotes(RepositoryConnection con, Resource note, Mo
 			Resource graph = null;
 			while (statements.hasNext()) {
 				Statement st = statements.next();
-				String predicat = RdfUtils.toString(st.getPredicate());
+                String predicat = st.getPredicate().toString();
 				subject = publicationUtils.tranformBaseURIToPublish(st.getSubject());
 				graph = st.getContext();
 				if (predicat.endsWith("conceptVersion")) {
@@ -214,12 +211,12 @@ public void publishCollection(JSONArray collectionsToValidate) throws RmesExcept
 					while (statements.hasNext()) {
 						Statement st = statements.next();
 						// Other URI to transform
-						if (RdfUtils.toString(st.getPredicate()).endsWith("member")) {
+                        if (st.getPredicate().toString().endsWith("member")) {
 							model.add(publicationUtils.tranformBaseURIToPublish(st.getSubject()), st.getPredicate(),
 									publicationUtils.tranformBaseURIToPublish((Resource) st.getObject()), st.getContext());
-						} else if (RdfUtils.toString(st.getPredicate()).endsWith("isValidated")
-								|| (RdfUtils.toString(st.getPredicate()).endsWith(Constants.CREATOR))
-								|| (RdfUtils.toString(st.getPredicate()).endsWith(Constants.CONTRIBUTOR))) {
+						} else if (st.getPredicate().toString().endsWith("isValidated")
+								|| (st.getPredicate().toString().endsWith(Constants.CREATOR))
+								|| (st.getPredicate().toString().endsWith(Constants.CONTRIBUTOR))) {
 							// nothing, wouldn't copy this attr
 						}
 						// Literals

diff --git a/src/main/java/fr/insee/rmes/bauhaus_services/geography/GeographyServiceImpl.java b/src/main/java/fr/insee/rmes/bauhaus_services/geography/GeographyServiceImpl.java
@@ -195,7 +195,7 @@ public String createRdfGeoFeature(GeoFeature geoFeature) throws RmesException {
 		});
 		repoGestion.loadSimpleObject(geoIRI, model);
 
-		return RdfUtils.toString(geoIRI);
-	}
+        return geoIRI.toString();
+    }
 
 }
diff --git a/src/main/java/fr/insee/rmes/bauhaus_services/operations/ParentUtils.java b/src/main/java/fr/insee/rmes/bauhaus_services/operations/ParentUtils.java
@@ -67,7 +67,7 @@ public boolean checkIfSeriesHasSims(String uriSeries) throws RmesException {
 	}
 
 	public void checkIfParentIsASeriesWithOperations(String idParent) throws RmesException {
-		String uriParent = RdfUtils.toString(RdfUtils.objectIRI(ObjectType.SERIES, idParent));
+        String uriParent = RdfUtils.objectIRI(ObjectType.SERIES, idParent).toString();
 		if (checkIfParentExists(uriParent) && checkIfSeriesHasOperation(uriParent)) throw new RmesNotAcceptableException(ErrorCodes.SERIES_OPERATION_OR_SIMS,
 				"Cannot create Sims for a series which already has operations", idParent);
 	}
@@ -117,7 +117,7 @@ public JSONArray getSeriesCreators(String id) throws RmesException {
 	}
 
 	public JSONArray getSeriesCreators(IRI iri) throws RmesException {
-		return repoGestion.getResponseAsJSONList(OpSeriesQueries.getCreatorsBySeriesUri("<" + RdfUtils.toString(iri) + ">"));
+        return repoGestion.getResponseAsJSONList(OpSeriesQueries.getCreatorsBySeriesUri("<" + iri.toString() + ">"));
 	}
 
 	public String[] getDocumentationTargetTypeAndId(String idSims) throws RmesException {