From 905872cf2360988ff690789b8bb6cc0e492e75b9 Mon Sep 17 00:00:00 2001 From: LSmyrnaios Date: Mon, 27 Nov 2023 16:45:53 +0200 Subject: [PATCH] - Fix throwing an error when the storage directory already exists. - Add the "testCustomInputOutputWithIdDocFileNames" test. - Update dependencies. - Code polishing. --- pom.xml | 8 +++--- .../PublicationsRetriever.java | 2 +- .../util/file/FileUtils.java | 28 ++++++++++--------- .../util/http/ConnSupportUtils.java | 3 +- .../test/TestNonStandardInputOutput.java | 16 +++++++++++ 5 files changed, 38 insertions(+), 19 deletions(-) diff --git a/pom.xml b/pom.xml index d0c87d0..48ae523 100644 --- a/pom.xml +++ b/pom.xml @@ -61,7 +61,7 @@ org.apache.maven.plugins maven-surefire-plugin - 3.2.1 + 3.2.2 @@ -141,7 +141,7 @@ io.minio minio - 8.5.6 + 8.5.7 com.google.guava @@ -178,7 +178,7 @@ org.junit.jupiter junit-jupiter-engine - 5.10.0 + 5.10.1 test diff --git a/src/main/java/eu/openaire/publications_retriever/PublicationsRetriever.java b/src/main/java/eu/openaire/publications_retriever/PublicationsRetriever.java index 94325da..95edc1e 100644 --- a/src/main/java/eu/openaire/publications_retriever/PublicationsRetriever.java +++ b/src/main/java/eu/openaire/publications_retriever/PublicationsRetriever.java @@ -53,7 +53,7 @@ public class PublicationsRetriever public static Instant startTime = null; public static String targetUrlType = "docOrDatasetUrl"; // docUrl, documentUrl, docOrDatasetUrl ; this is set by the args-parser, and it's used when outputting data. - public static DecimalFormat df = new DecimalFormat("0.00"); + public static final DecimalFormat df = new DecimalFormat("0.00"); public static ExecutorService executor; public static int workerThreadsCount = 0; diff --git a/src/main/java/eu/openaire/publications_retriever/util/file/FileUtils.java b/src/main/java/eu/openaire/publications_retriever/util/file/FileUtils.java index e80b613..54aec23 100644 --- a/src/main/java/eu/openaire/publications_retriever/util/file/FileUtils.java +++ b/src/main/java/eu/openaire/publications_retriever/util/file/FileUtils.java @@ -140,19 +140,21 @@ public static void handleStoreDocFileDirectory() // If the directory doesn't exist, try to (re)create it. try { - if ( !dir.mkdirs() ) { // Try to create the directory(-ies) if they don't exist. - String errorMessage; - if ( PublicationsRetriever.docFilesStorageGivenByUser ) - errorMessage = "Problem when creating the \"storeDocFilesDir\": \"" + FileUtils.storeDocFilesDir + "\"." - + "\nPlease give a valid Directory-path."; - else // User has left the storageDir to be the default one. - errorMessage = "Problem when creating the default \"storeDocFilesDir\": \"" + FileUtils.storeDocFilesDir + "\"." - + "\nPlease verify you have the necessary privileges in the directory you are running the program from or specify the directory you want to save the files to." - + "\nIf the above is not an option, then you can set to retrieve just the " + PublicationsRetriever.targetUrlType + "s and download the full-texts later (on your own)."; - System.err.println(errorMessage); - logger.error(errorMessage); - FileUtils.closeIO(); - System.exit(-3); + if ( !dir.exists() ) { + if ( !dir.mkdirs() ) { // Try to create the directory(-ies) if they don't exist. If they exist OR if sth went wrong, the result os the same: "false". + String errorMessage; + if ( PublicationsRetriever.docFilesStorageGivenByUser ) + errorMessage = "Problem when creating the \"storeDocFilesDir\": \"" + FileUtils.storeDocFilesDir + "\"." + + "\nPlease give a valid Directory-path."; + else // User has left the storageDir to be the default one. + errorMessage = "Problem when creating the default \"storeDocFilesDir\": \"" + FileUtils.storeDocFilesDir + "\"." + + "\nPlease verify you have the necessary privileges in the directory you are running the program from or specify the directory you want to save the files to." + + "\nIf the above is not an option, then you can set to retrieve just the " + PublicationsRetriever.targetUrlType + "s and download the full-texts later (on your own)."; + System.err.println(errorMessage); + logger.error(errorMessage); + FileUtils.closeIO(); + System.exit(-3); + } } } catch (SecurityException se) { logger.error(se.getMessage(), se); diff --git a/src/main/java/eu/openaire/publications_retriever/util/http/ConnSupportUtils.java b/src/main/java/eu/openaire/publications_retriever/util/http/ConnSupportUtils.java index 6b8b5ba..5f43af0 100644 --- a/src/main/java/eu/openaire/publications_retriever/util/http/ConnSupportUtils.java +++ b/src/main/java/eu/openaire/publications_retriever/util/http/ConnSupportUtils.java @@ -15,8 +15,9 @@ import eu.openaire.publications_retriever.util.url.LoaderAndChecker; import eu.openaire.publications_retriever.util.url.UrlUtils; import org.apache.commons.io.FileDeleteStrategy; -import org.jsoup.Jsoup; import org.apache.commons.lang3.StringUtils; + +import org.jsoup.Jsoup; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/test/java/eu/openaire/publications_retriever/test/TestNonStandardInputOutput.java b/src/test/java/eu/openaire/publications_retriever/test/TestNonStandardInputOutput.java index 3a92db8..5065257 100644 --- a/src/test/java/eu/openaire/publications_retriever/test/TestNonStandardInputOutput.java +++ b/src/test/java/eu/openaire/publications_retriever/test/TestNonStandardInputOutput.java @@ -87,6 +87,22 @@ public void testCustomInputOutputWithOriginalDocFileNames() } + @Disabled + @Test + public void testCustomInputOutputWithIdDocFileNames() + { + String[] args = new String[7]; + args[0] = "-retrieveDataType"; + args[1] = "document"; // "document" OR "dataset" OR "all" + args[2] = "-downloadDocFiles"; + args[3] = "-docFileNameType"; + args[4] = "idName"; + args[5] = "-docFilesStorage"; + args[6] = "docFiles"; + main(args); + } + + @Disabled @Test public void testCustomInputOutputWithIdDocFileNamesToS3()