diff --git a/CHANGELOG.md b/CHANGELOG.md index 996f2b50d84..f61e34de33e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv ### Changed +- When a communication error with an [online service](https://docs.jabref.org/collect/import-using-online-bibliographic-database) occurs, JabRef displays the HTTP error. [#11223](https://github.com/JabRef/jabref/issues/11223) - The Pubmed/Medline Plain importer now imports the PMID field as well [#11488](https://github.com/JabRef/jabref/issues/11488) - The 'Check for updates' menu bar button is now always enabled. [#11485](https://github.com/JabRef/jabref/pull/11485) - JabRef respects the [configuration for storing files relative to the .bib file](https://docs.jabref.org/finding-sorting-and-cleaning-entries/filelinks#directories-for-files) in more cases. [#11492](https://github.com/JabRef/jabref/pull/11492) diff --git a/src/main/java/org/jabref/cli/ArgumentProcessor.java b/src/main/java/org/jabref/cli/ArgumentProcessor.java index 909fc46c6e2..6ce14604a64 100644 --- a/src/main/java/org/jabref/cli/ArgumentProcessor.java +++ b/src/main/java/org/jabref/cli/ArgumentProcessor.java @@ -1,6 +1,7 @@ package org.jabref.cli; import java.io.IOException; +import java.net.MalformedURLException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -133,7 +134,7 @@ private Optional importFile(String argument) { // Download web resource to temporary file try { file = new URLDownload(address).toTemporaryFile(); - } catch (IOException e) { + } catch (FetcherException | MalformedURLException e) { System.err.println(Localization.lang("Problem downloading from %1", address) + e.getLocalizedMessage()); return Optional.empty(); } diff --git a/src/main/java/org/jabref/gui/DialogService.java b/src/main/java/org/jabref/gui/DialogService.java index 3865f5d000e..5a6cc7d7919 100644 --- a/src/main/java/org/jabref/gui/DialogService.java +++ b/src/main/java/org/jabref/gui/DialogService.java @@ -20,7 +20,7 @@ import org.jabref.gui.util.BaseDialog; import org.jabref.gui.util.DirectoryDialogConfiguration; import org.jabref.gui.util.FileDialogConfiguration; -import org.jabref.logic.l10n.Localization; +import org.jabref.logic.importer.FetcherException; import org.controlsfx.control.textfield.CustomPasswordField; import org.controlsfx.dialog.ProgressDialog; @@ -97,9 +97,9 @@ default Optional showEditableChoiceDialogAndWait(String title, String con * * @param exception the exception causing the error */ - default void showErrorDialogAndWait(Exception exception) { - showErrorDialogAndWait(Localization.lang("Unhandled exception occurred."), exception); - } + void showErrorDialogAndWait(Exception exception); + + void showErrorDialogAndWait(FetcherException fetcherException); /** * Create and display error dialog displaying the given exception. diff --git a/src/main/java/org/jabref/gui/JabRefDialogService.java b/src/main/java/org/jabref/gui/JabRefDialogService.java index 0ec2acdddd5..b92954bff36 100644 --- a/src/main/java/org/jabref/gui/JabRefDialogService.java +++ b/src/main/java/org/jabref/gui/JabRefDialogService.java @@ -47,6 +47,10 @@ import org.jabref.gui.util.FileDialogConfiguration; import org.jabref.gui.util.UiTaskExecutor; import org.jabref.gui.util.ZipFileChooser; +import org.jabref.http.dto.SimpleHttpResponse; +import org.jabref.logic.importer.FetcherClientException; +import org.jabref.logic.importer.FetcherException; +import org.jabref.logic.importer.FetcherServerException; import org.jabref.logic.l10n.Localization; import com.tobiasdiez.easybind.EasyBind; @@ -87,6 +91,7 @@ private FXDialog createDialog(AlertType type, String title, String content) { alert.setResizable(true); TextArea area = new TextArea(content); + area.setWrapText(true); alert.getDialogPane().setContent(area); alert.initOwner(mainWindow); @@ -204,6 +209,42 @@ public void showErrorDialogAndWait(String message, Throwable exception) { exceptionDialog.showAndWait(); } + @Override + public void showErrorDialogAndWait(Exception exception) { + if (exception instanceof FetcherException fetcherException) { + // Somehow, Java does not route correctly to the other method + showErrorDialogAndWait(fetcherException); + } else { + showErrorDialogAndWait(Localization.lang("Unhandled exception occurred."), exception); + } + } + + @Override + public void showErrorDialogAndWait(FetcherException fetcherException) { + String failedTitle = Localization.lang("Failed to download from URL"); + String localizedMessage = fetcherException.getLocalizedMessage(); + Optional httpResponse = fetcherException.getHttpResponse(); + if (httpResponse.isPresent()) { + int statusCode = httpResponse.get().statusCode(); + if (statusCode == 401) { + this.showInformationDialogAndWait(failedTitle, Localization.lang("Access denied. You are not authorized to access this resource. Please check your credentials and try again. If you believe you should have access, please contact the administrator for assistance.") + "\n\n" + localizedMessage); + } else if (statusCode == 403) { + this.showInformationDialogAndWait(failedTitle, Localization.lang("Access denied. You do not have permission to access this resource. Please contact the administrator for assistance or try a different action.") + "\n\n" + localizedMessage); + } else if (statusCode == 404) { + this.showInformationDialogAndWait(failedTitle, Localization.lang("The requested resource could not be found. It seems that the file you are trying to download is not available or has been moved. Please verify the URL and try again. If you believe this is an error, please contact the administrator for further assistance.") + "\n\n" + localizedMessage); + } else { + this.showErrorDialogAndWait(failedTitle, Localization.lang("Something is wrong on JabRef side. Please check the URL and try again.") + "\n\n" + localizedMessage); + } + } else if (fetcherException instanceof FetcherClientException) { + this.showErrorDialogAndWait(failedTitle, Localization.lang("Something is wrong on JabRef side. Please check the URL and try again.") + "\n\n" + localizedMessage); + } else if (fetcherException instanceof FetcherServerException) { + this.showInformationDialogAndWait(failedTitle, + Localization.lang("Error downloading from URL. Cause is likely the server side.\nPlease try again later or contact the server administrator.") + "\n\n" + localizedMessage); + } else { + this.showErrorDialogAndWait(failedTitle, localizedMessage); + } + } + @Override public void showErrorDialogAndWait(String title, String content, Throwable exception) { ExceptionDialog exceptionDialog = new ExceptionDialog(exception); diff --git a/src/main/java/org/jabref/gui/entryeditor/SciteTabViewModel.java b/src/main/java/org/jabref/gui/entryeditor/SciteTabViewModel.java index 2ac46e5cc8a..2a8216ea0c4 100644 --- a/src/main/java/org/jabref/gui/entryeditor/SciteTabViewModel.java +++ b/src/main/java/org/jabref/gui/entryeditor/SciteTabViewModel.java @@ -1,6 +1,5 @@ package org.jabref.gui.entryeditor; -import java.io.IOException; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; @@ -96,25 +95,24 @@ private void cancelSearch() { } public SciteTallyModel fetchTallies(DOI doi) throws FetcherException { + URL url; try { - URL url = new URI(BASE_URL + "tallies/" + doi.getDOI()).toURL(); - LOGGER.debug("Fetching tallies from {}", url); - URLDownload download = new URLDownload(url); - String response = download.asString(); - LOGGER.debug("Response {}", response); - JSONObject tallies = new JSONObject(response); - if (tallies.has("detail")) { - String message = tallies.getString("detail"); - throw new FetcherException(message); - } else if (!tallies.has("total")) { - throw new FetcherException("Unexpected result data."); - } - return SciteTallyModel.fromJSONObject(tallies); + url = new URI(BASE_URL + "tallies/" + doi.getDOI()).toURL(); } catch (MalformedURLException | URISyntaxException ex) { throw new FetcherException("Malformed URL for DOI", ex); - } catch (IOException ioex) { - throw new FetcherException("Failed to retrieve tallies for DOI - I/O Exception", ioex); } + LOGGER.debug("Fetching tallies from {}", url); + URLDownload download = new URLDownload(url); + String response = download.asString(); + LOGGER.debug("Response {}", response); + JSONObject tallies = new JSONObject(response); + if (tallies.has("detail")) { + String message = tallies.getString("detail"); + throw new FetcherException(message); + } else if (!tallies.has("total")) { + throw new FetcherException("Unexpected result data."); + } + return SciteTallyModel.fromJSONObject(tallies); } public ObjectProperty statusProperty() { diff --git a/src/main/java/org/jabref/gui/entryeditor/citationrelationtab/semanticscholar/SemanticScholarFetcher.java b/src/main/java/org/jabref/gui/entryeditor/citationrelationtab/semanticscholar/SemanticScholarFetcher.java index 1463d22aa9c..82ad2767f6a 100644 --- a/src/main/java/org/jabref/gui/entryeditor/citationrelationtab/semanticscholar/SemanticScholarFetcher.java +++ b/src/main/java/org/jabref/gui/entryeditor/citationrelationtab/semanticscholar/SemanticScholarFetcher.java @@ -1,6 +1,6 @@ package org.jabref.gui.entryeditor.citationrelationtab.semanticscholar; -import java.io.IOException; +import java.net.MalformedURLException; import java.net.URI; import java.net.URL; import java.util.List; @@ -33,50 +33,55 @@ public String getAPIUrl(String entry_point, BibEntry entry) { @Override public List searchCitedBy(BibEntry entry) throws FetcherException { - if (entry.getDOI().isPresent()) { - try { - URL citationsUrl = URI.create(getAPIUrl("citations", entry)).toURL(); - URLDownload urlDownload = new URLDownload(citationsUrl); - - String apiKey = getApiKey(); - if (!apiKey.isEmpty()) { - urlDownload.addHeader("x-api-key", apiKey); - } - CitationsResponse citationsResponse = new Gson() - .fromJson(urlDownload.asString(), CitationsResponse.class); - - return citationsResponse.getData() - .stream().filter(citationDataItem -> citationDataItem.getCitingPaper() != null) - .map(citationDataItem -> citationDataItem.getCitingPaper().toBibEntry()).toList(); - } catch (IOException e) { - throw new FetcherException("Could not fetch", e); - } + if (!entry.getDOI().isPresent()) { + return List.of(); } - return List.of(); + + URL citationsUrl; + try { + citationsUrl = URI.create(getAPIUrl("citations", entry)).toURL(); + } catch (MalformedURLException e) { + throw new FetcherException("Malformed URL", e); + } + URLDownload urlDownload = new URLDownload(citationsUrl); + + String apiKey = getApiKey(); + if (!apiKey.isEmpty()) { + urlDownload.addHeader("x-api-key", apiKey); + } + CitationsResponse citationsResponse = new Gson() + .fromJson(urlDownload.asString(), CitationsResponse.class); + + return citationsResponse.getData() + .stream().filter(citationDataItem -> citationDataItem.getCitingPaper() != null) + .map(citationDataItem -> citationDataItem.getCitingPaper().toBibEntry()).toList(); } @Override public List searchCiting(BibEntry entry) throws FetcherException { - if (entry.getDOI().isPresent()) { - try { - URL referencesUrl = URI.create(getAPIUrl("references", entry)).toURL(); - URLDownload urlDownload = new URLDownload(referencesUrl); - String apiKey = getApiKey(); - if (!apiKey.isEmpty()) { - urlDownload.addHeader("x-api-key", apiKey); - } - ReferencesResponse referencesResponse = new Gson() - .fromJson(urlDownload.asString(), ReferencesResponse.class); - - return referencesResponse.getData() - .stream() - .filter(citationDataItem -> citationDataItem.getCitedPaper() != null) - .map(referenceDataItem -> referenceDataItem.getCitedPaper().toBibEntry()).toList(); - } catch (IOException e) { - throw new FetcherException("Could not fetch", e); - } + if (!entry.getDOI().isPresent()) { + return List.of(); } - return List.of(); + + URL referencesUrl; + try { + referencesUrl = URI.create(getAPIUrl("references", entry)).toURL(); + } catch (MalformedURLException e) { + throw new FetcherException("Malformed URL", e); + } + + URLDownload urlDownload = new URLDownload(referencesUrl); + String apiKey = getApiKey(); + if (!apiKey.isEmpty()) { + urlDownload.addHeader("x-api-key", apiKey); + } + ReferencesResponse referencesResponse = new Gson() + .fromJson(urlDownload.asString(), ReferencesResponse.class); + + return referencesResponse.getData() + .stream() + .filter(citationDataItem -> citationDataItem.getCitedPaper() != null) + .map(referenceDataItem -> referenceDataItem.getCitedPaper().toBibEntry()).toList(); } @Override diff --git a/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java b/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java index 943e3ceddc0..e532df7a635 100644 --- a/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java +++ b/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java @@ -144,17 +144,22 @@ public void search() { } SearchBasedFetcher activeFetcher = getSelectedFetcher(); - Callable parserResultCallable = () -> new ParserResult(activeFetcher.performSearch(query)); + + Callable parserResultCallable; + String fetcherName = activeFetcher.getName(); if (CompositeIdFetcher.containsValidId(query)) { CompositeIdFetcher compositeIdFetcher = new CompositeIdFetcher(preferencesService.getImportFormatPreferences()); parserResultCallable = () -> new ParserResult(OptionalUtil.toList(compositeIdFetcher.performSearchById(query))); fetcherName = Localization.lang("Identifier-based Web Search"); + } else { + // Exceptions are handled below at "task.onFailure(dialogService::showErrorDialogAndWait)" + parserResultCallable = () -> new ParserResult(activeFetcher.performSearch(query)); } BackgroundTask task = BackgroundTask.wrap(parserResultCallable) - .withInitialMessage(Localization.lang("Processing %0", query)); + .withInitialMessage(Localization.lang("Processing \"%0\"...", query)); task.onFailure(dialogService::showErrorDialogAndWait); ImportEntriesDialog dialog = new ImportEntriesDialog(stateManager.getActiveDatabase().get(), task); diff --git a/src/main/java/org/jabref/gui/linkedfile/DownloadLinkedFileAction.java b/src/main/java/org/jabref/gui/linkedfile/DownloadLinkedFileAction.java index 8fc11e21d52..54695f9577b 100644 --- a/src/main/java/org/jabref/gui/linkedfile/DownloadLinkedFileAction.java +++ b/src/main/java/org/jabref/gui/linkedfile/DownloadLinkedFileAction.java @@ -30,8 +30,7 @@ import org.jabref.gui.util.BackgroundTask; import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.externalfiles.LinkedFileHandler; -import org.jabref.logic.importer.FetcherClientException; -import org.jabref.logic.importer.FetcherServerException; +import org.jabref.logic.importer.FetcherException; import org.jabref.logic.l10n.Localization; import org.jabref.logic.net.ProgressInputStream; import org.jabref.logic.net.URLDownload; @@ -199,24 +198,12 @@ private void onSuccess(Path targetDirectory, Path downloadedFile) { private void onFailure(URLDownload urlDownload, Exception ex) { LOGGER.error("Error downloading from URL: {}", urlDownload, ex); - String fetcherExceptionMessage = ex.getMessage(); - String failedTitle = Localization.lang("Failed to download from URL"); - int statusCode; - if (ex instanceof FetcherClientException clientException) { - statusCode = clientException.getStatusCode(); - if (statusCode == 401) { - dialogService.showInformationDialogAndWait(failedTitle, Localization.lang("401 Unauthorized: Access Denied. You are not authorized to access this resource. Please check your credentials and try again. If you believe you should have access, please contact the administrator for assistance.\nURL: %0 \n %1", urlDownload.getSource(), fetcherExceptionMessage)); - } else if (statusCode == 403) { - dialogService.showInformationDialogAndWait(failedTitle, Localization.lang("403 Forbidden: Access Denied. You do not have permission to access this resource. Please contact the administrator for assistance or try a different action.\nURL: %0 \n %1", urlDownload.getSource(), fetcherExceptionMessage)); - } else if (statusCode == 404) { - dialogService.showInformationDialogAndWait(failedTitle, Localization.lang("404 Not Found Error: The requested resource could not be found. It seems that the file you are trying to download is not available or has been moved. Please verify the URL and try again. If you believe this is an error, please contact the administrator for further assistance.\nURL: %0 \n %1", urlDownload.getSource(), fetcherExceptionMessage)); - } - } else if (ex instanceof FetcherServerException serverException) { - statusCode = serverException.getStatusCode(); - dialogService.showInformationDialogAndWait(failedTitle, - Localization.lang("Error downloading from URL. Cause is likely the server side. HTTP Error %0 \n %1 \nURL: %2 \nPlease try again later or contact the server administrator.", statusCode, fetcherExceptionMessage, urlDownload.getSource())); + if (ex instanceof FetcherException fetcherException) { + dialogService.showErrorDialogAndWait(fetcherException); } else { - dialogService.showErrorDialogAndWait(failedTitle, Localization.lang("Error message: %0 \nURL: %1 \nPlease check the URL and try again.", fetcherExceptionMessage, urlDownload.getSource())); + String fetcherExceptionMessage = ex.getLocalizedMessage(); + String failedTitle = Localization.lang("Failed to download from URL"); + dialogService.showErrorDialogAndWait(failedTitle, Localization.lang("Please check the URL and try again.\nURL: %0\nDetails: %1", urlDownload.getSource(), fetcherExceptionMessage)); } } diff --git a/src/main/java/org/jabref/gui/preview/PreviewViewer.java b/src/main/java/org/jabref/gui/preview/PreviewViewer.java index d17400cd6e1..b8e260f1d89 100644 --- a/src/main/java/org/jabref/gui/preview/PreviewViewer.java +++ b/src/main/java/org/jabref/gui/preview/PreviewViewer.java @@ -264,7 +264,7 @@ private void update() { final BibEntry theEntry = entry.get(); BackgroundTask .wrap(() -> layout.generatePreview(theEntry, database)) - .onRunning(() -> setPreviewText("" + Localization.lang("Processing %0", Localization.lang("Citation Style")) + ": " + layout.getDisplayName() + " ..." + "")) + .onRunning(() -> setPreviewText("" + Localization.lang("Processing Citation Style \"%0\"...", layout.getDisplayName()) + "")) .onSuccess(this::setPreviewText) .onFailure(exception -> { LOGGER.error("Error while generating citation style", exception); diff --git a/src/main/java/org/jabref/http/dto/SimpleHttpResponse.java b/src/main/java/org/jabref/http/dto/SimpleHttpResponse.java new file mode 100644 index 00000000000..a07a9b304d9 --- /dev/null +++ b/src/main/java/org/jabref/http/dto/SimpleHttpResponse.java @@ -0,0 +1,109 @@ +package org.jabref.http.dto; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.nio.charset.StandardCharsets; + +import org.jsoup.HttpStatusException; + +public record SimpleHttpResponse(int statusCode, String responseMessage, String responseBody) { + private static final int MAX_RESPONSE_LENGTH = 1024; // 1 KB + + public SimpleHttpResponse(int statusCode, String responseMessage, String responseBody) { + this.statusCode = statusCode; + this.responseBody = truncateResponseBody(responseBody); + this.responseMessage = responseMessage; + } + + public SimpleHttpResponse(HttpURLConnection connection) { + this(getStatusCode(connection), getResponseMessage(connection), getResponseBodyDefaultEmpty(connection)); + } + + public SimpleHttpResponse(HttpStatusException e) { + this(e.getStatusCode(), "", ""); + } + + @Override + public String toString() { + return "SimpleHttpResponse{" + + "statusCode=" + statusCode + + ", responseMessage='" + responseMessage + '\'' + + ", responseBody='" + responseBody + '\'' + + '}'; + } + + private static int getStatusCode(HttpURLConnection connection) { + int statusCode; + try { + statusCode = connection.getResponseCode(); + } catch (IOException e) { + statusCode = -1; + } + return statusCode; + } + + private static String getResponseMessage(HttpURLConnection connection) { + String responseMessage; + try { + responseMessage = connection.getResponseMessage(); + } catch (IOException e) { + responseMessage = ""; + } + return responseMessage; + } + + private static String getResponseBodyDefaultEmpty(HttpURLConnection connection) { + String responseBody; + try { + responseBody = getResponseBody(connection); + } catch (IOException e) { + responseBody = ""; + } + return responseBody; + } + + /** + * Truncates the response body to 1 KB if it exceeds that size. + * Appends "... (truncated)" to indicate truncation. + * + * @param responseBody the original response body + * @return the truncated response body + */ + private static String truncateResponseBody(String responseBody) { + byte[] bytes = responseBody.getBytes(StandardCharsets.UTF_8); + if (bytes.length > MAX_RESPONSE_LENGTH) { + // Truncate the response body to 1 KB and append "... (truncated)" + // Response is in English, thus we append English text - and not localized text + return new String(bytes, 0, MAX_RESPONSE_LENGTH, StandardCharsets.UTF_8) + "... (truncated)"; + } + // Return the original response body if it's within the 1 KB limit + return responseBody; + } + + /** + * Reads the response body from the HttpURLConnection and returns it as a string. + * This method is used to retrieve the response body from the connection, + * which may contain error messages or other information from the server. + * + * @param connection the HttpURLConnection to read the response body from + * @return the response body as a string + * @throws IOException if an I/O error occurs while reading the response body + */ + private static String getResponseBody(HttpURLConnection connection) throws IOException { + InputStream errorStream = connection.getErrorStream(); + if (errorStream == null) { + return ""; + } + try (BufferedReader in = new BufferedReader(new InputStreamReader(errorStream))) { + String inputLine; + StringBuilder content = new StringBuilder(); + while ((content.length() < MAX_RESPONSE_LENGTH) && (inputLine = in.readLine()) != null) { + content.append(inputLine); + } + return truncateResponseBody(content.toString()); + } + } +} diff --git a/src/main/java/org/jabref/logic/JabRefException.java b/src/main/java/org/jabref/logic/JabRefException.java index 687383f9823..4caad91fa1d 100644 --- a/src/main/java/org/jabref/logic/JabRefException.java +++ b/src/main/java/org/jabref/logic/JabRefException.java @@ -33,8 +33,9 @@ public JabRefException(Throwable cause) { @Override public String getLocalizedMessage() { if (localizedMessage == null) { - LOGGER.debug("No localized exception message defined. Falling back to getMessage()."); - return getMessage(); + String message = getMessage(); + LOGGER.debug("No localized exception message defined. Falling back to getMessage() ({}).", message); + return message; } else { return localizedMessage; } diff --git a/src/main/java/org/jabref/logic/importer/EntryBasedParserFetcher.java b/src/main/java/org/jabref/logic/importer/EntryBasedParserFetcher.java index 28795882d36..38a010c62ce 100644 --- a/src/main/java/org/jabref/logic/importer/EntryBasedParserFetcher.java +++ b/src/main/java/org/jabref/logic/importer/EntryBasedParserFetcher.java @@ -12,6 +12,8 @@ import org.jabref.model.entry.BibEntry; +import org.slf4j.LoggerFactory; + /** * Provides a convenient interface for entry-based fetcher, which follow the usual three-step procedure: * 1. Open a URL based on the entry @@ -36,16 +38,16 @@ public interface EntryBasedParserFetcher extends EntryBasedFetcher, ParserFetche default List performSearch(BibEntry entry) throws FetcherException { Objects.requireNonNull(entry); - URL UrlForEntry; + URL urlForEntry; try { - if ((UrlForEntry = getURLForEntry(entry)) == null) { + if ((urlForEntry = getURLForEntry(entry)) == null) { return Collections.emptyList(); } } catch (MalformedURLException | URISyntaxException e) { throw new FetcherException("Search URI is malformed", e); } - try (InputStream stream = new BufferedInputStream(UrlForEntry.openStream())) { + try (InputStream stream = new BufferedInputStream(urlForEntry.openStream())) { List fetchedEntries = getParser().parseEntries(stream); // Post-cleanup @@ -54,9 +56,11 @@ default List performSearch(BibEntry entry) throws FetcherException { return fetchedEntries; } catch (IOException e) { // TODO: Catch HTTP Response 401 errors and report that user has no rights to access resource - throw new FetcherException("A network error occurred", e); + // Same TODO as in org.jabref.logic.net.URLDownload.openConnection. Code should be reused. + LoggerFactory.getLogger(EntryBasedParserFetcher.class).error("Could not fetch from URL {}", urlForEntry, e); + throw new FetcherException(urlForEntry, "A network error occurred", e); } catch (ParseException e) { - throw new FetcherException("An internal parser error occurred", e); + throw new FetcherException(urlForEntry, "An internal parser error occurred", e); } } } diff --git a/src/main/java/org/jabref/logic/importer/FetcherClientException.java b/src/main/java/org/jabref/logic/importer/FetcherClientException.java index ada85e02903..a5780cc08ff 100644 --- a/src/main/java/org/jabref/logic/importer/FetcherClientException.java +++ b/src/main/java/org/jabref/logic/importer/FetcherClientException.java @@ -1,29 +1,14 @@ package org.jabref.logic.importer; +import java.net.URL; + +import org.jabref.http.dto.SimpleHttpResponse; + /** * Should be thrown when you encounter an HTTP status code error >= 400 and < 500. */ public class FetcherClientException extends FetcherException { - private int statusCode; - - public FetcherClientException(String errorMessage, Throwable cause) { - super(errorMessage, cause); - } - - public FetcherClientException(String errorMessage, Throwable cause, int statusCode) { - super(errorMessage, cause); - this.statusCode = statusCode; - } - - public FetcherClientException(String errorMessage) { - super(errorMessage); - } - - public FetcherClientException(String errorMessage, String localizedMessage, Throwable cause) { - super(errorMessage, localizedMessage, cause); - } - - public int getStatusCode() { - return statusCode; + public FetcherClientException(URL url, SimpleHttpResponse httpResponse) { + super(url, httpResponse); } } diff --git a/src/main/java/org/jabref/logic/importer/FetcherException.java b/src/main/java/org/jabref/logic/importer/FetcherException.java index 03557eef2bc..47f672a8c51 100644 --- a/src/main/java/org/jabref/logic/importer/FetcherException.java +++ b/src/main/java/org/jabref/logic/importer/FetcherException.java @@ -1,18 +1,118 @@ package org.jabref.logic.importer; +import java.io.IOException; +import java.net.URL; +import java.util.Optional; +import java.util.regex.Pattern; + +import org.jabref.http.dto.SimpleHttpResponse; import org.jabref.logic.JabRefException; +import org.jabref.model.strings.StringUtil; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class FetcherException extends JabRefException { + private static final Logger LOGGER = LoggerFactory.getLogger(FetcherException.class); + + Pattern API_KEY_PATTERN = Pattern.compile("(?i)(api|key|api[-_]?key)=[^&]*"); + + String REDACTED_STRING = "[REDACTED]"; + + private final String url; + private final SimpleHttpResponse httpResponse; + + public FetcherException(String url, SimpleHttpResponse httpResponse) { + // Empty string handled at org.jabref.logic.importer.FetcherException.getPrefix. + super(""); + this.url = url; + this.httpResponse = httpResponse; + } + + protected FetcherException(URL url, SimpleHttpResponse httpResponse) { + this(url.toString(), httpResponse); + } + + public FetcherException(URL url, Throwable cause) { + super(cause); + httpResponse = null; + this.url = url.toString(); + } + + public FetcherException(URL url, String errorMessage, Throwable cause) { + super(errorMessage, cause); + httpResponse = null; + this.url = url.toString(); + } public FetcherException(String errorMessage, Throwable cause) { + // TODO: Convert IOException e to FetcherClientException + // Same TODO as in org.jabref.logic.importer.EntryBasedParserFetcher.performSearch. + // Maybe do this in org.jabref.logic.importer.FetcherException.getLocalizedMessage + + // Idea (from org.jabref.logic.importer.fetcher.GoogleScholar.performSearchPaged) + // if (e.getMessage().contains("Server returned HTTP response code: 503 for URL")) { super(errorMessage, cause); + if (LOGGER.isDebugEnabled() && (cause instanceof IOException)) { + LOGGER.debug("I/O exception", cause); + } + httpResponse = null; + url = null; } public FetcherException(String errorMessage) { super(errorMessage); + httpResponse = null; + url = null; } public FetcherException(String errorMessage, String localizedMessage, Throwable cause) { super(errorMessage, localizedMessage, cause); + httpResponse = null; + url = null; + } + + @Override + public String getLocalizedMessage() { + // TODO: This should be moved to a separate class converting "any" exception object to a localized message + // TODO: 5% of the "new-ers" pass a "real" localized message. See org.jabref.logic.importer.fetcher.GoogleScholar.addHitsFromQuery. We should maybe make use of this (and rewrite the whole message handling) + // TODO: Try to convert IOException to some more meaningful information here (or at org.jabref.gui.DialogService.showErrorDialogAndWait(org.jabref.logic.importer.FetcherException)). See also org.jabref.logic.net.URLDownload.openConnection + if (httpResponse != null) { + // We decided to not "translate" technical terms (URL, HTTP) + return getPrefix() + "URL: %s\nHTTP %d %s\n%s".formatted(getRedactedUrl(), httpResponse.statusCode(), httpResponse.responseMessage(), httpResponse.responseBody()); + } else if (url != null) { + return getPrefix() + "URL: %s".formatted(getRedactedUrl()); + } else { + return super.getLocalizedMessage(); + } + } + + private String getRedactedUrl() { + return API_KEY_PATTERN.matcher(url).replaceAll(REDACTED_STRING); + } + + private String getPrefix() { + String superLocalizedMessage = super.getLocalizedMessage(); + if (!StringUtil.isBlank(superLocalizedMessage)) { + return superLocalizedMessage + "\n"; + } else { + return ""; + } + } + + public Optional getHttpResponse() { + return Optional.ofNullable(httpResponse); + } + + public Optional getUrl() { + return Optional.ofNullable(url); + } + + public static FetcherException of(URL url, SimpleHttpResponse simpleHttpResponse) { + if (simpleHttpResponse.statusCode() >= 500) { + return new FetcherServerException(url, simpleHttpResponse); + } else { + return new FetcherClientException(url, simpleHttpResponse); + } } } diff --git a/src/main/java/org/jabref/logic/importer/FetcherServerException.java b/src/main/java/org/jabref/logic/importer/FetcherServerException.java index 7b657553b5f..ed67992b65d 100644 --- a/src/main/java/org/jabref/logic/importer/FetcherServerException.java +++ b/src/main/java/org/jabref/logic/importer/FetcherServerException.java @@ -1,28 +1,14 @@ package org.jabref.logic.importer; + +import java.net.URL; + +import org.jabref.http.dto.SimpleHttpResponse; + /** * Should be thrown when you encounter a http status code error >= 500 */ public class FetcherServerException extends FetcherException { - private int statusCode; - - public FetcherServerException(String errorMessage, Throwable cause) { - super(errorMessage, cause); - } - - public FetcherServerException(String errorMessage, Throwable cause, int statusCode) { - super(errorMessage, cause); - this.statusCode = statusCode; - } - - public FetcherServerException(String errorMessage) { - super(errorMessage); - } - - public FetcherServerException(String errorMessage, String localizedMessage, Throwable cause) { - super(errorMessage, localizedMessage, cause); - } - - public int getStatusCode() { - return statusCode; + public FetcherServerException(URL source, SimpleHttpResponse httpResponse) { + super(source, httpResponse); } } diff --git a/src/main/java/org/jabref/logic/importer/IdBasedParserFetcher.java b/src/main/java/org/jabref/logic/importer/IdBasedParserFetcher.java index a938f8ef6f5..55a7e1e615a 100644 --- a/src/main/java/org/jabref/logic/importer/IdBasedParserFetcher.java +++ b/src/main/java/org/jabref/logic/importer/IdBasedParserFetcher.java @@ -29,7 +29,7 @@ public interface IdBasedParserFetcher extends IdBasedFetcher, ParserFetcher { * * @param identifier the ID */ - URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException, FetcherException; + URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException; /** * Returns the parser used to convert the response to a list of {@link BibEntry}. @@ -41,34 +41,31 @@ default Optional performSearchById(String identifier) throws FetcherEx if (StringUtil.isBlank(identifier)) { return Optional.empty(); } - - try (InputStream stream = getUrlDownload(getUrlForIdentifier(identifier)).asInputStream()) { + URL urlForIdentifier; + try { + urlForIdentifier = getUrlForIdentifier(identifier); + } catch (URISyntaxException | MalformedURLException e) { + throw new FetcherException("Search URI is malformed", e); + } + try (InputStream stream = getUrlDownload(urlForIdentifier).asInputStream()) { List fetchedEntries = getParser().parseEntries(stream); - if (fetchedEntries.isEmpty()) { return Optional.empty(); } - if (fetchedEntries.size() > 1) { LOGGER.info("Fetcher {} found more than one result for identifier {}. We will use the first entry.", getName(), identifier); } - BibEntry entry = fetchedEntries.getFirst(); - - // Post-cleanup doPostCleanup(entry); - return Optional.of(entry); - } catch (URISyntaxException e) { - throw new FetcherException("Search URI is malformed", e); } catch (IOException e) { // check for the case where we already have a FetcherException from UrlDownload if (e.getCause() instanceof FetcherException fe) { throw fe; } - throw new FetcherException("A network error occurred", e); + throw new FetcherException(urlForIdentifier, "A network error occurred", e); } catch (ParseException e) { - throw new FetcherException("An internal parser error occurred", e); + throw new FetcherException(urlForIdentifier, "An internal parser error occurred", e); } } } diff --git a/src/main/java/org/jabref/logic/importer/IdParserFetcher.java b/src/main/java/org/jabref/logic/importer/IdParserFetcher.java index 5907d81d1a3..fe6245455eb 100644 --- a/src/main/java/org/jabref/logic/importer/IdParserFetcher.java +++ b/src/main/java/org/jabref/logic/importer/IdParserFetcher.java @@ -52,7 +52,13 @@ public interface IdParserFetcher extends IdFetcher, Par default Optional findIdentifier(BibEntry entry) throws FetcherException { Objects.requireNonNull(entry); - try (InputStream stream = new BufferedInputStream(getURLForEntry(entry).openStream())) { + URL urlForEntry; + try { + urlForEntry = getURLForEntry(entry); + } catch (URISyntaxException | MalformedURLException e) { + throw new FetcherException("Search URL is malformed", e); + } + try (InputStream stream = new BufferedInputStream(urlForEntry.openStream())) { List fetchedEntries = getParser().parseEntries(stream); if (fetchedEntries.isEmpty()) { @@ -63,8 +69,6 @@ default Optional findIdentifier(BibEntry entry) throws FetcherException { fetchedEntries.forEach(this::doPostCleanup); return extractIdentifier(entry, fetchedEntries); - } catch (URISyntaxException e) { - throw new FetcherException("Search URI is malformed", e); } catch (FileNotFoundException e) { LOGGER.debug("Id not found"); return Optional.empty(); @@ -73,9 +77,9 @@ default Optional findIdentifier(BibEntry entry) throws FetcherException { if (e.getCause() instanceof FetcherException fe) { throw fe; } - throw new FetcherException("An I/O exception occurred", e); + throw new FetcherException(urlForEntry, "An I/O exception occurred", e); } catch (ParseException e) { - throw new FetcherException("An internal parser error occurred", e); + throw new FetcherException(urlForEntry, "An internal parser error occurred", e); } } } diff --git a/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java b/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java index 5c2fa3d3dbb..db8bae8ddf0 100644 --- a/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java +++ b/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java @@ -32,9 +32,9 @@ private List getBibEntries(URL urlForQuery) throws FetcherException { fetchedEntries.forEach(this::doPostCleanup); return fetchedEntries; } catch (IOException e) { - throw new FetcherException("A network error occurred while fetching from " + urlForQuery, e); + throw new FetcherException(urlForQuery, "A network error occurred", e); } catch (ParseException e) { - throw new FetcherException("An internal parser error occurred while fetching from " + urlForQuery, e); + throw new FetcherException(urlForQuery, "An internal parser error occurred", e); } } @@ -44,10 +44,10 @@ private List getBibEntries(URL urlForQuery) throws FetcherException { * @param luceneQuery the search query * @param pageNumber the number of the page indexed from 0 */ - URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException; + URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException; @Override - default URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + default URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { return getURLForQuery(luceneQuery, 0); } diff --git a/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java b/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java index 16f8a33598a..4df75fb981f 100644 --- a/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java +++ b/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java @@ -32,6 +32,7 @@ * We need multi inheritance, because a fetcher might implement multiple query types (such as id fetching {@link IdBasedFetcher}), complete entry {@link EntryBasedFetcher}, and search-based fetcher (this class). *

*/ + public interface SearchBasedParserFetcher extends SearchBasedFetcher, ParserFetcher { /** @@ -59,9 +60,11 @@ private List getBibEntries(URL urlForQuery) throws FetcherException { fetchedEntries.forEach(this::doPostCleanup); return fetchedEntries; } catch (IOException e) { - throw new FetcherException("A network error occurred while fetching from " + urlForQuery, e); + // Regular expression to redact API keys from the error message + throw new FetcherException(urlForQuery, e); } catch (ParseException e) { - throw new FetcherException("An internal parser error occurred while fetching from " + urlForQuery, e); + // Regular expression to redact API keys from the error message + throw new FetcherException(urlForQuery, "An internal parser error occurred while fetching", e); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java index 2d36d11e9d6..93572f9b041 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java @@ -8,7 +8,6 @@ import java.util.Optional; import org.jabref.logic.help.HelpFile; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; import org.jabref.logic.importer.fetcher.transformers.DefaultQueryTransformer; @@ -49,7 +48,7 @@ private static String createQueryString(QueryNode query) { * @return query URL */ @Override - public URL getURLForQuery(QueryNode query) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode query) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(SEARCH_URL); uriBuilder.addParameter("AllField", createQueryString(query)); return uriBuilder.build().toURL(); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/AbstractIsbnFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/AbstractIsbnFetcher.java index efa37343f95..80899dd511b 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/AbstractIsbnFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/AbstractIsbnFetcher.java @@ -1,14 +1,13 @@ package org.jabref.logic.importer.fetcher; +import java.net.URISyntaxException; import java.util.Optional; import org.jabref.logic.help.HelpFile; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.IdBasedParserFetcher; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.fileformat.BibtexParser; -import org.jabref.logic.l10n.Localization; import org.jabref.model.entry.identifier.ISBN; public abstract class AbstractIsbnFetcher implements IdBasedParserFetcher { @@ -24,10 +23,14 @@ public Optional getHelpPage() { return Optional.of(HelpFile.FETCHER_ISBN); } - protected void ensureThatIsbnIsValid(String identifier) throws FetcherException { + /** + * @throws URISyntaxException if the ISBN is invalid + * @implNote We could have created a new exception (which causes much implementation efforts) or we could have used "FetcherException", which is currently more used for I/O errors than syntax errors (thus also more WTF). Moreover, a ISBN is "kind of" an URI (even if the isbn: prefix is missing) + */ + protected void ensureThatIsbnIsValid(String identifier) throws URISyntaxException { ISBN isbn = new ISBN(identifier); if (!isbn.isValid()) { - throw new FetcherException(Localization.lang("Invalid ISBN: '%0'.", identifier)); + throw new URISyntaxException(identifier, "Invalid ISBN"); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ArXivFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/ArXivFetcher.java index 1e2fb5000b6..cf487d4ec6b 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ArXivFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ArXivFetcher.java @@ -529,20 +529,30 @@ private Document callApi(String searchQuery, List ids, int star throw new IllegalArgumentException("The arXiv API limits the number of maximal results to be 2000"); } + URIBuilder uriBuilder; try { - URIBuilder uriBuilder = new URIBuilder(API_URL); - // The arXiv API has problems with accents, so we remove them (i.e. Fréchet -> Frechet) - if (StringUtil.isNotBlank(searchQuery)) { - uriBuilder.addParameter("search_query", StringUtil.stripAccents(searchQuery)); - } - if (!ids.isEmpty()) { - uriBuilder.addParameter("id_list", - ids.stream().map(ArXivIdentifier::getNormalized).collect(Collectors.joining(","))); - } - uriBuilder.addParameter("start", String.valueOf(start)); - uriBuilder.addParameter("max_results", String.valueOf(maxResults)); - URL url = uriBuilder.build().toURL(); + uriBuilder = new URIBuilder(API_URL); + } catch (URISyntaxException e) { + throw new FetcherException("Invalid URL", e); + } + // The arXiv API has problems with accents, so we remove them (i.e. Fréchet -> Frechet) + if (StringUtil.isNotBlank(searchQuery)) { + uriBuilder.addParameter("search_query", StringUtil.stripAccents(searchQuery)); + } + if (!ids.isEmpty()) { + uriBuilder.addParameter("id_list", + ids.stream().map(ArXivIdentifier::getNormalized).collect(Collectors.joining(","))); + } + uriBuilder.addParameter("start", String.valueOf(start)); + uriBuilder.addParameter("max_results", String.valueOf(maxResults)); + URL url = null; + try { + url = uriBuilder.build().toURL(); + } catch (MalformedURLException | URISyntaxException e) { + throw new FetcherException("Invalid URL", e); + } + try { DocumentBuilder builder = DOCUMENT_BUILDER_FACTORY.newDocumentBuilder(); HttpURLConnection connection = (HttpURLConnection) url.openConnection(); @@ -552,8 +562,8 @@ private Document callApi(String searchQuery, List ids, int star } else { return builder.parse(connection.getInputStream()); } - } catch (SAXException | ParserConfigurationException | IOException | URISyntaxException exception) { - throw new FetcherException("arXiv API request failed", exception); + } catch (SAXException | ParserConfigurationException | IOException exception) { + throw new FetcherException(url, "arXiv API request failed", exception); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java index 147fd4866e5..373cd064427 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java @@ -1,6 +1,5 @@ package org.jabref.logic.importer.fetcher; -import java.io.IOException; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; @@ -88,7 +87,7 @@ public String getName() { * @return URL which points to a search request for given query */ @Override - public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException { URIBuilder builder = new URIBuilder(API_SEARCH_URL); builder.addParameter("q", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); builder.addParameter("fl", "bibcode"); @@ -129,7 +128,7 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR * @return URL which points to a search URL for given identifier */ @Override - public URL getUrlForIdentifier(String identifier) throws FetcherException, URISyntaxException, MalformedURLException { + public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException { String query = "doi:\"" + identifier + "\" OR " + "bibcode:\"" + identifier + "\""; URIBuilder builder = new URIBuilder(API_SEARCH_URL); builder.addParameter("q", query); @@ -178,14 +177,15 @@ public List performSearch(BibEntry entry) throws FetcherException { return Collections.emptyList(); } + URL urlForEntry = null; try { - List bibcodes = fetchBibcodes(getURLForEntry(entry)); - return performSearchByIds(bibcodes); - } catch (URISyntaxException e) { + urlForEntry = getURLForEntry(entry); + } catch (URISyntaxException | MalformedURLException e) { throw new FetcherException("Search URI is malformed", e); - } catch (IOException e) { - throw new FetcherException("A network error occurred", e); } + + List bibcodes = fetchBibcodes(urlForEntry); + return performSearchByIds(bibcodes); } /** @@ -203,9 +203,8 @@ private List fetchBibcodes(URL url) throws FetcherException { bibcodes.add(codes.getJSONObject(i).getString("bibcode")); } return bibcodes; - } catch (IOException e) { - throw new FetcherException("A network error occurred", e); } catch (JSONException e) { + LOGGER.error("Error while parsing JSON", e); return Collections.emptyList(); } } @@ -216,24 +215,24 @@ public Optional performSearchById(String identifier) throws FetcherExc return Optional.empty(); } + URL urlForIdentifier; try { - List bibcodes = fetchBibcodes(getUrlForIdentifier(identifier)); - List fetchedEntries = performSearchByIds(bibcodes); - - if (fetchedEntries.isEmpty()) { - return Optional.empty(); - } - if (fetchedEntries.size() > 1) { - LOGGER.info("Fetcher " + getName() + "found more than one result for identifier " + identifier - + ". We will use the first entry."); - } - BibEntry entry = fetchedEntries.getFirst(); - return Optional.of(entry); - } catch (URISyntaxException e) { + urlForIdentifier = getUrlForIdentifier(identifier); + } catch (URISyntaxException | MalformedURLException e) { throw new FetcherException("Search URI is malformed", e); - } catch (IOException e) { - throw new FetcherException("A network error occurred", e); } + + List bibcodes = fetchBibcodes(urlForIdentifier); + List fetchedEntries = performSearchByIds(bibcodes); + + if (fetchedEntries.isEmpty()) { + return Optional.empty(); + } + if (fetchedEntries.size() > 1) { + LOGGER.info("Fetcher {} found more than one result for identifier {}. We will use the first entry.", getName(), identifier); + } + BibEntry entry = fetchedEntries.getFirst(); + return Optional.of(entry); } /** @@ -245,9 +244,17 @@ private List performSearchByIds(Collection identifiers) throws if (ids.isEmpty()) { return Collections.emptyList(); } + + URL urLforExport; + try { + urLforExport = getURLforExport(); + } catch (URISyntaxException | MalformedURLException e) { + throw new FetcherException("Search URI is malformed", e); + } + try { String postData = buildPostData(ids); - URLDownload download = new URLDownload(getURLforExport()); + URLDownload download = new URLDownload(urLforExport); download.addHeader("Authorization", "Bearer " + importerPreferences.getApiKey(getName()).orElse(API_KEY)); download.addHeader("ContentType", "application/json"); download.setPostData(postData); @@ -264,14 +271,11 @@ private List performSearchByIds(Collection identifiers) throws return fetchedEntries; } catch (JSONException e) { + LOGGER.error("Error while parsing JSON", e); return Collections.emptyList(); } - } catch (URISyntaxException e) { - throw new FetcherException("Search URI is malformed", e); - } catch (IOException e) { - throw new FetcherException("A network error occurred", e); } catch (ParseException e) { - throw new FetcherException("An internal parser error occurred", e); + throw new FetcherException(urLforExport, "An internal parser error occurred", e); } } @@ -280,10 +284,8 @@ public List performSearch(QueryNode luceneQuery) throws FetcherExcepti URL urlForQuery; try { urlForQuery = getURLForQuery(luceneQuery); - } catch (URISyntaxException e) { + } catch (URISyntaxException | MalformedURLException e) { throw new FetcherException("Search URI is malformed", e); - } catch (IOException e) { - throw new FetcherException("A network error occurred", e); } List bibCodes = fetchBibcodes(urlForQuery); return performSearchByIds(bibCodes); @@ -294,10 +296,8 @@ public Page performSearchPaged(QueryNode luceneQuery, int pageNumber) URL urlForQuery; try { urlForQuery = getURLForQuery(luceneQuery, pageNumber); - } catch (URISyntaxException e) { + } catch (URISyntaxException | MalformedURLException e) { throw new FetcherException("Search URI is malformed", e); - } catch (IOException e) { - throw new FetcherException("A network error occurred", e); } // This is currently just interpreting the complex query as a default string query List bibCodes = fetchBibcodes(urlForQuery); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/BibsonomyScraper.java b/src/main/java/org/jabref/logic/importer/fetcher/BibsonomyScraper.java index f506d0128f2..724ac0a6f0a 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/BibsonomyScraper.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/BibsonomyScraper.java @@ -4,6 +4,7 @@ import java.net.URL; import java.util.Optional; +import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.fileformat.BibtexParser; @@ -38,7 +39,7 @@ public static Optional getEntry(String entryUrl, ImportFormatPreferenc URL url = new URL(BibsonomyScraper.BIBSONOMY_SCRAPER + cleanURL + BibsonomyScraper.BIBSONOMY_SCRAPER_POST); String bibtex = new URLDownload(url).asString(); return BibtexParser.singleFromString(bibtex, importFormatPreferences); - } catch (IOException ex) { + } catch (IOException | FetcherException ex) { LOGGER.warn("Could not download entry", ex); return Optional.empty(); } catch (ParseException ex) { diff --git a/src/main/java/org/jabref/logic/importer/fetcher/BiodiversityLibrary.java b/src/main/java/org/jabref/logic/importer/fetcher/BiodiversityLibrary.java index 8b2a0fec78d..99ce61c4ca9 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/BiodiversityLibrary.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/BiodiversityLibrary.java @@ -1,6 +1,5 @@ package org.jabref.logic.importer.fetcher; -import java.io.IOException; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; @@ -24,6 +23,7 @@ import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.StandardEntryType; +import com.google.common.annotations.VisibleForTesting; import kong.unirest.core.json.JSONArray; import kong.unirest.core.json.JSONException; import kong.unirest.core.json.JSONObject; @@ -80,7 +80,8 @@ public URL getItemMetadataURL(String identifier) throws URISyntaxException, Malf return uriBuilder.build().toURL(); } - public URL getPartMetadataURL(String identifier) throws URISyntaxException, MalformedURLException { + @VisibleForTesting + URL getPartMetadataURL(String identifier) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(getBaseURL().toURI()); uriBuilder.addParameter("op", "GetPartMetadata"); uriBuilder.addParameter("pages", "f"); @@ -90,17 +91,22 @@ public URL getPartMetadataURL(String identifier) throws URISyntaxException, Malf return uriBuilder.build().toURL(); } - public JSONObject getDetails(URL url) throws IOException { + public JSONObject getDetails(URL url) throws FetcherException { URLDownload download = new URLDownload(url); String response = download.asString(); Logger.debug("Response {}", response); return new JSONObject(response).getJSONArray("Result").getJSONObject(0); } - public BibEntry parseBibJSONtoBibtex(JSONObject item, BibEntry entry) throws IOException, URISyntaxException { + public BibEntry parseBibJSONtoBibtex(JSONObject item, BibEntry entry) throws FetcherException { if (item.has("BHLType")) { if ("Part".equals(item.getString("BHLType"))) { - URL url = getPartMetadataURL(item.getString("PartID")); + URL url; + try { + url = getPartMetadataURL(item.getString("PartID")); + } catch (URISyntaxException | MalformedURLException e) { + throw new FetcherException("Malformed URL", e); + } JSONObject itemsDetails = getDetails(url); entry.setField(StandardField.LANGUAGE, itemsDetails.optString("Language", "")); @@ -113,7 +119,12 @@ public BibEntry parseBibJSONtoBibtex(JSONObject item, BibEntry entry) throws IOE } if ("Item".equals(item.getString("BHLType"))) { - URL url = getItemMetadataURL(item.getString("ItemID")); + URL url = null; + try { + url = getItemMetadataURL(item.getString("ItemID")); + } catch (URISyntaxException | MalformedURLException e) { + throw new FetcherException("Malformed URL", e); + } JSONObject itemsDetails = getDetails(url); entry.setField(StandardField.EDITOR, itemsDetails.optString("Sponsor", "")); entry.setField(StandardField.PUBLISHER, itemsDetails.optString("HoldingInstitution", "")); @@ -186,7 +197,7 @@ public Parser getParser() { BibEntry entry = jsonResultToBibEntry(item); try { entry = parseBibJSONtoBibtex(item, entry); - } catch (JSONException | IOException | URISyntaxException exception) { + } catch (JSONException | FetcherException exception) { throw new ParseException("Error when parsing entry", exception); } entries.add(entry); @@ -197,7 +208,7 @@ public Parser getParser() { } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(getBaseURL().toURI()); BiodiversityLibraryTransformer transformer = new BiodiversityLibraryTransformer(); uriBuilder.addParameter("op", "PublicationSearch"); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java b/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java index 4d63622b68f..20e73b6bb1b 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java @@ -6,6 +6,7 @@ import java.util.Objects; import java.util.Optional; +import org.jabref.http.dto.SimpleHttpResponse; import org.jabref.logic.help.HelpFile; import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.FulltextFetcher; @@ -16,14 +17,19 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; +import kong.unirest.core.HttpResponse; import kong.unirest.core.JsonNode; import kong.unirest.core.Unirest; import kong.unirest.core.json.JSONArray; import kong.unirest.core.json.JSONElement; import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class CiteSeer implements SearchBasedFetcher, FulltextFetcher { + private static final Logger LOGGER = LoggerFactory.getLogger(CiteSeer.class); + private static final String BASE_URL = "citeseerx.ist.psu.edu"; private static final String API_URL = "https://citeseerx.ist.psu.edu/api/search"; @@ -50,21 +56,27 @@ public List performSearch(QueryNode luceneQuery) throws FetcherExcepti // ADR-0014 try { JSONElement payload = getPayloadJSON(luceneQuery); - JsonNode requestResponse = Unirest.post(API_URL) - .header("authority", BASE_URL) - .header("accept", "application/json, text/plain, */*") - .header("content-type", "application/json;charset=UTF-8") - .header("origin", "https://" + BASE_URL) - .body(payload) - .asJson().getBody(); - - Optional jsonResponse = Optional.of(requestResponse) - .map(JsonNode::getObject) - .filter(Objects::nonNull) - .map(response -> response.optJSONArray("response")) - .filter(Objects::nonNull); - - if (!jsonResponse.isPresent()) { + HttpResponse httpResponse = Unirest.post(API_URL) + .header("authority", BASE_URL) + .header("accept", "application/json, text/plain, */*") + .header("content-type", "application/json;charset=UTF-8") + .header("origin", "https://" + BASE_URL) + .body(payload) + .asJson(); + if (!httpResponse.isSuccess()) { + LOGGER.debug("No success"); + // TODO: body needs to be added to the exception, but we currently only have JSON available, but the error is most probably simple text (or HTML) + SimpleHttpResponse simpleHttpResponse = new SimpleHttpResponse(httpResponse.getStatus(), httpResponse.getStatusText(), ""); + throw new FetcherException(API_URL, simpleHttpResponse); + } + + JsonNode requestResponse = httpResponse.getBody(); + Optional jsonResponse = Optional.ofNullable(requestResponse) + .map(JsonNode::getObject) + .map(response -> response.optJSONArray("response")); + + if (jsonResponse.isEmpty()) { + LOGGER.debug("No entries found for query: {}", luceneQuery); return List.of(); } @@ -72,7 +84,7 @@ public List performSearch(QueryNode luceneQuery) throws FetcherExcepti List fetchedEntries = parser.parseCiteSeerResponse(jsonResponse.orElse(new JSONArray())); return fetchedEntries; } catch (ParseException ex) { - throw new FetcherException("An internal parser error occurred while parsing CiteSeer entries, ", ex); + throw new FetcherException("An internal parser error occurred while parsing CiteSeer entries", ex); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java index 797b2bc1860..018a9b4c9c5 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java @@ -10,7 +10,6 @@ import org.jabref.logic.formatter.bibtexfields.RemoveNewlinesFormatter; import org.jabref.logic.formatter.bibtexfields.RemoveRedundantSpacesFormatter; import org.jabref.logic.formatter.bibtexfields.ReplaceTabsBySpaceFormater; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; @@ -35,7 +34,7 @@ public CollectionOfComputerScienceBibliographiesFetcher(ImportFormatPreferences } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { return new URIBuilder(BASIC_SEARCH_URL) .addParameter("query", new CollectionOfComputerScienceBibliographiesQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")) .addParameter("sort", "score") diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesParser.java b/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesParser.java index 4a351e40cfc..c2f8aebe2be 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesParser.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesParser.java @@ -9,6 +9,7 @@ import java.util.stream.Collectors; import org.jabref.logic.formatter.bibtexfields.HtmlToUnicodeFormatter; +import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.Parser; @@ -38,7 +39,7 @@ public List parseEntries(InputStream inputStream) throws ParseExceptio .collect(Collectors.joining()); return bibtexParser.parseEntries(bibtexDataString); - } catch (IOException e) { + } catch (IOException | FetcherException e) { throw new ParseException(e); } } @@ -51,7 +52,7 @@ private List matchRegexFromInputStreamHtml(InputStream inputStream, Patt } } - private List parseBibtexStringsFromLinks(List links) throws IOException { + private List parseBibtexStringsFromLinks(List links) throws IOException, FetcherException { List bibtexStringsFromAllLinks = new ArrayList<>(); for (String link : links) { try (InputStream inputStream = new URLDownload(link).asInputStream()) { diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java b/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java index 35f85e651e5..7a3e20f19a7 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java @@ -67,14 +67,14 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(API_URL); uriBuilder.addParameter("query", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); return uriBuilder.build().toURL(); } @Override - public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(API_URL + "/" + identifier); return uriBuilder.build().toURL(); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java index cca32ab6b7d..188dc6c9158 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java @@ -12,7 +12,6 @@ import org.jabref.logic.cleanup.FieldFormatterCleanups; import org.jabref.logic.formatter.bibtexfields.ClearFormatter; import org.jabref.logic.help.HelpFile; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; @@ -44,7 +43,7 @@ public DBLPFetcher(ImportFormatPreferences importFormatPreferences) { } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(BASIC_SEARCH_URL); uriBuilder.addParameter("q", new DBLPQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); uriBuilder.addParameter("h", String.valueOf(100)); // number of hits diff --git a/src/main/java/org/jabref/logic/importer/fetcher/DOABFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/DOABFetcher.java index 33f70d7dc61..3408c631142 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/DOABFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/DOABFetcher.java @@ -8,7 +8,6 @@ import java.util.List; import java.util.StringJoiner; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; import org.jabref.logic.importer.fetcher.transformers.DefaultQueryTransformer; @@ -39,7 +38,7 @@ public String getName() { } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { URIBuilder builder = new URIBuilder(SEARCH_URL); String query = new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse(""); // adding quotations for the query for more specified results diff --git a/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java index 81b7e9af2b7..42f8152f387 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java @@ -12,7 +12,6 @@ import java.util.stream.Collectors; import org.jabref.logic.help.HelpFile; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; @@ -185,7 +184,7 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(SEARCH_URL); DOAJFetcher.addPath(uriBuilder, new DefaultLuceneQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); // Number of results diff --git a/src/main/java/org/jabref/logic/importer/fetcher/DiVA.java b/src/main/java/org/jabref/logic/importer/fetcher/DiVA.java index 88dcd6ebf2f..241c58a3d86 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/DiVA.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/DiVA.java @@ -6,7 +6,6 @@ import java.util.Optional; import org.jabref.logic.help.HelpFile; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.IdBasedParserFetcher; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; @@ -37,7 +36,7 @@ public Optional getHelpPage() { } @Override - public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder("http://www.diva-portal.org/smash/getreferences"); uriBuilder.addParameter("referenceFormat", "BibTex"); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/DoiFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/DoiFetcher.java index 9fa428e5ead..49a28e140ba 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/DoiFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/DoiFetcher.java @@ -2,6 +2,7 @@ import java.io.IOException; import java.net.HttpURLConnection; +import java.net.MalformedURLException; import java.net.URL; import java.net.URLConnection; import java.util.Collections; @@ -94,7 +95,7 @@ private void doAPILimiting(String identifier) { LOGGER.trace("Thread %s, searching for DOI '%s', waited %.2fs because of API rate limiter".formatted( Thread.currentThread().threadId(), identifier, waitingTime)); } - } catch (IOException e) { + } catch (FetcherException | MalformedURLException e) { LOGGER.warn("Could not limit DOI API access rate", e); } } @@ -114,64 +115,63 @@ protected CompletableFuture> asyncPerformSearchById(String id public Optional performSearchById(String identifier) throws FetcherException { Optional doi = DOI.parse(identifier); + if (doi.isEmpty()) { + throw new FetcherException(Localization.lang("Invalid DOI: '%0'.", identifier)); + } + + URL doiURL; + try { + doiURL = new URL(doi.get().getURIAsASCIIString()); + } catch (MalformedURLException e) { + throw new FetcherException("Malformed URL", e); + } + try { - if (doi.isPresent()) { - Optional fetchedEntry; + Optional fetchedEntry; - // mEDRA does not return a parsable bibtex string - Optional agency = getAgency(doi.get()); - if (agency.isPresent() && "medra".equalsIgnoreCase(agency.get())) { - return new Medra().performSearchById(identifier); - } - URL doiURL = new URL(doi.get().getURIAsASCIIString()); - - // BibTeX data - URLDownload download = getUrlDownload(doiURL); - download.addHeader("Accept", MediaTypes.APPLICATION_BIBTEX); - - String bibtexString; - URLConnection openConnection; - try { - openConnection = download.openConnection(); - bibtexString = URLDownload.asString(openConnection).trim(); - } catch (IOException e) { - // an IOException with a nested FetcherException will be thrown when you encounter a 400x or 500x http status code - if (e.getCause() instanceof FetcherException fe) { - throw fe; - } - throw e; - } + // mEDRA does not return a parsable bibtex string + Optional agency = getAgency(doi.get()); + if (agency.isPresent() && "medra".equalsIgnoreCase(agency.get())) { + return new Medra().performSearchById(identifier); + } - // BibTeX entry - fetchedEntry = BibtexParser.singleFromString(bibtexString, preferences); - fetchedEntry.ifPresent(this::doPostCleanup); + // BibTeX data + URLDownload download = getUrlDownload(doiURL); + download.addHeader("Accept", MediaTypes.APPLICATION_BIBTEX); - // Crossref has a dynamic API rate limit - if (agency.isPresent() && "crossref".equalsIgnoreCase(agency.get())) { - updateCrossrefAPIRate(openConnection); - } + String bibtexString; + URLConnection openConnection; - // Check if the entry is an APS journal and add the article id as the page count if page field is missing - if (fetchedEntry.isPresent() && fetchedEntry.get().hasField(StandardField.DOI)) { - BibEntry entry = fetchedEntry.get(); - if (isAPSJournal(entry, entry.getField(StandardField.DOI).get()) && !entry.hasField(StandardField.PAGES)) { - setPageCountToArticleId(entry, entry.getField(StandardField.DOI).get()); - } - } + openConnection = download.openConnection(); + bibtexString = URLDownload.asString(openConnection).trim(); + + // BibTeX entry + fetchedEntry = BibtexParser.singleFromString(bibtexString, preferences); + fetchedEntry.ifPresent(this::doPostCleanup); - if (openConnection instanceof HttpURLConnection connection) { - connection.disconnect(); + // Crossref has a dynamic API rate limit + if (agency.isPresent() && "crossref".equalsIgnoreCase(agency.get())) { + updateCrossrefAPIRate(openConnection); + } + + // Check if the entry is an APS journal and add the article id as the page count if page field is missing + if (fetchedEntry.isPresent() && fetchedEntry.get().hasField(StandardField.DOI)) { + BibEntry entry = fetchedEntry.get(); + if (isAPSJournal(entry, entry.getField(StandardField.DOI).get()) && !entry.hasField(StandardField.PAGES)) { + setPageCountToArticleId(entry, entry.getField(StandardField.DOI).get()); } - return fetchedEntry; - } else { - throw new FetcherException(Localization.lang("Invalid DOI: '%0'.", identifier)); } + + if (openConnection instanceof HttpURLConnection connection) { + connection.disconnect(); + } + return fetchedEntry; } catch (IOException e) { - throw new FetcherException(Localization.lang("Connection error"), e); + throw new FetcherException(doiURL, Localization.lang("Connection error"), e); } catch (ParseException e) { - throw new FetcherException("Could not parse BibTeX entry", e); + throw new FetcherException(doiURL, "Could not parse BibTeX entry", e); } catch (JSONException e) { - throw new FetcherException("Could not retrieve Registration Agency", e); + throw new FetcherException(doiURL, "Could not retrieve Registration Agency", e); } } @@ -215,7 +215,7 @@ public List performSearch(BibEntry entry) throws FetcherException { * * @param doi the DOI to be searched */ - public Optional getAgency(DOI doi) throws IOException { + public Optional getAgency(DOI doi) throws FetcherException, MalformedURLException { Optional agency = Optional.empty(); try { URLDownload download = getUrlDownload(new URL(DOI.AGENCY_RESOLVER + "/" + doi.getDOI())); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java b/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java index b2de8a817d1..f0461e8338e 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.io.StringReader; import java.net.HttpCookie; +import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; @@ -132,7 +133,8 @@ private void addHitsFromQuery(List entryList, String queryURL) throws String content = new URLDownload(queryURL).asString(); if (needsCaptcha(content)) { - throw new FetcherException("Fetching from Google Scholar failed: Captacha hit at " + queryURL + ".", + // TODO: Remove "null" + throw new FetcherException(queryURL, "Fetching from Google Scholar failed: Captacha hit." + Localization.lang("This might be caused by reaching the traffic limitation of Google Scholar (see 'Help' for details)."), null); } @@ -153,7 +155,7 @@ private BibEntry downloadEntry(String link) throws IOException, FetcherException } else { Collection entries = result.getDatabase().getEntries(); if (entries.size() != 1) { - LOGGER.debug(entries.size() + " entries found! (" + link + ")"); + LOGGER.debug("{} entries found! ({})", entries.size(), link); throw new FetcherException("Parsing entries from Google Scholar bib file failed."); } else { BibEntry entry = entries.iterator().next(); @@ -179,40 +181,49 @@ private void obtainAndModifyCookie() throws FetcherException { public Page performSearchPaged(QueryNode luceneQuery, int pageNumber) throws FetcherException { ScholarQueryTransformer queryTransformer = new ScholarQueryTransformer(); String transformedQuery = queryTransformer.transformLuceneQuery(luceneQuery).orElse(""); + + obtainAndModifyCookie(); + + URIBuilder uriBuilder; try { - obtainAndModifyCookie(); - List foundEntries = new ArrayList<>(10); - URIBuilder uriBuilder = new URIBuilder(BASIC_SEARCH_URL); - uriBuilder.addParameter("hl", "en"); - uriBuilder.addParameter("btnG", "Search"); - uriBuilder.addParameter("q", transformedQuery); - uriBuilder.addParameter("start", String.valueOf(pageNumber * getPageSize())); - uriBuilder.addParameter("num", String.valueOf(getPageSize())); - uriBuilder.addParameter("as_ylo", String.valueOf(queryTransformer.getStartYear())); - uriBuilder.addParameter("as_yhi", String.valueOf(queryTransformer.getEndYear())); - - try { - addHitsFromQuery(foundEntries, uriBuilder.toString()); + uriBuilder = new URIBuilder(BASIC_SEARCH_URL); + } catch (URISyntaxException e) { + throw new FetcherException("Building URI failed.", e); + } + uriBuilder.addParameter("hl", "en"); + uriBuilder.addParameter("btnG", "Search"); + uriBuilder.addParameter("q", transformedQuery); + uriBuilder.addParameter("start", String.valueOf(pageNumber * getPageSize())); + uriBuilder.addParameter("num", String.valueOf(getPageSize())); + uriBuilder.addParameter("as_ylo", String.valueOf(queryTransformer.getStartYear())); + uriBuilder.addParameter("as_yhi", String.valueOf(queryTransformer.getEndYear())); - if (foundEntries.size() == 10) { - uriBuilder.addParameter("start", "10"); - addHitsFromQuery(foundEntries, uriBuilder.toString()); - } - } catch (IOException e) { - LOGGER.info("IOException for URL {}", uriBuilder.toString()); - // if there are too much requests from the same IP adress google is answering with a 503 and redirecting to a captcha challenge - // The caught IOException looks for example like this: - // java.io.IOException: Server returned HTTP response code: 503 for URL: https://ipv4.google.com/sorry/index?continue=https://scholar.google.com/scholar%3Fhl%3Den%26btnG%3DSearch%26q%3Dbpmn&hl=en&q=CGMSBI0NBDkYuqy9wAUiGQDxp4NLQCWbIEY1HjpH5zFJhv4ANPGdWj0 - if (e.getMessage().contains("Server returned HTTP response code: 503 for URL")) { - throw new FetcherException("Fetching from Google Scholar failed.", - Localization.lang("This might be caused by reaching the traffic limitation of Google Scholar (see 'Help' for details)."), e); - } else { - throw new FetcherException("Error while fetching from " + getName(), e); + List foundEntries = new ArrayList<>(10); + + try { + addHitsFromQuery(foundEntries, uriBuilder.toString()); + if (foundEntries.size() == 10) { + uriBuilder.addParameter("start", "10"); + addHitsFromQuery(foundEntries, uriBuilder.toString()); + } + } catch (IOException e) { + LOGGER.info("IOException for URL {}", uriBuilder); + // if there are too much requests from the same IP adress google is answering with a 503 and redirecting to a captcha challenge + // The caught IOException looks for example like this: + // java.io.IOException: Server returned HTTP response code: 503 for URL: https://ipv4.google.com/sorry/index?continue=https://scholar.google.com/scholar%3Fhl%3Den%26btnG%3DSearch%26q%3Dbpmn&hl=en&q=CGMSBI0NBDkYuqy9wAUiGQDxp4NLQCWbIEY1HjpH5zFJhv4ANPGdWj0 + if (e.getMessage().contains("Server returned HTTP response code: 503 for URL")) { + throw new FetcherException("Fetching from Google Scholar failed.", + Localization.lang("This might be caused by reaching the traffic limitation of Google Scholar (see 'Help' for details)."), e); + } else { + URL url; + try { + url = uriBuilder.build().toURL(); + } catch (URISyntaxException | MalformedURLException ex) { + throw new FetcherException("Wrong URL syntax", e); } + throw new FetcherException(url, "Error while fetching from " + getName(), e); } - return new Page<>(transformedQuery, pageNumber, foundEntries); - } catch (URISyntaxException e) { - throw new FetcherException("Error while fetching from " + getName(), e); } + return new Page<>(transformedQuery, pageNumber, foundEntries); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java index b2479cc90fb..0576c90e6ea 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java @@ -8,6 +8,7 @@ import java.util.Optional; import java.util.stream.Collectors; +import org.jabref.http.dto.SimpleHttpResponse; import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.ParseException; @@ -49,17 +50,15 @@ private Optional parseUsingGrobid(String plainText) throws FetcherExce try { return grobidService.processCitation(plainText, importFormatPreferences, GrobidService.ConsolidateCitations.WITH_METADATA); } catch (HttpStatusException e) { - String msg = "Connection failure."; - LOGGER.debug(msg, e); - throw new FetcherException(msg, e.getCause()); + LOGGER.debug("Could not connect to Grobid", e); + throw new FetcherException("{grobid}", new SimpleHttpResponse(e)); } catch (SocketTimeoutException e) { String msg = "Connection timed out."; LOGGER.debug(msg, e); throw new FetcherException(msg, e.getCause()); } catch (IOException | ParseException e) { - String msg = "Could not process citation. " + e.getMessage(); - LOGGER.debug(msg, e); - return Optional.empty(); + LOGGER.debug("Could not process citation", e); + throw new FetcherException("Could not process citation", e); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java index 24513e0084d..ae463ba3391 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java @@ -11,7 +11,6 @@ import org.jabref.logic.formatter.bibtexfields.NormalizeNamesFormatter; import org.jabref.logic.formatter.bibtexfields.NormalizePagesFormatter; import org.jabref.logic.help.HelpFile; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; @@ -48,7 +47,7 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(URL_PATTERN); uriBuilder.addParameter("version", "1.1"); uriBuilder.addParameter("operation", "searchRetrieve"); @@ -62,7 +61,7 @@ public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, Malf } @Override - public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException { this.ensureThatIsbnIsValid(identifier); URIBuilder uriBuilder = new URIBuilder(URL_PATTERN); uriBuilder.addParameter("version", "1.1"); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java b/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java index e9293e65065..185e9e3906b 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java @@ -1,7 +1,6 @@ package org.jabref.logic.importer.fetcher; import java.io.BufferedReader; -import java.io.IOException; import java.io.InputStreamReader; import java.net.MalformedURLException; import java.net.URISyntaxException; @@ -135,7 +134,7 @@ private static BibEntry parseJsonResponse(JSONObject jsonEntry, Character keywor } @Override - public Optional findFullText(BibEntry entry) throws IOException { + public Optional findFullText(BibEntry entry) throws FetcherException { Objects.requireNonNull(entry); String stampString = ""; @@ -162,7 +161,12 @@ public Optional findFullText(BibEntry entry) throws IOException { Optional doi = entry.getField(StandardField.DOI).flatMap(DOI::parse); if (doi.isPresent() && doi.get().getDOI().startsWith(IEEE_DOI) && doi.get().getExternalURI().isPresent()) { // Download the HTML page from IEEE - URLDownload urlDownload = new URLDownload(doi.get().getExternalURI().get().toURL()); + URLDownload urlDownload = null; + try { + urlDownload = new URLDownload(doi.get().getExternalURI().get().toURL()); + } catch (MalformedURLException e) { + throw new FetcherException("Malformed URL", e); + } // We don't need to modify the cookies, but we need support for them urlDownload.getCookieFromUrl(); @@ -182,7 +186,12 @@ public Optional findFullText(BibEntry entry) throws IOException { } // Download the HTML page containing a frame with the PDF - URLDownload urlDownload = new URLDownload(BASE_URL + stampString); + URLDownload urlDownload; + try { + urlDownload = new URLDownload(BASE_URL + stampString); + } catch (MalformedURLException e) { + throw new FetcherException("Malformed URL", e); + } // We don't need to modify the cookies, but we need support for them urlDownload.getCookieFromUrl(); @@ -192,7 +201,13 @@ public Optional findFullText(BibEntry entry) throws IOException { if (matcher.find()) { // The PDF was found LOGGER.debug("Full text document found on IEEE Xplore"); - return Optional.of(new URL(matcher.group(1))); + URL value; + try { + value = new URL(matcher.group(1)); + } catch (MalformedURLException e) { + throw new FetcherException("Malformed URL", e); + } + return Optional.of(value); } return Optional.empty(); } @@ -258,7 +273,7 @@ public String getTestUrl() { } @Override - public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException { // transformer is stored globally, because we need to filter out the bib entries by the year manually // the transformer stores the min and max year transformer = new IEEEQueryTransformer(); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java index 71cf9817a3a..8038ea2edd0 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java @@ -1,11 +1,9 @@ package org.jabref.logic.importer.fetcher; -import java.io.IOException; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; -import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -57,7 +55,7 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(INSPIRE_HOST); uriBuilder.addParameter("q", new DefaultLuceneQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); return uriBuilder.build().toURL(); @@ -88,27 +86,33 @@ public Parser getParser() { @Override public List performSearch(BibEntry entry) throws FetcherException { - List results = new ArrayList<>(); Optional doi = entry.getField(StandardField.DOI); Optional archiveprefix = entry.getFieldOrAlias(StandardField.ARCHIVEPREFIX); Optional eprint = entry.getField(StandardField.EPRINT); - String url; + String urlString; if (archiveprefix.filter("arxiv"::equals).isPresent() && eprint.isPresent()) { - url = INSPIRE_ARXIV_HOST + eprint.get(); + urlString = INSPIRE_ARXIV_HOST + eprint.get(); } else if (doi.isPresent()) { - url = INSPIRE_DOI_HOST + doi.get(); + urlString = INSPIRE_DOI_HOST + doi.get(); } else { - return results; + return List.of(); + } + + URL url; + try { + url = new URI(urlString).toURL(); + } catch (MalformedURLException | URISyntaxException e) { + throw new FetcherException("Invalid URL", e); } try { - URLDownload download = getUrlDownload(new URI(url).toURL()); - results = getParser().parseEntries(download.asInputStream()); + URLDownload download = getUrlDownload(url); + List results = getParser().parseEntries(download.asInputStream()); results.forEach(this::doPostCleanup); return results; - } catch (IOException | ParseException | URISyntaxException e) { - throw new FetcherException("Error occurred during fetching", e); + } catch (ParseException e) { + throw new FetcherException(url, e); } } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ISIDOREFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/ISIDOREFetcher.java index d882a05d0f0..208ac577542 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ISIDOREFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ISIDOREFetcher.java @@ -7,7 +7,6 @@ import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.StringJoiner; @@ -76,15 +75,13 @@ public Parser getParser() { Element entryElement = document.getDocumentElement(); if (entryElement == null) { - return Collections.emptyList(); + return List.of(); } return parseXMl(entryElement); } catch (FetcherException e) { Unchecked.throwChecked(e); - } catch (ParserConfigurationException | - IOException | - SAXException e) { + } catch (ParserConfigurationException | IOException | SAXException e) { Unchecked.throwChecked(new FetcherException("Issue with parsing link", e)); } return null; @@ -99,7 +96,7 @@ public URLDownload getUrlDownload(URL url) { } @Override - public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException { ISIDOREQueryTransformer queryTransformer = new ISIDOREQueryTransformer(); String transformedQuery = queryTransformer.transformLuceneQuery(luceneQuery).orElse(""); URIBuilder uriBuilder = new URIBuilder(SOURCE_WEB_SEARCH); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/IacrEprintFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/IacrEprintFetcher.java index 7233c998ae3..d1d630ffe21 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/IacrEprintFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/IacrEprintFetcher.java @@ -1,6 +1,8 @@ package org.jabref.logic.importer.fetcher; import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URI; import java.net.URL; import java.util.Objects; import java.util.Optional; @@ -19,10 +21,14 @@ import org.jabref.model.entry.field.StandardField; import org.jabref.model.strings.StringUtil; -public class IacrEprintFetcher implements FulltextFetcher, IdBasedFetcher { +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +public class IacrEprintFetcher implements FulltextFetcher, IdBasedFetcher { public static final String NAME = "IACR eprints"; + private static final Logger LOGGER = LoggerFactory.getLogger(IacrEprintFetcher.class); + private static final Pattern WITHOUT_LETTERS_SPACE = Pattern.compile("[^0-9/]"); private static final Predicate IDENTIFIER_PREDICATE = Pattern.compile("\\d{4}/\\d{3,5}").asPredicate(); @@ -55,7 +61,13 @@ public Optional performSearchById(String identifier) throws FetcherExc } private Optional createEntryFromIacrCitation(String validIdentifier) throws FetcherException { - String bibtexCitationHtml = getHtml(CITATION_URL_PREFIX + validIdentifier); + URL url; + try { + url = URI.create(CITATION_URL_PREFIX + validIdentifier).toURL(); + } catch (MalformedURLException e) { + throw new FetcherException("Invalid URL", e); + } + String bibtexCitationHtml = getHtml(url); if (bibtexCitationHtml.contains("No such report found")) { throw new FetcherException(Localization.lang("No results found.")); } @@ -69,7 +81,13 @@ private Optional createEntryFromIacrCitation(String validIdentifier) t } private void setAdditionalFields(BibEntry entry, String identifier) throws FetcherException { - String entryUrl = DESCRIPTION_URL_PREFIX + identifier; + URL entryUrl; + try { + entryUrl = URI.create(DESCRIPTION_URL_PREFIX + identifier).toURL(); + } catch (MalformedURLException e) { + throw new FetcherException("Invalid URL", e); + } + String descriptiveHtml = getHtml(entryUrl); entry.setField(StandardField.ABSTRACT, getAbstract(descriptiveHtml)); @@ -77,8 +95,12 @@ private void setAdditionalFields(BibEntry entry, String identifier) throws Fetch // Version information for entries after year 2000 if (isFromOrAfterYear2000(entry)) { - String entryVersion = VERSION_URL_PREFIX + identifier; - String versionHtml = getHtml(entryVersion); + try { + entryUrl = URI.create(VERSION_URL_PREFIX + identifier).toURL(); + } catch (MalformedURLException e) { + throw new FetcherException("Invalid URL", e); + } + String versionHtml = getHtml(entryUrl); String version = getVersion(identifier, versionHtml); entry.setField(StandardField.VERSION, version); entry.setField(StandardField.URL, entryUrl + "/" + version); @@ -103,13 +125,9 @@ private String getDate(String descriptiveHtml) throws FetcherException { return dateStringAsInHtml; } - private String getHtml(String url) throws FetcherException { - try { - URLDownload download = new URLDownload(url); - return download.asString(); - } catch (IOException e) { - throw new FetcherException(Localization.lang("Could not retrieve entry data from '%0'.", url), e); - } + private String getHtml(URL url) throws FetcherException { + URLDownload download = new URLDownload(url); + return download.asString(); } private String getRequiredValueBetween(String from, String to, String haystack) throws FetcherException { @@ -140,7 +158,15 @@ public Optional findFullText(BibEntry entry) throws IOException, FetcherExc Optional urlField = entry.getField(StandardField.URL); if (urlField.isPresent()) { - String descriptiveHtml = getHtml(urlField.get()); + URL url; + try { + url = URI.create(urlField.get()).toURL(); + } catch (MalformedURLException e) { + LOGGER.warn("Invalid URL {}", urlField.get(), e); + return Optional.empty(); + } + + String descriptiveHtml = getHtml(url); String startOfFulltextLink = " entries; try { Document doc = Jsoup.parse(inputStream, null, HOST); @@ -100,7 +97,7 @@ public Parser getParser() { stringBuilder.append(data); } entries = new ArrayList<>(parser.parseEntries(stringBuilder.toString())); - } catch (IOException e) { + } catch (IOException | FetcherException e) { throw new ParseException("Could not download data from jstor.org", e); } return entries; @@ -113,7 +110,7 @@ public String getName() { } @Override - public Optional findFullText(BibEntry entry) throws IOException { + public Optional findFullText(BibEntry entry) throws FetcherException, IOException { if (entry.getField(StandardField.URL).isEmpty()) { return Optional.empty(); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/LOBIDFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/LOBIDFetcher.java index 5920b605943..c34d7cc4860 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/LOBIDFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/LOBIDFetcher.java @@ -12,7 +12,6 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImporterPreferences; import org.jabref.logic.importer.PagedSearchBasedParserFetcher; import org.jabref.logic.importer.Parser; @@ -58,12 +57,12 @@ public LOBIDFetcher(ImporterPreferences importerPreferences) { * @return URL */ @Override - public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(API_URL); uriBuilder.addParameter("q", new LOBIDQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); // search query uriBuilder.addParameter("from", String.valueOf(getPageSize() * pageNumber)); // from entry number, starts indexing at 0 - uriBuilder.addParameter("size", String.valueOf(getPageSize())); // page size - uriBuilder.addParameter("format", "json"); // response format + uriBuilder.addParameter("size", String.valueOf(getPageSize())); + uriBuilder.addParameter("format", "json"); return uriBuilder.build().toURL(); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/LibraryOfCongress.java b/src/main/java/org/jabref/logic/importer/fetcher/LibraryOfCongress.java index 8ffa700df18..6471c36aae6 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/LibraryOfCongress.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/LibraryOfCongress.java @@ -4,7 +4,6 @@ import java.net.URISyntaxException; import java.net.URL; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.IdBasedParserFetcher; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; @@ -29,7 +28,7 @@ public String getName() { } @Override - public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder("https://lccn.loc.gov/" + identifier + "/mods"); return uriBuilder.build().toURL(); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java b/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java index f18b74c9fd7..2af1bebbf42 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java @@ -97,7 +97,7 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder("https://mathscinet.ams.org/mathscinet/api/publications/search"); uriBuilder.addParameter("query", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); // query uriBuilder.addParameter("currentPage", "1"); // start index @@ -106,7 +106,7 @@ public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, Malf } @Override - public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder("https://mathscinet.ams.org/mathscinet/api/publications/format"); uriBuilder.addParameter("formats", "bib"); uriBuilder.addParameter("ids", identifier); // identifier diff --git a/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java index 25b6116294a..a3444ad3530 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java @@ -126,7 +126,7 @@ public Optional getHelpPage() { } @Override - public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(ID_URL); uriBuilder.addParameter("db", "pubmed"); uriBuilder.addParameter("retmode", "xml"); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/Medra.java b/src/main/java/org/jabref/logic/importer/fetcher/Medra.java index e6329656927..2c52645baf7 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/Medra.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/Medra.java @@ -8,7 +8,6 @@ import java.util.stream.IntStream; import org.jabref.logic.cleanup.DoiCleanup; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.IdBasedParserFetcher; import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.Parser; @@ -103,7 +102,7 @@ public URLDownload getUrlDownload(URL url) { } @Override - public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException { return new URL(API_URL + "/" + identifier); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/MrDLibFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/MrDLibFetcher.java index e0d7310c52b..ba12ac0d9aa 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/MrDLibFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/MrDLibFetcher.java @@ -1,9 +1,10 @@ package org.jabref.logic.importer.fetcher; import java.io.IOException; +import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; -import java.util.ArrayList; +import java.net.URL; import java.util.Calendar; import java.util.List; import java.util.Optional; @@ -54,31 +55,38 @@ public String getName() { @Override public List performSearch(BibEntry entry) throws FetcherException { Optional title = entry.getFieldLatexFree(StandardField.TITLE); - if (title.isPresent()) { - String response = makeServerRequest(title.get()); - MrDLibImporter importer = new MrDLibImporter(); - ParserResult parserResult; - try { - if (importer.isRecognizedFormat(response)) { - parserResult = importer.importDatabase(response); - heading = importer.getRecommendationsHeading(); - description = importer.getRecommendationsDescription(); - recommendationSetId = importer.getRecommendationSetId(); - } else { - // For displaying An ErrorMessage - description = DEFAULT_MRDLIB_ERROR_MESSAGE; - BibDatabase errorBibDataBase = new BibDatabase(); - parserResult = new ParserResult(errorBibDataBase); - } - } catch (IOException e) { - LOGGER.error(e.getMessage(), e); - throw new FetcherException("JSON Parser IOException."); - } - return parserResult.getDatabase().getEntries(); - } else { + if (!title.isPresent()) { // without a title there is no reason to ask MrDLib - return new ArrayList<>(0); + return List.of(); + } + + URL url; + try { + url = constructQuery(title.get()); + } catch (URISyntaxException | MalformedURLException e) { + throw new FetcherException("Invalid URL", e); + } + + String response = makeServerRequest(url); + MrDLibImporter importer = new MrDLibImporter(); + ParserResult parserResult; + try { + if (importer.isRecognizedFormat(response)) { + parserResult = importer.importDatabase(response); + heading = importer.getRecommendationsHeading(); + description = importer.getRecommendationsDescription(); + recommendationSetId = importer.getRecommendationSetId(); + } else { + // For displaying An ErrorMessage + description = DEFAULT_MRDLIB_ERROR_MESSAGE; + BibDatabase errorBibDataBase = new BibDatabase(); + parserResult = new ParserResult(errorBibDataBase); + } + } catch (IOException e) { + LOGGER.error("Error while fetching", e); + throw new FetcherException(url, e); } + return parserResult.getDatabase().getEntries(); } public String getHeading() { @@ -92,30 +100,25 @@ public String getDescription() { /** * Contact the server with the title of the selected item * - * @param queryByTitle the query holds the title of the selected entry. Used to make a query to the MDL Server * @return Returns the server response. This is an XML document as a String. */ - private String makeServerRequest(String queryByTitle) throws FetcherException { - try { - URLDownload urlDownload = new URLDownload(constructQuery(queryByTitle)); - String response = urlDownload.asString(); + private String makeServerRequest(URL url) throws FetcherException { + URLDownload urlDownload = new URLDownload(url); + String response = urlDownload.asString(); - // Conversion of < and > - response = response.replace(">", ">"); - response = response.replace("<", "<"); - return response; - } catch (IOException e) { - throw new FetcherException("Problem downloading", e); - } + // Conversion of < and > + response = response.replace(">", ">"); + response = response.replace("<", "<"); + return response; } /** * Constructs the query based on title of the BibEntry. Adds statistical stuff to the url. * - * @param queryWithTitle the title of the bib entry. + * @param queryWithTitle the query holds the title of the selected entry. Used to make a query to the MDL Server * @return the string used to make the query at mdl server */ - private String constructQuery(String queryWithTitle) { + private URL constructQuery(String queryWithTitle) throws URISyntaxException, MalformedURLException { // The encoding does not work for / so we convert them by our own queryWithTitle = queryWithTitle.replace("/", " "); URIBuilder builder = new URIBuilder(); @@ -136,13 +139,8 @@ private String constructQuery(String queryWithTitle) { builder.addParameter("timezone", Calendar.getInstance().getTimeZone().getID()); } - try { - URI uri = builder.build(); - LOGGER.trace("Request: " + uri.toString()); - return uri.toString(); - } catch (URISyntaxException e) { - LOGGER.error(e.getMessage(), e); - } - return ""; + URI uri = builder.build(); + LOGGER.trace("Request: {}", uri.toString()); + return uri.toURL(); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ResearchGate.java b/src/main/java/org/jabref/logic/importer/fetcher/ResearchGate.java index e9f584e0e3a..1f1f0196ed1 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ResearchGate.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ResearchGate.java @@ -3,6 +3,7 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; +import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; @@ -116,7 +117,7 @@ private Document getHTML(BibEntry entry) throws FetcherException, IOException { throw new FetcherException("Could not find a pdf"); } - Optional getURLByString(String query) throws IOException, NullPointerException { + Optional getURLByString(String query) throws IOException, FetcherException { URIBuilder source; String link; try { @@ -169,26 +170,28 @@ Optional getURLByDoi(DOI doi) throws IOException, NullPointerException { return Optional.of(link); } + private Document getPage(URL url) throws IOException { + return Jsoup.connect(url.toString()) + .userAgent(URLDownload.USER_AGENT) + .referrer("www.google.com") + .ignoreHttpErrors(true) + .get(); + } + /** * Constructs a URL based on the query, size and page number. - *

* Extract the numerical internal ID and add it to the URL to receive a link to a {@link BibEntry} + *

* * @param luceneQuery the search query. * @return A URL that lets us download a .bib file - * @throws URISyntaxException from {@link URIBuilder}'s build() method - * @throws IOException from {@link Connection}'s get() method */ - private Document getPage(QueryNode luceneQuery) throws URISyntaxException, IOException { + private static URL getUrlForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { String query = new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse(""); URIBuilder source = new URIBuilder(SEARCH); source.addParameter("type", "publication"); source.addParameter("query", query); - return Jsoup.connect(source.build().toString()) - .userAgent(URLDownload.USER_AGENT) - .referrer("www.google.com") - .ignoreHttpErrors(true) - .get(); + return source.build().toURL(); } @Override @@ -206,14 +209,22 @@ public TrustLevel getTrustLevel() { @Override public List performSearch(QueryNode luceneQuery) throws FetcherException { Document html; + + URL url; try { - html = getPage(luceneQuery); + url = getUrlForQuery(luceneQuery); + } catch (URISyntaxException | MalformedURLException e) { + throw new FetcherException("Invalid URL", e); + } + + try { + html = getPage(url); // ResearchGate's server blocks when too many request are made if (!html.getElementsByClass("nova-legacy-v-publication-item__title").hasText()) { - throw new FetcherException("ResearchGate server unavailable"); + throw new FetcherException(url, "Required HTML element not found", null); } - } catch (URISyntaxException | IOException e) { - throw new FetcherException("URL is not correct", e); + } catch (IOException e) { + throw new FetcherException(url, e); } Elements sol = html.getElementsByClass("nova-legacy-v-publication-item__title"); @@ -234,7 +245,7 @@ public List performSearch(QueryNode luceneQuery) throws FetcherExcepti entry = parser.parseSingleEntry(bib); entry.ifPresent(list::add); } catch (ParseException e) { - LOGGER.debug("Entry is not convertible to Bibtex", e); + LOGGER.debug("Entry is not convertible to BibTeX", e); } } return list; @@ -245,7 +256,7 @@ private BufferedReader getInputStream(String urlString) { URL url = new URL(urlString); return new BufferedReader(new InputStreamReader(url.openStream())); } catch (IOException e) { - LOGGER.debug("Wrong URL:", e); + LOGGER.debug("Wrong URL", e); } return null; } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/RfcFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/RfcFetcher.java index db1df209765..43849ff3319 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/RfcFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/RfcFetcher.java @@ -7,7 +7,6 @@ import java.util.Optional; import org.jabref.logic.help.HelpFile; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.IdBasedParserFetcher; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; @@ -46,7 +45,7 @@ public Optional getHelpPage() { * @return the URL of the RFC resource */ @Override - public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException { String prefixedIdentifier = identifier.toLowerCase(Locale.ENGLISH); // if not a "draft" version if ((!prefixedIdentifier.startsWith(DRAFT_PREFIX)) && (!prefixedIdentifier.startsWith("rfc"))) { diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ScholarArchiveFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/ScholarArchiveFetcher.java index 27a5273f45b..3c1521e1861 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ScholarArchiveFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ScholarArchiveFetcher.java @@ -8,7 +8,6 @@ import java.util.Optional; import java.util.stream.IntStream; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.PagedSearchBasedParserFetcher; import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.Parser; @@ -46,7 +45,7 @@ public class ScholarArchiveFetcher implements PagedSearchBasedParserFetcher { * @return URL */ @Override - public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(API_URL); uriBuilder.addParameter("q", new ScholarArchiveQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); uriBuilder.addParameter("from", String.valueOf(getPageSize() * pageNumber)); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/SemanticScholar.java b/src/main/java/org/jabref/logic/importer/fetcher/SemanticScholar.java index ab993deeeb1..09f8534e6a3 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/SemanticScholar.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/SemanticScholar.java @@ -133,15 +133,16 @@ String getURLBySource(String source) throws IOException, FetcherException { } @Override - public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(SOURCE_WEB_SEARCH); uriBuilder.addParameter("query", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); uriBuilder.addParameter("offset", String.valueOf(pageNumber * getPageSize())); uriBuilder.addParameter("limit", String.valueOf(Math.min(getPageSize(), 10000 - pageNumber * getPageSize()))); // All fields need to be specified uriBuilder.addParameter("fields", "paperId,externalIds,url,title,abstract,venue,year,authors"); - LOGGER.debug("URL for query: {}", uriBuilder.build().toURL()); - return uriBuilder.build().toURL(); + URL result = uriBuilder.build().toURL(); + LOGGER.debug("URL for query: {}", result); + return result; } /** diff --git a/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java index b271b113c39..33be2ee890e 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java @@ -11,7 +11,6 @@ import java.util.stream.Collectors; import org.jabref.logic.help.HelpFile; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImporterPreferences; import org.jabref.logic.importer.PagedSearchBasedParserFetcher; import org.jabref.logic.importer.Parser; @@ -185,8 +184,7 @@ public String getTestUrl() { * @return URL */ @Override - public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException { - + public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(API_URL); uriBuilder.addParameter("q", new SpringerQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); // Search query uriBuilder.addParameter("api_key", importerPreferences.getApiKey(getName()).orElse(API_KEY)); // API key diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java b/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java index 8f88bdf0675..b02c19d9025 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java @@ -104,7 +104,7 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder("https://zbmath.org/bibtexoutput/"); uriBuilder.addParameter("q", new ZbMathQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); // search all fields uriBuilder.addParameter("start", "0"); // start index @@ -113,7 +113,7 @@ public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, Malf } @Override - public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder("https://zbmath.org/bibtexoutput/"); String query = "an:".concat(identifier); // use an: to search for a zbMATH identifier uriBuilder.addParameter("q", query); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/isbntobibtex/DoiToBibtexConverterComIsbnFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/isbntobibtex/DoiToBibtexConverterComIsbnFetcher.java index 1ebc813c7bc..66bd0235ee6 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/isbntobibtex/DoiToBibtexConverterComIsbnFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/isbntobibtex/DoiToBibtexConverterComIsbnFetcher.java @@ -8,7 +8,6 @@ import java.util.stream.IntStream; import java.util.stream.Stream; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.Parser; @@ -40,7 +39,7 @@ public String getName() { } @Override - public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException { this.ensureThatIsbnIsValid(identifier); return new URIBuilder(BASE_URL) .setPathSegments("getInfo.php") diff --git a/src/main/java/org/jabref/logic/importer/fetcher/isbntobibtex/EbookDeIsbnFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/isbntobibtex/EbookDeIsbnFetcher.java index 42effb2bf05..f55ba6eaa76 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/isbntobibtex/EbookDeIsbnFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/isbntobibtex/EbookDeIsbnFetcher.java @@ -7,7 +7,6 @@ import org.jabref.logic.cleanup.FieldFormatterCleanup; import org.jabref.logic.formatter.bibtexfields.NormalizeNamesFormatter; import org.jabref.logic.formatter.bibtexfields.NormalizePagesFormatter; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.fetcher.AbstractIsbnFetcher; import org.jabref.model.entry.BibEntry; @@ -31,7 +30,7 @@ public String getName() { } @Override - public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException { this.ensureThatIsbnIsValid(identifier); return new URIBuilder(BASE_URL) .addParameter("isbn", identifier) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/isbntobibtex/OpenLibraryIsbnFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/isbntobibtex/OpenLibraryIsbnFetcher.java index 3283a169cf8..cb8e7267d77 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/isbntobibtex/OpenLibraryIsbnFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/isbntobibtex/OpenLibraryIsbnFetcher.java @@ -11,7 +11,6 @@ import java.util.stream.Stream; import org.jabref.logic.importer.AuthorListParser; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.Parser; @@ -53,7 +52,7 @@ public String getName() { } @Override - public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException { this.ensureThatIsbnIsValid(identifier); return new URIBuilder(BASE_URL) .setPathSegments("isbn", identifier + ".json") diff --git a/src/main/java/org/jabref/logic/importer/fileformat/ACMPortalParser.java b/src/main/java/org/jabref/logic/importer/fileformat/ACMPortalParser.java index fb60e4a92f4..e1eef008b2c 100644 --- a/src/main/java/org/jabref/logic/importer/fileformat/ACMPortalParser.java +++ b/src/main/java/org/jabref/logic/importer/fileformat/ACMPortalParser.java @@ -83,16 +83,16 @@ public List parseDoiSearchPage(InputStream stream) throws ParseException return doiList; } - /** - * Obtain BibEntry according to DOI - * - * @param doiList DOI List - * @return BibEntry List - */ public List getBibEntriesFromDoiList(List doiList) throws FetcherException { List bibEntries = new ArrayList<>(); CookieHandler.setDefault(new CookieManager()); - try (InputStream stream = new URLDownload(getUrlFromDoiList(doiList)).asInputStream()) { + URL urlFromDoiList; + try { + urlFromDoiList = getUrlFromDoiList(doiList); + } catch (URISyntaxException | MalformedURLException e) { + throw new FetcherException("Wrong URL", e); + } + try (InputStream stream = new URLDownload(urlFromDoiList).asInputStream()) { String jsonString = new String((stream.readAllBytes()), StandardCharsets.UTF_8); JsonElement jsonElement = JsonParser.parseString(jsonString); @@ -104,8 +104,8 @@ public List getBibEntriesFromDoiList(List doiList) throws Fetc } } } - } catch (IOException | URISyntaxException e) { - throw new FetcherException("A network error occurred while fetching from ", e); + } catch (IOException e) { + throw new FetcherException(urlFromDoiList, e); } return bibEntries; diff --git a/src/main/java/org/jabref/logic/importer/util/ShortDOIService.java b/src/main/java/org/jabref/logic/importer/util/ShortDOIService.java index f10996f9b4f..e9c6446fc19 100644 --- a/src/main/java/org/jabref/logic/importer/util/ShortDOIService.java +++ b/src/main/java/org/jabref/logic/importer/util/ShortDOIService.java @@ -1,11 +1,11 @@ package org.jabref.logic.importer.util; -import java.io.IOException; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ParseException; import org.jabref.logic.net.URLDownload; import org.jabref.model.entry.identifier.DOI; @@ -36,8 +36,8 @@ public DOI getShortDOI(DOI doi) throws ShortDOIServiceException { private JSONObject makeRequest(DOI doi) throws ShortDOIServiceException { - URIBuilder uriBuilder = null; - URL url = null; + URIBuilder uriBuilder; + URL url; try { uriBuilder = new URIBuilder(BASIC_URL); @@ -58,7 +58,7 @@ private JSONObject makeRequest(DOI doi) throws ShortDOIServiceException { throw new ShortDOIServiceException("Cannot get short DOI"); } return resultAsJSON; - } catch (ParseException | IOException | JSONException e) { + } catch (ParseException | JSONException | FetcherException e) { throw new ShortDOIServiceException("Cannot get short DOI", e); } } diff --git a/src/main/java/org/jabref/logic/net/URLDownload.java b/src/main/java/org/jabref/logic/net/URLDownload.java index 19d4d2e4bc1..db9e3b4d9f7 100644 --- a/src/main/java/org/jabref/logic/net/URLDownload.java +++ b/src/main/java/org/jabref/logic/net/URLDownload.java @@ -2,7 +2,6 @@ import java.io.BufferedInputStream; import java.io.BufferedReader; -import java.io.ByteArrayInputStream; import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; @@ -40,7 +39,9 @@ import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; +import org.jabref.http.dto.SimpleHttpResponse; import org.jabref.logic.importer.FetcherClientException; +import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.FetcherServerException; import org.jabref.logic.util.io.FileUtil; @@ -240,7 +241,7 @@ public void setPostData(String postData) { * * @return the downloaded string */ - public String asString() throws IOException { + public String asString() throws FetcherException { return asString(StandardCharsets.UTF_8, this.openConnection()); } @@ -250,7 +251,7 @@ public String asString() throws IOException { * @param encoding the desired String encoding * @return the downloaded string */ - public String asString(Charset encoding) throws IOException { + public String asString(Charset encoding) throws FetcherException { return asString(encoding, this.openConnection()); } @@ -260,7 +261,7 @@ public String asString(Charset encoding) throws IOException { * @param existingConnection an existing connection * @return the downloaded string */ - public static String asString(URLConnection existingConnection) throws IOException { + public static String asString(URLConnection existingConnection) throws FetcherException { return asString(StandardCharsets.UTF_8, existingConnection); } @@ -271,16 +272,17 @@ public static String asString(URLConnection existingConnection) throws IOExcepti * @param connection an existing connection * @return the downloaded string */ - public static String asString(Charset encoding, URLConnection connection) throws IOException { - + public static String asString(Charset encoding, URLConnection connection) throws FetcherException { try (InputStream input = new BufferedInputStream(connection.getInputStream()); Writer output = new StringWriter()) { copy(input, output, encoding); return output.toString(); + } catch (IOException e) { + throw new FetcherException("Error downloading", e); } } - public List getCookieFromUrl() throws IOException { + public List getCookieFromUrl() throws FetcherException { CookieManager cookieManager = new CookieManager(); CookieHandler.setDefault(cookieManager); cookieManager.setCookiePolicy(CookiePolicy.ACCEPT_ALL); @@ -301,27 +303,41 @@ public List getCookieFromUrl() throws IOException { * * @param destination the destination file path. */ - public void toFile(Path destination) throws IOException { + public void toFile(Path destination) throws FetcherException { try (InputStream input = new BufferedInputStream(this.openConnection().getInputStream())) { Files.copy(input, destination, StandardCopyOption.REPLACE_EXISTING); } catch (IOException e) { LOGGER.warn("Could not copy input", e); - throw e; + throw new FetcherException("Could not copy input", e); } } /** * Takes the web resource as the source for a monitored input stream. */ - public ProgressInputStream asInputStream() throws IOException { + public ProgressInputStream asInputStream() throws FetcherException { HttpURLConnection urlConnection = (HttpURLConnection) this.openConnection(); - if ((urlConnection.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) || (urlConnection.getResponseCode() == HttpURLConnection.HTTP_BAD_REQUEST)) { - LOGGER.error("Response message {} returned for url {}", urlConnection.getResponseMessage(), urlConnection.getURL()); - return new ProgressInputStream(new ByteArrayInputStream(new byte[0]), 0); + int responseCode; + try { + responseCode = urlConnection.getResponseCode(); + } catch (IOException e) { + throw new FetcherException("Error getting response code", e); + } + LOGGER.debug("Response code: {}", responseCode); // We could check for != 200, != 204 + if (responseCode >= 300) { + SimpleHttpResponse simpleHttpResponse = new SimpleHttpResponse(urlConnection); + LOGGER.error("Failed to read from url: {}", simpleHttpResponse); + throw FetcherException.of(this.source, simpleHttpResponse); } long fileSize = urlConnection.getContentLengthLong(); - return new ProgressInputStream(new BufferedInputStream(urlConnection.getInputStream()), fileSize); + InputStream inputStream; + try { + inputStream = urlConnection.getInputStream(); + } catch (IOException e) { + throw new FetcherException("Error getting input stream", e); + } + return new ProgressInputStream(new BufferedInputStream(inputStream), fileSize); } /** @@ -329,7 +345,7 @@ public ProgressInputStream asInputStream() throws IOException { * * @return the path of the temporary file. */ - public Path toTemporaryFile() throws IOException { + public Path toTemporaryFile() throws FetcherException { // Determine file name and extension from source url String sourcePath = source.getPath(); @@ -339,7 +355,12 @@ public Path toTemporaryFile() throws IOException { String extension = "." + FileUtil.getFileExtension(fileNameWithExtension).orElse("tmp"); // Create temporary file and download to it - Path file = Files.createTempFile(fileName, extension); + Path file = null; + try { + file = Files.createTempFile(fileName, extension); + } catch (IOException e) { + throw new FetcherException("Could not create temporary file", e); + } file.toFile().deleteOnExit(); toFile(file); @@ -363,44 +384,65 @@ private static void copy(InputStream in, Writer out, Charset encoding) throws IO } /** - * Open a connection to this object's URL (with specified settings). If accessing an HTTP URL, don't forget - * to close the resulting connection after usage. + * Open a connection to this object's URL (with specified settings). + *

+ * If accessing an HTTP URL, remeber to close the resulting connection after usage. * * @return an open connection */ - public URLConnection openConnection() throws IOException { - URLConnection connection = this.source.openConnection(); - connection.setConnectTimeout((int) connectTimeout.toMillis()); - for (Entry entry : this.parameters.entrySet()) { - connection.setRequestProperty(entry.getKey(), entry.getValue()); - } - if (!this.postData.isEmpty()) { - connection.setDoOutput(true); - try (DataOutputStream wr = new DataOutputStream(connection.getOutputStream())) { - wr.writeBytes(this.postData); - } + public URLConnection openConnection() throws FetcherException { + URLConnection connection; + try { + connection = getUrlConnection(); + } catch (IOException e) { + throw new FetcherException("Error opening connection", e); } - if (connection instanceof HttpURLConnection lConnection) { - // this does network i/o: GET + read returned headers - int status = lConnection.getResponseCode(); + if (connection instanceof HttpURLConnection httpURLConnection) { + int status; + try { + // this does network i/o: GET + read returned headers + status = httpURLConnection.getResponseCode(); + } catch (IOException e) { + LOGGER.error("Error getting response code", e); + throw new FetcherException("Error getting response code", e); + } - // normally, 3xx is redirect if ((status == HttpURLConnection.HTTP_MOVED_TEMP) || (status == HttpURLConnection.HTTP_MOVED_PERM) || (status == HttpURLConnection.HTTP_SEE_OTHER)) { // get redirect url from "location" header field String newUrl = connection.getHeaderField("location"); // open the new connection again - connection = new URLDownload(newUrl).openConnection(); + try { + connection = new URLDownload(newUrl).openConnection(); + } catch (MalformedURLException e) { + throw new FetcherException("Could not open URL Download", e); + } + } else if (status >= 400) { + // in case of an error, propagate the error message + SimpleHttpResponse httpResponse = new SimpleHttpResponse(httpURLConnection); + LOGGER.info("{}", httpResponse); + if ((status >= 400) && (status < 500)) { + throw new FetcherClientException(this.source, httpResponse); + } else if (status >= 500) { + throw new FetcherServerException(this.source, httpResponse); + } } - if ((status >= 400) && (status < 500)) { - LOGGER.info("HTTP {}, details: {}, {}", status, lConnection.getResponseMessage(), lConnection.getContentLength() > 0 ? lConnection.getContent() : ""); - throw new IOException(new FetcherClientException("Encountered HTTP %s %s".formatted(status, lConnection.getResponseMessage()))); - } - if (status >= 500) { - LOGGER.info("HTTP {}, details: {}, {}", status, lConnection.getResponseMessage(), lConnection.getContentLength() > 0 ? lConnection.getContent() : ""); - throw new IOException(new FetcherServerException("Encountered HTTP %s %s".formatted(status, lConnection.getResponseMessage()))); + } + return connection; + } + + private URLConnection getUrlConnection() throws IOException { + URLConnection connection = this.source.openConnection(); + connection.setConnectTimeout((int) connectTimeout.toMillis()); + for (Entry entry : this.parameters.entrySet()) { + connection.setRequestProperty(entry.getKey(), entry.getValue()); + } + if (!this.postData.isEmpty()) { + connection.setDoOutput(true); + try (DataOutputStream wr = new DataOutputStream(connection.getOutputStream())) { + wr.writeBytes(this.postData); } } return connection; diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index f32e815d31c..9f1c5fa38ef 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -329,8 +329,14 @@ Extract\ references\ from\ file\ (offline)=Extract references from file (offline Extract\ references\ from\ file\ (online)=Extract references from file (online) Extract\ References\ (offline)=Extract References (offline) Extract\ References\ (online)=Extract References (online) + +Processing...=Processing... +Processing\ "%0"...=Processing "%0"... +Processing\ Citation\ Style\ "%0"...=Processing Citation Style "%0"... Processing\ PDF(s)=Processing PDF(s) +Processing\ file\ %0=Processing file %0 Processing\ a\ large\ number\ of\ files=Processing a large number of files + You\ are\ about\ to\ process\ %0\ files.\ Continue?=You are about to process %0 files. Continue? Will\ write\ metadata\ to\ the\ PDFs\ linked\ from\ selected\ entries.=Will write metadata to the PDFs linked from selected entries. @@ -691,7 +697,6 @@ Preferences=Preferences Preferences\ recorded.=Preferences recorded. Preview=Preview -Citation\ Style=Citation Style Current\ Preview=Current Preview Add\ BST\ file=Add BST file @@ -707,7 +712,6 @@ Selected\ Layouts\ can\ not\ be\ empty=Selected Layouts can not be empty Reset\ default\ preview\ style=Reset default preview style Previous\ entry=Previous entry Problem\ with\ parsing\ entry=Problem with parsing entry -Processing\ %0=Processing %0 Pull\ changes\ from\ shared\ database=Pull changes from shared database Pushed\ citations\ to\ %0=Pushed citations to %0 @@ -1841,7 +1845,6 @@ Fetcher\ '%0'\ did\ not\ find\ an\ entry\ for\ id\ '%1'.=Fetcher '%0' did not fi Select\ first\ entry=Select first entry Select\ last\ entry=Select last entry -Invalid\ ISBN\:\ '%0'.=Invalid ISBN: '%0'. should\ be\ an\ integer\ or\ normalized=should be an integer or normalized should\ be\ normalized=should be normalized @@ -1917,7 +1920,6 @@ Removes\ all\ line\ breaks\ in\ the\ field\ content.=Removes all line breaks in Remove\ hyphenated\ line\ breaks=Remove hyphenated line breaks Removes\ all\ hyphenated\ line\ breaks\ in\ the\ field\ content.=Removes all hyphenated line breaks in the field content. -Could\ not\ retrieve\ entry\ data\ from\ '%0'.=Could not retrieve entry data from '%0'. Entry\ from\ %0\ could\ not\ be\ parsed.=Entry from %0 could not be parsed. Invalid\ identifier\:\ '%0'.=Invalid identifier: '%0'. empty\ citation\ key=empty citation key @@ -2383,7 +2385,6 @@ Error\ reading\ PDF\ content\:\ %0=Error reading PDF content\: %0 Bib\ entry\ was\ successfully\ imported=Bib entry was successfully imported File\ was\ successfully\ imported\ as\ a\ new\ entry=File was successfully imported as a new entry No\ BibTeX\ data\ was\ found.\ An\ empty\ entry\ was\ created\ with\ file\ link.=No BibTeX data was found. An empty entry was created with file link. -Processing\ file\ %0=Processing file %0 Export\ selected=Export selected Separate\ merged\ citations=Separate merged citations @@ -2709,16 +2710,16 @@ File\ "%0"\ cannot\ be\ added\!=File "%0" cannot be added! Illegal\ characters\ in\ the\ file\ name\ detected.\nFile\ will\ be\ renamed\ to\ "%0"\ and\ added.=Illegal characters in the file name detected.\nFile will be renamed to "%0" and added. Rename\ and\ add=Rename and add -401\ Unauthorized\:\ Access\ Denied.\ You\ are\ not\ authorized\ to\ access\ this\ resource.\ Please\ check\ your\ credentials\ and\ try\ again.\ If\ you\ believe\ you\ should\ have\ access,\ please\ contact\ the\ administrator\ for\ assistance.\nURL\:\ %0\ \n\ %1=401 Unauthorized: Access Denied. You are not authorized to access this resource. Please check your credentials and try again. If you believe you should have access, please contact the administrator for assistance.\nURL: %0 \n %1 -403\ Forbidden\:\ Access\ Denied.\ You\ do\ not\ have\ permission\ to\ access\ this\ resource.\ Please\ contact\ the\ administrator\ for\ assistance\ or\ try\ a\ different\ action.\nURL\:\ %0\ \n\ %1=403 Forbidden: Access Denied. You do not have permission to access this resource. Please contact the administrator for assistance or try a different action.\nURL: %0 \n %1 -404\ Not\ Found\ Error\:\ The\ requested\ resource\ could\ not\ be\ found.\ It\ seems\ that\ the\ file\ you\ are\ trying\ to\ download\ is\ not\ available\ or\ has\ been\ moved.\ Please\ verify\ the\ URL\ and\ try\ again.\ If\ you\ believe\ this\ is\ an\ error,\ please\ contact\ the\ administrator\ for\ further\ assistance.\nURL\:\ %0\ \n\ %1=404 Not Found Error: The requested resource could not be found. It seems that the file you are trying to download is not available or has been moved. Please verify the URL and try again. If you believe this is an error, please contact the administrator for further assistance.\nURL: %0 \n %1 -Error\ downloading\ from\ URL.\ Cause\ is\ likely\ the\ server\ side.\ HTTP\ Error\ %0\ \n\ %1\ \nURL\:\ %2\ \nPlease\ try\ again\ later\ or\ contact\ the\ server\ administrator.=Error downloading from URL. Cause is likely the server side. HTTP Error %0 \n %1 \nURL: %2 \nPlease try again later or contact the server administrator. -Error\ message\:\ %0\ \nURL\:\ %1\ \nPlease\ check\ the\ URL\ and\ try\ again.=Error message: %0 \nURL: %1 \nPlease check the URL and try again. Failed\ to\ download\ from\ URL=Failed to download from URL +Access\ denied.\ You\ are\ not\ authorized\ to\ access\ this\ resource.\ Please\ check\ your\ credentials\ and\ try\ again.\ If\ you\ believe\ you\ should\ have\ access,\ please\ contact\ the\ administrator\ for\ assistance.=Access denied. You are not authorized to access this resource. Please check your credentials and try again. If you believe you should have access, please contact the administrator for assistance. +Access\ denied.\ You\ do\ not\ have\ permission\ to\ access\ this\ resource.\ Please\ contact\ the\ administrator\ for\ assistance\ or\ try\ a\ different\ action.=Access denied. You do not have permission to access this resource. Please contact the administrator for assistance or try a different action. +The\ requested\ resource\ could\ not\ be\ found.\ It\ seems\ that\ the\ file\ you\ are\ trying\ to\ download\ is\ not\ available\ or\ has\ been\ moved.\ Please\ verify\ the\ URL\ and\ try\ again.\ If\ you\ believe\ this\ is\ an\ error,\ please\ contact\ the\ administrator\ for\ further\ assistance.=The requested resource could not be found. It seems that the file you are trying to download is not available or has been moved. Please verify the URL and try again. If you believe this is an error, please contact the administrator for further assistance. +Something\ is\ wrong\ on\ JabRef\ side.\ Please\ check\ the\ URL\ and\ try\ again.=Something is wrong on JabRef side. Please check the URL and try again. +Error\ downloading\ from\ URL.\ Cause\ is\ likely\ the\ server\ side.\nPlease\ try\ again\ later\ or\ contact\ the\ server\ administrator.=Error downloading from URL. Cause is likely the server side.\nPlease try again later or contact the server administrator. +Please\ check\ the\ URL\ and\ try\ again.\nURL\:\ %0\nDetails\:\ %1=Please check the URL and try again.\nURL: %0\nDetails: %1 Finished=Finished Finished\ writing\ metadata\ for\ library\ %0\ (%1\ succeeded,\ %2\ skipped,\ %3\ errors).=Finished writing metadata for library %0 (%1 succeeded, %2 skipped, %3 errors). -Processing...=Processing... Writing\ metadata\ to\ %0=Writing metadata to %0 Get\ more\ themes...=Get more themes... diff --git a/src/test/java/org/jabref/logic/importer/fetcher/GoogleScholarTest.java b/src/test/java/org/jabref/logic/importer/fetcher/GoogleScholarTest.java index 4432e20a19b..a3a96315a11 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/GoogleScholarTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/GoogleScholarTest.java @@ -38,7 +38,7 @@ void setUp() { } @Test - void linkFound() throws IOException, FetcherException { + void linkFound() throws Exception { entry.setField(StandardField.TITLE, "Towards Application Portability in Platform as a Service"); assertEquals( diff --git a/src/test/java/org/jabref/logic/importer/fetcher/ResearchGateTest.java b/src/test/java/org/jabref/logic/importer/fetcher/ResearchGateTest.java index 5fdccc287c6..8ccc6f6f1af 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/ResearchGateTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/ResearchGateTest.java @@ -56,7 +56,7 @@ void fullTextNotFoundByDOI() throws IOException, FetcherException { } @Test - void getDocumentByTitle() throws IOException, NullPointerException { + void getDocumentByTitle() throws Exception { Optional source = fetcher.getURLByString(entry.getTitle().get()); assertTrue(source.isPresent() && source.get().startsWith(URL_PAGE)); } diff --git a/src/test/java/org/jabref/logic/importer/fileformat/ACMPortalParserTest.java b/src/test/java/org/jabref/logic/importer/fileformat/ACMPortalParserTest.java index fc1672af602..d9f641ab855 100644 --- a/src/test/java/org/jabref/logic/importer/fileformat/ACMPortalParserTest.java +++ b/src/test/java/org/jabref/logic/importer/fileformat/ACMPortalParserTest.java @@ -1,6 +1,5 @@ package org.jabref.logic.importer.fileformat; -import java.io.IOException; import java.net.CookieHandler; import java.net.CookieManager; import java.net.MalformedURLException; @@ -11,7 +10,6 @@ import java.util.Optional; import org.jabref.logic.importer.FetcherException; -import org.jabref.logic.importer.ParseException; import org.jabref.logic.net.URLDownload; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; @@ -69,7 +67,7 @@ void setUp() throws URISyntaxException, MalformedURLException { } @Test - void parseEntries() throws IOException, ParseException { + void parseEntries() throws Exception { CookieHandler.setDefault(new CookieManager()); List bibEntries = parser.parseEntries(new URLDownload(searchUrl).asInputStream()); for (BibEntry bibEntry : bibEntries) { @@ -79,7 +77,7 @@ void parseEntries() throws IOException, ParseException { } @Test - void parseDoiSearchPage() throws ParseException, IOException { + void parseDoiSearchPage() throws Exception { String testDoi = "10.1145/3129790.3129810"; CookieHandler.setDefault(new CookieManager()); List doiList = parser.parseDoiSearchPage(new URLDownload(searchUrl).asInputStream()); @@ -139,7 +137,7 @@ void parseBibEntryWithFamilyAuthorOnly() { } @Test - void noEntryFound() throws URISyntaxException, IOException, ParseException { + void noEntryFound() throws Exception { CookieHandler.setDefault(new CookieManager()); URL url = new URIBuilder("https://dl.acm.org/action/doSearch?AllField=10.1145/3129790.31298").build().toURL(); List bibEntries = parser.parseEntries(new URLDownload(url).asInputStream()); diff --git a/src/test/java/org/jabref/logic/net/URLDownloadTest.java b/src/test/java/org/jabref/logic/net/URLDownloadTest.java index 926d5c67c38..38b0846d8a5 100644 --- a/src/test/java/org/jabref/logic/net/URLDownloadTest.java +++ b/src/test/java/org/jabref/logic/net/URLDownloadTest.java @@ -1,7 +1,6 @@ package org.jabref.logic.net; import java.io.File; -import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.nio.charset.StandardCharsets; @@ -17,7 +16,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -28,21 +26,21 @@ public class URLDownloadTest { private static final Logger LOGGER = LoggerFactory.getLogger(URLDownloadTest.class); @Test - public void stringDownloadWithSetEncoding() throws IOException { + public void stringDownloadWithSetEncoding() throws Exception { URLDownload dl = new URLDownload(new URL("http://www.google.com")); assertTrue(dl.asString().contains("Google"), "google.com should contain google"); } @Test - public void stringDownload() throws IOException { + public void stringDownload() throws Exception { URLDownload dl = new URLDownload(new URL("http://www.google.com")); assertTrue(dl.asString(StandardCharsets.UTF_8).contains("Google"), "google.com should contain google"); } @Test - public void fileDownload() throws IOException { + public void fileDownload() throws Exception { File destination = File.createTempFile("jabref-test", ".html"); try { URLDownload dl = new URLDownload(new URL("http://www.google.com")); @@ -57,14 +55,14 @@ public void fileDownload() throws IOException { } @Test - public void determineMimeType() throws IOException { + public void determineMimeType() throws Exception { URLDownload dl = new URLDownload(new URL("http://www.google.com")); assertTrue(dl.getMimeType().startsWith("text/html")); } @Test - public void downloadToTemporaryFilePathWithoutFileSavesAsTmpFile() throws IOException { + public void downloadToTemporaryFilePathWithoutFileSavesAsTmpFile() throws Exception { URLDownload google = new URLDownload(new URL("http://www.google.com")); String path = google.toTemporaryFile().toString(); @@ -72,7 +70,7 @@ public void downloadToTemporaryFilePathWithoutFileSavesAsTmpFile() throws IOExce } @Test - public void downloadToTemporaryFileKeepsName() throws IOException { + public void downloadToTemporaryFileKeepsName() throws Exception { URLDownload google = new URLDownload(new URL("https://github.com/JabRef/jabref/blob/main/LICENSE")); String path = google.toTemporaryFile().toString(); @@ -81,7 +79,7 @@ public void downloadToTemporaryFileKeepsName() throws IOException { @Test @DisabledOnCIServer("CI Server is apparently blocked") - public void downloadOfFTPSucceeds() throws IOException { + public void downloadOfFTPSucceeds() throws Exception { URLDownload ftp = new URLDownload(new URL("ftp://ftp.informatik.uni-stuttgart.de/pub/library/ncstrl.ustuttgart_fi/INPROC-2016-15/INPROC-2016-15.pdf")); Path path = ftp.toTemporaryFile(); @@ -89,7 +87,7 @@ public void downloadOfFTPSucceeds() throws IOException { } @Test - public void downloadOfHttpSucceeds() throws IOException { + public void downloadOfHttpSucceeds() throws Exception { URLDownload ftp = new URLDownload(new URL("http://www.jabref.org")); Path path = ftp.toTemporaryFile(); @@ -97,7 +95,7 @@ public void downloadOfHttpSucceeds() throws IOException { } @Test - public void downloadOfHttpsSucceeds() throws IOException { + public void downloadOfHttpsSucceeds() throws Exception { URLDownload ftp = new URLDownload(new URL("https://www.jabref.org")); Path path = ftp.toTemporaryFile(); @@ -128,18 +126,14 @@ public void connectTimeoutIsNeverNull() throws MalformedURLException { } @Test - public void test503ErrorThrowsNestedIOExceptionWithFetcherServerException() throws Exception { + public void test503ErrorThrowsFetcherServerException() throws Exception { URLDownload urlDownload = new URLDownload(new URL("http://httpstat.us/503")); - - Exception exception = assertThrows(IOException.class, urlDownload::asString); - assertInstanceOf(FetcherServerException.class, exception.getCause()); + assertThrows(FetcherServerException.class, urlDownload::asString); } @Test - public void test429ErrorThrowsNestedIOExceptionWithFetcherServerException() throws Exception { + public void test429ErrorThrowsFetcherClientException() throws Exception { URLDownload urlDownload = new URLDownload(new URL("http://httpstat.us/429")); - - Exception exception = assertThrows(IOException.class, urlDownload::asString); - assertInstanceOf(FetcherClientException.class, exception.getCause()); + Exception exception = assertThrows(FetcherClientException.class, urlDownload::asString); } }