Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

reduce log size when duplicate entry in zipping #2006

Merged
merged 5 commits into from
Jan 9, 2024
Merged

reduce log size when duplicate entry in zipping #2006

Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
cfe667c
reduce log size
julien-louis Jan 8, 2024
6f44f3f
just ignore the url
julien-louis Jan 8, 2024
d66cd44
add a log
julien-louis Jan 8, 2024
d6969a1
typo
julien-louis Jan 9, 2024
8aa2707
missing space
julien-louis Jan 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 40 additions & 27 deletions shanoir-ng-datasets/src/main/java/org/shanoir/ng/download/WADODownloaderService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,12 @@
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipException;
import java.util.zip.ZipOutputStream;

import javax.json.Json;
Expand Down Expand Up @@ -150,39 +153,49 @@ public void initRestTemplate() {
public List<String> downloadDicomFilesForURLsAsZip(final List<URL> urls, final ZipOutputStream zipOutputStream, String subjectName, Dataset dataset, String datasetFilePath, List<SerieError> serieErrors) throws IOException, MessagingException, RestServiceException {
int i = 0;
List<String> files = new ArrayList<>();
Set<String> zippedUrls= new HashSet<>();
long duplicates = 0;
for (Iterator<URL> iterator = urls.iterator(); iterator.hasNext(); i++) {
String url = ((URL) iterator.next()).toString();
// handle and check at first for WADO-RS URLs by "/instances/"
int indexInstanceUID = url.lastIndexOf(WADO_REQUEST_TYPE_WADO_RS);
// WADO-URI link found in database
if (indexInstanceUID <= 0) {
// handle and check secondly for WADO-URI URLs by "objectUID="
// instanceUID == objectUID
indexInstanceUID = url.lastIndexOf(WADO_REQUEST_TYPE_WADO_URI);
if (!zippedUrls.contains(url)) {
zippedUrls.add(url);
// handle and check at first for WADO-RS URLs by "/instances/"
int indexInstanceUID = url.lastIndexOf(WADO_REQUEST_TYPE_WADO_RS);
// WADO-URI link found in database
if (indexInstanceUID <= 0) {
LOG.error("URL for download is neither in WADO-RS nor in WADO-URI format. URL : " + url + " - Dataset id : " + dataset.getId());
String errorDetails = "URL for download is neither in WADO-RS nor in WADO-URI format";
writeErrorFileInZip(zipOutputStream, subjectName, indexInstanceUID, errorDetails);
// in case an old WADO-URI is found in the database: convert it to WADO-RS
} else {
url = wadoURItoWadoRS(url);
indexInstanceUID = url.lastIndexOf(WADO_REQUEST_TYPE_WADO_RS); // calculate new index
// handle and check secondly for WADO-URI URLs by "objectUID="
// instanceUID == objectUID
indexInstanceUID = url.lastIndexOf(WADO_REQUEST_TYPE_WADO_URI);
if (indexInstanceUID <= 0) {
LOG.error("URL for download is neither in WADO-RS nor in WADO-URI format. URL : " + url + " - Dataset id : " + dataset.getId());
String errorDetails = "URL for download is neither in WADO-RS nor in WADO-URI format";
writeErrorFileInZip(zipOutputStream, subjectName, indexInstanceUID, errorDetails);
// in case an old WADO-URI is found in the database: convert it to WADO-RS
} else {
url = wadoURItoWadoRS(url);
indexInstanceUID = url.lastIndexOf(WADO_REQUEST_TYPE_WADO_RS); // calculate new index
}
}
}
String instanceUID = url.substring(indexInstanceUID + WADO_REQUEST_TYPE_WADO_RS.length());
// Build name
String name = buildFileName(subjectName, dataset, datasetFilePath, instanceUID);
// Download and zip
try {
String zipedFile = downloadAndWriteFileInZip(url, zipOutputStream, name);
if (zipedFile != null) {
files.add(zipedFile);
String instanceUID = url.substring(indexInstanceUID + WADO_REQUEST_TYPE_WADO_RS.length());
// Build name
String name = buildFileName(subjectName, dataset, datasetFilePath, instanceUID);
// Download and zip
try {
String zipedFile = downloadAndWriteFileInZip(url, zipOutputStream, name);
if (zipedFile != null) {
files.add(zipedFile);
}
} catch (ZipPacsFileException e) {
writeErrorFileInZip(zipOutputStream, name, i, e.getMessage());
if (serieErrors != null) serieErrors.add(new SerieError(i, url, e.getMessage()));
}
} catch (ZipPacsFileException e) {
writeErrorFileInZip(zipOutputStream, name, i, e.getMessage());
if (serieErrors != null) serieErrors.add(new SerieError(i, url, e.getMessage()));
} else {
duplicates++;
}
}
if (duplicates > 0) {
LOG.error("There were " + duplicates + " duplicate dataset_files when zipping dataset n°" + dataset.getId() + ", they were ignored.");
}
return files;
}

Expand Down Expand Up @@ -228,7 +241,7 @@ private String downloadAndWriteFileInZip(String url, ZipOutputStream zipOutputSt
this.extractDICOMZipFromMHTMLFile(responseBody, extractInstanceUID(url), name, zipOutputStream);
return name + DCM;
} catch (IOException | MessagingException e) {
LOG.error("A dicom file could not be downloaded from the pacs:", e);
LOG.error("Error in downloading/wring a file from pacs to zip", e);
throw new ZipPacsFileException(e);
} catch (HttpClientErrorException e) {
//LOG.error("A dicom file could not be downloaded from the pacs:", e);
Expand Down
Loading