diff --git a/src/main/java/com/drunkendev/confluence/plugins/attachments/PurgeAttachmentsJob.java b/src/main/java/com/drunkendev/confluence/plugins/attachments/PurgeAttachmentsJob.java
index 8e8f1f4..2e7c59c 100644
--- a/src/main/java/com/drunkendev/confluence/plugins/attachments/PurgeAttachmentsJob.java
+++ b/src/main/java/com/drunkendev/confluence/plugins/attachments/PurgeAttachmentsJob.java
@@ -24,18 +24,18 @@
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
+import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.ArrayDeque;
import java.util.ArrayList;
-import java.util.Calendar;
import java.util.Collections;
import java.util.Comparator;
+import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Supplier;
-import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,7 +48,8 @@
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toMap;
import static java.util.stream.Collectors.toSet;
-import static org.apache.commons.lang.StringUtils.isNotBlank;
+import static org.apache.commons.lang3.StringUtils.equalsIgnoreCase;
+import static org.apache.commons.lang3.StringUtils.isNotBlank;
import static org.apache.commons.lang3.StringUtils.repeat;
@@ -64,8 +65,8 @@ public class PurgeAttachmentsJob implements JobRunner {
private static final Comparator COMP_ATTACHMENT_VERSION
= nullsFirst(comparingInt(n -> n.getVersion()));
private static final Comparator COMP_MAILLOG_SPACE_TITLE
- = comparing((MailLogEntry n) -> n.getAttachment().getSpace().getName(), nullsFirst(naturalOrder()))
- .thenComparing((MailLogEntry n) -> n.getAttachment().getDisplayTitle(), nullsFirst(naturalOrder()));
+ = comparing((MailLogEntry n) -> n.getSpaceName(), nullsFirst(naturalOrder()))
+ .thenComparing((MailLogEntry n) -> n.getDisplayTitle(), nullsFirst(naturalOrder()));
private static final int BATCH_SIZE = 50;
@@ -95,7 +96,6 @@ public PurgeAttachmentsJob(AttachmentManager attachmentManager,
MultiQueueTaskManager mailQueueTaskManager,
SettingsManager settingsManager,
TransactionTemplate transactionTemplate) {
- LOG.debug("Creating purge-old-attachment-job instance.");
this.attachmentManager = attachmentManager;
this.spaceManager = spaceManager;
this.settingSvc = purgeAttachmentsSettingsService;
@@ -152,9 +152,9 @@ public static Duration time(Runnable r) {
}
@Override
- public JobRunnerResponse runJob(JobRunnerRequest jrr) {
+ public JobRunnerResponse runJob(JobRunnerRequest req) {
+ LOG.info("Purge attachment revisions started.");
try {
- LOG.info("Purge old attachments started.");
LocalDateTime start = LocalDateTime.now();
PurgeAttachmentSettings systemSettings = getSystemSettings();
@@ -173,99 +173,35 @@ public JobRunnerResponse runJob(JobRunnerRequest jrr) {
= time(() -> attachmentManager.getAttachmentDao().findAll()
.stream()
.map(Attachment::getId).collect(toCollection(ArrayDeque::new)));
- LOG.debug("FIND ALL ({}) : {}", findAll.left, findAll.right.size());
+ LOG.debug("Got {} attachments in {}.", findAll.right.size(), findAll.left);
ArrayDeque ids = findAll.right;
- while (!ids.isEmpty()) {
- LOG.debug("Processing batch {}; {} remain", ++counters[IDX_BATCHES], ids.size());
+
+ while (!ids.isEmpty() && !req.isCancellationRequested()) {
+ LOG.debug("Processing batch {}; {} atttachments remain", ++counters[IDX_BATCHES], ids.size());
transactionTemplate.execute(() -> {
AttachmentDao dao = attachmentManager.getAttachmentDao();
- try {
- for (int i = 0; i < BATCH_SIZE && !ids.isEmpty(); i++) {
- Attachment attachment = attachmentManager.getAttachment(ids.poll());
- counters[IDX_CURRENT_VERSIONS]++;
-
- if (attachment.getVersion() > 1 && spaceSettings.containsKey(attachment.getSpaceKey())) {
- counters[IDX_CURRENT_VISITED]++;
-
- PurgeAttachmentSettings settings = spaceSettings.get(attachment.getSpaceKey());
-
- List prior = attachmentManager.getPreviousVersions(attachment);
- counters[IDX_PRIOR_VERSIONS] += prior.size();
-
- List toDelete = findDeletions(prior, settings);
- Set badVersions = toDelete.stream()
- .filter(n -> n.getVersion() >= attachment.getVersion())
- .map(n -> n.getVersion())
- .collect(toSet());
- if (badVersions.size() > 0) {
- LOG.error("Attachment bas invalid prior versions: {}:{} :- {} ({}) :: {}",
- attachment.getSpaceKey(),
- attachment.getSpace().getName(),
- attachment.getDisplayTitle(),
- attachment.getVersion(),
- badVersions);
- } else if (!toDelete.isEmpty()) {
- boolean canUpdate;
- if (!settings.isReportOnly() && !systemSettings.isReportOnly()) {
- canUpdate = systemSettings.getDeleteLimit() == 0 ||
- counters[IDX_PROCESS_LIMIT] < systemSettings.getDeleteLimit();
- } else {
- canUpdate = false;
- }
-
- if (canUpdate) {
- counters[IDX_PROCESS_LIMIT]++;
- }
-
- long spaceSaved = toDelete.stream().map(p -> {
- LOG.debug("Attachment to remove {}", p.getId());
- if (canUpdate) {
-// attachmentManager.removeAttachmentVersionFromServer(p);
- Duration dur = time(() -> dao.removeAttachmentVersionFromServer(p));
- counters[IDX_DELETED]++;
- counters[IDX_DELETED_TIME] += dur.toMillis();
- } else {
- counters[IDX_DELETE_AVAIL]++;
- }
- return p.getFileSize();
- }).reduce(0L, (a, b) -> a + b);
-
- MailLogEntry mle = new MailLogEntry(
- attachment,
- toDelete.stream().map(Attachment::getVersion).collect(toList()),
- !canUpdate,
- settings == systemSettings,
- spaceSaved);
-
- if (settings != systemSettings && StringUtils.isNotBlank(settings.getReportEmailAddress())) {
- if (!mailEntries.containsKey(settings.getReportEmailAddress())) {
- mailEntries.put(settings.getReportEmailAddress(), new ArrayList<>());
- }
- mailEntries.get(settings.getReportEmailAddress()).add(mle);
- }
- //TODO: I know this will log twice if system email and space
- // email are the same, will fix later, just hacking atm.
- if (isNotBlank(systemSettings.getReportEmailAddress())) {
- if (!mailEntries.containsKey(systemSettings.getReportEmailAddress())) {
- mailEntries.put(systemSettings.getReportEmailAddress(), new ArrayList<>());
- }
- mailEntries.get(systemSettings.getReportEmailAddress()).add(mle);
- }
- }
- }
- } // for id
- return null;
- } catch (Throwable ex) {
- throw new RuntimeException(ex);
+ for (int i = 0; i < BATCH_SIZE && !ids.isEmpty() && !req.isCancellationRequested(); i++) {
+ Attachment attachment = attachmentManager.getAttachment(ids.poll());
+ process(attachment,
+ counters,
+ spaceSettings.get(attachment.getSpaceKey()),
+ systemSettings,
+ dao,
+ mailEntries);
}
+ return null;
});
}
+ if (req.isCancellationRequested()) {
+ LOG.warn("Attachment purging has been cancelled.");
+ }
+
LocalDateTime end = LocalDateTime.now();
long ms = Duration.between(start, end).toMillis();
- LOG.info("{} processable prior versions found for {} attachments.",
+ LOG.info("{} prior versions visited for {} attachments.",
counters[IDX_PRIOR_VERSIONS],
counters[IDX_CURRENT_VERSIONS]);
if (counters[IDX_CURRENT_VISITED] > 0) {
@@ -281,30 +217,113 @@ public JobRunnerResponse runJob(JobRunnerRequest jrr) {
LOG.info("A further {} versions are available for deleting.",
counters[IDX_DELETE_AVAIL]);
LOG.info("Attachment purging completed in {} ms.", ms);
- try {
- if (systemSettings.isSendPlainTextMail()) {
- mailResultsPlain(mailEntries,
- start,
- end,
- counters);
- } else {
- mailResultsHtml(mailEntries,
- start,
- end,
- counters);
- }
- } catch (MailException ex) {
- LOG.error("Exception raised while trying to mail results.", ex);
- return JobRunnerResponse.failed("Task completed but could not email.");
+
+ if (systemSettings.isSendPlainTextMail()) {
+ mailResultsPlain(mailEntries,
+ start,
+ end,
+ counters,
+ req.isCancellationRequested());
+ } else {
+ mailResultsHtml(mailEntries,
+ start,
+ end,
+ counters,
+ req.isCancellationRequested());
}
- LOG.debug("Purge attachments complete.");
+ } catch (MailException ex) {
+ LOG.error("Exception raised while trying to mail results.", ex);
+ return JobRunnerResponse.failed("Task completed but could not email.");
} catch (Throwable ex) {
- LOG.error("Exception in job: {}", ex.getMessage(), ex);
+ LOG.error("Purge attachment revisions failed: {}", ex.getMessage(), ex);
return JobRunnerResponse.failed(ex);
}
+ LOG.info("Purge attachment revisions completed.");
return JobRunnerResponse.success();
}
+ private void process(Attachment attachment,
+ long[] counters,
+ PurgeAttachmentSettings settings,
+ PurgeAttachmentSettings systemSettings,
+ AttachmentDao dao,
+ Map> mailEntries) {
+ counters[IDX_CURRENT_VERSIONS]++;
+
+ if (attachment.getVersion() == 1) {
+ LOG.trace("Skipping only attachment version {}", attachment.getId());
+ return;
+ }
+
+ if (settings == null) {
+ settings = systemSettings;
+ }
+
+ counters[IDX_CURRENT_VISITED]++;
+
+ List prior = attachmentManager.getPreviousVersions(attachment);
+ counters[IDX_PRIOR_VERSIONS] += prior.size();
+
+ List toDelete = findDeletions(prior, settings);
+ Set badVersions = toDelete.stream()
+ .filter(n -> n.getVersion() >= attachment.getVersion())
+ .map(n -> n.getVersion())
+ .collect(toSet());
+ if (badVersions.size() > 0) {
+ LOG.error("Attachment with versions to delete > current version: {}:{} :- {} ({}) :: {}",
+ attachment.getSpaceKey(),
+ attachment.getSpace().getName(),
+ attachment.getDisplayTitle(),
+ attachment.getVersion(),
+ badVersions);
+ } else if (!toDelete.isEmpty()) {
+ boolean canUpdate
+ = !settings.isReportOnly() &&
+ !systemSettings.isReportOnly() &&
+ (systemSettings.getDeleteLimit() == 0 ||
+ counters[IDX_PROCESS_LIMIT] < systemSettings.getDeleteLimit());
+
+ if (canUpdate) {
+ counters[IDX_PROCESS_LIMIT]++;
+ }
+
+ long spaceSaved = toDelete.stream().map(p -> {
+ LOG.debug("Attachment to remove {}", p.getId());
+ if (canUpdate) {
+ Duration dur = time(() -> dao.removeAttachmentVersionFromServer(p));
+ counters[IDX_DELETED]++;
+ counters[IDX_DELETED_TIME] += dur.toMillis();
+ } else {
+ counters[IDX_DELETE_AVAIL]++;
+ }
+ return p.getFileSize();
+ }).reduce(0L, (a, b) -> a + b);
+
+ if (isNotBlank(settings.getReportEmailAddress()) || isNotBlank(systemSettings.getReportEmailAddress())) {
+ MailLogEntry mle = new MailLogEntry(
+ attachment,
+ toDelete.stream().map(Attachment::getVersion).collect(toList()),
+ !canUpdate,
+ settings == systemSettings,
+ spaceSaved);
+
+ if (isNotBlank(settings.getReportEmailAddress())) {
+ if (!mailEntries.containsKey(settings.getReportEmailAddress())) {
+ mailEntries.put(settings.getReportEmailAddress(), new ArrayList<>());
+ }
+ mailEntries.get(settings.getReportEmailAddress()).add(mle);
+ }
+ if (isNotBlank(systemSettings.getReportEmailAddress()) && !equalsIgnoreCase(settings.getReportEmailAddress(), systemSettings.getReportEmailAddress())) {
+ if (!mailEntries.containsKey(systemSettings.getReportEmailAddress())) {
+ mailEntries.put(systemSettings.getReportEmailAddress(), new ArrayList<>());
+ }
+ mailEntries.get(systemSettings.getReportEmailAddress()).add(mle);
+ }
+ }
+
+ }
+ }
+
private List findDeletions(List prior, PurgeAttachmentSettings stng) {
if (prior == null || prior.isEmpty()) {
return Collections.emptyList();
@@ -341,22 +360,31 @@ private List findDeletions(List prior, PurgeAttachmentSe
}
private int filterRevisionCount(List prior, int maxRevisions) {
- if (prior.size() > maxRevisions) {
- return (prior.size() - maxRevisions) - 1;
- } else {
- return -1;
- }
+ return prior.size() > maxRevisions
+ ? (prior.size() - maxRevisions) - 1
+ : -1;
+ }
+
+ public static LocalDateTime toLocalDateTime(Date value) {
+ return toLocalDateTime(value, ZoneId.systemDefault());
+ }
+
+ public static LocalDateTime toLocalDateTime(Date value, ZoneId zoneId) {
+ // NOTE: java.sql.Date does not support toInstant. To prevent an UnsupportedOperationException
+ // do not use toInstant on dates.
+ //return value == null ? null : LocalDateTime.ofInstant(value.toInstant(), zoneId);
+ return value == null ? null
+ : LocalDateTime.ofInstant(Instant.ofEpochMilli(value.getTime()),
+ zoneId);
}
private int filterAge(List prior, int maxDaysOld) {
- Calendar dateFrom = Calendar.getInstance();
- dateFrom.add(Calendar.DAY_OF_MONTH, -(maxDaysOld));
- Calendar modDate = Calendar.getInstance();
+ LocalDateTime from = LocalDateTime.now().minusDays(maxDaysOld);
for (int i = prior.size() - 1; i >= 0; i--) {
if (prior.get(i).getLastModificationDate() != null) {
- modDate.setTime(prior.get(i).getLastModificationDate());
- if (dateFrom.after(modDate)) {
+ LocalDateTime mod = toLocalDateTime(prior.get(i).getLastModificationDate());
+ if (from.isAfter(mod)) {
return i;
}
}
@@ -380,7 +408,8 @@ private int filterSize(List prior, long maxTotalSize) {
private void mailResultsPlain(Map> entries,
LocalDateTime started,
LocalDateTime ended,
- long[] counters) throws MailException {
+ long[] counters,
+ boolean cancellationRequested) throws MailException {
String p = settingsManager.getGlobalSettings().getBaseUrl();
entries.forEach((emailAddress, entryList) -> {
@@ -400,6 +429,9 @@ private void mailResultsPlain(Map> entries,
deleted += me.getSpaceSaved();
}
}
+ if (cancellationRequested) {
+ sb.append("CANCELLED: Job has had an early cancellation request.");
+ }
if (deleted > 0) {
sb.append("A total of ").append(FileSize.format(deleted))
.append(" space has been reclaimed.\n");
@@ -413,15 +445,14 @@ private void mailResultsPlain(Map> entries,
String ps = null;
for (MailLogEntry me : entryList) {
- Attachment a = me.getAttachment();
- if (ps == null || !ps.equalsIgnoreCase(a.getSpaceKey())) {
+ if (ps == null || !ps.equalsIgnoreCase(me.getSpaceKey())) {
sb.append("\n");
- ps = a.getSpaceKey();
- String sp = ps + ":" + a.getSpace().getName() + " (" + p + a.getSpace().getUrlPath() + ")";
+ ps = me.getSpaceKey();
+ String sp = ps + ":" + me.getSpaceName() + " (" + p + me.getSpaceUrlPath() + ")";
sb.append(sp).append('\n').append(repeat('-', sp.length())).append('\n');
}
- sb.append(a.getDisplayTitle()).append(" (").append(a.getVersion()).append(") ");
+ sb.append(me.getDisplayTitle()).append(" (").append(me.getVersion()).append(") ");
sb.append(me.isReportOnly() ? "TO_DELETE:" : "DELETED:");
me.getDeletedVersions().stream().forEach(ver -> sb.append(" ").append(ver));
sb.append(" [").append(me.getSpaceSavedPretty()).append("]\n");
@@ -460,14 +491,15 @@ private void mailResultsPlain(Map> entries,
});
}
- private void mailResultsHtml(Map> mailEntries1,
+ private void mailResultsHtml(Map> mailEntries,
LocalDateTime started,
LocalDateTime ended,
- long[] counters) throws MailException {
+ long[] counters,
+ boolean cancellationRequested) throws MailException {
String p = settingsManager.getGlobalSettings().getBaseUrl();
String subject = "Purged old attachments";
- mailEntries1.forEach((emailAddress, entryList) -> {
+ mailEntries.forEach((emailAddress, entryList) -> {
Collections.sort(entryList, COMP_MAILLOG_SPACE_TITLE);
StringBuilder sb = new StringBuilder();
@@ -506,9 +538,12 @@ private void mailResultsHtml(Map> mailEntries1,
sb.append(" other rows are in report-only mode.");
sb.append("
");
+ if (cancellationRequested) {
+ sb.append("CANCELLED: Job has had an early cancellation request.
");
+ }
sb.append("Started: ")
.append(started.format(DateTimeFormatter.ISO_DATE_TIME))
- .append(" Ended: ")
+ .append("
Ended: ")
.append(ended.format(DateTimeFormatter.ISO_DATE_TIME))
.append("
");
@@ -548,8 +583,6 @@ private void mailResultsHtml(Map> mailEntries1,
sb.append("");
for (MailLogEntry me : entryList) {
- Attachment a = me.getAttachment();
-
sb.append("> mailEntries1,
sb.append(">");
sb.append("");
- sb.append("")
- .append(a.getSpace().getName()).append("");
+ sb.append("")
+ .append(me.getSpaceName()).append("");
sb.append(" | ");
sb.append("");
- sb.append("")
- .append(a.getDisplayTitle()).append("");
+ sb.append("")
+ .append(me.getDisplayTitle()).append("");
sb.append(" | ");
sb.append("").append(me.getSpaceSavedPretty()).append(" | ");
//sb.append("").append(me.isGlobalSettings() ? "Yes" : "No").append(" | ");
- sb.append("").append(a.getVersion()).append(" | ");
+ sb.append("").append(me.getVersion()).append(" | ");
- sb.append("");
- int c = 0;
- for (Integer dl : me.getDeletedVersions()) {
- if (c++ > 0) {
- sb.append(", ");
- }
- sb.append(dl);
- }
- sb.append(" | ");
+ sb.append("").append(me.getDeletedVersionsRanged()).append(" | ");
sb.append("
");
}
sb.append("");
sb.append("").append(counters[IDX_PRIOR_VERSIONS])
- .append("processable prior versions found for ")
+ .append(" prior versions found for ")
.append(counters[IDX_CURRENT_VERSIONS]).append(" attachments.
");
long ms = Duration.between(started, ended).toMillis();
@@ -602,8 +627,10 @@ private void mailResultsHtml(Map> mailEntries1,
.append(Math.round(counters[IDX_DELETED_TIME] / (double) counters[IDX_DELETED]))
.append(" ms per deletion.");
}
- sb.append("A further ").append(counters[IDX_DELETE_AVAIL])
- .append(" versions are available for deleting.
");
+ if (counters[IDX_DELETE_AVAIL] > 0) {
+ sb.append("A further ").append(counters[IDX_DELETE_AVAIL])
+ .append(" versions are available for deleting.
");
+ }
sb.append("Attachment purging completed in ").append(ms).append(" ms.
");
sb.append("This message has been sent by Attachment Tools - Purge Attachment Versions
");
@@ -626,22 +653,60 @@ private void mailResultsHtml(Map> mailEntries1,
*/
private class MailLogEntry {
- private final Attachment attachment;
+ private final String spaceKey;
+ private final String spaceName;
+ private final String spaceUrlPath;
+
+ private final String displayTitle;
+ private final String attachmentsUrlPath;
+ private final int version;
+
private final List deletedVersions;
private final boolean reportOnly;
private final boolean globalSettings;
private final long spaceSaved;
- private MailLogEntry(Attachment a, List deletedVersions, boolean reportOnly, boolean globalSettings, long spaceSaved) {
- this.attachment = a;
+ private MailLogEntry(Attachment a,
+ List deletedVersions,
+ boolean reportOnly,
+ boolean globalSettings,
+ long spaceSaved) {
+ this.spaceKey = a.getSpaceKey();
+ this.spaceName = a.getSpace() == null ? null : a.getSpace().getName();
+ this.spaceUrlPath = a.getSpace() == null ? null : a.getSpace().getUrlPath();
+
+ this.displayTitle = a.getDisplayTitle();
+ this.attachmentsUrlPath = a.getAttachmentsUrlPath();
+ this.version = a.getVersion();
+
this.deletedVersions = deletedVersions;
this.reportOnly = reportOnly;
this.globalSettings = globalSettings;
this.spaceSaved = spaceSaved;
}
- private Attachment getAttachment() {
- return attachment;
+ public String getSpaceKey() {
+ return spaceKey;
+ }
+
+ public String getSpaceName() {
+ return spaceName;
+ }
+
+ public String getSpaceUrlPath() {
+ return spaceUrlPath;
+ }
+
+ public String getDisplayTitle() {
+ return displayTitle;
+ }
+
+ public String getAttachmentsUrlPath() {
+ return attachmentsUrlPath;
+ }
+
+ public int getVersion() {
+ return version;
}
private List getDeletedVersions() {
@@ -664,6 +729,42 @@ private String getSpaceSavedPretty() {
return FileSize.format(spaceSaved);
}
+ private void append(StringBuilder res, int a, int b) {
+ if (a == -1 || b == -1) {
+ return;
+ }
+ if (res.length() > 0) {
+ res.append(", ");
+ }
+ if (a == b) {
+ res.append(a);
+// } else if (b - a == 1) {
+// res.append(a).append(", ").append(b);
+ } else {
+ res.append("[").append(a).append("-").append(b).append("]");
+ }
+ }
+
+ public String getDeletedVersionsRanged() {
+ StringBuilder res = new StringBuilder();
+
+ int first = -1;
+ int prior = -1;
+ for (int i : deletedVersions) {
+ if (first == -1) {
+ first = prior = i;
+ } else if (prior - i > 1) {
+ append(res, first, prior);
+ first = prior = i;
+ } else {
+ prior = i;
+ }
+ }
+ append(res, first, prior);
+
+ return res.toString();
+ }
+
}
}