diff --git a/src/main/java/com/drunkendev/confluence/plugins/attachments/PurgeAttachmentsJob.java b/src/main/java/com/drunkendev/confluence/plugins/attachments/PurgeAttachmentsJob.java index 8e8f1f4..2e7c59c 100644 --- a/src/main/java/com/drunkendev/confluence/plugins/attachments/PurgeAttachmentsJob.java +++ b/src/main/java/com/drunkendev/confluence/plugins/attachments/PurgeAttachmentsJob.java @@ -24,18 +24,18 @@ import java.time.Duration; import java.time.Instant; import java.time.LocalDateTime; +import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.ArrayDeque; import java.util.ArrayList; -import java.util.Calendar; import java.util.Collections; import java.util.Comparator; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Supplier; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.ImmutablePair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,7 +48,8 @@ import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toSet; -import static org.apache.commons.lang.StringUtils.isNotBlank; +import static org.apache.commons.lang3.StringUtils.equalsIgnoreCase; +import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.commons.lang3.StringUtils.repeat; @@ -64,8 +65,8 @@ public class PurgeAttachmentsJob implements JobRunner { private static final Comparator COMP_ATTACHMENT_VERSION = nullsFirst(comparingInt(n -> n.getVersion())); private static final Comparator COMP_MAILLOG_SPACE_TITLE - = comparing((MailLogEntry n) -> n.getAttachment().getSpace().getName(), nullsFirst(naturalOrder())) - .thenComparing((MailLogEntry n) -> n.getAttachment().getDisplayTitle(), nullsFirst(naturalOrder())); + = comparing((MailLogEntry n) -> n.getSpaceName(), nullsFirst(naturalOrder())) + .thenComparing((MailLogEntry n) -> n.getDisplayTitle(), nullsFirst(naturalOrder())); private static final int BATCH_SIZE = 50; @@ -95,7 +96,6 @@ public PurgeAttachmentsJob(AttachmentManager attachmentManager, MultiQueueTaskManager mailQueueTaskManager, SettingsManager settingsManager, TransactionTemplate transactionTemplate) { - LOG.debug("Creating purge-old-attachment-job instance."); this.attachmentManager = attachmentManager; this.spaceManager = spaceManager; this.settingSvc = purgeAttachmentsSettingsService; @@ -152,9 +152,9 @@ public static Duration time(Runnable r) { } @Override - public JobRunnerResponse runJob(JobRunnerRequest jrr) { + public JobRunnerResponse runJob(JobRunnerRequest req) { + LOG.info("Purge attachment revisions started."); try { - LOG.info("Purge old attachments started."); LocalDateTime start = LocalDateTime.now(); PurgeAttachmentSettings systemSettings = getSystemSettings(); @@ -173,99 +173,35 @@ public JobRunnerResponse runJob(JobRunnerRequest jrr) { = time(() -> attachmentManager.getAttachmentDao().findAll() .stream() .map(Attachment::getId).collect(toCollection(ArrayDeque::new))); - LOG.debug("FIND ALL ({}) : {}", findAll.left, findAll.right.size()); + LOG.debug("Got {} attachments in {}.", findAll.right.size(), findAll.left); ArrayDeque ids = findAll.right; - while (!ids.isEmpty()) { - LOG.debug("Processing batch {}; {} remain", ++counters[IDX_BATCHES], ids.size()); + + while (!ids.isEmpty() && !req.isCancellationRequested()) { + LOG.debug("Processing batch {}; {} atttachments remain", ++counters[IDX_BATCHES], ids.size()); transactionTemplate.execute(() -> { AttachmentDao dao = attachmentManager.getAttachmentDao(); - try { - for (int i = 0; i < BATCH_SIZE && !ids.isEmpty(); i++) { - Attachment attachment = attachmentManager.getAttachment(ids.poll()); - counters[IDX_CURRENT_VERSIONS]++; - - if (attachment.getVersion() > 1 && spaceSettings.containsKey(attachment.getSpaceKey())) { - counters[IDX_CURRENT_VISITED]++; - - PurgeAttachmentSettings settings = spaceSettings.get(attachment.getSpaceKey()); - - List prior = attachmentManager.getPreviousVersions(attachment); - counters[IDX_PRIOR_VERSIONS] += prior.size(); - - List toDelete = findDeletions(prior, settings); - Set badVersions = toDelete.stream() - .filter(n -> n.getVersion() >= attachment.getVersion()) - .map(n -> n.getVersion()) - .collect(toSet()); - if (badVersions.size() > 0) { - LOG.error("Attachment bas invalid prior versions: {}:{} :- {} ({}) :: {}", - attachment.getSpaceKey(), - attachment.getSpace().getName(), - attachment.getDisplayTitle(), - attachment.getVersion(), - badVersions); - } else if (!toDelete.isEmpty()) { - boolean canUpdate; - if (!settings.isReportOnly() && !systemSettings.isReportOnly()) { - canUpdate = systemSettings.getDeleteLimit() == 0 || - counters[IDX_PROCESS_LIMIT] < systemSettings.getDeleteLimit(); - } else { - canUpdate = false; - } - - if (canUpdate) { - counters[IDX_PROCESS_LIMIT]++; - } - - long spaceSaved = toDelete.stream().map(p -> { - LOG.debug("Attachment to remove {}", p.getId()); - if (canUpdate) { -// attachmentManager.removeAttachmentVersionFromServer(p); - Duration dur = time(() -> dao.removeAttachmentVersionFromServer(p)); - counters[IDX_DELETED]++; - counters[IDX_DELETED_TIME] += dur.toMillis(); - } else { - counters[IDX_DELETE_AVAIL]++; - } - return p.getFileSize(); - }).reduce(0L, (a, b) -> a + b); - - MailLogEntry mle = new MailLogEntry( - attachment, - toDelete.stream().map(Attachment::getVersion).collect(toList()), - !canUpdate, - settings == systemSettings, - spaceSaved); - - if (settings != systemSettings && StringUtils.isNotBlank(settings.getReportEmailAddress())) { - if (!mailEntries.containsKey(settings.getReportEmailAddress())) { - mailEntries.put(settings.getReportEmailAddress(), new ArrayList<>()); - } - mailEntries.get(settings.getReportEmailAddress()).add(mle); - } - //TODO: I know this will log twice if system email and space - // email are the same, will fix later, just hacking atm. - if (isNotBlank(systemSettings.getReportEmailAddress())) { - if (!mailEntries.containsKey(systemSettings.getReportEmailAddress())) { - mailEntries.put(systemSettings.getReportEmailAddress(), new ArrayList<>()); - } - mailEntries.get(systemSettings.getReportEmailAddress()).add(mle); - } - } - } - } // for id - return null; - } catch (Throwable ex) { - throw new RuntimeException(ex); + for (int i = 0; i < BATCH_SIZE && !ids.isEmpty() && !req.isCancellationRequested(); i++) { + Attachment attachment = attachmentManager.getAttachment(ids.poll()); + process(attachment, + counters, + spaceSettings.get(attachment.getSpaceKey()), + systemSettings, + dao, + mailEntries); } + return null; }); } + if (req.isCancellationRequested()) { + LOG.warn("Attachment purging has been cancelled."); + } + LocalDateTime end = LocalDateTime.now(); long ms = Duration.between(start, end).toMillis(); - LOG.info("{} processable prior versions found for {} attachments.", + LOG.info("{} prior versions visited for {} attachments.", counters[IDX_PRIOR_VERSIONS], counters[IDX_CURRENT_VERSIONS]); if (counters[IDX_CURRENT_VISITED] > 0) { @@ -281,30 +217,113 @@ public JobRunnerResponse runJob(JobRunnerRequest jrr) { LOG.info("A further {} versions are available for deleting.", counters[IDX_DELETE_AVAIL]); LOG.info("Attachment purging completed in {} ms.", ms); - try { - if (systemSettings.isSendPlainTextMail()) { - mailResultsPlain(mailEntries, - start, - end, - counters); - } else { - mailResultsHtml(mailEntries, - start, - end, - counters); - } - } catch (MailException ex) { - LOG.error("Exception raised while trying to mail results.", ex); - return JobRunnerResponse.failed("Task completed but could not email."); + + if (systemSettings.isSendPlainTextMail()) { + mailResultsPlain(mailEntries, + start, + end, + counters, + req.isCancellationRequested()); + } else { + mailResultsHtml(mailEntries, + start, + end, + counters, + req.isCancellationRequested()); } - LOG.debug("Purge attachments complete."); + } catch (MailException ex) { + LOG.error("Exception raised while trying to mail results.", ex); + return JobRunnerResponse.failed("Task completed but could not email."); } catch (Throwable ex) { - LOG.error("Exception in job: {}", ex.getMessage(), ex); + LOG.error("Purge attachment revisions failed: {}", ex.getMessage(), ex); return JobRunnerResponse.failed(ex); } + LOG.info("Purge attachment revisions completed."); return JobRunnerResponse.success(); } + private void process(Attachment attachment, + long[] counters, + PurgeAttachmentSettings settings, + PurgeAttachmentSettings systemSettings, + AttachmentDao dao, + Map> mailEntries) { + counters[IDX_CURRENT_VERSIONS]++; + + if (attachment.getVersion() == 1) { + LOG.trace("Skipping only attachment version {}", attachment.getId()); + return; + } + + if (settings == null) { + settings = systemSettings; + } + + counters[IDX_CURRENT_VISITED]++; + + List prior = attachmentManager.getPreviousVersions(attachment); + counters[IDX_PRIOR_VERSIONS] += prior.size(); + + List toDelete = findDeletions(prior, settings); + Set badVersions = toDelete.stream() + .filter(n -> n.getVersion() >= attachment.getVersion()) + .map(n -> n.getVersion()) + .collect(toSet()); + if (badVersions.size() > 0) { + LOG.error("Attachment with versions to delete > current version: {}:{} :- {} ({}) :: {}", + attachment.getSpaceKey(), + attachment.getSpace().getName(), + attachment.getDisplayTitle(), + attachment.getVersion(), + badVersions); + } else if (!toDelete.isEmpty()) { + boolean canUpdate + = !settings.isReportOnly() && + !systemSettings.isReportOnly() && + (systemSettings.getDeleteLimit() == 0 || + counters[IDX_PROCESS_LIMIT] < systemSettings.getDeleteLimit()); + + if (canUpdate) { + counters[IDX_PROCESS_LIMIT]++; + } + + long spaceSaved = toDelete.stream().map(p -> { + LOG.debug("Attachment to remove {}", p.getId()); + if (canUpdate) { + Duration dur = time(() -> dao.removeAttachmentVersionFromServer(p)); + counters[IDX_DELETED]++; + counters[IDX_DELETED_TIME] += dur.toMillis(); + } else { + counters[IDX_DELETE_AVAIL]++; + } + return p.getFileSize(); + }).reduce(0L, (a, b) -> a + b); + + if (isNotBlank(settings.getReportEmailAddress()) || isNotBlank(systemSettings.getReportEmailAddress())) { + MailLogEntry mle = new MailLogEntry( + attachment, + toDelete.stream().map(Attachment::getVersion).collect(toList()), + !canUpdate, + settings == systemSettings, + spaceSaved); + + if (isNotBlank(settings.getReportEmailAddress())) { + if (!mailEntries.containsKey(settings.getReportEmailAddress())) { + mailEntries.put(settings.getReportEmailAddress(), new ArrayList<>()); + } + mailEntries.get(settings.getReportEmailAddress()).add(mle); + } + if (isNotBlank(systemSettings.getReportEmailAddress()) && !equalsIgnoreCase(settings.getReportEmailAddress(), systemSettings.getReportEmailAddress())) { + if (!mailEntries.containsKey(systemSettings.getReportEmailAddress())) { + mailEntries.put(systemSettings.getReportEmailAddress(), new ArrayList<>()); + } + mailEntries.get(systemSettings.getReportEmailAddress()).add(mle); + } + } + + } + } + private List findDeletions(List prior, PurgeAttachmentSettings stng) { if (prior == null || prior.isEmpty()) { return Collections.emptyList(); @@ -341,22 +360,31 @@ private List findDeletions(List prior, PurgeAttachmentSe } private int filterRevisionCount(List prior, int maxRevisions) { - if (prior.size() > maxRevisions) { - return (prior.size() - maxRevisions) - 1; - } else { - return -1; - } + return prior.size() > maxRevisions + ? (prior.size() - maxRevisions) - 1 + : -1; + } + + public static LocalDateTime toLocalDateTime(Date value) { + return toLocalDateTime(value, ZoneId.systemDefault()); + } + + public static LocalDateTime toLocalDateTime(Date value, ZoneId zoneId) { + // NOTE: java.sql.Date does not support toInstant. To prevent an UnsupportedOperationException + // do not use toInstant on dates. + //return value == null ? null : LocalDateTime.ofInstant(value.toInstant(), zoneId); + return value == null ? null + : LocalDateTime.ofInstant(Instant.ofEpochMilli(value.getTime()), + zoneId); } private int filterAge(List prior, int maxDaysOld) { - Calendar dateFrom = Calendar.getInstance(); - dateFrom.add(Calendar.DAY_OF_MONTH, -(maxDaysOld)); - Calendar modDate = Calendar.getInstance(); + LocalDateTime from = LocalDateTime.now().minusDays(maxDaysOld); for (int i = prior.size() - 1; i >= 0; i--) { if (prior.get(i).getLastModificationDate() != null) { - modDate.setTime(prior.get(i).getLastModificationDate()); - if (dateFrom.after(modDate)) { + LocalDateTime mod = toLocalDateTime(prior.get(i).getLastModificationDate()); + if (from.isAfter(mod)) { return i; } } @@ -380,7 +408,8 @@ private int filterSize(List prior, long maxTotalSize) { private void mailResultsPlain(Map> entries, LocalDateTime started, LocalDateTime ended, - long[] counters) throws MailException { + long[] counters, + boolean cancellationRequested) throws MailException { String p = settingsManager.getGlobalSettings().getBaseUrl(); entries.forEach((emailAddress, entryList) -> { @@ -400,6 +429,9 @@ private void mailResultsPlain(Map> entries, deleted += me.getSpaceSaved(); } } + if (cancellationRequested) { + sb.append("CANCELLED: Job has had an early cancellation request."); + } if (deleted > 0) { sb.append("A total of ").append(FileSize.format(deleted)) .append(" space has been reclaimed.\n"); @@ -413,15 +445,14 @@ private void mailResultsPlain(Map> entries, String ps = null; for (MailLogEntry me : entryList) { - Attachment a = me.getAttachment(); - if (ps == null || !ps.equalsIgnoreCase(a.getSpaceKey())) { + if (ps == null || !ps.equalsIgnoreCase(me.getSpaceKey())) { sb.append("\n"); - ps = a.getSpaceKey(); - String sp = ps + ":" + a.getSpace().getName() + " (" + p + a.getSpace().getUrlPath() + ")"; + ps = me.getSpaceKey(); + String sp = ps + ":" + me.getSpaceName() + " (" + p + me.getSpaceUrlPath() + ")"; sb.append(sp).append('\n').append(repeat('-', sp.length())).append('\n'); } - sb.append(a.getDisplayTitle()).append(" (").append(a.getVersion()).append(") "); + sb.append(me.getDisplayTitle()).append(" (").append(me.getVersion()).append(") "); sb.append(me.isReportOnly() ? "TO_DELETE:" : "DELETED:"); me.getDeletedVersions().stream().forEach(ver -> sb.append(" ").append(ver)); sb.append(" [").append(me.getSpaceSavedPretty()).append("]\n"); @@ -460,14 +491,15 @@ private void mailResultsPlain(Map> entries, }); } - private void mailResultsHtml(Map> mailEntries1, + private void mailResultsHtml(Map> mailEntries, LocalDateTime started, LocalDateTime ended, - long[] counters) throws MailException { + long[] counters, + boolean cancellationRequested) throws MailException { String p = settingsManager.getGlobalSettings().getBaseUrl(); String subject = "Purged old attachments"; - mailEntries1.forEach((emailAddress, entryList) -> { + mailEntries.forEach((emailAddress, entryList) -> { Collections.sort(entryList, COMP_MAILLOG_SPACE_TITLE); StringBuilder sb = new StringBuilder(); @@ -506,9 +538,12 @@ private void mailResultsHtml(Map> mailEntries1, sb.append(" other rows are in report-only mode."); sb.append("

"); + if (cancellationRequested) { + sb.append("

CANCELLED: Job has had an early cancellation request.

"); + } sb.append("

Started: ") .append(started.format(DateTimeFormatter.ISO_DATE_TIME)) - .append(" Ended: ") + .append("
Ended: ") .append(ended.format(DateTimeFormatter.ISO_DATE_TIME)) .append("

"); @@ -548,8 +583,6 @@ private void mailResultsHtml(Map> mailEntries1, sb.append(""); for (MailLogEntry me : entryList) { - Attachment a = me.getAttachment(); - sb.append("> mailEntries1, sb.append(">"); sb.append(""); - sb.append("") - .append(a.getSpace().getName()).append(""); + sb.append("") + .append(me.getSpaceName()).append(""); sb.append(""); sb.append(""); - sb.append("") - .append(a.getDisplayTitle()).append(""); + sb.append("") + .append(me.getDisplayTitle()).append(""); sb.append(""); sb.append("").append(me.getSpaceSavedPretty()).append(""); //sb.append("").append(me.isGlobalSettings() ? "Yes" : "No").append(""); - sb.append("").append(a.getVersion()).append(""); + sb.append("").append(me.getVersion()).append(""); - sb.append(""); - int c = 0; - for (Integer dl : me.getDeletedVersions()) { - if (c++ > 0) { - sb.append(", "); - } - sb.append(dl); - } - sb.append(""); + sb.append("").append(me.getDeletedVersionsRanged()).append(""); sb.append(""); } sb.append(""); sb.append("

").append(counters[IDX_PRIOR_VERSIONS]) - .append("processable prior versions found for ") + .append(" prior versions found for ") .append(counters[IDX_CURRENT_VERSIONS]).append(" attachments.

"); long ms = Duration.between(started, ended).toMillis(); @@ -602,8 +627,10 @@ private void mailResultsHtml(Map> mailEntries1, .append(Math.round(counters[IDX_DELETED_TIME] / (double) counters[IDX_DELETED])) .append(" ms per deletion.

"); } - sb.append("

A further ").append(counters[IDX_DELETE_AVAIL]) - .append(" versions are available for deleting.

"); + if (counters[IDX_DELETE_AVAIL] > 0) { + sb.append("

A further ").append(counters[IDX_DELETE_AVAIL]) + .append(" versions are available for deleting.

"); + } sb.append("

Attachment purging completed in ").append(ms).append(" ms.

"); sb.append("

This message has been sent by Attachment Tools - Purge Attachment Versions

"); @@ -626,22 +653,60 @@ private void mailResultsHtml(Map> mailEntries1, */ private class MailLogEntry { - private final Attachment attachment; + private final String spaceKey; + private final String spaceName; + private final String spaceUrlPath; + + private final String displayTitle; + private final String attachmentsUrlPath; + private final int version; + private final List deletedVersions; private final boolean reportOnly; private final boolean globalSettings; private final long spaceSaved; - private MailLogEntry(Attachment a, List deletedVersions, boolean reportOnly, boolean globalSettings, long spaceSaved) { - this.attachment = a; + private MailLogEntry(Attachment a, + List deletedVersions, + boolean reportOnly, + boolean globalSettings, + long spaceSaved) { + this.spaceKey = a.getSpaceKey(); + this.spaceName = a.getSpace() == null ? null : a.getSpace().getName(); + this.spaceUrlPath = a.getSpace() == null ? null : a.getSpace().getUrlPath(); + + this.displayTitle = a.getDisplayTitle(); + this.attachmentsUrlPath = a.getAttachmentsUrlPath(); + this.version = a.getVersion(); + this.deletedVersions = deletedVersions; this.reportOnly = reportOnly; this.globalSettings = globalSettings; this.spaceSaved = spaceSaved; } - private Attachment getAttachment() { - return attachment; + public String getSpaceKey() { + return spaceKey; + } + + public String getSpaceName() { + return spaceName; + } + + public String getSpaceUrlPath() { + return spaceUrlPath; + } + + public String getDisplayTitle() { + return displayTitle; + } + + public String getAttachmentsUrlPath() { + return attachmentsUrlPath; + } + + public int getVersion() { + return version; } private List getDeletedVersions() { @@ -664,6 +729,42 @@ private String getSpaceSavedPretty() { return FileSize.format(spaceSaved); } + private void append(StringBuilder res, int a, int b) { + if (a == -1 || b == -1) { + return; + } + if (res.length() > 0) { + res.append(", "); + } + if (a == b) { + res.append(a); +// } else if (b - a == 1) { +// res.append(a).append(", ").append(b); + } else { + res.append("[").append(a).append("-").append(b).append("]"); + } + } + + public String getDeletedVersionsRanged() { + StringBuilder res = new StringBuilder(); + + int first = -1; + int prior = -1; + for (int i : deletedVersions) { + if (first == -1) { + first = prior = i; + } else if (prior - i > 1) { + append(res, first, prior); + first = prior = i; + } else { + prior = i; + } + } + append(res, first, prior); + + return res.toString(); + } + } }