Skip to content

Commit

Permalink
Released with v5.10 support.
Browse files Browse the repository at this point in the history 
- Migrated AbstractJob to JobRunner.
- Batch removals to stop hogging the hibernate session.
- No longer raising delete event exceptions.
- Performance enhancement (2,500 ms average down to 7).
  • Loading branch information
@brettryan
brettryan committed Oct 17, 2016
1 parent b4d9e51 commit 4c87820
Show file tree
Hide file tree
Showing 4 changed files with 147 additions and 109 deletions.
8 changes: 4 additions & 4 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,12 @@
<groupId>com.drunkendev.confluence.plugins</groupId>
<artifactId>attachment-tools-plugin</artifactId>
<name>Attachment Tools Plugin</name>
<version>1.2.0-5.9-alpha-3</version>
<version>1.2.0</version>

<properties>
<confluence.version>5.9.6</confluence.version>
<confluence.data.version>5.9.6</confluence.data.version>
<amps.version>6.2.3</amps.version>
<confluence.version>5.10.7</confluence.version>
<confluence.data.version>5.10.7</confluence.data.version>
<amps.version>6.2.9</amps.version>
<plugin.testrunner.version>1.2.0</plugin.testrunner.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<netbeans.hint.license>dd</netbeans.hint.license>
Expand Down
229 changes: 134 additions & 95 deletions src/main/java/com/drunkendev/confluence/plugins/attachments/PurgeAttachmentsJob.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,38 +10,41 @@
import com.atlassian.confluence.mail.template.ConfluenceMailQueueItem;
import com.atlassian.confluence.pages.Attachment;
import com.atlassian.confluence.pages.AttachmentManager;
import com.atlassian.confluence.pages.persistence.dao.AttachmentDao;
import com.atlassian.confluence.setup.settings.SettingsManager;
import com.atlassian.confluence.spaces.SpaceManager;
import com.atlassian.confluence.spaces.SpaceStatus;
import com.atlassian.core.task.MultiQueueTaskManager;
import com.atlassian.core.util.FileSize;
import com.atlassian.mail.MailException;
import com.atlassian.quartz.jobs.AbstractJob;
import com.atlassian.sal.api.transaction.TransactionTemplate;
import com.atlassian.scheduler.JobRunner;
import com.atlassian.scheduler.JobRunnerRequest;
import com.atlassian.scheduler.JobRunnerResponse;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Supplier;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static java.util.Comparator.comparing;
import static java.util.Comparator.comparingInt;
import static java.util.Comparator.naturalOrder;
import static java.util.Comparator.nullsFirst;
import static java.util.stream.Collectors.toCollection;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toMap;
import static java.util.stream.Collectors.toSet;
Expand All @@ -54,15 +57,17 @@
*
* @author Brett Ryan
*/
public class PurgeAttachmentsJob extends AbstractJob {
public class PurgeAttachmentsJob implements JobRunner {

private static final Logger LOG = LoggerFactory.getLogger(PurgeAttachmentsJob.class);

private static final Comparator<Attachment> COMP_ATTACHMENT_VERSION
= nullsFirst(comparingInt(n -> n.getVersion()));
private static final Comparator<MailLogEntry> COMP_MAILLOG_SPACE_TITLE
= comparing((MailLogEntry n) -> n.getAttachment().getSpace().getName(), nullsFirst(naturalOrder()))
.thenComparing((MailLogEntry n) -> n.getAttachment().getDisplayTitle(), nullsFirst(naturalOrder()));
.thenComparing((MailLogEntry n) -> n.getAttachment().getDisplayTitle(), nullsFirst(naturalOrder()));

private static final int BATCH_SIZE = 50;

private static final int IDX_PRIOR_VERSIONS = 0;
private static final int IDX_DELETED = 1;
Expand All @@ -71,6 +76,8 @@ public class PurgeAttachmentsJob extends AbstractJob {
private static final int IDX_CURRENT_VERSIONS = 4;
private static final int IDX_CURRENT_VISITED = 5;
private static final int IDX_PROCESS_LIMIT = 6;
private static final int IDX_BATCHES = 7;
private static final int COUNTER_ARRAY_SIZE = 8;

private final AttachmentManager attachmentManager;
private final SpaceManager spaceManager;
Expand Down Expand Up @@ -132,99 +139,127 @@ private PurgeAttachmentSettings getSystemSettings() {
return res;
}

public static <T> ImmutablePair<Duration, T> time(Supplier<T> r) {
Instant start = Instant.now();
T res = r.get();
return new ImmutablePair<>(Duration.between(start, Instant.now()), res);
}

public static Duration time(Runnable r) {
Instant start = Instant.now();
r.run();
return Duration.between(start, Instant.now());
}

@Override
public void doExecute(JobExecutionContext jec) throws JobExecutionException {
LOG.debug("Purge old attachments started.");
LocalDateTime start = LocalDateTime.now();
public JobRunnerResponse runJob(JobRunnerRequest jrr) {
try {
LOG.info("Purge old attachments started.");
LocalDateTime start = LocalDateTime.now();

PurgeAttachmentSettings systemSettings = getSystemSettings();
Map<String, PurgeAttachmentSettings> spaceSettings = getAllSpaceSettings(systemSettings);
PurgeAttachmentSettings systemSettings = getSystemSettings();
Map<String, PurgeAttachmentSettings> spaceSettings = getAllSpaceSettings(systemSettings);

if (LOG.isDebugEnabled()) {
LOG.debug("System settings: {}", systemSettings);
spaceSettings.forEach((k, v) -> LOG.debug("Space Settings: {} -> {}", k, v));
}
if (LOG.isDebugEnabled()) {
LOG.debug("System settings: {}", systemSettings);
spaceSettings.forEach((k, v) -> LOG.debug("Space Settings: {} -> {}", k, v));
}

Map<String, List<MailLogEntry>> mailEntries = new HashMap<>();

long[] counters = new long[7];

transactionTemplate.execute(() -> {
Iterator<Attachment> attIter = attachmentManager.getAttachmentDao().findLatestVersionsIterator();

while (attIter.hasNext()) {
Attachment attachment = attIter.next();
counters[IDX_CURRENT_VERSIONS]++;

if (attachment.getVersion() > 1 && spaceSettings.containsKey(attachment.getSpaceKey())) {
counters[IDX_CURRENT_VISITED]++;

PurgeAttachmentSettings settings = spaceSettings.get(attachment.getSpaceKey());

List<Attachment> prior = attachmentManager.getPreviousVersions(attachment);
counters[IDX_PRIOR_VERSIONS] += prior.size();

List<Attachment> toDelete = findDeletions(prior, settings);
Set<Integer> badVersions = toDelete.stream()
.filter(n -> n.getVersion() >= attachment.getVersion())
.map(n -> n.getVersion())
.collect(toSet());
if (badVersions.size() > 0) {
LOG.error("Attachment bas invalid prior versions: {}:{} :- {} ({}) :: {}",
attachment.getSpaceKey(),
attachment.getSpace().getName(),
attachment.getDisplayTitle(),
attachment.getVersion(),
badVersions);
} else if (!toDelete.isEmpty()) {
boolean canUpdate;
if (!settings.isReportOnly() && !systemSettings.isReportOnly()) {
canUpdate = systemSettings.getDeleteLimit() == 0 ||
counters[IDX_PROCESS_LIMIT] < systemSettings.getDeleteLimit();
} else {
canUpdate = false;
}

if (canUpdate) {
counters[IDX_PROCESS_LIMIT]++;
}

long spaceSaved = toDelete.stream().map(p -> {
LOG.debug("Attachment to remove {}", p.getId());
if (canUpdate) {
Instant s = Instant.now();
attachmentManager.removeAttachmentVersionFromServer(p);
counters[IDX_DELETED]++;
counters[IDX_DELETED_TIME] += Duration.between(s, Instant.now()).toMillis();
} else {
counters[IDX_DELETE_AVAIL]++;
Map<String, List<MailLogEntry>> mailEntries = new HashMap<>();

long[] counters = new long[COUNTER_ARRAY_SIZE];

ImmutablePair<Duration, ArrayDeque<Long>> findAll
= time(() -> attachmentManager.getAttachmentDao().findAll()
.stream()
.map(Attachment::getId).collect(toCollection(ArrayDeque::new)));
LOG.debug("FIND ALL ({}) : {}", findAll.left, findAll.right.size());

ArrayDeque<Long> ids = findAll.right;
while (!ids.isEmpty()) {
LOG.debug("Processing batch {}; {} remain", ++counters[IDX_BATCHES], ids.size());
transactionTemplate.execute(() -> {
AttachmentDao dao = attachmentManager.getAttachmentDao();
try {
for (int i = 0; i < BATCH_SIZE && !ids.isEmpty(); i++) {
Attachment attachment = attachmentManager.getAttachment(ids.poll());
counters[IDX_CURRENT_VERSIONS]++;

if (attachment.getVersion() > 1 && spaceSettings.containsKey(attachment.getSpaceKey())) {
counters[IDX_CURRENT_VISITED]++;

PurgeAttachmentSettings settings = spaceSettings.get(attachment.getSpaceKey());

List<Attachment> prior = attachmentManager.getPreviousVersions(attachment);
counters[IDX_PRIOR_VERSIONS] += prior.size();

List<Attachment> toDelete = findDeletions(prior, settings);
Set<Integer> badVersions = toDelete.stream()
.filter(n -> n.getVersion() >= attachment.getVersion())
.map(n -> n.getVersion())
.collect(toSet());
if (badVersions.size() > 0) {
LOG.error("Attachment bas invalid prior versions: {}:{} :- {} ({}) :: {}",
attachment.getSpaceKey(),
attachment.getSpace().getName(),
attachment.getDisplayTitle(),
attachment.getVersion(),
badVersions);
} else if (!toDelete.isEmpty()) {
boolean canUpdate;
if (!settings.isReportOnly() && !systemSettings.isReportOnly()) {
canUpdate = systemSettings.getDeleteLimit() == 0 ||
counters[IDX_PROCESS_LIMIT] < systemSettings.getDeleteLimit();
} else {
canUpdate = false;
}

if (canUpdate) {
counters[IDX_PROCESS_LIMIT]++;
}

long spaceSaved = toDelete.stream().map(p -> {
LOG.debug("Attachment to remove {}", p.getId());
if (canUpdate) {
// attachmentManager.removeAttachmentVersionFromServer(p);
Duration dur = time(() -> dao.removeAttachmentVersionFromServer(p));
counters[IDX_DELETED]++;
counters[IDX_DELETED_TIME] += dur.toMillis();
} else {
counters[IDX_DELETE_AVAIL]++;
}
return p.getFileSize();
}).reduce(0L, (a, b) -> a + b);

MailLogEntry mle = new MailLogEntry(
attachment,
toDelete.stream().map(Attachment::getVersion).collect(toList()),
!canUpdate,
settings == systemSettings,
spaceSaved);

if (settings != systemSettings && StringUtils.isNotBlank(settings.getReportEmailAddress())) {
if (!mailEntries.containsKey(settings.getReportEmailAddress())) {
mailEntries.put(settings.getReportEmailAddress(), new ArrayList<>());
}
mailEntries.get(settings.getReportEmailAddress()).add(mle);
}
//TODO: I know this will log twice if system email and space
// email are the same, will fix later, just hacking atm.
if (isNotBlank(systemSettings.getReportEmailAddress())) {
if (!mailEntries.containsKey(systemSettings.getReportEmailAddress())) {
mailEntries.put(systemSettings.getReportEmailAddress(), new ArrayList<>());
}
mailEntries.get(systemSettings.getReportEmailAddress()).add(mle);
}
}
}
return p.getFileSize();
}).reduce(0L, (a, b) -> a + b);

MailLogEntry mle = new MailLogEntry(
attachment,
toDelete.stream().map(Attachment::getVersion).collect(toList()),
!canUpdate,
settings == systemSettings,
spaceSaved);

if (settings != systemSettings && StringUtils.isNotBlank(settings.getReportEmailAddress())) {
if (!mailEntries.containsKey(settings.getReportEmailAddress())) {
mailEntries.put(settings.getReportEmailAddress(), new ArrayList<>());
}
mailEntries.get(settings.getReportEmailAddress()).add(mle);
}
//TODO: I know this will log twice if system email and space
// email are the same, will fix later, just hacking atm.
if (isNotBlank(systemSettings.getReportEmailAddress())) {
if (!mailEntries.containsKey(systemSettings.getReportEmailAddress())) {
mailEntries.put(systemSettings.getReportEmailAddress(), new ArrayList<>());
}
mailEntries.get(systemSettings.getReportEmailAddress()).add(mle);
}
} // for id
return null;
} catch (Throwable ex) {
throw new RuntimeException(ex);
}
}
});
}

LocalDateTime end = LocalDateTime.now();
Expand Down Expand Up @@ -260,10 +295,14 @@ public void doExecute(JobExecutionContext jec) throws JobExecutionException {
}
} catch (MailException ex) {
LOG.error("Exception raised while trying to mail results.", ex);
return JobRunnerResponse.failed("Task completed but could not email.");
}
LOG.debug("Purge attachments complete.");
return null;
});
} catch (Throwable ex) {
LOG.error("Exception in job: {}", ex.getMessage(), ex);
return JobRunnerResponse.failed(ex);
}
return JobRunnerResponse.success();
}

private List<Attachment> findDeletions(List<Attachment> prior, PurgeAttachmentSettings stng) {
Expand Down
16 changes: 7 additions & 9 deletions src/main/resources/atlassian-plugin.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
class="com.drunkendev.confluence.plugins.attachments.PurgeAttachmentsSettingsService"/>

<web-resource key="purge-attachment-images" name="Purge Attachment Images">
<resource type="download" name="images/" location="images"/>
<resource type="download" name="images/" location="images"/>
</web-resource>

<web-item key="configure-purge-attachments-space"
Expand All @@ -44,21 +44,19 @@

<resource type="i18n" name="i18n" location="i18n" />

<job key="purge-old-attachments-job"
name="Purge Attachment Versions"
class="com.drunkendev.confluence.plugins.attachments.PurgeAttachmentsJob"
perClusterJob="false" />
<component key="purge-old-attachments-job"
class="com.drunkendev.confluence.plugins.attachments.PurgeAttachmentsJob"/>

<trigger key="purge-old-attachments-trigger"
name="Purge Attachment Versions - Trigger">
<job key="purge-old-attachments-job" />
<job-config key="purge-old-attachments-trigger"
name="Purge Attachment Versions - Trigger">
<job key="purge-old-attachments-job" perClusterJob="false" />
<!-- Run once a day -->
<schedule cron-expression="0 0 0 * * ?"/>
<managed editable="true"
keepingHistory="true"
canRunAdhoc="true"
canDisable="true" />
</trigger>
</job-config>

<xwork key="configure-purge-attachments" name="Configure Purge Attachments">

Expand Down
3 changes: 2 additions & 1 deletion src/main/resources/i18n.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@
# Author: Brett Ryan

# Attachment Purging - Job name.
scheduledjob.desc.purge-old-attachments-job=Purge Attachment Revisions
#scheduledjob.desc.purge-old-attachments-job=Purge Attachment Revisions
scheduledjob.desc.purge-old-attachments-trigger=Purge Attachment Revisions

# Attachment Purging - Space menu-item config text.
com.drunkendev.confluence.plugins.attachment-tools-plugin.config.purge=Attachment Purging
Expand Down

0 comments on commit 4c87820

Please sign in to comment.