From 743fbd229bc5a127a3b1b69caaf7a56e0e4b4acb Mon Sep 17 00:00:00 2001 From: Marcono1234 Date: Thu, 9 Dec 2021 19:39:19 +0100 Subject: [PATCH] Redact access tokens in attachments (#709) * Redact access tokens in attachments Redact access tokens in attachments (e.g. JVM crash reports) by deleting the original attachment and uploading a redacted version. Additionally add a comment to let the user know that their attachment has been redacted. * Log where PrivacyModule found sensitive data This hopefully makes it easier to understand why an issue has been made private and allows improving or removing imprecise regex patterns. --- config/local.template.yml | 2 +- docs/Modules.md | 14 +- .../io/github/mojira/arisa/ModuleExecutor.kt | 2 +- .../github/mojira/arisa/domain/Attachment.kt | 8 +- .../mojira/arisa/domain/ChangeLogItem.kt | 4 + .../io/github/mojira/arisa/domain/Issue.kt | 1 + .../io/github/mojira/arisa/domain/User.kt | 3 +- .../arisa/infrastructure/AttachmentUtils.kt | 10 +- .../arisa/infrastructure/HelperMessages.kt | 10 +- .../arisa/infrastructure/jira/Mappers.kt | 64 ++-- .../mojira/arisa/modules/AttachmentModule.kt | 18 +- .../io/github/mojira/arisa/modules/Helpers.kt | 5 +- .../arisa/modules/KeepPlatformModule.kt | 6 +- .../mojira/arisa/modules/PrivacyModule.kt | 90 ----- .../arisa/modules/ReopenAwaitingModule.kt | 4 +- .../mojira/arisa/modules/ReplaceTextModule.kt | 13 +- .../modules/commands/DeobfuscateCommand.kt | 11 +- .../modules/privacy/AttachmentRedactor.kt | 48 +++ .../arisa/modules/privacy/PrivacyModule.kt | 246 +++++++++++++ .../modules/privacy/TextRangeLocation.kt | 84 +++++ .../arisa/registry/InstantModuleRegistry.kt | 27 +- .../infrastructure/HelperMessagesTest.kt | 6 + .../mojira/arisa/modules/CommandModuleTest.kt | 2 +- .../mojira/arisa/modules/CrashModuleTest.kt | 14 +- .../arisa/modules/ReopenAwaitingModuleTest.kt | 4 +- .../modules/RevokeConfirmationModuleTest.kt | 48 +-- .../commands/DeobfuscateCommandTest.kt | 2 +- .../arguments/EnumArgumentTypeTest.kt | 2 + .../privacy/AccessTokenRedactorTest.kt | 33 ++ .../{ => privacy}/PrivacyModuleTest.kt | 330 +++++++++++++++++- .../modules/privacy/TextRangeLocationTest.kt | 77 ++++ .../github/mojira/arisa/utils/MockDomain.kt | 12 +- 32 files changed, 994 insertions(+), 206 deletions(-) delete mode 100644 src/main/kotlin/io/github/mojira/arisa/modules/PrivacyModule.kt create mode 100644 src/main/kotlin/io/github/mojira/arisa/modules/privacy/AttachmentRedactor.kt create mode 100644 src/main/kotlin/io/github/mojira/arisa/modules/privacy/PrivacyModule.kt create mode 100644 src/main/kotlin/io/github/mojira/arisa/modules/privacy/TextRangeLocation.kt create mode 100644 src/test/kotlin/io/github/mojira/arisa/modules/privacy/AccessTokenRedactorTest.kt rename src/test/kotlin/io/github/mojira/arisa/modules/{ => privacy}/PrivacyModuleTest.kt (59%) create mode 100644 src/test/kotlin/io/github/mojira/arisa/modules/privacy/TextRangeLocationTest.kt diff --git a/config/local.template.yml b/config/local.template.yml index f6541e74b..a9b711a81 100644 --- a/config/local.template.yml +++ b/config/local.template.yml @@ -7,6 +7,6 @@ arisa: # These configs are optional and can be omitted dandelionToken: discordLogWebhook: - discordErrorWebhook: + discordErrorLogWebhook: debug: # You can add some debug options here, see `ArisaConfig.kt` for details diff --git a/docs/Modules.md b/docs/Modules.md index a74d5563f..761e87038 100644 --- a/docs/Modules.md +++ b/docs/Modules.md @@ -229,12 +229,14 @@ Resolves tickets about pirated games as `Invalid`. - Any of the description, environment, and/or summary contains any of the `piracySignatures` defined in the [config](../config/config.yml). ## Privacy -| Entry | Value | -| ----- | -------------------------------------------------------------------------- | -| Name | `Privacy` | -| Class | [Link](../src/main/kotlin/io/github/mojira/arisa/modules/PrivacyModule.kt) | +| Entry | Value | +| ----- | ---------------------------------------------------------------------------------- | +| Name | `Privacy` | +| Class | [Link](../src/main/kotlin/io/github/mojira/arisa/modules/privacy/PrivacyModule.kt) | -Hides privacy information like Email addresses in tickets or comments. +Makes tickets and comments, which contain sensitive data like Email addresses, private. +Additionally in some cases it redacts sensitive data from attachments by deleting the original attachment, +reuploading it with redacted content and its name prefixed with `redacted_` and adding a comment informing the uploader. ### Checks - The ticket is not set to private. @@ -242,7 +244,7 @@ Hides privacy information like Email addresses in tickets or comments. - Any of the following matches one of the regex patterns specified by `sensitiveTextRegexes` in the [config](../config/config.yml), or contains an email address which does not match one of the `allowedEmailRegexes`: - summary, environment, description (if ticket was created after last run) - - text attachment (if it was created after last run) + - text attachment (if it was created after last run), and the module was unable to redact the sensitive data - changelog entry string value (if it was created after last run) - Or any of the attachments created after the last run has a name which matches one of `sensitiveFileNameRegexes` in the [config](../config/config.yml). diff --git a/src/main/kotlin/io/github/mojira/arisa/ModuleExecutor.kt b/src/main/kotlin/io/github/mojira/arisa/ModuleExecutor.kt index a6f059bd5..30148cfdd 100644 --- a/src/main/kotlin/io/github/mojira/arisa/ModuleExecutor.kt +++ b/src/main/kotlin/io/github/mojira/arisa/ModuleExecutor.kt @@ -62,6 +62,6 @@ class ModuleExecutor( return allIssues .filter { it.project.key in projects } .filter { it.status.toLowerCase() !in excludedStatuses } - .filter { it.resolution?.toLowerCase() ?: "unresolved" in resolutions } + .filter { (it.resolution?.toLowerCase() ?: "unresolved") in resolutions } } } diff --git a/src/main/kotlin/io/github/mojira/arisa/domain/Attachment.kt b/src/main/kotlin/io/github/mojira/arisa/domain/Attachment.kt index 5682bd8ac..917269470 100644 --- a/src/main/kotlin/io/github/mojira/arisa/domain/Attachment.kt +++ b/src/main/kotlin/io/github/mojira/arisa/domain/Attachment.kt @@ -12,4 +12,10 @@ data class Attachment( val openContentStream: () -> InputStream, val getContent: () -> ByteArray, val uploader: User? -) +) { + /** Returns whether the type of the content is text */ + fun hasTextContent() = mimeType.startsWith("text/") + + /** Decodes the content as UTF-8 String */ + fun getTextContent() = String(getContent()) +} diff --git a/src/main/kotlin/io/github/mojira/arisa/domain/ChangeLogItem.kt b/src/main/kotlin/io/github/mojira/arisa/domain/ChangeLogItem.kt index 01f6c0f26..2671b107d 100644 --- a/src/main/kotlin/io/github/mojira/arisa/domain/ChangeLogItem.kt +++ b/src/main/kotlin/io/github/mojira/arisa/domain/ChangeLogItem.kt @@ -3,6 +3,10 @@ package io.github.mojira.arisa.domain import java.time.Instant data class ChangeLogItem( + /** ID of the enclosing change log entry */ + val entryId: String, + /** 0-based index of this change log item within the enclosing change log entry */ + val itemIndex: Int, val created: Instant, val field: String, val changedFrom: String?, diff --git a/src/main/kotlin/io/github/mojira/arisa/domain/Issue.kt b/src/main/kotlin/io/github/mojira/arisa/domain/Issue.kt index 6820a8046..0d752f7fb 100644 --- a/src/main/kotlin/io/github/mojira/arisa/domain/Issue.kt +++ b/src/main/kotlin/io/github/mojira/arisa/domain/Issue.kt @@ -50,6 +50,7 @@ data class Issue( val addRestrictedComment: (options: CommentOptions) -> Unit, val addNotEnglishComment: (language: String) -> Unit, val addRawRestrictedComment: (body: String, restriction: String) -> Unit, + val addRawBotComment: (rawMessage: String) -> Unit, val markAsFixedWithSpecificVersion: (fixVersionName: String) -> Unit, val changeReporter: (reporter: String) -> Unit, val addAttachmentFromFile: (file: File, cleanupCallback: () -> Unit) -> Unit, diff --git a/src/main/kotlin/io/github/mojira/arisa/domain/User.kt b/src/main/kotlin/io/github/mojira/arisa/domain/User.kt index ac70e3a36..f1028d578 100644 --- a/src/main/kotlin/io/github/mojira/arisa/domain/User.kt +++ b/src/main/kotlin/io/github/mojira/arisa/domain/User.kt @@ -4,5 +4,6 @@ data class User( val name: String?, val displayName: String?, val getGroups: () -> List?, - val isNewUser: () -> Boolean + val isNewUser: () -> Boolean, + val isBotUser: () -> Boolean ) diff --git a/src/main/kotlin/io/github/mojira/arisa/infrastructure/AttachmentUtils.kt b/src/main/kotlin/io/github/mojira/arisa/infrastructure/AttachmentUtils.kt index 096edd629..ee2a390eb 100644 --- a/src/main/kotlin/io/github/mojira/arisa/infrastructure/AttachmentUtils.kt +++ b/src/main/kotlin/io/github/mojira/arisa/infrastructure/AttachmentUtils.kt @@ -13,8 +13,7 @@ fun getDeobfName(name: String): String = "deobf_$name" class AttachmentUtils( private val crashReportExtensions: List, - private val crashReader: CrashReader, - private val botUserName: String + private val crashReader: CrashReader ) { private val mappingsDir by lazy { val file = File("mc-mappings") @@ -37,7 +36,7 @@ class AttachmentUtils( // Get crashes from issue attachments val textDocuments = attachments // Ignore attachments from Arisa (e.g. deobfuscated crash reports) - .filterNot { it.uploader?.name == botUserName } + .filterNot { it.uploader?.isBotUser?.invoke() == true } // Only check attachments with allowed extensions .filter { isCrashAttachment(it.name) } @@ -61,10 +60,7 @@ class AttachmentUtils( crashReportExtensions.any { it == fileName.substring(fileName.lastIndexOf(".") + 1) } fun fetchAttachment(attachment: Attachment): TextDocument { - val getText = { - val data = attachment.getContent() - String(data) - } + val getText = attachment::getTextContent return TextDocument(getText, attachment.created, attachment.name) } diff --git a/src/main/kotlin/io/github/mojira/arisa/infrastructure/HelperMessages.kt b/src/main/kotlin/io/github/mojira/arisa/infrastructure/HelperMessages.kt index 8ac16ed01..39bda5c69 100644 --- a/src/main/kotlin/io/github/mojira/arisa/infrastructure/HelperMessages.kt +++ b/src/main/kotlin/io/github/mojira/arisa/infrastructure/HelperMessages.kt @@ -1,6 +1,7 @@ package io.github.mojira.arisa.infrastructure import arrow.core.Either +import arrow.core.getOrElse import arrow.core.left import arrow.core.right import arrow.core.rightIfNotNull @@ -119,12 +120,19 @@ object HelperMessageService { } } + private const val BOT_SIGNATURE_KEY = "i-am-a-bot" fun getMessageWithBotSignature(project: String, key: String, filledText: String? = null, lang: String = "en") = - getMessage(project, listOf(key, "i-am-a-bot"), listOf(filledText), lang) + getMessage(project, listOf(key, BOT_SIGNATURE_KEY), listOf(filledText), lang) fun getMessageWithDupeBotSignature(project: String, key: String, filledText: String? = null, lang: String = "en") = getMessage(project, listOf(key, "i-am-a-bot-dupe"), listOf(filledText), lang) + fun getRawMessageWithBotSignature(rawMessage: String): String { + // Note: Project does not matter, message is (currently) the same for all projects + val botSignature = getSingleMessage("MC", BOT_SIGNATURE_KEY).getOrElse { "" } + return "$rawMessage\n$botSignature" + } + fun setHelperMessages(json: String) = data.fromJSON(json) ?: throw IOException("Couldn't deserialize helper messages from setHelperMessages()") diff --git a/src/main/kotlin/io/github/mojira/arisa/infrastructure/jira/Mappers.kt b/src/main/kotlin/io/github/mojira/arisa/infrastructure/jira/Mappers.kt index 11d16961c..ffd741d33 100644 --- a/src/main/kotlin/io/github/mojira/arisa/infrastructure/jira/Mappers.kt +++ b/src/main/kotlin/io/github/mojira/arisa/infrastructure/jira/Mappers.kt @@ -39,19 +39,20 @@ import net.rcarz.jiraclient.Project as JiraProject import net.rcarz.jiraclient.User as JiraUser import net.rcarz.jiraclient.Version as JiraVersion -fun JiraAttachment.toDomain(jiraClient: JiraClient, issue: JiraIssue) = Attachment( +fun JiraAttachment.toDomain(jiraClient: JiraClient, issue: JiraIssue, config: Config) = Attachment( id, fileName, getCreationDate(issue, id, issue.createdDate.toInstant()), mimeType, ::deleteAttachment.partially1(issue.getUpdateContext(jiraClient)).partially1(this), { openAttachmentStream(jiraClient, this) }, - this::download, - author?.toDomain(jiraClient) + // Cache attachment content once it has been downloaded + lazy { this.download() }::value, + author?.toDomain(jiraClient, config) ) fun getCreationDate(issue: JiraIssue, id: String, default: Instant) = issue.changeLog.entries - .filter { it.items.any { it.field == "Attachment" && it.to == id } } + .filter { changeLogEntry -> changeLogEntry.items.any { it.field == "Attachment" && it.to == id } } .maxByOrNull { it.created } ?.created ?.toInstant() ?: default @@ -93,7 +94,7 @@ fun JiraIssue.toDomain( description, getEnvironment(), security?.id, - reporter?.toDomain(jiraClient), + reporter?.toDomain(jiraClient, config), resolution?.name, createdDate.toInstant(), updatedDate.toInstant(), @@ -108,10 +109,10 @@ fun JiraIssue.toDomain( getDungeonsPlatform(config), mapVersions(), mapFixVersions(), - mapAttachments(jiraClient), - mapComments(jiraClient), + mapAttachments(jiraClient, config), + mapComments(jiraClient, config), mapLinks(jiraClient, config), - getChangeLogEntries(jiraClient), + getChangeLogEntries(jiraClient, config), ::reopen.partially1(context), ::resolveAs.partially1(context).partially1("Awaiting Response"), ::resolveAs.partially1(context).partially1("Invalid"), @@ -155,12 +156,19 @@ fun JiraIssue.toDomain( }, addNotEnglishComment = { language -> createComment( - context, HelperMessageService.getMessageWithBotSignature( + context, + HelperMessageService.getMessageWithBotSignature( project.key, config[Arisa.Modules.Language.message], lang = language ) ) }, addRawRestrictedComment = ::addRestrictedComment.partially1(context), + addRawBotComment = { rawMessage -> + createComment( + context, + HelperMessageService.getRawMessageWithBotSignature(rawMessage) + ) + }, ::markAsFixedWithSpecificVersion.partially1(context), ::changeReporter.partially1(context), addAttachmentFromFile, @@ -191,13 +199,14 @@ fun JiraProject.toDomain( fun JiraComment.toDomain( jiraClient: JiraClient, - issue: JiraIssue + issue: JiraIssue, + config: Config ): Comment { val context = issue.getUpdateContext(jiraClient) return Comment( id, body, - author.toDomain(jiraClient), + author.toDomain(jiraClient, config), { getGroups(jiraClient, author.name).fold({ null }, { it }) }, createdDate.toInstant(), updatedDate.toInstant(), @@ -209,11 +218,15 @@ fun JiraComment.toDomain( ) } -fun JiraUser.toDomain(jiraClient: JiraClient) = User( +fun JiraUser.toDomain(jiraClient: JiraClient, config: Config) = User( name, displayName, ::getUserGroups.partially1(jiraClient).partially1(name), ::isNewUser.partially1(jiraClient).partially1(name) -) +) { + // Check case insensitively because it apparently does not matter when logging in, so `username` might have + // incorrect capitalization + name.equals(config[Arisa.Credentials.username], ignoreCase = true) +} private fun getUserGroups(jiraClient: JiraClient, username: String) = getGroups( jiraClient, @@ -265,14 +278,21 @@ fun JiraIssueLink.toDomain( ::deleteLink.partially1(issue.getUpdateContext(jiraClient)).partially1(this) ) -fun JiraChangeLogItem.toDomain(jiraClient: JiraClient, entry: JiraChangeLogEntry) = ChangeLogItem( +fun JiraChangeLogItem.toDomain( + jiraClient: JiraClient, + entry: JiraChangeLogEntry, + itemIndex: Int, + config: Config +) = ChangeLogItem( + entry.id, + itemIndex, entry.created.toInstant(), field, from, fromString, to, toString, - entry.author.toDomain(jiraClient), + entry.author.toDomain(jiraClient, config), ::getUserGroups.partially1(jiraClient).partially1(entry.author.name) ) @@ -284,11 +304,11 @@ private fun JiraIssue.mapLinks( it.toDomain(jiraClient, this, config) } -private fun JiraIssue.mapComments(jiraClient: JiraClient) = - comments.map { it.toDomain(jiraClient, this) } +private fun JiraIssue.mapComments(jiraClient: JiraClient, config: Config) = + comments.map { it.toDomain(jiraClient, this, config) } -private fun JiraIssue.mapAttachments(jiraClient: JiraClient) = - attachments.map { it.toDomain(jiraClient, this) } +private fun JiraIssue.mapAttachments(jiraClient: JiraClient, config: Config) = + attachments.map { it.toDomain(jiraClient, this, config) } private fun JiraIssue.mapVersions() = versions.map { it.toDomain() } @@ -296,10 +316,10 @@ private fun JiraIssue.mapVersions() = private fun JiraIssue.mapFixVersions() = fixVersions.map { it.toDomain() } -private fun JiraIssue.getChangeLogEntries(jiraClient: JiraClient) = +private fun JiraIssue.getChangeLogEntries(jiraClient: JiraClient, config: Config) = changeLog.entries.flatMap { e -> - e.items.map { i -> - i.toDomain(jiraClient, e) + e.items.mapIndexed { index, item -> + item.toDomain(jiraClient, e, index, config) } } diff --git a/src/main/kotlin/io/github/mojira/arisa/modules/AttachmentModule.kt b/src/main/kotlin/io/github/mojira/arisa/modules/AttachmentModule.kt index d49d04cfe..bbb252b16 100644 --- a/src/main/kotlin/io/github/mojira/arisa/modules/AttachmentModule.kt +++ b/src/main/kotlin/io/github/mojira/arisa/modules/AttachmentModule.kt @@ -7,8 +7,11 @@ import io.github.mojira.arisa.domain.Attachment import io.github.mojira.arisa.domain.CommentOptions import io.github.mojira.arisa.domain.Issue import io.github.mojira.arisa.infrastructure.jira.sanitizeCommentArg +import org.slf4j.LoggerFactory import java.time.Instant +private val log = LoggerFactory.getLogger("AttachmentModule") + class AttachmentModule( private val extensionBlackList: List, private val attachmentRemovedMessage: String @@ -17,13 +20,16 @@ class AttachmentModule( override fun invoke(issue: Issue, lastRun: Instant): Either = with(issue) { Either.fx { val endsWithBlacklistedExtensionAdapter = ::endsWithBlacklistedExtensions.partially1(extensionBlackList) - val functions = attachments + val attachmentsToDelete = attachments .filter { endsWithBlacklistedExtensionAdapter(it.name) } - assertNotEmpty(functions).bind() - val commentInfo = functions.getCommentInfo() - functions - .map { it.remove } - .forEach { it.invoke() } + assertNotEmpty(attachmentsToDelete).bind() + val commentInfo = attachmentsToDelete.getCommentInfo() + + val attachmentsString = attachmentsToDelete.joinToString(separator = ", ", transform = Attachment::id) + log.info("Deleting attachments of issue $key because they have forbidden extensions: $attachmentsString") + attachmentsToDelete + .forEach { it.remove() } + addComment(CommentOptions(attachmentRemovedMessage)) addRawRestrictedComment("Removed attachments:\n$commentInfo", "helper") } diff --git a/src/main/kotlin/io/github/mojira/arisa/modules/Helpers.kt b/src/main/kotlin/io/github/mojira/arisa/modules/Helpers.kt index 25e491139..3ea0accbe 100644 --- a/src/main/kotlin/io/github/mojira/arisa/modules/Helpers.kt +++ b/src/main/kotlin/io/github/mojira/arisa/modules/Helpers.kt @@ -126,10 +126,7 @@ fun String?.getOrDefault(default: String) = this fun String?.getOrDefaultNull(default: String) = - if (this == null) - default - else - this + this ?: default fun MutableList.splitElemsByCommas() { val newList = this.flatMap { s -> diff --git a/src/main/kotlin/io/github/mojira/arisa/modules/KeepPlatformModule.kt b/src/main/kotlin/io/github/mojira/arisa/modules/KeepPlatformModule.kt index cb3e111e9..a8e727156 100644 --- a/src/main/kotlin/io/github/mojira/arisa/modules/KeepPlatformModule.kt +++ b/src/main/kotlin/io/github/mojira/arisa/modules/KeepPlatformModule.kt @@ -64,11 +64,7 @@ class KeepPlatformModule( val userChange = firstOrNull { it.created.isAfter(markedTime) } - if (userChange != null) { - userChange.changedFromString.getOrDefault("None") - } else { - null - } + userChange?.changedFromString?.getOrDefault("None") } } } diff --git a/src/main/kotlin/io/github/mojira/arisa/modules/PrivacyModule.kt b/src/main/kotlin/io/github/mojira/arisa/modules/PrivacyModule.kt deleted file mode 100644 index 82e36789a..000000000 --- a/src/main/kotlin/io/github/mojira/arisa/modules/PrivacyModule.kt +++ /dev/null @@ -1,90 +0,0 @@ -package io.github.mojira.arisa.modules - -import arrow.core.Either -import arrow.core.extensions.fx -import io.github.mojira.arisa.domain.Attachment -import io.github.mojira.arisa.domain.CommentOptions -import io.github.mojira.arisa.domain.Issue -import java.time.Instant - -private fun Iterable.anyMatches(string: String) = any { it.matches(string) } - -class PrivacyModule( - private val message: String, - private val commentNote: String, - private val allowedEmailRegexes: List, - private val sensitiveTextRegexes: List, - private val sensitiveFileNameRegexes: List -) : Module { - // Matches an email address, which is not part of a user mention ([~name]) - private val emailRegex = "(? = with(issue) { - Either.fx { - assertNull(securityLevel).bind() - - var string = "" - - if (created.isAfter(lastRun)) { - string += "$summary $environment $description " - } - - val newAttachments = attachments.filter { it.created.isAfter(lastRun) } - newAttachments - .asSequence() - .filter { it.mimeType.startsWith("text/") } - .forEach { string += "${String(it.getContent())} " } - - changeLog - .filter { it.created.isAfter(lastRun) } - .filter { it.field != "Attachment" } - .filter { it.changedFromString == null } - .forEach { string += "${it.changedToString} " } - - val doesStringMatchPatterns = string.containsMatch(sensitiveTextRegexes) - val doesEmailMatches = matchesEmail(string) - - val doesAttachmentNameMatch = newAttachments - .asSequence() - .map(Attachment::name) - .any(sensitiveFileNameRegexes::anyMatches) - - val restrictCommentFunctions = comments - .asSequence() - .filter { it.created.isAfter(lastRun) } - .filter { it.visibilityType == null } - .filter { it.body?.containsMatch(sensitiveTextRegexes) ?: false || matchesEmail(it.body ?: "") } - .filterNot { - it.getAuthorGroups()?.any { group -> - listOf("helper", "global-moderators", "staff").contains(group) - } ?: false - } - .map { { it.restrict("${it.body}$commentNote") } } - .toList() - - assertAny( - assertTrue(doesStringMatchPatterns), - assertTrue(doesEmailMatches), - assertTrue(doesAttachmentNameMatch), - assertNotEmpty(restrictCommentFunctions) - ).bind() - - if (doesStringMatchPatterns || doesEmailMatches || doesAttachmentNameMatch) { - setPrivate() - addComment(CommentOptions(message)) - } - - restrictCommentFunctions.forEach { it.invoke() } - } - } - - private fun matchesEmail(string: String): Boolean { - return emailRegex - .findAll(string) - .map(MatchResult::value) - .filterNot(allowedEmailRegexes::anyMatches) - .any() - } - - private fun String.containsMatch(patterns: List) = patterns.any { it.containsMatchIn(this) } -} diff --git a/src/main/kotlin/io/github/mojira/arisa/modules/ReopenAwaitingModule.kt b/src/main/kotlin/io/github/mojira/arisa/modules/ReopenAwaitingModule.kt index e61b2266f..412d54dca 100644 --- a/src/main/kotlin/io/github/mojira/arisa/modules/ReopenAwaitingModule.kt +++ b/src/main/kotlin/io/github/mojira/arisa/modules/ReopenAwaitingModule.kt @@ -17,8 +17,8 @@ import java.time.temporal.ChronoUnit const val TWO_SECONDS_IN_MILLIS = 2000 class ReopenAwaitingModule( - private val blacklistedRoles: List, - private val blacklistedVisibilities: List, + private val blacklistedRoles: Set, + private val blacklistedVisibilities: Set, private val softArPeriod: Long, private val keepARTag: String, private val onlyOPTag: String, diff --git a/src/main/kotlin/io/github/mojira/arisa/modules/ReplaceTextModule.kt b/src/main/kotlin/io/github/mojira/arisa/modules/ReplaceTextModule.kt index 7a1525d3d..df587dfe1 100644 --- a/src/main/kotlin/io/github/mojira/arisa/modules/ReplaceTextModule.kt +++ b/src/main/kotlin/io/github/mojira/arisa/modules/ReplaceTextModule.kt @@ -2,7 +2,6 @@ package io.github.mojira.arisa.modules import arrow.core.Either import arrow.core.extensions.fx -import io.github.mojira.arisa.domain.Comment import io.github.mojira.arisa.domain.Issue import java.time.Instant @@ -15,13 +14,6 @@ class ReplaceTextModule( "(http://i.imgur.com)".toRegex() to "https://i.imgur.com" ) ) : Module { - data class Request( - val lastRun: Instant, - val description: String?, - val comments: List, - val updateDescription: (description: String) -> Either - ) - override fun invoke(issue: Issue, lastRun: Instant): Either = with(issue) { Either.fx { val needUpdateDescription = created.isAfter(lastRun) && @@ -52,7 +44,6 @@ class ReplaceTextModule( private fun needReplacement(text: String?) = replacements.any { (regex, _) -> text?.contains(regex) ?: false } private fun replace(text: String): String = replacements.fold( - text, - { str, (regex, replacement) -> str.replace(regex, replacement) } - ) + text + ) { str, (regex, replacement) -> str.replace(regex, replacement) } } diff --git a/src/main/kotlin/io/github/mojira/arisa/modules/commands/DeobfuscateCommand.kt b/src/main/kotlin/io/github/mojira/arisa/modules/commands/DeobfuscateCommand.kt index 4d5de66c3..9c7d11064 100644 --- a/src/main/kotlin/io/github/mojira/arisa/modules/commands/DeobfuscateCommand.kt +++ b/src/main/kotlin/io/github/mojira/arisa/modules/commands/DeobfuscateCommand.kt @@ -33,8 +33,7 @@ class DeobfuscateCommand(private val attachmentUtils: AttachmentUtils) { throw CommandExceptions.ATTACHMENT_ALREADY_EXISTS.create(deobfuscatedName) } - @Suppress("VariableNaming") - var minecraftVersionId_ = minecraftVersionId + var resolvedMinecraftVersionId = minecraftVersionId var isClientCrash = when (crashReportType) { CrashReportType.CLIENT -> true CrashReportType.SERVER -> false @@ -42,13 +41,13 @@ class DeobfuscateCommand(private val attachmentUtils: AttachmentUtils) { } // If version or crash report type are not specified try to obtain them from crash report - if (minecraftVersionId_ == null || isClientCrash == null) { + if (resolvedMinecraftVersionId == null || isClientCrash == null) { val parsedCrashReport = attachmentUtils.processCrash(attachmentUtils.fetchAttachment(attachment)) ?.crash as? Crash.Minecraft ?: throw MISSING_DEOBFUSCATION_ARGUMENTS.create() - if (minecraftVersionId_ == null) { - minecraftVersionId_ = parsedCrashReport.minecraftVersion + if (resolvedMinecraftVersionId == null) { + resolvedMinecraftVersionId = parsedCrashReport.minecraftVersion ?: throw MISSING_DEOBFUSCATION_ARGUMENTS.create() } if (isClientCrash == null) { @@ -59,7 +58,7 @@ class DeobfuscateCommand(private val attachmentUtils: AttachmentUtils) { val deobfuscated = try { attachmentUtils.deobfuscate( String(attachment.getContent()), - minecraftVersionId_, + resolvedMinecraftVersionId, isClientCrash ) } catch (@Suppress("TooGenericExceptionCaught") e: Exception) { diff --git a/src/main/kotlin/io/github/mojira/arisa/modules/privacy/AttachmentRedactor.kt b/src/main/kotlin/io/github/mojira/arisa/modules/privacy/AttachmentRedactor.kt new file mode 100644 index 000000000..3cbe7d5ed --- /dev/null +++ b/src/main/kotlin/io/github/mojira/arisa/modules/privacy/AttachmentRedactor.kt @@ -0,0 +1,48 @@ +package io.github.mojira.arisa.modules.privacy + +import io.github.mojira.arisa.domain.Attachment + +private const val REDACTED_REPLACEMENT = "###REDACTED###" + +/** + * Important: Redactor implementations should be as specific as possible. They irrecoverably remove information + * from attachments which can only be recovered when the uploader provides the attachment again (in case they + * still have the original). Redactors should therefore only remove sensitive data when it is definitely not + * needed, such as access tokens in JVM crash reports. A redactor should for example not blindly remove (what it + * assumes to be) e-mail addresses from attachments without actually knowing what kind of attachment it is processing. + */ +interface AttachmentRedactor { + /** + * Redacts sensitive data from the attachment content. + * + * @return + * information about the redacted attachment, or `null` + * if nothing was redacted + */ + fun redact(attachment: Attachment): RedactedAttachment? +} + +/** Redacts access tokens passed as command line argument, as found in JVM crash reports. */ +object AccessTokenRedactor : AttachmentRedactor { + // Use lookbehind to only redact the token itself + private val pattern = Regex("""(?<=(^|\s)--accessToken )[a-zA-Z0-9.+/=\-_]+(?=(\s|$))""") + + override fun redact(attachment: Attachment): RedactedAttachment? { + if (attachment.hasTextContent()) { + val original = attachment.getTextContent() + val redacted = original.replace(pattern, REDACTED_REPLACEMENT) + if (redacted != original) { + return RedactedAttachment(attachment, redacted) + } + } + + return null + } +} + +data class RedactedAttachment( + /** The original attachment containing sensitive data */ + val attachment: Attachment, + /** Attachment content with sensitive data redacted */ + val redactedContent: String +) diff --git a/src/main/kotlin/io/github/mojira/arisa/modules/privacy/PrivacyModule.kt b/src/main/kotlin/io/github/mojira/arisa/modules/privacy/PrivacyModule.kt new file mode 100644 index 000000000..68fd7019a --- /dev/null +++ b/src/main/kotlin/io/github/mojira/arisa/modules/privacy/PrivacyModule.kt @@ -0,0 +1,246 @@ +package io.github.mojira.arisa.modules.privacy + +import arrow.core.Either +import arrow.core.extensions.fx +import com.urielsalis.mccrashlib.deobfuscator.getSafeChildPath +import io.github.mojira.arisa.domain.Attachment +import io.github.mojira.arisa.domain.CommentOptions +import io.github.mojira.arisa.domain.Issue +import io.github.mojira.arisa.infrastructure.jira.sanitizeCommentArg +import io.github.mojira.arisa.modules.Module +import io.github.mojira.arisa.modules.ModuleError +import io.github.mojira.arisa.modules.ModuleResponse +import io.github.mojira.arisa.modules.assertAny +import io.github.mojira.arisa.modules.assertNotEmpty +import io.github.mojira.arisa.modules.assertNull +import io.github.mojira.arisa.modules.assertTrue +import org.slf4j.LoggerFactory +import java.nio.file.Files +import java.time.Instant + +private fun Iterable.anyMatches(string: String) = any { it.matches(string) } + +private val log = LoggerFactory.getLogger("PrivacyModule") + +class PrivacyModule( + private val message: String, + private val commentNote: String, + private val allowedEmailRegexes: List, + private val sensitiveTextRegexes: List, + private val attachmentRedactor: AttachmentRedactor, + private val sensitiveFileNameRegexes: List +) : Module { + // Matches an email address, which is not part of a user mention ([~name]) + private val emailRegex = "(? = with(issue) { + Either.fx { + assertNull(securityLevel).bind() + + val (foundNonRedactableSensitiveData, attachmentsToRedact) = checkAttachments(lastRun) + val issueContainsSensitiveData = foundNonRedactableSensitiveData || containsIssueSensitiveData(lastRun) + + val restrictCommentActions = getRestrictCommentActions(lastRun) + + assertAny( + assertNotEmpty(attachmentsToRedact), + assertTrue(issueContainsSensitiveData), + assertNotEmpty(restrictCommentActions) + ).bind() + + // Always try to redact attachments, even if issue would be made private anyways + // So in case issue was made private erroneously, it can easily be made public manually + val redactingSucceeded = redactAttachments(issue, attachmentsToRedact) + + if (!redactingSucceeded || issueContainsSensitiveData) { + setPrivate() + addComment(CommentOptions(message)) + } + + restrictCommentActions.forEach { it.invoke() } + } + } + + private fun containsSensitiveData(string: String): TextRangeLocation? = + containsEmailMatch(string) + ?: sensitiveTextRegexes.asSequence() + .mapNotNull { it.find(string) } + .map { TextRangeLocation.fromMatchResult(string, it) } + .firstOrNull() + + private fun containsEmailMatch(string: String): TextRangeLocation? { + return emailRegex + .findAll(string) + .filterNot { match -> allowedEmailRegexes.anyMatches(match.value) } + .map { TextRangeLocation.fromMatchResult(string, it) } + .firstOrNull() + } + + private data class AttachmentsCheckResult( + /** Whether any of the attachments contains sensitive data which cannot be redacted */ + val foundNonRedactableSensitiveData: Boolean, + val attachmentsToRedact: List + ) + + private fun Issue.checkAttachments(lastRun: Instant): AttachmentsCheckResult { + var foundNonRedactableSensitiveData: Boolean + val newAttachments = attachments.filter { it.created.isAfter(lastRun) } + + foundNonRedactableSensitiveData = newAttachments + .map(Attachment::name) + .any(sensitiveFileNameRegexes::anyMatches) + + val attachmentsToRedact = newAttachments + .filter { it.hasTextContent() } + .mapNotNull { + // Don't redact bot attachments to guard against infinite loop + // But still check bot attachments for sensitive data, e.g. when deobfuscated crash report + // contains sensitive data + val redacted = if (it.uploader?.isBotUser?.invoke() == true) null else attachmentRedactor.redact(it) + if (redacted == null) { + // No redaction necessary / possible; check if attachment contains sensitive data + if (!foundNonRedactableSensitiveData) { + containsSensitiveData(it.getTextContent())?.let { matchResult -> + foundNonRedactableSensitiveData = true + logFoundSensitiveData("in attachment with ID ${it.id}", matchResult) + } + } + return@mapNotNull null + } + // Check if attachment content still contains sensitive data after redacting + else if (containsSensitiveData(redacted.redactedContent) != null) { + if (!foundNonRedactableSensitiveData) { + // Because attachment won't be redacted get match result in original attachment for logging + containsSensitiveData(it.getTextContent())?.let { matchResult -> + logFoundSensitiveData("in attachment with ID ${it.id}", matchResult) + } ?: run { + // Redactor might have produced malformed output, or sensitive data regex pattern + // is imprecise; marking attachment as containing sensitive data nonetheless + log.warn("$key: Sensitive data was detected in redacted content for attachment with " + + "ID ${it.id}, but original attachment content was not detect") + } + } + + foundNonRedactableSensitiveData = true + // Don't redact if attachment content would still contain sensitive data + return@mapNotNull null + } else { + return@mapNotNull redacted + } + } + + return AttachmentsCheckResult(foundNonRedactableSensitiveData, attachmentsToRedact) + } + + @Suppress("ReturnCount") + private fun Issue.containsIssueSensitiveData(lastRun: Instant): Boolean { + if (created.isAfter(lastRun)) { + summary?.let(::containsSensitiveData)?.let { + logFoundSensitiveData("in summary", it) + return true + } + environment?.let(::containsSensitiveData)?.let { + logFoundSensitiveData("in environment", it) + return true + } + description?.let(::containsSensitiveData)?.let { + logFoundSensitiveData("in description", it) + return true + } + } + + return changeLog + .asSequence() + .filter { it.created.isAfter(lastRun) } + .filter { it.field != "Attachment" } + .filter { it.changedFromString == null } + .any { + it.changedToString?.let(::containsSensitiveData)?.let { matchResult -> + logFoundSensitiveData("in change log item ${it.entryId}[${it.itemIndex}]", matchResult) + return true + } + return false + } + } + + private fun Issue.getRestrictCommentActions(lastRun: Instant) = comments + .asSequence() + .filter { it.created.isAfter(lastRun) } + .filter { it.visibilityType == null } + .filterNot { + it.getAuthorGroups()?.any { group -> + listOf("helper", "global-moderators", "staff").contains(group) + } ?: false + } + .filter { + it.body?.let(::containsSensitiveData)?.let { matchResult -> + logFoundSensitiveData("in comment with ID ${it.id}", matchResult) + return@filter true + } + return@filter false + } + .map { { it.restrict("${it.body}$commentNote") } } + .toList() + + private fun Issue.logFoundSensitiveData(location: String, matchResult: TextRangeLocation) { + // Important: Don't log value (i.e. sensitive data) of match result + log.info("$key: Found sensitive data $location at ${matchResult.getLocationDescription()}") + } + + private fun Issue.hasAnyAttachmentName(name: String) = attachments.any { it.name == name } + + /** + * @return true if all provided attachments have been redacted; false if at least one attachment + * still contains sensitive data + */ + private fun redactAttachments(issue: Issue, attachments: Collection): Boolean { + var redactedAll = true + attachments + // Group by uploader in case they uploaded multiple attachments at once + .groupBy { it.attachment.uploader?.name!! } + .forEach { (uploader, userAttachments) -> + val fileNames = mutableSetOf() + userAttachments.forEach { + val attachment = it.attachment + val tempDir = Files.createTempDirectory("arisa-redaction-upload").toFile() + val fileName = "redacted_${attachment.name}" + val filePath = getSafeChildPath(tempDir, fileName) + + if (filePath == null || issue.hasAnyAttachmentName(fileName) || !fileNames.add(fileName)) { + redactedAll = false + // Note: Don't log file name to avoid log injection + log.warn("Cannot redact attachment with ID ${attachment.id} of issue ${issue.key}; file name " + + "is malformed or would clash with other attachment") + tempDir.delete() + } else { + log.info("Redacting attachment with ID ${attachment.id} of issue ${issue.key} because it " + + "contains sensitive data") + filePath.writeText(it.redactedContent) + issue.addAttachmentFromFile(filePath) { + // Once uploaded, delete the temp directory containing the attachment + tempDir.deleteRecursively() + } + // Remove the original attachment + attachment.remove() + } + } + + if (fileNames.isNotEmpty()) { + // Use postfix line break to prevent bot signature from appearing as part of last list item + val fileNamesString = fileNames.joinToString(separator = "", postfix = "\n") { + // Use link for attachments + "\n- [^${sanitizeCommentArg(it)}]" + } + val sanitizedUploaderName = sanitizeCommentArg(uploader) + // Does not use helper message because message will only be used by bot and helper messages + // currently only support one placeholder + issue.addRawBotComment( + "@[~$sanitizedUploaderName], sensitive data has been removed from your " + + "attachment(s) and they have been re-uploaded as:$fileNamesString" + ) + } + } + + return redactedAll + } +} diff --git a/src/main/kotlin/io/github/mojira/arisa/modules/privacy/TextRangeLocation.kt b/src/main/kotlin/io/github/mojira/arisa/modules/privacy/TextRangeLocation.kt new file mode 100644 index 000000000..63bd072d1 --- /dev/null +++ b/src/main/kotlin/io/github/mojira/arisa/modules/privacy/TextRangeLocation.kt @@ -0,0 +1,84 @@ +package io.github.mojira.arisa.modules.privacy + +data class TextRangeLocation( + private val completeText: String, + /** Global start index, beginning at 0 */ + private val startIndex: Int, + /** Global end index (inclusive), beginning at 0 */ + private val endIndex: Int +) { + companion object { + fun fromMatchResult(completeText: String, matchResult: MatchResult): TextRangeLocation { + val range = matchResult.range + return TextRangeLocation(completeText, range.first, range.last) + } + } + + private fun getIndexBehindNextLineTerminator(string: String, startIndex: Int): Int? { + val nextLfIndex = string.indexOf('\n', startIndex) + val nextCrIndex = string.indexOf('\r', startIndex) + + return ( + if (nextLfIndex != -1 && (nextCrIndex == -1 || nextLfIndex < nextCrIndex)) { + // LF + nextLfIndex + 1 + } else if (nextCrIndex != -1) { + if (nextLfIndex == nextCrIndex + 1) { + // CR LF + nextLfIndex + 1 + } else { + // CR + nextCrIndex + 1 + } + } else { + null + } + ) + } + + fun getLocationDescription(): String { + var currentLineStartIndex = 0 + // Start line numbering at 1 + var lineNumber = 1 + + var startLine: Int? = null + var relativeStartIndex = 0 + var endLine: Int? = null + var relativeEndIndex = 0 + + @Suppress("LoopWithTooManyJumpStatements") + while (currentLineStartIndex < completeText.length) { + val previousLineStartIndex = currentLineStartIndex + currentLineStartIndex = getIndexBehindNextLineTerminator(completeText, previousLineStartIndex) + ?: break // reached end of last line + + if (startIndex in previousLineStartIndex until currentLineStartIndex) { + startLine = lineNumber + relativeStartIndex = startIndex - previousLineStartIndex + } + if (endIndex in previousLineStartIndex until currentLineStartIndex) { + endLine = lineNumber + relativeEndIndex = endIndex - previousLineStartIndex + } + + // Found both line numbers, can stop iteration + if (startLine != null && endLine != null) { + break + } + + lineNumber++ + } + + // If startLine or endLine are still null they are in the last line + if (startLine == null) { + startLine = lineNumber + relativeStartIndex = startIndex - currentLineStartIndex + } + if (endLine == null) { + endLine = lineNumber + relativeEndIndex = endIndex - currentLineStartIndex + } + + return "$startIndex - $endIndex ($startLine:$relativeStartIndex - $endLine:$relativeEndIndex)" + } +} diff --git a/src/main/kotlin/io/github/mojira/arisa/registry/InstantModuleRegistry.kt b/src/main/kotlin/io/github/mojira/arisa/registry/InstantModuleRegistry.kt index 63466a227..c4628a049 100644 --- a/src/main/kotlin/io/github/mojira/arisa/registry/InstantModuleRegistry.kt +++ b/src/main/kotlin/io/github/mojira/arisa/registry/InstantModuleRegistry.kt @@ -6,6 +6,7 @@ import io.github.mojira.arisa.ExecutionTimeframe import io.github.mojira.arisa.infrastructure.AttachmentUtils import io.github.mojira.arisa.infrastructure.LanguageDetectionApi import io.github.mojira.arisa.infrastructure.config.Arisa +import io.github.mojira.arisa.modules.privacy.AccessTokenRedactor import io.github.mojira.arisa.modules.AffectedVersionMessageModule import io.github.mojira.arisa.modules.AttachmentModule import io.github.mojira.arisa.modules.CHKModule @@ -20,7 +21,7 @@ import io.github.mojira.arisa.modules.KeepPrivateModule import io.github.mojira.arisa.modules.LanguageModule import io.github.mojira.arisa.modules.MultiplePlatformsModule import io.github.mojira.arisa.modules.PiracyModule -import io.github.mojira.arisa.modules.PrivacyModule +import io.github.mojira.arisa.modules.privacy.PrivacyModule import io.github.mojira.arisa.modules.PrivateDuplicateModule import io.github.mojira.arisa.modules.RemoveIdenticalLinkModule import io.github.mojira.arisa.modules.RemoveNonStaffTagsModule @@ -92,8 +93,7 @@ class InstantModuleRegistry(config: Config) : ModuleRegistry(config) { val attachmentUtils = AttachmentUtils( config[Arisa.Modules.Crash.crashExtensions], - CrashReader(), - config[Arisa.Credentials.username] + CrashReader() ) register( Arisa.Modules.Crash, @@ -159,6 +159,7 @@ class InstantModuleRegistry(config: Config) : ModuleRegistry(config) { config[Arisa.Modules.Privacy.commentNote], config[Arisa.Modules.Privacy.allowedEmailRegexes].map(String::toRegex), config[Arisa.Modules.Privacy.sensitiveTextRegexes].map(String::toRegex), + AccessTokenRedactor, config[Arisa.Modules.Privacy.sensitiveFileNameRegexes].map(String::toRegex) ) ) @@ -196,8 +197,8 @@ class InstantModuleRegistry(config: Config) : ModuleRegistry(config) { register( Arisa.Modules.ReopenAwaiting, ReopenAwaitingModule( - config[Arisa.Modules.ReopenAwaiting.blacklistedRoles], - config[Arisa.Modules.ReopenAwaiting.blacklistedVisibilities], + config[Arisa.Modules.ReopenAwaiting.blacklistedRoles].toSetNoDuplicates(), + config[Arisa.Modules.ReopenAwaiting.blacklistedVisibilities].toSetNoDuplicates(), config[Arisa.Modules.ReopenAwaiting.softARDays], config[Arisa.Modules.ReopenAwaiting.keepARTag], config[Arisa.Modules.ReopenAwaiting.onlyOPTag], @@ -259,4 +260,20 @@ class InstantModuleRegistry(config: Config) : ModuleRegistry(config) { ShadowbanModule() ) } + + private fun List.toSetNoDuplicates(): Set { + val result = toMutableSet() + if (result.size != size) { + val duplicates = mutableSetOf() + for (element in this) { + // If removal fails it is a duplicate element because it has already been removed + if (!result.remove(element)) { + duplicates.add(element) + } + } + throw IllegalArgumentException("Contains these duplicate elements: $duplicates") + } + + return result + } } diff --git a/src/test/kotlin/io/github/mojira/arisa/infrastructure/HelperMessagesTest.kt b/src/test/kotlin/io/github/mojira/arisa/infrastructure/HelperMessagesTest.kt index cfe0bbde5..cf07aba53 100644 --- a/src/test/kotlin/io/github/mojira/arisa/infrastructure/HelperMessagesTest.kt +++ b/src/test/kotlin/io/github/mojira/arisa/infrastructure/HelperMessagesTest.kt @@ -246,4 +246,10 @@ class HelperMessagesTest : StringSpec({ result shouldBe "Normal message\n~{color:#888}-- I am a bot.{color}~" } + + "should append the bot signature to raw message correctly" { + val result = HelperMessageService.getRawMessageWithBotSignature("some message") + + result shouldBe "some message\n~{color:#888}-- I am a bot.{color}~" + } }) diff --git a/src/test/kotlin/io/github/mojira/arisa/modules/CommandModuleTest.kt b/src/test/kotlin/io/github/mojira/arisa/modules/CommandModuleTest.kt index 7d8faad12..a1a57e52c 100644 --- a/src/test/kotlin/io/github/mojira/arisa/modules/CommandModuleTest.kt +++ b/src/test/kotlin/io/github/mojira/arisa/modules/CommandModuleTest.kt @@ -25,7 +25,7 @@ private const val PREFIX = "ARISA" private const val BOT_USER_NAME = "botName" class CommandModuleTest : StringSpec({ - val attachmentUtils = AttachmentUtils(emptyList(), CrashReader(), BOT_USER_NAME) + val attachmentUtils = AttachmentUtils(emptyList(), CrashReader()) val module = CommandModule(PREFIX, BOT_USER_NAME, true, attachmentUtils, ::getDispatcher) "should return OperationNotNeededModuleResponse when no comments" { diff --git a/src/test/kotlin/io/github/mojira/arisa/modules/CrashModuleTest.kt b/src/test/kotlin/io/github/mojira/arisa/modules/CrashModuleTest.kt index 20674c2c5..d72ca323d 100644 --- a/src/test/kotlin/io/github/mojira/arisa/modules/CrashModuleTest.kt +++ b/src/test/kotlin/io/github/mojira/arisa/modules/CrashModuleTest.kt @@ -273,11 +273,9 @@ private val A_MINUTE_AGO = NOW.minusSeconds(60) private const val UNCONFIRMED = "Unconfirmed" private val NO_PRIORITY = null -private const val BOT_USER_NAME = "testBot" - class CrashModuleTest : StringSpec({ val crashReader = CrashReader() - val attachmentUtils = AttachmentUtils(listOf("txt"), crashReader, BOT_USER_NAME) + val attachmentUtils = AttachmentUtils(listOf("txt"), crashReader) "should return OperationNotNeededModuleResponse when issue does not contain any valid crash report" { val module = CrashModule( @@ -1245,7 +1243,7 @@ class CrashModuleTest : StringSpec({ val issue = mockIssue( attachments = listOf( getAttachment(OBFUSCATED_CRASH, "obfuscated.txt", A_MINUTE_AGO), - getAttachment(OBFUSCATED_CRASH, "deobf_obfuscated.txt", A_SECOND_AGO, uploader = BOT_USER_NAME) + getAttachment(OBFUSCATED_CRASH, "deobf_obfuscated.txt", A_SECOND_AGO, isUploadedByBot = true) ), description = "hello please verify my crash thanks", created = A_MINUTE_AGO, @@ -1274,11 +1272,15 @@ private fun getAttachment( name: String = "crash.txt", created: Instant = NOW, remove: () -> Unit = { }, - uploader: String = "someRandomUser" + uploader: String = "someRandomUser", + isUploadedByBot: Boolean = false ) = mockAttachment( name = name, created = created, remove = remove, getContent = { content.toByteArray() }, - uploader = mockUser(uploader) + uploader = mockUser( + name = uploader, + isBotUser = { isUploadedByBot } + ) ) diff --git a/src/test/kotlin/io/github/mojira/arisa/modules/ReopenAwaitingModuleTest.kt b/src/test/kotlin/io/github/mojira/arisa/modules/ReopenAwaitingModuleTest.kt index af0d41de7..c28fcf423 100644 --- a/src/test/kotlin/io/github/mojira/arisa/modules/ReopenAwaitingModuleTest.kt +++ b/src/test/kotlin/io/github/mojira/arisa/modules/ReopenAwaitingModuleTest.kt @@ -28,8 +28,8 @@ private val TEN_SECONDS_AGO = RIGHT_NOW.minusSeconds(10) private val TWO_YEARS_AGO = RIGHT_NOW.minus(730, ChronoUnit.DAYS) private val MODULE = ReopenAwaitingModule( - listOf("staff", "global-moderators"), - listOf("helper", "staff", "global-moderators"), + setOf("staff", "global-moderators"), + setOf("helper", "staff", "global-moderators"), 365, "MEQS_KEEP_AR", "ARISA_REOPEN_OP", diff --git a/src/test/kotlin/io/github/mojira/arisa/modules/RevokeConfirmationModuleTest.kt b/src/test/kotlin/io/github/mojira/arisa/modules/RevokeConfirmationModuleTest.kt index 8797b17a7..76bfc079f 100644 --- a/src/test/kotlin/io/github/mojira/arisa/modules/RevokeConfirmationModuleTest.kt +++ b/src/test/kotlin/io/github/mojira/arisa/modules/RevokeConfirmationModuleTest.kt @@ -12,6 +12,18 @@ import java.time.Instant import java.time.temporal.ChronoUnit class RevokeConfirmationModuleTest : StringSpec({ + fun mockChangeLogItem( + created: Instant = RIGHT_NOW, + field: String = "Confirmation Status", + value: String = "Confirmed", + getAuthorGroups: () -> List? = { emptyList() } + ) = mockChangeLogItem( + created = created, + field = field, + changedToString = value, + getAuthorGroups = getAuthorGroups + ) + "should return OperationNotNeededModuleResponse when Ticket is unconfirmed and confirmation was never changed" { val module = RevokeConfirmationModule() val issue = mockIssue( @@ -25,7 +37,7 @@ class RevokeConfirmationModuleTest : StringSpec({ "should return OperationNotNeededModuleResponse when Ticket is confirmed and was changed by staff" { val module = RevokeConfirmationModule() - val changeLogItem = io.github.mojira.arisa.modules.mockChangeLogItem { listOf("staff") } + val changeLogItem = mockChangeLogItem { listOf("staff") } val issue = mockIssue( confirmationStatus = "Confirmed", changeLog = listOf(changeLogItem) @@ -38,7 +50,7 @@ class RevokeConfirmationModuleTest : StringSpec({ "should return OperationNotNeededModuleResponse when Ticket is confirmed and was changed by helper" { val module = RevokeConfirmationModule() - val changeLogItem = io.github.mojira.arisa.modules.mockChangeLogItem { listOf("helper") } + val changeLogItem = mockChangeLogItem { listOf("helper") } val issue = mockIssue( confirmationStatus = "Confirmed", changeLog = listOf(changeLogItem) @@ -51,7 +63,7 @@ class RevokeConfirmationModuleTest : StringSpec({ "should return OperationNotNeededModuleResponse when Ticket is confirmed and was changed by global-moderator" { val module = RevokeConfirmationModule() - val changeLogItem = io.github.mojira.arisa.modules.mockChangeLogItem { listOf("global-moderators") } + val changeLogItem = mockChangeLogItem { listOf("global-moderators") } val issue = mockIssue( confirmationStatus = "Confirmed", changeLog = listOf(changeLogItem) @@ -64,7 +76,7 @@ class RevokeConfirmationModuleTest : StringSpec({ "should return OperationNotNeededModuleResponse when Ticket was confirmed more than a day ago by a user who is no longer staff" { val module = RevokeConfirmationModule() - val changeLogItem = io.github.mojira.arisa.modules.mockChangeLogItem(Instant.now().minus(2, ChronoUnit.DAYS)) + val changeLogItem = mockChangeLogItem(Instant.now().minus(2, ChronoUnit.DAYS)) val issue = mockIssue( confirmationStatus = "Confirmed", changeLog = listOf(changeLogItem) @@ -77,7 +89,7 @@ class RevokeConfirmationModuleTest : StringSpec({ "should return OperationNotNeededModuleResponse when Ticket is confirmed and groups are unknown" { val module = RevokeConfirmationModule() - val changeLogItem = io.github.mojira.arisa.modules.mockChangeLogItem { null } + val changeLogItem = mockChangeLogItem { null } val issue = mockIssue( confirmationStatus = "Confirmed", changeLog = listOf(changeLogItem) @@ -90,7 +102,7 @@ class RevokeConfirmationModuleTest : StringSpec({ "should return OperationNotNeededModuleResponse when multiple volunteers changed the confirmation status" { val module = RevokeConfirmationModule() - val volunteerChange = io.github.mojira.arisa.modules.mockChangeLogItem { listOf("staff") } + val volunteerChange = mockChangeLogItem { listOf("staff") } val otherVolunteerChange = mockChangeLogItem(value = "Unconfirmed") { listOf("helper") } val issue = mockIssue( confirmationStatus = "Unconfirmed", @@ -106,7 +118,7 @@ class RevokeConfirmationModuleTest : StringSpec({ var changedConfirmation = "Unconfirmed" val module = RevokeConfirmationModule() - val volunteerChange = io.github.mojira.arisa.modules.mockChangeLogItem { listOf("staff") } + val volunteerChange = mockChangeLogItem { listOf("staff") } val otherVolunteerChange = mockChangeLogItem(value = "") { listOf("helper") } val issue = mockIssue( confirmationStatus = "Unconfirmed", @@ -122,7 +134,7 @@ class RevokeConfirmationModuleTest : StringSpec({ "should return OperationNotNeededModuleResponse when confirmation status is empty and was unset" { val module = RevokeConfirmationModule() - val volunteerChange = io.github.mojira.arisa.modules.mockChangeLogItem { listOf("staff") } + val volunteerChange = mockChangeLogItem { listOf("staff") } val otherVolunteerChange = mockChangeLogItem(value = "") { listOf("helper") } val issue = mockIssue( confirmationStatus = "", @@ -154,7 +166,7 @@ class RevokeConfirmationModuleTest : StringSpec({ val module = RevokeConfirmationModule() val changeLogItem = - io.github.mojira.arisa.modules.mockChangeLogItem(field = "Totally Not Confirmation Status") { listOf("staff") } + mockChangeLogItem(field = "Totally Not Confirmation Status") { listOf("staff") } val issue = mockIssue( confirmationStatus = "Confirmed", changeLog = listOf(changeLogItem), @@ -171,7 +183,7 @@ class RevokeConfirmationModuleTest : StringSpec({ var changedConfirmation = "" val module = RevokeConfirmationModule() - val changeLogItem = io.github.mojira.arisa.modules.mockChangeLogItem() + val changeLogItem = mockChangeLogItem() val issue = mockIssue( confirmationStatus = "Confirmed", changeLog = listOf(changeLogItem), @@ -205,7 +217,7 @@ class RevokeConfirmationModuleTest : StringSpec({ var changedConfirmation = "" val module = RevokeConfirmationModule() - val volunteerChange = io.github.mojira.arisa.modules.mockChangeLogItem { listOf("staff") } + val volunteerChange = mockChangeLogItem { listOf("staff") } val userChange = mockChangeLogItem(value = "Unconfirmed") { listOf("users") } val issue = mockIssue( confirmationStatus = "Unconfirmed", @@ -223,7 +235,7 @@ class RevokeConfirmationModuleTest : StringSpec({ var changedConfirmation = "" val module = RevokeConfirmationModule() - val volunteerChange = io.github.mojira.arisa.modules.mockChangeLogItem(Instant.now().minus(2, ChronoUnit.DAYS)) + val volunteerChange = mockChangeLogItem(Instant.now().minus(2, ChronoUnit.DAYS)) val userChange = mockChangeLogItem(value = "Unconfirmed") { listOf("users") } val issue = mockIssue( confirmationStatus = "Unconfirmed", @@ -237,15 +249,3 @@ class RevokeConfirmationModuleTest : StringSpec({ changedConfirmation.shouldBe("Confirmed") } }) - -private fun mockChangeLogItem( - created: Instant = RIGHT_NOW, - field: String = "Confirmation Status", - value: String = "Confirmed", - getAuthorGroups: () -> List? = { emptyList() } -) = mockChangeLogItem( - created = created, - field = field, - changedToString = value, - getAuthorGroups = getAuthorGroups -) diff --git a/src/test/kotlin/io/github/mojira/arisa/modules/commands/DeobfuscateCommandTest.kt b/src/test/kotlin/io/github/mojira/arisa/modules/commands/DeobfuscateCommandTest.kt index 797b443ff..a45c22798 100644 --- a/src/test/kotlin/io/github/mojira/arisa/modules/commands/DeobfuscateCommandTest.kt +++ b/src/test/kotlin/io/github/mojira/arisa/modules/commands/DeobfuscateCommandTest.kt @@ -134,7 +134,7 @@ Details: """.trim() class DeobfuscateCommandTest : StringSpec({ - val command = DeobfuscateCommand(AttachmentUtils(emptyList(), CrashReader(), "bot")) + val command = DeobfuscateCommand(AttachmentUtils(emptyList(), CrashReader())) "should throw for unknown attachment ID" { val issue = mockIssue() diff --git a/src/test/kotlin/io/github/mojira/arisa/modules/commands/arguments/EnumArgumentTypeTest.kt b/src/test/kotlin/io/github/mojira/arisa/modules/commands/arguments/EnumArgumentTypeTest.kt index 2782e1012..06d711702 100644 --- a/src/test/kotlin/io/github/mojira/arisa/modules/commands/arguments/EnumArgumentTypeTest.kt +++ b/src/test/kotlin/io/github/mojira/arisa/modules/commands/arguments/EnumArgumentTypeTest.kt @@ -16,7 +16,9 @@ private enum class MyEnum { WORLD } +@Suppress("unused") private enum class EnumWithNameClashes { + @Suppress("EnumEntryName") a, A } diff --git a/src/test/kotlin/io/github/mojira/arisa/modules/privacy/AccessTokenRedactorTest.kt b/src/test/kotlin/io/github/mojira/arisa/modules/privacy/AccessTokenRedactorTest.kt new file mode 100644 index 000000000..0a892413d --- /dev/null +++ b/src/test/kotlin/io/github/mojira/arisa/modules/privacy/AccessTokenRedactorTest.kt @@ -0,0 +1,33 @@ +package io.github.mojira.arisa.modules.privacy + +import io.github.mojira.arisa.utils.mockAttachment +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.nulls.shouldBeNull +import io.kotest.matchers.shouldBe +import io.kotest.matchers.types.shouldBeSameInstanceAs + +class AccessTokenRedactorTest : StringSpec({ + "AccessTokenRedactor should return null if nothing to redact" { + val attachment = mockAttachment( + mimeType = "text/plain", + getContent = { "some text".toByteArray() } + ) + val redactedAttachment = AccessTokenRedactor.redact(attachment) + redactedAttachment.shouldBeNull() + } + + "AccessTokenRedactor should redact access token" { + val attachment = mockAttachment( + mimeType = "text/plain", + getContent = { + // Example JWT token from https://jwt.io/ + ("some text --accessToken eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6" + + "IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c more text" + ).toByteArray() + } + ) + val redactedAttachment = AccessTokenRedactor.redact(attachment) + redactedAttachment!!.attachment shouldBeSameInstanceAs attachment + redactedAttachment.redactedContent shouldBe "some text --accessToken ###REDACTED### more text" + } +}) diff --git a/src/test/kotlin/io/github/mojira/arisa/modules/PrivacyModuleTest.kt b/src/test/kotlin/io/github/mojira/arisa/modules/privacy/PrivacyModuleTest.kt similarity index 59% rename from src/test/kotlin/io/github/mojira/arisa/modules/PrivacyModuleTest.kt rename to src/test/kotlin/io/github/mojira/arisa/modules/privacy/PrivacyModuleTest.kt index 735bcfb16..68ca14ac7 100644 --- a/src/test/kotlin/io/github/mojira/arisa/modules/PrivacyModuleTest.kt +++ b/src/test/kotlin/io/github/mojira/arisa/modules/privacy/PrivacyModuleTest.kt @@ -1,14 +1,19 @@ -package io.github.mojira.arisa.modules +package io.github.mojira.arisa.modules.privacy +import io.github.mojira.arisa.domain.Attachment import io.github.mojira.arisa.domain.CommentOptions +import io.github.mojira.arisa.modules.ModuleResponse +import io.github.mojira.arisa.modules.OperationNotNeededModuleResponse import io.github.mojira.arisa.utils.RIGHT_NOW import io.github.mojira.arisa.utils.mockAttachment import io.github.mojira.arisa.utils.mockChangeLogItem import io.github.mojira.arisa.utils.mockComment import io.github.mojira.arisa.utils.mockIssue +import io.github.mojira.arisa.utils.mockUser import io.kotest.assertions.arrow.either.shouldBeLeft import io.kotest.assertions.arrow.either.shouldBeRight import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.collections.shouldContainExactlyInAnyOrder import io.kotest.matchers.shouldBe private val TWO_SECONDS_AGO = RIGHT_NOW.minusSeconds(2) @@ -17,17 +22,25 @@ private val TEN_SECONDS_AGO = RIGHT_NOW.minusSeconds(10) private const val MADE_PRIVATE_MESSAGE = "made-private" private const val COMMENT_NOTE = "\n----\nRestricted by PrivacyModule ??[~arisabot]??" +private val NOOP_REDACTOR = object : AttachmentRedactor { + override fun redact(attachment: Attachment): RedactedAttachment? { + return null + } +} + private fun createModule( message: String = MADE_PRIVATE_MESSAGE, commentNote: String = COMMENT_NOTE, allowedEmailRegexes: List = emptyList(), sensitiveTextRegexes: List = emptyList(), + attachmentRedactor: AttachmentRedactor = NOOP_REDACTOR, sensitiveFileNameRegexes: List = emptyList() ) = PrivacyModule( message, commentNote, allowedEmailRegexes, sensitiveTextRegexes, + attachmentRedactor, sensitiveFileNameRegexes ) @@ -523,4 +536,319 @@ class PrivacyModuleTest : FunSpec({ addedComment shouldBe CommentOptions(MADE_PRIVATE_MESSAGE) } } + + context("attachment redaction") { + // Example JWT token from https://jwt.io/ + val accessTokenText = "... --accessToken " + + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c" + + " ..." + val otherSensitiveText = "sensitive" + val module = createModule( + sensitiveTextRegexes = listOf(Regex.fromLiteral("--accessToken ey"), Regex.fromLiteral(otherSensitiveText)), + attachmentRedactor = AccessTokenRedactor + ) + + test("should redact access tokens") { + val uploader = "some-\n-user" + val attachmentName = "my-\u202E-attachment.txt" + + var hasSetPrivate = false + var hasDeletedAttachment = false + var newAttachmentName: String? = null + var newAttachmentContent: String? = null + var addedComment: String? = null + + val issue = mockIssue( + attachments = listOf( + mockAttachment( + name = attachmentName, + uploader = mockUser( + name = uploader + ), + getContent = { accessTokenText.toByteArray() }, + remove = { hasDeletedAttachment = true } + ) + ), + setPrivate = { hasSetPrivate = true }, + addAttachmentFromFile = { file, cleanupCallback -> + newAttachmentName = file.name + newAttachmentContent = file.readText() + cleanupCallback() + }, + addRawBotComment = { addedComment = it } + ) + + val result = module(issue, TWO_SECONDS_AGO) + result.shouldBeRight(ModuleResponse) + + // Attachment was redacted; issue should not have been made private + hasSetPrivate shouldBe false + hasDeletedAttachment shouldBe true + newAttachmentName shouldBe "redacted_$attachmentName" + newAttachmentContent shouldBe "... --accessToken ###REDACTED### ..." + + // Should also have sanitized user name and attachment name + addedComment shouldBe "@[~some-?-user], sensitive data has been removed from your attachment(s) and they have " + + "been re-uploaded as:\n" + + "- [^redacted_my-?-attachment.txt]\n" + } + + test("should add multiple comments when redacting attachments of multiple users") { + val uploader1 = "some-user" + val uploader1User = mockUser( + name = uploader1 + ) + @Suppress("LocalVariableName") + val attachmentName1_1 = "my-attachment.txt" + @Suppress("LocalVariableName") + val attachmentName1_2 = "my-attachment2.txt" + + val uploader2 = "some-other-user" + val attachmentName2 = "other-attachment.txt" + + var hasSetPrivate = false + val deletedAttachmentNames = mutableListOf() + val newAttachmentNames = mutableListOf() + val addedComments = mutableListOf() + + val issue = mockIssue( + attachments = listOf( + mockAttachment( + name = attachmentName1_1, + uploader = uploader1User, + getContent = { accessTokenText.toByteArray() }, + remove = { deletedAttachmentNames.add(attachmentName1_1) } + ), + mockAttachment( + name = attachmentName1_2, + uploader = uploader1User, + getContent = { accessTokenText.toByteArray() }, + remove = { deletedAttachmentNames.add(attachmentName1_2) } + ), + mockAttachment( + name = attachmentName2, + uploader = mockUser(name = uploader2), + getContent = { accessTokenText.toByteArray() }, + remove = { deletedAttachmentNames.add(attachmentName2) } + ) + ), + setPrivate = { hasSetPrivate = true }, + addAttachmentFromFile = { file, cleanupCallback -> + newAttachmentNames.add(file.name) + cleanupCallback() + }, + addRawBotComment = { addedComments.add(it) } + ) + + val result = module(issue, TWO_SECONDS_AGO) + result.shouldBeRight(ModuleResponse) + + // Attachment was redacted; issue should not have been made private + hasSetPrivate shouldBe false + val expectedDeletedAttachmentNames = listOf(attachmentName1_1, attachmentName1_2, attachmentName2) + deletedAttachmentNames shouldContainExactlyInAnyOrder expectedDeletedAttachmentNames + newAttachmentNames shouldContainExactlyInAnyOrder expectedDeletedAttachmentNames.map { "redacted_$it" } + + addedComments shouldContainExactlyInAnyOrder listOf( + "@[~some-user], sensitive data has been removed from your attachment(s) and they " + + "have been re-uploaded as:\n" + + "- [^redacted_my-attachment.txt]\n" + + "- [^redacted_my-attachment2.txt]\n", + "@[~some-other-user], sensitive data has been removed from your attachment(s) " + + "and they have been re-uploaded as:\n" + + "- [^redacted_other-attachment.txt]\n" + ) + } + + test("should not redact bot attachments") { + var hasSetPrivate = false + var hasDeletedAttachment = false + + val issue = mockIssue( + attachments = listOf( + mockAttachment( + uploader = mockUser( + isBotUser = { true } + ), + getContent = { accessTokenText.toByteArray() }, + remove = { hasDeletedAttachment = true } + ) + ), + setPrivate = { hasSetPrivate = true } + ) + + val result = module(issue, TWO_SECONDS_AGO) + result.shouldBeRight(ModuleResponse) + + hasSetPrivate shouldBe true + hasDeletedAttachment shouldBe false + } + + test("should not redact attachments with non-redactable content") { + var hasSetPrivate = false + var hasDeletedAttachment = false + + val issue = mockIssue( + attachments = listOf( + mockAttachment( + // Redactor does not handle this; but it represents sensitive data + getContent = { "some $otherSensitiveText and more text".toByteArray() }, + remove = { hasDeletedAttachment = true } + ) + ), + setPrivate = { hasSetPrivate = true } + ) + + val result = module(issue, TWO_SECONDS_AGO) + result.shouldBeRight(ModuleResponse) + + hasSetPrivate shouldBe true + hasDeletedAttachment shouldBe false + } + + test("should redact attachments even if issue is made private") { + var hasSetPrivate = false + var hasDeletedAttachment = false + var hasAddedNewAttachment = false + var privateComment: CommentOptions? = null + var hasAddedRedactionComment = false + + val issue = mockIssue( + // Issue description cannot be redacted (would still be in history) + description = "some $otherSensitiveText and more text", + attachments = listOf( + mockAttachment( + getContent = { accessTokenText.toByteArray() }, + remove = { hasDeletedAttachment = true } + ) + ), + setPrivate = { hasSetPrivate = true }, + addAttachmentFromFile = { _, cleanupCallback -> + hasAddedNewAttachment = true + cleanupCallback() + }, + addComment = { privateComment = it }, + addRawBotComment = { hasAddedRedactionComment = true } + ) + + val result = module(issue, TWO_SECONDS_AGO) + result.shouldBeRight(ModuleResponse) + + hasSetPrivate shouldBe true + privateComment shouldBe CommentOptions(MADE_PRIVATE_MESSAGE) + // Issue was made private, but attachment should have been redacted anyways + hasDeletedAttachment shouldBe true + hasAddedNewAttachment shouldBe true + hasAddedRedactionComment shouldBe true + } + + test("should make private if redacting does not remove all sensitive data") { + var hasSetPrivate = false + var hasDeletedAttachment = false + var hasAddedNewAttachment = false + var privateComment: CommentOptions? = null + var hasAddedRedactionComment = false + + val issue = mockIssue( + attachments = listOf( + mockAttachment( + // Other sensitive text cannot be redacted + getContent = { "$accessTokenText \n $otherSensitiveText".toByteArray() }, + remove = { hasDeletedAttachment = true } + ) + ), + setPrivate = { hasSetPrivate = true }, + addAttachmentFromFile = { _, cleanupCallback -> + hasAddedNewAttachment = true + cleanupCallback() + }, + addComment = { privateComment = it }, + addRawBotComment = { hasAddedRedactionComment = true } + ) + + val result = module(issue, TWO_SECONDS_AGO) + result.shouldBeRight(ModuleResponse) + + hasSetPrivate shouldBe true + privateComment shouldBe CommentOptions(MADE_PRIVATE_MESSAGE) + hasDeletedAttachment shouldBe false + hasAddedNewAttachment shouldBe false + hasAddedRedactionComment shouldBe false + } + + test("should make private if attachment to redact has malformed name") { + var hasSetPrivate = false + var hasDeletedAttachment = false + var hasAddedNewAttachment = false + var privateComment: CommentOptions? = null + var hasAddedRedactionComment = false + + val issue = mockIssue( + attachments = listOf( + mockAttachment( + // Malformed name + name = "../../text.txt", + getContent = { accessTokenText.toByteArray() }, + remove = { hasDeletedAttachment = true } + ) + ), + setPrivate = { hasSetPrivate = true }, + addAttachmentFromFile = { _, cleanupCallback -> + hasAddedNewAttachment = true + cleanupCallback() + }, + addComment = { privateComment = it }, + addRawBotComment = { hasAddedRedactionComment = true } + ) + + val result = module(issue, TWO_SECONDS_AGO) + result.shouldBeRight(ModuleResponse) + + hasSetPrivate shouldBe true + privateComment shouldBe CommentOptions(MADE_PRIVATE_MESSAGE) + hasDeletedAttachment shouldBe false + hasAddedNewAttachment shouldBe false + hasAddedRedactionComment shouldBe false + } + + test("should make private if attachment to redact has name clash") { + val attachmentName = "text.txt" + + var hasSetPrivate = false + var hasDeletedAttachment = false + var hasAddedNewAttachment = false + var privateComment: CommentOptions? = null + var hasAddedRedactionComment = false + + val issue = mockIssue( + attachments = listOf( + mockAttachment( + name = "redacted_$attachmentName" + ), + mockAttachment( + // Name would clash with other attachment name + name = attachmentName, + getContent = { accessTokenText.toByteArray() }, + remove = { hasDeletedAttachment = true } + ) + ), + setPrivate = { hasSetPrivate = true }, + addAttachmentFromFile = { _, cleanupCallback -> + hasAddedNewAttachment = true + cleanupCallback() + }, + addComment = { privateComment = it }, + addRawBotComment = { hasAddedRedactionComment = true } + ) + + val result = module(issue, TWO_SECONDS_AGO) + result.shouldBeRight(ModuleResponse) + + hasSetPrivate shouldBe true + privateComment shouldBe CommentOptions(MADE_PRIVATE_MESSAGE) + hasDeletedAttachment shouldBe false + hasAddedNewAttachment shouldBe false + hasAddedRedactionComment shouldBe false + } + } }) diff --git a/src/test/kotlin/io/github/mojira/arisa/modules/privacy/TextRangeLocationTest.kt b/src/test/kotlin/io/github/mojira/arisa/modules/privacy/TextRangeLocationTest.kt new file mode 100644 index 000000000..426467512 --- /dev/null +++ b/src/test/kotlin/io/github/mojira/arisa/modules/privacy/TextRangeLocationTest.kt @@ -0,0 +1,77 @@ +package io.github.mojira.arisa.modules.privacy + +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.shouldBe + +class TextRangeLocationTest : FunSpec({ + data class TestData( + val description: String, + val textRangeLocation: TextRangeLocation, + val expectedLocationString: String + ) + + listOf( + TestData( + "in empty string", + // Range 0..-1 is returned for 0-length match at start, e.g. `Regex("^").find("test")?.range` + TextRangeLocation("", 0, -1), + "0 - -1 (1:0 - 1:-1)" + ), + TestData( + "at end of string", + // Range 4..3 is returned for 0-length match at end, e.g. `Regex("$").find("test")?.range` + TextRangeLocation("test", 4, 3), + "4 - 3 (1:4 - 1:3)" + ), + TestData( + "in first line", + TextRangeLocation("first\nsecond", 2, 3), + "2 - 3 (1:2 - 1:3)" + ), + TestData( + "in middle line", + TextRangeLocation("first\nsecond\nthird", 9, 10), + "9 - 10 (2:3 - 2:4)" + ), + TestData( + "in last line", + TextRangeLocation("first\nsecond", 9, 10), + "9 - 10 (2:3 - 2:4)" + ), + TestData( + "spanning multiple lines", + TextRangeLocation("first\nsecond", 2, 9), + "2 - 9 (1:2 - 2:3)" + ), + TestData( + "with CR", + TextRangeLocation("first\rsecond", 9, 10), + "9 - 10 (2:3 - 2:4)" + ), + TestData( + "with CR LF", + TextRangeLocation("first\r\nsecond", 9, 10), + "9 - 10 (2:2 - 2:3)" + ), + TestData( + "with mixed CR and LF", + TextRangeLocation("first\nsecond\rthird\r\nfourth", 23, 24), + "23 - 24 (4:3 - 4:4)" + ), + TestData( + "with trailing LF", + TextRangeLocation("test\n", 2, 3), + "2 - 3 (1:2 - 1:3)" + ), + TestData( + "behind trailing LF", + // Range 5..4 is returned for 0-length match at end, e.g. `Regex("\\z").find("test\n")?.range` + TextRangeLocation("test\n", 5, 4), + "5 - 4 (2:0 - 1:4)" + ) + ).forEach { + test(it.description) { + it.textRangeLocation.getLocationDescription() shouldBe it.expectedLocationString + } + } +}) diff --git a/src/test/kotlin/io/github/mojira/arisa/utils/MockDomain.kt b/src/test/kotlin/io/github/mojira/arisa/utils/MockDomain.kt index 4eb371106..27f2c0898 100644 --- a/src/test/kotlin/io/github/mojira/arisa/utils/MockDomain.kt +++ b/src/test/kotlin/io/github/mojira/arisa/utils/MockDomain.kt @@ -40,6 +40,8 @@ fun mockAttachment( ) fun mockChangeLogItem( + entryId: String = "1", + itemIndex: Int = 0, created: Instant = RIGHT_NOW, field: String = "", changedFrom: String? = null, @@ -49,6 +51,8 @@ fun mockChangeLogItem( author: User = mockUser(), getAuthorGroups: () -> List? = { emptyList() } ) = ChangeLogItem( + entryId, + itemIndex, created, field, changedFrom, @@ -132,6 +136,7 @@ fun mockIssue( addRestrictedComment: (options: CommentOptions) -> Unit = { }, addNotEnglishComment: (language: String) -> Unit = { }, addRawRestrictedComment: (body: String, restriction: String) -> Unit = { _, _ -> }, + addRawBotComment: (rawBody: String) -> Unit = { }, markAsFixedInASpecificVersion: (versionName: String) -> Unit = { }, changeReporter: (reporter: String) -> Unit = { }, addAttachmentFromFile: (file: File, cleanupCallback: () -> Unit) -> Unit = { _, cleanupCallback -> cleanupCallback() }, @@ -183,6 +188,7 @@ fun mockIssue( addRestrictedComment, addNotEnglishComment, addRawRestrictedComment, + addRawBotComment, markAsFixedInASpecificVersion, changeReporter, addAttachmentFromFile, @@ -227,12 +233,14 @@ fun mockUser( name: String = "user", displayName: String = "User", getGroups: () -> List? = { null }, - isNewUser: () -> Boolean = { false } + isNewUser: () -> Boolean = { false }, + isBotUser: () -> Boolean = { false } ) = User( name, displayName, getGroups, - isNewUser + isNewUser, + isBotUser ) fun mockVersion(