From 84e52e5c9252006e88d860417d13c836835b9844 Mon Sep 17 00:00:00 2001 From: Dmitry Pavlov Date: Sun, 5 Nov 2023 16:58:52 +0400 Subject: [PATCH 1/2] Working on logging improvements: 1. Explicitly configured logback 2. Log level management via cli parameter 3. by default printing summary for every updated/crated page only --- CHANGELOG.md | 12 +- .../zeldigas/text2confl/cli/CliOptions.kt | 6 + .../zeldigas/text2confl/cli/ClicktExt.kt | 14 +- .../zeldigas/text2confl/cli/DumpToMarkdown.kt | 1 + .../github/zeldigas/text2confl/cli/Logging.kt | 52 ++++++ .../github/zeldigas/text2confl/cli/Main.kt | 5 + .../cli/PrintingUploadOperationsTracker.kt | 154 +++++++++++++++++ .../github/zeldigas/text2confl/cli/Upload.kt | 12 +- cli/src/main/resources/logback.xml | 21 +++ .../zeldigas/text2confl/cli/UploadTest.kt | 8 +- .../zeldigas/confclient/ConfluenceClient.kt | 2 + .../confclient/ConfluenceClientImpl.kt | 18 +- .../convert/confluence/ReferenceProvider.kt | 7 +- .../confluence/ReferenceProviderImplTest.kt | 18 ++ .../text2confl/core/ServiceProvider.kt | 10 +- .../zeldigas/text2confl/core/config/IO.kt | 32 ++-- .../core/upload/ContentUploadErrors.kt | 8 + .../text2confl/core/upload/ContentUploader.kt | 33 ++-- .../text2confl/core/upload/DryRunClient.kt | 12 ++ .../core/upload/PageUploadOperations.kt | 38 ++++- .../core/upload/PageUploadOperationsImpl.kt | 147 ++++++++++++---- .../core/upload/UploadOperationTracker.kt | 26 +++ .../core/ServiceProviderImplTest.kt | 8 +- .../core/upload/ContentUploaderTest.kt | 98 +++++++---- .../upload/PageUploadOperationsImplTest.kt | 157 +++++++++++------- 25 files changed, 723 insertions(+), 176 deletions(-) create mode 100644 cli/src/main/kotlin/com/github/zeldigas/text2confl/cli/Logging.kt create mode 100644 cli/src/main/kotlin/com/github/zeldigas/text2confl/cli/PrintingUploadOperationsTracker.kt create mode 100644 cli/src/main/resources/logback.xml create mode 100644 core/src/main/kotlin/com/github/zeldigas/text2confl/core/upload/ContentUploadErrors.kt create mode 100644 core/src/main/kotlin/com/github/zeldigas/text2confl/core/upload/UploadOperationTracker.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d86dacd..31250436 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,13 +11,15 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - now in `upload` and `export-to-md` commands you can enable logging of http requests/responses and configure request timeout +- now configuration file can be named as `text2confl.yml` or `text2confl.yaml` in addition to dot-prefixed + names (`.text2confl.yml`, `.text2confl.yaml`). ### Changed - dependency updates: - - migrated to `io.github.oshai:kotlin-logging-jvm` - - plantuml to 1.2023.12 - + - migrated to `io.github.oshai:kotlin-logging-jvm` + - plantuml to 1.2023.12 + ### Fixed - Non-local links detection (may cause crash on Windows) @@ -38,8 +40,8 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - `cli` model is split into 2: `core` and `cli`. Contributed by @dgautier. - dependency updates - - Asciidoctor diagram to 2.2.13 - - plantuml to 1.2023.11 + - Asciidoctor diagram to 2.2.13 + - plantuml to 1.2023.11 ## 0.13.0 - 2023-08-28 diff --git a/cli/src/main/kotlin/com/github/zeldigas/text2confl/cli/CliOptions.kt b/cli/src/main/kotlin/com/github/zeldigas/text2confl/cli/CliOptions.kt index dc7ab283..7bc587d9 100644 --- a/cli/src/main/kotlin/com/github/zeldigas/text2confl/cli/CliOptions.kt +++ b/cli/src/main/kotlin/com/github/zeldigas/text2confl/cli/CliOptions.kt @@ -96,6 +96,12 @@ internal interface WithConfluenceServerOptions { requestTimeout = httpRequestTimeout ) fun askForSecret(prompt: String, requireConfirmation: Boolean = true): String? + + fun configureRequestLoggingIfEnabled() { + if (httpLogLevel != LogLevel.NONE) { + enableHttpLogging() + } + } } internal interface WithConversionOptions { diff --git a/cli/src/main/kotlin/com/github/zeldigas/text2confl/cli/ClicktExt.kt b/cli/src/main/kotlin/com/github/zeldigas/text2confl/cli/ClicktExt.kt index caa72d97..bf1785a9 100644 --- a/cli/src/main/kotlin/com/github/zeldigas/text2confl/cli/ClicktExt.kt +++ b/cli/src/main/kotlin/com/github/zeldigas/text2confl/cli/ClicktExt.kt @@ -11,6 +11,7 @@ import com.github.ajalt.mordant.terminal.StringPrompt import com.github.zeldigas.text2confl.convert.ConversionFailedException import com.github.zeldigas.text2confl.convert.FileDoesNotExistException import com.github.zeldigas.text2confl.core.ContentValidationFailedException +import com.github.zeldigas.text2confl.core.upload.ContentUploadException import com.github.zeldigas.text2confl.core.upload.InvalidTenantException fun parameterMissing(what: String, cliOption: String, fileOption: String): Nothing { @@ -31,8 +32,9 @@ fun RawOption.optionalFlag(vararg secondaryNames: String): NullableOption throw PrintMessage(ex.message!!, printError = true) - is FileDoesNotExistException -> throw PrintMessage(ex.message!!, printError = true) + is InvalidTenantException -> error(ex.message!!) + is FileDoesNotExistException -> error(ex.message!!) + is ContentUploadException -> error(ex.message!!) is ConversionFailedException -> { val reason = buildString { append(ex.message) @@ -40,18 +42,22 @@ fun tryHandleException(ex: Exception): Nothing { append(" (cause: ${ex.cause})") } } - throw PrintMessage("Failed to convert ${ex.file}: $reason", printError = true) + error("Failed to convert ${ex.file}: $reason") } is ContentValidationFailedException -> { val issues = ex.errors.mapIndexed { index, error -> "${index + 1}. $error" }.joinToString(separator = "\n") - throw PrintMessage("Some pages content is invalid:\n${issues}", printError = true) + error("Some pages content is invalid:\n${issues}") } else -> throw ex } } +private fun error(message: String): Nothing { + throw PrintMessage(message, printError = true, statusCode = 1) +} + fun CliktCommand.promptForSecret(prompt: String, requireConfirmation: Boolean): String? { return if (requireConfirmation) { ConfirmationPrompt.create(prompt, "Repeat for confirmation: ") { diff --git a/cli/src/main/kotlin/com/github/zeldigas/text2confl/cli/DumpToMarkdown.kt b/cli/src/main/kotlin/com/github/zeldigas/text2confl/cli/DumpToMarkdown.kt index 591c4780..8923ce3f 100644 --- a/cli/src/main/kotlin/com/github/zeldigas/text2confl/cli/DumpToMarkdown.kt +++ b/cli/src/main/kotlin/com/github/zeldigas/text2confl/cli/DumpToMarkdown.kt @@ -43,6 +43,7 @@ class DumpToMarkdown : CliktCommand(name = "export-to-md", help = "Exports confl override fun run() { try { + configureRequestLoggingIfEnabled() runBlocking { dumpPage() } } catch (ex: Exception) { tryHandleException(ex) diff --git a/cli/src/main/kotlin/com/github/zeldigas/text2confl/cli/Logging.kt b/cli/src/main/kotlin/com/github/zeldigas/text2confl/cli/Logging.kt new file mode 100644 index 00000000..da296fdf --- /dev/null +++ b/cli/src/main/kotlin/com/github/zeldigas/text2confl/cli/Logging.kt @@ -0,0 +1,52 @@ +package com.github.zeldigas.text2confl.cli + +import ch.qos.logback.classic.Level +import ch.qos.logback.classic.Logger +import ch.qos.logback.classic.Logger.ROOT_LOGGER_NAME +import ch.qos.logback.classic.spi.ILoggingEvent +import ch.qos.logback.core.filter.Filter +import ch.qos.logback.core.spi.FilterReply +import org.slf4j.LoggerFactory + + +class StdOutFilter : Filter() { + override fun decide(event: ILoggingEvent): FilterReply { + return if (event.level.isGreaterOrEqual(Level.WARN)) { + FilterReply.DENY + } else { + FilterReply.ACCEPT + } + } +} + +class StdErrFilter : Filter() { + override fun decide(event: ILoggingEvent): FilterReply { + return if (event.level.isGreaterOrEqual(Level.WARN)) { + FilterReply.ACCEPT + } else { + FilterReply.DENY + } + } +} + +fun configureLogging(verbosity: Int) { + if (verbosity == 0) return + + val rootLogger = LoggerFactory.getLogger(ROOT_LOGGER_NAME) as Logger + rootLogger.level = when (verbosity) { + 1 -> rootLogger.level + 2 -> Level.INFO + else -> Level.DEBUG + } + + val text2conflRoot = LoggerFactory.getLogger("com.github.zeldigas.text2confl") as Logger + text2conflRoot.level = when (verbosity) { + 1 -> Level.INFO + else -> Level.DEBUG + } +} + +fun enableHttpLogging() { + val rootLogger = LoggerFactory.getLogger("io.ktor.client") as Logger + rootLogger.level = Level.INFO +} \ No newline at end of file diff --git a/cli/src/main/kotlin/com/github/zeldigas/text2confl/cli/Main.kt b/cli/src/main/kotlin/com/github/zeldigas/text2confl/cli/Main.kt index 1bef06d0..4f14b00c 100644 --- a/cli/src/main/kotlin/com/github/zeldigas/text2confl/cli/Main.kt +++ b/cli/src/main/kotlin/com/github/zeldigas/text2confl/cli/Main.kt @@ -3,6 +3,8 @@ package com.github.zeldigas.text2confl.cli import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.core.context import com.github.ajalt.clikt.core.subcommands +import com.github.ajalt.clikt.parameters.options.counted +import com.github.ajalt.clikt.parameters.options.option import com.github.ajalt.clikt.sources.ChainedValueSource import com.github.ajalt.clikt.sources.PropertiesValueSource import com.github.zeldigas.text2confl.core.ServiceProviderImpl @@ -23,7 +25,10 @@ class ConfluencePublisher : CliktCommand() { } } + val verbosityLevel by option("-v", help = "Enable verbose output").counted() + override fun run() { + configureLogging(verbosityLevel) currentContext.obj = ServiceProviderImpl() } diff --git a/cli/src/main/kotlin/com/github/zeldigas/text2confl/cli/PrintingUploadOperationsTracker.kt b/cli/src/main/kotlin/com/github/zeldigas/text2confl/cli/PrintingUploadOperationsTracker.kt new file mode 100644 index 00000000..eaca7ce4 --- /dev/null +++ b/cli/src/main/kotlin/com/github/zeldigas/text2confl/cli/PrintingUploadOperationsTracker.kt @@ -0,0 +1,154 @@ +import com.github.ajalt.mordant.rendering.TextColors.* +import com.github.zeldigas.confclient.model.ConfluencePage +import com.github.zeldigas.text2confl.convert.Page +import com.github.zeldigas.text2confl.core.upload.* +import io.ktor.http.* +import java.util.concurrent.atomic.AtomicLong + +class PrintingUploadOperationsTracker( + val server: Url, + val prefix: String = "", + val printer: (msg: String) -> Unit = ::println +) : UploadOperationTracker { + + companion object { + const val UILINK = "tinyui" + } + + private val updatedCount = AtomicLong(0L) + + + override fun pageUpdated( + pageResult: PageOperationResult, + labelUpdate: LabelsUpdateResult, + attachmentsUpdated: AttachmentsUpdateResult + ) { + if (pageResult is PageOperationResult.NotModified + && labelUpdate == LabelsUpdateResult.NotChanged + && attachmentsUpdated == AttachmentsUpdateResult.NotChanged + ) return; + + updatedCount.incrementAndGet() + + when (pageResult) { + is PageOperationResult.Created -> { + printWithPrefix("${green("Created:")} ${pageInfo(pageResult.serverPage, pageResult.local)}") + } + + is PageOperationResult.ContentModified, + is PageOperationResult.LocationModified-> { + describeModifiedPage("Updated:", pageResult.serverPage, pageResult.local, labelUpdate, attachmentsUpdated) + } + + is PageOperationResult.NotModified -> { + if (labelUpdate != LabelsUpdateResult.NotChanged || attachmentsUpdated != AttachmentsUpdateResult.NotChanged) { + describeModifiedPage("Updated labels/attachments:", pageResult.serverPage, pageResult.local, labelUpdate, attachmentsUpdated) + } + } + } + } + + private fun describeModifiedPage( + operation: String, + serverPage: ServerPage, + local: Page, + labelUpdate: LabelsUpdateResult, + attachmentsUpdated: AttachmentsUpdateResult + ) { + val labelsAttachmentsInfo = labelsAttachmentsInfo(labelUpdate, attachmentsUpdated) + printWithPrefix( + "${cyan(operation)} ${ + pageInfo( + serverPage, + local + ) + }${if (labelsAttachmentsInfo.isNotBlank()) ". $labelsAttachmentsInfo" else "" }" + ) + } + + private fun pageInfo(serverPage: ServerPage, page: Page): String = buildString { + append('"') + append(blue(serverPage.title)) + append('"') + append(" from - ") + append(page.source.normalize()) + append(".") + val uiLink = serverPage.links[UILINK] + if (uiLink != null) { + append(" URL - ") + append(URLBuilder(server).appendPathSegments(uiLink).buildString()) + } + } + + private fun labelsAttachmentsInfo(labelsUpdateResult: LabelsUpdateResult, attachmentsUpdated: AttachmentsUpdateResult): String { + val labelsInfo = buildString { + if (labelsUpdateResult is LabelsUpdateResult.Updated) { + append("Labels ") + if (labelsUpdateResult.added.isNotEmpty()) { + append(green("added ")) + append("[") + append(labelsUpdateResult.added.joinToString(", ")) + append("]") + if (labelsUpdateResult.removed.isNotEmpty()) { + append("; ") + } + } + if (labelsUpdateResult.removed.isNotEmpty()) { + append(red("removed ")) + append("[") + append(labelsUpdateResult.removed.joinToString(", ")) + append("]") + } + } + } + val attachmentsInfo = buildString { + if (attachmentsUpdated is AttachmentsUpdateResult.Updated) { + append(" attachments: ") + append(green("added ${attachmentsUpdated.added.size}, ")) + append(cyan("modified ${attachmentsUpdated.modified.size}, ")) + append(red("removed ${attachmentsUpdated.removed.size}. ")) + } + } + return listOf(labelsInfo, attachmentsInfo).filter { it.isNotBlank() }.joinToString(", ") + } + + override fun uploadsCompleted() { + val updated = updatedCount.get() + if (updated == 0L) { + printer(green("All pages are up to date")) + } + } + + override fun pagesDeleted(root: ConfluencePage, allDeletedPages: List) { + if (allDeletedPages.isEmpty()) return + + printWithPrefix(buildString { + append(red("Deleted page")) + append(" ") + append(deletedPage(allDeletedPages[0])) + if (allDeletedPages.size > 1) { + append(" with subpages:") + } + }) + + val tail = allDeletedPages.drop(1) + if (tail.isNotEmpty()) { + tail.forEach { page -> + printWithPrefix("${red(" Deleted page")} ${deletedPage(page)}") + } + } + } + + private fun deletedPage(confluencePage: ConfluencePage): String { + return buildString { + append(blue(confluencePage.title)) + append(" (") + append(confluencePage.id) + append(")") + } + } + + private fun printWithPrefix(msg: String) { + printer("$prefix$msg") + } +} \ No newline at end of file diff --git a/cli/src/main/kotlin/com/github/zeldigas/text2confl/cli/Upload.kt b/cli/src/main/kotlin/com/github/zeldigas/text2confl/cli/Upload.kt index 90a26600..5966f83e 100644 --- a/cli/src/main/kotlin/com/github/zeldigas/text2confl/cli/Upload.kt +++ b/cli/src/main/kotlin/com/github/zeldigas/text2confl/cli/Upload.kt @@ -1,8 +1,10 @@ package com.github.zeldigas.text2confl.cli +import PrintingUploadOperationsTracker import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.core.PrintMessage import com.github.ajalt.clikt.core.requireObject +import com.github.ajalt.clikt.core.terminal import com.github.ajalt.clikt.parameters.options.flag import com.github.ajalt.clikt.parameters.options.option import com.github.ajalt.clikt.parameters.types.enum @@ -13,6 +15,7 @@ import com.github.zeldigas.text2confl.convert.EditorVersion import com.github.zeldigas.text2confl.core.ServiceProvider import com.github.zeldigas.text2confl.core.config.* import com.github.zeldigas.text2confl.core.upload.ChangeDetector +import com.github.zeldigas.text2confl.core.upload.UploadOperationTracker import io.ktor.client.plugins.logging.* import io.ktor.http.* import kotlinx.coroutines.Dispatchers @@ -66,6 +69,7 @@ class Upload : CliktCommand(name = "upload", help = "Converts source files and u override fun run() = runBlocking { try { + configureRequestLoggingIfEnabled() tryUpload() } catch (ex: Exception) { tryHandleException(ex) @@ -87,12 +91,18 @@ class Upload : CliktCommand(name = "upload", help = "Converts source files and u val confluenceClient = serviceProvider.createConfluenceClient(clientConfig, dryRun) val publishUnder = resolveParent(confluenceClient, uploadConfig, directoryStoredParams) - val contentUploader = serviceProvider.createUploader(confluenceClient, uploadConfig, conversionConfig) + val contentUploader = serviceProvider.createUploader(confluenceClient, uploadConfig, conversionConfig, operationsTracker(clientConfig.server)) withContext(Dispatchers.Default) { contentUploader.uploadPages(pages = result, uploadConfig.space, publishUnder) } } + private fun operationsTracker(server: Url): UploadOperationTracker = PrintingUploadOperationsTracker( + server = server, + printer = terminal::println, + prefix = if (dryRun) "(dryrun) " else "" + ) + private fun createUploadConfig(configuration: DirectoryConfig): UploadConfig { val orphanRemoval = if (docs.isFile) { Cleanup.None diff --git a/cli/src/main/resources/logback.xml b/cli/src/main/resources/logback.xml new file mode 100644 index 00000000..deb44c6b --- /dev/null +++ b/cli/src/main/resources/logback.xml @@ -0,0 +1,21 @@ + + + System.err + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger - %msg%n + + + + System.out + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger - %msg%n + + + + + + + + diff --git a/cli/src/test/kotlin/com/github/zeldigas/text2confl/cli/UploadTest.kt b/cli/src/test/kotlin/com/github/zeldigas/text2confl/cli/UploadTest.kt index ff36788a..dc15bedc 100644 --- a/cli/src/test/kotlin/com/github/zeldigas/text2confl/cli/UploadTest.kt +++ b/cli/src/test/kotlin/com/github/zeldigas/text2confl/cli/UploadTest.kt @@ -51,7 +51,7 @@ internal class UploadTest( internal fun setUp() { every { serviceProvider.createConverter(any(), any()) } returns converter every { serviceProvider.createConfluenceClient(any(), any()) } returns confluenceClient - every { serviceProvider.createUploader(confluenceClient, any(), any()) } returns contentUploader + every { serviceProvider.createUploader(confluenceClient, any(), any(), any()) } returns contentUploader every { serviceProvider.createContentValidator() } returns contentValidator command.context { @@ -100,7 +100,8 @@ internal class UploadTest( confluenceClient, UploadConfig( "TR", Cleanup.All, "Automated upload by text2confl", true, ChangeDetector.HASH, "test" ), - expectedConverterConfig + expectedConverterConfig, + any() ) } } @@ -154,7 +155,8 @@ internal class UploadTest( directoryConfig.modificationCheck, "test1" ), - converterConfig + converterConfig, + any() ) } } diff --git a/confluence-client/src/main/kotlin/com/github/zeldigas/confclient/ConfluenceClient.kt b/confluence-client/src/main/kotlin/com/github/zeldigas/confclient/ConfluenceClient.kt index 615a163b..a8a65023 100644 --- a/confluence-client/src/main/kotlin/com/github/zeldigas/confclient/ConfluenceClient.kt +++ b/confluence-client/src/main/kotlin/com/github/zeldigas/confclient/ConfluenceClient.kt @@ -42,6 +42,8 @@ interface ConfluenceClient { suspend fun updatePage(pageId: String, value: PageContentInput, updateParameters: PageUpdateOptions): ConfluencePage + suspend fun renamePage(serverPage: ConfluencePage, newTitle: String, updateParameters: PageUpdateOptions) : ConfluencePage + suspend fun changeParent( pageId: String, title: String, diff --git a/confluence-client/src/main/kotlin/com/github/zeldigas/confclient/ConfluenceClientImpl.kt b/confluence-client/src/main/kotlin/com/github/zeldigas/confclient/ConfluenceClientImpl.kt index b88f1857..cf74e6f4 100644 --- a/confluence-client/src/main/kotlin/com/github/zeldigas/confclient/ConfluenceClientImpl.kt +++ b/confluence-client/src/main/kotlin/com/github/zeldigas/confclient/ConfluenceClientImpl.kt @@ -161,6 +161,20 @@ class ConfluenceClientImpl( ) ) + override suspend fun renamePage( + serverPage: ConfluencePage, + newTitle: String, + updateParameters: PageUpdateOptions + ): ConfluencePage = + performPageUpdate( + serverPage.id, mapOf( + "type" to "page", + "title" to newTitle, + "version" to versionNode(serverPage.version!!.number + 1, updateParameters) + ) + ) + + private suspend fun performPageUpdate(pageId: String, body: Map): ConfluencePage { val response = httpClient.put("$apiBase/content/$pageId") { contentType(ContentType.Application.Json) @@ -366,6 +380,6 @@ fun confluenceClient( } } - - return ConfluenceClientImpl(config.server, "${config.server}/rest/api", client) + val baseUrl = URLBuilder(config.server).appendPathSegments("rest", "api").build().toString() + return ConfluenceClientImpl(config.server, baseUrl, client) } \ No newline at end of file diff --git a/convert/src/main/kotlin/com/github/zeldigas/text2confl/convert/confluence/ReferenceProvider.kt b/convert/src/main/kotlin/com/github/zeldigas/text2confl/convert/confluence/ReferenceProvider.kt index 22fc69db..5ea7aedf 100644 --- a/convert/src/main/kotlin/com/github/zeldigas/text2confl/convert/confluence/ReferenceProvider.kt +++ b/convert/src/main/kotlin/com/github/zeldigas/text2confl/convert/confluence/ReferenceProvider.kt @@ -1,6 +1,7 @@ package com.github.zeldigas.text2confl.convert.confluence import com.github.zeldigas.text2confl.convert.PageHeader +import io.github.oshai.kotlinlogging.KotlinLogging import java.nio.file.Path import kotlin.io.path.relativeTo @@ -40,13 +41,17 @@ class ReferenceProviderImpl(private val basePath: Path, documents: Map path.relativeTo(basePath).normalize() to header }.toMap() override fun resolveReference(source: Path, refTo: String): Reference? { - if (URI_DETECTOR.find(refTo) != null) return null + if (URI_DETECTOR.find(refTo) != null) { + log.debug { "$refTo detected as link in $source" } + return null + } if (refTo.startsWith("#")) return Anchor(refTo.substring(1)) val parts = refTo.split("#", limit = 2) diff --git a/convert/src/test/kotlin/com/github/zeldigas/text2confl/convert/confluence/ReferenceProviderImplTest.kt b/convert/src/test/kotlin/com/github/zeldigas/text2confl/convert/confluence/ReferenceProviderImplTest.kt index 8e1192bf..ee5e004d 100644 --- a/convert/src/test/kotlin/com/github/zeldigas/text2confl/convert/confluence/ReferenceProviderImplTest.kt +++ b/convert/src/test/kotlin/com/github/zeldigas/text2confl/convert/confluence/ReferenceProviderImplTest.kt @@ -3,6 +3,7 @@ package com.github.zeldigas.text2confl.convert.confluence import assertk.assertThat import assertk.assertions.isEqualTo import assertk.assertions.isNotNull +import assertk.assertions.isNull import com.github.zeldigas.text2confl.convert.PageHeader import org.junit.jupiter.api.Test import org.junit.jupiter.params.ParameterizedTest @@ -68,4 +69,21 @@ internal class ReferenceProviderImplTest { assertThat(result).isNotNull().isEqualTo(Xref("Sub Title One", "test")) } + + @CsvSource( + value = [ + "mailto:john@example.org", + "http://example.org", + "https://example.org", + "https://example.org/docs/one.md", + "file://example.org/docs/one.md", //for now this one is filtered out as well + "ftp://example.org/docs/one.md" + ] + ) + @ParameterizedTest + fun `Links of varios kinds are filtered out`(link:String) { + val result = providerImpl.resolveReference(Path("docs/one.md"), link) + + assertThat(result).isNull() + } } \ No newline at end of file diff --git a/core/src/main/kotlin/com/github/zeldigas/text2confl/core/ServiceProvider.kt b/core/src/main/kotlin/com/github/zeldigas/text2confl/core/ServiceProvider.kt index 155725d0..d645569b 100644 --- a/core/src/main/kotlin/com/github/zeldigas/text2confl/core/ServiceProvider.kt +++ b/core/src/main/kotlin/com/github/zeldigas/text2confl/core/ServiceProvider.kt @@ -12,6 +12,7 @@ import com.github.zeldigas.text2confl.core.config.UploadConfig import com.github.zeldigas.text2confl.core.export.PageExporter import com.github.zeldigas.text2confl.core.upload.ContentUploader import com.github.zeldigas.text2confl.core.upload.DryRunClient +import com.github.zeldigas.text2confl.core.upload.UploadOperationTracker interface ServiceProvider { fun createConverter(space: String, config: ConverterConfig): Converter @@ -19,7 +20,8 @@ interface ServiceProvider { fun createUploader( client: ConfluenceClient, uploadConfig: UploadConfig, - converterConfig: ConverterConfig + converterConfig: ConverterConfig, + uploadOperationTracker: UploadOperationTracker ): ContentUploader fun createContentValidator(): ContentValidator @@ -51,13 +53,15 @@ class ServiceProviderImpl : ServiceProvider { override fun createUploader( client: ConfluenceClient, uploadConfig: UploadConfig, - converterConfig: ConverterConfig + converterConfig: ConverterConfig, + uploadOperationTracker: UploadOperationTracker ): ContentUploader { return ContentUploader( client, uploadConfig.uploadMessage, uploadConfig.notifyWatchers, uploadConfig.modificationCheck, converterConfig.editorVersion, uploadConfig.removeOrphans, - uploadConfig.tenant + uploadConfig.tenant, + uploadOperationTracker ) } diff --git a/core/src/main/kotlin/com/github/zeldigas/text2confl/core/config/IO.kt b/core/src/main/kotlin/com/github/zeldigas/text2confl/core/config/IO.kt index 50592282..01e014b6 100644 --- a/core/src/main/kotlin/com/github/zeldigas/text2confl/core/config/IO.kt +++ b/core/src/main/kotlin/com/github/zeldigas/text2confl/core/config/IO.kt @@ -2,24 +2,28 @@ package com.github.zeldigas.text2confl.core.config import com.fasterxml.jackson.databind.DeserializationFeature import com.fasterxml.jackson.databind.MapperFeature -import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.PropertyNamingStrategies +import com.fasterxml.jackson.databind.json.JsonMapper import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import com.fasterxml.jackson.module.kotlin.kotlinModule import com.fasterxml.jackson.module.kotlin.readValue -import com.fasterxml.jackson.module.kotlin.registerKotlinModule +import io.github.oshai.kotlinlogging.KotlinLogging import java.nio.file.Path import kotlin.io.path.absolute import kotlin.io.path.exists import kotlin.io.path.isRegularFile -private val mapper = ObjectMapper(YAMLFactory()) - .registerKotlinModule() - .setPropertyNamingStrategy(PropertyNamingStrategies.KEBAB_CASE) +private val mapper = JsonMapper.builder(YAMLFactory()) + .addModule(kotlinModule()) + .propertyNamingStrategy(PropertyNamingStrategies.KEBAB_CASE) .enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS) .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) + .build() -private val CONFIG_FILE_NAMES = listOf(".text2confl.yml", ".text2confl.yaml") +private val CONFIG_FILE_NAMES = listOf(".text2confl", "text2confl") + +private val log = KotlinLogging.logger { } fun readDirectoryConfig(dirOfFile: Path): DirectoryConfig { val resolver: (String) -> Path = if (dirOfFile.isRegularFile()) { @@ -27,12 +31,20 @@ fun readDirectoryConfig(dirOfFile: Path): DirectoryConfig { } else { dirOfFile::resolve } + val docsDir = if (dirOfFile.isRegularFile()) dirOfFile.absolute().parent else dirOfFile - val directoryConfig = CONFIG_FILE_NAMES.asSequence() + val configFile = CONFIG_FILE_NAMES.asSequence() + .flatMap { listOf("$it.yml", "$it.yaml") } .map(resolver) .filter { it.exists() } - .map { mapper.readValue(it.toFile()) } - .firstOrNull() ?: DirectoryConfig() - directoryConfig.docsDir = if (dirOfFile.isRegularFile()) dirOfFile.absolute().parent else dirOfFile + .firstOrNull() + val directoryConfig:DirectoryConfig = if (configFile != null) { + log.debug { "Found config file $configFile" } + mapper.readValue(configFile.toFile()) + } else { + log.debug { "No config file found in $docsDir. Using defaults" } + DirectoryConfig() + } + directoryConfig.docsDir = docsDir return directoryConfig } \ No newline at end of file diff --git a/core/src/main/kotlin/com/github/zeldigas/text2confl/core/upload/ContentUploadErrors.kt b/core/src/main/kotlin/com/github/zeldigas/text2confl/core/upload/ContentUploadErrors.kt new file mode 100644 index 00000000..a09972a7 --- /dev/null +++ b/core/src/main/kotlin/com/github/zeldigas/text2confl/core/upload/ContentUploadErrors.kt @@ -0,0 +1,8 @@ +package com.github.zeldigas.text2confl.core.upload + +import java.nio.file.Path + +open class ContentUploadException(message: String, cause: Throwable? = null) : RuntimeException(message, cause) + +class VirtualPageNotFound(val source: Path, val title: String, val space: String) : + ContentUploadException("${source} defined as virtual, but $space space does not have page with title: $title") \ No newline at end of file diff --git a/core/src/main/kotlin/com/github/zeldigas/text2confl/core/upload/ContentUploader.kt b/core/src/main/kotlin/com/github/zeldigas/text2confl/core/upload/ContentUploader.kt index 800a05b6..638650b7 100644 --- a/core/src/main/kotlin/com/github/zeldigas/text2confl/core/upload/ContentUploader.kt +++ b/core/src/main/kotlin/com/github/zeldigas/text2confl/core/upload/ContentUploader.kt @@ -17,7 +17,8 @@ class ContentUploader( val pageUploadOperations: PageUploadOperations, val client: ConfluenceClient, val cleanup: Cleanup, - val tenant: String? + val tenant: String?, + val tracker: UploadOperationTracker = NOP ) { constructor( @@ -27,7 +28,8 @@ class ContentUploader( pageContentChangeDetector: ChangeDetector, editorVersion: EditorVersion, cleanup: Cleanup, - tenant: String? + tenant: String?, + tracker: UploadOperationTracker = NOP ) : this( PageUploadOperationsImpl( client, @@ -39,7 +41,8 @@ class ContentUploader( ), client, cleanup, - tenant + tenant, + tracker ) companion object { @@ -48,6 +51,7 @@ class ContentUploader( suspend fun uploadPages(pages: List, space: String, parentPageId: String) { val uploadedPages = uploadPagesRecursive(pages, space, parentPageId) + tracker.uploadsCompleted() val uploadedPagesByParent = buildOrphanedRemovalRegistry(uploadedPages) deleteOrphans(uploadedPagesByParent) } @@ -75,14 +79,21 @@ class ContentUploader( private suspend fun uploadPage(page: Page, space: String, defaultParentPage: String): PageUploadResult { val parentId = customPageParent(page, space) ?: defaultParentPage return if (!page.virtual) { - logger.info { "Uploading page: title=${page.title}" } - val serverPage = pageUploadOperations.createOrUpdatePageContent(page, space, parentId) - pageUploadOperations.updatePageLabels(serverPage, page.content) - pageUploadOperations.updatePageAttachments(serverPage, page.content) + logger.info { "Uploading page: title=${page.title}, src=${page.source}" } + val pageResult = pageUploadOperations.createOrUpdatePageContent(page, space, parentId) + val serverPage = pageResult.serverPage + val labelUpdate = pageUploadOperations.updatePageLabels(serverPage, page.content) + val attachmentsUpdated = pageUploadOperations.updatePageAttachments(serverPage, page.content) + tracker.pageUpdated(pageResult, labelUpdate, attachmentsUpdated) + logger.info { "Page uploaded: title=${page.title}, src=${page.source}: id=${serverPage.id}" } PageUploadResult(parentId, serverPage, virtual = false) } else { logger.info { "Checking that virtual page exists and properly located: ${page.title}" } - val virtualPage = pageUploadOperations.checkPageAndUpdateParentIfRequired(page.title, space, parentId) + val virtualPage = try { + pageUploadOperations.checkPageAndUpdateParentIfRequired(page.title, space, parentId) + } catch (ex: PageNotFoundException) { + throw VirtualPageNotFound(page.source, page.title, space) + } PageUploadResult(parentId, virtualPage, true) } } @@ -108,6 +119,7 @@ class ContentUploader( private suspend fun deleteOrphans(uploadedPagesByParent: Map>) { logger.debug { "Running cleanup operation using strategy: $cleanup" } + logger.debug { "Cleanup operation: $cleanup" } coroutineScope { for ((parent, children) in uploadedPagesByParent) { launch { deleteOrphanedChildren(parent, children) } @@ -127,8 +139,9 @@ class ContentUploader( coroutineScope { for (page in pagesForDeletion) { launch { - logger.info { "Deleting orphaned page: title=${page.title}, id=${page.id}" } - pageUploadOperations.deletePageWithChildren(page.id) + logger.info { "Deleting orphaned page and subpages: title=${page.title}, id=${page.id}" } + val deletedPages = pageUploadOperations.deletePageWithChildren(page) + tracker.pagesDeleted(page, deletedPages) } } } diff --git a/core/src/main/kotlin/com/github/zeldigas/text2confl/core/upload/DryRunClient.kt b/core/src/main/kotlin/com/github/zeldigas/text2confl/core/upload/DryRunClient.kt index 4297482c..b54651da 100644 --- a/core/src/main/kotlin/com/github/zeldigas/text2confl/core/upload/DryRunClient.kt +++ b/core/src/main/kotlin/com/github/zeldigas/text2confl/core/upload/DryRunClient.kt @@ -71,6 +71,18 @@ class DryRunClient(private val realClient: ConfluenceClient) : ConfluenceClient ) } + override suspend fun renamePage( + serverPage: ConfluencePage, + newTitle: String, + updateParameters: PageUpdateOptions + ): ConfluencePage { + log.info { "(dryrun) Changing title of page with ${serverPage.id}: ${serverPage.title} -> $newTitle" } + return serverPage.copy( + title = newTitle, + version = PageVersionInfo(serverPage.version!!.number + 1, true, ZonedDateTime.now()) + ) + } + override suspend fun deletePage(pageId: String) { log.info { "(dryrun) Deleting page $pageId" } } diff --git a/core/src/main/kotlin/com/github/zeldigas/text2confl/core/upload/PageUploadOperations.kt b/core/src/main/kotlin/com/github/zeldigas/text2confl/core/upload/PageUploadOperations.kt index c6745506..328a6505 100644 --- a/core/src/main/kotlin/com/github/zeldigas/text2confl/core/upload/PageUploadOperations.kt +++ b/core/src/main/kotlin/com/github/zeldigas/text2confl/core/upload/PageUploadOperations.kt @@ -12,18 +12,45 @@ const val EDITOR_PROPERTY = "editor" interface PageUploadOperations { - suspend fun createOrUpdatePageContent(page: Page, space: String, parentPageId: String): ServerPage + suspend fun createOrUpdatePageContent(page: Page, space: String, parentPageId: String): PageOperationResult suspend fun checkPageAndUpdateParentIfRequired(title: String, space: String, parentId: String): ServerPage - suspend fun updatePageLabels(serverPage: ServerPage, content: PageContent) + suspend fun updatePageLabels(serverPage: ServerPage, content: PageContent): LabelsUpdateResult + + suspend fun updatePageAttachments(serverPage: ServerPage, content: PageContent): AttachmentsUpdateResult - suspend fun updatePageAttachments(serverPage: ServerPage, content: PageContent) suspend fun findChildPages(pageId: String): List - suspend fun deletePageWithChildren(pageId: String) + suspend fun deletePageWithChildren(page: ConfluencePage): List + +} + +sealed class PageOperationResult { + data class NotModified(override val local: Page, override val serverPage: ServerPage) : PageOperationResult() + data class LocationModified(override val local: Page, override val serverPage: ServerPage, val previousParent: String, val previousTitle: String) : PageOperationResult() + data class Created(override val local: Page, override val serverPage: ServerPage) : PageOperationResult() + data class ContentModified(override val local: Page, override val serverPage: ServerPage, val parentChanged: Boolean = false) : PageOperationResult() + + abstract val local: Page + abstract val serverPage: ServerPage +} + +sealed class LabelsUpdateResult { + data class Updated(val added: List, val removed: List): LabelsUpdateResult() + object NotChanged: LabelsUpdateResult() +} +sealed class AttachmentsUpdateResult { + data class Updated(val added: List, + val modified:List, + val removed: List): AttachmentsUpdateResult() + object NotChanged: AttachmentsUpdateResult() } +abstract class PageOperationException(message: String, cause: Exception? = null) : RuntimeException(message, cause) + +data class PageNotFoundException(val space: String, val title: String): PageOperationException("Page $title in space $space not found") + enum class ChangeDetector( val extraData: Set, val strategy: (serverPage: ConfluencePage, content: PageContent) -> Boolean @@ -37,7 +64,8 @@ enum class ChangeDetector( } data class ServerPage( - val id: String, val title: String, val parent: String, val labels: List