GH-458 Initialize basic storage providers tests with Testcontainers a…

…nd bump dependencies

gradle/wrapper/gradle-wrapper.properties

@@ -1,21 +1,5 @@
-#
-# Copyright (c) 2021 dzikoysk
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

gradlew

@@ -1,13 +1,13 @@
 #!/usr/bin/env sh
 
 #
-# Copyright (c) 2021 dzikoysk
+# Copyright 2015 the original author or authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
+#      https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -72,7 +72,7 @@ case "`uname`" in
   Darwin* )
     darwin=true
     ;;
-  MINGW* )
+  MSYS* | MINGW* )
     msys=true
     ;;
   NONSTOP* )

reposilite-backend/reposilite-backend.gradle.kts

@@ -73,11 +73,11 @@ dependencies {
     implementation("org.panda-lang:expressible-kt:$expressible")
     testImplementation("org.panda-lang:expressible-junit:$expressible")
 
-    val awssdk = "2.15.15"
+    val awssdk = "2.17.31"
     implementation("software.amazon.awssdk:bom:$awssdk")
     implementation("software.amazon.awssdk:s3:$awssdk")
 
-    val exposed = "0.32.1"
+    val exposed = "0.34.1"
     implementation("org.jetbrains.exposed:exposed-core:$exposed")
     implementation("org.jetbrains.exposed:exposed-dao:$exposed")
     implementation("org.jetbrains.exposed:exposed-jdbc:$exposed")
@@ -134,10 +134,13 @@ dependencies {
 
     /* Tests */
 
-    val testcontainers = "1.15.3"
+    val testcontainers = "1.16.0"
     testImplementation("org.testcontainers:mariadb:$testcontainers")
     testImplementation("org.testcontainers:testcontainers:$testcontainers")
     testImplementation("org.testcontainers:junit-jupiter:$testcontainers")
+    testImplementation("org.testcontainers:localstack:$testcontainers")
+
+    testImplementation("com.amazonaws:aws-java-sdk-s3:1.12.59")
 
     testImplementation("com.google.http-client:google-http-client-jackson2:1.39.2")
     testImplementation("org.junit.jupiter:junit-jupiter-params:5.7.1")

reposilite-backend/src/main/kotlin/com/reposilite/config/ConfigurationLoader.kt

@@ -19,34 +19,12 @@ import com.reposilite.journalist.Journalist
 import com.reposilite.journalist.Logger
 import net.dzikoysk.cdn.CdnFactory
 import panda.utilities.StringUtils
-import picocli.CommandLine
 import java.nio.file.Path
 
 const val DEFAULT_CONFIGURATION_FILE = "reposilite.cdn"
 
 class ConfigurationLoader(private val journalist: Journalist) : Journalist {
 
-    companion object {
-
-        fun <CONFIGURATION : Runnable> loadConfiguration(configuration: CONFIGURATION, description: String): Pair<String, CONFIGURATION> =
-            description.split(" ", limit = 2)
-                .let { Pair(it[0], it.getOrNull(1) ?: "") }
-                .also {
-                    val commandLine = CommandLine(configuration)
-
-                    val args =
-                        if (it.second.isEmpty())
-                            arrayOf()
-                        else
-                            it.second.split(" ").toTypedArray()
-
-                    commandLine.execute(*args)
-                }
-                .let { Pair(it.first, configuration) }
-                .also { it.second.run() }
-
-    }
-
     fun tryLoad(customConfigurationFile: Path): Configuration {
         return try {
             load(customConfigurationFile)

reposilite-backend/src/main/kotlin/com/reposilite/console/LevelCommand.kt

@@ -22,7 +22,7 @@ import com.reposilite.journalist.Channel
 import picocli.CommandLine.Command
 import picocli.CommandLine.Parameters
 
-@Command(name = "version", description = ["Change current level of visible logging"])
+@Command(name = "level", description = ["Change current level of visible logging"])
 internal class LevelCommand(private val journalist: ReposiliteJournalist) : ReposiliteCommand {
 
     @Parameters(index = "0", paramLabel = "<level>", description = ["the new threshold"], defaultValue = "info")

reposilite-backend/src/main/kotlin/com/reposilite/maven/RepositoryFactory.kt

@@ -18,9 +18,9 @@ package com.reposilite.maven
 
 import com.reposilite.config.Configuration.RepositoryConfiguration
 import com.reposilite.config.Configuration.RepositoryConfiguration.ProxiedHostConfiguration
-import com.reposilite.config.ConfigurationLoader
 import com.reposilite.journalist.Journalist
 import com.reposilite.maven.MavenFacade.Companion.REPOSITORIES
+import com.reposilite.shared.loadCommandBasedConfiguration
 import com.reposilite.shared.safeResolve
 import com.reposilite.storage.StorageProviderFactory.createStorageProvider
 import java.nio.file.Path
@@ -45,7 +45,7 @@ internal class RepositoryFactory(
         )
 
     private fun createProxiedHostConfiguration(configuration: String): Pair<String, ProxiedHostConfiguration> =
-        with(ConfigurationLoader.loadConfiguration(ProxiedHostConfiguration(), configuration)) {
+        with(loadCommandBasedConfiguration(ProxiedHostConfiguration(), configuration)) {
             if (first.endsWith("/"))
                 Pair(first.substring(0, first.length - 1), second)
             else

reposilite-backend/src/main/kotlin/com/reposilite/shared/PicocliExtensions.kt

@@ -0,0 +1,20 @@
+package com.reposilite.shared
+
+import picocli.CommandLine
+
+fun <CONFIGURATION : Runnable> loadCommandBasedConfiguration(configuration: CONFIGURATION, description: String): Pair<String, CONFIGURATION> =
+    description.split(" ", limit = 2)
+        .let { Pair(it[0], it.getOrNull(1) ?: "") }
+        .also {
+            val commandLine = CommandLine(configuration)
+
+            val args =
+                if (it.second.isEmpty())
+                    arrayOf()
+                else
+                    it.second.split(" ").toTypedArray()
+
+            commandLine.execute(*args)
+        }
+        .let { Pair(it.first, configuration) }
+        .also { it.second.run() }

reposilite-backend/src/main/kotlin/com/reposilite/storage/StorageProviderFactory.kt

@@ -16,11 +16,14 @@
 
 package com.reposilite.storage
 
-import com.reposilite.config.ConfigurationLoader
 import com.reposilite.journalist.Journalist
+import com.reposilite.shared.loadCommandBasedConfiguration
 import com.reposilite.storage.infrastructure.FileSystemStorageProviderFactory
 import com.reposilite.storage.infrastructure.S3StorageProvider
 import com.reposilite.storage.infrastructure.S3StorageProviderSettings
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider
+import software.amazon.awssdk.regions.Region
+import software.amazon.awssdk.services.s3.S3Client
 import java.nio.file.Files
 import java.nio.file.Path
 
@@ -33,13 +36,19 @@ internal object StorageProviderFactory {
         }
         else if (storageDescription.startsWith("s3")) {
             // Implement quota
-            val settings = ConfigurationLoader.loadConfiguration(S3StorageProviderSettings(), storageDescription).second
-            S3StorageProvider(journalist, settings.bucketName, settings.region)
+            val settings = loadCommandBasedConfiguration(S3StorageProviderSettings(), storageDescription).second
+            S3StorageProvider(journalist, createUnauthenticatedS3Client(settings.region), settings.bucketName)
         }
         // else if (storageDescription.equals("rest", ignoreCase = true)) {
         // TOFIX REST API storage endpoint
         //    null
         //}
         else throw UnsupportedOperationException("Unknown storage provider: $storageDescription")
 
+    private fun createUnauthenticatedS3Client(region: String): S3Client =
+         S3Client.builder()
+            .region(Region.of(region))
+            .credentialsProvider(AnonymousCredentialsProvider.create())
+            .build()
+
 }

reposilite-backend/src/main/kotlin/com/reposilite/storage/infrastructure/S3StorageProvider.kt

@@ -26,19 +26,18 @@ import com.reposilite.shared.getExtension
 import com.reposilite.shared.getSimpleName
 import com.reposilite.shared.safeResolve
 import com.reposilite.storage.StorageProvider
+import com.reposilite.web.http.ErrorResponse
+import com.reposilite.web.http.errorResponse
 import io.javalin.http.ContentType
 import io.javalin.http.ContentType.APPLICATION_OCTET_STREAM
 import io.javalin.http.ContentType.Companion.OCTET_STREAM
-import com.reposilite.web.http.ErrorResponse
-import com.reposilite.web.http.errorResponse
 import io.javalin.http.HttpCode
 import io.javalin.http.HttpCode.NOT_FOUND
 import panda.std.Result
 import panda.std.asSuccess
-import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider
 import software.amazon.awssdk.core.sync.RequestBody
-import software.amazon.awssdk.regions.Region
 import software.amazon.awssdk.services.s3.S3Client
+import software.amazon.awssdk.services.s3.model.BucketAlreadyExistsException
 import software.amazon.awssdk.services.s3.model.DeleteObjectRequest
 import software.amazon.awssdk.services.s3.model.GetObjectRequest
 import software.amazon.awssdk.services.s3.model.HeadObjectRequest
@@ -54,14 +53,19 @@ import java.nio.file.attribute.FileTime
 
 internal class S3StorageProvider(
     private val journalist: Journalist,
+    private val s3: S3Client,
     private val bucket: String,
-    region: String
 ) : StorageProvider, Journalist {
 
-    private val s3: S3Client = S3Client.builder()
-        .region(Region.of(region))
-        .credentialsProvider(AnonymousCredentialsProvider.create())
-        .build()
+    init {
+        try {
+            s3.createBucket {
+                it.bucket(bucket)
+            }
+        } catch (bucketExists: BucketAlreadyExistsException) {
+            // ignored
+        }
+    }
 
     override fun putFile(file: Path, bytes: ByteArray): Result<DocumentInfo, ErrorResponse> =
         try {

reposilite-backend/src/test/kotlin/com/reposilite/storage/FileSystemStorageProviderTest.kt

@@ -0,0 +1,18 @@
+package com.reposilite.storage
+
+import com.reposilite.storage.infrastructure.FileSystemStorageProviderFactory
+import org.junit.jupiter.api.BeforeEach
+import org.junit.jupiter.api.io.TempDir
+import java.io.File
+
+internal class FileSystemStorageProviderTest : StorageProviderTest() {
+
+    @TempDir
+    lateinit var rootDirectory: File
+
+    @BeforeEach
+    fun setup() {
+        super.storageProvider = FileSystemStorageProviderFactory.of(rootDirectory.toPath(), 1000L)
+    }
+
+}

reposilite-backend/src/test/kotlin/com/reposilite/storage/S3StorageProviderTest.kt

@@ -0,0 +1,40 @@
+package com.reposilite.storage
+
+import com.reposilite.journalist.backend.InMemoryLogger
+import com.reposilite.storage.infrastructure.S3StorageProvider
+import org.junit.jupiter.api.BeforeEach
+import org.testcontainers.containers.localstack.LocalStackContainer
+import org.testcontainers.containers.localstack.LocalStackContainer.Service.S3
+import org.testcontainers.junit.jupiter.Container
+import org.testcontainers.junit.jupiter.Testcontainers
+import org.testcontainers.utility.DockerImageName
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider
+import software.amazon.awssdk.regions.Region
+import software.amazon.awssdk.services.s3.S3Client
+
+@Testcontainers
+internal class S3StorageProviderTest : StorageProviderTest() {
+
+    @Container
+    val localstack: LocalStackContainer = LocalStackContainer(DockerImageName.parse("localstack/localstack:0.12.17"))
+        .withServices(S3)
+
+    @BeforeEach
+    fun setup() {
+        val s3 = S3Client.builder()
+            .endpointOverride(localstack.getEndpointOverride(S3))
+            .credentialsProvider(
+                StaticCredentialsProvider.create(
+                    AwsBasicCredentials.create(
+                        localstack.accessKey, localstack.secretKey
+                    )
+                )
+            )
+            .region(Region.of(localstack.region))
+            .build()
+
+        this.storageProvider = S3StorageProvider(InMemoryLogger(), s3, "repositories")
+    }
+
+}

reposilite-backend/src/test/kotlin/com/reposilite/storage/StorageProviderTest.kt

@@ -0,0 +1,39 @@
+package com.reposilite.storage
+
+import com.reposilite.shared.toPath
+import org.junit.jupiter.api.Assertions.assertArrayEquals
+import org.junit.jupiter.api.Test
+import panda.std.ResultAssertions.assertError
+import panda.std.ResultAssertions.assertOk
+
+internal abstract class StorageProviderTest {
+
+    protected lateinit var storageProvider: StorageProvider
+
+    @Test
+    fun `should store and return valid resource` () {
+        // given: a destination path to the resource and its content
+        val path = "/directory/file.data".toPath()
+        val content = "content".toByteArray()
+
+        // when: non-existing resource is requested
+        val nonExistingResource = storageProvider.getFile(path)
+
+        // then: response should contain error
+        assertError(nonExistingResource)
+
+        // when: resource is put in storage and then requested
+        val putResponse = storageProvider.putFile(path, content)
+
+        // then: put request should succeed
+        assertOk(putResponse)
+
+        // when: stored resource is requested
+        val fetchResponse = storageProvider.getFile(path)
+
+        // then: provider should return proper resource
+        assertOk(fetchResponse)
+        assertArrayEquals(content, fetchResponse.get().readBytes())
+    }
+
+}