Add Redis cache and multi-level caching (#50)

.github/workflows/ci.yml

@@ -80,6 +80,7 @@ jobs:
         run: |
           gcloud run deploy ${{ vars.SERVICE_NAME }} \
             --image ${{ vars.CONTAINER_IMAGE }}:${{ env.SHA_SHORT }} \
+            --update-secrets=KS_REDIS_REST_URL=redis-rest-url:latest,KS_REDIS_REST_TOKEN=redis-rest-token:latest \
             --region ${{ secrets.GCP_REGION }} \
             --cpu ${{ vars.CONTAINER_CPU }} \
             --memory ${{ vars.CONTAINER_MEMORY }} \

build.gradle.kts

@@ -22,6 +22,7 @@ dependencyManagement {
     imports {
         mavenBom(libs.dgs.bom.get().toString())
     }
+    applyMavenExclusions(false)
 }
 
 graalvmNative {
@@ -40,6 +41,18 @@ graalvmNative {
     }
 }
 
+tasks.bootRun {
+    environment(
+        envVar("KS_REDIS_REST_URL"),
+        envVar("KS_REDIS_REST_TOKEN"),
+    )
+}
+
+fun envVar(name: String): Pair<String, String> {
+    return name to (providers.environmentVariable(name).orElse(providers.gradleProperty(name)).orNull
+        ?: error("Missing environment variable or Gradle property: $name"))
+}
+
 tasks.withType<GenerateJavaTask>().configureEach {
     packageName = "io.github.reactivecircus.kstreamlined.backend.schema.generated"
 }
@@ -51,9 +64,7 @@ kotlin {
     }
     compilerOptions {
         freeCompilerArgs.addAll(
-            "-opt-in=kotlin.RequiresOptIn",
             "-Xjsr305=strict",
-            "-Xcontext-receivers",
         )
     }
 }
@@ -89,8 +100,9 @@ dependencies {
     implementation(libs.spring.boot.starter)
     implementation(libs.dgs.starter)
     implementation(libs.ktor.client.core)
-    implementation(libs.ktor.client.cio)
+    implementation(libs.ktor.client.okhttp)
     implementation(libs.ktor.client.contentNegotiation)
+    implementation(libs.ktor.serialization.json)
     implementation(libs.ktor.serialization.xml)
     implementation(libs.apacheCommonsText)
     implementation(libs.apacheCommonsLang3)
@@ -100,7 +112,7 @@ dependencies {
     implementation(libs.jsoup)
     implementation(libs.xalan)
 
-    testImplementation(libs.spring.boot.starter.test)
     testImplementation(kotlin("test"))
+    testImplementation(libs.spring.boot.starter.test)
     testImplementation(libs.ktor.client.mock)
 }

gradle/libs.versions.toml

@@ -32,9 +32,10 @@ detektFormatting = { module = "io.gitlab.arturbosch.detekt:detekt-formatting", v
 spring-boot-starter = { module = "org.springframework.boot:spring-boot-starter-webflux" }
 spring-boot-starter-test = { module = "org.springframework.boot:spring-boot-starter-test" }
 ktor-client-core = { module = "io.ktor:ktor-client-core", version.ref = "ktor" }
-ktor-client-cio = { module = "io.ktor:ktor-client-cio", version.ref = "ktor" }
+ktor-client-okhttp = { module = "io.ktor:ktor-client-okhttp", version.ref = "ktor" }
 ktor-client-contentNegotiation = { module = "io.ktor:ktor-client-content-negotiation", version.ref = "ktor" }
 ktor-client-mock = { module = "io.ktor:ktor-client-mock", version.ref = "ktor" }
+ktor-serialization-json = { module = "io.ktor:ktor-serialization-kotlinx-json", version.ref = "ktor" }
 ktor-serialization-xml = { module = "io.ktor:ktor-serialization-kotlinx-xml", version.ref = "ktor" }
 dgs-bom = { module = "com.netflix.graphql.dgs:graphql-dgs-platform-dependencies", version.ref = "dgsBom" }
 dgs-starter = { module = "com.netflix.graphql.dgs:graphql-dgs-spring-graphql-starter"}

src/main/kotlin/io/github/reactivecircus/kstreamlined/backend/KSConfiguration.kt

@@ -1,56 +1,79 @@
 package io.github.reactivecircus.kstreamlined.backend
 
-import io.github.reactivecircus.kstreamlined.backend.client.ClientConfigs
-import io.github.reactivecircus.kstreamlined.backend.client.FeedClient
-import io.github.reactivecircus.kstreamlined.backend.client.KotlinWeeklyIssueClient
-import io.github.reactivecircus.kstreamlined.backend.client.RealFeedClient
-import io.github.reactivecircus.kstreamlined.backend.client.RealKotlinWeeklyIssueClient
+import io.github.reactivecircus.kstreamlined.backend.datasource.DataLoader
+import io.github.reactivecircus.kstreamlined.backend.datasource.FeedDataSource
+import io.github.reactivecircus.kstreamlined.backend.datasource.FeedDataSourceConfig
+import io.github.reactivecircus.kstreamlined.backend.datasource.KotlinWeeklyIssueDataSource
+import io.github.reactivecircus.kstreamlined.backend.datasource.RealFeedDataSource
+import io.github.reactivecircus.kstreamlined.backend.datasource.RealKotlinWeeklyIssueDataSource
+import io.github.reactivecircus.kstreamlined.backend.redis.RedisClient
 import io.ktor.client.engine.HttpClientEngine
-import io.ktor.client.engine.cio.CIO
+import io.ktor.client.engine.okhttp.OkHttp
 import org.springframework.beans.factory.annotation.Value
 import org.springframework.context.annotation.Bean
 import org.springframework.context.annotation.Configuration
+import kotlin.time.Duration.Companion.hours
+import kotlin.time.Duration.Companion.minutes
 
 @Configuration
 class KSConfiguration {
 
     @Bean
-    fun feedClient(
+    fun feedDataSource(
         engine: HttpClientEngine,
-        clientConfigs: ClientConfigs,
-    ): FeedClient {
-        return RealFeedClient(
+        dataSourceConfig: FeedDataSourceConfig,
+        redisClient: RedisClient,
+    ): FeedDataSource {
+        return RealFeedDataSource(
             engine = engine,
-            clientConfigs = clientConfigs,
+            dataSourceConfig = dataSourceConfig,
+            cacheConfig = DataLoader.CacheConfig(
+                localExpiry = 10.minutes,
+                remoteExpiry = 1.hours,
+            ),
+            redisClient = redisClient,
         )
     }
 
     @Bean
-    fun kotlinWeeklyIssueClient(
+    fun feedDataSourceConfig(
+        @Value("\${ks.kotlin-blog-feed-url}") kotlinBlogFeedUrl: String,
+        @Value("\${ks.kotlin-youtube-feed-url}") kotlinYouTubeFeedUrl: String,
+        @Value("\${ks.talking-kotlin-feed-url}") talkingKotlinFeedUrl: String,
+        @Value("\${ks.kotlin-weekly-feed-url}") kotlinWeeklyFeedUrl: String,
+    ): FeedDataSourceConfig {
+        return FeedDataSourceConfig(
+            kotlinBlogFeedUrl = kotlinBlogFeedUrl,
+            kotlinYouTubeFeedUrl = kotlinYouTubeFeedUrl,
+            talkingKotlinFeedUrl = talkingKotlinFeedUrl,
+            kotlinWeeklyFeedUrl = kotlinWeeklyFeedUrl,
+        )
+    }
+
+    @Bean
+    fun kotlinWeeklyIssueDataSource(
         engine: HttpClientEngine
-    ): KotlinWeeklyIssueClient {
-        return RealKotlinWeeklyIssueClient(
+    ): KotlinWeeklyIssueDataSource {
+        return RealKotlinWeeklyIssueDataSource(
             engine = engine,
         )
     }
 
     @Bean
     fun httpClientEngine(): HttpClientEngine {
-        return CIO.create()
+        return OkHttp.create()
     }
 
     @Bean
-    fun clientConfigs(
-        @Value("\${ks.kotlin-blog-feed-url}") kotlinBlogFeedUrl: String,
-        @Value("\${ks.kotlin-youtube-feed-url}") kotlinYouTubeFeedUrl: String,
-        @Value("\${ks.talking-kotlin-feed-url}") talkingKotlinFeedUrl: String,
-        @Value("\${ks.kotlin-weekly-feed-url}") kotlinWeeklyFeedUrl: String,
-    ): ClientConfigs {
-        return ClientConfigs(
-            kotlinBlogFeedUrl = kotlinBlogFeedUrl,
-            kotlinYouTubeFeedUrl = kotlinYouTubeFeedUrl,
-            talkingKotlinFeedUrl = talkingKotlinFeedUrl,
-            kotlinWeeklyFeedUrl = kotlinWeeklyFeedUrl,
+    fun redisClient(
+        engine: HttpClientEngine,
+        @Value("\${KS_REDIS_REST_URL}") redisUrl: String,
+        @Value("\${KS_REDIS_REST_TOKEN}") redisToken: String,
+    ): RedisClient {
+        return RedisClient(
+            engine = engine,
+            url = redisUrl,
+            token = redisToken,
         )
     }
 }