From 78662da8bbd766fa496b500fb099b97ee152d2a0 Mon Sep 17 00:00:00 2001 From: Yang Date: Sat, 24 Aug 2024 20:26:52 +1000 Subject: [PATCH] `syncFeeds` mutation. (#52) --- .../datafetcher/FeedEntryDataFetcher.kt | 27 +++++++++++ .../backend/datasource/DataLoader.kt | 7 +++ .../backend/datasource/FeedDataSource.kt | 24 +++++----- .../META-INF/native-image/reflect-config.json | 46 ++++++++++++------- .../native-image/resource-config.json | 4 +- .../resources/schema/kstreamlined.graphqls | 5 ++ .../datafetcher/FeedEntryDataFetcherTest.kt | 46 +++++++++++++++++++ .../backend/datasource/DataLoaderTest.kt | 28 +++++++++++ .../backend/datasource/FakeFeedDataSource.kt | 8 ++-- 9 files changed, 160 insertions(+), 35 deletions(-) diff --git a/src/main/kotlin/io/github/reactivecircus/kstreamlined/backend/datafetcher/FeedEntryDataFetcher.kt b/src/main/kotlin/io/github/reactivecircus/kstreamlined/backend/datafetcher/FeedEntryDataFetcher.kt index bac8e89..8f89ff5 100644 --- a/src/main/kotlin/io/github/reactivecircus/kstreamlined/backend/datafetcher/FeedEntryDataFetcher.kt +++ b/src/main/kotlin/io/github/reactivecircus/kstreamlined/backend/datafetcher/FeedEntryDataFetcher.kt @@ -1,6 +1,7 @@ package io.github.reactivecircus.kstreamlined.backend.datafetcher import com.netflix.graphql.dgs.DgsComponent +import com.netflix.graphql.dgs.DgsMutation import com.netflix.graphql.dgs.DgsQuery import com.netflix.graphql.dgs.DgsTypeResolver import com.netflix.graphql.dgs.InputArgument @@ -60,6 +61,32 @@ class FeedEntryDataFetcher( } } + @DgsMutation(field = DgsConstants.MUTATION.SyncFeeds) + suspend fun syncFeeds(): Boolean = coroutineScope { + FeedSourceKey.entries.map { source -> + async(coroutineDispatcher) { + when (source) { + FeedSourceKey.KOTLIN_BLOG -> { + dataSource.loadKotlinBlogFeed(skipCache = true) + } + + FeedSourceKey.KOTLIN_YOUTUBE_CHANNEL -> { + dataSource.loadKotlinYouTubeFeed(skipCache = true) + } + + FeedSourceKey.TALKING_KOTLIN_PODCAST -> { + dataSource.loadTalkingKotlinFeed(skipCache = true) + } + + FeedSourceKey.KOTLIN_WEEKLY -> { + dataSource.loadKotlinWeeklyFeed(skipCache = true) + } + } + } + }.awaitAll() + true + } + @DgsTypeResolver(name = DgsConstants.FEEDENTRY.TYPE_NAME) internal fun resolveFeedEntry(feedEntry: FeedEntry): String { return when (feedEntry) { diff --git a/src/main/kotlin/io/github/reactivecircus/kstreamlined/backend/datasource/DataLoader.kt b/src/main/kotlin/io/github/reactivecircus/kstreamlined/backend/datasource/DataLoader.kt index 2b928d7..7f739c9 100644 --- a/src/main/kotlin/io/github/reactivecircus/kstreamlined/backend/datasource/DataLoader.kt +++ b/src/main/kotlin/io/github/reactivecircus/kstreamlined/backend/datasource/DataLoader.kt @@ -23,8 +23,11 @@ class DataLoader private constructor( @Suppress("ReturnCount") suspend fun load( key: String, + sotOnly: Boolean = false, sot: suspend () -> List ): List { + if (sotOnly) return loadFromSot(key, sot) + // L1 cache - local val l1Value = localCache.getIfPresent(key) if (l1Value != null) { @@ -40,6 +43,10 @@ class DataLoader private constructor( } // Source of truth + return loadFromSot(key, sot) + } + + private suspend fun loadFromSot(key: String, sot: suspend () -> List): List { val sotValue = sot() localCache.put(key, sotValue) redisClient.set( diff --git a/src/main/kotlin/io/github/reactivecircus/kstreamlined/backend/datasource/FeedDataSource.kt b/src/main/kotlin/io/github/reactivecircus/kstreamlined/backend/datasource/FeedDataSource.kt index 62e606e..6d4c983 100644 --- a/src/main/kotlin/io/github/reactivecircus/kstreamlined/backend/datasource/FeedDataSource.kt +++ b/src/main/kotlin/io/github/reactivecircus/kstreamlined/backend/datasource/FeedDataSource.kt @@ -26,10 +26,10 @@ import nl.adaptivity.xmlutil.serialization.XmlConfig import org.apache.commons.text.StringEscapeUtils interface FeedDataSource { - suspend fun loadKotlinBlogFeed(): List - suspend fun loadKotlinYouTubeFeed(): List - suspend fun loadTalkingKotlinFeed(): List - suspend fun loadKotlinWeeklyFeed(): List + suspend fun loadKotlinBlogFeed(skipCache: Boolean = false): List + suspend fun loadKotlinYouTubeFeed(skipCache: Boolean = false): List + suspend fun loadTalkingKotlinFeed(skipCache: Boolean = false): List + suspend fun loadKotlinWeeklyFeed(skipCache: Boolean = false): List } class FeedDataSourceConfig( @@ -73,8 +73,8 @@ class RealFeedDataSource( } } - override suspend fun loadKotlinBlogFeed(): List { - return kotlinBlogFeedDataLoader.load("kotlin-blog") { + override suspend fun loadKotlinBlogFeed(skipCache: Boolean): List { + return kotlinBlogFeedDataLoader.load("kotlin-blog", sotOnly = skipCache) { httpClient.get(dataSourceConfig.kotlinBlogFeedUrl).body().channel.items.map { it.copy( description = StringEscapeUtils.unescapeXml(it.description).trim() @@ -83,16 +83,16 @@ class RealFeedDataSource( } } - override suspend fun loadKotlinYouTubeFeed(): List { - return kotlinYouTubeFeedDataLoader.load("kotlin-youtube") { + override suspend fun loadKotlinYouTubeFeed(skipCache: Boolean): List { + return kotlinYouTubeFeedDataLoader.load("kotlin-youtube", sotOnly = skipCache) { httpClient.get(dataSourceConfig.kotlinYouTubeFeedUrl).bodyAsText().let { DefaultXml.decodeFromString(it.replace("&(?!.{2,4};)".toRegex(), "&")).entries } } } - override suspend fun loadTalkingKotlinFeed(): List { - return talkingKotlinFeedDataLoader.load("talking-kotlin") { + override suspend fun loadTalkingKotlinFeed(skipCache: Boolean): List { + return talkingKotlinFeedDataLoader.load("talking-kotlin", sotOnly = skipCache) { httpClient.get(dataSourceConfig.talkingKotlinFeedUrl).body().channel.items .take(TalkingKotlinFeedSize) .map { @@ -103,8 +103,8 @@ class RealFeedDataSource( } } - override suspend fun loadKotlinWeeklyFeed(): List { - return kotlinWeeklyFeedDataLoader.load("kotlin-weekly") { + override suspend fun loadKotlinWeeklyFeed(skipCache: Boolean): List { + return kotlinWeeklyFeedDataLoader.load("kotlin-weekly", sotOnly = skipCache) { httpClient.get(dataSourceConfig.kotlinWeeklyFeedUrl).body().channel.items } } diff --git a/src/main/resources/META-INF/native-image/reflect-config.json b/src/main/resources/META-INF/native-image/reflect-config.json index a193d0d..0fe8e95 100644 --- a/src/main/resources/META-INF/native-image/reflect-config.json +++ b/src/main/resources/META-INF/native-image/reflect-config.json @@ -296,6 +296,10 @@ { "name":"com.netflix.graphql.dgs.DgsFederationResolver" }, +{ + "name":"com.netflix.graphql.dgs.DgsMutation", + "queryAllDeclaredMethods":true +}, { "name":"com.netflix.graphql.dgs.DgsQuery", "queryAllDeclaredMethods":true @@ -850,7 +854,7 @@ "allDeclaredFields":true, "queryAllDeclaredMethods":true, "queryAllDeclaredConstructors":true, - "methods":[{"name":"","parameterTypes":["io.github.reactivecircus.kstreamlined.backend.datasource.FeedDataSource"] }, {"name":"feedEntries","parameterTypes":["java.util.List","kotlin.coroutines.Continuation"] }, {"name":"resolveFeedEntry$kstreamlined_backend","parameterTypes":["io.github.reactivecircus.kstreamlined.backend.schema.generated.types.FeedEntry"] }] + "methods":[{"name":"","parameterTypes":["io.github.reactivecircus.kstreamlined.backend.datasource.FeedDataSource"] }, {"name":"feedEntries","parameterTypes":["java.util.List","kotlin.coroutines.Continuation"] }, {"name":"resolveFeedEntry$kstreamlined_backend","parameterTypes":["io.github.reactivecircus.kstreamlined.backend.schema.generated.types.FeedEntry"] }, {"name":"syncFeeds","parameterTypes":["kotlin.coroutines.Continuation"] }] }, { "name":"io.github.reactivecircus.kstreamlined.backend.datafetcher.FeedSourceDataFetcher", @@ -893,7 +897,7 @@ "name":"io.github.reactivecircus.kstreamlined.backend.datasource.RealFeedDataSource", "allDeclaredFields":true, "queryAllDeclaredMethods":true, - "methods":[{"name":"close","parameterTypes":[] }, {"name":"loadKotlinBlogFeed","parameterTypes":["kotlin.coroutines.Continuation"] }, {"name":"loadKotlinWeeklyFeed","parameterTypes":["kotlin.coroutines.Continuation"] }, {"name":"loadKotlinYouTubeFeed","parameterTypes":["kotlin.coroutines.Continuation"] }, {"name":"loadTalkingKotlinFeed","parameterTypes":["kotlin.coroutines.Continuation"] }, {"name":"shutdown","parameterTypes":[] }] + "methods":[{"name":"close","parameterTypes":[] }, {"name":"loadKotlinBlogFeed","parameterTypes":["kotlin.coroutines.Continuation"] }, {"name":"loadKotlinBlogFeed","parameterTypes":["boolean","kotlin.coroutines.Continuation"] }, {"name":"loadKotlinWeeklyFeed","parameterTypes":["kotlin.coroutines.Continuation"] }, {"name":"loadKotlinWeeklyFeed","parameterTypes":["boolean","kotlin.coroutines.Continuation"] }, {"name":"loadKotlinYouTubeFeed","parameterTypes":["kotlin.coroutines.Continuation"] }, {"name":"loadKotlinYouTubeFeed","parameterTypes":["boolean","kotlin.coroutines.Continuation"] }, {"name":"loadTalkingKotlinFeed","parameterTypes":["kotlin.coroutines.Continuation"] }, {"name":"loadTalkingKotlinFeed","parameterTypes":["boolean","kotlin.coroutines.Continuation"] }, {"name":"shutdown","parameterTypes":[] }] }, { "name":"io.github.reactivecircus.kstreamlined.backend.datasource.RealKotlinWeeklyIssueDataSource", @@ -936,36 +940,36 @@ }, { "name":"io.github.reactivecircus.kstreamlined.backend.schema.generated.types.FeedSource", - "allDeclaredFields":true, - "allDeclaredMethods":true + "allDeclaredFields":true, + "allDeclaredMethods":true }, { "name":"io.github.reactivecircus.kstreamlined.backend.schema.generated.types.FeedSourceKey" }, { "name":"io.github.reactivecircus.kstreamlined.backend.schema.generated.types.KotlinBlog", - "allDeclaredFields":true, - "allDeclaredMethods":true + "allDeclaredFields":true, + "allDeclaredMethods":true }, { "name":"io.github.reactivecircus.kstreamlined.backend.schema.generated.types.KotlinWeekly", - "allDeclaredFields":true, - "allDeclaredMethods":true + "allDeclaredFields":true, + "allDeclaredMethods":true }, { "name":"io.github.reactivecircus.kstreamlined.backend.schema.generated.types.KotlinWeeklyIssueEntry", - "allDeclaredFields":true, - "allDeclaredMethods":true + "allDeclaredFields":true, + "allDeclaredMethods":true }, { "name":"io.github.reactivecircus.kstreamlined.backend.schema.generated.types.KotlinYouTube", - "allDeclaredFields":true, - "allDeclaredMethods":true + "allDeclaredFields":true, + "allDeclaredMethods":true }, { "name":"io.github.reactivecircus.kstreamlined.backend.schema.generated.types.TalkingKotlin", - "allDeclaredFields":true, - "allDeclaredMethods":true + "allDeclaredFields":true, + "allDeclaredMethods":true }, { "name":"io.ktor.client.HttpClient", @@ -1298,7 +1302,7 @@ }, { "name":"java.io.Console", - "methods":[{"name":"isTerminal","parameterTypes":[] }] + "methods":[{"name":"charset","parameterTypes":[] }, {"name":"isTerminal","parameterTypes":[] }] }, { "name":"java.io.FileDescriptor" @@ -1323,7 +1327,7 @@ { "name":"java.lang.Class", "queryAllDeclaredMethods":true, - "methods":[{"name":"getPermittedSubclasses","parameterTypes":[] }, {"name":"getRecordComponents","parameterTypes":[] }, {"name":"isRecord","parameterTypes":[] }, {"name":"isSealed","parameterTypes":[] }] + "methods":[{"name":"accessFlags","parameterTypes":[] }, {"name":"getPermittedSubclasses","parameterTypes":[] }, {"name":"getRecordComponents","parameterTypes":[] }, {"name":"isRecord","parameterTypes":[] }, {"name":"isSealed","parameterTypes":[] }] }, { "name":"java.lang.ClassLoader", @@ -1472,7 +1476,12 @@ "name":"java.security.interfaces.RSAPublicKey" }, { - "name":"java.time.Duration" + "name":"java.text.NumberFormat", + "methods":[{"name":"isStrict","parameterTypes":[] }] +}, +{ + "name":"java.time.Duration", + "methods":[{"name":"isPositive","parameterTypes":[] }] }, { "name":"java.time.DurationEditor" @@ -1563,6 +1572,9 @@ { "name":"javax.inject.Named" }, +{ + "name":"javax.inject.Qualifier" +}, { "name":"javax.money.MonetaryAmount" }, diff --git a/src/main/resources/META-INF/native-image/resource-config.json b/src/main/resources/META-INF/native-image/resource-config.json index 74acb41..a591403 100644 --- a/src/main/resources/META-INF/native-image/resource-config.json +++ b/src/main/resources/META-INF/native-image/resource-config.json @@ -1,8 +1,6 @@ { "resources":{ "includes":[{ - "pattern": "schema/.*\\.graphqls$"} - ,{ "pattern":"\\QMETA-INF/build-info.properties\\E" }, { "pattern":"\\QMETA-INF/resources/index.html\\E" @@ -474,6 +472,8 @@ "pattern":"jdk.internal.le:\\Qjdk/internal/org/jline/utils/capabilities.txt\\E" }, { "pattern":"jdk.internal.le:\\Qjdk/internal/org/jline/utils/xterm-256color.caps\\E" + }, { + "pattern":"schema/.*\\.graphqls$" }]}, "bundles":[{ "name":"i18n.Parsing", diff --git a/src/main/resources/schema/kstreamlined.graphqls b/src/main/resources/schema/kstreamlined.graphqls index 9d5e9a6..ae54b7e 100644 --- a/src/main/resources/schema/kstreamlined.graphqls +++ b/src/main/resources/schema/kstreamlined.graphqls @@ -7,6 +7,11 @@ type Query { kotlinWeeklyIssue(url: String!): [KotlinWeeklyIssueEntry!]! } +type Mutation { + "Syncs feeds from all sources." + syncFeeds: Boolean! +} + type FeedSource { "Unique identifier of the feed source." key: FeedSourceKey! diff --git a/src/test/kotlin/io/github/reactivecircus/kstreamlined/backend/datafetcher/FeedEntryDataFetcherTest.kt b/src/test/kotlin/io/github/reactivecircus/kstreamlined/backend/datafetcher/FeedEntryDataFetcherTest.kt index 2d23603..62ef3a6 100644 --- a/src/test/kotlin/io/github/reactivecircus/kstreamlined/backend/datafetcher/FeedEntryDataFetcherTest.kt +++ b/src/test/kotlin/io/github/reactivecircus/kstreamlined/backend/datafetcher/FeedEntryDataFetcherTest.kt @@ -61,6 +61,12 @@ class FeedEntryDataFetcherTest { } """.trimIndent() + private val syncFeedsMutation = """ + mutation SyncFeeds { + syncFeeds + } + """.trimIndent() + @Test fun `feedEntries() query returns expected feed entries ordered by publish time when operation succeeds`() { (feedDataSource as FakeFeedDataSource).nextKotlinBlogFeedResponse = { @@ -173,5 +179,45 @@ class FeedEntryDataFetcherTest { assert(context.read("data.feedEntries[1].description") == dummyKotlinYouTubeEntry.description) } + @Test + fun `syncFeeds mutation returns true when operation succeeds`() { + (feedDataSource as FakeFeedDataSource).nextKotlinBlogFeedResponse = { + listOf(DummyKotlinBlogItem) + } + (feedDataSource as FakeFeedDataSource).nextKotlinYouTubeFeedResponse = { + listOf(DummyKotlinYouTubeItem) + } + (feedDataSource as FakeFeedDataSource).nextTalkingKotlinFeedResponse = { + listOf(DummyTalkingKotlinItem) + } + (feedDataSource as FakeFeedDataSource).nextKotlinWeeklyFeedResponse = { + listOf(DummyKotlinWeeklyItem) + } + + val context = dgsQueryExecutor.executeAndGetDocumentContext(syncFeedsMutation) + + assert(context.read("data.syncFeeds") == true) + } + + @Test + fun `syncFeeds mutation returns error response when failed to load data from any feed sources`() { + (feedDataSource as FakeFeedDataSource).nextKotlinBlogFeedResponse = { + throw GraphqlErrorException.newErrorException().build() + } + (feedDataSource as FakeFeedDataSource).nextKotlinYouTubeFeedResponse = { + listOf(DummyKotlinYouTubeItem) + } + (feedDataSource as FakeFeedDataSource).nextTalkingKotlinFeedResponse = { + listOf(DummyTalkingKotlinItem) + } + (feedDataSource as FakeFeedDataSource).nextKotlinWeeklyFeedResponse = { + listOf(DummyKotlinWeeklyItem) + } + + val result = dgsQueryExecutor.execute(syncFeedsMutation) + + assert(result.errors[0].extensions["errorType"] == "INTERNAL") + } + private fun String.toInstant(): Instant = Instant.parse(this) } diff --git a/src/test/kotlin/io/github/reactivecircus/kstreamlined/backend/datasource/DataLoaderTest.kt b/src/test/kotlin/io/github/reactivecircus/kstreamlined/backend/datasource/DataLoaderTest.kt index 04d5449..cd3c567 100644 --- a/src/test/kotlin/io/github/reactivecircus/kstreamlined/backend/datasource/DataLoaderTest.kt +++ b/src/test/kotlin/io/github/reactivecircus/kstreamlined/backend/datasource/DataLoaderTest.kt @@ -129,6 +129,34 @@ class DataLoaderTest { assert(redisMockEngine.responseHistory.last().statusCode == HttpStatusCode.OK) } + @Test + fun `skips both local and remote caches when loading with sotOnly = true`() = runBlocking { + localCache.put("key", listOf(1, 2)) + + val redisMockEngine = MockEngine { + if (it.url.encodedPath.contains("get")) { + respond(content = "{ \"result\": \"[1, 2, 3]\" }") + } else { + respond(content = "{ \"result\": \"OK\" }") + } + } + val dataLoader = createDataLoader( + localCache = localCache, + remoteCacheExpiry = 1.hours, + redisMockEngine = redisMockEngine, + ) + + dataLoader.load("key", sotOnly = true) { + listOf(1, 2, 3, 4) + } + + assert(localCache.getIfPresent("key") == listOf(1, 2, 3, 4)) + assert(redisMockEngine.requestHistory.last().url.pathSegments.last() == "key") + assert(redisMockEngine.requestHistory.last().url.encodedQuery == "EX=${1.hours.inWholeSeconds}") + assert(redisMockEngine.requestHistory.last().body.toString() == "TextContent[application/json] \"[1,2,3,4]\"") + assert(redisMockEngine.responseHistory.last().statusCode == HttpStatusCode.OK) + } + private fun createDataLoader( localCache: Cache>, remoteCacheExpiry: Duration = 1.days, diff --git a/src/test/kotlin/io/github/reactivecircus/kstreamlined/backend/datasource/FakeFeedDataSource.kt b/src/test/kotlin/io/github/reactivecircus/kstreamlined/backend/datasource/FakeFeedDataSource.kt index 141b5d5..25896e2 100644 --- a/src/test/kotlin/io/github/reactivecircus/kstreamlined/backend/datasource/FakeFeedDataSource.kt +++ b/src/test/kotlin/io/github/reactivecircus/kstreamlined/backend/datasource/FakeFeedDataSource.kt @@ -27,19 +27,19 @@ object FakeFeedDataSource : FeedDataSource { listOf(DummyKotlinWeeklyItem) } - override suspend fun loadKotlinBlogFeed(): List { + override suspend fun loadKotlinBlogFeed(skipCache: Boolean): List { return nextKotlinBlogFeedResponse() } - override suspend fun loadKotlinYouTubeFeed(): List { + override suspend fun loadKotlinYouTubeFeed(skipCache: Boolean): List { return nextKotlinYouTubeFeedResponse() } - override suspend fun loadTalkingKotlinFeed(): List { + override suspend fun loadTalkingKotlinFeed(skipCache: Boolean): List { return nextTalkingKotlinFeedResponse() } - override suspend fun loadKotlinWeeklyFeed(): List { + override suspend fun loadKotlinWeeklyFeed(skipCache: Boolean): List { return nextKotlinWeeklyFeedResponse() } }