Kotlin Blog summarization

.github/workflows/ci.yml

@@ -87,7 +87,7 @@ jobs:
         run: |
           gcloud run deploy ${{ vars.SERVICE_NAME }} \
             --image ${{ vars.CONTAINER_IMAGE }}:${{ env.SHA_SHORT }} \
-            --update-secrets=KS_REDIS_REST_URL=redis-rest-url:latest,KS_REDIS_REST_TOKEN=redis-rest-token:latest,KS_GCLOUD_PROJECT_ID=gcloud-project-id:latest \
+            --update-secrets=KS_REDIS_REST_URL=redis-rest-url:latest,KS_REDIS_REST_TOKEN=redis-rest-token:latest,KS_GCLOUD_PROJECT_ID=gcloud-project-id:latest,KS_CF_BASE_URL=cf-base-url:latest,KS_CF_ACCOUNT_ID=cf-account-id:latest,KS_CF_API_TOKEN=cf-api-token:latest \
             --region ${{ secrets.GCP_REGION }} \
             --cpu ${{ vars.CONTAINER_CPU }} \
             --memory ${{ vars.CONTAINER_MEMORY }} \

README.md

@@ -26,6 +26,9 @@ Setup the required environment variables:
 KS_REDIS_REST_URL
 KS_REDIS_REST_TOKEN
 KS_GCLOUD_PROJECT_ID
+KS_CF_BASE_URL
+KS_CF_ACCOUNT_ID
+KS_CF_API_TOKEN
 ```
 
 Run `gcloud auth application-default login` to authenticate Firestore.

build.gradle.kts

@@ -64,6 +64,9 @@ tasks.bootRun {
         envVar("KS_REDIS_REST_URL"),
         envVar("KS_REDIS_REST_TOKEN"),
         envVar("KS_GCLOUD_PROJECT_ID"),
+        envVar("KS_CF_BASE_URL"),
+        envVar("KS_CF_ACCOUNT_ID"),
+        envVar("KS_CF_API_TOKEN"),
     )
 }
 
@@ -147,6 +150,7 @@ dependencies {
     implementation(libs.gcloud.firestore)
     implementation(libs.caffeine)
     implementation(libs.scrapeit)
+    implementation(libs.ksoup)
 
     testImplementation(kotlin("test"))
     testImplementation(libs.spring.boot.starter.test)

detekt.yml

@@ -4,6 +4,8 @@ complexity:
   LongParameterList:
     active: false
   TooManyFunctions:
+    allowedFunctionsPerClass: 20
+    allowedFunctionsPerFile: 20
     excludes: ["**/test/**"]
 
 ktlint:

gradle/libs.versions.toml

@@ -9,9 +9,10 @@ kotlinxCoroutines = "1.11.0"
 kotlinxSerializationBom = "1.11.0"
 gcloud-firestore = "3.47.0"
 detekt = "2.0.0-alpha.6"
-graalvmNative = "1.1.8"
+graalvmNative = "1.1.13"
 caffeine = "3.2.4"
 scrapeit = "1.3.0-alpha.2"
+ksoup = "0.2.6"
 toolchainsResolver = "1.0.0"
 
 [plugins]
@@ -43,3 +44,4 @@ kotlinx-coroutines-bom = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-bo
 kotlinx-serialization-bom = { module = "org.jetbrains.kotlinx:kotlinx-serialization-bom", version.ref = "kotlinxSerializationBom" }
 caffeine = { module = "com.github.ben-manes.caffeine:caffeine", version.ref = "caffeine" }
 scrapeit = { module = "it.skrape:skrapeit", version.ref = "scrapeit" }
+ksoup = { module = "com.fleeksoft.ksoup:ksoup", version.ref = "ksoup" }

src/main/kotlin/io/github/reactivecircus/kstreamlined/backend/KSConfiguration.kt

@@ -3,15 +3,21 @@ package io.github.reactivecircus.kstreamlined.backend
 import com.google.auth.oauth2.GoogleCredentials
 import com.google.cloud.firestore.Firestore
 import com.google.cloud.firestore.FirestoreOptions
+import io.github.reactivecircus.kstreamlined.backend.cloudflare.CloudflareAiClient
 import io.github.reactivecircus.kstreamlined.backend.datasource.DataLoader
 import io.github.reactivecircus.kstreamlined.backend.datasource.FeedDataSource
 import io.github.reactivecircus.kstreamlined.backend.datasource.FeedDataSourceConfig
-import io.github.reactivecircus.kstreamlined.backend.datasource.FeedPersister
-import io.github.reactivecircus.kstreamlined.backend.datasource.FirestoreFeedPersister
+import io.github.reactivecircus.kstreamlined.backend.datasource.KotlinBlogTldrDataSource
 import io.github.reactivecircus.kstreamlined.backend.datasource.KotlinWeeklyIssueDataSource
 import io.github.reactivecircus.kstreamlined.backend.datasource.RealFeedDataSource
+import io.github.reactivecircus.kstreamlined.backend.datasource.RealKotlinBlogTldrDataSource
 import io.github.reactivecircus.kstreamlined.backend.datasource.RealKotlinWeeklyIssueDataSource
+import io.github.reactivecircus.kstreamlined.backend.datasource.persister.FeedPersister
+import io.github.reactivecircus.kstreamlined.backend.datasource.persister.FirestoreFeedPersister
+import io.github.reactivecircus.kstreamlined.backend.datasource.persister.FirestoreKotlinBlogContentPersister
+import io.github.reactivecircus.kstreamlined.backend.datasource.persister.KotlinBlogContentPersister
 import io.github.reactivecircus.kstreamlined.backend.redis.RedisClient
+import io.github.reactivecircus.kstreamlined.backend.tldr.TldrGenerator
 import io.ktor.client.engine.HttpClientEngine
 import io.ktor.client.engine.okhttp.OkHttp
 import org.springframework.beans.factory.annotation.Value
@@ -28,6 +34,7 @@ class KSConfiguration {
         dataSourceConfig: FeedDataSourceConfig,
         redisClient: RedisClient,
         feedPersister: FeedPersister,
+        kotlinBlogContentPersister: KotlinBlogContentPersister,
     ): FeedDataSource {
         return RealFeedDataSource(
             engine = engine,
@@ -38,6 +45,7 @@ class KSConfiguration {
             ),
             redisClient = redisClient,
             feedPersister = feedPersister,
+            kotlinBlogContentPersister = kotlinBlogContentPersister,
         )
     }
 
@@ -63,6 +71,24 @@ class KSConfiguration {
         return FirestoreFeedPersister(firestore = firestore)
     }
 
+    @Bean
+    fun kotlinBlogContentPersister(
+        firestore: Firestore,
+    ): KotlinBlogContentPersister {
+        return FirestoreKotlinBlogContentPersister(firestore = firestore)
+    }
+
+    @Bean
+    fun kotlinBlogTldrDataSource(
+        kotlinBlogContentPersister: KotlinBlogContentPersister,
+        tldrGenerator: TldrGenerator,
+    ): KotlinBlogTldrDataSource {
+        return RealKotlinBlogTldrDataSource(
+            kotlinBlogContentPersister = kotlinBlogContentPersister,
+            tldrGenerator = tldrGenerator,
+        )
+    }
+
     @Bean
     fun kotlinWeeklyIssueDataSource(
         engine: HttpClientEngine
@@ -72,6 +98,13 @@ class KSConfiguration {
         )
     }
 
+    @Bean
+    fun tldrGenerator(
+        cloudflareAiClient: CloudflareAiClient,
+    ): TldrGenerator {
+        return TldrGenerator(cloudflareAiClient = cloudflareAiClient)
+    }
+
     @Bean
     fun httpClientEngine(): HttpClientEngine {
         return OkHttp.create()
@@ -90,6 +123,21 @@ class KSConfiguration {
         )
     }
 
+    @Bean
+    fun cloudflareAiClient(
+        engine: HttpClientEngine,
+        @Value("\${KS_CF_BASE_URL}") baseUrl: String,
+        @Value("\${KS_CF_ACCOUNT_ID}") accountId: String,
+        @Value("\${KS_CF_API_TOKEN}") apiToken: String,
+    ): CloudflareAiClient {
+        return CloudflareAiClient(
+            engine = engine,
+            baseUrl = baseUrl,
+            accountId = accountId,
+            apiToken = apiToken,
+        )
+    }
+
     @Bean
     fun firestore(
         @Value("\${KS_GCLOUD_PROJECT_ID}") projectId: String,

src/main/kotlin/io/github/reactivecircus/kstreamlined/backend/cloudflare/CloudflareAiClient.kt

@@ -0,0 +1,156 @@
+package io.github.reactivecircus.kstreamlined.backend.cloudflare
+
+import io.ktor.client.HttpClient
+import io.ktor.client.call.body
+import io.ktor.client.engine.HttpClientEngine
+import io.ktor.client.plugins.HttpTimeout
+import io.ktor.client.plugins.contentnegotiation.ContentNegotiation
+import io.ktor.client.request.bearerAuth
+import io.ktor.client.request.post
+import io.ktor.client.request.setBody
+import io.ktor.http.ContentType
+import io.ktor.http.contentType
+import io.ktor.serialization.kotlinx.json.json
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.json.Json
+
+class CloudflareAiClient(
+    engine: HttpClientEngine,
+    private val baseUrl: String,
+    private val accountId: String,
+    private val apiToken: String,
+) {
+    private val httpClient = HttpClient(engine) {
+        expectSuccess = true
+        install(ContentNegotiation) {
+            json(CloudflareAiJson)
+        }
+        install(HttpTimeout) {
+            requestTimeoutMillis = 30_000L
+            socketTimeoutMillis = 30_000L
+        }
+    }
+
+    suspend fun run(
+        model: String,
+        request: CloudflareAiRequest,
+    ): CloudflareAiResult {
+        val response = httpClient.post("$baseUrl/accounts/$accountId/ai/run/$model") {
+            bearerAuth(apiToken)
+            contentType(ContentType.Application.Json)
+            setBody(request)
+        }.body<CloudflareAiResponse>()
+
+        if (!response.success || response.errors.isNotEmpty()) {
+            val errorCodes = response.errors
+                .map(CloudflareAiError::code)
+                .distinct()
+                .joinToString()
+            val errorSuffix = errorCodes.takeIf(String::isNotEmpty)?.let { " Error codes: $it." }.orEmpty()
+            throw CloudflareAiException("Cloudflare Workers AI rejected the request.$errorSuffix")
+        }
+        return response.result
+            ?: throw CloudflareAiException("Cloudflare Workers AI response did not contain a result.")
+    }
+}
+
+@Serializable
+data class CloudflareAiRequest(
+    val messages: List<Message>,
+    val temperature: Double,
+    @SerialName("top_p")
+    val topP: Double,
+    val seed: Long,
+    @SerialName("max_tokens")
+    val maxTokens: Int,
+    @SerialName("reasoning_effort")
+    val reasoningEffort: ReasoningEffort? = null,
+) {
+    @Serializable
+    data class Message(
+        val role: Role,
+        val content: String,
+    ) {
+        @Serializable
+        enum class Role {
+            @SerialName("system")
+            System,
+
+            @SerialName("user")
+            User,
+
+            @SerialName("assistant")
+            Assistant,
+        }
+    }
+
+    @Serializable
+    enum class ReasoningEffort {
+        @SerialName("low")
+        Low,
+
+        @SerialName("medium")
+        Medium,
+
+        @SerialName("high")
+        High,
+    }
+}
+
+@Serializable
+private data class CloudflareAiResponse(
+    val result: CloudflareAiResult?,
+    val success: Boolean,
+    val errors: List<CloudflareAiError>,
+)
+
+@Serializable
+private data class CloudflareAiError(
+    val code: Int,
+    val message: String,
+)
+
+@Serializable
+data class CloudflareAiResult(
+    val id: String,
+    @SerialName("object")
+    val objectType: String,
+    val created: Long,
+    val model: String,
+    val choices: List<Choice>,
+    val usage: Usage? = null,
+) {
+    @Serializable
+    data class Choice(
+        val index: Int,
+        val message: Message,
+        @SerialName("finish_reason")
+        val finishReason: String,
+    ) {
+        @Serializable
+        data class Message(
+            val role: String,
+            val content: String?,
+        )
+    }
+
+    @Serializable
+    data class Usage(
+        @SerialName("prompt_tokens")
+        val promptTokens: Int,
+        @SerialName("completion_tokens")
+        val completionTokens: Int,
+        @SerialName("total_tokens")
+        val totalTokens: Int,
+        val neurons: Double? = null,
+    )
+}
+
+class CloudflareAiException(
+    message: String,
+) : RuntimeException(message)
+
+private val CloudflareAiJson = Json {
+    ignoreUnknownKeys = true
+}

src/main/kotlin/io/github/reactivecircus/kstreamlined/backend/datafetcher/FeedEntryDataFetcher.kt

@@ -22,7 +22,6 @@ import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.async
 import kotlinx.coroutines.awaitAll
 import kotlinx.coroutines.coroutineScope
-import kotlin.time.measureTimedValue
 
 @DgsComponent
 class FeedEntryDataFetcher(
@@ -32,37 +31,34 @@ class FeedEntryDataFetcher(
 
     @DgsQuery(field = DgsConstants.QUERY.FeedEntries)
     suspend fun feedEntries(@InputArgument filters: List<FeedSourceKey>?): List<FeedEntry> = coroutineScope {
-        val (value, time) = measureTimedValue {
-            FeedSourceKey.entries.filter {
-                filters == null || filters.contains(it)
-            }.map { source ->
-                async(coroutineDispatcher) {
-                    when (source) {
-                        FeedSourceKey.KOTLIN_BLOG -> {
-                            dataSource.loadKotlinBlogFeed().map { it.toKotlinBlogEntry() }
-                        }
+        FeedSourceKey.entries.filter {
+            filters == null || filters.contains(it)
+        }.map { source ->
+            async(coroutineDispatcher) {
+                when (source) {
+                    FeedSourceKey.KOTLIN_BLOG -> {
+                        dataSource.loadKotlinBlogFeed().map { it.toKotlinBlogEntry() }
+                    }
 
-                        FeedSourceKey.KOTLIN_YOUTUBE_CHANNEL -> {
-                            dataSource.loadKotlinYouTubeFeed().map { it.toKotlinYouTubeEntry() }
-                        }
+                    FeedSourceKey.KOTLIN_YOUTUBE_CHANNEL -> {
+                        dataSource.loadKotlinYouTubeFeed().map { it.toKotlinYouTubeEntry() }
+                    }
 
-                        FeedSourceKey.TALKING_KOTLIN_PODCAST -> {
-                            dataSource.loadTalkingKotlinFeed().map { it.toTalkingKotlinEntry() }
-                        }
+                    FeedSourceKey.TALKING_KOTLIN_PODCAST -> {
+                        dataSource.loadTalkingKotlinFeed().map { it.toTalkingKotlinEntry() }
+                    }
 
-                        FeedSourceKey.KOTLIN_WEEKLY -> {
-                            dataSource.loadKotlinWeeklyFeed().map { it.toKotlinWeeklyEntry() }
-                        }
+                    FeedSourceKey.KOTLIN_WEEKLY -> {
+                        dataSource.loadKotlinWeeklyFeed().map { it.toKotlinWeeklyEntry() }
                     }
                 }
             }
-                .awaitAll()
-                .flatten()
-                .sortedByDescending {
-                    it.publishTime
-                }
         }
-        value
+            .awaitAll()
+            .flatten()
+            .sortedByDescending {
+                it.publishTime
+            }
     }
 
     @DgsMutation(field = DgsConstants.MUTATION.SyncFeeds)