feat: add podcasts and channel sorting

build.gradle.kts

@@ -31,7 +31,7 @@ dependencies {
     implementation("io.ktor:ktor-server-call-logging-jvm")
     implementation("io.ktor:ktor-server-rate-limit-jvm")
     implementation("ch.qos.logback:logback-classic:1.5.32")
-    implementation("com.github.InfinityLoop1308.PipePipeExtractor:extractor:a69bcc15d146d391a695210a52cbde7b3fff1137")
+    implementation("com.github.InfinityLoop1308.PipePipeExtractor:extractor:3e06f36ec130ca452dcf107f223eedbc30a07dc2")
     implementation("com.squareup.okhttp3:okhttp:5.3.2")
     implementation("io.lettuce:lettuce-core:7.5.2.RELEASE")
     implementation("org.jetbrains.exposed:exposed-core:1.3.0")
@@ -74,6 +74,7 @@ tasks.test {
     useJUnitPlatform {
         excludeTags("network")
     }
+    jvmArgs("-XX:+EnableDynamicAgentLoading", "-Xshare:off")
     finalizedBy(tasks.jacocoTestReport)
 }
 

openapi.yaml

@@ -21,6 +21,10 @@ paths:
     $ref: ./openapi/paths/search.yaml#/Search
   /channel:
     $ref: ./openapi/paths/channel.yaml#/Channel
+  /podcasts:
+    $ref: ./openapi/paths/podcasts.yaml#/Podcasts
+  /podcasts/episodes:
+    $ref: ./openapi/paths/podcasts.yaml#/PodcastEpisodes
   /comments:
     $ref: ./openapi/paths/comments.yaml#/Comments
   /comments/replies:
@@ -51,6 +55,10 @@ components:
       $ref: ./openapi/components/media.yaml#/SearchPageResponse
     ChannelResponse:
       $ref: ./openapi/components/media.yaml#/ChannelResponse
+    PodcastPageResponse:
+      $ref: ./openapi/components/media.yaml#/PodcastPageResponse
+    PodcastEpisodesResponse:
+      $ref: ./openapi/components/media.yaml#/PodcastEpisodesResponse
     CommentsPageResponse:
       $ref: ./openapi/components/media.yaml#/CommentsPageResponse
     DownloadJob:

openapi/components/media.yaml

@@ -38,6 +38,33 @@ ChannelResponse:
     isVerified: { type: boolean }
     videos: { type: array, items: { $ref: '#/VideoItem' } }
     nextpage: { type: string, nullable: true }
+PodcastItem:
+  type: object
+  required: [id, title, url, thumbnailUrl, uploaderName, streamCount, playlistType]
+  properties:
+    id: { type: string }
+    title: { type: string }
+    url: { type: string }
+    thumbnailUrl: { type: string }
+    uploaderName: { type: string }
+    streamCount: { type: integer, format: int64 }
+    playlistType: { type: string }
+PodcastPageResponse:
+  type: object
+  required: [channelName, channelUrl, podcasts, episodes, nextpage]
+  properties:
+    channelName: { type: string }
+    channelUrl: { type: string }
+    podcasts: { type: array, items: { $ref: '#/PodcastItem' } }
+    episodes: { type: array, items: { $ref: '#/VideoItem' } }
+    nextpage: { type: string, nullable: true }
+PodcastEpisodesResponse:
+  type: object
+  required: [podcast, episodes, nextpage]
+  properties:
+    podcast: { $ref: '#/PodcastItem' }
+    episodes: { type: array, items: { $ref: '#/VideoItem' } }
+    nextpage: { type: string, nullable: true }
 CommentItem:
   type: object
   required: [id, text, author, authorUrl, authorAvatarUrl, likeCount, textualLikeCount, publishedTime, isHeartedByUploader, isPinned, uploaderVerified, replyCount, repliesPage]

openapi/paths/channel.yaml

@@ -11,6 +11,12 @@ Channel:
         in: query
         required: false
         schema: { type: string }
+      - name: sort
+        in: query
+        required: false
+        schema:
+          type: string
+          enum: [latest, popular, oldest]
     responses:
       '200':
         description: Channel metadata and video page.

openapi/paths/podcasts.yaml

@@ -0,0 +1,54 @@
+Podcasts:
+  get:
+    tags: [extraction]
+    summary: Get YouTube channel podcasts and episodes
+    parameters:
+      - name: url
+        in: query
+        required: true
+        schema: { type: string }
+      - name: nextpage
+        in: query
+        required: false
+        schema: { type: string }
+    responses:
+      '200':
+        description: YouTube podcasts tab page.
+        headers:
+          X-Request-ID:
+            $ref: ../components/common.yaml#/RequestIdHeader
+        content:
+          application/json:
+            schema:
+              $ref: ../components/media.yaml#/PodcastPageResponse
+      '400':
+        $ref: ../components/common.yaml#/JsonError
+      '422':
+        $ref: ../components/common.yaml#/JsonError
+PodcastEpisodes:
+  get:
+    tags: [extraction]
+    summary: Get YouTube podcast playlist episodes
+    parameters:
+      - name: url
+        in: query
+        required: true
+        schema: { type: string }
+      - name: nextpage
+        in: query
+        required: false
+        schema: { type: string }
+    responses:
+      '200':
+        description: YouTube podcast playlist episodes.
+        headers:
+          X-Request-ID:
+            $ref: ../components/common.yaml#/RequestIdHeader
+        content:
+          application/json:
+            schema:
+              $ref: ../components/media.yaml#/PodcastEpisodesResponse
+      '400':
+        $ref: ../components/common.yaml#/JsonError
+      '422':
+        $ref: ../components/common.yaml#/JsonError

src/main/kotlin/dev/typetype/server/Application.kt

@@ -11,6 +11,7 @@ import dev.typetype.server.routes.downloaderGatewayRoutes
 import dev.typetype.server.routes.internalObservabilityRoutes
 import dev.typetype.server.routes.manifestRoutes
 import dev.typetype.server.routes.nicoVideoProxyRoutes
+import dev.typetype.server.routes.podcastRoutes
 import dev.typetype.server.routes.proxyRoutes
 import dev.typetype.server.routes.storyboardProxyRoutes
 import dev.typetype.server.routes.searchRoutes
@@ -94,6 +95,7 @@ fun Application.module() {
         }
         rateLimit(CHANNEL_ZONE) {
             channelRoutes(svc.channelService)
+            podcastRoutes(svc.podcastService)
         }
         rateLimit(PROXY_ZONE) {
             proxyRoutes(svc.proxyService)

src/main/kotlin/dev/typetype/server/ExtractorLifecycle.kt

@@ -16,14 +16,14 @@ private val log = LoggerFactory.getLogger("ExtractorLifecycle")
 internal fun CoroutineScope.launchExtractorLifecycle() {
     launch(Dispatchers.IO) {
         runCatching { StreamInfo.getInfo(WARMUP_URL) }
-            .onFailure { log.warn("Warmup extraction failed: ${it.message}") }
+            .onFailure { log.info("Warmup extraction failed: ${it.message}") }
     }
 
     launch(Dispatchers.IO) {
         while (true) {
             delay(THROTTLE_CLEANUP_INTERVAL_MS)
             runCatching { YoutubeJavaScriptPlayerManager.clearThrottlingParametersCache() }
-                .onFailure { log.warn("Throttling cache cleanup failed: ${it.message}") }
+                .onFailure { log.info("Throttling cache cleanup failed: ${it.message}") }
         }
     }
 }

src/main/kotlin/dev/typetype/server/ServiceRegistry.kt

@@ -8,6 +8,7 @@ import dev.typetype.server.services.CachedChannelService
 import dev.typetype.server.services.CachedCommentService
 import dev.typetype.server.services.CachedManifestService
 import dev.typetype.server.services.CachedNativeManifestService
+import dev.typetype.server.services.CachedPodcastService
 import dev.typetype.server.services.CachedSearchService
 import dev.typetype.server.services.CachedStreamService
 import dev.typetype.server.services.CachedSuggestionService
@@ -25,6 +26,7 @@ import dev.typetype.server.services.OkHttpProxyService
 import dev.typetype.server.services.PipePipeBulletCommentService
 import dev.typetype.server.services.PipePipeChannelService
 import dev.typetype.server.services.PipePipeCommentService
+import dev.typetype.server.services.PipePipePodcastService
 import dev.typetype.server.services.PipePipeSearchService
 import dev.typetype.server.services.PipePipeStreamService
 import dev.typetype.server.services.PipePipeSuggestionService
@@ -58,6 +60,7 @@ internal class ServiceRegistry(cache: DragonflyService, subtitleServiceUrl: Stri
     val commentService = CachedCommentService(PipePipeCommentService(), cache)
     val bulletCommentService = PipePipeBulletCommentService()
     val channelService = CachedChannelService(PipePipeChannelService(), cache)
+    val podcastService = CachedPodcastService(PipePipePodcastService(), cache)
     val proxyService = OkHttpProxyService(proxyHttpClient)
     val nicoVideoProxyService = NicoVideoProxyService()
     val manifestService = CachedManifestService(ManifestService(streamService), cache)

src/main/kotlin/dev/typetype/server/models/PodcastEpisodesResponse.kt

@@ -0,0 +1,10 @@
+package dev.typetype.server.models
+
+import kotlinx.serialization.Serializable
+
+@Serializable
+data class PodcastEpisodesResponse(
+    val podcast: PodcastItem,
+    val episodes: List<VideoItem>,
+    val nextpage: String?,
+)

src/main/kotlin/dev/typetype/server/models/PodcastItem.kt

@@ -0,0 +1,14 @@
+package dev.typetype.server.models
+
+import kotlinx.serialization.Serializable
+
+@Serializable
+data class PodcastItem(
+    val id: String,
+    val title: String,
+    val url: String,
+    val thumbnailUrl: String,
+    val uploaderName: String,
+    val streamCount: Long,
+    val playlistType: String,
+)

src/main/kotlin/dev/typetype/server/models/PodcastPageResponse.kt

@@ -0,0 +1,12 @@
+package dev.typetype.server.models
+
+import kotlinx.serialization.Serializable
+
+@Serializable
+data class PodcastPageResponse(
+    val channelName: String,
+    val channelUrl: String,
+    val podcasts: List<PodcastItem>,
+    val episodes: List<VideoItem>,
+    val nextpage: String?,
+)

src/main/kotlin/dev/typetype/server/routes/AdminRoutes.kt

@@ -59,7 +59,7 @@ fun Route.adminRoutes(
         call.withAdminAuth(authService) { adminId ->
             val id = call.parameters["id"] ?: return@withAdminAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing id"))
             if (id == adminId) {
-                adminRouteLog.warn("Admin self-suspend blocked for userId={}", adminId)
+                adminRouteLog.info("Admin self-suspend blocked for userId={}", adminId)
                 return@withAdminAuth call.respond(HttpStatusCode.Forbidden, ErrorResponse("Cannot suspend your own account"))
             }
             val ok = userAdminService.suspendUser(id)
@@ -71,7 +71,7 @@ fun Route.adminRoutes(
         call.withAdminAuth(authService) { adminId ->
             val id = call.parameters["id"] ?: return@withAdminAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing id"))
             if (id == adminId) {
-                adminRouteLog.warn("Admin self-unsuspend blocked for userId={}", adminId)
+                adminRouteLog.info("Admin self-unsuspend blocked for userId={}", adminId)
                 return@withAdminAuth call.respond(HttpStatusCode.Forbidden, ErrorResponse("Cannot suspend your own account"))
             }
             val ok = userAdminService.unsuspendUser(id)
@@ -83,7 +83,7 @@ fun Route.adminRoutes(
         call.withAdminAuth(authService) { adminId ->
             val id = call.parameters["id"] ?: return@withAdminAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing id"))
             if (id == adminId) {
-                adminRouteLog.warn("Admin role self-change blocked for userId={}", adminId)
+                adminRouteLog.info("Admin role self-change blocked for userId={}", adminId)
                 return@withAdminAuth call.respond(HttpStatusCode.Forbidden, ErrorResponse("Cannot modify your own role"))
             }
             val body = runCatching { call.receive<RoleBody>() }.getOrElse {

src/main/kotlin/dev/typetype/server/routes/ChannelRoutes.kt

@@ -13,8 +13,9 @@ fun Route.channelRoutes(channelService: ChannelService) {
         val url = call.request.queryParameters["url"]
             ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing 'url' parameter"))
         val nextpage = call.request.queryParameters["nextpage"]
+        val sort = call.request.queryParameters["sort"]?.takeIf { it.isNotBlank() }
 
-        when (val result = channelService.getChannel(url = url, nextpage = nextpage)) {
+        when (val result = channelService.getChannel(url = url, nextpage = nextpage, sort = sort)) {
             is ExtractionResult.Success -> call.respond(result.data)
             is ExtractionResult.BadRequest -> call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message))
             is ExtractionResult.Failure -> call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message))

src/main/kotlin/dev/typetype/server/routes/PodcastRoutes.kt

@@ -0,0 +1,34 @@
+package dev.typetype.server.routes
+
+import dev.typetype.server.models.ErrorResponse
+import dev.typetype.server.models.ExtractionResult
+import dev.typetype.server.services.PodcastService
+import io.ktor.http.HttpStatusCode
+import io.ktor.server.response.respond
+import io.ktor.server.routing.Route
+import io.ktor.server.routing.get
+
+fun Route.podcastRoutes(podcastService: PodcastService) {
+    get("/podcasts") {
+        val url = call.request.queryParameters["url"]
+            ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing 'url' parameter"))
+        val nextpage = call.request.queryParameters["nextpage"]
+
+        when (val result = podcastService.getPodcasts(url = url, nextpage = nextpage)) {
+            is ExtractionResult.Success -> call.respond(result.data)
+            is ExtractionResult.BadRequest -> call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message))
+            is ExtractionResult.Failure -> call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message))
+        }
+    }
+    get("/podcasts/episodes") {
+        val url = call.request.queryParameters["url"]
+            ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing 'url' parameter"))
+        val nextpage = call.request.queryParameters["nextpage"]
+
+        when (val result = podcastService.getPodcastEpisodes(url = url, nextpage = nextpage)) {
+            is ExtractionResult.Success -> call.respond(result.data)
+            is ExtractionResult.BadRequest -> call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message))
+            is ExtractionResult.Failure -> call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message))
+        }
+    }
+}

src/main/kotlin/dev/typetype/server/routes/RestoreRoutes.kt

@@ -49,7 +49,7 @@ fun Route.restoreRoutes(restoreService: PipePipeBackupImporterService, authServi
                 val result = restoreService.restore(userId, tmp, timeMode)
                 call.respond(result)
             } catch (e: Exception) {
-                call.application.environment.log.warn("Restore backup failed", e)
+                call.application.environment.log.info("Restore backup failed", e)
                 return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid backup archive"))
             } finally {
                 Files.deleteIfExists(tmp)

src/main/kotlin/dev/typetype/server/services/CachedChannelService.kt

@@ -14,14 +14,14 @@ class CachedChannelService(
         private const val CHANNEL_CACHE_TTL_SECONDS = 1800L
     }
 
-    override suspend fun getChannel(url: String, nextpage: String?): ExtractionResult<ChannelResponse> {
-        val key = "channel:$url:${nextpage ?: "null"}"
+    override suspend fun getChannel(url: String, nextpage: String?, sort: String?): ExtractionResult<ChannelResponse> {
+        val key = "channel:$url:${nextpage ?: "null"}:${sort ?: "default"}"
         runCatching { cache.get(key) }.getOrNull()?.let { cached ->
             return runCatching { ExtractionResult.Success(CacheJson.decodeFromString<ChannelResponse>(cached)) }.getOrElse {
-                delegate.getChannel(url, nextpage)
+                delegate.getChannel(url, nextpage, sort)
             }
         }
-        val result = delegate.getChannel(url, nextpage)
+        val result = delegate.getChannel(url, nextpage, sort)
         if (result is ExtractionResult.Success) {
             runCatching { cache.set(key, CacheJson.encodeToString(ChannelResponse.serializer(), result.data), CHANNEL_CACHE_TTL_SECONDS) }
         }

src/main/kotlin/dev/typetype/server/services/CachedPodcastService.kt

@@ -0,0 +1,45 @@
+package dev.typetype.server.services
+
+import dev.typetype.server.cache.CacheJson
+import dev.typetype.server.cache.CacheService
+import dev.typetype.server.models.ExtractionResult
+import dev.typetype.server.models.PodcastEpisodesResponse
+import dev.typetype.server.models.PodcastPageResponse
+
+class CachedPodcastService(
+    private val delegate: PodcastService,
+    private val cache: CacheService,
+) : PodcastService {
+
+    companion object {
+        private const val PODCAST_CACHE_TTL_SECONDS = 1800L
+    }
+
+    override suspend fun getPodcasts(url: String, nextpage: String?): ExtractionResult<PodcastPageResponse> {
+        val key = "podcasts:$url:${nextpage ?: "null"}"
+        runCatching { cache.get(key) }.getOrNull()?.let { cached ->
+            return runCatching { ExtractionResult.Success(CacheJson.decodeFromString<PodcastPageResponse>(cached)) }.getOrElse {
+                delegate.getPodcasts(url, nextpage)
+            }
+        }
+        val result = delegate.getPodcasts(url, nextpage)
+        if (result is ExtractionResult.Success) {
+            runCatching { cache.set(key, CacheJson.encodeToString(PodcastPageResponse.serializer(), result.data), PODCAST_CACHE_TTL_SECONDS) }
+        }
+        return result
+    }
+
+    override suspend fun getPodcastEpisodes(url: String, nextpage: String?): ExtractionResult<PodcastEpisodesResponse> {
+        val key = "podcast-episodes:$url:${nextpage ?: "null"}"
+        runCatching { cache.get(key) }.getOrNull()?.let { cached ->
+            return runCatching { ExtractionResult.Success(CacheJson.decodeFromString<PodcastEpisodesResponse>(cached)) }.getOrElse {
+                delegate.getPodcastEpisodes(url, nextpage)
+            }
+        }
+        val result = delegate.getPodcastEpisodes(url, nextpage)
+        if (result is ExtractionResult.Success) {
+            runCatching { cache.set(key, CacheJson.encodeToString(PodcastEpisodesResponse.serializer(), result.data), PODCAST_CACHE_TTL_SECONDS) }
+        }
+        return result
+    }
+}