Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions src/main/kotlin/dev/typetype/downloader/models/JobOptions.kt
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,14 @@ data class JobOptions(
val mode: DownloadMode = DownloadMode.VIDEO,
val quality: String = "best",
val format: String = "",
val videoItag: String = "",
val audioItag: String = "",
val height: Int? = null,
val fps: Int? = null,
val videoCodec: String = "",
val audioCodec: String = "",
val bitrate: Int? = null,
val allowQualityFallback: Boolean = false,
val sponsorBlock: Boolean = false,
val sponsorBlockCategories: List<String> = emptyList(),
val thumbnailOnly: Boolean = false,
Expand Down
7 changes: 7 additions & 0 deletions src/main/kotlin/dev/typetype/downloader/models/JobResponse.kt
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,13 @@ data class JobResponse(
val durationMs: Long,
val title: String,
val error: String? = null,
val errorCode: String? = null,
val artifactUrl: String? = null,
val artifactExpiresAt: String? = null,
val resolved: ResolvedOutput? = null,
val progressPercent: Int? = null,
val downloadedBytes: Long? = null,
val totalBytes: Long? = null,
val etaSeconds: Long? = null,
val stage: String? = null,
)
16 changes: 16 additions & 0 deletions src/main/kotlin/dev/typetype/downloader/models/ResolvedOutput.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package dev.typetype.downloader.models

import kotlinx.serialization.Serializable

@Serializable
data class ResolvedOutput(
val videoItag: String? = null,
val audioItag: String? = null,
val height: Int? = null,
val fps: Int? = null,
val videoCodec: String? = null,
val audioCodec: String? = null,
val container: String? = null,
val bitrate: Int? = null,
val fileName: String? = null,
)
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,12 @@ class GarageStorageService(config: AppConfig) {
s3.putObject(request, RequestBody.fromFile(filePath))
}

fun presignGet(objectKey: String, duration: Duration): String {
val getRequest = GetObjectRequest.builder().bucket(bucket).key(objectKey).build()
fun presignGet(objectKey: String, duration: Duration, fileName: String? = null): String {
val requestBuilder = GetObjectRequest.builder().bucket(bucket).key(objectKey)
if (!fileName.isNullOrBlank()) {
requestBuilder.responseContentDisposition("attachment; filename=\"$fileName\"")
}
val getRequest = requestBuilder.build()
val presignRequest = GetObjectPresignRequest.builder().signatureDuration(duration).getObjectRequest(getRequest).build()
return presigner.presignGetObject(presignRequest).url().toString()
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,27 +24,54 @@ object JobOptionsNormalizer {
private val allowedSponsorBlockCategories = defaultSponsorBlockCategories.toSet()

fun normalize(options: JobOptions): JobOptions {
val quality = normalizeQuality(options)
val format = normalizeFormat(options)
val quality = normalizeQuality(options)
val videoItag = normalizeItag(options.videoItag)
val audioItag = normalizeItag(options.audioItag)
val height = normalizePositive(options.height, max = 4320)
val fps = normalizePositive(options.fps, max = 240)
val bitrate = normalizePositive(options.bitrate, max = 500_000)
val videoCodec = normalizeCodec(options.videoCodec)
val audioCodec = normalizeCodec(options.audioCodec)
val subtitles = normalizeSubtitles(options.subtitles)
val sponsorBlockCategories = normalizeSponsorBlockCategories(options)
if (options.thumbnailOnly) {
return options.copy(
quality = "best",
format = "",
videoItag = "",
audioItag = "",
height = null,
fps = null,
videoCodec = "",
audioCodec = "",
bitrate = null,
subtitles = SubtitlesOptions(),
sponsorBlockCategories = emptyList(),
)
}
return options.copy(
quality = quality,
format = format,
videoItag = videoItag,
audioItag = audioItag,
height = height,
fps = fps,
videoCodec = videoCodec,
audioCodec = audioCodec,
bitrate = bitrate,
subtitles = subtitles,
sponsorBlockCategories = sponsorBlockCategories,
)
}

private fun normalizeQuality(options: JobOptions): String {
if (options.thumbnailOnly) return "best"
val raw = options.quality.trim().lowercase().ifBlank { "best" }
val allowed = if (options.mode == DownloadMode.AUDIO) allowedAudioQualities else allowedVideoQualities
return if (raw in allowed) raw else "best"
}

private fun normalizeFormat(options: JobOptions): String {
if (options.thumbnailOnly) return ""
val raw = options.format.trim().lowercase()
if (options.mode == DownloadMode.AUDIO) {
if (raw.isBlank()) return "mp3"
Expand All @@ -54,6 +81,23 @@ object JobOptionsNormalizer {
return if (raw in allowedVideoFormats) raw else "mp4"
}

private fun normalizeItag(raw: String): String {
val value = raw.trim()
return if (value.all { it.isDigit() }) value else ""
}

private fun normalizeCodec(raw: String): String {
val value = raw.trim().lowercase()
if (value.isBlank()) return ""
return if (value.all { it.isLetterOrDigit() || it == '.' || it == '_' || it == '-' }) value else ""
}

private fun normalizePositive(value: Int?, max: Int): Int? {
val current = value ?: return null
if (current <= 0) return null
return if (current > max) max else current
}

private fun normalizeSubtitles(input: SubtitlesOptions): SubtitlesOptions {
if (!input.enabled) return SubtitlesOptions()
val langs = input.languages.map { it.trim() }.filter { it.isNotBlank() }.ifEmpty { listOf("en") }
Expand All @@ -62,7 +106,7 @@ object JobOptionsNormalizer {
}

private fun normalizeSponsorBlockCategories(options: JobOptions): List<String> {
if (!options.sponsorBlock) return emptyList()
if (!options.sponsorBlock || options.thumbnailOnly) return emptyList()
val custom = options.sponsorBlockCategories
.map { it.trim().lowercase() }
.filter { it in allowedSponsorBlockCategories }
Expand Down
25 changes: 1 addition & 24 deletions src/main/kotlin/dev/typetype/downloader/services/JobService.kt
Original file line number Diff line number Diff line change
@@ -1,16 +1,13 @@
package dev.typetype.downloader.services

import dev.typetype.downloader.config.AppConfig
import dev.typetype.downloader.db.JobRow
import dev.typetype.downloader.db.JobsRepository
import dev.typetype.downloader.models.CreateJobResponse
import dev.typetype.downloader.models.JobOptions
import dev.typetype.downloader.models.JobResponse
import dev.typetype.downloader.models.JobStatus
import redis.clients.jedis.JedisPooled
import java.net.URI
import java.time.Duration
import java.time.Instant
import java.util.UUID

enum class CancelJobResult { NOT_FOUND, NOT_CANCELLABLE, CANCELLED }
Expand Down Expand Up @@ -49,7 +46,7 @@ class JobService(

fun get(id: String): JobResponse? {
val row = jobsRepository.getById(id) ?: return null
return row.toResponse(presignUrl(row), row.artifactExpiresAt?.toString())
return JobViewBuilder.build(row, storageService)
}

fun cancel(id: String): CancelJobResult {
Expand Down Expand Up @@ -97,24 +94,4 @@ class JobService(
}

private fun redisJobKey(id: String): String = "downloader:job:$id"

private fun presignUrl(row: JobRow): String? {
val key = row.artifactKey ?: return null
val expiresAt = row.artifactExpiresAt ?: return null
val now = Instant.now()
if (expiresAt <= now) return null
val seconds = Duration.between(now, expiresAt).seconds.coerceIn(1, 900)
return storageService.presignGet(key, Duration.ofSeconds(seconds))
}

private fun JobRow.toResponse(artifactUrl: String?, artifactExpiresAt: String?): JobResponse = JobResponse(
id = id,
url = url,
status = status,
durationMs = durationMs,
title = title,
error = error,
artifactUrl = artifactUrl,
artifactExpiresAt = artifactExpiresAt,
)
}
93 changes: 93 additions & 0 deletions src/main/kotlin/dev/typetype/downloader/services/JobViewBuilder.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
package dev.typetype.downloader.services

import dev.typetype.downloader.db.JobRow
import dev.typetype.downloader.models.JobOptions
import dev.typetype.downloader.models.JobResponse
import dev.typetype.downloader.models.JobStatus
import dev.typetype.downloader.models.ResolvedOutput
import java.time.Duration
import java.time.Instant

object JobViewBuilder {
fun build(row: JobRow, storageService: GarageStorageService): JobResponse {
val options = runCatching { JobOptionsCodec.decode(row.optionsJson) }.map(JobOptionsNormalizer::normalize).getOrElse { JobOptions() }
val fileName = stableFileName(row, options)
val artifactUrl = presignUrl(storageService, row, fileName)
return JobResponse(
id = row.id,
url = row.url,
status = row.status,
durationMs = row.durationMs,
title = row.title,
error = row.error,
errorCode = errorCode(row.error),
artifactUrl = artifactUrl,
artifactExpiresAt = row.artifactExpiresAt?.toString(),
resolved = resolved(row, options, fileName),
progressPercent = progressPercent(row.status),
downloadedBytes = null,
totalBytes = null,
etaSeconds = null,
stage = stage(row.status),
)
}

private fun presignUrl(storageService: GarageStorageService, row: JobRow, fileName: String?): String? {
val key = row.artifactKey ?: return null
val expiresAt = row.artifactExpiresAt ?: return null
val now = Instant.now()
if (expiresAt <= now) return null
val seconds = Duration.between(now, expiresAt).seconds.coerceIn(1, 900)
return storageService.presignGet(key, Duration.ofSeconds(seconds), fileName)
}

private fun resolved(row: JobRow, options: JobOptions, fileName: String?): ResolvedOutput = ResolvedOutput(
videoItag = options.videoItag.ifBlank { null },
audioItag = options.audioItag.ifBlank { null },
height = options.height ?: qualityHeight(options.quality),
fps = options.fps,
videoCodec = options.videoCodec.ifBlank { null },
audioCodec = options.audioCodec.ifBlank { null },
container = row.artifactKey?.substringAfterLast('.', "").orEmpty().ifBlank { null },
bitrate = options.bitrate,
fileName = fileName,
)

private fun stableFileName(row: JobRow, options: JobOptions): String? {
val extension = row.artifactKey?.substringAfterLast('.', "")?.ifBlank { null } ?: return null
val base = row.title.takeIf { it.isNotBlank() } ?: row.id
val clean = base.map { if (it.isLetterOrDigit() || it == '-' || it == '_') it else '_' }
.joinToString("")
.replace("__", "_")
.trim('_')
.ifBlank { row.id }
val name = if (options.thumbnailOnly) "${clean}_thumbnail" else clean
return "$name.$extension"
}

private fun qualityHeight(quality: String): Int? = when (quality.lowercase()) {
"1080p" -> 1080
"720p" -> 720
"480p" -> 480
else -> null
}

private fun stage(status: JobStatus): String = when (status) {
JobStatus.QUEUED -> "queued"
JobStatus.RUNNING -> "downloading"
JobStatus.DONE -> "finalizing"
JobStatus.FAILED -> "failed"
}

private fun progressPercent(status: JobStatus): Int? = when (status) {
JobStatus.QUEUED -> 0
JobStatus.RUNNING -> 50
JobStatus.DONE -> 100
JobStatus.FAILED -> null
}

private fun errorCode(error: String?): String? {
val value = error?.lowercase() ?: return null
return if ("exact selection unavailable" in value) "exact_selection_unavailable" else null
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,52 @@ object YtDlpOptionResolver {
private val audioFormats = setOf("mp3", "m4a", "aac", "opus", "flac", "wav")
private val videoFormats = setOf("mp4", "webm", "mkv", "mov")

fun audioSelector(quality: String): String = if (quality.lowercase() == "worst") "worstaudio/worst" else "bestaudio/best"
fun audioSelector(options: JobOptions): String {
val audioItag = options.audioItag
if (audioItag.isNotBlank()) {
val strict = "ba[format_id=$audioItag]/b[format_id=$audioItag]"
return if (options.allowQualityFallback) "$strict/bestaudio/best" else strict
}
return if (options.quality.lowercase() == "worst") "worstaudio/worst" else "bestaudio/best"
}

fun videoSelector(options: JobOptions): String {
if (options.videoItag.isNotBlank() || options.audioItag.isNotBlank() || hasTupleSelection(options)) {
val exact = exactVideoSelector(options)
if (!options.allowQualityFallback) return exact
return "$exact/${qualityVideoSelector(options.quality, allowFallback = true)}"
}
return qualityVideoSelector(options.quality, options.allowQualityFallback)
}

private fun hasTupleSelection(options: JobOptions): Boolean {
return options.height != null || options.fps != null || options.videoCodec.isNotBlank() ||
options.audioCodec.isNotBlank() || options.bitrate != null
}

private fun exactVideoSelector(options: JobOptions): String {
val videoFilters = mutableListOf<String>()
val audioFilters = mutableListOf<String>()
options.videoItag.takeIf { it.isNotBlank() }?.let { videoFilters += "[format_id=$it]" }
options.audioItag.takeIf { it.isNotBlank() }?.let { audioFilters += "[format_id=$it]" }
options.height?.let { videoFilters += "[height=$it]" }
options.fps?.let { videoFilters += "[fps=$it]" }
options.videoCodec.takeIf { it.isNotBlank() }?.let { videoFilters += "[vcodec^=$it]" }
options.audioCodec.takeIf { it.isNotBlank() }?.let { audioFilters += "[acodec^=$it]" }
options.bitrate?.let { videoFilters += "[tbr>=$it][tbr<=${it + 300}]" }
val video = "bv*${videoFilters.joinToString("")}"
val audio = "ba${audioFilters.joinToString("")}"
return if (options.videoItag.isNotBlank() && options.audioItag.isBlank()) {
"$video+$audio/b[format_id=${options.videoItag}]"
} else {
"$video+$audio"
}
}

fun videoSelector(quality: String): String = when (quality.lowercase()) {
"1080p" -> "bv*[height<=1080]+ba/b[height<=1080]"
"720p" -> "bv*[height<=720]+ba/b[height<=720]"
"480p" -> "bv*[height<=480]+ba/b[height<=480]"
private fun qualityVideoSelector(quality: String, allowFallback: Boolean): String = when (quality.lowercase()) {
"1080p" -> if (allowFallback) "bv*[height<=1080]+ba/b[height<=1080]" else "bv*[height=1080]+ba/b[height=1080]"
"720p" -> if (allowFallback) "bv*[height<=720]+ba/b[height<=720]" else "bv*[height=720]+ba/b[height=720]"
"480p" -> if (allowFallback) "bv*[height<=480]+ba/b[height<=480]" else "bv*[height=480]+ba/b[height=480]"
"worst" -> "worst"
else -> "bv*+ba/b"
}
Expand Down
10 changes: 7 additions & 3 deletions src/main/kotlin/dev/typetype/downloader/services/YtDlpService.kt
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,11 @@ class YtDlpService(private val config: AppConfig) {
val output = process.inputStream.bufferedReader().readLines()
if (process.exitValue() != 0) {
deleteDirectory(workDir)
val error = output.lastOrNull { it.isNotBlank() } ?: "yt-dlp failed"
val error = if (output.any { it.contains("Requested format is not available", ignoreCase = true) }) {
"exact selection unavailable: requested format is not available"
} else {
output.lastOrNull { it.isNotBlank() } ?: "yt-dlp failed"
}
return YtDlpResult(title = "", filePath = null, error = error)
}
val filePath = selectOutputFile(workDir, options)
Expand All @@ -61,12 +65,12 @@ class YtDlpService(private val config: AppConfig) {
when {
options.thumbnailOnly -> command.addAll(listOf("--skip-download", "--write-thumbnail"))
options.mode == DownloadMode.AUDIO -> {
val selector = YtDlpOptionResolver.audioSelector(options.quality)
val selector = YtDlpOptionResolver.audioSelector(options)
val audioFormat = YtDlpOptionResolver.audioFormat(options.format)
command.addAll(listOf("-f", selector, "--extract-audio", "--audio-format", audioFormat))
}
else -> {
val selector = YtDlpOptionResolver.videoSelector(options.quality)
val selector = YtDlpOptionResolver.videoSelector(options)
val videoFormat = YtDlpOptionResolver.videoFormat(options.format)
command.addAll(listOf("-f", selector, "--merge-output-format", videoFormat))
}
Expand Down
Loading