diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2eeb0b00..413eb721 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: build: runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || fromJSON('["self-hosted","Linux","X64","arko"]') }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Isolate Gradle user home run: echo "GRADLE_USER_HOME=${RUNNER_TEMP}/gradle-user-home" >> "$GITHUB_ENV" diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index bc6db65a..7b38c40d 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -15,7 +15,7 @@ jobs: coverage: runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || fromJSON('["self-hosted","Linux","X64","arko"]') }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Isolate Gradle user home run: echo "GRADLE_USER_HOME=${RUNNER_TEMP}/gradle-user-home" >> "$GITHUB_ENV" diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 70306f40..07c4ac15 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -33,7 +33,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Resolve build metadata id: build-info @@ -92,7 +92,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Isolate Docker credentials run: echo "DOCKER_CONFIG=$RUNNER_TEMP/docker-config" >> "$GITHUB_ENV" @@ -106,7 +106,7 @@ jobs: - name: Set up QEMU if: matrix.arch == 'arm64' - uses: docker/setup-qemu-action@v4.1.0 + uses: docker/setup-qemu-action@v4.2.0 with: platforms: arm64 cache-image: false diff --git a/.github/workflows/openapi.yml b/.github/workflows/openapi.yml index 890fbddb..4abbc6c3 100644 --- a/.github/workflows/openapi.yml +++ b/.github/workflows/openapi.yml @@ -17,7 +17,7 @@ jobs: validate: runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || fromJSON('["self-hosted","Linux","X64","arko"]') }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Isolate Gradle user home run: echo "GRADLE_USER_HOME=${RUNNER_TEMP}/gradle-user-home" >> "$GITHUB_ENV" - uses: actions/setup-java@v5 diff --git a/Dockerfile b/Dockerfile index 02193ceb..db8e4f19 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,9 @@ COPY src/ ./src/ RUN ./gradlew shadowJar --no-daemon -q -PappVersion="$APP_VERSION" FROM eclipse-temurin:25-jre-alpine AS runner -RUN addgroup -S typetype && adduser -S typetype -G typetype +RUN apk upgrade --no-cache \ + && addgroup -S typetype \ + && adduser -S typetype -G typetype WORKDIR /app COPY --from=builder /app/build/libs/typetype-server-all.jar app.jar USER typetype diff --git a/build.gradle.kts b/build.gradle.kts index 997fee8a..ae7b3e35 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,8 +1,8 @@ import java.time.Instant plugins { - kotlin("jvm") version "2.4.0" - kotlin("plugin.serialization") version "2.4.0" + kotlin("jvm") version "2.4.10" + kotlin("plugin.serialization") version "2.4.10" id("io.ktor.plugin") version "3.5.1" id("jacoco") } @@ -25,6 +25,8 @@ repositories { } dependencies { + implementation(platform("com.fasterxml.jackson:jackson-bom:2.22.1")) + implementation(platform("io.netty:netty-bom:4.2.16.Final")) implementation("io.ktor:ktor-server-core-jvm") implementation("io.ktor:ktor-server-netty-jvm") implementation("io.ktor:ktor-server-content-negotiation-jvm") @@ -35,20 +37,20 @@ dependencies { implementation("io.ktor:ktor-server-status-pages-jvm") implementation("io.ktor:ktor-server-call-logging-jvm") implementation("io.ktor:ktor-server-rate-limit-jvm") - implementation("ch.qos.logback:logback-classic:1.5.38") + implementation("ch.qos.logback:logback-classic:1.6.1") implementation("com.github.Priveetee.PipePipeExtractor:extractor:21ab9b6ac415d23ef88df19c5ad126c7952bb543") compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") - implementation("org.json:json:20260522") + implementation("org.json:json:20260719") implementation("com.squareup.okhttp3:okhttp:5.4.0") implementation("io.lettuce:lettuce-core:7.6.0.RELEASE") implementation("org.jetbrains.exposed:exposed-core:1.3.1") implementation("org.jetbrains.exposed:exposed-jdbc:1.3.1") implementation("com.zaxxer:HikariCP:7.1.0") implementation("org.postgresql:postgresql:42.7.13") - implementation("org.xerial:sqlite-jdbc:3.53.2.0") + implementation("org.xerial:sqlite-jdbc:3.53.2.1") implementation("com.password4j:password4j:1.8.4") - implementation("com.auth0:java-jwt:4.5.2") - testImplementation("org.junit.jupiter:junit-jupiter:6.1.1") + implementation("com.auth0:java-jwt:4.6.0") + testImplementation("org.junit.jupiter:junit-jupiter:6.1.2") testRuntimeOnly("org.junit.platform:junit-platform-launcher") testImplementation("io.mockk:mockk:1.14.11") testImplementation("io.ktor:ktor-server-test-host-jvm") @@ -112,7 +114,7 @@ tasks.test { } jacoco { - toolVersion = "0.8.14" + toolVersion = "0.8.15" } tasks.jacocoTestReport { diff --git a/docker-compose.arm64.yml b/docker-compose.arm64.yml index ba02be94..541b2ecb 100644 --- a/docker-compose.arm64.yml +++ b/docker-compose.arm64.yml @@ -1,4 +1,4 @@ services: dragonfly: - image: docker.dragonflydb.io/dragonflydb/dragonfly:latest + image: docker.dragonflydb.io/dragonflydb/dragonfly:v1.39.0 platform: linux/arm64 diff --git a/docker-compose.dev-mirror.yml b/docker-compose.dev-mirror.yml index ab2a4493..5071bcc7 100644 --- a/docker-compose.dev-mirror.yml +++ b/docker-compose.dev-mirror.yml @@ -104,7 +104,7 @@ services: fi restart: "no" dragonfly: - image: docker.dragonflydb.io/dragonflydb/dragonfly:latest + image: docker.dragonflydb.io/dragonflydb/dragonfly:v1.39.0 ulimits: { memlock: -1 } restart: unless-stopped garage: diff --git a/docker-compose.yml b/docker-compose.yml index 64e500da..12ef9261 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ services: dragonfly: - image: docker.dragonflydb.io/dragonflydb/dragonfly:latest + image: docker.dragonflydb.io/dragonflydb/dragonfly:v1.39.0 ports: - "6379:6379" ulimits: diff --git a/docs/android-playback.md b/docs/android-playback.md deleted file mode 100644 index 4ae8252a..00000000 --- a/docs/android-playback.md +++ /dev/null @@ -1,43 +0,0 @@ -# Android playback contract - -TypeType-Android discovers this contract through `androidPlayback` in -`GET /api/instance`. Contract version 4 supports completed YouTube VODs only; -active livestreams remain explicitly unsupported. - -Android playback sessions use `/api/android/youtube/playback/*` and are isolated -from the web player's `/api/sabr/playback/*` sessions, generations, caches, and -window protocol. A ready VOD manifest is a complete static DASH presentation -from time zero. The server requires exact audio and video segment indexes, but -continues to fetch media bytes on demand. - -Creating a session may return `202` while one shared background task obtains the -exact initialization indexes. The response includes `preparationStage` and -`retryAfterMs`; manifest polling only reads that task's current state and never -starts duplicate network work. Preparation has an eight-second server deadline. -It ends with a complete static MPD, `422 android_playback_invalid_index`, or a -typed `503` (`android_playback_preparation_timeout` or -`android_playback_preparation_failed`). A client should retry only the same -manifest URL after `retryAfterMs` and create a new session after a terminal -error. - -The index task fetches only the selected formats' bounded initialization ranges. -It does not preload media. The SABR pump starts later when Media3 requests media -segments, so segment delivery remains demand-driven. - -A seek keeps the session ID and selected itags stable, increments the generation, -and makes older media URLs return `409`. Unknown sessions return `404`; recently -expired sessions return `410`. Expiration cancels unfinished preparation work. -Session manifests and media remain same-origin and use `Cache-Control: no-store`. - -Every successful creation response contains the complete authoritative -`subtitles` descriptor catalog. Videos without captions return an empty list. -The catalog is copied into the playback session and remains unchanged across -seek generations, so Android can attach every `SubtitleConfiguration` before -the first Media3 preparation. - -Descriptor discovery reads caption metadata only. It does not fetch or convert -a subtitle document. Each descriptor points to a session-scoped -`/api/android/youtube/playback/{sessionId}/subtitles/{trackId}.vtt` resource, -and the server obtains that WebVTT content only when the resource is requested. -The `deferredSubtitleContent` and `bootstrapSubtitleDescriptors` capabilities -advertise these separate guarantees. diff --git a/gradle.properties b/gradle.properties index 33c2324d..0a6bd48a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ org.gradle.jvmargs=-Xmx2g -XX:+UseG1GC kotlin.code.style=official -appVersion=1.2.4 +appVersion=1.3.0 systemProp.sun.net.client.defaultReadTimeout=180000 systemProp.sun.net.client.defaultConnectTimeout=60000 diff --git a/openapi.yaml b/openapi.yaml index 0c906cad..ab456f14 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -15,26 +15,17 @@ tags: - name: playback - name: downloader - name: youtube-session - - name: android-playback - name: user-data paths: /health: { $ref: ./openapi/paths/health.yaml#/Health } /instance: { $ref: ./openapi/paths/metadata.yaml#/Instance } - /streams: { $ref: ./openapi/paths/streams.yaml#/Streams } - /streams/legacy: { $ref: ./openapi/paths/streams.yaml#/LegacyStreams } /streams/youtube/sabr: { $ref: ./openapi/paths/streams.yaml#/YoutubeSabrStreams } /streams/youtube/sabr/bootstrap: { $ref: ./openapi/paths/streams.yaml#/YoutubeSabrBootstrap } - /streams/youtube/legacy: { $ref: ./openapi/paths/streams.yaml#/YoutubeLegacyStreams } /streams/niconico: { $ref: ./openapi/paths/streams.yaml#/NicoNicoStreams } /streams/bilibili: { $ref: ./openapi/paths/streams.yaml#/BiliBiliStreams } /streams/audio-only: { $ref: ./openapi/paths/streams.yaml#/AudioOnly } /streams/audio-only/source: { $ref: ./openapi/paths/streams.yaml#/AudioOnlySource } - /android/youtube/playback/{videoId}: { $ref: ./openapi/paths/android-playback.yaml#/Create } - /android/youtube/playback/{sessionId}/seek: { $ref: ./openapi/paths/android-playback.yaml#/Seek } - /android/youtube/playback/{sessionId}/manifest.mpd: { $ref: ./openapi/paths/android-playback.yaml#/Manifest } - /android/youtube/playback/{sessionId}/subtitles/{trackId}.vtt: { $ref: ./openapi/paths/android-playback.yaml#/Subtitle } - /android/youtube/playback/{sessionId}/{itag}/init: { $ref: ./openapi/paths/android-playback.yaml#/Initialization } - /android/youtube/playback/{sessionId}/{itag}/segment/{sequence}: { $ref: ./openapi/paths/android-playback.yaml#/Segment } + /sabr/download/{videoId}: { $ref: ./openapi/paths/sabr-download.yaml#/SabrDownload } /sabr/playback/{sessionId}/position: { $ref: ./openapi/paths/sabr-playback.yaml#/SabrPlaybackPosition } /sabr/playback/{sessionId}/window: { $ref: ./openapi/paths/sabr-playback.yaml#/SabrPlaybackWindow } /sabr/playback/{sessionId}/prefetch: { $ref: ./openapi/paths/sabr-playback.yaml#/SabrPlaybackPrefetch } @@ -51,6 +42,10 @@ paths: /saved-playlists/{id}: { $ref: ./openapi/paths/saved-playlists.yaml#/SavedPlaylist } /subscriptions/feed: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionFeed } /settings: { $ref: ./openapi/paths/access-control.yaml#/Settings } + /backup/typetype: { $ref: ./openapi/paths/user-backup.yaml#/TypeTypeBackup } + /restore/typetype: { $ref: ./openapi/paths/user-backup.yaml#/TypeTypeRestore } + /blocked/keywords: { $ref: ./openapi/paths/user-backup.yaml#/BlockedKeywords } + /blocked/keywords/{keyword}: { $ref: ./openapi/paths/user-backup.yaml#/BlockedKeyword } /allowed/channels: { $ref: ./openapi/paths/access-control.yaml#/AllowedChannels } /allowed/channels/{channelUrl}: { $ref: ./openapi/paths/access-control.yaml#/AllowedChannel } /admin/settings: { $ref: ./openapi/paths/access-control.yaml#/AdminSettings } @@ -101,12 +96,6 @@ components: $ref: ./openapi/components/instance.yaml#/InstanceMinClientVersion InstanceResponse: $ref: ./openapi/components/instance.yaml#/InstanceResponse - AndroidPlaybackCreateRequest: - $ref: ./openapi/components/android-playback.yaml#/AndroidPlaybackCreateRequest - AndroidPlaybackSeekRequest: - $ref: ./openapi/components/android-playback.yaml#/AndroidPlaybackSeekRequest - AndroidPlaybackResponse: - $ref: ./openapi/components/android-playback.yaml#/AndroidPlaybackResponse StreamResponse: $ref: ./openapi/components/streams.yaml#/StreamResponse AudioOnlyStreamResponse: @@ -144,6 +133,9 @@ components: SubscriptionFeedResponse: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionFeedResponse } SubscriptionFeedPreparingResponse: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionFeedPreparingResponse } SettingsItem: { $ref: ./openapi/components/access-control.yaml#/SettingsItem } + TypeTypeBackupItem: { $ref: ./openapi/components/user-backup.yaml#/TypeTypeBackupItem } + TypeTypeRestoreSummary: { $ref: ./openapi/components/user-backup.yaml#/TypeTypeRestoreSummary } + BlockedKeywordItem: { $ref: ./openapi/components/user-backup.yaml#/BlockedKeywordItem } AdminSettingsItem: { $ref: ./openapi/components/access-control.yaml#/AdminSettingsItem } AllowedChannelItem: { $ref: ./openapi/components/access-control.yaml#/AllowedChannelItem } CommentsPageResponse: { $ref: ./openapi/components/media.yaml#/CommentsPageResponse } diff --git a/openapi/components/access-control.yaml b/openapi/components/access-control.yaml index 3e04f364..f3e34bab 100644 --- a/openapi/components/access-control.yaml +++ b/openapi/components/access-control.yaml @@ -3,6 +3,12 @@ SettingsItem: properties: defaultService: { type: integer, default: 0 } defaultQuality: { type: string, default: 1080p } + defaultPlaybackSpeed: + type: number + format: double + minimum: 0.25 + maximum: 4 + default: 1 defaultLandingPage: { type: string, default: home } autoplay: { type: boolean, default: true } volume: { type: number, format: double, default: 1.0 } diff --git a/openapi/components/android-playback.yaml b/openapi/components/android-playback.yaml deleted file mode 100644 index 9ab0d961..00000000 --- a/openapi/components/android-playback.yaml +++ /dev/null @@ -1,56 +0,0 @@ -AndroidPlaybackCreateRequest: - type: object - properties: - videoItag: { type: integer, nullable: true } - audioItag: { type: integer, nullable: true } - audioTrackId: { type: string, nullable: true } -AndroidPlaybackSeekRequest: - type: object - required: [generation, playerTimeMs] - properties: - generation: { type: integer, format: int64, minimum: 0 } - playerTimeMs: { type: integer, format: int64, minimum: 0 } -AndroidPlaybackResponse: - type: object - required: - - sessionId - - videoId - - manifestUrl - - videoItag - - audioItag - - generation - - ready - - status - - subtitles - properties: - sessionId: { type: string } - videoId: { type: string } - manifestUrl: { type: string } - videoItag: { type: integer } - audioItag: { type: integer } - audioTrackId: { type: string, nullable: true } - generation: { type: integer, format: int64, minimum: 0 } - ready: { type: boolean } - status: { type: string, enum: [ready, preparing] } - preparationStage: - type: string - nullable: true - enum: [audio_index, video_index, audio_video_index, media_bytes] - description: Current bounded preparation stage when ready is false. - subtitles: - type: array - description: Complete authoritative descriptor catalog. Subtitle content remains demand-driven. - items: { $ref: '#/AndroidSubtitle' } - retryAfterMs: { type: integer, format: int64, nullable: true, minimum: 100, maximum: 2000 } -AndroidSubtitle: - type: object - required: [id, mimeType, languageTag, displayLanguageName, isAutoGenerated, url] - properties: - id: { type: string } - mimeType: { type: string, enum: [text/vtt] } - languageTag: { type: string } - displayLanguageName: { type: string } - isAutoGenerated: { type: boolean } - url: - type: string - description: Session-scoped same-origin resource that remains stable across playback generations. diff --git a/openapi/components/instance.yaml b/openapi/components/instance.yaml index 7d8db7f2..676c77c6 100644 --- a/openapi/components/instance.yaml +++ b/openapi/components/instance.yaml @@ -20,7 +20,6 @@ InstanceResponse: - oidcAutoRedirect - youtubeRemoteLoginEnabled - youtubeRemoteLoginReady - - androidPlayback properties: name: { type: string, example: TypeType } tagline: { type: string, nullable: true } @@ -50,28 +49,3 @@ InstanceResponse: type: string nullable: true enum: [disabled, not_configured, token_unreachable] - androidPlayback: - $ref: '#/AndroidPlaybackCapability' -AndroidPlaybackCapability: - type: object - required: [supported, contractVersion, youtube] - properties: - supported: { type: boolean, example: true } - contractVersion: { type: integer, enum: [4] } - youtube: - $ref: '#/AndroidYoutubePlaybackCapability' -AndroidYoutubePlaybackCapability: - type: object - required: [vod, live, subtitles, deferredSubtitleContent, bootstrapSubtitleDescriptors] - properties: - vod: { type: boolean, example: true } - live: { type: boolean, example: false } - subtitles: { type: boolean, example: true } - deferredSubtitleContent: - type: boolean - example: true - description: Subtitle documents are fetched only when a session-scoped WebVTT resource is requested. - bootstrapSubtitleDescriptors: - type: boolean - example: true - description: Every successful playback creation response contains the complete subtitle descriptor catalog. diff --git a/openapi/components/user-backup.yaml b/openapi/components/user-backup.yaml new file mode 100644 index 00000000..36f5a1d4 --- /dev/null +++ b/openapi/components/user-backup.yaml @@ -0,0 +1,71 @@ +BlockedKeywordItem: + type: object + required: [keyword] + properties: + keyword: { type: string, minLength: 1, maxLength: 100 } + blockedAt: { type: integer, format: int64, default: 0 } + global: { type: boolean, nullable: true } +TypeTypeContentFiltersBackup: + type: object + properties: + blockedChannels: + type: array + items: { type: object, additionalProperties: true } + blockedVideos: + type: array + items: { type: object, additionalProperties: true } + blockedKeywords: + type: array + items: { $ref: '#/BlockedKeywordItem' } + allowedChannels: + type: array + items: { $ref: ./access-control.yaml#/AllowedChannelItem } + allowedPlaylists: + type: array + items: { $ref: ./access-control.yaml#/AllowedPlaylistItem } +TypeTypeBackupItem: + type: object + required: [format, version, exportedAt, categories] + properties: + format: { type: string, enum: [typetype-backup] } + version: { type: integer, enum: [1] } + exportedAt: { type: integer, format: int64 } + categories: + type: array + uniqueItems: true + items: + type: string + enum: + - subscriptions + - history + - playlists + - watchLater + - favorites + - progress + - searchHistory + - savedPlaylists + - settings + - contentFilters + subscriptions: { type: array, nullable: true, items: { type: object, additionalProperties: true } } + history: { type: array, nullable: true, items: { type: object, additionalProperties: true } } + playlists: { type: array, nullable: true, items: { type: object, additionalProperties: true } } + watchLater: { type: array, nullable: true, items: { type: object, additionalProperties: true } } + favorites: { type: array, nullable: true, items: { type: object, additionalProperties: true } } + progress: { type: array, nullable: true, items: { type: object, additionalProperties: true } } + searchHistory: { type: array, nullable: true, items: { type: object, additionalProperties: true } } + savedPlaylists: { type: array, nullable: true, items: { type: object, additionalProperties: true } } + settings: + allOf: + - { $ref: ./access-control.yaml#/SettingsItem } + nullable: true + contentFilters: + allOf: + - { $ref: '#/TypeTypeContentFiltersBackup' } + nullable: true +TypeTypeRestoreSummary: + type: object + required: [restored] + properties: + restored: + type: object + additionalProperties: { type: integer, minimum: 0 } diff --git a/openapi/paths/android-playback.yaml b/openapi/paths/android-playback.yaml deleted file mode 100644 index 431a1a63..00000000 --- a/openapi/paths/android-playback.yaml +++ /dev/null @@ -1,163 +0,0 @@ -Create: - parameters: - - name: videoId - in: path - required: true - schema: { type: string } - post: - tags: [android-playback] - summary: Create an Android-only YouTube VOD playback session - description: Active livestreams are rejected. Every successful response contains the complete authoritative subtitle descriptor catalog, including an empty list when the video has no captions. Descriptor discovery does not fetch subtitle documents; media and WebVTT content remain demand-driven. A 202 response represents one shared session preparation with an eight-second server deadline. - requestBody: - required: false - content: - application/json: - schema: { $ref: ../components/android-playback.yaml#/AndroidPlaybackCreateRequest } - responses: - '200': { $ref: '#/components/responses/ReadySession' } - '202': { $ref: '#/components/responses/PreparingSession' } - '400': { $ref: ../components/common.yaml#/JsonError } - '401': { $ref: ../components/common.yaml#/JsonError } - '403': { $ref: ../components/common.yaml#/JsonError } - '422': { $ref: ../components/common.yaml#/JsonError } - '503': { $ref: ../components/common.yaml#/JsonError } -Seek: - parameters: - - $ref: '#/components/parameters/SessionId' - post: - tags: [android-playback] - summary: Reposition an Android VOD playback session - description: The session ID and selected formats stay stable while the generation increments. - requestBody: - required: true - content: - application/json: - schema: { $ref: ../components/android-playback.yaml#/AndroidPlaybackSeekRequest } - responses: - '200': { $ref: '#/components/responses/ReadySession' } - '202': { $ref: '#/components/responses/PreparingSession' } - '400': { $ref: ../components/common.yaml#/JsonError } - '404': { $ref: ../components/common.yaml#/JsonError } - '409': { $ref: ../components/common.yaml#/JsonError } - '410': { $ref: ../components/common.yaml#/JsonError } - '422': { $ref: ../components/common.yaml#/JsonError } - '503': { $ref: ../components/common.yaml#/JsonError } -Manifest: - parameters: - - $ref: '#/components/parameters/SessionId' - get: - tags: [android-playback] - summary: Get the complete static DASH presentation for an Android VOD session - description: Polling reads the current session state only and does not restart index extraction. Preparation ends as a complete MPD or a typed 422 or 503 response. - responses: - '200': - description: Complete standards-compatible static DASH MPD. - headers: - Cache-Control: - schema: { type: string, example: no-store } - content: - application/dash+xml: - schema: { type: string } - '202': { $ref: '#/components/responses/PreparingSession' } - '404': { $ref: ../components/common.yaml#/JsonError } - '410': { $ref: ../components/common.yaml#/JsonError } - '422': { $ref: ../components/common.yaml#/JsonError } - '503': { $ref: ../components/common.yaml#/JsonError } -Subtitle: - parameters: - - $ref: '#/components/parameters/SessionId' - - $ref: '#/components/parameters/TrackId' - get: - tags: [android-playback] - summary: Get one session-scoped Android subtitle as UTF-8 WebVTT - description: The caller must use the same account context that created the playback session. The resource remains valid across seek generations and subtitle content is fetched only when this route is requested. - responses: - '200': - description: Complete UTF-8 WebVTT subtitle document. - headers: - Cache-Control: - schema: { type: string, example: no-store } - content: - text/vtt: - schema: { type: string } - '401': { $ref: ../components/common.yaml#/JsonError } - '404': { $ref: ../components/common.yaml#/JsonError } - '410': { $ref: ../components/common.yaml#/JsonError } - '422': { $ref: ../components/common.yaml#/JsonError } - '503': { $ref: ../components/common.yaml#/JsonError } -Initialization: - parameters: - - $ref: '#/components/parameters/SessionId' - - $ref: '#/components/parameters/Itag' - - $ref: '#/components/parameters/SessionQuery' - - $ref: '#/components/parameters/Generation' - get: - tags: [android-playback] - summary: Get initialization bytes for a selected Android playback track - responses: - '200': { description: Initialization bytes. } - '202': { $ref: '#/components/responses/PreparingSession' } - '206': { description: Partial initialization bytes. } - '400': { $ref: ../components/common.yaml#/JsonError } - '404': { $ref: ../components/common.yaml#/JsonError } - '409': { $ref: ../components/common.yaml#/JsonError } - '410': { $ref: ../components/common.yaml#/JsonError } -Segment: - parameters: - - $ref: '#/components/parameters/SessionId' - - $ref: '#/components/parameters/Itag' - - name: sequence - in: path - required: true - schema: { type: integer, minimum: 1 } - - $ref: '#/components/parameters/SessionQuery' - - $ref: '#/components/parameters/Generation' - get: - tags: [android-playback] - summary: Get one demand-driven media segment for an Android playback track - responses: - '200': { description: Media segment bytes. } - '202': { $ref: '#/components/responses/PreparingSession' } - '206': { description: Partial media segment bytes. } - '400': { $ref: ../components/common.yaml#/JsonError } - '404': { $ref: ../components/common.yaml#/JsonError } - '409': { $ref: ../components/common.yaml#/JsonError } - '410': { $ref: ../components/common.yaml#/JsonError } -components: - parameters: - SessionId: - name: sessionId - in: path - required: true - schema: { type: string } - Itag: - name: itag - in: path - required: true - schema: { type: integer } - TrackId: - name: trackId - in: path - required: true - schema: { type: string } - SessionQuery: - name: session - in: query - required: true - schema: { type: string } - Generation: - name: generation - in: query - required: true - schema: { type: integer, format: int64, minimum: 0 } - responses: - ReadySession: - description: Playback session with a complete VOD presentation. - content: - application/json: - schema: { $ref: ../components/android-playback.yaml#/AndroidPlaybackResponse } - PreparingSession: - description: The shared exact virtual index or requested bytes are still preparing. Retry after retryAfterMs without creating another session. - content: - application/json: - schema: { $ref: ../components/android-playback.yaml#/AndroidPlaybackResponse } diff --git a/openapi/paths/sabr-download.yaml b/openapi/paths/sabr-download.yaml new file mode 100644 index 00000000..5a17c1e3 --- /dev/null +++ b/openapi/paths/sabr-download.yaml @@ -0,0 +1,56 @@ +SabrDownload: + get: + tags: [playback] + summary: Stream selected YouTube SABR tracks for the Downloader + operationId: streamSabrDownload + security: + - bearerAuth: [] + parameters: + - in: path + name: videoId + required: true + schema: { type: string } + - in: query + name: audioItag + required: true + schema: { type: integer } + - in: query + name: videoItag + schema: { type: integer } + - in: query + name: audioTrackId + schema: { type: string } + - in: query + name: audioOnly + schema: { type: boolean, default: false } + - in: query + name: part + description: Zero-based temporal part to stream + schema: { type: integer, minimum: 0, default: 0 } + - in: query + name: parts + description: Number of independent SABR parts + schema: { type: integer, minimum: 1, maximum: 12, default: 1 } + responses: + '200': + description: Ordered framed SABR track stream + content: + application/vnd.typetype.sabr-download: + schema: + type: string + format: binary + '400': + description: Invalid request + content: + application/json: + schema: { $ref: ../components/common.yaml#/ErrorResponse } + '401': + description: Authentication required + content: + application/json: + schema: { $ref: ../components/common.yaml#/ErrorResponse } + '422': + description: SABR preparation failed + content: + application/json: + schema: { $ref: ../components/common.yaml#/ErrorResponse } diff --git a/openapi/paths/streams.yaml b/openapi/paths/streams.yaml index 90d30a43..743342b7 100644 --- a/openapi/paths/streams.yaml +++ b/openapi/paths/streams.yaml @@ -1,26 +1,3 @@ -Streams: - get: - tags: [extraction] - summary: Extract classic stream metadata with the historical provider-neutral contract - parameters: - - name: url - in: query - required: true - schema: { type: string } - responses: - '200': - description: Stream metadata containing no SABR sources. - headers: - X-Request-ID: - $ref: ../components/common.yaml#/RequestIdHeader - content: - application/json: - schema: - $ref: ../components/streams.yaml#/StreamResponse - '400': - $ref: ../components/common.yaml#/JsonError - '422': - $ref: ../components/common.yaml#/JsonError YoutubeSabrStreams: get: tags: [extraction] @@ -67,31 +44,6 @@ YoutubeSabrBootstrap: $ref: ../components/common.yaml#/JsonError '422': $ref: ../components/common.yaml#/JsonError -LegacyStreams: - get: - tags: [extraction] - summary: Extract classic DASH, HLS, or progressive stream metadata - parameters: - - name: url - in: query - required: true - schema: { type: string } - responses: - '200': - description: Stream metadata containing no SABR sources. - headers: - X-Request-ID: - $ref: ../components/common.yaml#/RequestIdHeader - content: - application/json: - schema: - $ref: ../components/streams.yaml#/StreamResponse - '400': - $ref: ../components/common.yaml#/JsonError - '422': - $ref: ../components/common.yaml#/JsonError -YoutubeLegacyStreams: - $ref: '#/LegacyStreams' NicoNicoStreams: get: tags: [extraction] diff --git a/openapi/paths/user-backup.yaml b/openapi/paths/user-backup.yaml new file mode 100644 index 00000000..59c5f006 --- /dev/null +++ b/openapi/paths/user-backup.yaml @@ -0,0 +1,84 @@ +TypeTypeBackup: + get: + tags: [user-data] + summary: Export selected TypeType user data + parameters: + - name: categories + in: query + description: Comma-separated categories, or all. All categories are exported when omitted. + schema: { type: string, default: all } + responses: + '200': + description: Versioned TypeType backup + headers: + Content-Disposition: + schema: { type: string } + content: + application/json: + schema: { $ref: ../components/user-backup.yaml#/TypeTypeBackupItem } + '400': { description: Invalid categories } + '401': { description: Missing or invalid token } +TypeTypeRestore: + post: + tags: [user-data] + summary: Restore categories from a TypeType backup + description: Each category present in the backup replaces that category for the current user. + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + required: [file] + properties: + file: { type: string, format: binary } + responses: + '200': + description: Restored item counts + content: + application/json: + schema: { $ref: ../components/user-backup.yaml#/TypeTypeRestoreSummary } + '400': { description: Invalid TypeType backup } + '401': { description: Missing or invalid token } +BlockedKeywords: + get: + tags: [user-data] + summary: List blocked title keywords + responses: + '200': + description: Blocked keywords + content: + application/json: + schema: + type: array + items: { $ref: ../components/user-backup.yaml#/BlockedKeywordItem } + '401': { description: Missing or invalid token } + post: + tags: [user-data] + summary: Block a title keyword + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/user-backup.yaml#/BlockedKeywordItem } + responses: + '201': + description: Created blocked keyword + content: + application/json: + schema: { $ref: ../components/user-backup.yaml#/BlockedKeywordItem } + '400': { description: Invalid keyword } + '401': { description: Missing or invalid token } +BlockedKeyword: + delete: + tags: [user-data] + summary: Delete a blocked title keyword + parameters: + - name: keyword + in: path + required: true + schema: { type: string } + responses: + '204': { description: Deleted } + '401': { description: Missing or invalid token } + '404': { description: Not found } diff --git a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt b/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt index 174bbc62..e1bef03f 100644 --- a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt @@ -7,7 +7,6 @@ import dev.typetype.server.routes.adminIdentityRoutes import dev.typetype.server.routes.adminSessionRoutes import dev.typetype.server.routes.authRoutes import dev.typetype.server.routes.avatarRoutes -import dev.typetype.server.routes.androidPlaybackRoutes import dev.typetype.server.routes.bulletCommentRoutes import dev.typetype.server.routes.channelRoutes import dev.typetype.server.routes.commentRoutes @@ -27,7 +26,6 @@ import dev.typetype.server.routes.userDataRoutes import dev.typetype.server.routes.youtubeRemoteBrowserRoutes import dev.typetype.server.services.ActiveSessionService import dev.typetype.server.services.AdminSettingsService -import dev.typetype.server.services.AndroidSubtitleInventoryCoordinator import dev.typetype.server.services.AuthService import dev.typetype.server.services.AvatarService import dev.typetype.server.services.DownloaderGatewayService @@ -63,7 +61,6 @@ internal fun Application.installApplicationRoutes( restoreService: PipePipeBackupImporterService, youtubeRemoteBrowserService: YoutubeRemoteBrowserService, ) { - val androidSubtitleCoordinator = AndroidSubtitleInventoryCoordinator(svc.androidSubtitleService, this) routing { internalObservabilityRoutes(internalHealthService::check) publicMetadataRoutes(instanceService::getInstance) @@ -83,18 +80,9 @@ internal fun Application.installApplicationRoutes( } installProxyRoutes(svc) rateLimit(PROXY_ZONE) { - androidPlaybackRoutes( - svc.androidSabrSessionStore, - svc.streamService, - svc.androidSubtitleService, - androidSubtitleCoordinator, - authService, - svc.accessControlService, - adminSettingsService, - ) sabrRoutes( svc.sabrSessionStore, - svc.streamService, + svc.youtubeSabrStreamService, authService, svc.accessControlService, adminSettingsService, diff --git a/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt b/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt index 1f40dea1..b965446a 100644 --- a/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt @@ -5,7 +5,6 @@ import dev.typetype.server.routes.audioOnlySourceRoutes import dev.typetype.server.routes.manifestRoutes import dev.typetype.server.routes.nicoVideoProxyRoutes import dev.typetype.server.routes.proxyRoutes -import dev.typetype.server.routes.sabrAudioOnlyUnplayableReason import dev.typetype.server.routes.storyboardProxyRoutes import dev.typetype.server.routes.streamRoutes import dev.typetype.server.routes.withPlayableSabrStreams @@ -22,16 +21,11 @@ internal fun Route.installStreamRoutes( rateLimit(STREAMS_ZONE) { streamRoutes( streamService = svc.youtubeSabrStreamService, - legacyStreamService = svc.legacyStreamService, - genericLegacyStreamService = svc.streamService, nicoNicoStreamService = svc.nicoNicoStreamService, bilibiliStreamService = svc.bilibiliStreamService, sabrBootstrapStreamService = svc.youtubeSabrBootstrapStreamService, authService = authService, accessControlService = svc.accessControlService, - youtubeSessionStreamInfo = svc.youtubeSessionStreamService?.let { service -> - { userId, url -> service.getStreamInfo(userId, url) } - }, adminSettingsService = adminSettingsService, publicHlsManifestTokenService = svc.publicHlsManifestTokenService, sabrStreamContractFilter = { url, data -> data.withPlayableSabrStreams(url, svc.sabrSessionStore) }, @@ -47,9 +41,6 @@ internal fun Route.installStreamRoutes( adminSettingsService = adminSettingsService, publicHlsManifestTokenService = svc.publicHlsManifestTokenService, proxyService = svc.proxyService, - sabrAudioOnlyUnavailableReason = { url, userId, selection -> - svc.sabrSessionStore.sabrAudioOnlyUnplayableReason(url, userId, selection) - }, ) manifestRoutes( svc.manifestService, @@ -73,7 +64,6 @@ internal fun Route.installProxyRoutes(svc: ServiceRegistry) { youtubeSessionStreamInfo = svc.youtubeSessionStreamService?.let { service -> { userId, url -> service.getStreamInfo(userId, url) } }, - sabrSessionStore = svc.sabrSessionStore, ) nicoVideoProxyRoutes(svc.nicoVideoProxyService) } diff --git a/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt index 45183642..6182e354 100644 --- a/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt @@ -1,8 +1,6 @@ package dev.typetype.server import dev.typetype.server.cache.DragonflyService -import dev.typetype.server.services.AndroidSubtitleHttpClient -import dev.typetype.server.services.AndroidSubtitleService import dev.typetype.server.services.BilibiliRelatedService import dev.typetype.server.services.BilibiliTrendingService import dev.typetype.server.services.CachedChannelService @@ -53,9 +51,7 @@ import dev.typetype.server.services.YoutubeSessionStreamService import okhttp3.ConnectionPool import okhttp3.Dispatcher import okhttp3.OkHttpClient -import java.net.Proxy import java.net.ProxySelector -import java.time.Duration import java.util.concurrent.TimeUnit internal class ExtractionServiceRegistry( @@ -78,29 +74,8 @@ internal class ExtractionServiceRegistry( .followRedirects(true) .build() val sabrSessionStore = SabrSessionStore(subtitleServiceUrl, initCache = cache) - val androidSabrSessionStore = SabrSessionStore( - subtitleServiceUrl, - idleEviction = Duration.ofMinutes(6), - initCache = cache, - ) val youtubeSubtitleService = YouTubeSubtitleService(httpClient, subtitleServiceUrl) - val androidSubtitleService = AndroidSubtitleService( - youtubeSubtitleService, - AndroidSubtitleHttpClient( - httpClient.newBuilder() - .followRedirects(false) - .callTimeout(10, TimeUnit.SECONDS) - .build(), - youtubeProxySelector?.let { - httpClient.newBuilder() - .proxy(Proxy.NO_PROXY) - .followRedirects(false) - .callTimeout(10, TimeUnit.SECONDS) - .build() - }, - ), - ) - private val classicPipePipeStreamService = PipePipeStreamService( + private val directPipePipeStreamService = PipePipeStreamService( cache, youtubeSubtitleService, BilibiliRelatedService(), @@ -111,12 +86,12 @@ internal class ExtractionServiceRegistry( BilibiliRelatedService(), sabrSessionStore::rememberExtractedInfo, ) - private val classicPublicStreamService = YoutubePlayerClientFallbackStreamService( - classicPipePipeStreamService, - listOf(YoutubePlayerClient.ANDROID_VR, YoutubePlayerClient.WEB_SAFARI, YoutubePlayerClient.TV_SIMPLY), + private val publicStreamService = YoutubePlayerClientStreamService( + directPipePipeStreamService, + YoutubePlayerClient.WEB_SAFARI, ) - private val classicAuthenticatedStreamService = YoutubePlayerClientFallbackStreamService( - classicPipePipeStreamService, + private val authenticatedStreamService = YoutubePlayerClientFallbackStreamService( + directPipePipeStreamService, listOf(YoutubePlayerClient.TV_DOWNGRADED, YoutubePlayerClient.WEB_SAFARI), ) private val sabrPublicStreamService = YoutubePlayerClientStreamService( @@ -127,13 +102,8 @@ internal class ExtractionServiceRegistry( private val hlsTokenService = youtubeSessionSecret?.let(::SignedHlsManifestTokenService) private val tokenYoutubeSessionClient = TypetypeTokenYoutubeSessionClient(subtitleServiceUrl, httpClient) val youtubeSessionStreamService = hlsTokenService?.let { - YoutubeSessionStreamService(classicAuthenticatedStreamService, youtubeSessionService, cache, it) + YoutubeSessionStreamService(authenticatedStreamService, youtubeSessionService, cache, it) } - val legacyStreamService = CachedStreamService( - YoutubeScopedStreamService(classicPublicStreamService), - cache, - "stream-youtube-legacy:v5", - ) val youtubeSabrStreamService = CachedStreamService( YoutubeScopedStreamService( SabrFallbackStreamService(sabrPublicStreamService, sabrSessionStore, tokenYoutubeSessionClient), @@ -146,9 +116,9 @@ internal class ExtractionServiceRegistry( cache, "stream-youtube-sabr-bootstrap:v1", ) - val nicoNicoStreamService = CachedStreamService(classicPipePipeStreamService, cache, "stream-niconico:v1") - val bilibiliStreamService = CachedStreamService(classicPipePipeStreamService, cache, "stream-bilibili:v1") - val streamService = CachedStreamService(classicPublicStreamService, cache, "stream-classic:v2") + val nicoNicoStreamService = CachedStreamService(directPipePipeStreamService, cache, "stream-niconico:v1") + val bilibiliStreamService = CachedStreamService(directPipePipeStreamService, cache, "stream-bilibili:v1") + val streamService = CachedStreamService(publicStreamService, cache, "stream-direct:v1") val searchService = CachedSearchService(YoutubeScopedSearchService(PipePipeSearchService()), cache) val trendingService = CachedTrendingService( YoutubeScopedTrendingService(PipePipeTrendingService(BilibiliTrendingService(), NicoNicoTrendingService(httpClient))), diff --git a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt index 8e7e325f..0ec40261 100644 --- a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt @@ -29,6 +29,7 @@ import dev.typetype.server.services.SubscriptionShortsFeedService import dev.typetype.server.services.SubscriptionsService import dev.typetype.server.services.SubscriptionFeedCacheInvalidation import dev.typetype.server.services.SubscriptionFeedCacheInvalidator +import dev.typetype.server.services.TypeTypeBackupService import dev.typetype.server.services.UserVideoMetadataRepairService import dev.typetype.server.services.VideoMetadataResolver import dev.typetype.server.services.WatchLaterService @@ -55,7 +56,6 @@ internal class ServiceRegistry( ) val youtubeSessionService = extraction.youtubeSessionService val youtubeSessionStreamService = extraction.youtubeSessionStreamService - val legacyStreamService = extraction.legacyStreamService val youtubeSabrStreamService = extraction.youtubeSabrStreamService val youtubeSabrBootstrapStreamService = extraction.youtubeSabrBootstrapStreamService val nicoNicoStreamService = extraction.nicoNicoStreamService @@ -77,8 +77,6 @@ internal class ServiceRegistry( val audioOnlyMediaTokenService = AudioOnlyMediaTokenService(jwtSecret) val suggestionService = extraction.suggestionService val sabrSessionStore = extraction.sabrSessionStore - val androidSabrSessionStore = extraction.androidSabrSessionStore - val androidSubtitleService = extraction.androidSubtitleService val historyService = HistoryService() val subscriptionsService = SubscriptionsService() val subscriptionFeedService = SubscriptionFeedService(subscriptionsService, channelService, cache) @@ -108,6 +106,20 @@ internal class ServiceRegistry( val adminUserLookupService = AdminUserLookupService() val accessControlService = AccessControlService(settingsService, allowedChannelsService, allowedPlaylistsService, adminSettingsService) val blockedService = BlockedService() + val typeTypeBackupService = TypeTypeBackupService( + subscriptionsService, + historyService, + playlistService, + watchLaterService, + favoritesService, + progressService, + searchHistoryService, + savedPlaylistService, + settingsService, + blockedService, + allowedChannelsService, + allowedPlaylistsService, + ) val bugReportService = BugReportService() val youtubeTakeoutImportService = YoutubeTakeoutFactory.create(subscriptionsService, playlistService, historyService, favoritesService, watchLaterService) val recommendationPoolResolverDependencies = HomeRecommendationPoolResolverDependencies( diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt b/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt index dbb29632..e99acb31 100644 --- a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt +++ b/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt @@ -3,6 +3,7 @@ package dev.typetype.server.db import com.zaxxer.hikari.HikariConfig import com.zaxxer.hikari.HikariDataSource import dev.typetype.server.db.tables.BlockedChannelsTable +import dev.typetype.server.db.tables.BlockedKeywordsTable import dev.typetype.server.db.tables.BlockedVideosTable import dev.typetype.server.db.tables.BugReportsTable import dev.typetype.server.db.tables.HistoryTable @@ -63,6 +64,7 @@ object DatabaseFactory { dev.typetype.server.db.tables.AllowedPlaylistsTable, SearchHistoryTable, BlockedChannelsTable, + BlockedKeywordsTable, BlockedVideosTable, PasswordResetTable, YoutubeTakeoutImportJobsTable, diff --git a/src/main/kotlin/dev/typetype/server/db/DatabasePrimaryKeyMigrations.kt b/src/main/kotlin/dev/typetype/server/db/DatabasePrimaryKeyMigrations.kt index 9cc008bd..b80ae613 100644 --- a/src/main/kotlin/dev/typetype/server/db/DatabasePrimaryKeyMigrations.kt +++ b/src/main/kotlin/dev/typetype/server/db/DatabasePrimaryKeyMigrations.kt @@ -9,6 +9,7 @@ object DatabasePrimaryKeyMigrations { recreatePrimaryKey(table = "favorites", keyName = "favorites_pkey", columns = "user_id, video_url") recreatePrimaryKey(table = "watch_later", keyName = "watch_later_pkey", columns = "user_id, url") recreatePrimaryKey(table = "blocked_channels", keyName = "blocked_channels_pkey", columns = "user_id, channel_url") + recreatePrimaryKey(table = "blocked_keywords", keyName = "blocked_keywords_pkey", columns = "user_id, keyword") recreatePrimaryKey(table = "blocked_videos", keyName = "blocked_videos_pkey", columns = "user_id, video_url") recreatePrimaryKey(table = "settings", keyName = "settings_pkey", columns = "user_id") } diff --git a/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt b/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt index 5ba7b1ae..cb816f2f 100644 --- a/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt +++ b/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt @@ -5,6 +5,7 @@ import org.jetbrains.exposed.v1.jdbc.transactions.TransactionManager object SettingsSchemaMigrations { fun apply() { exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS subtitles_enabled BOOLEAN NOT NULL DEFAULT false") + exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS default_playback_speed DOUBLE PRECISION NOT NULL DEFAULT 1.0") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS default_subtitle_language TEXT NOT NULL DEFAULT ''") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS default_audio_language TEXT NOT NULL DEFAULT ''") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS prefer_original_language BOOLEAN NOT NULL DEFAULT false") diff --git a/src/main/kotlin/dev/typetype/server/db/tables/BlockedKeywordsTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/BlockedKeywordsTable.kt new file mode 100644 index 00000000..62ae4442 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/BlockedKeywordsTable.kt @@ -0,0 +1,11 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object BlockedKeywordsTable : Table("blocked_keywords") { + val userId = text("user_id") + val scope = text("scope").default("user") + val keyword = text("keyword") + val blockedAt = long("blocked_at") + override val primaryKey = PrimaryKey(userId, keyword) +} diff --git a/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt index 79eaf515..d4b78c50 100644 --- a/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt +++ b/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt @@ -6,6 +6,7 @@ object SettingsTable : Table("settings") { val userId = text("user_id") val defaultService = integer("default_service").default(0) val defaultQuality = text("default_quality").default("1080p") + val defaultPlaybackSpeed = double("default_playback_speed").default(1.0) val defaultLandingPage = text("default_landing_page").default("home") val autoplay = bool("autoplay").default(true) val skipPlaylistAutoplayScreen = bool("skip_playlist_autoplay_screen").default(false) diff --git a/src/main/kotlin/dev/typetype/server/models/BlockedKeywordItem.kt b/src/main/kotlin/dev/typetype/server/models/BlockedKeywordItem.kt new file mode 100644 index 00000000..ad46ad2e --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/BlockedKeywordItem.kt @@ -0,0 +1,10 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class BlockedKeywordItem( + val keyword: String, + val blockedAt: Long = 0L, + val global: Boolean? = null, +) diff --git a/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt b/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt index a4cd5e69..2ec0c25a 100644 --- a/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt +++ b/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt @@ -24,21 +24,4 @@ data class InstanceResponse( val youtubeRemoteLoginEnabled: Boolean = false, val youtubeRemoteLoginReady: Boolean = false, val youtubeRemoteLoginUnavailableReason: String? = null, - val androidPlayback: AndroidPlaybackCapability = AndroidPlaybackCapability(), -) - -@Serializable -data class AndroidPlaybackCapability( - val supported: Boolean = true, - val contractVersion: Int = 4, - val youtube: AndroidYoutubePlaybackCapability = AndroidYoutubePlaybackCapability(), -) - -@Serializable -data class AndroidYoutubePlaybackCapability( - val vod: Boolean = true, - val live: Boolean = false, - val subtitles: Boolean = true, - val deferredSubtitleContent: Boolean = true, - val bootstrapSubtitleDescriptors: Boolean = true, ) diff --git a/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt b/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt index 0d8e04c6..e471b347 100644 --- a/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt +++ b/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt @@ -6,6 +6,7 @@ import kotlinx.serialization.Serializable data class SettingsItem( val defaultService: Int = 0, val defaultQuality: String = "1080p", + val defaultPlaybackSpeed: Double = 1.0, val defaultLandingPage: String = "home", val autoplay: Boolean = true, val skipPlaylistAutoplayScreen: Boolean = false, diff --git a/src/main/kotlin/dev/typetype/server/models/TypeTypeBackupItem.kt b/src/main/kotlin/dev/typetype/server/models/TypeTypeBackupItem.kt new file mode 100644 index 00000000..4c9300b9 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/TypeTypeBackupItem.kt @@ -0,0 +1,38 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class TypeTypeBackupItem( + val format: String = TYPE_TYPE_BACKUP_FORMAT, + val version: Int = TYPE_TYPE_BACKUP_VERSION, + val exportedAt: Long, + val categories: List, + val subscriptions: List? = null, + val history: List? = null, + val playlists: List? = null, + val watchLater: List? = null, + val favorites: List? = null, + val progress: List? = null, + val searchHistory: List? = null, + val savedPlaylists: List? = null, + val settings: SettingsItem? = null, + val contentFilters: TypeTypeContentFiltersBackup? = null, +) + +@Serializable +data class TypeTypeContentFiltersBackup( + val blockedChannels: List = emptyList(), + val blockedVideos: List = emptyList(), + val blockedKeywords: List = emptyList(), + val allowedChannels: List = emptyList(), + val allowedPlaylists: List = emptyList(), +) + +@Serializable +data class TypeTypeRestoreSummary( + val restored: Map, +) + +const val TYPE_TYPE_BACKUP_FORMAT = "typetype-backup" +const val TYPE_TYPE_BACKUP_VERSION = 1 diff --git a/src/main/kotlin/dev/typetype/server/routes/AndroidPlaybackCallResponses.kt b/src/main/kotlin/dev/typetype/server/routes/AndroidPlaybackCallResponses.kt deleted file mode 100644 index 5115de06..00000000 --- a/src/main/kotlin/dev/typetype/server/routes/AndroidPlaybackCallResponses.kt +++ /dev/null @@ -1,36 +0,0 @@ -package dev.typetype.server.routes - -import dev.typetype.server.models.ErrorResponse -import dev.typetype.server.services.AndroidPlaybackSessionLookup -import dev.typetype.server.services.AndroidPlaybackSession -import dev.typetype.server.services.AndroidPlaybackService -import dev.typetype.server.services.SabrSessionHolder -import io.ktor.http.HttpStatusCode -import io.ktor.server.application.ApplicationCall -import io.ktor.server.response.respond - -internal suspend fun ApplicationCall.androidPlaybackHolder( - service: AndroidPlaybackService, - sessionId: String, -): SabrSessionHolder? = androidPlaybackSession(service, sessionId)?.holder - -internal suspend fun ApplicationCall.androidPlaybackSession( - service: AndroidPlaybackService, - sessionId: String, -): AndroidPlaybackSession? = when (val result = service.lookup(sessionId)) { - is AndroidPlaybackSessionLookup.Active -> result.session - AndroidPlaybackSessionLookup.Expired -> { - respondAndroidError(HttpStatusCode.Gone, "android_playback_expired", "Android playback session expired") - null - } - AndroidPlaybackSessionLookup.Unknown -> { - respondAndroidError(HttpStatusCode.NotFound, "android_playback_not_found", "Android playback session not found") - null - } -} - -internal suspend fun ApplicationCall.respondAndroidError( - status: HttpStatusCode, - code: String, - message: String, -): Unit = respond(status, ErrorResponse(message, code)) diff --git a/src/main/kotlin/dev/typetype/server/routes/AndroidPlaybackHandler.kt b/src/main/kotlin/dev/typetype/server/routes/AndroidPlaybackHandler.kt deleted file mode 100644 index 0d25ee60..00000000 --- a/src/main/kotlin/dev/typetype/server/routes/AndroidPlaybackHandler.kt +++ /dev/null @@ -1,198 +0,0 @@ -package dev.typetype.server.routes - -import dev.typetype.server.models.ExtractionResult -import dev.typetype.server.services.AccessControlService -import dev.typetype.server.services.AdminSettingsService -import dev.typetype.server.services.AndroidDashManifestResult -import dev.typetype.server.services.AndroidPlaybackCreateResult -import dev.typetype.server.services.AndroidPlaybackSeekResult -import dev.typetype.server.services.AndroidPlaybackService -import dev.typetype.server.services.AndroidSubtitleInventoryCoordinator -import dev.typetype.server.services.AndroidSubtitleInventorySnapshot -import dev.typetype.server.services.AuthService -import dev.typetype.server.services.SabrSessionStore -import dev.typetype.server.services.StreamService -import dev.typetype.server.services.runCatchingNonCancellation -import io.ktor.http.HttpStatusCode -import io.ktor.server.application.ApplicationCall -import io.ktor.server.request.receiveNullable -import io.ktor.server.response.respond -import io.ktor.server.response.respondText - -internal class AndroidPlaybackHandler( - private val store: SabrSessionStore, - private val streamService: StreamService, - private val authService: AuthService?, - private val accessControlService: AccessControlService?, - private val adminSettingsService: AdminSettingsService?, - private val subtitleCoordinator: AndroidSubtitleInventoryCoordinator, - val service: AndroidPlaybackService = AndroidPlaybackService(store), -) { - suspend fun create(call: ApplicationCall, videoId: String) { - val access = call.accessProfileOrRespond(authService, accessControlService, adminSettingsService) ?: return - if (!validateAccess(call, videoId, access)) return - val requestResult = runCatchingNonCancellation { call.receiveNullable() } - if (requestResult.isFailure) { - return call.respondAndroidError( - HttpStatusCode.BadRequest, - "android_playback_invalid_request", - "Invalid playback request", - ) - } - val request = requestResult.getOrNull() ?: AndroidPlaybackCreateRequest() - val subtitleInventory = subtitleCoordinator.start(videoId) - val prepared = store.fetchInfo(videoId, cachedFirst = true) - prepared - ?: return call.respondAndroidError( - HttpStatusCode.UnprocessableEntity, - "android_playback_probe_failed", - "SABR probe failed", - ) - val subtitles = when (val inventory = subtitleInventory.await()) { - is AndroidSubtitleInventorySnapshot.Ready -> inventory.tracks - AndroidSubtitleInventorySnapshot.Preparing, - AndroidSubtitleInventorySnapshot.TemporaryFailure, - -> return call.respondAndroidError( - HttpStatusCode.ServiceUnavailable, - "android_subtitle_inventory_unavailable", - "Android subtitle inventory is temporarily unavailable", - ) - } - val audio = SabrFormatSelector.audio(prepared.info, request.audioItag, request.audioTrackId, requireAac = true) - ?: return call.respondAndroidError( - HttpStatusCode.UnprocessableEntity, - "android_playback_audio_unavailable", - "No compatible SABR audio for this video", - ) - val video = SabrFormatSelector.androidVideo(prepared.info, request.videoItag) - ?: return call.respondAndroidError( - HttpStatusCode.UnprocessableEntity, - "android_playback_video_unavailable", - "No compatible SABR video for this video", - ) - when ( - val result = service.create( - videoId, - access.userId ?: "guest", - prepared, - audio, - video, - subtitles, - ) - ) { - is AndroidPlaybackCreateResult.Created -> call.respondSession( - result.session.toAndroidPlaybackResponse(result.manifest), - result.manifest, - ) - AndroidPlaybackCreateResult.UnsupportedLive -> call.respondAndroidError( - HttpStatusCode.UnprocessableEntity, - "android_live_playback_unsupported", - "Android live playback is not supported", - ) - } - } - - suspend fun seek(call: ApplicationCall, sessionId: String) { - val session = call.androidPlaybackSession(service, sessionId) ?: return - val holder = session.holder - val requestResult = runCatchingNonCancellation { call.receiveNullable() } - val request = requestResult.getOrNull() - if (requestResult.isFailure || request == null) { - return call.respondAndroidError( - HttpStatusCode.BadRequest, - "android_playback_invalid_request", - "Invalid seek request", - ) - } - if (request.playerTimeMs < 0L) { - return call.respondAndroidError( - HttpStatusCode.BadRequest, - "android_playback_invalid_seek", - "Invalid seek position", - ) - } - when (val result = service.seek(session, request.generation, request.playerTimeMs)) { - is AndroidPlaybackSeekResult.Ready -> call.respondSession( - session.withHolder(result.holder).toAndroidPlaybackResponse(result.manifest), - result.manifest, - ) - AndroidPlaybackSeekResult.StaleGeneration -> call.respondAndroidError( - HttpStatusCode.Conflict, - "android_playback_stale_generation", - "Stale Android playback generation", - ) - } - } - - suspend fun manifest(call: ApplicationCall, sessionId: String) { - call.response.headers.append("Cache-Control", "no-store") - val session = call.androidPlaybackSession(service, sessionId) ?: return - when (val result = service.manifest(session)) { - is AndroidDashManifestResult.Ready -> { - call.respondText(result.manifest, DASH_CONTENT_TYPE) - } - is AndroidDashManifestResult.Preparing -> call.respond( - HttpStatusCode.Accepted, - session.toAndroidPlaybackResponse(result), - ) - AndroidDashManifestResult.UnsupportedLive -> call.respondAndroidError( - HttpStatusCode.UnprocessableEntity, - "android_live_playback_unsupported", - "Android live playback is not supported", - ) - is AndroidDashManifestResult.Invalid -> call.respondAndroidError( - HttpStatusCode.UnprocessableEntity, - "android_playback_invalid_index", - result.reason, - ) - is AndroidDashManifestResult.TemporaryFailure -> call.respondAndroidError( - HttpStatusCode.ServiceUnavailable, - result.code, - result.reason, - ) - } - } - - private suspend fun ApplicationCall.respondSession( - response: AndroidPlaybackResponse, - manifest: AndroidDashManifestResult, - ): Unit = when (manifest) { - is AndroidDashManifestResult.Ready -> respond(HttpStatusCode.OK, response) - is AndroidDashManifestResult.Preparing -> respond(HttpStatusCode.Accepted, response) - AndroidDashManifestResult.UnsupportedLive -> respondAndroidError( - HttpStatusCode.UnprocessableEntity, - "android_live_playback_unsupported", - "Android live playback is not supported", - ) - is AndroidDashManifestResult.Invalid -> respondAndroidError( - HttpStatusCode.UnprocessableEntity, - "android_playback_invalid_index", - manifest.reason, - ) - is AndroidDashManifestResult.TemporaryFailure -> respondAndroidError( - HttpStatusCode.ServiceUnavailable, - manifest.code, - manifest.reason, - ) - } - - private suspend fun validateAccess(call: ApplicationCall, videoId: String, access: AccessRouteProfile): Boolean { - if (!access.profile.enabled) return true - return when (val result = streamService.getStreamInfo("https://www.youtube.com/watch?v=$videoId")) { - is ExtractionResult.Success -> if (access.profile.allowsUploader(result.data.uploaderUrl, result.data.uploaderName)) { - true - } else { - call.respondAndroidError(HttpStatusCode.Forbidden, "channel_not_allowed", "Channel is not allowed") - false - } - is ExtractionResult.Failure -> { - call.respondAndroidError(HttpStatusCode.UnprocessableEntity, "android_playback_extraction_failed", result.message) - false - } - is ExtractionResult.BadRequest -> { - call.respondAndroidError(HttpStatusCode.BadRequest, "android_playback_invalid_video", result.message) - false - } - } - } -} diff --git a/src/main/kotlin/dev/typetype/server/routes/AndroidPlaybackMediaHandler.kt b/src/main/kotlin/dev/typetype/server/routes/AndroidPlaybackMediaHandler.kt deleted file mode 100644 index 18faa069..00000000 --- a/src/main/kotlin/dev/typetype/server/routes/AndroidPlaybackMediaHandler.kt +++ /dev/null @@ -1,73 +0,0 @@ -package dev.typetype.server.routes - -import dev.typetype.server.services.AndroidPlaybackMediaResult -import dev.typetype.server.services.AndroidPlaybackPreparationStage -import dev.typetype.server.services.AndroidPlaybackService -import io.ktor.http.HttpStatusCode -import io.ktor.server.application.ApplicationCall -import io.ktor.server.response.respond - -internal class AndroidPlaybackMediaHandler(private val service: AndroidPlaybackService) { - suspend fun initialization(call: ApplicationCall, sessionId: String, itag: Int) { - val session = call.androidPlaybackSession(service, sessionId) ?: return - val generation = call.validGeneration(sessionId) ?: return - call.respondMediaResult(service.initialization(session.holder, itag, generation), session) - } - - suspend fun segment(call: ApplicationCall, sessionId: String, itag: Int, sequence: Int) { - val session = call.androidPlaybackSession(service, sessionId) ?: return - val generation = call.validGeneration(sessionId) ?: return - call.respondMediaResult(service.segment(session.holder, itag, sequence, generation), session) - } - - private suspend fun ApplicationCall.validGeneration(sessionId: String): Long? { - if (request.queryParameters["session"] != sessionId) { - respondAndroidError( - HttpStatusCode.BadRequest, - "android_playback_invalid_session", - "Manifest session does not match the request path", - ) - return null - } - val generation = request.queryParameters["generation"]?.toLongOrNull() - if (generation == null || generation < 0L) { - respondAndroidError( - HttpStatusCode.BadRequest, - "android_playback_invalid_generation", - "Missing or invalid Android playback generation", - ) - return null - } - return generation - } - - private suspend fun ApplicationCall.respondMediaResult( - result: AndroidPlaybackMediaResult, - session: dev.typetype.server.services.AndroidPlaybackSession, - ): Unit = when (result) { - is AndroidPlaybackMediaResult.Ready -> respondSabrMediaBytes(result.mimeType, result.bytes) - AndroidPlaybackMediaResult.Preparing -> respond( - HttpStatusCode.Accepted, - session.toAndroidPlaybackResponse( - dev.typetype.server.services.AndroidDashManifestResult.Preparing( - AndroidPlaybackPreparationStage.MEDIA_BYTES, - ), - ), - ) - AndroidPlaybackMediaResult.StaleGeneration -> respondAndroidError( - HttpStatusCode.Conflict, - "android_playback_stale_generation", - "Stale Android playback generation", - ) - AndroidPlaybackMediaResult.TrackNotFound -> respondAndroidError( - HttpStatusCode.NotFound, - "android_playback_track_not_found", - "Android playback track not found", - ) - AndroidPlaybackMediaResult.InvalidSequence -> respondAndroidError( - HttpStatusCode.BadRequest, - "android_playback_invalid_sequence", - "Invalid Android playback segment sequence", - ) - } -} diff --git a/src/main/kotlin/dev/typetype/server/routes/AndroidPlaybackModels.kt b/src/main/kotlin/dev/typetype/server/routes/AndroidPlaybackModels.kt deleted file mode 100644 index d21e3928..00000000 --- a/src/main/kotlin/dev/typetype/server/routes/AndroidPlaybackModels.kt +++ /dev/null @@ -1,74 +0,0 @@ -package dev.typetype.server.routes - -import dev.typetype.server.services.AndroidDashManifestResult -import dev.typetype.server.services.AndroidPlaybackSession -import dev.typetype.server.services.AndroidSubtitleTrack -import kotlinx.serialization.Serializable - -@Serializable -internal data class AndroidPlaybackCreateRequest( - val videoItag: Int? = null, - val audioItag: Int? = null, - val audioTrackId: String? = null, -) - -@Serializable -internal data class AndroidPlaybackSeekRequest( - val generation: Long, - val playerTimeMs: Long, -) - -@Serializable -internal data class AndroidPlaybackResponse( - val sessionId: String, - val videoId: String, - val manifestUrl: String, - val videoItag: Int, - val audioItag: Int, - val audioTrackId: String? = null, - val generation: Long, - val ready: Boolean, - val status: String, - val subtitles: List, - val retryAfterMs: Long? = null, - val preparationStage: String? = null, -) - -@Serializable -internal data class AndroidSubtitleResponse( - val id: String, - val mimeType: String, - val languageTag: String, - val displayLanguageName: String, - val isAutoGenerated: Boolean, - val url: String, -) - -internal fun AndroidPlaybackSession.toAndroidPlaybackResponse( - manifest: AndroidDashManifestResult, -): AndroidPlaybackResponse = AndroidPlaybackResponse( - sessionId = holder.sessionToken, - videoId = holder.key.videoId, - manifestUrl = "/api/android/youtube/playback/${holder.sessionToken}/manifest.mpd", - videoItag = holder.videoFormat.itag, - audioItag = holder.audioFormat.itag, - audioTrackId = holder.audioFormat.audioTrackId, - generation = holder.activeGeneration(), - ready = manifest is AndroidDashManifestResult.Ready, - status = if (manifest is AndroidDashManifestResult.Ready) "ready" else "preparing", - subtitles = subtitles.map { it.toResponse(holder.sessionToken) }, - retryAfterMs = if (manifest is AndroidDashManifestResult.Ready) null else ANDROID_RETRY_AFTER_MS, - preparationStage = (manifest as? AndroidDashManifestResult.Preparing)?.stage?.wireValue, -) - -internal fun AndroidSubtitleTrack.toResponse(sessionId: String): AndroidSubtitleResponse = - AndroidSubtitleResponse( - id = id, - mimeType = "text/vtt", - languageTag = languageTag, - displayLanguageName = displayLanguageName, - isAutoGenerated = isAutoGenerated, - url = "/api/android/youtube/playback/$sessionId/subtitles/$id.vtt", - ) - -internal const val ANDROID_RETRY_AFTER_MS = 500L diff --git a/src/main/kotlin/dev/typetype/server/routes/AndroidPlaybackRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/AndroidPlaybackRoutes.kt deleted file mode 100644 index ef2536b9..00000000 --- a/src/main/kotlin/dev/typetype/server/routes/AndroidPlaybackRoutes.kt +++ /dev/null @@ -1,80 +0,0 @@ -package dev.typetype.server.routes - -import dev.typetype.server.models.ErrorResponse -import dev.typetype.server.services.AccessControlService -import dev.typetype.server.services.AdminSettingsService -import dev.typetype.server.services.AndroidSubtitleService -import dev.typetype.server.services.AndroidSubtitleInventoryCoordinator -import dev.typetype.server.services.AuthService -import dev.typetype.server.services.SabrSessionStore -import dev.typetype.server.services.StreamService -import io.ktor.http.HttpStatusCode -import io.ktor.server.response.respond -import io.ktor.server.routing.Route -import io.ktor.server.routing.get -import io.ktor.server.routing.post - -internal fun Route.androidPlaybackRoutes( - store: SabrSessionStore, - streamService: StreamService, - subtitleService: AndroidSubtitleService, - subtitleCoordinator: AndroidSubtitleInventoryCoordinator, - authService: AuthService?, - accessControlService: AccessControlService?, - adminSettingsService: AdminSettingsService?, -) { - val handler = AndroidPlaybackHandler( - store, - streamService, - authService, - accessControlService, - adminSettingsService, - subtitleCoordinator = subtitleCoordinator, - ) - val media = AndroidPlaybackMediaHandler(handler.service) - val subtitles = AndroidSubtitleHandler( - handler.service, - subtitleService, - authService, - accessControlService, - adminSettingsService, - ) - post("/android/youtube/playback/{videoId}") { - val videoId = call.parameters["videoId"] - ?: return@post call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing videoId")) - handler.create(call, videoId) - } - post("/android/youtube/playback/{sessionId}/seek") { - val sessionId = call.parameters["sessionId"] - ?: return@post call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing sessionId")) - handler.seek(call, sessionId) - } - get("/android/youtube/playback/{sessionId}/manifest.mpd") { - val sessionId = call.parameters["sessionId"] - ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing sessionId")) - handler.manifest(call, sessionId) - } - get("/android/youtube/playback/{sessionId}/subtitles/{trackId}.vtt") { - val sessionId = call.parameters["sessionId"] - ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing sessionId")) - val trackId = call.parameters["trackId"] - ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing trackId")) - subtitles.content(call, sessionId, trackId) - } - get("/android/youtube/playback/{sessionId}/{itag}/init") { - val sessionId = call.parameters["sessionId"] - ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing sessionId")) - val itag = call.parameters["itag"]?.toIntOrNull() - ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid itag")) - media.initialization(call, sessionId, itag) - } - get("/android/youtube/playback/{sessionId}/{itag}/segment/{sequence}") { - val sessionId = call.parameters["sessionId"] - ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing sessionId")) - val itag = call.parameters["itag"]?.toIntOrNull() - ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid itag")) - val sequence = call.parameters["sequence"]?.toIntOrNull() - ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid sequence")) - media.segment(call, sessionId, itag, sequence) - } -} diff --git a/src/main/kotlin/dev/typetype/server/routes/AndroidSubtitleHandler.kt b/src/main/kotlin/dev/typetype/server/routes/AndroidSubtitleHandler.kt deleted file mode 100644 index af15ac13..00000000 --- a/src/main/kotlin/dev/typetype/server/routes/AndroidSubtitleHandler.kt +++ /dev/null @@ -1,61 +0,0 @@ -package dev.typetype.server.routes - -import dev.typetype.server.services.AccessControlService -import dev.typetype.server.services.AdminSettingsService -import dev.typetype.server.services.AndroidPlaybackService -import dev.typetype.server.services.AndroidPlaybackSession -import dev.typetype.server.services.AndroidSubtitleContentResult -import dev.typetype.server.services.AndroidSubtitleService -import dev.typetype.server.services.AuthService -import io.ktor.http.ContentType -import io.ktor.http.HttpStatusCode -import io.ktor.server.application.ApplicationCall -import io.ktor.server.response.respondBytes - -internal class AndroidSubtitleHandler( - private val playbackService: AndroidPlaybackService, - private val subtitleService: AndroidSubtitleService, - private val authService: AuthService?, - private val accessControlService: AccessControlService?, - private val adminSettingsService: AdminSettingsService?, -) { - suspend fun content(call: ApplicationCall, sessionId: String, trackId: String) { - call.response.headers.append("Cache-Control", "no-store") - val session = call.authorizedSession(sessionId) ?: return - val track = session.subtitles.firstOrNull { it.id == trackId } - ?: return call.notFound() - when (val result = subtitleService.content(session.holder.key.videoId, track)) { - is AndroidSubtitleContentResult.Ready -> call.respondBytes( - result.bytes, - ContentType.parse("text/vtt; charset=utf-8"), - HttpStatusCode.OK, - ) - AndroidSubtitleContentResult.TemporaryFailure -> call.respondAndroidError( - HttpStatusCode.ServiceUnavailable, - "android_subtitle_upstream_unavailable", - "Android subtitle is temporarily unavailable", - ) - AndroidSubtitleContentResult.Unavailable -> call.respondAndroidError( - HttpStatusCode.UnprocessableEntity, - "android_subtitle_unavailable", - "Android subtitle cannot be produced", - ) - } - } - - private suspend fun ApplicationCall.authorizedSession(sessionId: String): AndroidPlaybackSession? { - val access = accessProfileOrRespond(authService, accessControlService, adminSettingsService) ?: return null - val session = androidPlaybackSession(playbackService, sessionId) ?: return null - if (session.holder.key.userId == (access.userId ?: "guest")) return session - notFound() - return null - } - - private suspend fun ApplicationCall.notFound() { - respondAndroidError( - HttpStatusCode.NotFound, - "android_subtitle_not_found", - "Android subtitle track not found", - ) - } -} diff --git a/src/main/kotlin/dev/typetype/server/routes/AudioOnlyResponseMapper.kt b/src/main/kotlin/dev/typetype/server/routes/AudioOnlyResponseMapper.kt index 1d82fd3d..6162efd2 100644 --- a/src/main/kotlin/dev/typetype/server/routes/AudioOnlyResponseMapper.kt +++ b/src/main/kotlin/dev/typetype/server/routes/AudioOnlyResponseMapper.kt @@ -19,8 +19,7 @@ internal fun AudioOnlyStreamSelection.toResponse( ): ExtractionResult { val token = tokenService.createToken(userId, url, preferOriginal, preferredLocale, stream.itag, stream.audioTrackId) val src = when (kind) { - AudioOnlyStreamKind.Progressive, AudioOnlyStreamKind.SabrProgressive -> - "/streams/audio-only/source?token=${encode(token)}" + AudioOnlyStreamKind.Progressive -> "/streams/audio-only/source?token=${encode(token)}" AudioOnlyStreamKind.Hls -> hlsTokenService?.createPath(stream.url) ?: return ExtractionResult.Failure("No audio-only stream is available") AudioOnlyStreamKind.Dash -> stream.manifestUrl?.let { audioOnlyDashManifest(it) } @@ -34,7 +33,7 @@ internal fun AudioOnlyStreamSelection.toResponse( mimeType = when (kind) { AudioOnlyStreamKind.Hls, AudioOnlyStreamKind.SabrHls -> HLS_MIME_TYPE AudioOnlyStreamKind.Dash -> DASH_MIME_TYPE - AudioOnlyStreamKind.Progressive, AudioOnlyStreamKind.SabrProgressive -> stream.mimeType + AudioOnlyStreamKind.Progressive -> stream.mimeType }, codec = stream.codec, bitrate = stream.bitrate, diff --git a/src/main/kotlin/dev/typetype/server/routes/AudioOnlyRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/AudioOnlyRoutes.kt index d7c9b102..761e5f02 100644 --- a/src/main/kotlin/dev/typetype/server/routes/AudioOnlyRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/AudioOnlyRoutes.kt @@ -7,13 +7,10 @@ import dev.typetype.server.services.AccessControlService import dev.typetype.server.services.AdminSettingsService import dev.typetype.server.services.AudioOnlyMediaTokenResult import dev.typetype.server.services.AudioOnlyMediaTokenService -import dev.typetype.server.services.AudioOnlyStreamKind import dev.typetype.server.services.AudioOnlyStreamResolver -import dev.typetype.server.services.AudioOnlyStreamSelection import dev.typetype.server.services.AuthService import dev.typetype.server.services.PublicHlsManifestTokenService import dev.typetype.server.services.ProxyService -import dev.typetype.server.services.SabrSessionStore import dev.typetype.server.services.StreamService import dev.typetype.server.services.isYoutubeUrl import io.ktor.http.HttpHeaders @@ -33,7 +30,6 @@ fun Route.audioOnlyContractRoutes( adminSettingsService: AdminSettingsService? = null, publicHlsManifestTokenService: PublicHlsManifestTokenService? = null, proxyService: ProxyService? = null, - sabrAudioOnlyUnavailableReason: (suspend (String, String?, AudioOnlyStreamSelection) -> String?)? = null, ) { get("/streams/audio-only") { val url = call.request.queryParameters["url"] @@ -55,15 +51,6 @@ fun Route.audioOnlyContractRoutes( if (!access.profile.allowsUploader(result.data.response.uploaderUrl, result.data.response.uploaderName)) { return@get call.respond(HttpStatusCode.Forbidden, ErrorResponse("Channel is not allowed")) } - val sabrReason = if (result.data.kind == AudioOnlyStreamKind.SabrProgressive) { - sabrAudioOnlyUnavailableReason?.invoke(url, access.userId, result.data) - } else null - if (sabrReason != null) { - return@get call.respond( - HttpStatusCode.UnprocessableEntity, - ErrorResponse(sabrReason), - ) - } call.response.headers.append(HttpHeaders.CacheControl, "no-store") when (val response = result.data.toResponse( tokenService, @@ -89,46 +76,29 @@ internal fun Route.audioOnlySourceRoutes( proxyService: ProxyService, tokenService: AudioOnlyMediaTokenService, youtubeSessionStreamInfo: (suspend (String, String) -> ExtractionResult?)? = null, - sabrSessionStore: SabrSessionStore? = null, ) { head("/streams/audio-only/source") { - val result = call.resolveAudioOnlySource(tokenService, streamService, youtubeSessionStreamInfo, sabrSessionStore) + val result = call.resolveAudioOnlySource(tokenService, streamService, youtubeSessionStreamInfo) ?: return@head - if (result.result is ExtractionResult.Success && result.result.data.kind == AudioOnlyStreamKind.SabrProgressive) { - val store = sabrSessionStore ?: return@head call.respond( - HttpStatusCode.UnprocessableEntity, - ErrorResponse("No audio-only stream is available"), - ) - return@head call.respondSabrAudioOnlyHead(store, result.token, result.result.data) - } call.respondAudioOnlyHead(result, proxyService) } get("/streams/audio-only/source") { - val source = call.resolveAudioOnlySource(tokenService, streamService, youtubeSessionStreamInfo, sabrSessionStore) + val source = call.resolveAudioOnlySource(tokenService, streamService, youtubeSessionStreamInfo) ?: return@get when (val result = source.result) { - is ExtractionResult.Success -> when (result.data.kind) { - AudioOnlyStreamKind.SabrProgressive -> { - val store = sabrSessionStore ?: return@get call.respond( - HttpStatusCode.UnprocessableEntity, - ErrorResponse("No audio-only stream is available"), - ) - call.respondSabrAudioOnlySource(store, source.token, result.data) - } - else -> call.respondProxyResult( - result = proxyService.pipe( - url = result.data.stream.url, - rangeHeader = proxyService.resolveAudioOnlyRangeHeader( - call.request.headers, - result.data.stream.url, - result.data.stream.contentLength, - ), - domandBid = null, - ) - .ensureProgressiveAudio(), - contentTypeOverride = result.data.stream.mimeType, + is ExtractionResult.Success -> call.respondProxyResult( + result = proxyService.pipe( + url = result.data.stream.url, + rangeHeader = proxyService.resolveAudioOnlyRangeHeader( + call.request.headers, + result.data.stream.url, + result.data.stream.contentLength, + ), + domandBid = null, ) - } + .ensureProgressiveAudio(), + contentTypeOverride = result.data.stream.mimeType, + ) is ExtractionResult.BadRequest -> call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) is ExtractionResult.Failure -> call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) } @@ -139,7 +109,6 @@ private suspend fun ApplicationCall.resolveAudioOnlySource( tokenService: AudioOnlyMediaTokenService, streamService: StreamService, youtubeSessionStreamInfo: (suspend (String, String) -> ExtractionResult?)?, - sabrSessionStore: SabrSessionStore?, ): AudioOnlySourceResolution? { val raw = request.queryParameters["token"] ?: return respond(HttpStatusCode.BadRequest, ErrorResponse("Missing 'token' parameter")).let { null } @@ -163,7 +132,7 @@ private suspend fun ApplicationCall.resolveAudioOnlySource( token.preferOriginal, token.preferredLocale, allowHls = false, - allowSabr = sabrSessionStore != null, + allowSabr = false, selectedItag = token.selectedItag, selectedAudioTrackId = token.selectedAudioTrackId, ), diff --git a/src/main/kotlin/dev/typetype/server/routes/AudioOnlySabrBodyCache.kt b/src/main/kotlin/dev/typetype/server/routes/AudioOnlySabrBodyCache.kt deleted file mode 100644 index c802004e..00000000 --- a/src/main/kotlin/dev/typetype/server/routes/AudioOnlySabrBodyCache.kt +++ /dev/null @@ -1,53 +0,0 @@ -package dev.typetype.server.routes - -internal object AudioOnlySabrBodyCache { - private val bodies = LinkedHashMap(8, 0.75f, true) - private var totalBytes = 0L - - @Synchronized - fun get(videoId: String, itag: Int, audioTrackId: String?): ByteArray? { - val entry = bodies[key(videoId, itag, audioTrackId)] ?: return null - if (System.currentTimeMillis() > entry.expiresAtMs) { - remove(key(videoId, itag, audioTrackId)) - return null - } - return entry.body - } - - @Synchronized - fun put(videoId: String, itag: Int, audioTrackId: String?, body: ByteArray): Unit { - remove(key(videoId, itag, audioTrackId)) - bodies[key(videoId, itag, audioTrackId)] = Entry( - body = body, - expiresAtMs = System.currentTimeMillis() + TTL_MS, - ) - totalBytes += body.size - trim() - } - - @Synchronized - private fun remove(key: String): Unit { - bodies.remove(key)?.let { totalBytes -= it.body.size } - } - - @Synchronized - private fun trim(): Unit { - val iterator = bodies.iterator() - val now = System.currentTimeMillis() - while (iterator.hasNext()) { - val entry = iterator.next().value - if (entry.expiresAtMs <= now || totalBytes > MAX_BYTES) { - totalBytes -= entry.body.size - iterator.remove() - } - } - } - - private fun key(videoId: String, itag: Int, audioTrackId: String?): String = - listOf(videoId, itag.toString(), audioTrackId.orEmpty()).joinToString(":") - - private data class Entry(val body: ByteArray, val expiresAtMs: Long) - - private const val TTL_MS = 5 * 60 * 1_000L - private const val MAX_BYTES = 64L * 1024L * 1024L -} diff --git a/src/main/kotlin/dev/typetype/server/routes/AudioOnlySabrMaterializer.kt b/src/main/kotlin/dev/typetype/server/routes/AudioOnlySabrMaterializer.kt deleted file mode 100644 index 7dd0eb4f..00000000 --- a/src/main/kotlin/dev/typetype/server/routes/AudioOnlySabrMaterializer.kt +++ /dev/null @@ -1,114 +0,0 @@ -package dev.typetype.server.routes - -import dev.typetype.server.services.SabrSessionHolder -import dev.typetype.server.services.SabrSessionStore -import kotlinx.coroutines.withTimeoutOrNull -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import java.io.ByteArrayOutputStream - -internal suspend fun materializeSabrAudioOnlyBody( - sabrSessionStore: SabrSessionStore, - holder: SabrSessionHolder, - init: ByteArray, -): ByteArray? { - val videoWasActive = holder.isVideoActive() - val audioWasActive = holder.isAudioActive() - holder.setActiveTracks(videoActive = false, audioActive = true) - return try { - withTimeoutOrNull(SABR_AUDIO_ONLY_MATERIALIZE_TIMEOUT_MS) { - materializeSabrAudioOnlyBodyActive(sabrSessionStore, holder, init) - } - } finally { - holder.setActiveTracks(videoActive = videoWasActive, audioActive = audioWasActive) - } -} - -private suspend fun materializeSabrAudioOnlyBodyActive( - sabrSessionStore: SabrSessionStore, - holder: SabrSessionHolder, - init: ByteArray, -): ByteArray? { - val output = ByteArrayOutputStream() - output.write(init) - var sequence = 1 - while (sequence <= MAX_SABR_AUDIO_ONLY_SEGMENTS) { - val endSegment = holder.session.streamState.getEndSegment(holder.audioFormat).toInt() - if (endSegment > 0 && sequence > endSegment) return output.toByteArray() - val request = SabrSegmentRequest.media(holder.audioFormat, sequence) - val bytes = sabrSessionStore.fetchAudioBytes(holder, request, sequence) - ?: return if (holder.session.isBeyondEnd(request) && sequence > 1) output.toByteArray() else null - output.write(bytes) - sequence++ - } - return null -} - -private suspend fun SabrSessionStore.fetchAudioBytes( - holder: SabrSessionHolder, - request: SabrSegmentRequest, - sequence: Int, -): ByteArray? { - cachedSegment(holder, request)?.let { cached -> - if (!cached.init && cached.itag == holder.audioFormat.itag && cached.sequence == sequence) return cached.bytes - } - val startMs = holder.audioTargetTimeMs(sequence) - fetchMediaAt(holder, startMs)?.firstOrNull { segment -> - !segment.header.isInitSegment && - segment.header.itag == holder.audioFormat.itag && - segment.header.sequenceNumber == sequence - }?.let { return it.data } - return fetchSegment(holder, request)?.takeIf { segment -> - !segment.header.isInitSegment && - segment.header.itag == holder.audioFormat.itag && - segment.header.sequenceNumber == sequence - }?.data ?: fetchAudioBytesFromFreshHolder(holder, sequence, startMs) -} - -private suspend fun SabrSessionStore.fetchAudioBytesFromFreshHolder( - holder: SabrSessionHolder, - sequence: Int, - targetMs: Long, -): ByteArray? { - val prepared = fetchInfo(holder.key.videoId, startTimeMs = targetMs, cachedFirst = false) ?: return null - val audio = SabrFormatSelector.audio( - prepared.info, - holder.audioFormat.itag, - holder.audioFormat.audioTrackId, - requireAac = true, - ) ?: return null - val video = SabrFormatSelector.lightestVideo(prepared.info) ?: return null - val freshHolder = getOrCreate( - holder.key.videoId, - holder.key.userId, - prepared.info, - audio, - video, - prepared.initialToken, - startTimeMs = targetMs, - startPump = false, - audioOnly = true, - ) - freshHolder.setActiveTracks(videoActive = true, audioActive = true) - fetchMediaAt(freshHolder, targetMs)?.firstOrNull { segment -> - !segment.header.isInitSegment && - segment.header.itag == audio.itag && - segment.header.sequenceNumber == sequence - }?.let { return it.data } - val request = SabrSegmentRequest.media(audio, sequence) - return fetchSegment(freshHolder, request)?.takeIf { segment -> - !segment.header.isInitSegment && - segment.header.itag == audio.itag && - segment.header.sequenceNumber == sequence - }?.data -} - -private fun SabrSessionHolder.audioTargetTimeMs(sequence: Int): Long { - val startMs = session.streamState.getSegmentStartMs(audioFormat, sequence).coerceAtLeast(0L) - val nextStartMs = session.streamState.getSegmentStartMs(audioFormat, sequence + 1).coerceAtLeast(0L) - if (nextStartMs <= startMs + 1L) return startMs - return minOf(startMs + SABR_AUDIO_ONLY_TARGET_OFFSET_MS, nextStartMs - 1L) -} - -private const val MAX_SABR_AUDIO_ONLY_SEGMENTS = 10_000 -private const val SABR_AUDIO_ONLY_TARGET_OFFSET_MS = 1_000L -private const val SABR_AUDIO_ONLY_MATERIALIZE_TIMEOUT_MS = 60_000L diff --git a/src/main/kotlin/dev/typetype/server/routes/AudioOnlySabrPreflight.kt b/src/main/kotlin/dev/typetype/server/routes/AudioOnlySabrPreflight.kt deleted file mode 100644 index 39e75a97..00000000 --- a/src/main/kotlin/dev/typetype/server/routes/AudioOnlySabrPreflight.kt +++ /dev/null @@ -1,53 +0,0 @@ -package dev.typetype.server.routes - -import dev.typetype.server.services.AudioOnlyStreamSelection -import dev.typetype.server.services.SabrSessionStore -import kotlinx.coroutines.withTimeoutOrNull - -internal suspend fun SabrSessionStore.isSabrAudioOnlyPlayable( - videoUrl: String, - userId: String?, - selection: AudioOnlyStreamSelection, -): Boolean = sabrAudioOnlyUnplayableReason(videoUrl, userId, selection) == null - -internal suspend fun SabrSessionStore.sabrAudioOnlyUnplayableReason( - videoUrl: String, - userId: String?, - selection: AudioOnlyStreamSelection, -): String? { - return withTimeoutOrNull(SABR_AUDIO_ONLY_PREFLIGHT_TIMEOUT_MS) { - sabrAudioOnlyUnplayableReasonWithinTimeout(videoUrl, userId, selection) - } ?: "SABR audio-only preflight timed out before complete audio body" -} - -private suspend fun SabrSessionStore.sabrAudioOnlyUnplayableReasonWithinTimeout( - videoUrl: String, - userId: String?, - selection: AudioOnlyStreamSelection, -): String? { - val videoId = videoUrl.youtubeVideoId() ?: return "Invalid YouTube URL" - if (AudioOnlySabrBodyCache.get(videoId, selection.stream.itag, selection.stream.audioTrackId) != null) { - return null - } - val prepared = fetchInfo(videoId, cachedFirst = true) ?: return "SABR probe failed" - val audio = SabrFormatSelector.audio(prepared.info, selection.stream.itag, selection.stream.audioTrackId, requireAac = true) - ?: return "No SABR audio for this video" - val video = SabrFormatSelector.lightestVideo(prepared.info) ?: return "No SABR video context for this audio" - val holder = getOrCreate( - videoId, - userId ?: videoId, - prepared.info, - audio, - video, - prepared.initialToken, - startPump = false, - audioOnly = true, - ) - holder.setActiveTracks(videoActive = true, audioActive = true) - val init = fetchInitializationData(holder, audio) ?: return "SABR audio initialization failed" - val body = materializeSabrAudioOnlyBody(this, holder, init) ?: return "SABR audio-only body is incomplete" - AudioOnlySabrBodyCache.put(videoId, audio.itag, audio.audioTrackId, body) - return null -} - -private const val SABR_AUDIO_ONLY_PREFLIGHT_TIMEOUT_MS = 25_000L diff --git a/src/main/kotlin/dev/typetype/server/routes/AudioOnlySabrSource.kt b/src/main/kotlin/dev/typetype/server/routes/AudioOnlySabrSource.kt deleted file mode 100644 index 7bb9d497..00000000 --- a/src/main/kotlin/dev/typetype/server/routes/AudioOnlySabrSource.kt +++ /dev/null @@ -1,133 +0,0 @@ -package dev.typetype.server.routes - -import dev.typetype.server.models.ErrorResponse -import dev.typetype.server.services.AudioOnlyMediaToken -import dev.typetype.server.services.AudioOnlyStreamSelection -import dev.typetype.server.services.SabrSessionStore -import io.ktor.http.HttpHeaders -import io.ktor.http.HttpStatusCode -import io.ktor.server.application.ApplicationCall -import io.ktor.server.response.respond -import io.ktor.server.response.respondBytes -import io.ktor.server.response.respondOutputStream - -internal suspend fun ApplicationCall.respondSabrAudioOnlySource( - sabrSessionStore: SabrSessionStore, - token: AudioOnlyMediaToken, - selection: AudioOnlyStreamSelection, -) { - val videoId = token.videoUrl.youtubeVideoId() - ?: return respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid YouTube URL")) - AudioOnlySabrBodyCache.get(videoId, selection.stream.itag, token.selectedAudioTrackId)?.let { body -> - return respondSabrAudioOnlyBytes(selection.stream.mimeType.orEmpty(), body, request.headers[HttpHeaders.Range]) - } - val prepared = sabrSessionStore.fetchInfo(videoId, cachedFirst = true) - ?: return respond(HttpStatusCode.UnprocessableEntity, ErrorResponse("SABR probe failed")) - val audio = SabrFormatSelector.audio(prepared.info, selection.stream.itag, token.selectedAudioTrackId, requireAac = true) - ?: return respond(HttpStatusCode.UnprocessableEntity, ErrorResponse("No SABR audio for this video")) - val video = SabrFormatSelector.lightestVideo(prepared.info) - ?: return respond(HttpStatusCode.UnprocessableEntity, ErrorResponse("No SABR video for this video")) - val holder = sabrSessionStore.getOrCreate( - videoId, - token.userId ?: videoId, - prepared.info, - audio, - video, - prepared.initialToken, - startPump = false, - audioOnly = true, - ) - holder.setActiveTracks(videoActive = true, audioActive = true) - val init = sabrSessionStore.fetchInitializationData(holder, audio) - ?: return respond(HttpStatusCode.NotFound, ErrorResponse("Segment not available")) - val body = materializeSabrAudioOnlyBody(sabrSessionStore, holder, init) - ?: return respond(HttpStatusCode.UnprocessableEntity, ErrorResponse("No complete SABR audio for this video")) - AudioOnlySabrBodyCache.put(videoId, audio.itag, audio.audioTrackId, body) - respondSabrAudioOnlyBytes(audio.mimeType.orEmpty(), body, request.headers[HttpHeaders.Range]) -} - -internal suspend fun ApplicationCall.respondSabrAudioOnlyHead( - sabrSessionStore: SabrSessionStore, - token: AudioOnlyMediaToken, - selection: AudioOnlyStreamSelection, -) { - val videoId = token.videoUrl.youtubeVideoId() - ?: return respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid YouTube URL")) - AudioOnlySabrBodyCache.get(videoId, selection.stream.itag, token.selectedAudioTrackId)?.let { body -> - return respondSabrAudioOnlyHeadBytes(selection.stream.mimeType.orEmpty(), body) - } - val prepared = sabrSessionStore.fetchInfo(videoId, cachedFirst = true) - ?: return respond(HttpStatusCode.UnprocessableEntity, ErrorResponse("SABR probe failed")) - val audio = SabrFormatSelector.audio(prepared.info, selection.stream.itag, token.selectedAudioTrackId, requireAac = true) - ?: return respond(HttpStatusCode.UnprocessableEntity, ErrorResponse("No SABR audio for this video")) - val video = SabrFormatSelector.lightestVideo(prepared.info) - ?: return respond(HttpStatusCode.UnprocessableEntity, ErrorResponse("No SABR video for this video")) - val holder = sabrSessionStore.getOrCreate( - videoId, - token.userId ?: videoId, - prepared.info, - audio, - video, - prepared.initialToken, - startPump = false, - audioOnly = true, - ) - holder.setActiveTracks(videoActive = true, audioActive = true) - val init = sabrSessionStore.fetchInitializationData(holder, audio) - ?: return respond(HttpStatusCode.NotFound, ErrorResponse("Segment not available")) - val body = materializeSabrAudioOnlyBody(sabrSessionStore, holder, init) - ?: return respond(HttpStatusCode.UnprocessableEntity, ErrorResponse("No complete SABR audio for this video")) - AudioOnlySabrBodyCache.put(videoId, audio.itag, audio.audioTrackId, body) - respondSabrAudioOnlyHeadBytes(audio.mimeType.orEmpty(), body) -} - -private suspend fun ApplicationCall.respondSabrAudioOnlyHeadBytes( - mimeType: String, - body: ByteArray, -): Unit { - val total = body.size.toLong() - response.headers.append(HttpHeaders.CacheControl, "no-store") - response.headers.append(HttpHeaders.AcceptRanges, "bytes") - when (val range = parseAudioOnlyByteRange(request.headers[HttpHeaders.Range], total)) { - is AudioOnlyByteRange.Satisfiable -> { - response.headers.append(HttpHeaders.ContentRange, "bytes ${range.first}-${range.last}/${range.total}") - respondOutputStream( - containerMime(mimeType), - HttpStatusCode.PartialContent, - range.last - range.first + 1L, - ) {} - } - is AudioOnlyByteRange.Unsatisfiable -> { - response.headers.append(HttpHeaders.ContentRange, "bytes */${range.total}") - respond(HttpStatusCode.RequestedRangeNotSatisfiable) - } - null -> { - respondOutputStream(containerMime(mimeType), HttpStatusCode.OK, total) {} - } - } -} - -private suspend fun ApplicationCall.respondSabrAudioOnlyBytes( - mimeType: String, - body: ByteArray, - rangeHeader: String?, -): Unit { - val total = body.size.toLong() - response.headers.append(HttpHeaders.CacheControl, "no-store") - response.headers.append(HttpHeaders.AcceptRanges, "bytes") - when (val range = parseAudioOnlyByteRange(rangeHeader, total)) { - is AudioOnlyByteRange.Satisfiable -> { - response.headers.append(HttpHeaders.ContentRange, "bytes ${range.first}-${range.last}/${range.total}") - respondBytes( - body.copyOfRange(range.first.toInt(), (range.last + 1L).toInt()), - containerMime(mimeType), - HttpStatusCode.PartialContent, - ) - } - is AudioOnlyByteRange.Unsatisfiable -> { - response.headers.append(HttpHeaders.ContentRange, "bytes */${range.total}") - respond(HttpStatusCode.RequestedRangeNotSatisfiable) - } - null -> respondBytes(body, containerMime(mimeType), HttpStatusCode.OK) - } -} diff --git a/src/main/kotlin/dev/typetype/server/routes/BlockedRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/BlockedRoutes.kt index 486136c6..ba55a069 100644 --- a/src/main/kotlin/dev/typetype/server/routes/BlockedRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/BlockedRoutes.kt @@ -1,6 +1,7 @@ package dev.typetype.server.routes import dev.typetype.server.models.BlockedItem +import dev.typetype.server.models.BlockedKeywordItem import dev.typetype.server.models.ErrorResponse import dev.typetype.server.services.AuthService import dev.typetype.server.services.BlockedService @@ -56,4 +57,31 @@ fun Route.blockedRoutes(blockedService: BlockedService, authService: AuthService if (deleted) call.respond(HttpStatusCode.NoContent) else call.respond(HttpStatusCode.NotFound, ErrorResponse("Not found")) } } + get("/blocked/keywords") { + call.withJwtAuth(authService) { userId -> call.respond(blockedService.getKeywords(userId)) } + } + post("/blocked/keywords") { + call.withJwtAuth(authService) { userId -> + val item = runCatching { call.receive() }.getOrElse { + return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) + } + val keyword = item.keyword.trim() + if (keyword.isEmpty() || keyword.length > 100) { + return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Keyword must contain 1 to 100 characters")) + } + val role = authService.getUserRole(userId) ?: "user" + val global = (role == "admin" || role == "moderator") && item.global == true + call.respond(HttpStatusCode.Created, blockedService.addKeyword(userId, keyword, global)) + } + } + delete("/blocked/keywords/{keyword...}") { + call.withJwtAuth(authService) { userId -> + val keyword = call.urlTailParameter("keyword") + ?: return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing keyword")) + val role = authService.getUserRole(userId) ?: "user" + val deleted = blockedService.deleteKeyword(userId, keyword, role) + if (deleted) call.respond(HttpStatusCode.NoContent) + else call.respond(HttpStatusCode.NotFound, ErrorResponse("Not found")) + } + } } diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrDownloadRangeParser.kt b/src/main/kotlin/dev/typetype/server/routes/SabrDownloadRangeParser.kt new file mode 100644 index 00000000..d271c443 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/SabrDownloadRangeParser.kt @@ -0,0 +1,10 @@ +package dev.typetype.server.routes + +import dev.typetype.server.services.SabrDownloadRange +import io.ktor.server.application.ApplicationCall + +internal fun ApplicationCall.sabrDownloadRange(): Result = runCatching { + val parts = request.queryParameters["parts"]?.toIntOrNull() ?: 1 + val part = request.queryParameters["part"]?.toIntOrNull() ?: 0 + SabrDownloadRange(part, parts) +} diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrFormatSelector.kt b/src/main/kotlin/dev/typetype/server/routes/SabrFormatSelector.kt index 47b9c700..1e8e5cdc 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrFormatSelector.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrFormatSelector.kt @@ -14,13 +14,6 @@ internal object SabrFormatSelector { info.formats.filter { it.isSupportedVideo() } .minWithOrNull(compareBy { it.height }.thenBy { it.bitrate }) - fun androidVideo(info: YoutubeSabrInfo, itag: Int?): YoutubeSabrFormat? { - if (itag != null) return video(info, itag) - val supported = info.formats.filter { it.isSupportedVideo() } - return supported.filter { it.videoCodec().contains("avc1") }.maxByOrNull { it.height } - ?: supported.maxByOrNull { it.height } - } - fun audio(info: YoutubeSabrInfo, itag: Int?, trackId: String?, requireAac: Boolean): YoutubeSabrFormat? { if (itag != null) return info.formats.filter { it.matchesAudio(itag, trackId, requireAac) } .maxWithOrNull(audioComparator) diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrManifestHandler.kt b/src/main/kotlin/dev/typetype/server/routes/SabrManifestHandler.kt index 5fa2817f..f24a01e4 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrManifestHandler.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrManifestHandler.kt @@ -29,9 +29,17 @@ internal class SabrManifestHandler( private val audioOnlyTokenService: AudioOnlyMediaTokenService?, ) { private val accessResolver = SabrManifestAccessResolver(audioOnlyTokenService) + private val responder = SabrManifestResponder(sabrSessionStore) - suspend fun handle(call: ApplicationCall, videoId: String) { + suspend fun handle(call: ApplicationCall, videoId: String, download: Boolean = false) { val audioOnly = call.request.queryParameters["audioOnly"].equals("true", ignoreCase = true) + val downloadRange = if (download) { + call.sabrDownloadRange().getOrElse { + return call.respond(HttpStatusCode.BadRequest, ErrorResponse(it.message ?: "Invalid download range")) + } + } else { + null + } val hls = call.request.queryParameters["format"].equals("hls", ignoreCase = true) val playlist = call.request.queryParameters["playlist"] val sessionToken = call.request.queryParameters["session"] @@ -39,10 +47,12 @@ internal class SabrManifestHandler( return handleHlsPlaylist(call, videoId, playlist) } if (sessionToken != null) { + if (download) { + return call.respond(HttpStatusCode.BadRequest, ErrorResponse("Download streams require a new SABR session")) + } val holder = sabrSessionStore.lookupByToken(videoId, sessionToken) ?: return call.respond(HttpStatusCode.NotFound, ErrorResponse("No active SABR session for this request")) - sabrSessionStore.startPump(holder) - return call.respondSabrManifest(holder, videoId, audioOnly, hls) + return responder.respond(call, holder, videoId, audioOnly, hls, download = false) } val manifestAccess = accessResolver.resolve(call, videoId) ?: return val access = when (manifestAccess) { @@ -64,10 +74,15 @@ internal class SabrManifestHandler( return call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) } } - val startTimeMs = call.request.queryParameters["playerTimeMs"]?.toLongOrNull()?.coerceAtLeast(0L) + val requestedStartTimeMs = call.request.queryParameters["playerTimeMs"]?.toLongOrNull()?.coerceAtLeast(0L) ?: call.request.queryParameters["startTimeMs"]?.toLongOrNull()?.coerceAtLeast(0L) ?: 0L - val prepared = sabrSessionStore.fetchInfo(videoId, startTimeMs, cachedFirst = true) + val prepared = sabrSessionStore.fetchInfo( + videoId, + if (downloadRange == null) requestedStartTimeMs else 0L, + cachedFirst = !download, + isolatedPlayback = download, + ) ?: return call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse("SABR probe failed")) val audioToken = (manifestAccess as? SabrManifestAccess.AudioOnlyToken)?.token val audio = SabrFormatSelector.audio( @@ -88,13 +103,26 @@ internal class SabrManifestHandler( HttpStatusCode.UnprocessableEntity, ErrorResponse("No SABR video for this video"), ) + val startTimeMs = downloadRange?.startTimeMs(audio, video, audioOnly) ?: requestedStartTimeMs val userId = audioToken?.userId ?: access?.userId ?: videoId - val purpose = if (call.request.queryParameters["workload"] == "download") { + val purpose = if (download || call.request.queryParameters["workload"] == "download") { SabrSessionPurpose.DOWNLOAD } else { SabrSessionPurpose.MANIFEST } val holder = createHolder(videoId, userId, prepared, audio, video, startTimeMs, purpose, audioOnly) + if (download) { + holder.setActiveTracks(videoActive = !audioOnly, audioActive = true) + return responder.respond( + call, + holder, + videoId, + audioOnly, + hls = false, + download = true, + downloadRange = downloadRange, + ) + } if (audioOnly) { holder.setActiveTracks(videoActive = false, audioActive = true) sabrSessionStore.ensureWarmed(holder) @@ -103,11 +131,9 @@ internal class SabrManifestHandler( if (readyHolder == null) { return call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse("SABR preflight failed")) } - sabrSessionStore.startPump(readyHolder) - return call.respondSabrManifest(readyHolder, videoId, audioOnly, hls) + return responder.respond(call, readyHolder, videoId, audioOnly, hls, download = false) } - sabrSessionStore.startPump(holder) - call.respondSabrManifest(holder, videoId, audioOnly, hls) + responder.respond(call, holder, videoId, audioOnly, hls, download = false) } private suspend fun preflightOrRecreate( diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrManifestResponder.kt b/src/main/kotlin/dev/typetype/server/routes/SabrManifestResponder.kt new file mode 100644 index 00000000..b1c917b3 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/SabrManifestResponder.kt @@ -0,0 +1,41 @@ +package dev.typetype.server.routes + +import dev.typetype.server.services.SabrDownloadStreamer +import dev.typetype.server.services.SabrDownloadRange +import dev.typetype.server.services.SabrSessionHolder +import dev.typetype.server.services.SabrSessionStore +import io.ktor.http.ContentType +import io.ktor.http.HttpStatusCode +import io.ktor.server.application.ApplicationCall +import io.ktor.server.response.respondOutputStream + +internal class SabrManifestResponder(private val store: SabrSessionStore) { + private val downloadStreamer = SabrDownloadStreamer(store) + + suspend fun respond( + call: ApplicationCall, + holder: SabrSessionHolder, + videoId: String, + audioOnly: Boolean, + hls: Boolean, + download: Boolean, + downloadRange: SabrDownloadRange? = null, + ) { + if (!download) { + store.startPump(holder) + return call.respondSabrManifest(holder, videoId, audioOnly, hls) + } + call.response.headers.append("Cache-Control", "no-store") + try { + call.respondOutputStream(DOWNLOAD_CONTENT_TYPE, HttpStatusCode.OK) { + downloadStreamer.stream(holder, this, requireNotNull(downloadRange)) + } + } finally { + store.release(holder) + } + } + + private companion object { + val DOWNLOAD_CONTENT_TYPE = ContentType.parse("application/vnd.typetype.sabr-download") + } +} diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/SabrRoutes.kt index 463235ea..2f4abe7f 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrRoutes.kt @@ -124,6 +124,11 @@ internal fun Route.sabrRoutes( ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing videoId")) manifestHandler.handle(call, videoId) } + get("/sabr/download/{videoId}") { + val videoId = call.parameters["videoId"] + ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing videoId")) + manifestHandler.handle(call, videoId, download = true) + } get("/sabr/{videoId}/{itag}/init") { segmentHandler.handle(call, isInit = true, seq = 0) } diff --git a/src/main/kotlin/dev/typetype/server/routes/StreamDeliveryMode.kt b/src/main/kotlin/dev/typetype/server/routes/StreamDeliveryMode.kt index 1ab37c08..6ab72c9e 100644 --- a/src/main/kotlin/dev/typetype/server/routes/StreamDeliveryMode.kt +++ b/src/main/kotlin/dev/typetype/server/routes/StreamDeliveryMode.kt @@ -3,8 +3,6 @@ package dev.typetype.server.routes import java.net.URI internal enum class StreamDeliveryMode { - GenericLegacy, - YoutubeLegacy, YoutubeSabr, NicoNico, BiliBili; @@ -12,21 +10,11 @@ internal enum class StreamDeliveryMode { fun accepts(url: String): Boolean { val host = runCatching { URI(url).host?.lowercase() }.getOrNull() ?: return false return when (this) { - GenericLegacy -> true - YoutubeLegacy, YoutubeSabr -> host == "youtu.be" || host.endsWith(".youtube.com") || host == "youtube.com" + YoutubeSabr -> host == "youtu.be" || host.endsWith(".youtube.com") || host == "youtube.com" NicoNico -> host == "nico.ms" || host.endsWith(".nicovideo.jp") || host == "nicovideo.jp" BiliBili -> host == "b23.tv" || host.endsWith(".bilibili.com") || host == "bilibili.com" } } - fun usesYoutubeSession(url: String): Boolean = when (this) { - GenericLegacy -> runCatching { URI(url).host?.lowercase() }.getOrNull()?.let { - it == "youtu.be" || it.endsWith(".youtube.com") || it == "youtube.com" - } == true - YoutubeLegacy -> true - YoutubeSabr -> false - NicoNico, BiliBili -> false - } - fun isSabr(): Boolean = this == YoutubeSabr } diff --git a/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt b/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt index ca4d54c1..04bcb785 100644 --- a/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt +++ b/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt @@ -1,6 +1,5 @@ package dev.typetype.server.routes -import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.StreamResponse import dev.typetype.server.services.AccessControlService import dev.typetype.server.services.AdminSettingsService @@ -9,7 +8,6 @@ import dev.typetype.server.services.PublicHlsManifestTokenService internal data class StreamRouteDependencies( val authService: AuthService?, - val youtubeSessionStreamInfo: (suspend (String, String) -> ExtractionResult?)?, val accessControlService: AccessControlService?, val adminSettingsService: AdminSettingsService?, val publicHlsManifestTokenService: PublicHlsManifestTokenService?, diff --git a/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt index 4be035bd..872aa9f2 100644 --- a/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt @@ -7,13 +7,11 @@ import dev.typetype.server.services.AccessControlService import dev.typetype.server.services.AdminSettingsService import dev.typetype.server.services.AuthService import dev.typetype.server.services.PublicHlsManifestTokenService -import dev.typetype.server.services.SignedHlsManifestCookie import dev.typetype.server.services.StreamService -import dev.typetype.server.services.YOUTUBE_SESSION_RECONNECT_ERROR import dev.typetype.server.services.filterAllowed import dev.typetype.server.services.withSabrManifestUrls -import io.ktor.http.HttpStatusCode import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode import io.ktor.server.response.respond import io.ktor.server.routing.Route import io.ktor.server.routing.get @@ -24,35 +22,28 @@ private const val AUTHENTICATED_STREAMS_CACHE_CONTROL = "no-store" fun Route.streamRoutes( streamService: StreamService, authService: AuthService? = null, - youtubeSessionStreamInfo: (suspend (String, String) -> ExtractionResult?)? = null, accessControlService: AccessControlService? = null, adminSettingsService: AdminSettingsService? = null, publicHlsManifestTokenService: PublicHlsManifestTokenService? = null, - legacyStreamService: StreamService = streamService, - genericLegacyStreamService: StreamService = legacyStreamService, - nicoNicoStreamService: StreamService = legacyStreamService, - bilibiliStreamService: StreamService = legacyStreamService, + nicoNicoStreamService: StreamService = streamService, + bilibiliStreamService: StreamService = streamService, sabrBootstrapStreamService: StreamService = streamService, sabrStreamContractFilter: (suspend (String, StreamResponse) -> StreamResponse)? = null, -): Unit { +) { val dependencies = StreamRouteDependencies( authService = authService, - youtubeSessionStreamInfo = youtubeSessionStreamInfo, accessControlService = accessControlService, adminSettingsService = adminSettingsService, publicHlsManifestTokenService = publicHlsManifestTokenService, sabrStreamContractFilter = sabrStreamContractFilter, ) - streamRoute("/streams", StreamDeliveryMode.GenericLegacy, genericLegacyStreamService, dependencies) streamRoute("/streams/youtube/sabr", StreamDeliveryMode.YoutubeSabr, streamService, dependencies) streamRoute( "/streams/youtube/sabr/bootstrap", StreamDeliveryMode.YoutubeSabr, sabrBootstrapStreamService, - dependencies.copy(youtubeSessionStreamInfo = null), + dependencies, ) - streamRoute("/streams/legacy", StreamDeliveryMode.GenericLegacy, genericLegacyStreamService, dependencies) - streamRoute("/streams/youtube/legacy", StreamDeliveryMode.YoutubeLegacy, legacyStreamService, dependencies) streamRoute("/streams/niconico", StreamDeliveryMode.NicoNico, nicoNicoStreamService, dependencies) streamRoute("/streams/bilibili", StreamDeliveryMode.BiliBili, bilibiliStreamService, dependencies) } @@ -62,7 +53,7 @@ private fun Route.streamRoute( deliveryMode: StreamDeliveryMode, streamService: StreamService, dependencies: StreamRouteDependencies, -): Unit { +) { get(path) { val url = call.request.queryParameters["url"] ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing 'url' parameter")) @@ -72,23 +63,13 @@ private fun Route.streamRoute( ErrorResponse("URL does not match stream endpoint", "provider_mismatch"), ) } - val access = call.accessProfileOrRespond( dependencies.authService, dependencies.accessControlService, dependencies.adminSettingsService, ) ?: return@get - val userId = access.userId - val sessionResult = fetchYoutubeSession(userId, url, deliveryMode, dependencies.youtubeSessionStreamInfo) - .forDeliveryMode(deliveryMode) - val publicResult = if (sessionResult.hasPlayableSource()) { - null - } else { - streamService.getStreamInfo(url) - } - val usedYoutubeSession = sessionResult != null val accessProfile = access.profile - when (val result = publicResult.resolveWith(sessionResult)) { + when (val result = streamService.getStreamInfo(url)) { is ExtractionResult.Success -> { if (!accessProfile.allowsUploader(result.data.uploaderUrl, result.data.uploaderName)) { return@get call.respond(HttpStatusCode.Forbidden, ErrorResponse("Channel is not allowed")) @@ -101,7 +82,7 @@ private fun Route.streamRoute( val filtered = selected .filterAllowed(accessProfile) .withSignedPublicHlsUrl( - deliveryMode.isSabr() && selected.isLive || userId != null && !access.allowGuest, + deliveryMode.isSabr() && selected.isLive || access.userId != null && !access.allowGuest, dependencies.publicHlsManifestTokenService, ) val data = if (!deliveryMode.isSabr() || dependencies.sabrStreamContractFilter == null) { @@ -117,70 +98,24 @@ private fun Route.streamRoute( } call.response.headers.append( HttpHeaders.CacheControl, - if (usedYoutubeSession || accessProfile.enabled) AUTHENTICATED_STREAMS_CACHE_CONTROL else STREAMS_CACHE_CONTROL, + if (accessProfile.enabled) AUTHENTICATED_STREAMS_CACHE_CONTROL else STREAMS_CACHE_CONTROL, ) - if (usedYoutubeSession) { - SignedHlsManifestCookie.tokenFromPath(data.hlsUrl) - ?.let { SignedHlsManifestCookie.append(call.response, url, it) } - } call.respond(data) } - is ExtractionResult.BadRequest -> call.respond(HttpStatusCode.BadRequest, result.toErrorResponse()) - is ExtractionResult.Failure -> call.respond(HttpStatusCode.UnprocessableEntity, result.toErrorResponse()) + is ExtractionResult.BadRequest -> + call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message, result.code)) + is ExtractionResult.Failure -> + call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message, result.code)) } } } -private suspend fun fetchYoutubeSession( - userId: String?, - url: String, - deliveryMode: StreamDeliveryMode, - youtubeSessionStreamInfo: (suspend (String, String) -> ExtractionResult?)?, -): ExtractionResult? = - if (deliveryMode.usesYoutubeSession(url) && userId != null && youtubeSessionStreamInfo != null) { - youtubeSessionStreamInfo(userId, url) - } else { - null - } - -private fun ExtractionResult?.forDeliveryMode( - deliveryMode: StreamDeliveryMode, -): ExtractionResult? = when { - this !is ExtractionResult.Success -> this - deliveryMode.isSabr() -> ExtractionResult.Success(data.withSabrManifestUrls().onlySabrStreams()) - else -> ExtractionResult.Success(data.withoutSabrStreams()) -} - -private fun ExtractionResult?.resolveWith( - sessionResult: ExtractionResult?, -): ExtractionResult { - if (this == null && sessionResult != null) return sessionResult - if (this == null) return ExtractionResult.Failure("Stream extraction failed") - if (sessionResult == null) return this - if (this is ExtractionResult.Success && sessionResult is ExtractionResult.Success) { - return ExtractionResult.Success(sessionResult.data.mergeWithPublic(data)) - } - if (this is ExtractionResult.Success) return this - return sessionResult -} - -private fun StreamResponse.mergeWithPublic(public: StreamResponse): StreamResponse = - copy( - hlsUrl = hlsUrl.ifBlank { public.hlsUrl }, - dashMpdUrl = dashMpdUrl.ifBlank { public.dashMpdUrl }, - videoStreams = videoStreams.ifEmpty { public.videoStreams }, - videoOnlyStreams = videoOnlyStreams.ifEmpty { public.videoOnlyStreams }, - audioStreams = audioStreams.ifEmpty { public.audioStreams }, - ) - -private fun ExtractionResult?.hasPlayableSource(): Boolean = - this is ExtractionResult.Success && data.hasPlayableSource() - private fun StreamResponse.hasPlayableSource(): Boolean = - playableStreamCount() > 0 || hlsUrl.isNotBlank() || dashMpdUrl.isNotBlank() - -private fun StreamResponse.playableStreamCount(): Int = - videoStreams.size + videoOnlyStreams.size + audioStreams.size + videoStreams.isNotEmpty() || + videoOnlyStreams.isNotEmpty() || + audioStreams.isNotEmpty() || + hlsUrl.isNotBlank() || + dashMpdUrl.isNotBlank() private fun StreamResponse.withSignedPublicHlsUrl( shouldSign: Boolean, @@ -190,12 +125,3 @@ private fun StreamResponse.withSignedPublicHlsUrl( if (hlsUrl.startsWith("/streams/hls-manifest?token=")) return this return copy(hlsUrl = tokenService.createPath(hlsUrl)) } - -private fun ExtractionResult.BadRequest.toErrorResponse(): ErrorResponse = - if (message == YOUTUBE_SESSION_RECONNECT_ERROR) { - ErrorResponse(message, "youtube_session_needs_reconnect") - } else { - ErrorResponse(message, code) - } - -private fun ExtractionResult.Failure.toErrorResponse(): ErrorResponse = ErrorResponse(message, code) diff --git a/src/main/kotlin/dev/typetype/server/routes/TypeTypeBackupRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/TypeTypeBackupRoutes.kt new file mode 100644 index 00000000..6ffa9751 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/TypeTypeBackupRoutes.kt @@ -0,0 +1,79 @@ +package dev.typetype.server.routes + +import dev.typetype.server.cache.CacheJson +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.models.TypeTypeBackupItem +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.PipePipeBackupUploadWriter +import dev.typetype.server.services.TypeTypeBackupCategory +import dev.typetype.server.services.TypeTypeBackupService +import io.ktor.http.ContentDisposition +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.http.content.PartData +import io.ktor.server.application.call +import io.ktor.server.request.receiveMultipart +import io.ktor.server.response.header +import io.ktor.server.response.respond +import io.ktor.server.routing.Route +import io.ktor.server.routing.get +import io.ktor.server.routing.post +import kotlinx.coroutines.CancellationException +import java.nio.file.Files +import java.time.LocalDate +import java.time.ZoneOffset + +private const val MAX_TYPE_TYPE_BACKUP_BYTES = 128L * 1024 * 1024 + +fun Route.typeTypeBackupRoutes(service: TypeTypeBackupService, authService: AuthService) { + get("/backup/typetype") { + call.withJwtAuth(authService) { userId -> + val categories = TypeTypeBackupCategory.parse(call.request.queryParameters["categories"]) + ?: return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid backup categories")) + val filename = "typetype-backup-${LocalDate.now(ZoneOffset.UTC)}.json" + call.response.header( + HttpHeaders.ContentDisposition, + ContentDisposition.Attachment + .withParameter(ContentDisposition.Parameters.FileName, filename) + .toString(), + ) + call.respond(service.export(userId, categories)) + } + } + post("/restore/typetype") { + call.withJwtAuth(authService) { userId -> + val tmp = Files.createTempFile("typetype-backup-", ".json") + try { + val multipart = call.receiveMultipart(MAX_TYPE_TYPE_BACKUP_BYTES) + var fileCount = 0 + while (true) { + val part = multipart.readPart() ?: break + if (part is PartData.FileItem && part.name == "file") { + PipePipeBackupUploadWriter.writeWithLimit( + part.provider(), + tmp, + MAX_TYPE_TYPE_BACKUP_BYTES, + ) + fileCount += 1 + } + part.release() + } + if (fileCount != 1) { + return@withJwtAuth call.respond( + HttpStatusCode.BadRequest, + ErrorResponse("Exactly one backup file is required"), + ) + } + val backup = CacheJson.decodeFromString(Files.readString(tmp)) + call.respond(service.restore(userId, backup)) + } catch (error: CancellationException) { + throw error + } catch (error: Exception) { + call.application.environment.log.info("TypeType restore failed", error) + call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid TypeType backup")) + } finally { + Files.deleteIfExists(tmp) + } + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt index f1f07980..0c169e0c 100644 --- a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt @@ -37,6 +37,7 @@ internal fun Route.userDataRoutes( accountIdentityRoutes(svc.accountIdentityService, authService) bugReportRoutes(bugReportService, authService) restoreRoutes(restoreService, authService) + typeTypeBackupRoutes(svc.typeTypeBackupService, authService) homeRecommendationRoutes(svc.homeRecommendationService, authService, svc.accessControlService) homeRecommendationShortsRoutes(svc.homeRecommendationService, authService, svc.accessControlService) } diff --git a/src/main/kotlin/dev/typetype/server/services/AndroidDashManifestBuilder.kt b/src/main/kotlin/dev/typetype/server/services/AndroidDashManifestBuilder.kt deleted file mode 100644 index f366385c..00000000 --- a/src/main/kotlin/dev/typetype/server/services/AndroidDashManifestBuilder.kt +++ /dev/null @@ -1,112 +0,0 @@ -package dev.typetype.server.services - -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import java.io.StringWriter -import javax.xml.stream.XMLOutputFactory -import javax.xml.stream.XMLStreamWriter - -internal object AndroidDashManifestBuilder { - fun build( - sessionId: String, - generation: Long, - audio: AndroidDashTrack, - video: AndroidDashTrack, - durationMs: Long, - ): String { - val output = StringWriter() - val xml = XMLOutputFactory.newFactory().createXMLStreamWriter(output) - xml.writeStartDocument("UTF-8", "1.0") - xml.writeStartElement("MPD") - xml.writeDefaultNamespace(DASH_NAMESPACE) - xml.writeAttribute("profiles", "urn:mpeg:dash:profile:full:2011") - xml.writeAttribute("type", "static") - xml.writeAttribute("mediaPresentationDuration", duration(durationMs)) - xml.writeAttribute("minBufferTime", "PT2S") - xml.writeStartElement("Period") - xml.writeAttribute("id", "0") - xml.writeAttribute("start", "PT0S") - xml.writeTrack(sessionId, generation, video) - xml.writeTrack(sessionId, generation, audio) - xml.writeEndElement() - xml.writeEndElement() - xml.writeEndDocument() - xml.close() - return output.toString() - } - - private fun XMLStreamWriter.writeTrack( - sessionId: String, - generation: Long, - track: AndroidDashTrack, - ) { - val format = track.format - val (mimeType, codecs) = splitMime(format.mimeType.orEmpty()) - writeStartElement("AdaptationSet") - writeAttribute("id", format.itag.toString()) - writeAttribute("contentType", if (format.isAudio) "audio" else "video") - writeAttribute("mimeType", mimeType) - writeAttribute("segmentAlignment", "true") - writeAttribute("startWithSAP", "1") - writeStartElement("Representation") - writeAttribute("id", format.itag.toString()) - writeAttribute("bandwidth", format.bitrate.coerceAtLeast(1).toString()) - if (codecs.isNotBlank()) writeAttribute("codecs", codecs) - if (format.isVideo) { - writeAttribute("width", format.width.coerceAtLeast(1).toString()) - writeAttribute("height", format.height.coerceAtLeast(1).toString()) - } - writeSegmentTemplate(sessionId, generation, track) - writeEndElement() - writeEndElement() - } - - private fun XMLStreamWriter.writeSegmentTemplate( - sessionId: String, - generation: Long, - track: AndroidDashTrack, - ) { - val format = track.format - val base = "/api/android/youtube/playback/$sessionId/${format.itag}" - val query = "?session=$sessionId&generation=$generation" - writeStartElement("SegmentTemplate") - writeAttribute("timescale", "1000") - writeAttribute("startNumber", track.timeline.startNumber.toString()) - writeAttribute("initialization", "$base/init$query") - writeAttribute("media", "$base/segment/\$Number\$$query") - writeStartElement("SegmentTimeline") - compressed(track.timeline.segments).forEach { segment -> - writeEmptyElement("S") - writeAttribute("t", segment.startMs.toString()) - writeAttribute("d", segment.durationMs.toString()) - if (segment.repeat > 0) writeAttribute("r", segment.repeat.toString()) - } - writeEndElement() - writeEndElement() - } - - private fun compressed(segments: List): List { - val result = ArrayList() - for (segment in segments) { - val previous = result.lastOrNull() - if (previous != null && previous.durationMs == segment.durationMs && previous.endMs == segment.startMs) { - previous.repeat++ - } else { - result += CompressedSegment(segment.startMs, segment.durationMs) - } - } - return result - } - - private fun duration(ms: Long): String = "PT${ms / 1_000}.${(ms % 1_000).toString().padStart(3, '0')}S" - - private class CompressedSegment(val startMs: Long, val durationMs: Long, var repeat: Int = 0) { - val endMs: Long get() = startMs + durationMs * (repeat + 1L) - } - - private const val DASH_NAMESPACE = "urn:mpeg:dash:schema:mpd:2011" -} - -internal data class AndroidDashTrack( - val format: YoutubeSabrFormat, - val timeline: AndroidDashTimeline, -) diff --git a/src/main/kotlin/dev/typetype/server/services/AndroidDashManifestService.kt b/src/main/kotlin/dev/typetype/server/services/AndroidDashManifestService.kt deleted file mode 100644 index 3a1eb105..00000000 --- a/src/main/kotlin/dev/typetype/server/services/AndroidDashManifestService.kt +++ /dev/null @@ -1,61 +0,0 @@ -package dev.typetype.server.services - -internal class AndroidDashManifestService { - fun build(holder: SabrSessionHolder): AndroidDashManifestResult { - if (holder.expectsLive()) return AndroidDashManifestResult.UnsupportedLive - val state = holder.session.streamState - val audio = AndroidDashTimelineReader.read(state, holder.audioFormat) - val video = AndroidDashTimelineReader.read(state, holder.videoFormat) - val audioPending = audio is AndroidDashTimelineResult.Pending - val videoPending = video is AndroidDashTimelineResult.Pending - if (audioPending || videoPending) { - val stage = when { - audioPending && videoPending -> AndroidPlaybackPreparationStage.AUDIO_VIDEO_INDEX - audioPending -> AndroidPlaybackPreparationStage.AUDIO_INDEX - else -> AndroidPlaybackPreparationStage.VIDEO_INDEX - } - return AndroidDashManifestResult.Preparing(stage) - } - if (audio is AndroidDashTimelineResult.Invalid) return AndroidDashManifestResult.Invalid(audio.reason) - if (video is AndroidDashTimelineResult.Invalid) return AndroidDashManifestResult.Invalid(video.reason) - audio as AndroidDashTimelineResult.Ready - video as AndroidDashTimelineResult.Ready - val durationMs = maxOf(audio.timeline.endMs, video.timeline.endMs) - if (durationMs <= 0L || durationMs - audio.timeline.endMs > COVERAGE_TOLERANCE_MS || - durationMs - video.timeline.endMs > COVERAGE_TOLERANCE_MS - ) { - return AndroidDashManifestResult.Invalid("Audio and video timelines do not cover the same presentation") - } - val manifest = AndroidDashManifestBuilder.build( - sessionId = holder.sessionToken, - generation = holder.activeGeneration(), - audio = AndroidDashTrack(holder.audioFormat, audio.timeline), - video = AndroidDashTrack(holder.videoFormat, video.timeline), - durationMs = durationMs, - ) - if (manifest.toByteArray(Charsets.UTF_8).size > MAX_MANIFEST_BYTES) { - return AndroidDashManifestResult.Invalid("DASH manifest exceeds the size limit") - } - return AndroidDashManifestResult.Ready(manifest, durationMs) - } - - private companion object { - const val COVERAGE_TOLERANCE_MS = 1_500L - const val MAX_MANIFEST_BYTES = 2 * 1024 * 1024 - } -} - -internal sealed interface AndroidDashManifestResult { - data class Ready(val manifest: String, val durationMs: Long) : AndroidDashManifestResult - data class Preparing(val stage: AndroidPlaybackPreparationStage) : AndroidDashManifestResult - data object UnsupportedLive : AndroidDashManifestResult - data class Invalid(val reason: String) : AndroidDashManifestResult - data class TemporaryFailure(val code: String, val reason: String) : AndroidDashManifestResult -} - -internal enum class AndroidPlaybackPreparationStage(val wireValue: String) { - AUDIO_INDEX("audio_index"), - VIDEO_INDEX("video_index"), - AUDIO_VIDEO_INDEX("audio_video_index"), - MEDIA_BYTES("media_bytes"), -} diff --git a/src/main/kotlin/dev/typetype/server/services/AndroidDashTimeline.kt b/src/main/kotlin/dev/typetype/server/services/AndroidDashTimeline.kt deleted file mode 100644 index 5e8118ab..00000000 --- a/src/main/kotlin/dev/typetype/server/services/AndroidDashTimeline.kt +++ /dev/null @@ -1,55 +0,0 @@ -package dev.typetype.server.services - -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState - -internal data class AndroidDashTimeline( - val startNumber: Int, - val segments: List, -) { - val endMs: Long get() = segments.last().endMs -} - -internal data class AndroidDashTimelineSegment( - val startMs: Long, - val durationMs: Long, -) { - val endMs: Long get() = startMs + durationMs -} - -internal sealed interface AndroidDashTimelineResult { - data class Ready(val timeline: AndroidDashTimeline) : AndroidDashTimelineResult - data object Pending : AndroidDashTimelineResult - data class Invalid(val reason: String) : AndroidDashTimelineResult -} - -internal object AndroidDashTimelineReader { - fun read( - state: YoutubeSabrStreamState, - format: YoutubeSabrFormat, - ): AndroidDashTimelineResult { - if (!state.hasSegmentIndex(format)) return AndroidDashTimelineResult.Pending - val endSegment = state.getEndSegment(format) - if (endSegment !in 1..MAX_SEGMENTS) { - return AndroidDashTimelineResult.Invalid("Invalid exact segment count for itag ${format.itag}") - } - val segments = ArrayList(endSegment.toInt()) - for (sequence in 1..endSegment.toInt()) { - val startMs = state.getSegmentStartMs(format, sequence) - val endMs = state.getSegmentEndMs(format, sequence) - if (startMs < 0L || endMs <= startMs) { - return AndroidDashTimelineResult.Invalid("Invalid exact timeline for itag ${format.itag}") - } - if (sequence == 1 && startMs != 0L) { - return AndroidDashTimelineResult.Invalid("Timeline does not begin at zero for itag ${format.itag}") - } - if (segments.lastOrNull()?.startMs?.let { startMs <= it } == true) { - return AndroidDashTimelineResult.Invalid("Timeline is not monotonic for itag ${format.itag}") - } - segments += AndroidDashTimelineSegment(startMs, endMs - startMs) - } - return AndroidDashTimelineResult.Ready(AndroidDashTimeline(startNumber = 1, segments)) - } - - private const val MAX_SEGMENTS = 100_000L -} diff --git a/src/main/kotlin/dev/typetype/server/services/AndroidPlaybackPreparation.kt b/src/main/kotlin/dev/typetype/server/services/AndroidPlaybackPreparation.kt deleted file mode 100644 index a6b6daed..00000000 --- a/src/main/kotlin/dev/typetype/server/services/AndroidPlaybackPreparation.kt +++ /dev/null @@ -1,115 +0,0 @@ -package dev.typetype.server.services - -import kotlinx.coroutines.CancellationException -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.CoroutineStart -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.Job -import kotlinx.coroutines.SupervisorJob -import kotlinx.coroutines.TimeoutCancellationException -import kotlinx.coroutines.launch -import kotlinx.coroutines.withTimeout -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.slf4j.LoggerFactory -import java.io.IOException -import java.util.concurrent.atomic.AtomicReference - -internal class AndroidPlaybackPreparation { - private val job = AtomicReference() - private val failure = AtomicReference() - - fun start(scope: CoroutineScope, block: suspend () -> Unit): Unit { - val candidate = scope.launch(start = CoroutineStart.LAZY) { block() } - if (job.compareAndSet(null, candidate)) candidate.start() else candidate.cancel() - } - - fun fail(result: AndroidDashManifestResult.TemporaryFailure): Unit { - failure.compareAndSet(null, result) - } - - fun result(holder: SabrSessionHolder, manifests: AndroidDashManifestService): AndroidDashManifestResult = - failure.get() ?: manifests.build(holder) - - fun cancel(): Unit { - job.getAndSet(null)?.cancel() - } -} - -internal class AndroidPlaybackPreparationCoordinator( - private val store: SabrSessionStore, - private val manifests: AndroidDashManifestService, - private val timeoutMs: Long = PREPARATION_TIMEOUT_MS, - private val initializationTimeoutMs: Long = INITIALIZATION_TIMEOUT_MS, - private val scope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO), - private val exactInitializer: suspend (SabrSessionHolder, YoutubeSabrFormat, Long) -> ByteArray = - store::fetchExactInitializationData, -) { - fun start(session: AndroidPlaybackSession): Unit { - if (manifests.build(session.holder) !is AndroidDashManifestResult.Preparing) return - session.preparation.start(scope) { prepare(session) } - } - - private suspend fun prepare(session: AndroidPlaybackSession): Unit { - val holder = session.holder - val startedAt = System.currentTimeMillis() - try { - withTimeout(timeoutMs) { - initializeIfMissing(holder, holder.videoFormat) - initializeIfMissing(holder, holder.audioFormat) - } - val result = manifests.build(holder) - if (result is AndroidDashManifestResult.Preparing) { - session.preparation.fail(temporaryFailure(PREPARATION_FAILED)) - } - logger.info( - "android_playback_preparation videoId={} videoItag={} audioItag={} elapsedMs={} result={}", - holder.key.videoId, - holder.videoFormat.itag, - holder.audioFormat.itag, - System.currentTimeMillis() - startedAt, - result::class.simpleName, - ) - } catch (error: TimeoutCancellationException) { - session.preparation.fail(temporaryFailure(PREPARATION_TIMEOUT)) - logFailure(holder, startedAt, error) - } catch (error: CancellationException) { - throw error - } catch (error: IOException) { - session.preparation.fail(temporaryFailure(PREPARATION_FAILED)) - logFailure(holder, startedAt, error) - } catch (error: RuntimeException) { - session.preparation.fail(temporaryFailure(PREPARATION_FAILED)) - logFailure(holder, startedAt, error) - } - } - - private suspend fun initializeIfMissing(holder: SabrSessionHolder, format: YoutubeSabrFormat): Unit { - if (holder.session.streamState.hasSegmentIndex(format)) return - exactInitializer(holder, format, initializationTimeoutMs) - if (!holder.session.streamState.hasSegmentIndex(format)) { - throw IOException("Exact initialization did not produce a segment index for itag ${format.itag}") - } - } - - private fun logFailure(holder: SabrSessionHolder, startedAt: Long, error: Throwable): Unit { - logger.warn( - "android_playback_preparation_failed videoId={} videoItag={} audioItag={} elapsedMs={} errorClass={}", - holder.key.videoId, - holder.videoFormat.itag, - holder.audioFormat.itag, - System.currentTimeMillis() - startedAt, - error::class.simpleName, - ) - } - - private fun temporaryFailure(code: String): AndroidDashManifestResult.TemporaryFailure = - AndroidDashManifestResult.TemporaryFailure(code, "Android playback preparation is temporarily unavailable") - - private companion object { - val logger = LoggerFactory.getLogger(AndroidPlaybackPreparationCoordinator::class.java) - const val PREPARATION_TIMEOUT_MS = 8_000L - const val INITIALIZATION_TIMEOUT_MS = 3_500L - const val PREPARATION_TIMEOUT = "android_playback_preparation_timeout" - const val PREPARATION_FAILED = "android_playback_preparation_failed" - } -} diff --git a/src/main/kotlin/dev/typetype/server/services/AndroidPlaybackService.kt b/src/main/kotlin/dev/typetype/server/services/AndroidPlaybackService.kt deleted file mode 100644 index 357c30ad..00000000 --- a/src/main/kotlin/dev/typetype/server/services/AndroidPlaybackService.kt +++ /dev/null @@ -1,118 +0,0 @@ -package dev.typetype.server.services - -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat - -internal class AndroidPlaybackService( - private val store: SabrSessionStore, - private val sessions: AndroidPlaybackSessionRegistry = AndroidPlaybackSessionRegistry(store), - private val manifests: AndroidDashManifestService = AndroidDashManifestService(), - private val preparation: AndroidPlaybackPreparationCoordinator = - AndroidPlaybackPreparationCoordinator(store, manifests), -) { - private val playback = SabrPlaybackSessionService(store, SabrSessionPurpose.ANDROID_PLAYBACK) - - suspend fun create( - videoId: String, - userId: String, - prepared: SabrPreparedInfo, - audio: YoutubeSabrFormat, - video: YoutubeSabrFormat, - subtitles: List, - ): AndroidPlaybackCreateResult { - if (prepared.isLive || prepared.isLiveContent) return AndroidPlaybackCreateResult.UnsupportedLive - val result = playback.prepare( - videoId = videoId, - userId = userId, - prepared = prepared, - audio = audio, - video = video, - startTimeMs = 0L, - preloadInitialization = false, - startPumpOnPrepare = false, - ) - val session = sessions.register(result.holder, subtitles) - preparation.start(session) - return AndroidPlaybackCreateResult.Created(session, session.preparation.result(result.holder, manifests)) - } - - fun lookup(sessionId: String): AndroidPlaybackSessionLookup = sessions.lookup(sessionId) - - fun seek( - session: AndroidPlaybackSession, - generation: Long, - playerTimeMs: Long, - ): AndroidPlaybackSeekResult { - val holder = session.holder - if (generation != holder.activeGeneration()) return AndroidPlaybackSeekResult.StaleGeneration - playback.seekExisting(holder, playerTimeMs.coerceAtLeast(0L)) - return AndroidPlaybackSeekResult.Ready(holder, session.preparation.result(holder, manifests)) - } - - fun manifest(session: AndroidPlaybackSession): AndroidDashManifestResult = - session.preparation.result(session.holder, manifests) - - suspend fun initialization( - holder: SabrSessionHolder, - itag: Int, - generation: Long, - ): AndroidPlaybackMediaResult { - val format = holder.formatForItag(itag) ?: return AndroidPlaybackMediaResult.TrackNotFound - if (generation != holder.activeGeneration()) return AndroidPlaybackMediaResult.StaleGeneration - return playback.fetchInitialization(holder, format, MEDIA_TIMEOUT_MS, generation).toAndroidResult() - } - - suspend fun segment( - holder: SabrSessionHolder, - itag: Int, - sequence: Int, - generation: Long, - ): AndroidPlaybackMediaResult { - val format = holder.formatForItag(itag) ?: return AndroidPlaybackMediaResult.TrackNotFound - if (generation != holder.activeGeneration()) return AndroidPlaybackMediaResult.StaleGeneration - return playback.fetchMedia(holder, format, sequence, MEDIA_TIMEOUT_MS, generation).toAndroidResult() - } - - private fun SabrSessionHolder.formatForItag(itag: Int): YoutubeSabrFormat? = when (itag) { - audioFormat.itag -> audioFormat - videoFormat.itag -> videoFormat - else -> null - } - - private fun SabrPlaybackSegmentResult.toAndroidResult(): AndroidPlaybackMediaResult = when (this) { - is SabrPlaybackSegmentResult.Ready -> AndroidPlaybackMediaResult.Ready(mimeType, bytes) - is SabrPlaybackSegmentResult.Retry -> AndroidPlaybackMediaResult.Preparing - is SabrPlaybackSegmentResult.Stale -> AndroidPlaybackMediaResult.StaleGeneration - SabrPlaybackSegmentResult.InvalidGeneration -> AndroidPlaybackMediaResult.StaleGeneration - SabrPlaybackSegmentResult.InvalidSequence -> AndroidPlaybackMediaResult.InvalidSequence - } - - private companion object { - const val MEDIA_TIMEOUT_MS = 4_000L - } -} - -internal sealed interface AndroidPlaybackCreateResult { - data class Created( - val session: AndroidPlaybackSession, - val manifest: AndroidDashManifestResult, - ) : AndroidPlaybackCreateResult - - data object UnsupportedLive : AndroidPlaybackCreateResult -} - -internal sealed interface AndroidPlaybackSeekResult { - data class Ready( - val holder: SabrSessionHolder, - val manifest: AndroidDashManifestResult, - ) : AndroidPlaybackSeekResult - - data object StaleGeneration : AndroidPlaybackSeekResult -} - -internal sealed interface AndroidPlaybackMediaResult { - data class Ready(val mimeType: String, val bytes: ByteArray) : AndroidPlaybackMediaResult - data object Preparing : AndroidPlaybackMediaResult - data object StaleGeneration : AndroidPlaybackMediaResult - data object TrackNotFound : AndroidPlaybackMediaResult - data object InvalidSequence : AndroidPlaybackMediaResult -} diff --git a/src/main/kotlin/dev/typetype/server/services/AndroidPlaybackSessionRegistry.kt b/src/main/kotlin/dev/typetype/server/services/AndroidPlaybackSessionRegistry.kt deleted file mode 100644 index e117d369..00000000 --- a/src/main/kotlin/dev/typetype/server/services/AndroidPlaybackSessionRegistry.kt +++ /dev/null @@ -1,98 +0,0 @@ -package dev.typetype.server.services - -import java.time.Duration -import java.time.Instant -import java.util.concurrent.ConcurrentHashMap - -internal class AndroidPlaybackSessionRegistry( - private val store: SabrSessionStore, - private val idleTimeout: Duration = Duration.ofMinutes(4), - private val tombstoneTtl: Duration = Duration.ofMinutes(10), - private val now: () -> Instant = Instant::now, -) { - private val sessions = ConcurrentHashMap() - private val tombstones = ConcurrentHashMap() - - fun register( - holder: SabrSessionHolder, - subtitles: List, - ): AndroidPlaybackSession { - val current = now() - cleanup(current) - val session = AndroidPlaybackSession(holder, subtitles) - sessions[holder.sessionToken] = Lease(session, current) - tombstones.remove(holder.sessionToken) - return session - } - - fun lookup(sessionId: String): AndroidPlaybackSessionLookup { - val current = now() - cleanup(current) - val lease = sessions[sessionId] - ?: return if (tombstones.containsKey(sessionId)) { - AndroidPlaybackSessionLookup.Expired - } else { - AndroidPlaybackSessionLookup.Unknown - } - if (!lease.lastAccess.plus(idleTimeout).isAfter(current)) { - expire(sessionId, lease, current) - return AndroidPlaybackSessionLookup.Expired - } - val holder = store.lookupByToken(sessionId) - if (holder == null || holder.key.purpose != SabrSessionPurpose.ANDROID_PLAYBACK) { - expire(sessionId, lease, current) - return AndroidPlaybackSessionLookup.Expired - } - lease.lastAccess = current - return AndroidPlaybackSessionLookup.Active(lease.session.withHolder(holder)) - } - - internal fun expire(sessionId: String): Unit { - val current = now() - sessions[sessionId]?.let { expire(sessionId, it, current) } - cleanup(current) - } - - private fun expire(sessionId: String, lease: Lease, current: Instant): Unit { - if (!sessions.remove(sessionId, lease)) return - tombstones[sessionId] = current.plus(tombstoneTtl) - lease.session.cancelPreparation() - store.release(lease.session.holder) - } - - private fun cleanup(current: Instant): Unit { - sessions.entries - .filter { !it.value.lastAccess.plus(idleTimeout).isAfter(current) } - .forEach { expire(it.key, it.value, current) } - tombstones.entries.removeIf { !it.value.isAfter(current) } - if (tombstones.size <= MAX_TOMBSTONES) return - tombstones.entries.sortedBy { it.value } - .take(tombstones.size - MAX_TOMBSTONES) - .forEach { tombstones.remove(it.key, it.value) } - } - - private class Lease(val session: AndroidPlaybackSession, @Volatile var lastAccess: Instant) - - private companion object { - const val MAX_TOMBSTONES = 2_048 - } -} - -internal class AndroidPlaybackSession( - val holder: SabrSessionHolder, - subtitles: List, - val preparation: AndroidPlaybackPreparation = AndroidPlaybackPreparation(), -) { - val subtitles: List = subtitles.toList() - - fun withHolder(current: SabrSessionHolder): AndroidPlaybackSession = - AndroidPlaybackSession(current, subtitles, preparation) - - fun cancelPreparation(): Unit = preparation.cancel() -} - -internal sealed interface AndroidPlaybackSessionLookup { - data class Active(val session: AndroidPlaybackSession) : AndroidPlaybackSessionLookup - data object Expired : AndroidPlaybackSessionLookup - data object Unknown : AndroidPlaybackSessionLookup -} diff --git a/src/main/kotlin/dev/typetype/server/services/AndroidSubtitleHttpClient.kt b/src/main/kotlin/dev/typetype/server/services/AndroidSubtitleHttpClient.kt deleted file mode 100644 index 77feaf80..00000000 --- a/src/main/kotlin/dev/typetype/server/services/AndroidSubtitleHttpClient.kt +++ /dev/null @@ -1,91 +0,0 @@ -package dev.typetype.server.services - -import kotlinx.coroutines.suspendCancellableCoroutine -import okhttp3.Call -import okhttp3.Callback -import okhttp3.HttpUrl -import okhttp3.OkHttpClient -import okhttp3.Request -import okhttp3.Response -import java.io.IOException -import java.nio.ByteBuffer -import java.nio.charset.CodingErrorAction -import java.nio.charset.StandardCharsets -import kotlin.coroutines.resume - -internal class AndroidSubtitleHttpClient( - private val primaryClient: OkHttpClient, - private val directClient: OkHttpClient?, -) { - suspend fun fetch(sourceUrl: HttpUrl): AndroidSubtitleUpstreamResult { - val url = sourceUrl.newBuilder() - .removeAllQueryParameters("fmt") - .addQueryParameter("fmt", "vtt") - .build() - val primary = request(primaryClient, url) - if (primary !is AndroidSubtitleUpstreamResult.TemporaryFailure) return primary - return directClient?.let { request(it, url) } ?: primary - } - - private suspend fun request(client: OkHttpClient, url: HttpUrl): AndroidSubtitleUpstreamResult = - suspendCancellableCoroutine { continuation -> - val request = Request.Builder() - .url(url) - .header("Accept", "text/vtt") - .header("Origin", YOUTUBE_ORIGIN) - .header("Referer", "$YOUTUBE_ORIGIN/") - .header("User-Agent", ANDROID_VR_USER_AGENT) - .build() - val call = client.newCall(request) - continuation.invokeOnCancellation { call.cancel() } - call.enqueue(object : Callback { - override fun onFailure(call: Call, e: IOException) { - if (continuation.isActive) { - continuation.resume(AndroidSubtitleUpstreamResult.TemporaryFailure) - } - } - - override fun onResponse(call: Call, response: Response) { - val result = runCatching { response.use(::readResponse) } - .getOrDefault(AndroidSubtitleUpstreamResult.TemporaryFailure) - if (continuation.isActive) continuation.resume(result) - } - }) - } - - private fun readResponse(response: Response): AndroidSubtitleUpstreamResult { - if (response.code == 403 || response.code == 429 || response.code >= 500) { - return AndroidSubtitleUpstreamResult.TemporaryFailure - } - if (!response.isSuccessful) return AndroidSubtitleUpstreamResult.Unavailable - val body = response.body - if (body.contentLength() > MAX_SUBTITLE_BYTES) return AndroidSubtitleUpstreamResult.Unavailable - val bytes = body.byteStream().readNBytes(MAX_SUBTITLE_BYTES + 1) - if (bytes.size > MAX_SUBTITLE_BYTES) return AndroidSubtitleUpstreamResult.Unavailable - val text = runCatching { - StandardCharsets.UTF_8.newDecoder() - .onMalformedInput(CodingErrorAction.REPORT) - .onUnmappableCharacter(CodingErrorAction.REPORT) - .decode(ByteBuffer.wrap(bytes)) - .toString() - }.getOrNull() ?: return AndroidSubtitleUpstreamResult.Unavailable - if (!text.removePrefix("\uFEFF").startsWith("WEBVTT")) { - return AndroidSubtitleUpstreamResult.Unavailable - } - return AndroidSubtitleUpstreamResult.Ready(bytes) - } - - private companion object { - const val MAX_SUBTITLE_BYTES = 16 * 1024 * 1024 - const val YOUTUBE_ORIGIN = "https://www.youtube.com" - const val ANDROID_VR_USER_AGENT = - "com.google.android.apps.youtube.vr.oculus/1.65.10 " + - "(Linux; U; Android 12L; eureka-user Build/SQ3A.220605.009.A1) gzip" - } -} - -internal sealed interface AndroidSubtitleUpstreamResult { - data class Ready(val bytes: ByteArray) : AndroidSubtitleUpstreamResult - data object TemporaryFailure : AndroidSubtitleUpstreamResult - data object Unavailable : AndroidSubtitleUpstreamResult -} diff --git a/src/main/kotlin/dev/typetype/server/services/AndroidSubtitleInventory.kt b/src/main/kotlin/dev/typetype/server/services/AndroidSubtitleInventory.kt deleted file mode 100644 index 38b73343..00000000 --- a/src/main/kotlin/dev/typetype/server/services/AndroidSubtitleInventory.kt +++ /dev/null @@ -1,107 +0,0 @@ -package dev.typetype.server.services - -import kotlinx.coroutines.CompletableDeferred -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.launch -import java.util.concurrent.atomic.AtomicBoolean -import java.util.concurrent.atomic.AtomicReference - -internal sealed interface AndroidSubtitleInventorySnapshot { - data object Preparing : AndroidSubtitleInventorySnapshot - data class Ready(val tracks: List) : AndroidSubtitleInventorySnapshot - data object TemporaryFailure : AndroidSubtitleInventorySnapshot -} - -internal class AndroidSubtitleInventoryHandle private constructor( - initial: AndroidSubtitleInventorySnapshot, -) { - private val state = AtomicReference(initial) - private val completed = AtomicBoolean(initial !is AndroidSubtitleInventorySnapshot.Preparing) - private val completion = CompletableDeferred().also { - if (initial !is AndroidSubtitleInventorySnapshot.Preparing) it.complete(initial) - } - - fun snapshot(): AndroidSubtitleInventorySnapshot = state.get() - - suspend fun await(): AndroidSubtitleInventorySnapshot = completion.await() - - internal fun complete(result: AndroidSubtitleInventorySnapshot) { - if (!completed.compareAndSet(false, true)) return - state.set(result) - completion.complete(result) - } - - companion object { - fun preparing(): AndroidSubtitleInventoryHandle = - AndroidSubtitleInventoryHandle(AndroidSubtitleInventorySnapshot.Preparing) - - fun ready(tracks: List): AndroidSubtitleInventoryHandle = - AndroidSubtitleInventoryHandle(AndroidSubtitleInventorySnapshot.Ready(tracks)) - - fun temporaryFailure(): AndroidSubtitleInventoryHandle = - AndroidSubtitleInventoryHandle(AndroidSubtitleInventorySnapshot.TemporaryFailure) - } -} - -internal class AndroidSubtitleInventoryCoordinator( - private val service: AndroidSubtitleService, - private val scope: CoroutineScope, - private val maxEntries: Int = 256, - private val cacheTtlMs: Long = 120_000L, - private val nowMs: () -> Long = System::currentTimeMillis, -) { - private val lock = Any() - private val entries = LinkedHashMap(16, 0.75f, true) - - init { - require(maxEntries > 0) - require(cacheTtlMs >= 0L) - } - - fun start(videoId: String): AndroidSubtitleInventoryHandle { - var created: AndroidSubtitleInventoryHandle? = null - val handle = synchronized(lock) { - cleanupCompleted(nowMs()) - entries[videoId]?.handle ?: allocate(videoId)?.also { created = it } - ?: AndroidSubtitleInventoryHandle.temporaryFailure() - } - created?.let { launchInventory(videoId, it) } - return handle - } - - private fun allocate(videoId: String): AndroidSubtitleInventoryHandle? { - if (entries.size >= maxEntries) { - entries.entries.firstOrNull { it.value.completedAtMs != null }?.let { entries.remove(it.key) } - } - if (entries.size >= maxEntries) return null - return AndroidSubtitleInventoryHandle.preparing().also { entries[videoId] = Entry(it) } - } - - private fun launchInventory(videoId: String, handle: AndroidSubtitleInventoryHandle) { - scope.launch { - val result = runCatchingNonCancellation { service.inventory(videoId) } - .getOrDefault(AndroidSubtitleInventoryResult.Unavailable) - .toSnapshot() - handle.complete(result) - synchronized(lock) { - entries[videoId]?.takeIf { it.handle === handle }?.completedAtMs = nowMs() - } - } - } - - private fun cleanupCompleted(currentMs: Long) { - entries.entries.removeIf { (_, entry) -> - entry.completedAtMs?.let { currentMs - it >= cacheTtlMs } == true - } - } - - private fun AndroidSubtitleInventoryResult.toSnapshot(): AndroidSubtitleInventorySnapshot = when (this) { - is AndroidSubtitleInventoryResult.Ready -> AndroidSubtitleInventorySnapshot.Ready(tracks) - AndroidSubtitleInventoryResult.Unavailable -> AndroidSubtitleInventorySnapshot.TemporaryFailure - } - - private class Entry( - val handle: AndroidSubtitleInventoryHandle, - var completedAtMs: Long? = null, - ) -} diff --git a/src/main/kotlin/dev/typetype/server/services/AndroidSubtitleService.kt b/src/main/kotlin/dev/typetype/server/services/AndroidSubtitleService.kt deleted file mode 100644 index 1cd84cd5..00000000 --- a/src/main/kotlin/dev/typetype/server/services/AndroidSubtitleService.kt +++ /dev/null @@ -1,48 +0,0 @@ -package dev.typetype.server.services - -internal class AndroidSubtitleService( - private val inventorySource: YouTubeSubtitleService, - private val httpClient: AndroidSubtitleHttpClient, -) { - suspend fun inventory(videoId: String): AndroidSubtitleInventoryResult = - when (val result = inventorySource.fetchSubtitleInventory(videoId)) { - is YouTubeSubtitleInventoryResult.Ready -> { - val tracks = AndroidSubtitleTrackFactory.create(videoId, result.tracks) - ?: return AndroidSubtitleInventoryResult.Unavailable - AndroidSubtitleInventoryResult.Ready(tracks) - } - YouTubeSubtitleInventoryResult.Unavailable -> AndroidSubtitleInventoryResult.Unavailable - } - - suspend fun content( - videoId: String, - track: AndroidSubtitleTrack, - ): AndroidSubtitleContentResult { - return when (val initial = httpClient.fetch(track.sourceUrl)) { - is AndroidSubtitleUpstreamResult.Ready -> AndroidSubtitleContentResult.Ready(initial.bytes) - AndroidSubtitleUpstreamResult.Unavailable -> AndroidSubtitleContentResult.Unavailable - AndroidSubtitleUpstreamResult.TemporaryFailure -> retryWithFreshInventory(videoId, track.id) - } - } - - private suspend fun retryWithFreshInventory( - videoId: String, - trackId: String, - ): AndroidSubtitleContentResult { - val refreshed = inventory(videoId) as? AndroidSubtitleInventoryResult.Ready - ?: return AndroidSubtitleContentResult.TemporaryFailure - val track = refreshed.tracks.firstOrNull { it.id == trackId } - ?: return AndroidSubtitleContentResult.Unavailable - return when (val retry = httpClient.fetch(track.sourceUrl)) { - is AndroidSubtitleUpstreamResult.Ready -> AndroidSubtitleContentResult.Ready(retry.bytes) - AndroidSubtitleUpstreamResult.Unavailable -> AndroidSubtitleContentResult.Unavailable - AndroidSubtitleUpstreamResult.TemporaryFailure -> AndroidSubtitleContentResult.TemporaryFailure - } - } -} - -internal sealed interface AndroidSubtitleContentResult { - data class Ready(val bytes: ByteArray) : AndroidSubtitleContentResult - data object TemporaryFailure : AndroidSubtitleContentResult - data object Unavailable : AndroidSubtitleContentResult -} diff --git a/src/main/kotlin/dev/typetype/server/services/AndroidSubtitleTrack.kt b/src/main/kotlin/dev/typetype/server/services/AndroidSubtitleTrack.kt deleted file mode 100644 index 75415c90..00000000 --- a/src/main/kotlin/dev/typetype/server/services/AndroidSubtitleTrack.kt +++ /dev/null @@ -1,62 +0,0 @@ -package dev.typetype.server.services - -import dev.typetype.server.models.SubtitleItem -import okhttp3.HttpUrl -import okhttp3.HttpUrl.Companion.toHttpUrlOrNull -import java.nio.charset.StandardCharsets -import java.security.MessageDigest - -internal data class AndroidSubtitleTrack( - val id: String, - val languageTag: String, - val displayLanguageName: String, - val isAutoGenerated: Boolean, - val sourceUrl: HttpUrl, -) - -internal object AndroidSubtitleTrackFactory { - fun create(videoId: String, items: List): List? { - val occurrences = mutableMapOf() - return items.map { item -> - val sourceUrl = item.url.toHttpUrlOrNull() - ?.takeIf(::isYoutubeCaptionUrl) - ?: return null - val baseId = stableId(videoId, item, sourceUrl) - val occurrence = occurrences.merge(baseId, 1, Int::plus) ?: 1 - AndroidSubtitleTrack( - id = if (occurrence == 1) baseId else "$baseId-$occurrence", - languageTag = item.languageTag, - displayLanguageName = item.displayLanguageName, - isAutoGenerated = item.isAutoGenerated, - sourceUrl = sourceUrl, - ) - } - } - - private fun stableId(videoId: String, item: SubtitleItem, url: HttpUrl): String { - val sourceIdentity = listOf("vssId", "lang", "kind", "tlang") - .joinToString("&") { key -> "$key=${url.queryParameter(key).orEmpty()}" } - val material = listOf( - videoId, - item.languageTag, - item.isAutoGenerated.toString(), - sourceIdentity, - ).joinToString("\u0000") - return MessageDigest.getInstance("SHA-256") - .digest(material.toByteArray(StandardCharsets.UTF_8)) - .take(ID_BYTES) - .joinToString("") { "%02x".format(it) } - } - - private fun isYoutubeCaptionUrl(url: HttpUrl): Boolean = - url.isHttps && - (url.host == "youtube.com" || url.host.endsWith(".youtube.com")) && - url.encodedPath == "/api/timedtext" - - private const val ID_BYTES = 12 -} - -internal sealed interface AndroidSubtitleInventoryResult { - data class Ready(val tracks: List) : AndroidSubtitleInventoryResult - data object Unavailable : AndroidSubtitleInventoryResult -} diff --git a/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamKind.kt b/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamKind.kt index 8cb12644..39b3b5fc 100644 --- a/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamKind.kt +++ b/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamKind.kt @@ -2,7 +2,6 @@ package dev.typetype.server.services enum class AudioOnlyStreamKind(val apiValue: String) { Progressive("progressive"), - SabrProgressive("progressive"), Hls("hls"), Dash("dash"), SabrHls("hls"), diff --git a/src/main/kotlin/dev/typetype/server/services/BlockedContentFilters.kt b/src/main/kotlin/dev/typetype/server/services/BlockedContentFilters.kt index f1773c69..fcba974b 100644 --- a/src/main/kotlin/dev/typetype/server/services/BlockedContentFilters.kt +++ b/src/main/kotlin/dev/typetype/server/services/BlockedContentFilters.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.models.SearchPageResponse internal fun SearchPageResponse.filterBlocked(profile: BlockedContentProfile): SearchPageResponse = copy( - items = items.filter { profile.allowsVideo(it.url, it.uploaderUrl, it.uploaderName) }, + items = items.filter { profile.allowsVideo(it.url, it.title, it.uploaderUrl, it.uploaderName) }, channels = channels.filter { profile.allowsChannel(url = it.url, name = it.name) }, playlists = playlists.filter { profile.allowsChannel(url = "", name = it.uploaderName) }, ) diff --git a/src/main/kotlin/dev/typetype/server/services/BlockedContentProfile.kt b/src/main/kotlin/dev/typetype/server/services/BlockedContentProfile.kt index 9eb013bf..07581898 100644 --- a/src/main/kotlin/dev/typetype/server/services/BlockedContentProfile.kt +++ b/src/main/kotlin/dev/typetype/server/services/BlockedContentProfile.kt @@ -1,13 +1,17 @@ package dev.typetype.server.services import dev.typetype.server.models.BlockedItem +import dev.typetype.server.models.BlockedKeywordItem data class BlockedContentProfile( val videos: List, val channels: List, + val keywords: List, ) { - fun allowsVideo(url: String, uploaderUrl: String, uploaderName: String): Boolean = - videos.none { normalizeUrl(it.url) == normalizeUrl(url) } && allowsChannel(uploaderUrl, uploaderName) + fun allowsVideo(url: String, title: String, uploaderUrl: String, uploaderName: String): Boolean = + videos.none { normalizeUrl(it.url) == normalizeUrl(url) } && + keywords.none { containsBlockedKeyword(title, it.keyword) } && + allowsChannel(uploaderUrl, uploaderName) fun allowsChannel(url: String, name: String): Boolean = channels.none { item -> val blockedUrl = normalizeChannelKey(item.url) @@ -17,7 +21,7 @@ data class BlockedContentProfile( } companion object { - val empty = BlockedContentProfile(videos = emptyList(), channels = emptyList()) + val empty = BlockedContentProfile(videos = emptyList(), channels = emptyList(), keywords = emptyList()) } } diff --git a/src/main/kotlin/dev/typetype/server/services/BlockedService.kt b/src/main/kotlin/dev/typetype/server/services/BlockedService.kt index f035903f..3d91f801 100644 --- a/src/main/kotlin/dev/typetype/server/services/BlockedService.kt +++ b/src/main/kotlin/dev/typetype/server/services/BlockedService.kt @@ -2,8 +2,10 @@ package dev.typetype.server.services import dev.typetype.server.db.DatabaseFactory import dev.typetype.server.db.tables.BlockedChannelsTable +import dev.typetype.server.db.tables.BlockedKeywordsTable import dev.typetype.server.db.tables.BlockedVideosTable import dev.typetype.server.models.BlockedItem +import dev.typetype.server.models.BlockedKeywordItem import org.jetbrains.exposed.v1.core.SortOrder import org.jetbrains.exposed.v1.core.and import org.jetbrains.exposed.v1.core.eq @@ -11,6 +13,7 @@ import org.jetbrains.exposed.v1.core.or import org.jetbrains.exposed.v1.jdbc.deleteWhere import org.jetbrains.exposed.v1.jdbc.insert import org.jetbrains.exposed.v1.jdbc.selectAll +import java.text.Normalizer private const val SCOPE_USER = "user" private const val SCOPE_GLOBAL = "global" @@ -19,6 +22,7 @@ class BlockedService { suspend fun profileFor(userId: String): BlockedContentProfile = BlockedContentProfile( videos = getVideos(userId), channels = getChannels(userId), + keywords = getKeywords(userId), ) suspend fun getChannels(userId: String): List = DatabaseFactory.query { @@ -31,6 +35,7 @@ class BlockedService { name = row[BlockedChannelsTable.channelName], thumbnailUrl = row[BlockedChannelsTable.channelThumbnailUrl], blockedAt = row[BlockedChannelsTable.blockedAt], + global = row[BlockedChannelsTable.scope] == SCOPE_GLOBAL, ) } } @@ -39,9 +44,37 @@ class BlockedService { BlockedVideosTable.selectAll() .where { (BlockedVideosTable.userId eq userId) or (BlockedVideosTable.scope eq SCOPE_GLOBAL) } .orderBy(BlockedVideosTable.blockedAt to SortOrder.DESC) - .map { row -> BlockedItem(url = row[BlockedVideosTable.videoUrl], blockedAt = row[BlockedVideosTable.blockedAt]) } + .map { row -> + BlockedItem( + url = row[BlockedVideosTable.videoUrl], + blockedAt = row[BlockedVideosTable.blockedAt], + global = row[BlockedVideosTable.scope] == SCOPE_GLOBAL, + ) + } } + suspend fun getKeywords(userId: String): List = DatabaseFactory.query { + BlockedKeywordsTable.selectAll() + .where { (BlockedKeywordsTable.userId eq userId) or (BlockedKeywordsTable.scope eq SCOPE_GLOBAL) } + .orderBy(BlockedKeywordsTable.blockedAt to SortOrder.DESC) + .map { row -> + BlockedKeywordItem( + keyword = row[BlockedKeywordsTable.keyword], + blockedAt = row[BlockedKeywordsTable.blockedAt], + global = row[BlockedKeywordsTable.scope] == SCOPE_GLOBAL, + ) + } + } + + suspend fun getUserChannels(userId: String): List = + getChannels(userId).filter { it.global != true } + + suspend fun getUserVideos(userId: String): List = + getVideos(userId).filter { it.global != true } + + suspend fun getUserKeywords(userId: String): List = + getKeywords(userId).filter { it.global != true } + suspend fun addChannel( userId: String, url: String, @@ -76,6 +109,25 @@ class BlockedService { return BlockedItem(url = url, blockedAt = now) } + suspend fun addKeyword(userId: String, keyword: String, global: Boolean = false): BlockedKeywordItem { + val normalized = normalizeBlockedKeyword(keyword) + require(normalized.isNotEmpty()) { "Keyword cannot be empty" } + val now = System.currentTimeMillis() + DatabaseFactory.query { + BlockedKeywordsTable.deleteWhere { + (BlockedKeywordsTable.keyword eq normalized) and + (BlockedKeywordsTable.userId eq userId) + } + BlockedKeywordsTable.insert { + it[BlockedKeywordsTable.userId] = userId + it[scope] = if (global) SCOPE_GLOBAL else SCOPE_USER + it[BlockedKeywordsTable.keyword] = normalized + it[blockedAt] = now + } + } + return BlockedKeywordItem(keyword = normalized, blockedAt = now, global = global) + } + suspend fun deleteChannel(userId: String, url: String, role: String): Boolean = DatabaseFactory.query { val canDeleteGlobal = role == "admin" || role == "moderator" val scopeClause = if (canDeleteGlobal) { @@ -95,4 +147,24 @@ class BlockedService { } BlockedVideosTable.deleteWhere { (BlockedVideosTable.videoUrl eq url) and scopeClause } > 0 } + + suspend fun deleteKeyword(userId: String, keyword: String, role: String): Boolean = DatabaseFactory.query { + val canDeleteGlobal = role == "admin" || role == "moderator" + val scopeClause = if (canDeleteGlobal) { + (BlockedKeywordsTable.scope eq SCOPE_GLOBAL) or (BlockedKeywordsTable.userId eq userId) + } else { + (BlockedKeywordsTable.scope eq SCOPE_USER) and (BlockedKeywordsTable.userId eq userId) + } + BlockedKeywordsTable.deleteWhere { + (BlockedKeywordsTable.keyword eq normalizeBlockedKeyword(keyword)) and scopeClause + } > 0 + } +} + +internal fun normalizeBlockedKeyword(value: String): String = + Normalizer.normalize(value, Normalizer.Form.NFKC).trim().lowercase() + +internal fun containsBlockedKeyword(title: String, keyword: String): Boolean { + val normalizedKeyword = normalizeBlockedKeyword(keyword) + return normalizedKeyword.isNotEmpty() && normalizeBlockedKeyword(title).contains(normalizedKeyword) } diff --git a/src/main/kotlin/dev/typetype/server/services/HistoryProgressMapper.kt b/src/main/kotlin/dev/typetype/server/services/HistoryProgressMapper.kt index f2bbc09a..766b240c 100644 --- a/src/main/kotlin/dev/typetype/server/services/HistoryProgressMapper.kt +++ b/src/main/kotlin/dev/typetype/server/services/HistoryProgressMapper.kt @@ -13,7 +13,12 @@ internal object HistoryProgressMapper { fun toHistoryItems(userId: String, rows: List): List { val items = rows.map { it.toHistoryItem() } val savedProgress = savedProgressSeconds(userId, items.map { it.url }) - return items.map { YoutubeTypeTypeMapper.historyItem(it).withSavedProgress(savedProgress[it.url]) } + return mergeSavedProgress(items, savedProgress) + } + + fun toHistoryItemsForExport(userId: String, rows: List): List { + val items = rows.map { it.toHistoryItem() } + return mergeSavedProgress(items, savedProgressSeconds(userId)) } fun savedProgressSeconds(userId: String, videoUrl: String): Long? = savedProgressSeconds(userId, listOf(videoUrl))[videoUrl] @@ -21,11 +26,28 @@ internal object HistoryProgressMapper { fun savedProgressSeconds(userId: String, videoUrls: List): Map { val urls = videoUrls.distinct() if (urls.isEmpty()) return emptyMap() - return ProgressTable.selectAll() - .where { (ProgressTable.userId eq userId) and (ProgressTable.videoUrl inList urls) } + return progressLookupBatches(urls) + .flatMap { batch -> progressRows(userId, batch) } .associate { it[ProgressTable.videoUrl] to it[ProgressTable.position].toSeconds() } } + fun progressLookupBatches(videoUrls: List): List> = + videoUrls.distinct().chunked(PROGRESS_LOOKUP_BATCH_SIZE) + + private fun savedProgressSeconds(userId: String): Map = + ProgressTable.selectAll() + .where { ProgressTable.userId eq userId } + .associate { it[ProgressTable.videoUrl] to it[ProgressTable.position].toSeconds() } + + private fun progressRows(userId: String, videoUrls: List) = ProgressTable.selectAll() + .where { (ProgressTable.userId eq userId) and (ProgressTable.videoUrl inList videoUrls) } + .toList() + + private fun mergeSavedProgress( + items: List, + savedProgress: Map, + ) = items.map { YoutubeTypeTypeMapper.historyItem(it).withSavedProgress(savedProgress[it.url]) } + private fun HistoryItem.withSavedProgress(savedProgress: Long?): HistoryItem = copy(progress = maxOf(progress, savedProgress ?: 0L)) @@ -44,3 +66,5 @@ internal object HistoryProgressMapper { watchedAt = this[HistoryTable.watchedAt], ) } + +private const val PROGRESS_LOOKUP_BATCH_SIZE = 1_000 diff --git a/src/main/kotlin/dev/typetype/server/services/HistoryService.kt b/src/main/kotlin/dev/typetype/server/services/HistoryService.kt index 0dd760d2..4d065d51 100644 --- a/src/main/kotlin/dev/typetype/server/services/HistoryService.kt +++ b/src/main/kotlin/dev/typetype/server/services/HistoryService.kt @@ -20,6 +20,14 @@ import org.jetbrains.exposed.v1.jdbc.selectAll import java.util.UUID class HistoryService { + suspend fun getAll(userId: String): List = DatabaseFactory.query { + val rows = HistoryTable.selectAll() + .where { HistoryTable.userId eq userId } + .orderBy(HistoryTable.watchedAt to SortOrder.DESC, HistoryTable.id to SortOrder.DESC) + .toList() + HistoryProgressMapper.toHistoryItemsForExport(userId, rows) + } + suspend fun search(userId: String, q: String?, from: Long?, to: Long?, limit: Int, offset: Int): Pair, Long> = DatabaseFactory.query { val query = HistoryTable.selectAll().where { HistoryTable.userId eq userId } if (!q.isNullOrBlank()) { diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolBuilder.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolBuilder.kt index d2550f1f..2fb091b7 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolBuilder.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolBuilder.kt @@ -51,6 +51,7 @@ class HomeRecommendationPoolBuilder { val video = tagged.video if (video.url.isBlank()) return@forEach if (video.url in profile.seenUrls || video.url in profile.blockedVideos) return@forEach + if (profile.blockedKeywords.any { containsBlockedKeyword(video.title, it) }) return@forEach if (shortsOnly && !(video.isShortFormContent || video.duration in 1L..85L)) return@forEach if (video.url in profile.feedbackBlockedVideos || video.url in profile.implicitBlockedVideos) return@forEach if (video.uploaderUrl.isNotBlank() && video.uploaderUrl in profile.blockedChannels) return@forEach diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationProfile.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationProfile.kt index cf981be7..c0f71adf 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationProfile.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationProfile.kt @@ -4,6 +4,7 @@ data class HomeRecommendationProfile( val seenUrls: Set, val blockedVideos: Set, val blockedChannels: Set, + val blockedKeywords: Set = emptySet(), val feedbackBlockedVideos: Set, val feedbackBlockedChannels: Set, val subscriptionChannels: Set, diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationUserSignalService.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationUserSignalService.kt index 8ba1ef95..f26a0246 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationUserSignalService.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationUserSignalService.kt @@ -19,12 +19,14 @@ class HomeRecommendationUserSignalService( } val blockedVideosDeferred = async { blockedService.getVideos(userId).map { it.url }.toSet() } val blockedChannelsDeferred = async { blockedService.getChannels(userId).map { it.url }.toSet() } + val blockedKeywordsDeferred = async { blockedService.getKeywords(userId).map { it.keyword }.toSet() } val subscriptions = subscriptionsDeferred.await() val favorites = favoritesDeferred.await() val watchLater = watchLaterDeferred.await() val historyItems = historyDeferred.await() val blockedVideos = blockedVideosDeferred.await() val blockedChannels = blockedChannelsDeferred.await() + val blockedKeywords = blockedKeywordsDeferred.await() val seenUrls = historyItems.map { it.url }.toSet() val favoriteUrls = favorites.map { it.videoUrl }.toSet() val watchLaterUrls = watchLater.map { it.url }.toSet() @@ -33,6 +35,7 @@ class HomeRecommendationUserSignalService( seenUrls = seenUrls, blockedVideos = blockedVideos, blockedChannels = blockedChannels, + blockedKeywords = blockedKeywords, feedbackBlockedVideos = emptySet(), feedbackBlockedChannels = emptySet(), subscriptionChannels = subscriptionChannels, diff --git a/src/main/kotlin/dev/typetype/server/services/ProgressService.kt b/src/main/kotlin/dev/typetype/server/services/ProgressService.kt index 2e5e73b0..e73dea44 100644 --- a/src/main/kotlin/dev/typetype/server/services/ProgressService.kt +++ b/src/main/kotlin/dev/typetype/server/services/ProgressService.kt @@ -8,9 +8,23 @@ import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.jdbc.insert import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.update +import org.jetbrains.exposed.v1.core.SortOrder class ProgressService { + suspend fun getAll(userId: String): List = DatabaseFactory.query { + ProgressTable.selectAll() + .where { ProgressTable.userId eq userId } + .orderBy(ProgressTable.updatedAt to SortOrder.DESC) + .map { + ProgressItem( + videoUrl = it[ProgressTable.videoUrl], + position = it[ProgressTable.position], + updatedAt = it[ProgressTable.updatedAt], + ) + } + } + suspend fun get(userId: String, videoUrl: String): ProgressItem? = DatabaseFactory.query { ProgressTable.selectAll() .where { (ProgressTable.videoUrl eq videoUrl) and (ProgressTable.userId eq userId) } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDownloadFrameWriter.kt b/src/main/kotlin/dev/typetype/server/services/SabrDownloadFrameWriter.kt new file mode 100644 index 00000000..5d2eeee6 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SabrDownloadFrameWriter.kt @@ -0,0 +1,64 @@ +package dev.typetype.server.services + +import java.io.InputStream +import java.io.OutputStream + +internal class SabrDownloadFrameWriter(private val output: OutputStream) { + private val header = ByteArray(FRAME_HEADER_SIZE) + private val copyBuffer = ByteArray(COPY_BUFFER_SIZE) + + fun start(): Unit = output.write(MAGIC) + + fun initialization(itag: Int, data: ByteArray) { + writeHeader(FRAME_INITIALIZATION, itag, 0, data.size.toLong()) + output.write(data) + } + + fun media(itag: Int, sequence: Int, length: Int, input: InputStream) { + require(length >= 0) { "Negative SABR frame length" } + writeHeader(FRAME_MEDIA, itag, sequence, length.toLong()) + var remaining = length + while (remaining > 0) { + val read = input.read(copyBuffer, 0, minOf(copyBuffer.size, remaining)) + if (read < 0) error("SABR segment ended before its declared length") + if (read == 0) continue + output.write(copyBuffer, 0, read) + remaining -= read + } + check(input.read() == -1) { "SABR segment exceeds its declared length" } + } + + fun finish() { + writeHeader(FRAME_COMPLETE, 0, 0, 0) + output.flush() + } + + private fun writeHeader(type: Int, itag: Int, sequence: Int, length: Long) { + header[0] = type.toByte() + header.putInt(1, itag) + header.putInt(5, sequence) + header.putLong(9, length) + output.write(header) + } + + private fun ByteArray.putInt(offset: Int, value: Int) { + for (index in 0 until Int.SIZE_BYTES) { + this[offset + index] = (value ushr (24 - index * 8)).toByte() + } + } + + private fun ByteArray.putLong(offset: Int, value: Long) { + for (index in 0 until Long.SIZE_BYTES) { + this[offset + index] = (value ushr (56 - index * 8)).toByte() + } + } + + internal companion object { + val MAGIC = "TTSABR1\n".encodeToByteArray() + const val FRAME_INITIALIZATION = 1 + const val FRAME_MEDIA = 2 + const val FRAME_COMPLETE = 3 + const val FRAME_HEADER_SIZE = 17 + const val COPY_BUFFER_SIZE = 256 * 1024 + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDownloadInitialization.kt b/src/main/kotlin/dev/typetype/server/services/SabrDownloadInitialization.kt new file mode 100644 index 00000000..8941e0f3 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SabrDownloadInitialization.kt @@ -0,0 +1,35 @@ +package dev.typetype.server.services + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.runInterruptible +import org.schabi.newpipe.extractor.localization.Localization +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat + +internal object SabrDownloadInitialization { + private val localization = Localization("en", "US") + + suspend fun fetch( + store: SabrSessionStore, + holder: SabrSessionHolder, + format: YoutubeSabrFormat, + ): ByteArray? { + SabrInitializationData.fetch(holder.key.videoId, format, store.initCache)?.let { + holder.session.streamState.ingestInitializationData(format, it) + return it + } + val poToken = holder.session.streamState.poToken?.takeIf { it.isNotEmpty() } + ?: holder.playerContextToken?.streamingPoTokenBytesFor(holder.info) + ?: return store.fetchInitializationData(holder, format) + val direct = runCatchingNonCancellation { + runInterruptible(Dispatchers.IO) { + holder.withPlayerContext { + fetchInitializationData(format, localization, DIRECT_TIMEOUT_MS, poToken) + } + } + }.getOrNull() ?: return store.fetchInitializationData(holder, format) + SabrInitializationData.remember(holder.key.videoId, format, direct, store.initCache) + return direct + } + + private const val DIRECT_TIMEOUT_MS = 5_000L +} diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDownloadRange.kt b/src/main/kotlin/dev/typetype/server/services/SabrDownloadRange.kt new file mode 100644 index 00000000..5117cc1b --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SabrDownloadRange.kt @@ -0,0 +1,46 @@ +package dev.typetype.server.services + +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState + +internal data class SabrDownloadRange( + val part: Int = 0, + val parts: Int = 1, +) { + init { + require(parts in 1..MAX_PARTS) { "parts must be between 1 and $MAX_PARTS" } + require(part in 0 until parts) { "part must be between 0 and parts - 1" } + } + + fun startTimeMs(audio: YoutubeSabrFormat, video: YoutubeSabrFormat, audioOnly: Boolean): Long { + if (part == 0) return 0L + val durationMs = if (audioOnly) audio.approxDurationMs else maxOf(audio.approxDurationMs, video.approxDurationMs) + return durationMs.coerceAtLeast(0L) * part / parts + } + + fun startSequence(state: YoutubeSabrStreamState, format: YoutubeSabrFormat): Int = + boundarySequence(state, format, part) + + fun endSequenceExclusive(state: YoutubeSabrStreamState, format: YoutubeSabrFormat): Int? { + if (parts == 1) return null + if (part + 1 < parts) return boundarySequence(state, format, part + 1) + val end = state.getEndSegment(format) + return end.takeIf { it in 1 until Int.MAX_VALUE }?.toInt()?.plus(1) + } + + private fun boundarySequence( + state: YoutubeSabrStreamState, + format: YoutubeSabrFormat, + boundaryPart: Int, + ): Int { + if (parts == 1 || boundaryPart == 0) return 1 + val durationMs = format.approxDurationMs.coerceAtLeast(0L) + require(durationMs > 0L) { "SABR duration is required for multipart downloads" } + val boundaryMs = durationMs * boundaryPart / parts + return state.getSegmentNumberAtOrAfterTimeMs(format, boundaryMs).coerceAtLeast(1) + } + + private companion object { + const val MAX_PARTS = 12 + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDownloadStreamer.kt b/src/main/kotlin/dev/typetype/server/services/SabrDownloadStreamer.kt new file mode 100644 index 00000000..6024be9d --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SabrDownloadStreamer.kt @@ -0,0 +1,195 @@ +package dev.typetype.server.services + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay +import kotlinx.coroutines.runInterruptible +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeoutOrNull +import org.schabi.newpipe.extractor.localization.Localization +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import org.slf4j.LoggerFactory +import java.io.IOException +import java.io.OutputStream + +internal class SabrDownloadStreamer( + private val store: SabrSessionStore, + private val pumpTimeoutMs: Long = PUMP_TIMEOUT_MS, +) { + private val localization = Localization("en", "US") + private val unauthorizedRecovery = SabrUnauthorizedResponseRecovery(store::refreshVideoPoToken) + + suspend fun stream( + holder: SabrSessionHolder, + output: OutputStream, + range: SabrDownloadRange = SabrDownloadRange(), + ) { + val startedNs = System.nanoTime() + val writer = SabrDownloadFrameWriter(output) + var pumpCount = 0 + writer.start() + val formats = selectedFormats(holder) + writeInitializations(holder, formats, writer) + val tracks = selectedTracks(holder, formats, range) + discardBeforeRange(holder, tracks) + var emptyRounds = 0 + while (!complete(holder, tracks)) { + val before = drain(holder, tracks, writer) + if (complete(holder, tracks)) break + val target = nextMissingRequest(holder, tracks) + ?: throw IOException("SABR download could not identify the next missing segment") + val playerTimeMs = holder.session.streamState + .getSegmentStartMs(target.format, target.sequenceNumber) + .coerceAtLeast(0L) + val pumped = pump(holder, playerTimeMs, target) + pumpCount++ + val after = drain(holder, tracks, writer) + if (holder.session.cachedBytes > MAX_SESSION_CACHE_BYTES) { + throw IOException("SABR download exceeded its bounded media cache") + } + emptyRounds = if (before + after + pumped > 0) 0 else emptyRounds + 1 + if (emptyRounds > MAX_EMPTY_ROUNDS) { + val status = SabrUnauthorizedResponseRecovery.latestUnauthorizedStatus( + holder.session.diagnosticTrace, + ) + logger.warn( + "sabr_download_stall videoId={} part={}/{} request={} upstreamStatus={}", + holder.key.videoId, + range.part + 1, + range.parts, + holder.session.requestNumber, + status, + ) + throw IOException("SABR download stalled without receiving media") + } + if (emptyRounds > 0) delay(EMPTY_ROUND_DELAY_MS) + } + writer.finish() + logger.info( + "sabr_download videoId={} part={}/{} tracks={} durationMs={} pumps={} upstreamBytes={} peakResponseBytes={}", + holder.key.videoId, + range.part + 1, + range.parts, + tracks.size, + (System.nanoTime() - startedNs) / 1_000_000, + pumpCount, + holder.session.totalResponseBytes, + holder.session.maxResponseBytes, + ) + } + + private suspend fun writeInitializations( + holder: SabrSessionHolder, + formats: List, + writer: SabrDownloadFrameWriter, + ) { + for (format in formats) { + val data = SabrDownloadInitialization.fetch(store, holder, format) + ?: throw IOException("Missing SABR initialization for itag ${format.itag}") + writer.initialization(format.itag, data) + holder.session.discardCachedSegment(SabrSegmentRequest.initialization(format)) + } + } + + private fun drain( + holder: SabrSessionHolder, + tracks: List, + writer: SabrDownloadFrameWriter, + ): Int { + var written = 0 + for (track in tracks) { + while (true) { + if (track.endSequenceExclusive?.let { track.nextSequence >= it } == true) break + val request = SabrSegmentRequest.media(track.format, track.nextSequence) + val segment = holder.session.getCachedSegment(request) ?: break + segment.openStream().use { + writer.media(track.format.itag, track.nextSequence, segment.length, it) + } + holder.session.discardCachedSegment(request) + track.nextSequence++ + written++ + } + } + return written + } + + private fun discardBeforeRange(holder: SabrSessionHolder, tracks: List) { + for (track in tracks) { + for (sequence in 1 until track.nextSequence) { + holder.session.discardCachedSegment(SabrSegmentRequest.media(track.format, sequence)) + } + } + } + + private fun nextMissingRequest( + holder: SabrSessionHolder, + tracks: List, + ): SabrSegmentRequest? = tracks + .asSequence() + .filterNot { track -> track.endSequenceExclusive?.let { track.nextSequence >= it } == true } + .map { track -> SabrSegmentRequest.media(track.format, track.nextSequence) } + .filter { request -> + holder.session.getCachedSegment(request) == null && !holder.session.isBeyondEnd(request) + } + .minByOrNull { request -> + holder.session.streamState.getSegmentStartMs(request.format, request.sequenceNumber) + } + + private suspend fun pump( + holder: SabrSessionHolder, + playerTimeMs: Long, + target: SabrSegmentRequest, + ): Int { + val pumped = withContext(Dispatchers.IO) { + withTimeoutOrNull(pumpTimeoutMs) { + runInterruptible { + holder.withPlayerContext { + prepareForForwardJump(target, playerTimeMs.coerceAtLeast(0L)) + pumpOnceStreamingUntilCached(localization, target) + } + } + } + } ?: throw IOException("SABR download upstream pump timed out") + return pumped.also { unauthorizedRecovery.verify(holder) } + } + + private fun complete(holder: SabrSessionHolder, tracks: List): Boolean = + tracks.all { track -> + if (track.endSequenceExclusive?.let { track.nextSequence >= it } == true) return@all true + val request = SabrSegmentRequest.media(track.format, track.nextSequence) + holder.session.getCachedSegment(request) == null && + (holder.session.streamState.isComplete(track.format) || holder.session.isBeyondEnd(request)) + } + + private fun selectedTracks( + holder: SabrSessionHolder, + formats: List, + range: SabrDownloadRange, + ): List = formats.map { format -> + DownloadTrack( + format, + range.startSequence(holder.session.streamState, format), + range.endSequenceExclusive(holder.session.streamState, format), + ) + } + + private fun selectedFormats(holder: SabrSessionHolder): List = + buildList { + if (holder.isAudioActive()) add(holder.audioFormat) + if (holder.isVideoActive()) add(holder.videoFormat) + } + + private data class DownloadTrack( + val format: YoutubeSabrFormat, + var nextSequence: Int, + val endSequenceExclusive: Int?, + ) + + private companion object { + val logger = LoggerFactory.getLogger(SabrDownloadStreamer::class.java) + const val MAX_EMPTY_ROUNDS = 12 + const val EMPTY_ROUND_DELAY_MS = 50L + const val MAX_SESSION_CACHE_BYTES = 64L * 1024 * 1024 + const val PUMP_TIMEOUT_MS = 30_000L + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt b/src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt index efaa11c4..c9c5b71c 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt @@ -22,8 +22,13 @@ internal class SabrInfoFetcher( videoId: String, startTimeMs: Long = 0L, cachedFirst: Boolean = false, + isolatedPlayback: Boolean = false, ): SabrPreparedInfo? = withContext(Dispatchers.IO) { val startedAt = System.currentTimeMillis() + if (isolatedPlayback) { + return@withContext fetchInfoOnce(videoId, startTimeMs, isolatedPlayback = true) + ?.also { logFetch(videoId, startTimeMs, startedAt, "isolated_network") } + } if (cachedFirst) repository.local(videoId, startTimeMs)?.let { logFetch(videoId, startTimeMs, startedAt, "cache") return@withContext it @@ -66,9 +71,13 @@ internal class SabrInfoFetcher( private suspend fun fetchPlayable(videoId: String, startTimeMs: Long): SabrPreparedInfo? = fetchInfoOnce(videoId, startTimeMs)?.let { repository.putPrepared(videoId, startTimeMs, it) } - private suspend fun fetchInfoOnce(videoId: String, startTimeMs: Long): SabrPreparedInfo? = + private suspend fun fetchInfoOnce( + videoId: String, + startTimeMs: Long, + isolatedPlayback: Boolean = false, + ): SabrPreparedInfo? = withTimeoutOrNull(SabrSessionStoreDefaults.INFO_TIMEOUT_MS) { - val tokenSession = sessionClient?.fetchPlaybackSession(videoId) + val tokenSession = sessionClient?.fetchPlaybackSession(videoId, isolatedPlayback) tokenSession?.token ?.takeIf { it.visitorData == tokenSession.info.visitorData } ?.let { sessionToken -> diff --git a/src/main/kotlin/dev/typetype/server/services/SabrInitializationData.kt b/src/main/kotlin/dev/typetype/server/services/SabrInitializationData.kt index 5c8409b2..966e84d0 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrInitializationData.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrInitializationData.kt @@ -1,13 +1,10 @@ package dev.typetype.server.services import dev.typetype.server.cache.CacheService -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.runInterruptible import kotlinx.coroutines.sync.withLock import org.schabi.newpipe.extractor.localization.Localization import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import java.io.IOException import java.security.MessageDigest import java.util.Base64 import java.util.Collections @@ -79,32 +76,6 @@ internal object SabrInitializationData { return initialized.firstOrNull { (candidate) -> candidate.matches(format) }?.second } - suspend fun fetchExact( - holder: SabrSessionHolder, - format: YoutubeSabrFormat, - timeoutMs: Long, - cache: CacheService? = null, - ): ByteArray { - fetch(holder.key.videoId, format, cache)?.let { cached -> - if (holder.session.streamState.ingestInitializationData(format, cached) && - holder.session.streamState.hasSegmentIndex(format) - ) { - return cached - } - } - val poToken = holder.session.streamState.poToken - ?.takeIf { it.isNotEmpty() } - ?: holder.playerContextToken?.streamingPoTokenBytesFor(holder.info) - ?: throw IOException("Missing video-bound PO token for Android playback initialization") - val bytes = runInterruptible(Dispatchers.IO) { - holder.withPlayerContext { - fetchInitializationData(format, Localization("en", "US"), timeoutMs, poToken) - } - } - remember(holder.key.videoId, format, bytes, cache) - return bytes - } - private suspend fun CacheService.getBytes(key: String): ByteArray? = runCatching { get(key)?.let { Base64.getDecoder().decode(it) } }.getOrNull() @@ -133,9 +104,3 @@ internal object SabrInitializationData { .digest(value.toByteArray()) .joinToString("") { byte -> "%02x".format(byte) } } - -internal suspend fun SabrSessionStore.fetchExactInitializationData( - holder: SabrSessionHolder, - format: YoutubeSabrFormat, - timeoutMs: Long, -): ByteArray = SabrInitializationData.fetchExact(holder, format, timeoutMs, initCache) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt index ae2cbef4..2bd6732d 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt @@ -5,10 +5,7 @@ import kotlinx.coroutines.withTimeoutOrNull import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -internal class SabrPlaybackSessionService( - private val sessionStore: SabrSessionStore, - private val purpose: SabrSessionPurpose = SabrSessionPurpose.PLAYBACK, -) { +internal class SabrPlaybackSessionService(private val sessionStore: SabrSessionStore) { suspend fun prepare( videoId: String, userId: String, @@ -19,8 +16,6 @@ internal class SabrPlaybackSessionService( audioOnly: Boolean = false, isLive: Boolean = false, initialGeneration: Long = 0L, - preloadInitialization: Boolean = true, - startPumpOnPrepare: Boolean = true, ): SabrPlaybackPreparation { val holder = sessionStore.getOrCreate( videoId = videoId, @@ -31,7 +26,7 @@ internal class SabrPlaybackSessionService( initialToken = prepared.initialToken, startTimeMs = startTimeMs, startPump = false, - purpose = purpose, + purpose = SabrSessionPurpose.PLAYBACK, audioOnly = audioOnly, initialGeneration = initialGeneration, ) @@ -44,7 +39,7 @@ internal class SabrPlaybackSessionService( holder.session.streamState.setWriteTopLevelPlayerTimeMs(false) } sessionStore.ensureWarmed(holder, LIVE_INITIAL_PUMPS) - } else if (preloadInitialization) { + } else { val initialization = SabrPlaybackInitializationPreloader.preload( sessionStore, holder, @@ -64,7 +59,7 @@ internal class SabrPlaybackSessionService( holder, holder.resolvePlaybackStartMs(startTimeMs), audioOnly, - startPumpOnPrepare, + startPump = true, ) } @@ -106,7 +101,7 @@ internal class SabrPlaybackSessionService( } fun lookup(sessionId: String): SabrSessionHolder? = sessionStore.lookupByToken(sessionId) - ?.takeIf { it.key.purpose == purpose } + ?.takeIf { it.key.purpose == SabrSessionPurpose.PLAYBACK } fun startPump(holder: SabrSessionHolder): Unit = sessionStore.startPump(holder) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionIdentity.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionIdentity.kt new file mode 100644 index 00000000..f1e86285 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionIdentity.kt @@ -0,0 +1,25 @@ +package dev.typetype.server.services + +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper +import org.schabi.newpipe.extractor.services.youtube.sabr.TypeTypeYoutubeSabrInfoFactory +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo + +internal object SabrSessionIdentity { + fun fresh(info: YoutubeSabrInfo): YoutubeSabrInfo { + val cpn = YoutubeParsingHelper.generateContentPlaybackNonce() + val url = requireNotNull(info.serverAbrStreamingUrl) + .toHttpUrl() + .newBuilder() + .setQueryParameter("cpn", cpn) + .build() + .toString() + return TypeTypeYoutubeSabrInfoFactory.withPlaybackIdentity( + info, + url, + info.clientVersion, + cpn, + info.visitorData, + ) + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionPurpose.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionPurpose.kt index 71467c01..83f64b5e 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionPurpose.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionPurpose.kt @@ -3,6 +3,5 @@ package dev.typetype.server.services internal enum class SabrSessionPurpose { MANIFEST, PLAYBACK, - ANDROID_PLAYBACK, DOWNLOAD, } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt index e9aa476e..c4dd1f57 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt @@ -52,10 +52,9 @@ internal class SabrSessionStore( audioOnly: Boolean = false, initialGeneration: Long = 0L, ): SabrSessionHolder { - val playbackToken = if (purpose == SabrSessionPurpose.PLAYBACK || purpose == SabrSessionPurpose.ANDROID_PLAYBACK) { - SabrSessionTokenGenerator.newToken() - } else { - null + val sessionToken = SabrSessionTokenGenerator.newToken() + val isolatedSourceId = sessionToken.takeIf { + purpose == SabrSessionPurpose.PLAYBACK || purpose == SabrSessionPurpose.DOWNLOAD } val key = SabrSessionKey( videoId, @@ -66,23 +65,24 @@ internal class SabrSessionStore( startTimeMs.coerceAtLeast(0L), purpose, audioOnly, - playbackToken, + isolatedSourceId, ) registry.getReusable(key)?.let { return it } registry.ensureCapacity(maxSessions) val provider = TypetypeTokenSabrPoTokenProvider(tokenClient, initialToken) - val session = YoutubeSabrSession(info, audioFormat, videoFormat, provider) + val sessionInfo = if (isolatedSourceId == null) info else SabrSessionIdentity.fresh(info) + val session = YoutubeSabrSession(sessionInfo, audioFormat, videoFormat, provider) val normalizedStartTimeMs = startTimeMs.coerceAtLeast(0L) session.streamState.setPlayerTimeMs(normalizedStartTimeMs) - runCatching { provider.getPoToken(info, session.streamState) } + runCatching { provider.getPoToken(sessionInfo, session.streamState) } .getOrNull() ?.let { session.streamState.setPoToken(it) } val holder = SabrSessionHolder( session, - info, + sessionInfo, audioFormat, videoFormat, - playbackToken ?: SabrSessionTokenGenerator.newToken(), + sessionToken, key, Instant.now(), initialToken, @@ -151,13 +151,17 @@ internal class SabrSessionStore( videoId: String, startTimeMs: Long = 0L, cachedFirst: Boolean = false, - ): SabrPreparedInfo? = infoFetcher.fetchInfo(videoId, startTimeMs, cachedFirst) + isolatedPlayback: Boolean = false, + ): SabrPreparedInfo? = infoFetcher.fetchInfo(videoId, startTimeMs, cachedFirst, isolatedPlayback) internal suspend fun rememberExtractedInfo(videoId: String, info: YoutubeSabrInfo): Unit = infoFetcher.rememberExtractedInfo(videoId, info) internal suspend fun invalidatePlaybackInfo(videoId: String): Unit = infoFetcher.invalidatePlayback(videoId) + internal fun refreshVideoPoToken(videoId: String): SabrTokenBundle? = + tokenClient.fetch(videoId, refreshVideo = true) + internal suspend fun fetchSegment( holder: SabrSessionHolder, request: SabrSegmentRequest, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecovery.kt b/src/main/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecovery.kt index 6f8d83de..393988a8 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecovery.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecovery.kt @@ -6,18 +6,28 @@ internal class SabrUnauthorizedResponseRecovery( private val refreshPoToken: (String) -> SabrTokenBundle?, ) { fun verify(holder: SabrSessionHolder): Unit { - val response = holder.session.diagnosticTrace.substringAfterLast("response n=", missingDelimiterValue = "") - if (!response.contains(" http=403 ")) return - if (!holder.markUnauthorizedRefreshAttempted()) throw unauthorized() - val refreshed = refreshPoToken(holder.key.videoId) ?: throw unauthorized() + val status = latestUnauthorizedStatus(holder.session.diagnosticTrace) ?: return + if (!holder.markUnauthorizedRefreshAttempted()) throw unauthorized(status) + val refreshed = refreshPoToken(holder.key.videoId) ?: throw unauthorized(status) val token = refreshed.streamingPoTokenBytesFor(holder.info) ?.takeUnless { holder.session.streamState.poToken?.contentEquals(it) == true } - ?: throw unauthorized() + ?: throw unauthorized(status) holder.playerContextToken = refreshed holder.session.streamState.setPoToken(token) - holder.session.addDiagnosticEvent("upstream 403 refreshed TypeType PO token") + holder.session.addDiagnosticEvent("upstream $status refreshed TypeType PO token") } - private fun unauthorized(): SabrRecoverableException = - SabrRecoverableException("SABR upstream unauthorized HTTP 403 after TypeType token refresh") + private fun unauthorized(status: Int): SabrRecoverableException = + SabrRecoverableException("SABR upstream unauthorized HTTP $status after TypeType token refresh") + + companion object { + fun latestUnauthorizedStatus(trace: String): Int? { + val response = trace.substringAfterLast("response n=", missingDelimiterValue = "") + return when { + response.contains(" http=401 ") -> 401 + response.contains(" http=403 ") -> 403 + else -> null + } + } + } } diff --git a/src/main/kotlin/dev/typetype/server/services/SearchHistoryService.kt b/src/main/kotlin/dev/typetype/server/services/SearchHistoryService.kt index b6d5eafc..45047516 100644 --- a/src/main/kotlin/dev/typetype/server/services/SearchHistoryService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SearchHistoryService.kt @@ -13,6 +13,13 @@ import java.util.UUID class SearchHistoryService { + suspend fun getAll(userId: String): List = DatabaseFactory.query { + SearchHistoryTable.selectAll() + .where { SearchHistoryTable.userId eq userId } + .orderBy(SearchHistoryTable.searchedAt to SortOrder.DESC) + .map { it.toItem() } + } + suspend fun getPage(userId: String, page: Int, limit: Int): Pair, Long> = DatabaseFactory.query { val query = SearchHistoryTable.selectAll() .where { SearchHistoryTable.userId eq userId } diff --git a/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt b/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt index 1395d835..451019b7 100644 --- a/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt +++ b/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt @@ -19,6 +19,7 @@ private val CAPTION_STYLES_SERIALIZER = CaptionStylesItem.serializer() internal fun ResultRow.toSettingsItem(): SettingsItem = SettingsItem( defaultService = this[SettingsTable.defaultService], defaultQuality = this[SettingsTable.defaultQuality], + defaultPlaybackSpeed = this[SettingsTable.defaultPlaybackSpeed], defaultLandingPage = this[SettingsTable.defaultLandingPage], autoplay = this[SettingsTable.autoplay], skipPlaylistAutoplayScreen = this[SettingsTable.skipPlaylistAutoplayScreen], @@ -55,6 +56,7 @@ internal fun ResultRow.toSettingsItem(): SettingsItem = SettingsItem( internal fun UpdateBuilder<*>.writeSettings(settings: SettingsItem) { this[SettingsTable.defaultService] = settings.defaultService this[SettingsTable.defaultQuality] = settings.defaultQuality + this[SettingsTable.defaultPlaybackSpeed] = settings.defaultPlaybackSpeed this[SettingsTable.defaultLandingPage] = settings.defaultLandingPage this[SettingsTable.autoplay] = settings.autoplay this[SettingsTable.skipPlaylistAutoplayScreen] = settings.skipPlaylistAutoplayScreen @@ -92,6 +94,8 @@ internal fun UpdateBuilder<*>.writeSettings(settings: SettingsItem) { internal fun SettingsItem.normalized(): SettingsItem = copy( defaultLandingPage = defaultLandingPage.ifBlank { "home" }, + defaultPlaybackSpeed = defaultPlaybackSpeed.takeIf { it.isFinite() } + ?.coerceIn(MIN_PLAYBACK_SPEED, MAX_PLAYBACK_SPEED) ?: 1.0, accessMode = accessMode.toAccessMode(), sponsorBlockCategoryActions = sponsorBlockCategoryActions.withDefaultSponsorBlockCategoryActions(), sponsorBlockMinimumDuration = sponsorBlockMinimumDuration.coerceAtLeast(0), @@ -101,6 +105,8 @@ internal fun SettingsItem.normalized(): SettingsItem = copy( ) private val DEARROW_TITLE_MODES = setOf("original", "dearrow") +private const val MIN_PLAYBACK_SPEED = 0.25 +private const val MAX_PLAYBACK_SPEED = 4.0 private val DEARROW_THUMBNAIL_MODES = setOf("original", "dearrow", "random", "dearrow_or_random") private val DEARROW_TRUST_MODES = setOf("accepted", "locked") diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt index c0b46aea..16d87f35 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt @@ -26,7 +26,7 @@ internal class SubscriptionFeedBuilder(private val channelService: ChannelServic } } }.map { it.await() } - val videos = outcomes.flatMap { it.videos }.deduplicated().sortedWith(FEED_ORDER) + val videos = outcomes.flatMap { it.videos }.deduplicated() SubscriptionFeedBuildResult( videos = videos, successfulSources = outcomes.sumOf { it.successfulSources }, @@ -64,14 +64,12 @@ internal class SubscriptionFeedBuilder(private val channelService: ChannelServic private fun List.deduplicated(): List = buildMap { this@deduplicated.forEach { video -> - val key = video.feedKey() + val key = video.subscriptionFeedKey() val current = get(key) if (current == null || video.isLive && !current.isLive) put(key, video) } }.values.toList() - private fun VideoItem.feedKey(): String = url.ifBlank { id.ifBlank { "$uploaderUrl|$title" } } - private fun String.toLivestreamsTabUrl(): String { val uri = URI(this) val path = uri.path.trimEnd('/') @@ -95,8 +93,6 @@ internal class SubscriptionFeedBuilder(private val channelService: ChannelServic private const val MAX_CONCURRENT_FETCHES = 20 private const val CHANNEL_TIMEOUT_MS = 15_000L private val YOUTUBE_CHANNEL_TABS = setOf("featured", "videos", "shorts", "streams", "playlists", "community", "about") - private val FEED_ORDER = compareByDescending { it.isLive } - .thenByDescending { if (it.uploaded < 0L) Long.MIN_VALUE else it.uploaded } } } diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedOrderer.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedOrderer.kt new file mode 100644 index 00000000..c9cb2f66 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedOrderer.kt @@ -0,0 +1,46 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.VideoItem + +internal class SubscriptionFeedOrderer { + fun order( + videos: List, + previous: SubscriptionFeedSnapshot?, + refreshedAt: Long, + ): SubscriptionFeedOrdering { + val previousByKey = previous?.videos.orEmpty().associateBy(VideoItem::subscriptionFeedKey) + val promotedAt = buildMap { + videos.filter(VideoItem::isLive).forEach { video -> + val key = video.subscriptionFeedKey() + val previousVideo = previousByKey[key] + val promotion = when { + previousVideo?.isLive == true -> previous?.livePromotedAt?.get(key) + ?: previous?.generatedAt + ?: refreshedAt + else -> refreshedAt + } + put(key, promotion) + } + } + val ordered = videos.sortedWith( + compareByDescending { video -> + promotedAt[video.subscriptionFeedKey()] ?: video.feedTimestamp() + }.thenBy(VideoItem::subscriptionFeedKey), + ) + return SubscriptionFeedOrdering(ordered, promotedAt) + } + + private fun VideoItem.feedTimestamp(): Long = when { + uploaded >= 0L -> uploaded + publishedAt != null && publishedAt >= 0L -> publishedAt + else -> Long.MIN_VALUE + } +} + +internal data class SubscriptionFeedOrdering( + val videos: List, + val livePromotedAt: Map, +) + +internal fun VideoItem.subscriptionFeedKey(): String = + url.ifBlank { id.ifBlank { "$uploaderUrl|$title" } } diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt index 56f357b5..48a09fc2 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt @@ -27,6 +27,7 @@ class SubscriptionFeedService( ) { private val store = SubscriptionFeedSnapshotStore(cache, clock) private val builder = SubscriptionFeedBuilder(channelService) + private val orderer = SubscriptionFeedOrderer() private val refreshJobs = ConcurrentHashMap() private val generation = AtomicLong(clock()) @@ -121,7 +122,7 @@ class SubscriptionFeedService( val subscriptions = subscriptionsService.getAll(userId) val result = builder.build(subscriptions) if (store.invalidationToken(userId) != invalidation) return true - val valid = result.failedSources == 0 || previous == null && result.successfulSources > 0 || subscriptions.isEmpty() + val valid = result.successfulSources > 0 || subscriptions.isEmpty() if (!valid) { logger.warn( "subscription_feed event=refresh_kept_previous user={} durationMs={} failedSources={}", @@ -129,8 +130,16 @@ class SubscriptionFeedService( ) return false } + val refreshedAt = clock() + val ordering = orderer.order(result.videos, previous, refreshedAt) val nextGeneration = generation.updateAndGet { maxOf(it + 1, clock(), (previous?.generation ?: 0L) + 1) } - val snapshot = SubscriptionFeedSnapshot(nextGeneration, clock(), stale = false, result.videos) + val snapshot = SubscriptionFeedSnapshot( + generation = nextGeneration, + generatedAt = refreshedAt, + stale = false, + videos = ordering.videos, + livePromotedAt = ordering.livePromotedAt, + ) runCatching { store.publish(userId, snapshot) }.onFailure { logger.warn("subscription_feed event=publish_failed user={} error={}", userKey(userId), it.message) return false @@ -141,7 +150,7 @@ class SubscriptionFeedService( } logger.info( "subscription_feed event=refresh_completed user={} requestId={} generation={} videos={} failedSources={} durationMs={}", - userKey(userId), requestId ?: "none", nextGeneration, result.videos.size, result.failedSources, clock() - startedAt, + userKey(userId), requestId ?: "none", nextGeneration, ordering.videos.size, result.failedSources, clock() - startedAt, ) return false } diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt index 89ab75f1..b48aa4a8 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt @@ -12,6 +12,7 @@ internal data class SubscriptionFeedSnapshot( val generatedAt: Long, val stale: Boolean, val videos: List, + val livePromotedAt: Map = emptyMap(), ) @Serializable diff --git a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCategory.kt b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCategory.kt new file mode 100644 index 00000000..7ba90ef9 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCategory.kt @@ -0,0 +1,26 @@ +package dev.typetype.server.services + +enum class TypeTypeBackupCategory(val wireName: String) { + SUBSCRIPTIONS("subscriptions"), + HISTORY("history"), + PLAYLISTS("playlists"), + WATCH_LATER("watchLater"), + FAVORITES("favorites"), + PROGRESS("progress"), + SEARCH_HISTORY("searchHistory"), + SAVED_PLAYLISTS("savedPlaylists"), + SETTINGS("settings"), + CONTENT_FILTERS("contentFilters"); + + companion object { + val all: Set = entries.toSet() + + fun parse(raw: String?): Set? { + if (raw.isNullOrBlank() || raw == "all") return all + val requested = raw.split(',').map(String::trim).filter(String::isNotEmpty) + if (requested.isEmpty()) return null + val byName = entries.associateBy(TypeTypeBackupCategory::wireName) + return requested.map { byName[it] ?: return null }.toSet() + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCoreRestore.kt b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCoreRestore.kt new file mode 100644 index 00000000..f8f5b715 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCoreRestore.kt @@ -0,0 +1,82 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.tables.HistoryTable +import dev.typetype.server.db.tables.PlaylistVideosTable +import dev.typetype.server.db.tables.PlaylistsTable +import dev.typetype.server.db.tables.SubscriptionsTable +import dev.typetype.server.models.HistoryItem +import dev.typetype.server.models.PlaylistItem +import dev.typetype.server.models.SubscriptionItem +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.batchInsert +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import java.util.UUID + +internal object TypeTypeBackupCoreRestore { + fun subscriptions(userId: String, items: List): Int { + SubscriptionsTable.deleteWhere { SubscriptionsTable.userId eq userId } + SubscriptionsTable.batchInsert(items, shouldReturnGeneratedValues = false) { item -> + this[SubscriptionsTable.userId] = userId + this[SubscriptionsTable.channelUrl] = ChannelUrlCanonicalizer.canonicalize(item.channelUrl) + this[SubscriptionsTable.name] = item.name + this[SubscriptionsTable.avatarUrl] = item.avatarUrl + this[SubscriptionsTable.subscribedAt] = item.subscribedAt + } + return items.size + } + + fun history(userId: String, items: List): Int { + HistoryTable.deleteWhere { HistoryTable.userId eq userId } + HistoryTable.batchInsert(items, shouldReturnGeneratedValues = false) { item -> + this[HistoryTable.id] = UUID.randomUUID().toString() + this[HistoryTable.userId] = userId + this[HistoryTable.url] = item.url + this[HistoryTable.title] = item.title + this[HistoryTable.thumbnail] = item.thumbnail + this[HistoryTable.channelName] = item.channelName + this[HistoryTable.channelUrl] = item.channelUrl + this[HistoryTable.channelAvatar] = item.channelAvatar + this[HistoryTable.duration] = item.duration + this[HistoryTable.progress] = item.progress + this[HistoryTable.watchedAt] = item.watchedAt + } + return items.size + } + + fun playlists(userId: String, items: List): Pair { + PlaylistVideosTable.deleteWhere { PlaylistVideosTable.userId eq userId } + PlaylistsTable.deleteWhere { PlaylistsTable.userId eq userId } + var videoCount = 0 + items.forEach { playlist -> + val playlistId = UUID.randomUUID().toString() + PlaylistsTable.batchInsert(listOf(playlist), shouldReturnGeneratedValues = false) { + this[PlaylistsTable.id] = playlistId + this[PlaylistsTable.userId] = userId + this[PlaylistsTable.name] = it.name + this[PlaylistsTable.description] = it.description + this[PlaylistsTable.createdAt] = it.createdAt + } + PlaylistVideosTable.batchInsert( + playlist.videos, + shouldReturnGeneratedValues = false, + ) { video -> + this[PlaylistVideosTable.id] = UUID.randomUUID().toString() + this[PlaylistVideosTable.playlistId] = playlistId + this[PlaylistVideosTable.userId] = userId + this[PlaylistVideosTable.url] = video.url + this[PlaylistVideosTable.title] = video.title + this[PlaylistVideosTable.thumbnail] = video.thumbnail + this[PlaylistVideosTable.duration] = video.duration + this[PlaylistVideosTable.position] = video.position + this[PlaylistVideosTable.channelName] = video.channelName + this[PlaylistVideosTable.channelUrl] = video.channelUrl + this[PlaylistVideosTable.channelAvatar] = video.channelAvatar + this[PlaylistVideosTable.viewCount] = video.viewCount + this[PlaylistVideosTable.addedAt] = video.addedAt + this[PlaylistVideosTable.publishedAt] = video.publishedAt + } + videoCount += playlist.videos.size + } + return items.size to videoCount + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupFilterRestore.kt b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupFilterRestore.kt new file mode 100644 index 00000000..a105e0a9 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupFilterRestore.kt @@ -0,0 +1,82 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.tables.AllowedChannelsTable +import dev.typetype.server.db.tables.AllowedPlaylistsTable +import dev.typetype.server.db.tables.BlockedChannelsTable +import dev.typetype.server.db.tables.BlockedKeywordsTable +import dev.typetype.server.db.tables.BlockedVideosTable +import dev.typetype.server.models.TypeTypeContentFiltersBackup +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.batchInsert +import org.jetbrains.exposed.v1.jdbc.deleteWhere + +private const val USER_SCOPE = "user" + +internal object TypeTypeBackupFilterRestore { + fun restore(userId: String, backup: TypeTypeContentFiltersBackup): Map { + deleteUserFilters(userId) + BlockedChannelsTable.batchInsert(backup.blockedChannels, false) { item -> + this[BlockedChannelsTable.userId] = userId + this[BlockedChannelsTable.scope] = USER_SCOPE + this[BlockedChannelsTable.channelUrl] = item.url + this[BlockedChannelsTable.channelName] = item.name + this[BlockedChannelsTable.channelThumbnailUrl] = item.thumbnailUrl + this[BlockedChannelsTable.blockedAt] = item.blockedAt + } + BlockedVideosTable.batchInsert(backup.blockedVideos, false) { item -> + this[BlockedVideosTable.userId] = userId + this[BlockedVideosTable.scope] = USER_SCOPE + this[BlockedVideosTable.videoUrl] = item.url + this[BlockedVideosTable.blockedAt] = item.blockedAt + } + BlockedKeywordsTable.batchInsert(backup.blockedKeywords, false) { item -> + this[BlockedKeywordsTable.userId] = userId + this[BlockedKeywordsTable.scope] = USER_SCOPE + this[BlockedKeywordsTable.keyword] = normalizeBlockedKeyword(item.keyword) + this[BlockedKeywordsTable.blockedAt] = item.blockedAt + } + AllowedChannelsTable.batchInsert(backup.allowedChannels, false) { item -> + this[AllowedChannelsTable.userId] = userId + this[AllowedChannelsTable.scope] = USER_SCOPE + this[AllowedChannelsTable.channelUrl] = normalizeChannelKey(item.url) + this[AllowedChannelsTable.channelName] = item.name + this[AllowedChannelsTable.channelThumbnailUrl] = item.thumbnailUrl + this[AllowedChannelsTable.allowedAt] = item.allowedAt + } + AllowedPlaylistsTable.batchInsert(backup.allowedPlaylists, false) { item -> + this[AllowedPlaylistsTable.userId] = userId + this[AllowedPlaylistsTable.scope] = USER_SCOPE + this[AllowedPlaylistsTable.playlistUrl] = normalizePlaylistKey(item.url) + this[AllowedPlaylistsTable.title] = item.title + this[AllowedPlaylistsTable.thumbnailUrl] = item.thumbnailUrl + this[AllowedPlaylistsTable.uploaderName] = item.uploaderName + this[AllowedPlaylistsTable.allowedAt] = item.allowedAt + } + return linkedMapOf( + "blockedChannels" to backup.blockedChannels.size, + "blockedVideos" to backup.blockedVideos.size, + "blockedKeywords" to backup.blockedKeywords.size, + "allowedChannels" to backup.allowedChannels.size, + "allowedPlaylists" to backup.allowedPlaylists.size, + ) + } + + private fun deleteUserFilters(userId: String) { + BlockedChannelsTable.deleteWhere { + (BlockedChannelsTable.userId eq userId) and (BlockedChannelsTable.scope eq USER_SCOPE) + } + BlockedVideosTable.deleteWhere { + (BlockedVideosTable.userId eq userId) and (BlockedVideosTable.scope eq USER_SCOPE) + } + BlockedKeywordsTable.deleteWhere { + (BlockedKeywordsTable.userId eq userId) and (BlockedKeywordsTable.scope eq USER_SCOPE) + } + AllowedChannelsTable.deleteWhere { + (AllowedChannelsTable.userId eq userId) and (AllowedChannelsTable.scope eq USER_SCOPE) + } + AllowedPlaylistsTable.deleteWhere { + (AllowedPlaylistsTable.userId eq userId) and (AllowedPlaylistsTable.scope eq USER_SCOPE) + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupLibraryRestore.kt b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupLibraryRestore.kt new file mode 100644 index 00000000..febf7710 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupLibraryRestore.kt @@ -0,0 +1,111 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.tables.FavoritesTable +import dev.typetype.server.db.tables.ProgressTable +import dev.typetype.server.db.tables.SavedPlaylistsTable +import dev.typetype.server.db.tables.SearchHistoryTable +import dev.typetype.server.db.tables.SettingsTable +import dev.typetype.server.db.tables.WatchLaterTable +import dev.typetype.server.models.FavoriteItem +import dev.typetype.server.models.ProgressItem +import dev.typetype.server.models.SavedPlaylistItem +import dev.typetype.server.models.SearchHistoryItem +import dev.typetype.server.models.SettingsItem +import dev.typetype.server.models.WatchLaterItem +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.batchInsert +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.update +import java.util.UUID + +internal object TypeTypeBackupLibraryRestore { + fun watchLater(userId: String, items: List): Int { + WatchLaterTable.deleteWhere { WatchLaterTable.userId eq userId } + WatchLaterTable.batchInsert(items, shouldReturnGeneratedValues = false) { item -> + this[WatchLaterTable.userId] = userId + this[WatchLaterTable.url] = item.url + this[WatchLaterTable.title] = item.title + this[WatchLaterTable.thumbnail] = item.thumbnail + this[WatchLaterTable.duration] = item.duration + this[WatchLaterTable.addedAt] = item.addedAt + this[WatchLaterTable.channelName] = item.channelName + this[WatchLaterTable.channelUrl] = item.channelUrl + this[WatchLaterTable.channelAvatar] = item.channelAvatar + this[WatchLaterTable.viewCount] = item.viewCount + this[WatchLaterTable.publishedAt] = item.publishedAt + } + return items.size + } + + fun favorites(userId: String, items: List): Int { + FavoritesTable.deleteWhere { FavoritesTable.userId eq userId } + FavoritesTable.batchInsert(items, shouldReturnGeneratedValues = false) { item -> + this[FavoritesTable.userId] = userId + this[FavoritesTable.videoUrl] = item.videoUrl + this[FavoritesTable.favoritedAt] = item.favoritedAt + this[FavoritesTable.title] = item.title + this[FavoritesTable.thumbnail] = item.thumbnail + this[FavoritesTable.duration] = item.duration + this[FavoritesTable.channelName] = item.channelName + this[FavoritesTable.channelUrl] = item.channelUrl + this[FavoritesTable.channelAvatar] = item.channelAvatar + this[FavoritesTable.viewCount] = item.viewCount + this[FavoritesTable.publishedAt] = item.publishedAt + } + return items.size + } + + fun progress(userId: String, items: List): Int { + ProgressTable.deleteWhere { ProgressTable.userId eq userId } + ProgressTable.batchInsert(items, shouldReturnGeneratedValues = false) { item -> + this[ProgressTable.userId] = userId + this[ProgressTable.videoUrl] = item.videoUrl + this[ProgressTable.position] = item.position.coerceAtLeast(0) + this[ProgressTable.updatedAt] = item.updatedAt + } + return items.size + } + + fun searchHistory(userId: String, items: List): Int { + SearchHistoryTable.deleteWhere { SearchHistoryTable.userId eq userId } + SearchHistoryTable.batchInsert(items, shouldReturnGeneratedValues = false) { item -> + this[SearchHistoryTable.id] = UUID.randomUUID().toString() + this[SearchHistoryTable.userId] = userId + this[SearchHistoryTable.term] = item.term + this[SearchHistoryTable.searchedAt] = item.searchedAt + } + return items.size + } + + fun savedPlaylists(userId: String, items: List): Int { + SavedPlaylistsTable.deleteWhere { SavedPlaylistsTable.userId eq userId } + SavedPlaylistsTable.batchInsert(items, shouldReturnGeneratedValues = false) { item -> + this[SavedPlaylistsTable.id] = UUID.randomUUID().toString() + this[SavedPlaylistsTable.userId] = userId + this[SavedPlaylistsTable.publicPlaylistId] = item.publicPlaylistId + this[SavedPlaylistsTable.url] = item.url + this[SavedPlaylistsTable.title] = item.title + this[SavedPlaylistsTable.thumbnailUrl] = item.thumbnailUrl + this[SavedPlaylistsTable.uploaderName] = item.uploaderName + this[SavedPlaylistsTable.streamCount] = item.streamCount + this[SavedPlaylistsTable.playlistType] = item.playlistType + this[SavedPlaylistsTable.savedAt] = item.savedAt + } + return items.size + } + + fun settings(userId: String, item: SettingsItem): Int { + val normalized = item.normalized() + val updated = SettingsTable.update({ SettingsTable.userId eq userId }) { + it.writeSettings(normalized) + } + if (updated == 0) { + SettingsTable.insert { + it[SettingsTable.userId] = userId + it.writeSettings(normalized) + } + } + return 1 + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupRestoreWriter.kt b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupRestoreWriter.kt new file mode 100644 index 00000000..ad10644a --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupRestoreWriter.kt @@ -0,0 +1,80 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.models.TypeTypeBackupItem +import dev.typetype.server.models.TypeTypeRestoreSummary + +internal object TypeTypeBackupRestoreWriter { + suspend fun restore( + userId: String, + backup: TypeTypeBackupItem, + categories: Set, + ): TypeTypeRestoreSummary = DatabaseFactory.query { + val restored = linkedMapOf() + if (TypeTypeBackupCategory.SUBSCRIPTIONS in categories) { + restored["subscriptions"] = TypeTypeBackupCoreRestore.subscriptions( + userId, + requireNotNull(backup.subscriptions), + ) + } + if (TypeTypeBackupCategory.HISTORY in categories) { + restored["history"] = TypeTypeBackupCoreRestore.history( + userId, + requireNotNull(backup.history), + ) + } + if (TypeTypeBackupCategory.PLAYLISTS in categories) { + val counts = TypeTypeBackupCoreRestore.playlists( + userId, + requireNotNull(backup.playlists), + ) + restored["playlists"] = counts.first + restored["playlistVideos"] = counts.second + } + if (TypeTypeBackupCategory.WATCH_LATER in categories) { + restored["watchLater"] = TypeTypeBackupLibraryRestore.watchLater( + userId, + requireNotNull(backup.watchLater), + ) + } + if (TypeTypeBackupCategory.FAVORITES in categories) { + restored["favorites"] = TypeTypeBackupLibraryRestore.favorites( + userId, + requireNotNull(backup.favorites), + ) + } + if (TypeTypeBackupCategory.PROGRESS in categories) { + restored["progress"] = TypeTypeBackupLibraryRestore.progress( + userId, + requireNotNull(backup.progress), + ) + } + if (TypeTypeBackupCategory.SEARCH_HISTORY in categories) { + restored["searchHistory"] = TypeTypeBackupLibraryRestore.searchHistory( + userId, + requireNotNull(backup.searchHistory), + ) + } + if (TypeTypeBackupCategory.SAVED_PLAYLISTS in categories) { + restored["savedPlaylists"] = TypeTypeBackupLibraryRestore.savedPlaylists( + userId, + requireNotNull(backup.savedPlaylists), + ) + } + if (TypeTypeBackupCategory.SETTINGS in categories) { + restored["settings"] = TypeTypeBackupLibraryRestore.settings( + userId, + requireNotNull(backup.settings), + ) + } + if (TypeTypeBackupCategory.CONTENT_FILTERS in categories) { + restored.putAll( + TypeTypeBackupFilterRestore.restore( + userId, + requireNotNull(backup.contentFilters), + ), + ) + } + TypeTypeRestoreSummary(restored) + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupService.kt b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupService.kt new file mode 100644 index 00000000..1c98ddfb --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupService.kt @@ -0,0 +1,99 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.TYPE_TYPE_BACKUP_FORMAT +import dev.typetype.server.models.TYPE_TYPE_BACKUP_VERSION +import dev.typetype.server.models.TypeTypeBackupItem +import dev.typetype.server.models.TypeTypeContentFiltersBackup +import dev.typetype.server.models.TypeTypeRestoreSummary + +class TypeTypeBackupService( + private val subscriptions: SubscriptionsService, + private val history: HistoryService, + private val playlists: PlaylistService, + private val watchLater: WatchLaterService, + private val favorites: FavoritesService, + private val progress: ProgressService, + private val searchHistory: SearchHistoryService, + private val savedPlaylists: SavedPlaylistService, + private val settings: SettingsService, + private val blocked: BlockedService, + private val allowedChannels: AllowedChannelsService, + private val allowedPlaylists: AllowedPlaylistsService, +) { + suspend fun export( + userId: String, + categories: Set, + ): TypeTypeBackupItem { + fun includes(category: TypeTypeBackupCategory) = category in categories + val fullPlaylists = if (includes(TypeTypeBackupCategory.PLAYLISTS)) { + playlists.getAll(userId).mapNotNull { playlists.getById(userId, it.id) } + } else { + null + } + return TypeTypeBackupItem( + exportedAt = System.currentTimeMillis(), + categories = categories.map(TypeTypeBackupCategory::wireName).sorted(), + subscriptions = if (includes(TypeTypeBackupCategory.SUBSCRIPTIONS)) subscriptions.getAll(userId) else null, + history = if (includes(TypeTypeBackupCategory.HISTORY)) history.getAll(userId) else null, + playlists = fullPlaylists, + watchLater = if (includes(TypeTypeBackupCategory.WATCH_LATER)) watchLater.getAll(userId) else null, + favorites = if (includes(TypeTypeBackupCategory.FAVORITES)) favorites.getAll(userId) else null, + progress = if (includes(TypeTypeBackupCategory.PROGRESS)) progress.getAll(userId) else null, + searchHistory = if (includes(TypeTypeBackupCategory.SEARCH_HISTORY)) searchHistory.getAll(userId) else null, + savedPlaylists = if (includes(TypeTypeBackupCategory.SAVED_PLAYLISTS)) savedPlaylists.getAll(userId) else null, + settings = if (includes(TypeTypeBackupCategory.SETTINGS)) settings.get(userId) else null, + contentFilters = if (includes(TypeTypeBackupCategory.CONTENT_FILTERS)) contentFilters(userId) else null, + ) + } + + suspend fun restore(userId: String, backup: TypeTypeBackupItem): TypeTypeRestoreSummary { + require(backup.format == TYPE_TYPE_BACKUP_FORMAT) { "Unsupported backup format" } + require(backup.version == TYPE_TYPE_BACKUP_VERSION) { "Unsupported backup version" } + require(backup.categories.isNotEmpty()) { "Backup has no categories" } + val categories = TypeTypeBackupCategory.parse(backup.categories.joinToString(",")) + ?: throw IllegalArgumentException("Invalid backup categories") + validateSections(backup, categories) + validateContentFilters(backup, categories) + return TypeTypeBackupRestoreWriter.restore(userId, backup, categories) + } + + private suspend fun contentFilters(userId: String) = TypeTypeContentFiltersBackup( + blockedChannels = blocked.getUserChannels(userId), + blockedVideos = blocked.getUserVideos(userId), + blockedKeywords = blocked.getUserKeywords(userId), + allowedChannels = allowedChannels.getUserChannels(userId), + allowedPlaylists = allowedPlaylists.getUserPlaylists(userId), + ) +} + +private fun validateSections( + backup: TypeTypeBackupItem, + categories: Set, +) { + val missing = categories.filter { category -> + when (category) { + TypeTypeBackupCategory.SUBSCRIPTIONS -> backup.subscriptions == null + TypeTypeBackupCategory.HISTORY -> backup.history == null + TypeTypeBackupCategory.PLAYLISTS -> backup.playlists == null + TypeTypeBackupCategory.WATCH_LATER -> backup.watchLater == null + TypeTypeBackupCategory.FAVORITES -> backup.favorites == null + TypeTypeBackupCategory.PROGRESS -> backup.progress == null + TypeTypeBackupCategory.SEARCH_HISTORY -> backup.searchHistory == null + TypeTypeBackupCategory.SAVED_PLAYLISTS -> backup.savedPlaylists == null + TypeTypeBackupCategory.SETTINGS -> backup.settings == null + TypeTypeBackupCategory.CONTENT_FILTERS -> backup.contentFilters == null + } + } + require(missing.isEmpty()) { "Backup is missing selected data" } +} + +private fun validateContentFilters( + backup: TypeTypeBackupItem, + categories: Set, +) { + if (TypeTypeBackupCategory.CONTENT_FILTERS !in categories) return + val keywords = requireNotNull(backup.contentFilters).blockedKeywords + .map { normalizeBlockedKeyword(it.keyword) } + require(keywords.all { it.length in 1..100 }) { "Blocked keywords must contain 1 to 100 characters" } + require(keywords.distinct().size == keywords.size) { "Backup contains duplicate blocked keywords" } +} diff --git a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClient.kt b/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClient.kt index 99abec11..0629ee93 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClient.kt +++ b/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClient.kt @@ -32,7 +32,10 @@ internal class TypetypeTokenYoutubeSessionClient( session.toSabrInfo(videoId) } - suspend fun fetchPlaybackSession(videoId: String): TokenYoutubeSession? = fetchSession(videoId)?.let { session -> + suspend fun fetchPlaybackSession( + videoId: String, + isolated: Boolean = false, + ): TokenYoutubeSession? = fetchSession(videoId, isolated)?.let { session -> session.toPlaybackSession(videoId) } @@ -60,9 +63,10 @@ internal class TypetypeTokenYoutubeSessionClient( ) } - private suspend fun fetchSession(videoId: String): JSONObject? { + private suspend fun fetchSession(videoId: String, isolated: Boolean = false): JSONObject? { val encodedVideoId = URLEncoder.encode(videoId, StandardCharsets.UTF_8) - val url = "${tokenServiceUrl.trimEnd('/')}/youtube/sabr/session?videoId=$encodedVideoId&client=MWEB" + val url = "${tokenServiceUrl.trimEnd('/')}/youtube/sabr/session?videoId=$encodedVideoId&client=MWEB" + + if (isolated) "&isolated=true" else "" return suspendCancellableCoroutine { continuation -> val call = client.newCall(Request.Builder().url(url).get().build()) continuation.invokeOnCancellation { call.cancel() } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClient.kt b/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClient.kt index 73c9740f..62d276cf 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClient.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClient.kt @@ -2,8 +2,6 @@ package dev.typetype.server.services internal enum class YoutubePlayerClient(val value: String) { MWEB("mweb"), - TV_SIMPLY("tv_simply"), WEB_SAFARI("web_safari"), - ANDROID_VR("android_vr"), TV_DOWNGRADED("tv_downgraded"), } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt index b36ee98f..4c3fdabe 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt @@ -1,19 +1,20 @@ package dev.typetype.server.services +import dev.typetype.server.models.FavoriteItem import dev.typetype.server.models.SubscriptionItem import java.nio.file.Path import java.util.zip.ZipFile object YoutubeTakeoutActivitySignalService { private val subscribedRegex = Regex("""(?:${YoutubeTakeoutActivityClassifier.subscribedPattern})\s*([\s\S]*?)
""", RegexOption.IGNORE_CASE) - private val likedRegex = Regex("""(?:${YoutubeTakeoutActivityClassifier.likedPattern})\s*([\s\S]*?)
""", RegexOption.IGNORE_CASE) + private val likedRegex = Regex("""(?:${YoutubeTakeoutActivityClassifier.likedPattern})\s*([\s\S]*?)
\s*(?:([\s\S]*?)
\s*)?([\s\S]*?)
""", RegexOption.IGNORE_CASE) private val watchUrlRegex = Regex("""https?://(?:www\.)?(?:youtube\.com/(?:watch\?v=|shorts/)|youtu\.be/)[A-Za-z0-9_-]{6,}""") private val channelUrlRegex = Regex("""https?://www\.youtube\.com/(?:channel/[A-Za-z0-9_-]+|@[A-Za-z0-9._-]+)""") private val spacesRegex = Regex("""\s+""") - fun parse(zipPath: Path): Pair, List> { + fun parse(zipPath: Path): Pair, List> { val subscriptions = mutableListOf() - val favorites = mutableListOf() + val favorites = mutableListOf() ZipFile(zipPath.toFile()).use { zip -> zip.entries().asSequence().forEach { entry -> val normalized = entry.name.lowercase() @@ -23,7 +24,7 @@ object YoutubeTakeoutActivitySignalService { favorites += parseFavorites(html) } } - return subscriptions.distinctBy { it.channelUrl } to favorites.distinct() + return subscriptions.distinctBy { it.channelUrl } to favorites.distinctBy { it.videoUrl } } private fun parseSubscriptions(html: String): List { @@ -34,13 +35,26 @@ object YoutubeTakeoutActivitySignalService { }.toList() } - private fun parseFavorites(html: String): List { + private fun parseFavorites(html: String): List { return likedRegex.findAll(html).mapNotNull { match -> - val source = decode(match.groupValues[1]) + " " + decode(match.groupValues[2]) - watchUrlRegex.find(source)?.value?.replace("http://", "https://") + val title = decode(match.groupValues[2]) + if (isUnavailable(title)) return@mapNotNull null + val source = decode(match.groupValues[1]) + " " + title + val videoUrl = watchUrlRegex.find(source)?.value?.replace("http://", "https://") + ?: return@mapNotNull null + FavoriteItem( + videoUrl = videoUrl, + favoritedAt = YoutubeTakeoutDateParser.parseEpochMillis(decode(match.groupValues[5])) ?: 0L, + title = title, + channelName = decode(match.groupValues[4]), + channelUrl = decode(match.groupValues[3]).replace("http://", "https://"), + ) }.toList() } + private fun isUnavailable(title: String): Boolean = + YoutubeTakeoutTextNormalizer.normalize(title) in unavailableTitles + private fun decode(value: String): String { return value .replace(" ", " ") @@ -52,4 +66,14 @@ object YoutubeTakeoutActivitySignalService { .replace(spacesRegex, " ") .trim() } + + private val unavailableTitles = setOf( + "deleted video", + "private video", + "video unavailable", + "video deleted", + "video indisponible", + "video privee", + "video supprimee", + ) } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt index bd30edff..2364cdc1 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt @@ -4,14 +4,27 @@ import java.time.Instant import java.time.LocalDate import java.time.OffsetDateTime import java.time.ZoneOffset +import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter +import java.util.Locale object YoutubeTakeoutDateParser { + private val activityFormatters = listOf( + Locale.ENGLISH, + Locale.FRENCH, + ).flatMap { locale -> + listOf("d MMM yyyy, HH:mm:ss z", "d MMMM yyyy, HH:mm:ss z").map { pattern -> + DateTimeFormatter.ofPattern(pattern, locale) + } + } + fun parseEpochMillis(value: String): Long? { - val trimmed = value.trim() + val trimmed = value.replace("\u00a0", " ").trim() if (trimmed.isBlank()) return null parseOffset(trimmed)?.let { return it } parseInstant(trimmed)?.let { return it } - return parseDate(trimmed) + parseDate(trimmed)?.let { return it } + return parseActivityDate(trimmed) } private fun parseOffset(value: String): Long? = runCatching { @@ -25,4 +38,13 @@ object YoutubeTakeoutDateParser { private fun parseDate(value: String): Long? = runCatching { LocalDate.parse(value).atStartOfDay().toInstant(ZoneOffset.UTC).toEpochMilli() }.getOrNull() + + private fun parseActivityDate(value: String): Long? { + activityFormatters.forEach { formatter -> + runCatching { ZonedDateTime.parse(value, formatter).toInstant().toEpochMilli() } + .getOrNull() + ?.let { return it } + } + return null + } } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt index 389d8575..472a3b34 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt @@ -1,18 +1,9 @@ package dev.typetype.server.services import dev.typetype.server.models.HistoryItem -import java.time.ZonedDateTime -import java.time.format.DateTimeFormatter -import java.util.Locale object YoutubeTakeoutHistoryParser { - private val dateFormatters = listOf( - DateTimeFormatter.ofPattern("d MMM yyyy, HH:mm:ss z", Locale.ENGLISH), - DateTimeFormatter.ofPattern("d MMM yyyy, HH:mm:ss z", Locale.FRENCH), - DateTimeFormatter.ofPattern("d MMMM yyyy, HH:mm:ss z", Locale.FRENCH), - DateTimeFormatter.ofPattern("d MMMM yyyy, HH:mm:ss z", Locale.ENGLISH), - ) - private val rowRegex = Regex("""(?:${YoutubeTakeoutActivityClassifier.watchedPattern})\s*([\s\S]*?)
(?:([\s\S]*?)
)?([\s\S]*?)
""", RegexOption.IGNORE_CASE) + private val rowRegex = Regex("""(?:${YoutubeTakeoutActivityClassifier.watchedPattern})\s*([\s\S]*?)
\s*(?:([\s\S]*?)
\s*)?([\s\S]*?)
""", RegexOption.IGNORE_CASE) private val urlRegex = Regex("""https?://(?:www\.)?(?:youtube\.com/(?:watch\?v=|shorts/)|youtu\.be/)[A-Za-z0-9_-]{6,}""") private val tagRegex = Regex("<[^>]+>") private val spacesRegex = Regex("\\s+") @@ -40,13 +31,7 @@ object YoutubeTakeoutHistoryParser { } private fun parseDate(value: String): Long { - val raw = value.replace("\u00a0", " ") - for (formatter in dateFormatters) { - runCatching { ZonedDateTime.parse(raw, formatter).toInstant().toEpochMilli() } - .getOrNull() - ?.let { return it } - } - return 0L + return YoutubeTakeoutDateParser.parseEpochMillis(value) ?: 0L } private fun extractUrl(value: String): String? { diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutParserService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutParserService.kt index 47e5a5c4..562e56c9 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutParserService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutParserService.kt @@ -46,7 +46,7 @@ class YoutubeTakeoutParserService { val favorites = playlistItems.filterKeys { isLikedPlaylistKey(it) }.values.flatten().map { it.toFavorite() } val activitySignals = YoutubeTakeoutActivitySignalService.parse(zipPath) val mergedSubscriptions = dedupSubscriptions(subscriptions + activitySignals.first) - val mergedFavorites = dedupFavorites(favorites + activitySignals.second.map { FavoriteItem(videoUrl = it).withYoutubeFallbackTitle() }) + val mergedFavorites = dedupFavorites(favorites + activitySignals.second.map { it.withYoutubeFallbackTitle() }) if (mergedSubscriptions.isEmpty()) warnings += "No subscription rows detected" return YoutubeTakeoutParsedData( subscriptions = mergedSubscriptions, diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutRowParser.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutRowParser.kt index 089d0fb3..0fa8fd81 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutRowParser.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutRowParser.kt @@ -49,7 +49,9 @@ object YoutubeTakeoutRowParser { val duration = values.pickExact("duration")?.toLongOrNull() ?: 0L val position = values.pickExact("position")?.toIntOrNull() ?: 0 val addedAt = values.pickHeader(YoutubeTakeoutSchemaHints::isPlaylistItemAddedAtHeader) - ?.let { YoutubeTakeoutDateParser.parseEpochMillis(it) } + ?.let(YoutubeTakeoutDateParser::parseEpochMillis) + ?: values.pickValue { YoutubeTakeoutDateParser.parseEpochMillis(it) != null } + ?.let(YoutubeTakeoutDateParser::parseEpochMillis) ?: 0L return playlistKey to PlaylistVideoItem( url = videoUrl, diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaHints.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaHints.kt index ea4eee78..e2a784c7 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaHints.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaHints.kt @@ -81,10 +81,10 @@ object YoutubeTakeoutSchemaHints { fun isPlaylistItemAddedAtHeader(value: String): Boolean { val normalized = normalize(value) val parts = words(normalized) - return ("added" in parts && "at" in parts) || + return ("added" in parts && parts.any { it == "at" || it == "date" || it == "time" || it == "timestamp" }) || ("creation" in parts && "video" in parts) || ("created" in parts && "at" in parts) || - ("timestamp" in parts && "playlist" in normalized) + ("timestamp" in parts && ("playlist" in normalized || "video" in normalized)) } fun isUrlHeader(value: String): Boolean = "url" in words(normalize(value)) diff --git a/src/test/kotlin/dev/typetype/server/AccessControlledExtractionRoutesTest.kt b/src/test/kotlin/dev/typetype/server/AccessControlledExtractionRoutesTest.kt index bad11482..d40a36c5 100644 --- a/src/test/kotlin/dev/typetype/server/AccessControlledExtractionRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/AccessControlledExtractionRoutesTest.kt @@ -96,11 +96,11 @@ class AccessControlledExtractionRoutesTest { } @Test - fun `streams blocks direct video from non allowed channel`() = testApplication { + fun `youtube sabr streams block video from non allowed channel`() = testApplication { enableAllowList() coEvery { streams.getStreamInfo(any()) } returns ExtractionResult.Success(testStreamResponse().copy(uploaderName = "Blocked", uploaderUrl = "https://youtube.com/@blocked")) application { install(ContentNegotiation) { json() }; routing { streamRoutes(streams, auth, accessControlService = access) } } - val response = client.get("/streams?url=https://youtube.com/watch?v=x") { headers.append(HttpHeaders.Authorization, "Bearer test-jwt") } + val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=x") { headers.append(HttpHeaders.Authorization, "Bearer test-jwt") } assertEquals(HttpStatusCode.Forbidden, response.status) } diff --git a/src/test/kotlin/dev/typetype/server/AudioOnlyProgressiveContractRoutesTest.kt b/src/test/kotlin/dev/typetype/server/AudioOnlyProgressiveContractRoutesTest.kt index 8c0c55f8..66aee30c 100644 --- a/src/test/kotlin/dev/typetype/server/AudioOnlyProgressiveContractRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/AudioOnlyProgressiveContractRoutesTest.kt @@ -5,7 +5,6 @@ import dev.typetype.server.models.ProxyResponse import dev.typetype.server.routes.audioOnlyContractRoutes import dev.typetype.server.routes.audioOnlySourceRoutes import dev.typetype.server.services.AudioOnlyMediaTokenService -import dev.typetype.server.services.AudioOnlyStreamSelection import dev.typetype.server.services.PublicHlsManifestTokenService import dev.typetype.server.services.ProxyService import dev.typetype.server.services.StreamService @@ -96,7 +95,7 @@ class AudioOnlyProgressiveContractRoutesTest { coEvery { streamService.getStreamInfo(any()) } returns ExtractionResult.Success( testStreamResponse(audioStreams = listOf(sabrAudio)) ) - installContractApp { _, _, _ -> "SABR audio-only body is incomplete" } + installContractApp() val response = client.get("/streams/audio-only?url=https://youtube.com/watch?v=test") val body = response.bodyAsText() @@ -131,16 +130,13 @@ class AudioOnlyProgressiveContractRoutesTest { assertTrue(response.bodyAsText().contains("Audio-only source did not return progressive audio")) } - private fun io.ktor.server.testing.ApplicationTestBuilder.installContractApp( - sabrAudioOnlyUnavailableReason: (suspend (String, String?, AudioOnlyStreamSelection) -> String?)? = null, - ): Unit = application { + private fun io.ktor.server.testing.ApplicationTestBuilder.installContractApp(): Unit = application { install(ContentNegotiation) { json() } routing { audioOnlyContractRoutes( streamService, tokenService, publicHlsManifestTokenService = hlsTokenService, - sabrAudioOnlyUnavailableReason = sabrAudioOnlyUnavailableReason, ) } } diff --git a/src/test/kotlin/dev/typetype/server/AudioOnlySabrMaterializerTest.kt b/src/test/kotlin/dev/typetype/server/AudioOnlySabrMaterializerTest.kt deleted file mode 100644 index 0849a882..00000000 --- a/src/test/kotlin/dev/typetype/server/AudioOnlySabrMaterializerTest.kt +++ /dev/null @@ -1,117 +0,0 @@ -package dev.typetype.server - -import dev.typetype.server.routes.materializeSabrAudioOnlyBody -import dev.typetype.server.services.SabrSessionHolder -import dev.typetype.server.services.SabrSessionKey -import dev.typetype.server.services.SabrSessionStore -import io.mockk.coEvery -import io.mockk.every -import io.mockk.mockk -import kotlinx.coroutines.test.runTest -import org.junit.jupiter.api.Assertions.assertArrayEquals -import org.junit.jupiter.api.Assertions.assertNull -import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState -import java.time.Instant - -class AudioOnlySabrMaterializerTest { - @Test - fun `materializer writes init and every audio sequence`() = runTest { - val audio = sabrFormat(140, isAudio = true) - val video = sabrFormat(137, isAudio = false) - val holder = holder(audio, video, endSegments = listOf(0L, 0L, 0L, 3L)) - val store = mockk() - coEvery { store.cachedSegment(holder, any()) } returns null - coEvery { store.fetchMediaAt(holder, 1_000L) } returns listOf(segment(140, 1, byteArrayOf(1))) - coEvery { store.fetchMediaAt(holder, 11_000L) } returns listOf(segment(140, 2, byteArrayOf(2))) - coEvery { store.fetchMediaAt(holder, 21_000L) } returns listOf(segment(140, 3, byteArrayOf(3))) - coEvery { store.fetchSegment(holder, any()) } returns null - - val body = materializeSabrAudioOnlyBody(store, holder, byteArrayOf(9, 8)) - - assertArrayEquals(byteArrayOf(9, 8, 1, 2, 3), body) - } - - @Test - fun `materializer falls back when time fetch returns only video`() = runTest { - val audio = sabrFormat(140, isAudio = true) - val video = sabrFormat(160, isAudio = false) - val holder = holder(audio, video, endSegments = listOf(0L, 0L, 2L)) - val store = mockk() - coEvery { store.cachedSegment(holder, any()) } returns null - coEvery { store.fetchMediaAt(holder, 1_000L) } returns listOf(segment(140, 1, byteArrayOf(1))) - coEvery { store.fetchMediaAt(holder, 11_000L) } returns listOf(segment(160, 2, byteArrayOf(7))) - coEvery { store.fetchSegment(holder, match { it.sequenceNumber == 1 }) } returns null - coEvery { store.fetchSegment(holder, match { it.sequenceNumber == 2 }) } returns segment(140, 2, byteArrayOf(2)) - - val body = materializeSabrAudioOnlyBody(store, holder, byteArrayOf(9, 8)) - - assertArrayEquals(byteArrayOf(9, 8, 1, 2), body) - } - - @Test - fun `materializer fails instead of returning partial body`() = runTest { - val audio = sabrFormat(140, isAudio = true) - val video = sabrFormat(137, isAudio = false) - val holder = holder(audio, video, endSegments = listOf(0L, 0L, 0L)) - val store = mockk() - coEvery { store.cachedSegment(holder, any()) } returns null - coEvery { store.fetchMediaAt(holder, any()) } returns null - coEvery { store.fetchSegment(holder, match { it.sequenceNumber == 1 }) } returns segment(140, 1, byteArrayOf(1)) - coEvery { store.fetchSegment(holder, match { it.sequenceNumber == 2 }) } returns null - coEvery { store.fetchInfo(any(), any(), any()) } returns null - - val body = materializeSabrAudioOnlyBody(store, holder, byteArrayOf(9, 8)) - - assertNull(body) - } - - private fun holder(audio: YoutubeSabrFormat, video: YoutubeSabrFormat, endSegments: List): SabrSessionHolder { - val session = mockk() - val streamState = mockk() - every { session.streamState } returns streamState - every { session.isBeyondEnd(any()) } returns false - every { streamState.setActiveTrackTypes(any(), any()) } returns Unit - every { streamState.getEndSegment(audio) } returnsMany endSegments - every { streamState.getSegmentStartMs(audio, any()) } answers { - (secondArg() - 1).coerceAtLeast(0) * 10_000L - } - return SabrSessionHolder( - session = session, - info = mockk(), - audioFormat = audio, - videoFormat = video, - sessionToken = "session", - key = SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), - lastRequestAt = Instant.EPOCH, - ) - } - - private fun sabrFormat(itag: Int, isAudio: Boolean): YoutubeSabrFormat { - val format = mockk() - every { format.itag } returns itag - every { format.isAudio } returns isAudio - every { format.isVideo } returns !isAudio - every { format.audioTrackId } returns null - return format - } - - private fun segment(itag: Int, sequence: Int, data: ByteArray): SabrMediaSegment { - val header = mockk() - every { header.isInitSegment } returns false - every { header.itag } returns itag - every { header.sequenceNumber } returns sequence - val segment = mockk() - every { segment.header } returns header - every { segment.data } returns data - return segment - } - - private val SabrSegmentRequest.sequenceNumber: Int get() = getSequenceNumber() -} diff --git a/src/test/kotlin/dev/typetype/server/BlockedRoutesTest.kt b/src/test/kotlin/dev/typetype/server/BlockedRoutesTest.kt index 447ac22b..771f1d64 100644 --- a/src/test/kotlin/dev/typetype/server/BlockedRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/BlockedRoutesTest.kt @@ -107,4 +107,31 @@ class BlockedRoutesTest { fun `DELETE blocked-videos returns 404 when not found`() = withApp { assertEquals(HttpStatusCode.NotFound, client.delete("/blocked/videos/https%3A%2F%2Fyt.com") { headers.append(HttpHeaders.Authorization, "Bearer test-jwt") }.status) } + + @Test + fun `blocked keywords are normalized persisted and deleted`() = withApp { + val created = client.post("/blocked/keywords") { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + headers.append(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + setBody("""{"keyword":" ClickBait "}""") + } + assertEquals(HttpStatusCode.Created, created.status) + assertTrue(created.bodyAsText().contains("\"keyword\":\"clickbait\"")) + assertTrue(client.get("/blocked/keywords") { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + }.bodyAsText().contains("\"keyword\":\"clickbait\"")) + assertEquals(HttpStatusCode.NoContent, client.delete("/blocked/keywords/clickbait") { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + }.status) + } + + @Test + fun `POST blocked keyword rejects blank values`() = withApp { + val response = client.post("/blocked/keywords") { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + headers.append(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + setBody("""{"keyword":" "}""") + } + assertEquals(HttpStatusCode.BadRequest, response.status) + } } diff --git a/src/test/kotlin/dev/typetype/server/ExtractionTest.kt b/src/test/kotlin/dev/typetype/server/ExtractionTest.kt index 2b1bbe9b..52483ea5 100644 --- a/src/test/kotlin/dev/typetype/server/ExtractionTest.kt +++ b/src/test/kotlin/dev/typetype/server/ExtractionTest.kt @@ -27,8 +27,6 @@ private object NoOpCache : CacheService { class ExtractionTest { private val service = PipePipeStreamService(NoOpCache, YouTubeSubtitleService(OkHttpClient(), "http://localhost:8081"), BilibiliRelatedService()) - private val classicService = YoutubePlayerClientStreamService(service, YoutubePlayerClient.ANDROID_VR) - private val tvClassicService = YoutubePlayerClientStreamService(service, YoutubePlayerClient.TV_SIMPLY) private val safariClassicService = YoutubePlayerClientStreamService(service, YoutubePlayerClient.WEB_SAFARI) @BeforeAll @@ -36,37 +34,6 @@ class ExtractionTest { NewPipe.init(OkHttpDownloader.instance()) } - @Test - fun `YouTube rickroll has non-empty videoOnlyStreams`() = kotlinx.coroutines.runBlocking { - val result = classicService.getStreamInfo("https://www.youtube.com/watch?v=dQw4w9WgXcQ") - assertTrue(result is ExtractionResult.Success) - val data = (result as ExtractionResult.Success).data - assertTrue(data.videoOnlyStreams.isNotEmpty()) - } - - @Test - fun `YouTube rickroll streams have positive indexStart`() = kotlinx.coroutines.runBlocking { - val result = classicService.getStreamInfo("https://www.youtube.com/watch?v=dQw4w9WgXcQ") - val data = (result as ExtractionResult.Success).data - assertTrue(data.videoOnlyStreams.any { it.indexStart > 0 }) - } - - @Test - fun `YouTube rickroll has non-empty audioStreams`() = kotlinx.coroutines.runBlocking { - val result = classicService.getStreamInfo("https://www.youtube.com/watch?v=dQw4w9WgXcQ") - val data = (result as ExtractionResult.Success).data - assertTrue(data.audioStreams.isNotEmpty()) - } - - @Test - fun `YouTube classic streams use Android VR URLs without SABR`() = kotlinx.coroutines.runBlocking { - val result = classicService.getStreamInfo("https://www.youtube.com/watch?v=dQw4w9WgXcQ") - val data = (result as ExtractionResult.Success).data - val streams = data.videoStreams + data.videoOnlyStreams - assertTrue(streams.none { it.deliveryMethod == "sabr" }) - assertTrue(streams.any { "c=ANDROID_VR" in it.url }) - } - @Test fun `YouTube classic Safari path has playable streams without SABR`() = kotlinx.coroutines.runBlocking { val result = safariClassicService.getStreamInfo("https://www.youtube.com/watch?v=dQw4w9WgXcQ") @@ -77,16 +44,6 @@ class ExtractionTest { assertTrue(streams.none { it.deliveryMethod == "sabr" }) } - @Test - fun `YouTube classic TV path has playable streams without SABR`() = kotlinx.coroutines.runBlocking { - val result = tvClassicService.getStreamInfo("https://www.youtube.com/watch?v=dQw4w9WgXcQ") - assertTrue(result is ExtractionResult.Success) - val data = (result as ExtractionResult.Success).data - val streams = data.videoStreams + data.videoOnlyStreams - assertTrue(streams.isNotEmpty()) - assertTrue(streams.none { it.deliveryMethod == "sabr" }) - } - @Test fun `BiliBili video has non-empty videoOnlyStreams`() = kotlinx.coroutines.runBlocking { val result = service.getStreamInfo("https://www.bilibili.com/video/BV1xx411c7mD") diff --git a/src/test/kotlin/dev/typetype/server/HistoryClearIsolationRoutesTest.kt b/src/test/kotlin/dev/typetype/server/HistoryClearIsolationRoutesTest.kt new file mode 100644 index 00000000..3188fef0 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/HistoryClearIsolationRoutesTest.kt @@ -0,0 +1,78 @@ +package dev.typetype.server + +import dev.typetype.server.models.HistoryItem +import dev.typetype.server.models.PlaylistItem +import dev.typetype.server.models.SettingsItem +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.routes.historyRoutes +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.HistoryService +import dev.typetype.server.services.PlaylistService +import dev.typetype.server.services.ProgressService +import dev.typetype.server.services.SettingsService +import dev.typetype.server.services.SubscriptionsService +import io.ktor.client.request.delete +import io.ktor.client.request.headers +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.ApplicationTestBuilder +import io.ktor.server.testing.testApplication +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +private const val CLEAR_HISTORY_VIDEO_URL = "https://yt.test/watch?v=clear-history" +private const val CLEAR_HISTORY_CHANNEL_URL = "https://www.youtube.com/channel/UCclear" + +class HistoryClearIsolationRoutesTest { + private val history = HistoryService() + private val progress = ProgressService() + private val settings = SettingsService() + private val subscriptions = SubscriptionsService() + private val playlists = PlaylistService() + private val auth = AuthService.fixed(TEST_USER_ID) + + companion object { @BeforeAll @JvmStatic fun initDb() { TestDatabase.setup() } } + + @BeforeEach + fun clean() { TestDatabase.truncateAll() } + + @Test + fun `DELETE history preserves unrelated user data`() = withApp { + history.add(TEST_USER_ID, historyItem()) + progress.upsert(TEST_USER_ID, CLEAR_HISTORY_VIDEO_URL, 42_000L) + settings.upsert(TEST_USER_ID, SettingsItem(defaultQuality = "720p")) + subscriptions.add(TEST_USER_ID, SubscriptionItem(channelUrl = CLEAR_HISTORY_CHANNEL_URL, name = "Channel", avatarUrl = "")) + playlists.create(TEST_USER_ID, PlaylistItem(name = "Saved")) + + val response = client.delete("/history") { headers.append(HttpHeaders.Authorization, "Bearer test-jwt") } + + assertEquals(HttpStatusCode.NoContent, response.status) + assertEquals(emptyList(), history.search(TEST_USER_ID, null, null, null, 20, 0).first) + assertNull(progress.get(TEST_USER_ID, CLEAR_HISTORY_VIDEO_URL)) + assertEquals("720p", settings.get(TEST_USER_ID).defaultQuality) + assertEquals(1, subscriptions.getAll(TEST_USER_ID).size) + assertEquals(1, playlists.getAll(TEST_USER_ID).size) + } + + private fun withApp(block: suspend ApplicationTestBuilder.() -> Unit) = testApplication { + application { install(ContentNegotiation) { json() }; routing { historyRoutes(history, auth) } } + block() + } + + private fun historyItem(): HistoryItem = HistoryItem( + url = CLEAR_HISTORY_VIDEO_URL, + title = "Video", + thumbnail = "", + channelName = "Channel", + channelUrl = CLEAR_HISTORY_CHANNEL_URL, + duration = 120L, + progress = 0L, + ) +} diff --git a/src/test/kotlin/dev/typetype/server/HistoryProgressMapperTest.kt b/src/test/kotlin/dev/typetype/server/HistoryProgressMapperTest.kt new file mode 100644 index 00000000..e64b2d80 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/HistoryProgressMapperTest.kt @@ -0,0 +1,18 @@ +package dev.typetype.server + +import dev.typetype.server.services.HistoryProgressMapper +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class HistoryProgressMapperTest { + @Test + fun `progress lookups stay below the postgres parameter limit`() { + val urls = List(71_037) { index -> "https://youtube.com/watch?v=$index" } + + val batches = HistoryProgressMapper.progressLookupBatches(urls) + + assertEquals(urls, batches.flatten()) + assertTrue(batches.all { it.size <= 1_000 }) + } +} diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationItemFixtures.kt b/src/test/kotlin/dev/typetype/server/HomeRecommendationItemFixtures.kt index aa685849..73902f88 100644 --- a/src/test/kotlin/dev/typetype/server/HomeRecommendationItemFixtures.kt +++ b/src/test/kotlin/dev/typetype/server/HomeRecommendationItemFixtures.kt @@ -39,6 +39,7 @@ internal object HomeRecommendationItemFixtures { seenUrls: Set = emptySet(), blockedVideos: Set = emptySet(), blockedChannels: Set = emptySet(), + blockedKeywords: Set = emptySet(), subscriptionChannels: Set = emptySet(), keywordAffinity: Set = emptySet(), subscriptionEngagement: Double = 0.0, @@ -48,6 +49,7 @@ internal object HomeRecommendationItemFixtures { seenUrls = seenUrls, blockedVideos = blockedVideos, blockedChannels = blockedChannels, + blockedKeywords = blockedKeywords, feedbackBlockedVideos = emptySet(), feedbackBlockedChannels = emptySet(), subscriptionChannels = subscriptionChannels, diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationPoolBuilderTest.kt b/src/test/kotlin/dev/typetype/server/HomeRecommendationPoolBuilderTest.kt index 717b5f99..d5da8d72 100644 --- a/src/test/kotlin/dev/typetype/server/HomeRecommendationPoolBuilderTest.kt +++ b/src/test/kotlin/dev/typetype/server/HomeRecommendationPoolBuilderTest.kt @@ -51,6 +51,19 @@ class HomeRecommendationPoolBuilderTest { assertTrue(pool.discovery.first().url.endsWith("/normal1")) } + @Test + fun `pool builder excludes titles containing a blocked keyword`() { + val profile = profile(blockedKeywords = setOf("ＳＰＯＮＳＯＲＥＤ")) + val discovery = listOf( + tagged(video("blocked", "a", title = "A sponsored review"), HomeRecommendationSourceTag.DISCOVERY_THEME), + tagged(video("allowed", "b", title = "A regular review"), HomeRecommendationSourceTag.DISCOVERY_THEME), + ) + + val pool = HomeRecommendationPoolBuilder().build(profile, emptyList(), discovery, context) + + assertEquals(listOf("https://yt.com/v/allowed"), pool.discovery.map { it.url }) + } + @Test fun `pool builder keeps neutral home source weights`() { val profile = profile( diff --git a/src/test/kotlin/dev/typetype/server/InstanceRoutesTest.kt b/src/test/kotlin/dev/typetype/server/InstanceRoutesTest.kt index f07a2fd5..3881e19a 100644 --- a/src/test/kotlin/dev/typetype/server/InstanceRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/InstanceRoutesTest.kt @@ -74,20 +74,7 @@ class InstanceRoutesTest { assertEquals(false, root["youtubeRemoteLoginReady"]?.jsonPrimitive?.boolean) assertEquals("disabled", root["youtubeRemoteLoginUnavailableReason"]?.jsonPrimitive?.contentOrNull) assertEquals(listOf(0, 3, 4, 5, 6), root["supportedServices"]?.jsonArray?.map { it.jsonPrimitive.int }) - val androidPlayback = root["androidPlayback"]?.jsonObject - assertEquals(true, androidPlayback?.get("supported")?.jsonPrimitive?.boolean) - assertEquals(4, androidPlayback?.get("contractVersion")?.jsonPrimitive?.int) - assertEquals(true, androidPlayback?.get("youtube")?.jsonObject?.get("vod")?.jsonPrimitive?.boolean) - assertEquals(false, androidPlayback?.get("youtube")?.jsonObject?.get("live")?.jsonPrimitive?.boolean) - assertEquals(true, androidPlayback?.get("youtube")?.jsonObject?.get("subtitles")?.jsonPrimitive?.boolean) - assertEquals( - true, - androidPlayback?.get("youtube")?.jsonObject?.get("deferredSubtitleContent")?.jsonPrimitive?.boolean, - ) - assertEquals( - true, - androidPlayback?.get("youtube")?.jsonObject?.get("bootstrapSubtitleDescriptors")?.jsonPrimitive?.boolean, - ) + assertEquals(null, root["androidPlayback"]) } @Test diff --git a/src/test/kotlin/dev/typetype/server/SabrBootstrapStreamRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SabrBootstrapStreamRoutesTest.kt index a496ba51..017d327d 100644 --- a/src/test/kotlin/dev/typetype/server/SabrBootstrapStreamRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SabrBootstrapStreamRoutesTest.kt @@ -26,7 +26,6 @@ class SabrBootstrapStreamRoutesTest { fun `bootstrap endpoint uses isolated sabr service`() = testApplication { val full = mockk() val bootstrap = mockk() - var youtubeSessionCalls = 0 coEvery { bootstrap.getStreamInfo(VIDEO_URL) } returns ExtractionResult.Success(sabrResponse()) application { install(ContentNegotiation) { json() } @@ -34,10 +33,6 @@ class SabrBootstrapStreamRoutesTest { streamRoutes( streamService = full, authService = AuthService.fixed(TEST_USER_ID), - youtubeSessionStreamInfo = { _, _ -> - youtubeSessionCalls += 1 - ExtractionResult.Failure("unexpected session extraction") - }, sabrBootstrapStreamService = bootstrap, ) } @@ -49,7 +44,6 @@ class SabrBootstrapStreamRoutesTest { assertEquals(HttpStatusCode.OK, response.status) assertTrue(response.bodyAsText().contains("\"deliveryMethod\":\"sabr\"")) - assertEquals(0, youtubeSessionCalls) coVerify(exactly = 1) { bootstrap.getStreamInfo(VIDEO_URL) } coVerify(exactly = 0) { full.getStreamInfo(any()) } } diff --git a/src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt index 90729efe..883e3808 100644 --- a/src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt @@ -60,6 +60,7 @@ class SettingsRoutesTest { assertTrue(body.contains("\"volume\":1.0")) assertTrue(body.contains("\"muted\":false")) assertTrue(body.contains("\"defaultLandingPage\":\"home\"")) + assertTrue(body.contains("\"defaultPlaybackSpeed\":1.0")) } @Test @@ -116,4 +117,17 @@ class SettingsRoutesTest { setBody("""not json""") }.status) } + + @Test + fun `PUT settings persists and bounds default playback speed`() = withApp { + client.put("/settings") { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + headers.append(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + setBody("""{"defaultPlaybackSpeed":8.0}""") + } + val body = client.get("/settings") { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + }.bodyAsText() + assertTrue(body.contains("\"defaultPlaybackSpeed\":4.0")) + } } diff --git a/src/test/kotlin/dev/typetype/server/StreamFieldsTest.kt b/src/test/kotlin/dev/typetype/server/StreamFieldsTest.kt index 6c64b6ba..ea9b7445 100644 --- a/src/test/kotlin/dev/typetype/server/StreamFieldsTest.kt +++ b/src/test/kotlin/dev/typetype/server/StreamFieldsTest.kt @@ -42,7 +42,7 @@ class StreamFieldsTest { hlsUrl = "https://example.com/live.m3u8", ) ) - val body = client.get("/streams/legacy?url=https://youtube.com/watch?v=test").bodyAsText() + val body = client.get(STREAM_URL).bodyAsText() assertTrue(body.contains("\"streamType\":\"live_stream\"")) assertTrue(body.contains("\"isLive\":true")) assertTrue(body.contains("\"isPostLive\":false")) @@ -60,7 +60,7 @@ class StreamFieldsTest { startPosition = 42L, ) ) - val body = client.get("/streams/legacy?url=https://youtube.com/shorts/test").bodyAsText() + val body = client.get(STREAM_URL).bodyAsText() assertTrue(body.contains("\"isShortFormContent\":true")) assertTrue(body.contains("\"requiresMembership\":true")) assertTrue(body.contains("\"startPosition\":42")) @@ -77,7 +77,7 @@ class StreamFieldsTest { ) coEvery { streamService.getStreamInfo(any()) } returns ExtractionResult.Success(testStreamResponse().copy(streamSegments = listOf(segment))) - val body = client.get("/streams/legacy?url=https://youtube.com/watch?v=test").bodyAsText() + val body = client.get(STREAM_URL).bodyAsText() assertTrue(body.contains("\"streamSegments\"")) assertTrue(body.contains("\"title\":\"Intro\"")) assertTrue(body.contains("\"startTimeSeconds\":0")) @@ -93,7 +93,7 @@ class StreamFieldsTest { preferredDefaultAudioTrackId = "en.0", ) ) - val body = client.get("/streams/legacy?url=https://youtube.com/watch?v=test").bodyAsText() + val body = client.get(STREAM_URL).bodyAsText() assertTrue(body.contains("\"audioLocale\":\"en\"")) assertTrue(body.contains("\"isOriginal\":true")) assertTrue(body.contains("\"originalAudioTrackId\":\"en.0\"")) @@ -110,10 +110,14 @@ class StreamFieldsTest { ) coEvery { streamService.getStreamInfo(any()) } returns ExtractionResult.Success(testStreamResponse().copy(sponsorBlockSegments = listOf(segment))) - val body = client.get("/streams/legacy?url=https://youtube.com/watch?v=test").bodyAsText() + val body = client.get(STREAM_URL).bodyAsText() assertTrue(body.contains("\"sponsorBlockSegments\"")) assertTrue(body.contains("\"category\":\"sponsor\"")) assertTrue(body.contains("\"action\":\"skip\"")) assertTrue(body.contains("55649.0")) } + + private companion object { + const val STREAM_URL = "/streams/niconico?url=https://www.nicovideo.jp/watch/sm9" + } } diff --git a/src/test/kotlin/dev/typetype/server/StreamRoutesDeliveryModeTest.kt b/src/test/kotlin/dev/typetype/server/StreamRoutesDeliveryModeTest.kt index b8d4388d..a719ef4c 100644 --- a/src/test/kotlin/dev/typetype/server/StreamRoutesDeliveryModeTest.kt +++ b/src/test/kotlin/dev/typetype/server/StreamRoutesDeliveryModeTest.kt @@ -3,12 +3,9 @@ package dev.typetype.server import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.StreamResponse import dev.typetype.server.routes.streamRoutes -import dev.typetype.server.services.AuthService import dev.typetype.server.services.StreamService import io.ktor.client.request.get -import io.ktor.client.request.headers import io.ktor.client.statement.bodyAsText -import io.ktor.http.HttpHeaders import io.ktor.http.HttpStatusCode import io.ktor.serialization.kotlinx.json.json import io.ktor.server.application.Application @@ -26,40 +23,21 @@ import org.junit.jupiter.api.Test class StreamRoutesDeliveryModeTest { private val sabrService: StreamService = mockk() - private val legacyService: StreamService = mockk() - private val genericLegacyService: StreamService = mockk() private val nicoNicoService: StreamService = mockk() private val bilibiliService: StreamService = mockk() @Test - fun `generic streams endpoint preserves classic contract`() = testApplication { - coEvery { genericLegacyService.getStreamInfo(any()) } returns ExtractionResult.Success(mixedResponse()) + fun `removed classic endpoints are not registered`() = testApplication { application { installRoutes() } - val response = client.get("/streams?url=$VIDEO_URL") - val body = response.bodyAsText() - - assertEquals(HttpStatusCode.OK, response.status) - assertTrue(body.contains("\"itag\":18")) - assertFalse(body.contains("\"deliveryMethod\":\"sabr\"")) - coVerify(exactly = 1) { genericLegacyService.getStreamInfo(VIDEO_URL) } - coVerify(exactly = 0) { sabrService.getStreamInfo(any()) } + assertEquals(HttpStatusCode.NotFound, client.get("/streams?url=$VIDEO_URL").status) + assertEquals(HttpStatusCode.NotFound, client.get("/streams/legacy?url=$VIDEO_URL").status) + assertEquals(HttpStatusCode.NotFound, client.get("/streams/youtube/legacy?url=$VIDEO_URL").status) } @Test - fun `generic legacy endpoint remains provider neutral`() = testApplication { - coEvery { genericLegacyService.getStreamInfo(any()) } returns ExtractionResult.Success(testStreamResponse()) - application { installRoutes() } - - val response = client.get("/streams/legacy?url=$NICONICO_URL") - - assertEquals(HttpStatusCode.OK, response.status) - coVerify(exactly = 1) { genericLegacyService.getStreamInfo(NICONICO_URL) } - } - - @Test - fun `legacy endpoint uses isolated service and strips sabr streams`() = testApplication { - coEvery { legacyService.getStreamInfo(any()) } returns ExtractionResult.Success(mixedResponse()) + fun `sabr endpoint returns only sabr streams`() = testApplication { + coEvery { sabrService.getStreamInfo(any()) } returns ExtractionResult.Success(mixedResponse()) var sabrFilterCalled = false application { installRoutes { _, data -> @@ -68,75 +46,34 @@ class StreamRoutesDeliveryModeTest { } } - val response = client.get("/streams/youtube/legacy?url=$VIDEO_URL") + val response = client.get("/streams/youtube/sabr?url=$VIDEO_URL") val body = response.bodyAsText() assertEquals(HttpStatusCode.OK, response.status) - assertTrue(body.contains("\"itag\":18")) - assertFalse(body.contains("\"deliveryMethod\":\"sabr\"")) - assertFalse(sabrFilterCalled) - coVerify(exactly = 1) { legacyService.getStreamInfo(VIDEO_URL) } - coVerify(exactly = 0) { sabrService.getStreamInfo(any()) } - } - - @Test - fun `legacy endpoint short circuits extraction on authenticated hls`() = testApplication { - val hls = testStreamResponse(videoOnlyStreams = emptyList(), audioStreams = emptyList()) - .copy(hlsUrl = "/streams/hls-manifest?token=legacy") - application { installRoutes(session = { _, _ -> ExtractionResult.Success(hls) }) } - - val response = client.get("/streams/youtube/legacy?url=$VIDEO_URL") { - headers.append(HttpHeaders.Authorization, "Bearer test-jwt") - } - - assertEquals(HttpStatusCode.OK, response.status) - assertTrue(response.bodyAsText().contains("\"hlsUrl\":\"/streams/hls-manifest?token=legacy\"")) - coVerify(exactly = 0) { legacyService.getStreamInfo(any()) } - coVerify(exactly = 0) { sabrService.getStreamInfo(any()) } - } - - @Test - fun `sabr endpoint never uses the connected YouTube session`() = testApplication { - coEvery { sabrService.getStreamInfo(any()) } returns ExtractionResult.Success(sabrResponse()) - var sessionCalls = 0 - application { - installRoutes(session = { _, _ -> - sessionCalls += 1 - ExtractionResult.Success(sabrResponse()) - }) - } - - val response = client.get("/streams/youtube/sabr?url=$VIDEO_URL") { - headers.append(HttpHeaders.Authorization, "Bearer test-jwt") - } - - assertEquals(HttpStatusCode.OK, response.status) - assertEquals(0, sessionCalls) + assertTrue(body.contains("\"deliveryMethod\":\"sabr\"")) + assertFalse(body.contains("\"itag\":18")) + assertTrue(sabrFilterCalled) coVerify(exactly = 1) { sabrService.getStreamInfo(VIDEO_URL) } - coVerify(exactly = 0) { legacyService.getStreamInfo(any()) } } @Test - fun `sabr endpoint ignores authenticated hls and returns only sabr streams`() = testApplication { - val hls = testStreamResponse(videoOnlyStreams = emptyList(), audioStreams = emptyList()) - .copy(hlsUrl = "/streams/hls-manifest?token=classic") - coEvery { sabrService.getStreamInfo(any()) } returns ExtractionResult.Success(mixedResponse()) - application { installRoutes(session = { _, _ -> ExtractionResult.Success(hls) }) } + fun `sabr endpoint rejects direct only extraction`() = testApplication { + coEvery { sabrService.getStreamInfo(any()) } returns ExtractionResult.Success( + testStreamResponse( + videoOnlyStreams = listOf(testVideoStream()), + audioStreams = listOf(testAudioStream()), + ), + ) + application { installRoutes() } - val response = client.get("/streams/youtube/sabr?url=$VIDEO_URL") { - headers.append(HttpHeaders.Authorization, "Bearer test-jwt") - } - val body = response.bodyAsText() + val response = client.get("/streams/youtube/sabr?url=$VIDEO_URL") - assertEquals(HttpStatusCode.OK, response.status) - assertTrue(body.contains("\"deliveryMethod\":\"sabr\"")) - assertFalse(body.contains("\"itag\":18")) - assertFalse(body.contains("classic")) - coVerify(exactly = 1) { sabrService.getStreamInfo(VIDEO_URL) } + assertEquals(HttpStatusCode.UnprocessableEntity, response.status) + assertTrue(response.bodyAsText().contains("\"code\":\"no_playable_streams\"")) } @Test - fun `sabr endpoint removes hls for live streams`() = testApplication { + fun `sabr endpoint removes direct live manifest`() = testApplication { val live = sabrResponse().copy( hlsUrl = "/streams/hls-manifest?url=live", isLive = true, @@ -153,30 +90,6 @@ class StreamRoutesDeliveryModeTest { assertFalse(response.bodyAsText().contains("hls-manifest")) } - @Test - fun `legacy endpoint rejects sabr only extraction`() = testApplication { - coEvery { legacyService.getStreamInfo(any()) } returns ExtractionResult.Success(sabrResponse()) - application { installRoutes() } - - val response = client.get("/streams/youtube/legacy?url=$VIDEO_URL") - - assertEquals(HttpStatusCode.UnprocessableEntity, response.status) - assertTrue(response.bodyAsText().contains("\"code\":\"no_playable_streams\"")) - } - - @Test - fun `sabr endpoint rejects classic only extraction`() = testApplication { - coEvery { sabrService.getStreamInfo(any()) } returns ExtractionResult.Success( - testStreamResponse(videoOnlyStreams = listOf(testVideoStream()), audioStreams = listOf(testAudioStream())), - ) - application { installRoutes() } - - val response = client.get("/streams/youtube/sabr?url=$VIDEO_URL") - - assertEquals(HttpStatusCode.UnprocessableEntity, response.status) - assertTrue(response.bodyAsText().contains("\"code\":\"no_playable_streams\"")) - } - @Test fun `provider endpoints use isolated services`() = testApplication { coEvery { nicoNicoService.getStreamInfo(any()) } returns ExtractionResult.Success(testStreamResponse()) @@ -191,7 +104,6 @@ class StreamRoutesDeliveryModeTest { coVerify(exactly = 1) { nicoNicoService.getStreamInfo(NICONICO_URL) } coVerify(exactly = 1) { bilibiliService.getStreamInfo(BILIBILI_URL) } coVerify(exactly = 0) { sabrService.getStreamInfo(any()) } - coVerify(exactly = 0) { legacyService.getStreamInfo(any()) } } @Test @@ -207,17 +119,12 @@ class StreamRoutesDeliveryModeTest { } private fun Application.installRoutes( - session: (suspend (String, String) -> ExtractionResult?)? = null, sabrFilter: suspend (String, StreamResponse) -> StreamResponse = { _, data -> data }, - ): Unit { + ) { install(ContentNegotiation) { json() } routing { streamRoutes( streamService = sabrService, - authService = AuthService.fixed(TEST_USER_ID), - youtubeSessionStreamInfo = session, - legacyStreamService = legacyService, - genericLegacyStreamService = genericLegacyService, nicoNicoStreamService = nicoNicoService, bilibiliStreamService = bilibiliService, sabrStreamContractFilter = sabrFilter, @@ -227,7 +134,7 @@ class StreamRoutesDeliveryModeTest { private fun mixedResponse(): StreamResponse = testStreamResponse( videoOnlyStreams = listOf(testVideoStream(itag = 18), sabrVideo()), - audioStreams = listOf(testAudioStream(itag = 140, deliveryMethod = "sabr")), + audioStreams = listOf(testAudioStream(), testAudioStream(itag = 140, deliveryMethod = "sabr")), ) private fun sabrResponse(): StreamResponse = testStreamResponse( diff --git a/src/test/kotlin/dev/typetype/server/StreamRoutesSignedHlsTest.kt b/src/test/kotlin/dev/typetype/server/StreamRoutesSignedHlsTest.kt index b3d97e3a..d2fa50ba 100644 --- a/src/test/kotlin/dev/typetype/server/StreamRoutesSignedHlsTest.kt +++ b/src/test/kotlin/dev/typetype/server/StreamRoutesSignedHlsTest.kt @@ -49,7 +49,7 @@ class StreamRoutesSignedHlsTest { coEvery { streamService.getStreamInfo(any()) } returns ExtractionResult.Success(publicHlsStream()) installApp() - val response = client.get("/streams/legacy?url=https://youtube.com/watch?v=test") { + val response = client.get("/streams/niconico?url=https://www.nicovideo.jp/watch/sm9") { headers.append(HttpHeaders.Authorization, "Bearer test-jwt") } val body = response.bodyAsText() @@ -93,7 +93,7 @@ class StreamRoutesSignedHlsTest { adminSettings.upsert(AdminSettingsItem(allowGuest = false)) installApp() - val response = client.get("/streams?url=https://youtube.com/watch?v=test") + val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=test") assertEquals(HttpStatusCode.Unauthorized, response.status) } diff --git a/src/test/kotlin/dev/typetype/server/StreamRoutesTest.kt b/src/test/kotlin/dev/typetype/server/StreamRoutesTest.kt index 411da2a0..69794325 100644 --- a/src/test/kotlin/dev/typetype/server/StreamRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/StreamRoutesTest.kt @@ -28,19 +28,19 @@ class StreamRoutesTest { install(ContentNegotiation) { json() } routing { streamRoutes(streamService) } } - val response = client.get("/streams") + val response = client.get("/streams/youtube/sabr") assertEquals(HttpStatusCode.BadRequest, response.status) } @Test fun `GET streams with valid url returns 200 on Success`() = testApplication { coEvery { streamService.getStreamInfo(any()) } returns - ExtractionResult.Success(testStreamResponse()) + ExtractionResult.Success(sabrResponse()) application { install(ContentNegotiation) { json() } routing { streamRoutes(streamService) } } - val response = client.get("/streams/legacy?url=https://youtube.com/watch?v=test") + val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=test") assertEquals(HttpStatusCode.OK, response.status) assertEquals("public, max-age=21600, stale-while-revalidate=3600", response.headers[HttpHeaders.CacheControl]) } @@ -53,7 +53,7 @@ class StreamRoutesTest { install(ContentNegotiation) { json() } routing { streamRoutes(streamService) } } - val response = client.get("/streams?url=https://youtube.com/watch?v=bad") + val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=bad") assertEquals(HttpStatusCode.UnprocessableEntity, response.status) assertTrue(response.bodyAsText().contains("\"code\":\"scheduled_premiere\"")) } @@ -66,7 +66,7 @@ class StreamRoutesTest { install(ContentNegotiation) { json() } routing { streamRoutes(streamService) } } - val response = client.get("/streams?url=https://unsupported.com/video") + val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=paid") assertEquals(HttpStatusCode.BadRequest, response.status) assertTrue(response.bodyAsText().contains("\"code\":\"paid_content\"")) } @@ -96,4 +96,9 @@ class StreamRoutesTest { assertEquals(HttpStatusCode.UnprocessableEntity, response.status) assertTrue(body.contains("\"code\":\"no_playable_streams\"")) } + + private fun sabrResponse() = testStreamResponse( + videoOnlyStreams = listOf(testVideoStream().copy(deliveryMethod = "sabr")), + audioStreams = listOf(testAudioStream(deliveryMethod = "sabr")), + ) } diff --git a/src/test/kotlin/dev/typetype/server/StreamRoutesYoutubeSessionTest.kt b/src/test/kotlin/dev/typetype/server/StreamRoutesYoutubeSessionTest.kt deleted file mode 100644 index f27d32ee..00000000 --- a/src/test/kotlin/dev/typetype/server/StreamRoutesYoutubeSessionTest.kt +++ /dev/null @@ -1,95 +0,0 @@ -package dev.typetype.server - -import dev.typetype.server.models.ExtractionResult -import dev.typetype.server.services.SignedHlsManifestCookie -import dev.typetype.server.routes.streamRoutes -import dev.typetype.server.services.AuthService -import dev.typetype.server.services.StreamService -import dev.typetype.server.services.YOUTUBE_SESSION_RECONNECT_ERROR -import io.ktor.client.request.get -import io.ktor.client.request.headers -import io.ktor.client.statement.bodyAsText -import io.ktor.http.HttpHeaders -import io.ktor.http.HttpStatusCode -import io.ktor.serialization.kotlinx.json.json -import io.ktor.server.application.install -import io.ktor.server.plugins.contentnegotiation.ContentNegotiation -import io.ktor.server.routing.routing -import io.ktor.server.testing.testApplication -import io.mockk.coEvery -import io.mockk.coVerify -import io.mockk.mockk -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.Test - -class StreamRoutesYoutubeSessionTest { - private val streamService: StreamService = mockk() - - @Test - fun `GET streams uses authenticated YouTube session when bearer is valid`() = testApplication { - coEvery { streamService.getStreamInfo(any()) } returns ExtractionResult.Failure("public path") - application { testRoutes { userId, _ -> if (userId == TEST_USER_ID) ExtractionResult.Success(testStreamResponse()) else null } } - val response = client.get("/streams/legacy?url=https://youtube.com/watch?v=test") { - headers.append(HttpHeaders.Authorization, "Bearer test-jwt") - } - assertEquals(HttpStatusCode.OK, response.status) - assertEquals("no-store", response.headers[HttpHeaders.CacheControl]) - coVerify(exactly = 0) { streamService.getStreamInfo(any()) } - } - - @Test - fun `GET streams uses authenticated HLS without public extraction`() = testApplication { - coEvery { streamService.getStreamInfo(any()) } returns ExtractionResult.Success( - testStreamResponse(videoOnlyStreams = listOf(testVideoStream()), audioStreams = listOf(testAudioStream())) - ) - application { - testRoutes { _, _ -> - ExtractionResult.Success( - testStreamResponse(videoOnlyStreams = emptyList(), audioStreams = emptyList()) - .copy(hlsUrl = "/streams/hls-manifest?token=signed-hls") - ) - } - } - val response = client.get("/streams/legacy?url=https://youtube.com/watch?v=test") { - headers.append(HttpHeaders.Authorization, "Bearer test-jwt") - } - assertEquals(HttpStatusCode.OK, response.status) - assertEquals("no-store", response.headers[HttpHeaders.CacheControl]) - val body = response.bodyAsText() - assertTrue(body.contains("\"hlsUrl\":\"/streams/hls-manifest?token=signed-hls\"")) - assertTrue(response.headers.getAll(HttpHeaders.SetCookie).orEmpty().any { - it.startsWith("${SignedHlsManifestCookie.name("https://youtube.com/watch?v=test")}=signed-hls") - }) - assertTrue(body.contains("\"videoOnlyStreams\":[")) - assertTrue(body.contains("\"audioStreams\":[")) - coVerify(exactly = 0) { streamService.getStreamInfo(any()) } - } - - @Test - fun `GET streams ignores authenticated path without valid bearer`() = testApplication { - coEvery { streamService.getStreamInfo(any()) } returns ExtractionResult.Success(testStreamResponse()) - application { testRoutes { _, _ -> ExtractionResult.Failure("unexpected session path") } } - val response = client.get("/streams/legacy?url=https://youtube.com/watch?v=test") - assertEquals(HttpStatusCode.OK, response.status) - assertEquals("public, max-age=21600, stale-while-revalidate=3600", response.headers[HttpHeaders.CacheControl]) - } - - @Test - fun `GET streams returns stable code when YouTube session needs reconnect`() = testApplication { - coEvery { streamService.getStreamInfo(any()) } returns ExtractionResult.Failure("public path") - application { testRoutes { _, _ -> ExtractionResult.BadRequest(YOUTUBE_SESSION_RECONNECT_ERROR) } } - val response = client.get("/streams/legacy?url=https://youtube.com/watch?v=test") { - headers.append(HttpHeaders.Authorization, "Bearer test-jwt") - } - assertEquals(HttpStatusCode.BadRequest, response.status) - assertTrue(response.bodyAsText().contains("\"code\":\"youtube_session_needs_reconnect\"")) - } - - private fun io.ktor.server.application.Application.testRoutes( - block: suspend (String, String) -> ExtractionResult?, - ): Unit { - install(ContentNegotiation) { json() } - routing { streamRoutes(streamService, AuthService.fixed(TEST_USER_ID), block) } - } -} diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedOrdererTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedOrdererTest.kt new file mode 100644 index 00000000..8dd8dd1c --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedOrdererTest.kt @@ -0,0 +1,71 @@ +package dev.typetype.server + +import dev.typetype.server.SubscriptionFeedTestFixtures.video +import dev.typetype.server.services.SubscriptionFeedOrderer +import dev.typetype.server.services.SubscriptionFeedSnapshot +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class SubscriptionFeedOrdererTest { + private val orderer = SubscriptionFeedOrderer() + + @Test + fun `scheduled livestream follows normal chronology`() { + val scheduled = video(2_000L, url = "scheduled") + val recent = video(3_000L, url = "recent") + + val result = orderer.order(listOf(scheduled, recent), previous = null, refreshedAt = 10_000L) + + assertEquals(listOf("recent", "scheduled"), result.videos.map { it.url }) + assertEquals(emptyMap(), result.livePromotedAt) + } + + @Test + fun `scheduled to live transition is promoted once`() { + val scheduled = video(2_000L, url = "live") + val previous = snapshot(5_000L, listOf(scheduled)) + val live = video(-1L, url = "live", live = true) + + val promoted = orderer.order(listOf(video(8_000L), live), previous, refreshedAt = 10_000L) + + assertEquals("live", promoted.videos.first().url) + assertEquals(10_000L, promoted.livePromotedAt["live"]) + } + + @Test + fun `active livestream keeps its promotion while newer uploads pass it`() { + val live = video(-1L, url = "live", live = true) + val previous = snapshot( + generatedAt = 10_000L, + videos = listOf(live), + livePromotedAt = mapOf("live" to 10_000L), + ) + + val result = orderer.order(listOf(live, video(12_000L, url = "new")), previous, refreshedAt = 20_000L) + + assertEquals(listOf("new", "live"), result.videos.map { it.url }) + assertEquals(10_000L, result.livePromotedAt["live"]) + } + + @Test + fun `old snapshot gives an existing live one migration promotion`() { + val live = video(-1L, url = "live", live = true) + val previous = snapshot(7_000L, listOf(live)) + + val result = orderer.order(listOf(live), previous, refreshedAt = 20_000L) + + assertEquals(7_000L, result.livePromotedAt["live"]) + } + + private fun snapshot( + generatedAt: Long, + videos: List, + livePromotedAt: Map = emptyMap(), + ) = SubscriptionFeedSnapshot( + generation = 1L, + generatedAt = generatedAt, + stale = false, + videos = videos, + livePromotedAt = livePromotedAt, + ) +} diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedRoutesTest.kt index e22484a2..1bcddd7a 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionFeedRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedRoutesTest.kt @@ -31,7 +31,6 @@ import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.delay import kotlinx.serialization.json.Json import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertNotNull import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeAll @@ -103,7 +102,7 @@ class SubscriptionFeedRoutesTest { } @Test - fun `ready feed sorts live first and unknown dates last`() = withApp { + fun `ready feed promotes a newly observed live and keeps unknown dates last`() = withApp { val channelUrl = "https://www.youtube.com/channel/UC1" subscriptionsService.add(TEST_USER_ID, subscription(channelUrl, "Live channel")) coEvery { channelService.getChannel(channelUrl, null) } returns channel( @@ -183,7 +182,7 @@ class SubscriptionFeedRoutesTest { } @Test - fun `partial refresh keeps the previous complete snapshot`() = withApp { + fun `partial refresh publishes videos from successful sources`() = withApp { subscriptionsService.add(TEST_USER_ID, subscription(1)) subscriptionsService.add(TEST_USER_ID, subscription(2)) var rebuilding = false @@ -197,10 +196,25 @@ class SubscriptionFeedRoutesTest { rebuilding = true feedService.invalidate(TEST_USER_ID) feedService.awaitRefresh(TEST_USER_ID) + val refreshed = Json.decodeFromString(requestFeed().bodyAsText()) + assertTrue(refreshed.generation != original.generation) + assertEquals(listOf(9000L), refreshed.videos.map { it.uploaded }) + } + + @Test + fun `refresh keeps previous snapshot when every source fails`() = withApp { + subscriptionsService.add(TEST_USER_ID, subscription(1)) + var rebuilding = false + coEvery { channelService.getChannel(any(), null) } coAnswers { + if (rebuilding) error("channel unavailable") else channel(video(1000L)) + } + val original = buildAndRead() + rebuilding = true + feedService.invalidate(TEST_USER_ID) + feedService.awaitRefresh(TEST_USER_ID) val retained = Json.decodeFromString(requestFeed().bodyAsText()) assertEquals(original.generation, retained.generation) - assertEquals(setOf(1000L, 2000L), retained.videos.map { it.uploaded }.toSet()) - assertFalse(retained.videos.any { it.uploaded == 9000L }) + assertEquals(listOf(1000L), retained.videos.map { it.uploaded }) } @Test diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedSnapshotStoreTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedSnapshotStoreTest.kt index 4093e46d..a3ebd00c 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionFeedSnapshotStoreTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedSnapshotStoreTest.kt @@ -46,6 +46,18 @@ class SubscriptionFeedSnapshotStoreTest { assertEquals(86_400L, cache.ttls[SubscriptionFeedCacheKeys.feed("user")]) } + @Test + fun `snapshot without live promotion metadata remains readable`() = runTest { + val cache = RecordingCache() + val raw = """ + {"generation":1,"generatedAt":1000,"stale":false,"videos":[]} + """.trimIndent() + cache.set(SubscriptionFeedCacheKeys.feed("user"), raw, 60) + val store = SubscriptionFeedSnapshotStore(cache) { 10_000L } + + assertEquals(emptyMap(), store.current("user")?.livePromotedAt) + } + private class RecordingCache : CacheService { val values = mutableMapOf() val ttls = mutableMapOf() diff --git a/src/test/kotlin/dev/typetype/server/TestDatabase.kt b/src/test/kotlin/dev/typetype/server/TestDatabase.kt index af3b1fec..f55367ae 100644 --- a/src/test/kotlin/dev/typetype/server/TestDatabase.kt +++ b/src/test/kotlin/dev/typetype/server/TestDatabase.kt @@ -4,6 +4,7 @@ import dev.typetype.server.db.DatabaseFactory import dev.typetype.server.db.tables.AllowedChannelsTable import dev.typetype.server.db.tables.AllowedPlaylistsTable import dev.typetype.server.db.tables.BlockedChannelsTable +import dev.typetype.server.db.tables.BlockedKeywordsTable import dev.typetype.server.db.tables.BlockedVideosTable import dev.typetype.server.db.tables.BugReportsTable import dev.typetype.server.db.tables.FavoritesTable @@ -105,6 +106,7 @@ object TestDatabase { AllowedChannelsTable.deleteAll() AllowedPlaylistsTable.deleteAll() BlockedChannelsTable.deleteAll() + BlockedKeywordsTable.deleteAll() BlockedVideosTable.deleteAll() YoutubeTakeoutImportJobsTable.deleteAll() YoutubeTakeoutPlaylistKeysTable.deleteAll() diff --git a/src/test/kotlin/dev/typetype/server/TypeTypeBackupServiceTest.kt b/src/test/kotlin/dev/typetype/server/TypeTypeBackupServiceTest.kt new file mode 100644 index 00000000..8cce3567 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/TypeTypeBackupServiceTest.kt @@ -0,0 +1,180 @@ +package dev.typetype.server + +import dev.typetype.server.models.AllowedChannelItem +import dev.typetype.server.models.AllowedPlaylistItem +import dev.typetype.server.models.FavoriteItem +import dev.typetype.server.models.HistoryItem +import dev.typetype.server.models.PlaylistItem +import dev.typetype.server.models.PlaylistVideoItem +import dev.typetype.server.models.PublicPlaylistItem +import dev.typetype.server.models.SettingsItem +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.models.TypeTypeBackupItem +import dev.typetype.server.models.TypeTypeContentFiltersBackup +import dev.typetype.server.models.WatchLaterItem +import dev.typetype.server.models.BlockedKeywordItem +import dev.typetype.server.services.AllowedChannelsService +import dev.typetype.server.services.AllowedPlaylistsService +import dev.typetype.server.services.BlockedService +import dev.typetype.server.services.FavoritesService +import dev.typetype.server.services.HistoryService +import dev.typetype.server.services.PlaylistService +import dev.typetype.server.services.ProgressService +import dev.typetype.server.services.SavedPlaylistService +import dev.typetype.server.services.SearchHistoryService +import dev.typetype.server.services.SettingsService +import dev.typetype.server.services.SubscriptionsService +import dev.typetype.server.services.TypeTypeBackupCategory +import dev.typetype.server.services.TypeTypeBackupService +import dev.typetype.server.services.WatchLaterService +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class TypeTypeBackupServiceTest { + private val subscriptions = SubscriptionsService() + private val history = HistoryService() + private val playlists = PlaylistService() + private val watchLater = WatchLaterService() + private val favorites = FavoritesService() + private val progress = ProgressService() + private val searchHistory = SearchHistoryService() + private val savedPlaylists = SavedPlaylistService() + private val settings = SettingsService() + private val blocked = BlockedService() + private val allowedChannels = AllowedChannelsService() + private val allowedPlaylists = AllowedPlaylistsService() + private val service = TypeTypeBackupService( + subscriptions, + history, + playlists, + watchLater, + favorites, + progress, + searchHistory, + savedPlaylists, + settings, + blocked, + allowedChannels, + allowedPlaylists, + ) + + companion object { + private const val SOURCE = "backup-source" + private const val TARGET = "backup-target" + + @BeforeAll + @JvmStatic + fun initDb() = TestDatabase.setup() + } + + @BeforeEach + fun clean() = TestDatabase.truncateAll() + + @Test + fun `full backup restores every user data category`() = runTest { + subscriptions.add(SOURCE, SubscriptionItem("https://youtube.com/channel/source", "Source", "avatar")) + history.addImported(SOURCE, videoHistory()) + val playlist = playlists.create(SOURCE, PlaylistItem(name = "Saved videos")) + playlists.addVideo(SOURCE, playlist.id, playlistVideo()) + watchLater.add(SOURCE, watchLaterItem()) + favorites.add(SOURCE, FavoriteItem(videoUrl = VIDEO_URL, title = "Favorite")) + progress.upsert(SOURCE, VIDEO_URL, 90_000) + searchHistory.add(SOURCE, "kotlin") + savedPlaylists.save( + SOURCE, + PublicPlaylistItem("PL1", "Public", "https://youtube.com/playlist?list=PL1", "", "Uploader", 3, "playlist"), + ) + settings.upsert(SOURCE, SettingsItem(defaultPlaybackSpeed = 1.75)) + blocked.addChannel(SOURCE, "https://youtube.com/channel/blocked") + blocked.addVideo(SOURCE, VIDEO_URL) + blocked.addKeyword(SOURCE, "spoiler") + blocked.addKeyword("admin", "global policy", global = true) + allowedChannels.addChannel(SOURCE, "https://youtube.com/channel/allowed") + allowedPlaylists.addPlaylist( + SOURCE, + AllowedPlaylistItem("https://youtube.com/playlist?list=allowed"), + global = false, + ) + + val backup = service.export(SOURCE, TypeTypeBackupCategory.all) + assertFalse(backup.contentFilters.orEmptyKeywords().contains("global policy")) + assertEquals(90, backup.history?.single()?.progress) + val result = service.restore(TARGET, backup) + + assertEquals(1, result.restored["subscriptions"]) + assertEquals(1, result.restored["history"]) + assertEquals(1, result.restored["playlists"]) + assertEquals(1, result.restored["playlistVideos"]) + assertEquals(1, result.restored["watchLater"]) + assertEquals(1, result.restored["favorites"]) + assertEquals(1, result.restored["progress"]) + assertEquals(1, result.restored["searchHistory"]) + assertEquals(1, result.restored["savedPlaylists"]) + assertEquals(1.75, settings.get(TARGET).defaultPlaybackSpeed) + assertEquals(1, playlists.getById(TARGET, playlists.getAll(TARGET).single().id)?.videos?.size) + assertEquals(setOf("spoiler", "global policy"), blocked.getKeywords(TARGET).map { it.keyword }.toSet()) + } + + @Test + fun `selective restore leaves categories absent from backup untouched`() = runTest { + favorites.add(TARGET, FavoriteItem(videoUrl = "existing")) + subscriptions.add(SOURCE, SubscriptionItem("https://youtube.com/channel/source", "Source", "")) + val backup = service.export(SOURCE, setOf(TypeTypeBackupCategory.SUBSCRIPTIONS)) + + service.restore(TARGET, backup) + + assertEquals(1, subscriptions.getAll(TARGET).size) + assertEquals("existing", favorites.getAll(TARGET).single().videoUrl) + assertTrue(backup.history == null) + } + + @Test + fun `restore rejects empty normalized blocked keywords`() = runTest { + val backup = TypeTypeBackupItem( + exportedAt = 1, + categories = listOf(TypeTypeBackupCategory.CONTENT_FILTERS.wireName), + contentFilters = TypeTypeContentFiltersBackup( + blockedKeywords = listOf(BlockedKeywordItem(" ", 1)), + ), + ) + + val error = runCatching { service.restore(TARGET, backup) }.exceptionOrNull() + + assertTrue(error is IllegalArgumentException) + } + + private fun videoHistory() = HistoryItem( + url = VIDEO_URL, + title = "Video", + thumbnail = "", + channelName = "Channel", + channelUrl = "https://youtube.com/channel/source", + duration = 120, + progress = 42, + watchedAt = 1234, + ) + + private fun playlistVideo() = PlaylistVideoItem( + url = VIDEO_URL, + title = "Video", + thumbnail = "", + duration = 120, + ) + + private fun watchLaterItem() = WatchLaterItem( + url = VIDEO_URL, + title = "Video", + thumbnail = "", + duration = 120, + ) + + private fun dev.typetype.server.models.TypeTypeContentFiltersBackup?.orEmptyKeywords() = + this?.blockedKeywords?.map { it.keyword }.orEmpty() +} + +private const val VIDEO_URL = "https://youtube.com/watch?v=backup" diff --git a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutActivitySignalServiceTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutActivitySignalServiceTest.kt index a05ac46c..845f712d 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutActivitySignalServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutActivitySignalServiceTest.kt @@ -22,23 +22,44 @@ class YoutubeTakeoutActivitySignalServiceTest { "https://www.youtube.com/watch?v=like456", "https://www.youtube.com/watch?v=like789", ), - result.second, + result.second.map { it.videoUrl }, ) + assertEquals(1_783_530_662_000L, result.second.first().favoritedAt) + assertEquals("A video", result.second.first().title) + assertEquals("Channel Name", result.second.first().channelName) + assertEquals("https://www.youtube.com/channel/UC999", result.second.first().channelUrl) Files.deleteIfExists(zip) } - private fun createZip(): Path { + @Test + fun `parse omits unavailable activity favorites`() { + val zip = createZip( + """ + You liked Deleted Video
+ 8 Jul 2026, 19:11:02 CEST
+ """.trimIndent(), + ) + + assertTrue(YoutubeTakeoutActivitySignalService.parse(zip).second.isEmpty()) + Files.deleteIfExists(zip) + } + + private fun createZip( + html: String = """ + Vous vous êtes abonné à Channel Name
+ Vous avez aimé A video
+ Channel Name
+ 8 juil. 2026, 19:11:02 CEST
+ A aimé Another video
+ 5 juil. 2026, 19:10:19 CEST
+ BeÄŸendiniz Turkish video
+ 4 Jul 2026, 12:00:00 CEST
+ """.trimIndent(), + ): Path { val zip = Files.createTempFile("yt-activity-signals-", ".zip") ZipOutputStream(Files.newOutputStream(zip)).use { out -> out.putNextEntry(ZipEntry("Takeout/Mon activité/YouTube/MonActivité.html")) - out.write(""" - - Vous vous êtes abonné à Channel Name
- Vous avez aimé A video
- A aimé Another video
- BeÄŸendiniz Turkish video
- - """.trimIndent().toByteArray()) + out.write("$html".toByteArray()) out.closeEntry() } return zip diff --git a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt index a2644aff..55970acc 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt @@ -102,6 +102,41 @@ class YoutubeTakeoutParserServiceTest { Files.deleteIfExists(zip) } + @Test + fun `parse preserves favorite dates from video added timestamp header`() { + val zip = Files.createTempFile("yt-takeout-favorite-order-", ".zip") + ZipOutputStream(Files.newOutputStream(zip)).use { out -> + out.putNextEntry(ZipEntry("Takeout/YouTube/playlists/Liked videos.csv")) + out.write( + "Video ID,Video Added Timestamp\nnewer000001,2026-01-02T00:00:00Z\nolder000001,2026-01-01T00:00:00Z\n" + .toByteArray(), + ) + out.closeEntry() + } + + val favorites = YoutubeTakeoutParserService().parse(zip).favorites + + assertEquals(listOf("newer000001", "older000001"), favorites.map { it.videoUrl.substringAfter("v=") }) + assertEquals(listOf(1_767_312_000_000L, 1_767_225_600_000L), favorites.map { it.favoritedAt }) + Files.deleteIfExists(zip) + } + + @Test + fun `parse preserves favorite dates from a localized timestamp header`() { + val zip = Files.createTempFile("yt-takeout-localized-favorite-order-", ".zip") + ZipOutputStream(Files.newOutputStream(zip)).use { out -> + out.putNextEntry(ZipEntry("Takeout/YouTube/playlists/Liked videos.csv")) + out.write("Video ID,Ajoutee a la playlist\nvideo000001,2026-01-03T00:00:00Z\n".toByteArray()) + out.closeEntry() + } + + val favorite = YoutubeTakeoutParserService().parse(zip).favorites.single() + + assertEquals("video000001", favorite.videoUrl.substringAfter("v=")) + assertEquals(1_767_398_400_000L, favorite.favoritedAt) + Files.deleteIfExists(zip) + } + private fun createZip(): Path { val zip = Files.createTempFile("yt-takeout-parser-", ".zip") ZipOutputStream(Files.newOutputStream(zip)).use { out -> diff --git a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutSystemPlaylistImportTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutSystemPlaylistImportTest.kt index 1d1ecfcf..2370b125 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutSystemPlaylistImportTest.kt +++ b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutSystemPlaylistImportTest.kt @@ -68,6 +68,27 @@ class YoutubeTakeoutSystemPlaylistImportTest { assertEquals(listOf(newer.addedAt, older.addedAt), imported.map { it.addedAt }) } + @Test + fun `commit returns imported favorites newest first`() = runBlocking { + val parsed = YoutubeTakeoutParsedData( + subscriptions = emptyList(), + playlists = emptyList(), + playlistItems = emptyMap(), + favorites = listOf( + FavoriteItem(videoUrl = "https://www.youtube.com/watch?v=older", favoritedAt = 1_700_000_000_000L), + FavoriteItem(videoUrl = "https://www.youtube.com/watch?v=newer", favoritedAt = 1_700_000_100_000L), + ), + watchLater = emptyList(), + history = emptyList(), + warnings = emptyList(), + errors = emptyList(), + ) + + importer.commit(TEST_USER_ID, parsed, YoutubeTakeoutCommitPlan(false, false, false, true, false, false)) + + assertEquals(listOf("newer", "older"), favorites.getAll(TEST_USER_ID).map { it.videoUrl.substringAfter("v=") }) + } + private fun parsed(): YoutubeTakeoutParsedData = YoutubeTakeoutParsedData( subscriptions = emptyList(), playlists = listOf(PlaylistItem(id = "WL", name = "Watch later"), PlaylistItem(id = "LL", name = "Liked videos")), diff --git a/src/test/kotlin/dev/typetype/server/routes/AndroidPlaybackRouteContractTest.kt b/src/test/kotlin/dev/typetype/server/routes/AndroidPlaybackRouteContractTest.kt deleted file mode 100644 index 1d598d2d..00000000 --- a/src/test/kotlin/dev/typetype/server/routes/AndroidPlaybackRouteContractTest.kt +++ /dev/null @@ -1,203 +0,0 @@ -package dev.typetype.server.routes - -import dev.typetype.server.services.AndroidDashManifestResult -import dev.typetype.server.services.AndroidPlaybackMediaResult -import dev.typetype.server.services.AndroidPlaybackPreparationStage -import dev.typetype.server.services.AndroidPlaybackService -import dev.typetype.server.services.AndroidPlaybackSession -import dev.typetype.server.services.AndroidPlaybackSessionLookup -import dev.typetype.server.services.AndroidSubtitleInventoryCoordinator -import dev.typetype.server.services.SabrSessionHolder -import io.ktor.client.request.get -import io.ktor.client.statement.bodyAsText -import io.ktor.http.ContentType -import io.ktor.http.HttpHeaders -import io.ktor.http.HttpStatusCode -import io.ktor.http.contentType -import io.ktor.serialization.kotlinx.json.json -import io.ktor.server.application.install -import io.ktor.server.plugins.contentnegotiation.ContentNegotiation -import io.ktor.server.routing.get -import io.ktor.server.routing.routing -import io.ktor.server.testing.testApplication -import io.mockk.coEvery -import io.mockk.every -import io.mockk.mockk -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat - -class AndroidPlaybackRouteContractTest { - @Test - fun `dedicated Android manifest path does not use the web SABR namespace`() = testApplication { - application { - install(ContentNegotiation) { json() } - routing { - androidPlaybackRoutes( - mockk(relaxed = true), - mockk(), - mockk(), - mockk(), - null, - null, - null, - ) - } - } - - val response = client.get("/android/youtube/playback/unknown/manifest.mpd") - - assertEquals(HttpStatusCode.NotFound, response.status) - assertEquals("no-store", response.headers[HttpHeaders.CacheControl]) - } - - @Test - fun `ready manifest returns dash xml with no store`() = testApplication { - val fixture = fixture() - every { fixture.service.lookup(SESSION_ID) } returns fixture.lookup - every { fixture.service.manifest(fixture.lookup.session) } returns AndroidDashManifestResult.Ready(MPD, 1_000L) - application { - install(ContentNegotiation) { json() } - routing { get("/manifest") { fixture.handler.manifest(call, SESSION_ID) } } - } - - val response = client.get("/manifest") - - assertEquals(HttpStatusCode.OK, response.status) - assertEquals("no-store", response.headers[HttpHeaders.CacheControl]) - assertEquals(ContentType.parse("application/dash+xml"), response.contentType()?.withoutParameters()) - assertEquals(MPD, response.bodyAsText()) - } - - @Test - fun `preparing manifest returns bounded json retry`() = testApplication { - val fixture = fixture() - every { fixture.service.lookup(SESSION_ID) } returns fixture.lookup - every { fixture.service.manifest(fixture.lookup.session) } returns AndroidDashManifestResult.Preparing( - AndroidPlaybackPreparationStage.AUDIO_VIDEO_INDEX, - ) - application { - install(ContentNegotiation) { json() } - routing { get("/manifest") { fixture.handler.manifest(call, SESSION_ID) } } - } - - val response = client.get("/manifest") - - assertEquals(HttpStatusCode.Accepted, response.status) - assertTrue(response.bodyAsText().contains("\"retryAfterMs\":500")) - assertTrue(response.bodyAsText().contains("\"preparationStage\":\"audio_video_index\"")) - } - - @Test - fun `unknown and expired sessions are distinguished`() = testApplication { - val fixture = fixture() - every { fixture.service.lookup("unknown") } returns AndroidPlaybackSessionLookup.Unknown - every { fixture.service.lookup("expired") } returns AndroidPlaybackSessionLookup.Expired - application { - install(ContentNegotiation) { json() } - routing { - get("/unknown") { fixture.handler.manifest(call, "unknown") } - get("/expired") { fixture.handler.manifest(call, "expired") } - } - } - - assertEquals(HttpStatusCode.NotFound, client.get("/unknown").status) - assertEquals(HttpStatusCode.Gone, client.get("/expired").status) - } - - @Test - fun `stale media generation returns conflict`() = testApplication { - val fixture = fixture() - every { fixture.service.lookup(SESSION_ID) } returns fixture.lookup - coEvery { fixture.service.segment(fixture.holder, 137, 1, 0L) } returns AndroidPlaybackMediaResult.StaleGeneration - val media = AndroidPlaybackMediaHandler(fixture.service) - application { - install(ContentNegotiation) { json() } - routing { - get("/segment") { media.segment(call, SESSION_ID, 137, 1) } - } - } - - val response = client.get("/segment?session=$SESSION_ID&generation=0") - - assertEquals(HttpStatusCode.Conflict, response.status) - assertTrue(response.bodyAsText().contains("android_playback_stale_generation")) - } - - @Test - fun `invalid complete index returns unprocessable entity`() = testApplication { - val fixture = fixture() - every { fixture.service.lookup(SESSION_ID) } returns fixture.lookup - every { fixture.service.manifest(fixture.lookup.session) } returns AndroidDashManifestResult.Invalid("invalid index") - application { - install(ContentNegotiation) { json() } - routing { get("/manifest") { fixture.handler.manifest(call, SESSION_ID) } } - } - - val response = client.get("/manifest") - - assertEquals(HttpStatusCode.UnprocessableEntity, response.status) - assertTrue(response.bodyAsText().contains("android_playback_invalid_index")) - } - - @Test - fun `preparation failure returns a typed service unavailable response`() = testApplication { - val fixture = fixture() - every { fixture.service.lookup(SESSION_ID) } returns fixture.lookup - every { fixture.service.manifest(fixture.lookup.session) } returns AndroidDashManifestResult.TemporaryFailure( - "android_playback_preparation_timeout", - "Android playback preparation is temporarily unavailable", - ) - application { - install(ContentNegotiation) { json() } - routing { get("/manifest") { fixture.handler.manifest(call, SESSION_ID) } } - } - - val response = client.get("/manifest") - - assertEquals(HttpStatusCode.ServiceUnavailable, response.status) - assertTrue(response.bodyAsText().contains("android_playback_preparation_timeout")) - } - - private fun fixture(): Fixture { - val service = mockk() - val audio = format(140, true) - val video = format(137, false) - val holder = mockk { - every { sessionToken } returns SESSION_ID - every { key.videoId } returns "video" - every { audioFormat } returns audio - every { videoFormat } returns video - every { activeGeneration() } returns 0L - } - val handler = AndroidPlaybackHandler( - mockk(), - mockk(), - null, - null, - null, - mockk(), - service, - ) - return Fixture(service, holder, handler, AndroidPlaybackSessionLookup.Active(AndroidPlaybackSession(holder, emptyList()))) - } - - private fun format(itag: Int, audio: Boolean): YoutubeSabrFormat = mockk { - every { this@mockk.itag } returns itag - every { audioTrackId } returns null - every { isAudio } returns audio - } - - private data class Fixture( - val service: AndroidPlaybackService, - val holder: SabrSessionHolder, - val handler: AndroidPlaybackHandler, - val lookup: AndroidPlaybackSessionLookup.Active, - ) - - private companion object { - const val SESSION_ID = "android-session" - const val MPD = "" - } -} diff --git a/src/test/kotlin/dev/typetype/server/routes/AndroidPlaybackSubtitleBootstrapTest.kt b/src/test/kotlin/dev/typetype/server/routes/AndroidPlaybackSubtitleBootstrapTest.kt deleted file mode 100644 index 526ae548..00000000 --- a/src/test/kotlin/dev/typetype/server/routes/AndroidPlaybackSubtitleBootstrapTest.kt +++ /dev/null @@ -1,202 +0,0 @@ -package dev.typetype.server.routes - -import dev.typetype.server.services.AndroidDashManifestResult -import dev.typetype.server.services.AndroidPlaybackPreparationStage -import dev.typetype.server.services.AndroidPlaybackCreateResult -import dev.typetype.server.services.AndroidPlaybackService -import dev.typetype.server.services.AndroidPlaybackSession -import dev.typetype.server.services.AndroidSubtitleInventoryCoordinator -import dev.typetype.server.services.AndroidSubtitleInventoryHandle -import dev.typetype.server.services.AndroidSubtitleInventorySnapshot -import dev.typetype.server.services.AndroidSubtitleTrack -import dev.typetype.server.services.SabrPreparedInfo -import dev.typetype.server.services.SabrSessionHolder -import dev.typetype.server.services.SabrSessionStore -import dev.typetype.server.services.StreamService -import io.ktor.client.request.post -import io.ktor.client.request.setBody -import io.ktor.client.statement.bodyAsText -import io.ktor.http.ContentType -import io.ktor.http.HttpStatusCode -import io.ktor.http.contentType -import io.ktor.serialization.kotlinx.json.json -import io.ktor.server.application.install -import io.ktor.server.plugins.contentnegotiation.ContentNegotiation -import io.ktor.server.routing.post -import io.ktor.server.routing.routing -import io.ktor.server.testing.testApplication -import io.mockk.coEvery -import io.mockk.coVerify -import io.mockk.every -import io.mockk.mockk -import kotlinx.coroutines.async -import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.yield -import kotlinx.serialization.json.Json -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertFalse -import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo - -class AndroidPlaybackSubtitleBootstrapTest { - @Test - fun `creation waits for the authoritative descriptor catalog`() = testApplication { - val inventory = AndroidSubtitleInventoryHandle.preparing() - val fixture = fixture(inventory) - fixture.stubCreated(listOf(TRACK), AndroidDashManifestResult.Ready(MPD, 213_000L)) - application { installRoute(fixture.handler) } - - coroutineScope { - val pending = async { - client.post("/playback") { - contentType(ContentType.Application.Json) - setBody("""{"subtitleMode":"deferred"}""") - } - } - yield() - assertFalse(pending.isCompleted) - - inventory.complete(AndroidSubtitleInventorySnapshot.Ready(listOf(TRACK))) - val response = pending.await() - val body = response.bodyAsText() - - assertEquals(HttpStatusCode.OK, response.status) - assertTrue(body.contains("\"subtitles\":[{\"id\":\"track\"")) - assertTrue(body.contains("/subtitles/track.vtt")) - assertFalse(body.contains("subtitleInventory")) - } - } - - @Test - fun `creation fails when a complete descriptor catalog is unavailable`() = testApplication { - val fixture = fixture(AndroidSubtitleInventoryHandle.temporaryFailure()) - application { installRoute(fixture.handler) } - - val response = client.post("/playback") { - contentType(ContentType.Application.Json) - setBody("{}") - } - - assertEquals(HttpStatusCode.ServiceUnavailable, response.status) - assertTrue(response.bodyAsText().contains("android_subtitle_inventory_unavailable")) - coVerify(exactly = 0) { fixture.service.create(any(), any(), any(), any(), any(), any()) } - } - - @Test - fun `video without captions returns an authoritative empty catalog`() = testApplication { - val fixture = fixture(AndroidSubtitleInventoryHandle.ready(emptyList())) - fixture.stubCreated(emptyList(), AndroidDashManifestResult.Ready(MPD, 213_000L)) - application { installRoute(fixture.handler) } - - val response = client.post("/playback") { - contentType(ContentType.Application.Json) - setBody("{}") - } - - assertEquals(HttpStatusCode.OK, response.status) - assertTrue(response.bodyAsText().contains("\"subtitles\":[]")) - } - - @Test - fun `preparing media response still contains every subtitle descriptor`() = testApplication { - val fixture = fixture(AndroidSubtitleInventoryHandle.ready(listOf(TRACK))) - fixture.stubCreated( - listOf(TRACK), - AndroidDashManifestResult.Preparing(AndroidPlaybackPreparationStage.AUDIO_VIDEO_INDEX), - ) - application { installRoute(fixture.handler) } - - val response = client.post("/playback") { - contentType(ContentType.Application.Json) - setBody("{}") - } - - assertEquals(HttpStatusCode.Accepted, response.status) - assertTrue(response.bodyAsText().contains("\"id\":\"track\"")) - assertTrue(response.bodyAsText().contains("\"retryAfterMs\":500")) - } - - private fun io.ktor.server.application.Application.installRoute(handler: AndroidPlaybackHandler) { - install(ContentNegotiation) { - json(Json { ignoreUnknownKeys = true; encodeDefaults = true }) - } - routing { post("/playback") { handler.create(call, VIDEO_ID) } } - } - - private fun fixture(inventory: AndroidSubtitleInventoryHandle): Fixture { - val audio = format(140, audio = true, "audio/mp4; codecs=\"mp4a.40.2\"") - val video = format(137, audio = false, "video/mp4; codecs=\"avc1.4d4028\"") - val info = mockk { - every { formats } returns listOf(audio, video) - } - val prepared = SabrPreparedInfo(info, null) - val store = mockk() - coEvery { store.fetchInfo(VIDEO_ID, cachedFirst = true) } returns prepared - val coordinator = mockk() - every { coordinator.start(VIDEO_ID) } returns inventory - val service = mockk() - val holder = mockk { - every { sessionToken } returns SESSION_ID - every { key.videoId } returns VIDEO_ID - every { audioFormat } returns audio - every { videoFormat } returns video - every { activeGeneration() } returns 0L - } - val handler = AndroidPlaybackHandler( - store, - mockk(), - null, - null, - null, - coordinator, - service, - ) - return Fixture(handler, service, prepared, holder, audio, video) - } - - private fun format(itag: Int, audio: Boolean, mimeType: String): YoutubeSabrFormat = - mockk(relaxed = true) { - every { this@mockk.itag } returns itag - every { isAudio } returns audio - every { isVideo } returns !audio - every { this@mockk.mimeType } returns mimeType - every { height } returns if (audio) 0 else 1080 - every { bitrate } returns if (audio) 128_000 else 4_000_000 - every { audioTrackId } returns null - } - - private data class Fixture( - val handler: AndroidPlaybackHandler, - val service: AndroidPlaybackService, - val prepared: SabrPreparedInfo, - val holder: SabrSessionHolder, - val audio: YoutubeSabrFormat, - val video: YoutubeSabrFormat, - ) { - fun stubCreated( - subtitles: List, - manifest: AndroidDashManifestResult, - ) { - coEvery { - service.create(VIDEO_ID, "guest", prepared, audio, video, subtitles) - } returns AndroidPlaybackCreateResult.Created( - AndroidPlaybackSession(holder, subtitles), - manifest, - ) - } - } - - private companion object { - const val VIDEO_ID = "dQw4w9WgXcQ" - const val SESSION_ID = "android-session" - const val MPD = "" - val TRACK = mockk(relaxed = true) { - every { id } returns "track" - every { languageTag } returns "en" - every { displayLanguageName } returns "English" - every { isAutoGenerated } returns false - } - } -} diff --git a/src/test/kotlin/dev/typetype/server/routes/AndroidPlaybackSubtitleModelsTest.kt b/src/test/kotlin/dev/typetype/server/routes/AndroidPlaybackSubtitleModelsTest.kt deleted file mode 100644 index 880232dc..00000000 --- a/src/test/kotlin/dev/typetype/server/routes/AndroidPlaybackSubtitleModelsTest.kt +++ /dev/null @@ -1,55 +0,0 @@ -package dev.typetype.server.routes - -import dev.typetype.server.services.AndroidDashManifestResult -import dev.typetype.server.services.AndroidSubtitleTrack -import dev.typetype.server.services.AndroidPlaybackSession -import dev.typetype.server.services.SabrSessionHolder -import io.mockk.every -import io.mockk.mockk -import okhttp3.HttpUrl.Companion.toHttpUrl -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat - -class AndroidPlaybackSubtitleModelsTest { - @Test - fun `subtitle resources stay stable when the playback generation changes`() { - var generation = 0L - val holder = mockk { - every { sessionToken } returns "session" - every { key.videoId } returns "video" - every { audioFormat } returns format(140) - every { videoFormat } returns format(137) - every { activeGeneration() } answers { generation } - } - val track = AndroidSubtitleTrack( - "track-id", - "en", - "English", - false, - "https://www.youtube.com/api/timedtext?v=video&lang=en".toHttpUrl(), - ) - - val session = AndroidPlaybackSession(holder, listOf(track)) - val initial = session.toAndroidPlaybackResponse(MANIFEST) - generation = 1L - val afterSeek = session.toAndroidPlaybackResponse(MANIFEST) - - assertEquals(0L, initial.generation) - assertEquals(1L, afterSeek.generation) - assertEquals(initial.subtitles, afterSeek.subtitles) - assertEquals( - "/api/android/youtube/playback/session/subtitles/track-id.vtt", - initial.subtitles.single().url, - ) - } - - private fun format(itag: Int): YoutubeSabrFormat = mockk { - every { this@mockk.itag } returns itag - every { audioTrackId } returns null - } - - private companion object { - val MANIFEST = AndroidDashManifestResult.Ready("", 1_000L) - } -} diff --git a/src/test/kotlin/dev/typetype/server/routes/AndroidSubtitleRouteTest.kt b/src/test/kotlin/dev/typetype/server/routes/AndroidSubtitleRouteTest.kt deleted file mode 100644 index 5574d3cf..00000000 --- a/src/test/kotlin/dev/typetype/server/routes/AndroidSubtitleRouteTest.kt +++ /dev/null @@ -1,128 +0,0 @@ -package dev.typetype.server.routes - -import dev.typetype.server.services.AndroidPlaybackService -import dev.typetype.server.services.AndroidPlaybackSession -import dev.typetype.server.services.AndroidPlaybackSessionLookup -import dev.typetype.server.services.AndroidSubtitleContentResult -import dev.typetype.server.services.AndroidSubtitleService -import dev.typetype.server.services.AndroidSubtitleTrack -import dev.typetype.server.services.AuthService -import dev.typetype.server.services.SabrSessionHolder -import dev.typetype.server.services.SabrSessionKey -import dev.typetype.server.services.SabrSessionPurpose -import io.ktor.client.request.bearerAuth -import io.ktor.client.request.get -import io.ktor.client.statement.bodyAsText -import io.ktor.http.ContentType -import io.ktor.http.HttpHeaders -import io.ktor.http.HttpStatusCode -import io.ktor.http.contentType -import io.ktor.serialization.kotlinx.json.json -import io.ktor.server.application.install -import io.ktor.server.plugins.contentnegotiation.ContentNegotiation -import io.ktor.server.routing.get -import io.ktor.server.routing.routing -import io.ktor.server.testing.testApplication -import io.mockk.coEvery -import io.mockk.coVerify -import io.mockk.every -import io.mockk.mockk -import okhttp3.HttpUrl.Companion.toHttpUrl -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.Test - -class AndroidSubtitleRouteTest { - @Test - fun `session owner receives complete WebVTT with no store`() = testApplication { - val fixture = fixture("user") - coEvery { fixture.subtitles.content(VIDEO_ID, TRACK) } returns AndroidSubtitleContentResult.Ready(VTT) - application { installRoute(fixture.handler) } - - val response = client.get("/subtitle") { bearerAuth("test-jwt") } - - assertEquals(HttpStatusCode.OK, response.status) - assertEquals(ContentType.parse("text/vtt"), response.contentType()?.withoutParameters()) - assertEquals("no-store", response.headers[HttpHeaders.CacheControl]) - assertEquals(VTT.decodeToString(), response.bodyAsText()) - } - - @Test - fun `another account cannot read a session subtitle`() = testApplication { - val fixture = fixture("owner", authenticatedUser = "other") - application { installRoute(fixture.handler) } - - val response = client.get("/subtitle") { bearerAuth("test-jwt") } - - assertEquals(HttpStatusCode.NotFound, response.status) - assertTrue(response.bodyAsText().contains("android_subtitle_not_found")) - coVerify(exactly = 0) { fixture.subtitles.content(any(), any()) } - } - - @Test - fun `bounded upstream failure returns typed service unavailable`() = testApplication { - val fixture = fixture("user") - coEvery { fixture.subtitles.content(VIDEO_ID, TRACK) } returns AndroidSubtitleContentResult.TemporaryFailure - application { installRoute(fixture.handler) } - - val response = client.get("/subtitle") { bearerAuth("test-jwt") } - - assertEquals(HttpStatusCode.ServiceUnavailable, response.status) - assertTrue(response.bodyAsText().contains("android_subtitle_upstream_unavailable")) - } - - private fun io.ktor.server.application.Application.installRoute(handler: AndroidSubtitleHandler) { - install(ContentNegotiation) { json() } - routing { - get("/subtitle") { handler.content(call, SESSION_ID, TRACK.id) } - } - } - - private fun fixture( - owner: String, - authenticatedUser: String = owner, - ): Fixture { - val playback = mockk() - val holder = mockk { - every { key } returns SabrSessionKey( - VIDEO_ID, - owner, - 140, - null, - 137, - 0L, - SabrSessionPurpose.ANDROID_PLAYBACK, - ) - } - every { playback.lookup(SESSION_ID) } returns - AndroidPlaybackSessionLookup.Active(AndroidPlaybackSession(holder, listOf(TRACK))) - val subtitles = mockk() - val handler = AndroidSubtitleHandler( - playback, - subtitles, - AuthService.fixed(authenticatedUser), - null, - null, - ) - return Fixture(handler, playback, subtitles) - } - - private data class Fixture( - val handler: AndroidSubtitleHandler, - val playback: AndroidPlaybackService, - val subtitles: AndroidSubtitleService, - ) - - private companion object { - const val SESSION_ID = "android-session" - const val VIDEO_ID = "dQw4w9WgXcQ" - val VTT = "WEBVTT\n\n00:00.000 --> 00:01.000\nHello\n".toByteArray() - val TRACK = AndroidSubtitleTrack( - "track", - "en", - "English", - false, - "https://www.youtube.com/api/timedtext?v=$VIDEO_ID&lang=en".toHttpUrl(), - ) - } -} diff --git a/src/test/kotlin/dev/typetype/server/services/AndroidDashManifestBuilderTest.kt b/src/test/kotlin/dev/typetype/server/services/AndroidDashManifestBuilderTest.kt deleted file mode 100644 index c71257c5..00000000 --- a/src/test/kotlin/dev/typetype/server/services/AndroidDashManifestBuilderTest.kt +++ /dev/null @@ -1,96 +0,0 @@ -package dev.typetype.server.services - -import io.mockk.every -import io.mockk.mockk -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertFalse -import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import java.io.ByteArrayInputStream -import javax.xml.parsers.DocumentBuilderFactory - -class AndroidDashManifestBuilderTest { - @Test - fun `five minute vod describes the complete presentation from zero`() { - val manifest = manifest( - audioSegments = timeline(300, 1_000L), - videoSegments = timeline(150, 2_000L), - durationMs = 300_000L, - ) - - val document = parse(manifest) - val mpd = document.documentElement - assertEquals("static", mpd.getAttribute("type")) - assertEquals("PT300.000S", mpd.getAttribute("mediaPresentationDuration")) - assertEquals("PT0S", document.getElementsByTagNameNS(DASH_NAMESPACE, "Period").item(0).attributes - .getNamedItem("start").nodeValue) - assertEquals(2, document.getElementsByTagNameNS(DASH_NAMESPACE, "SegmentTimeline").length) - } - - @Test - fun `manifest resources contain the session and generation with valid XML escaping`() { - val manifest = manifest(timeline(3, 1_000L), timeline(3, 1_000L), 3_000L, generation = 7L) - - assertTrue(manifest.contains("session=session-token&generation=7")) - val template = parse(manifest).getElementsByTagNameNS(DASH_NAMESPACE, "SegmentTemplate").item(0) - assertEquals( - "/api/android/youtube/playback/session-token/137/segment/\$Number\$?session=session-token&generation=7", - template.attributes.getNamedItem("media").nodeValue, - ) - } - - @Test - fun `audio and video may have different exact segment counts`() { - val manifest = manifest(timeline(6, 500L), timeline(2, 1_500L), 3_000L) - val document = parse(manifest) - val templates = document.getElementsByTagNameNS(DASH_NAMESPACE, "SegmentTemplate") - - assertEquals("1", templates.item(0).attributes.getNamedItem("startNumber").nodeValue) - assertEquals("1", templates.item(1).attributes.getNamedItem("startNumber").nodeValue) - assertEquals(2, document.getElementsByTagNameNS(DASH_NAMESPACE, "S").length) - } - - @Test - fun `long vod manifest remains compact`() { - val manifest = manifest(timeline(100_000, 1_000L), timeline(100_000, 1_000L), 100_000_000L) - - assertTrue(manifest.toByteArray().size < 2 * 1024 * 1024) - assertFalse(manifest.contains("SegmentURL")) - assertEquals(2, parse(manifest).getElementsByTagNameNS(DASH_NAMESPACE, "S").length) - } - - private fun manifest( - audioSegments: List, - videoSegments: List, - durationMs: Long, - generation: Long = 0L, - ): String = AndroidDashManifestBuilder.build( - sessionId = "session-token", - generation = generation, - audio = AndroidDashTrack(format(140, audio = true), AndroidDashTimeline(1, audioSegments)), - video = AndroidDashTrack(format(137, audio = false), AndroidDashTimeline(1, videoSegments)), - durationMs = durationMs, - ) - - private fun timeline(count: Int, durationMs: Long): List = - List(count) { index -> AndroidDashTimelineSegment(index * durationMs, durationMs) } - - private fun format(itag: Int, audio: Boolean): YoutubeSabrFormat = mockk { - every { this@mockk.itag } returns itag - every { isAudio } returns audio - every { isVideo } returns !audio - every { mimeType } returns if (audio) "audio/mp4; codecs=\"mp4a.40.2\"" else "video/mp4; codecs=\"avc1.640028\"" - every { bitrate } returns if (audio) 128_000 else 2_000_000 - every { width } returns if (audio) 0 else 1920 - every { height } returns if (audio) 0 else 1080 - } - - private fun parse(manifest: String) = DocumentBuilderFactory.newInstance().apply { - isNamespaceAware = true - }.newDocumentBuilder().parse(ByteArrayInputStream(manifest.toByteArray())) - - private companion object { - const val DASH_NAMESPACE = "urn:mpeg:dash:schema:mpd:2011" - } -} diff --git a/src/test/kotlin/dev/typetype/server/services/AndroidDashManifestServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/AndroidDashManifestServiceTest.kt deleted file mode 100644 index 3870304a..00000000 --- a/src/test/kotlin/dev/typetype/server/services/AndroidDashManifestServiceTest.kt +++ /dev/null @@ -1,117 +0,0 @@ -package dev.typetype.server.services - -import io.mockk.every -import io.mockk.mockk -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertInstanceOf -import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState - -class AndroidDashManifestServiceTest { - @Test - fun `complete exact indexes produce a full static vod manifest`() { - val fixture = fixture(audioCount = 150, videoCount = 150, segmentMs = 2_000L) - - val result = AndroidDashManifestService().build(fixture.holder) as AndroidDashManifestResult.Ready - - assertEquals(300_000L, result.durationMs) - assertTrue(result.manifest.contains("mediaPresentationDuration=\"PT300.000S\"")) - assertTrue(result.manifest.contains("() - every { state.hasSegmentIndex(audio) } returns true - every { state.hasSegmentIndex(video) } returns true - every { state.getEndSegment(audio) } returns audioCount.toLong() - every { state.getEndSegment(video) } returns videoCount.toLong() - every { state.getSegmentStartMs(any(), any()) } answers { (secondArg() - 1L) * segmentMs } - every { state.getSegmentEndMs(any(), any()) } answers { secondArg() * segmentMs } - val session = mockk { every { streamState } returns state } - val holder = mockk { - every { this@mockk.session } returns session - every { audioFormat } returns audio - every { videoFormat } returns video - every { sessionToken } returns "session-token" - every { activeGeneration() } returns 0L - every { expectsLive() } returns false - } - return Fixture(holder, state, audio, video) - } - - private fun format(itag: Int, audio: Boolean): YoutubeSabrFormat = mockk { - every { this@mockk.itag } returns itag - every { isAudio } returns audio - every { isVideo } returns !audio - every { mimeType } returns if (audio) "audio/mp4; codecs=\"mp4a.40.2\"" else "video/mp4; codecs=\"avc1.640028\"" - every { bitrate } returns if (audio) 128_000 else 2_000_000 - every { width } returns if (audio) 0 else 1920 - every { height } returns if (audio) 0 else 1080 - } - - private data class Fixture( - val holder: SabrSessionHolder, - val state: YoutubeSabrStreamState, - val audio: YoutubeSabrFormat, - val video: YoutubeSabrFormat, - ) -} diff --git a/src/test/kotlin/dev/typetype/server/services/AndroidDashTimelineReaderTest.kt b/src/test/kotlin/dev/typetype/server/services/AndroidDashTimelineReaderTest.kt deleted file mode 100644 index 1f17b316..00000000 --- a/src/test/kotlin/dev/typetype/server/services/AndroidDashTimelineReaderTest.kt +++ /dev/null @@ -1,66 +0,0 @@ -package dev.typetype.server.services - -import io.mockk.every -import io.mockk.mockk -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertInstanceOf -import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState - -class AndroidDashTimelineReaderTest { - private val format = mockk { - every { itag } returns 137 - } - - @Test - fun `unknown exact index stays preparing`() { - val state = mockk { - every { hasSegmentIndex(format) } returns false - } - - assertEquals(AndroidDashTimelineResult.Pending, AndroidDashTimelineReader.read(state, format)) - } - - @Test - fun `exact index preserves complete segment timing`() { - val state = indexedState(count = 3, durationMs = 2_000L) - - val ready = AndroidDashTimelineReader.read(state, format) as AndroidDashTimelineResult.Ready - - assertEquals(1, ready.timeline.startNumber) - assertEquals(3, ready.timeline.segments.size) - assertEquals(0L, ready.timeline.segments.first().startMs) - assertEquals(6_000L, ready.timeline.endMs) - } - - @Test - fun `timeline beginning after zero is rejected`() { - val state = indexedState(count = 2, durationMs = 1_000L, offsetMs = 500L) - - assertInstanceOf( - AndroidDashTimelineResult.Invalid::class.java, - AndroidDashTimelineReader.read(state, format), - ) - } - - @Test - fun `unbounded exact segment count is rejected`() { - val state = mockk { - every { hasSegmentIndex(format) } returns true - every { getEndSegment(format) } returns 100_001L - } - - assertInstanceOf( - AndroidDashTimelineResult.Invalid::class.java, - AndroidDashTimelineReader.read(state, format), - ) - } - - private fun indexedState(count: Int, durationMs: Long, offsetMs: Long = 0L): YoutubeSabrStreamState = mockk { - every { hasSegmentIndex(format) } returns true - every { getEndSegment(format) } returns count.toLong() - every { getSegmentStartMs(format, any()) } answers { offsetMs + (secondArg() - 1L) * durationMs } - every { getSegmentEndMs(format, any()) } answers { offsetMs + secondArg() * durationMs } - } -} diff --git a/src/test/kotlin/dev/typetype/server/services/AndroidPlaybackPreparationTest.kt b/src/test/kotlin/dev/typetype/server/services/AndroidPlaybackPreparationTest.kt deleted file mode 100644 index a2437d64..00000000 --- a/src/test/kotlin/dev/typetype/server/services/AndroidPlaybackPreparationTest.kt +++ /dev/null @@ -1,166 +0,0 @@ -package dev.typetype.server.services - -import io.mockk.every -import io.mockk.mockk -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.ExperimentalCoroutinesApi -import kotlinx.coroutines.awaitCancellation -import kotlinx.coroutines.delay -import kotlinx.coroutines.test.advanceUntilIdle -import kotlinx.coroutines.test.runCurrent -import kotlinx.coroutines.test.runTest -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertInstanceOf -import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState -import java.io.IOException -import java.time.Instant -import java.util.concurrent.atomic.AtomicBoolean - -@OptIn(ExperimentalCoroutinesApi::class) -class AndroidPlaybackPreparationTest { - @Test - fun `one shared preparation publishes the complete manifest`() = runTest { - val fixture = fixture() - val calls = mutableListOf() - val coordinator = coordinator(fixture, this) { _, format, _ -> - calls += format.itag - fixture.ready[format.itag] = true - byteArrayOf(1) - } - - coordinator.start(fixture.session) - coordinator.start(fixture.session) - advanceUntilIdle() - - assertEquals(listOf(137, 140), calls) - assertInstanceOf( - AndroidDashManifestResult.Ready::class.java, - fixture.session.preparation.result(fixture.holder, fixture.manifests), - ) - } - - @Test - fun `preparation deadline becomes a typed temporary failure`() = runTest { - val fixture = fixture() - val coordinator = coordinator(fixture, this, timeoutMs = 100L) { _, _, _ -> - delay(1_000L) - byteArrayOf(1) - } - - coordinator.start(fixture.session) - advanceUntilIdle() - - val result = fixture.session.preparation.result(fixture.holder, fixture.manifests) - as AndroidDashManifestResult.TemporaryFailure - assertEquals("android_playback_preparation_timeout", result.code) - } - - @Test - fun `initialization failure becomes a typed temporary failure`() = runTest { - val fixture = fixture() - val coordinator = coordinator(fixture, this) { _, _, _ -> throw IOException("upstream failed") } - - coordinator.start(fixture.session) - advanceUntilIdle() - - val result = fixture.session.preparation.result(fixture.holder, fixture.manifests) - as AndroidDashManifestResult.TemporaryFailure - assertEquals("android_playback_preparation_failed", result.code) - } - - @Test - fun `session cancellation stops preparation work`() = runTest { - val fixture = fixture() - val cancelled = AtomicBoolean() - val coordinator = coordinator(fixture, this) { _, _, _ -> - try { - awaitCancellation() - } finally { - cancelled.set(true) - } - } - - coordinator.start(fixture.session) - runCurrent() - fixture.session.cancelPreparation() - runCurrent() - - assertTrue(cancelled.get()) - } - - private fun coordinator( - fixture: Fixture, - scope: CoroutineScope, - timeoutMs: Long = 1_000L, - initializer: suspend (SabrSessionHolder, YoutubeSabrFormat, Long) -> ByteArray, - ): AndroidPlaybackPreparationCoordinator = AndroidPlaybackPreparationCoordinator( - store = mockk(relaxed = true), - manifests = fixture.manifests, - timeoutMs = timeoutMs, - initializationTimeoutMs = 500L, - scope = scope, - exactInitializer = initializer, - ) - - private fun fixture(): Fixture { - val audio = format(140, true) - val video = format(137, false) - val ready = mutableMapOf(140 to false, 137 to false) - val state = mockk(relaxed = true) { - every { hasSegmentIndex(any()) } answers { ready[firstArg().itag] == true } - every { getEndSegment(any()) } returns 3L - every { getSegmentStartMs(any(), any()) } answers { (secondArg() - 1L) * 1_000L } - every { getSegmentEndMs(any(), any()) } answers { secondArg() * 1_000L } - } - val sabr = mockk { every { streamState } returns state } - val holder = SabrSessionHolder( - sabr, - mockk(), - audio, - video, - "android-session", - SabrSessionKey( - "video", - "user", - 140, - null, - 137, - 0L, - SabrSessionPurpose.ANDROID_PLAYBACK, - ), - Instant.EPOCH, - ) - return Fixture( - holder, - AndroidPlaybackSession(holder, emptyList()), - AndroidDashManifestService(), - ready, - ) - } - - private fun format(itag: Int, audio: Boolean): YoutubeSabrFormat = mockk { - every { this@mockk.itag } returns itag - every { isAudio } returns audio - every { isVideo } returns !audio - every { audioTrackId } returns null - every { mimeType } returns if (audio) { - "audio/mp4; codecs=\"mp4a.40.2\"" - } else { - "video/mp4; codecs=\"avc1.640028\"" - } - every { bitrate } returns if (audio) 128_000 else 2_000_000 - every { width } returns if (audio) 0 else 1920 - every { height } returns if (audio) 0 else 1080 - } - - private data class Fixture( - val holder: SabrSessionHolder, - val session: AndroidPlaybackSession, - val manifests: AndroidDashManifestService, - val ready: MutableMap, - ) -} diff --git a/src/test/kotlin/dev/typetype/server/services/AndroidPlaybackServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/AndroidPlaybackServiceTest.kt deleted file mode 100644 index 8c07e64c..00000000 --- a/src/test/kotlin/dev/typetype/server/services/AndroidPlaybackServiceTest.kt +++ /dev/null @@ -1,119 +0,0 @@ -package dev.typetype.server.services - -import io.mockk.coVerify -import io.mockk.every -import io.mockk.mockk -import io.mockk.verify -import kotlinx.coroutines.test.runTest -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertSame -import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState -import java.time.Instant - -class AndroidPlaybackServiceTest { - @Test - fun `seek keeps session and complete presentation while advancing generation`() { - val audio = format(140, audio = true) - val video = format(137, audio = false) - val state = state(audio, video) - val holder = holder(audio, video, state) - val store = mockk { - every { startPump(holder) } returns Unit - } - val service = AndroidPlaybackService(store, mockk(relaxed = true)) - - val result = service.seek( - AndroidPlaybackSession(holder, emptyList()), - generation = 0L, - playerTimeMs = 45_000L, - ) as AndroidPlaybackSeekResult.Ready - - assertSame(holder, result.holder) - assertEquals(1L, holder.activeGeneration()) - val manifest = result.manifest as AndroidDashManifestResult.Ready - assertEquals(60_000L, manifest.durationMs) - assertEquals(true, manifest.manifest.contains("(relaxed = true) - val service = AndroidPlaybackService(store, mockk(relaxed = true)) - val prepared = SabrPreparedInfo(mockk(), null, isLive = true) - - val result = service.create( - "video", - "user", - prepared, - format(140, true), - format(137, false), - emptyList(), - ) - - assertEquals(AndroidPlaybackCreateResult.UnsupportedLive, result) - coVerify(exactly = 0) { store.fetchInitializationData(any(), any()) } - verify(exactly = 0) { store.getOrCreate(any(), any(), any(), any(), any(), any(), any(), any(), any(), any()) } - } - - private fun holder( - audio: YoutubeSabrFormat, - video: YoutubeSabrFormat, - state: YoutubeSabrStreamState, - ): SabrSessionHolder { - val session = mockk(relaxed = true) { - every { streamState } returns state - every { getCachedSegment(any()) } returns null - } - return SabrSessionHolder( - session, - mockk(), - audio, - video, - "android-session", - SabrSessionKey("video", "user", 140, null, 137, 0L, SabrSessionPurpose.ANDROID_PLAYBACK), - Instant.EPOCH, - ) - } - - private fun state(audio: YoutubeSabrFormat, video: YoutubeSabrFormat): YoutubeSabrStreamState = mockk(relaxed = true) { - every { hasSegmentIndex(audio) } returns true - every { hasSegmentIndex(video) } returns true - every { getEndSegment(any()) } returns 60L - every { getSegmentStartMs(any(), any()) } answers { (secondArg() - 1L) * 1_000L } - every { getSegmentEndMs(any(), any()) } answers { secondArg() * 1_000L } - every { getSegmentNumberAtOrAfterTimeMs(any(), 45_000L) } returns 46 - every { getSegmentStartMs(any(), 46) } returns 45_000L - every { getMinBufferedEndMs() } returns 0L - } - - private fun format(itag: Int, audio: Boolean): YoutubeSabrFormat = mockk { - every { this@mockk.itag } returns itag - every { isAudio } returns audio - every { isVideo } returns !audio - every { mimeType } returns if (audio) "audio/mp4; codecs=\"mp4a.40.2\"" else "video/mp4; codecs=\"avc1.640028\"" - every { bitrate } returns if (audio) 128_000 else 2_000_000 - every { width } returns if (audio) 0 else 1920 - every { height } returns if (audio) 0 else 1080 - } -} diff --git a/src/test/kotlin/dev/typetype/server/services/AndroidPlaybackSessionRegistryTest.kt b/src/test/kotlin/dev/typetype/server/services/AndroidPlaybackSessionRegistryTest.kt deleted file mode 100644 index 60a3acb7..00000000 --- a/src/test/kotlin/dev/typetype/server/services/AndroidPlaybackSessionRegistryTest.kt +++ /dev/null @@ -1,111 +0,0 @@ -package dev.typetype.server.services - -import io.mockk.every -import io.mockk.mockk -import io.mockk.verify -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertSame -import org.junit.jupiter.api.Test -import java.time.Duration -import java.time.Instant - -class AndroidPlaybackSessionRegistryTest { - @Test - fun `registered Android session resolves and touches the isolated store`() { - val current = MutableNow(Instant.parse("2026-07-20T10:00:00Z")) - val store = mockk() - val holder = holder() - val subtitle = mockk() - every { store.lookupByToken("android-session") } returns holder - val registry = registry(store, current) - registry.register(holder, listOf(subtitle)) - - val result = registry.lookup("android-session") as AndroidPlaybackSessionLookup.Active - - assertSame(holder, result.session.holder) - assertSame(subtitle, result.session.subtitles.single()) - verify(exactly = 1) { store.lookupByToken("android-session") } - } - - @Test - fun `registered subtitle catalog is an immutable snapshot`() { - val current = MutableNow(Instant.parse("2026-07-20T10:00:00Z")) - val store = mockk() - val holder = holder() - val track = mockk() - every { store.lookupByToken("android-session") } returns holder - val source = mutableListOf(track) - val registry = registry(store, current) - registry.register(holder, source) - - source.clear() - val active = registry.lookup("android-session") as AndroidPlaybackSessionLookup.Active - - assertSame(track, active.session.subtitles.single()) - } - - @Test - fun `idle session returns gone while unknown session returns not found`() { - val current = MutableNow(Instant.parse("2026-07-20T10:00:00Z")) - val store = mockk(relaxed = true) - val holder = holder() - val registry = registry(store, current) - registry.register(holder, emptyList()) - current.value = current.value.plus(Duration.ofMinutes(5)) - - assertEquals(AndroidPlaybackSessionLookup.Expired, registry.lookup("android-session")) - assertEquals(AndroidPlaybackSessionLookup.Unknown, registry.lookup("never-created")) - verify(exactly = 1) { store.release(holder) } - } - - @Test - fun `expired session tombstone is temporary`() { - val current = MutableNow(Instant.parse("2026-07-20T10:00:00Z")) - val store = mockk(relaxed = true) - val registry = registry(store, current) - registry.register(holder(), emptyList()) - current.value = current.value.plus(Duration.ofMinutes(5)) - assertEquals(AndroidPlaybackSessionLookup.Expired, registry.lookup("android-session")) - - current.value = current.value.plus(Duration.ofMinutes(11)) - - assertEquals(AndroidPlaybackSessionLookup.Unknown, registry.lookup("android-session")) - } - - @Test - fun `missing backing Android session becomes expired`() { - val current = MutableNow(Instant.parse("2026-07-20T10:00:00Z")) - val store = mockk(relaxed = true) - val holder = holder() - every { store.lookupByToken("android-session") } returns null - val registry = registry(store, current) - registry.register(holder, emptyList()) - - assertEquals(AndroidPlaybackSessionLookup.Expired, registry.lookup("android-session")) - verify(exactly = 1) { store.release(holder) } - } - - private fun registry(store: SabrSessionStore, current: MutableNow) = AndroidPlaybackSessionRegistry( - store = store, - idleTimeout = Duration.ofMinutes(4), - tombstoneTtl = Duration.ofMinutes(10), - now = current::get, - ) - - private fun holder(): SabrSessionHolder = mockk { - every { sessionToken } returns "android-session" - every { key } returns SabrSessionKey( - "video", - "user", - 140, - null, - 137, - 0L, - SabrSessionPurpose.ANDROID_PLAYBACK, - ) - } - - private class MutableNow(var value: Instant) { - fun get(): Instant = value - } -} diff --git a/src/test/kotlin/dev/typetype/server/services/AndroidSubtitleHttpClientTest.kt b/src/test/kotlin/dev/typetype/server/services/AndroidSubtitleHttpClientTest.kt deleted file mode 100644 index ca564889..00000000 --- a/src/test/kotlin/dev/typetype/server/services/AndroidSubtitleHttpClientTest.kt +++ /dev/null @@ -1,104 +0,0 @@ -package dev.typetype.server.services - -import kotlinx.coroutines.test.runTest -import okhttp3.MediaType.Companion.toMediaType -import okhttp3.OkHttpClient -import okhttp3.Protocol -import okhttp3.Request -import okhttp3.Response -import okhttp3.ResponseBody.Companion.toResponseBody -import okhttp3.HttpUrl.Companion.toHttpUrl -import org.junit.jupiter.api.Assertions.assertArrayEquals -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Test -import java.util.concurrent.atomic.AtomicInteger - -class AndroidSubtitleHttpClientTest { - @Test - fun `HTTP 429 falls back to direct egress and requests WebVTT`() = runTest { - val primaryCalls = AtomicInteger() - val directCalls = AtomicInteger() - val primary = client { - primaryCalls.incrementAndGet() - response(it, 429, "") - } - val direct = client { - directCalls.incrementAndGet() - assertEquals("vtt", it.url.queryParameter("fmt")) - assertEquals("text/vtt", it.header("Accept")) - response(it, 200, VTT.decodeToString()) - } - - val result = AndroidSubtitleHttpClient(primary, direct).fetch(SOURCE_URL) - - assertArrayEquals(VTT, (result as AndroidSubtitleUpstreamResult.Ready).bytes) - assertEquals(1, primaryCalls.get()) - assertEquals(1, directCalls.get()) - } - - @Test - fun `permanent upstream rejection does not retry another egress`() = runTest { - val directCalls = AtomicInteger() - val primary = client { response(it, 404, "") } - val direct = client { - directCalls.incrementAndGet() - response(it, 200, VTT.decodeToString()) - } - - val result = AndroidSubtitleHttpClient(primary, direct).fetch(SOURCE_URL) - - assertEquals(AndroidSubtitleUpstreamResult.Unavailable, result) - assertEquals(0, directCalls.get()) - } - - @Test - fun `invalid successful body is not exposed as WebVTT`() = runTest { - val client = client { response(it, 200, "") } - - val result = AndroidSubtitleHttpClient(client, client).fetch(SOURCE_URL) - - assertEquals(AndroidSubtitleUpstreamResult.Unavailable, result) - } - - @Test - fun `temporary failure on both egress paths remains typed`() = runTest { - val client = client { response(it, 503, "") } - - val result = AndroidSubtitleHttpClient(client, client).fetch(SOURCE_URL) - - assertEquals(AndroidSubtitleUpstreamResult.TemporaryFailure, result) - } - - @Test - fun `temporary failure is not duplicated without a configured proxy`() = runTest { - val calls = AtomicInteger() - val client = client { - calls.incrementAndGet() - response(it, 429, "") - } - - val result = AndroidSubtitleHttpClient(client, null).fetch(SOURCE_URL) - - assertEquals(AndroidSubtitleUpstreamResult.TemporaryFailure, result) - assertEquals(1, calls.get()) - } - - private fun client(responder: (Request) -> Response): OkHttpClient = - OkHttpClient.Builder() - .addInterceptor { chain -> responder(chain.request()) } - .build() - - private fun response(request: Request, status: Int, body: String): Response = - Response.Builder() - .request(request) - .protocol(Protocol.HTTP_1_1) - .code(status) - .message("test") - .body(body.toResponseBody("text/vtt".toMediaType())) - .build() - - private companion object { - val SOURCE_URL = "https://www.youtube.com/api/timedtext?v=dQw4w9WgXcQ&fmt=ttml".toHttpUrl() - val VTT = "WEBVTT\n\n00:00.000 --> 00:01.000\nHello\n".toByteArray() - } -} diff --git a/src/test/kotlin/dev/typetype/server/services/AndroidSubtitleInventoryCoordinatorTest.kt b/src/test/kotlin/dev/typetype/server/services/AndroidSubtitleInventoryCoordinatorTest.kt deleted file mode 100644 index 23a44f04..00000000 --- a/src/test/kotlin/dev/typetype/server/services/AndroidSubtitleInventoryCoordinatorTest.kt +++ /dev/null @@ -1,79 +0,0 @@ -package dev.typetype.server.services - -import io.mockk.coEvery -import io.mockk.coVerify -import io.mockk.mockk -import kotlinx.coroutines.CompletableDeferred -import kotlinx.coroutines.ExperimentalCoroutinesApi -import kotlinx.coroutines.async -import kotlinx.coroutines.test.runCurrent -import kotlinx.coroutines.test.runTest -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertSame -import org.junit.jupiter.api.Test - -@OptIn(ExperimentalCoroutinesApi::class) -class AndroidSubtitleInventoryCoordinatorTest { - @Test - fun `concurrent requests share one in flight inventory extraction`() = runTest { - val release = CompletableDeferred() - val track = mockk() - val service = mockk() - coEvery { service.inventory(VIDEO_ID) } coAnswers { - release.await() - AndroidSubtitleInventoryResult.Ready(listOf(track)) - } - val coordinator = AndroidSubtitleInventoryCoordinator(service, this) - - val first = coordinator.start(VIDEO_ID) - val second = coordinator.start(VIDEO_ID) - runCurrent() - - assertSame(first, second) - assertEquals(AndroidSubtitleInventorySnapshot.Preparing, first.snapshot()) - release.complete(Unit) - val completed = async { first.await() }.await() as AndroidSubtitleInventorySnapshot.Ready - assertSame(track, completed.tracks.single()) - coVerify(exactly = 1) { service.inventory(VIDEO_ID) } - } - - @Test - fun `completed inventory is reused until its cache entry expires`() = runTest { - var now = 1_000L - val service = mockk() - coEvery { service.inventory(VIDEO_ID) } returns AndroidSubtitleInventoryResult.Ready(emptyList()) - val coordinator = AndroidSubtitleInventoryCoordinator( - service, - this, - cacheTtlMs = 500L, - nowMs = { now }, - ) - - val first = coordinator.start(VIDEO_ID) - runCurrent() - assertEquals(AndroidSubtitleInventorySnapshot.Ready(emptyList()), first.await()) - assertSame(first, coordinator.start(VIDEO_ID)) - now += 501L - val refreshed = coordinator.start(VIDEO_ID) - runCurrent() - - refreshed.await() - coVerify(exactly = 2) { service.inventory(VIDEO_ID) } - } - - @Test - fun `unavailable inventory completes with a typed temporary failure`() = runTest { - val service = mockk() - coEvery { service.inventory(VIDEO_ID) } returns AndroidSubtitleInventoryResult.Unavailable - val coordinator = AndroidSubtitleInventoryCoordinator(service, this) - - val result = coordinator.start(VIDEO_ID) - runCurrent() - - assertEquals(AndroidSubtitleInventorySnapshot.TemporaryFailure, result.await()) - } - - private companion object { - const val VIDEO_ID = "dQw4w9WgXcQ" - } -} diff --git a/src/test/kotlin/dev/typetype/server/services/AndroidSubtitleServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/AndroidSubtitleServiceTest.kt deleted file mode 100644 index a996a2b9..00000000 --- a/src/test/kotlin/dev/typetype/server/services/AndroidSubtitleServiceTest.kt +++ /dev/null @@ -1,83 +0,0 @@ -package dev.typetype.server.services - -import dev.typetype.server.models.SubtitleItem -import io.mockk.coEvery -import io.mockk.coVerify -import io.mockk.mockk -import kotlinx.coroutines.awaitCancellation -import kotlinx.coroutines.test.runTest -import org.junit.jupiter.api.Assertions.assertArrayEquals -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Test - -class AndroidSubtitleServiceTest { - @Test - fun `inventory exposes all Token tracks without fetching slow subtitle content`() = runTest { - val source = mockk() - val http = mockk() - val items = listOf(item("en", false), item("en", true), item("fr", false)) - coEvery { source.fetchSubtitleInventory(VIDEO_ID) } returns YouTubeSubtitleInventoryResult.Ready(items) - coEvery { http.fetch(any()) } coAnswers { awaitCancellation() } - val service = AndroidSubtitleService(source, http) - - val result = service.inventory(VIDEO_ID) as AndroidSubtitleInventoryResult.Ready - - assertEquals(3, result.tracks.size) - assertEquals(3, result.tracks.map { it.id }.distinct().size) - coVerify(exactly = 0) { http.fetch(any()) } - } - - @Test - fun `temporary upstream failure refreshes inventory once`() = runTest { - val source = mockk() - val http = mockk() - val oldTrack = track(item("en", false, "signature=old")) - val refreshedItem = item("en", false, "signature=new") - val refreshedTrack = track(refreshedItem) - coEvery { http.fetch(oldTrack.sourceUrl) } returns AndroidSubtitleUpstreamResult.TemporaryFailure - coEvery { source.fetchSubtitleInventory(VIDEO_ID) } returns - YouTubeSubtitleInventoryResult.Ready(listOf(refreshedItem)) - coEvery { http.fetch(refreshedTrack.sourceUrl) } returns AndroidSubtitleUpstreamResult.Ready(VTT) - val service = AndroidSubtitleService(source, http) - - val result = service.content(VIDEO_ID, oldTrack) as AndroidSubtitleContentResult.Ready - - assertArrayEquals(VTT, result.bytes) - coVerify(exactly = 1) { source.fetchSubtitleInventory(VIDEO_ID) } - } - - @Test - fun `repeated temporary failure remains typed and bounded`() = runTest { - val source = mockk() - val http = mockk() - val track = track(item("en", false)) - coEvery { http.fetch(any()) } returns AndroidSubtitleUpstreamResult.TemporaryFailure - coEvery { source.fetchSubtitleInventory(VIDEO_ID) } returns - YouTubeSubtitleInventoryResult.Ready(listOf(item("en", false))) - val service = AndroidSubtitleService(source, http) - - assertEquals(AndroidSubtitleContentResult.TemporaryFailure, service.content(VIDEO_ID, track)) - coVerify(exactly = 2) { http.fetch(any()) } - coVerify(exactly = 1) { source.fetchSubtitleInventory(VIDEO_ID) } - } - - private fun track(item: SubtitleItem): AndroidSubtitleTrack = - AndroidSubtitleTrackFactory.create(VIDEO_ID, listOf(item))!!.single() - - private fun item( - language: String, - auto: Boolean, - extra: String = "signature=test", - ) = SubtitleItem( - url = "https://www.youtube.com/api/timedtext?v=$VIDEO_ID&lang=$language&kind=${if (auto) "asr" else ""}&$extra", - mimeType = "application/ttml+xml", - languageTag = language, - displayLanguageName = language, - isAutoGenerated = auto, - ) - - private companion object { - const val VIDEO_ID = "dQw4w9WgXcQ" - val VTT = "WEBVTT\n\n00:00.000 --> 00:01.000\nHello\n".toByteArray() - } -} diff --git a/src/test/kotlin/dev/typetype/server/services/AndroidSubtitleTrackFactoryTest.kt b/src/test/kotlin/dev/typetype/server/services/AndroidSubtitleTrackFactoryTest.kt deleted file mode 100644 index b1fc14dc..00000000 --- a/src/test/kotlin/dev/typetype/server/services/AndroidSubtitleTrackFactoryTest.kt +++ /dev/null @@ -1,66 +0,0 @@ -package dev.typetype.server.services - -import dev.typetype.server.models.SubtitleItem -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertNotEquals -import org.junit.jupiter.api.Assertions.assertNull -import org.junit.jupiter.api.Test - -class AndroidSubtitleTrackFactoryTest { - @Test - fun `track identity ignores expiring upstream parameters`() { - val first = track(url("signature=old&expire=1"), "English") - val refreshed = track(url("signature=new&expire=2"), "Anglais") - - assertEquals(first.id, refreshed.id) - } - - @Test - fun `manual and generated tracks in one language remain distinct`() { - val tracks = AndroidSubtitleTrackFactory.create( - VIDEO_ID, - listOf( - item(url("vssId=.en"), auto = false), - item(url("vssId=a.en&kind=asr"), auto = true), - ), - )!! - - assertEquals(2, tracks.size) - assertNotEquals(tracks[0].id, tracks[1].id) - } - - @Test - fun `non YouTube caption URL rejects the inventory`() { - val tracks = AndroidSubtitleTrackFactory.create( - VIDEO_ID, - listOf(item("https://example.com/subtitles.vtt", auto = false)), - ) - - assertNull(tracks) - } - - private fun track(sourceUrl: String, displayName: String): AndroidSubtitleTrack = - AndroidSubtitleTrackFactory.create( - VIDEO_ID, - listOf(item(sourceUrl, false, displayName)), - )!!.single() - - private fun item( - sourceUrl: String, - auto: Boolean, - displayName: String = "English", - ) = SubtitleItem( - url = sourceUrl, - mimeType = "application/ttml+xml", - languageTag = "en", - displayLanguageName = displayName, - isAutoGenerated = auto, - ) - - private fun url(extra: String): String = - "https://www.youtube.com/api/timedtext?v=$VIDEO_ID&lang=en&$extra" - - private companion object { - const val VIDEO_ID = "dQw4w9WgXcQ" - } -} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrDownloadFrameWriterTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrDownloadFrameWriterTest.kt new file mode 100644 index 00000000..542b20ed --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/SabrDownloadFrameWriterTest.kt @@ -0,0 +1,56 @@ +package dev.typetype.server.services + +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.Assertions.assertArrayEquals +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertThrows +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream +import java.nio.ByteBuffer +import java.nio.ByteOrder + +class SabrDownloadFrameWriterTest { + @Test + fun `writes deterministic framed stream`() { + val output = ByteArrayOutputStream() + val writer = SabrDownloadFrameWriter(output) + + writer.start() + writer.initialization(140, byteArrayOf(1, 2)) + writer.media(140, 1, 3, ByteArrayInputStream(byteArrayOf(3, 4, 5))) + writer.finish() + + val input = ByteBuffer.wrap(output.toByteArray()).order(ByteOrder.BIG_ENDIAN) + val magic = ByteArray(SabrDownloadFrameWriter.MAGIC.size) + input.get(magic) + assertArrayEquals(SabrDownloadFrameWriter.MAGIC, magic) + assertFrame(input, SabrDownloadFrameWriter.FRAME_INITIALIZATION, 140, 0, byteArrayOf(1, 2)) + assertFrame(input, SabrDownloadFrameWriter.FRAME_MEDIA, 140, 1, byteArrayOf(3, 4, 5)) + assertFrame(input, SabrDownloadFrameWriter.FRAME_COMPLETE, 0, 0, byteArrayOf()) + assertEquals(0, input.remaining()) + } + + @Test + fun `rejects payload shorter than declared frame`() { + val writer = SabrDownloadFrameWriter(ByteArrayOutputStream()) + assertThrows(IllegalStateException::class.java) { + writer.media(137, 1, 3, ByteArrayInputStream(byteArrayOf(1, 2))) + } + } + + private fun assertFrame( + input: ByteBuffer, + expectedType: Int, + expectedItag: Int, + expectedSequence: Int, + expectedPayload: ByteArray, + ) { + assertEquals(expectedType, input.get().toInt()) + assertEquals(expectedItag, input.int) + assertEquals(expectedSequence, input.int) + assertEquals(expectedPayload.size.toLong(), input.long) + val payload = ByteArray(expectedPayload.size) + input.get(payload) + assertArrayEquals(expectedPayload, payload) + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrDownloadRangeTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrDownloadRangeTest.kt new file mode 100644 index 00000000..3cea29a5 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/SabrDownloadRangeTest.kt @@ -0,0 +1,60 @@ +package dev.typetype.server.services + +import io.mockk.every +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState + +class SabrDownloadRangeTest { + @Test + fun `adjacent parts share an exclusive segment boundary`() { + val state = mockk() + val format = format(durationMs = 1_000L) + every { state.getSegmentNumberAtOrAfterTimeMs(format, 250L) } returns 26 + every { state.getSegmentNumberAtOrAfterTimeMs(format, 500L) } returns 51 + + val first = SabrDownloadRange(part = 0, parts = 4) + val second = SabrDownloadRange(part = 1, parts = 4) + + assertEquals(1, first.startSequence(state, format)) + assertEquals(26, first.endSequenceExclusive(state, format)) + assertEquals(26, second.startSequence(state, format)) + assertEquals(51, second.endSequenceExclusive(state, format)) + } + + @Test + fun `single part keeps natural stream completion`() { + val state = mockk() + val format = format(durationMs = 0L) + val range = SabrDownloadRange() + + assertEquals(1, range.startSequence(state, format)) + assertNull(range.endSequenceExclusive(state, format)) + } + + @Test + fun `part start time follows selected duration`() { + val audio = format(durationMs = 1_800_000L) + val video = format(durationMs = 900_000L) + val range = SabrDownloadRange(part = 2, parts = 4) + + assertEquals(900_000L, range.startTimeMs(audio, video, audioOnly = true)) + assertEquals(900_000L, range.startTimeMs(audio, video, audioOnly = false)) + } + + @Test + fun `part count is bounded`() { + SabrDownloadRange(part = 11, parts = 12) + assertThrows(IllegalArgumentException::class.java) { + SabrDownloadRange(part = 0, parts = 13) + } + } + + private fun format(durationMs: Long): YoutubeSabrFormat = mockk { + every { approxDurationMs } returns durationMs + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrDownloadStreamerTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrDownloadStreamerTest.kt new file mode 100644 index 00000000..c6c9225c --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/SabrDownloadStreamerTest.kt @@ -0,0 +1,235 @@ +package dev.typetype.server.services + +import io.mockk.coEvery +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.Assertions.assertArrayEquals +import org.junit.jupiter.api.Assertions.assertEquals +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream +import java.io.IOException + +class SabrDownloadStreamerTest { + @Test + fun `streams cached track in order and discards each segment`() = runTest { + val store = mockk() + val holder = mockk() + val session = mockk() + val state = mockk() + val audio = format(140) + val media = segment(byteArrayOf(3, 4, 5)) + val cached = mutableMapOf(1 to media) + every { holder.session } returns session + every { holder.audioFormat } returns audio + every { holder.isAudioActive() } returns true + every { holder.isVideoActive() } returns false + every { holder.key } returns SabrSessionKey("video", "user", 140, null, 137, 0, SabrSessionPurpose.DOWNLOAD) + every { holder.playerContextToken } returns null + every { store.initCache } returns null + every { session.streamState } returns state + every { state.poToken } returns null + every { state.getSegmentNumberAtOrAfterTimeMs(audio, 0) } returns 1 + every { state.isComplete(audio) } answers { cached.isEmpty() } + every { session.isBeyondEnd(any()) } returns false + every { session.cachedBytes } returns 0 + every { session.totalResponseBytes } returns 3 + every { session.maxResponseBytes } returns 3 + every { session.getCachedSegment(any()) } answers { + val request = firstArg() + cached[request.sequenceNumber] + } + every { session.discardCachedSegment(any()) } answers { + cached.remove(firstArg().sequenceNumber) + } + coEvery { store.fetchInitializationData(holder, audio) } returns byteArrayOf(1, 2) + val output = ByteArrayOutputStream() + + SabrDownloadStreamer(store).stream(holder, output) + + val expected = ByteArrayOutputStream().also { + val writer = SabrDownloadFrameWriter(it) + writer.start() + writer.initialization(140, byteArrayOf(1, 2)) + writer.media(140, 1, 3, ByteArrayInputStream(byteArrayOf(3, 4, 5))) + writer.finish() + }.toByteArray() + assertArrayEquals(expected, output.toByteArray()) + verify { session.discardCachedSegment(match { it.sequenceNumber == 1 }) } + } + + @Test + fun `discards cached media before multipart boundary`() = runTest { + val store = mockk() + val holder = mockk() + val session = mockk() + val state = mockk() + val audio = format(140) + val stale = segment(byteArrayOf(9)) + val current = segment(byteArrayOf(3, 4)) + val cached = mutableMapOf(1 to stale, 5 to current) + every { holder.session } returns session + every { holder.audioFormat } returns audio + every { holder.isAudioActive() } returns true + every { holder.isVideoActive() } returns false + every { holder.key } returns SabrSessionKey( + "video", + "user", + 140, + null, + 137, + 50, + SabrSessionPurpose.DOWNLOAD, + ) + every { holder.playerContextToken } returns null + every { store.initCache } returns null + every { session.streamState } returns state + every { state.poToken } returns null + every { state.getSegmentNumberAtOrAfterTimeMs(audio, 50) } returns 5 + every { state.getEndSegment(audio) } returns 5 + every { session.isBeyondEnd(any()) } returns false + every { session.cachedBytes } returns 0 + every { session.totalResponseBytes } returns 2 + every { session.maxResponseBytes } returns 2 + every { session.getCachedSegment(any()) } answers { + cached[firstArg().sequenceNumber] + } + every { session.discardCachedSegment(any()) } answers { + cached.remove(firstArg().sequenceNumber) + } + coEvery { store.fetchInitializationData(holder, audio) } returns byteArrayOf(1, 2) + val output = ByteArrayOutputStream() + + SabrDownloadStreamer(store).stream(holder, output, SabrDownloadRange(part = 1, parts = 2)) + + verify { session.discardCachedSegment(match { it.sequenceNumber == 1 }) } + verify { session.discardCachedSegment(match { it.sequenceNumber == 5 }) } + } + + @Test + fun `positions SABR state before pumping a missing segment`() = runTest { + val store = mockk() + val holder = mockk() + val session = mockk() + val state = mockk() + val audio = format(140) + val media = segment(byteArrayOf(3, 4)) + val cached = mutableMapOf() + var complete = false + every { holder.session } returns session + every { holder.audioFormat } returns audio + every { holder.isAudioActive() } returns true + every { holder.isVideoActive() } returns false + every { holder.key } returns SabrSessionKey( + "video", + "user", + 140, + null, + 137, + 50, + SabrSessionPurpose.DOWNLOAD, + ) + every { holder.playerContextToken } returns null + every { store.initCache } returns null + every { session.streamState } returns state + every { state.poToken } returns null + every { state.getSegmentNumberAtOrAfterTimeMs(audio, 50) } returns 5 + every { state.getSegmentStartMs(audio, 5) } returns 50 + every { state.getEndSegment(audio) } returns 5 + every { state.isComplete(audio) } answers { complete } + every { session.isBeyondEnd(any()) } returns false + every { session.cachedBytes } returns 0 + every { session.totalResponseBytes } returns 2 + every { session.maxResponseBytes } returns 2 + every { session.diagnosticTrace } returns "" + every { session.getCachedSegment(any()) } answers { + cached[firstArg().sequenceNumber] + } + every { session.prepareForForwardJump(any(), any()) } returns Unit + every { session.pumpOnceStreamingUntilCached(any(), any()) } answers { + cached[5] = media + 1 + } + every { session.discardCachedSegment(any()) } answers { + if (cached.remove(firstArg().sequenceNumber) != null) complete = true + } + coEvery { store.fetchInitializationData(holder, audio) } returns byteArrayOf(1, 2) + + SabrDownloadStreamer(store).stream( + holder, + ByteArrayOutputStream(), + SabrDownloadRange(part = 1, parts = 2), + ) + + verify { session.prepareForForwardJump(match { it.sequenceNumber == 5 }, 50) } + } + + @Test + fun `fails when an upstream pump stops responding`() = runTest { + val store = mockk() + val holder = mockk() + val session = mockk() + val state = mockk() + val audio = format(140) + every { holder.session } returns session + every { holder.audioFormat } returns audio + every { holder.isAudioActive() } returns true + every { holder.isVideoActive() } returns false + every { holder.key } returns SabrSessionKey( + "video", + "user", + 140, + null, + 137, + 0, + SabrSessionPurpose.DOWNLOAD, + ) + every { holder.playerContextToken } returns null + every { store.initCache } returns null + every { session.streamState } returns state + every { state.poToken } returns null + every { state.getSegmentNumberAtOrAfterTimeMs(audio, 0) } returns 1 + every { state.getSegmentStartMs(audio, 1) } returns 0 + every { state.isComplete(audio) } returns false + every { session.isBeyondEnd(any()) } returns false + every { session.cachedBytes } returns 0 + every { session.getCachedSegment(any()) } returns null + every { session.discardCachedSegment(any()) } returns Unit + every { session.prepareForForwardJump(any(), any()) } returns Unit + every { session.pumpOnceStreamingUntilCached(any(), any()) } answers { + Thread.sleep(10_000) + 0 + } + coEvery { store.fetchInitializationData(holder, audio) } returns byteArrayOf(1, 2) + + val error = try { + SabrDownloadStreamer(store, pumpTimeoutMs = 25).stream(holder, ByteArrayOutputStream()) + null + } catch (caught: IOException) { + caught + } + + assertEquals("SABR download upstream pump timed out", error?.message) + } + + private fun format(itag: Int): YoutubeSabrFormat = mockk { + every { this@mockk.itag } returns itag + every { lastModified } returns 0 + every { xtags } returns null + every { mimeType } returns "audio/mp4" + every { audioTrackId } returns null + every { approxDurationMs } returns 100 + } + + private fun segment(data: ByteArray): SabrMediaSegment = mockk { + every { length } returns data.size + every { openStream() } answers { ByteArrayInputStream(data) } + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrExactInitializationDataTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrExactInitializationDataTest.kt deleted file mode 100644 index 59df2907..00000000 --- a/src/test/kotlin/dev/typetype/server/services/SabrExactInitializationDataTest.kt +++ /dev/null @@ -1,74 +0,0 @@ -package dev.typetype.server.services - -import io.mockk.every -import io.mockk.mockk -import io.mockk.verify -import kotlinx.coroutines.test.runTest -import org.junit.jupiter.api.Assertions.assertArrayEquals -import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.localization.Localization -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState -import java.time.Instant -import java.util.concurrent.atomic.AtomicBoolean - -class SabrExactInitializationDataTest { - @Test - fun `exact Android initialization uses the video-bound token and publishes the index`() = runTest { - val audio = format(140, true) - val video = format(137, false) - val token = byteArrayOf(1, 2, 3) - val bytes = byteArrayOf(4, 5, 6) - val indexed = AtomicBoolean() - val state = mockk(relaxed = true) { - every { poToken } returns token - every { hasSegmentIndex(video) } answers { indexed.get() } - } - val sabr = mockk(relaxed = true) { - every { streamState } returns state - every { - fetchInitializationData(video, any(), 3_500L, match { it.contentEquals(token) }) - } answers { - indexed.set(true) - bytes - } - } - val holder = SabrSessionHolder( - sabr, - mockk(), - audio, - video, - "android-session", - SabrSessionKey( - "exact-init-video", - "user", - 140, - null, - 137, - 0L, - SabrSessionPurpose.ANDROID_PLAYBACK, - ), - Instant.EPOCH, - ) - - val result = SabrInitializationData.fetchExact(holder, video, 3_500L) - - assertArrayEquals(bytes, result) - assertTrue(state.hasSegmentIndex(video)) - verify(exactly = 1) { - sabr.fetchInitializationData(video, any(), 3_500L, match { it.contentEquals(token) }) - } - } - - private fun format(itag: Int, audio: Boolean): YoutubeSabrFormat = mockk { - every { this@mockk.itag } returns itag - every { isAudio } returns audio - every { lastModified } returns 1L - every { xtags } returns null - every { mimeType } returns if (audio) "audio/mp4" else "video/mp4" - every { audioTrackId } returns null - } -} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackDeferredPreparationTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackDeferredPreparationTest.kt deleted file mode 100644 index 00f055b5..00000000 --- a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackDeferredPreparationTest.kt +++ /dev/null @@ -1,75 +0,0 @@ -package dev.typetype.server.services - -import io.mockk.coVerify -import io.mockk.every -import io.mockk.mockk -import io.mockk.verify -import kotlinx.coroutines.test.runTest -import org.junit.jupiter.api.Assertions.assertSame -import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState -import java.time.Instant - -class SabrPlaybackDeferredPreparationTest { - @Test - fun `android preparation can defer initialization and pump startup`() = runTest { - val audio = mockk { every { itag } returns 140 } - val video = mockk { every { itag } returns 137 } - val state = mockk(relaxed = true) - val session = mockk(relaxed = true) { every { streamState } returns state } - val holder = SabrSessionHolder( - session, - mockk(), - audio, - video, - "android-session", - SabrSessionKey( - "video", - "user", - 140, - null, - 137, - 0L, - SabrSessionPurpose.ANDROID_PLAYBACK, - ), - Instant.EPOCH, - ) - val info = mockk() - val prepared = SabrPreparedInfo(info, null) - val store = mockk { - every { - getOrCreate( - "video", - "user", - info, - audio, - video, - null, - 0L, - false, - SabrSessionPurpose.ANDROID_PLAYBACK, - false, - 0L, - ) - } returns holder - } - - val result = SabrPlaybackSessionService(store, SabrSessionPurpose.ANDROID_PLAYBACK).prepare( - "video", - "user", - prepared, - audio, - video, - 0L, - preloadInitialization = false, - startPumpOnPrepare = false, - ) - - assertSame(holder, result.holder) - coVerify(exactly = 0) { store.fetchInitializationData(any(), any()) } - verify(exactly = 0) { store.startPump(any()) } - } -} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionIsolationTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionIsolationTest.kt index bfab455b..e36b8f97 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionIsolationTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionIsolationTest.kt @@ -2,9 +2,11 @@ package dev.typetype.server.services import io.mockk.every import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertNotEquals import org.junit.jupiter.api.Assertions.assertNotSame import org.junit.jupiter.api.Assertions.assertSame import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrClientProfile import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo @@ -43,6 +45,7 @@ class SabrPlaybackSessionIsolationTest { ) assertNotSame(first, second) + assertNotEquals(first.info.cpn, second.info.cpn) assertSame(first, store.lookupByToken(first.sessionToken)) assertSame(second, store.lookupByToken(second.sessionToken)) store.release() @@ -51,8 +54,13 @@ class SabrPlaybackSessionIsolationTest { private fun info(): YoutubeSabrInfo { val info = mockk() every { info.videoId } returns "video" + every { info.profile } returns YoutubeSabrClientProfile.WEB + every { info.clientVersion } returns "1.2.3" + every { info.cpn } returns "source-cpn" every { info.visitorData } returns "visitor" every { info.serverAbrStreamingUrl } returns "https://example.com/sabr" + every { info.videoPlaybackUstreamerConfig } returns "config" + every { info.formats } returns emptyList() return info } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSessionIdentityTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSessionIdentityTest.kt new file mode 100644 index 00000000..1545d8a8 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/SabrSessionIdentityTest.kt @@ -0,0 +1,36 @@ +package dev.typetype.server.services + +import io.mockk.every +import io.mockk.mockk +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotEquals +import org.junit.jupiter.api.Assertions.assertSame +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrClientProfile +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo + +class SabrSessionIdentityTest { + @Test + fun `fresh identities use unique coherent playback nonces`() { + val formats = listOf(mockk()) + val source = mockk() + every { source.profile } returns YoutubeSabrClientProfile.WEB + every { source.videoId } returns "video" + every { source.clientVersion } returns "1.2.3" + every { source.visitorData } returns "visitor" + every { source.serverAbrStreamingUrl } returns "https://example.com/sabr?cpn=stale&foo=bar" + every { source.videoPlaybackUstreamerConfig } returns "config" + every { source.formats } returns formats + + val first = SabrSessionIdentity.fresh(source) + val second = SabrSessionIdentity.fresh(source) + + assertNotEquals(first.cpn, second.cpn) + assertEquals(first.cpn, first.serverAbrStreamingUrl!!.toHttpUrl().queryParameter("cpn")) + assertEquals(second.cpn, second.serverAbrStreamingUrl!!.toHttpUrl().queryParameter("cpn")) + assertEquals("bar", first.serverAbrStreamingUrl!!.toHttpUrl().queryParameter("foo")) + assertSame(formats[0], first.formats[0]) + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecoveryTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecoveryTest.kt index 8e87c440..e398e8b6 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecoveryTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecoveryTest.kt @@ -45,9 +45,25 @@ class SabrUnauthorizedResponseRecoveryTest { verify(exactly = 1) { holder.playerContextToken = refreshed } } + @Test + fun refreshesAfterUpstreamHttp401(): Unit { + val freshToken = byteArrayOf(2) + val refreshed = bundle("visitor-a", freshToken) + val (holder, state) = holder( + visitorData = "visitor-a", + currentToken = byteArrayOf(1), + responseStatus = 401, + ) + + SabrUnauthorizedResponseRecovery { refreshed }.verify(holder) + + verify(exactly = 1) { state.setPoToken(match { it.contentEquals(freshToken) }) } + } + private fun holder( visitorData: String, currentToken: ByteArray, + responseStatus: Int = 403, ): Pair { val state = mockk(relaxed = true) val session = mockk(relaxed = true) @@ -55,7 +71,7 @@ class SabrUnauthorizedResponseRecoveryTest { val holder = mockk(relaxed = true) every { state.poToken } returns currentToken every { session.streamState } returns state - every { session.diagnosticTrace } returns "response n=4 http=403 segments=count=0" + every { session.diagnosticTrace } returns "response n=4 http=$responseStatus segments=count=0" every { info.videoId } returns "video" every { info.visitorData } returns visitorData every { holder.session } returns session diff --git a/src/test/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClientTest.kt b/src/test/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClientTest.kt index 15694f8a..a19abd2f 100644 --- a/src/test/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClientTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClientTest.kt @@ -36,6 +36,26 @@ class TypetypeTokenYoutubeSessionClientTest { assertEquals("MWEB", requestedClient) } + @Test + fun `requests isolated playback material for download sessions`() = runTest { + var isolated: String? = null + val httpClient = OkHttpClient.Builder().addInterceptor(Interceptor { chain -> + isolated = chain.request().url.queryParameter("isolated") + Response.Builder() + .request(chain.request()) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body("{}".toResponseBody()) + .build() + }).build() + + TypetypeTokenYoutubeSessionClient("https://token.example", httpClient) + .fetchPlaybackSession("video-id", isolated = true) + + assertEquals("true", isolated) + } + @Test fun `reconstructs pipepipe sabr info from token session`() = runTest { val body = """ diff --git a/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamServiceTest.kt index 9615412e..be2e9474 100644 --- a/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamServiceTest.kt @@ -19,19 +19,19 @@ class YoutubePlayerClientFallbackStreamServiceTest { override suspend fun getStreamInfo(url: String): ExtractionResult { val client = NewPipe.getYoutubePlayerClient() observed += client - return if (client == YoutubePlayerClient.ANDROID_VR.value) success else ExtractionResult.Failure("blocked") + return if (client == YoutubePlayerClient.TV_DOWNGRADED.value) success else ExtractionResult.Failure("blocked") } } val service = YoutubePlayerClientFallbackStreamService( delegate, - listOf(YoutubePlayerClient.WEB_SAFARI, YoutubePlayerClient.ANDROID_VR), + listOf(YoutubePlayerClient.WEB_SAFARI, YoutubePlayerClient.TV_DOWNGRADED), ) val result = service.getStreamInfo(YOUTUBE_URL) assertSame(success, result) assertEquals( - listOf(YoutubePlayerClient.WEB_SAFARI.value, YoutubePlayerClient.ANDROID_VR.value), + listOf(YoutubePlayerClient.WEB_SAFARI.value, YoutubePlayerClient.TV_DOWNGRADED.value), observed, ) assertEquals(YoutubePlayerClient.MWEB.value, NewPipe.getYoutubePlayerClient()) @@ -49,7 +49,7 @@ class YoutubePlayerClientFallbackStreamServiceTest { } val service = YoutubePlayerClientFallbackStreamService( delegate, - listOf(YoutubePlayerClient.WEB_SAFARI, YoutubePlayerClient.ANDROID_VR), + listOf(YoutubePlayerClient.WEB_SAFARI, YoutubePlayerClient.TV_DOWNGRADED), ) val result = service.getStreamInfo(YOUTUBE_URL) diff --git a/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamServiceTest.kt index 8c027ac2..edc77c56 100644 --- a/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamServiceTest.kt @@ -18,15 +18,15 @@ import org.schabi.newpipe.extractor.NewPipe class YoutubePlayerClientStreamServiceTest { @Test - fun `classic extraction selects android vr and restores mweb`() = runBlocking { + fun `classic extraction selects web safari and restores mweb`() = runBlocking { NewPipe.setYoutubePlayerClient(YoutubePlayerClient.MWEB.value) val observed = mutableListOf() val delegate = recordingService(observed) - val service = YoutubePlayerClientStreamService(delegate, YoutubePlayerClient.ANDROID_VR) + val service = YoutubePlayerClientStreamService(delegate, YoutubePlayerClient.WEB_SAFARI) service.getStreamInfo(YOUTUBE_URL) - assertEquals(listOf(YoutubePlayerClient.ANDROID_VR.value), observed) + assertEquals(listOf(YoutubePlayerClient.WEB_SAFARI.value), observed) assertEquals(YoutubePlayerClient.MWEB.value, NewPipe.getYoutubePlayerClient()) } @@ -52,7 +52,7 @@ class YoutubePlayerClientStreamServiceTest { } } val sabr = YoutubePlayerClientStreamService(delegate, YoutubePlayerClient.MWEB) - val classic = YoutubePlayerClientStreamService(delegate, YoutubePlayerClient.ANDROID_VR) + val classic = YoutubePlayerClientStreamService(delegate, YoutubePlayerClient.WEB_SAFARI) val sabrJobs = List(2) { launch { sabr.getStreamInfo(YOUTUBE_URL) } } repeat(2) { sabrEntered.receive() } @@ -65,7 +65,7 @@ class YoutubePlayerClientStreamServiceTest { assertTrue(classicEntered.get()) assertEquals(4, observations.count { it == YoutubePlayerClient.MWEB.value }) - assertEquals(2, observations.count { it == YoutubePlayerClient.ANDROID_VR.value }) + assertEquals(2, observations.count { it == YoutubePlayerClient.WEB_SAFARI.value }) assertEquals(YoutubePlayerClient.MWEB.value, NewPipe.getYoutubePlayerClient()) }