From 1ea2927f6cf93ed945740abdceb07c16bc525d5f Mon Sep 17 00:00:00 2001 From: tripsittr <160368852+tripsittr@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:54:41 -0700 Subject: [PATCH] Let a client choose which audio track to hear A rip routinely carries the original language, a dub or two and a commentary. The probe has recorded all of them since it was written -- codec, channels, language, the default flag -- and none of it was ever sent to a client, so every player played whichever track came first with no way to change it. The transcode made that permanent: -map 0:a:0 was hardcoded, so even a client that could choose had nothing to choose between. audio_tracks now reaches the detail endpoint, ?audio=N threads through to -map 0:a:N, and the track is part of the session id so switching language produces a different stream rather than reusing the old audio. Labels are language plus layout, because "English 5.1" against "English Stereo" is the other reason for the menu. A nameless track falls back to its position, since an empty row is not a choice. A track beyond the end is clamped rather than refused: mapping nothing produces a silent stream, which reads as a broken encode. Verified against a real two-track MKV -- English at 300Hz, Japanese at 900Hz -- by running the server's own command for each and measuring the output. --- app/Http/Controllers/Api/MediaController.php | 5 + app/Http/Controllers/HlsController.php | 24 +++ app/Models/MediaProbe.php | 104 ++++++++++ app/Services/Streaming/HlsSegmenter.php | 29 ++- changelog/306-audio-track-selection.md | 52 +++++ tests/Feature/AudioTrackSelectionTest.php | 206 +++++++++++++++++++ 6 files changed, 413 insertions(+), 7 deletions(-) create mode 100644 changelog/306-audio-track-selection.md create mode 100644 tests/Feature/AudioTrackSelectionTest.php diff --git a/app/Http/Controllers/Api/MediaController.php b/app/Http/Controllers/Api/MediaController.php index 5109819..4459a95 100644 --- a/app/Http/Controllers/Api/MediaController.php +++ b/app/Http/Controllers/Api/MediaController.php @@ -252,6 +252,11 @@ public function details(MediaItem $item): JsonResponse // "not measured", and inventing "HD" from a filename would be a // guess presented as a fact. 'capabilities' => $item->probe?->capabilities() ?? [], + // Every audio track, so a client can offer the choice. A rip + // routinely carries the original language, a dub or two and a + // commentary; the probe has recorded them all along and nothing + // was ever sent, so every client played whichever came first. + 'audio_tracks' => $item->probe?->audioTracks() ?? [], // The facts a detail page is built from. Flat and null-free, so a // client renders what is present and skips what is not rather than diff --git a/app/Http/Controllers/HlsController.php b/app/Http/Controllers/HlsController.php index 7426820..95d7382 100644 --- a/app/Http/Controllers/HlsController.php +++ b/app/Http/Controllers/HlsController.php @@ -76,10 +76,16 @@ public function playlist(Request $request, MediaItem $item): Response $from = max(0.0, (float) $request->float('from')); $signed = $this->signsSegments($request); + // Which audio track to encode. Clamped to what the file actually + // has: a request for track 9 of a two-track file would otherwise map + // nothing and produce a silent stream. + $audioTrack = $this->audioTrackFor($request, $item); + $session = $this->segmenter->start( $item, $decision->maxHeight ?? (int) config('transcode.video.max_height', 1080), $from, + $audioTrack, ); abort_if($session === null, 503, 'Could not start the stream.'); @@ -136,6 +142,24 @@ public function playlist(Request $request, MediaItem $item): Response * Keyed on the request reaching an API route rather than on the guard, so * the web player is unaffected by anything that changes about tokens. */ + /** + * The requested audio track, clamped to what the file has. + * + * A request for track 9 of a two-track file maps nothing and produces a + * **silent** stream -- which looks like a broken encode rather than a bad + * parameter, so it is clamped rather than refused. + */ + private function audioTrackFor(Request $request, MediaItem $item): int + { + $requested = max(0, $request->integer('audio', 0)); + + $item->loadMissing('probe'); + + $available = count($item->probe?->audioTracks() ?? []); + + return $available > 0 ? min($requested, $available - 1) : 0; + } + private function signsSegments(Request $request): bool { return $request->is('api/*'); diff --git a/app/Models/MediaProbe.php b/app/Models/MediaProbe.php index ed41598..e6b0762 100644 --- a/app/Models/MediaProbe.php +++ b/app/Models/MediaProbe.php @@ -237,6 +237,110 @@ public function playsDirectly(): ?bool return $audio->contains(fn (string $codec): bool => in_array($codec, self::PLAYABLE_AUDIO, true)); } + /** + * The audio tracks, as something a client can put in a menu. + * + * A rip of a film routinely carries half a dozen: the original language, + * a dub or two, a commentary, a descriptive track. The probe has recorded + * all of them all along and nothing was ever sent, so every client played + * whichever one happened to be first and offered no way to change it. + * + * The index is what matters functionally -- it is what `-map 0:a:N` and + * `AVMediaSelectionOption` both key on -- and the label is what makes the + * menu readable. + * + * @return array + */ + public function audioTracks(): array + { + $tracks = []; + + foreach (array_values((array) ($this->audio_streams ?? [])) as $index => $stream) { + $stream = (array) $stream; + + $tracks[] = [ + 'index' => $index, + 'label' => $this->audioTrackLabel($stream, $index), + 'language' => $stream['language'] ?? null, + 'channels' => isset($stream['channels']) ? (int) $stream['channels'] : null, + 'codec' => $stream['codec'] ?? null, + 'default' => (bool) ($stream['default'] ?? false), + ]; + } + + return $tracks; + } + + /** + * What to call one track in a menu. + * + * Language first, because that is what somebody is choosing between, and + * the channel layout after it -- "English 5.1" distinguishes the surround + * mix from the stereo one, which is the other reason to have this menu. + * + * A track with no language at all falls back to its position, because + * "Track 2" is still a choice somebody can make and an empty row is not. + */ + private function audioTrackLabel(array $stream, int $index): string + { + $parts = []; + + $language = is_string($stream['language'] ?? null) ? trim($stream['language']) : ''; + + if ($language !== '' && strtolower($language) !== 'und') { + $parts[] = self::LANGUAGE_NAMES[strtolower($language)] ?? strtoupper($language); + } + + $channels = isset($stream['channels']) ? (int) $stream['channels'] : 0; + + $layout = match (true) { + $channels >= 8 => '7.1', + $channels >= 6 => '5.1', + $channels === 2 => 'Stereo', + $channels === 1 => 'Mono', + default => null, + }; + + if ($layout !== null) { + $parts[] = $layout; + } + + return $parts === [] ? 'Track '.($index + 1) : implode(' ', $parts); + } + + /** + * The handful of languages worth spelling out. + * + * Deliberately short: a three-letter code is readable enough for the + * unusual ones, and a complete ISO table is a lot of rows to carry for a + * menu that mostly says "English". + */ + private const LANGUAGE_NAMES = [ + 'eng' => 'English', + 'en' => 'English', + 'spa' => 'Spanish', + 'es' => 'Spanish', + 'fra' => 'French', + 'fre' => 'French', + 'fr' => 'French', + 'deu' => 'German', + 'ger' => 'German', + 'de' => 'German', + 'ita' => 'Italian', + 'it' => 'Italian', + 'jpn' => 'Japanese', + 'ja' => 'Japanese', + 'kor' => 'Korean', + 'ko' => 'Korean', + 'por' => 'Portuguese', + 'pt' => 'Portuguese', + 'rus' => 'Russian', + 'ru' => 'Russian', + 'zho' => 'Chinese', + 'chi' => 'Chinese', + 'zh' => 'Chinese', + ]; + public function audioLabel(): ?string { $channels = collect($this->audio_streams ?? []) diff --git a/app/Services/Streaming/HlsSegmenter.php b/app/Services/Streaming/HlsSegmenter.php index 37d529c..2ae3ab7 100644 --- a/app/Services/Streaming/HlsSegmenter.php +++ b/app/Services/Streaming/HlsSegmenter.php @@ -47,7 +47,7 @@ class HlsSegmenter * Returns null when ffmpeg could not be started at all, which the caller * turns into a direct-play fallback rather than a broken player. */ - public function start(MediaItem $item, int $maxHeight, float $from = 0.0): ?string + public function start(MediaItem $item, int $maxHeight, float $from = 0.0, int $audioTrack = 0): ?string { $source = $item->playbackPath(); @@ -55,7 +55,7 @@ public function start(MediaItem $item, int $maxHeight, float $from = 0.0): ?stri return null; } - $session = $this->sessionId($item, $maxHeight, $from); + $session = $this->sessionId($item, $maxHeight, $from, $audioTrack); $directory = $this->directoryFor($session); // Already running or finished for this exact request: reuse it. A @@ -71,7 +71,7 @@ public function start(MediaItem $item, int $maxHeight, float $from = 0.0): ?stri return null; } - $command = $this->command($source, $directory, $maxHeight, $from); + $command = $this->command($source, $directory, $maxHeight, $from, $audioTrack); // Started detached: this runs for as long as the film does, and the // request that asked for it must return as soon as the first segment @@ -325,13 +325,17 @@ public function fileIn(string $session, string $name): ?string * backwards past what has been produced starts a new one rather than * waiting for a segment that will never come. */ - private function sessionId(MediaItem $item, int $maxHeight, float $from): string + private function sessionId(MediaItem $item, int $maxHeight, float $from, int $audioTrack = 0): string { return substr(hash('sha256', implode(':', [ $item->id, $item->updated_at?->timestamp ?? 0, $maxHeight, (int) round($from), + // The track is part of the identity: switching language has to + // produce a different stream, not reuse the one already encoded + // with the old audio. + $audioTrack, ])), 0, 32); } @@ -372,8 +376,13 @@ private function frameRate(string $source): float } /** @return array */ - private function command(string $source, string $directory, int $maxHeight, float $from): array - { + private function command( + string $source, + string $directory, + int $maxHeight, + float $from, + int $audioTrack = 0, + ): array { $seconds = max(2, (int) config('transcode.hls.segment_seconds', 6)); // Keyframes at segment boundaries, in *frames* — so the frame rate has @@ -394,7 +403,13 @@ private function command(string $source, string $directory, int $maxHeight, floa // One video and one audio stream: many rips carry a dozen audio // tracks, and HLS players pick badly among them. '-map', '0:v:0', - '-map', '0:a:0?', + // The chosen track, not always the first. + // + // This was hardcoded to `0:a:0`, so a rip carrying an original + // language, a dub and a commentary always played whichever + // happened to be first -- and no amount of choosing in the app + // could change it, because the stream only ever contained one. + '-map', '0:a:'.$audioTrack.'?', '-c:v', 'libx264', // **The output profile has to be pinned.** // diff --git a/changelog/306-audio-track-selection.md b/changelog/306-audio-track-selection.md new file mode 100644 index 0000000..ac853a1 --- /dev/null +++ b/changelog/306-audio-track-selection.md @@ -0,0 +1,52 @@ +# Choosing which audio track to hear + +A rip routinely carries the original language, a dub or two and a commentary. +The probe has recorded all of them since it was written — codec, channels, +**language**, the default flag — and **none of it was ever sent to a client**. +So every player played whichever track happened to be first, and offered no +way to change it. + +The transcode made that permanent. `-map 0:a:0` was hardcoded, so even a +client that *could* choose had nothing to choose between: the stream contained +exactly one track. + +## What changed + +- **`audio_tracks` on the detail endpoint** — index, label, language, channels, + codec and the default flag. +- **`?audio=N` on the HLS playlist**, threaded through to `-map 0:a:N`. +- **The track is part of the session id**, so switching language produces a + different stream rather than reusing the one already encoded with the old + audio. + +## Labels + +"English 5.1" and "English Stereo" in the same menu is the other reason this +exists — picking the surround mix over the downmix. So the label is language +plus layout, not language alone. + +A track with no language falls back to "Track 2", because an empty row is not +a choice and that is. + +## Clamping rather than refusing + +A request for track 9 of a two-track file maps nothing and produces a +**silent** stream — which reads as a broken encode rather than a bad +parameter. It is clamped to what the file has. + +## Verified against a real file + +Built a two-track MKV (English at 300Hz, Japanese at 900Hz) and ran the +server's own command for each: track 0 produced the 300Hz tone, track 1 the +900Hz one. The selection genuinely changes the audio, not just the arguments. + +## Testing + +9 new tests, **394 pass**. Three mutations confirmed to fail: the hardcoded +`0:a:0`, the track left out of the session id, and the tracks never reaching +the client. + +## Next + +The iOS side — a track menu in the player — is the follow-up, and is the first +piece of the custom player work (#517). diff --git a/tests/Feature/AudioTrackSelectionTest.php b/tests/Feature/AudioTrackSelectionTest.php new file mode 100644 index 0000000..98768a8 --- /dev/null +++ b/tests/Feature/AudioTrackSelectionTest.php @@ -0,0 +1,206 @@ +user = User::factory()->create(); + } + + /** @param array> $streams */ + private function film(array $streams): MediaItem + { + $item = MediaItem::unresolved()->create([ + 'user_id' => $this->user->id, + 'title' => 'A Film', + 'type' => MediaItemType::Movie, + 'file_path' => 'films/a.mkv', + 'processing_status' => ProcessingStatus::Complete, + ]); + + MediaProbe::create([ + 'media_item_id' => $item->id, + 'probed_at' => now(), + 'video_codec' => 'h264', + 'audio_streams' => $streams, + ]); + + return $item->fresh()->load('probe'); + } + + /* ----------------------------------------------------- the labels --- */ + + public function test_a_track_is_named_by_language_and_layout(): void + { + $film = $this->film([ + ['codec' => 'aac', 'channels' => 6, 'language' => 'eng'], + ['codec' => 'aac', 'channels' => 2, 'language' => 'jpn'], + ]); + + $tracks = $film->probe->audioTracks(); + + $this->assertSame('English 5.1', $tracks[0]['label']); + $this->assertSame('Japanese Stereo', $tracks[1]['label']); + } + + /** + * "English 5.1" against "English Stereo" is the other reason this menu + * exists — picking the surround mix over the downmix. + */ + public function test_two_tracks_in_one_language_are_told_apart(): void + { + $film = $this->film([ + ['codec' => 'ac3', 'channels' => 6, 'language' => 'eng'], + ['codec' => 'aac', 'channels' => 2, 'language' => 'eng'], + ]); + + $tracks = $film->probe->audioTracks(); + + $this->assertNotSame($tracks[0]['label'], $tracks[1]['label']); + } + + /** + * A nameless track still has to be choosable: an empty row is not an + * option, and "Track 2" is. + */ + public function test_a_track_with_no_language_falls_back_to_its_position(): void + { + $film = $this->film([ + ['codec' => 'aac', 'language' => 'eng', 'channels' => 2], + ['codec' => 'aac'], + ]); + + $this->assertSame('Track 2', $film->probe->audioTracks()[1]['label']); + } + + public function test_an_undetermined_language_is_not_printed(): void + { + $film = $this->film([['codec' => 'aac', 'channels' => 2, 'language' => 'und']]); + + $this->assertSame('Stereo', $film->probe->audioTracks()[0]['label']); + } + + /* -------------------------------------------------- over the wire --- */ + + public function test_the_tracks_reach_the_client(): void + { + $film = $this->film([ + ['codec' => 'aac', 'channels' => 6, 'language' => 'eng', 'default' => true], + ['codec' => 'aac', 'channels' => 2, 'language' => 'jpn'], + ]); + + $response = $this->actingAs($this->user) + ->getJson("/api/v1/items/{$film->id}/details") + ->assertOk(); + + $this->assertCount(2, (array) $response->json('audio_tracks')); + $this->assertSame('English 5.1', $response->json('audio_tracks.0.label')); + $this->assertTrue($response->json('audio_tracks.0.default')); + } + + /* --------------------------------------------------- the transcode -- */ + + /** + * The choice has to reach ffmpeg, or the menu is decoration. + */ + public function test_the_transcode_maps_the_chosen_track(): void + { + $film = $this->film([['codec' => 'aac'], ['codec' => 'aac']]); + + $method = new ReflectionMethod(HlsSegmenter::class, 'command'); + + $first = $method->invoke(app(HlsSegmenter::class), '/x/a.mkv', '/tmp/x', 1080, 0.0, 0); + $second = $method->invoke(app(HlsSegmenter::class), '/x/a.mkv', '/tmp/x', 1080, 0.0, 1); + + $this->assertContains('0:a:0?', $first); + $this->assertContains('0:a:1?', $second); + } + + /** + * Switching track must produce a *different* stream, not reuse the one + * already encoded with the old audio. + */ + public function test_each_track_gets_its_own_session(): void + { + $film = $this->film([['codec' => 'aac'], ['codec' => 'aac']]); + + $method = new ReflectionMethod(HlsSegmenter::class, 'sessionId'); + + $first = $method->invoke(app(HlsSegmenter::class), $film, 1080, 0.0, 0); + $second = $method->invoke(app(HlsSegmenter::class), $film, 1080, 0.0, 1); + + $this->assertNotSame($first, $second); + } + + /** + * A request for track 9 of a two-track file maps nothing and produces a + * **silent** stream — which reads as a broken encode rather than a bad + * parameter, so it is clamped. + */ + public function test_a_track_beyond_the_end_is_clamped(): void + { + $film = $this->film([['codec' => 'aac'], ['codec' => 'aac']]); + + $this->actingAs($this->user); + + $controller = app(HlsController::class); + $method = new ReflectionMethod($controller, 'audioTrackFor'); + + $request = Request::create('/x', 'GET', ['audio' => 9]); + + $this->assertSame(1, $method->invoke($controller, $request, $film)); + } + + public function test_an_unprobed_file_falls_back_to_the_first_track(): void + { + $item = MediaItem::unresolved()->create([ + 'user_id' => $this->user->id, + 'title' => 'Unprobed', + 'type' => MediaItemType::Movie, + 'file_path' => 'films/b.mkv', + 'processing_status' => ProcessingStatus::Complete, + ]); + + $controller = app(HlsController::class); + $method = new ReflectionMethod($controller, 'audioTrackFor'); + + $request = Request::create('/x', 'GET', ['audio' => 3]); + + $this->assertSame(0, $method->invoke($controller, $request, $item)); + } +}