Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 6 additions & 12 deletions src/Sleezer/Core/PostProcessing/CorruptionFailureHandler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,10 @@

namespace NzbDrone.Plugin.Sleezer.Core.PostProcessing;

public record CorruptionStrike(string File, string? Reason);

public interface ICorruptionFailureHandler
{
// Universal post-corruption workflow shared across every Sleezer download
// client: wipe the download folder, locate the grab history for this
// client: log each failed file, wipe the download folder, locate the grab history for this
// download, and publish DownloadFailedEvent so Lidarr blocklists the
// release and searches for a different one. Client-specific pre-steps
// (e.g. slskd retry-budget zeroing, peer strike tracking) live in the
Expand Down Expand Up @@ -57,6 +55,10 @@ public Task HandleAsync(
if (strikes.Count == 0)
return Task.CompletedTask;

// Every file, including those the message only counts: it points here.
foreach (CorruptionStrike strike in strikes)
_logger.Warn("[post-process] Corrupt file in download {DownloadId}: {Strike}", downloadId, CorruptionReport.Describe(strike));

try
{
// 1. Wipe the whole download folder. One corrupt file poisons the
Expand Down Expand Up @@ -131,7 +133,7 @@ public Task HandleAsync(
SourceTitle = historyItem.SourceTitle,
DownloadClient = downloadClientType,
DownloadId = historyItem.DownloadId,
Message = BuildFailureMessage(strikes),
Message = CorruptionReport.FailureMessage(strikes),
Data = historyItem.Data,
TrackedDownload = tracked,
SkipRedownload = false,
Expand All @@ -149,12 +151,4 @@ public Task HandleAsync(

return Task.CompletedTask;
}

private static string BuildFailureMessage(IReadOnlyList<CorruptionStrike> strikes)
{
string firstReason = strikes[0].Reason ?? "(unknown)";
return strikes.Count == 1
? $"NzbDrone.Plugin.Sleezer: corrupt file detected — {firstReason}"
: $"NzbDrone.Plugin.Sleezer: {strikes.Count} corrupt files detected; first: {firstReason}";
}
}
46 changes: 46 additions & 0 deletions src/Sleezer/Core/PostProcessing/CorruptionReport.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
using System.Text;

namespace NzbDrone.Plugin.Sleezer.Core.PostProcessing;

public record CorruptionStrike(string File, string? Reason);

/// <summary>Builds the failure message Lidarr shows in History and the Blocklist.</summary>
public static class CorruptionReport
{
// Lidarr sends this text to notifications: Pushover's limit is 1,024 characters, Discord's 2,000.
public const int MaxMessageLength = 1000;

/// <summary>Lists the failed files by name until the message is full, then counts the rest.</summary>
public static string FailureMessage(IReadOnlyList<CorruptionStrike> strikes)
{
List<CorruptionStrike> byName = [.. strikes.OrderBy(s => s.File, StringComparer.OrdinalIgnoreCase)];
string count = byName.Count == 1 ? "1 file" : $"{byName.Count} files";
StringBuilder message = new($"Sleezer: {count} failed the corruption check — ");
for (int i = 0; i < byName.Count; i++)
{
string entry = $"{(i > 0 ? "; " : "")}{Describe(byName[i])}";
string rest = MoreSuffix(byName.Count - i - 1);
if (message.Length + entry.Length + rest.Length > MaxMessageLength)
{
if (i > 0)
return message.Append(MoreSuffix(byName.Count - i)).ToString();
int cut = MaxMessageLength - message.Length - rest.Length - 1;
if (char.IsHighSurrogate(entry[cut - 1]))
cut--;
entry = entry[..cut] + "…";
}

message.Append(entry);
}

return message.ToString();
}

/// <summary>One failed file and why, as the message and the log show it.</summary>
public static string Describe(CorruptionStrike strike) => $"{strike.File}: {strike.Reason ?? "no reason recorded"}";

/// <summary>The file's path inside the download folder, so a disc folder (CD1/, CD2/) stays in the name.</summary>
public static string FileLabel(string folder, string path) => Path.GetRelativePath(folder, path);

private static string MoreSuffix(int left) => left > 0 ? $"; +{left} more (see the Lidarr log)" : string.Empty;
}
18 changes: 9 additions & 9 deletions src/Sleezer/Core/PostProcessing/CorruptionScanner.cs
Original file line number Diff line number Diff line change
Expand Up @@ -102,9 +102,8 @@ public async Task<Result> ScanAsync(string path, int timeoutSeconds, Cancellatio
if (FfmpegErrorFormatter.IsAttachedPictureFailure(stderr))
return await ReverifyPastAttachedPictureAsync(path, timeoutSeconds, sw, ct);

string reason = FfmpegErrorFormatter.CleanFfmpegErrors(stderr);
_logger.Debug("Corruption scan {Path}: ffmpeg verdict corrupt (exit={ExitCode}) \u2014 {Reason}", path, exitCode, reason);
return new Result(true, reason);
_logger.Debug("Corruption scan {Path}: ffmpeg verdict corrupt (exit={ExitCode}) \u2014 {Stderr}", path, exitCode, FfmpegErrorFormatter.LogExcerpt(stderr));
return DecodeFailure(stderr, exitCode);
}

// Defence in depth: ffmpeg sometimes exits 0 even with `-xerror -err_detect explode`
Expand All @@ -120,9 +119,8 @@ public async Task<Result> ScanAsync(string path, int timeoutSeconds, Cancellatio
string significant = FfmpegErrorFormatter.StripBenignMetadataNoise(stderr);
if (!string.IsNullOrWhiteSpace(significant))
{
string reason = FfmpegErrorFormatter.CleanFfmpegErrors(significant);
_logger.Debug("Corruption scan {Path}: ffmpeg exited 0 but emitted error-level output \u2014 {Reason}", path, reason);
return new Result(true, reason);
_logger.Debug("Corruption scan {Path}: ffmpeg exited 0 but emitted error-level output \u2014 {Stderr}", path, FfmpegErrorFormatter.LogExcerpt(significant));
return DecodeFailure(stderr, exitCode);
}

if (!string.IsNullOrWhiteSpace(stderr))
Expand Down Expand Up @@ -166,15 +164,17 @@ private async Task<Result> ReverifyPastAttachedPictureAsync(string path, int tim
string significant = FfmpegErrorFormatter.StripBenignMetadataNoise(stderr);
if (exitCode != 0 || !string.IsNullOrWhiteSpace(significant))
{
string reason = FfmpegErrorFormatter.CleanFfmpegErrors(string.IsNullOrWhiteSpace(significant) ? stderr : significant);
_logger.Debug("Corruption scan {Path}: corrupt beyond the attached picture (exit={ExitCode}) — {Reason}", path, exitCode, reason);
return new Result(true, reason);
_logger.Debug("Corruption scan {Path}: corrupt beyond the attached picture (exit={ExitCode}) — {Stderr}", path, exitCode, FfmpegErrorFormatter.LogExcerpt(stderr));
return DecodeFailure(stderr, exitCode);
}

_logger.Info("Corruption scan {Path}: embedded cover art is malformed but the audio decoded clean in {ElapsedMs}ms — keeping the file", path, sw.ElapsedMilliseconds);
return new Result(false, null);
}

private static Result DecodeFailure(string stderr, int exitCode) =>
new(true, FfmpegErrorFormatter.DecodeFailureReason(stderr, exitCode));

/// <summary>
/// Resolve the ffmpeg binary the scanner will invoke. Public so the post-import
/// converter can reuse the exact same resolution + decode logic to verify its
Expand Down
102 changes: 82 additions & 20 deletions src/Sleezer/Core/PostProcessing/FfmpegErrorFormatter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,34 +4,102 @@ namespace NzbDrone.Plugin.Sleezer.Core.PostProcessing;

/// <summary>
/// Pure formatter for ffmpeg stderr output emitted during corruption scans.
/// Strips codec/address prefixes, deduplicates errors, and joins them with pipes
/// Strips codec/address prefixes and ffmpeg's follow-on lines, deduplicates errors, and joins them with pipes
/// so the corrupt-file quarantine log stays readable instead of a 200-line dump.
/// Kept separate from CorruptionScanner so it can be unit-tested without dragging
/// in NLog, TagLib, and Xabe.FFmpeg.
/// </summary>
public static class FfmpegErrorFormatter
{
private static readonly Regex CodecPrefixPattern = new(@"\[[\w:#/]+ @ 0x[0-9a-fA-F]+\]\s*", RegexOptions.Compiled);
private static readonly Regex PipeSeparatorPattern = new(@"\s*\|\s*", RegexOptions.Compiled);
private static readonly Regex CodecPrefixPattern = new(@"\[[^\]\s]+ @ 0x[0-9a-fA-F]+\]\s*", RegexOptions.Compiled);
private static readonly Regex WhitespacePattern = new(@"\s+", RegexOptions.Compiled);

private const int LogExcerptLines = 20;

// The muxer's complaint about packets a bad frame left behind; it follows the decoder's error.
private const string MuxerDtsMarker = "non monotonically increasing dts";

// Lines ffmpeg prints after the decoder's own error (n9.0.2; "Error while decoding stream" is 4.x-6.x).
private static readonly string[] FollowOnMarkers =
{
"decode_frame() failed",
"Decoding error:",
"Error submitting packet to decoder:",
"Error processing packet in decoder:",
"Task finished with error code:",
"Terminating thread with return code",
"Error while decoding stream",
MuxerDtsMarker,
};

// Image decoders: under -map 0:a they only ever read the embedded cover art.
private static readonly string[] CoverArtDecoderMarkers =
{
"mjpeg @ 0x", "png @ 0x", "bmp @ 0x", "gif @ 0x", "webp @ 0x", "tiff @ 0x",
};

/// <summary>The decoder's own errors from ffmpeg stderr, deduplicated and joined with " | "; one follow-on line if that is all there is.</summary>
public static string CleanFfmpegErrors(string stderr)
{
if (string.IsNullOrWhiteSpace(stderr))
List<string> errors = DistinctErrors(stderr);
if (errors.Count == 0)
return "Non-zero exit code";

string cleaned = CodecPrefixPattern.Replace(stderr, string.Empty);
cleaned = PipeSeparatorPattern.Replace(cleaned, " | ");
cleaned = WhitespacePattern.Replace(cleaned, " ").Trim();
List<string> specific = errors.FindAll(e => !IsFollowOn(e));
if (specific.Count > 0)
return string.Join(" | ", specific);

return errors.Find(e => !e.Contains(MuxerDtsMarker, System.StringComparison.Ordinal)) ?? errors[0];
}

/// <summary>The corruption reason for a file that failed ffmpeg's decode check.</summary>
public static string DecodeFailureReason(string stderr, int exitCode)
{
List<string> lines = [.. (stderr ?? string.Empty).Split('\n').Where(l => !string.IsNullOrWhiteSpace(l))];
List<string> audio = lines.FindAll(l => !IsBenignMetadataNoise(l) && !ContainsAny(l, CoverArtDecoderMarkers));
if (audio.Count > 0)
return $"{(exitCode == 0 ? "Audio has decode errors" : "Audio won't decode")} ({CleanFfmpegErrors(string.Join('\n', audio))})";

return lines.Count > 0
? $"Damaged tag or cover art ({CleanFfmpegErrors(string.Join('\n', lines))})"
: $"ffmpeg failed with exit code {exitCode}";
}

/// <summary>The first lines of ffmpeg stderr for the Debug log, with a count of the rest.</summary>
public static string LogExcerpt(string stderr)
{
string[] lines = stderr.Trim().Split('\n');
if (lines.Length <= LogExcerptLines)
return string.Join('\n', lines);

return string.Join('\n', lines.Take(LogExcerptLines)) + $"\n({lines.Length - LogExcerptLines} more lines)";
}

private static List<string> DistinctErrors(string stderr)
{
List<string> unique = new();
if (string.IsNullOrWhiteSpace(stderr))
return unique;

string[] parts = cleaned.Split(" | ", System.StringSplitOptions.RemoveEmptyEntries | System.StringSplitOptions.TrimEntries);
HashSet<string> seen = new(System.StringComparer.Ordinal);
List<string> unique = new(parts.Length);
foreach (string p in parts)
if (seen.Add(p))
unique.Add(p);
foreach (string line in CodecPrefixPattern.Replace(stderr, string.Empty).Split('\n'))
{
string error = WhitespacePattern.Replace(line, " ").Trim();
if (error.Length > 0 && seen.Add(error))
unique.Add(error);
}

return unique.Count > 0 ? string.Join(" | ", unique) : "Non-zero exit code";
return unique;
}

private static bool IsFollowOn(string error) => ContainsAny(error, FollowOnMarkers);

private static bool ContainsAny(string line, string[] markers)
{
foreach (string marker in markers)
if (line.Contains(marker, System.StringComparison.Ordinal))
return true;
return false;
}

/// <summary>
Expand Down Expand Up @@ -69,13 +137,7 @@ public static bool IsAttachedPictureFailure(string stderr) =>
/// True if an ffmpeg stderr line is an ID3 tag-parse error the demuxer recovered
/// from by skipping the tag - a metadata defect, not audio corruption.
/// </summary>
public static bool IsBenignMetadataNoise(string line)
{
foreach (string marker in BenignMetadataMarkers)
if (line.Contains(marker, System.StringComparison.Ordinal))
return true;
return false;
}
public static bool IsBenignMetadataNoise(string line) => ContainsAny(line, BenignMetadataMarkers);

/// <summary>
/// Return <paramref name="stderr"/> with benign ID3 tag-parse lines removed. If
Expand Down
7 changes: 3 additions & 4 deletions src/Sleezer/Core/PostProcessing/PostProcessRunner.cs
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ public async Task<bool> RunAsync(PostProcessRequest request, CancellationToken c
await EnsureFFmpegResolvedAsync(ct);

var sw = Stopwatch.StartNew();
List<CorruptionStrike> strikes = await ScanForCorruptAsync(request.Folder, request.Client.ToString(), ct);
List<CorruptionStrike> strikes = await ScanForCorruptAsync(request.Folder, ct);
logger.Info("[post-process] {Client} item {ID}: scan completed in {ElapsedMs}ms — {StrikeCount} strike(s)",
request.Client, request.DownloadId, sw.ElapsedMilliseconds, strikes.Count);

Expand Down Expand Up @@ -153,7 +153,7 @@ await preImportTagger.TagCompletedDownloadAsync(
logger.Info("[post-process] {Client} item {ID}: tagging completed in {ElapsedMs}ms", request.Client, request.DownloadId, tagSw.ElapsedMilliseconds);
}

private async Task<List<CorruptionStrike>> ScanForCorruptAsync(string folder, string clientName, CancellationToken ct)
private async Task<List<CorruptionStrike>> ScanForCorruptAsync(string folder, CancellationToken ct)
{
List<CorruptionStrike> strikes = [];

Expand All @@ -168,8 +168,7 @@ private async Task<List<CorruptionStrike>> ScanForCorruptAsync(string folder, st
if (!result.IsCorrupt)
continue;

logger.Warn("[post-process] {Client} corrupt file: {File} — {Reason}", clientName, Path.GetFileName(path), result.Reason);
strikes.Add(new CorruptionStrike(Path.GetFileName(path), result.Reason));
strikes.Add(new CorruptionStrike(CorruptionReport.FileLabel(folder, path), result.Reason));
}

return strikes;
Expand Down
18 changes: 3 additions & 15 deletions src/Sleezer/Download/Clients/Soulseek/SlskdCorruptionHandler.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
using NLog;
using NzbDrone.Core.Indexers;
using NzbDrone.Plugin.Sleezer.Core.PostProcessing;
using NzbDrone.Plugin.Sleezer.Download.Clients.Soulseek.Models;
Expand All @@ -22,22 +21,19 @@ Task HandleCorruptDownloadAsync(
// Slskd-internal strike shape: carries the SlskdFileState so the handler can
// zero its retry budget. Projected to the shared CorruptionStrike before
// delegating folder deletion / event publishing to ICorruptionFailureHandler.
public record SlskdCorruptionStrike(SlskdFileState? FileState, string? Reason);
public record SlskdCorruptionStrike(SlskdFileState? FileState, string File, string? Reason);

public class SlskdCorruptionHandler : ISlskdCorruptionHandler
{
private readonly ICorruptionFailureHandler _failureHandler;
private readonly ISlskdCorruptUserTracker _userTracker;
private readonly Logger _logger;

public SlskdCorruptionHandler(
ICorruptionFailureHandler failureHandler,
ISlskdCorruptUserTracker userTracker,
Logger logger)
ISlskdCorruptUserTracker userTracker)
{
_failureHandler = failureHandler;
_userTracker = userTracker;
_logger = logger;
}

public void UseSettings(int definitionId, SlskdProviderSettings settings) => _userTracker.UseThreshold(definitionId, settings.BanUserAfterCorruptCount);
Expand All @@ -57,11 +53,7 @@ public async Task HandleCorruptDownloadAsync(
// so zeroing the budget prevents the re-enqueue path from firing even if a
// later state transition comes through.
foreach (SlskdCorruptionStrike strike in strikes)
{
strike.FileState?.UpdateMaxRetryCount(0);
string fn = strike.FileState != null ? SlskdTextProcessor.GetFileNameFromFilename(strike.FileState.File.Filename) : "(unmapped)";
_logger.Info("Corruption strike: {File} | reason: {Reason}", fn, strike.Reason ?? "(unknown)");
}

// 2. Record per-user strikes so repeat offenders can be auto-banned.
if (!string.IsNullOrEmpty(item.Username) && settings.BanUserAfterCorruptCount > 0)
Expand All @@ -71,11 +63,7 @@ public async Task HandleCorruptDownloadAsync(

// 3. Delegate the universal steps (wipe folder, publish DownloadFailedEvent
// so Lidarr blocklists and re-searches) to the shared handler.
CorruptionStrike[] shared = strikes
.Select(s => new CorruptionStrike(
File: s.FileState != null ? SlskdTextProcessor.GetFileNameFromFilename(s.FileState.File.Filename) : "(unmapped)",
Reason: s.Reason))
.ToArray();
CorruptionStrike[] shared = strikes.Select(s => new CorruptionStrike(s.File, s.Reason)).ToArray();

await _failureHandler.HandleAsync(
downloadId: item.ID,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1735,7 +1735,7 @@ private async Task<List<SlskdCorruptionStrike>> ScanFolderForCorruptionAsync(
if (fileState == null)
_logger.Warn("[post-process] Corrupt file {Filename} not mapped to any tracked slskd transfer; counting as corruption but skipping retry short-circuit.", filename);

strikes.Add(new SlskdCorruptionStrike(fileState, result.Reason));
strikes.Add(new SlskdCorruptionStrike(fileState, CorruptionReport.FileLabel(folderPath, path), result.Reason));
}

watch.Stop();
Expand Down
14 changes: 1 addition & 13 deletions tests/Sleezer.Tests/AlbumWideFailureTests.cs
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
using NzbDrone.Common.Messaging;
using NzbDrone.Core.Datastore;
using NzbDrone.Core.Download;
using NzbDrone.Core.Download.TrackedDownloads;
using NzbDrone.Core.History;
using NzbDrone.Core.MediaFiles.Events;
using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Plugin.Sleezer.Blocklisting;
using NzbDrone.Plugin.Sleezer.Core.Qobuz;
using Sleezer.Tests.Fakes;
using Xunit;

namespace Sleezer.Tests;
Expand Down Expand Up @@ -67,17 +66,6 @@ public void A_failure_with_no_reason_blocks_only_the_grabbed_tier()
public void Unstreamable_tracks_rule_out_the_album_only_when_it_must_be_complete(int unstreamable, bool requireCompleteAlbum, bool albumWide) =>
Assert.Equal(albumWide, AlbumWideFailure.Covers(QobuzAlbumFailure.Reason(unstreamable, requireCompleteAlbum)));

private sealed class CapturingEvents : IEventAggregator
{
public List<DownloadFailedEvent> Failed { get; } = [];

public void PublishEvent<TEvent>(TEvent @event) where TEvent : class, IEvent
{
if (@event is DownloadFailedEvent failed)
Failed.Add(failed);
}
}

private sealed class GrabHistory(EntityHistory grab) : IHistoryService
{
public List<EntityHistory> Find(string downloadId, EntityHistoryEventType eventType) =>
Expand Down
Loading
Loading