diff --git a/src/NzbDrone.Core/Lidarr.Core.csproj b/src/NzbDrone.Core/Lidarr.Core.csproj index b36b2a8f7e..b0719a3fd1 100644 --- a/src/NzbDrone.Core/Lidarr.Core.csproj +++ b/src/NzbDrone.Core/Lidarr.Core.csproj @@ -4,6 +4,7 @@ + diff --git a/src/NzbDrone.Core/MediaFiles/CueSheetService.cs b/src/NzbDrone.Core/MediaFiles/CueSheetService.cs index 28a8f160b1..5c55a53c9c 100644 --- a/src/NzbDrone.Core/MediaFiles/CueSheetService.cs +++ b/src/NzbDrone.Core/MediaFiles/CueSheetService.cs @@ -3,7 +3,9 @@ using System.IO; using System.IO.Abstractions; using System.Linq; +using System.Text; using System.Text.RegularExpressions; +using Diacritics.Extensions; using NLog; using NzbDrone.Core.MediaFiles.TrackImport; using NzbDrone.Core.Music; @@ -19,6 +21,15 @@ public class CueSheetInfo public IdentificationOverrides IdOverrides { get; set; } public CueSheet CueSheet { get; set; } public bool IsForMediaFile(string path) => CueSheet != null && CueSheet.Files.Count > 0 && CueSheet.Files.Any(x => Path.GetFileName(path) == x.Name); + public CueSheet.FileEntry TryToGetFileEntryForMediaFile(string path) + { + if (CueSheet != null && CueSheet.Files.Count > 0) + { + return CueSheet.Files.Find(x => Path.GetFileName(path) == x.Name); + } + + return null; + } } public interface ICueSheetService @@ -50,6 +61,39 @@ public CueSheetService(IParsingService parsingService, _logger = logger; } + private class PunctuationReplacer + { + private readonly Dictionary _replacements = new Dictionary + { + { '‘', '\'' }, { '’', '\'' }, // Single quotes + { '“', '"' }, { '”', '"' }, // Double quotes + { '‹', '<' }, { '›', '>' }, // Angle quotes + { '«', '<' }, { '»', '>' }, // Guillemets + { '–', '-' }, { '—', '-' }, // Dashes + { '…', '.' }, // Ellipsis + { '¡', '!' }, { '¿', '?' }, // Inverted punctuation (Spanish) + }; + + public string ReplacePunctuation(string input) + { + var output = new StringBuilder(input.Length); + + foreach (var c in input) + { + if (_replacements.TryGetValue(c, out var replacement)) + { + output.Append(replacement); + } + else + { + output.Append(c); + } + } + + return output.ToString(); + } + } + public List> GetImportDecisions(ref List mediaFileList, IdentificationOverrides idOverrides, ImportDecisionMakerInfo itemInfo, ImportDecisionMakerConfig config) { var decisions = new List>(); @@ -119,7 +163,6 @@ public List> GetImportDecisions(ref List m } } - var addedTracks = new List(); decisions.ForEach(decision => { if (!decision.Item.IsSingleFileRelease) @@ -156,14 +199,53 @@ public List> GetImportDecisions(ref List m return; } - // TODO diacritics could cause false positives here - decision.Item.Tracks = tracksFromRelease.Where(trackFromRelease => !addedTracks.Contains(trackFromRelease) && tracksFromCueSheet.Any(trackFromCueSheet => string.Equals(trackFromCueSheet.Title, trackFromRelease.Title, StringComparison.OrdinalIgnoreCase))).ToList(); - addedTracks.AddRange(decision.Item.Tracks); + var replacer = new PunctuationReplacer(); + var i = 0; + while (i < tracksFromRelease.Count) + { + var trackFromRelease = tracksFromRelease[i]; + var trackFromReleaseTitle = NormalizeTitle(replacer, trackFromRelease.Title); + + var j = 0; + var anyMatch = false; + while (j < tracksFromCueSheet.Count) + { + var trackFromCueSheet = tracksFromCueSheet[j]; + var trackFromCueSheetTitle = NormalizeTitle(replacer, trackFromCueSheet.Title); + anyMatch = string.Equals(trackFromReleaseTitle, trackFromCueSheetTitle, StringComparison.InvariantCultureIgnoreCase); + + if (anyMatch) + { + decision.Item.Tracks.Add(trackFromRelease); + tracksFromRelease.RemoveAt(i); + tracksFromCueSheet.RemoveAt(j); + + break; + } + else + { + j++; + } + } + + if (!anyMatch) + { + i++; + } + } }); return decisions; } + private static string NormalizeTitle(PunctuationReplacer replacer, string title) + { + title.Normalize(NormalizationForm.FormKD); + title = title.RemoveDiacritics(); + title = replacer.ReplacePunctuation(title); + return title; + } + private CueSheet LoadCueSheet(IFileInfo fileInfo) { using (var fs = fileInfo.OpenRead()) @@ -337,7 +419,7 @@ private Artist GetArtist(List performers) } else if (performers.Count > 1) { - return _parsingService.GetArtist("Various Artist"); + return _parsingService.GetArtist("various artists"); } return null; diff --git a/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/DistanceCalculator.cs b/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/DistanceCalculator.cs index 4b53efff45..d951a3938a 100644 --- a/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/DistanceCalculator.cs +++ b/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/DistanceCalculator.cs @@ -30,6 +30,11 @@ private static bool TrackIndexIncorrect(LocalTrack localTrack, Track mbTrack, in localTrack.FileTrackInfo.TrackNumbers[0] != totalTrackNumber; } + private static bool TrackIndexIncorrect(CueSheet.TrackEntry cuesheetTrack, Track mbTrack, int totalTrackNumber) + { + return cuesheetTrack.Number != mbTrack.AbsoluteTrackNumber; + } + public static int GetTotalTrackNumber(Track track, List allTracks) { return track.AbsoluteTrackNumber + allTracks.Count(t => t.MediumNumber < track.MediumNumber); @@ -79,6 +84,28 @@ public static Distance TrackDistance(LocalTrack localTrack, Track mbTrack, int t return dist; } + public static Distance TrackDistance(CueSheet.TrackEntry cuesheetTrack, Track mbTrack, int totalTrackNumber, bool includeArtist = false) + { + var dist = new Distance(); + + // musicbrainz never has 'featuring' in the track title + // see https://musicbrainz.org/doc/Style/Artist_Credits + dist.AddString("track_title", cuesheetTrack.Title ?? "", mbTrack.Title); + + if (includeArtist && cuesheetTrack.Performers.Count == 1 + && !VariousArtistNames.Any(x => x.Equals(cuesheetTrack.Performers[0], StringComparison.InvariantCultureIgnoreCase))) + { + dist.AddString("track_artist", cuesheetTrack.Performers[0], mbTrack.ArtistMetadata.Value.Name); + } + + if (mbTrack.AbsoluteTrackNumber > 0) + { + dist.AddBool("track_index", TrackIndexIncorrect(cuesheetTrack, mbTrack, totalTrackNumber)); + } + + return dist; + } + public static Distance AlbumReleaseDistance(List localTracks, AlbumRelease release, TrackMapping mapping) { var dist = new Distance(); @@ -179,9 +206,14 @@ public static Distance AlbumReleaseDistance(List localTracks, AlbumR } // tracks - if (localTracks.All(x => x.IsSingleFileRelease == true)) + if (mapping.CuesheetTrackMapping.Count != 0) { - dist.Add("tracks", 0); + foreach (var pair in mapping.CuesheetTrackMapping) + { + dist.Add("tracks", pair.Value.Item2.NormalizedDistance()); + } + + Logger.Trace("after trackMapping: {0}", dist.NormalizedDistance()); } else { @@ -192,14 +224,6 @@ public static Distance AlbumReleaseDistance(List localTracks, AlbumR Logger.Trace("after trackMapping: {0}", dist.NormalizedDistance()); - // missing tracks - foreach (var track in mapping.MBExtra.Take(localTracks.Count)) - { - dist.Add("missing_tracks", 1.0); - } - - Logger.Trace("after missing tracks: {0}", dist.NormalizedDistance()); - // unmatched tracks foreach (var track in mapping.LocalExtra.Take(localTracks.Count)) { @@ -209,6 +233,14 @@ public static Distance AlbumReleaseDistance(List localTracks, AlbumR Logger.Trace("after unmatched tracks: {0}", dist.NormalizedDistance()); } + // missing tracks + foreach (var track in mapping.MBExtra.Take(localTracks.Count)) + { + dist.Add("missing_tracks", 1.0); + } + + Logger.Trace("after missing tracks: {0}", dist.NormalizedDistance()); + return dist; } } diff --git a/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/IdentificationService.cs b/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/IdentificationService.cs index 058648eb15..b1b7dff9f0 100644 --- a/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/IdentificationService.cs +++ b/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/IdentificationService.cs @@ -323,7 +323,8 @@ private void GetBestRelease(LocalAlbumRelease localAlbumRelease, List extraTrackPaths.Contains(x.Path)).ToList(); var allLocalTracks = localAlbumRelease.LocalTracks.Concat(extraTracks).DistinctBy(x => x.Path).ToList(); - var mapping = MapReleaseTracks(allLocalTracks, release.Tracks.Value); + var isSingleFileRelease = allLocalTracks.All(x => x.IsSingleFileRelease == true); + var mapping = isSingleFileRelease ? MapSingleFileReleaseTracks(allLocalTracks, release.Tracks.Value) : MapReleaseTracks(allLocalTracks, release.Tracks.Value); var distance = DistanceCalculator.AlbumReleaseDistance(allLocalTracks, release, mapping); var currDistance = distance.NormalizedDistance(); @@ -356,12 +357,6 @@ private void GetBestRelease(LocalAlbumRelease localAlbumRelease, List localTracks, List mbTracks) { var result = new TrackMapping(); - result.IsSingleFileRelease = localTracks.All(x => x.IsSingleFileRelease == true); - if (result.IsSingleFileRelease) - { - return result; - } - var distances = new Distance[localTracks.Count, mbTracks.Count]; var costs = new double[localTracks.Count, mbTracks.Count]; @@ -392,5 +387,46 @@ public TrackMapping MapReleaseTracks(List localTracks, List m return result; } + + public TrackMapping MapSingleFileReleaseTracks(List localTracks, List mbTracks) + { + var result = new TrackMapping(); + + var cuesheetTracks = new List(); + foreach (var localTrack in localTracks) + { + if (localTrack.CueSheetFileEntry != null) + { + cuesheetTracks.AddRange(localTrack.CueSheetFileEntry.Tracks); + } + } + + var distances = new Distance[cuesheetTracks.Count, mbTracks.Count]; + var costs = new double[cuesheetTracks.Count, mbTracks.Count]; + + for (var col = 0; col < mbTracks.Count; col++) + { + var totalTrackNumber = DistanceCalculator.GetTotalTrackNumber(mbTracks[col], mbTracks); + for (var row = 0; row < cuesheetTracks.Count; row++) + { + distances[row, col] = DistanceCalculator.TrackDistance(cuesheetTracks[row], mbTracks[col], totalTrackNumber, false); + costs[row, col] = distances[row, col].NormalizedDistance(); + } + } + + var m = new Munkres(costs); + m.Run(); + + foreach (var pair in m.Solution) + { + result.CuesheetTrackMapping.Add(cuesheetTracks[pair.Item1], Tuple.Create(mbTracks[pair.Item2], distances[pair.Item1, pair.Item2])); + _logger.Trace("Mapped {0} to {1}, dist: {2}", cuesheetTracks[pair.Item1], mbTracks[pair.Item2], costs[pair.Item1, pair.Item2]); + } + + result.MBExtra = mbTracks.Except(result.CuesheetTrackMapping.Values.Select(x => x.Item1)).ToList(); + _logger.Trace($"Missing tracks:\n{string.Join("\n", result.MBExtra)}"); + + return result; + } } } diff --git a/src/NzbDrone.Core/MediaFiles/TrackImport/ImportDecisionMaker.cs b/src/NzbDrone.Core/MediaFiles/TrackImport/ImportDecisionMaker.cs index 07cf098c02..df7b1a19c4 100644 --- a/src/NzbDrone.Core/MediaFiles/TrackImport/ImportDecisionMaker.cs +++ b/src/NzbDrone.Core/MediaFiles/TrackImport/ImportDecisionMaker.cs @@ -159,11 +159,12 @@ public List> GetImportDecisions(List music { localTracks.ForEach(localTrack => { - var cueSheetFindResult = itemInfo.CueSheetInfos.Find(x => x.IsForMediaFile(localTrack.Path)); - var cueSheet = cueSheetFindResult?.CueSheet; + var cueSheetInfo = itemInfo.CueSheetInfos.Find(x => x.IsForMediaFile(localTrack.Path)); + var cueSheet = cueSheetInfo?.CueSheet; if (cueSheet != null) { localTrack.IsSingleFileRelease = cueSheet.IsSingleFileRelease; + localTrack.CueSheetFileEntry = cueSheetInfo.TryToGetFileEntryForMediaFile(localTrack.Path); localTrack.Artist = idOverrides.Artist; localTrack.Album = idOverrides.Album; } diff --git a/src/NzbDrone.Core/Parser/Model/LocalAlbumRelease.cs b/src/NzbDrone.Core/Parser/Model/LocalAlbumRelease.cs index 0d1dd69c61..2ed3aa7737 100644 --- a/src/NzbDrone.Core/Parser/Model/LocalAlbumRelease.cs +++ b/src/NzbDrone.Core/Parser/Model/LocalAlbumRelease.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using NzbDrone.Core.MediaFiles; using NzbDrone.Core.MediaFiles.TrackImport.Identification; using NzbDrone.Core.Music; @@ -70,11 +71,12 @@ public class TrackMapping public TrackMapping() { Mapping = new Dictionary>(); + CuesheetTrackMapping = new Dictionary>(); } public Dictionary> Mapping { get; set; } public List LocalExtra { get; set; } public List MBExtra { get; set; } - public bool IsSingleFileRelease { get; set; } + public Dictionary> CuesheetTrackMapping { get; set; } } } diff --git a/src/NzbDrone.Core/Parser/Model/LocalTrack.cs b/src/NzbDrone.Core/Parser/Model/LocalTrack.cs index 422ac2b878..9796fe0fbc 100644 --- a/src/NzbDrone.Core/Parser/Model/LocalTrack.cs +++ b/src/NzbDrone.Core/Parser/Model/LocalTrack.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using NzbDrone.Core.MediaFiles; using NzbDrone.Core.MediaFiles.TrackImport.Identification; using NzbDrone.Core.Music; using NzbDrone.Core.Qualities; @@ -32,6 +33,7 @@ public LocalTrack() public string ReleaseGroup { get; set; } public string SceneName { get; set; } public bool IsSingleFileRelease { get; set; } + public CueSheet.FileEntry CueSheetFileEntry { get; set; } public string CueSheetPath { get; set; } public override string ToString() {