Skip to content

Commit

Permalink
Image-only Libraries + Library Fixes (#2427)
Browse files Browse the repository at this point in the history
  • Loading branch information
@majora2007
majora2007 authored Nov 11, 2023
1 parent 5963ea5 commit fe2b9b8
Show file tree
Hide file tree
Showing 8 changed files with 193 additions and 105 deletions.
109 changes: 84 additions & 25 deletions API.Tests/Parser/DefaultParserTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,15 +239,6 @@ public void Parse_ParseInfo_Manga()
FullFilePath = filepath, IsSpecial = false
});

// Note: Fallback to folder will parse Monster #8 and get Monster
filepath = @"E:\Manga\Monster #8\Ch. 001-016 [MangaPlus] [Digital] [amit34521]\Monster #8 Ch. 001 [MangaPlus] [Digital] [amit34521]\13.jpg";
expected.Add(filepath, new ParserInfo
{
Series = "Monster", Volumes = "0", Edition = "",
Chapters = "1", Filename = "13.jpg", Format = MangaFormat.Image,
FullFilePath = filepath, IsSpecial = false
});

filepath = @"E:\Manga\Air Gear\Air Gear Omnibus v01 (2016) (Digital) (Shadowcat-Empire).cbz";
expected.Add(filepath, new ParserInfo
{
Expand All @@ -256,22 +247,6 @@ public void Parse_ParseInfo_Manga()
FullFilePath = filepath, IsSpecial = false
});

filepath = @"E:\Manga\Extra layer for no reason\Just Images the second\Vol19\ch186\Vol. 19 p106.gif";
expected.Add(filepath, new ParserInfo
{
Series = "Just Images the second", Volumes = "19", Edition = "",
Chapters = "186", Filename = "Vol. 19 p106.gif", Format = MangaFormat.Image,
FullFilePath = filepath, IsSpecial = false
});

filepath = @"E:\Manga\Extra layer for no reason\Just Images the second\Blank Folder\Vol19\ch186\Vol. 19 p106.gif";
expected.Add(filepath, new ParserInfo
{
Series = "Just Images the second", Volumes = "19", Edition = "",
Chapters = "186", Filename = "Vol. 19 p106.gif", Format = MangaFormat.Image,
FullFilePath = filepath, IsSpecial = false
});

filepath = @"E:\Manga\Harrison, Kim - The Good, The Bad, and the Undead - Hollows Vol 2.5.epub";
expected.Add(filepath, new ParserInfo
{
Expand Down Expand Up @@ -308,6 +283,90 @@ public void Parse_ParseInfo_Manga()
}
}

[Fact]
public void Parse_ParseInfo_Manga_ImageOnly()
{
// Images don't have root path as E:\Manga, but rather as the path of the folder

// Note: Fallback to folder will parse Monster #8 and get Monster
var filepath = @"E:\Manga\Monster #8\Ch. 001-016 [MangaPlus] [Digital] [amit34521]\Monster #8 Ch. 001 [MangaPlus] [Digital] [amit34521]\13.jpg";
var expectedInfo2 = new ParserInfo
{
Series = "Monster #8", Volumes = "0", Edition = "",
Chapters = "1", Filename = "13.jpg", Format = MangaFormat.Image,
FullFilePath = filepath, IsSpecial = false
};
var actual2 = _defaultParser.Parse(filepath, @"E:\Manga\Monster #8");
Assert.NotNull(actual2);
_testOutputHelper.WriteLine($"Validating {filepath}");
Assert.Equal(expectedInfo2.Format, actual2.Format);
_testOutputHelper.WriteLine("Format ✓");
Assert.Equal(expectedInfo2.Series, actual2.Series);
_testOutputHelper.WriteLine("Series ✓");
Assert.Equal(expectedInfo2.Chapters, actual2.Chapters);
_testOutputHelper.WriteLine("Chapters ✓");
Assert.Equal(expectedInfo2.Volumes, actual2.Volumes);
_testOutputHelper.WriteLine("Volumes ✓");
Assert.Equal(expectedInfo2.Edition, actual2.Edition);
_testOutputHelper.WriteLine("Edition ✓");
Assert.Equal(expectedInfo2.Filename, actual2.Filename);
_testOutputHelper.WriteLine("Filename ✓");
Assert.Equal(expectedInfo2.FullFilePath, actual2.FullFilePath);
_testOutputHelper.WriteLine("FullFilePath ✓");

filepath = @"E:\Manga\Extra layer for no reason\Just Images the second\Vol19\ch186\Vol. 19 p106.gif";
expectedInfo2 = new ParserInfo
{
Series = "Just Images the second", Volumes = "19", Edition = "",
Chapters = "186", Filename = "Vol. 19 p106.gif", Format = MangaFormat.Image,
FullFilePath = filepath, IsSpecial = false
};

actual2 = _defaultParser.Parse(filepath, @"E:\Manga\Extra layer for no reason\");
Assert.NotNull(actual2);
_testOutputHelper.WriteLine($"Validating {filepath}");
Assert.Equal(expectedInfo2.Format, actual2.Format);
_testOutputHelper.WriteLine("Format ✓");
Assert.Equal(expectedInfo2.Series, actual2.Series);
_testOutputHelper.WriteLine("Series ✓");
Assert.Equal(expectedInfo2.Chapters, actual2.Chapters);
_testOutputHelper.WriteLine("Chapters ✓");
Assert.Equal(expectedInfo2.Volumes, actual2.Volumes);
_testOutputHelper.WriteLine("Volumes ✓");
Assert.Equal(expectedInfo2.Edition, actual2.Edition);
_testOutputHelper.WriteLine("Edition ✓");
Assert.Equal(expectedInfo2.Filename, actual2.Filename);
_testOutputHelper.WriteLine("Filename ✓");
Assert.Equal(expectedInfo2.FullFilePath, actual2.FullFilePath);
_testOutputHelper.WriteLine("FullFilePath ✓");

filepath = @"E:\Manga\Extra layer for no reason\Just Images the second\Blank Folder\Vol19\ch186\Vol. 19 p106.gif";
expectedInfo2 = new ParserInfo
{
Series = "Just Images the second", Volumes = "19", Edition = "",
Chapters = "186", Filename = "Vol. 19 p106.gif", Format = MangaFormat.Image,
FullFilePath = filepath, IsSpecial = false
};

actual2 = _defaultParser.Parse(filepath, @"E:\Manga\Extra layer for no reason\");
Assert.NotNull(actual2);
_testOutputHelper.WriteLine($"Validating {filepath}");
Assert.Equal(expectedInfo2.Format, actual2.Format);
_testOutputHelper.WriteLine("Format ✓");
Assert.Equal(expectedInfo2.Series, actual2.Series);
_testOutputHelper.WriteLine("Series ✓");
Assert.Equal(expectedInfo2.Chapters, actual2.Chapters);
_testOutputHelper.WriteLine("Chapters ✓");
Assert.Equal(expectedInfo2.Volumes, actual2.Volumes);
_testOutputHelper.WriteLine("Volumes ✓");
Assert.Equal(expectedInfo2.Edition, actual2.Edition);
_testOutputHelper.WriteLine("Edition ✓");
Assert.Equal(expectedInfo2.Filename, actual2.Filename);
_testOutputHelper.WriteLine("Filename ✓");
Assert.Equal(expectedInfo2.FullFilePath, actual2.FullFilePath);
_testOutputHelper.WriteLine("FullFilePath ✓");
}

[Fact]
public void Parse_ParseInfo_Manga_WithSpecialsFolder()
{
Expand Down
5 changes: 5 additions & 0 deletions API/Entities/Enums/LibraryType.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,9 @@ public enum LibraryType
/// </summary>
[Description("Book")]
Book = 2,
/// <summary>
/// Uses a different type of grouping and parsing mechanism
/// </summary>
[Description("Image")]
Image = 3,
}
3 changes: 3 additions & 0 deletions API/Entities/Library.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ public class Library : IEntityDate
/// </summary>
/// <remarks>Scrobbling requires a valid LicenseKey</remarks>
public bool AllowScrobbling { get; set; } = true;



public DateTime Created { get; set; }
public DateTime LastModified { get; set; }
public DateTime CreatedUtc { get; set; }
Expand Down
4 changes: 2 additions & 2 deletions API/Helpers/Builders/ChapterBuilder.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ public static ChapterBuilder FromParserInfo(ParserInfo info)
{
var specialTreatment = info.IsSpecialInfo();
var specialTitle = specialTreatment ? info.Filename : info.Chapters;
var builder = new ChapterBuilder(Services.Tasks.Scanner.Parser.Parser.DefaultChapter);
return builder.WithNumber(specialTreatment ? Services.Tasks.Scanner.Parser.Parser.DefaultChapter : Services.Tasks.Scanner.Parser.Parser.MinNumberFromRange(info.Chapters) + string.Empty)
var builder = new ChapterBuilder(Parser.DefaultChapter);
return builder.WithNumber(specialTreatment ? Parser.DefaultChapter : Parser.MinNumberFromRange(info.Chapters) + string.Empty)
.WithRange(specialTreatment ? info.Filename : info.Chapters)
.WithTitle((specialTreatment && info.Format == MangaFormat.Epub)
? info.Title
Expand Down
76 changes: 44 additions & 32 deletions API/Services/Tasks/Scanner/Parser/DefaultParser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,43 +35,38 @@ public DefaultParser(IDirectoryService directoryService)
{
var fileName = _directoryService.FileSystem.Path.GetFileNameWithoutExtension(filePath);
// TODO: Potential Bug: This will return null, but on Image libraries, if all images, we would want to include this. (we can probably remove this and have users use kavitaignore)
if (Parser.IsCoverImage(_directoryService.FileSystem.Path.GetFileName(filePath))) return null;
if (type != LibraryType.Image && Parser.IsCoverImage(_directoryService.FileSystem.Path.GetFileName(filePath))) return null;

ParserInfo ret;

if (Parser.IsEpub(filePath)) // NOTE: Will this ever be called? Because we use ReadingService to handle parse
var ret = new ParserInfo()
{
ret = new ParserInfo
{
Chapters = Parser.ParseChapter(fileName) ?? Parser.ParseComicChapter(fileName),
Series = Parser.ParseSeries(fileName) ?? Parser.ParseComicSeries(fileName),
Volumes = Parser.ParseVolume(fileName) ?? Parser.ParseComicVolume(fileName),
Filename = Path.GetFileName(filePath),
Format = Parser.ParseFormat(filePath),
FullFilePath = filePath
};
}
else
Filename = Path.GetFileName(filePath),
Format = Parser.ParseFormat(filePath),
Title = Path.GetFileNameWithoutExtension(fileName),
FullFilePath = filePath,
Series = string.Empty
};

// If library type is Image or this is not a cover image in a non-image library, then use dedicated parsing mechanism
if (type == LibraryType.Image || Parser.IsImage(filePath))
{
ret = new ParserInfo
{
Chapters = type == LibraryType.Comic ? Parser.ParseComicChapter(fileName) : Parser.ParseChapter(fileName),
Series = type == LibraryType.Comic ? Parser.ParseComicSeries(fileName) : Parser.ParseSeries(fileName),
Volumes = type == LibraryType.Comic ? Parser.ParseComicVolume(fileName) : Parser.ParseVolume(fileName),
Filename = Path.GetFileName(filePath),
Format = Parser.ParseFormat(filePath),
Title = Path.GetFileNameWithoutExtension(fileName),
FullFilePath = filePath
};
return ParseImage(filePath, rootPath, ret);
}


if (Parser.IsImage(filePath))
// This will be called if the epub is already parsed once then we call and merge the information, if the
if (Parser.IsEpub(filePath))
{
ret.Chapters = Parser.ParseChapter(fileName);
ret.Series = Parser.ParseSeries(fileName);
ret.Volumes = Parser.ParseVolume(fileName);
}
else
{
// Reset Chapters, Volumes, and Series as images are not good to parse information out of. Better to use folders.
ret.Volumes = Parser.DefaultVolume;
ret.Chapters = Parser.DefaultChapter;
ret.Series = string.Empty;
ret.Chapters = type == LibraryType.Comic
? Parser.ParseComicChapter(fileName)
: Parser.ParseChapter(fileName);
ret.Series = type == LibraryType.Comic ? Parser.ParseComicSeries(fileName) : Parser.ParseSeries(fileName);
ret.Volumes = type == LibraryType.Comic ? Parser.ParseComicVolume(fileName) : Parser.ParseVolume(fileName);
}

if (ret.Series == string.Empty || Parser.IsImage(filePath))
Expand Down Expand Up @@ -120,6 +115,23 @@ public DefaultParser(IDirectoryService directoryService)
return ret.Series == string.Empty ? null : ret;
}

private ParserInfo ParseImage(string filePath, string rootPath, ParserInfo ret)
{
ret.Volumes = Parser.DefaultVolume;
ret.Chapters = Parser.DefaultChapter;
// Next we need to see if the image has a folder between rootPath and filePath.
// if so, take that folder as a volume 0 chapter 0 special and group everything under there (if we can't parse a volume/chapter)
ParseFromFallbackFolders(filePath, rootPath, LibraryType.Image, ref ret);
if ((string.IsNullOrEmpty(ret.Chapters) || ret.Chapters == Parser.DefaultChapter) &&
(string.IsNullOrEmpty(ret.Volumes) || ret.Volumes == Parser.DefaultVolume))
{
ret.IsSpecial = true;
}

ret.Series = _directoryService.FileSystem.DirectoryInfo.New(rootPath).Name;
return ret;
}

/// <summary>
/// Fills out <see cref="ParserInfo"/> by trying to parse volume, chapters, and series from folders
/// </summary>
Expand Down Expand Up @@ -160,11 +172,11 @@ public void ParseFromFallbackFolders(string filePath, string rootPath, LibraryTy

if (!parsedVolume.Equals(Parser.DefaultVolume) || !parsedChapter.Equals(Parser.DefaultChapter))
{
if ((string.IsNullOrEmpty(ret.Volumes) || ret.Volumes.Equals(Parser.DefaultVolume)) && !parsedVolume.Equals(Parser.DefaultVolume))
if ((string.IsNullOrEmpty(ret.Volumes) || ret.Volumes.Equals(Parser.DefaultVolume)) && !string.IsNullOrEmpty(parsedVolume) && !parsedVolume.Equals(Parser.DefaultVolume))
{
ret.Volumes = parsedVolume;
}
if ((string.IsNullOrEmpty(ret.Chapters) || ret.Chapters.Equals(Parser.DefaultChapter)) && !parsedChapter.Equals(Parser.DefaultChapter))
if ((string.IsNullOrEmpty(ret.Chapters) || ret.Chapters.Equals(Parser.DefaultChapter)) && !string.IsNullOrEmpty(parsedChapter) && !parsedChapter.Equals(Parser.DefaultChapter))
{
ret.Chapters = parsedChapter;
}
Expand Down
38 changes: 13 additions & 25 deletions API/Services/Tasks/Scanner/ProcessSeries.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,9 +62,6 @@ public class ProcessSeries : IProcessSeries
private IList<Person> _people;
private Dictionary<string, Tag> _tags;
private Dictionary<string, CollectionTag> _collectionTags;
private readonly object _peopleLock = new object();
private readonly object _genreLock = new object();
private readonly object _tagLock = new object();

public ProcessSeries(IUnitOfWork unitOfWork, ILogger<ProcessSeries> logger, IEventHub eventHub,
IDirectoryService directoryService, ICacheHelper cacheHelper, IReadingItemService readingItemService,
Expand Down Expand Up @@ -845,23 +842,20 @@ private static IList<string> GetTagValues(string comicInfoTagSeparatedByComma)
/// <param name="action"></param>
private void UpdatePeople(IEnumerable<string> names, PersonRole role, Action<Person> action)
{
lock (_peopleLock)
var allPeopleTypeRole = _people.Where(p => p.Role == role).ToList();

foreach (var name in names)
{
var allPeopleTypeRole = _people.Where(p => p.Role == role).ToList();
var normalizedName = name.ToNormalized();
var person = allPeopleTypeRole.Find(p =>
p.NormalizedName != null && p.NormalizedName.Equals(normalizedName));

foreach (var name in names)
if (person == null)
{
var normalizedName = name.ToNormalized();
var person = allPeopleTypeRole.Find(p =>
p.NormalizedName != null && p.NormalizedName.Equals(normalizedName));

if (person == null)
{
person = new PersonBuilder(name, role).Build();
_people.Add(person);
}
action(person);
person = new PersonBuilder(name, role).Build();
_people.Add(person);
}
action(person);
}
}

Expand All @@ -882,11 +876,8 @@ private void UpdateGenre(IEnumerable<string> names, Action<Genre, bool> action)
if (newTag)
{
genre = new GenreBuilder(name).Build();
lock (_genreLock)
{
_genres.Add(normalizedName, genre);
_unitOfWork.GenreRepository.Attach(genre);
}
_genres.Add(normalizedName, genre);
_unitOfWork.GenreRepository.Attach(genre);
}

action(genre!, newTag);
Expand All @@ -911,10 +902,7 @@ private void UpdateTag(IEnumerable<string> names, Action<Tag, bool> action)
if (tag == null)
{
tag = new TagBuilder(name).Build();
lock (_tagLock)
{
_tags.Add(normalizedName, tag);
}
_tags.Add(normalizedName, tag);
}

action(tag, added);
Expand Down
Loading

0 comments on commit fe2b9b8

Please sign in to comment.