diff --git a/src/download.py b/src/download.py index 079138d..138ebe0 100644 --- a/src/download.py +++ b/src/download.py @@ -142,240 +142,244 @@ def get_new_story_id(bytestring): ) -def downloader(args): - url, inout_file, fanficfare_config, path, force, live = args - output = "" - output += log(f"Working with url {url}", Bcolors.HEADER, live) +def do_download(path, loc, url, fanficfare_config, output, force, live): + if not path: + # We have no path to a Calibre library, so just download the story. + fff_update_result = check_subprocess_output( + f'cd "{loc}" && fanficfare -u "{url}" --update-cover' + ) + check_fff_output(fff_update_result) + cur = get_files(loc, ".epub", True)[0] + name = get_files(loc, ".epub", False)[0] + rename(cur, name) + output += log( + f"\tDownloaded story {story_name.search(name).group(1)} to {name}", + Bcolors.OKGREEN, + live, + ) - try: - url = get_url_without_chapter(url) - except BadDataException as e: - output += log(f"\tException: {e}", Bcolors.FAIL, live) - if not live: - print(output.strip()) return - loc = mkdtemp() story_id = None - + cur = url try: - if not path: - # We have no path to a Calibre library, so just download the story. - res = check_subprocess_output( - f'cd "{loc}" && fanficfare -u "{url}" --update-cover' - ) - check_fff_output(res) + lock.acquire() + story_id = check_subprocess_output( + f'calibredb search "Identifiers:url:={url}" {path}' + ) + lock.release() + except CalledProcessError: + # story is not in Calibre + lock.release() + + if story_id is not None: + # Story is in Calibre + story_id = story_id.decode("utf-8").replace("Initialized urlfixer\n", "") + output += log( + f"\tStory is in Calibre with id {story_id}", + Bcolors.OKBLUE, + live, + ) + output += log("\tExporting file", Bcolors.OKBLUE, live) + output += log( + f"\tcalibredb export {story_id} --dont-save-cover --dont-write-opf " + f'--single-dir --to-dir "{loc}" {path}', + Bcolors.OKBLUE, + live, + ) + lock.acquire() + check_subprocess_output( + f"calibredb export {story_id} --dont-save-cover --dont-write-opf " + f'--single-dir --to-dir "{loc}" {path}', + ) + lock.release() + + try: cur = get_files(loc, ".epub", True)[0] - name = get_files(loc, ".epub", False)[0] - rename(cur, name) output += log( - f"\tDownloaded story {story_name.search(name).group(1)} to {name}", + f'\tDownloading with fanficfare, updating file "{cur}"', Bcolors.OKGREEN, live, ) - - return - - try: - lock.acquire() - story_id = check_subprocess_output( - f'calibredb search "Identifiers:url:={url}" {path}' - ) - lock.release() - except CalledProcessError: - # story is not in Calibre - lock.release() - cur = url - - if story_id is not None: - story_id = story_id.decode("utf-8").replace("Initialized urlfixer\n", "") + except IndexError: + # Calibre doesn't have this story in epub format. + # The ebook-convert and ebook-meta CLIs can't save an epub with a source + # url in the way fanficfare expects, so we'll download a new copy as if we + # didn't have it at all output += log( - f"\tStory is in Calibre with id {story_id}", + f'\tNo epub for story id "{story_id}" in Calibre', Bcolors.OKBLUE, live, ) - output += log("\tExporting file", Bcolors.OKBLUE, live) + + check_subprocess_output(f'cp "{fanficfare_config}" {loc}/personal.ini') + + command = f'cd "{loc}" && fanficfare -j -u "{cur}" --update-cover' + output += log( + f"\tRunning: {command}", + Bcolors.OKBLUE, + live, + ) + fff_update_result = "" + try: + fff_update_result = check_subprocess_output(command) + if len(fff_update_result) == 0: + raise EmptyCalibreResponseException(command) + except CalledProcessError as e: + if ( + "AttributeError: 'NoneType' object has no attribute 'get_text'" + in e.output.decode("utf-8") + ): + # This is an uncaught error fanficfare returns when it can't make + # the expected BeautifulSoup out of the story page, e.g. when a + # story has been added to a hidden AO3 collection. + raise BadDataException( + "No story found at this url. It might have been hidden." + ) + + try: + # Throws an exception if we couldn't/shouldn't update the epub + check_fff_output(fff_update_result) + except Exception as e: + if isinstance(e, TempFileUpdatedMoreRecentlyException) or ( + force and isinstance(e, StoryUpToDateException) + ): output += log( - f"\tcalibredb export {story_id} --dont-save-cover --dont-write-opf " - f'--single-dir --to-dir "{loc}" {path}', - Bcolors.OKBLUE, + "\tForcing download update. FanFicFare error message:", + Bcolors.WARNING, live, ) - lock.acquire() - res = check_subprocess_output( - f"calibredb export {story_id} --dont-save-cover --dont-write-opf " - f'--single-dir --to-dir "{loc}" {path}', + for line in fff_update_result.split(b"\n"): + if line == b"{": + break + output += log(f"\t\t{str(line)}", Bcolors.WARNING, live) + fff_update_result = check_subprocess_output( + f'cd "{loc}" && fanficfare -u -j "{cur}" --force "--update-cover"' ) - lock.release() + check_fff_output(fff_update_result) + else: + raise e - try: - cur = get_files(loc, ".epub", True)[0] - output += log( - f'\tDownloading with fanficfare, updating file "{cur}"', - Bcolors.OKGREEN, - live, - ) - except IndexError: - # Calibre doesn't have this story in epub format. - # the ebook-convert and ebook-meta CLIs can't save an epub - # with a source url in the way fanficfare expects, so - # we'll download a new copy as if we didn't have it at all - cur = url - output += log( - f'\tNo epub for story id "{story_id}" in Calibre', - Bcolors.OKBLUE, - live, - ) + metadata = get_metadata(fff_update_result) + series_options = get_series_options(metadata) + word_count = get_word_count(metadata) + cur = get_files(loc, ".epub", True)[0] - check_subprocess_output(f'cp "{fanficfare_config}" {loc}/personal.ini') + output += log(f"\tAdding {cur} to library", Bcolors.OKBLUE, live) + try: + lock.acquire() + check_subprocess_output(f'calibredb add -d {path} "{cur}" {series_options}') + lock.release() + except Exception as e: + lock.release() + output += log(e) + if not live: + print(output.strip()) + raise + try: + lock.acquire() + calibre_search_result = check_subprocess_output( + f'calibredb search "Identifiers:url:={url}" {path}' + ) + lock.release() + new_story_id = get_new_story_id(calibre_search_result) + output += log( + f"\tAdded {cur} to library with id {new_story_id}", + Bcolors.OKGREEN, + live, + ) + except CalledProcessError as e: + lock.release() + output += log( + "\tIt's been added to library, but not sure what the ID is.", + Bcolors.WARNING, + live, + ) + output += log("\tAdded /Story-file to library with id 0", Bcolors.OKGREEN, live) + output += log(f"\t{e.output}") + raise - command = f'cd "{loc}" && fanficfare -j -u "{cur}" --update-cover' + if new_story_id: output += log( - f"\tRunning: {command}", + f"\tSetting word count of {word_count} on story {new_story_id}", Bcolors.OKBLUE, live, ) - try: - res = check_subprocess_output(command) - if len(res) == 0: - raise EmptyCalibreResponseException(command) - except CalledProcessError as e: - if ( - "AttributeError: 'NoneType' object has no attribute 'get_text'" - in e.output.decode("utf-8") - ): - # This is an uncaught error fanficfare returns when it can't make - # the expected BeautifulSoup out of the story page, e.g. when a - # story has been added to a hidden AO3 collection. - raise BadDataException( - "No story found at this url. It might have been hidden." - ) - - try: - # Throws an exception if we couldn't/shouldn't update the epub - check_fff_output(res) - except Exception as e: - if isinstance(e, TempFileUpdatedMoreRecentlyException) or ( - force and isinstance(e, StoryUpToDateException) - ): - output += log( - "\tForcing download update. FanFicFare error message:", - Bcolors.WARNING, - live, - ) - for line in res.split(b"\n"): - if line == b"{": - break - output += log(f"\t\t{str(line)}", Bcolors.WARNING, live) - res = check_subprocess_output( - f'cd "{loc}" && fanficfare -u -j "{cur}" --force "--update-cover"' - ) - check_fff_output(res) - else: - raise e - - metadata = get_metadata(res) - series_options = get_series_options(metadata) - word_count = get_word_count(metadata) - cur = get_files(loc, ".epub", True)[0] - - output += log(f"\tAdding {cur} to library", Bcolors.OKBLUE, live) try: lock.acquire() - check_subprocess_output(f'calibredb add -d {path} "{cur}" {series_options}') + check_subprocess_output( + f"calibredb set_custom {path} words {new_story_id} '{word_count}'" + ) lock.release() - except Exception as e: + except CalledProcessError as e: lock.release() - output += log(e) - if not live: - print(output.strip()) - raise + output += log( + "\tError setting word count.", + Bcolors.WARNING, + live, + ) + output += log(f"\t{e.output}") + + extra_series_options = get_extra_series_options(metadata) + tags_options = get_tags_options(metadata) try: lock.acquire() - res = check_subprocess_output( - f'calibredb search "Identifiers:url:={url}" {path}' - ) - lock.release() - new_story_id = get_new_story_id(res) output += log( - f"\tAdded {cur} to library with id {new_story_id}", - Bcolors.OKGREEN, + f"\tSetting custom fields on story {new_story_id}", + Bcolors.OKBLUE, live, ) + update_command = ( + f"calibredb set_metadata {str(new_story_id)} " + f"{path} {tags_options} {extra_series_options}" + ) + output += log(update_command, Bcolors.OKBLUE, live) + check_subprocess_output(update_command) + lock.release() except CalledProcessError as e: lock.release() output += log( - "\tIt's been added to library, but not sure what the ID is.", + "\tError setting custom data.", Bcolors.WARNING, live, ) - output += log( - "\tAdded /Story-file to library with id 0", Bcolors.OKGREEN, live - ) output += log(f"\t{e.output}") + + if story_id: + output += log(f"\tRemoving {story_id} from library", Bcolors.OKBLUE, live) + try: + lock.acquire() + check_subprocess_output(f"calibredb remove {path} {story_id}") + lock.release() + except BaseException: + lock.release() + if not live: + print(output.strip()) raise - if new_story_id: - output += log( - f"\tSetting word count of {word_count} on story {new_story_id}", - Bcolors.OKBLUE, - live, - ) - try: - lock.acquire() - check_subprocess_output( - f"calibredb set_custom {path} words {new_story_id} '{word_count}'" - ) - lock.release() - except CalledProcessError as e: - lock.release() - output += log( - "\tError setting word count.", - Bcolors.WARNING, - live, - ) - output += log(f"\t{e.output}") - - extra_series_options = get_extra_series_options(metadata) - tags_options = get_tags_options(metadata) - try: - lock.acquire() - output += log( - f"\tSetting custom fields on story {new_story_id}", - Bcolors.OKBLUE, - live, - ) - update_command = ( - f"calibredb set_metadata {str(new_story_id)} " - f"{path} {tags_options} {extra_series_options}" - ) - output += log(update_command, Bcolors.OKBLUE, live) - check_subprocess_output(update_command) - lock.release() - except CalledProcessError as e: - lock.release() - output += log( - "\tError setting custom data.", - Bcolors.WARNING, - live, - ) - output += log(f"\t{e.output}") - - if story_id: - output += log(f"\tRemoving {story_id} from library", Bcolors.OKBLUE, live) - try: - lock.acquire() - check_subprocess_output(f"calibredb remove {path} {story_id}") - lock.release() - except BaseException: - lock.release() - if not live: - print(output.strip()) - raise + if not live: + print(output.strip()) + rmtree(loc) + +def downloader(args): + url, inout_file, fanficfare_config, path, force, live = args + output = "" + output += log(f"Working with url {url}", Bcolors.HEADER, live) + + try: + url = get_url_without_chapter(url) + except BadDataException as e: + output += log(f"\tException: {e}", Bcolors.FAIL, live) if not live: print(output.strip()) - rmtree(loc) + return + + loc = mkdtemp() + story_id = None + + try: + do_download(path, loc, url, fanficfare_config, output, force, live) except Exception as e: output += log(f"\tException: {e}", Bcolors.FAIL, live) if isinstance(e, CalledProcessError):