From c4be677a6209a1530942538ee21f462d1553bd19 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 15 Jan 2019 17:22:26 -0500 Subject: [PATCH 1/8] Fix git subprocess --- core/version_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/version_check.py b/core/version_check.py index 8fac2d374..53945e1f4 100644 --- a/core/version_check.py +++ b/core/version_check.py @@ -116,7 +116,7 @@ def _find_working_git(self): test_cmd = 'version' if core.GIT_PATH: - main_git = '\'{git}\''.format(git=core.GIT_PATH) + main_git = '"{git}"'.format(git=core.GIT_PATH) else: main_git = 'git' From 7185e0b31ba84cb1bae3174d8aa64ab7291a97e2 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 15 Jan 2019 17:35:41 -0500 Subject: [PATCH 2/8] Add docstring --- cleanup.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cleanup.py b/cleanup.py index 02bdf6482..3e1559cf8 100644 --- a/cleanup.py +++ b/cleanup.py @@ -37,6 +37,13 @@ def __exit__(self, exc_type, exc_val, exc_tb): def module_path(module=__file__, parent=False): + """ + Detect path for a module. + + :param module: The module who's path is being detected. Defaults to current module. + :param parent: True to return the parent folder of the current module. + :return: The absolute normalized path to the module or its parent. + """ try: path = module.__file__ except AttributeError: @@ -122,6 +129,12 @@ def clean_folders(*paths): def force_clean_folder(path, required): + """ + Force clean a folder and exclude any required subfolders. + + :param path: Target folder to remove subfolders + :param required: Keep only the required subfolders + """ root, dirs, files = next(os.walk(path)) required = sorted(required) if required: From 844c1d15e98f43df705563933e2455d919e17550 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 15 Jan 2019 17:47:22 -0500 Subject: [PATCH 3/8] Fix cleanup script output --- cleanup.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/cleanup.py b/cleanup.py index 3e1559cf8..b78df372e 100644 --- a/cleanup.py +++ b/cleanup.py @@ -151,6 +151,11 @@ def force_clean_folder(path, required): def clean(paths): """Clean up bytecode and obsolete folders.""" + def _report_error(msg): + print('WARNING: Automatic cleanup could not be executed.') + print(' If errors occur, manual cleanup may be required.') + print('REASON : {}'.format(msg)) + with WorkingDirectory(module_path()) as cwd: if cwd.working_directory != cwd.original_directory: print('Changing to directory:', cwd.working_directory) @@ -159,7 +164,7 @@ def clean(paths): try: result = clean_bytecode() except SystemExit as error: - print(error) + _report_error(error) else: print(result or 'No bytecode to clean') @@ -168,7 +173,7 @@ def clean(paths): try: result = clean_folders(*paths) except SystemExit as error: - print(error) + _report_error(error) else: print(result or 'No folders to clean\n') else: @@ -176,7 +181,7 @@ def clean(paths): try: items = paths.items() except AttributeError: - print('Failed to clean, no subfolder structure given') + _report_error('Failed to clean, no subfolder structure given') else: for folder, subfolders in items: print('\nForce cleaning folder:', folder) From aa057e65d5809b8b261a32b47f413cc9b655dbb6 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 15 Jan 2019 17:55:43 -0500 Subject: [PATCH 4/8] Refactor common utils to utils.common --- core/utils/__init__.py | 122 +---------------------------------------- core/utils/common.py | 114 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 120 deletions(-) create mode 100644 core/utils/common.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 176c92db0..ee2eba058 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -1,15 +1,9 @@ # coding=utf-8 -from __future__ import print_function, unicode_literals - -import os - import requests -from six import text_type -import core -from core import logger from core.utils import shutil_custom +from core.utils.common import clean_dir, flatten, get_dirs, process_dir from core.utils.download_info import get_download_info, update_download_info_status from core.utils.encoding import char_replace, convert_to_ascii from core.utils.files import ( @@ -24,7 +18,7 @@ from core.utils.identification import category_search, find_imdbid from core.utils.links import copy_link, replace_links from core.utils.naming import clean_file_name, is_sample, sanitize_name -from core.utils.network import find_download, test_connection, wake_on_lan, wake_up, server_responding +from core.utils.network import find_download, server_responding, test_connection, wake_on_lan, wake_up from core.utils.notifications import plex_update from core.utils.nzbs import get_nzoid, report_nzb from core.utils.parsers import ( @@ -52,117 +46,5 @@ from core.utils.subtitles import import_subs from core.utils.torrents import create_torrent_class, pause_torrent, remove_torrent, resume_torrent -try: - import jaraco -except ImportError: - if os.name == 'nt': - raise - requests.packages.urllib3.disable_warnings() - shutil_custom.monkey_patch() - - -def flatten(output_destination): - return flatten_dir(output_destination, list_media_files(output_destination)) - - -def clean_dir(path, section, subsection): - cfg = dict(core.CFG[section][subsection]) - min_size = int(cfg.get('minSize', 0)) - delete_ignored = int(cfg.get('delete_ignored', 0)) - try: - files = list_media_files(path, min_size=min_size, delete_ignored=delete_ignored) - except Exception: - files = [] - return clean_directory(path, files) - - -def process_dir(path, link): - folders = [] - - logger.info('Searching {0} for mediafiles to post-process ...'.format(path)) - dir_contents = os.listdir(text_type(path)) - - # search for single files and move them into their own folder for post-processing - - # Generate list of sync files - sync_files = ( - item for item in dir_contents - if os.path.splitext(item)[1] in ['.!sync', '.bts'] - ) - - # Generate a list of file paths - filepaths = ( - os.path.join(path, item) for item in dir_contents - if item not in ['Thumbs.db', 'thumbs.db'] - ) - - # Generate a list of media files - mediafiles = ( - item for item in filepaths - if os.path.isfile(item) - ) - - if any(sync_files): - logger.info('') - else: - for mediafile in mediafiles: - try: - move_file(mediafile, path, link) - except Exception as e: - logger.error('Failed to move {0} to its own directory: {1}'.format(os.path.split(mediafile)[1], e)) - - # removeEmptyFolders(path, removeRoot=False) - - # Generate all path contents - path_contents = ( - os.path.join(path, item) - for item in os.listdir(text_type(path)) - ) - - # Generate all directories from path contents - directories = ( - path for path in path_contents - if os.path.isdir(path) - ) - - for directory in directories: - dir_contents = os.listdir(directory) - sync_files = ( - item for item in dir_contents - if os.path.splitext(item)[1] in ['.!sync', '.bts'] - ) - if not any(dir_contents) or any(sync_files): - continue - folders.append(directory) - - return folders - - -def get_dirs(section, subsection, link='hard'): - to_return = [] - - watch_directory = core.CFG[section][subsection]['watch_dir'] - directory = os.path.join(watch_directory, subsection) - - if not os.path.exists(directory): - directory = watch_directory - - try: - to_return.extend(process_dir(directory, link)) - except Exception as e: - logger.error('Failed to add directories from {0} for post-processing: {1}'.format(watch_directory, e)) - - if core.USELINK == 'move': - try: - output_directory = os.path.join(core.OUTPUTDIRECTORY, subsection) - if os.path.exists(output_directory): - to_return.extend(process_dir(output_directory, link)) - except Exception as e: - logger.error('Failed to add directories from {0} for post-processing: {1}'.format(core.OUTPUTDIRECTORY, e)) - - if not to_return: - logger.debug('No directories identified in {0}:{1} for post-processing'.format(section, subsection)) - - return list(set(to_return)) diff --git a/core/utils/common.py b/core/utils/common.py new file mode 100644 index 000000000..f352fb877 --- /dev/null +++ b/core/utils/common.py @@ -0,0 +1,114 @@ + +import os.path + +from six import text_type + +import core +from core import logger +from core.utils.files import list_media_files, move_file +from core.utils.paths import clean_directory, flatten_dir + + +def flatten(output_destination): + return flatten_dir(output_destination, list_media_files(output_destination)) + + +def clean_dir(path, section, subsection): + cfg = dict(core.CFG[section][subsection]) + min_size = int(cfg.get('minSize', 0)) + delete_ignored = int(cfg.get('delete_ignored', 0)) + try: + files = list_media_files(path, min_size=min_size, delete_ignored=delete_ignored) + except Exception: + files = [] + return clean_directory(path, files) + + +def process_dir(path, link): + folders = [] + + logger.info('Searching {0} for mediafiles to post-process ...'.format(path)) + dir_contents = os.listdir(text_type(path)) + + # search for single files and move them into their own folder for post-processing + + # Generate list of sync files + sync_files = ( + item for item in dir_contents + if os.path.splitext(item)[1] in ['.!sync', '.bts'] + ) + + # Generate a list of file paths + filepaths = ( + os.path.join(path, item) for item in dir_contents + if item not in ['Thumbs.db', 'thumbs.db'] + ) + + # Generate a list of media files + mediafiles = ( + item for item in filepaths + if os.path.isfile(item) + ) + + if any(sync_files): + logger.info('') + else: + for mediafile in mediafiles: + try: + move_file(mediafile, path, link) + except Exception as e: + logger.error('Failed to move {0} to its own directory: {1}'.format(os.path.split(mediafile)[1], e)) + + # removeEmptyFolders(path, removeRoot=False) + + # Generate all path contents + path_contents = ( + os.path.join(path, item) + for item in os.listdir(text_type(path)) + ) + + # Generate all directories from path contents + directories = ( + path for path in path_contents + if os.path.isdir(path) + ) + + for directory in directories: + dir_contents = os.listdir(directory) + sync_files = ( + item for item in dir_contents + if os.path.splitext(item)[1] in ['.!sync', '.bts'] + ) + if not any(dir_contents) or any(sync_files): + continue + folders.append(directory) + + return folders + + +def get_dirs(section, subsection, link='hard'): + to_return = [] + + watch_directory = core.CFG[section][subsection]['watch_dir'] + directory = os.path.join(watch_directory, subsection) + + if not os.path.exists(directory): + directory = watch_directory + + try: + to_return.extend(process_dir(directory, link)) + except Exception as e: + logger.error('Failed to add directories from {0} for post-processing: {1}'.format(watch_directory, e)) + + if core.USELINK == 'move': + try: + output_directory = os.path.join(core.OUTPUTDIRECTORY, subsection) + if os.path.exists(output_directory): + to_return.extend(process_dir(output_directory, link)) + except Exception as e: + logger.error('Failed to add directories from {0} for post-processing: {1}'.format(core.OUTPUTDIRECTORY, e)) + + if not to_return: + logger.debug('No directories identified in {0}:{1} for post-processing'.format(section, subsection)) + + return list(set(to_return)) From d960c432eb982f229433805475d1de54714b98a0 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 15 Jan 2019 18:02:36 -0500 Subject: [PATCH 5/8] Refactor rchmod to utils.paths.rchmod --- core/__init__.py | 38 +++++++++++++++++++++----------------- core/utils/__init__.py | 1 + core/utils/paths.py | 13 +++++++++++++ 3 files changed, 35 insertions(+), 17 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 9958308b6..7dcd4fd9d 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -46,10 +46,27 @@ from core import logger, main_db, version_check, databases, transcoder from core.configuration import config from core.utils import ( - RunningProcess, wake_up, category_search, clean_dir, clean_dir, copy_link, - create_torrent_class, extract_files, flatten, get_dirs, get_download_info, - list_media_files, make_dir, parse_args, pause_torrent, remove_torrent, - resume_torrent, remove_dir, remove_read_only, sanitize_name, update_download_info_status, + RunningProcess, + category_search, + clean_dir, + copy_link, + create_torrent_class, + extract_files, + flatten, + get_dirs, + get_download_info, + list_media_files, + make_dir, + parse_args, + pause_torrent, + rchmod, + remove_dir, + remove_read_only, + remove_torrent, + resume_torrent, + sanitize_name, + update_download_info_status, + wake_up, ) __version__ = '12.0.6' @@ -866,16 +883,3 @@ def restart(): status = p.returncode os._exit(status) - - -def rchmod(path, mod): - logger.log('Changing file mode of {0} to {1}'.format(path, oct(mod))) - os.chmod(path, mod) - if not os.path.isdir(path): - return # Skip files - - for root, dirs, files in os.walk(path): - for d in dirs: - os.chmod(os.path.join(root, d), mod) - for f in files: - os.chmod(os.path.join(root, f), mod) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index ee2eba058..99bb03769 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -37,6 +37,7 @@ get_dir_size, make_dir, onerror, + rchmod, remote_dir, remove_dir, remove_empty_folders, diff --git a/core/utils/paths.py b/core/utils/paths.py index 5576206fa..ec99a2467 100644 --- a/core/utils/paths.py +++ b/core/utils/paths.py @@ -149,3 +149,16 @@ def clean_directory(path, files): shutil.rmtree(path, onerror=onerror) except Exception: logger.error('Unable to delete directory {0}'.format(path)) + + +def rchmod(path, mod): + logger.log('Changing file mode of {0} to {1}'.format(path, oct(mod))) + os.chmod(path, mod) + if not os.path.isdir(path): + return # Skip files + + for root, dirs, files in os.walk(path): + for d in dirs: + os.chmod(os.path.join(root, d), mod) + for f in files: + os.chmod(os.path.join(root, f), mod) From b6672ccf09825fc7cdf2d589ac3ce022fd2bc548 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 15 Jan 2019 18:06:05 -0500 Subject: [PATCH 6/8] Refactor restart to utils.processes.restart --- core/__init__.py | 21 +-------------------- core/utils/__init__.py | 2 +- core/utils/processes.py | 23 +++++++++++++++++++++++ 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index 7dcd4fd9d..593c4a5a5 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -63,6 +63,7 @@ remove_dir, remove_read_only, remove_torrent, + restart, resume_torrent, sanitize_name, update_download_info_status, @@ -863,23 +864,3 @@ def initialize(section=None): # finished initalizing return True - - -def restart(): - install_type = version_check.CheckVersion().install_type - - status = 0 - popen_list = [] - - if install_type in ('git', 'source'): - popen_list = [sys.executable, APP_FILENAME] - - if popen_list: - popen_list += SYS_ARGV - logger.log(u'Restarting nzbToMedia with {args}'.format(args=popen_list)) - logger.close() - p = subprocess.Popen(popen_list, cwd=os.getcwd()) - p.wait() - status = p.returncode - - os._exit(status) diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 99bb03769..0083fbbd4 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -43,7 +43,7 @@ remove_empty_folders, remove_read_only, ) -from core.utils.processes import RunningProcess +from core.utils.processes import RunningProcess, restart from core.utils.subtitles import import_subs from core.utils.torrents import create_torrent_class, pause_torrent, remove_torrent, resume_torrent diff --git a/core/utils/processes.py b/core/utils/processes.py index 6513a12c1..6edcfa44a 100644 --- a/core/utils/processes.py +++ b/core/utils/processes.py @@ -1,7 +1,10 @@ import os import socket +import subprocess +import sys import core +from core import logger, version_check, APP_FILENAME, SYS_ARGV if os.name == 'nt': from win32event import CreateMutex @@ -90,3 +93,23 @@ def __del__(self): RunningProcess = WindowsProcess else: RunningProcess = PosixProcess + + +def restart(): + install_type = version_check.CheckVersion().install_type + + status = 0 + popen_list = [] + + if install_type in ('git', 'source'): + popen_list = [sys.executable, APP_FILENAME] + + if popen_list: + popen_list += SYS_ARGV + logger.log(u'Restarting nzbToMedia with {args}'.format(args=popen_list)) + logger.close() + p = subprocess.Popen(popen_list, cwd=os.getcwd()) + p.wait() + status = p.returncode + + os._exit(status) From d9436603abb7ada9895f0b717e23a8b4be7e2b62 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 15 Jan 2019 18:11:31 -0500 Subject: [PATCH 7/8] =?UTF-8?q?Bump=20version:=2012.0.6=20=E2=86=92=2012.0?= =?UTF-8?q?.7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- README.md | 2 +- core/__init__.py | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index f9e4c7da6..a1d61c634 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 12.0.6 +current_version = 12.0.7 commit = True tag = False diff --git a/README.md b/README.md index bf3819987..c78ffe838 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -nzbToMedia v12.0.6 +nzbToMedia v12.0.7 ================== Provides an [efficient](https://github.com/clinton-hall/nzbToMedia/wiki/Efficient-on-demand-post-processing) way to handle postprocessing for [CouchPotatoServer](https://couchpota.to/ "CouchPotatoServer") and [SickBeard](http://sickbeard.com/ "SickBeard") (and its [forks](https://github.com/clinton-hall/nzbToMedia/wiki/Failed-Download-Handling-%28FDH%29#sick-beard-and-its-forks)) diff --git a/core/__init__.py b/core/__init__.py index 593c4a5a5..e7280ded5 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -70,7 +70,7 @@ wake_up, ) -__version__ = '12.0.6' +__version__ = '12.0.7' # Client Agents NZB_CLIENTS = ['sabnzbd', 'nzbget', 'manual'] diff --git a/setup.py b/setup.py index 665c5705d..941b44772 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ def read(*names, **kwargs): setup( name='nzbToMedia', - version='12.0.6', + version='12.0.7', license='GPLv3', description='Efficient on demand post processing', long_description=""" From 30872db797e813f9a0906ee41e59a0af331c6f88 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 15 Jan 2019 18:37:45 -0500 Subject: [PATCH 8/8] Update changelog --- changelog.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/changelog.txt b/changelog.txt index 380dca71d..b197e7fa8 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,13 @@ Change_LOG / History +V12.0.7 + +Refactor utils +Fix git subprocess +Fix cleanup script output +Add extra logging for fork detection +Additional code clean up + V12.0.6 Hotfix for Manual Torrent run results.