From 07c9b40e7bfd5b907a777960db124ac63012a326 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=BCnsebeck?= Date: Thu, 23 Jun 2022 14:17:09 +0200 Subject: [PATCH 1/8] Use importlib.metadata.version instead of pkg_resources.get_distribution --- ocrd/ocrd/workspace_bagger.py | 12 ++++++++---- ocrd_utils/ocrd_utils/constants.py | 7 +++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/ocrd/ocrd/workspace_bagger.py b/ocrd/ocrd/workspace_bagger.py index 28ae155b9a..6ba6db7160 100644 --- a/ocrd/ocrd/workspace_bagger.py +++ b/ocrd/ocrd/workspace_bagger.py @@ -6,8 +6,6 @@ import re import tempfile import sys - -from pkg_resources import get_distribution from bagit import Bag, make_manifests # pylint: disable=no-name-in-module from ocrd_utils import ( @@ -25,6 +23,12 @@ from .workspace import Workspace +try: + from importlib.metadata import version +except ImportError: + from importlib_metadata import version + + tempfile.tempdir = '/tmp' # TODO hard-coded BACKUPDIR = join('/tmp', TMP_BAGIT_PREFIX + 'backup') @@ -123,8 +127,8 @@ def _set_bag_info(self, bag, total_bytes, total_files, ocrd_identifier, ocrd_man bag.info['BagIt-Profile-Identifier'] = OCRD_BAGIT_PROFILE_URL bag.info['Bag-Software-Agent'] = 'ocrd/core %s (bagit.py %s, bagit_profile %s) [cmdline: "%s"]' % ( VERSION, # TODO - get_distribution('bagit').version, - get_distribution('bagit_profile').version, + version('bagit'), + version('bagit_profile'), ' '.join(sys.argv)) bag.info['Ocrd-Identifier'] = ocrd_identifier diff --git a/ocrd_utils/ocrd_utils/constants.py b/ocrd_utils/ocrd_utils/constants.py index 121e5df612..6f0202dde1 100644 --- a/ocrd_utils/ocrd_utils/constants.py +++ b/ocrd_utils/ocrd_utils/constants.py @@ -1,10 +1,13 @@ """ Constants for ocrd_utils. """ -from pkg_resources import get_distribution from re import compile as regex_compile from os import environ from os.path import join, expanduser +try: + from importlib.metadata import version +except ImportError: + from importlib_metadata import version __all__ = [ 'EXT_TO_MIME', @@ -22,7 +25,7 @@ 'XDG_DATA_HOME', ] -VERSION = get_distribution('ocrd_utils').version +VERSION = version('ocrd_utils') MIMETYPE_PAGE = 'application/vnd.prima.page+xml' From 0b7c125804213f17e1a2f6754575ff28aa13b855 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=BCnsebeck?= Date: Thu, 23 Jun 2022 14:18:36 +0200 Subject: [PATCH 2/8] Use importlib based replacements instead of pkg_resources.resource_* --- ocrd/ocrd/constants.py | 2 +- ocrd/requirements.txt | 2 + ocrd_models/ocrd_models/constants.py | 2 +- ocrd_utils/ocrd_utils/package_resources.py | 45 ++++++++++++++++++++ ocrd_validators/ocrd_validators/constants.py | 2 +- 5 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 ocrd_utils/ocrd_utils/package_resources.py diff --git a/ocrd/ocrd/constants.py b/ocrd/ocrd/constants.py index 1d436a7fa9..2e9c17c649 100644 --- a/ocrd/ocrd/constants.py +++ b/ocrd/ocrd/constants.py @@ -1,7 +1,7 @@ """ Constants for ocrd. """ -from pkg_resources import resource_filename +from ocrd_utils.package_resources import resource_filename __all__ = [ 'TMP_PREFIX', diff --git a/ocrd/requirements.txt b/ocrd/requirements.txt index 2da0163b74..42e6c16a99 100644 --- a/ocrd/requirements.txt +++ b/ocrd/requirements.txt @@ -8,3 +8,5 @@ Flask jsonschema pyyaml Deprecated == 1.2.0 +importlib_metadata;python_version<'3.8' +importlib_resources;python_version<'3.8' diff --git a/ocrd_models/ocrd_models/constants.py b/ocrd_models/ocrd_models/constants.py index 6c8b0e1017..b3fe89a4c9 100644 --- a/ocrd_models/ocrd_models/constants.py +++ b/ocrd_models/ocrd_models/constants.py @@ -1,7 +1,7 @@ """ Constants for ocrd_models. """ -from pkg_resources import resource_string +from ocrd_utils.package_resources import resource_string import re __all__ = [ diff --git a/ocrd_utils/ocrd_utils/package_resources.py b/ocrd_utils/ocrd_utils/package_resources.py new file mode 100644 index 0000000000..0f268793ea --- /dev/null +++ b/ocrd_utils/ocrd_utils/package_resources.py @@ -0,0 +1,45 @@ +import atexit +from contextlib import ExitStack +from pathlib import Path + +try: + from importlib.resources import path, read_binary +except ImportError: + from importlib_resources import path, read_binary # type: ignore + + +_file_manager = ExitStack() +atexit.register(_file_manager.close) + + +def resource_filename(package: str, resource: str) -> Path: + """ + Reimplementation of the function with the same name from pkg_resources + + Using importlib for better performance + + package : str + The package from where to start looking for resource (often __name__) + resource : str + The resource to look up + """ + parent_package = package.rsplit('.',1)[0] + return _file_manager.enter_context(path(parent_package, resource)) + + +def resource_string(package: str, resource: str) -> bytes: + """ + Reimplementation of the function with the same name from pkg_resources + + Using importlib for better performance + + package : str + The package from where to start looking for resource (often __name__) + resource : str + The resource to look up + """ + parent_package = package.rsplit('.',1)[0] + return read_binary(parent_package, resource) + + +__all__ = ['resource_filename', 'resource_string'] \ No newline at end of file diff --git a/ocrd_validators/ocrd_validators/constants.py b/ocrd_validators/ocrd_validators/constants.py index 25d2e0e53b..5497102f25 100644 --- a/ocrd_validators/ocrd_validators/constants.py +++ b/ocrd_validators/ocrd_validators/constants.py @@ -2,7 +2,7 @@ Constants for ocrd_validators. """ import yaml -from pkg_resources import resource_string, resource_filename +from ocrd_utils.package_resources import resource_string, resource_filename __all__ = [ 'OCRD_TOOL_SCHEMA', From 6f0728e0f1e97b686f3e4c23388c22ef5425cb25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=BCnsebeck?= Date: Thu, 23 Jun 2022 14:19:19 +0200 Subject: [PATCH 3/8] Use importlib based replacements instead of pkg_resources.resource_* and move ocrd-tool.json to a importlib findable place --- ocrd/ocrd/processor/builtin/dummy_processor.py | 4 ++-- ocrd/ocrd/processor/builtin/{dummy => }/ocrd-tool.json | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename ocrd/ocrd/processor/builtin/{dummy => }/ocrd-tool.json (100%) diff --git a/ocrd/ocrd/processor/builtin/dummy_processor.py b/ocrd/ocrd/processor/builtin/dummy_processor.py index 9a1ad511e7..cce37a7430 100644 --- a/ocrd/ocrd/processor/builtin/dummy_processor.py +++ b/ocrd/ocrd/processor/builtin/dummy_processor.py @@ -1,6 +1,6 @@ # pylint: disable=missing-module-docstring,invalid-name from os.path import join, basename -from pkg_resources import resource_string +from ocrd_utils.package_resources import resource_string import click @@ -17,7 +17,7 @@ ) from ocrd_modelfactory import page_from_file -OCRD_TOOL = parse_json_string_with_comments(resource_string(__name__, 'dummy/ocrd-tool.json').decode('utf8')) +OCRD_TOOL = parse_json_string_with_comments(resource_string(__name__, 'ocrd-tool.json').decode('utf8')) class DummyProcessor(Processor): """ diff --git a/ocrd/ocrd/processor/builtin/dummy/ocrd-tool.json b/ocrd/ocrd/processor/builtin/ocrd-tool.json similarity index 100% rename from ocrd/ocrd/processor/builtin/dummy/ocrd-tool.json rename to ocrd/ocrd/processor/builtin/ocrd-tool.json From 79651f24d9c3f8bef4e6c8e34bd789a26763c185 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=BCnsebeck?= Date: Thu, 23 Jun 2022 14:20:00 +0200 Subject: [PATCH 4/8] upgrade setuptools fpr faster console_scripts entrypoint generation --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6da911a0f2..54396a0d13 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,7 @@ deps-test: # (Re)install the tool install: - $(PIP) install -U pip wheel + $(PIP) install -U pip wheel setuptools for mod in $(BUILD_ORDER);do (cd $$mod ; $(PIP_INSTALL) .);done # Install with pip install -e From 50eae8b02b98d296ba600efcff7543c159daf0a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=BCnsebeck?= Date: Thu, 23 Jun 2022 14:20:40 +0200 Subject: [PATCH 5/8] Use fastentrypoints --- Makefile | 2 +- ocrd/setup.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 54396a0d13..2d5ebda7e9 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,7 @@ deps-test: # (Re)install the tool install: - $(PIP) install -U pip wheel setuptools + $(PIP) install -U pip wheel setuptools fastentrypoints for mod in $(BUILD_ORDER);do (cd $$mod ; $(PIP_INSTALL) .);done # Install with pip install -e diff --git a/ocrd/setup.py b/ocrd/setup.py index 0c8c0fa2ae..0269893e28 100644 --- a/ocrd/setup.py +++ b/ocrd/setup.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import fastentrypoints from setuptools import setup, find_packages from ocrd_utils import VERSION From cf3eb36133a1ae6cf1fef58d183b861b16f08b0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=BCnsebeck?= Date: Wed, 29 Jun 2022 20:48:21 +0200 Subject: [PATCH 6/8] Move importlib_* requirements to correct module --- ocrd/requirements.txt | 4 +--- ocrd_utils/requirements.txt | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ocrd/requirements.txt b/ocrd/requirements.txt index 42e6c16a99..ca62ed9370 100644 --- a/ocrd/requirements.txt +++ b/ocrd/requirements.txt @@ -7,6 +7,4 @@ opencv-python-headless Flask jsonschema pyyaml -Deprecated == 1.2.0 -importlib_metadata;python_version<'3.8' -importlib_resources;python_version<'3.8' +Deprecated == 1.2.0 \ No newline at end of file diff --git a/ocrd_utils/requirements.txt b/ocrd_utils/requirements.txt index 300ed90949..de4e7adee3 100644 --- a/ocrd_utils/requirements.txt +++ b/ocrd_utils/requirements.txt @@ -3,3 +3,5 @@ Pillow >= 7.2.0 # tensorflow versions might require different versions numpy atomicwrites >= 1.3.0 +importlib_metadata;python_version<'3.8' +importlib_resources;python_version<'3.8' From 0caa1a1f955c801aa759b04c61c7a5be62175284 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=BCnsebeck?= Date: Wed, 29 Jun 2022 20:48:58 +0200 Subject: [PATCH 7/8] Use importlib based replacement for pkg_resources get_distribution --- ocrd/ocrd/workspace_bagger.py | 11 +++-------- ocrd_utils/ocrd_utils/constants.py | 8 +++----- ocrd_utils/ocrd_utils/package_resources.py | 6 +++++- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/ocrd/ocrd/workspace_bagger.py b/ocrd/ocrd/workspace_bagger.py index 6ba6db7160..ac215fa7e7 100644 --- a/ocrd/ocrd/workspace_bagger.py +++ b/ocrd/ocrd/workspace_bagger.py @@ -20,15 +20,10 @@ from ocrd_validators.constants import BAGIT_TXT, TMP_BAGIT_PREFIX, OCRD_BAGIT_PROFILE_URL from ocrd_modelfactory import page_from_file from ocrd_models.ocrd_page import to_xml +from ocrd_utils.package_resources import get_distribution from .workspace import Workspace -try: - from importlib.metadata import version -except ImportError: - from importlib_metadata import version - - tempfile.tempdir = '/tmp' # TODO hard-coded BACKUPDIR = join('/tmp', TMP_BAGIT_PREFIX + 'backup') @@ -127,8 +122,8 @@ def _set_bag_info(self, bag, total_bytes, total_files, ocrd_identifier, ocrd_man bag.info['BagIt-Profile-Identifier'] = OCRD_BAGIT_PROFILE_URL bag.info['Bag-Software-Agent'] = 'ocrd/core %s (bagit.py %s, bagit_profile %s) [cmdline: "%s"]' % ( VERSION, # TODO - version('bagit'), - version('bagit_profile'), + get_distribution('bagit').version, + get_distribution('bagit_profile').version, ' '.join(sys.argv)) bag.info['Ocrd-Identifier'] = ocrd_identifier diff --git a/ocrd_utils/ocrd_utils/constants.py b/ocrd_utils/ocrd_utils/constants.py index 6f0202dde1..1164025081 100644 --- a/ocrd_utils/ocrd_utils/constants.py +++ b/ocrd_utils/ocrd_utils/constants.py @@ -4,10 +4,8 @@ from re import compile as regex_compile from os import environ from os.path import join, expanduser -try: - from importlib.metadata import version -except ImportError: - from importlib_metadata import version + +from ocrd_utils.package_resources import get_distribution __all__ = [ 'EXT_TO_MIME', @@ -25,7 +23,7 @@ 'XDG_DATA_HOME', ] -VERSION = version('ocrd_utils') +VERSION = get_distribution('ocrd_utils').version MIMETYPE_PAGE = 'application/vnd.prima.page+xml' diff --git a/ocrd_utils/ocrd_utils/package_resources.py b/ocrd_utils/ocrd_utils/package_resources.py index 0f268793ea..57644ba379 100644 --- a/ocrd_utils/ocrd_utils/package_resources.py +++ b/ocrd_utils/ocrd_utils/package_resources.py @@ -7,6 +7,10 @@ except ImportError: from importlib_resources import path, read_binary # type: ignore +try: + from importlib.metadata import distribution as get_distribution +except ImportError: + from importlib_metadata import distribution as get_distribution _file_manager = ExitStack() atexit.register(_file_manager.close) @@ -42,4 +46,4 @@ def resource_string(package: str, resource: str) -> bytes: return read_binary(parent_package, resource) -__all__ = ['resource_filename', 'resource_string'] \ No newline at end of file +__all__ = ['resource_filename', 'resource_string', 'get_distribution'] \ No newline at end of file From b7e575499d9b30f3c39ca02a03dfee0415c66b91 Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Fri, 15 Jul 2022 15:50:42 +0200 Subject: [PATCH 8/8] utils.package_resources: reference migration guide --- ocrd_utils/ocrd_utils/package_resources.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ocrd_utils/ocrd_utils/package_resources.py b/ocrd_utils/ocrd_utils/package_resources.py index 57644ba379..ee01d046f4 100644 --- a/ocrd_utils/ocrd_utils/package_resources.py +++ b/ocrd_utils/ocrd_utils/package_resources.py @@ -12,6 +12,7 @@ except ImportError: from importlib_metadata import distribution as get_distribution +# See https://importlib-resources.readthedocs.io/en/latest/migration.html#pkg-resources-resource-filename _file_manager = ExitStack() atexit.register(_file_manager.close) @@ -46,4 +47,4 @@ def resource_string(package: str, resource: str) -> bytes: return read_binary(parent_package, resource) -__all__ = ['resource_filename', 'resource_string', 'get_distribution'] \ No newline at end of file +__all__ = ['resource_filename', 'resource_string', 'get_distribution']