From 59c06b40cb84d1cc1a80f487147517ff7e922d20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Fri, 13 Nov 2020 19:47:29 +0000 Subject: [PATCH 01/13] refactor requirement parsing in setup.py --- setup.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 226e0c50..dd8f0c93 100644 --- a/setup.py +++ b/setup.py @@ -47,15 +47,11 @@ def _resolve_requirement(req: typing.Any) -> str: def _read_requirements( filename: str="requirements.txt" -) -> typing.Dict[str, typing.List[str]]: +) -> typing.List[str]: reqs = list(parse_requirements(filename, session="libioc")) - return dict( - install_requires=[_resolve_requirement(req) for req in reqs] - ) + return [_resolve_requirement(req) for req in reqs] -ioc_requirements = _read_requirements("requirements.txt") - if sys.version_info < (3, 6): exit("Only Python 3.6 and higher is supported.") @@ -75,7 +71,7 @@ def _read_requirements( packages=find_packages(include=["libioc", "libioc.*"]), package_data={'': ['VERSION']}, include_package_data=True, - install_requires=ioc_requirements["install_requires"], + install_requires=_read_requirements("requirements.txt"), # setup_requires=['pytest-runner'], tests_require=['pytest', 'pytest-cov', 'pytest-pep8'] ) From e0eb867e02493c23171af0d797db9e96928de6df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Fri, 13 Nov 2020 23:35:30 +0000 Subject: [PATCH 02/13] adopt setup.cfg configuration from @igalic https://github.com/bsdci/libioc/pull/764#issuecomment-727064416 --- MANIFEST.in | 1 - Makefile | 26 ++++------------------ libioc/__init__.py | 2 +- requirements.txt | 3 --- setup.cfg | 35 ++++++++++++++++++++++++++++- setup.py | 55 ++-------------------------------------------- 6 files changed, 41 insertions(+), 81 deletions(-) delete mode 100644 requirements.txt diff --git a/MANIFEST.in b/MANIFEST.in index 8feea28b..368ef6fb 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,3 @@ -include requirements.txt include LICENSE.txt include *.md include Makefile diff --git a/Makefile b/Makefile index 47667ba9..7f24e06b 100644 --- a/Makefile +++ b/Makefile @@ -14,26 +14,10 @@ pyver= ${PYTHON:S/^python//:S/.//:C/\([0-9]+\)/\1/} . error "libioc cannot run with a Python version < 3.5" .endif -install: install-deps install-python-requirements - $(PYTHON) -m pip install -U . -install-python-requirements: - $(PYTHON) -m ensurepip - $(PYTHON) -m pip install -Ur requirements.txt -install-python-requirements-dev: install-python-requirements - $(PYTHON) -m pip install -Ur requirements-dev.txt -install-deps: - pkg install -q -y libucl rsync git py$(pyver)-ucl py$(pyver)-libzfs -install-deps-dev: install-deps - if [ "`uname`" = "FreeBSD" ]; then pkg install -y gmake py$(pyver)-setuptools py$(pyver)-sqlite3; fi -install-dev: install-deps-dev install-python-requirements-dev - $(PYTHON) -m pip install -e . +install: + $(PYTHON) setup.py install install-travis: python$(TRAVIS_PYTHON_VERSION) -m pip install -IU flake8-mutable flake8-docstrings flake8-builtins flake8-mypy bandit==1.5.1 bandit-high-entropy-string -uninstall: - $(PYTHON) -m pip uninstall -y ioc - @if [ -f /usr/local/etc/rc.d/ioc ]; then \ - rm /usr/local/etc/rc.d/ioc; \ - fi check: flake8 --version mypy --version @@ -47,11 +31,9 @@ docs: help: @echo " install" @echo " Installs libioc" - @echo " uninstall" - @echo " Removes libioc" @echo " test" @echo " Run unit tests with pytest" @echo " check" @echo " Run static linters & other static analysis tests" - @echo " install-dev" - @echo " Install dependencies needed to run `check`" + @echo " docs" + @echo " Generate documentation" diff --git a/libioc/__init__.py b/libioc/__init__.py index 21547bcd..a6523740 100644 --- a/libioc/__init__.py +++ b/libioc/__init__.py @@ -59,7 +59,7 @@ class _IocModule(sys.modules["libioc"].__class__): ] def __getattribute__(self, key: str) -> typing.Any: - if key == "VERSION": + if (key == "VERSION") or (key == "__version__"): return _get_version() if key.startswith("_") is True: return super().__getattribute__(key) diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 44da16ed..00000000 --- a/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -gitpython -freebsd_sysctl==0.0.7 -jail==0.0.12 diff --git a/setup.cfg b/setup.cfg index f7fa6d93..03455ec1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,37 @@ [metadata] -description-file = README.md +name = libioc +version = attr: libioc.__version__ +description = A Python library to manage jails with ioc{age,cell}. +long_description = file: README.md +long_description_content_type = text/markdown +url = https://github.com/bsdci/libioc +author = Stefan Grönke +author_email = stefan@gronke.net +python_requires = >=3.6 +platforms = + FreeBSD +license = BSD 2-Clause License +keywords = + FreeBSD + jail + ioc +classifiers = + License :: OSI Approved :: BSD 2-Clause License + Operating System :: POSIX :: BSD :: FreeBSD + Programming Language :: Python :: 3 :: Only + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Typing :: Typed +install_requires = + gitpython + freebsd_sysctl==0.0.7 + jail==0.0.11 +tests_require = + pytest-runner + pytest + pytest-cov + pytest-pep8 [mypy] python_version = 3.7 @@ -41,6 +73,7 @@ disallow_untyped_defs=True addopts = -v -x -rs --ignore=setup.py --pep8 --cov-report term-missing --no-cov-on-fail --cov=libioc libioc tests pep8maxlinelength = 80 pep8ignore = * ALL + [aliases] test=pytest diff --git a/setup.py b/setup.py index dd8f0c93..130a9f77 100644 --- a/setup.py +++ b/setup.py @@ -23,56 +23,5 @@ # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. """Installs libioc using setuptools.""" -import sys -import typing -from setuptools import find_packages, setup -try: - from pip._internal.req import parse_requirements -except ModuleNotFoundError: - from pip.req import parse_requirements - -try: - import setuptools_scm.integration - setuptools_scm.integration.find_files = lambda _: [] -except ImportError: - pass - - -def _resolve_requirement(req: typing.Any) -> str: - if req.__class__.__name__ == "ParsedRequirement": - return str(req.requirement) - else: - return f"{req.name}{req.specifier}" - - -def _read_requirements( - filename: str="requirements.txt" -) -> typing.List[str]: - reqs = list(parse_requirements(filename, session="libioc")) - return [_resolve_requirement(req) for req in reqs] - - -if sys.version_info < (3, 6): - exit("Only Python 3.6 and higher is supported.") - -with open("libioc/VERSION", "r") as f: - version = f.read().split()[0] - -setup( - name='libioc', - license='BSD', - version=version, - description='A Python library to manage jails with ioc{age,cell}', - keywords='FreeBSD jail ioc', - author='ioc Contributors', - author_email='authors@libioc.io', - url='https://github.com/bsdci/libioc', - python_requires='>=3.6', - packages=find_packages(include=["libioc", "libioc.*"]), - package_data={'': ['VERSION']}, - include_package_data=True, - install_requires=_read_requirements("requirements.txt"), - # setup_requires=['pytest-runner'], - tests_require=['pytest', 'pytest-cov', 'pytest-pep8'] -) - +from setuptools import setup +setup() From 8ad7d29813f1f5e2309f14249d4a1cd731d5cc5a Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Thu, 21 May 2020 10:37:31 -0700 Subject: [PATCH 03/13] Include missing files for sdist Closes #759 --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 368ef6fb..89e61910 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,4 @@ include *.md include Makefile include *.py recursive-include libioc *.py +include libioc/VERSION From 4dc6ba0637b4eef34cab01555cb8fb589e75837d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Sat, 14 Nov 2020 00:10:02 +0000 Subject: [PATCH 04/13] set setup.cfg options --- setup.cfg | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/setup.cfg b/setup.cfg index 03455ec1..7321b797 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,6 +23,11 @@ classifiers = Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Typing :: Typed + +[options] +zip_safe = False +include_package_data = True +packages = find: install_requires = gitpython freebsd_sysctl==0.0.7 @@ -33,6 +38,9 @@ tests_require = pytest-cov pytest-pep8 +[options.package_data] +* = VERSION + [mypy] python_version = 3.7 # override flake8-mypy defaults, since we provide (missing) types From ab08376e2c7e076a1db2485f3b88d17c45e67add Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Mon, 21 Jun 2021 14:27:24 +0200 Subject: [PATCH 05/13] chore: alias lambda function to prevent T484 --- libioc/Jail.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libioc/Jail.py b/libioc/Jail.py index 5f525b84..32157797 100644 --- a/libioc/Jail.py +++ b/libioc/Jail.py @@ -763,6 +763,9 @@ def _start_dependant_jails( yield jailDependantsStartEvent.skip("No dependant jails") return + def _sort(x: JailResource) -> int: + return int(x.config["priority"]) + dependant_jails = sorted( libioc.Jails.JailsGenerator( filters=_depends, @@ -770,7 +773,7 @@ def _start_dependant_jails( logger=self.logger, zfs=self.zfs ), - key=lambda x: x.config["priority"] + key=_sort ) for dependant_jail in dependant_jails: From 6881facffb06da446f441407fab9dbea542e9c74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Wed, 23 Jun 2021 13:10:50 +0200 Subject: [PATCH 06/13] add iocage/iocell to package keywords addresses https://github.com/bsdci/libioc/pull/764#discussion_r523384533 --- setup.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 7321b797..095515bf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,7 +14,8 @@ license = BSD 2-Clause License keywords = FreeBSD jail - ioc + iocage + iocell classifiers = License :: OSI Approved :: BSD 2-Clause License Operating System :: POSIX :: BSD :: FreeBSD From 0e05beb3adb95cd470c156a64229d3b820575353 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Wed, 23 Jun 2021 13:21:38 +0200 Subject: [PATCH 07/13] cirrus.yml uses make install --- .cirrus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index d0eb71ed..e17b8493 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -21,6 +21,6 @@ task: - sysctl -a - mount -t fdescfs null /dev/fd - pkg install -y python3 - - make install-dev + - make install test_script: - env ZPOOL="ioc-test-`uname -r`" make test From ff3c060eb40303bd2caec7bcb65d1a7847c2ace8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Wed, 23 Jun 2021 13:29:33 +0200 Subject: [PATCH 08/13] install python-setuptools on Cirrus CI --- .cirrus.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.cirrus.yml b/.cirrus.yml index e17b8493..71b083ab 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -21,6 +21,7 @@ task: - sysctl -a - mount -t fdescfs null /dev/fd - pkg install -y python3 + - python3 -m pip install -U setuptools - make install test_script: - env ZPOOL="ioc-test-`uname -r`" make test From 98267036a5abb3fd5cbaddbcb71783abf79ab024 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Wed, 23 Jun 2021 13:32:07 +0200 Subject: [PATCH 09/13] enable FreeBSD 13.0 builds on Cirrus CI --- .cirrus.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.cirrus.yml b/.cirrus.yml index 71b083ab..95ee8cc8 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -2,6 +2,7 @@ task: freebsd_instance: matrix: image: freebsd-12-1-release-amd64 + image: freebsd-13-0-release-amd64 pip_cache: folder: ~/.cache/pip fingerprint_script: cat requirements* From bc96b129ab0fe68a069f40033838fa8e77263477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Wed, 23 Jun 2021 13:36:53 +0200 Subject: [PATCH 10/13] ensure pip is installed on Cirrus CI --- .cirrus.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.cirrus.yml b/.cirrus.yml index 95ee8cc8..97f1d66c 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -22,6 +22,7 @@ task: - sysctl -a - mount -t fdescfs null /dev/fd - pkg install -y python3 + - python3 -m ensurepip - python3 -m pip install -U setuptools - make install test_script: From d09f149d0e9f53059afcafacdedf53572ca3ef0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Tue, 13 Jul 2021 17:24:44 +0200 Subject: [PATCH 11/13] upgrade python jail module to 0.0.14 fixes ParsedRequirement error installing libioc --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 095515bf..b65b571d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,7 +32,7 @@ packages = find: install_requires = gitpython freebsd_sysctl==0.0.7 - jail==0.0.11 + jail==0.0.14 tests_require = pytest-runner pytest From 6936f902fe0731c18890db9e0916f45bbfc02f19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Tue, 13 Jul 2021 16:08:16 +0000 Subject: [PATCH 12/13] pull freebsd-update scripts from git instead of SVN --- libioc/ResourceUpdater.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/libioc/ResourceUpdater.py b/libioc/ResourceUpdater.py index f0618562..0d73c899 100644 --- a/libioc/ResourceUpdater.py +++ b/libioc/ResourceUpdater.py @@ -589,14 +589,13 @@ def _get_release_trunk_file_url( filename: str ) -> str: - if release.name == "11.0-RELEASE": - release_name = "11.0.1" - else: - fragments = release.name.split("-", maxsplit=1) - release_name = f"{fragments[0]}.0" + fragments = release.name.split("-", maxsplit=1) + release_name = f"{fragments[0]}" + + base_url = "https://cgit.freebsd.org/src/plain/" + query_string = f"?h=releng/{release_name}" - base_url = "https://svn.freebsd.org/base/release" - return f"{base_url}/{release_name}/{filename}" + return f"{base_url}{filename}{query_string}" @property def _base_release_symlink_location(self) -> str: From 1d6a4e60878e8f85c1674aeed0401d5aaa235a69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Wed, 14 Jul 2021 14:19:49 +0200 Subject: [PATCH 13/13] only delete snapshots while cloning when an error occurred --- libioc/ZFS.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libioc/ZFS.py b/libioc/ZFS.py index 449a5c6c..de7fc89d 100644 --- a/libioc/ZFS.py +++ b/libioc/ZFS.py @@ -170,7 +170,7 @@ def clone_dataset( except libzfs.ZFSException as e: snapshot_error = e - if delete_snapshot is True: + if (delete_snapshot is True) and (snapshot_error is not None): snapshot.delete(recursive=True) if snapshot_error is not None: