Migrate Makefile to tox 4

* Add tox >= 4.0 as a dev dependency
* Add tox configuration to setup.cfg
  - I chose setup.cfg because pyproject.toml support in tox is awkward (tox-dev/tox#999)
  - Specified flake8 < 6.0.0 until we can get around to replacing type comments with type annotations.
    PyCQA/pyflakes#747
* Modify Makefile to use tox instead of running the commands directly
* Update .gitignore for relevant new files and directories
* Update requirements files
  - Newer tarball in freeze-requirements.sh because the older one was sometimes breaking in tox.
* Set pylint configuration to ignore tests/legacy rather than manually excluding it

A future commit will configure tox in Github actions.

.gitignore

@@ -1,6 +1,7 @@
 build
 Cargo.lock
 .coverage**
+coverage-*
 demos/*/parts/
 demos/*/prime/
 demos/**/*.snap
@@ -21,6 +22,7 @@ pip-wheel-metadata/
 prime
 *.pyc
 __pycache__
+.pytest_cache
 *.snap
 snap/.snapcraft/
 .spread-reuse.*
@@ -29,6 +31,8 @@ stage
 target
 tests/unit/snap/
 tests/unit/stage/
+test-results*
+.tox
 .vscode
 venv
 .venv

Makefile

@@ -3,58 +3,55 @@ SOURCES_LEGACY=snapcraft_legacy tests/legacy
 
 .PHONY: autoformat-black
 autoformat-black:
-	black $(SOURCES) $(SOURCES_LEGACY)
+	tox run -e format
 
 .PHONY: freeze-requirements
 freeze-requirements:
 	tools/freeze-requirements.sh
 
 .PHONY: test-black
 test-black:
-	black --check --diff $(SOURCES) $(SOURCES_LEGACY)
+	tox run -e black
 
 .PHONY: test-codespell
 test-codespell:
-	codespell
+	tox run -e codespell
 
 .PHONY: test-flake8
 test-flake8:
-	python3 -m flake8 $(SOURCES) $(SOURCES_LEGACY)
+	tox run -e flake
 
 .PHONY: test-isort
 test-isort:
-	isort --check $(SOURCES) $(SOURCES_LEGACY)
+	tox run -e isort
 
 .PHONY: test-mypy
 test-mypy:
-	mypy $(SOURCES)
+	tox run -e mypy
 
 .PHONY: test-pydocstyle
 test-pydocstyle:
-	pydocstyle snapcraft
+	tox run -e docstyle
 
 .PHONY: test-pylint
 test-pylint:
-	pylint snapcraft
-	pylint tests/*.py tests/unit --disable=invalid-name,missing-module-docstring,missing-function-docstring,duplicate-code,protected-access,unspecified-encoding,too-many-public-methods,too-many-arguments,too-many-lines
+	tox run -e pylint
 
 .PHONY: test-pyright
 test-pyright:
-	pyright $(SOURCES)
+	tox run -e pyright
 
 .PHONY: test-shellcheck
 test-shellcheck:
-# Skip third-party gradlew script.
-	find . \( -name .git -o -name gradlew \) -prune -o -print0 | xargs -0 file -N | grep shell.script | cut -f1 -d: | xargs shellcheck
-	./tools/spread-shellcheck.py spread.yaml tests/spread/
+	tox run -e spellcheck
 
 .PHONY: test-legacy-units
 test-legacy-units:
-	pytest --cov-report=xml --cov=snapcraft tests/legacy/unit
+	tox run -e py38-dev-legacy
 
 .PHONY: test-units
 test-units: test-legacy-units
-	pytest --cov-report=xml --cov=snapcraft tests/unit
+	tox run -e py38-dev-unit
 
 .PHONY: tests
 tests: tests-static test-units

pyproject.toml

@@ -20,6 +20,9 @@ exclude = '''
   | ^/prime
 )/
 '''
+# Targeting future versions as well so we don't have black reformatting code
+# en masse later.
+target_version = ["py38", "py310", "py311"]
 
 [tool.isort]
 # black-compatible isort configuration
@@ -30,6 +33,9 @@ use_parentheses = true
 ensure_newline_before_comments = true
 line_length = 88
 
+[tool.pylint.main]
+ignore-paths = ["tests/legacy"]
+
 [tool.pylint.messages_control]
 # duplicate-code can't be disabled locally: https://github.com/PyCQA/pylint/issues/214
 disable = "too-few-public-methods,fixme,use-implicit-booleaness-not-comparison,duplicate-code,unnecessary-lambda-assignment"
@@ -51,3 +57,28 @@ load-plugins = "pylint_fixme_info,pylint_pytest"
 
 [tool.pylint.SIMILARITIES]
 min-similarity-lines=10
+
+[tool.mypy]
+python_version = 3.8
+ignore_missing_imports = true
+follow_imports = "silent"
+exclude = [
+    "build",
+    "snapcraft_legacy",
+    "tests/spread",
+    "tests/legacy",
+    "tools",
+]
+
+[tool.pyright]
+include = ["snapcraft", "tests"]
+exclude = ["tests/legacy", "tests/spread", "build"]
+pythonVersion = "3.8"
+
+[tool.pytest.ini_options]
+minversion = 7.0
+required_plugins = ["pytest-cov>=4.0", "pytest-mock>=3.10", "pytest-subprocess>=1.4"]
+
+[tool.coverage.run]
+branch = true
+source = ["snapcraft"]

requirements-devel.txt

@@ -1,6 +1,7 @@
 astroid==2.13.2
 attrs==22.2.0
 black==22.12.0
+cachetools==5.2.0
 catkin-pkg==0.5.2
 certifi==2022.12.7
 cffi==1.15.1
@@ -9,6 +10,7 @@ charset-normalizer==2.1.1
 click==8.1.3
 codespell==2.2.2
 coverage==7.0.4
+colorama==0.4.6
 craft-cli==1.2.0
 craft-grammar==1.1.1
 craft-parts==1.17.1
@@ -17,10 +19,12 @@ craft-store==2.3.0
 cryptography==3.4
 Deprecated==1.2.13
 dill==0.3.6
+distlib==0.3.6
 distro==1.8.0
 docutils==0.19
 exceptiongroup==1.1.0
 extras==1.0.0
+filelock==3.8.2
 fixtures==4.0.1
 flake8==6.0.0
 gnupg==2.3.1
@@ -73,6 +77,7 @@ pylint-pytest==1.1.2
 pylxd==2.3.1
 pymacaroons==0.13.0
 pyparsing==3.0.9
+pyproject_api==1.2.1
 pyramid==2.0
 pyRFC3339==1.1
 pytest==7.2.0
@@ -102,6 +107,8 @@ tinydb==4.7.0
 toml==0.10.2
 tomli==2.0.1
 tomlkit==0.11.6
+tox==4.0.10
+tomlkit==0.11.6
 translationstring==1.4
 types-Deprecated==1.2.9
 types-docutils==0.19.1.1
@@ -113,15 +120,16 @@ types-urllib3==1.26.25.4
 typing_extensions==4.4.0
 urllib3==1.26.13
 venusian==3.0.0
+virtualenv==20.17.1
 wadllib==1.3.6
 WebOb==1.8.7
 wrapt==1.14.1
 ws4py==0.5.1
 zipp==3.11.0
 zope.deprecation==4.4.0
 zope.interface==5.5.2
-python-apt @ https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/python-apt/2.0.0ubuntu0.20.04.6/python-apt_2.0.0ubuntu0.20.04.6.tar.xz; sys.platform == "linux"
+python-apt @ https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/python-apt/2.0.0ubuntu0.20.04.8/python-apt_2.0.0ubuntu0.20.04.8.tar.xz; sys.platform == "linux"
 PyNaCl==1.4.0; sys.platform != "linux"
 PyNaCl @ https://files.pythonhosted.org/packages/61/ab/2ac6dea8489fa713e2b4c6c5b549cc962dd4a842b5998d9e80cf8440b7cd/PyNaCl-1.3.0.tar.gz; sys.platform == "linux"
 setuptools==49.6.0
-pyinstaller==4.10; sys.platform == "win32"
+pyinstaller==4.3; sys.platform == "win32"

requirements.txt

@@ -69,7 +69,7 @@ wadllib==1.3.6
 wrapt==1.14.1
 ws4py==0.5.1
 zipp==3.11.0
-python-apt @ https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/python-apt/2.0.0ubuntu0.20.04.6/python-apt_2.0.0ubuntu0.20.04.6.tar.xz; sys.platform == "linux"
+python-apt @ https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/python-apt/2.0.0ubuntu0.20.04.8/python-apt_2.0.0ubuntu0.20.04.8.tar.xz; sys.platform == "linux"
 PyNaCl==1.4.0; sys.platform != "linux"
 PyNaCl @ https://files.pythonhosted.org/packages/61/ab/2ac6dea8489fa713e2b4c6c5b549cc962dd4a842b5998d9e80cf8440b7cd/PyNaCl-1.3.0.tar.gz; sys.platform == "linux"
 setuptools==49.6.0

setup.cfg

@@ -30,11 +30,6 @@ exclude =
     stage,
     prime
 
-[mypy]
-python_version = 3.8
-ignore_missing_imports = True
-follow_imports = silent
-
 [pycodestyle]
 max-line-length = 88
 ignore = E203,E501
@@ -45,3 +40,138 @@ ignore = E203,E501
 # D213 Multi-line docstring summary should start at the second line (reason: pep257 default)
 ignore = D107, D203, D213
 ignore_decorators = overrides
+
+[tox:tox]
+min_version = 4.0
+env_list =
+    # Parametrized environments.
+    # First parameter allows us to choose Python 3.8 or 3.10.
+    # Second parameter chooses how to define the environment:
+    #  dev: using requirements-devel.txt
+    #  prod: using requirements.txt (but additionally installing pytest, pytest-cov, pytest-mock and pytest-subprocess)
+    #  noreq: Without either requirements file (but including dev requirements)
+    # Third parameter selects the current unit tests (unit) or the legacy unit tests (legacy)
+    py{38,310}-{dev,prod,noreq}-{unit,legacy}
+skip_missing_interpreters = true
+labels =
+    # Minimal testing environments. run with `tox run-parallel -m test`
+    test = py38-dev-unit,py38-dev-legacy
+    # Test in Python 3.10 from an empty environment.
+    future-test = py310-noreq-unit,py310-noreq-legacy
+
+[testenv]
+package = wheel
+wheel_build_env = .pkg
+deps =
+    dev,pylint: -r{tox_root}/requirements-devel.txt
+    prod: -r{tox_root}/requirements.txt
+    noreq,pyright: python-apt@ https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/python-apt/2.0.0ubuntu0.20.04.8/python-apt_2.0.0ubuntu0.20.04.8.tar.xz
+
+[testenv:py{38,310}-noreq-unit]
+description = Run the unit tests based on only the package's specifications
+platform = linux
+install_command = pip install --no-binary PyNaCl {opts} {packages}
+extras = dev
+commands =
+    pytest {tty:--color=yes} --cov-report=xml:coverage-{env_name}.xml --junit-xml=test-results-{env_name}.xml tests/unit
+
+[testenv:py{38,310}-noreq-legacy]
+description = Run the legacy unit tests based on only the package's specifications
+platform = linux
+install_command = pip install --no-binary PyNaCl {opts} {packages}
+extras = dev
+commands =
+    pytest {tty:--color=yes} --cov-report=xml:coverage-{env_name}.xml --junit-xml=test-results-{env_name}.xml tests/legacy/unit {posargs}
+
+[testenv:py{38,310}-{dev,prod}-unit]
+description = Run the unit tests based on the relevant requirements file
+commands =
+    pytest {tty:--color=yes} --cov-report=xml:coverage-{env_name}.xml --junit-xml=test-results-{env_name}.xml tests/unit {posargs}
+
+[testenv:py{38,310}-{dev,prod}-legacy]
+description = Run the legacy unit tests based on the relevant requirements file
+commands =
+    pytest {tty:--color=yes} --cov-report=xml:coverage-{env_name}.xml --junit-xml=test-results-{env_name}.xml tests/legacy/unit {posargs}
+
+[testenv:format]
+description = Format with black
+labels = fix
+deps = black
+skip_install = true
+# Note: this does not include `snapcraft_legacy` as it contains several files that need reformatting.
+commands = black setup.py snapcraft tests
+
+[testenv:mypy]
+description = Run mypy
+skip_install = true
+labels = type, lint
+deps = mypy
+commands = mypy --install-types --non-interactive .
+
+[testenv:pyright]
+description = run PyRight
+labels = type, lint
+use_develop = true
+extras = dev
+allowlist_externals = pyright
+commands = pyright snapcraft tests
+
+[testenv:black]
+description = run black in checking mode
+skip_install = true
+labels = lint
+deps = black
+commands = black --check --diff setup.py snapcraft tests
+
+[testenv:codespell]
+description = Check spelling with codespell
+skip_install = true
+labels = lint
+deps = codespell
+commands = codespell --quiet-level 4 --ignore-words-list crate,keyserver,comandos,ro,buildd --skip '*.tar,*.xz,*.zip,*.bz2,*.7z,*.gz,*.deb,*.rpm,*.snap,*.gpg,*.pyc,*.png,*.ico,*.jar,*.so,changelog,.git,.hg,.mypy_cache,.tox,.venv,venv,_build,buck-out,__pycache__,build,dist,.vscode,parts,stage,prime,test_appstream.py,./snapcraft.spec,./.direnv,./.pytest_cache,./tests/spread/plugins/v1/waf/snaps/waf-hello/waf'
+
+[testenv:flake]
+description = Lint with flake8
+skip_install = true
+labels = lint
+deps = flake8<6.0
+commands =
+    flake8 .
+
+[testenv:isort]
+description = Check import order with isort
+skip_install = true
+labels = lint
+deps = isort
+commands = isort --check .
+
+[testenv:docstyle]
+description = Check documentation style with pydocstyle
+skip_install = true
+labels = lint
+deps = pydocstyle
+commands = pydocstyle snapcraft
+
+[testenv:pylint]
+description = Lint with pylint
+labels = lint
+skip_install = true
+commands =
+    pylint -j 0 snapcraft
+	pylint -j 0 tests --disable=invalid-name,missing-module-docstring,missing-function-docstring,duplicate-code,protected-access,unspecified-encoding,too-many-public-methods,too-many-arguments,too-many-lines,redefined-outer-name
+
+[testenv:spellcheck]
+description = Check spelling using spread-shellcheck.py
+labels = lint
+deps = pyyaml
+skip_install = true
+commands =
+    python3 tools/spread-shellcheck.py spread.yaml tests/spread/
+
+[testenv:shellcheck]
+description = Check spelling with shellcheck
+labels = lint
+skip_install = true
+allowlist_externals = bash
+commands =
+    bash -c "find . \( -name .git -o -name gradlew -o -name .tox \) -prune -o -print0 | xargs -0 file -N | grep shell.script | cut -f1 -d: | xargs shellcheck"