tests: Full Python matrix tests (#478)

Full matrix of feature tests for py-spy & PyPerf across all desired versions.
This is what I suggested in #287 and finally got around to implement. I rather have
these in place before performing any other changes on PyPerf/py-spy.

tests/conftest.py

@@ -224,6 +224,28 @@ def application_docker_images(docker_client: DockerClient) -> Mapping[str, Image
         "python": {
             "": {},
             "libpython": dict(dockerfile="libpython.Dockerfile"),
+            "2.7-glibc-python": dict(dockerfile="matrix.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "2.7-slim"}),
+            "2.7-musl-python": dict(dockerfile="matrix.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "2.7-alpine"}),
+            "3.5-glibc-python": dict(dockerfile="matrix.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "3.5-slim"}),
+            "3.5-musl-python": dict(dockerfile="matrix.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "3.5-alpine"}),
+            "3.6-glibc-python": dict(dockerfile="matrix.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "3.6-slim"}),
+            "3.6-musl-python": dict(dockerfile="matrix.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "3.6-alpine"}),
+            "3.7-glibc-python": dict(dockerfile="matrix.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "3.7-slim"}),
+            "3.7-musl-python": dict(dockerfile="matrix.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "3.7-alpine"}),
+            "3.8-glibc-python": dict(dockerfile="matrix.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "3.8-slim"}),
+            "3.8-musl-python": dict(dockerfile="matrix.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "3.8-alpine"}),
+            "3.9-glibc-python": dict(dockerfile="matrix.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "3.9-slim"}),
+            "3.9-musl-python": dict(dockerfile="matrix.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "3.9-alpine"}),
+            "3.10-glibc-python": dict(dockerfile="matrix.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "3.10-slim"}),
+            "3.10-musl-python": dict(dockerfile="matrix.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "3.10-alpine"}),
+            "2.7-glibc-uwsgi": dict(
+                dockerfile="uwsgi.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "2.7"}
+            ),  # not slim - need gcc
+            "2.7-musl-uwsgi": dict(dockerfile="uwsgi.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "2.7-alpine"}),
+            "3.7-glibc-uwsgi": dict(
+                dockerfile="uwsgi.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "3.7"}
+            ),  # not slim - need gcc
+            "3.7-musl-uwsgi": dict(dockerfile="uwsgi.Dockerfile", buildargs={"PYTHON_IMAGE_TAG": "3.7-alpine"}),
         },
         "ruby": {"": {}},
     }

tests/containers/python/lister.py

@@ -6,22 +6,27 @@
 import os
 from threading import Thread
 
-import yaml
-
 
 class Lister(object):
     @classmethod
-    def lister(cls) -> None:
+    def lister(cls):
+        # type: () -> None
         os.listdir("/")  # have some kernel stacks & Python stacks from a class method
 
 
 class Burner(object):
-    def burner(self) -> None:
+    def burner(self):
+        # type: () -> None
         while True:  # have some Python stacks from an instance method
             pass
 
 
-def parser() -> None:
+def parser():
+    # type: () -> None
+    try:
+        import yaml
+    except ImportError:
+        return  # not required in this test
     while True:
         # Have some package stacks.
         # Notice the name of the package name (PyYAML) is different from the name of the module (yaml)

tests/containers/python/matrix.Dockerfile

@@ -0,0 +1,6 @@
+ARG PYTHON_IMAGE_TAG
+FROM python:${PYTHON_IMAGE_TAG}
+
+WORKDIR /app
+ADD lister.py /app
+CMD ["python", "lister.py"]

tests/containers/python/uwsgi.Dockerfile

@@ -0,0 +1,12 @@
+ARG PYTHON_IMAGE_TAG
+FROM python:${PYTHON_IMAGE_TAG}
+
+WORKDIR /app
+
+# to build uwsgi
+RUN if grep -q Alpine /etc/os-release; then apk add gcc libc-dev linux-headers; fi
+
+RUN pip install uwsgi
+
+ADD lister.py /app
+CMD ["uwsgi", "--py", "lister.py"]

tests/test_python.py

@@ -2,15 +2,17 @@
 # Copyright (c) Granulate. All rights reserved.
 # Licensed under the AGPL3 License. See LICENSE.md in the project root for license information.
 #
+import os
 from pathlib import Path
 from threading import Event
 
+import psutil
 import pytest
-from docker.models.containers import Container
+from granulate_utils.linux.process import is_musl
 
 from gprofiler.profilers.python import PythonProfiler
 from tests.conftest import AssertInCollapsed
-from tests.utils import snapshot_pid_collapsed
+from tests.utils import assert_function_in_collapsed, snapshot_pid_collapsed, snapshot_pid_profile
 
 
 @pytest.fixture
@@ -22,7 +24,7 @@ def runtime() -> str:
 @pytest.mark.parametrize("application_image_tag", ["libpython"])
 def test_python_select_by_libpython(
     tmp_path: Path,
-    application_docker_container: Container,
+    application_pid: int,
     assert_collapsed: AssertInCollapsed,
 ) -> None:
     """
@@ -32,6 +34,80 @@ def test_python_select_by_libpython(
     This test runs a Python named "shmython".
     """
     with PythonProfiler(1000, 1, Event(), str(tmp_path), False, "pyspy", True, None) as profiler:
-        process_collapsed = snapshot_pid_collapsed(profiler, application_docker_container.attrs["State"]["Pid"])
+        process_collapsed = snapshot_pid_collapsed(profiler, application_pid)
     assert_collapsed(process_collapsed)
     assert all(stack.startswith("shmython") for stack in process_collapsed.keys())
+
+
+@pytest.mark.parametrize("in_container", [True])
+@pytest.mark.parametrize(
+    "application_image_tag",
+    [
+        "2.7-glibc-python",
+        "2.7-musl-python",
+        "3.5-glibc-python",
+        "3.5-musl-python",
+        "3.6-glibc-python",
+        "3.6-musl-python",
+        "3.7-glibc-python",
+        "3.7-musl-python",
+        "3.8-glibc-python",
+        "3.8-musl-python",
+        "3.9-glibc-python",
+        "3.9-musl-python",
+        "3.10-glibc-python",
+        "3.10-musl-python",
+        "2.7-glibc-uwsgi",
+        "2.7-musl-uwsgi",
+        "3.7-glibc-uwsgi",
+        "3.7-musl-uwsgi",
+    ],
+)
+@pytest.mark.parametrize("profiler_type", ["py-spy", "pyperf"])
+def test_python_matrix(
+    tmp_path: Path,
+    application_pid: int,
+    assert_collapsed: AssertInCollapsed,
+    profiler_type: str,
+    application_image_tag: str,
+) -> None:
+    python_version, libc, app = application_image_tag.split("-")
+
+    if python_version == "3.5" and profiler_type == "pyperf":
+        pytest.skip("PyPerf doesn't support Python 3.5!")
+
+    if python_version == "2.7" and profiler_type == "pyperf" and app == "uwsgi":
+        pytest.xfail("This combination fails, see https://github.com/Granulate/gprofiler/issues/485")
+
+    with PythonProfiler(1000, 2, Event(), str(tmp_path), False, profiler_type, True, None) as profiler:
+        profile = snapshot_pid_profile(profiler, application_pid)
+
+    collapsed = profile.stacks
+
+    assert_collapsed(collapsed)
+    # searching for "python_version.", because ours is without the patchlevel.
+    assert_function_in_collapsed(f"standard-library=={python_version}.", collapsed)
+
+    assert libc in ("musl", "glibc")
+    assert (libc == "musl") == is_musl(psutil.Process(application_pid))
+
+    if profiler_type == "pyperf":
+        # we expect to see kernel code
+        assert_function_in_collapsed("do_syscall_64_[k]", collapsed)
+        # and native user code
+        assert_function_in_collapsed(
+            "PyEval_EvalFrameEx_[pn]" if python_version == "2.7" else "_PyEval_EvalFrameDefault_[pn]", collapsed
+        )
+        # ensure class name exists for instance methods
+        assert_function_in_collapsed("lister.Burner.burner", collapsed)
+        # ensure class name exists for class methods
+        assert_function_in_collapsed("lister.Lister.lister", collapsed)
+
+    assert profile.app_metadata is not None
+    assert os.path.basename(profile.app_metadata["execfn"]) == app
+    # searching for "python_version.", because ours is without the patchlevel.
+    assert profile.app_metadata["python_version"].startswith(f"Python {python_version}.")
+    if python_version == "2.7" and app == "python":
+        assert profile.app_metadata["sys_maxunicode"] == "1114111"
+    else:
+        assert profile.app_metadata["sys_maxunicode"] is None