diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 83f04ed..40ca50a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,9 +32,37 @@ on: jobs: + get-changed-files: + name: Get Changed Files + runs-on: ubuntu-latest + permissions: + contents: read # for dorny/paths-filter to fetch a list of changed files + pull-requests: read # for dorny/paths-filter to read pull requests + outputs: + changed-files: ${{ toJSON(steps.changed-files.outputs) }} + steps: + - uses: actions/checkout@v3 + - name: Get Changed Files + id: changed-files + uses: dorny/paths-filter@v2 + with: + token: ${{ github.token }} + list-files: json + filters: | + repo: + - added|modified: + - '**' + deleted: + - deleted: + - '**' + pre-commit: name: Pre-Commit uses: ./.github/workflows/pre-commit-action.yml + needs: + - get-changed-files + with: + changed-files: ${{ needs.get-changed-files.outputs.changed-files }} build-python-package: name: Python Package @@ -44,5 +72,72 @@ jobs: - pre-commit with: kind: "${{ inputs.kind }}" - cmd: "${{ inputs.package_command }}" + cmd: python -m build + #cmd: "${{ inputs.package_command }}" + + test: + name: Test + needs: + - get-changed-files + uses: ./.github/workflows/test-action.yml + with: + changed-files: ${{ needs.get-changed-files.outputs.changed-files }} + + deploy-python-package: + name: Deploy Python Package + uses: ./.github/workflows/deploy-package-action.yml + if: ${{ inputs.kind == 'release' && success() }} + needs: + - pre-commit + - test + - build-python-package + secrets: + PYPI_API_TOKEN: "${{ secrets.PYPI_API_TOKEN }}" + + push-tag: + name: Push Version Tag + runs-on: ubuntu-latest + permissions: + contents: write + if: ${{ inputs.kind == 'release' && success() }} + needs: + - build-python-package + - deploy-python-package + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Push Tag + uses: rickstaa/action-create-tag@v1 + with: + tag: "v${{ needs.build-python-package.outputs.version }}" + message: "Version ${{ needs.build-python-package.outputs.version }}" + + set-pipeline-exit-status: + # This step is just so we can make github require this step, to pass checks + # on a pull request instead of requiring all + name: Set the CI Pipeline Exit Status + runs-on: ubuntu-latest + if: always() + needs: + - pre-commit + - test + - deploy-python-package + - push-tag + steps: + - name: Get workflow information + id: get-workflow-info + uses: technote-space/workflow-conclusion-action@v3 + + - name: Set Pipeline Exit Status + shell: bash + run: | + if [ "${{ steps.get-workflow-info.outputs.conclusion }}" != "success" ]; then + exit 1 + else + exit 0 + fi + - name: Done + if: always() + run: + echo "All workflows finished" diff --git a/.github/workflows/deploy-package-action.yml b/.github/workflows/deploy-package-action.yml new file mode 100644 index 0000000..27605e1 --- /dev/null +++ b/.github/workflows/deploy-package-action.yml @@ -0,0 +1,22 @@ +name: Relenv Python Package + +on: + workflow_call: + secrets: + PYPI_API_TOKEN: + required: true + +jobs: + build: + name: Publish Python Wheel + runs-on: ubuntu-latest + steps: + - name: Download Python Package Artifacts + uses: actions/download-artifact@v3 + with: + name: dist + path: dist + - name: Publish distribution to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/package-action.yml b/.github/workflows/package-action.yml index 729a025..da1d3ce 100644 --- a/.github/workflows/package-action.yml +++ b/.github/workflows/package-action.yml @@ -27,78 +27,69 @@ jobs: name: Build Python Wheel strategy: matrix: - host: - - x86_64 - - aarch64 + host: + - x86_64 + - aarch64 + runs-on: - self-hosted - linux - src-build - ${{ matrix.host }} + + container: + image: debian:11 outputs: version: ${{ steps.version.outputs.version }} - steps: - - uses: actions/checkout@master + - uses: actions/checkout@master -# - name: Set up Python 3.10 -# uses: actions/setup-python@v4 -# with: -# python-version: "3.10" + - name: Update Apt + run: >- + apt-get update - - name: Install pypa/build - run: >- - python -m - pip install - build - --user + - name: Install OS Dependencies + run: >- + apt-get install -y python3 + python3-pip python3-venv patchelf build-essential m4 texinfo - - name: Install pypa/pkginffo - run: >- - python -m - pip install - pkginfo - --user + - name: Create virtualenv + run: >- + python3 -m venv venv - - name: Install relenv - run: >- - python -m - pip install - relenv - --user + - name: Activate virtualenv + run: | + . venv/bin/activate + echo PATH=$PATH >> $GITHUB_ENV - - name: Install relenv toolchain - run: >- - python -m - relenv - toolchain - fetch + - name: Python Version + run: >- + python3 --version - - name: Install relenv fetch - run: >- - python -m - relenv - fetch + - name: Install Python Dependencies + run: >- + pip install build wheel setuptools pkginfo - - name: Echo Build Wheel Command - run: echo "${{ inputs.cmd }}" + - name: Echo Build Wheel Command + run: echo "${{ inputs.cmd }}" - - name: Build Wheel - run: "${{ inputs.cmd }}" + - name: Build Wheel + run: | + ${{ inputs.cmd }} - - name: Wheel Artifact ${{matrix.host}} - uses: actions/upload-artifact@v3 - if: always() - with: - name: relenv-gdb-debug-${{ matrix.host }}-wheel - path: dist/*.whl - retention-days: 5 + - name: Python Build Artifact + uses: actions/upload-artifact@v3 + if: always() + with: + name: dist-${{ matrix.host }} + path: dist/* + retention-days: 5 - - name: Read Version - run: >- - python3 - -c - "from pkginfo import Wheel; s = Wheel('dist/$(ls dist/)'); print(f'version={s.version}')" - >> - $GITHUB_OUTPUT - id: version + - name: Read Version + run: >- + python3 + -c + "from pkginfo import Wheel; s = Wheel('''$(ls dist/*.whl)'''); print('version='+str(s.version))" + >> + $GITHUB_OUTPUT + id: version diff --git a/.github/workflows/pre-commit-action.yml b/.github/workflows/pre-commit-action.yml index 86324c1..12d334b 100644 --- a/.github/workflows/pre-commit-action.yml +++ b/.github/workflows/pre-commit-action.yml @@ -4,7 +4,7 @@ on: workflow_call: inputs: changed-files: - required: false + required: true type: string description: JSON string containing information about changed files @@ -30,5 +30,11 @@ jobs: pre-commit install --install-hooks - name: Check ALL Files On Branch + if: github.event_name != 'pull_request' run: | pre-commit run --show-diff-on-failure --color=always --all-files + + - name: Check Changed Files On PR + if: github.event_name == 'pull_request' && fromJSON(inputs.changed-files)['repo'] == 'true' + run: | + pre-commit run --show-diff-on-failure --color=always --files ${{ join(fromJSON(inputs.changed-files)['repo_files'], ' ') }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..5c458a7 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,32 @@ +name: Build and Release + +on: + workflow_dispatch: + inputs: + kind: + required: false + type: string + default: dev + package_command: + required: false + type: string + description: Command used to build python package + default: >- + python -m + build + --wheel + --outdir dist/ + +jobs: + ci: + name: CI + permissions: + contents: write + pull-requests: read + uses: ./.github/workflows/ci.yml + if: contains('["dwoz", "twangboy", "dmurphy18"]', github.actor) + with: + kind: "${{ inputs.kind }}" + package_command: "${{ inputs.package_command }}" + secrets: + PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/test-action.yml b/.github/workflows/test-action.yml new file mode 100644 index 0000000..e88027a --- /dev/null +++ b/.github/workflows/test-action.yml @@ -0,0 +1,39 @@ +name: Unit Tests + +on: + workflow_call: + inputs: + changed-files: + required: true + type: string + description: JSON string containing information about changed files + +jobs: + test: + strategy: + fail-fast: false + matrix: + runs-on: + - ubuntu-latest + - macos-12 + - macos-13-xlarge + - windows-latest + + name: Unit Test ${{ matrix.runs-on }} + runs-on: ${{ matrix.runs-on }} + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Install python dependencies + run: | + pip3 install pytest + + - name: Run tests + run: | + pytest -v diff --git a/.pre-commit-hooks/copyright_headers.py b/.pre-commit-hooks/copyright_headers.py index eb0fedb..aa65dee 100644 --- a/.pre-commit-hooks/copyright_headers.py +++ b/.pre-commit-hooks/copyright_headers.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2021-2023 VMware, Inc. +# Copyright 2021-2024 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 # # pylint: disable=invalid-name,missing-module-docstring,missing-function-docstring diff --git a/pyproject.toml b/pyproject.toml index 132b443..fb4a8be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ requires = [ "setuptools", "wheel", - "relenv@git+https://github.com/saltstack/relative-environment-for-python@buildenv", + "relenv", ] build-backend = "build" backend-path = ["src/relenv_gdb"] @@ -12,3 +12,8 @@ skip = "src/relenv_gdb/libpython.py" [tool.black] force_exclude = "libpython.py" + +[tool.pytest.ini_options] +pythonpath = [ + "src" +] diff --git a/scripts/dockerbuild.sh b/scripts/dockerbuild.sh new file mode 100755 index 0000000..4159216 --- /dev/null +++ b/scripts/dockerbuild.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# +# Build a ppt wheel in a docker container. +# +# This script is meant to build a wheel in the same docker container images as +# we do in CI/CD. Run it from the root of the github repository. +# +# `scripts/dockerbuild.sh1 +# +if [ -n "$UID" ] +then + CHOWN="" +else + CHOWN="" #chown -R $UID dist build src/ppbt/_toolchain" +fi + +export CHOWN + +CMD=" +apt-get update; +apt-get install -y gcc python3 \ + python3-pip python3-venv build-essential patchelf m4 texinfo +cd /src +python3 -m venv venv +venv/bin/pip install build wheel setuptools +venv/bin/python3 -m build +$CHOWN +" + +docker run --rm -v $(pwd):/src debian:11 /bin/sh -c "$CMD" diff --git a/setup.py b/setup.py index 376eb70..3cea5f9 100644 --- a/setup.py +++ b/setup.py @@ -1,14 +1,22 @@ # Copyright 2023 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 # +import platform + from setuptools import Distribution, setup -class BinaryDistribution(Distribution): - def has_ext_modules(self): - return True +GLIBC_VERSION = "2.17" + +def plat_name(): + return f"manylinux_{GLIBC_VERSION.replace('.', '_')}_{platform.machine()}" setup( - distclass=BinaryDistribution, + options={ + "bdist_wheel": { + "plat_name": f"{plat_name()}", + "python_tag": "py3", + } + } ) diff --git a/src/relenv_gdb/build.py b/src/relenv_gdb/build.py index bee6c4b..bd1edb4 100644 --- a/src/relenv_gdb/build.py +++ b/src/relenv_gdb/build.py @@ -1,19 +1,24 @@ -# Copyright 2023 VMware, Inc. +# Copyright 2023-2024 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 # """ Build our python wheel. """ import contextlib +import logging import os import pathlib +import pprint import shutil import subprocess -import tempfile +import sys +import relenv.build import relenv.buildenv import relenv.common import relenv.create +import relenv.fetch +import relenv.toolchain from setuptools.build_meta import * _build_wheel = build_wheel @@ -35,39 +40,81 @@ def pushd(path): def build_gdb(prefix): """Compile and install gdb to the prefix.""" src = prefix / "src" - src.mkdir() + src.mkdir(exist_ok=True) + + os.environ.update(relenv.buildenv.buildenv(prefix)) + os.environ["CFLAGS"] = ( + f"{os.environ['CFLAGS']} -I{os.environ['RELENV_PATH']}/include/ncursesw " + f"-I{os.environ['RELENV_PATH']}/include/readline " + f"-I{os.environ['RELENV_PATH']}/lib/python3.10/site-packages/relenv_gdb/gdb/include" + ) + os.environ["CPPFLAGS"] = ( + f"{os.environ['CPPFLAGS']} -I{os.environ['RELENV_PATH']}/include/ncursesw " + f"-I{os.environ['RELENV_PATH']}/include/readline " + f"-I{os.environ['RELENV_PATH']}/lib/python3.10/site-packages/relenv_gdb/gdb/include" + ) + os.environ["LDFLAGS"] = ( + f"{os.environ['LDFLAGS']} " + f"-L{os.environ['RELENV_PATH']}/lib/python3.10/site-packages/relenv_gdb/gdb/lib " + ) + + print(f"Build environment: {pprint.pformat(dict(os.environ))}") + sys.stdout.flush() url = "https://gmplib.org/download/gmp/gmp-6.3.0.tar.xz" relenv.common.download_url( url, src, ) - archive_name = str(src / pathlib.Path(url).name) relenv.common.extract_archive(str(src), archive_name) dir_name = archive_name.split(".tar")[0] - os.environ.update(relenv.buildenv.buildenv(prefix)) - os.environ[ - "CFLAGS" - ] = f"{os.environ['CFLAGS']} -I{os.environ['RELENV_PATH']}/include/ncursesw" - os.environ[ - "CPPFLAGS" - ] = f"{os.environ['CPPFLAGS']} -I{os.environ['RELENV_PATH']}/include/ncursesw" - import pprint - pprint.pprint(dict(os.environ)) + with pushd(src / dir_name): + subprocess.run( + [ + "./configure", + f"--prefix={os.environ['RELENV_PATH']}/lib/python3.10/site-packages/relenv_gdb/gdb", + ], + check=True, + ) + subprocess.run(["make"], check=True) + subprocess.run(["make", "install"], check=True) + + url = "https://www.mpfr.org/mpfr-current/mpfr-4.2.1.tar.xz" + relenv.common.download_url( + url, + src, + ) + archive_name = str(src / pathlib.Path(url).name) + relenv.common.extract_archive(str(src), archive_name) + dir_name = archive_name.split(".tar")[0] with pushd(src / dir_name): subprocess.run( [ "./configure", f"--prefix={os.environ['RELENV_PATH']}/lib/python3.10/site-packages/relenv_gdb/gdb", - ] + ], + check=True, ) - subprocess.run(["make"]) - subprocess.run(["make", "install"]) + subprocess.run(["make"], check=True) + subprocess.run(["make", "install"], check=True) + + # Newer patchelf for now + arch = relenv.common.build_arch() + url = f"https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-{arch}.tar.gz" + relenv.common.download_url( + url, + src, + ) + archive_name = str(src / pathlib.Path(url).name) + relenv.common.extract_archive(str(src), archive_name) + dir_name = archive_name.split(".tar")[0] + patchelf = (src / "bin" / "patchelf").resolve() - url = "https://ftp.gnu.org/gnu/gdb/gdb-13.2.tar.xz" + os.environ["LDFLAGS"] = f"{os.environ['LDFLAGS']} -lreadline" + url = "https://ftp.gnu.org/gnu/gdb/gdb-15.1.tar.xz" relenv.common.download_url( url, src, @@ -83,18 +130,22 @@ def build_gdb(prefix): f"--with-python={os.environ['RELENV_PATH']}/bin/python3", "--with-lzma", "--with-separate-debug-dir=/usr/lib/debug", - ] + ], + check=True, ) - subprocess.run(["make"]) + subprocess.run(["make"], check=True) bins = ["gdb/gdb", "gdbserver/gdbserver", "gdbserver/libinproctrace.so"] for _ in bins: + if not pathlib.Path(_).resolve().exists(): + print(f"File not found {_}") subprocess.run( [ - "patchelf", + str(patchelf), "--add-rpath", f"{os.environ['TOOLCHAIN_PATH']}/{os.environ['TRIPLET']}/sysroot/lib", _, - ] + ], + check=True, ) subprocess.run(["make", "install"]) @@ -111,27 +162,28 @@ def build_gdb(prefix): def build_wheel(wheel_directory, metadata_directory=None, config_settings=None): """PEP 517 wheel creation hook.""" - # relenv.fetch.fetch( - # relenv.common.__version__, - # relenv.common.get_triplet(relenv.common.build_arch()), - # ) - static_build_dir = os.environ.get("PY_STATIC_BUILD_DIR", "") - if static_build_dir: - relenvdir = (pathlib.Path(static_build_dir) / "gdb").resolve() - relenv.create.create(str(relenvdir)) - build_gdb(relenvdir) - try: - return _build_wheel(wheel_directory, metadata_directory, config_settings) - finally: - shutil.rmtree("src/relenv_gdb/gdb") - else: - with tempfile.TemporaryDirectory() as tmp_dist_dir: - relenvdir = pathlib.Path(tmp_dist_dir) / "gdb" - relenv.create.create(str(relenvdir)) - build_gdb(relenvdir) - try: - return _build_wheel( - wheel_directory, metadata_directory, config_settings - ) - finally: - shutil.rmtree("src/relenv_gdb/gdb") + logging.basicConfig(level=logging.DEBUG) + + dirs = relenv.common.work_dirs() + if not dirs.toolchain.exists(): + os.makedirs(dirs.toolchain) + if not dirs.build.exists(): + os.makedirs(dirs.build) + + arch = relenv.common.build_arch() + triplet = relenv.common.get_triplet(machine=arch) + + python = relenv.build.platform_versions()[0] + version = relenv.common.__version__ + + root = pathlib.Path(os.environ.get("PWD", os.getcwd())) + build = root / "build" + + relenvdir = (build / "gdb").resolve() + + relenv.toolchain.fetch(arch, dirs.toolchain) + relenv.fetch.fetch(version, triplet, python) + if not relenvdir.exists(): + relenv.create.create(str(relenvdir), version=python) + build_gdb(relenvdir) + return _build_wheel(wheel_directory, metadata_directory, config_settings) diff --git a/src/relenv_gdb/inject.py b/src/relenv_gdb/inject.py index 8710f82..3cf7214 100644 --- a/src/relenv_gdb/inject.py +++ b/src/relenv_gdb/inject.py @@ -26,24 +26,17 @@ INJ_TPL = """ set pagination off source {libpython} -source {s_path} echo acquire gill\\n call (char *) PyGILState_Ensure() -p $SCRIPT -call (void) PyRun_SimpleString($SCRIPT) +call (PyObject *) Py_BuildValue("s", "{inject_path}") +call (FILE *) _Py_fopen_obj($2, "r+") +call (void) PyRun_SimpleFile($3, "{inject_path}") echo release gill\\n call (void) PyGILState_Release($1) quit """ -SCRIPT = """#!/usr/bin/python -import gdb -with open("{}", "r") as fp: - gdb.set_convenience_variable("SCRIPT", fp.read()) -""" - - def main(): """ The inject program entrypoint. @@ -63,30 +56,30 @@ def main(): except IndexError: print("Please provide an input file as the second argument") - s_fd, s_path = tempfile.mkstemp(suffix=".py") - with open(s_path, "w") as fp: - fp.write(SCRIPT.format(file)) - - fd, path = tempfile.mkstemp() + s_fd, inject_path = tempfile.mkstemp(suffix=".py") + with open(file, "r") as fp: + with open(inject_path, "w") as fp2: + fp2.write(fp.read()) + fd, gdb_command = tempfile.mkstemp() try: - with open(path, "w") as fp: + with open(gdb_command, "w") as fp: fp.write( INJ_TPL.format( - s_path=s_path, + inject_path=inject_path, libpython=( pathlib.Path(__file__).parent / "libpython.py" ).resolve(), ) ) subprocess.run( - [str(find_relenv_gdb()), "-p", f"{pid}", "--command", path], + [str(find_relenv_gdb()), "-p", f"{pid}", "--command", gdb_command], capture_output=False, ) finally: os.close(fd) - os.remove(path) + os.remove(gdb_command) os.close(s_fd) - os.remove(s_path) + os.remove(inject_path) if __name__ == "__main__": diff --git a/tests/test_relenv_gdb.py b/tests/test_relenv_gdb.py new file mode 100644 index 0000000..6b53161 --- /dev/null +++ b/tests/test_relenv_gdb.py @@ -0,0 +1,4 @@ + + +def test_noop(): + return