From b53eabfa406a521440acb60a21d899b8c83c1ae8 Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Thu, 28 Jul 2022 15:05:20 -0500 Subject: [PATCH 1/9] build: add `connect-markdown-renderer` dependency This is a small library that makes use of `rich` to render markdown in a terminal shell. --- poetry.lock | 93 ++++++++++++++++++++++++++++++++++++++++++++++++-- pyproject.toml | 1 + 2 files changed, 92 insertions(+), 2 deletions(-) diff --git a/poetry.lock b/poetry.lock index 3cb7538b..b68ea7f3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -97,6 +97,29 @@ category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "commonmark" +version = "0.9.1" +description = "Python parser for the CommonMark Markdown spec" +category = "main" +optional = false +python-versions = "*" + +[package.extras] +test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] + +[[package]] +name = "connect-markdown-renderer" +version = "2.0.1" +description = "Connect Markdown Renderer" +category = "main" +optional = false +python-versions = ">=3.7,<4" + +[package.dependencies] +markdown-it-py = ">=2.1.0,<3.0.0" +rich = ">=12.4.1,<13.0.0" + [[package]] name = "cryptography" version = "37.0.4" @@ -261,6 +284,36 @@ SecretStorage = {version = ">=3.2", markers = "sys_platform == \"linux\""} docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "jaraco.tidelift (>=1.4)"] testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] +[[package]] +name = "markdown-it-py" +version = "2.1.0" +description = "Python port of markdown-it. Markdown parsing, done right!" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +mdurl = ">=0.1,<1.0" +typing_extensions = {version = ">=3.7.4", markers = "python_version < \"3.8\""} + +[package.extras] +testing = ["pytest-regressions", "pytest-cov", "pytest", "coverage"] +rtd = ["sphinx-book-theme", "sphinx-design", "sphinx-copybutton", "sphinx", "pyyaml", "myst-parser", "attrs"] +profiling = ["gprof2dot"] +plugins = ["mdit-py-plugins"] +linkify = ["linkify-it-py (>=1.0,<2.0)"] +compare = ["panflute (>=2.1.3,<2.2.0)", "mistune (>=2.0.2,<2.1.0)", "mistletoe (>=0.8.1,<0.9.0)", "markdown (>=3.3.6,<3.4.0)", "commonmark (>=0.9.1,<0.10.0)"] +code_style = ["pre-commit (==2.6)"] +benchmarking = ["pytest-benchmark (>=3.2,<4.0)", "pytest", "psutil"] + +[[package]] +name = "mdurl" +version = "0.1.1" +description = "Markdown URL utilities" +category = "main" +optional = false +python-versions = ">=3.7" + [[package]] name = "packaging" version = "21.3" @@ -330,7 +383,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" name = "pygments" version = "2.12.0" description = "Pygments is a syntax highlighting package written in Python." -category = "dev" +category = "main" optional = false python-versions = ">=3.6" @@ -485,6 +538,22 @@ python-versions = ">=3.7" [package.extras] idna2008 = ["idna"] +[[package]] +name = "rich" +version = "12.5.1" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +category = "main" +optional = false +python-versions = ">=3.6.3,<4.0.0" + +[package.dependencies] +commonmark = ">=0.9.0,<0.10.0" +pygments = ">=2.6.0,<3.0.0" +typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""} + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] + [[package]] name = "ruamel.yaml" version = "0.17.21" @@ -724,7 +793,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "27b1c1c9fddbf146d091a57659d47f39e7637deafcb77f380e001271d007c1d1" +content-hash = "22dcaa94a88956d99d31b680a2befe9781ae7033cca5d4d44c6f312bba3d5c39" [metadata.files] atomicwrites = [] @@ -742,6 +811,14 @@ click-log = [ {file = "click_log-0.4.0-py2.py3-none-any.whl", hash = "sha256:a43e394b528d52112af599f2fc9e4b7cf3c15f94e53581f74fa6867e68c91756"}, ] colorama = [] +commonmark = [ + {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, + {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, +] +connect-markdown-renderer = [ + {file = "connect-markdown-renderer-2.0.1.tar.gz", hash = "sha256:fbaefff195a6c347a7f89e75a09e78f6be35c903d72a2766c8f263ca75758757"}, + {file = "connect_markdown_renderer-2.0.1-py3-none-any.whl", hash = "sha256:8d726b947d877697a42f528799908481685bf0db6a17b4e6d715389bfae9cccd"}, +] cryptography = [] distlib = [] docutils = [] @@ -771,6 +848,14 @@ jeepney = [ {file = "jeepney-0.8.0.tar.gz", hash = "sha256:5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806"}, ] keyring = [] +markdown-it-py = [ + {file = "markdown-it-py-2.1.0.tar.gz", hash = "sha256:cf7e59fed14b5ae17c0006eff14a2d9a00ed5f3a846148153899a0224e2c07da"}, + {file = "markdown_it_py-2.1.0-py3-none-any.whl", hash = "sha256:93de681e5c021a432c63147656fe21790bc01231e0cd2da73626f1aa3ac0fe27"}, +] +mdurl = [ + {file = "mdurl-0.1.1-py3-none-any.whl", hash = "sha256:6a8f6804087b7128040b2fb2ebe242bdc2affaeaa034d5fc9feeed30b443651b"}, + {file = "mdurl-0.1.1.tar.gz", hash = "sha256:f79c9709944df218a4cdb0fcc0b0c7ead2f44594e3e84dc566606f04ad749c20"}, +] packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, @@ -818,6 +903,10 @@ rfc3986 = [ {file = "rfc3986-2.0.0-py2.py3-none-any.whl", hash = "sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd"}, {file = "rfc3986-2.0.0.tar.gz", hash = "sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c"}, ] +rich = [ + {file = "rich-12.5.1-py3-none-any.whl", hash = "sha256:2eb4e6894cde1e017976d2975ac210ef515d7548bc595ba20e195fb9628acdeb"}, + {file = "rich-12.5.1.tar.gz", hash = "sha256:63a5c5ce3673d3d5fbbf23cd87e11ab84b6b451436f1b7f19ec54b6bc36ed7ca"}, +] "ruamel.yaml" = [ {file = "ruamel.yaml-0.17.21-py3-none-any.whl", hash = "sha256:742b35d3d665023981bd6d16b3d24248ce5df75fdb4e2924e93a05c1f8b61ca7"}, {file = "ruamel.yaml-0.17.21.tar.gz", hash = "sha256:8b7ce697a2f212752a35c1ac414471dc16c424c9573be4926b56ff3f5d23b7af"}, diff --git a/pyproject.toml b/pyproject.toml index b0d06d7e..0838ff23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ requests = "*" cryptography = "*" packaging = "*" "ruamel.yaml" = "*" +connect-markdown-renderer = "*" [tool.poetry.dev-dependencies] pytest = "*" From 7f75731993ed49a27775da3f337e4e26e8336649 Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Thu, 28 Jul 2022 15:07:35 -0500 Subject: [PATCH 2/9] docs: update CONTRIBUTING.md to show how to add dependencies without constraints --- CONTRIBUTING.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index badce5cf..53bc266f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -129,13 +129,17 @@ Here's how to set up `phylum-ci` for local development. ```sh # Unless there is a reason to do so, prefer to add dependencies without constraints - poetry add new-dependency-name + poetry add "new-dependency-name==*" - # When a version constraint is not specified, poetry chooses one. For example (in pyproject.toml): + # When a version constraint is not specified, poetry chooses one. For example, the command: + # + # $ poetry add new-dependency-name + # + # results in a caret-style version constraint added to the dependency in pyproject.toml: # # new-dependency-name = "^1.2.3" # - # Unless the constraint was intentional, change the entry to remove the constraint: + # Unless the constraint was intentional, change the pyproject.toml entry to remove the constraint: # # new-dependency-name = "*" From 48b73a0f74487250e840f6aec46bb2887e27a1d6 Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Thu, 28 Jul 2022 15:17:20 -0500 Subject: [PATCH 3/9] feat: render markdown output in terminal for pre-commit and no-CI environments So far, the CI environments that have been implemented allow for output, in the form of review comments, to be posted as rendered markdown. The environments that don't use CI - pre-commit and 'no-CI' so far - display their output in the terminal. Instead of writing separate output for these environments, a conversion utility library is used to render the existing markdown output in the terminal. Additionally, the labels for these environments were shortened to be more readable...in both the output as a link and the Phylum UI in the label dropdown menu. --- src/phylum/ci/ci_base.py | 2 +- src/phylum/ci/ci_none.py | 8 +++----- src/phylum/ci/ci_precommit.py | 8 +++----- src/phylum/ci/constants.py | 2 +- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/phylum/ci/ci_base.py b/src/phylum/ci/ci_base.py index c231760e..53373cea 100644 --- a/src/phylum/ci/ci_base.py +++ b/src/phylum/ci/ci_base.py @@ -120,7 +120,7 @@ def cli_path(self) -> Optional[Path]: @property def analysis_output(self) -> str: - """Get the output from the overall analysis.""" + """Get the output from the overall analysis, in markdown format.""" return self._analysis_output @property diff --git a/src/phylum/ci/ci_none.py b/src/phylum/ci/ci_none.py index 8b63a661..b75b1cdc 100644 --- a/src/phylum/ci/ci_none.py +++ b/src/phylum/ci/ci_none.py @@ -9,7 +9,7 @@ from pathlib import Path from typing import Optional -from phylum.ci import SCRIPT_NAME +from connect.utils.terminal.markdown import render from phylum.ci.ci_base import CIBase @@ -60,7 +60,7 @@ def phylum_label(self) -> str: # Reference: https://git-scm.com/book/en/v2/Git-Internals-Git-Objects cmd = f"git hash-object {self.lockfile}".split() lockfile_hash_object = subprocess.run(cmd, check=True, text=True, capture_output=True).stdout.strip() - label = f"{SCRIPT_NAME}_{self.ci_platform_name}_{current_branch}_{lockfile_hash_object}" + label = f"{self.ci_platform_name}_{current_branch}_{lockfile_hash_object[:7]}" label = label.replace(" ", "-") return label @@ -104,6 +104,4 @@ def _is_lockfile_changed(self, lockfile: Path) -> bool: def post_output(self) -> None: """Post the output of the analysis in the means appropriate for the CI environment.""" - # This is a bit of a placeholder for now. The output works in that it is human readable. - # However, it is more meant for display on the web, as HTML and rendered Markdown. - print(f" [+] Analysis output:\n{self.analysis_output}") + print(f" [+] Analysis output:\n{render(self.analysis_output)}") diff --git a/src/phylum/ci/ci_precommit.py b/src/phylum/ci/ci_precommit.py index db260c05..369ba1c6 100644 --- a/src/phylum/ci/ci_precommit.py +++ b/src/phylum/ci/ci_precommit.py @@ -13,7 +13,7 @@ from pathlib import Path from typing import List, Optional -from phylum.ci import SCRIPT_NAME +from connect.utils.terminal.markdown import render from phylum.ci.ci_base import CIBase @@ -50,7 +50,7 @@ def phylum_label(self) -> str: # Reference: https://git-scm.com/book/en/v2/Git-Internals-Git-Objects cmd = f"git hash-object {self.lockfile}".split() lockfile_hash_object = subprocess.run(cmd, check=True, text=True, capture_output=True).stdout.strip() - label = f"{SCRIPT_NAME}_{self.ci_platform_name}_{current_branch}_{lockfile_hash_object}" + label = f"{self.ci_platform_name}_{current_branch}_{lockfile_hash_object[:7]}" label = label.replace(" ", "-") return label @@ -77,6 +77,4 @@ def _is_lockfile_changed(self, lockfile: Path) -> bool: def post_output(self) -> None: """Post the output of the analysis in the means appropriate for the CI environment.""" - # TODO: Change this placeholder when the real Python pre-commit hook is ready. - # https://github.com/phylum-dev/phylum-ci/issues/35 - print(f" [+] Analysis output:\n{self.analysis_output}") + print(f" [+] Analysis output:\n{render(self.analysis_output)}") diff --git a/src/phylum/ci/constants.py b/src/phylum/ci/constants.py index 944c2833..fa6b78f3 100644 --- a/src/phylum/ci/constants.py +++ b/src/phylum/ci/constants.py @@ -5,7 +5,7 @@ from phylum.ci.common import RiskDomain # The common Phylum header that must exist as the first text in the first line of all analysis output -PHYLUM_HEADER = "## Phylum OSS Supply Chain Risk Analysis" +PHYLUM_HEADER = "# Phylum OSS Supply Chain Risk Analysis" # NOTE: All multi-line strings are indented by two levels on purpose, to ensure they line up correctly when used with # each other in templates and are all fully left justified after applying `textwrap.dedent` for normalization. From 35b03ee47d1bb67cd8ca00a4ed3bca745251867e Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Fri, 29 Jul 2022 17:34:24 -0500 Subject: [PATCH 4/9] feat: add support for Python pre-commit hook integration --- .pre-commit-hooks.yaml | 12 ++++++++++++ src/phylum/ci/ci_base.py | 28 ++++++++++++++-------------- src/phylum/ci/ci_precommit.py | 17 +++++++++++++++++ 3 files changed, 43 insertions(+), 14 deletions(-) create mode 100644 .pre-commit-hooks.yaml diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml new file mode 100644 index 00000000..675f7417 --- /dev/null +++ b/.pre-commit-hooks.yaml @@ -0,0 +1,12 @@ +# This is the config for defining `pre-commit` hooks for use in other repositories. +# +# See https://pre-commit.com for more information +--- +- id: phylum + name: analyze lockfile with phylum + description: Run `phylum` on a dependency lockfile + entry: phylum-ci + language: python + stages: [commit] +# TODO: Add a hook for Docker image usage? +# - id: phylum-docker diff --git a/src/phylum/ci/ci_base.py b/src/phylum/ci/ci_base.py index 53373cea..f10cad95 100644 --- a/src/phylum/ci/ci_base.py +++ b/src/phylum/ci/ci_base.py @@ -64,6 +64,20 @@ def __init__(self, args: Namespace) -> None: self.args = args self._phylum_project_file = Path.cwd().joinpath(".phylum_project").resolve() + # The lockfile specified as a script argument will be used, if provided. + # Otherwise, an attempt will be made to automatically detect the lockfile. + provided_lockfile: Path = args.lockfile + if provided_lockfile and provided_lockfile.exists() and provided_lockfile.stat().st_size: + self._lockfile = provided_lockfile.resolve() + else: + detected_lockfile = detect_lockfile() + if detected_lockfile: + self._lockfile = detected_lockfile + else: + raise SystemExit( + " [!] A lockfile is required and was not detected. Consider specifying one with `--lockfile`." + ) + # Ensure all pre-requisites are met and bail at the earliest opportunity when they aren't self._check_prerequisites() print(" [+] All pre-requisites met") @@ -82,20 +96,6 @@ def __init__(self, args: Namespace) -> None: os.environ[TOKEN_ENVVAR_NAME] = args.token self.args.token = token - # The lockfile specified as a script argument will be used, if provided. - # Otherwise, an attempt will be made to automatically detect the lockfile. - provided_lockfile: Path = args.lockfile - if provided_lockfile and provided_lockfile.exists() and provided_lockfile.stat().st_size: - self._lockfile = provided_lockfile.resolve() - else: - detected_lockfile = detect_lockfile() - if detected_lockfile: - self._lockfile = detected_lockfile - else: - raise SystemExit( - " [!] A lockfile is required and was not detected. Consider specifying one with `--lockfile`." - ) - self._lockfile_changed = self._is_lockfile_changed(self.lockfile) @property diff --git a/src/phylum/ci/ci_precommit.py b/src/phylum/ci/ci_precommit.py index 369ba1c6..016a0f17 100644 --- a/src/phylum/ci/ci_precommit.py +++ b/src/phylum/ci/ci_precommit.py @@ -35,9 +35,26 @@ def _check_prerequisites(self) -> None: cmd = "git diff --cached --name-only".split() staged_files = subprocess.run(cmd, check=True, text=True, capture_output=True).stdout.strip().split("\n") + extra_arg_paths = (Path(extra_arg).resolve() for extra_arg in self.extra_args) + + print(" [*] Checking extra args for valid pre-commit scenarios ...") + # Allow for a pre-commit config set up to send all staged files to the hook if sorted(staged_files) == sorted(self.extra_args): print(" [+] The extra args provided exactly match the list of staged files") + # Allow for a pre-commit config set up to filter the files sent to the hook + elif all(extra_arg in staged_files for extra_arg in self.extra_args): + print(" [+] All the extra args are staged files") + # Allow for cases where the lockfile is included or explicitly specified (e.g., `pre-commit run --all-files`) + elif self.lockfile in extra_arg_paths: + print(" [+] The lockfile was included in the extra args") + # NOTE: There is still the case where the lockfile is "accidentally" included as an extra argument. For example, + # `phylum-ci poetry.lock` was used instead of `phylum-ci --lockfile poetry.lock`, which is bad syntax but + # nonetheless results in the `CIPreCommit` environment used instead of `CINone`. This is not terrible; it + # just might be a slightly confusing corner case. It might be possible to use a library like `psutil` to + # acquire the command line from the parent process and inspect it for `pre-commit` usage. That is a + # heavyweight solution and one that will not be pursued until the need for it is more clear. else: + print(" [+] No valid pre-commit scenario found. Bailing ...") raise SystemExit(f" [!] Unrecognized arguments: {' '.join(self.extra_args)}") @property From 55baece06e780114f52415c7302e6c6e1b857a75 Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Fri, 29 Jul 2022 18:18:39 -0500 Subject: [PATCH 5/9] WIP: update hook to use a single process instead of in parallel --- .pre-commit-hooks.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 675f7417..659631ba 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -7,6 +7,7 @@ description: Run `phylum` on a dependency lockfile entry: phylum-ci language: python + require_serial: true stages: [commit] # TODO: Add a hook for Docker image usage? # - id: phylum-docker From 6374ad5597f05f4d3e7366bb63af8ef21eaab56c Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Fri, 29 Jul 2022 18:23:21 -0500 Subject: [PATCH 6/9] docs: add pre-commit badge to README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index aa386995..81be9dbe 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ ![GitHub last commit](https://img.shields.io/github/last-commit/phylum-dev/phylum-ci) [![GitHub Workflow Status (branch)][workflow_shield]][workflow_test] [![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)][CoC] +[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)][pre-commit] Utilities for integrating Phylum into CI pipelines (and beyond) @@ -15,6 +16,7 @@ Utilities for integrating Phylum into CI pipelines (and beyond) [workflow_shield]: https://img.shields.io/github/workflow/status/phylum-dev/phylum-ci/Test/main?label=Test&logo=GitHub [workflow_test]: https://github.com/phylum-dev/phylum-ci/actions/workflows/test.yml [CoC]: https://github.com/phylum-dev/phylum-ci/blob/main/CODE_OF_CONDUCT.md +[pre-commit]: https://github.com/pre-commit/pre-commit [contributing]: https://github.com/phylum-dev/phylum-ci/blob/main/CONTRIBUTING.md [changelog]: https://github.com/phylum-dev/phylum-ci/blob/main/CHANGELOG.md [security]: https://github.com/phylum-dev/phylum-ci/blob/main/docs/security.md From c99e49367f04225e46dfffdd9e0400fd9a8cc054 Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Mon, 1 Aug 2022 11:15:24 -0500 Subject: [PATCH 7/9] ci: add local pre-commit config and update auto update workflow * Add a local pre-commit config file `.pre-commit-config.yaml` * Add a few basic hooks to ensure functionality and start on QA * Add a local hook for using `phylum` to analyze the poetry lockfile * Rename `poetry_update` workflow to `auto_updates` * Update the `auto_updates` workflow * Enable auto updates of the pre-commit hooks to the latest tags * Ensure commits by `phylum-bot` are signed * Rename the workflow and branch names --- .../{poetry_update.yml => auto_updates.yml} | 39 +++++++++++++------ .pre-commit-config.yaml | 34 ++++++++++++++++ 2 files changed, 61 insertions(+), 12 deletions(-) rename .github/workflows/{poetry_update.yml => auto_updates.yml} (60%) create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/poetry_update.yml b/.github/workflows/auto_updates.yml similarity index 60% rename from .github/workflows/poetry_update.yml rename to .github/workflows/auto_updates.yml index 9e02bad4..03aa3ddb 100644 --- a/.github/workflows/poetry_update.yml +++ b/.github/workflows/auto_updates.yml @@ -1,5 +1,6 @@ # This is a workflow for updating Python dependencies with Poetry. # Major version updates are handled separately, by Dependabot. +# It will also update the pre-commit hooks to use latest tags. --- name: Update Deps @@ -10,8 +11,8 @@ on: - cron: '35 14 * * 1' jobs: - poetry-update: - name: Update Python dependencies + workflow-auto-updates: + name: Update dependencies and hooks runs-on: ubuntu-latest strategy: matrix: @@ -24,8 +25,19 @@ jobs: - name: Checkout the repo uses: actions/checkout@v3 - - name: Install poetry - run: pipx install poetry + # This GPG key is for the `phylum-bot` account and used in order to ensure commits are signed/verified + - name: Import GPG key for bot account + uses: crazy-max/ghaction-import-gpg@v5 + with: + gpg_private_key: ${{ secrets.PHYLUM_BOT_GPG_PRIVATE_KEY }} + passphrase: ${{ secrets.PHYLUM_BOT_GPG_PASSPHRASE }} + git_user_signingkey: true + git_commit_gpgsign: true + + - name: Install poetry and pre-commit + run: | + pipx install poetry + pipx install pre-commit - name: Configure poetry run: poetry config virtualenvs.in-project true @@ -45,17 +57,20 @@ jobs: poetry env use python${{ matrix.python-version }} poetry install --verbose --no-root - - name: Poetry update + - name: Update Python dependencies run: poetry update -vv + - name: Update pre-commit hooks + run: pre-commit autoupdate + - name: Commit changes id: commit continue-on-error: true + # NOTE: The git user name and email used for commits is already configured, + # by the crazy-max/ghaction-import-gpg action. run: | - git config user.name 'Phylum Bot' - git config user.email 'phylum-bot@users.noreply.github.com' - git commit -a -m "build: Bump poetry.lock dependencies" - git push --force origin HEAD:auto-poetry-update + git commit -a -m "build: Bump poetry.lock dependencies and pre-commit hooks" + git push --force origin HEAD:workflow-auto-updates - name: Create Pull Request if: ${{ steps.commit.outcome == 'success' }} @@ -66,8 +81,8 @@ jobs: github.rest.pulls.create({ owner: context.repo.owner, repo: context.repo.repo, - head: "auto-poetry-update", + head: "workflow-auto-updates", base: context.ref, - title: "build: Bump poetry.lock dependencies", - body: "Bump dependencies in poetry.lock for all SemVer-compatible updates.", + title: "build: Bump poetry.lock dependencies and pre-commit hooks", + body: "Bump dependencies in `poetry.lock` and hooks in `.pre-commit-config.yaml`.", }); diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..c546a79d --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,34 @@ +# This is the config for using `pre-commit` on this repository. +# +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +--- +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.3.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + + - repo: https://github.com/psf/black + rev: 22.6.0 + hooks: + - id: black + + - repo: https://github.com/asottile/pyupgrade + rev: v2.37.3 + hooks: + - id: pyupgrade + args: [--py37-plus] + + # NOTE: don't use this config for your own repositories. Instead, see + # "Git pre-commit Integration" in `docs/sync/git_precommit.md` + - repo: local + hooks: + - id: phylum-ci + name: analyze lockfile with phylum + language: system + files: ^poetry\.lock$ + entry: poetry run phylum-ci From 07c8af8e2e5ac417ee6529a1523dda83e55e50e6 Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Tue, 2 Aug 2022 10:40:00 -0500 Subject: [PATCH 8/9] ci: update hooks to freeze them --- .github/workflows/auto_updates.yml | 2 +- .pre-commit-config.yaml | 11 +++++++---- .pre-commit-hooks.yaml | 2 -- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/auto_updates.yml b/.github/workflows/auto_updates.yml index 03aa3ddb..bad2461c 100644 --- a/.github/workflows/auto_updates.yml +++ b/.github/workflows/auto_updates.yml @@ -61,7 +61,7 @@ jobs: run: poetry update -vv - name: Update pre-commit hooks - run: pre-commit autoupdate + run: pre-commit autoupdate --freeze - name: Commit changes id: commit diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c546a79d..e8454ef3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,10 +2,13 @@ # # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks +# +# NOTE: Individual hook revisions are kept up to date automatically with +# the `auto_updates` workflow, which bumps hooks to the latest tag. --- repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: 3298ddab3c13dd77d6ce1fc0baf97691430d84b0 # frozen: v4.3.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -13,12 +16,12 @@ repos: - id: check-added-large-files - repo: https://github.com/psf/black - rev: 22.6.0 + rev: f6c139c5215ce04fd3e73a900f1372942d58eca0 # frozen: 22.6.0 hooks: - id: black - repo: https://github.com/asottile/pyupgrade - rev: v2.37.3 + rev: a78007c1e9de96e71d5fb3e720c2b9fae8ed8abf # frozen: v2.37.3 hooks: - id: pyupgrade args: [--py37-plus] @@ -28,7 +31,7 @@ repos: - repo: local hooks: - id: phylum-ci - name: analyze lockfile with phylum + name: analyze lockfile with phylum-ci language: system files: ^poetry\.lock$ entry: poetry run phylum-ci diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 659631ba..41cc80cc 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -9,5 +9,3 @@ language: python require_serial: true stages: [commit] -# TODO: Add a hook for Docker image usage? -# - id: phylum-docker From a44049e88179b86a8f1e7e0c8299fb265c6ed464 Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Tue, 2 Aug 2022 12:45:16 -0500 Subject: [PATCH 9/9] docs: add git pre-commit documentation --- CONTRIBUTING.md | 32 ++++-- README.md | 8 +- docs/sync/git_precommit.md | 162 +++++++++++++++++++++++++++++ docs/sync/integrations_overview.md | 6 ++ 4 files changed, 197 insertions(+), 11 deletions(-) create mode 100644 docs/sync/git_precommit.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 53bc266f..05e60356 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -87,7 +87,20 @@ Here's how to set up `phylum-ci` for local development. git clone git@github.com:phylum-dev/phylum-ci.git ``` -2. Ensure all supported Python versions are installed locally +2. Optional: Install [pre-commit](https://pre-commit.com/) and the local hooks + + ```sh + # If the `pre-commit` tool is not already installed, the recommended method is to use pipx + pipx install pre-commit + + # Installing with homebrew is another good option + brew install pre-commit + + # Use the `pre-commit` tool to install the git hooks used by the repository + pre-commit install + ``` + +3. Ensure all supported Python versions are installed locally 1. The strategy is to support all released minor versions of Python that are not end-of-life yet 2. The current list 1. at the time of this writing is 3.7, 3.8, 3.9, and 3.10 @@ -108,15 +121,15 @@ Here's how to set up `phylum-ci` for local development. pyenv global 3.10.x 3.9.x 3.8.x 3.7.x ``` -3. Ensure [poetry](https://python-poetry.org/docs/) is installed -4. Install dependencies with `poetry`, which will automatically create a virtual environment: +4. Ensure [poetry](https://python-poetry.org/docs/) is installed +5. Install dependencies with `poetry`, which will automatically create a virtual environment: ```sh cd phylum-ci poetry install ``` -5. Create a branch for local development: +6. Create a branch for local development: ```sh git checkout -b @@ -124,7 +137,7 @@ Here's how to set up `phylum-ci` for local development. Now you can make your changes locally. -6. If new dependencies are added, do so in a way that does not add upper version constraints and ensure +7. If new dependencies are added, do so in a way that does not add upper version constraints and ensure the `poetry.lock` file is updated (and committed): ```sh @@ -146,17 +159,18 @@ Here's how to set up `phylum-ci` for local development. # Update the lockfile and the local environment to get the latest versions of dependencies poetry update - # Dependencies will be checked automatically in CI during a PR, but checking locally is possible: + # Dependencies will be checked automatically in CI during a PR. They will also be checked + # with the local pre-commit hook, if enabled. Manually checking locally is also possible: phylum analyze poetry.lock ``` -7. When you're done making changes, check that your changes pass the tests: +8. When you're done making changes, check that your changes pass the tests: ```sh poetry run tox ``` -8. Commit your changes and push your branch to GitHub: +9. Commit your changes and push your branch to GitHub: ```sh git add . @@ -164,7 +178,7 @@ Here's how to set up `phylum-ci` for local development. git push --set-upstream origin ``` -9. Submit a pull request (PR) through the GitHub website +10. Submit a pull request (PR) through the GitHub website ## Pull Request Guidelines diff --git a/README.md b/README.md index 81be9dbe..9cdc9122 100644 --- a/README.md +++ b/README.md @@ -118,14 +118,18 @@ The current CI platforms/environments supported are: * GitHub Actions * See the [GitHub Actions Integration documentation][github_docs] for more info +* Git `pre-commit` Hooks + * See the [Git `pre-commit` Integration documentation][precommit_docs] for more info + * None (local use) * This is the "fall-through" case used when no other environment is detected * Can be useful to analyze lockfiles locally, prior to or after submitting a pull/merge request (PR/MR) to a CI system * Establishing a successful submission prior to submitting a PR/MR to a CI system * Troubleshooting after submitting a PR/MR to a CI system and getting unexpected results -[gitlab_docs]: https://github.com/phylum-dev/phylum-ci/blob/main/docs/gitlab_ci.md -[github_docs]: https://github.com/phylum-dev/phylum-ci/blob/main/docs/github_actions.md +[gitlab_docs]: https://github.com/phylum-dev/phylum-ci/blob/main/docs/sync/gitlab_ci.md +[github_docs]: https://github.com/phylum-dev/phylum-ci/blob/main/docs/sync/github_actions.md +[precommit_docs]: https://github.com/phylum-dev/phylum-ci/blob/main/docs/sync/git_precommit.md ## License diff --git a/docs/sync/git_precommit.md b/docs/sync/git_precommit.md new file mode 100644 index 00000000..0b1ea1a2 --- /dev/null +++ b/docs/sync/git_precommit.md @@ -0,0 +1,162 @@ +--- +title: Git pre-commit Integration +category: 62cdf6722c2c1602a4b69643 +hidden: false +--- +# Git `pre-commit` Integration + +## Overview + +[pre-commit] is a framework for managing and maintaining multi-language Git pre-commit hooks. + +[pre-commit]: https://pre-commit.com/ + +Phylum is available as a pre-commit hook. + +Once configured for a repository, the git `pre-commit` integration will provide analysis of project dependencies +from a lockfile during a commit containing that lockfile. The hook will fail and provide a report if any of the +newly added/modified dependencies from the commit fail to meet the project risk thresholds for any of the five +Phylum risk domains: + +* Vulnerability (aka `vul`) +* Malicious Code (aka `mal`) +* Engineering (aka `eng`) +* License (aka `lic`) +* Author (aka `aut`) + +See [Phylum Risk Domains documentation](https://docs.phylum.io/docs/phylum-package-score#risk-domains) for more detail. + +**NOTE**: It is not enough to have the total project threshold set. Individual risk domain threshold values +must be set, either in the Phylum web UI or with `phylum-ci` options, in order to enable analysis results. +Otherwise, the risk domain is considered disabled and the threshold value used will be zero (0). + +The hook will be skipped if no dependencies were added or modified for a given commit. +If one or more dependencies are still processing (no results available), then the hook will only fail if +dependencies that have _completed analysis results_ do not meet the specified project risk thresholds. + +## Prerequisites + +The pre-requisites for using the git `pre-commit` hook are: + +* The [pre-commit] package manager installed +* A [Phylum token][phylum_tokens] with API access + * [Contact Phylum][phylum_contact] or create an account and register to gain access + * See also [`phylum auth register`][phylum_register] command documentation + * Consider using a bot or group account for this token +* Access to the Phylum API endpoints + * That usually means a connection to the internet, optionally via a proxy + * Support for on-premises installs are not available at this time +* A `.phylum_project` file exists at the root of the repository + * See [`phylum project`](https://docs.phylum.io/docs/phylum_project) and + [`phylum project create`](https://docs.phylum.io/docs/phylum_project_create) command documentation + +[phylum_tokens]: https://docs.phylum.io/docs/api-keys +[phylum_contact]: https://phylum.io/contact-us/ +[phylum_register]: https://docs.phylum.io/docs/phylum_auth_register + +**NOTE: If the `phylum` CLI binary is installed locally, it will be used. Otherwise, the hook will install it.** + +## Configure `.pre-commit-config.yaml` + +Phylum analysis of dependencies can be added to existing `pre-commit` configurations or +on it's own with this minimal configuration: + +```yaml +# This is the config for using `pre-commit` on this repository. +# +# See https://pre-commit.com for more information +--- +repos: + - repo: https://github.com/phylum-dev/phylum-ci + rev: main + hooks: + - id: phylum + # Optional: Specify the lockfile pattern for your repository + files: '' + # Optional: Specify additional arguments to be passed to `phylum-ci` + args: [] +``` + +**NOTE**: This example configuration uses a mutable reference for `rev`, which is a bad practice +(and only done here to prevent old tags from being used through copy and paste implementations). +A best practice is to ensure the `rev` key for all hooks is updated to a valid and current immutable reference: + +```sh +pre-commit autoupdate --freeze +``` + +The hook can be customized with [optional keys][hook_config] in the config file. +Two common customization keys for the `phylum` hook are `files` and `args`: + +[hook_config]: https://pre-commit.com/index.html#pre-commit-configyaml---hooks + +### File Control + +The `files` key in the hook configuration file is the way to ensure the hook only runs when the specified +lockfile has changed, saving execution time. + +The value for the `files` key is a [Python regular expression][re] and are matched with `re.search`. + +[re]: https://docs.python.org/3/library/re.html#regular-expression-syntax + +```yaml + # NOTE: These are examples. Only one `files` key for the hook is expected + + # Specify `package-lock.json` + files: ^package-lock\.json$ + + # Specify `poetry.lock` + files: ^poetry\.lock$ + + # Specify `requirements-*.txt` files + # NOTE: An explicit lockfile should still be specified in the `args` key + files: ^requirements-.*\.txt$ +``` + +### Argument Control + +The `args` key is the way to exert control over the execution of the Phylum analysis. +The `phylum-ci` script entry point is called by the hook. It has a number of arguments that are all optional +and defaulted to secure values. To view the arguments, their description, and default values, run the script +with `--help` output as specified in the [Usage section of the top-level README.md][usage] or view the +[source code][src] directly. + +[usage]: https://github.com/phylum-dev/phylum-ci/blob/main/README.md#usage +[src]: https://github.com/phylum-dev/phylum-ci/blob/main/src/phylum/ci/cli.py + +```yaml + # NOTE: These are examples. Only one `args` key for the hook is expected + + # Use the defaults for all the arguments. + # The default behavior is to only analyze newly added dependencies against + # the risk domain threshold levels set at the Phylum project level. + # The key can be removed if the defaults are used. + args: [] + + # Consider all dependencies in analysis results instead of just the newly added ones. + # The default is to only analyze newly added dependencies, which can be useful for + # existing code bases that may not meet established project risk thresholds yet, + # but don't want to make things worse. Specifying `--all-deps` can be useful for + # casting the widest net for strict adherence to Quality Assurance (QA) standards. + args: [--all-deps] + + # Some lockfile types (e.g., Python/pip `requirements.txt`) are ambiguous in that + # they can be named differently and may or may not contain strict dependencies. + # In these cases, it is best to specify an explicit lockfile path. + args: [--lockfile=requirements-prod.txt] + + # Thresholds for the five risk domains may be set at the Phylum project level. + # They can be set differently for the hook. + # NOTE: The shortened form is used here for brevity, but the long form might be more + # descriptive for future readers. For instance `--vul-threshold` instead of `-u`. + args: [-u=60, -m=60, -e=70, -c=90, -o=80] + + # Ensure the latest Phylum CLI is installed. + args: [--force-install] + + # Install a specific version of the Phylum CLI. + args: [--phylum-release=3.3.0, --force-install] + + # Mix and match for your specific use case. + args: [-u=60, -m=60, -e=70, -c=90, -o=80, --lockfile=requirements-prod.txt, --all-deps] +``` diff --git a/docs/sync/integrations_overview.md b/docs/sync/integrations_overview.md index 434df061..b47a09c6 100644 --- a/docs/sync/integrations_overview.md +++ b/docs/sync/integrations_overview.md @@ -24,6 +24,12 @@ See the [GitLab CI Integration documentation][gitlab_docs] for more info. [gitlab_docs]: https://docs.phylum.io/docs/gitlab_ci +### Git `pre-commit` Hooks + +See the [Git `pre-commit` documentation][precommit_docs] for more info. + +[precommit_docs]: https://docs.phylum.io/docs/git_precommit + ## Future Integrations If there is an unsupported use case for managing the security of your dependencies, we want to know about it.