Merge branch 'main' of github.com:slalombuild/secureli into feature/s…

…ecureli-344-linter-config-file-creation

.github/workflows/publish.yml

@@ -146,7 +146,7 @@ jobs:
     name: Integration Testing
     needs: [ build-test, secureli-release, secureli-publish, deploy ]
     if: |
-      always() && 
+      always() &&
       (needs.secureli-publish.result == 'success' || needs.secureli-publish.result == 'skipped') &&
       (needs.deploy.result == 'success' || needs.deploy.result == 'skipped')
     uses: ./.github/workflows/integration_testing.yml

.pre-commit-config.yaml

@@ -1 +1,14 @@
-repos: []
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-yaml
+    -   id: check-added-large-files
+-   repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 23.11.0
+    hooks:
+    -   id: black

.secureli.yaml

@@ -1,5 +1,5 @@
 echo:
-  level: ERROR
+  level: DEBUG
 repo_files:
   exclude_file_patterns:
   - .idea/

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "secureli"
-version = "0.18.0"
+version = "0.19.0"
 description = "Secure Project Manager"
 authors = ["Caleb Tonn <[email protected]>"]
 license = "Apache-2.0"

secureli/abstractions/pre_commit.py

@@ -1,10 +1,18 @@
-import stat
-import subprocess
 from pathlib import Path
 
-from typing import Optional
+# Note that this import is pulling from the pre-commit tool's internals.
+# A cleaner approach would be to update pre-commit
+# by implementing a dry-run option for the `autoupdate` command
+from pre_commit.commands.autoupdate import RevInfo as HookRepoRevInfo
+from typing import Any, Optional
 
 import pydantic
+import re
+import stat
+import subprocess
+import yaml
+
+from secureli.repositories.settings import PreCommitSettings
 
 
 class InstallFailedError(Exception):
@@ -34,6 +42,16 @@ class ExecuteResult(pydantic.BaseModel):
     output: str
 
 
+class RevisionPair(pydantic.BaseModel):
+    """
+    Used for updating hooks.
+    This could alternatively be implemented as named tuple, but those can't subclass pydantic's BaseModel
+    """
+
+    oldRev: str
+    newRev: str
+
+
 class InstallResult(pydantic.BaseModel):
     """
     The results of calling install
@@ -105,6 +123,48 @@ def execute_hooks(
         else:
             return ExecuteResult(successful=True, output=output)
 
+    def check_for_hook_updates(
+        self,
+        config: PreCommitSettings,
+        tags_only: bool = True,
+        freeze: Optional[bool] = None,
+    ) -> dict[str, RevisionPair]:
+        """
+        Call's pre-commit's undocumented/internal functions to check for updates to repositories containing hooks
+        :param config: A model representing the contents of the .pre-commit-config.yaml file.
+        See :meth:`~get_pre_commit_config` to deserialize the config file into a model.
+        :param tags_only: Represents whether we should check for the latest git tag or the latest git commit.
+        This defaults to true since anyone who cares enough to be on the "bleeding edge" (tags_only=False) can manually
+        update with `secureli update`.
+        :param freeze: This indicates whether tags names should be converted to the corresponding commit hash,
+        in case a tag is updated to point to a different commit in the future. If not specified, we check
+        the existing revision in the .pre-commit-config.yaml file to see if it looks like a commit (40-character hex string),
+        and infer that we should replace the commit hash with another commit hash ("freezing" the tag ref).
+        :returns: A dictionary of outdated with repositories, where the key is the repository URL and the RevisionPair value
+        indicates the old and new revisions. If the result is empty/falsy, then no updates were found.
+        """
+
+        git_commit_sha_pattern = re.compile(r"^[a-f0-9]{40}$")
+
+        repos_to_update: dict[str, RevisionPair] = {}
+        for repo_config in config.repos:
+            repo_config_dict = repo_config.__dict__ | {
+                "repo": repo_config.url
+            }  # PreCommitSettings uses "url" instead of "repo", so we need to copy that value over
+            old_rev_info = HookRepoRevInfo.from_config(repo_config_dict)
+            # if the revision currently specified in .pre-commit-config.yaml looks like a full git SHA
+            # (40-character hex string), then set freeze to True
+            freeze = (
+                bool(git_commit_sha_pattern.fullmatch(repo_config.rev))
+                if freeze is None
+                else freeze
+            )
+            new_rev_info = old_rev_info.update(tags_only=tags_only, freeze=freeze)
+            revisions = RevisionPair(oldRev=old_rev_info.rev, newRev=new_rev_info.rev)
+            if revisions.oldRev != revisions.newRev:
+                repos_to_update[old_rev_info.repo] = revisions
+        return repos_to_update
+
     def autoupdate_hooks(
         self,
         folder_path: Path,
@@ -113,7 +173,7 @@ def autoupdate_hooks(
         repos: Optional[list] = None,
     ) -> ExecuteResult:
         """
-        Updates the precommit hooks but executing precommit's autoupdate command.  Additional info at
+        Updates the precommit hooks by executing precommit's autoupdate command. Additional info at
         https://pre-commit.com/#pre-commit-autoupdate
         :param folder_path: Indicates the git folder against which you run secureli
         :param bleeding_edge: True if updating to the bleeding edge of the default branch instead of
@@ -199,3 +259,17 @@ def remove_unused_hooks(self, folder_path: Path) -> ExecuteResult:
             return ExecuteResult(successful=False, output=output)
         else:
             return ExecuteResult(successful=True, output=output)
+
+    def get_pre_commit_config(self, folder_path: Path):
+        """
+        Gets the contents of the .pre-commit-config file and returns it as a dictionary
+        :return: Dictionary containing the contents of the .pre-commit-config.yaml file
+        """
+        path_to_config = folder_path / ".pre-commit-config.yaml"
+        with open(path_to_config, "r") as f:
+            # For some reason, the mocking causes an infinite loop when we try to use yaml.safe_load()
+            # directly on the file-like object f. Reading the contents of the file into a string as a workaround.
+            # return PreCommitSettings(**yaml.safe_load(f))  # TODO figure out why this isn't working
+            contents = f.read()
+            yaml_values = yaml.safe_load(contents)
+            return PreCommitSettings(**yaml_values)

secureli/actions/action.py

@@ -2,9 +2,6 @@
 from enum import Enum
 from pathlib import Path
 from typing import Optional
-
-import pydantic
-
 from secureli.abstractions.echo import EchoAbstraction, Color
 from secureli.abstractions.pre_commit import (
     InstallFailedError,
@@ -21,14 +18,13 @@
 from secureli.services.scanner import ScannerService, ScanMode
 from secureli.services.updater import UpdaterService
 
+import pydantic
+
 
 class VerifyOutcome(str, Enum):
     INSTALL_CANCELED = "install-canceled"
     INSTALL_FAILED = "install-failed"
     INSTALL_SUCCEEDED = "install-succeeded"
-    UPGRADE_CANCELED = "upgrade-canceled"
-    UPGRADE_SUCCEEDED = "upgrade-succeeded"
-    UPGRADE_FAILED = "upgrade-failed"
     UPDATE_CANCELED = "update-canceled"
     UPDATE_SUCCEEDED = "update-succeeded"
     UPDATE_FAILED = "update-failed"
@@ -249,7 +245,8 @@ def _update_secureli(self, always_yes: bool):
 
         update_result = self.action_deps.updater.update()
         details = update_result.output
-        self.action_deps.echo.print(details)
+        if details:
+            self.action_deps.echo.print(details)
 
         if update_result.successful:
             return VerifyResult(outcome=VerifyOutcome.UPDATE_SUCCEEDED)
@@ -259,7 +256,7 @@ def _update_secureli(self, always_yes: bool):
     def _update_secureli_config_only(self, always_yes: bool) -> VerifyResult:
         self.action_deps.echo.print("seCureLI is using an out-of-date config.")
         response = always_yes or self.action_deps.echo.confirm(
-            "Update config only now?",
+            "Update configuration now?",
             default_response=True,
         )
         if not response:

secureli/actions/scan.py

@@ -1,17 +1,25 @@
 import json
 import sys
 from pathlib import Path
+from time import time
 from typing import Optional
 
 from secureli.abstractions.echo import EchoAbstraction
-from secureli.actions.action import VerifyOutcome, Action, ActionDependencies
+from secureli.actions.action import (
+    VerifyOutcome,
+    Action,
+    ActionDependencies,
+    VerifyResult,
+)
 from secureli.services.logging import LoggingService, LogAction
 from secureli.services.scanner import (
     ScanMode,
     ScannerService,
 )
 from secureli.utilities.usage_stats import post_log, convert_failures_to_failure_count
 
+ONE_WEEK_IN_SECONDS: int = 7 * 24 * 60 * 60
+
 
 class ScanAction(Action):
     """The action for the secureli `scan` command, orchestrating services and outputs results"""
@@ -34,6 +42,34 @@ def __init__(
         self.echo = echo
         self.logging = logging
 
+    def _check_secureli_hook_updates(self, folder_path: Path) -> VerifyResult:
+        """
+        Queries repositories referenced by pre-commit hooks to check
+        if we have the latest revisions listed in the .pre-commit-config.yaml file
+        :param folder_path: The folder path containing the .pre-commit-config.yaml file
+        """
+
+        self.action_deps.echo.info("Checking for pre-commit hook updates...")
+        pre_commit_config = self.scanner.pre_commit.get_pre_commit_config(folder_path)
+
+        repos_to_update = self.scanner.pre_commit.check_for_hook_updates(
+            pre_commit_config
+        )
+
+        if not repos_to_update:
+            self.action_deps.echo.info("No hooks to update")
+            return VerifyResult(outcome=VerifyOutcome.UP_TO_DATE)
+
+        for repo, revs in repos_to_update.items():
+            self.action_deps.echo.debug(
+                f"Found update for {repo}: {revs.oldRev} -> {revs.newRev}"
+            )
+        self.action_deps.echo.warning(
+            "You have out-of-date pre-commit hooks. Run `secureli update` to update them."
+        )
+        # Since we don't actually perform the updates here, return an outcome of UPDATE_CANCELLED
+        return VerifyResult(outcome=VerifyOutcome.UPDATE_CANCELED)
+
     def scan_repo(
         self,
         folder_path: Path,
@@ -53,6 +89,14 @@ def scan_repo(
         """
         verify_result = self.verify_install(folder_path, False, always_yes)
 
+        # Check if pre-commit hooks are up-to-date
+        secureli_config = self.action_deps.secureli_config.load()
+        now: int = int(time())
+        if (secureli_config.last_hook_update_check or 0) + ONE_WEEK_IN_SECONDS < now:
+            self._check_secureli_hook_updates(folder_path)
+            secureli_config.last_hook_update_check = now
+            self.action_deps.secureli_config.save(secureli_config)
+
         if verify_result.outcome in self.halting_outcomes:
             return
 

secureli/main.py

@@ -62,14 +62,14 @@ def init(
         help="Say 'yes' to every prompt automatically without input",
     ),
     directory: Annotated[
-        Optional[Path],
+        Path,
         Option(
             ".",
             "--directory",
             "-d",
             help="Run secureli against a specific directory",
         ),
-    ] = ".",
+    ] = Path("."),
 ):
     """
     Detect languages and initialize pre-commit hooks and linters for the project
@@ -100,14 +100,14 @@ def scan(
         help="Limit the scan to a specific hook ID from your pre-commit config",
     ),
     directory: Annotated[
-        Optional[Path],
+        Path,
         Option(
             ".",
             "--directory",
             "-d",
             help="Run secureli against a specific directory",
         ),
-    ] = ".",
+    ] = Path("."),
 ):
     """
     Performs an explicit check of the repository to detect security issues without remote logging.
@@ -133,14 +133,14 @@ def update(
         help="Update the installed pre-commit hooks to their latest versions",
     ),
     directory: Annotated[
-        Optional[Path],
+        Path,
         Option(
             ".",
             "--directory",
             "-d",
             help="Run secureli against a specific directory",
         ),
-    ] = ".",
+    ] = Path("."),
 ):
     """
     Update linters, configuration, and all else needed to maintain a secure repository.

secureli/repositories/secureli_config.py

@@ -9,9 +9,10 @@
 
 
 class SecureliConfig(BaseModel):
-    languages: Optional[list[str]]
-    lint_languages: Optional[list[str]]
-    version_installed: Optional[str]
+    languages: Optional[list[str]] = None
+    lint_languages: Optional[list[str]] = None
+    version_installed: Optional[str] = None
+    last_hook_update_check: Optional[int] = 0
 
 
 class DeprecatedSecureliConfig(BaseModel):
@@ -94,10 +95,13 @@ def update(self) -> SecureliConfig:
         with open(secureli_config_path, "r") as f:
             data = yaml.safe_load(f)
             old_config = DeprecatedSecureliConfig.parse_obj(data)
+        languages: list[str] | None = (
+            [old_config.overall_language] if old_config.overall_language else None
+        )
 
         return SecureliConfig(
-            languages=[old_config.overall_language],
-            lint_languages=[old_config.overall_language],
+            languages=languages,
+            lint_languages=languages,
             version_installed=old_config.version_installed,
         )