diff --git a/detection_rules/devtools.py b/detection_rules/devtools.py index 2f2e181fad9..77dff79ef49 100644 --- a/detection_rules/devtools.py +++ b/detection_rules/devtools.py @@ -5,32 +5,45 @@ """CLI commands for internal detection_rules dev team.""" import dataclasses +import functools import hashlib import io import json import os import shutil import subprocess +import textwrap import time +import typing from pathlib import Path +from typing import Optional, Tuple import click -import typing from elasticsearch import Elasticsearch -from eql import load_dump from kibana.connector import Kibana from . import rule_loader from .cli_utils import single_collection from .eswrap import CollectEvents, add_range_to_dsl from .main import root -from .misc import PYTHON_LICENSE, add_client, GithubClient, Manifest, client_error, getdefault -from .packaging import PACKAGE_FILE, Package, manage_versions, RELEASE_DIR -from .rule import TOMLRule, QueryRuleData -from .rule_loader import production_filter, RuleCollection -from .utils import get_path, dict_hash +from .misc import GithubClient, Manifest, PYTHON_LICENSE, add_client, client_error, getdefault +from .packaging import PACKAGE_FILE, Package, RELEASE_DIR, current_stack_version, manage_versions +from .rule import QueryRuleData, TOMLRule +from .rule_loader import RuleCollection, production_filter +from .utils import dict_hash, get_path, load_dump RULES_DIR = get_path('rules') +GH_CONFIG = Path.home() / ".config" / "gh" / "hosts.yml" + + +def get_github_token() -> Optional[str]: + """Get the current user's GitHub token.""" + token = os.getenv("GITHUB_TOKEN") + + if token is None and GH_CONFIG.exists(): + token = load_dump(str(GH_CONFIG)).get("github.com", {}).get("oauth_token") + + return token @root.group('dev') @@ -129,17 +142,28 @@ def kibana_diff(rule_id, repo, branch, threads): return diff +def add_git_args(f): + @click.argument("local-repo", default=get_path("..", "kibana")) + @click.option("--kibana-directory", "-d", help="Directory to overwrite in Kibana", + default="x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules") + @click.option("--base-branch", "-b", help="Base branch in Kibana", default="master") + @click.option("--branch-name", "-n", help="New branch for the rules commit") + @click.option("--ssh/--http", is_flag=True, help="Method to use for cloning") + @click.option("--github-repo", "-r", help="Repository to use for the branch", default="elastic/kibana") + @click.option("--message", "-m", help="Override default commit message") + @functools.wraps(f) + def decorated(*args, **kwargs): + return f(*args, **kwargs) + + return decorated + + @dev_group.command("kibana-commit") -@click.argument("local-repo", default=get_path("..", "kibana")) -@click.option("--kibana-directory", "-d", help="Directory to overwrite in Kibana", - default="x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules") -@click.option("--base-branch", "-b", help="Base branch in Kibana", default="master") -@click.option("--branch-name", "-n", help="Head branch for rules (default: package name)") -@click.option("--ssh/--http", is_flag=True, help="Method to use for cloning") -@click.option("--github-repo", "-r", help="Repository to use for the branch", default="elastic/kibana") -@click.option("--message", "-m", help="Override default commit message") +@add_git_args +@click.option("--push", "-p", is_flag=True, help="Push the commit to the remote") @click.pass_context -def kibana_commit(ctx, local_repo, github_repo, ssh, kibana_directory, base_branch, branch_name, message): +def kibana_commit(ctx, local_repo: str, github_repo: str, ssh: bool, kibana_directory: str, base_branch: str, + branch_name: Optional[str], message: Optional[str], push: bool) -> (str, str): """Prep a commit and push to Kibana.""" git_exe = shutil.which("git") @@ -149,13 +173,17 @@ def kibana_commit(ctx, local_repo, github_repo, ssh, kibana_directory, base_bran if not os.path.exists(release_dir): click.secho("Release directory doesn't exist.", fg="red", err=True) - click.echo(f"Run {click.style('python -m detection_rules build-release', bold=True)} to populate", err=True) + click.echo(f"Run {click.style('python -m detection_rules dev build-release', bold=True)} to populate", err=True) ctx.exit(1) if not git_exe: click.secho("Unable to find git", err=True, fg="red") ctx.exit(1) + # Get the current hash of the repo + long_commit_hash = subprocess.check_output([git_exe, "rev-parse", "HEAD"], encoding="utf-8").strip() + short_commit_hash = subprocess.check_output([git_exe, "rev-parse", "--short", "HEAD"], encoding="utf-8").strip() + try: if not os.path.exists(local_repo): if not click.confirm(f"Kibana repository doesn't exist at {local_repo}. Clone?"): @@ -168,9 +196,11 @@ def git(*args, show_output=False): method = subprocess.call if show_output else subprocess.check_output return method([git_exe, "-C", local_repo] + list(args), encoding="utf-8") + branch_name = branch_name or f"detection-rules/{package_name}-{short_commit_hash}" + git("checkout", base_branch) git("pull") - git("checkout", "-b", f"rules/{branch_name or package_name}", show_output=True) + git("checkout", "-b", branch_name, show_output=True) git("rm", "-r", kibana_directory) source_dir = os.path.join(release_dir, "rules") @@ -186,14 +216,61 @@ def git(*args, show_output=False): git("add", kibana_directory) - git("commit", "-S", "-m", message) + git("commit", "--no-verify", "-m", message) git("status", show_output=True) + if push: + git("push", "origin", branch_name) + click.echo(f"Kibana repository {local_repo} prepped. Push changes when ready") click.secho(f"cd {local_repo}", bold=True) + return branch_name, long_commit_hash + except subprocess.CalledProcessError as e: - client_error(e.returncode, e, ctx=ctx) + client_error(str(e), e, ctx=ctx) + + +@dev_group.command("kibana-pr") +@click.option("--token", required=True, prompt=True, default=get_github_token(), + help="GitHub token to use for the PR", hide_input=True) +@click.option("--assign", multiple=True, help="GitHub users to assign the PR") +@click.option("--label", multiple=True, help="GitHub labels to add to the PR") +# Pending an official GitHub API +# @click.option("--automerge", is_flag=True, help="Enable auto-merge on the PR") +@click.option("--draft", is_flag=True, help="Open the PR as a draft") +@add_git_args +@click.pass_context +def kibana_pr(ctx: click.Context, label: Tuple[str, ...], assign: Tuple[str, ...], draft: bool, token: str, **kwargs): + """Create a pull request to Kibana.""" + branch_name, commit_hash = ctx.invoke(kibana_commit, push=True, **kwargs) + client = GithubClient(token).authenticated_client + repo = client.get_repo(kwargs["github_repo"]) + + title = f"[Detection Engine] Adds {current_stack_version()} rules" + body = textwrap.dedent(f""" + ## Summary + + Pull updates to detection rules from https://github.com/elastic/detection-rules/tree/{commit_hash}. + + ### Checklist + + Delete any items that are not applicable to this PR. + + - [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), + uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/master/packages/kbn-i18n/README.md) + """).strip() # noqa: E501 + pr = repo.create_pull(title, body, kwargs["base_branch"], branch_name, draft=draft) + + label = set(label) + if label: + pr.add_to_labels(*sorted(label)) + + if assign: + pr.add_to_assignees(*assign) + + click.echo("PR created:") + click.echo(pr.html_url) @dev_group.command('license-check') @@ -708,7 +785,7 @@ def validate_ml_detections_asset(directory): @dev_group.command("update-schemas") def update_schemas(): - from . rule import BaseRuleData, AnyRuleData + from .rule import BaseRuleData, AnyRuleData classes = [BaseRuleData] + list(typing.get_args(AnyRuleData)) for cls in classes: diff --git a/detection_rules/misc.py b/detection_rules/misc.py index afedc5a58bb..c5c86bd1cac 100644 --- a/detection_rules/misc.py +++ b/detection_rules/misc.py @@ -357,7 +357,7 @@ def __init__(self, message, original_error=None): def show(self, file=None, err=True): """Print the error to the console.""" err = f' {self.original_error_type}' if self.original_error else '' - msg = f'{click.style(f"CLI Error{self.original_error_type}", fg="red", bold=True)}: {self.format_message()}' + msg = f'{click.style(f"CLI Error {self.original_error_type}", fg="red", bold=True)}: {self.format_message()}' click.echo(msg, err=err, file=file) diff --git a/requirements-dev.txt b/requirements-dev.txt index 635cde85892..c8331448df7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1 +1 @@ -PyGithub==1.53 +PyGithub==1.55