From a2ba772b912191cfa46e8dc53a8f292d7c306533 Mon Sep 17 00:00:00 2001 From: George Hickman Date: Thu, 2 Nov 2023 10:09:44 +0000 Subject: [PATCH 1/3] Move GitHub subcommand down into the github module We're about to add more commands so we don't want to have them all in the same cli.py. This moves us to having a cli.py for each sub command, which is then registered in the top level cli.py. --- metrics/cli.py | 32 ++------------------------ metrics/github/{commands.py => cli.py} | 30 +++++++++++++++++++----- 2 files changed, 26 insertions(+), 36 deletions(-) rename metrics/github/{commands.py => cli.py} (59%) diff --git a/metrics/cli.py b/metrics/cli.py index 6fb7d1ed..ccec1226 100644 --- a/metrics/cli.py +++ b/metrics/cli.py @@ -1,6 +1,6 @@ import click -from .github import commands as github_commands +from .github.cli import github from .logs import setup_logging @@ -15,32 +15,4 @@ def cli(ctx, debug): ctx.obj["DEBUG"] = debug -@cli.group() -@click.option("--token", required=True, envvar="GITHUB_TOKEN") -@click.pass_context -def github(ctx, token): - ctx.ensure_object(dict) - - ctx.obj["TOKEN"] = token - - -@github.command() -@click.argument("org") -@click.argument("date", type=click.DateTime()) -@click.option("--days-threshold", type=int) -@click.pass_context -def pr_queue(ctx, org, date, days_threshold): - date = date.date() - - github_commands.pr_queue(org, date, days_threshold) - - -@github.command() -@click.argument("org") -@click.argument("date", type=click.DateTime()) -@click.option("--days", default=7, type=int) -@click.pass_context -def pr_throughput(ctx, org, date, days): - date = date.date() - - github_commands.pr_throughput(org, date, days) +cli.add_command(github) diff --git a/metrics/github/commands.py b/metrics/github/cli.py similarity index 59% rename from metrics/github/commands.py rename to metrics/github/cli.py index 990c5238..e55bdc1e 100644 --- a/metrics/github/commands.py +++ b/metrics/github/cli.py @@ -1,21 +1,34 @@ from datetime import timedelta +import click import structlog -from .. import influxdb, timescaledb # noqa: F401 +from .. import influxdb from . import api from .prs import process_prs +log = structlog.get_logger() writer = influxdb.write -# writer = timescaledb.write -log = structlog.get_logger() +@click.group() +@click.option("--token", required=True, envvar="GITHUB_TOKEN") +@click.pass_context +def github(ctx, token): + ctx.ensure_object(dict) + + ctx.obj["TOKEN"] = token -def pr_queue(org, date, days_threshold=None): +@github.command() +@click.argument("org") +@click.argument("date", type=click.DateTime()) +@click.option("--days-threshold", type=int) +@click.pass_context +def pr_queue(ctx, org, date, days_threshold): """The number of PRs open on the given date""" + date = date.date() prs = api.prs_open_on_date(org, date) if days_threshold is not None: @@ -32,10 +45,15 @@ def pr_queue(org, date, days_threshold=None): process_prs(writer, f"queue{suffix}", prs, date) -def pr_throughput(org, date, days): +@github.command() +@click.argument("org") +@click.argument("date", type=click.DateTime()) +@click.option("--days", default=7, type=int) +@click.pass_context +def pr_throughput(ctx, org, date, days): """PRs opened in the last number of days given""" + end = date.date() start = date - timedelta(days=days) - end = date prs = api.prs_opened_in_the_last_N_days(org, start, end) From de0df86ee57d4e79e7010b2ee0425b5af94b4c86 Mon Sep 17 00:00:00 2001 From: George Hickman Date: Thu, 2 Nov 2023 16:12:34 +0000 Subject: [PATCH 2/3] Allow tags to be empty when writing to influx --- metrics/influxdb.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/metrics/influxdb.py b/metrics/influxdb.py index 23d3c70c..ce4b34d5 100644 --- a/metrics/influxdb.py +++ b/metrics/influxdb.py @@ -36,6 +36,9 @@ def delete(key): def write(measurement, date, value, tags=None): + if tags is None: + tags = {} + # convert date to a timestamp # TODO: do we need to do any checking to make sure this is tz-aware and in # UTC? @@ -43,9 +46,8 @@ def write(measurement, date, value, tags=None): point = Point(measurement).field("number", value).time(dt) - if tags is not None: - for k, v in tags.items(): - point = point.tag(k, v) + for k, v in tags.items(): + point = point.tag(k, v) write_api.write(bucket=BUCKET, org=ORG, record=point) From 514e048c483701fbf89acd5ab2d36791b4ef2391 Mon Sep 17 00:00:00 2001 From: George Hickman Date: Fri, 3 Nov 2023 09:58:27 +0000 Subject: [PATCH 3/3] Add slack support for pulling tech support messages --- metrics/cli.py | 2 ++ metrics/slack/__init__.py | 0 metrics/slack/api.py | 27 ++++++++++++++++++++++ metrics/slack/cli.py | 48 +++++++++++++++++++++++++++++++++++++++ pyproject.toml | 1 + requirements.prod.txt | 8 +++++++ 6 files changed, 86 insertions(+) create mode 100644 metrics/slack/__init__.py create mode 100644 metrics/slack/api.py create mode 100644 metrics/slack/cli.py diff --git a/metrics/cli.py b/metrics/cli.py index ccec1226..789c49be 100644 --- a/metrics/cli.py +++ b/metrics/cli.py @@ -2,6 +2,7 @@ from .github.cli import github from .logs import setup_logging +from .slack.cli import slack @click.group() @@ -16,3 +17,4 @@ def cli(ctx, debug): cli.add_command(github) +cli.add_command(slack) diff --git a/metrics/slack/__init__.py b/metrics/slack/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/metrics/slack/api.py b/metrics/slack/api.py new file mode 100644 index 00000000..b5bd6227 --- /dev/null +++ b/metrics/slack/api.py @@ -0,0 +1,27 @@ +from datetime import datetime, time, timedelta + +from slack_bolt import App + + +bennet_bot_id = "B03UJ58MALV" + + +def get_app(signing_secret, token): + return App(token=token, signing_secret=signing_secret) + + +def iter_messages(app, channel_id, date=None): + start = end = 0 + if date: + start = datetime.combine(date, time()).timestamp() + end = (datetime.combine(date, time()) + timedelta(days=1)).timestamp() + + for page in app.client.conversations_history( + channel=channel_id, + include_all_metadata=True, + latest=end, + oldest=start, + ): + for message in page["messages"]: + if "bot_id" in message and message["bot_id"] == bennet_bot_id: + yield message diff --git a/metrics/slack/cli.py b/metrics/slack/cli.py new file mode 100644 index 00000000..d4467546 --- /dev/null +++ b/metrics/slack/cli.py @@ -0,0 +1,48 @@ +import itertools +from datetime import datetime + +import click + +from .. import influxdb +from .api import get_app, iter_messages + + +writer = influxdb.write + + +@click.group() +@click.option("--signing-secret", required=True, envvar="SLACK_SIGNING_SECRET") +@click.option("--token", required=True, envvar="SLACK_TOKEN") +@click.pass_context +def slack(ctx, signing_secret, token): + ctx.ensure_object(dict) + + ctx.obj["SLACK_SIGNING_SECRET"] = signing_secret + ctx.obj["SLACK_TOKEN"] = token + + +@slack.command() +@click.argument("date", type=click.DateTime(), required=False) +@click.option( + "--tech-support-channel-id", required=True, envvar="SLACK_TECH_SUPPORT_CHANNEL_ID" +) +@click.option("--backfill", is_flag=True) +@click.pass_context +def tech_support(ctx, date, tech_support_channel_id, backfill): + if backfill and date: + raise click.BadParameter("--backfill cannot be used with a date") + + day = None if backfill else date.date() + + app = get_app(ctx.obj["SLACK_SIGNING_SECRET"], ctx.obj["SLACK_TOKEN"]) + + messages = iter_messages(app, tech_support_channel_id, date=day) + + for date, messages in itertools.groupby( + messages, lambda m: datetime.fromtimestamp(float(m["ts"])).date() + ): + writer( + "slack_tech_support_requests", + date, + len(list(messages)), + ) diff --git a/pyproject.toml b/pyproject.toml index 4aa9ec38..30674dc2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ dependencies = [ "influxdb-client", "requests", "psycopg[binary]", + "slack-bolt", "structlog", ] dynamic = ["version"] diff --git a/requirements.prod.txt b/requirements.prod.txt index bc15aa7c..def51d7f 100644 --- a/requirements.prod.txt +++ b/requirements.prod.txt @@ -272,6 +272,14 @@ six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 # via python-dateutil +slack-bolt==1.18.0 \ + --hash=sha256:43b121acf78440303ce5129e53be36bdfe5d926a193daef7daf2860688e65dd3 \ + --hash=sha256:63089a401ae3900c37698890249acd008a4651d06e86194edc7b72a00819bbac + # via metrics (pyproject.toml) +slack-sdk==3.23.0 \ + --hash=sha256:2a8513505cced20ceee22b5b49c11d9545caa6234b56bf0ad47133ea5b357d10 \ + --hash=sha256:9d6ebc4ff74e7983e1b27dbdb0f2bb6fc3c2a2451694686eaa2be23bbb085a73 + # via slack-bolt sqlite-fts4==1.0.3 \ --hash=sha256:0359edd8dea6fd73c848989e1e2b1f31a50fe5f9d7272299ff0e8dbaa62d035f \ --hash=sha256:78b05eeaf6680e9dbed8986bde011e9c086a06cb0c931b3cf7da94c214e8930c