From 333ccd21cb65256102c815d592bd32e521d2106c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Maru=C5=A1in?= Date: Tue, 20 Aug 2019 09:38:11 +0200 Subject: [PATCH 01/15] fix webhooks.py after API<-->OGR replacement --- release_bot/webhooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release_bot/webhooks.py b/release_bot/webhooks.py index 6944d4f..1de039d 100644 --- a/release_bot/webhooks.py +++ b/release_bot/webhooks.py @@ -64,7 +64,7 @@ def handle_issue(self): self.release_bot.git.pull() try: self.release_bot.load_release_conf() - if (self.release_bot.new_release.get('trigger_on_issue') and + if (self.release_bot.new_release.trigger_on_issue and self.release_bot.find_open_release_issues()): if self.release_bot.new_release.labels is not None: self.release_bot.project.add_issue_labels( From 635a483de1c9f28f77f3165b9a99ad0981dbf97f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Maru=C5=A1in?= Date: Wed, 21 Aug 2019 15:38:20 +0200 Subject: [PATCH 02/15] first working code version with celery lib --- release_bot/celerizer.py | 47 ++++++++++++++++ release_bot/celery_task.py | 106 +++++++++++++++++++++++++++++++++++ release_bot/configuration.py | 26 ++++----- release_bot/github.py | 9 ++- release_bot/webhooks.py | 57 +------------------ requirements.txt | 4 +- 6 files changed, 177 insertions(+), 72 deletions(-) create mode 100644 release_bot/celerizer.py create mode 100644 release_bot/celery_task.py diff --git a/release_bot/celerizer.py b/release_bot/celerizer.py new file mode 100644 index 0000000..138759f --- /dev/null +++ b/release_bot/celerizer.py @@ -0,0 +1,47 @@ +# MIT License +# +# Copyright (c) 2018-2019 Red Hat, Inc. + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from celery import Celery +from os import getenv + + +class Celerizer: + def __init__(self): + self._celery_app = None + + @property + def celery_app(self): + if self._celery_app is None: + redis_host = getenv("REDIS_SERVICE_HOST", "localhost") + redis_port = getenv("REDIS_SERVICE_PORT", "6379") + redis_db = getenv("REDIS_SERVICE_DB", "0") + redis_url = "redis://{host}:{port}/{db}".format( + host=redis_host, port=redis_port, db=redis_db + ) + + # http://docs.celeryproject.org/en/latest/reference/celery.html#celery.Celery + self._celery_app = Celery(backend=redis_url, broker=redis_url) + return self._celery_app + + +celerizer = Celerizer() +celery_app = celerizer.celery_app diff --git a/release_bot/celery_task.py b/release_bot/celery_task.py new file mode 100644 index 0000000..39b56f3 --- /dev/null +++ b/release_bot/celery_task.py @@ -0,0 +1,106 @@ +# MIT License +# +# Copyright (c) 2018-2019 Red Hat, Inc. + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from pathlib import Path + +from release_bot.celerizer import celery_app +from release_bot.exceptions import ReleaseException +from release_bot.configuration import configuration +from release_bot.releasebot import ReleaseBot + + +@celery_app.task(bind=True, name="task.celery_task.parse_web_hook_payload") +def parse_web_hook_payload(self, webhook_payload): + """ + Parse json webhook payload callback + :param webhook_payload: json from github webhook + """ + if 'issue' in webhook_payload.keys(): + if webhook_payload['action'] == 'opened': + handle_issue(webhook_payload) + elif 'pull_request' in webhook_payload.keys(): + if webhook_payload['action'] == 'closed': + if webhook_payload['pull_request']['merged'] is True: + handle_pr(webhook_payload) + + +def handle_issue(webhook_payload): + """Handler for newly opened issues""" + configuration.configuration = Path( + '/Users/marusinm/Documents/Python/tmp/bot-test-conf/conf.yaml').resolve() + configuration.load_configuration() + # add configuration from Github webhook + configuration.repository_name = webhook_payload['repository']['name'] + configuration.repository_owner = webhook_payload['repository']['owner']['login'] + configuration.github_username = webhook_payload['issue']['user']['login'] + configuration.clone_url = webhook_payload['repository']['clone_url'] + + logger = configuration.logger + release_bot = ReleaseBot(configuration) + + logger.info("Resolving opened issue") + release_bot.git.pull() + try: + release_bot.load_release_conf() + if (release_bot.new_release.trigger_on_issue and + release_bot.find_open_release_issues()): + if release_bot.new_release.labels is not None: + release_bot.project.add_issue_labels( + release_bot.new_pr.issue_number, + release_bot.new_release.labels) + release_bot.make_release_pull_request() + except ReleaseException as exc: + logger.error(exc) + + +def handle_pr(webhook_payload): + """Handler for merged PR""" + configuration.configuration = Path( + '/Users/marusinm/Documents/Python/tmp/bot-test-conf/conf.yaml').resolve() + configuration.load_configuration() + # add configuration from Github webhook + configuration.repository_name = webhook_payload['repository']['name'] + configuration.repository_owner = webhook_payload['repository']['owner']['login'] + configuration.github_username = webhook_payload['pull_request']['user']['login'] + configuration.clone_url = webhook_payload['repository']['clone_url'] + + logger = configuration.logger + release_bot = ReleaseBot(configuration) + + logger.info("Resolving opened PR") + release_bot.git.pull() + try: + release_bot.load_release_conf() + if release_bot.find_newest_release_pull_request(): + release_bot.make_new_github_release() + # Try to do PyPi release regardless whether we just did github release + # for case that in previous iteration (of the 'while True' loop) + # we succeeded with github release, but failed with PyPi release + release_bot.make_new_pypi_release() + except ReleaseException as exc: + logger.error(exc) + + msg = ''.join(release_bot.github.comment) + release_bot.project.pr_comment(release_bot.new_release.pr_number, msg) + release_bot.github.comment = [] # clean up + + diff --git a/release_bot/configuration.py b/release_bot/configuration.py index 28ec49b..65abd16 100644 --- a/release_bot/configuration.py +++ b/release_bot/configuration.py @@ -26,9 +26,6 @@ class Configuration: - # note that required items need to reference strings as their length is checked - REQUIRED_ITEMS = {"conf": ['repository_name', 'repository_owner'], - "release-conf": []} def __init__(self): self.version = __version__ @@ -105,11 +102,6 @@ def load_configuration(self): for key, value in file.items(): if hasattr(self, key): setattr(self, key, value) - # check if required items are present - for item in self.REQUIRED_ITEMS['conf']: - if item not in file: - self.logger.error(f"Item {item!r} is required in configuration!") - sys.exit(1) # if user hasn't specified clone_url, use default if 'clone_url' not in file: self.clone_url = (f'https://github.com/{self.repository_owner}' @@ -128,8 +120,6 @@ def load_release_conf(self, conf): self.logger.error("No release-conf.yaml found in " f"{self.repository_owner}/{self.repository_name} repository root!\n" "You have to add one for releasing to PyPi") - if self.REQUIRED_ITEMS['release-conf']: - sys.exit(1) parsed_conf = yaml.safe_load(conf) or {} # If pypi option is not specified in release-conf.yaml, @@ -137,10 +127,6 @@ def load_release_conf(self, conf): parsed_conf.setdefault('pypi', True) parsed_conf = {k: v for (k, v) in parsed_conf.items() if v} - for item in self.REQUIRED_ITEMS['release-conf']: - if item not in parsed_conf: - self.logger.error(f"Item {item!r} is required in release-conf!") - sys.exit(1) for index, label in enumerate(parsed_conf.get('labels', [])): parsed_conf['labels'][index] = str(label) @@ -195,6 +181,18 @@ def get_project(self): Project instance is used for manipulating with Github/Pagure repo. :return: ogr Github/Pagure project instance or None """ + # Return instance for github app + if self.github_app_id != '': + with open(self.github_app_cert_path, 'r') as cert: + github_cert = cert.read() + return get_project(url=self.clone_url, + custom_instances=[ + GithubService(token=None, + github_app_id=self.github_app_id, + github_app_private_key=github_cert, + )]) + + # Return instance for regular user (local machine) return get_project(url=self.clone_url, custom_instances=[ GithubService(token=self.github_token), diff --git a/release_bot/github.py b/release_bot/github.py index 2cab4e1..5dd9c7c 100644 --- a/release_bot/github.py +++ b/release_bot/github.py @@ -323,9 +323,12 @@ def get_user_contact(self): """ name = 'Release bot' mail = 'bot@releasebot.bot' - if which_service(self.project) == GitService.Github: - name = self.project.service.user.get_username() - mail = self.project.service.user.get_email() + + # don't set in case of Github app instance, it uses defaults + if self.conf.github_app_id == '': + if which_service(self.project) == GitService.Github: + name = self.project.service.user.get_username() + mail = self.project.service.user.get_email() return name, mail def get_file(self, name): diff --git a/release_bot/webhooks.py b/release_bot/webhooks.py index 1de039d..d5db593 100644 --- a/release_bot/webhooks.py +++ b/release_bot/webhooks.py @@ -19,7 +19,7 @@ from flask import request, jsonify from flask.views import View -from release_bot.exceptions import ReleaseException +from release_bot.celerizer import celery_app class GithubWebhooksHandler(View): @@ -36,59 +36,8 @@ def dispatch_request(self): self.logger.info(f'New github webhook call from ' f'{self.conf.repository_owner}/{self.conf.repository_name}') if request.is_json: - self.parse_payload(request.get_json()) + celery_app.send_task(name="task.celery_task.parse_web_hook_payload", + kwargs={"webhook_payload": request.get_json()}) else: self.logger.error("This webhook doesn't contain JSON") return jsonify(result={"status": 200}) - - def parse_payload(self, webhook_payload): - """ - Parse json webhook payload callback - :param webhook_payload: json from github webhook - """ - self.logger.info(f"release-bot v{self.conf.version} reporting for duty!") - if 'issue' in webhook_payload.keys(): - if webhook_payload['action'] == 'opened': - self.handle_issue() - elif 'pull_request' in webhook_payload.keys(): - if webhook_payload['action'] == 'closed': - if webhook_payload['pull_request']['merged'] is True: - self.handle_pr() - else: - self.logger.info("This webhook doesn't contain opened issue or merged PR") - self.logger.debug("Done. Waiting for another github webhook callback") - - def handle_issue(self): - """Handler for newly opened issues""" - self.logger.info("Resolving opened issue") - self.release_bot.git.pull() - try: - self.release_bot.load_release_conf() - if (self.release_bot.new_release.trigger_on_issue and - self.release_bot.find_open_release_issues()): - if self.release_bot.new_release.labels is not None: - self.release_bot.project.add_issue_labels( - self.release_bot.new_pr.issue_number, - self.release_bot.new_release.labels) - self.release_bot.make_release_pull_request() - except ReleaseException as exc: - self.logger.error(exc) - - def handle_pr(self): - """Handler for merged PR""" - self.logger.info("Resolving opened PR") - self.release_bot.git.pull() - try: - self.release_bot.load_release_conf() - if self.release_bot.find_newest_release_pull_request(): - self.release_bot.make_new_github_release() - # Try to do PyPi release regardless whether we just did github release - # for case that in previous iteration (of the 'while True' loop) - # we succeeded with github release, but failed with PyPi release - self.release_bot.make_new_pypi_release() - except ReleaseException as exc: - self.logger.error(exc) - - msg = ''.join(self.release_bot.github.comment) - self.release_bot.project.pr_comment(self.release_bot.new_release.pr_number, msg) - self.release_bot.github.comment = [] # clean up diff --git a/requirements.txt b/requirements.txt index 227d24a..4a7f7e3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,6 @@ PyJWT flask gitchangelog pystache -ogr@git+https://github.com/packit-service/ogr@master#egg=ogr +celery[redis] +cryptography +ogr@git+https://github.com/packit-service/ogr@master#egg=ogr \ No newline at end of file From d5d9e36e499783d7035bfe81842c81e6af4f44c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Maru=C5=A1in?= Date: Wed, 21 Aug 2019 16:37:57 +0200 Subject: [PATCH 03/15] celery-webhooks bugfix --- release_bot/celery_task.py | 9 +++++++-- release_bot/releasebot.py | 8 ++++---- release_bot/webhooks.py | 7 ++----- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/release_bot/celery_task.py b/release_bot/celery_task.py index 39b56f3..255b69f 100644 --- a/release_bot/celery_task.py +++ b/release_bot/celery_task.py @@ -21,6 +21,7 @@ # SOFTWARE. from pathlib import Path +from os import getenv from release_bot.celerizer import celery_app from release_bot.exceptions import ReleaseException @@ -47,12 +48,14 @@ def handle_issue(webhook_payload): """Handler for newly opened issues""" configuration.configuration = Path( '/Users/marusinm/Documents/Python/tmp/bot-test-conf/conf.yaml').resolve() - configuration.load_configuration() + # configuration.configuration = Path(getenv("CONF_PATH", "secrets/prod/conf.yaml")).resolve() + # add configuration from Github webhook configuration.repository_name = webhook_payload['repository']['name'] configuration.repository_owner = webhook_payload['repository']['owner']['login'] configuration.github_username = webhook_payload['issue']['user']['login'] configuration.clone_url = webhook_payload['repository']['clone_url'] + configuration.load_configuration() # load the rest of configuration if there is any logger = configuration.logger release_bot = ReleaseBot(configuration) @@ -76,12 +79,14 @@ def handle_pr(webhook_payload): """Handler for merged PR""" configuration.configuration = Path( '/Users/marusinm/Documents/Python/tmp/bot-test-conf/conf.yaml').resolve() - configuration.load_configuration() + # configuration.configuration = Path(getenv("CONF_PATH", "secrets/prod/conf.yaml")).resolve() + # add configuration from Github webhook configuration.repository_name = webhook_payload['repository']['name'] configuration.repository_owner = webhook_payload['repository']['owner']['login'] configuration.github_username = webhook_payload['pull_request']['user']['login'] configuration.clone_url = webhook_payload['repository']['clone_url'] + configuration.load_configuration() # load the rest of configuration if there is any logger = configuration.logger release_bot = ReleaseBot(configuration) diff --git a/release_bot/releasebot.py b/release_bot/releasebot.py index 8917889..81d9f71 100644 --- a/release_bot/releasebot.py +++ b/release_bot/releasebot.py @@ -62,12 +62,12 @@ def cleanup(self): self.github.comment = [] self.git.cleanup() - def create_flask_instance(self): + @staticmethod + def create_flask_instance(configuration): """Create flask instance for receiving Github webhooks""" app = Flask(__name__) app.add_url_rule('/webhook-handler/', # route for github callbacks view_func=GithubWebhooksHandler.as_view('github_webhooks_handler', - release_bot=self, conf=configuration), methods=['POST', ]) app.run(host='0.0.0.0', port=8080) @@ -323,10 +323,10 @@ def main(): else: CLI.get_configuration(args) configuration.load_configuration() - rb = ReleaseBot(configuration) if configuration.webhook_handler: - rb.create_flask_instance() + ReleaseBot.create_flask_instance(configuration) else: + rb = ReleaseBot(configuration) rb.run() diff --git a/release_bot/webhooks.py b/release_bot/webhooks.py index d5db593..9559ce3 100644 --- a/release_bot/webhooks.py +++ b/release_bot/webhooks.py @@ -27,14 +27,11 @@ class GithubWebhooksHandler(View): Handler for github callbacks. """ - def __init__(self, release_bot, conf): - self.release_bot = release_bot - self.conf = conf + def __init__(self, conf): self.logger = conf.logger def dispatch_request(self): - self.logger.info(f'New github webhook call from ' - f'{self.conf.repository_owner}/{self.conf.repository_name}') + self.logger.info(f'New github webhook call from detected') if request.is_json: celery_app.send_task(name="task.celery_task.parse_web_hook_payload", kwargs={"webhook_payload": request.get_json()}) From 186da2ed20d172956b9deda3ae1afc72e74aafa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Maru=C5=A1in?= Date: Thu, 22 Aug 2019 15:33:31 +0200 Subject: [PATCH 04/15] add docker file for github app deployment --- Dockerfile.githubapp | 19 +++++++++++++++++++ release_bot/celery_task.py | 12 ++++++------ 2 files changed, 25 insertions(+), 6 deletions(-) create mode 100644 Dockerfile.githubapp diff --git a/Dockerfile.githubapp b/Dockerfile.githubapp new file mode 100644 index 0000000..e64ad3a --- /dev/null +++ b/Dockerfile.githubapp @@ -0,0 +1,19 @@ +FROM registry.fedoraproject.org/fedora:30 + +EXPOSE 8080 + +ENV HOME=/src \ + PATH=/src/bin:/bin:/src/.local/bin:$PATH + +RUN dnf install -y git nss_wrapper + +COPY . /src/release-bot/ + +WORKDIR $HOME + +USER 1001 + +RUN cd /src/release-bot && \ + pip3 install --user . + +CMD $HOME \ No newline at end of file diff --git a/release_bot/celery_task.py b/release_bot/celery_task.py index 255b69f..e034b68 100644 --- a/release_bot/celery_task.py +++ b/release_bot/celery_task.py @@ -46,9 +46,9 @@ def parse_web_hook_payload(self, webhook_payload): def handle_issue(webhook_payload): """Handler for newly opened issues""" - configuration.configuration = Path( - '/Users/marusinm/Documents/Python/tmp/bot-test-conf/conf.yaml').resolve() - # configuration.configuration = Path(getenv("CONF_PATH", "secrets/prod/conf.yaml")).resolve() + # configuration.configuration = Path( + # '/Users/marusinm/Documents/Python/tmp/bot-test-conf/conf.yaml').resolve() + configuration.configuration = Path(getenv("CONF_PATH", "/secrets/prod/conf.yaml")).resolve() # add configuration from Github webhook configuration.repository_name = webhook_payload['repository']['name'] @@ -77,9 +77,9 @@ def handle_issue(webhook_payload): def handle_pr(webhook_payload): """Handler for merged PR""" - configuration.configuration = Path( - '/Users/marusinm/Documents/Python/tmp/bot-test-conf/conf.yaml').resolve() - # configuration.configuration = Path(getenv("CONF_PATH", "secrets/prod/conf.yaml")).resolve() + # configuration.configuration = Path( + # '/Users/marusinm/Documents/Python/tmp/bot-test-conf/conf.yaml').resolve() + configuration.configuration = Path(getenv("CONF_PATH", "/secrets/prod/conf.yaml")).resolve() # add configuration from Github webhook configuration.repository_name = webhook_payload['repository']['name'] From 3c3eed3dd43a2b683a90548f3a861df736862a88 Mon Sep 17 00:00:00 2001 From: Radoslav Pitonak Date: Thu, 22 Aug 2019 16:31:10 +0200 Subject: [PATCH 05/15] Dockerfile for openshift --- Dockerfile.githubapp | 37 +++++++++++++------ Makefile | 6 ++-- files/install-rpm-packages.yaml | 35 ++++++++++++++++++ files/passwd | 14 ++++++++ files/recipe.yaml | 63 +++++++++++++++++++++++++++++++++ files/run.sh | 7 ++++ 6 files changed, 149 insertions(+), 13 deletions(-) create mode 100644 files/install-rpm-packages.yaml create mode 100644 files/passwd create mode 100644 files/recipe.yaml create mode 100644 files/run.sh diff --git a/Dockerfile.githubapp b/Dockerfile.githubapp index e64ad3a..090ec01 100644 --- a/Dockerfile.githubapp +++ b/Dockerfile.githubapp @@ -1,19 +1,36 @@ FROM registry.fedoraproject.org/fedora:30 -EXPOSE 8080 +ENV LANG=en_US.UTF-8 +# nicer output from the playbook run +ENV ANSIBLE_STDOUT_CALLBACK=debug +# Ansible doesn't like /tmp +COPY files/ /src/files/ -ENV HOME=/src \ - PATH=/src/bin:/bin:/src/.local/bin:$PATH +USER 0 -RUN dnf install -y git nss_wrapper +RUN mkdir /home/release-bot && chmod 0776 /home/release-bot +COPY files/passwd /home/release-bot/passwd +ENV LD_PRELOAD=libnss_wrapper.so +ENV NSS_WRAPPER_PASSWD=/home/release-bot/passwd +ENV NSS_WRAPPER_GROUP=/etc/group -COPY . /src/release-bot/ +# Install packages first and reuse the cache as much as possible +RUN dnf install -y ansible \ + && cd /src/ \ + && ansible-playbook -vv -c local -i localhost, files/install-rpm-packages.yaml \ + && dnf clean all -WORKDIR $HOME +COPY setup.py setup.cfg requirements.txt files/recipe.yaml /src/ +# setuptools-scm +COPY .git /src/.git +COPY release_bot/ /src/release_bot/ + +RUN cd /src/ \ + && ansible-playbook -vv -c local -i localhost, files/recipe.yaml -USER 1001 -RUN cd /src/release-bot && \ - pip3 install --user . +USER 1001 +ENV USER=release-bot +ENV HOME=/home/release-bot -CMD $HOME \ No newline at end of file +CMD ["release-bot -c /secrets/prod/conf.yaml"] \ No newline at end of file diff --git a/Makefile b/Makefile index e6679d0..5cbd4f7 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,11 @@ .PHONY: test -IMAGE_NAME := usercont/release-bot +IMAGE_NAME := rpitonak/release-bot IMAGE_NAME_DEV := usercont/release-bot:dev TEST_IMAGE_NAME := release-bot-tests -image: - docker build --tag=$(IMAGE_NAME) . +image: files/install-rpm-packages.yaml files/recipe.yaml + docker build --rm -f Dockerfile.githubapp --tag=$(IMAGE_NAME) . image-dev: docker build --tag=$(IMAGE_NAME_DEV) -f Dockerfile.dev . diff --git a/files/install-rpm-packages.yaml b/files/install-rpm-packages.yaml new file mode 100644 index 0000000..799e3e5 --- /dev/null +++ b/files/install-rpm-packages.yaml @@ -0,0 +1,35 @@ +--- +- name: Install dependencies for packit-service. + hosts: all + tasks: + - name: Install all RPM packages needed to run packit-service. + dnf: + name: + - fedpkg + - git-core + - mod_md + - nss_wrapper + - origin-clients + - python3-GitPython + - python3-gnupg + - python3-click + - python3-copr + - python3-devel + - python3-fedmsg + - python3-flask + - python3-ipdb # for easy debugging + - python3-jsonschema + - python3-libpagure + - python3-mod_wsgi + - python3-munch + - python3-ogr + - python3-packaging + - python3-pip + - python3-pyyaml + - python3-requests + - python3-setuptools + - python3-setuptools_scm + - python3-setuptools_scm_git_archive + - python3-wheel # for bdist_wheel + - nss_wrapper # openshift anyuid passwd madness + state: present diff --git a/files/passwd b/files/passwd new file mode 100644 index 0000000..e60e721 --- /dev/null +++ b/files/passwd @@ -0,0 +1,14 @@ +root:x:0:0:root:/root:/bin/bash +bin:x:1:1:bin:/bin:/sbin/nologin +daemon:x:2:2:daemon:/sbin:/sbin/nologin +adm:x:3:4:adm:/var/adm:/sbin/nologin +lp:x:4:7:lp:/var/spool/lpd:/sbin/nologin +sync:x:5:0:sync:/sbin:/bin/sync +operator:x:11:0:operator:/root:/sbin/nologin +nobody:x:65534:65534:Kernel Overflow User:/:/sbin/nologin +dbus:x:81:81:System message bus:/:/sbin/nologin +systemd-coredump:x:999:997:systemd Core Dumper:/:/sbin/nologin +systemd-network:x:192:192:systemd Network Management:/:/sbin/nologin +systemd-resolve:x:193:193:systemd Resolver:/:/sbin/nologin +apache:x:48:48:Apache:/usr/share/httpd:/sbin/nologin +unbound:x:998:996:Unbound DNS resolver:/etc/unbound:/sbin/nologin diff --git a/files/recipe.yaml b/files/recipe.yaml new file mode 100644 index 0000000..fab1397 --- /dev/null +++ b/files/recipe.yaml @@ -0,0 +1,63 @@ +--- +- name: This is a recipe for how to cook with release-bot + hosts: all + vars: + home_path: "{{ lookup('env', 'HOME') }}" + ansible_bender: + base_image: FROM registry.fedoraproject.org/fedora:30 + #base_image: fedora:29 + target_image: + name: docker.io/usercont/packit-service + environment: + FLASK_APP: packit.service.web_hook + cmd: flask-3 run -h 0.0.0.0 + working_container: + volumes: + - '{{ playbook_dir }}:/src:Z' + tasks: + - name: Create /usr/share/packit directory + file: + state: directory + path: '{{ item }}' + with_items: + - /usr/share/packit + - /packit-ssh # this is where we put ssh creds + - /sandcastle # working dir for the upstream git which is mapped to the sandbox pod + - name: Create some home dirs + file: + state: directory + path: '{{ home_path }}/{{ item }}' + mode: 0700 + with_items: + - .ssh + - .config + - name: Create /secrets + file: + state: directory + path: /secrets + mode: 0777 + - name: stat /src + stat: + path: /src + tags: + - no-cache + register: src_path + - name: Let's make sure /src is present + assert: + that: + - 'src_path.stat.isdir' + - name: Copy run.sh + copy: + src: run.sh + dest: /usr/bin/run.sh + mode: 0777 + - name: Install ogr from git master + pip: + name: git+https://github.com/packit-service/ogr.git + executable: pip3 + - name: Install release-bot from /src + pip: + name: /src + executable: pip3 + - name: Clean all the cache files (especially pip) + shell: rm -rf ~/.cache/* \ No newline at end of file diff --git a/files/run.sh b/files/run.sh new file mode 100644 index 0000000..6d83618 --- /dev/null +++ b/files/run.sh @@ -0,0 +1,7 @@ +#!/usr/bin/bash + +printf "release-bot:x:$(id -u):0:Release bot:/home/release-bot:/bin/bash\n" >>/home/release-bot/passwd + +export RELEASE_BOT_HOME=/home/release-bot + +exec release-bot -c /secrets/prod/conf.yaml From 6816c67946473b61a3586daf9b7406c6cc513afc Mon Sep 17 00:00:00 2001 From: Radoslav Pitonak Date: Fri, 23 Aug 2019 09:57:44 +0200 Subject: [PATCH 06/15] run celery --- Dockerfile.githubapp | 2 +- files/run.sh | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile.githubapp b/Dockerfile.githubapp index 090ec01..1b5298c 100644 --- a/Dockerfile.githubapp +++ b/Dockerfile.githubapp @@ -33,4 +33,4 @@ USER 1001 ENV USER=release-bot ENV HOME=/home/release-bot -CMD ["release-bot -c /secrets/prod/conf.yaml"] \ No newline at end of file +CMD ["release-bot -c /home/release-bot/.config/conf.yaml"] \ No newline at end of file diff --git a/files/run.sh b/files/run.sh index 6d83618..7b5758d 100644 --- a/files/run.sh +++ b/files/run.sh @@ -1,7 +1,8 @@ #!/usr/bin/bash -printf "release-bot:x:$(id -u):0:Release bot:/home/release-bot:/bin/bash\n" >>/home/release-bot/passwd +printf "release-bot:x:$(id -u):0:Release bot:/home/release-bot:/bin/bash\n" >> /home/release-bot/passwd export RELEASE_BOT_HOME=/home/release-bot -exec release-bot -c /secrets/prod/conf.yaml +exec release-bot -c /home/release-bot/.config/conf.yaml & +exec celery -A release_bot.celery_task worker -l info From d1f3bf1b0214a0562678c83cc9bcbcf481fec9e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Maru=C5=A1in?= Date: Fri, 23 Aug 2019 10:53:02 +0200 Subject: [PATCH 07/15] make test passing --- release_bot/celery_task.py | 12 +++++------ release_bot/configuration.py | 14 ++++++++++++ tests/test_webhooks.py | 29 ++----------------------- tests/unit/test_github.py | 41 ++++++++++++++++++------------------ 4 files changed, 43 insertions(+), 53 deletions(-) diff --git a/release_bot/celery_task.py b/release_bot/celery_task.py index e034b68..d549eca 100644 --- a/release_bot/celery_task.py +++ b/release_bot/celery_task.py @@ -46,9 +46,9 @@ def parse_web_hook_payload(self, webhook_payload): def handle_issue(webhook_payload): """Handler for newly opened issues""" - # configuration.configuration = Path( - # '/Users/marusinm/Documents/Python/tmp/bot-test-conf/conf.yaml').resolve() - configuration.configuration = Path(getenv("CONF_PATH", "/secrets/prod/conf.yaml")).resolve() + configuration.configuration = Path( + '/Users/marusinm/Documents/Python/tmp/bot-test-conf/conf.yaml').resolve() + # configuration.configuration = Path(getenv("CONF_PATH", "/secrets/prod/conf.yaml")).resolve() # add configuration from Github webhook configuration.repository_name = webhook_payload['repository']['name'] @@ -77,9 +77,9 @@ def handle_issue(webhook_payload): def handle_pr(webhook_payload): """Handler for merged PR""" - # configuration.configuration = Path( - # '/Users/marusinm/Documents/Python/tmp/bot-test-conf/conf.yaml').resolve() - configuration.configuration = Path(getenv("CONF_PATH", "/secrets/prod/conf.yaml")).resolve() + configuration.configuration = Path( + '/Users/marusinm/Documents/Python/tmp/bot-test-conf/conf.yaml').resolve() + # configuration.configuration = Path(getenv("CONF_PATH", "/secrets/prod/conf.yaml")).resolve() # add configuration from Github webhook configuration.repository_name = webhook_payload['repository']['name'] diff --git a/release_bot/configuration.py b/release_bot/configuration.py index 65abd16..d65f1cb 100644 --- a/release_bot/configuration.py +++ b/release_bot/configuration.py @@ -26,6 +26,9 @@ class Configuration: + # note that required items need to reference strings as their length is checked + REQUIRED_ITEMS = {"conf": [], + "release-conf": []} def __init__(self): self.version = __version__ @@ -102,6 +105,11 @@ def load_configuration(self): for key, value in file.items(): if hasattr(self, key): setattr(self, key, value) + # check if required items are present + for item in self.REQUIRED_ITEMS['conf']: + if item not in file: + self.logger.error(f"Item {item!r} is required in configuration!") + sys.exit(1) # if user hasn't specified clone_url, use default if 'clone_url' not in file: self.clone_url = (f'https://github.com/{self.repository_owner}' @@ -120,6 +128,8 @@ def load_release_conf(self, conf): self.logger.error("No release-conf.yaml found in " f"{self.repository_owner}/{self.repository_name} repository root!\n" "You have to add one for releasing to PyPi") + if self.REQUIRED_ITEMS['release-conf']: + sys.exit(1) parsed_conf = yaml.safe_load(conf) or {} # If pypi option is not specified in release-conf.yaml, @@ -127,6 +137,10 @@ def load_release_conf(self, conf): parsed_conf.setdefault('pypi', True) parsed_conf = {k: v for (k, v) in parsed_conf.items() if v} + for item in self.REQUIRED_ITEMS['release-conf']: + if item not in parsed_conf: + self.logger.error(f"Item {item!r} is required in release-conf!") + sys.exit(1) for index, label in enumerate(parsed_conf.get('labels', [])): parsed_conf['labels'][index] = str(label) diff --git a/tests/test_webhooks.py b/tests/test_webhooks.py index 6a1a9e8..8f55287 100644 --- a/tests/test_webhooks.py +++ b/tests/test_webhooks.py @@ -19,6 +19,7 @@ import json from release_bot.webhooks import GithubWebhooksHandler +from release_bot.celerizer import celery_app @pytest.fixture() @@ -29,7 +30,6 @@ def flask_instance(): :return: flask instance for tests """ - release_bot = flexmock() configuration = flexmock( logger=flexmock(), repository_owner='repo-owner', @@ -46,7 +46,6 @@ def flask_instance(): app = Flask(__name__) app.add_url_rule('/webhook-handler/', view_func=GithubWebhooksHandler.as_view('github_webhooks_handler', - release_bot=release_bot, conf=configuration), methods=['POST', ]) @@ -64,8 +63,7 @@ def test_bad_requests(flask_instance): def test_json_requests(flask_instance): """Test if POST method which contains JSON call correct methods""" - flexmock(GithubWebhooksHandler).should_receive("handle_issue").once() - flexmock(GithubWebhooksHandler).should_receive("handle_pr").once() + flexmock(celery_app).should_receive("send_task").and_return('vooosh!').once() json_dummy_dict = { 'dummy': 'dummy', @@ -75,26 +73,3 @@ def test_json_requests(flask_instance): content_type='application/json') assert response.status_code == 200 - json_expected_dict = { - 'action': 'opened', - 'issue': { - 'dummy': 'dummy' - } - } - # call handle_issue once within GithubWebhooksHandler instance - response = flask_instance.post('/webhook-handler/', data=json.dumps(json_expected_dict), - content_type='application/json') - assert response.status_code == 200 - - json_expected_dict = { - 'action': 'closed', - 'pull_request': { - 'merged': True, - 'dummy': 'dummy' - } - } - # call handle_pr once within GithubWebhooksHandler instance - response = flask_instance.post('/webhook-handler/', data=json.dumps(json_expected_dict), - content_type='application/json') - assert response.status_code == 200 - diff --git a/tests/unit/test_github.py b/tests/unit/test_github.py index 89677d0..cefa06a 100644 --- a/tests/unit/test_github.py +++ b/tests/unit/test_github.py @@ -12,26 +12,27 @@ def test_latest_release(): - mocked_releases = [ - GithubRelease( - tag_name="0.0.1", - url="", - created_at="", - tarball_url="", - git_tag=GitTag(name="0.0.1", commit_sha="123"), - project=flexmock(GitProject), - raw_release=flexmock(title="0.0.1", body=""), - ), - GithubRelease( - tag_name="0.0.2", - url="", - created_at="", - tarball_url="", - git_tag=GitTag(name="0.0.2", commit_sha="123"), - project=flexmock(GitProject), - raw_release=flexmock(title="0.0.2", body=""), - ), - ] + r1 = GithubRelease( + tag_name='', + url='', + created_at='', + tarball_url='', + git_tag=GitTag(name='0.0.1', commit_sha='123'), + project=flexmock(GitProject), + raw_release=flexmock(title='0.0.1') + ) + + r2 = GithubRelease( + tag_name='', + url='', + created_at='', + tarball_url='', + git_tag=GitTag(name='0.0.2', commit_sha='123'), + project=flexmock(GitProject), + raw_release=flexmock(title='0.0.2') + ) + + mocked_releases = [r1, r2] git = flexmock(Git) c = flexmock(configuration) From 160fb4d33bcc5751027da1c4bcd0b377c61b3435 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Maru=C5=A1in?= Date: Fri, 23 Aug 2019 11:38:57 +0200 Subject: [PATCH 08/15] return paths --- Makefile | 2 +- release_bot/celery_task.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 5cbd4f7..9fcdb89 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: test -IMAGE_NAME := rpitonak/release-bot +IMAGE_NAME := marusinm/release-bot IMAGE_NAME_DEV := usercont/release-bot:dev TEST_IMAGE_NAME := release-bot-tests diff --git a/release_bot/celery_task.py b/release_bot/celery_task.py index d549eca..e034b68 100644 --- a/release_bot/celery_task.py +++ b/release_bot/celery_task.py @@ -46,9 +46,9 @@ def parse_web_hook_payload(self, webhook_payload): def handle_issue(webhook_payload): """Handler for newly opened issues""" - configuration.configuration = Path( - '/Users/marusinm/Documents/Python/tmp/bot-test-conf/conf.yaml').resolve() - # configuration.configuration = Path(getenv("CONF_PATH", "/secrets/prod/conf.yaml")).resolve() + # configuration.configuration = Path( + # '/Users/marusinm/Documents/Python/tmp/bot-test-conf/conf.yaml').resolve() + configuration.configuration = Path(getenv("CONF_PATH", "/secrets/prod/conf.yaml")).resolve() # add configuration from Github webhook configuration.repository_name = webhook_payload['repository']['name'] @@ -77,9 +77,9 @@ def handle_issue(webhook_payload): def handle_pr(webhook_payload): """Handler for merged PR""" - configuration.configuration = Path( - '/Users/marusinm/Documents/Python/tmp/bot-test-conf/conf.yaml').resolve() - # configuration.configuration = Path(getenv("CONF_PATH", "/secrets/prod/conf.yaml")).resolve() + # configuration.configuration = Path( + # '/Users/marusinm/Documents/Python/tmp/bot-test-conf/conf.yaml').resolve() + configuration.configuration = Path(getenv("CONF_PATH", "/secrets/prod/conf.yaml")).resolve() # add configuration from Github webhook configuration.repository_name = webhook_payload['repository']['name'] From d7870fdd73decb5f4658f547abff90e32a53a7cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Maru=C5=A1in?= Date: Mon, 26 Aug 2019 13:00:13 +0200 Subject: [PATCH 09/15] create token for github app --- .gitignore | 4 ++++ Dockerfile.githubapp => Dockerfile.app | 0 Makefile | 4 ++-- files/gitconfig | 5 +++++ files/recipe.yaml | 4 ++++ release_bot/celery_task.py | 22 ++++++++++++++-------- release_bot/configuration.py | 8 ++++++++ 7 files changed, 37 insertions(+), 10 deletions(-) rename Dockerfile.githubapp => Dockerfile.app (100%) create mode 100644 files/gitconfig diff --git a/.gitignore b/.gitignore index bf8eae1..4b35793 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,8 @@ __pycache__/ # Editors and IDEs .idea/ .vscode/ +.DS_Store + +dump.rdb + diff --git a/Dockerfile.githubapp b/Dockerfile.app similarity index 100% rename from Dockerfile.githubapp rename to Dockerfile.app diff --git a/Makefile b/Makefile index 9fcdb89..0117d3b 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,11 @@ .PHONY: test -IMAGE_NAME := marusinm/release-bot +IMAGE_NAME := usercont/release-bot IMAGE_NAME_DEV := usercont/release-bot:dev TEST_IMAGE_NAME := release-bot-tests image: files/install-rpm-packages.yaml files/recipe.yaml - docker build --rm -f Dockerfile.githubapp --tag=$(IMAGE_NAME) . + docker build --rm -f Dockerfile.app --tag=$(IMAGE_NAME) . image-dev: docker build --tag=$(IMAGE_NAME_DEV) -f Dockerfile.dev . diff --git a/files/gitconfig b/files/gitconfig new file mode 100644 index 0000000..122a167 --- /dev/null +++ b/files/gitconfig @@ -0,0 +1,5 @@ +[user] + name = Release Bot + email = user-cont-team+release-bot@redhat.com +[push] + default = current \ No newline at end of file diff --git a/files/recipe.yaml b/files/recipe.yaml index fab1397..5590d5f 100644 --- a/files/recipe.yaml +++ b/files/recipe.yaml @@ -36,6 +36,10 @@ state: directory path: /secrets mode: 0777 + - name: Copy gitconfig + copy: + src: gitconfig + dest: '{{ home_path }}/.gitconfig' - name: stat /src stat: path: /src diff --git a/release_bot/celery_task.py b/release_bot/celery_task.py index e034b68..f10e519 100644 --- a/release_bot/celery_task.py +++ b/release_bot/celery_task.py @@ -46,17 +46,20 @@ def parse_web_hook_payload(self, webhook_payload): def handle_issue(webhook_payload): """Handler for newly opened issues""" - # configuration.configuration = Path( - # '/Users/marusinm/Documents/Python/tmp/bot-test-conf/conf.yaml').resolve() - configuration.configuration = Path(getenv("CONF_PATH", "/secrets/prod/conf.yaml")).resolve() + configuration.configuration = Path(getenv("CONF_PATH", + "/home/release-bot/.config/conf.yaml")).resolve() # add configuration from Github webhook configuration.repository_name = webhook_payload['repository']['name'] configuration.repository_owner = webhook_payload['repository']['owner']['login'] configuration.github_username = webhook_payload['issue']['user']['login'] - configuration.clone_url = webhook_payload['repository']['clone_url'] configuration.load_configuration() # load the rest of configuration if there is any + # create url for github app to enable access over http + configuration.clone_url = f'https://x-access-token:' \ + f'{configuration.github_token}@github.com/' \ + f'{configuration.repository_owner}/{configuration.repository_name}.git' + logger = configuration.logger release_bot = ReleaseBot(configuration) @@ -77,17 +80,20 @@ def handle_issue(webhook_payload): def handle_pr(webhook_payload): """Handler for merged PR""" - # configuration.configuration = Path( - # '/Users/marusinm/Documents/Python/tmp/bot-test-conf/conf.yaml').resolve() - configuration.configuration = Path(getenv("CONF_PATH", "/secrets/prod/conf.yaml")).resolve() + configuration.configuration = Path(getenv("CONF_PATH", + "/home/release-bot/.config/conf.yaml")).resolve() # add configuration from Github webhook configuration.repository_name = webhook_payload['repository']['name'] configuration.repository_owner = webhook_payload['repository']['owner']['login'] configuration.github_username = webhook_payload['pull_request']['user']['login'] - configuration.clone_url = webhook_payload['repository']['clone_url'] configuration.load_configuration() # load the rest of configuration if there is any + # create url for github app to enable access over http + configuration.clone_url = f'https://x-access-token:' \ + f'{configuration.github_token}@github.com/' \ + f'{configuration.repository_owner}/{configuration.repository_name}.git' + logger = configuration.logger release_bot = ReleaseBot(configuration) diff --git a/release_bot/configuration.py b/release_bot/configuration.py index d65f1cb..6d384d6 100644 --- a/release_bot/configuration.py +++ b/release_bot/configuration.py @@ -23,6 +23,7 @@ from ogr import GithubService, PagureService, get_project from release_bot.version import __version__ +from release_bot.github import GitHubApp class Configuration: @@ -199,6 +200,13 @@ def get_project(self): if self.github_app_id != '': with open(self.github_app_cert_path, 'r') as cert: github_cert = cert.read() + + # github token will be used as a credential over http (commit/push) + github_app = GitHubApp(self.github_app_id, self.github_app_cert_path) + self.github_token = github_app.get_installation_access_token( + self.github_app_installation_id + ) + return get_project(url=self.clone_url, custom_instances=[ GithubService(token=None, From 4ed9ac96163d5b4bda3a844e4293899758d6ded0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Maru=C5=A1in?= Date: Tue, 10 Sep 2019 15:20:54 +0200 Subject: [PATCH 10/15] remove packit left overs --- Dockerfile.app | 4 ---- Makefile | 2 +- files/install-rpm-packages.yaml | 20 ++------------------ files/passwd | 14 -------------- files/recipe.yaml | 12 ++---------- files/run.sh | 2 -- 6 files changed, 5 insertions(+), 49 deletions(-) delete mode 100644 files/passwd diff --git a/Dockerfile.app b/Dockerfile.app index 1b5298c..bb2bc6c 100644 --- a/Dockerfile.app +++ b/Dockerfile.app @@ -9,10 +9,6 @@ COPY files/ /src/files/ USER 0 RUN mkdir /home/release-bot && chmod 0776 /home/release-bot -COPY files/passwd /home/release-bot/passwd -ENV LD_PRELOAD=libnss_wrapper.so -ENV NSS_WRAPPER_PASSWD=/home/release-bot/passwd -ENV NSS_WRAPPER_GROUP=/etc/group # Install packages first and reuse the cache as much as possible RUN dnf install -y ansible \ diff --git a/Makefile b/Makefile index 0117d3b..9ce0706 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: test -IMAGE_NAME := usercont/release-bot +IMAGE_NAME := marusinm/release-bot IMAGE_NAME_DEV := usercont/release-bot:dev TEST_IMAGE_NAME := release-bot-tests diff --git a/files/install-rpm-packages.yaml b/files/install-rpm-packages.yaml index 799e3e5..fb12890 100644 --- a/files/install-rpm-packages.yaml +++ b/files/install-rpm-packages.yaml @@ -1,29 +1,14 @@ --- -- name: Install dependencies for packit-service. +- name: Install dependencies for release-bot hosts: all tasks: - - name: Install all RPM packages needed to run packit-service. + - name: Install all RPM packages needed to run release-bot. dnf: name: - - fedpkg - - git-core - - mod_md - - nss_wrapper - - origin-clients - - python3-GitPython - - python3-gnupg - - python3-click - - python3-copr - - python3-devel - - python3-fedmsg - python3-flask - python3-ipdb # for easy debugging - python3-jsonschema - - python3-libpagure - - python3-mod_wsgi - - python3-munch - python3-ogr - - python3-packaging - python3-pip - python3-pyyaml - python3-requests @@ -31,5 +16,4 @@ - python3-setuptools_scm - python3-setuptools_scm_git_archive - python3-wheel # for bdist_wheel - - nss_wrapper # openshift anyuid passwd madness state: present diff --git a/files/passwd b/files/passwd deleted file mode 100644 index e60e721..0000000 --- a/files/passwd +++ /dev/null @@ -1,14 +0,0 @@ -root:x:0:0:root:/root:/bin/bash -bin:x:1:1:bin:/bin:/sbin/nologin -daemon:x:2:2:daemon:/sbin:/sbin/nologin -adm:x:3:4:adm:/var/adm:/sbin/nologin -lp:x:4:7:lp:/var/spool/lpd:/sbin/nologin -sync:x:5:0:sync:/sbin:/bin/sync -operator:x:11:0:operator:/root:/sbin/nologin -nobody:x:65534:65534:Kernel Overflow User:/:/sbin/nologin -dbus:x:81:81:System message bus:/:/sbin/nologin -systemd-coredump:x:999:997:systemd Core Dumper:/:/sbin/nologin -systemd-network:x:192:192:systemd Network Management:/:/sbin/nologin -systemd-resolve:x:193:193:systemd Resolver:/:/sbin/nologin -apache:x:48:48:Apache:/usr/share/httpd:/sbin/nologin -unbound:x:998:996:Unbound DNS resolver:/etc/unbound:/sbin/nologin diff --git a/files/recipe.yaml b/files/recipe.yaml index 5590d5f..9ab810a 100644 --- a/files/recipe.yaml +++ b/files/recipe.yaml @@ -7,22 +7,14 @@ base_image: FROM registry.fedoraproject.org/fedora:30 #base_image: fedora:29 target_image: - name: docker.io/usercont/packit-service + name: docker.io/usercont/release-bot environment: - FLASK_APP: packit.service.web_hook + FLASK_APP: releasebot.service.web_hook cmd: flask-3 run -h 0.0.0.0 working_container: volumes: - '{{ playbook_dir }}:/src:Z' tasks: - - name: Create /usr/share/packit directory - file: - state: directory - path: '{{ item }}' - with_items: - - /usr/share/packit - - /packit-ssh # this is where we put ssh creds - - /sandcastle # working dir for the upstream git which is mapped to the sandbox pod - name: Create some home dirs file: state: directory diff --git a/files/run.sh b/files/run.sh index 7b5758d..bdb3d62 100644 --- a/files/run.sh +++ b/files/run.sh @@ -1,7 +1,5 @@ #!/usr/bin/bash -printf "release-bot:x:$(id -u):0:Release bot:/home/release-bot:/bin/bash\n" >> /home/release-bot/passwd - export RELEASE_BOT_HOME=/home/release-bot exec release-bot -c /home/release-bot/.config/conf.yaml & From 70daca16d7b8b567a9e91751cfbe3b3c535b398d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Maru=C5=A1in?= Date: Tue, 10 Sep 2019 15:39:19 +0200 Subject: [PATCH 11/15] delete s2i leftovers --- .s2i-dev/bin/assemble | 11 --- .s2i-dev/bin/run | 20 ------ .s2i-dev/bin/usage | 10 --- .s2i/bin/assemble | 11 --- .s2i/bin/run | 20 ------ .s2i/bin/usage | 10 --- Dockerfile | 38 ---------- Dockerfile.dev | 39 ---------- Makefile | 6 -- openshift-template-dev.yml | 142 ------------------------------------- openshift-template.yml | 138 ----------------------------------- 11 files changed, 445 deletions(-) delete mode 100755 .s2i-dev/bin/assemble delete mode 100755 .s2i-dev/bin/run delete mode 100755 .s2i-dev/bin/usage delete mode 100755 .s2i/bin/assemble delete mode 100755 .s2i/bin/run delete mode 100755 .s2i/bin/usage delete mode 100644 Dockerfile delete mode 100644 Dockerfile.dev delete mode 100644 openshift-template-dev.yml delete mode 100644 openshift-template.yml diff --git a/.s2i-dev/bin/assemble b/.s2i-dev/bin/assemble deleted file mode 100755 index 2fae085..0000000 --- a/.s2i-dev/bin/assemble +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -shopt -s dotglob -echo "---> Installing application source ..." -mv /tmp/src/* ./ - -# check for bot conf -if [ ! -f ./conf.yaml ]; then - echo "ERROR: no bot configuration found" - exit 1 -fi diff --git a/.s2i-dev/bin/run b/.s2i-dev/bin/run deleted file mode 100755 index d670c68..0000000 --- a/.s2i-dev/bin/run +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# check for bot ssh private key -if [ -f ./id_rsa ]; then - echo "---> Setting up ssh key..." - mkdir ./.ssh - cp ./id_rsa ./.ssh/id_rsa - chmod 600 ./.ssh/id_rsa - echo "StrictHostKeyChecking no" >> ./.ssh/config - eval "$(ssh-agent -s)" - ssh-add ./.ssh/id_rsa -fi - -# https://docs.openshift.com/enterprise/3.1/creating_images/guidelines.html#openshift-enterprise-specific-guidelines -sed -e "s|:1001:|:$(id -u):|g" /etc/passwd > "${HOME}/passwd" -export LD_PRELOAD=libnss_wrapper.so -export NSS_WRAPPER_PASSWD="${HOME}/passwd" -export NSS_WRAPPER_GROUP=/etc/group - -release-bot --debug --configuration="${HOME}/conf.yaml" diff --git a/.s2i-dev/bin/usage b/.s2i-dev/bin/usage deleted file mode 100755 index 79c11c7..0000000 --- a/.s2i-dev/bin/usage +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -cat < usercont/release-bot:dev release-bot-app:dev -You can then run the resulting image via: -docker run release-bot-app:dev -EOF diff --git a/.s2i/bin/assemble b/.s2i/bin/assemble deleted file mode 100755 index 2fae085..0000000 --- a/.s2i/bin/assemble +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -shopt -s dotglob -echo "---> Installing application source ..." -mv /tmp/src/* ./ - -# check for bot conf -if [ ! -f ./conf.yaml ]; then - echo "ERROR: no bot configuration found" - exit 1 -fi diff --git a/.s2i/bin/run b/.s2i/bin/run deleted file mode 100755 index d670c68..0000000 --- a/.s2i/bin/run +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# check for bot ssh private key -if [ -f ./id_rsa ]; then - echo "---> Setting up ssh key..." - mkdir ./.ssh - cp ./id_rsa ./.ssh/id_rsa - chmod 600 ./.ssh/id_rsa - echo "StrictHostKeyChecking no" >> ./.ssh/config - eval "$(ssh-agent -s)" - ssh-add ./.ssh/id_rsa -fi - -# https://docs.openshift.com/enterprise/3.1/creating_images/guidelines.html#openshift-enterprise-specific-guidelines -sed -e "s|:1001:|:$(id -u):|g" /etc/passwd > "${HOME}/passwd" -export LD_PRELOAD=libnss_wrapper.so -export NSS_WRAPPER_PASSWD="${HOME}/passwd" -export NSS_WRAPPER_GROUP=/etc/group - -release-bot --debug --configuration="${HOME}/conf.yaml" diff --git a/.s2i/bin/usage b/.s2i/bin/usage deleted file mode 100755 index 6f58114..0000000 --- a/.s2i/bin/usage +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -cat < usercont/release-bot release-bot-app -You can then run the resulting image via: -docker run release-bot-app -EOF diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index b4dcd0b..0000000 --- a/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM registry.fedoraproject.org/fedora:30 - -EXPOSE 8080 - -ENV STI_SCRIPTS_URL=image:///usr/libexec/s2i \ - STI_SCRIPTS_PATH=/usr/libexec/s2i \ - # The $HOME is not set by default, but some applications needs this variable - HOME=/opt/app-root/src \ - PATH=/opt/app-root/src/bin:/opt/app-root/bin:/opt/app-root/src/.local/bin:$PATH - -LABEL summary="Automated releasing from GitHub repositories" \ - name="release-bot" \ - description="Automated releasing from GitHub repositories" \ - io.k8s.description="Automated releasing from GitHub repositories" \ - io.k8s.display-name="Release Bot" \ - io.openshift.tags="builder" \ - io.openshift.s2i.scripts-url="$STI_SCRIPTS_URL" \ - usage="s2i build usercont/release-bot " - -RUN dnf install -y git nss_wrapper - -RUN mkdir -p ${HOME} && \ - useradd -u 1001 -r -g 0 -d ${HOME} -s /sbin/nologin \ - -c "Default Application User" default && \ - chown -R 1001:0 /opt/app-root - -USER 1001 - -RUN pip3 install --user release-bot && \ - chgrp -R 0 /opt/app-root && \ - chmod -R g=u /opt/app-root - -# S2I scripts -COPY ./.s2i/bin/ $STI_SCRIPTS_PATH - -WORKDIR $HOME - -CMD $STI_SCRIPTS_PATH/usage diff --git a/Dockerfile.dev b/Dockerfile.dev deleted file mode 100644 index a5e195b..0000000 --- a/Dockerfile.dev +++ /dev/null @@ -1,39 +0,0 @@ -FROM registry.fedoraproject.org/fedora:30 - -EXPOSE 8080 - -ENV STI_SCRIPTS_URL=image:///usr/libexec/s2i \ - STI_SCRIPTS_PATH=/usr/libexec/s2i \ - # The $HOME is not set by default, but some applications needs this variable - HOME=/opt/app-root/src \ - PATH=/opt/app-root/src/bin:/opt/app-root/bin:/opt/app-root/src/.local/bin:$PATH - -LABEL summary="Automated releasing from GitHub repositories" \ - name="release-bot" \ - description="Automated releasing from GitHub repositories" \ - io.k8s.description="Automated releasing from GitHub repositories" \ - io.k8s.display-name="Release Bot" \ - io.openshift.tags="builder" \ - io.openshift.s2i.scripts-url="$STI_SCRIPTS_URL" \ - usage="s2i build usercont/release-bot:dev " - -RUN dnf install -y git nss_wrapper - -RUN mkdir -p ${HOME} && \ - useradd -u 1001 -r -g 0 -d ${HOME} -s /sbin/nologin \ - -c "Default Application User" default && \ - chown -R 1001:0 /opt/app-root - -USER 1001 - -RUN git clone https://github.com/user-cont/release-bot.git $HOME/release-bot && \ - pip3 install --user $HOME/release-bot/ && \ - chgrp -R 0 /opt/app-root && \ - chmod -R g=u /opt/app-root - -# S2I scripts -COPY ./.s2i-dev/bin/ $STI_SCRIPTS_PATH - -WORKDIR $HOME - -CMD $STI_SCRIPTS_PATH/usage diff --git a/Makefile b/Makefile index 9ce0706..10763ac 100644 --- a/Makefile +++ b/Makefile @@ -7,12 +7,6 @@ TEST_IMAGE_NAME := release-bot-tests image: files/install-rpm-packages.yaml files/recipe.yaml docker build --rm -f Dockerfile.app --tag=$(IMAGE_NAME) . -image-dev: - docker build --tag=$(IMAGE_NAME_DEV) -f Dockerfile.dev . - -image-dev-no-cache: - docker build --no-cache --tag=$(IMAGE_NAME_DEV) -f Dockerfile.dev . - image-test: docker build --tag=$(TEST_IMAGE_NAME) -f Dockerfile.test . diff --git a/openshift-template-dev.yml b/openshift-template-dev.yml deleted file mode 100644 index f473f4f..0000000 --- a/openshift-template-dev.yml +++ /dev/null @@ -1,142 +0,0 @@ -kind: Template -apiVersion: v1 -metadata: - name: release-bot - annotations: - description: S2I Relase-bot image builder - tags: release-bot s2i - iconClass: icon-python -labels: - template: release-bot - role: releasebot_application_builder -objects: - - kind : ImageStream - apiVersion : v1 - metadata : - name : ${APP_NAME} - labels : - appid : release-bot-${APP_NAME} - - kind : ImageStream - apiVersion : v1 - metadata : - name : ${APP_NAME}-s2i - labels : - appid : release-bot-${APP_NAME} - spec : - tags : - - name : latest - from : - kind : DockerImage - name : ${DOCKER_IMAGE} - importPolicy: - scheduled: true - - kind : BuildConfig - apiVersion : v1 - metadata : - name : ${APP_NAME} - labels : - appid : release-bot-${APP_NAME} - spec : - triggers : - - type : ConfigChange - - type : ImageChange - source : - type : Git - git : - uri : ${CONFIGURATION_REPOSITORY} - contextDir : ${CONFIGURATION_REPOSITORY} - sourceSecret : - name : release-bot-secret - strategy : - type : Source - sourceStrategy : - from : - kind : ImageStreamTag - name : ${APP_NAME}-s2i:latest - output : - to : - kind : ImageStreamTag - name : ${APP_NAME}:latest - - kind : DeploymentConfig - apiVersion : v1 - metadata : - name: ${APP_NAME} - labels : - appid : release-bot-${APP_NAME} - spec : - strategy : - type : Rolling - triggers : - - type : ConfigChange - - type : ImageChange - imageChangeParams : - automatic : true - containerNames : - - ${APP_NAME} - from : - kind : ImageStreamTag - name : ${APP_NAME}:latest - replicas : 1 - selector : - deploymentconfig : ${APP_NAME} - template : - metadata : - labels : - appid: release-bot-${APP_NAME} - deploymentconfig : ${APP_NAME} - spec : - containers : - - name : ${APP_NAME} - image : ${APP_NAME}:latest - ports: - - containerPort: 8080 - protocol: TCP - resources: - requests: - memory: "64Mi" - cpu: "50m" - limits: - memory: "128Mi" - cpu: "100m" - - kind: Service - apiVersion: v1 - metadata: - name: release-bot - labels: - application: release-bot - annotations: - description: Exposes release-bot port for github webhook callbacks - spec: - ports: - - name: 8080-tcp - port: 8080 - protocol: TCP - targetPort: 8080 - selector: - deploymentconfig: ${APP_NAME} - - kind: Route - apiVersion: v1 - metadata: - name: release-bot - labels: - application: release-bot - annotations: - description: Github webhook route to reach release-bot - spec: - to: - kind: Service - name: release-bot - -parameters : - - name : APP_NAME - description : Name of application - value : - required : true - - name : CONFIGURATION_REPOSITORY - description : Git repository with configuration - value : - required : true - - name : DOCKER_IMAGE - description : DockerHub repository with release-bot development image - value : usercont/release-bot:dev - required : true diff --git a/openshift-template.yml b/openshift-template.yml deleted file mode 100644 index 0e62007..0000000 --- a/openshift-template.yml +++ /dev/null @@ -1,138 +0,0 @@ -kind: Template -apiVersion: v1 -metadata: - name: release-bot - annotations: - description: S2I Relase-bot image builder - tags: release-bot s2i - iconClass: icon-python -labels: - template: release-bot - role: releasebot_application_builder -objects: - - kind : ImageStream - apiVersion : v1 - metadata : - name : ${APP_NAME} - labels : - appid : release-bot-${APP_NAME} - - kind : ImageStream - apiVersion : v1 - metadata : - name : ${APP_NAME}-s2i - labels : - appid : release-bot-${APP_NAME} - spec : - tags : - - name : latest - from : - kind : DockerImage - name : usercont/release-bot:latest - #importPolicy: - # scheduled: true - - kind : BuildConfig - apiVersion : v1 - metadata : - name : ${APP_NAME} - labels : - appid : release-bot-${APP_NAME} - spec : - triggers : - - type : ConfigChange - - type : ImageChange - source : - type : Git - git : - uri : ${CONFIGURATION_REPOSITORY} - contextDir : ${CONFIGURATION_REPOSITORY} - sourceSecret : - name : release-bot-secret - strategy : - type : Source - sourceStrategy : - from : - kind : ImageStreamTag - name : ${APP_NAME}-s2i:latest - output : - to : - kind : ImageStreamTag - name : ${APP_NAME}:latest - - kind : DeploymentConfig - apiVersion : v1 - metadata : - name: ${APP_NAME} - labels : - appid : release-bot-${APP_NAME} - spec : - strategy : - type : Rolling - triggers : - - type : ConfigChange - - type : ImageChange - imageChangeParams : - automatic : true - containerNames : - - ${APP_NAME} - from : - kind : ImageStreamTag - name : ${APP_NAME}:latest - replicas : 1 - selector : - deploymentconfig : ${APP_NAME} - template : - metadata : - labels : - appid: release-bot-${APP_NAME} - deploymentconfig : ${APP_NAME} - spec : - containers : - - name : ${APP_NAME} - image : ${APP_NAME}:latest - ports: - - containerPort: 8080 - protocol: "TCP" - resources: - requests: - memory: "64Mi" - cpu: "50m" - limits: - memory: "128Mi" - cpu: "100m" - - kind: Service - apiVersion: v1 - metadata: - name: release-bot - labels: - application: release-bot - annotations: - description: Exposes release-bot port for github webhook callbacks - spec: - ports: - - name: 8080-tcp - port: 8080 - protocol: TCP - targetPort: 8080 - selector: - deploymentconfig: ${APP_NAME} - - kind: Route - apiVersion: v1 - metadata: - name: release-bot - labels: - application: release-bot - annotations: - description: Github webhook route to reach release-bot - spec: - to: - kind: Service - name: release-bot - -parameters : - - name : APP_NAME - description : Name of application - value : - required : true - - name : CONFIGURATION_REPOSITORY - description : Git repository with configuration - value : - required : true From 3a363497069a0c791d73eb2ced554aac13850fd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Maru=C5=A1in?= Date: Tue, 10 Sep 2019 15:52:47 +0200 Subject: [PATCH 12/15] Update README --- README.md | 59 ++++--------------------------------------------------- 1 file changed, 4 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index 429cd54..4d4fa1b 100644 --- a/README.md +++ b/README.md @@ -43,8 +43,7 @@ Note that you have to setup your login details (see [Requirements](#requirements ``` $ pip install release-bot ``` -Other possible installations are through -[Docker](#docker-image), [OpenShift](#openshift-template), [Arch User Repository](#arch-user-repository). +Other possible installations are through [Arch User Repository](#arch-user-repository) or install on repo as [Github Application](#github-application). First interaction with release bot may be automated releases on Github. Let's do it. @@ -187,60 +186,10 @@ Are specified in `requirements.txt`. You have to setup your PyPI login details in `$HOME/.pypirc` as described in [PyPI documentation](https://packaging.python.org/tutorials/distributing-packages/#create-an-account). -## Docker image -To make it easier to run this, release-bot is available as an - [source-to-image](https://github.com/openshift/source-to-image) builder image. +## Github Application - You can then create the final image like this: -``` -$ s2i build $CONFIGURATION_REPOSITORY_URL usercont/release-bot app-name -``` - -where $CONFIGURATION_REPOSITORY_URL is link to repository with conf.yaml and .pypirc files. - -To test it locally, you can the run the final image like this: - -``` -$ docker run -``` - -once all changes, configuration files exist in GitHub and git repository contains needed files, -you can try to create an issue in your GitHub repository with string like "X.Y.Z release" -and you can see log like this: -``` -$ docker run meta-test-family-bot ----> Setting up ssh key... -Agent pid 12 -Identity added: ./.ssh/id_rsa (./.ssh/id_rsa) -11:47:36.212 configuration.py DEBUG Loaded configuration for fedora-modularity/meta-test-family -11:47:36.212 releasebot.py INFO release-bot v0.4.1 reporting for duty! -11:47:36.212 github.py DEBUG Fetching release-conf.yaml -11:47:51.636 releasebot.py DEBUG No merged release PR found -11:47:52.196 releasebot.py INFO Found new release issue with version: 0.8.4 -11:47:55.578 releasebot.py DEBUG No more open issues found -11:47:56.098 releasebot.py INFO Making a new PR for release of version 0.8.5 based on an issue. -11:47:57.608 utils.py DEBUG ['git', 'clone', 'https://github.com/fedora-modularity/meta-test-family.git', '.'] -... -``` -## OpenShift template -You can also run this bot in OpenShift using [openshift-template.yml](openshift-template.yml) in this repository. -You must set two environment variables, the `$APP_NAME` is the name of your release-bot deployment, -and `$CONFIGURATION_REPOSITORY` which contains configuration for the release-bot. -The contents of the repository are described [above](#docker-image). -Note that if you use private repository (which you **absolutely** should), -you will need to set up a new [OpenShift secret](https://docs.openshift.com/container-platform/3.7/dev_guide/secrets.html) named -`release-bot-secret` to authenticate. It can be a ssh private key that you can use to access the repository -(for GitHub see [deploy keys](https://developer.github.com/v3/guides/managing-deploy-keys/)). -Here's an [guide](https://blog.openshift.com/deploy-private-git-repositories/) on -how to do that in OpenShift GUI, or another -[guide](https://blog.openshift.com/deploying-from-private-git-repositories/) -that uses `oc` commandline tool. - -By default, the release-bot builder image won't update itself when a -new version of this image is pushed to docker hub. -You can change it by uncommenting lines with `#importPolicy:` -and `#scheduled: true` in [openshift-template.yml](openshift-template.yml). -Then the image will be pulled on a new release. +Release-bot as Github Application is currently in testing and will be available soon in Github market. +Github application will speed-up configuration process. ## Arch User Repository For Arch or Arch based Linux distributions, you can install the bot from the [AUR Package](https://aur.archlinux.org/packages/release-bot). From 59e45f2b956a46e3eeee57c56c7ee618f1d1603a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Maru=C5=A1in?= Date: Thu, 12 Sep 2019 13:18:03 +0200 Subject: [PATCH 13/15] fix codacy issues and code review --- release_bot/celerizer.py | 32 ++++++--------- release_bot/celery_task.py | 78 ++++++++++++++++-------------------- release_bot/configuration.py | 8 ++-- release_bot/github.py | 2 +- 4 files changed, 51 insertions(+), 69 deletions(-) diff --git a/release_bot/celerizer.py b/release_bot/celerizer.py index 138759f..74f3115 100644 --- a/release_bot/celerizer.py +++ b/release_bot/celerizer.py @@ -1,30 +1,24 @@ -# MIT License +# -*- coding: utf-8 -*- # -# Copyright (c) 2018-2019 Red Hat, Inc. - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. # -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. # -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . -from celery import Celery from os import getenv +from celery import Celery class Celerizer: + """Creates instance used by celery lib""" def __init__(self): self._celery_app = None diff --git a/release_bot/celery_task.py b/release_bot/celery_task.py index f10e519..6c26690 100644 --- a/release_bot/celery_task.py +++ b/release_bot/celery_task.py @@ -1,24 +1,17 @@ -# MIT License +# -*- coding: utf-8 -*- # -# Copyright (c) 2018-2019 Red Hat, Inc. - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. # -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. # -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . from pathlib import Path from os import getenv @@ -28,9 +21,11 @@ from release_bot.configuration import configuration from release_bot.releasebot import ReleaseBot +DEFAULT_CONF_FILE = "/home/release-bot/.config/conf.yaml" -@celery_app.task(bind=True, name="task.celery_task.parse_web_hook_payload") -def parse_web_hook_payload(self, webhook_payload): + +@celery_app.task(name="task.celery_task.parse_web_hook_payload") +def parse_web_hook_payload(webhook_payload): """ Parse json webhook payload callback :param webhook_payload: json from github webhook @@ -44,15 +39,23 @@ def parse_web_hook_payload(self, webhook_payload): handle_pr(webhook_payload) -def handle_issue(webhook_payload): - """Handler for newly opened issues""" +def set_configuration(webhook_payload, issue=True): + """ + Prepare configuration from parsed web hook payload and return ReleaseBot instance with logger + :param webhook_payload: payload from web hook + :param issue: if true parse Github issue payload otherwise parse Github pull request payload + :return: ReleaseBot instance, configuration logger + """ configuration.configuration = Path(getenv("CONF_PATH", - "/home/release-bot/.config/conf.yaml")).resolve() + DEFAULT_CONF_FILE)).resolve() # add configuration from Github webhook configuration.repository_name = webhook_payload['repository']['name'] configuration.repository_owner = webhook_payload['repository']['owner']['login'] - configuration.github_username = webhook_payload['issue']['user']['login'] + if issue: + configuration.github_username = webhook_payload['issue']['user']['login'] + else: + configuration.github_username = webhook_payload['pull_request']['user']['login'] configuration.load_configuration() # load the rest of configuration if there is any # create url for github app to enable access over http @@ -60,8 +63,12 @@ def handle_issue(webhook_payload): f'{configuration.github_token}@github.com/' \ f'{configuration.repository_owner}/{configuration.repository_name}.git' - logger = configuration.logger - release_bot = ReleaseBot(configuration) + return ReleaseBot(configuration), configuration.logger + + +def handle_issue(webhook_payload): + """Handler for newly opened issues""" + release_bot, logger = set_configuration(webhook_payload, issue=True) logger.info("Resolving opened issue") release_bot.git.pull() @@ -80,22 +87,7 @@ def handle_issue(webhook_payload): def handle_pr(webhook_payload): """Handler for merged PR""" - configuration.configuration = Path(getenv("CONF_PATH", - "/home/release-bot/.config/conf.yaml")).resolve() - - # add configuration from Github webhook - configuration.repository_name = webhook_payload['repository']['name'] - configuration.repository_owner = webhook_payload['repository']['owner']['login'] - configuration.github_username = webhook_payload['pull_request']['user']['login'] - configuration.load_configuration() # load the rest of configuration if there is any - - # create url for github app to enable access over http - configuration.clone_url = f'https://x-access-token:' \ - f'{configuration.github_token}@github.com/' \ - f'{configuration.repository_owner}/{configuration.repository_name}.git' - - logger = configuration.logger - release_bot = ReleaseBot(configuration) + release_bot, logger = set_configuration(webhook_payload, issue=False) logger.info("Resolving opened PR") release_bot.git.pull() @@ -113,5 +105,3 @@ def handle_pr(webhook_payload): msg = ''.join(release_bot.github.comment) release_bot.project.pr_comment(release_bot.new_release.pr_number, msg) release_bot.github.comment = [] # clean up - - diff --git a/release_bot/configuration.py b/release_bot/configuration.py index 6d384d6..e400f4a 100644 --- a/release_bot/configuration.py +++ b/release_bot/configuration.py @@ -197,9 +197,8 @@ def get_project(self): :return: ogr Github/Pagure project instance or None """ # Return instance for github app - if self.github_app_id != '': - with open(self.github_app_cert_path, 'r') as cert: - github_cert = cert.read() + if self.github_app_id: + github_cert = Path(self.github_app_cert_path).read_text() # github token will be used as a credential over http (commit/push) github_app = GitHubApp(self.github_app_id, self.github_app_cert_path) @@ -211,8 +210,7 @@ def get_project(self): custom_instances=[ GithubService(token=None, github_app_id=self.github_app_id, - github_app_private_key=github_cert, - )]) + github_app_private_key=github_cert)]) # Return instance for regular user (local machine) return get_project(url=self.clone_url, diff --git a/release_bot/github.py b/release_bot/github.py index 5dd9c7c..942f4b6 100644 --- a/release_bot/github.py +++ b/release_bot/github.py @@ -325,7 +325,7 @@ def get_user_contact(self): mail = 'bot@releasebot.bot' # don't set in case of Github app instance, it uses defaults - if self.conf.github_app_id == '': + if not self.conf.github_app_id: if which_service(self.project) == GitService.Github: name = self.project.service.user.get_username() mail = self.project.service.user.get_email() From b48caf6cb4165c2fb60918494a7a4536aca796b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Maru=C5=A1in?= Date: Wed, 18 Sep 2019 17:23:19 +0200 Subject: [PATCH 14/15] saving installation_id from initial installation webhook into Redis --- Makefile | 2 +- release_bot/celery_task.py | 82 ++++++++++++++++++++++++++++++++---- release_bot/configuration.py | 11 ++--- 3 files changed, 80 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 10763ac..afb02c9 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: test -IMAGE_NAME := marusinm/release-bot +IMAGE_NAME := usercont/release-bot IMAGE_NAME_DEV := usercont/release-bot:dev TEST_IMAGE_NAME := release-bot-tests diff --git a/release_bot/celery_task.py b/release_bot/celery_task.py index 6c26690..a502252 100644 --- a/release_bot/celery_task.py +++ b/release_bot/celery_task.py @@ -15,6 +15,7 @@ from pathlib import Path from os import getenv +import redis from release_bot.celerizer import celery_app from release_bot.exceptions import ReleaseException @@ -30,16 +31,36 @@ def parse_web_hook_payload(webhook_payload): Parse json webhook payload callback :param webhook_payload: json from github webhook """ + db = get_redis_instance() if 'issue' in webhook_payload.keys(): if webhook_payload['action'] == 'opened': - handle_issue(webhook_payload) + handle_issue(webhook_payload, db) elif 'pull_request' in webhook_payload.keys(): if webhook_payload['action'] == 'closed': if webhook_payload['pull_request']['merged'] is True: - handle_pr(webhook_payload) + handle_pr(webhook_payload, db) + elif 'installation' in webhook_payload.keys(): + if webhook_payload['action'] == 'added': # detect new repo installation + installation_id = webhook_payload['installation']['id'] + repositories_added = webhook_payload['repositories_added'] + save_new_installations(installation_id, repositories_added, db) + if webhook_payload['action'] == 'removed': # detect when repo uninstall app + repositories_removed = webhook_payload['repositories_removed'] + delete_installations(repositories_removed, db) -def set_configuration(webhook_payload, issue=True): + +def get_redis_instance(): + db = redis.Redis( + host=getenv("REDIS_SERVICE_HOST", "localhost"), + port=getenv("REDIS_SERVICE_PORT", "6379"), + db=1, # 0 is used by Celery + decode_responses=True, + ) + return db + + +def set_configuration(webhook_payload, db, issue=True): """ Prepare configuration from parsed web hook payload and return ReleaseBot instance with logger :param webhook_payload: payload from web hook @@ -56,6 +77,10 @@ def set_configuration(webhook_payload, issue=True): configuration.github_username = webhook_payload['issue']['user']['login'] else: configuration.github_username = webhook_payload['pull_request']['user']['login'] + + repo_installation_id = db.get(webhook_payload['repository']['full_name']) + configuration.github_app_installation_id = repo_installation_id + configuration.load_configuration() # load the rest of configuration if there is any # create url for github app to enable access over http @@ -66,9 +91,14 @@ def set_configuration(webhook_payload, issue=True): return ReleaseBot(configuration), configuration.logger -def handle_issue(webhook_payload): - """Handler for newly opened issues""" - release_bot, logger = set_configuration(webhook_payload, issue=True) +def handle_issue(webhook_payload, db): + """ + Handler for newly opened issues + :param webhook_payload: json data from webhook + :param db: Redis instance + :return: + """ + release_bot, logger = set_configuration(webhook_payload, db=db, issue=True) logger.info("Resolving opened issue") release_bot.git.pull() @@ -85,9 +115,14 @@ def handle_issue(webhook_payload): logger.error(exc) -def handle_pr(webhook_payload): - """Handler for merged PR""" - release_bot, logger = set_configuration(webhook_payload, issue=False) +def handle_pr(webhook_payload, db): + """ + Handler for merged PR + :param webhook_payload: json data from webhook + :param db: Redis instance + :return: + """ + release_bot, logger = set_configuration(webhook_payload, db=db, issue=False) logger.info("Resolving opened PR") release_bot.git.pull() @@ -105,3 +140,32 @@ def handle_pr(webhook_payload): msg = ''.join(release_bot.github.comment) release_bot.project.pr_comment(release_bot.new_release.pr_number, msg) release_bot.github.comment = [] # clean up + + +def save_new_installations(installation_id, repositories_added, db): + """ + Save repo which installed release-bot github app with it installation id + :param installation_id: installation identifier from initial installation web hook + :param repositories_added: repositories which user choose for release-bot installation + :param db: Redis instance + :return: True if data was saved successfully into Redis + """ + with db.pipeline() as pipe: + for repo in repositories_added: + pipe.set(repo["full_name"], installation_id) + pipe.execute() + return db.save() + + +def delete_installations(repositories_removed, db): + """ + Delete repo from Redis when user uninstall release-bot app from such repo + :param repositories_removed: repositories which user choose to uninstall from the release-bot + :param db: Redis instance + :return: True if data was deleted successfully into Redis + """ + with db.pipeline() as pipe: + for repo in repositories_removed: + pipe.delete(repo["full_name"]) + pipe.execute() + return db.save() diff --git a/release_bot/configuration.py b/release_bot/configuration.py index e400f4a..e7eba2f 100644 --- a/release_bot/configuration.py +++ b/release_bot/configuration.py @@ -200,11 +200,12 @@ def get_project(self): if self.github_app_id: github_cert = Path(self.github_app_cert_path).read_text() - # github token will be used as a credential over http (commit/push) - github_app = GitHubApp(self.github_app_id, self.github_app_cert_path) - self.github_token = github_app.get_installation_access_token( - self.github_app_installation_id - ) + if self.github_app_installation_id: + # github token will be used as a credential over http (commit/push) + github_app = GitHubApp(self.github_app_id, self.github_app_cert_path) + self.github_token = github_app.get_installation_access_token( + self.github_app_installation_id + ) return get_project(url=self.clone_url, custom_instances=[ From f25fa7e883487d5bee91aa33a0fb05d9c8facf62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Maru=C5=A1in?= Date: Wed, 2 Oct 2019 16:30:53 +0200 Subject: [PATCH 15/15] fix review --- Dockerfile.app | 2 +- files/install-rpm-packages.yaml | 7 ++++++ files/recipe.yaml | 4 ---- requirements.txt | 2 +- setup.cfg | 42 ++++++++++++++++++++++++++++----- setup.py | 17 +++++++++++++ 6 files changed, 62 insertions(+), 12 deletions(-) diff --git a/Dockerfile.app b/Dockerfile.app index bb2bc6c..b1d7161 100644 --- a/Dockerfile.app +++ b/Dockerfile.app @@ -16,7 +16,7 @@ RUN dnf install -y ansible \ && ansible-playbook -vv -c local -i localhost, files/install-rpm-packages.yaml \ && dnf clean all -COPY setup.py setup.cfg requirements.txt files/recipe.yaml /src/ +COPY setup.py setup.cfg files/recipe.yaml /src/ # setuptools-scm COPY .git /src/.git COPY release_bot/ /src/release_bot/ diff --git a/files/install-rpm-packages.yaml b/files/install-rpm-packages.yaml index fb12890..1aa149e 100644 --- a/files/install-rpm-packages.yaml +++ b/files/install-rpm-packages.yaml @@ -16,4 +16,11 @@ - python3-setuptools_scm - python3-setuptools_scm_git_archive - python3-wheel # for bdist_wheel + - python3-celery + - python3-cryptography + - python3-jwt + - python3-pystache + - python3-redis + - python3-semantic_version + - twine state: present diff --git a/files/recipe.yaml b/files/recipe.yaml index 9ab810a..c062f08 100644 --- a/files/recipe.yaml +++ b/files/recipe.yaml @@ -47,10 +47,6 @@ src: run.sh dest: /usr/bin/run.sh mode: 0777 - - name: Install ogr from git master - pip: - name: git+https://github.com/packit-service/ogr.git - executable: pip3 - name: Install release-bot from /src pip: name: /src diff --git a/requirements.txt b/requirements.txt index 4a7f7e3..1f6cb3a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,4 @@ gitchangelog pystache celery[redis] cryptography -ogr@git+https://github.com/packit-service/ogr@master#egg=ogr \ No newline at end of file +ogr \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 36fa0db..18319ac 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,36 +1,66 @@ + [metadata] name = release-bot version = attr: release_bot.version.__version__ -url = https://github.com/user-cont/release-bot +url = https://github.com/user-cont/release-bot/ description = Automated releasing from GitHub repositories. long_description = file: README.md long_description_content_type = text/markdown author = Red Hat author_email = user-cont-team@redhat.com -license = GPLv3+ +license = MIT license_file = LICENSE classifiers = Development Status :: 4 - Beta + Environment :: Console Intended Audience :: Developers License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+) Operating System :: POSIX :: Linux - Programming Language :: Python :: 3 + Programming Language :: Python + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 Topic :: Software Development + Topic :: Utilities keywords = git - github - release - PyPI packaging + fedora + rpm + dist-git + [options] packages = find: python_requires = >=3.6 +include_package_data = True + +setup_requires = + setuptools_scm + setuptools_scm_git_archive + +install_requires = + PyYAML + requests + semantic_version + twine + wheel + PyJWT + flask + gitchangelog + pystache + celery + redis + cryptography + ogr>=0.7.0 [options.packages.find] exclude = tests* +[options.extras_require] +testing = + pytest + [options.entry_points] console_scripts = release-bot=release_bot.releasebot:main diff --git a/setup.py b/setup.py index 169ef40..f98b1bf 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,20 @@ +#!/usr/bin/python3 + +# -*- coding: utf-8 -*- +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + from pathlib import Path from setuptools import setup