Add initial logic

.github/workflows/ci.yml

@@ -0,0 +1,25 @@
+name: CI
+on: [push, pull_request]
+jobs:
+  pylama:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.x
+      uses: actions/setup-python@v2
+      with: {python-version: '3.9'}
+    - name: Install dependencies
+      run: pip install tox
+    - name: Run pylama
+      run: tox -e pylama
+  pyre:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.x
+      uses: actions/setup-python@v2
+      with: {python-version: '3.9'}
+    - name: Install dependencies
+      run: pip install tox
+    - name: Run pyre
+      run: tox -e pyre

.pyre_configuration

@@ -0,0 +1,4 @@
+{
+  "source_directories": ["pipup"],
+  "strict": true
+}

MANIFEST.in

@@ -0,0 +1 @@
+include requirements.txt

README.md

@@ -1,2 +1,39 @@
-# pipup
-Simple requirements updater
+# pipup - Simple requirements updater
+
+A few different tools exist today such as pyup, however they either require a 3rd party service, or are (semi) abandoned.
+
+pipup aims to achieve the goal of automatically updating requirements.txt files, with merging based on passing CI.
+
+## Example usage
+
+`.pipup.yaml`
+```yaml
+requirements:
+  - requirements.txt
+  - requirements-dev.txt
+  - requirements-prod.txt
+workflows:
+  - CI
+mirrors:
+  - https://pypi.org/pypi/{name}/json
+```
+
+_Note: Sane defaults are used without a config file_
+
+`requirements.txt`
+```text
+pyre-check==0.9.6 # pipup:ignore
+pylama==7.7.1 # pipup:version:>=7.7.0,<8.0.0
+```
+
+_Note: All dependencies are updated by default_
+
+`.github/workflows/pipup.yml`
+```yaml
+name: Update dependencies using pipup
+on: {schedule: [{cron: '13 6 * * *'}], push: {branches: [main]}}
+permissions: {contents: write, issues: write, pull-requests: write}
+jobs: {pipup: {runs-on: ubuntu-20.04, steps: [{uses: InfraBits/[email protected]}]}}
+```
+
+_Note: Change the schedule/branches to suit the local repository_

action.yaml

@@ -0,0 +1,19 @@
+---
+name: infrabits/pipup
+description: Automatic update of `pip` dependancies
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v2
+      with: { python-version: '3.9' }
+    - shell: bash
+      run: pip install git+https://github.com/InfraBits/pipup.git@main
+    - uses: actions/checkout@v2
+    - shell: bash
+      run: |
+        pipup \
+        --merge \
+        --repository ${{ github.repository }}
+      env:
+        GITHUB_TOKEN: ${{ github.token }}

dev-requirements.txt

@@ -0,0 +1,4 @@
+tox
+pytest
+pylama
+pyre-check

pipup/__init__.py

@@ -0,0 +1,26 @@
+'''
+pipup - Simple requirements updater
+
+MIT License
+
+Copyright (c) 2022 Infra Bits
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+'''
+__version__ = '0.0.1'

pipup/cli.py

@@ -0,0 +1,115 @@
+'''
+pipup - Simple requirements updater
+
+MIT License
+
+Copyright (c) 2022 Infra Bits
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+'''
+import logging
+import sys
+import uuid
+
+import click
+
+from .git import Git
+from .settings import Settings
+from .updater import Updater
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+@click.command()
+@click.option('--debug', is_flag=True, help='Increase logging level to debug')
+@click.option('--merge', is_flag=True, help='Merge changes into a GitHub repo')
+@click.option('--repository', help='Name of the GitHub repo these files belong to')
+def cli(debug: bool, merge: bool, repository: str) -> None:
+    '''pipup - Simple requirements updater.'''
+    logging.basicConfig(stream=sys.stderr,
+                        level=(logging.DEBUG if debug else logging.INFO),
+                        format='%(asctime)-15s %(message)s')
+
+    if merge and not repository:
+        click.echo("--merge requires --repository")
+        return
+
+    # Load the settings for our runtime
+    settings = Settings.load()
+    logger.info(f'Using settings: {settings}')
+
+    # Perform the actual updates
+    updater = Updater(settings)
+
+    logger.info('Resolving requirements')
+    updater.resolve_requirements()
+
+    logger.info('Updating requirements')
+    updated_requirements = updater.update_requirements()
+
+    if not all([
+        requirements.have_updates()
+        for requirements in updated_requirements
+    ]):
+        logger.info('No updates required')
+        return
+
+    logger.info('Saving updated requirements files')
+    for requirements in updated_requirements:
+        if requirements.have_updates():
+            logger.info(f' - {requirements.file_path}')
+            with requirements.file_path.open('w') as fh:
+                fh.write(requirements.export_requirements_txt())
+
+    # Create a pull request if required
+    if merge:
+        branch_name = f'pipup-{uuid.uuid4()}'
+        logger.info(f'Merging updated requirements files using {branch_name}')
+
+        # Handle the merging logic as required
+        git = Git(repository, branch_name)
+        head_ref, head_sha = git.get_head_ref()
+        if not head_ref or not head_sha:
+            logger.error('Failed to get head ref')
+            return
+
+        git.create_branch(head_sha)
+        for requirements in updated_requirements:
+            if requirements.have_updates():
+                commit_summary = requirements.update_summary()
+                commit_description = requirements.update_detail()
+
+                logger.info(f' - {requirements.file_path}')
+                logger.info(f'  Using commit summary: {commit_summary}')
+                logger.info(f'  Using commit description: {commit_description}')
+                git.update_branch_file(requirements.file_path,
+                                       requirements.export_requirements_txt(),
+                                       commit_summary,
+                                       commit_description)
+
+        logger.info(f'Creating pull request for {branch_name}')
+        if pull_request_id := git.create_pull_request(head_ref):
+            logger.info(f'Waiting for workflows to complete on {branch_name}')
+            if git.wait_for_workflows(settings.workflows):
+                logger.info(f'Merging pull request {pull_request_id}')
+                git.merge_pull_request(pull_request_id)
+
+
+if __name__ == '__main__':
+    cli()

pipup/git.py

@@ -0,0 +1,146 @@
+'''
+pipup - Simple requirements updater
+
+MIT License
+
+Copyright (c) 2022 Infra Bits
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+'''
+import base64
+import logging
+import os
+import time
+from pathlib import PosixPath
+from typing import Optional, List, Tuple
+
+import requests
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+class Git:
+    def __init__(self, repository: str, branch_name: str) -> None:
+        self.repository = repository
+        self._headers = {
+            'Authorization': f'token {os.environ.get("GITHUB_TOKEN", "")}',
+            'Accept': 'application/vnd.github.v3+json'
+        }
+        self._branch_name = branch_name
+
+    def get_default_branch(self) -> Tuple[str, str]:
+        r = requests.get(f'https://api.github.com/repos/{self.repository}',
+                         headers=self._headers)
+        r.raise_for_status()
+        return r.json()['default_branch']
+
+    def get_head_ref(self) -> Tuple[Optional[str], Optional[str]]:
+        r = requests.get('https://api.github.com/repos/'
+                         f'{self.repository}/git/refs',
+                         headers=self._headers)
+        r.raise_for_status()
+
+        default_branch = self.get_default_branch()
+        for branch in r.json():
+            if branch['ref'] == f'refs/heads/{default_branch}':
+                return branch['ref'], branch['object']['sha']
+        return None, None
+
+    def create_branch(self, base_sha: str) -> None:
+        r = requests.post('https://api.github.com/repos/'
+                          f'{self.repository}/git/refs',
+                          json={
+                              'ref': f'refs/heads/{self._branch_name}',
+                              'sha': base_sha,
+                          },
+                          headers=self._headers)
+        r.raise_for_status()
+
+    def get_file_sha(self, path: PosixPath) -> Optional[str]:
+        r = requests.get('https://api.github.com/repos/'
+                         f'{self.repository}/contents/{path}',
+                         json={'branch': self._branch_name},
+                         headers=self._headers)
+        if r.status_code == 404:
+            return None
+        r.raise_for_status()
+        return r.json()['sha']
+
+    def update_branch_file(self,
+                           path: PosixPath,
+                           contents: str,
+                           commit_summary: str,
+                           commit_body: Optional[str]) -> None:
+        r = requests.put('https://api.github.com/repos/'
+                         f'{self.repository}/contents/{path}',
+                         json={
+                             'message': (f'{commit_summary}\n{commit_body}'
+                                         if commit_body else commit_summary),
+                             'branch': self._branch_name,
+                             'content': base64.b64encode(contents.encode('utf-8')).decode('utf-8'),
+                             'sha': self.get_file_sha(path),
+                         },
+                         headers=self._headers)
+        r.raise_for_status()
+
+    def create_pull_request(self, head_ref: str) -> int:
+        r = requests.post('https://api.github.com/repos/'
+                          f'{self.repository}/pulls',
+                          json={
+                              'title': 'pipup',
+                              'head': self._branch_name,
+                              'base': head_ref,
+                          },
+                          headers=self._headers)
+        r.raise_for_status()
+        return r.json()['number']
+
+    def get_pull_request_actions(self) -> List[Tuple[str, bool]]:
+        r = requests.get('https://api.github.com/repos/'
+                         f'{self.repository}/actions/runs',
+                         json={'branch': self._branch_name},
+                         headers=self._headers)
+        r.raise_for_status()
+        return [
+            (action['name'], action['conclusion'] == 'success')
+            for action in r.json()['workflow_runs']
+        ]
+
+    def merge_pull_request(self, pull_request_id: int) -> None:
+        r = requests.put(f'https://api.github.com/repos/'
+                         f'{self.repository}/pulls/{pull_request_id}/merge',
+                         headers=self._headers)
+        r.raise_for_status()
+
+    def wait_for_workflows(self, required_workflows: List[str]) -> Optional[bool]:
+        while True:
+            successful_workflows = {
+                action_name.lower()
+                for action_name, action_success in self.get_pull_request_actions()
+                if action_success
+            }
+
+            logger.info(f'Found completed workflows: {successful_workflows}')
+            if all([required_workflow.lower() in successful_workflows
+                    for required_workflow in required_workflows]):
+                logger.info(f'All required jobs completed: {required_workflows}')
+                return True
+
+            logger.info('Missing required jobs, waiting....')
+            time.sleep(5)