diff --git a/.github/workflows/updatesnaptests.yml b/.github/workflows/updatesnaptests.yml index 3a72904..e65d00b 100644 --- a/.github/workflows/updatesnaptests.yml +++ b/.github/workflows/updatesnaptests.yml @@ -26,6 +26,7 @@ jobs: python3 -m pip install pyyaml python3 -m pip install python-debian python3 -m pip install packaging + python3 -m pip install gitpython - name: Code tests env: GITHUB_USER: ubuntu diff --git a/README.md b/README.md index c887adc..b23bb2d 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,14 @@ For example, to run it locally on another repo (gnome-calculator in this case) t ./updatesnap/updatesnapyaml.py --github-user GITHUB_USER --github-token GITHUB_TOKEN https://github.com/ubuntu/gnome-calculator.git ``` +This tool can also be used to automate version updates of snap based on a specified version schema. +When the `--version-schema` (optional) flag is provided as input, the tool will automatically increment the version according to the specified schema. + +To include this feature as a github worflow you need to pass an optional input in `with` command. + +``` +./updatesnap/updatesnapyaml.py --github-user GITHUB_USER --github-token GITHUB_TOKEN --version-schema VERSION_SCHEMA https://github.com/ubuntu/gnome-calculator.git +``` ### GitHub action This action should be utilized by other repos' workflows. The action checks out this repository to use updatesnapyaml.py and replaces the old snapcraft.yaml with the new one. @@ -63,3 +71,27 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} repo: ${{ github.repository }} ``` + +For example, to use snap version automation +``` +name: Push new tag update to stable branch + +on: + schedule: + # Daily for now + - cron: '9 7 * * *' + workflow_dispatch: + +jobs: + update-snapcraft-yaml: + runs-on: ubuntu-latest + steps: + - name: Checkout this repo + uses: actions/checkout@v3 + - name: Run desktop-snaps action + uses: ubuntu/desktop-snaps@stable + with: + token: ${{ secrets.GITHUB_TOKEN }} + repo: ${{ github.repository }} + version-schema: '^debian/(\d+\.\d+\.\d+)' +``` diff --git a/action.yml b/action.yml index d224d77..d3f0d18 100644 --- a/action.yml +++ b/action.yml @@ -12,6 +12,10 @@ inputs: description: 'The repo containing the snapcraft.yaml to be updated' required: true default: 'None' + version-schema: + description: 'Version schema of snapping repository' + required: false + default: 'None' # Outputs generated by running this action. outputs: @@ -22,6 +26,7 @@ outputs: # Global env var that can keep track of if there was a change or not. This determines if there are commits to be pushed. env: IS_CHANGE: false + IS_VERSION_CHANGE: false # The jobs description defining a composite action runs: @@ -36,12 +41,22 @@ runs: ref: stable path: desktop-snaps + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip + python3 -m pip install gitpython + shell: bash + # Step to run the script that will generate a new output_file with an updated tag, if one is available. If there was a change, then we move this to the snapcraft.yaml and note that there was a change. - name: run updatesnapyaml id: updatesnapyaml run: | - ./desktop-snaps/updatesnap/updatesnapyaml.py --github-user $GITHUB_USER --github-token $GITHUB_TOKEN https://github.com/${{ github.repository }} + ./desktop-snaps/updatesnap/updatesnapyaml.py --github-user $GITHUB_USER --github-token $GITHUB_TOKEN --version-schema $VERSION_SCHEMA https://github.com/${{ github.repository }} # Make sure to put the updated snapcraft.yaml file in the right location if it lives in a snap directory + if [ -f version_file ]; then + echo "IS_VERSION_CHANGE=true" >> $GITHUB_ENV + rm version_file + fi if [ -f output_file ]; then echo "IS_CHANGE=true" >> $GITHUB_ENV if [ -d snap ]; then @@ -53,6 +68,7 @@ runs: env: GITHUB_USER: ubuntu GITHUB_TOKEN: ${{ inputs.token }} + VERSION_SCHEMA: ${{ inputs.version-schema }} shell: bash # Step to remove the desktop-snaps folder so that when we commit changes in another repo, the desktop-snaps folder is not committed there. @@ -63,17 +79,24 @@ runs: # If there was a change detected, then let's commit the changes - name: Commit files - if: ${{ env.IS_CHANGE }} + if: ${{ env.IS_CHANGE || env.IS_VERSION_CHANGE}} run: | set -x git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" git config --local user.name "github-actions[bot]" - git commit -a -m "Update tag" + commit_msg="Update tag" + if [ $IS_VERSION_CHANGE = true ] && [ $IS_CHANGE = false ]; then + commit_msg="Update snap version" + fi + if [ $IS_VERSION_CHANGE = true ] && [ $IS_CHANGE = true ]; then + commit_msg="Update snap version & update tag" + fi + git commit -a -m "$commit_msg" shell: bash # If there was a change detected, then let's push the changes - name: Push changes - if: ${{ env.IS_CHANGE }} + if: ${{ env.IS_CHANGE || env.IS_VERSION_CHANGE }} uses: ad-m/github-push-action@master env: GITHUB_USER: ubuntu diff --git a/updatesnap/SnapModule/manageYAML.py b/updatesnap/SnapModule/manageYAML.py new file mode 100644 index 0000000..8a3e274 --- /dev/null +++ b/updatesnap/SnapModule/manageYAML.py @@ -0,0 +1,116 @@ +from typing import Optional + +class ManageYAML: + """ This class takes a YAML file and splits it in an array with each + block, preserving the child structure to allow to re-create it without + loosing any line. This can't be done by reading it with the YAML module + because it deletes things like comments. """ + def __init__(self, yaml_data: str): + self._original_data = yaml_data + self._tree = self._split_yaml(yaml_data.split('\n'))[1] + + def _split_yaml(self, contents: str, level: int = 0, clevel: int = 0, + separator: str = ' ') -> tuple[list, str]: + """ Transform a YAML text file into a tree + + Splits a YAML file in lines in a format that preserves the structure, + the order and the comments. """ + + data = [] + while len(contents) != 0: + if len(contents[0].lstrip()) == 0 or contents[0][0] == '#': + if data[-1]['child'] is None: + data[-1]['child'] = [] + data[-1]['child'].append({'separator': '', + 'data': contents[0].lstrip(), + 'child': None, + 'level': clevel + 1}) + contents = contents[1:] + continue + if not contents[0].startswith(separator * level): + return contents, data + if level == 0: + if contents[0][0] == ' ' or contents[0][0] == '\t': + separator = contents[0][0] + if contents[0][level] != separator: + data.append({'separator': separator * level, + 'data': contents[0].lstrip(), + 'child': None, + 'level': clevel}) + contents = contents[1:] + continue + old_level = level + while contents[0][level] == separator: + level += 1 + contents, inner_data = self._split_yaml(contents, level, clevel+1, separator) + level = old_level + if data[-1]['child'] is None: + data[-1]['child'] = inner_data + else: + data[-1]['child'] += inner_data + return [], data + + def get_part_data(self, part_name: str) -> Optional[dict]: + """ Returns all the entries of an specific part of the current + YAML file. For example, the 'glib' part from a YAML file + with several parts. It returns None if that part doesn't + exist """ + + for entry in self._tree: + if entry['data'] != 'parts:': + continue + if ('child' not in entry) or (entry['child'] is None): + continue + for entry2 in entry['child']: + if entry2['data'] != f'{part_name}:': + continue + return entry2['child'] + return None + + def get_part_element(self, part_name: str, element: str) -> Optional[dict]: + """ Returns an specific entry for an specific part in the YAML file. + For example, it can returns the 'source-tag' entry of the part + 'glib' from a YAML file with several parts. """ + + part_data = self.get_part_data(part_name) + if part_data: + for entry in part_data: + if entry['data'].startswith(element): + return entry + return None + + def _get_yaml_group(self, group): + data = "" + for entry in group: + data += entry['separator'] + data += entry['data'] + data += '\n' + if entry['child']: + data += self._get_yaml_group(entry['child']) + return data + + def get_yaml(self) -> str: + """ Returns the YAML file updated with the new versions """ + data = self._get_yaml_group(self._tree) + data = data.rstrip() + if data[-1] != '\n': + data += '\n' + return data + + def get_metadata(self) -> Optional[dict]: + """ Returns metadata in form of list """ + data = [] + for entry in self._tree: + if entry['data'] == 'part': + continue + data.append(entry) + return data + + def get_part_metadata(self, element: str) -> Optional[dict]: + """ Returns specific element of the metadata""" + metadata = self.get_metadata() + if metadata: + for entry in metadata: + if entry['data'].startswith(element): + return entry + return None diff --git a/updatesnap/SnapModule/snapmodule.py b/updatesnap/SnapModule/snapmodule.py index 20fa80b..ea697d3 100644 --- a/updatesnap/SnapModule/snapmodule.py +++ b/updatesnap/SnapModule/snapmodule.py @@ -662,7 +662,8 @@ def process_part(self, part: str) -> Optional[dict]: "use_tag": False, "missing_format": False, "updates": [], - "version_format": {} + "version_format": {}, + "source_url": None, } if self._config is None: @@ -705,6 +706,7 @@ def process_part(self, part: str) -> Optional[dict]: if "format" not in version_format: part_data['missing_format'] = True source = data['source'] + part_data["source_url"] = source if ((not source.startswith('http://')) and (not source.startswith('https://')) and @@ -888,100 +890,33 @@ def _sort_elements(self, part, current_version, elements, text): for element in newer_elements: self._print_message(part, f" {element}\n") + def process_metadata(self) -> Optional[dict]: + """ Returns metadata from Snapcraft.yaml file """ + metadata = { + "name": None, + "version": None, + "adopt-info": None, + "grade": None, + "upstream-version": None, + "upstream-url": None, + } -class ManageYAML: - """ This class takes a YAML file and splits it in an array with each - block, preserving the child structure to allow to re-create it without - loosing any line. This can't be done by reading it with the YAML module - because it deletes things like comments. """ - def __init__(self, yaml_data: str): - self._original_data = yaml_data - self._tree = self._split_yaml(yaml_data.split('\n'))[1] - - def _split_yaml(self, contents: str, level: int = 0, clevel: int = 0, - separator: str = ' ') -> tuple[list, str]: - """ Transform a YAML text file into a tree - - Splits a YAML file in lines in a format that preserves the structure, - the order and the comments. """ - - data = [] - while len(contents) != 0: - if len(contents[0].lstrip()) == 0 or contents[0][0] == '#': - if data[-1]['child'] is None: - data[-1]['child'] = [] - data[-1]['child'].append({'separator': '', - 'data': contents[0].lstrip(), - 'child': None, - 'level': clevel + 1}) - contents = contents[1:] - continue - if not contents[0].startswith(separator * level): - return contents, data - if level == 0: - if contents[0][0] == ' ' or contents[0][0] == '\t': - separator = contents[0][0] - if contents[0][level] != separator: - data.append({'separator': separator * level, - 'data': contents[0].lstrip(), - 'child': None, - 'level': clevel}) - contents = contents[1:] - continue - old_level = level - while contents[0][level] == separator: - level += 1 - contents, inner_data = self._split_yaml(contents, level, clevel+1, separator) - level = old_level - if data[-1]['child'] is None: - data[-1]['child'] = inner_data - else: - data[-1]['child'] += inner_data - return [], data - - def get_part_data(self, part_name: str) -> Optional[dict]: - """ Returns all the entries of an specific part of the current - YAML file. For example, the 'glib' part from a YAML file - with several parts. It returns None if that part doesn't - exist """ - - for entry in self._tree: - if entry['data'] != 'parts:': - continue - if ('child' not in entry) or (entry['child'] is None): - continue - for entry2 in entry['child']: - if entry2['data'] != f'{part_name}:': - continue - return entry2['child'] - return None - - def get_part_element(self, part_name: str, element: str) -> Optional[dict]: - """ Returns an specific entry for an specific part in the YAML file. - For example, it can returns the 'source-tag' entry of the part - 'glib' from a YAML file with several parts. """ - - part_data = self.get_part_data(part_name) - if part_data: - for entry in part_data: - if entry['data'].startswith(element): - return entry - return None - - def _get_yaml_group(self, group): - data = "" - for entry in group: - data += entry['separator'] - data += entry['data'] - data += '\n' - if entry['child']: - data += self._get_yaml_group(entry['child']) - return data - - def get_yaml(self) -> str: - """ Returns the YAML file updated with the new versions """ - data = self._get_yaml_group(self._tree) - data = data.rstrip() - if data[-1] != '\n': - data += '\n' - return data + if self._config is None: + return None + data = self._config + if 'name' in data: + metadata['name'] = data['name'] + + if 'version' in data: + metadata['version'] = data['version'] + + if 'adopt-info' in data: + metadata['adopt-info'] = data['adopt-info'] + upstream_data = self.process_part(data['adopt-info']) + metadata['upstream-url'] = upstream_data['source_url'] + if len(upstream_data['updates']) != 0: + metadata['upstream-version'] = upstream_data['updates'][0] + + if 'grade' in data: + metadata['grade'] = data['grade'] + return metadata diff --git a/updatesnap/SnapVersionModule/__init__.py b/updatesnap/SnapVersionModule/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/updatesnap/SnapVersionModule/snap_version_module.py b/updatesnap/SnapVersionModule/snap_version_module.py new file mode 100644 index 0000000..4dfaf07 --- /dev/null +++ b/updatesnap/SnapVersionModule/snap_version_module.py @@ -0,0 +1,107 @@ +""" Processes the snap version and in the event of a new release of the primary + component, the version number is incremented accordingly, with the package + release number being reset to 1. Furthermore, any other modifications + to the package result in an increment of the package release number by 1 """ + +import subprocess +import os +import shutil +import re +from datetime import datetime +import logging +import requests +import git + + +def process_snap_version_data(git_repo_url, snap_name, version_schema): + """ Returns processed snap version and grade """ + + # Time stamp of Snap build in Snap Store + response = requests.get(f"https://api.snapcraft.io/v2/snaps/info/{snap_name}", + headers={"Snap-Device-Series": "16", }, timeout=20) + snap_info = response.json() + + edge_channel_info = next((channel for channel in snap_info["channel-map"] + if channel["channel"]["name"] == "edge" + and channel["channel"]["architecture"] == "amd64"), None) + snapbuilddate = 0 + if edge_channel_info: + # Parse the date string using datetime + snapbuilddate = datetime.fromisoformat(edge_channel_info["created-at"] + .replace("Z", "+00:00")) + snapbuilddate = int(snapbuilddate.timestamp()) + + # Time stamp of the last GIT commit of the snapping repository + git_log_output = subprocess.run(['git', 'log', '-1', '--date=unix'], + stdout=subprocess.PIPE, text=True, check=True) + date_string = next(line for line in git_log_output.stdout.split('\n') + if line.startswith('Date:')) + date_string = date_string.split(':', 1)[1].strip() + + # Convert the date string to a Unix timestamp + gitcommitdate = int(date_string) + + prevversion = max( + next((channel["version"] for channel in snap_info["channel-map"] + if channel["channel"]["name"] == "stable")), + next((channel["version"] for channel in snap_info["channel-map"] + if channel["channel"]["name"] == "edge")) + ) + # Clone the leading upstream repository if it doesn't exist + repo_dir = os.path.basename(git_repo_url.rstrip('.git')) + if not os.path.exists(repo_dir): + try: + git.Repo.clone_from(git_repo_url, repo_dir) + except git.exc.GitError: + logging.warning('Some error occur in cloning leading upstream repo') + os.chdir('..') + return None + + os.chdir(repo_dir) + + upstreamversion = subprocess.run(["git", "describe", "--tags", "--always"], + stdout=subprocess.PIPE, + text=True, check=True).stdout.strip() + os.chdir('..') + shutil.rmtree(repo_dir) + match = re.match(version_schema, upstreamversion) + if not match: + logging.warning("Version schema does not match with snapping repository version") + return None + upstreamversion = match.group(1).replace('_', '.') + + if upstreamversion != prevversion: + return f"{upstreamversion}-1" + # Determine package release number + packagerelease = int( + prevversion.split('-')[-1]) + 1 if gitcommitdate > snapbuilddate \ + else prevversion.split('-')[-1] + + return f"{upstreamversion}-{packagerelease}" + + +def is_version_update(snap, manager_yaml, arguments): + """ Returns if snap version update available """ + has_version_update = False + if arguments.version_schema == 'None': + return False + metadata = snap.process_metadata() + if process_snap_version_data(metadata['upstream-url'], + metadata['name'], arguments.version_schema) is not None: + snap_version = process_snap_version_data( + metadata['upstream-url'], metadata['name'], arguments.version_schema) + if metadata['version'] != snap_version: + snap_version_data = manager_yaml.get_part_metadata('version') + if snap_version_data is not None: + logging.info("Updating snap version from %s to %s", + metadata['version'], snap_version) + snap_version_data['data'] = f"version: '{snap_version}'" + has_version_update = True + else: + logging.warning("Version is not defined in metadata") + + if has_version_update: + with open('version_file', 'w', encoding="utf8") as version_file: + version_file.write(f"{snap_version}") + + return has_version_update diff --git a/updatesnap/style_check.sh b/updatesnap/style_check.sh index 907e28f..aa5bb66 100755 --- a/updatesnap/style_check.sh +++ b/updatesnap/style_check.sh @@ -14,3 +14,4 @@ test_style updatesnap.py test_style updatesnapyaml.py test_style SnapModule/snapmodule.py test_style unittests.py +test_style SnapVersionModule/snap_version_module.py diff --git a/updatesnap/tests/test_no_version_in_metadata.yaml b/updatesnap/tests/test_no_version_in_metadata.yaml new file mode 100644 index 0000000..9ebb7fb --- /dev/null +++ b/updatesnap/tests/test_no_version_in_metadata.yaml @@ -0,0 +1,41 @@ +name: hplip-printer-app +base: core22 +summary: HPLIP Printer Application +description: | + The HPLIP Printer Application is a PAPPL (Printer Application + Framework) based Printer Application to support printers using the + printer driver of HPLIP. Loading the proprietary plugin from HP is + supported, support for scanning will be added later. + +confinement: strict +adopt-info: hplip + +# Only build on the architectures supported +architectures: + - build-on: amd64 + - build-on: arm64 + - build-on: armhf + +environment: + MIBDIRS: /snap/hplip-printer-app/current/usr/share/snmp/mibs:/snap/hplip-printer-app/current/usr/share/snmp/mibs/iana:/snap/hplip-printer-app/current/usr/share/snmp/mibs/ietf + +apps: + hplip-printer-app-server: + command: scripts/run-hplip-printer-app-server + daemon: simple + stop-timeout: 70s + plugs: + [avahi-control, home, network, network-bind, raw-usb, hardware-observe] + hplip-printer-app: + command: scripts/run-hplip-printer-app + plugs: [avahi-control, home, network, network-bind, raw-usb] + +parts: + hplip: + source: https://salsa.debian.org/printing-team/hplip.v2.git + source-type: git + source-tag: "debian/3.22.10+dfsg0-4" + source-depth: 1 +# ext:updatesnap +# version-format: +# format: 'debian/%V' diff --git a/updatesnap/tests/test_snap_version_automation.yaml b/updatesnap/tests/test_snap_version_automation.yaml new file mode 100644 index 0000000..620bb5c --- /dev/null +++ b/updatesnap/tests/test_snap_version_automation.yaml @@ -0,0 +1,40 @@ +name: gutenprint-printer-app +base: core22 +version: "5.3.4-1" +grade: "stable" +summary: Gutenprint Printer Application +description: | + The Gutenprint Printer Application is a PAPPL (Printer Application + Framework) based Printer Application to support printers using the + Gutenprint printer driver. + +confinement: strict +adopt-info: gutenprint + +# Only build on the architectures supported +architectures: + - build-on: amd64 + - build-on: arm64 + - build-on: armhf + +apps: + gutenprint-printer-app-server: + command: scripts/run-gutenprint-printer-app-server + daemon: simple + stop-timeout: 70s + plugs: [avahi-control, home, network, network-bind, raw-usb] + gutenprint-printer-app: + command: scripts/run-gutenprint-printer-app + plugs: [avahi-control, home, network, network-bind, raw-usb] + +parts: + gutenprint: + source: https://github.com/echiu64/gutenprint.git + source-type: git + source-tag: "gutenprint-5_3_3" + source-depth: 1 +# ext:updatesnap +# version-format: +# format: 'gutenprint-%M_%m_%R' +# lower-than: '6' +# no-9x-revisions: true diff --git a/updatesnap/unittests.py b/updatesnap/unittests.py index 3af1448..dfa832e 100755 --- a/updatesnap/unittests.py +++ b/updatesnap/unittests.py @@ -6,12 +6,16 @@ import os import datetime import sys +import logging +from argparse import Namespace import yaml from SnapModule.snapmodule import Snapcraft -from SnapModule.snapmodule import ManageYAML +from SnapModule.manageYAML import ManageYAML from SnapModule.snapmodule import ProcessVersion from SnapModule.snapmodule import Github from SnapModule.snapmodule import Gitlab +from SnapVersionModule import snap_version_module +from SnapVersionModule.snap_version_module import is_version_update class TestYAMLfiles(unittest.TestCase): @@ -423,6 +427,53 @@ def test_invalid_version_variation_and_beta_release(self): part['part_name'], part['version'], part['entry_format'], False) assert version is None + def test_snap_version_automation(self): + """ tests if snap version automation working correctly""" + data = self._base_load_test_file("test_snap_version_automation.yaml") + yaml_obj = ManageYAML(data) + snap, _, _, _ = self._load_test_file("test_snap_version_automation.yaml", + None) + args = Namespace( + version_schema=r'^gutenprint-(\d+_\d+_\d+)', + ) + logging.basicConfig(level=logging.ERROR) + test = is_version_update(snap, yaml_obj, args) + os.remove('version_file') + assert test is not None + + def test_no_version_in_metadata(self): + """ tests if snap version automation fails + if version is not present in metadata""" + data = self._base_load_test_file("test_no_version_in_metadata.yaml") + yaml_obj = ManageYAML(data) + snap, _, _, _ = self._load_test_file("test_no_version_in_metadata.yaml", + None) + args = Namespace( + version_schema=r'^debian/(\d+\.\d+\.\d+)', + ) + logging.basicConfig(level=logging.ERROR) + test = is_version_update(snap, yaml_obj, args) + assert not test + + def test_correct_snap_version(self): + """ tests if snap version number is correct or not""" + contents = self._base_load_test_file("test_snap_version_automation.yaml") + manager_yaml = ManageYAML(contents) + snap, _, _, _ = self._load_test_file("test_snap_version_automation.yaml", + None) + args = Namespace( + version_schema=r'^gutenprint-(\d+_\d+_\d+)', + ) + + def mock_process_version_data(_git_repo_url, _snap_name, _version_schema): + return "5.3.4-1" + + temp = snap_version_module.process_snap_version_data + snap_version_module.process_snap_version_data = mock_process_version_data + test = is_version_update(snap, manager_yaml, args) + snap_version_module.process_snap_version_data = temp + assert not test + class GitPose: """ Helper class. It emulates a GitClass class, to allow to test diff --git a/updatesnap/updatesnapyaml.py b/updatesnap/updatesnapyaml.py index 18c7b8e..f4fc8bf 100755 --- a/updatesnap/updatesnapyaml.py +++ b/updatesnap/updatesnapyaml.py @@ -4,10 +4,10 @@ import sys import argparse -from SnapModule.snapmodule import Snapcraft -from SnapModule.snapmodule import Github -from SnapModule.snapmodule import ManageYAML - +import logging +from SnapModule.snapmodule import Snapcraft, Github +from SnapModule.manageYAML import ManageYAML +from SnapVersionModule.snap_version_module import is_version_update UPDATE_BRANCH = 'update_versions' @@ -66,6 +66,8 @@ def main(): help='User name for accesing Github projects.') parser.add_argument('--github-token', action='store', default=None, help='Access token for accesing Github projects.') + parser.add_argument('--version-schema', action='store', default='None', + help='Version schema of snapping repository') parser.add_argument('--verbose', action='store_true', default=False) parser.add_argument('project', default='.', help='The project URI') arguments = parser.parse_args(sys.argv[1:]) @@ -115,7 +117,8 @@ def main(): version_data['data'] = f"source-tag: '{part['updates'][0]['name']}'" has_update = True - if has_update: + logging.basicConfig(level=logging.INFO) + if (is_version_update(snap, manager_yaml, arguments) or has_update): with open('output_file', 'w', encoding="utf8") as output_file: output_file.write(manager_yaml.get_yaml()) else: