From 32b538a9377ba9a0135f14936a72582cf4d17ca4 Mon Sep 17 00:00:00 2001 From: Adam Saleh Date: Thu, 8 Dec 2022 15:28:08 +0100 Subject: [PATCH 1/3] Add test that checks repo-test workload and view --- Dockerfile | 2 +- feedback_pipeline.py | 8 +-- test_configs/base-test.yaml | 76 ++++++++++++++++++++++++++++ test_configs/view-test.yaml | 13 +++++ test_feedback_pipeline.py | 98 ++++++++++++++++++++++++++++++++++++- 5 files changed, 190 insertions(+), 7 deletions(-) create mode 100644 test_configs/base-test.yaml create mode 100644 test_configs/view-test.yaml diff --git a/Dockerfile b/Dockerfile index b5cc377..dc24386 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ from registry.fedoraproject.org/fedora:37 run dnf -y update fedora-gpg-keys && \ - dnf -y install git python3-jinja2 python3-koji python3-yaml && \ + dnf -y install git python3-pytest python3-pytest-cov python3-jinja2 python3-koji python3-yaml && \ dnf clean all workdir /workspace diff --git a/feedback_pipeline.py b/feedback_pipeline.py index af76faf..7becd7b 100755 --- a/feedback_pipeline.py +++ b/feedback_pipeline.py @@ -150,7 +150,7 @@ def datetime_now_string(): return datetime.datetime.now().strftime("%m/%d/%Y, %H:%M:%S") -def load_settings(): +def load_settings(argv=None): settings = {} parser = argparse.ArgumentParser() @@ -159,7 +159,7 @@ def load_settings(): parser.add_argument("--use-cache", dest="use_cache", action='store_true', help="Use local data instead of pulling Content Resolver. Saves a lot of time! Needs a 'cache_data.json' file at the same location as the script is at.") parser.add_argument("--dev-buildroot", dest="dev_buildroot", action='store_true', help="Buildroot grows pretty quickly. Use a fake one for development.") parser.add_argument("--dnf-cache-dir", dest="dnf_cache_dir_override", help="Override the dnf cache_dir.") - args = parser.parse_args() + args = parser.parse_args(argv) settings["configs"] = args.configs settings["output"] = args.output @@ -7245,7 +7245,7 @@ def generate_historic_data(query): ############################################################################### -def main(): +def main(argv=None): # ------------------------------------------------- # Stage 1: Data collection and analysis using DNF @@ -7254,7 +7254,7 @@ def main(): # measuring time of execution time_started = datetime_now_string() - settings = load_settings() + settings = load_settings(argv) settings["global_refresh_time_started"] = datetime.datetime.now().strftime("%-d %B %Y %H:%M UTC") diff --git a/test_configs/base-test.yaml b/test_configs/base-test.yaml new file mode 100644 index 0000000..6282d42 --- /dev/null +++ b/test_configs/base-test.yaml @@ -0,0 +1,76 @@ +--- +# This configuration file defines an "Environment" in Feedback Pipeline. +# https://tiny.distro.builders +# +# Environments influence how a workload looks like when installed. +# That's achieved by including specific packages — like coreutils-single — that +# influence the result. +# Environments can also act as base images when monitoring container sizes. + +document: feedback-pipeline-environment +version: 1 +data: + # id is the filename — that automatically prevents collisions for free! + + + ### MANDATORY FIELDS ### + + # Name is an identifier for humans + # + # (mandatory field) + name: Test F34 Environment + + # A short description, perhaps hinting the purpose + # + # (mandatory field) + description: A base environment on top of which all test workloads are analyzed. + + # Who maintains it? This is just a freeform string + # for humans to read. In Fedora, a FAS nick is recommended. + # + # (mandatory field) + maintainer: asamalik + + # Different instances of the environment, one per repository. + # + # (mandatory field) + repositories: + - repo-test + + # Packages defining this environment. + # This list includes packages for all + # architectures — that's the one to use by default. + # + # (mandatory field) + packages: + - fedora-repos-eln + - fedora-release-eln + + # Labels connect things together. + # Workloads get installed in environments with the same label. + # They also get included in views with the same label. + # + # (mandatory field) + labels: + - eln + - eln-but-not-included + - eln-extras + + ### OPTIONAL FIELDS ### + + # Architecture-specific packages. + # + # (optional field) + #arch_packages: + # x86_64: + # - arch-specific-package + + # Extra installation options. + # The following are now supported: + # - "include-docs" - include documentation packages + # - "include-weak-deps" - automatically pull in "recommends" weak dependencies + # + # (optional field) + #options: + #- option + diff --git a/test_configs/view-test.yaml b/test_configs/view-test.yaml new file mode 100644 index 0000000..883a0bd --- /dev/null +++ b/test_configs/view-test.yaml @@ -0,0 +1,13 @@ +--- +document: feedback-pipeline-compose-view +version: 1 +data: + name: Test Package Set + description: Test package set based on F34 + maintainer: bakery + labels: + - eln + repository: repo-test + buildroot_strategy: dep_tracker + + diff --git a/test_feedback_pipeline.py b/test_feedback_pipeline.py index f6c355b..f74e693 100644 --- a/test_feedback_pipeline.py +++ b/test_feedback_pipeline.py @@ -2,5 +2,99 @@ import feedback_pipeline -def test_build_completion(): - assert 1 == 1 \ No newline at end of file +import json +import os +import pytest +import tempfile + +from shutil import rmtree + +@pytest.fixture(scope="module") +def feedback_pipeline_output(): + with tempfile.TemporaryDirectory() as tmp: + os.mkdir(f"{tmp}/history") + feedback_pipeline.main([ + "--dev-buildroot", "--dnf-cache-dir", + "/tmp/test_cr", "test_configs", tmp]) + yield tmp + + +def test_bash_test_repo_workload(feedback_pipeline_output): + expected_pkg_added_ids = set([ + 'filesystem-3.14-5.fc34.aarch64', + 'glibc-minimal-langpack-2.33-5.fc34.aarch64', + 'tzdata-2021a-1.fc34.noarch', + 'basesystem-11-11.fc34.noarch', + 'bash-5.1.0-2.fc34.aarch64', + 'ncurses-libs-6.2-4.20200222.fc34.aarch64', + 'ncurses-base-6.2-4.20200222.fc34.noarch', + 'libgcc-11.0.1-0.3.fc34.aarch64', + 'glibc-2.33-5.fc34.aarch64', + 'setup-2.13.7-3.fc34.noarch', + 'glibc-common-2.33-5.fc34.aarch64' + ]) + + with open(f"{feedback_pipeline_output}/workload--bash--base-test--repo-test--aarch64.json") as w: + workload = json.load(w) + assert set(workload["data"]["pkg_added_ids"]) == expected_pkg_added_ids + +def test_bash_test_repo_view(feedback_pipeline_output): + expected_pkgs = {'alternatives-1.15-2.fc34', + 'basesystem-11-11.fc34', + 'bash-5.1.0-2.fc34', + 'ca-certificates-2020.2.41-7.fc34', + 'coreutils-8.32-21.fc34', + 'coreutils-common-8.32-21.fc34', + 'crypto-policies-20210213-1.git5c710c0.fc34', + 'fedora-gpg-keys-34-1', + 'fedora-release-34-1', + 'fedora-release-common-34-1', + 'fedora-release-identity-basic-34-1', + 'fedora-repos-34-1', + 'fedora-repos-eln-34-1', + 'fedora-repos-rawhide-34-1', + 'filesystem-3.14-5.fc34', + 'gc-8.0.4-5.fc34', + 'glibc-2.33-5.fc34', + 'glibc-common-2.33-5.fc34', + 'glibc-minimal-langpack-2.33-5.fc34', + 'gmp-1:6.2.0-6.fc34', + 'grep-3.6-2.fc34', + 'guile22-2.2.7-2.fc34', + 'libacl-2.3.1-1.fc34', + 'libattr-2.5.1-1.fc34', + 'libcap-2.48-2.fc34', + 'libffi-3.1-28.fc34', + 'libgcc-11.0.1-0.3.fc34', + 'libselinux-3.2-1.fc34', + 'libsepol-3.2-1.fc34', + 'libstdc++-11.0.1-0.3.fc34', + 'libtasn1-4.16.0-4.fc34', + 'libtool-ltdl-2.4.6-40.fc34', + 'libunistring-0.9.10-10.fc34', + 'libxcrypt-4.4.18-1.fc34', + 'make-1:4.3-5.fc34', + 'ncurses-base-6.2-4.20200222.fc34', + 'ncurses-libs-6.2-4.20200222.fc34', + 'openssl-libs-1:1.1.1k-1.fc34', + 'p11-kit-0.23.22-3.fc34', + 'p11-kit-trust-0.23.22-3.fc34', + 'pcre-8.44-3.fc34.1', + 'pcre2-10.36-4.fc34', + 'pcre2-syntax-10.36-4.fc34', + 'pizza-package-000-placeholder', + 'readline-8.1-2.fc34', + 'sed-4.8-7.fc34', + 'setup-2.13.7-3.fc34', + 'tar-2:1.34-1.fc34', + 'tzdata-2021a-1.fc34', + 'zlib-1.2.11-26.fc34'} + + with open(f"{feedback_pipeline_output}/view-packages--view-test.json") as w: + view = json.load(w) + assert set(view['pkgs'].keys()) == expected_pkgs + + + +if __name__ == "__main__": + test_mock_argv() From 14880d3238a7e1dffcd83211370bb9dc0e7eaf9c Mon Sep 17 00:00:00 2001 From: Adam Saleh Date: Sat, 17 Dec 2022 12:20:25 +0100 Subject: [PATCH 2/3] Changed configs relevant to test not to reference eln. --- test_configs/base-test.yaml | 7 +-- test_configs/bash-test.yaml | 68 ++++++++++++++++++++++++ test_configs/view-test.yaml | 2 +- test_feedback_pipeline.py | 101 +++++++++++++----------------------- 4 files changed, 106 insertions(+), 72 deletions(-) create mode 100644 test_configs/bash-test.yaml diff --git a/test_configs/base-test.yaml b/test_configs/base-test.yaml index 6282d42..6e57c1d 100644 --- a/test_configs/base-test.yaml +++ b/test_configs/base-test.yaml @@ -43,8 +43,7 @@ data: # # (mandatory field) packages: - - fedora-repos-eln - - fedora-release-eln + - bash # Labels connect things together. # Workloads get installed in environments with the same label. @@ -52,9 +51,7 @@ data: # # (mandatory field) labels: - - eln - - eln-but-not-included - - eln-extras + - test ### OPTIONAL FIELDS ### diff --git a/test_configs/bash-test.yaml b/test_configs/bash-test.yaml new file mode 100644 index 0000000..823e801 --- /dev/null +++ b/test_configs/bash-test.yaml @@ -0,0 +1,68 @@ +--- +# This configuration file defines a "Workload" in Feedback Pipeline. +# https://tiny.distro.builders +# +# Workload is a set of packages with a purpouse that someone cares about. +# They might want to monitor it for the install size, dependencies that get +# pulled in, etc. + +document: feedback-pipeline-workload +version: 1 +data: + # id is the filename — that automatically prevents collisions for free! + + + ### MANDATORY FIELDS ### + + # Name is an identifier for humans + # + # (mandatory field) + name: Bash + + # A short description, perhaps hinting the purpose + # + # (mandatory field) + description: Bash + + # Who maintains it? This is just a freeform string + # for humans to read. In Fedora, a FAS nick is recommended. + # + # (mandatory field) + maintainer: asamalik + + # Packages defining this workload. + # This list includes packages for all + # architectures — that's the one to use by default. + # + # (mandatory field) + packages: + - bash + + # Labels connect things together. + # Workloads get installed in environments with the same label. + # They also get included in views with the same label. + # + # (mandatory field) + labels: + - test + options: + - strict + + + ### OPTIONAL FIELDS ### + + # Architecture-specific packages. + # + # (optional field) + #arch_packages: + # x86_64: + # - arch-specific-package + + # Extra installation options. + # The following are now supported: + # - "include-docs" - include documentation packages + # - "include-weak-deps" - automatically pull in "recommends" weak dependencies + # + # (optional field) + #options: + #- option diff --git a/test_configs/view-test.yaml b/test_configs/view-test.yaml index 883a0bd..12d9a35 100644 --- a/test_configs/view-test.yaml +++ b/test_configs/view-test.yaml @@ -6,7 +6,7 @@ data: description: Test package set based on F34 maintainer: bakery labels: - - eln + - test repository: repo-test buildroot_strategy: dep_tracker diff --git a/test_feedback_pipeline.py b/test_feedback_pipeline.py index f74e693..afd880b 100644 --- a/test_feedback_pipeline.py +++ b/test_feedback_pipeline.py @@ -20,81 +20,50 @@ def feedback_pipeline_output(): def test_bash_test_repo_workload(feedback_pipeline_output): - expected_pkg_added_ids = set([ - 'filesystem-3.14-5.fc34.aarch64', - 'glibc-minimal-langpack-2.33-5.fc34.aarch64', + expected_pkg_env_ids = set([ 'tzdata-2021a-1.fc34.noarch', - 'basesystem-11-11.fc34.noarch', - 'bash-5.1.0-2.fc34.aarch64', - 'ncurses-libs-6.2-4.20200222.fc34.aarch64', - 'ncurses-base-6.2-4.20200222.fc34.noarch', + 'fedora-gpg-keys-34-1.noarch', + 'fedora-release-common-34-1.noarch', + 'glibc-minimal-langpack-2.33-5.fc34.aarch64', 'libgcc-11.0.1-0.3.fc34.aarch64', - 'glibc-2.33-5.fc34.aarch64', 'setup-2.13.7-3.fc34.noarch', - 'glibc-common-2.33-5.fc34.aarch64' + 'basesystem-11-11.fc34.noarch', + 'glibc-2.33-5.fc34.aarch64', + 'fedora-release-34-1.noarch', + 'ncurses-base-6.2-4.20200222.fc34.noarch', + 'ncurses-libs-6.2-4.20200222.fc34.aarch64', + 'bash-5.1.0-2.fc34.aarch64', + 'filesystem-3.14-5.fc34.aarch64', + 'glibc-common-2.33-5.fc34.aarch64', + 'fedora-release-identity-basic-34-1.noarch', + 'fedora-repos-34-1.noarch' ]) - with open(f"{feedback_pipeline_output}/workload--bash--base-test--repo-test--aarch64.json") as w: + with open(f"{feedback_pipeline_output}/workload--bash-test--base-test--repo-test--aarch64.json") as w: workload = json.load(w) - assert set(workload["data"]["pkg_added_ids"]) == expected_pkg_added_ids + assert set(workload["data"]["pkg_env_ids"]) == expected_pkg_env_ids def test_bash_test_repo_view(feedback_pipeline_output): - expected_pkgs = {'alternatives-1.15-2.fc34', - 'basesystem-11-11.fc34', - 'bash-5.1.0-2.fc34', - 'ca-certificates-2020.2.41-7.fc34', - 'coreutils-8.32-21.fc34', - 'coreutils-common-8.32-21.fc34', - 'crypto-policies-20210213-1.git5c710c0.fc34', - 'fedora-gpg-keys-34-1', - 'fedora-release-34-1', - 'fedora-release-common-34-1', - 'fedora-release-identity-basic-34-1', - 'fedora-repos-34-1', - 'fedora-repos-eln-34-1', - 'fedora-repos-rawhide-34-1', - 'filesystem-3.14-5.fc34', - 'gc-8.0.4-5.fc34', - 'glibc-2.33-5.fc34', - 'glibc-common-2.33-5.fc34', - 'glibc-minimal-langpack-2.33-5.fc34', - 'gmp-1:6.2.0-6.fc34', - 'grep-3.6-2.fc34', - 'guile22-2.2.7-2.fc34', - 'libacl-2.3.1-1.fc34', - 'libattr-2.5.1-1.fc34', - 'libcap-2.48-2.fc34', - 'libffi-3.1-28.fc34', - 'libgcc-11.0.1-0.3.fc34', - 'libselinux-3.2-1.fc34', - 'libsepol-3.2-1.fc34', - 'libstdc++-11.0.1-0.3.fc34', - 'libtasn1-4.16.0-4.fc34', - 'libtool-ltdl-2.4.6-40.fc34', - 'libunistring-0.9.10-10.fc34', - 'libxcrypt-4.4.18-1.fc34', - 'make-1:4.3-5.fc34', - 'ncurses-base-6.2-4.20200222.fc34', - 'ncurses-libs-6.2-4.20200222.fc34', - 'openssl-libs-1:1.1.1k-1.fc34', - 'p11-kit-0.23.22-3.fc34', - 'p11-kit-trust-0.23.22-3.fc34', - 'pcre-8.44-3.fc34.1', - 'pcre2-10.36-4.fc34', - 'pcre2-syntax-10.36-4.fc34', - 'pizza-package-000-placeholder', - 'readline-8.1-2.fc34', - 'sed-4.8-7.fc34', - 'setup-2.13.7-3.fc34', - 'tar-2:1.34-1.fc34', - 'tzdata-2021a-1.fc34', - 'zlib-1.2.11-26.fc34'} + expected_pkgs = {} with open(f"{feedback_pipeline_output}/view-packages--view-test.json") as w: + expected_pkgs = { + 'fedora-gpg-keys-34-1', + 'fedora-release-34-1', + 'fedora-release-common-34-1', + 'fedora-release-identity-basic-34-1', + 'fedora-repos-34-1', + 'filesystem-3.14-5.fc34', + 'glibc-minimal-langpack-2.33-5.fc34', + 'tzdata-2021a-1.fc34', + 'basesystem-11-11.fc34', + 'bash-5.1.0-2.fc34', + 'ncurses-libs-6.2-4.20200222.fc34', + 'ncurses-base-6.2-4.20200222.fc34', + 'libgcc-11.0.1-0.3.fc34', + 'glibc-2.33-5.fc34', + 'setup-2.13.7-3.fc34', + 'glibc-common-2.33-5.fc34' + } view = json.load(w) assert set(view['pkgs'].keys()) == expected_pkgs - - - -if __name__ == "__main__": - test_mock_argv() From 374c433419d97bb97303bfe3fc44d8ca8202b77d Mon Sep 17 00:00:00 2001 From: Adam Saleh Date: Tue, 24 Jan 2023 10:54:18 +0100 Subject: [PATCH 3/3] Added htmlmin library to minify the web-pages as per #23 --- .github/workflows/docker-image.yml | 4 ++-- Dockerfile | 2 +- feedback_pipeline.py | 11 ++++++++++- test_feedback_pipeline.py | 5 +++-- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 05dbe59..989b6d5 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -18,8 +18,8 @@ jobs: run: dnf -y update fedora-gpg-keys - name: Install git and Python libraries - run: dnf -y install git-core python3-yaml python3-jinja2 python3-koji python3-pytest python3-flake8 - + run: dnf -y install git-core python3-yaml python3-htmlmin python3-jinja2 python3-koji python3-pytest python3-flake8 + - name: Clean up run: dnf clean all diff --git a/Dockerfile b/Dockerfile index dc24386..3e6101e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ from registry.fedoraproject.org/fedora:37 run dnf -y update fedora-gpg-keys && \ - dnf -y install git python3-pytest python3-pytest-cov python3-jinja2 python3-koji python3-yaml && \ + dnf -y install git python3-pytest python3-htmlmin python3-pytest-cov python3-jinja2 python3-koji python3-yaml && \ dnf clean all workdir /workspace diff --git a/feedback_pipeline.py b/feedback_pipeline.py index 7becd7b..64d6dee 100755 --- a/feedback_pipeline.py +++ b/feedback_pipeline.py @@ -1,6 +1,6 @@ #!/usr/bin/python3 -import argparse, yaml, tempfile, os, subprocess, json, jinja2, datetime, copy, re, dnf, pprint, urllib.request, sys, koji +import argparse, yaml, tempfile, os, subprocess, json, jinja2, datetime, copy, re, dnf, pprint, urllib.request, sys, koji, htmlmin import concurrent.futures import rpm_showme as showme from functools import lru_cache @@ -159,11 +159,13 @@ def load_settings(argv=None): parser.add_argument("--use-cache", dest="use_cache", action='store_true', help="Use local data instead of pulling Content Resolver. Saves a lot of time! Needs a 'cache_data.json' file at the same location as the script is at.") parser.add_argument("--dev-buildroot", dest="dev_buildroot", action='store_true', help="Buildroot grows pretty quickly. Use a fake one for development.") parser.add_argument("--dnf-cache-dir", dest="dnf_cache_dir_override", help="Override the dnf cache_dir.") + parser.add_argument("--htmlmin", dest="htmlmin", action='store_true', help="Run html minimiser while producing the pages.") args = parser.parse_args(argv) settings["configs"] = args.configs settings["output"] = args.output settings["use_cache"] = args.use_cache + settings["htmlmin"] = args.htmlmin settings["dev_buildroot"] = args.dev_buildroot settings["dnf_cache_dir_override"] = args.dnf_cache_dir_override @@ -5721,6 +5723,13 @@ def _generate_html_page(template_name, template_data, page_name, settings): page = template.render(**template_data) + if settings["htmlmin"]: + try: + page = htmlmin.minify(page, remove_empty_space=True) + except Exception as e: + log(" Minification failed for... ({})".format(filename)) + + filename = ("{page_name}.html".format( page_name=page_name.replace(":", "--") )) diff --git a/test_feedback_pipeline.py b/test_feedback_pipeline.py index afd880b..77f9945 100644 --- a/test_feedback_pipeline.py +++ b/test_feedback_pipeline.py @@ -14,8 +14,9 @@ def feedback_pipeline_output(): with tempfile.TemporaryDirectory() as tmp: os.mkdir(f"{tmp}/history") feedback_pipeline.main([ - "--dev-buildroot", "--dnf-cache-dir", - "/tmp/test_cr", "test_configs", tmp]) + "--dev-buildroot", "--htmlmin", + "--dnf-cache-dir", "/tmp/test_cr", + "test_configs", tmp]) yield tmp