diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 05dbe59..989b6d5 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -18,8 +18,8 @@ jobs: run: dnf -y update fedora-gpg-keys - name: Install git and Python libraries - run: dnf -y install git-core python3-yaml python3-jinja2 python3-koji python3-pytest python3-flake8 - + run: dnf -y install git-core python3-yaml python3-htmlmin python3-jinja2 python3-koji python3-pytest python3-flake8 + - name: Clean up run: dnf clean all diff --git a/Dockerfile b/Dockerfile index b5cc377..3e6101e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ from registry.fedoraproject.org/fedora:37 run dnf -y update fedora-gpg-keys && \ - dnf -y install git python3-jinja2 python3-koji python3-yaml && \ + dnf -y install git python3-pytest python3-htmlmin python3-pytest-cov python3-jinja2 python3-koji python3-yaml && \ dnf clean all workdir /workspace diff --git a/feedback_pipeline.py b/feedback_pipeline.py index af76faf..64d6dee 100755 --- a/feedback_pipeline.py +++ b/feedback_pipeline.py @@ -1,6 +1,6 @@ #!/usr/bin/python3 -import argparse, yaml, tempfile, os, subprocess, json, jinja2, datetime, copy, re, dnf, pprint, urllib.request, sys, koji +import argparse, yaml, tempfile, os, subprocess, json, jinja2, datetime, copy, re, dnf, pprint, urllib.request, sys, koji, htmlmin import concurrent.futures import rpm_showme as showme from functools import lru_cache @@ -150,7 +150,7 @@ def datetime_now_string(): return datetime.datetime.now().strftime("%m/%d/%Y, %H:%M:%S") -def load_settings(): +def load_settings(argv=None): settings = {} parser = argparse.ArgumentParser() @@ -159,11 +159,13 @@ def load_settings(): parser.add_argument("--use-cache", dest="use_cache", action='store_true', help="Use local data instead of pulling Content Resolver. Saves a lot of time! Needs a 'cache_data.json' file at the same location as the script is at.") parser.add_argument("--dev-buildroot", dest="dev_buildroot", action='store_true', help="Buildroot grows pretty quickly. Use a fake one for development.") parser.add_argument("--dnf-cache-dir", dest="dnf_cache_dir_override", help="Override the dnf cache_dir.") - args = parser.parse_args() + parser.add_argument("--htmlmin", dest="htmlmin", action='store_true', help="Run html minimiser while producing the pages.") + args = parser.parse_args(argv) settings["configs"] = args.configs settings["output"] = args.output settings["use_cache"] = args.use_cache + settings["htmlmin"] = args.htmlmin settings["dev_buildroot"] = args.dev_buildroot settings["dnf_cache_dir_override"] = args.dnf_cache_dir_override @@ -5721,6 +5723,13 @@ def _generate_html_page(template_name, template_data, page_name, settings): page = template.render(**template_data) + if settings["htmlmin"]: + try: + page = htmlmin.minify(page, remove_empty_space=True) + except Exception as e: + log(" Minification failed for... ({})".format(filename)) + + filename = ("{page_name}.html".format( page_name=page_name.replace(":", "--") )) @@ -7245,7 +7254,7 @@ def generate_historic_data(query): ############################################################################### -def main(): +def main(argv=None): # ------------------------------------------------- # Stage 1: Data collection and analysis using DNF @@ -7254,7 +7263,7 @@ def main(): # measuring time of execution time_started = datetime_now_string() - settings = load_settings() + settings = load_settings(argv) settings["global_refresh_time_started"] = datetime.datetime.now().strftime("%-d %B %Y %H:%M UTC") diff --git a/test_configs/base-test.yaml b/test_configs/base-test.yaml new file mode 100644 index 0000000..6e57c1d --- /dev/null +++ b/test_configs/base-test.yaml @@ -0,0 +1,73 @@ +--- +# This configuration file defines an "Environment" in Feedback Pipeline. +# https://tiny.distro.builders +# +# Environments influence how a workload looks like when installed. +# That's achieved by including specific packages — like coreutils-single — that +# influence the result. +# Environments can also act as base images when monitoring container sizes. + +document: feedback-pipeline-environment +version: 1 +data: + # id is the filename — that automatically prevents collisions for free! + + + ### MANDATORY FIELDS ### + + # Name is an identifier for humans + # + # (mandatory field) + name: Test F34 Environment + + # A short description, perhaps hinting the purpose + # + # (mandatory field) + description: A base environment on top of which all test workloads are analyzed. + + # Who maintains it? This is just a freeform string + # for humans to read. In Fedora, a FAS nick is recommended. + # + # (mandatory field) + maintainer: asamalik + + # Different instances of the environment, one per repository. + # + # (mandatory field) + repositories: + - repo-test + + # Packages defining this environment. + # This list includes packages for all + # architectures — that's the one to use by default. + # + # (mandatory field) + packages: + - bash + + # Labels connect things together. + # Workloads get installed in environments with the same label. + # They also get included in views with the same label. + # + # (mandatory field) + labels: + - test + + ### OPTIONAL FIELDS ### + + # Architecture-specific packages. + # + # (optional field) + #arch_packages: + # x86_64: + # - arch-specific-package + + # Extra installation options. + # The following are now supported: + # - "include-docs" - include documentation packages + # - "include-weak-deps" - automatically pull in "recommends" weak dependencies + # + # (optional field) + #options: + #- option + diff --git a/test_configs/bash-test.yaml b/test_configs/bash-test.yaml new file mode 100644 index 0000000..823e801 --- /dev/null +++ b/test_configs/bash-test.yaml @@ -0,0 +1,68 @@ +--- +# This configuration file defines a "Workload" in Feedback Pipeline. +# https://tiny.distro.builders +# +# Workload is a set of packages with a purpouse that someone cares about. +# They might want to monitor it for the install size, dependencies that get +# pulled in, etc. + +document: feedback-pipeline-workload +version: 1 +data: + # id is the filename — that automatically prevents collisions for free! + + + ### MANDATORY FIELDS ### + + # Name is an identifier for humans + # + # (mandatory field) + name: Bash + + # A short description, perhaps hinting the purpose + # + # (mandatory field) + description: Bash + + # Who maintains it? This is just a freeform string + # for humans to read. In Fedora, a FAS nick is recommended. + # + # (mandatory field) + maintainer: asamalik + + # Packages defining this workload. + # This list includes packages for all + # architectures — that's the one to use by default. + # + # (mandatory field) + packages: + - bash + + # Labels connect things together. + # Workloads get installed in environments with the same label. + # They also get included in views with the same label. + # + # (mandatory field) + labels: + - test + options: + - strict + + + ### OPTIONAL FIELDS ### + + # Architecture-specific packages. + # + # (optional field) + #arch_packages: + # x86_64: + # - arch-specific-package + + # Extra installation options. + # The following are now supported: + # - "include-docs" - include documentation packages + # - "include-weak-deps" - automatically pull in "recommends" weak dependencies + # + # (optional field) + #options: + #- option diff --git a/test_configs/view-test.yaml b/test_configs/view-test.yaml new file mode 100644 index 0000000..12d9a35 --- /dev/null +++ b/test_configs/view-test.yaml @@ -0,0 +1,13 @@ +--- +document: feedback-pipeline-compose-view +version: 1 +data: + name: Test Package Set + description: Test package set based on F34 + maintainer: bakery + labels: + - test + repository: repo-test + buildroot_strategy: dep_tracker + + diff --git a/test_feedback_pipeline.py b/test_feedback_pipeline.py index f6c355b..77f9945 100644 --- a/test_feedback_pipeline.py +++ b/test_feedback_pipeline.py @@ -2,5 +2,69 @@ import feedback_pipeline -def test_build_completion(): - assert 1 == 1 \ No newline at end of file +import json +import os +import pytest +import tempfile + +from shutil import rmtree + +@pytest.fixture(scope="module") +def feedback_pipeline_output(): + with tempfile.TemporaryDirectory() as tmp: + os.mkdir(f"{tmp}/history") + feedback_pipeline.main([ + "--dev-buildroot", "--htmlmin", + "--dnf-cache-dir", "/tmp/test_cr", + "test_configs", tmp]) + yield tmp + + +def test_bash_test_repo_workload(feedback_pipeline_output): + expected_pkg_env_ids = set([ + 'tzdata-2021a-1.fc34.noarch', + 'fedora-gpg-keys-34-1.noarch', + 'fedora-release-common-34-1.noarch', + 'glibc-minimal-langpack-2.33-5.fc34.aarch64', + 'libgcc-11.0.1-0.3.fc34.aarch64', + 'setup-2.13.7-3.fc34.noarch', + 'basesystem-11-11.fc34.noarch', + 'glibc-2.33-5.fc34.aarch64', + 'fedora-release-34-1.noarch', + 'ncurses-base-6.2-4.20200222.fc34.noarch', + 'ncurses-libs-6.2-4.20200222.fc34.aarch64', + 'bash-5.1.0-2.fc34.aarch64', + 'filesystem-3.14-5.fc34.aarch64', + 'glibc-common-2.33-5.fc34.aarch64', + 'fedora-release-identity-basic-34-1.noarch', + 'fedora-repos-34-1.noarch' + ]) + + with open(f"{feedback_pipeline_output}/workload--bash-test--base-test--repo-test--aarch64.json") as w: + workload = json.load(w) + assert set(workload["data"]["pkg_env_ids"]) == expected_pkg_env_ids + +def test_bash_test_repo_view(feedback_pipeline_output): + expected_pkgs = {} + + with open(f"{feedback_pipeline_output}/view-packages--view-test.json") as w: + expected_pkgs = { + 'fedora-gpg-keys-34-1', + 'fedora-release-34-1', + 'fedora-release-common-34-1', + 'fedora-release-identity-basic-34-1', + 'fedora-repos-34-1', + 'filesystem-3.14-5.fc34', + 'glibc-minimal-langpack-2.33-5.fc34', + 'tzdata-2021a-1.fc34', + 'basesystem-11-11.fc34', + 'bash-5.1.0-2.fc34', + 'ncurses-libs-6.2-4.20200222.fc34', + 'ncurses-base-6.2-4.20200222.fc34', + 'libgcc-11.0.1-0.3.fc34', + 'glibc-2.33-5.fc34', + 'setup-2.13.7-3.fc34', + 'glibc-common-2.33-5.fc34' + } + view = json.load(w) + assert set(view['pkgs'].keys()) == expected_pkgs