diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index 60c90d3..6fbb588 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -46,6 +46,9 @@ jobs: - name: Run tests run: | poetry run python -m pytest -sxv + - name: Generate coverage report + run: | + poetry run pytest -vv --cov=monopacker --cov-report=term-missing # poetry-update-check: # runs-on: ubuntu-latest # steps: diff --git a/.gitignore b/.gitignore index dc6c540..864422b 100644 --- a/.gitignore +++ b/.gitignore @@ -20,8 +20,7 @@ packer-artifacts.json output-vagrant* ### Secrets ### -real_secrets.yaml -**/*secret*.yaml +**/*secret* # Python cache *.pyc diff --git a/SBOMs/.gitignore b/SBOMs/.gitignore new file mode 100644 index 0000000..88b0632 --- /dev/null +++ b/SBOMs/.gitignore @@ -0,0 +1,2 @@ +*.md +!README.md diff --git a/SBOMs/README.md b/SBOMs/README.md new file mode 100644 index 0000000..e126638 --- /dev/null +++ b/SBOMs/README.md @@ -0,0 +1,5 @@ +# monopacker SBOMs + +## overview + +TBD diff --git a/TEMPLATING.md b/TEMPLATING.md index fc9c218..1c5d0e0 100644 --- a/TEMPLATING.md +++ b/TEMPLATING.md @@ -160,6 +160,27 @@ simply create a `.jinja2` file with the name of your choice under `./template/bu Ensure that your builder template has a `name` key set to `{{builder.vars.name}}`, as this is how `monopacker` templating maps `builders` to `provisioners` in the Packer template. +## Rebooting while building + +Monopacker has support for handling restarts during the build process. + +If a script's name includes 'reboot', Monopacker will add a pause after the step to ensure that the host is back up before continuing. + +## Software Bill of Materials (SBOMs) + +Monopacker has support for generating SBOMs. + +SBOM generation is enabled by adding a variable file to your builder that sets `monopacker_generate_sbom` to `true`. Here's an example variable file: + +``` +--- +monopacker_generate_sbom: true +# defaults to "" +monopacker_sbom_command_args: "-b $MONOPACKER_BUILDER_NAME -c $MONOPACKER_GIT_SHA" +# default vaule is "monopacker_ubuntu_sbom.py" +# monoopacker_sbom_script: monopacker_ubuntu_sbom.py +``` + # FAQ ## I'm getting `did not find expected key` in my template @@ -174,3 +195,9 @@ A number of things could be going wrong here. Ensure that the builder template properly references all variables as being namespaced under `builder.vars` and that your `builder_var_files` and `builder_vars` do _not_ have any namespacing. See above for a more thorough description. + +## What's the difference between variables and environment variaboles? + +Variables are set only in Packer's context. + +Environment variables are set when scripts run on the target host. diff --git a/builders/gw_fxci_gcp_l1.yaml b/builders/gw_fxci_gcp_l1.yaml index adbcf33..5eb93bb 100644 --- a/builders/gw_fxci_gcp_l1.yaml +++ b/builders/gw_fxci_gcp_l1.yaml @@ -7,6 +7,7 @@ builder_var_files: - default_gcp - googlecompute_jammy - ubuntu_amd64 + - monopacker_generate_sbom script_directories: - ubuntu-jammy-from-community diff --git a/builders/gw_fxci_gcp_l1_arm64.yaml b/builders/gw_fxci_gcp_l1_arm64.yaml index 99c8a84..beea8eb 100644 --- a/builders/gw_fxci_gcp_l1_arm64.yaml +++ b/builders/gw_fxci_gcp_l1_arm64.yaml @@ -7,6 +7,7 @@ builder_var_files: - default_gcp - googlecompute_jammy_arm64 - ubuntu_arm64 + - monopacker_generate_sbom script_directories: - ubuntu-jammy-from-community diff --git a/builders/gw_fxci_gcp_l1_arm64_gui.yaml b/builders/gw_fxci_gcp_l1_arm64_gui.yaml index d220462..c4d71d0 100644 --- a/builders/gw_fxci_gcp_l1_arm64_gui.yaml +++ b/builders/gw_fxci_gcp_l1_arm64_gui.yaml @@ -8,6 +8,7 @@ builder_var_files: - googlecompute_jammy_arm64 - ubuntu_arm64 - firefoxci_loopback + - monopacker_generate_sbom script_directories: - ubuntu-jammy-from-community diff --git a/builders/gw_fxci_gcp_l1_gui.yaml b/builders/gw_fxci_gcp_l1_gui.yaml index 2f4d965..55e7db0 100644 --- a/builders/gw_fxci_gcp_l1_gui.yaml +++ b/builders/gw_fxci_gcp_l1_gui.yaml @@ -8,6 +8,7 @@ builder_var_files: - googlecompute_jammy - ubuntu_amd64 - firefoxci_loopback + - monopacker_generate_sbom script_directories: - ubuntu-jammy-from-community diff --git a/builders/gw_fxci_gcp_l3.yaml b/builders/gw_fxci_gcp_l3.yaml index 34c4553..d78a13f 100644 --- a/builders/gw_fxci_gcp_l3.yaml +++ b/builders/gw_fxci_gcp_l3.yaml @@ -8,6 +8,7 @@ builder_var_files: - googlecompute_jammy - ubuntu_amd64 - firefoxci_gcp_l3 + - monopacker_generate_sbom script_directories: - ubuntu-jammy-from-community diff --git a/builders/gw_fxci_gcp_l3_arm64.yaml b/builders/gw_fxci_gcp_l3_arm64.yaml index 2909cc4..2106196 100644 --- a/builders/gw_fxci_gcp_l3_arm64.yaml +++ b/builders/gw_fxci_gcp_l3_arm64.yaml @@ -8,6 +8,7 @@ builder_var_files: - googlecompute_jammy_arm64 - ubuntu_arm64 - firefoxci_gcp_l3 + - monopacker_generate_sbom script_directories: - ubuntu-jammy-from-community diff --git a/builders/gw_fxci_gcp_l3_arm64_gui.yaml b/builders/gw_fxci_gcp_l3_arm64_gui.yaml index ae3e919..29321c8 100644 --- a/builders/gw_fxci_gcp_l3_arm64_gui.yaml +++ b/builders/gw_fxci_gcp_l3_arm64_gui.yaml @@ -9,6 +9,7 @@ builder_var_files: - ubuntu_arm64 - firefoxci_loopback - firefoxci_gcp_l3 + - monopacker_generate_sbom script_directories: - ubuntu-jammy-from-community diff --git a/builders/gw_fxci_gcp_l3_gui.yaml b/builders/gw_fxci_gcp_l3_gui.yaml index c07ea51..d6d3305 100644 --- a/builders/gw_fxci_gcp_l3_gui.yaml +++ b/builders/gw_fxci_gcp_l3_gui.yaml @@ -9,6 +9,7 @@ builder_var_files: - ubuntu_amd64 - firefoxci_loopback - firefoxci_gcp_l3 + - monopacker_generate_sbom script_directories: - ubuntu-jammy-from-community diff --git a/builders/gw_translations_gcp.yaml b/builders/gw_translations_gcp.yaml index 8e7dadb..99a7f6b 100644 --- a/builders/gw_translations_gcp.yaml +++ b/builders/gw_translations_gcp.yaml @@ -7,6 +7,7 @@ builder_var_files: - default_linux - translations_gcp # TODO: merge this and following? - googlecompute_translations + - monopacker_generate_sbom script_directories: - ubuntu-jammy diff --git a/builders/monopacker-testing-image.yaml b/builders/monopacker-testing-image.yaml new file mode 100644 index 0000000..23962a1 --- /dev/null +++ b/builders/monopacker-testing-image.yaml @@ -0,0 +1,13 @@ +# a barebones image used for testing monopacker +template: googlecompute +platform: linux + +builder_var_files: + - default_linux + - default_gcp + - googlecompute_jammy + - ubuntu_amd64 + - monopacker_generate_sbom + +script_directories: + - ubuntu-tc-barebones diff --git a/monopacker/post-processors/move_sbom_to_latest_artifact_name.py b/monopacker/post-processors/move_sbom_to_latest_artifact_name.py new file mode 100755 index 0000000..b126629 --- /dev/null +++ b/monopacker/post-processors/move_sbom_to_latest_artifact_name.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 + +import json +import sys +import argparse +import os +import shutil + +# Set up argparse +parser = argparse.ArgumentParser(description="Move temp_sbom.md to the last build's artifact name.md") +parser.add_argument('-d', '--debug', action='store_true', help='Print what would have happened instead of performing the move') +args = parser.parse_args() + +# Load the JSON data from the file in the current working directory +file_path = 'packer-artifacts.json' +with open(file_path, 'r') as file: + data = json.load(file) + +# Extract the last_run_uuid +last_run_uuid = data['last_run_uuid'] + +# Find the matching build in the builds array +matching_build = None +for build in data['builds']: + if build['packer_run_uuid'] == last_run_uuid: + matching_build = build + break + +# Handle the move operation or describe the action if in debug mode +if matching_build: + artifact_id = matching_build['artifact_id'] + source_path = 'SBOMs/temp_sbom.md' + destination_dir = 'SBOMs' + destination_path = f'{destination_dir}/{artifact_id}.md' + + if not os.path.exists(source_path): + print(f'File {source_path} not found.') + sys.exit(0) + + if args.debug: + print(f'Would move {source_path} to {destination_path}') + else: + try: + # Create the destination directory if it doesn't exist + os.makedirs(destination_dir, exist_ok=True) + # Move the file + shutil.move(source_path, destination_path) + print(f'Moved {source_path} to {destination_path}') + except Exception as e: + print(f'An error occurred: {e}') + sys.exit(1) +else: + print('No matching build found.') + sys.exit(1) \ No newline at end of file diff --git a/monopacker/template_packer.py b/monopacker/template_packer.py index 1192a03..1bf5377 100755 --- a/monopacker/template_packer.py +++ b/monopacker/template_packer.py @@ -12,6 +12,7 @@ TemplateError, TemplateSyntaxError, ) +import subprocess from ruamel.yaml import YAML from .filters import clean_gcp_image_name @@ -20,6 +21,38 @@ yaml = YAML(typ="safe") + +# TODO: move to a utils module +def get_short_git_commit(report_dirty=True): + try: + # Get the short SHA1 of the latest commit + sha1 = ( + subprocess.check_output(["git", "rev-parse", "--short", "HEAD"]) + .strip() + .decode("utf-8") + ) + + if report_dirty: + # Check if there are any tracked changes in the working directory + changes = ( + subprocess.check_output(["git", "status", "--porcelain"]) + .strip() + .decode("utf-8") + ) + tracked_changes = [ + line + for line in changes.split("\n") + if line and not line.startswith("??") + ] + if tracked_changes: + sha1 += "-dirty" + + return sha1 + + except subprocess.CalledProcessError as e: + return str(e) + + def get_files_from_subdirs(*args, root_dir=".", globs=["*"]): """Get an sorted list of files from a list of subdirectories @@ -59,8 +92,8 @@ def load_yaml_from_file(filename: str): def merge_vars(base_vars, override_vars): """Takes two dicts, returns a new dict. Values in the second dict take precedence. - If both dicts have a dictionary value for a key their subdicts are - merged, recursively. All other values (including lists) are overridden. + If both dicts have a dictionary value for a key their subdicts are + merged, recursively. All other values (including lists) are overridden. """ d = {**base_vars} for k, v in override_vars.items(): @@ -74,7 +107,7 @@ def merge_vars(base_vars, override_vars): def get_vars_from_files(files: Sequence[str]): """Takes a list of variable files - in `root_dir` of increasing precedence, returns a merged dict + in `root_dir` of increasing precedence, returns a merged dict """ d = {} for file in files: @@ -139,11 +172,21 @@ def get_builders_for_templating( builder_vars = merge_vars(builder_vars, override_vars) # packer takes environment_vars as an array of "key=value" strings + git_sha = get_short_git_commit() if "env_vars" in builder_vars: env_vars = builder_vars["env_vars"] exit_if_type_mismatch(env_vars, dict) env_vars = [f"{k}={v}" for k, v in env_vars.items()] builder_vars["env_vars"] = env_vars + # inject monopacker builder name and monopacker git sha + git_sha = get_short_git_commit() + env_vars.append(f"MONOPACKER_BUILDER_NAME={builder}") + env_vars.append(f"MONOPACKER_GIT_SHA={git_sha}") + else: + env_vars = [ + f"MONOPACKER_BUILDER_NAME={builder}", + f"MONOPACKER_GIT_SHA={git_sha}", + ] if "template" in builder_config: builder_template = builder_config["template"] @@ -174,51 +217,56 @@ def get_builders_for_templating( ) return builders + def generate_packer_template_params(fn): "Decorate a click function with options for generate_packer_template" params = [ - click.argument( - 'builders', - nargs=-1, - type=str, - required=True), + click.argument("builders", nargs=-1, type=str, required=True), click.option( "--builders_dir", type=str, help="directory for builder configuration", - default=os.environ.get("MONOPACKER_BUILDERS_DIR", "./builders")), + default=os.environ.get("MONOPACKER_BUILDERS_DIR", "./builders"), + ), click.option( "--var_files_dir", type=str, help="directory for builder var_files", - default=os.environ.get("MONOPACKER_VARS_DIR", "./template/vars")), + default=os.environ.get("MONOPACKER_VARS_DIR", "./template/vars"), + ), click.option( "--templates_dir", type=str, help="directory for builder templates", - default=os.environ.get("MONOPACKER_TEMPLATES_DIR", "./template/builders")), + default=os.environ.get("MONOPACKER_TEMPLATES_DIR", "./template/builders"), + ), click.option( "--scripts_dir", type=str, help="directory for builder templates", - default=os.environ.get("MONOPACKER_SCRIPTS_DIR", "./scripts")), + default=os.environ.get("MONOPACKER_SCRIPTS_DIR", "./scripts"), + ), click.option( "--files_dir", type=str, help="directory for binary files used in packer provisioners", - default=os.environ.get("MONOPACKER_FILES_DIR", "./files")), + default=os.environ.get("MONOPACKER_FILES_DIR", "./files"), + ), click.option( "--secrets_file", type=str, help="file containing secrets", - default='./fake_secrets.yaml'), - ] + default="./fake_secrets.yaml", + ), + ] params.reverse() for param in params: fn = param(fn) return fn -def generate_packer_template(*, + +def generate_packer_template( + *, builders, builders_dir, var_files_dir, @@ -226,9 +274,10 @@ def generate_packer_template(*, scripts_dir, files_dir, secrets_file, - **_): - pack_secrets(secrets_file, 'secrets.tar') - pack_files(files_dir, 'files.tar') + **_, +): + pack_secrets(secrets_file, "secrets.tar") + pack_files(files_dir, "files.tar") # variables namespaced per builder variables: Dict[str, Dict[str, Any]] = {} @@ -261,51 +310,63 @@ def generate_packer_template(*, # include some setup for linux and windows builders if linux_builders: - pkr["provisioners"].append({ - "type": "file", - "source": "./files.tar", - "destination": "/tmp/", - # TODO: only - }) - pkr["provisioners"].append({ - "type": "shell", - "inline": [ - # files.tar is two levels deep (/tmp/files) - "sudo tar xvf /tmp/files.tar -C / --strip-components=1", - "rm /tmp/files.tar", - ], - # TODO: only - }) - pkr["provisioners"].append({ - 'type': 'file', - 'source': './secrets.tar', - 'destination': '/tmp/', - # TODO: only - }) - pkr["provisioners"].append({ - 'type': 'shell', - 'inline': [ - 'sudo mkdir -p /etc/taskcluster/secrets', - 'sudo tar xvf /tmp/secrets.tar -C /', - 'sudo chown root:root -R /etc/taskcluster', - 'sudo chmod 0400 -R /etc/taskcluster/secrets', - 'rm /tmp/secrets.tar', - ], - 'only': linux_builders, - }) + pkr["provisioners"].append( + { + "type": "file", + "source": "./files.tar", + "destination": "/tmp/", + # TODO: only + } + ) + pkr["provisioners"].append( + { + "type": "shell", + "inline": [ + # files.tar is two levels deep (/tmp/files) + "sudo tar xvf /tmp/files.tar -C / --strip-components=1", + "rm /tmp/files.tar", + ], + # TODO: only + } + ) + pkr["provisioners"].append( + { + "type": "file", + "source": "./secrets.tar", + "destination": "/tmp/", + # TODO: only + } + ) + pkr["provisioners"].append( + { + "type": "shell", + "inline": [ + "sudo mkdir -p /etc/taskcluster/secrets", + "sudo tar xvf /tmp/secrets.tar -C /", + "sudo chown root:root -R /etc/taskcluster", + "sudo chmod 0400 -R /etc/taskcluster/secrets", + "rm /tmp/secrets.tar", + ], + "only": linux_builders, + } + ) # chmod/chown all secret files (above only gets /etc/taskcluster) - pkr["provisioners"].append({ - 'type': 'shell', - 'inline': generate_packer_secret_chmod_shell(secrets_file), - 'only': linux_builders, - }) - pkr["provisioners"].append({ - 'type': 'shell', - 'inline': [ - '/usr/bin/cloud-init status --wait', - ], - 'only': linux_builders, - }) + pkr["provisioners"].append( + { + "type": "shell", + "inline": generate_packer_secret_chmod_shell(secrets_file), + "only": linux_builders, + } + ) + pkr["provisioners"].append( + { + "type": "shell", + "inline": [ + "/usr/bin/cloud-init status --wait", + ], + "only": linux_builders, + } + ) e = Environment(loader=FileSystemLoader([templates_dir])) e.filters["clean_gcp_image_name"] = clean_gcp_image_name @@ -344,7 +405,9 @@ def generate_packer_template(*, sys.exit(1) if type(template_builders) != list: - print(f"Template {template_file} generated YAML that is not an array:\n{output}\n") + print( + f"Template {template_file} generated YAML that is not an array:\n{output}\n" + ) print(f"Packer template variables:\n{variables}\n") sys.exit(1) @@ -358,30 +421,125 @@ def generate_packer_template(*, # detect if previous script was a reboot (via name) # - if it was, add a pause before running the next step pause_before = "0s" - if 'reboot' in previous_script: + if "reboot" in previous_script: pause_before = "10s" - pkr["provisioners"].append({ - 'type': 'shell', - 'scripts': script, - 'pause_before': pause_before, - 'environment_vars': builder["vars"]["env_vars"] if "env_vars" in builder["vars"] else None, - 'execute_command': builder["vars"]["execute_command"] if "execute_command" in builder["vars"] else None, - 'expect_disconnect': True, - 'start_retry_timeout': builder["vars"]["ssh_timeout"] if "ssh_timeout" in builder["vars"] else None, - 'only': [builder["vars"]["name"]] if builder["platform"] == "linux" else [], - }) + pkr["provisioners"].append( + { + "type": "shell", + "scripts": script, + "pause_before": pause_before, + "environment_vars": ( + builder["vars"]["env_vars"] + if "env_vars" in builder["vars"] + else None + ), + "execute_command": ( + builder["vars"]["execute_command"] + if "execute_command" in builder["vars"] + else None + ), + "expect_disconnect": True, + "start_retry_timeout": ( + builder["vars"]["ssh_timeout"] + if "ssh_timeout" in builder["vars"] + else None + ), + "only": ( + [builder["vars"]["name"]] + if builder["platform"] == "linux" + else [] + ), + } + ) previous_script = script if windows_builders: - pkr["provisioners"].append({ - 'type': 'powershell', - 'scripts': builder["scripts"], - 'only': [builder["vars"]["name"]] if builder["platform"] == "windows" else [], - }) + pkr["provisioners"].append( + { + "type": "powershell", + "scripts": builder["scripts"], + "only": ( + [builder["vars"]["name"]] + if builder["platform"] == "windows" + else [] + ), + } + ) # ensure we output the expected artifacts.. - pkr["post-processors"] = [ - {'type': 'manifest', 'output': 'packer-artifacts.json', 'strip_path': True}, + pkr["post-processors"] = [ + {"type": "manifest", "output": "packer-artifacts.json", "strip_path": True}, ] + # if env has monopacker_generate_sboms=true, generate SBOMs + if 'monopacker_generate_sbom' in builder["vars"]: + if builder["vars"]['monopacker_generate_sbom']: + remote_temp_path_for_sbom_tool = "/tmp/monopacker_sbom_script" + # TODO: allow configuring where the SBOM is stored on the remote host + + # see if optional params are present + sbom_tool_args = "" + if 'monopacker_sbom_command_args' in builder["vars"]: + sbom_tool_args = builder["vars"]['monopacker_sbom_command_args'] + sbom_tool = "monopacker_ubuntu_sbom.py" + if 'monoopacker_sbom_script' in builder["vars"]: + sbom_tool = builder["vars"]['monopacker_sbom_script'] + + # build path relative to module's root based on + module_root_dir = Path(__file__).parent + full_path_to_sbom_tool = module_root_dir / 'utils' / sbom_tool + + # comments not working, wait for HCL migration + # pkr["provisioners"].append( + # { + # "//": "SBOM generation process: start", + # } + # ) + # copy script over to temp path + pkr["provisioners"].append( + { + "type": "file", + "direction": "upload", + "source": str(full_path_to_sbom_tool), + "destination": remote_temp_path_for_sbom_tool, + } + ) + # chmod the script, run the sbom tool, and remove script from temp path + pkr["provisioners"].append( + { + "type": "shell", + "inline": [ + f"chmod +x {remote_temp_path_for_sbom_tool}", + "cd /etc", + f"sudo {remote_temp_path_for_sbom_tool} {sbom_tool_args}", + f"rm {remote_temp_path_for_sbom_tool}", + ], + "environment_vars": builder['vars']['env_vars'], + # TODO: add only? + } + ) + # copy SBOM back to localhost + pkr["provisioners"].append( + { + "type": "file", + "direction": "download", + "source": "/etc/SBOM.md", + # will be copied to SBOMs/image_name.md in post-processor + "destination": "SBOMs/temp_sbom.md", + } + ) + # add post-processor that renames the SBOM to the artifact name + pkr["post-processors"].append( + { + "type": "shell-local", + "script": "monopacker/post-processors/move_sbom_to_latest_artifact_name.py", + # TODO: add 'only'? + } + ) + # pkr["provisioners"].append( + # { + # "//": "SBOM generation process: finish", + # } + # ) + return pkr diff --git a/monopacker/utils/monopacker_ubuntu_sbom.py b/monopacker/utils/monopacker_ubuntu_sbom.py new file mode 100755 index 0000000..656ca7d --- /dev/null +++ b/monopacker/utils/monopacker_ubuntu_sbom.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 + +import subprocess +import platform +import argparse +import os +from datetime import datetime +from zoneinfo import ZoneInfo + +VERSION = "1.0.1" + +def get_system_info(): + info = { + 'OS': platform.system(), + 'OS Version': platform.version(), + 'Architecture': platform.machine(), + 'Processor': platform.processor(), + 'Python Version': platform.python_version(), + 'Kernel Version': subprocess.run(['uname', '-r'], stdout=subprocess.PIPE, text=True).stdout.strip() + } + return info + +def get_lsb_release_info(): + lsb_info = {} + try: + with open('/etc/lsb-release', 'r') as file: + lsb_data = file.readlines() + for line in lsb_data: + key, value = line.strip().split('=') + lsb_info[key] = value + except FileNotFoundError: + lsb_info['LSB Release'] = 'Not available' + return lsb_info + +def get_os_release_info(): + os_info = {} + try: + with open('/etc/os-release', 'r') as file: + os_data = file.readlines() + for line in os_data: + key, value = line.strip().split('=') + os_info[key] = value.strip('"') + except FileNotFoundError: + os_info['OS Release'] = 'Not available' + return os_info + +def get_taskcluster_info(): + taskcluster_info = {} + commands = { + 'generic-worker': ['generic-worker', '--version'], + 'taskcluster-proxy': ['taskcluster-proxy', '--version'], + 'livelog': ['livelog', '--version'], + 'start-worker': ['start-worker', '--version'] + } + + for key, command in commands.items(): + try: + result = subprocess.run(command, stdout=subprocess.PIPE, text=True, timeout=1) + taskcluster_info[key] = result.stdout.strip() + except (FileNotFoundError, subprocess.TimeoutExpired): + taskcluster_info[key] = 'Not available' + + return taskcluster_info + +def get_installed_packages(): + result = subprocess.run(['dpkg', '-l'], stdout=subprocess.PIPE, text=True) + packages = [] + for line in result.stdout.split('\n')[5:]: + if line: + parts = line.split() + packages.append({ + 'Name': parts[1], + 'Version': parts[2], + 'Architecture': parts[3], + }) + return packages + +def get_pip_packages(): + result = subprocess.run(['pip3', 'list'], stdout=subprocess.PIPE, text=True) + packages = [] + for line in result.stdout.split('\n')[2:]: + if line: + parts = line.split() + name = parts[0] + version = parts[1] + packages.append({ + 'Name': name, + 'Version': version + }) + return packages + +def generate_markdown(sbom): + md = [] + md.append(f"# Software Bill of Materials (SBOM)") + + if sbom.get('monopacker_builder_name'): + md.append(f"- **Monopacker Builder Name**: {sbom['monopacker_builder_name']}") + if sbom.get('monopacker_commit'): + commit_url = f"https://github.com/search?q=repo%3Amozilla-platform-ops%2Fmonopacker+{sbom['monopacker_commit']}&type=code" + md.append(f"- **Monopacker Commit**: [{sbom['monopacker_commit']}]({commit_url})") + md.append(f"- **SBOM Generated By**: {sbom['generated_by']} {VERSION}") + md.append(f"- **SBOM Generation Datetime**: {sbom['generated_on']} {sbom['timezone']}") + + md.append(f"\n## System Information") + md.append(f"Information obtained using Python's `platform` module and `uname -r` command.") + for key, value in sbom['system_info'].items(): + md.append(f"- **{key}**: {value}") + + md.append(f"\n## LSB Release Information") + md.append(f"Information obtained from `/etc/lsb-release` file.") + for key, value in sbom['lsb_release_info'].items(): + md.append(f"- **{key}**: {value}") + + md.append(f"\n## OS Release Information") + md.append(f"Information obtained from `/etc/os-release` file.") + for key, value in sbom['os_release_info'].items(): + md.append(f"- **{key}**: {value}") + + if sbom.get('additional_info'): + md.append(f"\n## Additional Information") + for key, value in sbom['additional_info'].items(): + md.append(f"- **{key}**: {value}") + + md.append(f"\n## Taskcluster Information") + md.append(f"Information obtained using the respective tool's `--version` command with a 1-second timeout.") + md.append(f"| Tool | Version |") + md.append(f"|-----------------|-------------|") + for key, value in sbom['taskcluster_info'].items(): + md.append(f"| {key} | {value} |") + + md.append(f"\n## Installed Packages") + md.append(f"Information obtained using `dpkg -l` command.") + md.append(f"| Name | Version | Architecture |") + md.append(f"|------|---------|--------------|") + for pkg in sbom['packages']: + md.append(f"| {pkg['Name']} | {pkg['Version']} | {pkg['Architecture']} |") + + md.append(f"\n## Python Packages") + md.append(f"Information obtained using `pip3 list` command.") + md.append(f"| Name | Version |") + md.append(f"|------|---------|") + for pkg in sbom['pip_packages']: + md.append(f"| {pkg['Name']} | {pkg['Version']} |") + + return '\n'.join(md) + +def main(): + parser = argparse.ArgumentParser(description="Generate a markdown-formatted SBOM for a Ubuntu system.") + parser.add_argument('-b', '--monopacker_builder_name', type=str, required=True, help='Name of the monopacker builder') + parser.add_argument('-c', '--monopacker_commit', type=str, required=True, help='Commit of the monopacker') + parser.add_argument('-a', '--additional-key-value', action='append', help='Additional key-value pairs to add to system info in the format key,value') + + args = parser.parse_args() + + system_info = get_system_info() + lsb_release_info = get_lsb_release_info() + os_release_info = get_os_release_info() + taskcluster_info = get_taskcluster_info() + + additional_info = {} + if args.additional_key_value: + for item in args.additional_key_value: + key, value = item.split(',', 1) + additional_info[key.strip()] = value.strip() + + packages = get_installed_packages() + pip_packages = get_pip_packages() + + now = datetime.now(ZoneInfo("UTC")) + timezone = now.astimezone().strftime('%Z %z') + sbom = { + 'system_info': system_info, + 'lsb_release_info': lsb_release_info, + 'os_release_info': os_release_info, + 'taskcluster_info': taskcluster_info, + 'additional_info': additional_info, + 'packages': packages, + 'pip_packages': pip_packages, + 'monopacker_builder_name': args.monopacker_builder_name, + 'monopacker_commit': args.monopacker_commit, + 'generated_on': now.astimezone().strftime("%Y-%m-%d %H:%M:%S"), + 'timezone': timezone, + 'generated_by': os.path.basename(__file__) + } + + markdown_content = generate_markdown(sbom) + + with open('SBOM.md', 'w') as f: + f.write(markdown_content) + + print("SBOM has been generated and saved to SBOM.md") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 2c0cfb8..18cc475 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "click" @@ -23,38 +23,95 @@ files = [ ] [[package]] -name = "exceptiongroup" -version = "1.2.0" -description = "Backport of PEP 654 (exception groups)" +name = "coverage" +version = "7.6.0" +description = "Code coverage measurement for Python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, - {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, + {file = "coverage-7.6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dff044f661f59dace805eedb4a7404c573b6ff0cdba4a524141bc63d7be5c7fd"}, + {file = "coverage-7.6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a8659fd33ee9e6ca03950cfdcdf271d645cf681609153f218826dd9805ab585c"}, + {file = "coverage-7.6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7792f0ab20df8071d669d929c75c97fecfa6bcab82c10ee4adb91c7a54055463"}, + {file = "coverage-7.6.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d4b3cd1ca7cd73d229487fa5caca9e4bc1f0bca96526b922d61053ea751fe791"}, + {file = "coverage-7.6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7e128f85c0b419907d1f38e616c4f1e9f1d1b37a7949f44df9a73d5da5cd53c"}, + {file = "coverage-7.6.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a94925102c89247530ae1dab7dc02c690942566f22e189cbd53579b0693c0783"}, + {file = "coverage-7.6.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:dcd070b5b585b50e6617e8972f3fbbee786afca71b1936ac06257f7e178f00f6"}, + {file = "coverage-7.6.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d50a252b23b9b4dfeefc1f663c568a221092cbaded20a05a11665d0dbec9b8fb"}, + {file = "coverage-7.6.0-cp310-cp310-win32.whl", hash = "sha256:0e7b27d04131c46e6894f23a4ae186a6a2207209a05df5b6ad4caee6d54a222c"}, + {file = "coverage-7.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:54dece71673b3187c86226c3ca793c5f891f9fc3d8aa183f2e3653da18566169"}, + {file = "coverage-7.6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c7b525ab52ce18c57ae232ba6f7010297a87ced82a2383b1afd238849c1ff933"}, + {file = "coverage-7.6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bea27c4269234e06f621f3fac3925f56ff34bc14521484b8f66a580aacc2e7d"}, + {file = "coverage-7.6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed8d1d1821ba5fc88d4a4f45387b65de52382fa3ef1f0115a4f7a20cdfab0e94"}, + {file = "coverage-7.6.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:01c322ef2bbe15057bc4bf132b525b7e3f7206f071799eb8aa6ad1940bcf5fb1"}, + {file = "coverage-7.6.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:03cafe82c1b32b770a29fd6de923625ccac3185a54a5e66606da26d105f37dac"}, + {file = "coverage-7.6.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0d1b923fc4a40c5832be4f35a5dab0e5ff89cddf83bb4174499e02ea089daf57"}, + {file = "coverage-7.6.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:4b03741e70fb811d1a9a1d75355cf391f274ed85847f4b78e35459899f57af4d"}, + {file = "coverage-7.6.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a73d18625f6a8a1cbb11eadc1d03929f9510f4131879288e3f7922097a429f63"}, + {file = "coverage-7.6.0-cp311-cp311-win32.whl", hash = "sha256:65fa405b837060db569a61ec368b74688f429b32fa47a8929a7a2f9b47183713"}, + {file = "coverage-7.6.0-cp311-cp311-win_amd64.whl", hash = "sha256:6379688fb4cfa921ae349c76eb1a9ab26b65f32b03d46bb0eed841fd4cb6afb1"}, + {file = "coverage-7.6.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f7db0b6ae1f96ae41afe626095149ecd1b212b424626175a6633c2999eaad45b"}, + {file = "coverage-7.6.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bbdf9a72403110a3bdae77948b8011f644571311c2fb35ee15f0f10a8fc082e8"}, + {file = "coverage-7.6.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cc44bf0315268e253bf563f3560e6c004efe38f76db03a1558274a6e04bf5d5"}, + {file = "coverage-7.6.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:da8549d17489cd52f85a9829d0e1d91059359b3c54a26f28bec2c5d369524807"}, + {file = "coverage-7.6.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0086cd4fc71b7d485ac93ca4239c8f75732c2ae3ba83f6be1c9be59d9e2c6382"}, + {file = "coverage-7.6.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1fad32ee9b27350687035cb5fdf9145bc9cf0a094a9577d43e909948ebcfa27b"}, + {file = "coverage-7.6.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:044a0985a4f25b335882b0966625270a8d9db3d3409ddc49a4eb00b0ef5e8cee"}, + {file = "coverage-7.6.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:76d5f82213aa78098b9b964ea89de4617e70e0d43e97900c2778a50856dac605"}, + {file = "coverage-7.6.0-cp312-cp312-win32.whl", hash = "sha256:3c59105f8d58ce500f348c5b56163a4113a440dad6daa2294b5052a10db866da"}, + {file = "coverage-7.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:ca5d79cfdae420a1d52bf177de4bc2289c321d6c961ae321503b2ca59c17ae67"}, + {file = "coverage-7.6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d39bd10f0ae453554798b125d2f39884290c480f56e8a02ba7a6ed552005243b"}, + {file = "coverage-7.6.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:beb08e8508e53a568811016e59f3234d29c2583f6b6e28572f0954a6b4f7e03d"}, + {file = "coverage-7.6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b2e16f4cd2bc4d88ba30ca2d3bbf2f21f00f382cf4e1ce3b1ddc96c634bc48ca"}, + {file = "coverage-7.6.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6616d1c9bf1e3faea78711ee42a8b972367d82ceae233ec0ac61cc7fec09fa6b"}, + {file = "coverage-7.6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad4567d6c334c46046d1c4c20024de2a1c3abc626817ae21ae3da600f5779b44"}, + {file = "coverage-7.6.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:d17c6a415d68cfe1091d3296ba5749d3d8696e42c37fca5d4860c5bf7b729f03"}, + {file = "coverage-7.6.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:9146579352d7b5f6412735d0f203bbd8d00113a680b66565e205bc605ef81bc6"}, + {file = "coverage-7.6.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:cdab02a0a941af190df8782aafc591ef3ad08824f97850b015c8c6a8b3877b0b"}, + {file = "coverage-7.6.0-cp38-cp38-win32.whl", hash = "sha256:df423f351b162a702c053d5dddc0fc0ef9a9e27ea3f449781ace5f906b664428"}, + {file = "coverage-7.6.0-cp38-cp38-win_amd64.whl", hash = "sha256:f2501d60d7497fd55e391f423f965bbe9e650e9ffc3c627d5f0ac516026000b8"}, + {file = "coverage-7.6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7221f9ac9dad9492cecab6f676b3eaf9185141539d5c9689d13fd6b0d7de840c"}, + {file = "coverage-7.6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ddaaa91bfc4477d2871442bbf30a125e8fe6b05da8a0015507bfbf4718228ab2"}, + {file = "coverage-7.6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4cbe651f3904e28f3a55d6f371203049034b4ddbce65a54527a3f189ca3b390"}, + {file = "coverage-7.6.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:831b476d79408ab6ccfadaaf199906c833f02fdb32c9ab907b1d4aa0713cfa3b"}, + {file = "coverage-7.6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46c3d091059ad0b9c59d1034de74a7f36dcfa7f6d3bde782c49deb42438f2450"}, + {file = "coverage-7.6.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:4d5fae0a22dc86259dee66f2cc6c1d3e490c4a1214d7daa2a93d07491c5c04b6"}, + {file = "coverage-7.6.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:07ed352205574aad067482e53dd606926afebcb5590653121063fbf4e2175166"}, + {file = "coverage-7.6.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:49c76cdfa13015c4560702574bad67f0e15ca5a2872c6a125f6327ead2b731dd"}, + {file = "coverage-7.6.0-cp39-cp39-win32.whl", hash = "sha256:482855914928c8175735a2a59c8dc5806cf7d8f032e4820d52e845d1f731dca2"}, + {file = "coverage-7.6.0-cp39-cp39-win_amd64.whl", hash = "sha256:543ef9179bc55edfd895154a51792b01c017c87af0ebaae092720152e19e42ca"}, + {file = "coverage-7.6.0-pp38.pp39.pp310-none-any.whl", hash = "sha256:6fe885135c8a479d3e37a7aae61cbd3a0fb2deccb4dda3c25f92a49189f766d6"}, + {file = "coverage-7.6.0.tar.gz", hash = "sha256:289cc803fa1dc901f84701ac10c9ee873619320f2f9aff38794db4a4a0268d51"}, ] +[package.dependencies] +tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} + [package.extras] -test = ["pytest (>=6)"] +toml = ["tomli"] [[package]] -name = "importlib-metadata" -version = "6.7.0" -description = "Read metadata from Python packages" +name = "docopt" +version = "0.6.2" +description = "Pythonic argument parser, that will make you smile" optional = false -python-versions = ">=3.7" +python-versions = "*" files = [ - {file = "importlib_metadata-6.7.0-py3-none-any.whl", hash = "sha256:cb52082e659e97afc5dac71e79de97d8681de3aa07ff18578330904a9d18e5b5"}, - {file = "importlib_metadata-6.7.0.tar.gz", hash = "sha256:1aaf550d4f73e5d6783e7acb77aec43d49da8017410afae93822cc9cca98c4d4"}, + {file = "docopt-0.6.2.tar.gz", hash = "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"}, ] -[package.dependencies] -typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} -zipp = ">=0.5" +[[package]] +name = "exceptiongroup" +version = "1.2.2" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"}, + {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"}, +] [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -perf = ["ipython"] -testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)", "pytest-ruff"] +test = ["pytest (>=6)"] [[package]] name = "iniconfig" @@ -164,42 +221,39 @@ files = [ [[package]] name = "packaging" -version = "23.2" +version = "24.1" description = "Core utilities for Python packages" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, - {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, + {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, + {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, ] [[package]] name = "pluggy" -version = "1.2.0" +version = "1.5.0" description = "plugin and hook calling mechanisms for python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pluggy-1.2.0-py3-none-any.whl", hash = "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849"}, - {file = "pluggy-1.2.0.tar.gz", hash = "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"}, + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, ] -[package.dependencies] -importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} - [package.extras] dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] [[package]] name = "pyfakefs" -version = "5.3.4" +version = "5.6.0" description = "pyfakefs implements a fake file system that mocks the Python file system modules." optional = false python-versions = ">=3.7" files = [ - {file = "pyfakefs-5.3.4-py3-none-any.whl", hash = "sha256:fc375229f5417f197f0892a7d6dc49a411e67e10eb8142b19d80e60a9d52a13d"}, - {file = "pyfakefs-5.3.4.tar.gz", hash = "sha256:dadac1653195a4bfe4c26e9dfa7cc0c0286b1cd8e18706442c2464cae5542a17"}, + {file = "pyfakefs-5.6.0-py3-none-any.whl", hash = "sha256:1a45bba8615323ec29d65929d32dc66d7b59a1e60a02109950440edb0486c539"}, + {file = "pyfakefs-5.6.0.tar.gz", hash = "sha256:7a549b32865aa97d8ba6538285a93816941d9b7359be2954ac60ec36b277e879"}, ] [[package]] @@ -216,7 +270,6 @@ files = [ [package.dependencies] colorama = {version = "*", markers = "sys_platform == \"win32\""} exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} -importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} iniconfig = "*" packaging = "*" pluggy = ">=0.12,<2.0" @@ -225,6 +278,40 @@ tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +[[package]] +name = "pytest-cov" +version = "5.0.0" +description = "Pytest plugin for measuring coverage." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-cov-5.0.0.tar.gz", hash = "sha256:5837b58e9f6ebd335b0f8060eecce69b662415b16dc503883a02f45dfeb14857"}, + {file = "pytest_cov-5.0.0-py3-none-any.whl", hash = "sha256:4f0764a1219df53214206bf1feea4633c3b558a2925c8b59f144f682861ce652"}, +] + +[package.dependencies] +coverage = {version = ">=5.2.1", extras = ["toml"]} +pytest = ">=4.6" + +[package.extras] +testing = ["fields", "hunter", "process-tests", "pytest-xdist", "virtualenv"] + +[[package]] +name = "pytest-watch" +version = "4.2.0" +description = "Local continuous test runner with pytest and watchdog." +optional = false +python-versions = "*" +files = [ + {file = "pytest-watch-4.2.0.tar.gz", hash = "sha256:06136f03d5b361718b8d0d234042f7b2f203910d8568f63df2f866b547b3d4b9"}, +] + +[package.dependencies] +colorama = ">=0.3.3" +docopt = ">=0.4.0" +pytest = ">=2.6.4" +watchdog = ">=0.6.0" + [[package]] name = "ruamel-yaml" version = "0.16.13" @@ -314,32 +401,50 @@ files = [ ] [[package]] -name = "typing-extensions" -version = "4.7.1" -description = "Backported and Experimental Type Hints for Python 3.7+" +name = "watchdog" +version = "4.0.1" +description = "Filesystem events monitoring" optional = false -python-versions = ">=3.7" -files = [ - {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, - {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, -] - -[[package]] -name = "zipp" -version = "3.15.0" -description = "Backport of pathlib-compatible object wrapper for zip files" -optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "zipp-3.15.0-py3-none-any.whl", hash = "sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556"}, - {file = "zipp-3.15.0.tar.gz", hash = "sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b"}, + {file = "watchdog-4.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:da2dfdaa8006eb6a71051795856bedd97e5b03e57da96f98e375682c48850645"}, + {file = "watchdog-4.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e93f451f2dfa433d97765ca2634628b789b49ba8b504fdde5837cdcf25fdb53b"}, + {file = "watchdog-4.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ef0107bbb6a55f5be727cfc2ef945d5676b97bffb8425650dadbb184be9f9a2b"}, + {file = "watchdog-4.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:17e32f147d8bf9657e0922c0940bcde863b894cd871dbb694beb6704cfbd2fb5"}, + {file = "watchdog-4.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:03e70d2df2258fb6cb0e95bbdbe06c16e608af94a3ffbd2b90c3f1e83eb10767"}, + {file = "watchdog-4.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:123587af84260c991dc5f62a6e7ef3d1c57dfddc99faacee508c71d287248459"}, + {file = "watchdog-4.0.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:093b23e6906a8b97051191a4a0c73a77ecc958121d42346274c6af6520dec175"}, + {file = "watchdog-4.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:611be3904f9843f0529c35a3ff3fd617449463cb4b73b1633950b3d97fa4bfb7"}, + {file = "watchdog-4.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:62c613ad689ddcb11707f030e722fa929f322ef7e4f18f5335d2b73c61a85c28"}, + {file = "watchdog-4.0.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:d4925e4bf7b9bddd1c3de13c9b8a2cdb89a468f640e66fbfabaf735bd85b3e35"}, + {file = "watchdog-4.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cad0bbd66cd59fc474b4a4376bc5ac3fc698723510cbb64091c2a793b18654db"}, + {file = "watchdog-4.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a3c2c317a8fb53e5b3d25790553796105501a235343f5d2bf23bb8649c2c8709"}, + {file = "watchdog-4.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c9904904b6564d4ee8a1ed820db76185a3c96e05560c776c79a6ce5ab71888ba"}, + {file = "watchdog-4.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:667f3c579e813fcbad1b784db7a1aaa96524bed53437e119f6a2f5de4db04235"}, + {file = "watchdog-4.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d10a681c9a1d5a77e75c48a3b8e1a9f2ae2928eda463e8d33660437705659682"}, + {file = "watchdog-4.0.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0144c0ea9997b92615af1d94afc0c217e07ce2c14912c7b1a5731776329fcfc7"}, + {file = "watchdog-4.0.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:998d2be6976a0ee3a81fb8e2777900c28641fb5bfbd0c84717d89bca0addcdc5"}, + {file = "watchdog-4.0.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:e7921319fe4430b11278d924ef66d4daa469fafb1da679a2e48c935fa27af193"}, + {file = "watchdog-4.0.1-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:f0de0f284248ab40188f23380b03b59126d1479cd59940f2a34f8852db710625"}, + {file = "watchdog-4.0.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:bca36be5707e81b9e6ce3208d92d95540d4ca244c006b61511753583c81c70dd"}, + {file = "watchdog-4.0.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:ab998f567ebdf6b1da7dc1e5accfaa7c6992244629c0fdaef062f43249bd8dee"}, + {file = "watchdog-4.0.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:dddba7ca1c807045323b6af4ff80f5ddc4d654c8bce8317dde1bd96b128ed253"}, + {file = "watchdog-4.0.1-py3-none-manylinux2014_armv7l.whl", hash = "sha256:4513ec234c68b14d4161440e07f995f231be21a09329051e67a2118a7a612d2d"}, + {file = "watchdog-4.0.1-py3-none-manylinux2014_i686.whl", hash = "sha256:4107ac5ab936a63952dea2a46a734a23230aa2f6f9db1291bf171dac3ebd53c6"}, + {file = "watchdog-4.0.1-py3-none-manylinux2014_ppc64.whl", hash = "sha256:6e8c70d2cd745daec2a08734d9f63092b793ad97612470a0ee4cbb8f5f705c57"}, + {file = "watchdog-4.0.1-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:f27279d060e2ab24c0aa98363ff906d2386aa6c4dc2f1a374655d4e02a6c5e5e"}, + {file = "watchdog-4.0.1-py3-none-manylinux2014_s390x.whl", hash = "sha256:f8affdf3c0f0466e69f5b3917cdd042f89c8c63aebdb9f7c078996f607cdb0f5"}, + {file = "watchdog-4.0.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:ac7041b385f04c047fcc2951dc001671dee1b7e0615cde772e84b01fbf68ee84"}, + {file = "watchdog-4.0.1-py3-none-win32.whl", hash = "sha256:206afc3d964f9a233e6ad34618ec60b9837d0582b500b63687e34011e15bb429"}, + {file = "watchdog-4.0.1-py3-none-win_amd64.whl", hash = "sha256:7577b3c43e5909623149f76b099ac49a1a01ca4e167d1785c76eb52fa585745a"}, + {file = "watchdog-4.0.1-py3-none-win_ia64.whl", hash = "sha256:d7b9f5f3299e8dd230880b6c55504a1f69cf1e4316275d1b215ebdd8187ec88d"}, + {file = "watchdog-4.0.1.tar.gz", hash = "sha256:eebaacf674fa25511e8867028d281e602ee6500045b57f43b08778082f7f8b44"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +watchmedo = ["PyYAML (>=3.10)"] [metadata] lock-version = "2.0" -python-versions = "^3.7" -content-hash = "c439627d7bef8ac718a3b19f79ed173eda9661ec1c77d5c4e2fe95ad1a0a45ad" +python-versions = ">=3.8,<4.0" +content-hash = "35801562dcc08f231e953980cbbd50dcccd42faa91697c963840aea01c2b731f" diff --git a/pyproject.toml b/pyproject.toml index b3e3bd5..1f94dbd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ readme = "README.md" keywords = ["packer", "templating"] [tool.poetry.dependencies] -python = "^3.7" +python = ">=3.8,<4.0" ruamel-yaml = "^0.16.10" jinja2 = "^2.11.3" click = "^7.0" @@ -19,6 +19,8 @@ markupsafe = "2.0.1" [tool.poetry.group.dev.dependencies] pytest = "^7.2.2" pyfakefs = "^5.1.0" +pytest-watch = "^4.2.0" +pytest-cov = "^5.0.0" [build-system] requires = ["poetry-core"] diff --git a/scripts/ubuntu-tc-barebones/01-install-packages.sh b/scripts/ubuntu-tc-barebones/01-install-packages.sh new file mode 100644 index 0000000..14b500c --- /dev/null +++ b/scripts/ubuntu-tc-barebones/01-install-packages.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +set -exv + +# init helpers +helpers_dir=${MONOPACKER_HELPERS_DIR:-"/etc/monopacker/scripts"} +for h in ${helpers_dir}/*.sh; do + . $h; +done + +retry apt-get update +retry apt-get upgrade -y + +# docker wants these +retry apt-get install -y \ + apt-transport-https \ + build-essential \ + ca-certificates \ + curl \ + gnupg-agent \ + python3 \ + python3-pip \ + software-properties-common \ + vim + +MISC_PACKAGES=() +MISC_PACKAGES+=(zstd python3-pip jq) +# docker-worker needs this for unpacking lz4 images +MISC_PACKAGES+=(liblz4-tool) + +# misc +retry apt-get install -y ${MISC_PACKAGES[@]} + +# Remove apport because it prevents obtaining crashes from containers +# and because it may send data to Canonical. +apt-get purge -y apport diff --git a/scripts/ubuntu-tc-barebones/05-install-tc.sh b/scripts/ubuntu-tc-barebones/05-install-tc.sh new file mode 100644 index 0000000..4858e38 --- /dev/null +++ b/scripts/ubuntu-tc-barebones/05-install-tc.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -exv + +# init helpers +helpers_dir=${MONOPACKER_HELPERS_DIR:-"/etc/monopacker/scripts"} +for h in ${helpers_dir}/*.sh; do + . $h; +done + +# + +# define this here (vs in a env file) +export TASKCLUSTER_VERSION=67.1.0 + +# TODO: automate fetching the latest version +# curl -L -s https://api.github.com/repos/taskcluster/taskcluster/releases/latest | jq -r '.tag_name' | sed 's/^v//' + +# TODO: test that TASKCLUSTER_VERSION is defined or exit 1 + +cd /usr/local/bin +retry curl -fsSL "https://github.com/taskcluster/taskcluster/releases/download/v${TASKCLUSTER_VERSION}/generic-worker-multiuser-linux-${TC_ARCH}" > generic-worker +retry curl -fsSL "https://github.com/taskcluster/taskcluster/releases/download/v${TASKCLUSTER_VERSION}/start-worker-linux-${TC_ARCH}" > start-worker +retry curl -fsSL "https://github.com/taskcluster/taskcluster/releases/download/v${TASKCLUSTER_VERSION}/livelog-linux-${TC_ARCH}" > livelog +retry curl -fsSL "https://github.com/taskcluster/taskcluster/releases/download/v${TASKCLUSTER_VERSION}/taskcluster-proxy-linux-${TC_ARCH}" > taskcluster-proxy +chmod a+x generic-worker start-worker taskcluster-proxy livelog \ No newline at end of file diff --git a/scripts/ubuntu-tc-barebones/99-clean.sh b/scripts/ubuntu-tc-barebones/99-clean.sh new file mode 100644 index 0000000..ac3f9b0 --- /dev/null +++ b/scripts/ubuntu-tc-barebones/99-clean.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -exv + +# init helpers +helpers_dir=${MONOPACKER_HELPERS_DIR:-"/etc/monopacker/scripts"} +for h in ${helpers_dir}/*.sh; do + . $h; +done + +rm -rf /usr/src/* + +# Do one final package cleanup, just in case. +apt-get autoremove -y --purge diff --git a/template/vars/monopacker_generate_sbom.yaml b/template/vars/monopacker_generate_sbom.yaml new file mode 100644 index 0000000..924c494 --- /dev/null +++ b/template/vars/monopacker_generate_sbom.yaml @@ -0,0 +1,6 @@ +--- +monopacker_generate_sbom: true +# defaults to "" +monopacker_sbom_command_args: "-b $MONOPACKER_BUILDER_NAME -c $MONOPACKER_GIT_SHA" +# default vaule is "monopacker_ubuntu_sbom.py" +# monoopacker_sbom_script: monopacker_ubuntu_sbom.py \ No newline at end of file diff --git a/tests/test_template_packer.py b/tests/test_template_packer.py index fa1f29e..b399c08 100644 --- a/tests/test_template_packer.py +++ b/tests/test_template_packer.py @@ -201,6 +201,14 @@ def test_generate_packer_template(tmpdir): secrets_file=str(secrets_file), ) + # scan all 'evnvironment_vars' and checks for MONOPACKER_GIT_SHA and MONOPACKER_BUILDER_NAME + # and rewrite them to be deadbeef and linux respectively + for provisioner in packer_template['provisioners']: + if 'environment_vars' in provisioner and provisioner['environment_vars']: + for index, kv_pair in enumerate(provisioner['environment_vars']): + if kv_pair.startswith('MONOPACKER_GIT_SHA'): + provisioner['environment_vars'][index] = 'MONOPACKER_GIT_SHA=deadbeef' + assert(packer_template == { 'builders': [ { @@ -258,7 +266,8 @@ def test_generate_packer_template(tmpdir): { 'type': 'shell', 'scripts': str(scripts_dir.join("facebook-worker", "01-fb.sh")), - 'environment_vars': ["AN_ENV_VAR=env!"], + 'environment_vars': ["AN_ENV_VAR=env!",'MONOPACKER_BUILDER_NAME=linux', +'MONOPACKER_GIT_SHA=deadbeef'], 'execute_command': "do-it", 'expect_disconnect': True, 'start_retry_timeout': '30m', @@ -291,3 +300,166 @@ def test_generate_packer_template(tmpdir): ], }) + + + +def test_generate_packer_template_with_sboms(tmpdir): + builders_dir = tmpdir.mkdir("builders") + var_files_dir = tmpdir.mkdir("var_files") + templates_dir = tmpdir.mkdir("templates") + scripts_dir = tmpdir.mkdir("scripts") + files_dir = tmpdir.mkdir("files") + secrets_file = tmpdir.join("secrets.yml") + + builders_dir.join("linux.yaml").write(json.dumps({ + "template": "alibaba_linux", + "platform": "linux", + "builder_var_files": ["bv", "env"], + "script_directories": ["facebook-worker"], + "builder_vars": { + "execute_command": "do-it", + "ssh_timeout": "30m", + }, + })) + + templates_dir.join("alibaba_linux.jinja2").write(textwrap.dedent("""\ + - name: a packer builder + type: alibaba + a-is: {{builder.vars.a}} + """)) + + templates_dir.join("openstack_windows.jinja2").write(textwrap.dedent("""\ + - name: a packer builder + type: openstack + """)) + + # TODO: add a fake secret json... missing test coverage + # secrets_file.write(json.dumps([])) + secrets_file.write(json.dumps([{'name': 'blah_key', 'path': '/etc/taskcluster/secrets/test_blah', 'value': 'test123'}])) + + scripts_dir.mkdir("facebook-worker").join("01-fb.sh").write("echo hello") + + scripts_dir.mkdir("win-worker").join("01-win.ps1").write("ECHO hello") + + var_files_dir.join("bv.yaml").write(json.dumps({ + "a": 10, + "b": 20, + "monopacker_generate_sbom": True, + })) + + var_files_dir.join("env.yaml").write(json.dumps({ + "env_vars": { + "AN_ENV_VAR": 'env!', + }, + })) + + packer_template = generate_packer_template( + builders=["linux"], + builders_dir=str(builders_dir), + var_files_dir=str(var_files_dir), + templates_dir=str(templates_dir), + scripts_dir=str(scripts_dir), + files_dir=str(files_dir), + secrets_file=str(secrets_file), + ) + + # scan all 'evnvironment_vars' and checks for MONOPACKER_GIT_SHA and MONOPACKER_BUILDER_NAME + # and rewrite them to be deadbeef and linux respectively + for provisioner in packer_template['provisioners']: + if 'environment_vars' in provisioner and provisioner['environment_vars']: + for index, kv_pair in enumerate(provisioner['environment_vars']): + if kv_pair.startswith('MONOPACKER_GIT_SHA'): + provisioner['environment_vars'][index] = 'MONOPACKER_GIT_SHA=deadbeef' + + # scan all provisioners, find any that have type 'file' and destination '/tmp/monopacker_sbom_script' + # and rewrite that item's 'source' to be 'monopacker/utils/monopacker_ubuntu_sbom.py' + for index, provisioner in enumerate(packer_template['provisioners']): + if provisioner['type'] == 'file' and provisioner['destination'] == '/tmp/monopacker_sbom_script': + packer_template['provisioners'][index]['source'] = 'monopacker/utils/monopacker_ubuntu_sbom.py' + + assert(packer_template == { + 'builders': [ + { + 'name': 'a packer builder', + 'type': 'alibaba', + 'a-is': 10, + }, + ], + 'provisioners': [ + { + 'type': 'file', + 'source': './files.tar', + 'destination': '/tmp/', + }, + { + 'type': 'shell', + 'inline': [ + 'sudo tar xvf /tmp/files.tar -C / --strip-components=1', + 'rm /tmp/files.tar', + ], + }, + { + 'type': 'file', + 'source': './secrets.tar', + 'destination': '/tmp/', + }, + { + 'type': 'shell', + 'inline': [ + 'sudo mkdir -p /etc/taskcluster/secrets', + 'sudo tar xvf /tmp/secrets.tar -C /', + 'sudo chown root:root -R /etc/taskcluster', + 'sudo chmod 0400 -R /etc/taskcluster/secrets', + 'rm /tmp/secrets.tar', + ], + 'only': ['linux'], + }, + {'inline': ['sudo chown -R root:root ' + '/etc/taskcluster/secrets', + 'sudo chmod -R 0400 /etc/taskcluster/secrets'], + 'only': ['linux'], + 'type': 'shell', + }, + { + 'type': 'shell', + 'inline': [ + '/usr/bin/cloud-init status --wait', + ], + 'only': ['linux'], + }, + { + 'type': 'shell', + 'scripts': str(scripts_dir.join("facebook-worker", "01-fb.sh")), + 'environment_vars': ["AN_ENV_VAR=env!",'MONOPACKER_BUILDER_NAME=linux', +'MONOPACKER_GIT_SHA=deadbeef'], + 'execute_command': "do-it", + 'expect_disconnect': True, + 'start_retry_timeout': '30m', + 'only': ['linux'], + 'pause_before': '0s', + }, + {'destination': '/tmp/monopacker_sbom_script', + 'direction': 'upload', + 'source': 'monopacker/utils/monopacker_ubuntu_sbom.py', + 'type': 'file'}, + {'environment_vars': ['AN_ENV_VAR=env!', + 'MONOPACKER_BUILDER_NAME=linux', + 'MONOPACKER_GIT_SHA=deadbeef'], + 'inline': ['chmod +x /tmp/monopacker_sbom_script', + 'cd /etc', + 'sudo /tmp/monopacker_sbom_script ', + 'rm /tmp/monopacker_sbom_script'], + 'type': 'shell'}, + {'destination': 'SBOMs/temp_sbom.md', + 'direction': 'download', + 'source': '/etc/SBOM.md', + 'type': 'file'}, + + ], + 'post-processors': [ + {'type': 'manifest', 'output': 'packer-artifacts.json', 'strip_path': True}, + {'script': 'monopacker/post-processors/move_sbom_to_latest_artifact_name.py', + 'type': 'shell-local'}, + ], + }) +