refactor: Consolidating common MC package installation

docs/source/system/model_component/mc_install.rst

@@ -50,14 +50,21 @@ The expected ``INSTALL`` directory structure is::
   MC_DIR
   └── INSTALL
       ├── tasks.yml
-      └── vars.yml
+      ├── vars.yml
+      └── pkgs/ (optional)
+          ├── package1.yml
+          ├── package2.yml
+          └── ...
+
+Where:
+- ``tasks.yml`` is a required YAML list of Ansible tasks that will be `included <https://docs.ansible.com/ansible/latest/collections/ansible/builtin/include_tasks_module.html>`__ when installing the model component.
+- ``vars.yml`` is a required YAML dictionary of all the variable keys/values which will be used when installing the model component.
+- ``pkgs/`` is an optional directory containing metadata files for external packages that need to be downloaded, verified, compressed, and cleaned up during installation.
 
-Where ``tasks.yml`` is a YAML list of Ansible tasks that will be `included <https://docs.ansible.com/ansible/latest/collections/ansible/builtin/include_tasks_module.html>`__ when installing the model component.
-The ``vars.yml`` file should be a YAML dictionary of all the variable keys/values which will be used when installing the model component.
 
 ``tasks.yml``
 =============
-The tasks file should be a YAML list with any tasks needed to ensure that the model component can execute correctly as intended.
+The tasks file should be a YAML list with any tasks needed to ensure that the model component can execute correctly as intended. At a minimum, this file must exist with a `---` line at the beginning.
 
 .. code-block:: yaml
   :caption: This is an example ``tasks.yml`` file that collects, verifies, and compresses needed binaries.
@@ -159,6 +166,28 @@ The full definition for ``required_files`` is:
 
 .. _mc_install_cache:
 
+``pkgs/`` Directory
+===================
+The ``pkgs/`` directory is an optional feature used to define metadata for external packages that need to be processed during the installation of a model component. Each package is represented as a `.yml` file containing the following metadata:
+
+.. code-block:: yaml
+  :caption: Example package metadata file (htop.yml)
+
+  name: "htop"
+  target_subdir: "vm_resources/debs"
+  target: "htop-1_0_2_debs.tgz"
+  files:
+    - dest: "htop_1.0.2-3_amd64.deb"
+      url: "http://archive.ubuntu.com/ubuntu/pool/universe/h/htop/htop_1.0.2-3_amd64.deb"
+      sha256: "0311d8a26689935ca53e8e9252cb2d95a1fdc2f8278f4edb5733f555dad984a9"
+
+The tasks for processing these packages are handled by the ``pkgs_tasks.yml`` file, which performs the following steps:
+1. Loads package variables from the `.yml` file.
+2. Creates the target subdirectory if it does not exist.
+3. Downloads and verifies files for the package using `ansible.builtin.get_url`.
+4. Compresses the package directory into a `.tgz` tarball.
+5. Removes the package directory after compression.
+
 ***************************
 Setting up an Offline Cache
 ***************************

src/firewheel/control/ansible_playbooks/main.yml

@@ -77,6 +77,32 @@
         msg: "All cache file methods have failed and system does not appear to be online. Please provide valid cache information."
       when: (ping_result.rc | default(0)) != 0
 
+    # Process each package in INSTALL/pkgs
+    - name: Check if pkgs_dir exists
+      ansible.builtin.stat:
+        path: "{{ mc_dir }}/INSTALL/pkgs"
+      register: pkgs_dir_status
+      failed_when: false
+
+    - name: Debug if pkgs_dir is missing or undefined
+      ansible.builtin.debug:
+        msg: "Skipping package processing because pkgs_dir is not defined or does not exist."
+      when: not pkgs_dir_status.stat.exists
+
+    - name: Find package files in INSTALL/pkgs
+      ansible.builtin.find:
+        paths: "{{ mc_dir }}/INSTALL/pkgs"
+        patterns: "*.yml"
+      register: pkg_files
+      when: pkgs_dir_status.stat.exists
+
+    - name: Process each package
+      ansible.builtin.include_tasks: roles/firewheel/tasks/pkgs_tasks.yml
+      vars:
+        pkg_file: "{{ item.path }}"
+      loop: "{{ pkg_files.files }}"
+      when: pkg_files.files is defined and pkg_files.files|length > 0
+
     # If we don't have any cached files and are online
     # then we need to run the MC Install tasks
     - name: "Include tasks (if needed) for: {{ mc_name }}"

src/firewheel/control/ansible_playbooks/roles/firewheel/tasks/pkgs_tasks.yml

@@ -0,0 +1,74 @@
+---
+###################################################################################################
+# Tasks File: pkgs_tasks.yml
+# Description:
+#   **This is not a replacement for `required_files`. This replaces a common download pattern for
+#   external packages and packages included here should be entered into required_files separately.**
+#   This Ansible tasks file is designed to process individual packages defined in the `INSTALL/pkgs`
+#   directory. Each package is represented as a `.yml` file containing metadata about the package,
+#   including its name, target tarball, target subdirectory, and the files it requires.
+#
+#   The tasks perform the following steps:
+#   1. Load package variables from the `.yml` file.
+#   2. Create the target subdirectory if it does not exist.
+#   3. Download and verify files for the package using `ansible.builtin.get_url`.
+#   4. Compress the package directory into a `.tgz` tarball.
+#   5. Remove the package directory after compression.
+#
+# Package `.yml` File Structure:
+#   Each package `.yml` file should follow this structure:
+#   ---
+#   name: "<package_name>"
+#   target_subdir: "<target_subdirectory>"  # Optional; defaults to "vm_resources"
+#   target: "<tarball_name>.tgz"
+#   files:
+#     - dest: "<destination_path>"
+#       url: "<file_url>"
+#       sha256: "<checksum>"
+#
+# Example:
+#   ---
+#   name: "htop"
+#   target_subdir: "vm_resources/debs"
+#   target: "htop-1_0_2_debs.tgz"
+#   files:
+#     - dest: "htop_1.0.2-3_amd64.deb"
+#       url: "http://archive.ubuntu.com/ubuntu/pool/universe/h/htop/htop_1.0.2-3_amd64.deb"
+#       sha256: "0311d8a26689935ca53e8e9252cb2d95a1fdc2f8278f4edb5733f555dad984a9"
+###################################################################################################
+
+- name: Load package variables
+  ansible.builtin.include_vars:
+    file: "{{ pkg_file }}"
+  failed_when: pkg_file is not defined or pkg_file|length == 0
+
+- name: Set default target_subdir if not specified
+  ansible.builtin.set_fact:
+    target_subdir: "{{ target_subdir | default('vm_resources') }}"
+
+- name: Create target subdirectory
+  ansible.builtin.file:
+    path: "{{ mc_dir }}/{{ target_subdir }}/{{ name }}"
+    state: directory
+
+- name: "Download and verify files for package: {{ name }}"
+  ansible.builtin.get_url:
+    url: "{{ item.url }}"
+    dest: "{{ mc_dir }}/{{ target_subdir }}/{{ name }}/{{ item.dest }}"
+    checksum: "sha256:{{ item.sha256 }}"
+  loop: "{{ files }}"
+  when: files is defined and files|length > 0
+  failed_when: item.url is not defined or item.sha256 is not defined
+
+- name: Compress package directory into tarball
+  ansible.builtin.archive:
+    path: "{{ mc_dir }}/{{ target_subdir }}/{{ name }}"
+    dest: "{{ mc_dir }}/{{ target_subdir }}/{{ target }}"
+    format: gz
+  when: target is defined and target|length > 0
+
+- name: Remove package directory
+  ansible.builtin.file:
+    path: "{{ mc_dir }}/{{ target_subdir }}/{{ name }}"
+    state: absent
+  when: target_subdir is defined and name is defined and name|length > 0