diff --git a/docs/source/system/model_component/mc_install.rst b/docs/source/system/model_component/mc_install.rst index 4ea44c9..d7973ad 100644 --- a/docs/source/system/model_component/mc_install.rst +++ b/docs/source/system/model_component/mc_install.rst @@ -50,14 +50,21 @@ The expected ``INSTALL`` directory structure is:: MC_DIR └── INSTALL ├── tasks.yml - └── vars.yml + ├── vars.yml + └── pkgs/ (optional) + ├── package1.yml + ├── package2.yml + └── ... + +Where: +- ``tasks.yml`` is a required YAML list of Ansible tasks that will be `included `__ when installing the model component. +- ``vars.yml`` is a required YAML dictionary of all the variable keys/values which will be used when installing the model component. +- ``pkgs/`` is an optional directory containing metadata files for external packages that need to be downloaded, verified, compressed, and cleaned up during installation. -Where ``tasks.yml`` is a YAML list of Ansible tasks that will be `included `__ when installing the model component. -The ``vars.yml`` file should be a YAML dictionary of all the variable keys/values which will be used when installing the model component. ``tasks.yml`` ============= -The tasks file should be a YAML list with any tasks needed to ensure that the model component can execute correctly as intended. +The tasks file should be a YAML list with any tasks needed to ensure that the model component can execute correctly as intended. At a minimum, this file must exist with a `---` line at the beginning. .. code-block:: yaml :caption: This is an example ``tasks.yml`` file that collects, verifies, and compresses needed binaries. @@ -159,6 +166,28 @@ The full definition for ``required_files`` is: .. _mc_install_cache: +``pkgs/`` Directory +=================== +The ``pkgs/`` directory is an optional feature used to define metadata for external packages that need to be processed during the installation of a model component. Each package is represented as a `.yml` file containing the following metadata: + +.. code-block:: yaml + :caption: Example package metadata file (htop.yml) + + name: "htop" + target_subdir: "vm_resources/debs" + target: "htop-1_0_2_debs.tgz" + files: + - dest: "htop_1.0.2-3_amd64.deb" + url: "http://archive.ubuntu.com/ubuntu/pool/universe/h/htop/htop_1.0.2-3_amd64.deb" + sha256: "0311d8a26689935ca53e8e9252cb2d95a1fdc2f8278f4edb5733f555dad984a9" + +The tasks for processing these packages are handled by the ``pkgs_tasks.yml`` file, which performs the following steps: +1. Loads package variables from the `.yml` file. +2. Creates the target subdirectory if it does not exist. +3. Downloads and verifies files for the package using `ansible.builtin.get_url`. +4. Compresses the package directory into a `.tgz` tarball. +5. Removes the package directory after compression. + *************************** Setting up an Offline Cache *************************** diff --git a/src/firewheel/control/ansible_playbooks/main.yml b/src/firewheel/control/ansible_playbooks/main.yml index 2b15d12..8f44ced 100644 --- a/src/firewheel/control/ansible_playbooks/main.yml +++ b/src/firewheel/control/ansible_playbooks/main.yml @@ -77,6 +77,32 @@ msg: "All cache file methods have failed and system does not appear to be online. Please provide valid cache information." when: (ping_result.rc | default(0)) != 0 + # Process each package in INSTALL/pkgs + - name: Check if pkgs_dir exists + ansible.builtin.stat: + path: "{{ mc_dir }}/INSTALL/pkgs" + register: pkgs_dir_status + failed_when: false + + - name: Debug if pkgs_dir is missing or undefined + ansible.builtin.debug: + msg: "Skipping package processing because pkgs_dir is not defined or does not exist." + when: not pkgs_dir_status.stat.exists + + - name: Find package files in INSTALL/pkgs + ansible.builtin.find: + paths: "{{ mc_dir }}/INSTALL/pkgs" + patterns: "*.yml" + register: pkg_files + when: pkgs_dir_status.stat.exists + + - name: Process each package + ansible.builtin.include_tasks: roles/firewheel/tasks/pkgs_tasks.yml + vars: + pkg_file: "{{ item.path }}" + loop: "{{ pkg_files.files }}" + when: pkg_files.files is defined and pkg_files.files|length > 0 + # If we don't have any cached files and are online # then we need to run the MC Install tasks - name: "Include tasks (if needed) for: {{ mc_name }}" diff --git a/src/firewheel/control/ansible_playbooks/roles/firewheel/tasks/pkgs_tasks.yml b/src/firewheel/control/ansible_playbooks/roles/firewheel/tasks/pkgs_tasks.yml new file mode 100644 index 0000000..16eaa04 --- /dev/null +++ b/src/firewheel/control/ansible_playbooks/roles/firewheel/tasks/pkgs_tasks.yml @@ -0,0 +1,74 @@ +--- +################################################################################################### +# Tasks File: pkgs_tasks.yml +# Description: +# **This is not a replacement for `required_files`. This replaces a common download pattern for +# external packages and packages included here should be entered into required_files separately.** +# This Ansible tasks file is designed to process individual packages defined in the `INSTALL/pkgs` +# directory. Each package is represented as a `.yml` file containing metadata about the package, +# including its name, target tarball, target subdirectory, and the files it requires. +# +# The tasks perform the following steps: +# 1. Load package variables from the `.yml` file. +# 2. Create the target subdirectory if it does not exist. +# 3. Download and verify files for the package using `ansible.builtin.get_url`. +# 4. Compress the package directory into a `.tgz` tarball. +# 5. Remove the package directory after compression. +# +# Package `.yml` File Structure: +# Each package `.yml` file should follow this structure: +# --- +# name: "" +# target_subdir: "" # Optional; defaults to "vm_resources" +# target: ".tgz" +# files: +# - dest: "" +# url: "" +# sha256: "" +# +# Example: +# --- +# name: "htop" +# target_subdir: "vm_resources/debs" +# target: "htop-1_0_2_debs.tgz" +# files: +# - dest: "htop_1.0.2-3_amd64.deb" +# url: "http://archive.ubuntu.com/ubuntu/pool/universe/h/htop/htop_1.0.2-3_amd64.deb" +# sha256: "0311d8a26689935ca53e8e9252cb2d95a1fdc2f8278f4edb5733f555dad984a9" +################################################################################################### + +- name: Load package variables + ansible.builtin.include_vars: + file: "{{ pkg_file }}" + failed_when: pkg_file is not defined or pkg_file|length == 0 + +- name: Set default target_subdir if not specified + ansible.builtin.set_fact: + target_subdir: "{{ target_subdir | default('vm_resources') }}" + +- name: Create target subdirectory + ansible.builtin.file: + path: "{{ mc_dir }}/{{ target_subdir }}/{{ name }}" + state: directory + +- name: "Download and verify files for package: {{ name }}" + ansible.builtin.get_url: + url: "{{ item.url }}" + dest: "{{ mc_dir }}/{{ target_subdir }}/{{ name }}/{{ item.dest }}" + checksum: "sha256:{{ item.sha256 }}" + loop: "{{ files }}" + when: files is defined and files|length > 0 + failed_when: item.url is not defined or item.sha256 is not defined + +- name: Compress package directory into tarball + ansible.builtin.archive: + path: "{{ mc_dir }}/{{ target_subdir }}/{{ name }}" + dest: "{{ mc_dir }}/{{ target_subdir }}/{{ target }}" + format: gz + when: target is defined and target|length > 0 + +- name: Remove package directory + ansible.builtin.file: + path: "{{ mc_dir }}/{{ target_subdir }}/{{ name }}" + state: absent + when: target_subdir is defined and name is defined and name|length > 0