CTT-221 create an install-from-repo method

create prep script
new script ipf_configure_modules
config file for ipf_configure_modules removes need for cmdline options
all steps in QUICKSTART can be cut-n-paste
configure_extmodules looks for multiple conf files
wfm - WorkFlow Manager
wfm - use init.d files
wfm - update init-WORKFLOW template, assume running as non-root
new script to save_configs
prep will restore config links
add FAQ

.gitignore

@@ -4,3 +4,4 @@
 build/
 dist/
 ipf.egg-info/
+*.swp

FAQ.md

@@ -0,0 +1,53 @@
+# Frequently Asked Questions
+
+### What does `no pid file` mean when starting workflows?
+Not sure right now, but just check publishing status at
+https://operations-api.access-ci.org/wh2/state/v1/status/
+to see if the runs were published.
+
+Also, check local process status with
+```bash
+bash ~/ipf/bin/wfm status
+```
+
+
+### How do I upgrade to the latest version?
+1. Re-run the installer
+   * ```bash
+     bash ~/install_ipf.sh
+     ```
+
+
+### I messed up the install. Can I start over from scratch?
+Yes!
+1. (optional) Save customized configs
+   * (see the "Backup workflow configs" later in this document).
+1. Stop any running workflows
+   * ```bash
+     bash ~/ipf/bin/wfm stop
+     ```
+1. Remove the install directory
+   * ```bash
+     rm -rf ~/ipf
+     ```
+1. Follow through the QUICKSTART guide again starting from the top
+
+
+
+### Can I configure multiple workflows of the same type?
+Yes!  The `configure_extmodules` script will look for config files matching the
+naming convention `configure_extmodules*.conf`. You can create multiple config
+files and a workflow definition will be created for each one. Just make sure
+that `RESOURCE_NAME` is unique in each config file.
+
+
+### How can I backup my workflow configs?
+1. Backup workflow configs
+   * ```bash
+     bash ~/ipf/bin/save_configs.sh
+     ```
+This will do 2 things:
+* make backup copies in `~/.config/ipf/`
+* create symlinks to the backup copies in the ipf install dir.
+On a re-install, the IPF installer will look for any backed up
+config files and re-make the symlnks.

QUICKSTART.md

@@ -0,0 +1,72 @@
+# Install from git
+
+1. Get installer
+   * ```bash
+     curl -o ~/install_ipf.sh https://raw.githubusercontent.com/access-ci-org/ipf/refs/heads/CTT-221/aloftus/install_from_repo/go.sh
+     ```
+
+1. Run installer
+   * ```bash
+     bash ~/install_ipf.sh
+     ```
+   * Note: IPF will be installed into the current directory. All commands in
+     this guide assume the current directory is `~/`.
+
+1. Do first time setup
+   * ```bash
+     bash ~/ipf/bin/prep.sh
+     ```
+
+# Setup the extmodules workflow
+### Configure the extmodules workflow
+1. Set variables for your site
+   * ```bash
+     cp ~/ipf/etc/configure_extmodules.conf.sample ~/ipf/etc/configure_extmodules.conf
+     vim ~/ipf/etc/configure_extmodules.conf
+     cp ~/ipf/etc/amqp.conf.sample ~/ipf/etc/amqp.conf
+     vim ~/ipf/etc/amqp.conf
+     ```
+   * Note: for initial testing, leave the PUBLISH variable empty.
+1. Run the configure script
+   * ```bash
+     bash ~/ipf/bin/configure_extmodules
+     ```
+
+### Test the extmodules workflow
+1. Start the workflow
+   * ```bash
+     bash ~/ipf/bin/wfm start
+     ```
+1. Check the output
+   * ```bash
+     bash ~/ipf/bin/wfm list
+     ```
+   * Check the `OUTPUT` file that was listed above
+1. Stop the workflow
+   * ```bash
+     bash ~/ipf/bin/wfm stop
+     ```
+
+### Test the publishing setup
+1. Enable publishing
+   * ```bash
+     sed -ie '/PUBLISH=/cPUBLISH=1' ~/ipf/etc/configure_extmodules*.conf
+     ```
+1. Re-run the configure script
+   * ```bash
+     bash ~/ipf/bin/configure_extmodules
+     ```
+1. Start the workflow
+   * ```bash
+     bash ~/ipf/bin/wfm start
+     ```
+1. Check the published data
+   * Look for the resource name at: https://operations-api.access-ci.org/wh2/state/v1/status/
+   * The date in the `Processed at` column should be recent.
+
+# Setup recurring runs for production
+1. Create a scheduled task to restart the workflows after a system restart.
+   * Example crontab:
+     ```bash
+        @restart $HOME/ipf/bin/wfm start
+     ```

go.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+export QS_GIT_REPO=https://github.com/access-ci-org/ipf.git
+export QS_GIT_BRANCH=CTT-221/aloftus/install_from_repo
+curl https://raw.githubusercontent.com/andylytical/quickstart/main/quickstart.sh | bash

ipf/assert_py_ver.py

@@ -0,0 +1,11 @@
+# Require python 3
+
+import sys
+
+min_ver = ( 3, 6 )
+
+exitcode = 0
+py_ver = ( sys.version_info[0:2] )
+if py_ver < min_ver:
+    exitcode=1
+sys.exit( exitcode )

ipf/bin/configure_extmodules

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+INSTALL_DIR=___INSTALL_DIR___
+CLEANSE="$INSTALL_DIR"/etc/cleanse.conf
+CONFIGS=( $( ls "$INSTALL_DIR"/etc/configure_extmodules*.conf ) )
+PYTHON="$INSTALL_DIR"/.venv/bin/python
+PYTHONPATH=$(readlink -e "$INSTALL_DIR"/..)
+
+export PYTHONPATH
+
+# get amqp credentials from separate file if it exists
+amqp_fn="$INSTALL_DIR"/etc/amqp.conf
+[[ -r "$amqp_fn" ]] && source "$amqp_fn"
+
+# Loop through all config files found
+for cfg in "${CONFIGS[@]}"; do
+  source "$CLEANSE" #unset var settings from any previous run
+  source "$cfg" #read in new var settings for this specific run
+
+  # create workflow files
+  $PYTHON "$INSTALL_DIR"/configure/configure_workflows.py \
+    --base_dir "$INSTALL_DIR" \
+    ${AMQP_CERTIFICATE:+--amqp_certificate $AMQP_CERTIFICATE} \
+    ${AMQP_CERTIFICATE_KEY:+--amqp_certificate_key $AMQP_CERTIFICATE_KEY} \
+    ${AMQP_PASSWORD:+--amqp_password $AMQP_PASSWORD} \
+    ${AMQP_USERNAME:+--amqp_username $AMQP_USERNAME} \
+    ${LMOD_CACHE_FILE:+--lmod_cache_file $LMOD_CACHE_FILE} \
+    ${MODULEPATH:+--modulepath $MODULEPATH} \
+    ${MODULES:+--modules $MODULES} \
+    ${MODULES_EXCLUDE:+--modules_exclude $MODULES_EXCLUDE} \
+    ${MODULES_INTERVAL:+--modules_interval $MODULES_INTERVAL} \
+    ${PUBLISH:+--publish} \
+    ${RESOURCE_NAME:+--resource_name $RESOURCE_NAME} \
+    ${SUPPORT_CONTACT:+--support_contact $SUPPORT_CONTACT} \
+    ${WORKFLOWS:+--workflows $WORKFLOWS}
+
+done #for cfg in "${CONFIGS[@]}"; do

ipf/bin/prep.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+INSTALL_DIR=___INSTALL_DIR___
+. ${INSTALL_DIR}/lib/utils.sh
+
+BIN="$INSTALL_DIR"/bin
+ETC="$INSTALL_DIR"/etc
+VENV="$INSTALL_DIR"/.venv
+V_PYTHON="$VENV"/bin/python
+CONF="$HOME"/.config/ipf
+
+
+assert_python_minimum_version() {
+  [[ $DEBUG -eq $YES ]] && set -x
+  SYSTEM_PYTHON=$(which python3) 2>/dev/null
+  [[ -z "$SYSTEM_PYTHON" ]] && die "Unable to find Python on this system."
+  "$SYSTEM_PYTHON" "$INSTALL_DIR"/assert_py_ver.py || die "Python version is too old."
+  success "Python version check passed"
+}
+
+
+mk_venv() {
+  [[ $DEBUG -eq $YES ]] && set -x
+  [[ -d "$VENV" ]] || {
+    "$SYSTEM_PYTHON" -m venv "$VENV"
+  }
+  success "Python venv created at '$VENV'"
+}
+
+
+install_dependencies() {
+  [[ $DEBUG -eq $YES ]] && set -x
+  "$V_PYTHON" -m pip install --upgrade pip || die "Pip upgrade had a problem"
+  "$V_PYTHON" -m pip install -r "$ETC"/requirements.txt || die "Problem installing dependencies"
+  success "Dependencies installed"
+}
+
+
+restore_config_links() {
+  [[ $DEBUG -eq $YES ]] && set -x
+  /bin/bash "$BIN"/save_configs.sh
+}
+
+
+[[ $DEBUG -eq $YES ]] && set -x
+
+assert_python_minimum_version
+
+mk_venv
+
+install_dependencies
+
+restore_config_links

ipf/bin/run_workflow

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+INSTALL_DIR=___INSTALL_DIR___
+PYTHON="$INSTALL_DIR"/.venv/bin/python
+PYTHONPATH=$(readlink -e "$INSTALL_DIR"/..)
+WORKFLOW_BASE="$INSTALL_DIR"/etc/ipf/workflow
+
+WORKFLOW="$1"
+WORKFLOW_FULL_PATH="$WORKFLOW_BASE"/"$WORKFLOW"
+[[ -r "$WORKFLOW_FULL_PATH" ]] || {
+  echo "Workflow file not found: '$WORKFLOW_FULL_PATH'"
+  exit 1
+}
+
+export PYTHONPATH
+$PYTHON "$INSTALL_DIR"/run_workflow.py -d "$WORKFLOW"

ipf/bin/save_configs.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+INSTALL_DIR=___INSTALL_DIR___
+. ${INSTALL_DIR}/lib/utils.sh
+
+ETC="$INSTALL_DIR"/etc
+CONF="$HOME"/.config/ipf
+
+
+backup_configs() {
+  [[ $DEBUG -eq $YES ]] && set -x
+  mkdir -p "$CONF"
+  local _config_list=(
+    $( find "$ETC" -maxdepth 1 -type f -name 'configure*.conf' )
+    $( find "$ETC" -maxdepth 1 -type f -name 'amqp.conf' )
+  )
+  for fname in "${_config_list[@]}"; do
+    rsync --backup --suffix="$TS" --checksum "$fname" "$CONF"/
+  done
+}
+
+
+mk_symlinks() {
+  [[ $DEBUG -eq $YES ]] && set -x
+  local _config_list=(
+    $( find "$CONF" -maxdepth 1 -type f -name '*.conf' )
+  )
+  for src in "${_config_list[@]}"; do
+    fn=$( basename "$src" )
+    tgt="$ETC"/"$fn"
+    ln -sf "$src" "$tgt"
+  done
+}
+
+
+[[ $DEBUG -eq $YES ]] && set -x
+
+backup_configs
+
+mk_symlinks