Add script to retry postprocessing of pulsar ghost jobs

[cat-bro]

This is on pulsar-qld-high-mem0 and pulsar-qld-high-mem1 as /home/ubuntu/ghost_jobs.sh and works some of the time. Sometimes the restart of pulsar needs to be skipped.

jwd_path='/mnt/pulsar/files/staging'
active_jobs_path='/mnt/pulsar/files/persistent_data/_default_-active-jobs'

remove_final_status_file () {
    job_id="$1"
    [ ! $job_id ] && echo 'job ID is required for remove_final_status';
    rm "${jwd_path}/${job_id}/final_status";
}

set_final_status_file () {
    job_id="$1"
    [ ! $job_id ] && echo 'job ID is required for set_final_status';
    echo -n '"complete"' > "${jwd_path}/${job_id}/final_status";
}

remove_postprocessed_file () {
    job_id="$1"
    [ ! $job_id ] && echo 'job ID is required for remove_postprocessed_file';
    rm "${jwd_path}/${job_id}/postprocessed";
}

set_active_job_file () {
    job_id="$1"
    [ ! $job_id ] && echo 'job ID is required for set_active_job_file';
    touch "${active_jobs_path}/${job_id}"
}

retry_ghost_jobs () {
    for x in $@; do
        remove_postprocessed_file $x;
        remove_final_status_file $x;
        set_active_job_file $x;
    done
    sudo systemctl restart pulsar
    sleep 10;
    for x in $@; do
        set_final_status_file $x;
    done
}