diff --git a/ansible/nomad-client.yml b/ansible/nomad-client.yml index 67a4ee69..06d0e14d 100644 --- a/ansible/nomad-client.yml +++ b/ansible/nomad-client.yml @@ -30,6 +30,7 @@ runtimeArgs: [] docker_daemon_options_extras: "{{ docker_daemon_nvidia_options if nvidia_docker_flag else {} }}" shard_role: nomad-pool + nomad_autoscaler_instance_type: "{{ 'whisper' if nomad_pool_type=='whisper' else 'nomad' }}" nomad_node_role: client nomad_gpu_count: 0 nomad_node_class: "{{ nomad_pool_type }}" @@ -145,8 +146,10 @@ - { role: "logrotate", tags: "logrotate"} - { role: "journald", tags: "journald"} - { role: "rsyslog", rsyslog_install_flag: false, tags: "rsyslog", rsyslog_tcp_output_enabled: true} - - { role: "autoscaler-sidecar", tags: "autoscaler-sidecar", - autoscaler_instance_type: "nomad", cloud_provider: "oracle", autoscaler_install_flag: false, autoscaler_configure_flag: true, when: nomad_enable_jitsi_autoscaler} + - { role: "whisper", tags: "whisper", when: nomad_pool_type == "whisper" } + - { role: "autoscaler-sidecar", + tags: "autoscaler-sidecar", + autoscaler_instance_type: "{{ nomad_autoscaler_instance_type }}", cloud_provider: "oracle", autoscaler_install_flag: false, autoscaler_configure_flag: true, when: nomad_enable_jitsi_autoscaler} - { role: "consul-agent", tags: "consul", consul_install_flag: false} - { role: "nvidia-docker", tags: "nvidia-docker", when: nvidia_docker_flag } - { role: "docker-daemon-config", tags: "docker-daemon-config" } diff --git a/ansible/roles/autoscaler-sidecar/defaults/main.yml b/ansible/roles/autoscaler-sidecar/defaults/main.yml index 5cea487d..dfbf2471 100644 --- a/ansible/roles/autoscaler-sidecar/defaults/main.yml +++ b/ansible/roles/autoscaler-sidecar/defaults/main.yml @@ -16,6 +16,7 @@ autoscaler_force_shutdown_command_by_type: 'jigasi': 'service jigasi stop' 'sip-jibri': 'service jibri stop' 'nomad': 'nomad node drain -self -enable -force -detach -yes' + 'whisper': 'nomad node drain -self -enable -force -detach -yes' autoscaler_git_repo: https://github.com/jitsi/jitsi-autoscaler-sidecar.git autoscaler_graceful_shutdown_script: "{{ autoscaler_graceful_shutdown_scripts[autoscaler_instance_type] @@ -26,6 +27,7 @@ autoscaler_graceful_shutdown_scripts: 'jigasi': /usr/local/bin/graceful-shutdown-wrapper-jigasi.sh 'sip-jibri': /opt/jitsi/jibri/wait_graceful_shutdown.sh 'nomad': /usr/local/bin/nomad_graceful_shutdown.sh + 'whisper': /usr/local/bin/whisper-graceful-shutdown.sh autoscaler_graceful_shutdown_wrapper: /usr/local/bin/graceful_shutdown_wrapper.sh autoscaler_group: "default" autoscaler_groupname: jsidecar @@ -33,7 +35,8 @@ autoscaler_health_script: "/usr/local/bin/autoscaler-sidecar-health.sh" autoscaler_health_url: "http://localhost:{{ autoscaler_express_port }}/health" autoscaler_install_flag: true autoscaler_instance_id: "{{ - ansible_ec2_instance_id if ansible_ec2_instance_id is defined + cloud_instance_id if cloud_instance_id is defined + else ansible_ec2_instance_id if ansible_ec2_instance_id is defined else oracle_instance_id if oracle_instance_id is defined else 'default' }}" autoscaler_instance_type: 'JVB' autoscaler_key_id: "{{ asap_key['id'] if asap_key['id'] is defined else 'default' }}" @@ -67,6 +70,7 @@ autoscaler_sidecar_instance_version_command_by_type: 'jigasi': "dpkg -s jigasi | grep Version | awk '{print $2}' | sed 's/..$//'" 'sip-jibri': "dpkg -s jibri | grep Version | awk '{print $2}' | sed 's/..$//'" 'nomad': "nomad --version | head -1 | cut -d' ' -f2 | tr -d 'v'" + 'whisper': "nomad --version | head -1 | cut -d' ' -f2 | tr -d 'v'" autoscaler_sidecar_private_ip: "{{ ansible_default_ipv4.address }}" autoscaler_sidecar_public_ip: "{{ ansible_ec2_public_ipv4 if ansible_ec2_public_ipv4 is defined else false }}" autoscaler_sidecar_service_name: jitsi-autoscaler-sidecar @@ -80,6 +84,7 @@ autoscaler_stats_retrieve_urls: 'jibri': http://localhost:2222/jibri/api/v1.0/health 'jigasi': http://localhost:8788/about/stats 'sip-jibri': http://localhost:2222/jibri/api/v1.0/health + 'whisper': http://{{ ansible_default_ipv4.address }}:8003/state autoscaler_status_url: "https://{{ autoscaler_server_host }}/sidecar/status" autoscaler_shutdown_url: "https://{{ autoscaler_server_host }}/sidecar/shutdown" autoscaler_terminate_script: '/usr/local/bin/terminate_instance.sh' diff --git a/ansible/roles/jigasi/defaults/main.yml b/ansible/roles/jigasi/defaults/main.yml index 3209a067..025e81f0 100644 --- a/ansible/roles/jigasi/defaults/main.yml +++ b/ansible/roles/jigasi/defaults/main.yml @@ -170,7 +170,7 @@ jigasi_transcriber_remote_select_aud: "jitsi" jigasi_transcriber_oci_flag: false jigasi_transcriber_whisper_private_key: "replaceme" jigasi_transcriber_whisper_private_key_name: "replaceme" -jigasi_transcriber_whisper_websocket_url: "wss://{{ hcv_environment }}-{{ oracle_region }}-whisper.jitsi.net:443/streaming-whisper/ws/" +jigasi_transcriber_whisper_websocket_url: "wss://{{ hcv_environment }}-{{ oracle_region }}-whisper.jitsi.net:443/whisper/ws/" jigasi_transcription_path: /var/lib/jigasi/transcripts jigasi_trusted_domains: false jigasi_udp_buffer_max_backlog: 2000 diff --git a/ansible/roles/whisper/tasks/main.yml b/ansible/roles/whisper/tasks/main.yml new file mode 100644 index 00000000..d6ccab9d --- /dev/null +++ b/ansible/roles/whisper/tasks/main.yml @@ -0,0 +1,6 @@ +- name: Copy graceful shutdown script + ansible.builtin.template: + src: whisper-graceful-shutdown.sh.j2 + dest: /usr/local/bin/whisper-graceful-shutdown.sh + mode: 0755 + owner: root diff --git a/ansible/roles/whisper/templates/whisper-graceful-shutdown.sh.j2 b/ansible/roles/whisper/templates/whisper-graceful-shutdown.sh.j2 new file mode 100644 index 00000000..d0ade4ca --- /dev/null +++ b/ansible/roles/whisper/templates/whisper-graceful-shutdown.sh.j2 @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +# Sets the server to drain mode and waits for all the connections to be closed before exiting +# with code 0 (clean shutdown). If the connections are not closed within the specified time, +# the script will exit with code 1 (dirty) and the autoscaler will shut down the instance anyway. + +# It takes two parameters: WAIT_SECONDS and STATE_URL +# WAIT_SECONDS: Seconds to wait for the server to enter drain mode and close all connections. +# Defaults to 100. +# STATE_URL: The URL of the whisper state endpoint. Defaults to http://localhost:8003/state. + +# Example usage: ./shutdown_script.sh 100 http://:8003/state + + +WAIT_SECONDS=100 +STATE_URL="http://{{ ansible_default_ipv4.address }}:8003/state" + +[[ -z $1 ]] && echo "Parameter WAIT_SECONDS not set, using default $WAIT_SECONDS" || WAIT_SECONDS=$1 +[[ -z $2 ]] && echo "Parameter STATE_URL not set, using default $STATE_URL" || STATE_URL=$2 + +echo "Waiting for $WAIT_SECONDS seconds for the server to enter drain mode and close all connections." + +START_TIME=$(date +%s) + +while true; do + RESPONSE=$(curl -s $STATE_URL) + WHISPER_STATE=$(echo $RESPONSE | jq -r '.state') + ACTIVE_CONNECTIONS=$(echo $RESPONSE | jq -r '.connections') + if [[ $WHISPER_STATE != "drain" ]]; then + echo "Setting server in drain mode." + curl -s -XPOST $STATE_URL -d '{"state": "drain"}' | jq -r '.request_status' + else + echo "Server is in drain mode. Waiting for $ACTIVE_CONNECTIONS connections to be closed before shutting down" + if [[ $ACTIVE_CONNECTIONS -eq 0 ]]; then + echo "All connections are closed. Shutting down the instance." + exit 0 + fi + fi + CURRENT_TIME=$(date +%s) + ELAPSED_TIME=$((CURRENT_TIME - START_TIME)) + if [[ $ELAPSED_TIME -gt $WAIT_SECONDS ]]; then + break + fi + sleep 2 +done + +echo "Server did not close all connections within $WAIT_SECONDS seconds. Shutting down anyway." +exit 1