From 60cb5da9163b10592ab67e2c1df208b54da90bca Mon Sep 17 00:00:00 2001 From: Nicky Gerritsen Date: Sun, 21 Aug 2022 20:16:54 +0200 Subject: [PATCH 1/3] Allow to set up load balancing when installing DOMjudge. --- provision-contest/ansible/domserver.yml | 12 ++--- .../ansible/group_vars/all/all.yml | 4 ++ .../ansible/group_vars/all/secret.yml.example | 4 ++ .../ansible/roles/domserver/tasks/main.yml | 20 ++++++--- .../domserver/templates/dbpasswords.secret.j2 | 4 ++ .../templates/nginx-domjudge-inner.j2 | 5 +++ .../templates/nginx-domjudge.conf.j2 | 45 +++++++++++++++++++ .../ansible/roles/mysql_server/tasks/main.yml | 20 +++++++++ 8 files changed, 103 insertions(+), 11 deletions(-) diff --git a/provision-contest/ansible/domserver.yml b/provision-contest/ansible/domserver.yml index c0419f45..db13132a 100644 --- a/provision-contest/ansible/domserver.yml +++ b/provision-contest/ansible/domserver.yml @@ -26,18 +26,18 @@ tags: ssh - role: mysql_server tags: mysql_server - - role: domjudge_checkout - tags: domjudge_checkout - - role: domjudge_build - tags: domjudge_build - - role: domserver - tags: domserver - role: mysql_replication tags: mysql_replication when: REPLICATION_PASSWORD is defined - role: keepalived tags: keepalived when: KEEPALIVED_PRIORITY is defined + - role: domjudge_checkout + tags: domjudge_checkout + - role: domjudge_build + tags: domjudge_build + - role: domserver + tags: domserver - role: prometheus_target_web tags: prometheus_target_web vars: diff --git a/provision-contest/ansible/group_vars/all/all.yml b/provision-contest/ansible/group_vars/all/all.yml index a3523a60..fe6a0221 100644 --- a/provision-contest/ansible/group_vars/all/all.yml +++ b/provision-contest/ansible/group_vars/all/all.yml @@ -29,6 +29,10 @@ GRAPHICAL: false # server. WF_RESTRICTED_NETWORK: false +# Set this to true to enable load balancing instead of (automatic) failover +# DBA_PASSWORD needs to be set for this in secret.yml +DOMSERVER_LOADBALANCING: false + TIMEZONE: "Europe/Moscow" PHP_FPM_MAX_CHILDREN: 400 diff --git a/provision-contest/ansible/group_vars/all/secret.yml.example b/provision-contest/ansible/group_vars/all/secret.yml.example index b166b42b..aacf72d8 100644 --- a/provision-contest/ansible/group_vars/all/secret.yml.example +++ b/provision-contest/ansible/group_vars/all/secret.yml.example @@ -2,6 +2,10 @@ # Set this to enable master-master replication between two domservers. #REPLICATION_PASSWORD: some-replication-password +# Database administrator password. Needed when DOMSERVER_LOADBALANCING is true +# Note: this user will have access from the whole SERVER_IP_PREFIX range +#DBA_PASSWORD: some-dba-password + # Database user password. DB_PASSWORD: some-database-password diff --git a/provision-contest/ansible/roles/domserver/tasks/main.yml b/provision-contest/ansible/roles/domserver/tasks/main.yml index efe8f0d6..1f31b3b4 100644 --- a/provision-contest/ansible/roles/domserver/tasks/main.yml +++ b/provision-contest/ansible/roles/domserver/tasks/main.yml @@ -19,16 +19,26 @@ owner: domjudge notify: Fix permissions on domjudge inplace-install +- name: set the DBA credentials + set_fact: + dba_credentials: | + {% if DBA_PASSWORD is defined %} + -u domjudge_dba -p {{ DBA_PASSWORD }} + {% else %} + -s -u root + {% endif %} + # When using replication, the DB will be dropped and recreated on the slave later. -- name: Check if the database is configured - command: "{{ DJ_DIR }}/bin/dj_setup_database -s -u root status" +- name: check if the database is configured + command: "{{ DJ_DIR }}/bin/dj_setup_database {{ dba_credentials }} status" register: db_status ignore_errors: true changed_when: false + when: not DOMSERVER_LOADBALANCING or groups['domserver'][0] == inventory_hostname -- name: Make sure the database is configured - command: "{{ DJ_DIR }}/bin/dj_setup_database -s -u root bare-install" - when: "'failed' in db_status.stdout" +- name: make sure the database is configured + command: "{{ DJ_DIR }}/bin/dj_setup_database {{ dba_credentials }} bare-install" + when: "(not DOMSERVER_LOADBALANCING or groups['domserver'][0] == inventory_hostname) and 'failed' in db_status.stdout" - name: Install required packages apt: diff --git a/provision-contest/ansible/roles/domserver/templates/dbpasswords.secret.j2 b/provision-contest/ansible/roles/domserver/templates/dbpasswords.secret.j2 index 15706d84..711b48a1 100644 --- a/provision-contest/ansible/roles/domserver/templates/dbpasswords.secret.j2 +++ b/provision-contest/ansible/roles/domserver/templates/dbpasswords.secret.j2 @@ -1,3 +1,7 @@ # {{ansible_managed}} # Format: 'unused:::::' +{% if DOMSERVER_LOADBALANCING %} +unused:{{DOMSERVER_IP}}:domjudge:domjudge:{{DB_PASSWORD}}:3306 +{% else %} unused:localhost:domjudge:domjudge:{{DB_PASSWORD}}:3306 +{% endif %} diff --git a/provision-contest/ansible/roles/domserver/templates/nginx-domjudge-inner.j2 b/provision-contest/ansible/roles/domserver/templates/nginx-domjudge-inner.j2 index 2d19afe6..21ca9b2e 100644 --- a/provision-contest/ansible/roles/domserver/templates/nginx-domjudge-inner.j2 +++ b/provision-contest/ansible/roles/domserver/templates/nginx-domjudge-inner.j2 @@ -11,6 +11,11 @@ set $domjudgeRoot {{ DJ_DIR }}/webapp/public; set $prefix ''; location / { +{% if DOMSERVER_LOADBALANCING %} + if ($access_allowed = false) { + return 403; + } +{% endif %} root $domjudgeRoot; try_files $uri @domjudgeFront; } diff --git a/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2 b/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2 index d768e818..41a2a55f 100644 --- a/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2 +++ b/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2 @@ -7,6 +7,36 @@ upstream domjudge { server unix:/var/run/php-fpm-domjudge.sock; # if using with etc/domjudge-fpm.conf } +{% if DOMSERVER_LOADBALANCING %} +upstream domjudge-loadbalanced { + least_conn; +{% for host in groups['domserver'] %} + server {{ hostvars[host].ansible_host }}:81; +{% endfor %} +} + +server { + listen 81; + listen [::]:81; + server_name _default_; + + add_header Strict-Transport-Security max-age=31556952; + include /etc/nginx/snippets/domjudge-inner; + + set_real_ip_from {{ SERVER_IP_PREFIX }}.0/24; + real_ip_header X-Forwarded-For; + real_ip_recursive on; +} + +map $realip_remote_addr $access_allowed { + default false; +{% for host in groups['domserver'] %} + {{ hostvars[host].ansible_host }} true; +{% endfor %} +} + +{% endif %} + server { listen 80; listen [::]:80; @@ -26,5 +56,20 @@ server { add_header Strict-Transport-Security max-age=31556952; send_timeout 36000s; +{% if DOMSERVER_LOADBALANCING %} + location / { + proxy_pass http://domjudge-loadbalanced; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_request_buffering off; + proxy_buffering off; + } +{% else %} include /etc/nginx/snippets/domjudge-inner; +{% endif %} } diff --git a/provision-contest/ansible/roles/mysql_server/tasks/main.yml b/provision-contest/ansible/roles/mysql_server/tasks/main.yml index 58f04170..1cad5a7c 100644 --- a/provision-contest/ansible/roles/mysql_server/tasks/main.yml +++ b/provision-contest/ansible/roles/mysql_server/tasks/main.yml @@ -67,3 +67,23 @@ loop: - load-db - dump-db + +- name: create mysql user for for DOMjudge database administration + mysql_user: + name: domjudge_dba + host: '{{ SERVER_IP_PREFIX }}.%' + password: "{{ DBA_PASSWORD }}" + append_privs: true + priv: 'domjudge.*:ALL,GRANT/*.*:CREATE USER,RELOAD' + state: present + when: DBA_PASSWORD is defined + +- name: create mysql user for for DOMjudge when we are doing loadbalancing + mysql_user: + name: domjudge + host: '{{ SERVER_IP_PREFIX }}.%' + password: "{{ DB_PASSWORD }}" + append_privs: true + priv: 'domjudge.*:SELECT,INSERT,UPDATE,DELETE' + state: present + when: DOMSERVER_LOADBALANCING From 082ff02e8c6429b9075e845feac752b8dfbb42d9 Mon Sep 17 00:00:00 2001 From: Nicky Gerritsen Date: Mon, 22 Aug 2022 09:47:37 +0200 Subject: [PATCH 2/3] Make sure admin machines still work, add MySQL users only for needed hosts and use SSL for LB communication. --- .../ansible/roles/domserver/tasks/main.yml | 6 ++--- .../domserver/templates/dbpasswords.secret.j2 | 2 +- .../templates/nginx-domjudge-inner.j2 | 2 +- .../templates/nginx-domjudge.conf.j2 | 22 ++++++++++++------- .../ansible/roles/mysql_server/tasks/main.yml | 10 +++++---- 5 files changed, 25 insertions(+), 17 deletions(-) diff --git a/provision-contest/ansible/roles/domserver/tasks/main.yml b/provision-contest/ansible/roles/domserver/tasks/main.yml index 1f31b3b4..6400a5a1 100644 --- a/provision-contest/ansible/roles/domserver/tasks/main.yml +++ b/provision-contest/ansible/roles/domserver/tasks/main.yml @@ -22,7 +22,7 @@ - name: set the DBA credentials set_fact: dba_credentials: | - {% if DBA_PASSWORD is defined %} + {% if host_type == 'domserver' and DBA_PASSWORD is defined %} -u domjudge_dba -p {{ DBA_PASSWORD }} {% else %} -s -u root @@ -34,11 +34,11 @@ register: db_status ignore_errors: true changed_when: false - when: not DOMSERVER_LOADBALANCING or groups['domserver'][0] == inventory_hostname + when: not DOMSERVER_LOADBALANCING or groups['domserver'][0] == inventory_hostname or host_type != 'domserver' - name: make sure the database is configured command: "{{ DJ_DIR }}/bin/dj_setup_database {{ dba_credentials }} bare-install" - when: "(not DOMSERVER_LOADBALANCING or groups['domserver'][0] == inventory_hostname) and 'failed' in db_status.stdout" + when: "(not DOMSERVER_LOADBALANCING or groups['domserver'][0] == inventory_hostname or host_type != 'domserver') and 'failed' in db_status.stdout" - name: Install required packages apt: diff --git a/provision-contest/ansible/roles/domserver/templates/dbpasswords.secret.j2 b/provision-contest/ansible/roles/domserver/templates/dbpasswords.secret.j2 index 711b48a1..0b8b5865 100644 --- a/provision-contest/ansible/roles/domserver/templates/dbpasswords.secret.j2 +++ b/provision-contest/ansible/roles/domserver/templates/dbpasswords.secret.j2 @@ -1,6 +1,6 @@ # {{ansible_managed}} # Format: 'unused:::::' -{% if DOMSERVER_LOADBALANCING %} +{% if host_type == 'domserver' and DOMSERVER_LOADBALANCING %} unused:{{DOMSERVER_IP}}:domjudge:domjudge:{{DB_PASSWORD}}:3306 {% else %} unused:localhost:domjudge:domjudge:{{DB_PASSWORD}}:3306 diff --git a/provision-contest/ansible/roles/domserver/templates/nginx-domjudge-inner.j2 b/provision-contest/ansible/roles/domserver/templates/nginx-domjudge-inner.j2 index 21ca9b2e..d1ec5a05 100644 --- a/provision-contest/ansible/roles/domserver/templates/nginx-domjudge-inner.j2 +++ b/provision-contest/ansible/roles/domserver/templates/nginx-domjudge-inner.j2 @@ -11,7 +11,7 @@ set $domjudgeRoot {{ DJ_DIR }}/webapp/public; set $prefix ''; location / { -{% if DOMSERVER_LOADBALANCING %} +{% if host_type == 'domserver' and DOMSERVER_LOADBALANCING %} if ($access_allowed = false) { return 403; } diff --git a/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2 b/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2 index 41a2a55f..09dbd459 100644 --- a/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2 +++ b/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2 @@ -7,19 +7,25 @@ upstream domjudge { server unix:/var/run/php-fpm-domjudge.sock; # if using with etc/domjudge-fpm.conf } -{% if DOMSERVER_LOADBALANCING %} +{% if host_type == 'domserver' and DOMSERVER_LOADBALANCING %} upstream domjudge-loadbalanced { least_conn; + keepalive 100; {% for host in groups['domserver'] %} - server {{ hostvars[host].ansible_host }}:81; + server {{ hostvars[host].ansible_host }}:444; {% endfor %} } server { - listen 81; - listen [::]:81; + listen 444 ssl http2; + listen [::]:444 ssl http2; server_name _default_; + ssl_certificate {{DOMSERVER_SSL_CERT}}; + ssl_certificate_key {{DOMSERVER_SSL_KEY}}; + ssl_session_timeout 5m; + ssl_prefer_server_ciphers on; + add_header Strict-Transport-Security max-age=31556952; include /etc/nginx/snippets/domjudge-inner; @@ -30,6 +36,7 @@ server { map $realip_remote_addr $access_allowed { default false; + {{ DOMSERVER_IP }} true; {% for host in groups['domserver'] %} {{ hostvars[host].ansible_host }} true; {% endfor %} @@ -56,12 +63,11 @@ server { add_header Strict-Transport-Security max-age=31556952; send_timeout 36000s; -{% if DOMSERVER_LOADBALANCING %} +{% if host_type == 'domserver' and DOMSERVER_LOADBALANCING %} location / { - proxy_pass http://domjudge-loadbalanced; + proxy_pass https://domjudge-loadbalanced; proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; + proxy_set_header Connection ""; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; diff --git a/provision-contest/ansible/roles/mysql_server/tasks/main.yml b/provision-contest/ansible/roles/mysql_server/tasks/main.yml index 1cad5a7c..b7dcb842 100644 --- a/provision-contest/ansible/roles/mysql_server/tasks/main.yml +++ b/provision-contest/ansible/roles/mysql_server/tasks/main.yml @@ -71,19 +71,21 @@ - name: create mysql user for for DOMjudge database administration mysql_user: name: domjudge_dba - host: '{{ SERVER_IP_PREFIX }}.%' + host: '{{ item }}' password: "{{ DBA_PASSWORD }}" append_privs: true priv: 'domjudge.*:ALL,GRANT/*.*:CREATE USER,RELOAD' state: present - when: DBA_PASSWORD is defined + when: host_type == 'domserver' and DBA_PASSWORD is defined + loop: "{{ groups['domserver'] | map('extract', hostvars, 'ansible_host') + [DOMSERVER_IP] }}" - name: create mysql user for for DOMjudge when we are doing loadbalancing mysql_user: name: domjudge - host: '{{ SERVER_IP_PREFIX }}.%' + host: '{{ item }}' password: "{{ DB_PASSWORD }}" append_privs: true priv: 'domjudge.*:SELECT,INSERT,UPDATE,DELETE' state: present - when: DOMSERVER_LOADBALANCING + when: host_type == 'domserver' and DOMSERVER_LOADBALANCING + loop: "{{ groups['domserver'] | map('extract', hostvars, 'ansible_host') + [DOMSERVER_IP] }}" From 125f9240be197c74a506fdfe62c94326c1fa1251 Mon Sep 17 00:00:00 2001 From: Nicky Gerritsen Date: Tue, 27 Sep 2022 21:50:33 +0200 Subject: [PATCH 3/3] Remove wrongly typed space. --- .../ansible/roles/domserver/templates/nginx-domjudge.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2 b/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2 index 09dbd459..fda4d842 100644 --- a/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2 +++ b/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2 @@ -67,7 +67,7 @@ server { location / { proxy_pass https://domjudge-loadbalanced; proxy_http_version 1.1; - proxy_set_header Connection ""; + proxy_set_header Connection ""; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr;