From 47200d9e098dcce086937b7ad7f920a18ccb2a09 Mon Sep 17 00:00:00 2001 From: Amit Sagtani Date: Mon, 4 Nov 2024 15:13:17 +0100 Subject: [PATCH 01/23] add retry while doing skopeo copy --- .github/workflows/custom-artifact.yml | 2 +- .github/workflows/offline.yml | 2 +- nix/scripts/create-container-dump.sh | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/custom-artifact.yml b/.github/workflows/custom-artifact.yml index 9a246a77a..293033cab 100644 --- a/.github/workflows/custom-artifact.yml +++ b/.github/workflows/custom-artifact.yml @@ -48,7 +48,7 @@ jobs: - name: Build and upload wire-server-deploy container run: | container_image=$(nix-build --no-out-link -A container) - skopeo copy --dest-creds "$DOCKER_LOGIN" \ + skopeo copy --retry-times 10 --dest-creds "$DOCKER_LOGIN" \ docker-archive:"$container_image" \ "docker://quay.io/wire/wire-server-deploy:${{ steps.upload_name.outputs.UPLOAD_NAME }}" env: diff --git a/.github/workflows/offline.yml b/.github/workflows/offline.yml index 2bd9a518a..367b41cd7 100644 --- a/.github/workflows/offline.yml +++ b/.github/workflows/offline.yml @@ -51,7 +51,7 @@ jobs: run: | container_image=$(nix-build --no-out-link -A container) - skopeo copy --dest-creds "$DOCKER_LOGIN" \ + skopeo copy --retry-times 10 --dest-creds "$DOCKER_LOGIN" \ docker-archive:"$container_image" \ "docker://quay.io/wire/wire-server-deploy:${{ steps.upload_name.outputs.UPLOAD_NAME }}" env: diff --git a/nix/scripts/create-container-dump.sh b/nix/scripts/create-container-dump.sh index a098d0d79..5adeb8975 100644 --- a/nix/scripts/create-container-dump.sh +++ b/nix/scripts/create-container-dump.sh @@ -29,10 +29,10 @@ while IFS= read -r image; do # If an image has both a tag and digest, remove the tag. Return the original if there is no match. image_trimmed=$(echo "$image" | sed -E 's/(.+)(:.+(@.+))/\1\3/') if [[ -n "${DOCKER_LOGIN:-}" && "$image" =~ quay.io/wire ]];then - skopeo copy --insecure-policy --src-creds "$DOCKER_LOGIN" \ + skopeo copy --retry-times 10 --insecure-policy --src-creds "$DOCKER_LOGIN" \ docker://$image_trimmed docker-archive:${image_path} --additional-tag $image else - skopeo copy --insecure-policy \ + skopeo copy --retry-times 10 --insecure-policy \ docker://$image_trimmed docker-archive:${image_path} --additional-tag $image fi echo "${image_filename}.tar" >> $(realpath "$1")/index.txt From 17b37323c84ee44bf2b9f2b9654ba7faf535e8bd Mon Sep 17 00:00:00 2001 From: Amit Sagtani Date: Tue, 5 Nov 2024 12:44:18 +0100 Subject: [PATCH 02/23] remove retry while copying from assethost --- nix/scripts/create-container-dump.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/scripts/create-container-dump.sh b/nix/scripts/create-container-dump.sh index 5adeb8975..a098d0d79 100644 --- a/nix/scripts/create-container-dump.sh +++ b/nix/scripts/create-container-dump.sh @@ -29,10 +29,10 @@ while IFS= read -r image; do # If an image has both a tag and digest, remove the tag. Return the original if there is no match. image_trimmed=$(echo "$image" | sed -E 's/(.+)(:.+(@.+))/\1\3/') if [[ -n "${DOCKER_LOGIN:-}" && "$image" =~ quay.io/wire ]];then - skopeo copy --retry-times 10 --insecure-policy --src-creds "$DOCKER_LOGIN" \ + skopeo copy --insecure-policy --src-creds "$DOCKER_LOGIN" \ docker://$image_trimmed docker-archive:${image_path} --additional-tag $image else - skopeo copy --retry-times 10 --insecure-policy \ + skopeo copy --insecure-policy \ docker://$image_trimmed docker-archive:${image_path} --additional-tag $image fi echo "${image_filename}.tar" >> $(realpath "$1")/index.txt From 569b0813b935a94594190cb4ab2a71a8e19ae2fa Mon Sep 17 00:00:00 2001 From: Amit Sagtani Date: Tue, 5 Nov 2024 15:45:34 +0100 Subject: [PATCH 03/23] replace cx11 with cx22 machines --- terraform/examples/create-infrastructure.tf | 8 ++++---- .../examples/wire-server-deploy-offline-hetzner/main.tf | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/terraform/examples/create-infrastructure.tf b/terraform/examples/create-infrastructure.tf index 6d6d70058..9f68ceecf 100644 --- a/terraform/examples/create-infrastructure.tf +++ b/terraform/examples/create-infrastructure.tf @@ -42,7 +42,7 @@ resource "hcloud_server" "redis" { count = 0 name = "redis${count.index}" image = "ubuntu-22.04" - server_type = "cx11" + server_type = "cx22" ssh_keys = ["hetznerssh-key"] # Nuremberg (for choices see `hcloud datacenter list`) @@ -53,7 +53,7 @@ resource "hcloud_server" "restund" { count = 2 name = "restund${count.index}" image = "ubuntu-22.04" - server_type = "cx11" + server_type = "cx22" ssh_keys = ["hetznerssh-key"] # Nuremberg (for choices see `hcloud datacenter list`) @@ -64,7 +64,7 @@ resource "hcloud_server" "minio" { count = 3 name = "minio${count.index}" image = "ubuntu-22.04" - server_type = "cx11" + server_type = "cx22" ssh_keys = ["hetznerssh-key"] # Nuremberg (for choices see `hcloud datacenter list`) @@ -86,7 +86,7 @@ resource "hcloud_server" "elasticsearch" { count = 3 name = "elasticsearch${count.index}" image = "ubuntu-22.04" - server_type = "cx11" + server_type = "cx22" ssh_keys = ["hetznerssh-key"] # Nuremberg (for choices see `hcloud datacenter list`) diff --git a/terraform/examples/wire-server-deploy-offline-hetzner/main.tf b/terraform/examples/wire-server-deploy-offline-hetzner/main.tf index 30553a8ff..1c0ce8a85 100644 --- a/terraform/examples/wire-server-deploy-offline-hetzner/main.tf +++ b/terraform/examples/wire-server-deploy-offline-hetzner/main.tf @@ -118,7 +118,7 @@ resource "hcloud_server" "restund" { name = "restund-${random_pet.restund[count.index].id}" image = "ubuntu-22.04" ssh_keys = local.ssh_keys - server_type = "cx11" + server_type = "cx22" user_data = local.disable_network_cfg } @@ -160,7 +160,7 @@ resource "hcloud_server" "cassandra" { name = "cassandra-${random_pet.cassandra[count.index].id}" image = "ubuntu-22.04" ssh_keys = local.ssh_keys - server_type = "cx11" + server_type = "cx22" user_data = local.disable_network_cfg } @@ -181,7 +181,7 @@ resource "hcloud_server" "elasticsearch" { name = "elasticsearch-${random_pet.elasticsearch[count.index].id}" image = "ubuntu-22.04" ssh_keys = local.ssh_keys - server_type = "cx11" + server_type = "cx22" user_data = local.disable_network_cfg } @@ -202,7 +202,7 @@ resource "hcloud_server" "minio" { name = "minio-${random_pet.minio[count.index].id}" image = "ubuntu-22.04" ssh_keys = local.ssh_keys - server_type = "cx11" + server_type = "cx22" user_data = local.disable_network_cfg } From 0b3f4280f8657238f2f2c22f411bfa3160895272 Mon Sep 17 00:00:00 2001 From: Amit Sagtani Date: Thu, 7 Nov 2024 11:43:53 +0100 Subject: [PATCH 04/23] try cpx series resources for hetzner machines --- .../examples/wire-server-deploy-offline-hetzner/main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/examples/wire-server-deploy-offline-hetzner/main.tf b/terraform/examples/wire-server-deploy-offline-hetzner/main.tf index 1c0ce8a85..cdecb37a1 100644 --- a/terraform/examples/wire-server-deploy-offline-hetzner/main.tf +++ b/terraform/examples/wire-server-deploy-offline-hetzner/main.tf @@ -63,7 +63,7 @@ resource "hcloud_server" "adminhost" { name = "adminhost-${random_pet.adminhost.id}" image = "ubuntu-22.04" ssh_keys = local.ssh_keys - server_type = "cx42" + server_type = "cpx41" user_data = <<-EOF #cloud-config apt: @@ -98,7 +98,7 @@ resource "hcloud_server" "assethost" { name = "assethost-${random_pet.assethost.id}" image = "ubuntu-22.04" ssh_keys = local.ssh_keys - server_type = "cx42" + server_type = "cpx41" user_data = local.disable_network_cfg } @@ -139,7 +139,7 @@ resource "hcloud_server" "kubenode" { name = "kubenode-${random_pet.kubenode[count.index].id}" image = "ubuntu-22.04" ssh_keys = local.ssh_keys - server_type = "cx42" + server_type = "cpx41" user_data = local.disable_network_cfg } From a51bca6e9f9184aad2322dd151964e1278cb4eb2 Mon Sep 17 00:00:00 2001 From: Amit Sagtani Date: Thu, 7 Nov 2024 14:46:40 +0100 Subject: [PATCH 05/23] change interface --- .../examples/wire-server-deploy-offline-hetzner/outputs.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/examples/wire-server-deploy-offline-hetzner/outputs.tf b/terraform/examples/wire-server-deploy-offline-hetzner/outputs.tf index 87294ed5e..f7b85dfe2 100644 --- a/terraform/examples/wire-server-deploy-offline-hetzner/outputs.tf +++ b/terraform/examples/wire-server-deploy-offline-hetzner/outputs.tf @@ -65,7 +65,7 @@ output "static-inventory" { } } vars = { - cassandra_network_interface = "ens10" + cassandra_network_interface = "eth0" } } cassandra_seed = { @@ -93,7 +93,7 @@ output "static-inventory" { } } vars = { - minio_network_interface = "ens10" + minio_network_interface = "eth0" } } restund = { @@ -104,7 +104,7 @@ output "static-inventory" { } } vars = { - restund_network_interface = "ens10" + restund_network_interface = "eth0" } } From 56b6689df5b73598878167652cb0370ddb2a237e Mon Sep 17 00:00:00 2001 From: Amit Sagtani Date: Mon, 18 Nov 2024 15:28:32 +0100 Subject: [PATCH 06/23] undo net interface change --- .../examples/wire-server-deploy-offline-hetzner/outputs.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/examples/wire-server-deploy-offline-hetzner/outputs.tf b/terraform/examples/wire-server-deploy-offline-hetzner/outputs.tf index f7b85dfe2..87294ed5e 100644 --- a/terraform/examples/wire-server-deploy-offline-hetzner/outputs.tf +++ b/terraform/examples/wire-server-deploy-offline-hetzner/outputs.tf @@ -65,7 +65,7 @@ output "static-inventory" { } } vars = { - cassandra_network_interface = "eth0" + cassandra_network_interface = "ens10" } } cassandra_seed = { @@ -93,7 +93,7 @@ output "static-inventory" { } } vars = { - minio_network_interface = "eth0" + minio_network_interface = "ens10" } } restund = { @@ -104,7 +104,7 @@ output "static-inventory" { } } vars = { - restund_network_interface = "eth0" + restund_network_interface = "ens10" } } From e13e0fc87a0253a8214bf9c4376045dbfeae4465 Mon Sep 17 00:00:00 2001 From: Amit Sagtani Date: Mon, 18 Nov 2024 20:33:52 +0100 Subject: [PATCH 07/23] add verbosity and eth0 interface --- bin/offline-cluster.sh | 12 +++++++----- .../wire-server-deploy-offline-hetzner/outputs.tf | 8 ++++---- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/bin/offline-cluster.sh b/bin/offline-cluster.sh index fb263f19d..bcffcc4db 100755 --- a/bin/offline-cluster.sh +++ b/bin/offline-cluster.sh @@ -11,8 +11,10 @@ ls $ANSIBLE_DIR/inventory/offline if [ -f "$ANSIBLE_DIR/inventory/offline/hosts.ini" ]; then INVENTORY_FILE="$ANSIBLE_DIR/inventory/offline/hosts.ini" + cat "$INVENTORY_FILE" elif [ -f "$ANSIBLE_DIR/inventory/offline/inventory.yml" ]; then INVENTORY_FILE="$ANSIBLE_DIR/inventory/offline/inventory.yml" + cat "$INVENTORY_FILE" else echo "No inventory file in ansible/inventory/offline/. Please supply an $ANSIBLE_DIR/inventory/offline/inventory.yml or $ANSIBLE_DIR/inventory/offline/hosts.ini" exit -1 @@ -50,10 +52,10 @@ ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/sync_time.yml -v ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/kubernetes.yml --skip-tags bootstrap-os,preinstall,container-engine,multus # Deploy all other services which don't run in kubernetes. -ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/cassandra.yml -ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/elasticsearch.yml -ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/minio.yml -ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/restund.yml +ansible-playbook -i $INVENTORY_FILE -vvv $ANSIBLE_DIR/cassandra.yml +ansible-playbook -i $INVENTORY_FILE -vvv $ANSIBLE_DIR/elasticsearch.yml +ansible-playbook -i $INVENTORY_FILE -vvv $ANSIBLE_DIR/minio.yml +ansible-playbook -i $INVENTORY_FILE -vvv $ANSIBLE_DIR/restund.yml # create helm values that tell our helm charts what the IP addresses of cassandra, elasticsearch and minio are: -ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/helm_external.yml --skip-tags=rabbitmq-external +ansible-playbook -i $INVENTORY_FILE -vvv $ANSIBLE_DIR/helm_external.yml --skip-tags=rabbitmq-external diff --git a/terraform/examples/wire-server-deploy-offline-hetzner/outputs.tf b/terraform/examples/wire-server-deploy-offline-hetzner/outputs.tf index 87294ed5e..c1535555f 100644 --- a/terraform/examples/wire-server-deploy-offline-hetzner/outputs.tf +++ b/terraform/examples/wire-server-deploy-offline-hetzner/outputs.tf @@ -65,7 +65,7 @@ output "static-inventory" { } } vars = { - cassandra_network_interface = "ens10" + cassandra_network_interface = "eth0" } } cassandra_seed = { @@ -79,7 +79,7 @@ output "static-inventory" { } } vars = { - elasticsearch_network_interface = "ens10" + elasticsearch_network_interface = "eth0" } } elasticsearch_master = { @@ -93,7 +93,7 @@ output "static-inventory" { } } vars = { - minio_network_interface = "ens10" + minio_network_interface = "eth0" } } restund = { @@ -104,7 +104,7 @@ output "static-inventory" { } } vars = { - restund_network_interface = "ens10" + restund_network_interface = "eth0" } } From 525a55fda7c3e927004b5c9dc15304be99493c44 Mon Sep 17 00:00:00 2001 From: Amit Sagtani Date: Tue, 19 Nov 2024 12:02:29 +0100 Subject: [PATCH 08/23] update CX21 to CS22 --- terraform/examples/create-infrastructure.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/examples/create-infrastructure.tf b/terraform/examples/create-infrastructure.tf index 9f68ceecf..2d74ac9ec 100644 --- a/terraform/examples/create-infrastructure.tf +++ b/terraform/examples/create-infrastructure.tf @@ -75,7 +75,7 @@ resource "hcloud_server" "cassandra" { count = 3 name = "cassandra${count.index}" image = "ubuntu-22.04" - server_type = "cx21" + server_type = "cx22" ssh_keys = ["hetznerssh-key"] # Nuremberg (for choices see `hcloud datacenter list`) From 7a60f81b40720ab726e41ec105b6c9f28cfb2202 Mon Sep 17 00:00:00 2001 From: Amit Sagtani Date: Tue, 19 Nov 2024 16:33:53 +0100 Subject: [PATCH 09/23] temp: disable tf destroy to debug cassandra --- .github/workflows/offline.yml | 10 +++++----- offline/cd.sh | 12 +++++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/workflows/offline.yml b/.github/workflows/offline.yml index 367b41cd7..cdcd119f1 100644 --- a/.github/workflows/offline.yml +++ b/.github/workflows/offline.yml @@ -63,10 +63,10 @@ jobs: env: HCLOUD_TOKEN: '${{ secrets.HCLOUD_TOKEN }}' - - name: Clean up hetzner environment; just in case - if: always() - run: (cd terraform/examples/wire-server-deploy-offline-hetzner ; terraform init && terraform destroy -auto-approve) - env: - HCLOUD_TOKEN: '${{ secrets.HCLOUD_TOKEN }}' +# - name: Clean up hetzner environment; just in case +# if: always() +# run: (cd terraform/examples/wire-server-deploy-offline-hetzner ; terraform init && terraform destroy -auto-approve) +# env: +# HCLOUD_TOKEN: '${{ secrets.HCLOUD_TOKEN }}' diff --git a/offline/cd.sh b/offline/cd.sh index 6b8bee6f1..8ecbe5062 100755 --- a/offline/cd.sh +++ b/offline/cd.sh @@ -2,16 +2,18 @@ set -euo pipefail -function cleanup { - (cd terraform/examples/wire-server-deploy-offline-hetzner ; terraform destroy -auto-approve) - echo done -} -trap cleanup EXIT +# function cleanup { +# (cd terraform/examples/wire-server-deploy-offline-hetzner ; terraform destroy -auto-approve) +# echo done +# } +# trap cleanup EXIT (cd terraform/examples/wire-server-deploy-offline-hetzner ; terraform init ; terraform apply -auto-approve ) adminhost=$(cd terraform/examples/wire-server-deploy-offline-hetzner ; terraform output adminhost) adminhost="${adminhost//\"/}" # remove extra quotes around the returned string ssh_private_key=$(cd terraform/examples/wire-server-deploy-offline-hetzner ; terraform output ssh_private_key) +echo "adminhost: $adminhost" +echo "$ssh_private_key" eval `ssh-agent` ssh-add - <<< "$ssh_private_key" From 84152300d79a7f68c081de1862ff160d659d01f7 Mon Sep 17 00:00:00 2001 From: Amit Sagtani Date: Wed, 20 Nov 2024 15:04:21 +0100 Subject: [PATCH 10/23] add iptable rules --- .../.terraform.lock.hcl | 61 +++++++++++++++++++ .../main.tf | 43 +++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 terraform/examples/wire-server-deploy-offline-hetzner/.terraform.lock.hcl diff --git a/terraform/examples/wire-server-deploy-offline-hetzner/.terraform.lock.hcl b/terraform/examples/wire-server-deploy-offline-hetzner/.terraform.lock.hcl new file mode 100644 index 000000000..706f235ce --- /dev/null +++ b/terraform/examples/wire-server-deploy-offline-hetzner/.terraform.lock.hcl @@ -0,0 +1,61 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/random" { + version = "3.6.3" + hashes = [ + "h1:zG9uFP8l9u+yGZZvi5Te7PV62j50azpgwPunq2vTm1E=", + "zh:04ceb65210251339f07cd4611885d242cd4d0c7306e86dda9785396807c00451", + "zh:448f56199f3e99ff75d5c0afacae867ee795e4dfda6cb5f8e3b2a72ec3583dd8", + "zh:4b4c11ccfba7319e901df2dac836b1ae8f12185e37249e8d870ee10bb87a13fe", + "zh:4fa45c44c0de582c2edb8a2e054f55124520c16a39b2dfc0355929063b6395b1", + "zh:588508280501a06259e023b0695f6a18149a3816d259655c424d068982cbdd36", + "zh:737c4d99a87d2a4d1ac0a54a73d2cb62974ccb2edbd234f333abd079a32ebc9e", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:a357ab512e5ebc6d1fda1382503109766e21bbfdfaa9ccda43d313c122069b30", + "zh:c51bfb15e7d52cc1a2eaec2a903ac2aff15d162c172b1b4c17675190e8147615", + "zh:e0951ee6fa9df90433728b96381fb867e3db98f66f735e0c3e24f8f16903f0ad", + "zh:e3cdcb4e73740621dabd82ee6a37d6cfce7fee2a03d8074df65086760f5cf556", + "zh:eff58323099f1bd9a0bec7cb04f717e7f1b2774c7d612bf7581797e1622613a0", + ] +} + +provider "registry.terraform.io/hashicorp/tls" { + version = "4.0.6" + hashes = [ + "h1:n3M50qfWfRSpQV9Pwcvuse03pEizqrmYEryxKky4so4=", + "zh:10de0d8af02f2e578101688fd334da3849f56ea91b0d9bd5b1f7a243417fdda8", + "zh:37fc01f8b2bc9d5b055dc3e78bfd1beb7c42cfb776a4c81106e19c8911366297", + "zh:4578ca03d1dd0b7f572d96bd03f744be24c726bfd282173d54b100fd221608bb", + "zh:6c475491d1250050765a91a493ef330adc24689e8837a0f07da5a0e1269e11c1", + "zh:81bde94d53cdababa5b376bbc6947668be4c45ab655de7aa2e8e4736dfd52509", + "zh:abdce260840b7b050c4e401d4f75c7a199fafe58a8b213947a258f75ac18b3e8", + "zh:b754cebfc5184873840f16a642a7c9ef78c34dc246a8ae29e056c79939963c7a", + "zh:c928b66086078f9917aef0eec15982f2e337914c5c4dbc31dd4741403db7eb18", + "zh:cded27bee5f24de6f2ee0cfd1df46a7f88e84aaffc2ecbf3ff7094160f193d50", + "zh:d65eb3867e8f69aaf1b8bb53bd637c99c6b649ba3db16ded50fa9a01076d1a27", + "zh:ecb0c8b528c7a619fa71852bb3fb5c151d47576c5aab2bf3af4db52588722eeb", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hetznercloud/hcloud" { + version = "1.49.0" + hashes = [ + "h1:TVZ8DO6bWVWxSMpwFe1tNS8gei8WGCgqOkc7eNWami8=", + "zh:05a5430404384d59002e9bda022e9c3b993687558ca92baa3b0943642b056264", + "zh:1e8e74c8b99594b782e96f62afa86daf11c5b15581deedc0d88f08bb4760a27e", + "zh:3e6d10b2a1f5a4fa74733b4b833d29ab5e3592bfdb99833b9915db6e885e4670", + "zh:41c134a4466bdd12a164803263c25225d2c0d8ed5b9af554b8562d101e5874e0", + "zh:47c4c0f7a2b9e27d10dc98473513c04e25b079fa10d23814482ad2fb4fb30bc8", + "zh:517c5c2fa53695e36e2c8a385464f8708a8733c2ab81869d0e43e019cf2b156b", + "zh:5ceae1343501301fd4c861a1d9be16fd3b8f20dce0c3a73c74dafec27f0d022c", + "zh:61b8e30742bb347cdb0284dbd5b505c11603ccc7255c88758d22f3f8528d0059", + "zh:8b702609ca48b0e76e930a76244414b5f8ef0c93125a63bf147dc1cbf1b28f6a", + "zh:a5ee062a97b4ab7356062f8acc93824fce97c932e7b741eed7a4685fe57de9b9", + "zh:c744bec43cd2edcbf507bf0090125cd5500288a51bf0fde4515566a68a7dedda", + "zh:cb8c4e23879b49cd09a4374c165ee0ad077591473ec54a8df0b8948c5d583b91", + "zh:d3f6233b8c8217f318ce5a9b6c22e0adc84051dcb42fdb431d54ce3ef2e82513", + "zh:f724c9ea8d3e9cd6bdba393701eaf92506269b114e3e643d1abd0e1ca820bc16", + ] +} diff --git a/terraform/examples/wire-server-deploy-offline-hetzner/main.tf b/terraform/examples/wire-server-deploy-offline-hetzner/main.tf index cdecb37a1..f30f2d49d 100644 --- a/terraform/examples/wire-server-deploy-offline-hetzner/main.tf +++ b/terraform/examples/wire-server-deploy-offline-hetzner/main.tf @@ -20,6 +20,49 @@ locals { - iptables -A OUTPUT -o eth0 -p udp --dport 123 -j ACCEPT - ip6tables -A OUTPUT -o eth0 -p udp --dport 123 -j ACCEPT + # Cassandra (inbound and outbound) + - iptables -A OUTPUT -o eth0 -p tcp --dport 9042 -j ACCEPT + - ip6tables -A OUTPUT -o eth0 -p tcp --dport 9042 -j ACCEPT + - iptables -A INPUT -i eth0 -p tcp --sport 9042 -j ACCEPT + - ip6tables -A INPUT -i eth0 -p tcp --sport 9042 -j ACCEPT + + - iptables -A OUTPUT -o eth0 -p tcp --dport 9160 -j ACCEPT + - ip6tables -A OUTPUT -o eth0 -p tcp --dport 9160 -j ACCEPT + - iptables -A INPUT -i eth0 -p tcp --sport 9160 -j ACCEPT + - ip6tables -A INPUT -i eth0 -p tcp --sport 9160 -j ACCEPT + + - iptables -A OUTPUT -o eth0 -p tcp --dport 7000 -j ACCEPT + - ip6tables -A OUTPUT -o eth0 -p tcp --dport 7000 -j ACCEPT + - iptables -A INPUT -i eth0 -p tcp --sport 7000 -j ACCEPT + - ip6tables -A INPUT -i eth0 -p tcp --sport 7000 -j ACCEPT + + - iptables -A OUTPUT -o eth0 -p tcp --dport 7199 -j ACCEPT + - ip6tables -A OUTPUT -o eth0 -p tcp --dport 7199 -j ACCEPT + - iptables -A INPUT -i eth0 -p tcp --sport 7199 -j ACCEPT + - ip6tables -A INPUT -i eth0 -p tcp --sport 7199 -j ACCEPT + + # Elasticsearch (inbound and outbound) + - iptables -A OUTPUT -o eth0 -p tcp --dport 9300 -j ACCEPT + - ip6tables -A OUTPUT -o eth0 -p tcp --dport 9300 -j ACCEPT + - iptables -A INPUT -i eth0 -p tcp --sport 9300 -j ACCEPT + - ip6tables -A INPUT -i eth0 -p tcp --sport 9300 -j ACCEPT + + - iptables -A OUTPUT -o eth0 -p tcp --dport 9200 -j ACCEPT + - ip6tables -A OUTPUT -o eth0 -p tcp --dport 9200 -j ACCEPT + - iptables -A INPUT -i eth0 -p tcp --sport 9200 -j ACCEPT + - ip6tables -A INPUT -i eth0 -p tcp --sport 9200 -j ACCEPT + + # MinIO (inbound and outbound) + - iptables -A OUTPUT -o eth0 -p tcp --dport 9000 -j ACCEPT + - ip6tables -A OUTPUT -o eth0 -p tcp --dport 9000 -j ACCEPT + - iptables -A INPUT -i eth0 -p tcp --sport 9000 -j ACCEPT + - ip6tables -A INPUT -i eth0 -p tcp --sport 9000 -j ACCEPT + + - iptables -A OUTPUT -o eth0 -p tcp --dport 9092 -j ACCEPT + - ip6tables -A OUTPUT -o eth0 -p tcp --dport 9092 -j ACCEPT + - iptables -A INPUT -i eth0 -p tcp --sport 9092 -j ACCEPT + - ip6tables -A INPUT -i eth0 -p tcp --sport 9092 -j ACCEPT + # Drop all other traffic - iptables -A OUTPUT -o eth0 -j DROP - ip6tables -A OUTPUT -o eth0 -j DROP From 25859113a675ce2b69cdc97b1d9c9f4f4785edfe Mon Sep 17 00:00:00 2001 From: Amit Sagtani Date: Wed, 20 Nov 2024 17:56:21 +0100 Subject: [PATCH 11/23] remove firewall from external services --- .../examples/wire-server-deploy-offline-hetzner/main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/examples/wire-server-deploy-offline-hetzner/main.tf b/terraform/examples/wire-server-deploy-offline-hetzner/main.tf index f30f2d49d..3a48ccaa8 100644 --- a/terraform/examples/wire-server-deploy-offline-hetzner/main.tf +++ b/terraform/examples/wire-server-deploy-offline-hetzner/main.tf @@ -204,7 +204,7 @@ resource "hcloud_server" "cassandra" { image = "ubuntu-22.04" ssh_keys = local.ssh_keys server_type = "cx22" - user_data = local.disable_network_cfg + # user_data = local.disable_network_cfg } resource "hcloud_server_network" "cassandra" { @@ -225,7 +225,7 @@ resource "hcloud_server" "elasticsearch" { image = "ubuntu-22.04" ssh_keys = local.ssh_keys server_type = "cx22" - user_data = local.disable_network_cfg + # user_data = local.disable_network_cfg } resource "hcloud_server_network" "elasticsearch" { @@ -246,7 +246,7 @@ resource "hcloud_server" "minio" { image = "ubuntu-22.04" ssh_keys = local.ssh_keys server_type = "cx22" - user_data = local.disable_network_cfg + # user_data = local.disable_network_cfg } resource "hcloud_server_network" "minio" { From 4451d4ab0a7a30ec75d563fb8fb42633ca27177e Mon Sep 17 00:00:00 2001 From: Amit Sagtani Date: Thu, 21 Nov 2024 12:19:17 +0100 Subject: [PATCH 12/23] set rabbitmq secrets for background-worker --- values/wire-server/prod-secrets.example.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/values/wire-server/prod-secrets.example.yaml b/values/wire-server/prod-secrets.example.yaml index 311e4809d..dfa5439e7 100644 --- a/values/wire-server/prod-secrets.example.yaml +++ b/values/wire-server/prod-secrets.example.yaml @@ -73,12 +73,12 @@ nginz: basicAuth: ":" -# Uncomment for federation below. Set values accordingly -# background-worker: -# secrets: -# rabbitmq: -# username: wire-server -# password: verysecurepassword +# RabbitMQ credentials for background-worker. +background-worker: + secrets: + rabbitmq: + username: wire-server + password: verysecurepassword # Uncomment for legalhold. Set values accordingly From d125156b12d8bf9375824760935ee9be14b84aae Mon Sep 17 00:00:00 2001 From: Amit Sagtani Date: Thu, 21 Nov 2024 12:24:15 +0100 Subject: [PATCH 13/23] remove temp firewall rules --- .../.terraform.lock.hcl | 61 ------------------- .../main.tf | 43 ------------- 2 files changed, 104 deletions(-) delete mode 100644 terraform/examples/wire-server-deploy-offline-hetzner/.terraform.lock.hcl diff --git a/terraform/examples/wire-server-deploy-offline-hetzner/.terraform.lock.hcl b/terraform/examples/wire-server-deploy-offline-hetzner/.terraform.lock.hcl deleted file mode 100644 index 706f235ce..000000000 --- a/terraform/examples/wire-server-deploy-offline-hetzner/.terraform.lock.hcl +++ /dev/null @@ -1,61 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/random" { - version = "3.6.3" - hashes = [ - "h1:zG9uFP8l9u+yGZZvi5Te7PV62j50azpgwPunq2vTm1E=", - "zh:04ceb65210251339f07cd4611885d242cd4d0c7306e86dda9785396807c00451", - "zh:448f56199f3e99ff75d5c0afacae867ee795e4dfda6cb5f8e3b2a72ec3583dd8", - "zh:4b4c11ccfba7319e901df2dac836b1ae8f12185e37249e8d870ee10bb87a13fe", - "zh:4fa45c44c0de582c2edb8a2e054f55124520c16a39b2dfc0355929063b6395b1", - "zh:588508280501a06259e023b0695f6a18149a3816d259655c424d068982cbdd36", - "zh:737c4d99a87d2a4d1ac0a54a73d2cb62974ccb2edbd234f333abd079a32ebc9e", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:a357ab512e5ebc6d1fda1382503109766e21bbfdfaa9ccda43d313c122069b30", - "zh:c51bfb15e7d52cc1a2eaec2a903ac2aff15d162c172b1b4c17675190e8147615", - "zh:e0951ee6fa9df90433728b96381fb867e3db98f66f735e0c3e24f8f16903f0ad", - "zh:e3cdcb4e73740621dabd82ee6a37d6cfce7fee2a03d8074df65086760f5cf556", - "zh:eff58323099f1bd9a0bec7cb04f717e7f1b2774c7d612bf7581797e1622613a0", - ] -} - -provider "registry.terraform.io/hashicorp/tls" { - version = "4.0.6" - hashes = [ - "h1:n3M50qfWfRSpQV9Pwcvuse03pEizqrmYEryxKky4so4=", - "zh:10de0d8af02f2e578101688fd334da3849f56ea91b0d9bd5b1f7a243417fdda8", - "zh:37fc01f8b2bc9d5b055dc3e78bfd1beb7c42cfb776a4c81106e19c8911366297", - "zh:4578ca03d1dd0b7f572d96bd03f744be24c726bfd282173d54b100fd221608bb", - "zh:6c475491d1250050765a91a493ef330adc24689e8837a0f07da5a0e1269e11c1", - "zh:81bde94d53cdababa5b376bbc6947668be4c45ab655de7aa2e8e4736dfd52509", - "zh:abdce260840b7b050c4e401d4f75c7a199fafe58a8b213947a258f75ac18b3e8", - "zh:b754cebfc5184873840f16a642a7c9ef78c34dc246a8ae29e056c79939963c7a", - "zh:c928b66086078f9917aef0eec15982f2e337914c5c4dbc31dd4741403db7eb18", - "zh:cded27bee5f24de6f2ee0cfd1df46a7f88e84aaffc2ecbf3ff7094160f193d50", - "zh:d65eb3867e8f69aaf1b8bb53bd637c99c6b649ba3db16ded50fa9a01076d1a27", - "zh:ecb0c8b528c7a619fa71852bb3fb5c151d47576c5aab2bf3af4db52588722eeb", - "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - ] -} - -provider "registry.terraform.io/hetznercloud/hcloud" { - version = "1.49.0" - hashes = [ - "h1:TVZ8DO6bWVWxSMpwFe1tNS8gei8WGCgqOkc7eNWami8=", - "zh:05a5430404384d59002e9bda022e9c3b993687558ca92baa3b0943642b056264", - "zh:1e8e74c8b99594b782e96f62afa86daf11c5b15581deedc0d88f08bb4760a27e", - "zh:3e6d10b2a1f5a4fa74733b4b833d29ab5e3592bfdb99833b9915db6e885e4670", - "zh:41c134a4466bdd12a164803263c25225d2c0d8ed5b9af554b8562d101e5874e0", - "zh:47c4c0f7a2b9e27d10dc98473513c04e25b079fa10d23814482ad2fb4fb30bc8", - "zh:517c5c2fa53695e36e2c8a385464f8708a8733c2ab81869d0e43e019cf2b156b", - "zh:5ceae1343501301fd4c861a1d9be16fd3b8f20dce0c3a73c74dafec27f0d022c", - "zh:61b8e30742bb347cdb0284dbd5b505c11603ccc7255c88758d22f3f8528d0059", - "zh:8b702609ca48b0e76e930a76244414b5f8ef0c93125a63bf147dc1cbf1b28f6a", - "zh:a5ee062a97b4ab7356062f8acc93824fce97c932e7b741eed7a4685fe57de9b9", - "zh:c744bec43cd2edcbf507bf0090125cd5500288a51bf0fde4515566a68a7dedda", - "zh:cb8c4e23879b49cd09a4374c165ee0ad077591473ec54a8df0b8948c5d583b91", - "zh:d3f6233b8c8217f318ce5a9b6c22e0adc84051dcb42fdb431d54ce3ef2e82513", - "zh:f724c9ea8d3e9cd6bdba393701eaf92506269b114e3e643d1abd0e1ca820bc16", - ] -} diff --git a/terraform/examples/wire-server-deploy-offline-hetzner/main.tf b/terraform/examples/wire-server-deploy-offline-hetzner/main.tf index 3a48ccaa8..dfd9b7b61 100644 --- a/terraform/examples/wire-server-deploy-offline-hetzner/main.tf +++ b/terraform/examples/wire-server-deploy-offline-hetzner/main.tf @@ -20,49 +20,6 @@ locals { - iptables -A OUTPUT -o eth0 -p udp --dport 123 -j ACCEPT - ip6tables -A OUTPUT -o eth0 -p udp --dport 123 -j ACCEPT - # Cassandra (inbound and outbound) - - iptables -A OUTPUT -o eth0 -p tcp --dport 9042 -j ACCEPT - - ip6tables -A OUTPUT -o eth0 -p tcp --dport 9042 -j ACCEPT - - iptables -A INPUT -i eth0 -p tcp --sport 9042 -j ACCEPT - - ip6tables -A INPUT -i eth0 -p tcp --sport 9042 -j ACCEPT - - - iptables -A OUTPUT -o eth0 -p tcp --dport 9160 -j ACCEPT - - ip6tables -A OUTPUT -o eth0 -p tcp --dport 9160 -j ACCEPT - - iptables -A INPUT -i eth0 -p tcp --sport 9160 -j ACCEPT - - ip6tables -A INPUT -i eth0 -p tcp --sport 9160 -j ACCEPT - - - iptables -A OUTPUT -o eth0 -p tcp --dport 7000 -j ACCEPT - - ip6tables -A OUTPUT -o eth0 -p tcp --dport 7000 -j ACCEPT - - iptables -A INPUT -i eth0 -p tcp --sport 7000 -j ACCEPT - - ip6tables -A INPUT -i eth0 -p tcp --sport 7000 -j ACCEPT - - - iptables -A OUTPUT -o eth0 -p tcp --dport 7199 -j ACCEPT - - ip6tables -A OUTPUT -o eth0 -p tcp --dport 7199 -j ACCEPT - - iptables -A INPUT -i eth0 -p tcp --sport 7199 -j ACCEPT - - ip6tables -A INPUT -i eth0 -p tcp --sport 7199 -j ACCEPT - - # Elasticsearch (inbound and outbound) - - iptables -A OUTPUT -o eth0 -p tcp --dport 9300 -j ACCEPT - - ip6tables -A OUTPUT -o eth0 -p tcp --dport 9300 -j ACCEPT - - iptables -A INPUT -i eth0 -p tcp --sport 9300 -j ACCEPT - - ip6tables -A INPUT -i eth0 -p tcp --sport 9300 -j ACCEPT - - - iptables -A OUTPUT -o eth0 -p tcp --dport 9200 -j ACCEPT - - ip6tables -A OUTPUT -o eth0 -p tcp --dport 9200 -j ACCEPT - - iptables -A INPUT -i eth0 -p tcp --sport 9200 -j ACCEPT - - ip6tables -A INPUT -i eth0 -p tcp --sport 9200 -j ACCEPT - - # MinIO (inbound and outbound) - - iptables -A OUTPUT -o eth0 -p tcp --dport 9000 -j ACCEPT - - ip6tables -A OUTPUT -o eth0 -p tcp --dport 9000 -j ACCEPT - - iptables -A INPUT -i eth0 -p tcp --sport 9000 -j ACCEPT - - ip6tables -A INPUT -i eth0 -p tcp --sport 9000 -j ACCEPT - - - iptables -A OUTPUT -o eth0 -p tcp --dport 9092 -j ACCEPT - - ip6tables -A OUTPUT -o eth0 -p tcp --dport 9092 -j ACCEPT - - iptables -A INPUT -i eth0 -p tcp --sport 9092 -j ACCEPT - - ip6tables -A INPUT -i eth0 -p tcp --sport 9092 -j ACCEPT - # Drop all other traffic - iptables -A OUTPUT -o eth0 -j DROP - ip6tables -A OUTPUT -o eth0 -j DROP From 058a20466fc4a37de4a4873697197b63d566e0b9 Mon Sep 17 00:00:00 2001 From: Amit Sagtani Date: Thu, 21 Nov 2024 14:30:07 +0100 Subject: [PATCH 14/23] add rabbitmq secrets to ci --- bin/offline-secrets.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/offline-secrets.sh b/bin/offline-secrets.sh index 48df9d35c..d58d48072 100755 --- a/bin/offline-secrets.sh +++ b/bin/offline-secrets.sh @@ -63,6 +63,11 @@ team-settings: # container is pre-seeded # It is just the empty "{}" json hashmap configJson: "e30K" +background-worker: + secrets: + rabbitmq: + username: wire-server + password: verysecurepassword EOF fi From cbc379fc770001a684fab4f58e76584a3f135291 Mon Sep 17 00:00:00 2001 From: Amit Sagtani Date: Thu, 21 Nov 2024 14:41:01 +0100 Subject: [PATCH 15/23] remove debugging steps --- .github/workflows/offline.yml | 12 +++++------- bin/offline-cluster.sh | 12 +++++------- offline/cd.sh | 12 +++++------- 3 files changed, 15 insertions(+), 21 deletions(-) diff --git a/.github/workflows/offline.yml b/.github/workflows/offline.yml index cdcd119f1..5ad30d640 100644 --- a/.github/workflows/offline.yml +++ b/.github/workflows/offline.yml @@ -63,10 +63,8 @@ jobs: env: HCLOUD_TOKEN: '${{ secrets.HCLOUD_TOKEN }}' -# - name: Clean up hetzner environment; just in case -# if: always() -# run: (cd terraform/examples/wire-server-deploy-offline-hetzner ; terraform init && terraform destroy -auto-approve) -# env: -# HCLOUD_TOKEN: '${{ secrets.HCLOUD_TOKEN }}' - - + - name: Clean up hetzner environment; just in case + if: always() + run: (cd terraform/examples/wire-server-deploy-offline-hetzner ; terraform init && terraform destroy -auto-approve) + env: + HCLOUD_TOKEN: '${{ secrets.HCLOUD_TOKEN }}' diff --git a/bin/offline-cluster.sh b/bin/offline-cluster.sh index bcffcc4db..fb263f19d 100755 --- a/bin/offline-cluster.sh +++ b/bin/offline-cluster.sh @@ -11,10 +11,8 @@ ls $ANSIBLE_DIR/inventory/offline if [ -f "$ANSIBLE_DIR/inventory/offline/hosts.ini" ]; then INVENTORY_FILE="$ANSIBLE_DIR/inventory/offline/hosts.ini" - cat "$INVENTORY_FILE" elif [ -f "$ANSIBLE_DIR/inventory/offline/inventory.yml" ]; then INVENTORY_FILE="$ANSIBLE_DIR/inventory/offline/inventory.yml" - cat "$INVENTORY_FILE" else echo "No inventory file in ansible/inventory/offline/. Please supply an $ANSIBLE_DIR/inventory/offline/inventory.yml or $ANSIBLE_DIR/inventory/offline/hosts.ini" exit -1 @@ -52,10 +50,10 @@ ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/sync_time.yml -v ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/kubernetes.yml --skip-tags bootstrap-os,preinstall,container-engine,multus # Deploy all other services which don't run in kubernetes. -ansible-playbook -i $INVENTORY_FILE -vvv $ANSIBLE_DIR/cassandra.yml -ansible-playbook -i $INVENTORY_FILE -vvv $ANSIBLE_DIR/elasticsearch.yml -ansible-playbook -i $INVENTORY_FILE -vvv $ANSIBLE_DIR/minio.yml -ansible-playbook -i $INVENTORY_FILE -vvv $ANSIBLE_DIR/restund.yml +ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/cassandra.yml +ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/elasticsearch.yml +ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/minio.yml +ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/restund.yml # create helm values that tell our helm charts what the IP addresses of cassandra, elasticsearch and minio are: -ansible-playbook -i $INVENTORY_FILE -vvv $ANSIBLE_DIR/helm_external.yml --skip-tags=rabbitmq-external +ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/helm_external.yml --skip-tags=rabbitmq-external diff --git a/offline/cd.sh b/offline/cd.sh index 8ecbe5062..6b8bee6f1 100755 --- a/offline/cd.sh +++ b/offline/cd.sh @@ -2,18 +2,16 @@ set -euo pipefail -# function cleanup { -# (cd terraform/examples/wire-server-deploy-offline-hetzner ; terraform destroy -auto-approve) -# echo done -# } -# trap cleanup EXIT +function cleanup { + (cd terraform/examples/wire-server-deploy-offline-hetzner ; terraform destroy -auto-approve) + echo done +} +trap cleanup EXIT (cd terraform/examples/wire-server-deploy-offline-hetzner ; terraform init ; terraform apply -auto-approve ) adminhost=$(cd terraform/examples/wire-server-deploy-offline-hetzner ; terraform output adminhost) adminhost="${adminhost//\"/}" # remove extra quotes around the returned string ssh_private_key=$(cd terraform/examples/wire-server-deploy-offline-hetzner ; terraform output ssh_private_key) -echo "adminhost: $adminhost" -echo "$ssh_private_key" eval `ssh-agent` ssh-add - <<< "$ssh_private_key" From c66f9f5ad16200a25af17846579e7431a7b1e4ef Mon Sep 17 00:00:00 2001 From: Amit Sagtani Date: Tue, 26 Nov 2024 16:35:59 +0100 Subject: [PATCH 16/23] add wait time for ansible operations --- ansible/ansible.cfg | 3 ++- bin/autodeploy.sh | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg index c9d74e2da..15dded1c1 100644 --- a/ansible/ansible.cfg +++ b/ansible/ansible.cfg @@ -11,7 +11,8 @@ gathering = smart host_key_checking = no interpreter_python = /usr/bin/python3 - +timeout = 30 [privilege_escalation] become = yes +become_timeout = 30 diff --git a/bin/autodeploy.sh b/bin/autodeploy.sh index 7c6bfd18b..eb189ba21 100755 --- a/bin/autodeploy.sh +++ b/bin/autodeploy.sh @@ -202,6 +202,7 @@ ansnode3 ansible_host=192.168.122.33 [all:vars] ansible_user = $DEMO_USER +ansible_ssh_common_args = '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' [cassandra:vars] cassandra_network_interface = enp1s0 From b3cb1bd47b72e1497f3a78c12d99a01262c7d13e Mon Sep 17 00:00:00 2001 From: Amit Sagtani Date: Tue, 3 Dec 2024 14:19:20 +0100 Subject: [PATCH 17/23] use lxc containers --- ansible/hetzner-single-deploy.yml | 5 +- bin/autodeploy.sh | 6 +- bin/offline-vm-stackit-setup_old.sh | 143 ++++++++++++++++++++++++++++ bin/offline-vm-stackit.sh | 134 ++++++++++++++++++++++++++ 4 files changed, 283 insertions(+), 5 deletions(-) create mode 100755 bin/offline-vm-stackit-setup_old.sh create mode 100755 bin/offline-vm-stackit.sh diff --git a/ansible/hetzner-single-deploy.yml b/ansible/hetzner-single-deploy.yml index 4d086fe32..bc347afdf 100644 --- a/ansible/hetzner-single-deploy.yml +++ b/ansible/hetzner-single-deploy.yml @@ -1,9 +1,9 @@ - hosts: all become: true vars: - artifact_hash: d8fe36747614968ea73ebd43d47b99364c52f9c1 + artifact_hash: 4188e07fd369e282f8a5083e4ede7f7a2fc0df3d ubuntu_version: 22.04.5 - ssh_pubkey: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDPTGTo1lTqd3Ym/75MRyQvj8xZINO/GI6FzfIadSe5c backend+hetzner-dedicated-operator@wire.com" + ssh_pubkey: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCvYwUBNBINxRAJWO+zSzDWhmp1he65zW7RTVgKUatku0q3NvIpYFCFbu99EsGRm6fkC5tikT6ucbft+BXbCf5RRo3od4QdmwSTZtE0lyDHGczJCohwDVTMj2vAG7V9f6u/qPu8Pw0nbQTeIyx27B9XDZjCPnvb3nBh6vP7nb5dKTEGW83brtu9rM0sqeovZicxME+PiAHJCHkijE4w1IcMEjuaBBMsSnwOp3C3L1LVc+CJDvjMGiuNquNy+GBmP1R/3vZFXyblrtQfSLo9p39QIDhpwvRS465iaMo7mdWw7RQ9oOwFKCHHzd96zlQFHWzILfgHvFyRyB5b7GZ+bX1tYi9y66JbmCvhD/oSdjQ5EU+UKGdjqklcrnLdvrZq+YFYi1pyeRcLG/5OE+5C0IpH1aOHIeXI4Dq8HDqj4HWO6yvlEXTBrNcl0MI77hd77mT5vfhq8tBKoQdrY4IAjHB/bDhojL67qDcnGX42vsOzOzO+GZA1X9g+//0ehb3e8j/XP1u0qnlzApN+8ete5uR8sqlvWMoJfH62mmthEpIrCAuUpoBSiuNY5eM4arB6Jg3M6avcOjuTGEXEU0QVRhmISAPsTNsyc6KudTl/a3Z8IEMli8jhllENEROITsi4NJyeaCTVrGmqMbndUJObQluSmI+TaKDKJDv6aIZQox6iyQ==" tasks: - name: set ipv4 forward sysctl: @@ -41,6 +41,7 @@ - telnet - python3-lxml - qemu + - lcx - qemu-kvm - qemu-utils - libvirt-clients diff --git a/bin/autodeploy.sh b/bin/autodeploy.sh index eb189ba21..212dbe9f2 100755 --- a/bin/autodeploy.sh +++ b/bin/autodeploy.sh @@ -81,9 +81,9 @@ parse_params "$@" ARTIFACT_HASH="${ARTIFACT_HASH:-5c06158547bc57846eadaa2be5c813ec43be9b59}" TARGET_SYSTEM="${TARGET_SYSTEM:-wiab-autodeploy.wire.link}" FORCE_REDEPLOY="${FORCE_REDEPLOY:-0}" -SUBDOMAINS="account assets coturn federator inbucket nginz-https nginz-ssl sft teams webapp" +SUBDOMAINS="" SSH_PORT=22 -SSH_USER=root +SSH_USER=ubuntu DEMO_USER=demo SCRIPT_DIR=/home/"$DEMO_USER"/wire-server-deploy DO_SYSTEM_CLEANUP=false @@ -160,7 +160,7 @@ remote_deployment() { } cd $SCRIPT_DIR &>/dev/null || exit 1 - bash bin/offline-vm-setup.sh + bash bin/offline-vm-stackit.sh msg "" while sudo virsh list --all | grep -Fq running; do sleep 20 diff --git a/bin/offline-vm-stackit-setup_old.sh b/bin/offline-vm-stackit-setup_old.sh new file mode 100755 index 000000000..3ff437102 --- /dev/null +++ b/bin/offline-vm-stackit-setup_old.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +msg() { + echo >&2 -e "${1-}" +} + +if [[ $EUID -eq 0 ]]; then + msg "Please don't run me as root" 1>&2 + exit 1 +fi + +trap cleanup SIGINT SIGTERM ERR EXIT + +usage() { + cat < /dev/null 2>&1; then + adduser --disabled-password --gecos '' demo + usermod -aG sudo demo + fi + mkdir -p /home/demo/.ssh + echo \"$SSH_KEY\" > /home/demo/.ssh/authorized_keys + chown -R demo:demo /home/demo/.ssh + chmod 600 /home/demo/.ssh/authorized_keys + " + + msg "Starting container..." + lxc restart "$name" +} + +for ((i = 0; i < ${#CONTAINER_NAME[@]}; i++)); do + if lxc list | grep -q "${CONTAINER_NAME[i]}"; then + msg "Container ${CONTAINER_NAME[i]} already exists. Skipping..." + continue + else + create_container "${CONTAINER_NAME[i]}" "${CONTAINER_IP[i]}" "${CONTAINER_RAM[i]}" "${CONTAINER_CPU[i]}" + fi +done \ No newline at end of file diff --git a/bin/offline-vm-stackit.sh b/bin/offline-vm-stackit.sh new file mode 100755 index 000000000..5715bd5a9 --- /dev/null +++ b/bin/offline-vm-stackit.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +msg() { + echo >&2 -e "${1-}" +} + +if [[ $EUID -eq 0 ]]; then + msg "Please don't run me as root" 1>&2 + exit 1 +fi + +trap cleanup SIGINT SIGTERM ERR EXIT + +usage() { + cat < /etc/netplan/01-netcfg.yaml + netplan apply + " + + msg "Creating demo user and adding SSH key..." + lxc exec "$name" -- bash -c " + if ! id -u demo > /dev/null 2>&1; then + adduser --disabled-password --gecos '' demo + usermod -aG sudo demo + fi + mkdir -p /home/demo/.ssh + echo \"$SSH_KEY\" > /home/demo/.ssh/authorized_keys + chown -R demo:demo /home/demo/.ssh + chmod 600 /home/demo/.ssh/authorized_keys + " + + msg "Starting container..." + lxc restart "$name" +} + +for ((i = 0; i < ${#CONTAINER_NAME[@]}; i++)); do + if lxc list | grep -q "${CONTAINER_NAME[i]}"; then + msg "Container ${CONTAINER_NAME[i]} already exists. Skipping..." + continue + else + create_container "${CONTAINER_NAME[i]}" "${CONTAINER_IP[i]}" "${CONTAINER_RAM[i]}" "${CONTAINER_CPU[i]}" + fi +done From 5c4a6d941346bba85f3564063ae144c190980bb9 Mon Sep 17 00:00:00 2001 From: Amit Sagtani Date: Tue, 3 Dec 2024 14:39:46 +0100 Subject: [PATCH 18/23] fix lint issues --- bin/offline-vm-stackit-setup_old.sh | 2 +- bin/offline-vm-stackit.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/offline-vm-stackit-setup_old.sh b/bin/offline-vm-stackit-setup_old.sh index 3ff437102..5a02813fc 100755 --- a/bin/offline-vm-stackit-setup_old.sh +++ b/bin/offline-vm-stackit-setup_old.sh @@ -43,7 +43,7 @@ parse_params() { case "${1-}" in -h | --help) usage ;; -v | --verbose) set -x ;; - --deploy-container) DEPLOY_SINGLE_CONTAINER=1 ;; + --deploy-container) ;; -?*) die "Unknown option: $1" ;; *) break ;; esac diff --git a/bin/offline-vm-stackit.sh b/bin/offline-vm-stackit.sh index 5715bd5a9..4b3c42670 100755 --- a/bin/offline-vm-stackit.sh +++ b/bin/offline-vm-stackit.sh @@ -44,7 +44,7 @@ parse_params() { case "${1-}" in -h | --help) usage ;; -v | --verbose) set -x ;; - --deploy-container) DEPLOY_SINGLE_CONTAINER=1 ;; + --deploy-container) ;; -?*) die "Unknown option: $1" ;; *) break ;; esac From 72069e6afeda3634cb78629b5c44726211b02b52 Mon Sep 17 00:00:00 2001 From: Amit Sagtani Date: Fri, 6 Dec 2024 15:56:30 +0100 Subject: [PATCH 19/23] fix lint issyes --- bin/offline-vm-stackit.sh | 42 +++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/bin/offline-vm-stackit.sh b/bin/offline-vm-stackit.sh index 4b3c42670..2ef5d900a 100755 --- a/bin/offline-vm-stackit.sh +++ b/bin/offline-vm-stackit.sh @@ -1,7 +1,6 @@ #!/usr/bin/env bash set -Eeuo pipefail - msg() { echo >&2 -e "${1-}" } @@ -18,7 +17,7 @@ usage() { Usage: $(basename "${BASH_SOURCE[0]}") [-h] [-v] [--deploy-container name] Non-interactive script for deploying a standard set of Ubuntu Server containers using LXC. -All containers are created with static IPs assigned by DHCP from the `virbr0` bridge. +All containers are created with static IPs assigned by DHCP from the $(virbr0) bridge. Available options: -h, --help Print this help and exit @@ -81,17 +80,17 @@ create_container() { local cpu=$4 msg "Creating container: $name" - lxc launch ubuntu-daily:jammy "$name" --storage default + sudo lxc launch ubuntu-daily:jammy "$name" --storage default msg "Configuring container resources..." - lxc config set "$name" limits.memory "${ram}MB" - lxc config set "$name" limits.cpu "$cpu" + sudo lxc config set "$name" limits.memory "${ram}MB" + sudo lxc config set "$name" limits.cpu "$cpu" msg "Attaching network and configuring IP via DHCP..." - lxc network attach virbr0 "$name" eth0 + sudo lxc network attach virbr0 "$name" eth0 msg "Configuring static IP for $name..." - lxc exec "$name" -- bash -c " + sudo lxc exec "$name" -- bash -c " echo 'network: version: 2 ethernets: @@ -105,11 +104,17 @@ create_container() { - 8.8.8.8 - 8.8.4.4 ' > /etc/netplan/01-netcfg.yaml + chmod 600 /etc/netplan/01-netcfg.yaml + chown root:root /etc/netplan/01-netcfg.yaml netplan apply + systemctl daemon-reload + systemctl restart systemd-networkd + apt-get install -y systemd dbus + systemctl start dbus || echo 'dbus service already running' " msg "Creating demo user and adding SSH key..." - lxc exec "$name" -- bash -c " + sudo lxc exec "$name" -- bash -c " if ! id -u demo > /dev/null 2>&1; then adduser --disabled-password --gecos '' demo usermod -aG sudo demo @@ -118,14 +123,31 @@ create_container() { echo \"$SSH_KEY\" > /home/demo/.ssh/authorized_keys chown -R demo:demo /home/demo/.ssh chmod 600 /home/demo/.ssh/authorized_keys + echo 'demo ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/demo + chmod 440 /etc/sudoers.d/demo " msg "Starting container..." - lxc restart "$name" + sudo lxc restart "$name" } +sudo systemctl start snap.lxd.daemon +sudo systemctl enable snap.lxd.daemon +#sudo usermod -aG lxd "$USER" +#newgrp lxd + +STORAGE_NAME="default" + +# Check if the storage pool already exists +if sudo lxc storage list --format csv | grep -q "^$STORAGE_NAME,"; then + echo "Storage pool '$STORAGE_NAME' already exists. Skipping creation." +else + echo "Storage pool '$STORAGE_NAME' does not exist. Creating it..." + sudo lxc storage create "$STORAGE_NAME" dir +fi + for ((i = 0; i < ${#CONTAINER_NAME[@]}; i++)); do - if lxc list | grep -q "${CONTAINER_NAME[i]}"; then + if sudo lxc list | grep -q "${CONTAINER_NAME[i]}"; then msg "Container ${CONTAINER_NAME[i]} already exists. Skipping..." continue else From fdb44fa7306101be133aeedccf8b2474d4b63343 Mon Sep 17 00:00:00 2001 From: Amit Sagtani Date: Mon, 9 Dec 2024 14:51:05 +0100 Subject: [PATCH 20/23] add missing commands --- bin/offline-vm-stackit.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bin/offline-vm-stackit.sh b/bin/offline-vm-stackit.sh index 2ef5d900a..13f152945 100755 --- a/bin/offline-vm-stackit.sh +++ b/bin/offline-vm-stackit.sh @@ -126,6 +126,10 @@ create_container() { echo 'demo ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/demo chmod 440 /etc/sudoers.d/demo " + sudo lxc config set "$name" security.privileged true + sudo lxc config set "$name" linux.kernel_modules overlay,nf_nat,br_netfilter,ip_tables + lxc config device add "$name" modules disk source=/lib/modules path=/lib/modules + msg "Starting container..." sudo lxc restart "$name" @@ -135,7 +139,10 @@ sudo systemctl start snap.lxd.daemon sudo systemctl enable snap.lxd.daemon #sudo usermod -aG lxd "$USER" #newgrp lxd - +sudo apt update +# shellcheck disable=SC2046 +sudo apt install --reinstall linux-modules-$(uname -r) linux-modules-extra-$(uname -r) -y +sudo modprobe ip_vs ip_vs_rr ip_vs_wrr ip_vs_lc ip_vs_wlc ip_vs_sh nf_conntrack STORAGE_NAME="default" # Check if the storage pool already exists From 1d4a88c36d133f2cb98dad8681f85b96400e0a7b Mon Sep 17 00:00:00 2001 From: Amit Sagtani Date: Mon, 9 Dec 2024 17:21:17 +0100 Subject: [PATCH 21/23] fix missing sudo --- bin/offline-vm-stackit.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/offline-vm-stackit.sh b/bin/offline-vm-stackit.sh index 13f152945..d3290dd9b 100755 --- a/bin/offline-vm-stackit.sh +++ b/bin/offline-vm-stackit.sh @@ -128,11 +128,11 @@ create_container() { " sudo lxc config set "$name" security.privileged true sudo lxc config set "$name" linux.kernel_modules overlay,nf_nat,br_netfilter,ip_tables - lxc config device add "$name" modules disk source=/lib/modules path=/lib/modules + sudo lxc restart "$name" + sudo lxc config device add "$name" modules disk source=/lib/modules path=/lib/modules msg "Starting container..." - sudo lxc restart "$name" } sudo systemctl start snap.lxd.daemon From f543e4c82242bcdb9a144956dcd5057386461e00 Mon Sep 17 00:00:00 2001 From: Amit Sagtani Date: Thu, 12 Dec 2024 15:56:51 +0100 Subject: [PATCH 22/23] add missing commands --- bin/offline-vm-stackit.sh | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/bin/offline-vm-stackit.sh b/bin/offline-vm-stackit.sh index d3290dd9b..1056734aa 100755 --- a/bin/offline-vm-stackit.sh +++ b/bin/offline-vm-stackit.sh @@ -130,6 +130,8 @@ create_container() { sudo lxc config set "$name" linux.kernel_modules overlay,nf_nat,br_netfilter,ip_tables sudo lxc restart "$name" sudo lxc config device add "$name" modules disk source=/lib/modules path=/lib/modules + sudo lxc config device add "$name" kmsg unix-char path=/dev/kmsg + sudo lxc config device add "$name" proc-sys disk source=/proc/sys path=/proc/sys msg "Starting container..." @@ -142,7 +144,24 @@ sudo systemctl enable snap.lxd.daemon sudo apt update # shellcheck disable=SC2046 sudo apt install --reinstall linux-modules-$(uname -r) linux-modules-extra-$(uname -r) -y -sudo modprobe ip_vs ip_vs_rr ip_vs_wrr ip_vs_lc ip_vs_wlc ip_vs_sh nf_conntrack +sudo modprobe ip_vs +sudo modprobe ip_vs_rr +sudo modprobe ip_vs_wrr +sudo modprobe ip_vs_lc +sudo modprobe ip_vs_wlc +sudo modprobe ip_vs_sh +sudo modprobe nf_conntrack +sudo modprobe dummy +sudo modprobe ip_tables +sudo modprobe br_netfilter +sudo modprobe ip6_tables +sudo modprobe ip_vs_sed +sudo modprobe bridge +sudo modprobe vxlan +sudo modprobe overlay +sudo sysctl -w kernel.panic=10 +sudo sysctl -w kernel.panic_on_oops=1 +sudo sysctl -w vm.overcommit_memory=1 STORAGE_NAME="default" # Check if the storage pool already exists From e09be4ca90f175acc56b44217dc86bfd5309ff3d Mon Sep 17 00:00:00 2001 From: Amit Sagtani Date: Sun, 22 Dec 2024 03:41:46 +0100 Subject: [PATCH 23/23] temp changes --- ansible/hetzner-single-deploy.yml | 14 +++++++------- bin/autodeploy.sh | 28 ++++++++++++++-------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/ansible/hetzner-single-deploy.yml b/ansible/hetzner-single-deploy.yml index bc347afdf..61a3a92cb 100644 --- a/ansible/hetzner-single-deploy.yml +++ b/ansible/hetzner-single-deploy.yml @@ -1,7 +1,7 @@ - hosts: all become: true vars: - artifact_hash: 4188e07fd369e282f8a5083e4ede7f7a2fc0df3d + artifact_hash: 3a18fae3106968b48b5a3b35f1bd0817ca4e5569 ubuntu_version: 22.04.5 ssh_pubkey: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCvYwUBNBINxRAJWO+zSzDWhmp1he65zW7RTVgKUatku0q3NvIpYFCFbu99EsGRm6fkC5tikT6ucbft+BXbCf5RRo3od4QdmwSTZtE0lyDHGczJCohwDVTMj2vAG7V9f6u/qPu8Pw0nbQTeIyx27B9XDZjCPnvb3nBh6vP7nb5dKTEGW83brtu9rM0sqeovZicxME+PiAHJCHkijE4w1IcMEjuaBBMsSnwOp3C3L1LVc+CJDvjMGiuNquNy+GBmP1R/3vZFXyblrtQfSLo9p39QIDhpwvRS465iaMo7mdWw7RQ9oOwFKCHHzd96zlQFHWzILfgHvFyRyB5b7GZ+bX1tYi9y66JbmCvhD/oSdjQ5EU+UKGdjqklcrnLdvrZq+YFYi1pyeRcLG/5OE+5C0IpH1aOHIeXI4Dq8HDqj4HWO6yvlEXTBrNcl0MI77hd77mT5vfhq8tBKoQdrY4IAjHB/bDhojL67qDcnGX42vsOzOzO+GZA1X9g+//0ehb3e8j/XP1u0qnlzApN+8ete5uR8sqlvWMoJfH62mmthEpIrCAuUpoBSiuNY5eM4arB6Jg3M6avcOjuTGEXEU0QVRhmISAPsTNsyc6KudTl/a3Z8IEMli8jhllENEROITsi4NJyeaCTVrGmqMbndUJObQluSmI+TaKDKJDv6aIZQox6iyQ==" tasks: @@ -41,7 +41,7 @@ - telnet - python3-lxml - qemu - - lcx + - lxc - qemu-kvm - qemu-utils - libvirt-clients @@ -51,7 +51,7 @@ - name: generate german locales locale_gen: name: de_DE.UTF-8 - state: present + state: present - name: generate us locales locale_gen: name: en_US.UTF-8 @@ -101,8 +101,8 @@ src: files/hetzner_server_sshd_config dest: /etc/ssh/sshd_config mode: 0644 - owner: root - group: root + owner: ubuntu + group: ubuntu notify: sshd | restart - name: stop and disable dnsmasq service service: @@ -149,8 +149,8 @@ src: files/hetzner_server_nftables.conf.j2 dest: /etc/nftables.conf mode: 0750 - owner: root - group: root + owner: ubuntu + group: ubuntu notify: nftables | restart when: not nft_deployed.stat.exists - name: add local file flag after nftables deployment diff --git a/bin/autodeploy.sh b/bin/autodeploy.sh index 212dbe9f2..3dcacf9e3 100755 --- a/bin/autodeploy.sh +++ b/bin/autodeploy.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # shellcheck disable=SC2087 set -Eeuo pipefail - +set -x msg() { echo >&2 -e "${1-}" } @@ -160,18 +160,18 @@ remote_deployment() { } cd $SCRIPT_DIR &>/dev/null || exit 1 - bash bin/offline-vm-stackit.sh - msg "" - while sudo virsh list --all | grep -Fq running; do - sleep 20 - msg "INFO: VM deployment still in progress ..." - done - sleep 20 - msg "" - msg "INFO: VM deployment done. Starting all VMs:" - msg "" - for VM in $(sudo virsh list --all --name); do sudo virsh start "$VM"; done - sleep 60 + #bash bin/offline-vm-stackit.sh + # msg "" + # while sudo virsh list --all | grep -Fq running; do + # sleep 20 + # msg "INFO: VM deployment still in progress ..." + # done + # sleep 20 + # msg "" + # msg "INFO: VM deployment done. Starting all VMs:" + # msg "" + # for VM in $(sudo virsh list --all --name); do sudo virsh start "$VM"; done + # sleep 60 msg "" msg "INFO: Setting up offline environment (this will take a while)." @@ -418,7 +418,7 @@ if [ "$DO_SYSTEM_CLEANUP" = true ] && [ "$FORCE_REDEPLOY" = 1 ]; then fi msg "INFO: Commencing Wire-in-a-box deployment on $TARGET_SYSTEM." -preprovision_hetzner +#preprovision_hetzner ssh -p "$SSH_PORT" -o StrictHostKeyChecking=no -o ServerAliveInterval=30 -o ServerAliveCountMax=10 "$DEMO_USER"@webapp."$TARGET_SYSTEM" "bash -s" <