SN10

.github/workflows/deploy_environments.yaml

@@ -26,3 +26,12 @@ jobs:
       environment: prod1
     secrets: inherit
     if: github.ref == 'refs/heads/main'
+
+  deploy_prod2:
+    name: Deploy prod2
+    needs: deploy_prod1
+    uses: ./.github/workflows/deploy_k8s_cluster.yaml
+    with:
+      environment: prod2
+    secrets: inherit
+    if: github.ref == 'refs/heads/main'

ansible/roles/k8s-lb/tasks/main.yaml

@@ -42,6 +42,13 @@
     dest: /etc/netplan/dummy0.yaml
     mode: "600"
 
+- name: dummy1 interface
+  ansible.builtin.template:
+    src: ../templates/netplan_dummy1.yaml.j2
+    dest: /etc/netplan/dummy1.yaml
+    mode: "600"
+  when: EXTERNAL_LISTEN_IPS | split(';') | length == 2
+
 - name: eth0 interface
   ansible.builtin.template:
     src: ../templates/netplan_50_cloud_init.yaml.j2

ansible/roles/k8s-lb/templates/frr.conf.j2

@@ -12,6 +12,8 @@ exit
 !
 router ospf
  network {{ INTERNAL_NETWORK_BLOCK }} area 0
- network {{ EXTERNAL_LISTEN_IP }}/32 area 0
+{% for external_ip in EXTERNAL_LISTEN_IPS.split(';') %}
+ network {{ external_ip }}/32 area 0
+{% endfor %}
 exit
 !

ansible/roles/k8s-lb/templates/haproxy.cfg

@@ -34,8 +34,10 @@ defaults
 	errorfile 504 /etc/haproxy/errors/504.http
 
 frontend meshdb
-  bind {{ EXTERNAL_LISTEN_IP }}:80
-  bind {{ EXTERNAL_LISTEN_IP }}:443 ssl crt /etc/haproxy/ssl/
+{% for external_ip in EXTERNAL_LISTEN_IPS.split(';') %}
+  bind {{ external_ip }}:80
+  bind {{ external_ip }}:443 ssl crt /etc/haproxy/ssl/
+{% endfor %}
 
   # By default, pass reauests to k3s
   default_backend k8s

ansible/roles/k8s-lb/templates/iptables.j2

@@ -2,11 +2,25 @@
 :INPUT ACCEPT [0:0]
 :FORWARD ACCEPT [0:0]
 :OUTPUT ACCEPT [0:0]
+
+# Allow ICMP traffic from Mesh
+-A INPUT -p icmp -s 10.0.0.0/8 -j ACCEPT
+-A INPUT -p icmp -s 23.158.16.0/24 -j ACCEPT
+-A INPUT -p icmp -s 199.167.59.0/24 -j ACCEPT
+-A INPUT -p icmp -s 199.170.132.0/24 -j ACCEPT
+-A INPUT -p icmp -s 208.68.5.0/24 -j ACCEPT
+
+# Allow SSH to the mgt address only
 -A INPUT -d {{ inventory_hostname }}/{{ INTERNAL_NETWORK_RANGE }} -p tcp -m tcp --dport 22 -j ACCEPT
--A INPUT -d {{ EXTERNAL_LISTEN_IP }}/32 -p tcp -m tcp --dport 80 -j ACCEPT
--A INPUT -d {{ EXTERNAL_LISTEN_IP }}/32 -p tcp -m tcp --dport 443 -j ACCEPT
--A INPUT -d {{ EXTERNAL_LISTEN_IP }}/32 -j DROP
--A FORWARD -d {{ EXTERNAL_LISTEN_IP }}/32 -p tcp -m tcp --dport 80 -j ACCEPT
--A FORWARD -d {{ EXTERNAL_LISTEN_IP }}/32 -p tcp -m tcp --dport 443 -j ACCEPT
--A FORWARD -d {{ EXTERNAL_LISTEN_IP }}/32 -j DROP
+
+# Allow HTTP and HTTPS traffic
+{% for external_ip in EXTERNAL_LISTEN_IPS.split(';') %}
+-A INPUT -d {{ external_ip }}/32 -p tcp -m tcp --dport 80 -j ACCEPT
+-A INPUT -d {{ external_ip }}/32 -p tcp -m tcp --dport 443 -j ACCEPT
+-A INPUT -d {{ external_ip }}/32 -j DROP
+-A FORWARD -d {{ external_ip }}/32 -p tcp -m tcp --dport 80 -j ACCEPT
+-A FORWARD -d {{ external_ip }}/32 -p tcp -m tcp --dport 443 -j ACCEPT
+-A FORWARD -d {{ external_ip }}/32 -j DROP
+{% endfor %}
+
 COMMIT

ansible/roles/k8s-lb/templates/netplan_dummy0.yaml.j2

@@ -6,4 +6,4 @@ network:
       dhcp4: no
       dhcp6: no
       addresses:
-        - {{ EXTERNAL_LISTEN_IP }}/32
+        - {{ EXTERNAL_LISTEN_IPS.split(';')[0] }}/32

ansible/roles/k8s-lb/templates/netplan_dummy1.yaml.j2

@@ -0,0 +1,9 @@
+network:
+  version: 2
+  renderer: networkd
+  ethernets:
+    lo:
+      dhcp4: no
+      dhcp6: no
+      addresses:
+        - {{ EXTERNAL_LISTEN_IPS.split(';')[1] }}/32

terraform/cluster.tf

@@ -10,6 +10,9 @@ module "some_mesh_cluster" {
   mesh_mgr_cores                = var.mesh_mgr_cores
   mesh_mgr_ram_mb               = var.mesh_mgr_ram_mb
   mesh_mgr_disk_size            = var.mesh_mgr_disk_size
+  mesh_lb_cores                 = var.mesh_lb_cores
+  mesh_lb_ram_mb                = var.mesh_lb_ram_mb
+  mesh_lb_disk_size             = var.mesh_lb_disk_size
   mesh_ips                      = var.mesh_ips
   mesh_worker_cores             = var.mesh_worker_cores
   mesh_worker_ram_mb            = var.mesh_worker_ram_mb
@@ -18,7 +21,7 @@ module "some_mesh_cluster" {
   mesh_gateway                  = var.mesh_gateway
   mesh_networkrange             = var.mesh_networkrange
   mesh_net_block                = var.mesh_net_block
-  mesh_external_ip              = var.mesh_external_ip
+  mesh_external_ips             = var.mesh_external_ips
   meshdb_fqdn                   = var.meshdb_fqdn
   mesh_local_password           = var.mesh_local_password
   longhorn_passphrase           = var.longhorn_passphrase

terraform/dev3.tfvars

@@ -43,6 +43,7 @@ mesh_lb_ip        = "10.70.90.145"
 mesh_gateway      = "10.70.90.1"
 mesh_networkrange = "24"
 mesh_net_block    = "10.70.90.0"
-mesh_external_ip  = "199.170.132.46"
-# Add domains to the end
+mesh_external_ips = [
+  "199.170.132.46",
+]
 meshdb_fqdn = "devdb.nycmesh.net,map.devdb.nycmesh.net,adminmap.devdb.nycmesh.net,los-backend.devdb.nycmesh.net,los.devdb.nycmesh.net,forms.devdb.nycmesh.net,devmap.nycmesh.net"

terraform/mesh_cluster/ansible.tf

@@ -28,12 +28,12 @@ resource "ansible_group" "lb" {
     ansible_user                 = var.mesh_local_user
     ansible_ssh_private_key_file = "../terraform/${path.module}/mesh${var.mesh_env_name}"
     ansible_ssh_common_args      = "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
-    EXTERNAL_LISTEN_IP           = var.mesh_external_ip
+    EXTERNAL_LISTEN_IPS          = join(";", var.mesh_external_ips)
     LB_HOSTNAME                  = "k8s-lb-${var.mesh_env_name}"
     INTERNAL_NETWORK_BLOCK       = format("%s/%s", var.mesh_net_block, var.mesh_networkrange)
     INTERNAL_NETWORK_RANGE       = var.mesh_networkrange
     WORKER_IPS                   = join(";", var.mesh_ips)
-    NODE_PORT                    = "30303"
+    NODE_PORT                    = "80"
     MESHDB_FQDN                  = var.meshdb_fqdn
     MESH_DG                      = var.mesh_gateway
     DATADOG_API_KEY              = var.DATADOG_API_KEY
@@ -48,7 +48,7 @@ resource "ansible_host" "meshmgr" {
   groups = [ansible_group.mgrs.name]
   variables = {
     K3S_API_ENDPOINT    = var.mesh_mgr_ips[0]
-    TRAEFIK_TRUSTED_IPs = format("%s/32,%s/32,10.42.0.0/16", var.mesh_lb_ip, var.mesh_external_ip)
+    TRAEFIK_TRUSTED_IPs = format("%s/32,%s/32,10.42.0.0/16%s", var.mesh_lb_ip, var.mesh_external_ips[0], length(var.mesh_external_ips) > 1 ? format(",%s/32", var.mesh_external_ips[1]) : "")
   }
 }
 

terraform/mesh_cluster/lb.tf

@@ -5,9 +5,9 @@ resource "proxmox_vm_qemu" "k8s-lb" {
 
   clone = var.mesh_proxmox_template_image
 
-  cores                   = 2
+  cores                   = var.mesh_lb_cores
   sockets                 = 1
-  memory                  = 2560
+  memory                  = var.mesh_lb_ram_mb
   os_type                 = "cloud-init"
   agent                   = 1
   cloudinit_cdrom_storage = var.mesh_proxmox_storage_location
@@ -21,7 +21,7 @@ resource "proxmox_vm_qemu" "k8s-lb" {
       scsi0 {
         disk {
           backup  = false
-          size    = 10
+          size    = var.mesh_lb_disk_size
           storage = var.mesh_proxmox_storage_location
 
         }

terraform/mesh_cluster/vars.tf

@@ -65,8 +65,8 @@ variable "mesh_net_block" {
   default     = "10.70.90.0"
 }
 
-variable "mesh_external_ip" {
-  type        = string
+variable "mesh_external_ips" {
+  type        = list(string)
   description = "external ip for meshdb, assigned to the lb vm"
 }
 
@@ -128,3 +128,21 @@ variable "mesh_worker_disk_size" {
   type        = list(number)
   description = "list of disk sizes for worker node"
 }
+
+variable "mesh_lb_cores" {
+  type        = number
+  description = "number of cores for the lb"
+  default     = 2
+}
+
+variable "mesh_lb_ram_mb" {
+  type        = number
+  description = "ram size for the lb in mb"
+  default     = 2560
+}
+
+variable "mesh_lb_disk_size" {
+  type        = number
+  description = "disk size for the lb in gb"
+  default     = 10
+}

terraform/prod1.tfvars

@@ -44,6 +44,7 @@ mesh_lb_ip        = "10.70.90.156"
 mesh_gateway      = "10.70.90.1"
 mesh_networkrange = "24"
 mesh_net_block    = "10.70.90.0"
-mesh_external_ip  = "199.170.132.45"
-# Add domains to the end
-meshdb_fqdn = "wiki.nycmesh.net,wiki.mesh.nycmesh.net,db.nycmesh.net,map.db.nycmesh.net,adminmap.db.nycmesh.net,los-backend.db.nycmesh.net,los.nycmesh.net,forms.nycmesh.net,stats-new.nycmesh.net,jmstemp.mesh.nycmesh.net,map.nycmesh.net,stats.nycmesh.net"
+mesh_external_ips = [
+  "199.170.132.45",
+]
+meshdb_fqdn = "wiki.nycmesh.net,wiki.mesh.nycmesh.net,los-backend.db.nycmesh.net"

terraform/prod2.tfvars

@@ -0,0 +1,49 @@
+mesh_proxmox_node             = "nycmesh-10-r630-01"
+mesh_proxmox_template_image   = "debian-cloud"
+mesh_proxmox_storage_location = "local-lvm"
+mesh_env_name                 = "prod2"
+mesh_local_user               = "debian"
+vm_nic                        = "vmbr1"
+mesh_mgr_ips = [
+  "10.70.100.47",
+]
+mesh_mgr_cores = [
+  4,
+]
+mesh_mgr_ram_mb = [
+  6144,
+]
+mesh_mgr_disk_size = [
+  80,
+]
+mesh_ips = [
+  "10.70.100.48",
+  "10.70.100.49",
+  "10.70.100.50",
+]
+mesh_worker_cores = [
+  4,
+  4,
+  4,
+]
+mesh_worker_ram_mb = [
+  4096,
+  4096,
+  4096,
+]
+mesh_worker_disk_size = [
+  150,
+  150,
+  150,
+]
+mesh_lb_ip        = "10.70.100.51"
+mesh_lb_cores     = 2
+mesh_lb_ram_mb    = 6144
+mesh_lb_disk_size = 10
+mesh_gateway      = "10.70.100.1"
+mesh_networkrange = "24"
+mesh_net_block    = "10.70.100.0"
+mesh_external_ips = [
+  "23.158.16.22",
+]
+meshdb_fqdn = "db.nycmesh.net,map.db.nycmesh.net,adminmap.db.nycmesh.net,los.nycmesh.net,forms.nycmesh.net,stats-new.nycmesh.net,map.nycmesh.net,stats.nycmesh.net"

terraform/vars.tf

@@ -99,8 +99,8 @@ variable "mesh_net_block" {
   default     = "10.70.90.0"
 }
 
-variable "mesh_external_ip" {
-  type        = string
+variable "mesh_external_ips" {
+  type        = list(string)
   description = "external ip for meshdb, assigned to the lb vm"
 }
 
@@ -144,3 +144,21 @@ variable "mesh_worker_disk_size" {
   type        = list(number)
   description = "list of disk sizes for worker node"
 }
+
+variable "mesh_lb_cores" {
+  type        = number
+  description = "number of cores for the lb"
+  default     = 2
+}
+
+variable "mesh_lb_ram_mb" {
+  type        = number
+  description = "ram size for the lb in mb"
+  default     = 2560
+}
+
+variable "mesh_lb_disk_size" {
+  type        = number
+  description = "disk size for the lb in gb"
+  default     = 10
+}