metal-stack · iljarotar · Jun 6, 2024 · Jun 6, 2024 · Jun 13, 2024 · Jul 4, 2024
@@ -21,6 +21,7 @@ jobs:
         flavors:
           - name: cumulus
           - name: sonic
+          - name: mixed
 
     steps:
     - name: Gain back workspace permissions # https://github.com/actions/checkout/issues/211

@@ -25,11 +25,18 @@ MAX_RETRIES := 30
 ifeq ($(MINI_LAB_FLAVOR),cumulus)
 LAB_MACHINES=machine01,machine02
 LAB_TOPOLOGY=mini-lab.cumulus.yaml
+LAB_SWITCHES=leaf01 leaf02
 VRF=vrf20
 else ifeq ($(MINI_LAB_FLAVOR),sonic)
 LAB_MACHINES=machine01,machine02
 LAB_TOPOLOGY=mini-lab.sonic.yaml
+LAB_SWITCHES=leaf01 leaf02
 VRF=Vrf20
+else ifeq ($(MINI_LAB_FLAVOR),mixed)
+LAB_MACHINES=machine01,machine02
+LAB_TOPOLOGY=mini-lab.mixed.yaml
+LAB_SWITCHES=leaf01 leaf02 leaf02-sonic
+VRF=vrf20
 else
 $(error Unknown flavor $(MINI_LAB_FLAVOR))
 endif
@@ -54,8 +61,7 @@ up: env control-plane-bake partition-bake
 # without restarting the metal-core
 # TODO: should be investigated and fixed if possible
 	sleep 10
-	ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o "PubkeyAcceptedKeyTypes +ssh-rsa" root@leaf01 -i files/ssh/id_rsa 'systemctl restart metal-core'
-	ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o "PubkeyAcceptedKeyTypes +ssh-rsa" root@leaf02 -i files/ssh/id_rsa 'systemctl restart metal-core'
+	@$(MAKE) --no-print-directory restart-metal-core
 
 .PHONY: restart
 restart: down up
@@ -83,9 +89,7 @@ partition: partition-bake
 .PHONY: partition-bake
 partition-bake: external_network
 	docker pull $(MINI_LAB_VM_IMAGE)
-ifeq ($(MINI_LAB_FLAVOR),sonic)
 	docker pull $(MINI_LAB_SONIC_IMAGE)
-endif
 	@if ! sudo $(CONTAINERLAB) --topo $(LAB_TOPOLOGY) inspect | grep -i leaf01 > /dev/null; then \
 		sudo --preserve-env $(CONTAINERLAB) deploy --topo $(LAB_TOPOLOGY) --reconfigure && \
 		./scripts/deactivate_offloading.sh; fi
@@ -120,6 +124,7 @@ cleanup-partition:
 	mkdir -p clab-mini-lab
 	sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.cumulus.yaml
 	sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.sonic.yaml
+	sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.mixed.yaml
 	docker network rm --force mini_lab_ext
 
 .PHONY: _privatenet
@@ -166,6 +171,20 @@ ssh-leaf01:
 ssh-leaf02:
 	ssh -o StrictHostKeyChecking=no -o "PubkeyAcceptedKeyTypes +ssh-rsa" -i files/ssh/id_rsa root@leaf02
 
+.PHONY: restart-metal-core
+restart-metal-core:
+	$(foreach host,$(LAB_SWITCHES),ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o "PubkeyAcceptedKeyTypes +ssh-rsa" root@$(host) -i files/ssh/id_rsa 'systemctl restart metal-core';)
+
+.PHONY: migrate-cumulus-to-sonic
+migrate-cumulus-to-sonic:
+	docker exec vms /mini-lab/migrate.sh tap1 lan1 lan4
+	docker exec vms /mini-lab/migrate.sh tap3 lan3 lan5
+
+.PHONY: migrate-sonic-to-cumulus
+migrate-sonic-to-cumulus:
+	docker exec vms /mini-lab/migrate.sh tap1 lan4 lan1
+	docker exec vms /mini-lab/migrate.sh tap3 lan5 lan3
+
 ## MACHINE MANAGEMENT ##
 
 .PHONY: start-machines

@@ -11,8 +11,8 @@ The mini-lab is a small, virtual setup to locally run the metal-stack. It deploy
 - [Requirements](#requirements)
 - [Known Limitations](#known-limitations)
 - [Try it out](#try-it-out)
-    - [Reinstall machine](#reinstall-machine)
-    - [Free machine](#free-machine)
+  - [Reinstall machine](#reinstall-machine)
+  - [Free machine](#free-machine)
 - [Flavors](#flavors)
 
 <!-- /TOC -->
@@ -29,7 +29,7 @@ The mini-lab is a small, virtual setup to locally run the metal-stack. It deploy
 
 Here is some code that should help you to set up most of the requirements:
 
- ```bash
+```bash
 # If UFW enabled.
 # Disable the firewall or allow traffic through Docker network IP range.
 sudo ufw status
@@ -58,11 +58,11 @@ sudo chmod +x /usr/local/bin/kind
 The following ports are used statically on your host machine:
 
 | Port | Bind Address | Description                        |
-|:----:|:------------ |:---------------------------------- |
-| 6443 |   0.0.0.0    | kube-apiserver of the kind cluster |
-| 4443 |   0.0.0.0    | HTTPS ingress                      |
-| 4150 |   0.0.0.0    | nsqd                               |
-| 8080 |   0.0.0.0    | HTTP ingress                       |
+| :--: | :----------- | :--------------------------------- |
+| 6443 | 0.0.0.0      | kube-apiserver of the kind cluster |
+| 4443 | 0.0.0.0      | HTTPS ingress                      |
+| 4150 | 0.0.0.0      | nsqd                               |
+| 8080 | 0.0.0.0      | HTTP ingress                       |
 
 ## Known Limitations
 
@@ -111,7 +111,7 @@ make firewall
 make machine
 ```
 
-__Alternatively__, you may want to issue the `metalctl` commands on your own:
+**Alternatively**, you may want to issue the `metalctl` commands on your own:
 
 ```bash
 docker compose run --rm metalctl network allocate \
@@ -197,10 +197,26 @@ There are two versions, or flavors, of the mini-lab environment which differ in
 
 - `cumulus` -- runs 2 Cumulus switches.
 - `sonic` -- runs 2 SONiC switches
+- `mixed` -- starts 2 Cumulus switches and one SONiC switch, where the SONiC switch is not connected to any machines yet. Read the `Switch migration` section to see how this setup can be used to simulate switch migration from Cumulus to SONiC or vice versa.
 
 In order to start specific flavor, you can define the flavor as follows:
 
 ```bash
 export MINI_LAB_FLAVOR=sonic
 make
 ```
+
+# Switch migration
+
+The strategy for migrating a Cumulus rack to SONiC without downtime involves migrating one switch first creating a mixed setup with Cumulus and SONiC running alongside each other and then migrating the second one.
+To simulate the first half of this procedure in the mini-lab, the following steps are performed:
+
+- start the mini-lab with the `mixed` flavor.
+- run `eval $(make dev-env)` to get access to the lab via `metalctl`.
+- run `metalctl switch migrate mini-lab-leaf02 leaf02-sonic` to adjust machine connections in the database.
+- run `make migrate-cumulus-to-sonic` to adjust the redirect rules in the `vms` container. This simulates changing the cables of the machines from one switch to another.
+- run `make firewall machine` to allocate a firewall and a machine
+
+Wait for a few minutes and run `metalctl machine ls`. If everything works as expected you should see that both machines have `Phoned Home`.
+
+Although migrating from Cumulus to SONiC is the more important use case, the other direction is supported as well. Just run `metalctl switch migrate leaf02-sonic mini-lab-leaf02` followed by `make migrate-sonic-to-cumulus`.
@@ -85,6 +85,6 @@
       retries: 60
       delay: 3
       until:
-        - lookup('metal', 'search', 'switch', api_url=metal_partition_metal_api_protocol+'://'+metal_partition_metal_api_addr+':'+metal_partition_metal_api_port|string+metal_partition_metal_api_basepath, api_hmac=metal_partition_metal_api_hmac_edit_key) | length == 2
+        - lookup('metal', 'search', 'switch', api_url=metal_partition_metal_api_protocol+'://'+metal_partition_metal_api_addr+':'+metal_partition_metal_api_port|string+metal_partition_metal_api_basepath, api_hmac=metal_partition_metal_api_hmac_edit_key) | length >= 2
         - lookup('metal', 'search', 'switch', api_url=metal_partition_metal_api_protocol+'://'+metal_partition_metal_api_addr+':'+metal_partition_metal_api_port|string+metal_partition_metal_api_basepath, api_hmac=metal_partition_metal_api_hmac_edit_key)[0]["last_sync"] != None
         - lookup('metal', 'search', 'switch', api_url=metal_partition_metal_api_protocol+'://'+metal_partition_metal_api_addr+':'+metal_partition_metal_api_port|string+metal_partition_metal_api_basepath, api_hmac=metal_partition_metal_api_hmac_edit_key)[1]["last_sync"] != None
@@ -41,25 +41,25 @@
   "PORT": {
     "Ethernet0": {
       "lanes": "25,26,27,28",
-      "alias": "fortyGigE0/0",
-      "index": "0",
-      "speed": "40000",
+      "alias": "Eth1/1(Port1)",
+      "index": "1",
+      "speed": "25000",
       "admin_status": "up",
       "mtu": "9100"
     },
     "Ethernet4": {
       "lanes": "29,30,31,32",
-      "alias": "fortyGigE0/4",
-      "index": "1",
-      "speed": "40000",
+      "alias": "Eth2/1(Port2)",
+      "index": "2",
+      "speed": "25000",
       "admin_status": "up",
       "mtu": "9100"
     },
     "Ethernet120": {
       "lanes": "33,34,35,36",
-      "alias": "fortyGigE0/8",
-      "index": "2",
-      "speed": "40000",
+      "alias": "Eth31(Port31)",
+      "index": "31",
+      "speed": "100000",
       "admin_status": "up",
       "mtu": "9100"
     }

@@ -7,7 +7,7 @@ metal_stack_release_version: develop
 
 # metal_hammer_image_url: https://images.metal-stack.io/metal-hammer/pull-requests/<pr-number-and-title>/metal-hammer-initrd.img.lz4
 # metal_api_image_name:
-# metal_api_image_tag:
+metal_api_image_tag: switch-migrate
 # metal_metalctl_image_name:
 # metal_metalctl_image_tag:
 # metal_masterdata_api_image_name:

@@ -0,0 +1,15 @@
+lo: 10.0.0.11
+asn: 4200000011
+metal_core_cidr: 10.0.1.1/{{ metal_core_cidr_mask }}
+
+dhcp_default_lease_time: 600
+dhcp_max_lease_time: 600
+dhcp_subnets:
+  - network: 10.0.1.0
+    netmask: 255.255.255.0
+    range:
+      begin: 10.0.1.2
+      end: 10.0.1.255
+    options:
+      - routers {{ dhcp_server_ip }}
+      - domain-name-servers 1.1.1.1, 8.8.8.8
@@ -0,0 +1,3 @@
+lo: 10.0.0.13
+asn: 4200000013
+metal_core_cidr: 10.0.1.255/{{ metal_core_cidr_mask }}
@@ -0,0 +1,3 @@
+lo: 10.0.0.12
+asn: 4200000012
+metal_core_cidr: 10.0.1.128/{{ metal_core_cidr_mask }}
@@ -10,27 +10,9 @@ partition:
     leaves:
 
 leaves:
-  hosts:
-    leaf01:
-      lo: 10.0.0.11
-      asn: 4200000011
-      metal_core_cidr: 10.0.1.1/{{ metal_core_cidr_mask }}
-
-      dhcp_default_lease_time: 600
-      dhcp_max_lease_time: 600
-      dhcp_subnets:
-        - network: 10.0.1.0
-          netmask: 255.255.255.0
-          range:
-            begin: 10.0.1.2
-            end: 10.0.1.255
-          options:
-            - routers {{ dhcp_server_ip }}
-            - domain-name-servers 1.1.1.1, 8.8.8.8
-    leaf02:
-      lo: 10.0.0.12
-      asn: 4200000012
-      metal_core_cidr: 10.0.1.128/{{ metal_core_cidr_mask }}
+  children:
+    cumulus:
+    sonic:
   vars:
     pixie_server_ip: 10.0.1.1
     dhcp_server_ip: 10.0.1.1

@@ -0,0 +1,74 @@
+name: mini-lab
+prefix: ""
+
+mgmt:
+  network: bridge
+
+topology:
+  kinds:
+    cvx:
+      image: networkop/cx:3.7.0
+      kernel: docker.io/grigoriymikh/kernel:4.1.0
+      sandbox: grigoriymikh/sandbox:latest
+      labels:
+        ansible-group: cumulus
+      binds:
+        - apt-transport-https.tar.gz:/root/jessie-apt-transport-fix.tar.gz
+        - files/ssh/id_rsa.pub:/root/.ssh/authorized_keys
+    linux:
+      image: ${MINI_LAB_VM_IMAGE}
+
+  nodes:
+    mini_lab_ext:
+      kind: bridge
+    leaf01:
+      kind: cvx
+    leaf02:
+      kind: cvx
+    leaf02-sonic:
+      kind: linux
+      image: ${MINI_LAB_SONIC_IMAGE}
+      labels:
+        ansible-group: sonic
+      binds:
+        - /dev:/dev
+        - files/ssh/id_rsa.pub:/authorized_keys
+    inet:
+      kind: linux
+      image: quay.io/frrouting/frr:10.0.1
+      binds:
+        - files/inet/daemons:/etc/frr/daemons
+        - files/inet/frr.conf:/etc/frr/frr.conf
+        - files/inet/vtysh.conf:/etc/frr/vtysh.conf
+        - files/inet/network.sh:/root/network.sh
+      exec:
+        - sh /root/network.sh
+    vms:
+      kind: linux
+      binds:
+        - /dev:/dev
+        - scripts:/mini-lab
+    www:
+      kind: linux
+      image: docker.io/library/nginx:alpine-slim
+      network-mode: none
+      exec:
+        - ip addr add 203.0.113.3/24 dev ext
+        - ip route add 203.0.113.128/25 via 203.0.113.2 dev ext
+
+  links:
+    - endpoints: ["inet:ext", "mini_lab_ext:inet"]
+      mtu: 9000
+    - endpoints: ["www:ext", "mini_lab_ext:www"]
+
+    - endpoints: ["leaf01:swp1", "vms:lan0"]
+    - endpoints: ["leaf01:swp2", "vms:lan2"]
+    - endpoints: ["leaf01:swp31", "inet:eth1"]
+
+    - endpoints: ["leaf02:swp1", "vms:lan1"]
+    - endpoints: ["leaf02:swp2", "vms:lan3"]
+    - endpoints: ["leaf02:swp31", "inet:eth2"]
+
+    - endpoints: ["leaf02-sonic:eth1", "vms:lan4"]
+    - endpoints: ["leaf02-sonic:eth2", "vms:lan5"]
+    - endpoints: ["leaf02-sonic:eth3", "inet:eth3"]
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -o errexit
+TAP_IF=$1
+FROM_IF=$2
+TO_IF=$3
+
+tc qdisc del dev $FROM_IF ingress
+tc qdisc del dev $TAP_IF ingress
+
+tc qdisc add dev $TO_IF ingress
+tc filter add dev $TO_IF parent ffff: protocol all u32 match u8 0 0 action mirred egress redirect dev $TAP_IF
+
+tc qdisc add dev $TAP_IF ingress
+tc filter add dev $TAP_IF parent ffff: protocol all u32 match u8 0 0 action mirred egress redirect dev $TO_IF
@@ -1,6 +1,5 @@
 #!/bin/bash
 
-# Script is taken from https://netdevops.me/2021/transparently-redirecting-packets/frames-between-interfaces/
 # Script is taken from https://netdevops.me/2021/transparently-redirecting-packetsframes-between-interfaces/
 # Read it for better understanding