From da01bab31a93f20aeec814737e993366cf1cbdd1 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Tue, 7 May 2024 15:56:41 +0200 Subject: [PATCH] import other commands --- Makefile | 57 +- go.mod | 235 +- go.sum | 365 ++- .../server/v1alpha1/v1alpha1_cluster.go | 256 ++ .../server/v1alpha1/v1alpha1_images.go | 109 + .../server/v1alpha1/v1alpha1_inspect.go | 71 + .../internal/server/v1alpha1/v1alpha1_meta.go | 82 + .../server/v1alpha1/v1alpha1_monitoring.go | 324 +++ .../server/v1alpha1/v1alpha1_server.go | 2383 +++++++++++++++++ .../internal/server/v1alpha1/v1alpha1_time.go | 72 + .../server/v1alpha1/v1alpha1_time_test.go | 128 + internal/app/machined/main.go | 326 +++ .../machined/pkg/adapters/cluster/cluster.go | 6 + .../machined/pkg/adapters/cluster/identity.go | 55 + .../pkg/adapters/cluster/identity_test.go | 40 + .../pkg/adapters/hardware/hardware.go | 6 + .../pkg/adapters/hardware/memorymodule.go | 54 + .../pkg/adapters/hardware/processor.go | 50 + .../adapters/hardware/system_information.go | 41 + internal/app/machined/pkg/adapters/k8s/k8s.go | 6 + .../app/machined/pkg/adapters/k8s/manifest.go | 101 + .../pkg/adapters/k8s/manifest_test.go | 69 + .../machined/pkg/adapters/k8s/static_pod.go | 52 + .../pkg/adapters/k8s/static_pod_status.go | 52 + .../pkg/adapters/k8s/testdata/list.yaml | 28 + .../pkg/adapters/kubespan/identity.go | 75 + .../pkg/adapters/kubespan/identity_test.go | 34 + .../pkg/adapters/kubespan/kubespan.go | 6 + .../pkg/adapters/kubespan/peer_status.go | 153 ++ .../pkg/adapters/kubespan/peer_status_test.go | 134 + .../pkg/adapters/network/bond_master_spec.go | 197 ++ .../adapters/network/bond_master_spec_test.go | 33 + .../adapters/network/bridge_master_spec.go | 60 + .../machined/pkg/adapters/network/ipset.go | 39 + .../pkg/adapters/network/ipset_test.go | 58 + .../machined/pkg/adapters/network/network.go | 6 + .../pkg/adapters/network/nftables_rule.go | 697 +++++ .../adapters/network/nftables_rule_test.go | 715 +++++ .../pkg/adapters/network/vlan_spec.go | 64 + .../pkg/adapters/network/vlan_spec_test.go | 31 + .../pkg/adapters/network/wireguard_spec.go | 199 ++ .../adapters/network/wireguard_spec_test.go | 280 ++ .../app/machined/pkg/adapters/perf/cpu.go | 71 + .../app/machined/pkg/adapters/perf/mem.go | 79 + .../app/machined/pkg/adapters/perf/perf.go | 6 + .../pkg/adapters/wireguard/wireguard.go | 16 + .../machined/pkg/controllers/block/block.go | 6 + .../machined/pkg/controllers/block/devices.go | 246 ++ .../pkg/controllers/block/devices_test.go | 39 + .../pkg/controllers/block/discovery.go | 277 ++ .../block/internal/inotify/inotify.go | 278 ++ .../block/internal/inotify/inotify_test.go | 85 + .../block/internal/kobject/kobject.go | 92 + .../block/internal/kobject/kobject_test.go | 27 + .../block/internal/sysblock/sysblock.go | 144 + .../block/internal/sysblock/sysblock_test.go | 42 + .../controllers/cluster/affiliate_merge.go | 114 + .../cluster/affiliate_merge_test.go | 132 + .../pkg/controllers/cluster/cluster.go | 44 + .../pkg/controllers/cluster/cluster_test.go | 84 + .../pkg/controllers/cluster/config.go | 93 + .../pkg/controllers/cluster/config_test.go | 199 ++ .../controllers/cluster/discovery_service.go | 502 ++++ .../cluster/discovery_service_test.go | 371 +++ .../pkg/controllers/cluster/endpoint.go | 99 + .../pkg/controllers/cluster/endpoint_test.go | 83 + .../machined/pkg/controllers/cluster/info.go | 46 + .../pkg/controllers/cluster/info_test.go | 63 + .../controllers/cluster/kubernetes_pull.go | 176 ++ .../controllers/cluster/kubernetes_push.go | 147 + .../controllers/cluster/local_affiliate.go | 323 +++ .../cluster/local_affiliate_test.go | 228 ++ .../pkg/controllers/cluster/member.go | 115 + .../pkg/controllers/cluster/member_test.go | 104 + .../pkg/controllers/cluster/node_identity.go | 131 + .../controllers/cluster/node_identity_test.go | 117 + .../pkg/controllers/config/acquire.go | 465 ++++ .../pkg/controllers/config/acquire_test.go | 474 ++++ .../machined/pkg/controllers/config/config.go | 6 + .../pkg/controllers/config/machine_type.go | 82 + .../app/machined/pkg/controllers/cri/cri.go | 5 + .../machined/pkg/controllers/cri/cri_test.go | 5 + .../pkg/controllers/cri/seccomp_profile.go | 88 + .../controllers/cri/seccomp_profile_file.go | 179 ++ .../cri/seccomp_profile_file_test.go | 130 + .../controllers/cri/seccomp_profile_test.go | 135 + .../machined/pkg/controllers/ctest/assert.go | 67 + .../machined/pkg/controllers/ctest/ctest.go | 189 ++ .../pkg/controllers/etcd/advertised_peer.go | 177 ++ .../machined/pkg/controllers/etcd/config.go | 69 + .../pkg/controllers/etcd/config_test.go | 178 ++ .../app/machined/pkg/controllers/etcd/etcd.go | 6 + .../machined/pkg/controllers/etcd/member.go | 116 + .../pkg/controllers/etcd/member_test.go | 149 ++ .../app/machined/pkg/controllers/etcd/pki.go | 150 ++ .../app/machined/pkg/controllers/etcd/spec.go | 221 ++ .../pkg/controllers/etcd/spec_test.go | 220 ++ .../pkg/controllers/files/cri_config_parts.go | 94 + .../controllers/files/cri_registry_config.go | 201 ++ .../machined/pkg/controllers/files/etcfile.go | 204 ++ .../pkg/controllers/files/etcfile_test.go | 162 ++ .../machined/pkg/controllers/files/files.go | 6 + .../pkg/controllers/hardware/hardware.go | 6 + .../pkg/controllers/hardware/hardware_test.go | 78 + .../pkg/controllers/hardware/system.go | 156 ++ .../pkg/controllers/hardware/system_test.go | 173 ++ .../testdata/SuperMicro-Dual-Xeon.dmi | Bin 0 -> 6852 bytes .../pkg/controllers/k8s/address_filter.go | 120 + .../controllers/k8s/address_filter_test.go | 182 ++ .../pkg/controllers/k8s/control_plane.go | 432 +++ .../k8s/control_plane_static_pod.go | 865 ++++++ .../k8s/control_plane_static_pod_test.go | 789 ++++++ .../pkg/controllers/k8s/control_plane_test.go | 569 ++++ .../machined/pkg/controllers/k8s/endpoint.go | 301 +++ .../pkg/controllers/k8s/extra_manifest.go | 261 ++ .../controllers/k8s/extra_manifest_test.go | 162 ++ .../k8s/internal/nodename/nodename.go | 56 + .../k8s/internal/nodename/nodename_test.go | 73 + .../k8s/internal/nodewatch/nodewatch.go | 94 + .../app/machined/pkg/controllers/k8s/k8s.go | 16 + .../pkg/controllers/k8s/kubelet_config.go | 94 + .../controllers/k8s/kubelet_config_test.go | 265 ++ .../pkg/controllers/k8s/kubelet_service.go | 565 ++++ .../pkg/controllers/k8s/kubelet_spec.go | 367 +++ .../pkg/controllers/k8s/kubelet_spec_test.go | 474 ++++ .../pkg/controllers/k8s/kubelet_static_pod.go | 233 ++ .../machined/pkg/controllers/k8s/kubeprism.go | 284 ++ .../pkg/controllers/k8s/kubeprism_config.go | 81 + .../controllers/k8s/kubeprism_config_test.go | 142 + .../controllers/k8s/kubeprism_endpoints.go | 88 + .../k8s/kubeprism_endpoints_test.go | 130 + .../machined/pkg/controllers/k8s/manifest.go | 302 +++ .../pkg/controllers/k8s/manifest_apply.go | 308 +++ .../pkg/controllers/k8s/manifest_test.go | 362 +++ .../pkg/controllers/k8s/node_apply.go | 443 +++ .../pkg/controllers/k8s/node_apply_test.go | 407 +++ .../pkg/controllers/k8s/node_cordoned_spec.go | 105 + .../k8s/node_cordoned_spec_test.go | 76 + .../pkg/controllers/k8s/node_label_spec.go | 99 + .../controllers/k8s/node_label_spec_test.go | 135 + .../pkg/controllers/k8s/node_status.go | 215 ++ .../pkg/controllers/k8s/node_taint_spec.go | 112 + .../controllers/k8s/node_taint_spec_test.go | 118 + .../machined/pkg/controllers/k8s/nodeip.go | 159 ++ .../pkg/controllers/k8s/nodeip_config.go | 105 + .../pkg/controllers/k8s/nodeip_config_test.go | 225 ++ .../pkg/controllers/k8s/nodeip_test.go | 113 + .../machined/pkg/controllers/k8s/nodename.go | 123 + .../pkg/controllers/k8s/nodename_test.go | 114 + .../k8s/render_config_static_pods.go | 257 ++ .../k8s/render_secrets_static_pod.go | 346 +++ .../pkg/controllers/k8s/static_endpoint.go | 100 + .../controllers/k8s/static_endpoint_test.go | 71 + .../pkg/controllers/k8s/static_pod_config.go | 118 + .../controllers/k8s/static_pod_config_test.go | 206 ++ .../pkg/controllers/k8s/static_pod_server.go | 210 ++ .../controllers/k8s/static_pod_server_test.go | 192 ++ .../machined/pkg/controllers/k8s/templates.go | 608 +++++ .../pkg/controllers/kubeaccess/config.go | 59 + .../pkg/controllers/kubeaccess/config_test.go | 106 + .../pkg/controllers/kubeaccess/endpoint.go | 210 ++ .../pkg/controllers/kubeaccess/kubeaccess.go | 6 + .../controllers/kubeaccess/serviceaccount.go | 206 ++ .../serviceaccount/crd_controller.go | 651 +++++ .../pkg/controllers/kubespan/config.go | 60 + .../pkg/controllers/kubespan/config_test.go | 106 + .../pkg/controllers/kubespan/endpoint.go | 145 + .../pkg/controllers/kubespan/endpoint_test.go | 136 + .../pkg/controllers/kubespan/identity.go | 155 ++ .../pkg/controllers/kubespan/identity_test.go | 143 + .../pkg/controllers/kubespan/kubespan.go | 6 + .../pkg/controllers/kubespan/kubespan_test.go | 117 + .../pkg/controllers/kubespan/manager.go | 613 +++++ .../pkg/controllers/kubespan/manager_test.go | 408 +++ .../pkg/controllers/kubespan/peer_spec.go | 209 ++ .../controllers/kubespan/peer_spec_test.go | 257 ++ .../pkg/controllers/kubespan/routing_rules.go | 155 ++ .../kubespan/routing_rules_test.go | 31 + .../pkg/controllers/network/address_config.go | 312 +++ .../network/address_config_test.go | 249 ++ .../pkg/controllers/network/address_event.go | 116 + .../controllers/network/address_event_test.go | 191 ++ .../pkg/controllers/network/address_merge.go | 140 + .../controllers/network/address_merge_test.go | 315 +++ .../pkg/controllers/network/address_spec.go | 310 +++ .../controllers/network/address_spec_test.go | 256 ++ .../pkg/controllers/network/address_status.go | 145 + .../network/address_status_test.go | 84 + .../pkg/controllers/network/cmdline.go | 513 ++++ .../pkg/controllers/network/cmdline_test.go | 492 ++++ .../pkg/controllers/network/device_config.go | 244 ++ .../controllers/network/device_config_test.go | 251 ++ .../controllers/network/dns_resolve_cache.go | 345 +++ .../network/dns_resolve_cache_test.go | 286 ++ .../pkg/controllers/network/dns_upstream.go | 162 ++ .../pkg/controllers/network/etcfile.go | 261 ++ .../pkg/controllers/network/etcfile_test.go | 320 +++ .../pkg/controllers/network/hardware_addr.go | 108 + .../controllers/network/hardware_addr_test.go | 152 ++ .../pkg/controllers/network/hostdns_config.go | 216 ++ .../controllers/network/hostname_config.go | 270 ++ .../network/hostname_config_test.go | 259 ++ .../pkg/controllers/network/hostname_merge.go | 124 + .../network/hostname_merge_test.go | 131 + .../pkg/controllers/network/hostname_spec.go | 122 + .../controllers/network/hostname_spec_test.go | 124 + .../network/internal/probe/probe.go | 159 ++ .../network/internal/probe/probe_test.go | 152 ++ .../pkg/controllers/network/link_config.go | 453 ++++ .../controllers/network/link_config_test.go | 484 ++++ .../pkg/controllers/network/link_merge.go | 156 ++ .../controllers/network/link_merge_test.go | 377 +++ .../pkg/controllers/network/link_spec.go | 615 +++++ .../pkg/controllers/network/link_spec_test.go | 974 +++++++ .../pkg/controllers/network/link_status.go | 343 +++ .../controllers/network/link_status_test.go | 372 +++ .../pkg/controllers/network/network.go | 138 + .../pkg/controllers/network/nftables_chain.go | 213 ++ .../network/nftables_chain_config.go | 224 ++ .../network/nftables_chain_config_test.go | 283 ++ .../network/nftables_chain_test.go | 514 ++++ .../pkg/controllers/network/node_address.go | 342 +++ .../controllers/network/node_address_test.go | 418 +++ .../pkg/controllers/network/operator/dhcp4.go | 575 ++++ .../pkg/controllers/network/operator/dhcp6.go | 305 +++ .../controllers/network/operator/operator.go | 27 + .../pkg/controllers/network/operator/vip.go | 392 +++ .../network/operator/vip/equinix_metal.go | 133 + .../operator/vip/equinix_metal_test.go | 69 + .../network/operator/vip/hcloud.go | 208 ++ .../controllers/network/operator/vip/nop.go | 22 + .../controllers/network/operator/vip/vip.go | 14 + .../controllers/network/operator_config.go | 322 +++ .../network/operator_config_test.go | 436 +++ .../pkg/controllers/network/operator_merge.go | 140 + .../network/operator_merge_test.go | 314 +++ .../pkg/controllers/network/operator_spec.go | 429 +++ .../controllers/network/operator_spec_test.go | 553 ++++ .../network/operator_vip_config.go | 240 ++ .../network/operator_vip_config_test.go | 169 ++ .../controllers/network/platform_config.go | 607 +++++ .../network/platform_config_test.go | 846 ++++++ .../machined/pkg/controllers/network/probe.go | 154 ++ .../pkg/controllers/network/probe_test.go | 91 + .../controllers/network/resolver_config.go | 211 ++ .../network/resolver_config_test.go | 209 ++ .../pkg/controllers/network/resolver_merge.go | 132 + .../network/resolver_merge_test.go | 132 + .../pkg/controllers/network/resolver_spec.go | 106 + .../controllers/network/resolver_spec_test.go | 120 + .../pkg/controllers/network/route_config.go | 325 +++ .../controllers/network/route_config_test.go | 280 ++ .../pkg/controllers/network/route_merge.go | 138 + .../controllers/network/route_merge_test.go | 357 +++ .../pkg/controllers/network/route_spec.go | 309 +++ .../controllers/network/route_spec_test.go | 570 ++++ .../pkg/controllers/network/route_status.go | 147 + .../controllers/network/route_status_test.go | 93 + .../pkg/controllers/network/status.go | 178 ++ .../pkg/controllers/network/status_test.go | 113 + .../controllers/network/timeserver_config.go | 203 ++ .../network/timeserver_config_test.go | 196 ++ .../controllers/network/timeserver_merge.go | 132 + .../network/timeserver_merge_test.go | 131 + .../controllers/network/timeserver_spec.go | 114 + .../network/timeserver_spec_test.go | 119 + .../pkg/controllers/network/utils/utils.go | 66 + .../pkg/controllers/network/watch/ethtool.go | 77 + .../controllers/network/watch/rtnetlink.go | 55 + .../pkg/controllers/network/watch/trigger.go | 74 + .../controllers/network/watch/trigger_test.go | 45 + .../pkg/controllers/network/watch/watch.go | 16 + .../app/machined/pkg/controllers/perf/perf.go | 113 + .../pkg/controllers/perf/perf_test.go | 128 + .../pkg/controllers/runtime/common_test.go | 91 + .../pkg/controllers/runtime/cri_image_gc.go | 314 +++ .../controllers/runtime/cri_image_gc_test.go | 298 +++ .../pkg/controllers/runtime/devices_status.go | 68 + .../runtime/drop_upgrade_fallback.go | 94 + .../runtime/drop_upgrade_fallback_test.go | 98 + .../pkg/controllers/runtime/events_sink.go | 208 ++ .../controllers/runtime/events_sink_config.go | 101 + .../runtime/events_sink_config_test.go | 80 + .../controllers/runtime/events_sink_test.go | 305 +++ .../pkg/controllers/runtime/export_test.go | 8 + .../controllers/runtime/extension_service.go | 225 ++ .../runtime/extension_service_config.go | 94 + .../runtime/extension_service_config_files.go | 124 + .../extension_service_config_files_test.go | 129 + .../runtime/extension_service_config_test.go | 141 + .../runtime/extension_service_test.go | 248 ++ .../controllers/runtime/extension_status.go | 79 + .../runtime/kernel_module_config.go | 89 + .../runtime/kernel_module_config_test.go | 104 + .../controllers/runtime/kernel_module_spec.go | 90 + .../runtime/kernel_param_config.go | 100 + .../runtime/kernel_param_config_test.go | 121 + .../runtime/kernel_param_defaults.go | 151 ++ .../runtime/kernel_param_defaults_test.go | 115 + .../controllers/runtime/kernel_param_spec.go | 206 ++ .../runtime/kernel_param_spec_test.go | 116 + .../pkg/controllers/runtime/kmsg_log.go | 274 ++ .../controllers/runtime/kmsg_log_config.go | 113 + .../runtime/kmsg_log_config_test.go | 104 + .../pkg/controllers/runtime/kmsg_log_test.go | 270 ++ .../pkg/controllers/runtime/machine_status.go | 476 ++++ .../runtime/machine_status_publisher.go | 84 + .../runtime/machine_status_test.go | 159 ++ .../controllers/runtime/maintenance_config.go | 129 + .../runtime/maintenance_config_test.go | 64 + .../runtime/maintenance_service.go | 307 +++ .../runtime/maintenance_service_test.go | 299 +++ .../pkg/controllers/runtime/runtime.go | 6 + .../pkg/controllers/runtime/security_state.go | 136 + .../runtime/testdata/extservices/foo.bar | 0 .../runtime/testdata/extservices/frr.yaml | 10 + .../runtime/testdata/extservices/hello.yaml | 10 + .../runtime/testdata/extservices/invalid.yaml | 9 + .../testdata/extservices/zduplicate.yaml | 9 + .../pkg/controllers/runtime/unique_token.go | 56 + .../machined/pkg/controllers/runtime/utils.go | 78 + .../pkg/controllers/runtime/watchdog_timer.go | 174 ++ .../runtime/watchdog_timer_config.go | 84 + .../runtime/watchdog_timer_config_test.go | 60 + .../machined/pkg/controllers/secrets/api.go | 428 +++ .../pkg/controllers/secrets/api_cert_sans.go | 154 ++ .../controllers/secrets/api_cert_sans_test.go | 119 + .../pkg/controllers/secrets/api_test.go | 148 + .../machined/pkg/controllers/secrets/etcd.go | 221 ++ .../pkg/controllers/secrets/etcd_test.go | 175 ++ .../pkg/controllers/secrets/kubelet.go | 87 + .../pkg/controllers/secrets/kubelet_test.go | 100 + .../pkg/controllers/secrets/kubernetes.go | 245 ++ .../secrets/kubernetes_cert_sans.go | 167 ++ .../secrets/kubernetes_cert_sans_test.go | 152 ++ .../secrets/kubernetes_dynamic_certs.go | 248 ++ .../secrets/kubernetes_dynamic_certs_test.go | 200 ++ .../controllers/secrets/kubernetes_test.go | 113 + .../pkg/controllers/secrets/maintenance.go | 143 + .../secrets/maintenance_cert_sans.go | 107 + .../secrets/maintenance_cert_sans_test.go | 67 + .../controllers/secrets/maintenance_root.go | 61 + .../secrets/maintenance_root_test.go | 39 + .../controllers/secrets/maintenance_test.go | 89 + .../machined/pkg/controllers/secrets/root.go | 200 ++ .../pkg/controllers/secrets/root_test.go | 121 + .../pkg/controllers/secrets/secrets.go | 6 + .../pkg/controllers/secrets/trustd.go | 272 ++ .../pkg/controllers/secrets/trustd_test.go | 134 + .../pkg/controllers/siderolink/config.go | 117 + .../pkg/controllers/siderolink/config_test.go | 81 + .../pkg/controllers/siderolink/manager.go | 510 ++++ .../controllers/siderolink/manager_test.go | 176 ++ .../pkg/controllers/siderolink/siderolink.go | 6 + .../pkg/controllers/siderolink/userspace.go | 265 ++ .../pkg/controllers/time/adjtime_status.go | 97 + .../app/machined/pkg/controllers/time/sync.go | 244 ++ .../pkg/controllers/time/sync_test.go | 519 ++++ .../app/machined/pkg/controllers/time/time.go | 6 + .../app/machined/pkg/controllers/utils.go | 61 + .../pkg/controllers/v1alpha1/service.go | 104 + .../pkg/controllers/v1alpha1/utils.go | 65 + .../pkg/controllers/v1alpha1/v1alpha1.go | 6 + internal/app/machined/pkg/runtime/board.go | 31 + .../app/machined/pkg/runtime/controller.go | 64 + .../app/machined/pkg/runtime/disk/disk.go | 6 + .../app/machined/pkg/runtime/disk/options.go | 20 + internal/app/machined/pkg/runtime/doc.go | 7 + internal/app/machined/pkg/runtime/drainer.go | 116 + .../app/machined/pkg/runtime/drainer_test.go | 99 + .../pkg/runtime/emergency/emergency.go | 19 + internal/app/machined/pkg/runtime/errors.go | 39 + internal/app/machined/pkg/runtime/events.go | 158 ++ internal/app/machined/pkg/runtime/logging.go | 90 + .../machined/pkg/runtime/logging/circular.go | 320 +++ .../machined/pkg/runtime/logging/extract.go | 124 + .../pkg/runtime/logging/extract_test.go | 104 + .../app/machined/pkg/runtime/logging/file.go | 129 + .../machined/pkg/runtime/logging/logging.go | 6 + .../app/machined/pkg/runtime/logging/null.go | 45 + .../pkg/runtime/logging/sender_jsonlines.go | 144 + .../runtime/logging/sender_jsonlines_test.go | 336 +++ internal/app/machined/pkg/runtime/mode.go | 92 + .../app/machined/pkg/runtime/mode_test.go | 98 + internal/app/machined/pkg/runtime/platform.go | 63 + internal/app/machined/pkg/runtime/runtime.go | 29 + .../app/machined/pkg/runtime/sequencer.go | 166 ++ .../machined/pkg/runtime/sequencer_test.go | 141 + internal/app/machined/pkg/runtime/state.go | 76 + .../pkg/runtime/v1alpha1/acpi/acpi.go | 108 + .../pkg/runtime/v1alpha1/acpi/acpi_test.go | 76 + .../board/bananapi_m64/bananapi_m64.go | 102 + .../pkg/runtime/v1alpha1/board/board.go | 56 + .../v1alpha1/board/jetson_nano/jetson_nano.go | 90 + .../libretech_all_h3_cc_h5.go | 99 + .../v1alpha1/board/nanopi_r4s/nanopi_r4s.go | 88 + .../runtime/v1alpha1/board/pine64/pine64.go | 100 + .../runtime/v1alpha1/board/rock64/rock64.go | 99 + .../runtime/v1alpha1/board/rockpi4/rockpi4.go | 95 + .../v1alpha1/board/rockpi4c/rockpi4c.go | 94 + .../v1alpha1/board/rpi_generic/config.txt | 19 + .../v1alpha1/board/rpi_generic/rpi_generic.go | 60 + .../runtime/v1alpha1/bootloader/bootloader.go | 75 + .../v1alpha1/bootloader/grub/boot_label.go | 62 + .../v1alpha1/bootloader/grub/constants.go | 29 + .../v1alpha1/bootloader/grub/decode.go | 159 ++ .../v1alpha1/bootloader/grub/encode.go | 76 + .../runtime/v1alpha1/bootloader/grub/grub.go | 84 + .../v1alpha1/bootloader/grub/grub_test.go | 278 ++ .../v1alpha1/bootloader/grub/install.go | 120 + .../runtime/v1alpha1/bootloader/grub/probe.go | 35 + .../runtime/v1alpha1/bootloader/grub/quote.go | 29 + .../v1alpha1/bootloader/grub/quote_test.go | 105 + .../v1alpha1/bootloader/grub/revert.go | 79 + .../grub/testdata/grub_parse_test.cfg | 29 + .../testdata/grub_write_no_reset_test.cfg | 15 + .../grub/testdata/grub_write_test.cfg | 21 + .../v1alpha1/bootloader/mount/mount.go | 107 + .../v1alpha1/bootloader/options/options.go | 82 + .../v1alpha1/bootloader/sdboot/efivars.go | 83 + .../v1alpha1/bootloader/sdboot/sdboot.go | 233 ++ .../app/machined/pkg/runtime/v1alpha1/doc.go | 6 + .../v1alpha1/platform/akamai/akamai.go | 206 ++ .../v1alpha1/platform/akamai/akamai_test.go | 48 + .../platform/akamai/testdata/expected.yaml | 56 + .../platform/akamai/testdata/instance.json | 19 + .../platform/akamai/testdata/network.json | 20 + .../pkg/runtime/v1alpha1/platform/aws/aws.go | 171 ++ .../runtime/v1alpha1/platform/aws/aws_test.go | 39 + .../runtime/v1alpha1/platform/aws/metadata.go | 99 + .../platform/aws/testdata/expected.yaml | 19 + .../platform/aws/testdata/metadata.json | 7 + .../runtime/v1alpha1/platform/azure/azure.go | 346 +++ .../v1alpha1/platform/azure/azure_test.go | 56 + .../v1alpha1/platform/azure/metadata.go | 67 + .../v1alpha1/platform/azure/register.go | 214 ++ .../platform/azure/testdata/compute.json | 14 + .../platform/azure/testdata/expected.yaml | 39 + .../platform/azure/testdata/interfaces.json | 27 + .../platform/azure/testdata/loadbalancer.json | 31 + .../v1alpha1/platform/container/container.go | 91 + .../container/internal/files/hostname.go | 33 + .../container/internal/files/hostname_test.go | 23 + .../container/internal/files/resolv.go | 42 + .../container/internal/files/resolv_test.go | 26 + .../internal/files/testdata/hostname | 1 + .../internal/files/testdata/resolv.conf | 8 + .../platform/digitalocean/digitalocean.go | 287 ++ .../digitalocean/digitalocean_test.go | 39 + .../platform/digitalocean/metadata.go | 81 + .../digitalocean/testdata/expected.yaml | 97 + .../digitalocean/testdata/metadata.json | 70 + .../v1alpha1/platform/equinixmetal/equinix.go | 467 ++++ .../platform/equinixmetal/equinix_test.go | 57 + .../platform/equinixmetal/metadata.go | 17 + .../equinixmetal/testdata/expected.yaml | 137 + .../equinixmetal/testdata/metadata.json | 149 ++ .../v1alpha1/platform/errors/errors.go | 23 + .../v1alpha1/platform/exoscale/exoscale.go | 117 + .../platform/exoscale/exoscale_test.go | 39 + .../v1alpha1/platform/exoscale/metadata.go | 67 + .../platform/exoscale/testdata/expected.yaml | 18 + .../platform/exoscale/testdata/metadata.json | 6 + .../pkg/runtime/v1alpha1/platform/gcp/gcp.go | 236 ++ .../runtime/v1alpha1/platform/gcp/gcp_test.go | 46 + .../runtime/v1alpha1/platform/gcp/metadata.go | 98 + .../platform/gcp/testdata/expected.yaml | 57 + .../platform/gcp/testdata/interfaces.json | 32 + .../platform/gcp/testdata/metadata.json | 9 + .../v1alpha1/platform/hcloud/hcloud.go | 213 ++ .../v1alpha1/platform/hcloud/hcloud_test.go | 46 + .../v1alpha1/platform/hcloud/metadata.go | 91 + .../platform/hcloud/testdata/expected.yaml | 51 + .../platform/hcloud/testdata/metadata.yaml | 17 + .../platform/internal/address/address.go | 58 + .../platform/internal/netutils/netutils.go | 101 + .../runtime/v1alpha1/platform/metal/metal.go | 245 ++ .../v1alpha1/platform/metal/metal_test.go | 120 + .../v1alpha1/platform/metal/oauth2/oauth2.go | 204 ++ .../platform/metal/oauth2/oauth2_test.go | 117 + .../v1alpha1/platform/metal/url/map.go | 85 + .../v1alpha1/platform/metal/url/map_test.go | 106 + .../v1alpha1/platform/metal/url/url.go | 118 + .../v1alpha1/platform/metal/url/url_test.go | 176 ++ .../v1alpha1/platform/metal/url/value.go | 148 + .../v1alpha1/platform/metal/url/variable.go | 108 + .../platform/metal/url/variable_test.go | 156 ++ .../v1alpha1/platform/metal/url_test.go | 129 + .../v1alpha1/platform/nocloud/metadata.go | 677 +++++ .../v1alpha1/platform/nocloud/nocloud.go | 141 + .../v1alpha1/platform/nocloud/nocloud_test.go | 106 + .../nocloud/testdata/expected-v1.yaml | 104 + .../nocloud/testdata/expected-v2.yaml | 153 ++ .../nocloud/testdata/metadata-v1.yaml | 29 + .../nocloud/testdata/metadata-v2.yaml | 49 + .../nocloud/testdata/metadata-v3.yaml | 49 + .../v1alpha1/platform/opennebula/metadata.go | 76 + .../platform/opennebula/opennebula.go | 258 ++ .../platform/opennebula/opennebula_test.go | 40 + .../opennebula/testdata/expected.yaml | 45 + .../opennebula/testdata/metadata.yaml | 28 + .../v1alpha1/platform/openstack/metadata.go | 206 ++ .../v1alpha1/platform/openstack/openstack.go | 437 +++ .../platform/openstack/openstack_test.go | 79 + .../platform/openstack/testdata/expected.yaml | 200 ++ .../platform/openstack/testdata/metadata.json | 11 + .../platform/openstack/testdata/network.json | 153 ++ .../v1alpha1/platform/oracle/metadata.go | 54 + .../v1alpha1/platform/oracle/oracle.go | 205 ++ .../v1alpha1/platform/oracle/oracle_test.go | 46 + .../platform/oracle/testdata/expected.yaml | 43 + .../platform/oracle/testdata/metadata.json | 53 + .../oracle/testdata/metadatanetwork.json | 15 + .../pkg/runtime/v1alpha1/platform/platform.go | 162 ++ .../v1alpha1/platform/scaleway/metadata.go | 36 + .../v1alpha1/platform/scaleway/scaleway.go | 214 ++ .../platform/scaleway/scaleway_test.go | 40 + .../platform/scaleway/testdata/expected.yaml | 64 + .../platform/scaleway/testdata/metadata.json | 22 + .../v1alpha1/platform/upcloud/metadata.go | 63 + .../platform/upcloud/testdata/expected.yaml | 81 + .../platform/upcloud/testdata/metadata.json | 83 + .../v1alpha1/platform/upcloud/upcloud.go | 225 ++ .../v1alpha1/platform/upcloud/upcloud_test.go | 39 + .../v1alpha1/platform/vmware/metadata.go | 266 ++ .../testdata/expected-match-by-mac.yaml | 36 + .../testdata/expected-match-by-name.yaml | 36 + .../testdata/metadata-match-by-mac.yaml | 16 + .../testdata/metadata-match-by-name.yaml | 16 + .../v1alpha1/platform/vmware/vmware_amd64.go | 293 ++ .../v1alpha1/platform/vmware/vmware_other.go | 45 + .../v1alpha1/platform/vmware/vmware_test.go | 85 + .../v1alpha1/platform/vultr/metadata.go | 38 + .../platform/vultr/testdata/expected.yaml | 56 + .../platform/vultr/testdata/metadata.json | 61 + .../runtime/v1alpha1/platform/vultr/vultr.go | 206 ++ .../v1alpha1/platform/vultr/vultr_test.go | 40 + .../runtime/v1alpha1/v1alpha1_controller.go | 415 +++ .../v1alpha1/v1alpha1_controller_test.go | 215 ++ .../pkg/runtime/v1alpha1/v1alpha1_dbus.go | 95 + .../pkg/runtime/v1alpha1/v1alpha1_events.go | 234 ++ .../runtime/v1alpha1/v1alpha1_events_test.go | 306 +++ .../v1alpha1/v1alpha1_priority_lock.go | 122 + .../v1alpha1/v1alpha1_priority_lock_test.go | 154 ++ .../pkg/runtime/v1alpha1/v1alpha1_runtime.go | 234 ++ .../runtime/v1alpha1/v1alpha1_sequencer.go | 578 ++++ .../v1alpha1/v1alpha1_sequencer_tasks.go | 2348 ++++++++++++++++ .../v1alpha1/v1alpha1_sequencer_test.go | 112 + .../pkg/runtime/v1alpha1/v1alpha1_state.go | 346 +++ .../machined/pkg/runtime/v1alpha2/adapters.go | 48 + .../machined/pkg/runtime/v1alpha2/v1alpha2.go | 6 + .../runtime/v1alpha2/v1alpha2_controller.go | 512 ++++ .../pkg/runtime/v1alpha2/v1alpha2_state.go | 260 ++ .../app/machined/pkg/system/events/events.go | 148 + .../machined/pkg/system/events/events_test.go | 78 + .../app/machined/pkg/system/export_test.go | 18 + .../app/machined/pkg/system/health/check.go | 64 + .../machined/pkg/system/health/health_test.go | 208 ++ .../machined/pkg/system/health/settings.go | 23 + .../app/machined/pkg/system/health/status.go | 131 + .../machined/pkg/system/integration_test.go | 93 + .../app/machined/pkg/system/mocks_test.go | 150 ++ .../system/runner/containerd/containerd.go | 373 +++ .../runner/containerd/containerd_test.go | 439 +++ .../pkg/system/runner/containerd/import.go | 125 + .../pkg/system/runner/containerd/opts.go | 55 + .../pkg/system/runner/containerd/stdin.go | 38 + .../pkg/system/runner/goroutine/goroutine.go | 118 + .../system/runner/goroutine/goroutine_test.go | 173 ++ .../pkg/system/runner/process/process.go | 257 ++ .../pkg/system/runner/process/process_test.go | 232 ++ .../pkg/system/runner/restart/restart.go | 175 ++ .../pkg/system/runner/restart/restart_test.go | 182 ++ .../app/machined/pkg/system/runner/runner.go | 222 ++ .../machined/pkg/system/runner/runner_test.go | 14 + internal/app/machined/pkg/system/service.go | 55 + .../app/machined/pkg/system/service_events.go | 116 + .../app/machined/pkg/system/service_runner.go | 458 ++++ .../pkg/system/service_runner_test.go | 467 ++++ .../app/machined/pkg/system/services/apid.go | 226 ++ .../pkg/system/services/containerd.go | 140 + .../app/machined/pkg/system/services/cri.go | 134 + .../machined/pkg/system/services/dashboard.go | 75 + .../app/machined/pkg/system/services/etcd.go | 710 +++++ .../pkg/system/services/export_test.go | 17 + .../machined/pkg/system/services/extension.go | 272 ++ .../pkg/system/services/extension_test.go | 189 ++ .../machined/pkg/system/services/kubelet.go | 237 ++ .../machined/pkg/system/services/machined.go | 240 ++ .../pkg/system/services/machined_test.go | 75 + .../pkg/system/services/mocks/snapshotter.go | 249 ++ .../machined/pkg/system/services/syslogd.go | 68 + .../machined/pkg/system/services/trustd.go | 190 ++ .../app/machined/pkg/system/services/udevd.go | 138 + .../app/machined/pkg/system/services/utils.go | 36 + internal/app/machined/pkg/system/system.go | 473 ++++ .../app/machined/pkg/system/system_test.go | 69 + internal/app/machined/revert.go | 69 + internal/pkg/cgroup/cgroup.go | 58 + internal/pkg/cri/client.go | 58 + internal/pkg/cri/containers.go | 117 + internal/pkg/cri/cri.go | 9 + internal/pkg/cri/cri_test.go | 252 ++ internal/pkg/cri/images.go | 55 + internal/pkg/cri/pods.go | 248 ++ internal/pkg/dashboard/apidata/apidata.go | 6 + internal/pkg/dashboard/apidata/data.go | 38 + internal/pkg/dashboard/apidata/diff.go | 85 + internal/pkg/dashboard/apidata/node.go | 209 ++ internal/pkg/dashboard/apidata/source.go | 297 ++ .../pkg/dashboard/components/components.go | 93 + internal/pkg/dashboard/components/footer.go | 127 + internal/pkg/dashboard/components/gauges.go | 97 + internal/pkg/dashboard/components/graphs.go | 84 + internal/pkg/dashboard/components/header.go | 185 ++ .../dashboard/components/horizontalline.go | 35 + internal/pkg/dashboard/components/info.go | 173 ++ .../dashboard/components/kubernetesinfo.go | 231 ++ .../pkg/dashboard/components/logviewer.go | 66 + .../pkg/dashboard/components/networkinfo.go | 273 ++ .../pkg/dashboard/components/sparklines.go | 76 + internal/pkg/dashboard/components/tables.go | 114 + .../pkg/dashboard/components/tables_test.go | 39 + .../pkg/dashboard/components/talosinfo.go | 216 ++ internal/pkg/dashboard/components/termui.go | 100 + internal/pkg/dashboard/configurl.go | 211 ++ internal/pkg/dashboard/context.go | 28 + internal/pkg/dashboard/dashboard.go | 584 ++++ internal/pkg/dashboard/formdata.go | 246 ++ internal/pkg/dashboard/formdata_test.go | 245 ++ internal/pkg/dashboard/logdata/logdata.go | 133 + internal/pkg/dashboard/monitor.go | 129 + internal/pkg/dashboard/networkconfig.go | 451 ++++ internal/pkg/dashboard/options.go | 52 + .../dashboard/resourcedata/resourcedata.go | 207 ++ internal/pkg/dashboard/summary.go | 134 + internal/pkg/dashboard/util/util.go | 49 + internal/pkg/environment/environment.go | 44 + internal/pkg/environment/environment_test.go | 85 + internal/pkg/etcd/certs.go | 139 + internal/pkg/etcd/endpoints.go | 50 + internal/pkg/etcd/etcd.go | 289 ++ internal/pkg/etcd/local.go | 34 + internal/pkg/etcd/lock.go | 44 + internal/pkg/install/install.go | 300 +++ internal/pkg/install/options.go | 89 + internal/pkg/install/pull.go | 111 + internal/pkg/logind/broker.go | 272 ++ internal/pkg/logind/dbus.go | 22 + internal/pkg/logind/kubelet_mock_test.go | 148 + internal/pkg/logind/logind.go | 64 + internal/pkg/logind/logind_test.go | 85 + internal/pkg/logind/service.go | 96 + internal/pkg/meta/constants.go | 30 + internal/pkg/meta/internal/adv/adv.go | 30 + .../meta/internal/adv/syslinux/syslinux.go | 260 ++ .../internal/adv/syslinux/syslinux_test.go | 378 +++ .../internal/adv/syslinux/testdata/adv.sys | Bin 0 -> 60416 bytes internal/pkg/meta/internal/adv/talos/talos.go | 230 ++ .../pkg/meta/internal/adv/talos/talos_test.go | 92 + internal/pkg/meta/meta.go | 345 +++ internal/pkg/meta/meta_test.go | 98 + internal/pkg/mount/all.go | 124 + internal/pkg/mount/bpffs.go | 14 + internal/pkg/mount/cgroups.go | 65 + internal/pkg/mount/iter.go | 115 + internal/pkg/mount/mount.go | 504 ++++ internal/pkg/mount/mount_test.go | 120 + internal/pkg/mount/options.go | 143 + internal/pkg/mount/overlay.go | 24 + internal/pkg/mount/pseudo.go | 44 + internal/pkg/mount/squashfs.go | 27 + internal/pkg/mount/switchroot/switchroot.go | 170 ++ .../pkg/mount/switchroot/switchroot_test.go | 14 + internal/pkg/mount/system.go | 299 +++ internal/pkg/mount/unmount.go | 77 + internal/pkg/partition/constants.go | 42 + internal/pkg/partition/format.go | 103 + internal/pkg/partition/format_test.go | 157 ++ internal/pkg/partition/partition.go | 121 + internal/pkg/secureboot/database/database.go | 62 + .../measure/internal/pcr/bank_data.go | 100 + .../measure/internal/pcr/bank_data_test.go | 60 + .../secureboot/measure/internal/pcr/extend.go | 49 + .../measure/internal/pcr/extend_test.go | 36 + .../secureboot/measure/internal/pcr/sign.go | 37 + .../measure/internal/pcr/sign_test.go | 45 + .../measure/internal/pcr/testdata/a | 1 + .../measure/internal/pcr/testdata/b | 1 + .../measure/internal/pcr/testdata/c | 1 + internal/pkg/secureboot/measure/measure.go | 60 + .../pkg/secureboot/measure/measure_test.go | 143 + .../measure/testdata/pcr-signing-key.pem | 51 + internal/pkg/secureboot/pesign/pesign.go | 47 + internal/pkg/secureboot/pesign/pesign_test.go | 63 + .../pesign/testdata/systemd-bootx64.efi | Bin 0 -> 80896 bytes internal/pkg/secureboot/secureboot.go | 87 + internal/pkg/secureboot/tpm2/keys.go | 73 + internal/pkg/secureboot/tpm2/pcr.go | 194 ++ internal/pkg/secureboot/tpm2/pcr_test.go | 48 + internal/pkg/secureboot/tpm2/policy.go | 79 + internal/pkg/secureboot/tpm2/policy_test.go | 51 + internal/pkg/secureboot/tpm2/seal.go | 139 + internal/pkg/secureboot/tpm2/signature.go | 50 + .../tpm2/testdata/pcr-signing-crt.pem | 14 + internal/pkg/secureboot/tpm2/tpm2.go | 14 + internal/pkg/secureboot/tpm2/unseal.go | 269 ++ internal/pkg/secureboot/uki/assemble.go | 84 + internal/pkg/secureboot/uki/generate.go | 242 ++ internal/pkg/secureboot/uki/kernel.go | 69 + internal/pkg/secureboot/uki/kernel_test.go | 21 + internal/pkg/secureboot/uki/sbat.go | 35 + internal/pkg/secureboot/uki/sbat_test.go | 25 + internal/pkg/secureboot/uki/testdata/kernel | Bin 0 -> 16384 bytes internal/pkg/secureboot/uki/uki.go | 135 + internal/pkg/smbios/smbios.go | 27 + .../{bootstrap.go => imported_bootstrap.go} | 43 +- pkg/commands/imported_containers.go | 129 + .../{dashboard.go => imported_dashboard.go} | 52 +- pkg/commands/imported_disks.go | 168 ++ pkg/commands/imported_dmesg.go | 75 + pkg/commands/imported_etcd.go | 480 ++++ pkg/commands/imported_events.go | 152 ++ pkg/commands/{get.go => imported_get.go} | 40 +- pkg/commands/imported_health.go | 276 ++ pkg/commands/imported_image.go | 199 ++ pkg/commands/imported_kubeconfig.go | 203 ++ pkg/commands/imported_list.go | 210 ++ pkg/commands/imported_logs.go | 269 ++ pkg/commands/imported_memory.go | 177 ++ pkg/commands/imported_mounts.go | 71 + pkg/commands/imported_netstat.go | 436 +++ pkg/commands/imported_pcap.go | 285 ++ pkg/commands/imported_processes.go | 265 ++ pkg/commands/imported_read.go | 85 + pkg/commands/imported_reboot.go | 114 + pkg/commands/{reset.go => imported_reset.go} | 34 +- pkg/commands/imported_restart.go | 88 + pkg/commands/imported_rollback.go | 61 + pkg/commands/imported_root.go | 209 ++ pkg/commands/imported_service.go | 247 ++ pkg/commands/imported_shutdown.go | 106 + pkg/commands/imported_stats.go | 130 + pkg/commands/imported_time.go | 114 + pkg/commands/imported_version.go | 139 + tools/import_commands.go | 282 ++ tools/import_functions.go | 197 -- 747 files changed, 118600 insertions(+), 376 deletions(-) create mode 100644 internal/app/machined/internal/server/v1alpha1/v1alpha1_cluster.go create mode 100644 internal/app/machined/internal/server/v1alpha1/v1alpha1_images.go create mode 100644 internal/app/machined/internal/server/v1alpha1/v1alpha1_inspect.go create mode 100644 internal/app/machined/internal/server/v1alpha1/v1alpha1_meta.go create mode 100644 internal/app/machined/internal/server/v1alpha1/v1alpha1_monitoring.go create mode 100644 internal/app/machined/internal/server/v1alpha1/v1alpha1_server.go create mode 100644 internal/app/machined/internal/server/v1alpha1/v1alpha1_time.go create mode 100644 internal/app/machined/internal/server/v1alpha1/v1alpha1_time_test.go create mode 100644 internal/app/machined/main.go create mode 100644 internal/app/machined/pkg/adapters/cluster/cluster.go create mode 100644 internal/app/machined/pkg/adapters/cluster/identity.go create mode 100644 internal/app/machined/pkg/adapters/cluster/identity_test.go create mode 100644 internal/app/machined/pkg/adapters/hardware/hardware.go create mode 100644 internal/app/machined/pkg/adapters/hardware/memorymodule.go create mode 100644 internal/app/machined/pkg/adapters/hardware/processor.go create mode 100644 internal/app/machined/pkg/adapters/hardware/system_information.go create mode 100644 internal/app/machined/pkg/adapters/k8s/k8s.go create mode 100644 internal/app/machined/pkg/adapters/k8s/manifest.go create mode 100644 internal/app/machined/pkg/adapters/k8s/manifest_test.go create mode 100644 internal/app/machined/pkg/adapters/k8s/static_pod.go create mode 100644 internal/app/machined/pkg/adapters/k8s/static_pod_status.go create mode 100644 internal/app/machined/pkg/adapters/k8s/testdata/list.yaml create mode 100644 internal/app/machined/pkg/adapters/kubespan/identity.go create mode 100644 internal/app/machined/pkg/adapters/kubespan/identity_test.go create mode 100644 internal/app/machined/pkg/adapters/kubespan/kubespan.go create mode 100644 internal/app/machined/pkg/adapters/kubespan/peer_status.go create mode 100644 internal/app/machined/pkg/adapters/kubespan/peer_status_test.go create mode 100644 internal/app/machined/pkg/adapters/network/bond_master_spec.go create mode 100644 internal/app/machined/pkg/adapters/network/bond_master_spec_test.go create mode 100644 internal/app/machined/pkg/adapters/network/bridge_master_spec.go create mode 100644 internal/app/machined/pkg/adapters/network/ipset.go create mode 100644 internal/app/machined/pkg/adapters/network/ipset_test.go create mode 100644 internal/app/machined/pkg/adapters/network/network.go create mode 100644 internal/app/machined/pkg/adapters/network/nftables_rule.go create mode 100644 internal/app/machined/pkg/adapters/network/nftables_rule_test.go create mode 100644 internal/app/machined/pkg/adapters/network/vlan_spec.go create mode 100644 internal/app/machined/pkg/adapters/network/vlan_spec_test.go create mode 100644 internal/app/machined/pkg/adapters/network/wireguard_spec.go create mode 100644 internal/app/machined/pkg/adapters/network/wireguard_spec_test.go create mode 100644 internal/app/machined/pkg/adapters/perf/cpu.go create mode 100644 internal/app/machined/pkg/adapters/perf/mem.go create mode 100644 internal/app/machined/pkg/adapters/perf/perf.go create mode 100644 internal/app/machined/pkg/adapters/wireguard/wireguard.go create mode 100644 internal/app/machined/pkg/controllers/block/block.go create mode 100644 internal/app/machined/pkg/controllers/block/devices.go create mode 100644 internal/app/machined/pkg/controllers/block/devices_test.go create mode 100644 internal/app/machined/pkg/controllers/block/discovery.go create mode 100644 internal/app/machined/pkg/controllers/block/internal/inotify/inotify.go create mode 100644 internal/app/machined/pkg/controllers/block/internal/inotify/inotify_test.go create mode 100644 internal/app/machined/pkg/controllers/block/internal/kobject/kobject.go create mode 100644 internal/app/machined/pkg/controllers/block/internal/kobject/kobject_test.go create mode 100644 internal/app/machined/pkg/controllers/block/internal/sysblock/sysblock.go create mode 100644 internal/app/machined/pkg/controllers/block/internal/sysblock/sysblock_test.go create mode 100644 internal/app/machined/pkg/controllers/cluster/affiliate_merge.go create mode 100644 internal/app/machined/pkg/controllers/cluster/affiliate_merge_test.go create mode 100644 internal/app/machined/pkg/controllers/cluster/cluster.go create mode 100644 internal/app/machined/pkg/controllers/cluster/cluster_test.go create mode 100644 internal/app/machined/pkg/controllers/cluster/config.go create mode 100644 internal/app/machined/pkg/controllers/cluster/config_test.go create mode 100644 internal/app/machined/pkg/controllers/cluster/discovery_service.go create mode 100644 internal/app/machined/pkg/controllers/cluster/discovery_service_test.go create mode 100644 internal/app/machined/pkg/controllers/cluster/endpoint.go create mode 100644 internal/app/machined/pkg/controllers/cluster/endpoint_test.go create mode 100644 internal/app/machined/pkg/controllers/cluster/info.go create mode 100644 internal/app/machined/pkg/controllers/cluster/info_test.go create mode 100644 internal/app/machined/pkg/controllers/cluster/kubernetes_pull.go create mode 100644 internal/app/machined/pkg/controllers/cluster/kubernetes_push.go create mode 100644 internal/app/machined/pkg/controllers/cluster/local_affiliate.go create mode 100644 internal/app/machined/pkg/controllers/cluster/local_affiliate_test.go create mode 100644 internal/app/machined/pkg/controllers/cluster/member.go create mode 100644 internal/app/machined/pkg/controllers/cluster/member_test.go create mode 100644 internal/app/machined/pkg/controllers/cluster/node_identity.go create mode 100644 internal/app/machined/pkg/controllers/cluster/node_identity_test.go create mode 100644 internal/app/machined/pkg/controllers/config/acquire.go create mode 100644 internal/app/machined/pkg/controllers/config/acquire_test.go create mode 100644 internal/app/machined/pkg/controllers/config/config.go create mode 100644 internal/app/machined/pkg/controllers/config/machine_type.go create mode 100644 internal/app/machined/pkg/controllers/cri/cri.go create mode 100644 internal/app/machined/pkg/controllers/cri/cri_test.go create mode 100644 internal/app/machined/pkg/controllers/cri/seccomp_profile.go create mode 100644 internal/app/machined/pkg/controllers/cri/seccomp_profile_file.go create mode 100644 internal/app/machined/pkg/controllers/cri/seccomp_profile_file_test.go create mode 100644 internal/app/machined/pkg/controllers/cri/seccomp_profile_test.go create mode 100644 internal/app/machined/pkg/controllers/ctest/assert.go create mode 100644 internal/app/machined/pkg/controllers/ctest/ctest.go create mode 100644 internal/app/machined/pkg/controllers/etcd/advertised_peer.go create mode 100644 internal/app/machined/pkg/controllers/etcd/config.go create mode 100644 internal/app/machined/pkg/controllers/etcd/config_test.go create mode 100644 internal/app/machined/pkg/controllers/etcd/etcd.go create mode 100644 internal/app/machined/pkg/controllers/etcd/member.go create mode 100644 internal/app/machined/pkg/controllers/etcd/member_test.go create mode 100644 internal/app/machined/pkg/controllers/etcd/pki.go create mode 100644 internal/app/machined/pkg/controllers/etcd/spec.go create mode 100644 internal/app/machined/pkg/controllers/etcd/spec_test.go create mode 100644 internal/app/machined/pkg/controllers/files/cri_config_parts.go create mode 100644 internal/app/machined/pkg/controllers/files/cri_registry_config.go create mode 100644 internal/app/machined/pkg/controllers/files/etcfile.go create mode 100644 internal/app/machined/pkg/controllers/files/etcfile_test.go create mode 100644 internal/app/machined/pkg/controllers/files/files.go create mode 100644 internal/app/machined/pkg/controllers/hardware/hardware.go create mode 100644 internal/app/machined/pkg/controllers/hardware/hardware_test.go create mode 100644 internal/app/machined/pkg/controllers/hardware/system.go create mode 100644 internal/app/machined/pkg/controllers/hardware/system_test.go create mode 100644 internal/app/machined/pkg/controllers/hardware/testdata/SuperMicro-Dual-Xeon.dmi create mode 100644 internal/app/machined/pkg/controllers/k8s/address_filter.go create mode 100644 internal/app/machined/pkg/controllers/k8s/address_filter_test.go create mode 100644 internal/app/machined/pkg/controllers/k8s/control_plane.go create mode 100644 internal/app/machined/pkg/controllers/k8s/control_plane_static_pod.go create mode 100644 internal/app/machined/pkg/controllers/k8s/control_plane_static_pod_test.go create mode 100644 internal/app/machined/pkg/controllers/k8s/control_plane_test.go create mode 100644 internal/app/machined/pkg/controllers/k8s/endpoint.go create mode 100644 internal/app/machined/pkg/controllers/k8s/extra_manifest.go create mode 100644 internal/app/machined/pkg/controllers/k8s/extra_manifest_test.go create mode 100644 internal/app/machined/pkg/controllers/k8s/internal/nodename/nodename.go create mode 100644 internal/app/machined/pkg/controllers/k8s/internal/nodename/nodename_test.go create mode 100644 internal/app/machined/pkg/controllers/k8s/internal/nodewatch/nodewatch.go create mode 100644 internal/app/machined/pkg/controllers/k8s/k8s.go create mode 100644 internal/app/machined/pkg/controllers/k8s/kubelet_config.go create mode 100644 internal/app/machined/pkg/controllers/k8s/kubelet_config_test.go create mode 100644 internal/app/machined/pkg/controllers/k8s/kubelet_service.go create mode 100644 internal/app/machined/pkg/controllers/k8s/kubelet_spec.go create mode 100644 internal/app/machined/pkg/controllers/k8s/kubelet_spec_test.go create mode 100644 internal/app/machined/pkg/controllers/k8s/kubelet_static_pod.go create mode 100644 internal/app/machined/pkg/controllers/k8s/kubeprism.go create mode 100644 internal/app/machined/pkg/controllers/k8s/kubeprism_config.go create mode 100644 internal/app/machined/pkg/controllers/k8s/kubeprism_config_test.go create mode 100644 internal/app/machined/pkg/controllers/k8s/kubeprism_endpoints.go create mode 100644 internal/app/machined/pkg/controllers/k8s/kubeprism_endpoints_test.go create mode 100644 internal/app/machined/pkg/controllers/k8s/manifest.go create mode 100644 internal/app/machined/pkg/controllers/k8s/manifest_apply.go create mode 100644 internal/app/machined/pkg/controllers/k8s/manifest_test.go create mode 100644 internal/app/machined/pkg/controllers/k8s/node_apply.go create mode 100644 internal/app/machined/pkg/controllers/k8s/node_apply_test.go create mode 100644 internal/app/machined/pkg/controllers/k8s/node_cordoned_spec.go create mode 100644 internal/app/machined/pkg/controllers/k8s/node_cordoned_spec_test.go create mode 100644 internal/app/machined/pkg/controllers/k8s/node_label_spec.go create mode 100644 internal/app/machined/pkg/controllers/k8s/node_label_spec_test.go create mode 100644 internal/app/machined/pkg/controllers/k8s/node_status.go create mode 100644 internal/app/machined/pkg/controllers/k8s/node_taint_spec.go create mode 100644 internal/app/machined/pkg/controllers/k8s/node_taint_spec_test.go create mode 100644 internal/app/machined/pkg/controllers/k8s/nodeip.go create mode 100644 internal/app/machined/pkg/controllers/k8s/nodeip_config.go create mode 100644 internal/app/machined/pkg/controllers/k8s/nodeip_config_test.go create mode 100644 internal/app/machined/pkg/controllers/k8s/nodeip_test.go create mode 100644 internal/app/machined/pkg/controllers/k8s/nodename.go create mode 100644 internal/app/machined/pkg/controllers/k8s/nodename_test.go create mode 100644 internal/app/machined/pkg/controllers/k8s/render_config_static_pods.go create mode 100644 internal/app/machined/pkg/controllers/k8s/render_secrets_static_pod.go create mode 100644 internal/app/machined/pkg/controllers/k8s/static_endpoint.go create mode 100644 internal/app/machined/pkg/controllers/k8s/static_endpoint_test.go create mode 100644 internal/app/machined/pkg/controllers/k8s/static_pod_config.go create mode 100644 internal/app/machined/pkg/controllers/k8s/static_pod_config_test.go create mode 100644 internal/app/machined/pkg/controllers/k8s/static_pod_server.go create mode 100644 internal/app/machined/pkg/controllers/k8s/static_pod_server_test.go create mode 100644 internal/app/machined/pkg/controllers/k8s/templates.go create mode 100644 internal/app/machined/pkg/controllers/kubeaccess/config.go create mode 100644 internal/app/machined/pkg/controllers/kubeaccess/config_test.go create mode 100644 internal/app/machined/pkg/controllers/kubeaccess/endpoint.go create mode 100644 internal/app/machined/pkg/controllers/kubeaccess/kubeaccess.go create mode 100644 internal/app/machined/pkg/controllers/kubeaccess/serviceaccount.go create mode 100644 internal/app/machined/pkg/controllers/kubeaccess/serviceaccount/crd_controller.go create mode 100644 internal/app/machined/pkg/controllers/kubespan/config.go create mode 100644 internal/app/machined/pkg/controllers/kubespan/config_test.go create mode 100644 internal/app/machined/pkg/controllers/kubespan/endpoint.go create mode 100644 internal/app/machined/pkg/controllers/kubespan/endpoint_test.go create mode 100644 internal/app/machined/pkg/controllers/kubespan/identity.go create mode 100644 internal/app/machined/pkg/controllers/kubespan/identity_test.go create mode 100644 internal/app/machined/pkg/controllers/kubespan/kubespan.go create mode 100644 internal/app/machined/pkg/controllers/kubespan/kubespan_test.go create mode 100644 internal/app/machined/pkg/controllers/kubespan/manager.go create mode 100644 internal/app/machined/pkg/controllers/kubespan/manager_test.go create mode 100644 internal/app/machined/pkg/controllers/kubespan/peer_spec.go create mode 100644 internal/app/machined/pkg/controllers/kubespan/peer_spec_test.go create mode 100644 internal/app/machined/pkg/controllers/kubespan/routing_rules.go create mode 100644 internal/app/machined/pkg/controllers/kubespan/routing_rules_test.go create mode 100644 internal/app/machined/pkg/controllers/network/address_config.go create mode 100644 internal/app/machined/pkg/controllers/network/address_config_test.go create mode 100644 internal/app/machined/pkg/controllers/network/address_event.go create mode 100644 internal/app/machined/pkg/controllers/network/address_event_test.go create mode 100644 internal/app/machined/pkg/controllers/network/address_merge.go create mode 100644 internal/app/machined/pkg/controllers/network/address_merge_test.go create mode 100644 internal/app/machined/pkg/controllers/network/address_spec.go create mode 100644 internal/app/machined/pkg/controllers/network/address_spec_test.go create mode 100644 internal/app/machined/pkg/controllers/network/address_status.go create mode 100644 internal/app/machined/pkg/controllers/network/address_status_test.go create mode 100644 internal/app/machined/pkg/controllers/network/cmdline.go create mode 100644 internal/app/machined/pkg/controllers/network/cmdline_test.go create mode 100644 internal/app/machined/pkg/controllers/network/device_config.go create mode 100644 internal/app/machined/pkg/controllers/network/device_config_test.go create mode 100644 internal/app/machined/pkg/controllers/network/dns_resolve_cache.go create mode 100644 internal/app/machined/pkg/controllers/network/dns_resolve_cache_test.go create mode 100644 internal/app/machined/pkg/controllers/network/dns_upstream.go create mode 100644 internal/app/machined/pkg/controllers/network/etcfile.go create mode 100644 internal/app/machined/pkg/controllers/network/etcfile_test.go create mode 100644 internal/app/machined/pkg/controllers/network/hardware_addr.go create mode 100644 internal/app/machined/pkg/controllers/network/hardware_addr_test.go create mode 100644 internal/app/machined/pkg/controllers/network/hostdns_config.go create mode 100644 internal/app/machined/pkg/controllers/network/hostname_config.go create mode 100644 internal/app/machined/pkg/controllers/network/hostname_config_test.go create mode 100644 internal/app/machined/pkg/controllers/network/hostname_merge.go create mode 100644 internal/app/machined/pkg/controllers/network/hostname_merge_test.go create mode 100644 internal/app/machined/pkg/controllers/network/hostname_spec.go create mode 100644 internal/app/machined/pkg/controllers/network/hostname_spec_test.go create mode 100644 internal/app/machined/pkg/controllers/network/internal/probe/probe.go create mode 100644 internal/app/machined/pkg/controllers/network/internal/probe/probe_test.go create mode 100644 internal/app/machined/pkg/controllers/network/link_config.go create mode 100644 internal/app/machined/pkg/controllers/network/link_config_test.go create mode 100644 internal/app/machined/pkg/controllers/network/link_merge.go create mode 100644 internal/app/machined/pkg/controllers/network/link_merge_test.go create mode 100644 internal/app/machined/pkg/controllers/network/link_spec.go create mode 100644 internal/app/machined/pkg/controllers/network/link_spec_test.go create mode 100644 internal/app/machined/pkg/controllers/network/link_status.go create mode 100644 internal/app/machined/pkg/controllers/network/link_status_test.go create mode 100644 internal/app/machined/pkg/controllers/network/network.go create mode 100644 internal/app/machined/pkg/controllers/network/nftables_chain.go create mode 100644 internal/app/machined/pkg/controllers/network/nftables_chain_config.go create mode 100644 internal/app/machined/pkg/controllers/network/nftables_chain_config_test.go create mode 100644 internal/app/machined/pkg/controllers/network/nftables_chain_test.go create mode 100644 internal/app/machined/pkg/controllers/network/node_address.go create mode 100644 internal/app/machined/pkg/controllers/network/node_address_test.go create mode 100644 internal/app/machined/pkg/controllers/network/operator/dhcp4.go create mode 100644 internal/app/machined/pkg/controllers/network/operator/dhcp6.go create mode 100644 internal/app/machined/pkg/controllers/network/operator/operator.go create mode 100644 internal/app/machined/pkg/controllers/network/operator/vip.go create mode 100644 internal/app/machined/pkg/controllers/network/operator/vip/equinix_metal.go create mode 100644 internal/app/machined/pkg/controllers/network/operator/vip/equinix_metal_test.go create mode 100644 internal/app/machined/pkg/controllers/network/operator/vip/hcloud.go create mode 100644 internal/app/machined/pkg/controllers/network/operator/vip/nop.go create mode 100644 internal/app/machined/pkg/controllers/network/operator/vip/vip.go create mode 100644 internal/app/machined/pkg/controllers/network/operator_config.go create mode 100644 internal/app/machined/pkg/controllers/network/operator_config_test.go create mode 100644 internal/app/machined/pkg/controllers/network/operator_merge.go create mode 100644 internal/app/machined/pkg/controllers/network/operator_merge_test.go create mode 100644 internal/app/machined/pkg/controllers/network/operator_spec.go create mode 100644 internal/app/machined/pkg/controllers/network/operator_spec_test.go create mode 100644 internal/app/machined/pkg/controllers/network/operator_vip_config.go create mode 100644 internal/app/machined/pkg/controllers/network/operator_vip_config_test.go create mode 100644 internal/app/machined/pkg/controllers/network/platform_config.go create mode 100644 internal/app/machined/pkg/controllers/network/platform_config_test.go create mode 100644 internal/app/machined/pkg/controllers/network/probe.go create mode 100644 internal/app/machined/pkg/controllers/network/probe_test.go create mode 100644 internal/app/machined/pkg/controllers/network/resolver_config.go create mode 100644 internal/app/machined/pkg/controllers/network/resolver_config_test.go create mode 100644 internal/app/machined/pkg/controllers/network/resolver_merge.go create mode 100644 internal/app/machined/pkg/controllers/network/resolver_merge_test.go create mode 100644 internal/app/machined/pkg/controllers/network/resolver_spec.go create mode 100644 internal/app/machined/pkg/controllers/network/resolver_spec_test.go create mode 100644 internal/app/machined/pkg/controllers/network/route_config.go create mode 100644 internal/app/machined/pkg/controllers/network/route_config_test.go create mode 100644 internal/app/machined/pkg/controllers/network/route_merge.go create mode 100644 internal/app/machined/pkg/controllers/network/route_merge_test.go create mode 100644 internal/app/machined/pkg/controllers/network/route_spec.go create mode 100644 internal/app/machined/pkg/controllers/network/route_spec_test.go create mode 100644 internal/app/machined/pkg/controllers/network/route_status.go create mode 100644 internal/app/machined/pkg/controllers/network/route_status_test.go create mode 100644 internal/app/machined/pkg/controllers/network/status.go create mode 100644 internal/app/machined/pkg/controllers/network/status_test.go create mode 100644 internal/app/machined/pkg/controllers/network/timeserver_config.go create mode 100644 internal/app/machined/pkg/controllers/network/timeserver_config_test.go create mode 100644 internal/app/machined/pkg/controllers/network/timeserver_merge.go create mode 100644 internal/app/machined/pkg/controllers/network/timeserver_merge_test.go create mode 100644 internal/app/machined/pkg/controllers/network/timeserver_spec.go create mode 100644 internal/app/machined/pkg/controllers/network/timeserver_spec_test.go create mode 100644 internal/app/machined/pkg/controllers/network/utils/utils.go create mode 100644 internal/app/machined/pkg/controllers/network/watch/ethtool.go create mode 100644 internal/app/machined/pkg/controllers/network/watch/rtnetlink.go create mode 100644 internal/app/machined/pkg/controllers/network/watch/trigger.go create mode 100644 internal/app/machined/pkg/controllers/network/watch/trigger_test.go create mode 100644 internal/app/machined/pkg/controllers/network/watch/watch.go create mode 100644 internal/app/machined/pkg/controllers/perf/perf.go create mode 100644 internal/app/machined/pkg/controllers/perf/perf_test.go create mode 100644 internal/app/machined/pkg/controllers/runtime/common_test.go create mode 100644 internal/app/machined/pkg/controllers/runtime/cri_image_gc.go create mode 100644 internal/app/machined/pkg/controllers/runtime/cri_image_gc_test.go create mode 100644 internal/app/machined/pkg/controllers/runtime/devices_status.go create mode 100644 internal/app/machined/pkg/controllers/runtime/drop_upgrade_fallback.go create mode 100644 internal/app/machined/pkg/controllers/runtime/drop_upgrade_fallback_test.go create mode 100644 internal/app/machined/pkg/controllers/runtime/events_sink.go create mode 100644 internal/app/machined/pkg/controllers/runtime/events_sink_config.go create mode 100644 internal/app/machined/pkg/controllers/runtime/events_sink_config_test.go create mode 100644 internal/app/machined/pkg/controllers/runtime/events_sink_test.go create mode 100644 internal/app/machined/pkg/controllers/runtime/export_test.go create mode 100644 internal/app/machined/pkg/controllers/runtime/extension_service.go create mode 100644 internal/app/machined/pkg/controllers/runtime/extension_service_config.go create mode 100644 internal/app/machined/pkg/controllers/runtime/extension_service_config_files.go create mode 100644 internal/app/machined/pkg/controllers/runtime/extension_service_config_files_test.go create mode 100644 internal/app/machined/pkg/controllers/runtime/extension_service_config_test.go create mode 100644 internal/app/machined/pkg/controllers/runtime/extension_service_test.go create mode 100644 internal/app/machined/pkg/controllers/runtime/extension_status.go create mode 100644 internal/app/machined/pkg/controllers/runtime/kernel_module_config.go create mode 100644 internal/app/machined/pkg/controllers/runtime/kernel_module_config_test.go create mode 100644 internal/app/machined/pkg/controllers/runtime/kernel_module_spec.go create mode 100644 internal/app/machined/pkg/controllers/runtime/kernel_param_config.go create mode 100644 internal/app/machined/pkg/controllers/runtime/kernel_param_config_test.go create mode 100644 internal/app/machined/pkg/controllers/runtime/kernel_param_defaults.go create mode 100644 internal/app/machined/pkg/controllers/runtime/kernel_param_defaults_test.go create mode 100644 internal/app/machined/pkg/controllers/runtime/kernel_param_spec.go create mode 100644 internal/app/machined/pkg/controllers/runtime/kernel_param_spec_test.go create mode 100644 internal/app/machined/pkg/controllers/runtime/kmsg_log.go create mode 100644 internal/app/machined/pkg/controllers/runtime/kmsg_log_config.go create mode 100644 internal/app/machined/pkg/controllers/runtime/kmsg_log_config_test.go create mode 100644 internal/app/machined/pkg/controllers/runtime/kmsg_log_test.go create mode 100644 internal/app/machined/pkg/controllers/runtime/machine_status.go create mode 100644 internal/app/machined/pkg/controllers/runtime/machine_status_publisher.go create mode 100644 internal/app/machined/pkg/controllers/runtime/machine_status_test.go create mode 100644 internal/app/machined/pkg/controllers/runtime/maintenance_config.go create mode 100644 internal/app/machined/pkg/controllers/runtime/maintenance_config_test.go create mode 100644 internal/app/machined/pkg/controllers/runtime/maintenance_service.go create mode 100644 internal/app/machined/pkg/controllers/runtime/maintenance_service_test.go create mode 100644 internal/app/machined/pkg/controllers/runtime/runtime.go create mode 100644 internal/app/machined/pkg/controllers/runtime/security_state.go create mode 100644 internal/app/machined/pkg/controllers/runtime/testdata/extservices/foo.bar create mode 100644 internal/app/machined/pkg/controllers/runtime/testdata/extservices/frr.yaml create mode 100644 internal/app/machined/pkg/controllers/runtime/testdata/extservices/hello.yaml create mode 100644 internal/app/machined/pkg/controllers/runtime/testdata/extservices/invalid.yaml create mode 100644 internal/app/machined/pkg/controllers/runtime/testdata/extservices/zduplicate.yaml create mode 100644 internal/app/machined/pkg/controllers/runtime/unique_token.go create mode 100644 internal/app/machined/pkg/controllers/runtime/utils.go create mode 100644 internal/app/machined/pkg/controllers/runtime/watchdog_timer.go create mode 100644 internal/app/machined/pkg/controllers/runtime/watchdog_timer_config.go create mode 100644 internal/app/machined/pkg/controllers/runtime/watchdog_timer_config_test.go create mode 100644 internal/app/machined/pkg/controllers/secrets/api.go create mode 100644 internal/app/machined/pkg/controllers/secrets/api_cert_sans.go create mode 100644 internal/app/machined/pkg/controllers/secrets/api_cert_sans_test.go create mode 100644 internal/app/machined/pkg/controllers/secrets/api_test.go create mode 100644 internal/app/machined/pkg/controllers/secrets/etcd.go create mode 100644 internal/app/machined/pkg/controllers/secrets/etcd_test.go create mode 100644 internal/app/machined/pkg/controllers/secrets/kubelet.go create mode 100644 internal/app/machined/pkg/controllers/secrets/kubelet_test.go create mode 100644 internal/app/machined/pkg/controllers/secrets/kubernetes.go create mode 100644 internal/app/machined/pkg/controllers/secrets/kubernetes_cert_sans.go create mode 100644 internal/app/machined/pkg/controllers/secrets/kubernetes_cert_sans_test.go create mode 100644 internal/app/machined/pkg/controllers/secrets/kubernetes_dynamic_certs.go create mode 100644 internal/app/machined/pkg/controllers/secrets/kubernetes_dynamic_certs_test.go create mode 100644 internal/app/machined/pkg/controllers/secrets/kubernetes_test.go create mode 100644 internal/app/machined/pkg/controllers/secrets/maintenance.go create mode 100644 internal/app/machined/pkg/controllers/secrets/maintenance_cert_sans.go create mode 100644 internal/app/machined/pkg/controllers/secrets/maintenance_cert_sans_test.go create mode 100644 internal/app/machined/pkg/controllers/secrets/maintenance_root.go create mode 100644 internal/app/machined/pkg/controllers/secrets/maintenance_root_test.go create mode 100644 internal/app/machined/pkg/controllers/secrets/maintenance_test.go create mode 100644 internal/app/machined/pkg/controllers/secrets/root.go create mode 100644 internal/app/machined/pkg/controllers/secrets/root_test.go create mode 100644 internal/app/machined/pkg/controllers/secrets/secrets.go create mode 100644 internal/app/machined/pkg/controllers/secrets/trustd.go create mode 100644 internal/app/machined/pkg/controllers/secrets/trustd_test.go create mode 100644 internal/app/machined/pkg/controllers/siderolink/config.go create mode 100644 internal/app/machined/pkg/controllers/siderolink/config_test.go create mode 100644 internal/app/machined/pkg/controllers/siderolink/manager.go create mode 100644 internal/app/machined/pkg/controllers/siderolink/manager_test.go create mode 100644 internal/app/machined/pkg/controllers/siderolink/siderolink.go create mode 100644 internal/app/machined/pkg/controllers/siderolink/userspace.go create mode 100644 internal/app/machined/pkg/controllers/time/adjtime_status.go create mode 100644 internal/app/machined/pkg/controllers/time/sync.go create mode 100644 internal/app/machined/pkg/controllers/time/sync_test.go create mode 100644 internal/app/machined/pkg/controllers/time/time.go create mode 100644 internal/app/machined/pkg/controllers/utils.go create mode 100644 internal/app/machined/pkg/controllers/v1alpha1/service.go create mode 100644 internal/app/machined/pkg/controllers/v1alpha1/utils.go create mode 100644 internal/app/machined/pkg/controllers/v1alpha1/v1alpha1.go create mode 100644 internal/app/machined/pkg/runtime/board.go create mode 100644 internal/app/machined/pkg/runtime/controller.go create mode 100644 internal/app/machined/pkg/runtime/disk/disk.go create mode 100644 internal/app/machined/pkg/runtime/disk/options.go create mode 100644 internal/app/machined/pkg/runtime/doc.go create mode 100644 internal/app/machined/pkg/runtime/drainer.go create mode 100644 internal/app/machined/pkg/runtime/drainer_test.go create mode 100644 internal/app/machined/pkg/runtime/emergency/emergency.go create mode 100644 internal/app/machined/pkg/runtime/errors.go create mode 100644 internal/app/machined/pkg/runtime/events.go create mode 100644 internal/app/machined/pkg/runtime/logging.go create mode 100644 internal/app/machined/pkg/runtime/logging/circular.go create mode 100644 internal/app/machined/pkg/runtime/logging/extract.go create mode 100644 internal/app/machined/pkg/runtime/logging/extract_test.go create mode 100644 internal/app/machined/pkg/runtime/logging/file.go create mode 100644 internal/app/machined/pkg/runtime/logging/logging.go create mode 100644 internal/app/machined/pkg/runtime/logging/null.go create mode 100644 internal/app/machined/pkg/runtime/logging/sender_jsonlines.go create mode 100644 internal/app/machined/pkg/runtime/logging/sender_jsonlines_test.go create mode 100644 internal/app/machined/pkg/runtime/mode.go create mode 100644 internal/app/machined/pkg/runtime/mode_test.go create mode 100644 internal/app/machined/pkg/runtime/platform.go create mode 100644 internal/app/machined/pkg/runtime/runtime.go create mode 100644 internal/app/machined/pkg/runtime/sequencer.go create mode 100644 internal/app/machined/pkg/runtime/sequencer_test.go create mode 100644 internal/app/machined/pkg/runtime/state.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/acpi/acpi.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/acpi/acpi_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/board/bananapi_m64/bananapi_m64.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/board/board.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/board/jetson_nano/jetson_nano.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/board/libretech_all_h3_cc_h5/libretech_all_h3_cc_h5.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/board/nanopi_r4s/nanopi_r4s.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/board/pine64/pine64.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/board/rock64/rock64.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/board/rockpi4/rockpi4.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/board/rockpi4c/rockpi4c.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/board/rpi_generic/config.txt create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/board/rpi_generic/rpi_generic.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/bootloader/bootloader.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/boot_label.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/constants.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/decode.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/encode.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/grub.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/grub_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/install.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/probe.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/quote.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/quote_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/revert.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/testdata/grub_parse_test.cfg create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/testdata/grub_write_no_reset_test.cfg create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/testdata/grub_write_test.cfg create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/bootloader/mount/mount.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/bootloader/options/options.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/bootloader/sdboot/efivars.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/bootloader/sdboot/sdboot.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/doc.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/akamai/akamai.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/akamai/akamai_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/akamai/testdata/expected.yaml create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/akamai/testdata/instance.json create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/akamai/testdata/network.json create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/aws/aws.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/aws/aws_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/aws/metadata.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/aws/testdata/expected.yaml create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/aws/testdata/metadata.json create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/azure/azure.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/azure/azure_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/azure/metadata.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/azure/register.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/azure/testdata/compute.json create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/azure/testdata/expected.yaml create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/azure/testdata/interfaces.json create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/azure/testdata/loadbalancer.json create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/container/container.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/container/internal/files/hostname.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/container/internal/files/hostname_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/container/internal/files/resolv.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/container/internal/files/resolv_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/container/internal/files/testdata/hostname create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/container/internal/files/testdata/resolv.conf create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean/digitalocean.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean/digitalocean_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean/metadata.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean/testdata/expected.yaml create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean/testdata/metadata.json create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/equinixmetal/equinix.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/equinixmetal/equinix_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/equinixmetal/metadata.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/equinixmetal/testdata/expected.yaml create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/equinixmetal/testdata/metadata.json create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/errors/errors.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/exoscale/exoscale.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/exoscale/exoscale_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/exoscale/metadata.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/exoscale/testdata/expected.yaml create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/exoscale/testdata/metadata.json create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/gcp/gcp.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/gcp/gcp_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/gcp/metadata.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/gcp/testdata/expected.yaml create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/gcp/testdata/interfaces.json create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/gcp/testdata/metadata.json create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/hcloud/hcloud.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/hcloud/hcloud_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/hcloud/metadata.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/hcloud/testdata/expected.yaml create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/hcloud/testdata/metadata.yaml create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/internal/address/address.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/internal/netutils/netutils.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/metal/metal.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/metal/metal_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/metal/oauth2/oauth2.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/metal/oauth2/oauth2_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url/map.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url/map_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url/url.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url/url_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url/value.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url/variable.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url/variable_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/metadata.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/nocloud.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/nocloud_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/testdata/expected-v1.yaml create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/testdata/expected-v2.yaml create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/testdata/metadata-v1.yaml create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/testdata/metadata-v2.yaml create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/testdata/metadata-v3.yaml create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/opennebula/metadata.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/opennebula/opennebula.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/opennebula/opennebula_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/opennebula/testdata/expected.yaml create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/opennebula/testdata/metadata.yaml create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/openstack/metadata.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/openstack/openstack.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/openstack/openstack_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/openstack/testdata/expected.yaml create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/openstack/testdata/metadata.json create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/openstack/testdata/network.json create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/oracle/metadata.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/oracle/oracle.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/oracle/oracle_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/oracle/testdata/expected.yaml create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/oracle/testdata/metadata.json create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/oracle/testdata/metadatanetwork.json create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/platform.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/scaleway/metadata.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/scaleway/scaleway.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/scaleway/scaleway_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/scaleway/testdata/expected.yaml create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/scaleway/testdata/metadata.json create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/upcloud/metadata.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/upcloud/testdata/expected.yaml create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/upcloud/testdata/metadata.json create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/upcloud/upcloud.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/upcloud/upcloud_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/metadata.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/testdata/expected-match-by-mac.yaml create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/testdata/expected-match-by-name.yaml create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/testdata/metadata-match-by-mac.yaml create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/testdata/metadata-match-by-name.yaml create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/vmware_amd64.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/vmware_other.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/vmware_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/vultr/metadata.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/vultr/testdata/expected.yaml create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/vultr/testdata/metadata.json create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/vultr/vultr.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/platform/vultr/vultr_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_controller.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_controller_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_dbus.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_events.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_events_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_priority_lock.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_priority_lock_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_runtime.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_test.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_state.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha2/adapters.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha2/v1alpha2.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go create mode 100644 internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_state.go create mode 100644 internal/app/machined/pkg/system/events/events.go create mode 100644 internal/app/machined/pkg/system/events/events_test.go create mode 100644 internal/app/machined/pkg/system/export_test.go create mode 100644 internal/app/machined/pkg/system/health/check.go create mode 100644 internal/app/machined/pkg/system/health/health_test.go create mode 100644 internal/app/machined/pkg/system/health/settings.go create mode 100644 internal/app/machined/pkg/system/health/status.go create mode 100644 internal/app/machined/pkg/system/integration_test.go create mode 100644 internal/app/machined/pkg/system/mocks_test.go create mode 100644 internal/app/machined/pkg/system/runner/containerd/containerd.go create mode 100644 internal/app/machined/pkg/system/runner/containerd/containerd_test.go create mode 100644 internal/app/machined/pkg/system/runner/containerd/import.go create mode 100644 internal/app/machined/pkg/system/runner/containerd/opts.go create mode 100644 internal/app/machined/pkg/system/runner/containerd/stdin.go create mode 100644 internal/app/machined/pkg/system/runner/goroutine/goroutine.go create mode 100644 internal/app/machined/pkg/system/runner/goroutine/goroutine_test.go create mode 100644 internal/app/machined/pkg/system/runner/process/process.go create mode 100644 internal/app/machined/pkg/system/runner/process/process_test.go create mode 100644 internal/app/machined/pkg/system/runner/restart/restart.go create mode 100644 internal/app/machined/pkg/system/runner/restart/restart_test.go create mode 100644 internal/app/machined/pkg/system/runner/runner.go create mode 100644 internal/app/machined/pkg/system/runner/runner_test.go create mode 100644 internal/app/machined/pkg/system/service.go create mode 100644 internal/app/machined/pkg/system/service_events.go create mode 100644 internal/app/machined/pkg/system/service_runner.go create mode 100644 internal/app/machined/pkg/system/service_runner_test.go create mode 100644 internal/app/machined/pkg/system/services/apid.go create mode 100644 internal/app/machined/pkg/system/services/containerd.go create mode 100644 internal/app/machined/pkg/system/services/cri.go create mode 100644 internal/app/machined/pkg/system/services/dashboard.go create mode 100644 internal/app/machined/pkg/system/services/etcd.go create mode 100644 internal/app/machined/pkg/system/services/export_test.go create mode 100644 internal/app/machined/pkg/system/services/extension.go create mode 100644 internal/app/machined/pkg/system/services/extension_test.go create mode 100644 internal/app/machined/pkg/system/services/kubelet.go create mode 100644 internal/app/machined/pkg/system/services/machined.go create mode 100644 internal/app/machined/pkg/system/services/machined_test.go create mode 100644 internal/app/machined/pkg/system/services/mocks/snapshotter.go create mode 100644 internal/app/machined/pkg/system/services/syslogd.go create mode 100644 internal/app/machined/pkg/system/services/trustd.go create mode 100644 internal/app/machined/pkg/system/services/udevd.go create mode 100644 internal/app/machined/pkg/system/services/utils.go create mode 100644 internal/app/machined/pkg/system/system.go create mode 100644 internal/app/machined/pkg/system/system_test.go create mode 100644 internal/app/machined/revert.go create mode 100644 internal/pkg/cgroup/cgroup.go create mode 100644 internal/pkg/cri/client.go create mode 100644 internal/pkg/cri/containers.go create mode 100644 internal/pkg/cri/cri.go create mode 100644 internal/pkg/cri/cri_test.go create mode 100644 internal/pkg/cri/images.go create mode 100644 internal/pkg/cri/pods.go create mode 100644 internal/pkg/dashboard/apidata/apidata.go create mode 100644 internal/pkg/dashboard/apidata/data.go create mode 100644 internal/pkg/dashboard/apidata/diff.go create mode 100644 internal/pkg/dashboard/apidata/node.go create mode 100644 internal/pkg/dashboard/apidata/source.go create mode 100644 internal/pkg/dashboard/components/components.go create mode 100644 internal/pkg/dashboard/components/footer.go create mode 100644 internal/pkg/dashboard/components/gauges.go create mode 100644 internal/pkg/dashboard/components/graphs.go create mode 100644 internal/pkg/dashboard/components/header.go create mode 100644 internal/pkg/dashboard/components/horizontalline.go create mode 100644 internal/pkg/dashboard/components/info.go create mode 100644 internal/pkg/dashboard/components/kubernetesinfo.go create mode 100644 internal/pkg/dashboard/components/logviewer.go create mode 100644 internal/pkg/dashboard/components/networkinfo.go create mode 100644 internal/pkg/dashboard/components/sparklines.go create mode 100644 internal/pkg/dashboard/components/tables.go create mode 100644 internal/pkg/dashboard/components/tables_test.go create mode 100644 internal/pkg/dashboard/components/talosinfo.go create mode 100644 internal/pkg/dashboard/components/termui.go create mode 100644 internal/pkg/dashboard/configurl.go create mode 100644 internal/pkg/dashboard/context.go create mode 100644 internal/pkg/dashboard/dashboard.go create mode 100644 internal/pkg/dashboard/formdata.go create mode 100644 internal/pkg/dashboard/formdata_test.go create mode 100644 internal/pkg/dashboard/logdata/logdata.go create mode 100644 internal/pkg/dashboard/monitor.go create mode 100644 internal/pkg/dashboard/networkconfig.go create mode 100644 internal/pkg/dashboard/options.go create mode 100644 internal/pkg/dashboard/resourcedata/resourcedata.go create mode 100644 internal/pkg/dashboard/summary.go create mode 100644 internal/pkg/dashboard/util/util.go create mode 100644 internal/pkg/environment/environment.go create mode 100644 internal/pkg/environment/environment_test.go create mode 100644 internal/pkg/etcd/certs.go create mode 100644 internal/pkg/etcd/endpoints.go create mode 100644 internal/pkg/etcd/etcd.go create mode 100644 internal/pkg/etcd/local.go create mode 100644 internal/pkg/etcd/lock.go create mode 100644 internal/pkg/install/install.go create mode 100644 internal/pkg/install/options.go create mode 100644 internal/pkg/install/pull.go create mode 100644 internal/pkg/logind/broker.go create mode 100644 internal/pkg/logind/dbus.go create mode 100644 internal/pkg/logind/kubelet_mock_test.go create mode 100644 internal/pkg/logind/logind.go create mode 100644 internal/pkg/logind/logind_test.go create mode 100644 internal/pkg/logind/service.go create mode 100644 internal/pkg/meta/constants.go create mode 100644 internal/pkg/meta/internal/adv/adv.go create mode 100644 internal/pkg/meta/internal/adv/syslinux/syslinux.go create mode 100644 internal/pkg/meta/internal/adv/syslinux/syslinux_test.go create mode 100644 internal/pkg/meta/internal/adv/syslinux/testdata/adv.sys create mode 100644 internal/pkg/meta/internal/adv/talos/talos.go create mode 100644 internal/pkg/meta/internal/adv/talos/talos_test.go create mode 100644 internal/pkg/meta/meta.go create mode 100644 internal/pkg/meta/meta_test.go create mode 100644 internal/pkg/mount/all.go create mode 100644 internal/pkg/mount/bpffs.go create mode 100644 internal/pkg/mount/cgroups.go create mode 100644 internal/pkg/mount/iter.go create mode 100644 internal/pkg/mount/mount.go create mode 100644 internal/pkg/mount/mount_test.go create mode 100644 internal/pkg/mount/options.go create mode 100644 internal/pkg/mount/overlay.go create mode 100644 internal/pkg/mount/pseudo.go create mode 100644 internal/pkg/mount/squashfs.go create mode 100644 internal/pkg/mount/switchroot/switchroot.go create mode 100644 internal/pkg/mount/switchroot/switchroot_test.go create mode 100644 internal/pkg/mount/system.go create mode 100644 internal/pkg/mount/unmount.go create mode 100644 internal/pkg/partition/constants.go create mode 100644 internal/pkg/partition/format.go create mode 100644 internal/pkg/partition/format_test.go create mode 100644 internal/pkg/partition/partition.go create mode 100644 internal/pkg/secureboot/database/database.go create mode 100644 internal/pkg/secureboot/measure/internal/pcr/bank_data.go create mode 100644 internal/pkg/secureboot/measure/internal/pcr/bank_data_test.go create mode 100644 internal/pkg/secureboot/measure/internal/pcr/extend.go create mode 100644 internal/pkg/secureboot/measure/internal/pcr/extend_test.go create mode 100644 internal/pkg/secureboot/measure/internal/pcr/sign.go create mode 100644 internal/pkg/secureboot/measure/internal/pcr/sign_test.go create mode 100644 internal/pkg/secureboot/measure/internal/pcr/testdata/a create mode 100644 internal/pkg/secureboot/measure/internal/pcr/testdata/b create mode 100644 internal/pkg/secureboot/measure/internal/pcr/testdata/c create mode 100644 internal/pkg/secureboot/measure/measure.go create mode 100644 internal/pkg/secureboot/measure/measure_test.go create mode 100644 internal/pkg/secureboot/measure/testdata/pcr-signing-key.pem create mode 100644 internal/pkg/secureboot/pesign/pesign.go create mode 100644 internal/pkg/secureboot/pesign/pesign_test.go create mode 100644 internal/pkg/secureboot/pesign/testdata/systemd-bootx64.efi create mode 100644 internal/pkg/secureboot/secureboot.go create mode 100644 internal/pkg/secureboot/tpm2/keys.go create mode 100644 internal/pkg/secureboot/tpm2/pcr.go create mode 100644 internal/pkg/secureboot/tpm2/pcr_test.go create mode 100644 internal/pkg/secureboot/tpm2/policy.go create mode 100644 internal/pkg/secureboot/tpm2/policy_test.go create mode 100644 internal/pkg/secureboot/tpm2/seal.go create mode 100644 internal/pkg/secureboot/tpm2/signature.go create mode 100644 internal/pkg/secureboot/tpm2/testdata/pcr-signing-crt.pem create mode 100644 internal/pkg/secureboot/tpm2/tpm2.go create mode 100644 internal/pkg/secureboot/tpm2/unseal.go create mode 100644 internal/pkg/secureboot/uki/assemble.go create mode 100644 internal/pkg/secureboot/uki/generate.go create mode 100644 internal/pkg/secureboot/uki/kernel.go create mode 100644 internal/pkg/secureboot/uki/kernel_test.go create mode 100644 internal/pkg/secureboot/uki/sbat.go create mode 100644 internal/pkg/secureboot/uki/sbat_test.go create mode 100644 internal/pkg/secureboot/uki/testdata/kernel create mode 100644 internal/pkg/secureboot/uki/uki.go create mode 100644 internal/pkg/smbios/smbios.go rename pkg/commands/{bootstrap.go => imported_bootstrap.go} (83%) create mode 100644 pkg/commands/imported_containers.go rename pkg/commands/{dashboard.go => imported_dashboard.go} (59%) create mode 100644 pkg/commands/imported_disks.go create mode 100644 pkg/commands/imported_dmesg.go create mode 100644 pkg/commands/imported_etcd.go create mode 100644 pkg/commands/imported_events.go rename pkg/commands/{get.go => imported_get.go} (95%) create mode 100644 pkg/commands/imported_health.go create mode 100644 pkg/commands/imported_image.go create mode 100644 pkg/commands/imported_kubeconfig.go create mode 100644 pkg/commands/imported_list.go create mode 100644 pkg/commands/imported_logs.go create mode 100644 pkg/commands/imported_memory.go create mode 100644 pkg/commands/imported_mounts.go create mode 100644 pkg/commands/imported_netstat.go create mode 100644 pkg/commands/imported_pcap.go create mode 100644 pkg/commands/imported_processes.go create mode 100644 pkg/commands/imported_read.go create mode 100644 pkg/commands/imported_reboot.go rename pkg/commands/{reset.go => imported_reset.go} (86%) create mode 100644 pkg/commands/imported_restart.go create mode 100644 pkg/commands/imported_rollback.go create mode 100644 pkg/commands/imported_root.go create mode 100644 pkg/commands/imported_service.go create mode 100644 pkg/commands/imported_shutdown.go create mode 100644 pkg/commands/imported_stats.go create mode 100644 pkg/commands/imported_time.go create mode 100644 pkg/commands/imported_version.go create mode 100644 tools/import_commands.go delete mode 100644 tools/import_functions.go diff --git a/Makefile b/Makefile index 8698c78..5a6c123 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,60 @@ VERSION=$(shell git describe --tags) TALOS_VERSION=$(shell go list -m github.com/siderolabs/talos | awk '{sub(/^v/, "", $$NF); print $$NF}') -generate: update-dashboard +generate: go generate build: go build -ldflags="-X 'main.Version=$(VERSION)'" -update-dashboard: - rm -rf internal/pkg/dashboard internal/pkg/meta internal/app/machined/pkg/runtime - wget -O- https://github.com/siderolabs/talos/archive/refs/tags/v$(TALOS_VERSION).tar.gz | tar --strip=1 -xzf- talos-$(TALOS_VERSION)/internal/pkg/dashboard talos-$(TALOS_VERSION)/internal/pkg/meta talos-$(TALOS_VERSION)/internal/app/machined/pkg/runtime - sed -i 's|github.com/siderolabs/talos/internal|github.com/aenix-io/talm/internal|g' `grep -rl 'github.com/siderolabs/talos/internal' internal/pkg/dashboard internal/pkg/meta internal/app/machined/pkg/runtime` +import: import-internal import-commands + +import-commands: + go run tools/import_commands.go --talos-version v$(TALOS_VERSION) \ + bootstrap \ + containers \ + dashboard \ + disks \ + dmesg \ + events \ + get \ + health \ + image \ + kubeconfig \ + list \ + logs \ + memory \ + mounts \ + netstat \ + pcap \ + processes \ + read \ + reboot \ + reset \ + restart \ + rollback \ + service \ + shutdown \ + stats \ + time \ + version + +import-internal: + rm -rf internal + mkdir internal + wget -O- https://github.com/siderolabs/talos/archive/refs/tags/v$(TALOS_VERSION).tar.gz | tar --strip=1 -xzf- \ + talos-$(TALOS_VERSION)/internal/pkg/meta \ + talos-$(TALOS_VERSION)/internal/app/machined/ \ + talos-$(TALOS_VERSION)/internal/pkg/cri \ + talos-$(TALOS_VERSION)/internal/pkg/cgroup \ + talos-$(TALOS_VERSION)/internal/pkg/dashboard \ + talos-$(TALOS_VERSION)/internal/pkg/environment \ + talos-$(TALOS_VERSION)/internal/pkg/etcd \ + talos-$(TALOS_VERSION)/internal/pkg/install \ + talos-$(TALOS_VERSION)/internal/pkg/logind \ + talos-$(TALOS_VERSION)/internal/pkg/mount \ + talos-$(TALOS_VERSION)/internal/pkg/partition \ + talos-$(TALOS_VERSION)/internal/pkg/secureboot \ + talos-$(TALOS_VERSION)/internal/pkg/smbios + sed -i 's|github.com/siderolabs/talos/internal|github.com/aenix-io/talm/internal|g' `grep -rl 'github.com/siderolabs/talos/internal' internal` + diff --git a/go.mod b/go.mod index a63f143..86eeca9 100644 --- a/go.mod +++ b/go.mod @@ -3,28 +3,129 @@ module github.com/aenix-io/talm go 1.22.2 require ( + cloud.google.com/go/compute/metadata v0.3.0 github.com/BurntSushi/toml v1.3.2 github.com/Masterminds/sprig/v3 v3.2.3 + github.com/aws/aws-sdk-go-v2/config v1.27.11 + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 + github.com/aws/smithy-go v1.20.2 + github.com/beevik/ntp v1.4.1 + github.com/benbjohnson/clock v1.3.5 + github.com/cenkalti/backoff/v4 v4.3.0 + github.com/containerd/cgroups/v3 v3.0.3 + github.com/containerd/containerd v1.7.16 + github.com/coredns/coredns v1.11.3 github.com/cosi-project/runtime v0.4.2 + github.com/dustin/go-humanize v1.0.1 + github.com/ecks/uefi v0.0.0-20221116212947-caef65d070eb + github.com/fatih/color v1.16.0 + github.com/foxboron/go-uefi v0.0.0-20240128152106-48be911532c2 + github.com/freddierice/go-losetup/v2 v2.0.1 + github.com/gdamore/tcell/v2 v2.7.4 + github.com/gizak/termui/v3 v3.1.0 github.com/gobwas/glob v0.2.3 + github.com/godbus/dbus/v5 v5.1.0 + github.com/golang/mock v1.6.0 + github.com/google/go-cmp v0.6.0 + github.com/google/go-tpm v0.9.1-0.20230914180155-ee6cbcd136f8 + github.com/google/nftables v0.2.0 + github.com/google/uuid v1.6.0 + github.com/gopacket/gopacket v1.2.0 + github.com/gosuri/uiprogress v0.0.1 + github.com/hashicorp/go-cleanhttp v0.5.2 + github.com/hashicorp/go-envparse v0.1.0 + github.com/hashicorp/go-getter/v2 v2.2.1 github.com/hashicorp/go-multierror v1.1.1 + github.com/hetznercloud/hcloud-go/v2 v2.8.0 + github.com/insomniacslk/dhcp v0.0.0-20240419123447-f1cffa2c0c49 + github.com/jsimonetti/rtnetlink v1.4.1 + github.com/jxskiss/base62 v1.1.0 + github.com/linode/go-metadata v0.2.0 + github.com/martinlindhe/base36 v1.1.1 + github.com/mattn/go-isatty v0.0.20 + github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875 + github.com/mdlayher/ethtool v0.1.0 + github.com/mdlayher/genetlink v1.3.2 + github.com/mdlayher/kobject v0.0.0-20200520190114-19ca17470d7d + github.com/mdlayher/netlink v1.7.2 + github.com/mdlayher/netx v0.0.0-20230430222610-7e21880baee8 + github.com/mdp/qrterminal/v3 v3.2.0 + github.com/miekg/dns v1.1.59 + github.com/nberlee/go-netstat v0.1.2 + github.com/opencontainers/go-digest v1.0.0 + github.com/opencontainers/image-spec v1.1.0 + github.com/opencontainers/runtime-spec v1.2.0 + github.com/packethost/packngo v0.31.0 github.com/pkg/errors v0.9.1 + github.com/pmorjan/kmod v1.1.1 + github.com/prometheus/procfs v0.14.0 + github.com/rivo/tview v0.0.0-20240505185119-ed116790de0f + github.com/rs/xid v1.5.0 + github.com/ryanuber/columnize v2.1.2+incompatible + github.com/ryanuber/go-glob v1.0.0 + github.com/safchain/ethtool v0.3.0 + github.com/scaleway/scaleway-sdk-go v1.0.0-beta.26 + github.com/siderolabs/crypto v0.4.4 + github.com/siderolabs/discovery-api v0.1.4 + github.com/siderolabs/discovery-client v0.1.9 github.com/siderolabs/gen v0.4.8 + github.com/siderolabs/go-api-signature v0.3.2 + github.com/siderolabs/go-blockdevice v0.4.7 + github.com/siderolabs/go-blockdevice/v2 v2.0.0-20240405165836-3265299b0192 + github.com/siderolabs/go-circular v0.1.0 + github.com/siderolabs/go-cmd v0.1.1 + github.com/siderolabs/go-copy v0.1.0 + github.com/siderolabs/go-debug v0.3.0 + github.com/siderolabs/go-kmsg v0.1.4 + github.com/siderolabs/go-kubeconfig v0.1.0 + github.com/siderolabs/go-kubernetes v0.2.9 + github.com/siderolabs/go-loadbalancer v0.3.3 + github.com/siderolabs/go-pointer v1.0.0 + github.com/siderolabs/go-procfs v0.1.2 + github.com/siderolabs/go-retry v0.3.3 + github.com/siderolabs/go-smbios v0.3.2 + github.com/siderolabs/go-tail v0.1.0 + github.com/siderolabs/go-talos-support v0.1.0 + github.com/siderolabs/net v0.4.0 + github.com/siderolabs/siderolink v0.3.6 github.com/siderolabs/talos v1.7.1 github.com/siderolabs/talos/pkg/machinery v1.7.1 github.com/spf13/cobra v1.8.0 github.com/stretchr/testify v1.9.0 + github.com/vishvananda/netlink v1.2.1-beta.2 + github.com/vmware/vmw-guestinfo v0.0.0-20220317130741-510905f0efa3 + github.com/vultr/metadata v1.1.0 + go.etcd.io/etcd/api/v3 v3.5.13 + go.etcd.io/etcd/client/pkg/v3 v3.5.13 + go.etcd.io/etcd/client/v3 v3.5.13 go.etcd.io/etcd/etcdutl/v3 v3.5.13 + go.uber.org/zap v1.27.0 + go4.org/netipx v0.0.0-20231129151722-fdeea329fbba + golang.org/x/net v0.25.0 + golang.org/x/oauth2 v0.20.0 + golang.org/x/sync v0.7.0 + golang.org/x/sys v0.20.0 + golang.org/x/term v0.20.0 + golang.org/x/text v0.15.0 + golang.org/x/time v0.5.0 + golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6 google.golang.org/grpc v1.63.2 - google.golang.org/protobuf v1.34.0 + google.golang.org/protobuf v1.34.1 gopkg.in/yaml.v3 v3.0.1 helm.sh/helm/v3 v3.14.4 + k8s.io/api v0.30.0 k8s.io/apimachinery v0.30.0 + k8s.io/apiserver v0.30.0 k8s.io/client-go v0.30.0 + k8s.io/component-base v0.30.0 + k8s.io/cri-api v0.30.0 + k8s.io/kube-scheduler v0.30.0 + k8s.io/kubelet v0.30.0 sigs.k8s.io/yaml v1.4.0 ) require ( + github.com/0x5a17ed/itkit v0.6.0 // indirect github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 // indirect @@ -42,26 +143,26 @@ require ( github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect github.com/ProtonMail/gopenpgp/v2 v2.7.5 // indirect github.com/adrg/xdg v0.4.0 // indirect + github.com/apparentlymart/go-cidr v1.1.0 // indirect + github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 // indirect github.com/aws/aws-sdk-go-v2 v1.26.1 // indirect - github.com/aws/aws-sdk-go-v2/config v1.27.10 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.17.10 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.11 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect github.com/aws/aws-sdk-go-v2/service/kms v1.30.1 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.20.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect - github.com/aws/smithy-go v1.20.2 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect github.com/blang/semver/v4 v4.0.0 // indirect + github.com/briandowns/spinner v1.19.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cilium/ebpf v0.12.3 // indirect github.com/cloudflare/circl v1.3.7 // indirect - github.com/containerd/cgroups/v3 v3.0.3 // indirect - github.com/containerd/containerd v1.7.16 // indirect github.com/containerd/continuity v0.4.2 // indirect github.com/containerd/errdefs v0.1.0 // indirect github.com/containerd/fifo v1.1.0 // indirect @@ -71,8 +172,10 @@ require ( github.com/containerd/ttrpc v1.2.3 // indirect github.com/containerd/typeurl/v2 v2.1.1 // indirect github.com/containernetworking/cni v1.1.2 // indirect + github.com/coredns/caddy v1.1.1 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/docker/cli v24.0.6+incompatible // indirect @@ -80,22 +183,23 @@ require ( github.com/docker/docker v26.0.0+incompatible // indirect github.com/docker/docker-credential-helpers v0.7.0 // indirect github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect - github.com/dustin/go-humanize v1.0.1 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/emicklei/dot v1.6.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v5.9.0+incompatible // indirect - github.com/fatih/color v1.16.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/foxboron/go-uefi v0.0.0-20240128152106-48be911532c2 // indirect + github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gdamore/encoding v1.0.0 // indirect - github.com/gdamore/tcell/v2 v2.7.4 // indirect github.com/gertd/go-pluralize v0.2.1 // indirect github.com/ghodss/yaml v1.0.0 // indirect - github.com/gizak/termui/v3 v3.1.0 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect + github.com/go-resty/resty/v2 v2.9.1 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang-jwt/jwt/v5 v5.2.0 // indirect @@ -103,36 +207,43 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.2 // indirect github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.6.0 // indirect github.com/google/go-containerregistry v0.19.1 // indirect - github.com/google/go-tpm v0.9.1-0.20230914180155-ee6cbcd136f8 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/uuid v1.6.0 // indirect + github.com/google/pprof v0.0.0-20240402174815-29b9bb013b0f // indirect + github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect + github.com/gosuri/uilive v0.0.4 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect + github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-safetemp v1.0.0 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect github.com/huandu/xstrings v1.4.0 // indirect github.com/imdario/mergo v0.3.13 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jeromer/syslogparser v1.1.0 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/josharian/native v1.1.0 // indirect - github.com/jsimonetti/rtnetlink v1.4.1 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.7 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect - github.com/mdlayher/ethtool v0.1.0 // indirect - github.com/mdlayher/genetlink v1.3.2 // indirect - github.com/mdlayher/netlink v1.7.2 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118 // indirect + github.com/mdlayher/packet v1.1.2 // indirect github.com/mdlayher/socket v0.5.1 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/locker v1.0.1 // indirect github.com/moby/spdystream v0.2.0 // indirect @@ -142,84 +253,84 @@ require ( github.com/moby/sys/user v0.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/nsf/termbox-go v0.0.0-20190121233118-02980233997d // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0 // indirect - github.com/opencontainers/runtime-spec v1.2.0 // indirect + github.com/onsi/ginkgo/v2 v2.17.1 // indirect github.com/opencontainers/selinux v1.11.0 // indirect + github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pelletier/go-toml v1.9.5 // indirect + github.com/pelletier/go-toml/v2 v2.0.6 // indirect + github.com/pierrec/lz4/v4 v4.1.15 // indirect + github.com/pin/tftp/v3 v3.1.0 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/planetscale/vtprotobuf v0.6.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.19.0 // indirect github.com/prometheus/client_model v0.6.0 // indirect - github.com/prometheus/common v0.50.0 // indirect - github.com/prometheus/procfs v0.13.0 // indirect - github.com/rivo/tview v0.0.0-20240403142647-a22293bda944 // indirect + github.com/prometheus/common v0.53.0 // indirect + github.com/quic-go/quic-go v0.42.0 // indirect + github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 // indirect github.com/rivo/uniseg v0.4.7 // indirect - github.com/rs/xid v1.5.0 // indirect - github.com/ryanuber/go-glob v1.0.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect + github.com/sethgrid/pester v1.2.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect - github.com/siderolabs/crypto v0.4.4 // indirect - github.com/siderolabs/go-api-signature v0.3.2 // indirect - github.com/siderolabs/go-blockdevice v0.4.7 // indirect - github.com/siderolabs/go-circular v0.1.0 // indirect - github.com/siderolabs/go-pointer v1.0.0 // indirect - github.com/siderolabs/go-procfs v0.1.2 // indirect - github.com/siderolabs/go-retry v0.3.3 // indirect - github.com/siderolabs/net v0.4.0 // indirect + github.com/siderolabs/go-pcidb v0.2.0 // indirect + github.com/siderolabs/grpc-proxy v0.4.0 // indirect + github.com/siderolabs/kms-client v0.1.0 // indirect github.com/siderolabs/protoenc v0.2.1 // indirect + github.com/siderolabs/tcpproxy v0.1.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/afero v1.10.0 // indirect github.com/spf13/cast v1.5.0 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/viper v1.14.0 // indirect + github.com/subosito/gotenv v1.4.1 // indirect + github.com/u-root/u-root v0.14.0 // indirect + github.com/u-root/uio v0.0.0-20240209044354-b3d14b93376a // indirect + github.com/ulikunitz/xz v0.5.12 // indirect github.com/vbatts/tar-split v0.11.3 // indirect + github.com/vishvananda/netns v0.0.4 // indirect + github.com/vmware-tanzu/sonobuoy v0.57.1 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect go.etcd.io/bbolt v1.3.9 // indirect - go.etcd.io/etcd/api/v3 v3.5.13 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.13 // indirect go.etcd.io/etcd/client/v2 v2.305.13 // indirect - go.etcd.io/etcd/client/v3 v3.5.13 // indirect go.etcd.io/etcd/pkg/v3 v3.5.13 // indirect go.etcd.io/etcd/raft/v3 v3.5.13 // indirect go.etcd.io/etcd/server/v3 v3.5.13 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect - go.opentelemetry.io/otel v1.22.0 // indirect - go.opentelemetry.io/otel/metric v1.22.0 // indirect - go.opentelemetry.io/otel/trace v1.22.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect + go.uber.org/mock v0.4.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/mod v0.16.0 // indirect - golang.org/x/net v0.23.0 // indirect - golang.org/x/oauth2 v0.18.0 // indirect - golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/term v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect - golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.19.0 // indirect - golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6 // indirect - google.golang.org/appengine v1.6.8 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/tools v0.20.0 // indirect + golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect + golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/api v0.30.0 // indirect k8s.io/apiextensions-apiserver v0.29.0 // indirect - k8s.io/apiserver v0.30.0 // indirect - k8s.io/component-base v0.30.0 // indirect - k8s.io/cri-api v0.30.0 // indirect + k8s.io/klog v1.0.0 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect + kernel.org/pub/linux/libs/security/libcap/cap v1.2.69 // indirect + kernel.org/pub/linux/libs/security/libcap/psx v1.2.69 // indirect + rsc.io/qr v0.2.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) diff --git a/go.sum b/go.sum index 4e8c775..843bc5f 100644 --- a/go.sum +++ b/go.sum @@ -17,17 +17,14 @@ cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHOb cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= -cloud.google.com/go v0.112.0 h1:tpFCD7hpHFlQ8yPwT3x+QeXqc2T6+n6T+hmABHfDUSM= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.24.0 h1:phWcR2eWzRJaL/kOiJwfFsPs4BaKq1j6vnpZrc1YlVg= -cloud.google.com/go/compute v1.24.0/go.mod h1:kw1/T+h/+tK2LJK0wiPPx1intgdAM3j/g3hFDlscY40= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -41,6 +38,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/0x5a17ed/itkit v0.6.0 h1:g1SnJQM61e0nAEk0Qu7cGGiL4zOHrk7ta55KoKwRcCs= +github.com/0x5a17ed/itkit v0.6.0/go.mod h1:v22t2Uc3bKewFBwLkY2U1KM7Us8iiEWw3qGqJFU76rI= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 h1:59MxjQVfjXsBpLy+dbd2/ELV5ofnUkUZBvWSC85sheA= @@ -84,14 +83,19 @@ github.com/ProtonMail/gopenpgp/v2 v2.7.5 h1:STOY3vgES59gNgoOt2w0nyHBjKViB/qSg7Nj github.com/ProtonMail/gopenpgp/v2 v2.7.5/go.mod h1:IhkNEDaxec6NyzSI0PlxapinnwPVIESk8/76da3Ct3g= github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls= github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E= +github.com/apparentlymart/go-cidr v1.1.0 h1:2mAhrMoF+nhXqxTzSZMUzDHkLjmIHC+Zzn4tdgBZjnU= +github.com/apparentlymart/go-cidr v1.1.0/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc= +github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 h1:7Ip0wMmLHLRJdrloDxZfhMm0xrLXZS8+COSu2bXmEQs= +github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-proxyproto v0.0.0-20210323213023-7e956b284f0a/go.mod h1:QmP9hvJ91BbJmGVGSbutW19IC0Q9phDCLGaomwTJbgU= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA= github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= -github.com/aws/aws-sdk-go-v2/config v1.27.10 h1:PS+65jThT0T/snC5WjyfHHyUgG+eBoupSDV+f838cro= -github.com/aws/aws-sdk-go-v2/config v1.27.10/go.mod h1:BePM7Vo4OBpHreKRUMuDXX+/+JWP38FLkzl5m27/Jjs= -github.com/aws/aws-sdk-go-v2/credentials v1.17.10 h1:qDZ3EA2lv1KangvQB6y258OssCHD0xvaGiEDkG4X/10= -github.com/aws/aws-sdk-go-v2/credentials v1.17.10/go.mod h1:6t3sucOaYDwDssHQa0ojH1RpmVmF5/jArkye1b2FKMI= +github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA= +github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE= +github.com/aws/aws-sdk-go-v2/credentials v1.17.11 h1:YuIB1dJNf1Re822rriUOTxopaHHvIq0l/pX3fwO+Tzs= +github.com/aws/aws-sdk-go-v2/credentials v1.17.11/go.mod h1:AQtFPsDH9bI2O+71anW6EKL+NcD7LG3dpKGMV4SShgo= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg= @@ -106,21 +110,31 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/g github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk= github.com/aws/aws-sdk-go-v2/service/kms v1.30.1 h1:SBn4I0fJXF9FYOVRSVMWuhvEKoAHDikjGpS3wlmw5DE= github.com/aws/aws-sdk-go-v2/service/kms v1.30.1/go.mod h1:2snWQJQUKsbN66vAawJuOGX7dr37pfOq9hb0tZDGIqQ= -github.com/aws/aws-sdk-go-v2/service/sso v1.20.4 h1:WzFol5Cd+yDxPAdnzTA5LmpHYSWinhmSj4rQChV0ee8= -github.com/aws/aws-sdk-go-v2/service/sso v1.20.4/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 h1:vN8hEbpRnL7+Hopy9dzmRle1xmDc7o8tmY0klsr175w= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.5/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2KCuY61kzoCpvtvJJBtOE= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak= github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n2HZPkcKgPAi1phU= github.com/aws/aws-sdk-go-v2/service/sts v1.28.6/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw= github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q= github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= +github.com/beevik/ntp v1.4.1 h1:wbs1dC82GEcSk7LjDog3kMEZuiOk8M7tePUsZaTCvXY= +github.com/beevik/ntp v1.4.1/go.mod h1:zkATLTt8VUZuOfYX2KgOnir4yvtAxWbnUUA24umXFnc= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas= +github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/briandowns/spinner v1.19.0 h1:s8aq38H+Qju89yhp89b4iIiMzMm8YN3p6vGpwyh/a8E= +github.com/briandowns/spinner v1.19.0/go.mod h1:mQak9GHqbspjC/5iUx3qMlIho8xBS/ppAL/hX5SmPJU= github.com/brianvoe/gofakeit/v6 v6.24.0 h1:74yq7RRz/noddscZHRS2T84oHZisW9muwbb8sRnU52A= github.com/brianvoe/gofakeit/v6 v6.24.0/go.mod h1:Ow6qC71xtwm79anlwKRlWZW6zVq9D2XHE4QSSMP/rU8= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -162,6 +176,10 @@ github.com/containerd/typeurl/v2 v2.1.1 h1:3Q4Pt7i8nYwy2KmQWIw2+1hTvwTE/6w9Fqctt github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3HZj1hsSQlywkQ0= github.com/containernetworking/cni v1.1.2 h1:wtRGZVv7olUHMOqouPpn3cXJWpJgM6+EUl31EQbXALQ= github.com/containernetworking/cni v1.1.2/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw= +github.com/coredns/caddy v1.1.1 h1:2eYKZT7i6yxIfGP3qLJoJ7HAsDJqYB+X68g4NYjSrE0= +github.com/coredns/caddy v1.1.1/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+BawD4= +github.com/coredns/coredns v1.11.3 h1:8RjnpZc42db5th84/QJKH2i137ecJdzZK1HJwhetSPk= +github.com/coredns/coredns v1.11.3/go.mod h1:lqFkDsHjEUdY7LJ75Nib3lwqJGip6ewWOqNIf8OavIQ= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= @@ -169,6 +187,7 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/cosi-project/runtime v0.4.2 h1:kJkhorzDWierDDbXn1BDHS6iQ7ai9AdvQOnK5uG/g8g= github.com/cosi-project/runtime v0.4.2/go.mod h1:eXVAHf9QzzSVblLUtHHPFOZ7JBuz+GypHbao1vw+SdQ= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= @@ -189,8 +208,14 @@ github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryef github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ecks/uefi v0.0.0-20221116212947-caef65d070eb h1:LZBZtPpqHDydudNAs2sHmo4Zp9bxEyxHdGCk3Fr6tv8= +github.com/ecks/uefi v0.0.0-20221116212947-caef65d070eb/go.mod h1:jP/WitZVr91050NiqxEEp0ynBFbP2eUQC0CnxWPlQTA= +github.com/emicklei/dot v1.6.1 h1:ujpDlBkkwgWUY+qPId5IwapRW/xEoligRSYjioR6DFI= +github.com/emicklei/dot v1.6.1/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -203,16 +228,23 @@ github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/foxboron/go-uefi v0.0.0-20240128152106-48be911532c2 h1:qGlg/7H49H30Eu7nkCBA7YxNmW30ephqBf7xIxlAGuQ= github.com/foxboron/go-uefi v0.0.0-20240128152106-48be911532c2/go.mod h1:ffg/fkDeOYicEQLoO2yFFGt00KUTYVXI+rfnc8il6vQ= -github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= -github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= +github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= +github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/freddierice/go-losetup/v2 v2.0.1 h1:wPDx/Elu9nDV8y/CvIbEDz5Xi5Zo80y4h7MKbi3XaAI= +github.com/freddierice/go-losetup/v2 v2.0.1/go.mod h1:TEyBrvlOelsPEhfWD5rutNXDmUszBXuFnwT1kIQF4J8= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko= github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= github.com/gdamore/tcell/v2 v2.7.4 h1:sg6/UnTM9jGpZU+oFYAsDahfchWAFW8Xx2yFinNSAYU= @@ -226,6 +258,7 @@ github.com/gizak/termui/v3 v3.1.0/go.mod h1:bXQEBkJpzxUAKf0+xq9MSWAvWZlE7c+aidmy github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -237,12 +270,16 @@ github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2Kv github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-resty/resty/v2 v2.9.1 h1:PIgGx4VrHvag0juCJ4dDv3MiFRlDmP0vicBucwf+gLM= +github.com/go-resty/resty/v2 v2.9.1/go.mod h1:4/GYJVjh9nhkhGR6AUNW3XhpDYNUr+Uvy9gV/VGZIy4= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= @@ -262,6 +299,8 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -297,6 +336,8 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -310,6 +351,8 @@ github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/nftables v0.2.0 h1:PbJwaBmbVLzpeldoeUKGkE2RjstrjPKMl6oLrfEJ6/8= +github.com/google/nftables v0.2.0/go.mod h1:Beg6V6zZ3oEn0JuiUQ4wqwuyqqzasOltcoXPtgLbFp4= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -331,22 +374,52 @@ github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/gopacket/gopacket v1.2.0 h1:eXbzFad7f73P1n2EJHQlsKuvIMJjVXK5tXoSca78I3A= +github.com/gopacket/gopacket v1.2.0/go.mod h1:BrAKEy5EOGQ76LSqh7DMAr7z0NNPdczWm2GxCG7+I8M= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY= +github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI= +github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw= +github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI= github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is= github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM= +github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 h1:MJG/KsmcqMwFAkh8mTnAwhyKoB+sTAnY4CACC110tbU= +github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-envparse v0.1.0 h1:bE++6bhIsNCPLvgDZkYqo3nA+/PFI51pkrHdmPSDFPY= +github.com/hashicorp/go-envparse v0.1.0/go.mod h1:OHheN1GoygLlAkTlXLXvAdnXdZxy8JUweQ1rAXx1xnc= +github.com/hashicorp/go-getter/v2 v2.2.1 h1:2JXqPZs1Jej67RtdTi0YZaEB2hEFB3fkBA4cPYKQwFQ= +github.com/hashicorp/go-getter/v2 v2.2.1/go.mod h1:EcJx6oZE8hmGuRR1l38QrfnyiujQbwsEAn11eHv6l2M= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= +github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hetznercloud/hcloud-go/v2 v2.8.0 h1:vfbfL/JfV8dIZUX7ANHWEbKNqgFWsETqvt/EctvoFJ0= +github.com/hetznercloud/hcloud-go/v2 v2.8.0/go.mod h1:jvpP3qAWMIZ3WQwQLYa97ia6t98iPCgsJNwRts+Jnrk= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU= github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714 h1:/jC7qQFrv8CrSJVmaolDVOxTfS9kc36uB6H40kdbQq8= +github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714/go.mod h1:2Goc3h8EklBH5mspfHFxBnEoURQCGzQQH1ga9Myjvis= +github.com/hugelgupf/vmtest v0.0.0-20240216064925-0561770280a1 h1:jWoR2Yqg8tzM0v6LAiP7i1bikZJu3gxpgvu3g1Lw+a0= +github.com/hugelgupf/vmtest v0.0.0-20240216064925-0561770280a1/go.mod h1:B63hDJMhTupLWCHwopAyEo7wRFowx9kOc8m8j1sfOqE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= @@ -354,18 +427,27 @@ github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/insomniacslk/dhcp v0.0.0-20240419123447-f1cffa2c0c49 h1:/OuvSMGT9+xnyZ+7MZQ1zdngaCCAdPoSw8B/uurZ7pg= +github.com/insomniacslk/dhcp v0.0.0-20240419123447-f1cffa2c0c49/go.mod h1:KclMyHxX06VrVr0DJmeFSUb1ankt7xTfoOA35pCkoic= +github.com/jeromer/syslogparser v1.1.0 h1:HES0EviO9iPvCu56LjVFVhbM3o0BckDlIbQfkkaRJAw= +github.com/jeromer/syslogparser v1.1.0/go.mod h1:zfowyus/j2SEgW31bIntTvEBE2zCSndtFsCC6NcW4S4= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= +github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw= +github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ= github.com/jsimonetti/rtnetlink v1.4.1 h1:JfD4jthWBqZMEffc5RjgmlzpYttAVw1sdnmiNaPO3hE= github.com/jsimonetti/rtnetlink v1.4.1/go.mod h1:xJjT7t59UIZ62GLZbv6PLLo8VFrostJMPBAheR6OM8w= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jxskiss/base62 v1.1.0 h1:A5zbF8v8WXx2xixnAKD2w+abC+sIzYJX+nxmhA6HWFw= +github.com/jxskiss/base62 v1.1.0/go.mod h1:HhWAlUXvxKThfOlZbcuFzsqwtF5TcqS9ru3y5GfjWAc= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= @@ -381,34 +463,69 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/linode/go-metadata v0.2.0 h1:hlWzkYLa80ikA0NmFX2hcwhcnWFol8F3UIvJnOgdKw4= +github.com/linode/go-metadata v0.2.0/go.mod h1:XraDbSwms0+CtA7/Qh7agkSvGDc6H0s782kpX9MdMu0= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/martinlindhe/base36 v1.1.1 h1:1F1MZ5MGghBXDZ2KJ3QfxmiydlWOGB8HCEtkap5NkVg= +github.com/martinlindhe/base36 v1.1.1/go.mod h1:vMS8PaZ5e/jV9LwFKlm0YLnXl/hpOihiBxKkIoc3g08= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875 h1:ql8x//rJsHMjS+qqEag8n3i4azw1QneKh5PieH9UEbY= +github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875/go.mod h1:kfOoFJuHWp76v1RgZCb9/gVUc7XdY877S2uVYbNliGc= +github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118 h1:2oDp6OOhLxQ9JBoUuysVz9UZ9uI6oLUbvAZu0x8o+vE= +github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118/go.mod h1:ZFUnHIVchZ9lJoWoEGUg8Q3M4U8aNNWA3CVSUTkW4og= github.com/mdlayher/ethtool v0.1.0 h1:XAWHsmKhyPOo42qq/yTPb0eFBGUKKTR1rE0dVrWVQ0Y= github.com/mdlayher/ethtool v0.1.0/go.mod h1:fBMLn2UhfRGtcH5ZFjr+6GUiHEjZsItFD7fSn7jbZVQ= github.com/mdlayher/genetlink v1.3.2 h1:KdrNKe+CTu+IbZnm/GVUMXSqBBLqcGpRDa0xkQy56gw= github.com/mdlayher/genetlink v1.3.2/go.mod h1:tcC3pkCrPUGIKKsCsp0B3AdaaKuHtaxoJRz3cc+528o= +github.com/mdlayher/kobject v0.0.0-20200520190114-19ca17470d7d h1:JmrZTpS0GAyMV4ZQVVH/AS0Y6r2PbnYNSRUuRX+HOLA= +github.com/mdlayher/kobject v0.0.0-20200520190114-19ca17470d7d/go.mod h1:+SexPO1ZvdbbWUdUnyXEWv3+4NwHZjKhxOmQqHY4Pqc= +github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA= +github.com/mdlayher/netlink v1.0.0/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M= +github.com/mdlayher/netlink v1.1.0/go.mod h1:H4WCitaheIsdF9yOYu8CFmCgQthAPIWZmcKp9uZHgmY= github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g= github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw= +github.com/mdlayher/netx v0.0.0-20230430222610-7e21880baee8 h1:HMgSn3c16SXca3M+n6fLK2hXJLd4mhKAsZZh7lQfYmQ= +github.com/mdlayher/netx v0.0.0-20230430222610-7e21880baee8/go.mod h1:qhZhwMDNWwZglKfwuWm0U9pCr/YKX1QAEwwJk9qfiTQ= +github.com/mdlayher/packet v1.0.0/go.mod h1:eE7/ctqDhoiRhQ44ko5JZU2zxB88g+JH/6jmnjzPjOU= +github.com/mdlayher/packet v1.1.2 h1:3Up1NG6LZrsgDVn6X4L9Ge/iyRyxFEFD9o6Pr3Q1nQY= +github.com/mdlayher/packet v1.1.2/go.mod h1:GEu1+n9sG5VtiRE4SydOmX5GTwyyYlteZiFU+x0kew4= +github.com/mdlayher/socket v0.2.1/go.mod h1:QLlNPkFR88mRUNQIzRBMfXxwKal8H7u1h3bL1CV+f0E= github.com/mdlayher/socket v0.5.1 h1:VZaqt6RkGkt2OE9l3GcC6nZkqD3xKeQLyfleW/uBcos= github.com/mdlayher/socket v0.5.1/go.mod h1:TjPLHI1UgwEv5J1B5q0zTZq12A/6H7nKmtTanQE37IQ= +github.com/mdp/qrterminal/v3 v3.2.0 h1:qteQMXO3oyTK4IHwj2mWsKYYRBOp1Pj2WRYFYYNTCdk= +github.com/mdp/qrterminal/v3 v3.2.0/go.mod h1:XGGuua4Lefrl7TLEsSONiD+UEjQXJZ4mPzF+gWYIJkk= +github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs= +github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk= +github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721 h1:RlZweED6sbSArvlE924+mUcZuXKLBHA35U7LN621Bws= +github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721/go.mod h1:Ickgr2WtCLZ2MDGd4Gr0geeCH5HybhRJbonOgQpvSxc= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= +github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= @@ -429,17 +546,19 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/nberlee/go-netstat v0.1.2 h1:wgPV1YOUo+kDFypqiKgfxMtnSs1Wb42c7ahI4qyEUJc= +github.com/nberlee/go-netstat v0.1.2/go.mod h1:GvDCRLsUKMRN1wULkt7tpnDmjSIE6YGf5zeVq+mBO64= github.com/nsf/termbox-go v0.0.0-20190121233118-02980233997d h1:x3S6kxmy49zXVVyhcnrFqxvNVCBPb2KZ9hV2RBdS840= github.com/nsf/termbox-go v0.0.0-20190121233118-02980233997d/go.mod h1:IuKpRQcYE1Tfu+oAQqaLisqDeXgjyyltCfsaoYN18NQ= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.17.1 h1:V++EzdbhI4ZV4ev0UTIj0PzhzOcReJFyJaLjtSF55M8= @@ -457,8 +576,18 @@ github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU= github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/packethost/packngo v0.31.0 h1:LLH90ardhULWbagBIc3I3nl2uU75io0a7AwY6hyi0S4= +github.com/packethost/packngo v0.31.0/go.mod h1:Io6VJqzkiqmIEQbpOjeIw9v8q9PfcTEq8TEY/tMQsfw= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= +github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= +github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pin/tftp/v3 v3.1.0 h1:rQaxd4pGwcAJnpId8zC+O2NX3B2/NscjDZQaqEjuE7c= +github.com/pin/tftp/v3 v3.1.0/go.mod h1:xwQaN4viYL019tM4i8iecm++5cGxSqen6AJEOEyEI0w= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -468,17 +597,23 @@ github.com/planetscale/vtprotobuf v0.6.0 h1:nBeETjudeJ5ZgBHUz1fVHvbqUKnYOXNhsIEa github.com/planetscale/vtprotobuf v0.6.0/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmorjan/kmod v1.1.1 h1:Vfw6bMaOg/sYSBCqJPT9TbqHHf5zK00GbaL5JQLO4r0= +github.com/pmorjan/kmod v1.1.1/go.mod h1:jR4fVosEpQ6b5U0rpxaqoShTDPvCjLIP8vEESZyvnqQ= github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= -github.com/prometheus/common v0.50.0 h1:YSZE6aa9+luNa2da6/Tik0q0A5AbR+U003TItK57CPQ= -github.com/prometheus/common v0.50.0/go.mod h1:wHFBCEVWVmHMUpg7pYcOm2QUR/ocQdYSJVQJKnHc3xQ= -github.com/prometheus/procfs v0.13.0 h1:GqzLlQyfsPbaEHaQkO7tbDlriv/4o5Hudv6OXHGKX7o= -github.com/prometheus/procfs v0.13.0/go.mod h1:cd4PFCR54QLnGKPaKGA6l+cfuNXtht43ZKY6tow0Y1g= -github.com/rivo/tview v0.0.0-20240403142647-a22293bda944 h1:mtRO4FDhPX7UzxxhaO4Rsw3UKNj1NT6lRhGpw4hPPW8= -github.com/rivo/tview v0.0.0-20240403142647-a22293bda944/go.mod h1:02iFIz7K/A9jGCvrizLPvoqr4cEIx7q54RH5Qudkrss= +github.com/prometheus/common v0.53.0 h1:U2pL9w9nmJwJDa4qqLQ3ZaePJ6ZTwt7cMD3AG3+aLCE= +github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U= +github.com/prometheus/procfs v0.14.0 h1:Lw4VdGGoKEZilJsayHf0B+9YgLGREba2C6xr+Fdfq6s= +github.com/prometheus/procfs v0.14.0/go.mod h1:XL+Iwz8k8ZabyZfMFHPiilCniixqQarAy5Mu67pHlNQ= +github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= +github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M= +github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnHyvtYjDeq0zlVHn9K/ZXoy17ylucdo= +github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM= +github.com/rivo/tview v0.0.0-20240505185119-ed116790de0f h1:DAbaKhyPcZQp/TqlSdUd6Z445PkJb3bI0VccXg22oeg= +github.com/rivo/tview v0.0.0-20240505185119-ed116790de0f/go.mod h1:02iFIz7K/A9jGCvrizLPvoqr4cEIx7q54RH5Qudkrss= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= @@ -488,38 +623,85 @@ github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v2.1.2+incompatible h1:C89EOx/XBWwIXl8wm8OPJBd7kPF25UfsK2X7Ph/zCAk= +github.com/ryanuber/columnize v2.1.2+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= +github.com/safchain/ethtool v0.3.0 h1:gimQJpsI6sc1yIqP/y8GYgiXn/NjgvpM0RNoWLVVmP0= +github.com/safchain/ethtool v0.3.0/go.mod h1:SA9BwrgyAqNo7M+uaL6IYbxpm5wk3L7Mm6ocLW+CJUs= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= +github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= +github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.26 h1:F+GIVtGqCFxPxO46ujf8cEOP574MBoRm3gNbPXECbxs= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.26/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= +github.com/sethgrid/pester v1.2.0 h1:adC9RS29rRUef3rIKWPOuP1Jm3/MmB6ke+OhE5giENI= +github.com/sethgrid/pester v1.2.0/go.mod h1:hEUINb4RqvDxtoCaU0BNT/HV4ig5kfgOasrf1xcvr0A= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/siderolabs/crypto v0.4.4 h1:Q6EDBMR2Ub2oAZW5Xl8lrKB27bM3Sn8Gkfw3rngco5U= github.com/siderolabs/crypto v0.4.4/go.mod h1:hsR3tJ3aaeuhCChsLF4dBd9vlJVPvmhg4vvx2ez4aD4= +github.com/siderolabs/discovery-api v0.1.4 h1:2fMEFSMiWaD1zDiBDY5md8VxItvL1rDQRSOfeXNjYKc= +github.com/siderolabs/discovery-api v0.1.4/go.mod h1:kaBy+G42v2xd/uAF/NIe383sjNTBE2AhxPTyi9SZI0s= +github.com/siderolabs/discovery-client v0.1.9 h1:yDzvts++Nf/2qczdDUfU5GAibkEIgz/eo9RPG/k/rOc= +github.com/siderolabs/discovery-client v0.1.9/go.mod h1:Ew1z07eyJwqNwum84IKYH4S649KEKK5WUmRW49HlXS8= github.com/siderolabs/gen v0.4.8 h1:VNpbmDLhkXp7qcSEkKk1Ee7vU2afs3xvHrWLGR2UuiY= github.com/siderolabs/gen v0.4.8/go.mod h1:7ROKVHHB68R3Amrd4a1ZXz/oMrXWF3Mg3lSEgnkJY5c= github.com/siderolabs/go-api-signature v0.3.2 h1:blqrZF1GM7TWgq7mY7CsR+yQ93u6az0Kf0mfsw+hvf0= github.com/siderolabs/go-api-signature v0.3.2/go.mod h1:punhUOaXa7LELYBRCUhfgUGH6ieVz68GrP98apCKXj8= github.com/siderolabs/go-blockdevice v0.4.7 h1:2bk4WpEEflGxjrNwp57ye24Pr+cYgAiAeNMWiQOuWbQ= github.com/siderolabs/go-blockdevice v0.4.7/go.mod h1:4PeOuk71pReJj1JQEXDE7kIIQJPVe8a+HZQa+qjxSEA= +github.com/siderolabs/go-blockdevice/v2 v2.0.0-20240405165836-3265299b0192 h1:16/cHDGhTUDBtfIftOkuHWhJcQdpa/FwwWPcTq4aOxc= +github.com/siderolabs/go-blockdevice/v2 v2.0.0-20240405165836-3265299b0192/go.mod h1:UBbbc+L7hU0UggOQeKCA+Qp3ImGkSeaLfVOiCbxRxEI= github.com/siderolabs/go-circular v0.1.0 h1:zpBJNUbCZSh0odZxA4Dcj0d3ShLLR2WxKW6hTdAtoiE= github.com/siderolabs/go-circular v0.1.0/go.mod h1:14XnLf/I3J0VjzTgmwWNGjp58/bdIi4zXppAEx8plfw= +github.com/siderolabs/go-cmd v0.1.1 h1:nTouZUSxLeiiEe7hFexSVvaTsY/3O8k1s08BxPRrsps= +github.com/siderolabs/go-cmd v0.1.1/go.mod h1:6hY0JG34LxEEwYE8aH2iIHkHX/ir12VRLqfwAf2yJIY= +github.com/siderolabs/go-copy v0.1.0 h1:OIWCtSg+rhOtnIZTpT31Gfpn17rv5kwJqQHG+QUEgC8= +github.com/siderolabs/go-copy v0.1.0/go.mod h1:4bF2rZOZAR/ags/U4AVSpjFE5RPGdEeSkOq6yR9YOkU= +github.com/siderolabs/go-debug v0.3.0 h1:C8t7jbac5Va2eYu9QRXXEGsy3Vz5xOEVo0TDwVJH268= +github.com/siderolabs/go-debug v0.3.0/go.mod h1:DonqzIQOm3+qof020meFwJ2gXI5Jv/x4Dj27FyUW4aE= +github.com/siderolabs/go-kmsg v0.1.4 h1:RLAa90O9bWuhA3pXPAYAdrI+kzcqTshZASRA5yso/mo= +github.com/siderolabs/go-kmsg v0.1.4/go.mod h1:BLkt2N2DHT0wsFMz32lMw6vNEZL90c8ZnBjpIUoBb/M= +github.com/siderolabs/go-kubeconfig v0.1.0 h1:t/2oMWkLSdWHXglKPMz8ySXnx6ZjHckeGY79NaDcBTo= +github.com/siderolabs/go-kubeconfig v0.1.0/go.mod h1:eM3mO02Td6wYDvdi9zTbMrj1Q4WqEFN8XQ6pNjCUWkI= +github.com/siderolabs/go-kubernetes v0.2.9 h1:EtaOcni9P0etJz+UDlIKQkgsTjCg2MWI2p1fKeRTo8Q= +github.com/siderolabs/go-kubernetes v0.2.9/go.mod h1:AAydnLZrqG+MJrKTa82AszkWIytkqwDBt7PL+bfbupI= +github.com/siderolabs/go-loadbalancer v0.3.3 h1:D6ONnP9Erlh4TS6kV9L7ocnfrNYCA/58i6ZF0QweLJk= +github.com/siderolabs/go-loadbalancer v0.3.3/go.mod h1:7j4Q9peU/UFuTNSFfwhKLQ028CNkyMkAdGnSi1Dm7Jw= +github.com/siderolabs/go-pcidb v0.2.0 h1:ZCkF1cz6UjoEIHpP7+aeTI5BwmSxE627Jl1Wy2VZAwU= +github.com/siderolabs/go-pcidb v0.2.0/go.mod h1:XstZrp8xnganxzIc3UQKfCs1fQFgYWH2lqtWeqBwRok= github.com/siderolabs/go-pointer v1.0.0 h1:6TshPKep2doDQJAAtHUuHWXbca8ZfyRySjSBT/4GsMU= github.com/siderolabs/go-pointer v1.0.0/go.mod h1:HTRFUNYa3R+k0FFKNv11zgkaCLzEkWVzoYZ433P3kHc= github.com/siderolabs/go-procfs v0.1.2 h1:bDs9hHyYGE2HO1frpmUsD60yg80VIEDrx31fkbi4C8M= github.com/siderolabs/go-procfs v0.1.2/go.mod h1:dBzQXobsM7+TWRRI3DS9X7vAuj8Nkfgu3Z/U9iY3ZTY= github.com/siderolabs/go-retry v0.3.3 h1:zKV+S1vumtO72E6sYsLlmIdV/G/GcYSBLiEx/c9oCEg= github.com/siderolabs/go-retry v0.3.3/go.mod h1:Ff/VGc7v7un4uQg3DybgrmOWHEmJ8BzZds/XNn/BqMI= +github.com/siderolabs/go-smbios v0.3.2 h1:/9MCz1h3HYFcNdFG9rIL9EKwtQJsHRPuGuM2ESdao3A= +github.com/siderolabs/go-smbios v0.3.2/go.mod h1:AKzwL3QdFOgA81h65Hay2bs3BUnH+FBnXqNfgeChpEc= +github.com/siderolabs/go-tail v0.1.0 h1:U+ZClt7BXLGsxDNU/XQ12sz7lQElfFZBYEPdkW78Qro= +github.com/siderolabs/go-tail v0.1.0/go.mod h1:vWxumnRUS3eTZczORCJW3QMjxiTETN31vyuFdaW8rPw= +github.com/siderolabs/go-talos-support v0.1.0 h1:ulf+RI0Wo6UGzKQJZog1uvdQE/zstogs1R46jZpAmvU= +github.com/siderolabs/go-talos-support v0.1.0/go.mod h1:hiYQrdQSBH6ap7LZHyHUZLbYnL2KhC6hPrJ7utqm+P8= +github.com/siderolabs/grpc-proxy v0.4.0 h1:zYrhqLYs8JlYoLHYeel7/XwXDZ4OJ5XyP9wX7JlbPew= +github.com/siderolabs/grpc-proxy v0.4.0/go.mod h1:QDurYOwQD4H8BKyvCuUxMiuG/etYnb/++xaQB644NdU= +github.com/siderolabs/kms-client v0.1.0 h1:rCDWzcDDsNlp6zdyLngOuuhchVILn+vwUQy3tk6rQps= +github.com/siderolabs/kms-client v0.1.0/go.mod h1:4UQkRhuEh3kaK7VhJxez4YyJLv6lPEff7g3Pa6Y9okg= github.com/siderolabs/net v0.4.0 h1:1bOgVay/ijPkJz4qct98nHsiB/ysLQU0KLoBC4qLm7I= github.com/siderolabs/net v0.4.0/go.mod h1:/ibG+Hm9HU27agp5r9Q3eZicEfjquzNzQNux5uEk0kM= github.com/siderolabs/protoenc v0.2.1 h1:BqxEmeWQeMpNP3R6WrPqDatX8sM/r4t97OP8mFmg6GA= github.com/siderolabs/protoenc v0.2.1/go.mod h1:StTHxjet1g11GpNAWiATgc8K0HMKiFSEVVFOa/H0otc= +github.com/siderolabs/siderolink v0.3.6 h1:8oBE/JuYEKlPJsmrBLUw2vTttp9PCNkZP3huLe94pn8= +github.com/siderolabs/siderolink v0.3.6/go.mod h1:Vd5sMtQA0DVApcMJzR0chYUE09YVw+09f9BBkg6Wf0s= github.com/siderolabs/talos v1.7.1 h1:a4Z0k+qGavk+dhz4DY6lOZbf0CaG5F7UOzk3tcx+aa8= github.com/siderolabs/talos v1.7.1/go.mod h1:9wo6GSfKke3m/PU36rbOrEZ5kfw9RxYnhO/MB45w4hQ= github.com/siderolabs/talos/pkg/machinery v1.7.1 h1:sVFQ0lNE6+kOomSZA8iuktzG1A4zSW9KTsB2TLaTPsU= github.com/siderolabs/talos/pkg/machinery v1.7.1/go.mod h1:YBl9KDCD45Uc7N0rXBY1JqovUn1n46ekUPSNbEVZzQU= +github.com/siderolabs/tcpproxy v0.1.0 h1:IbkS9vRhjMOscc1US3M5P1RnsGKFgB6U5IzUk+4WkKA= +github.com/siderolabs/tcpproxy v0.1.0/go.mod h1:onn6CPPj/w1UNqQ0U97oRPF0CqbrgEApYCw4P9IiCW8= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -530,8 +712,12 @@ github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.14.0 h1:Rg7d3Lo706X9tHsJMUjdiwMpHB7W8WnSVOssIY+JElU= +github.com/spf13/viper v1.14.0/go.mod h1:WT//axPky3FdvXHzGw33dNdXXXfFQqmEalje+egj8As= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -541,15 +727,36 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= +github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/u-root/u-root v0.14.0 h1:Ka4T10EEML7dQ5XDvO9c3MBN8z4nuSnGjcd1jmU2ivg= +github.com/u-root/u-root v0.14.0/go.mod h1:hAyZorapJe4qzbLWlAkmSVCJGbfoU9Pu4jpJ1WMluqE= +github.com/u-root/uio v0.0.0-20240209044354-b3d14b93376a h1:BH1SOPEvehD2kVrndDnGJiUF0TrBpNs+iyYocu6h0og= +github.com/u-root/uio v0.0.0-20240209044354-b3d14b93376a/go.mod h1:P3a5rG4X7tI17Nn3aOIAYr5HbIMukwXG0urG0WuL8OA= +github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= +github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck= github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY= +github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= +github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= +github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= +github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= +github.com/vmware-tanzu/sonobuoy v0.57.1 h1:3P2/P5WOJhyspsCkx58eDr0hsBmy8AJS1KYzjo5Epic= +github.com/vmware-tanzu/sonobuoy v0.57.1/go.mod h1:TevlYITSKi7JvgRPhShECtQiJpDErY+8FHehlkgen9c= +github.com/vmware/vmw-guestinfo v0.0.0-20220317130741-510905f0efa3 h1:v6jG/tdl4O07LNVp74Nt7/OyL+1JsIW1M2f/nSvQheY= +github.com/vmware/vmw-guestinfo v0.0.0-20220317130741-510905f0efa3/go.mod h1:CSBTxrhePCm0cmXNKDGeu+6bOQzpaEklfCqEpn89JWk= +github.com/vultr/metadata v1.1.0 h1:RUjCnH5Mdlz7uuyfb1jOZNkU72zl/HwK76jLzVFdiOo= +github.com/vultr/metadata v1.1.0/go.mod h1:4yocaI6h2EFJzwN0m1KnnC/vDCx2axIqnyxmtF/LWoQ= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -563,6 +770,7 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI= go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE= @@ -594,24 +802,29 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 h1:sv9kVfal0MK0wBMCOGr+HeJm9v803BkJxGrk2au7j08= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0/go.mod h1:SK2UL73Zy1quvRPonmOmRDiWk1KBV3LyIeeIxcEApWw= -go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y= -go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI= -go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg= -go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY= -go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0= -go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M= +go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -619,8 +832,9 @@ golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -631,8 +845,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= +golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -656,10 +870,11 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= -golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -673,6 +888,8 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -694,14 +911,17 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -711,8 +931,8 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= -golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= +golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= +golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -723,6 +943,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= @@ -730,16 +951,20 @@ golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -747,6 +972,7 @@ golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -755,6 +981,7 @@ golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -764,10 +991,14 @@ golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -778,17 +1009,21 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -797,15 +1032,18 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -858,14 +1096,19 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= -golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY= +golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg= +golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI= +golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 h1:/jFs0duh4rdb8uIfPMv78iAJGcPKDeqAFnaLBropIC4= +golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173/go.mod h1:tkCQ4FQXmpAgYVh++1cq16/dH4QJtmvpRv19DWGAHSA= golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6 h1:CawjfCvYQH2OU3/TnxLx97WDSUDRABfT18pCOYwc2GE= golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6/go.mod h1:3rxYc4HtVcSG9gVaTs2GEBdehh+sYPOwKtyUWEOTb80= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= @@ -894,8 +1137,6 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -936,8 +1177,8 @@ google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUE google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda h1:b6F6WIV4xHHD0FA4oIyzU6mHWg2WI2X1RBehwa5QN38= google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda/go.mod h1:AHcE/gZH76Bk/ROZhQphlRoWo5xKDEtz3eVEO1LfA8c= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda h1:LI5DOvAxUPMv/50agcLLoo+AdWc1irS9Rzz4vPuD1V4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be h1:LG9vZxsWGOmUKieR8wPAUR3u3MpnYFQZROPIMaXh7/A= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -969,8 +1210,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.34.0 h1:Qo/qEd2RZPCf2nKuorzksSknv0d3ERwp1vFG38gSmH4= -google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= @@ -979,6 +1220,8 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -990,8 +1233,10 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= -gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= +gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= +gvisor.dev/gvisor v0.0.0-20230927004350-cbd86285d259 h1:TbRPT0HtzFP3Cno1zZo7yPzEEnfu8EjLfl6IU9VfqkQ= +gvisor.dev/gvisor v0.0.0-20230927004350-cbd86285d259/go.mod h1:AVgIgHMwK63XvmAzWG9vLQ41YnVHN0du0tEC46fI7yY= helm.sh/helm/v3 v3.14.4 h1:6FSpEfqyDalHq3kUr4gOMThhgY55kXUEjdQoyODYnrM= helm.sh/helm/v3 v3.14.4/go.mod h1:Tje7LL4gprZpuBNTbG34d1Xn5NmRT3OWfBRwpOSer9I= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1015,13 +1260,25 @@ k8s.io/component-base v0.30.0 h1:cj6bp38g0ainlfYtaOQuRELh5KSYjhKxM+io7AUIk4o= k8s.io/component-base v0.30.0/go.mod h1:V9x/0ePFNaKeKYA3bOvIbrNoluTSG+fSJKjLdjOoeXQ= k8s.io/cri-api v0.30.0 h1:hZqh3vH5JZdqeAyhD9nPXSbT6GDgrtPJkPiIzhWKVhk= k8s.io/cri-api v0.30.0/go.mod h1://4/umPJSW1ISNSNng4OwjpkvswJOQwU8rnkvO8P+xg= +k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= +k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/kube-scheduler v0.30.0 h1:wr2bcKy9MoN0VlfiM66KHYsgUXPJYhtr3b6LiVmKc94= +k8s.io/kube-scheduler v0.30.0/go.mod h1:C/yQb0WrPsxAA3LGwh+HB4sY5RMbH+2UMfdDpEQNR30= +k8s.io/kubelet v0.30.0 h1:/pqHVR2Rn8ExCpn211wL3pMtqRFpcBcJPl4+1INbIMk= +k8s.io/kubelet v0.30.0/go.mod h1:WukdKqbQxnj+csn3K8XOKeX7Sh60J/da25IILjvvB5s= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +kernel.org/pub/linux/libs/security/libcap/cap v1.2.69 h1:N0m3tKYbkRMmDobh/47ngz+AWeV7PcfXMDi8xu3Vrag= +kernel.org/pub/linux/libs/security/libcap/cap v1.2.69/go.mod h1:Tk5Ip2TuxaWGpccL7//rAsLRH6RQ/jfqTGxuN/+i/FQ= +kernel.org/pub/linux/libs/security/libcap/psx v1.2.69 h1:IdrOs1ZgwGw5CI+BH6GgVVlOt+LAXoPyh7enr8lfaXs= +kernel.org/pub/linux/libs/security/libcap/psx v1.2.69/go.mod h1:+l6Ee2F59XiJ2I6WR5ObpC1utCQJZ/VLsEbQCD8RG24= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/qr v0.2.0 h1:6vBLea5/NRMVTz8V66gipeLycZMl/+UlFmk8DvqQ6WY= +rsc.io/qr v0.2.0/go.mod h1:IF+uZjkb9fqyeF/4tlBoynqmQxUoPfWEKh921coOuXs= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= diff --git a/internal/app/machined/internal/server/v1alpha1/v1alpha1_cluster.go b/internal/app/machined/internal/server/v1alpha1/v1alpha1_cluster.go new file mode 100644 index 0000000..ba82794 --- /dev/null +++ b/internal/app/machined/internal/server/v1alpha1/v1alpha1_cluster.go @@ -0,0 +1,256 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package runtime provides the runtime implementation. +package runtime + +import ( + "context" + "fmt" + "log" + "net/netip" + "slices" + "strings" + + "github.com/cosi-project/runtime/pkg/safe" + "github.com/siderolabs/gen/xslices" + "google.golang.org/grpc/metadata" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/cluster" + "github.com/siderolabs/talos/pkg/cluster/check" + "github.com/siderolabs/talos/pkg/conditions" + "github.com/siderolabs/talos/pkg/grpc/middleware/authz" + clusterapi "github.com/siderolabs/talos/pkg/machinery/api/cluster" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/constants" + clusterres "github.com/siderolabs/talos/pkg/machinery/resources/cluster" +) + +// HealthCheck implements the cluster.ClusterServer interface. +func (s *Server) HealthCheck(in *clusterapi.HealthCheckRequest, srv clusterapi.ClusterService_HealthCheckServer) error { + clientProvider := &cluster.LocalClientProvider{} + defer clientProvider.Close() //nolint:errcheck + + k8sProvider := &cluster.KubernetesClient{ + ClientProvider: clientProvider, + ForceEndpoint: in.GetClusterInfo().GetForceEndpoint(), + } + defer k8sProvider.K8sClose() //nolint:errcheck + + checkCtx, checkCtxCancel := context.WithTimeout(srv.Context(), in.WaitTimeout.AsDuration()) + defer checkCtxCancel() + + md := metadata.New(nil) + authz.SetMetadata(md, authz.GetRoles(srv.Context())) + checkCtx = metadata.NewOutgoingContext(checkCtx, md) + + r := s.Controller.Runtime() + + clusterInfo, err := buildClusterInfo(checkCtx, in, r, *k8sProvider) + if err != nil { + return err + } + + state := struct { + cluster.ClientProvider + cluster.K8sProvider + cluster.Info + }{ + ClientProvider: clientProvider, + K8sProvider: k8sProvider, + Info: clusterInfo, + } + + nodeInternalIPs := xslices.Map(clusterInfo.Nodes(), func(info cluster.NodeInfo) string { + return info.InternalIP.String() + }) + + if err := srv.Send(&clusterapi.HealthCheckProgress{ + Message: fmt.Sprintf("discovered nodes: %q", nodeInternalIPs), + }); err != nil { + return err + } + + return check.Wait(checkCtx, &state, append(check.DefaultClusterChecks(), check.ExtraClusterChecks()...), &healthReporter{srv: srv}) +} + +type healthReporter struct { + srv clusterapi.ClusterService_HealthCheckServer + lastLine string +} + +func (hr *healthReporter) Update(condition conditions.Condition) { + line := fmt.Sprintf("waiting for %s", condition) + + if line != hr.lastLine { + hr.srv.Send(&clusterapi.HealthCheckProgress{ //nolint:errcheck + Message: strings.TrimSpace(line), + }) + + hr.lastLine = line + } +} + +type clusterState struct { + nodeInfos []cluster.NodeInfo + nodeInfosByType map[machine.Type][]cluster.NodeInfo +} + +func (cl *clusterState) Nodes() []cluster.NodeInfo { + return cl.nodeInfos +} + +func (cl *clusterState) NodesByType(t machine.Type) []cluster.NodeInfo { + return cl.nodeInfosByType[t] +} + +func (cl *clusterState) String() string { + return fmt.Sprintf("control plane: %q, worker: %q", + xslices.Map(cl.nodeInfosByType[machine.TypeControlPlane], func(info cluster.NodeInfo) string { + return info.InternalIP.String() + }), + xslices.Map(cl.nodeInfosByType[machine.TypeWorker], func(info cluster.NodeInfo) string { + return info.InternalIP.String() + })) +} + +//nolint:gocyclo +func buildClusterInfo(ctx context.Context, + req *clusterapi.HealthCheckRequest, + r runtime.Runtime, + cli cluster.KubernetesClient, +) (cluster.Info, error) { + controlPlaneNodes := req.GetClusterInfo().GetControlPlaneNodes() + workerNodes := req.GetClusterInfo().GetWorkerNodes() + + // if the node list is explicitly provided, use it + if len(controlPlaneNodes) != 0 || len(workerNodes) != 0 { + controlPlaneNodeInfos, err := cluster.IPsToNodeInfos(controlPlaneNodes) + if err != nil { + return nil, err + } + + workerNodeInfos, err := cluster.IPsToNodeInfos(workerNodes) + if err != nil { + return nil, err + } + + return &clusterState{ + nodeInfos: append(slices.Clone(controlPlaneNodeInfos), workerNodeInfos...), + nodeInfosByType: map[machine.Type][]cluster.NodeInfo{ + machine.TypeControlPlane: controlPlaneNodeInfos, + machine.TypeWorker: workerNodeInfos, + }, + }, nil + } + + // try to discover nodes using discovery service + discoveryMemberList, err := getDiscoveryMemberList(ctx, r) + if err != nil { + log.Printf("discovery service returned error: %v\n", err) + } + + // discovery service returned some nodes, use them + if len(discoveryMemberList) > 0 { + return check.NewDiscoveredClusterInfo(discoveryMemberList) + } + + // as the last resort, get the nodes from the cluster itself + k8sCli, err := cli.K8sClient(ctx) + if err != nil { + return nil, err + } + + nodeList, err := k8sCli.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + + nodeInfos := make([]cluster.NodeInfo, len(nodeList.Items)) + nodeInfosByType := map[machine.Type][]cluster.NodeInfo{} + + for i, node := range nodeList.Items { + nodeInfo, err2 := k8sNodeToNodeInfo(&node) + if err2 != nil { + return nil, err + } + + if isControlPlaneNode(&node) { + nodeInfosByType[machine.TypeControlPlane] = append(nodeInfosByType[machine.TypeControlPlane], *nodeInfo) + } else { + nodeInfosByType[machine.TypeWorker] = append(nodeInfosByType[machine.TypeWorker], *nodeInfo) + } + + nodeInfos[i] = *nodeInfo + } + + return &clusterState{ + nodeInfos: nodeInfos, + nodeInfosByType: nodeInfosByType, + }, nil +} + +func k8sNodeToNodeInfo(node *corev1.Node) (*cluster.NodeInfo, error) { + if node == nil { + return nil, nil + } + + var internalIP netip.Addr + + ips := make([]netip.Addr, 0, len(node.Status.Addresses)) + + for _, address := range node.Status.Addresses { + if address.Type == corev1.NodeInternalIP { + ip, err := netip.ParseAddr(address.Address) + if err != nil { + return nil, err + } + + internalIP = ip + ips = append(ips, ip) + } else if address.Type == corev1.NodeExternalIP { + ip, err := netip.ParseAddr(address.Address) + if err != nil { + return nil, err + } + + ips = append(ips, ip) + } + } + + return &cluster.NodeInfo{ + InternalIP: internalIP, + IPs: ips, + }, nil +} + +func getDiscoveryMemberList(ctx context.Context, runtime runtime.Runtime) ([]*clusterres.Member, error) { + res := runtime.State().V1Alpha2().Resources() + + list, err := safe.StateListAll[*clusterres.Member](ctx, res) + if err != nil { + return nil, err + } + + result := make([]*clusterres.Member, 0, list.Len()) + + for iter := list.Iterator(); iter.Next(); { + result = append(result, iter.Value()) + } + + return result, nil +} + +func isControlPlaneNode(node *corev1.Node) bool { + for key := range node.Labels { + if key == constants.LabelNodeRoleControlPlane { + return true + } + } + + return false +} diff --git a/internal/app/machined/internal/server/v1alpha1/v1alpha1_images.go b/internal/app/machined/internal/server/v1alpha1/v1alpha1_images.go new file mode 100644 index 0000000..fb40c07 --- /dev/null +++ b/internal/app/machined/internal/server/v1alpha1/v1alpha1_images.go @@ -0,0 +1,109 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + + containerdapi "github.com/containerd/containerd" + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/namespaces" + criconstants "github.com/containerd/containerd/pkg/cri/constants" + "github.com/containerd/containerd/platforms" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/aenix-io/talm/internal/pkg/containers/image" + "github.com/siderolabs/talos/pkg/machinery/api/common" + "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +func containerdNamespaceHelper(ctx context.Context, ns common.ContainerdNamespace) (context.Context, error) { + var namespaceName string + + switch ns { + case common.ContainerdNamespace_NS_CRI: + namespaceName = criconstants.K8sContainerdNamespace + case common.ContainerdNamespace_NS_SYSTEM: + namespaceName = constants.SystemContainerdNamespace + case common.ContainerdNamespace_NS_UNKNOWN: + fallthrough + default: + return nil, status.Errorf(codes.InvalidArgument, "invalid namespace %s", ns) + } + + return namespaces.WithNamespace(ctx, namespaceName), nil +} + +// ImageList lists the images in the CRI. +func (s *Server) ImageList(req *machine.ImageListRequest, srv machine.MachineService_ImageListServer) error { + client, err := containerdapi.New(constants.CRIContainerdAddress) + if err != nil { + return status.Errorf(codes.Unavailable, "error connecting to containerd: %s", err) + } + //nolint:errcheck + defer client.Close() + + ctx, err := containerdNamespaceHelper(srv.Context(), req.Namespace) + if err != nil { + return err + } + + images, err := client.ImageService().List(ctx) + if err != nil { + return err + } + + for _, image := range images { + item := &machine.ImageListResponse{ + Name: image.Name, + Digest: image.Target.Digest.String(), + CreatedAt: timestamppb.New(image.CreatedAt), + } + + size, err := image.Size(ctx, client.ContentStore(), platforms.Default()) + if err == nil { + item.Size = size + } + + if err = srv.Send(item); err != nil { + return err + } + } + + return nil +} + +// ImagePull pulls an image to the CRI. +func (s *Server) ImagePull(ctx context.Context, req *machine.ImagePullRequest) (*machine.ImagePullResponse, error) { + client, err := containerdapi.New(constants.CRIContainerdAddress) + if err != nil { + return nil, status.Errorf(codes.Unavailable, "error connecting to containerd: %s", err) + } + //nolint:errcheck + defer client.Close() + + ctx, err = containerdNamespaceHelper(ctx, req.Namespace) + if err != nil { + return nil, err + } + + _, err = image.Pull(ctx, s.Controller.Runtime().Config().Machine().Registries(), client, req.Reference, image.WithSkipIfAlreadyPulled()) + if err != nil { + if errdefs.IsNotFound(err) { + return nil, status.Errorf(codes.NotFound, "error pulling image: %s", err) + } + + return nil, err + } + + return &machine.ImagePullResponse{ + Messages: []*machine.ImagePull{ + {}, + }, + }, nil +} diff --git a/internal/app/machined/internal/server/v1alpha1/v1alpha1_inspect.go b/internal/app/machined/internal/server/v1alpha1/v1alpha1_inspect.go new file mode 100644 index 0000000..a9cea71 --- /dev/null +++ b/internal/app/machined/internal/server/v1alpha1/v1alpha1_inspect.go @@ -0,0 +1,71 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "google.golang.org/protobuf/types/known/emptypb" + + inspectapi "github.com/siderolabs/talos/pkg/machinery/api/inspect" +) + +// InspectServer implements InspectService API. +type InspectServer struct { + inspectapi.UnimplementedInspectServiceServer + + server *Server +} + +// ControllerRuntimeDependencies implements inspect.InspectService interface. +func (s *InspectServer) ControllerRuntimeDependencies(ctx context.Context, in *emptypb.Empty) (*inspectapi.ControllerRuntimeDependenciesResponse, error) { + graph, err := s.server.Controller.V1Alpha2().DependencyGraph() + if err != nil { + return nil, fmt.Errorf("error fetching dependency graph: %w", err) + } + + edges := make([]*inspectapi.ControllerDependencyEdge, 0, len(graph.Edges)) + + for i := range graph.Edges { + var edgeType inspectapi.DependencyEdgeType + + switch graph.Edges[i].EdgeType { + case controller.EdgeOutputExclusive: + edgeType = inspectapi.DependencyEdgeType_OUTPUT_EXCLUSIVE + case controller.EdgeOutputShared: + edgeType = inspectapi.DependencyEdgeType_OUTPUT_SHARED + case controller.EdgeInputStrong: + edgeType = inspectapi.DependencyEdgeType_INPUT_STRONG + case controller.EdgeInputWeak: + edgeType = inspectapi.DependencyEdgeType_INPUT_WEAK + case controller.EdgeInputDestroyReady: + edgeType = inspectapi.DependencyEdgeType_INPUT_DESTROY_READY + case controller.EdgeInputQPrimary, + controller.EdgeInputQMapped, + controller.EdgeInputQMappedDestroyReady: + return nil, fmt.Errorf("unexpected edge type: %v", graph.Edges[i].EdgeType) + } + + edges = append(edges, &inspectapi.ControllerDependencyEdge{ + ControllerName: graph.Edges[i].ControllerName, + + EdgeType: edgeType, + + ResourceNamespace: graph.Edges[i].ResourceNamespace, + ResourceType: graph.Edges[i].ResourceType, + ResourceId: graph.Edges[i].ResourceID, + }) + } + + return &inspectapi.ControllerRuntimeDependenciesResponse{ + Messages: []*inspectapi.ControllerRuntimeDependency{ + { + Edges: edges, + }, + }, + }, nil +} diff --git a/internal/app/machined/internal/server/v1alpha1/v1alpha1_meta.go b/internal/app/machined/internal/server/v1alpha1/v1alpha1_meta.go new file mode 100644 index 0000000..ea2e420 --- /dev/null +++ b/internal/app/machined/internal/server/v1alpha1/v1alpha1_meta.go @@ -0,0 +1,82 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + "os" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/api/machine" +) + +// MetaWrite implements the machine.MachineServer interface. +func (s *Server) MetaWrite(ctx context.Context, req *machine.MetaWriteRequest) (*machine.MetaWriteResponse, error) { + if err := s.checkSupported(runtime.MetaKV); err != nil { + return nil, err + } + + if uint32(uint8(req.Key)) != req.Key { + return nil, status.Errorf(codes.InvalidArgument, "key must be a uint8") + } + + ok, err := s.Controller.Runtime().State().Machine().Meta().SetTagBytes(ctx, uint8(req.Key), req.Value) + if err != nil { + return nil, err + } + + if !ok { + // META overflowed + return nil, status.Errorf(codes.ResourceExhausted, "meta write failed") + } + + err = s.Controller.Runtime().State().Machine().Meta().Flush() + if err != nil && !os.IsNotExist(err) { + // ignore not exist error, as it's possible that the meta partition is not created yet + return nil, err + } + + return &machine.MetaWriteResponse{ + Messages: []*machine.MetaWrite{ + {}, + }, + }, nil +} + +// MetaDelete implements the machine.MachineServer interface. +func (s *Server) MetaDelete(ctx context.Context, req *machine.MetaDeleteRequest) (*machine.MetaDeleteResponse, error) { + if err := s.checkSupported(runtime.MetaKV); err != nil { + return nil, err + } + + if uint32(uint8(req.Key)) != req.Key { + return nil, status.Errorf(codes.InvalidArgument, "key must be a uint8") + } + + ok, err := s.Controller.Runtime().State().Machine().Meta().DeleteTag(ctx, uint8(req.Key)) + if err != nil { + return nil, err + } + + if !ok { + // META key not found + return nil, status.Errorf(codes.NotFound, "meta key not found") + } + + err = s.Controller.Runtime().State().Machine().Meta().Flush() + if err != nil && !os.IsNotExist(err) { + // ignore not exist error, as it's possible that the meta partition is not created yet + return nil, err + } + + return &machine.MetaDeleteResponse{ + Messages: []*machine.MetaDelete{ + {}, + }, + }, nil +} diff --git a/internal/app/machined/internal/server/v1alpha1/v1alpha1_monitoring.go b/internal/app/machined/internal/server/v1alpha1/v1alpha1_monitoring.go new file mode 100644 index 0000000..11650e4 --- /dev/null +++ b/internal/app/machined/internal/server/v1alpha1/v1alpha1_monitoring.go @@ -0,0 +1,324 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "bufio" + "context" + "os" + "strconv" + "strings" + + "github.com/prometheus/procfs" + "github.com/siderolabs/gen/maps" + "github.com/siderolabs/gen/xslices" + "google.golang.org/protobuf/types/known/emptypb" + + "github.com/siderolabs/talos/pkg/machinery/api/machine" +) + +// Hostname implements the machine.MachineServer interface. +func (s *Server) Hostname(ctx context.Context, in *emptypb.Empty) (*machine.HostnameResponse, error) { + hostname, err := os.Hostname() + if err != nil { + return nil, err + } + + reply := &machine.HostnameResponse{ + Messages: []*machine.Hostname{ + { + Hostname: hostname, + }, + }, + } + + return reply, nil +} + +// LoadAvg implements the machine.MachineServer interface. +func (s *Server) LoadAvg(ctx context.Context, in *emptypb.Empty) (*machine.LoadAvgResponse, error) { + fs, err := procfs.NewDefaultFS() + if err != nil { + return nil, err + } + + loadAvg, err := fs.LoadAvg() + if err != nil { + return nil, err + } + + reply := &machine.LoadAvgResponse{ + Messages: []*machine.LoadAvg{ + { + Load1: loadAvg.Load1, + Load5: loadAvg.Load5, + Load15: loadAvg.Load15, + }, + }, + } + + return reply, nil +} + +// SystemStat implements the machine.MachineServer interface. +func (s *Server) SystemStat(ctx context.Context, in *emptypb.Empty) (*machine.SystemStatResponse, error) { + fs, err := procfs.NewDefaultFS() + if err != nil { + return nil, err + } + + stat, err := fs.Stat() + if err != nil { + return nil, err + } + + translateCPUStat := func(in procfs.CPUStat) *machine.CPUStat { + return &machine.CPUStat{ + User: in.User, + Nice: in.Nice, + System: in.System, + Idle: in.Idle, + Iowait: in.Iowait, + Irq: in.IRQ, + SoftIrq: in.SoftIRQ, + Steal: in.Steal, + Guest: in.Guest, + GuestNice: in.GuestNice, + } + } + + translateListOfCPUStat := func(in map[int64]procfs.CPUStat) []*machine.CPUStat { + maxCore := int64(-1) + + for core := range in { + if core > maxCore { + maxCore = core + } + } + + slc := make([]*machine.CPUStat, maxCore+1) + + for core, stat := range in { + slc[core] = translateCPUStat(stat) + } + + return slc + } + + translateSoftIRQ := func(in procfs.SoftIRQStat) *machine.SoftIRQStat { + return &machine.SoftIRQStat{ + Hi: in.Hi, + Timer: in.Timer, + NetTx: in.NetTx, + NetRx: in.NetRx, + Block: in.Block, + BlockIoPoll: in.BlockIoPoll, + Tasklet: in.Tasklet, + Sched: in.Sched, + Hrtimer: in.Hrtimer, + Rcu: in.Rcu, + } + } + + reply := &machine.SystemStatResponse{ + Messages: []*machine.SystemStat{ + { + BootTime: stat.BootTime, + CpuTotal: translateCPUStat(stat.CPUTotal), + Cpu: translateListOfCPUStat(stat.CPU), + IrqTotal: stat.IRQTotal, + Irq: stat.IRQ, + ContextSwitches: stat.ContextSwitches, + ProcessCreated: stat.ProcessCreated, + ProcessRunning: stat.ProcessesRunning, + ProcessBlocked: stat.ProcessesBlocked, + SoftIrqTotal: stat.SoftIRQTotal, + SoftIrq: translateSoftIRQ(stat.SoftIRQ), + }, + }, + } + + return reply, nil +} + +// CPUInfo implements the machine.MachineServer interface. +func (s *Server) CPUInfo(ctx context.Context, in *emptypb.Empty) (*machine.CPUInfoResponse, error) { + fs, err := procfs.NewDefaultFS() + if err != nil { + return nil, err + } + + info, err := fs.CPUInfo() + if err != nil { + return nil, err + } + + translateCPUInfo := func(in procfs.CPUInfo) *machine.CPUInfo { + return &machine.CPUInfo{ + Processor: uint32(in.Processor), + VendorId: in.VendorID, + CpuFamily: in.CPUFamily, + Model: in.Model, + ModelName: in.ModelName, + Stepping: in.Stepping, + Microcode: in.Microcode, + CpuMhz: in.CPUMHz, + CacheSize: in.CacheSize, + PhysicalId: in.PhysicalID, + Siblings: uint32(in.Siblings), + CoreId: in.CoreID, + ApicId: in.APICID, + InitialApicId: in.InitialAPICID, + Fpu: in.FPU, + FpuException: in.FPUException, + CpuIdLevel: uint32(in.CPUIDLevel), + Wp: in.WP, + Flags: in.Flags, + Bugs: in.Bugs, + BogoMips: in.BogoMips, + ClFlushSize: uint32(in.CLFlushSize), + CacheAlignment: uint32(in.CacheAlignment), + AddressSizes: in.AddressSizes, + PowerManagement: in.PowerManagement, + } + } + + reply := &machine.CPUInfoResponse{ + Messages: []*machine.CPUsInfo{ + { + CpuInfo: xslices.Map(info, translateCPUInfo), + }, + }, + } + + return reply, nil +} + +// NetworkDeviceStats implements the machine.MachineServer interface. +func (s *Server) NetworkDeviceStats(ctx context.Context, in *emptypb.Empty) (*machine.NetworkDeviceStatsResponse, error) { + fs, err := procfs.NewDefaultFS() + if err != nil { + return nil, err + } + + info, err := fs.NetDev() + if err != nil { + return nil, err + } + + translateNetDevLine := func(in procfs.NetDevLine) *machine.NetDev { + return &machine.NetDev{ + Name: in.Name, + RxBytes: in.RxBytes, + RxPackets: in.RxPackets, + RxErrors: in.RxErrors, + RxDropped: in.RxDropped, + RxFifo: in.RxFIFO, + RxFrame: in.RxFrame, + RxCompressed: in.RxCompressed, + RxMulticast: in.RxMulticast, + TxBytes: in.TxBytes, + TxPackets: in.TxPackets, + TxErrors: in.TxErrors, + TxDropped: in.TxDropped, + TxFifo: in.TxFIFO, + TxCollisions: in.TxCollisions, + TxCarrier: in.TxCarrier, + TxCompressed: in.TxCompressed, + } + } + + reply := &machine.NetworkDeviceStatsResponse{ + Messages: []*machine.NetworkDeviceStats{ + { + Devices: maps.ValuesFunc(info, translateNetDevLine), + Total: translateNetDevLine(info.Total()), + }, + }, + } + + return reply, nil +} + +// DiskStats implements the machine.MachineServer interface. +func (s *Server) DiskStats(ctx context.Context, in *emptypb.Empty) (*machine.DiskStatsResponse, error) { + f, err := os.Open("/proc/diskstats") + if err != nil { + return nil, err + } + + defer f.Close() //nolint:errcheck + + resp := machine.DiskStats{ + Devices: []*machine.DiskStat{}, + Total: &machine.DiskStat{}, + } + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + + if len(fields) < 18 { + continue + } + + values := make([]uint64, 15) + for i := range values { + values[i], err = strconv.ParseUint(fields[3+i], 10, 64) + if err != nil { + return nil, err + } + } + + stat := &machine.DiskStat{ + Name: fields[2], + ReadCompleted: values[0], + ReadMerged: values[1], + ReadSectors: values[2], + ReadTimeMs: values[3], + WriteCompleted: values[4], + WriteMerged: values[5], + WriteSectors: values[6], + WriteTimeMs: values[7], + IoInProgress: values[8], + IoTimeMs: values[9], + IoTimeWeightedMs: values[10], + DiscardCompleted: values[11], + DiscardMerged: values[12], + DiscardSectors: values[13], + DiscardTimeMs: values[14], + } + + resp.Devices = append(resp.Devices, stat) + + resp.Total.ReadCompleted += stat.ReadCompleted + resp.Total.ReadMerged += stat.ReadMerged + resp.Total.ReadSectors += stat.ReadSectors + resp.Total.ReadTimeMs += stat.ReadTimeMs + resp.Total.WriteCompleted += stat.WriteCompleted + resp.Total.WriteMerged += stat.WriteMerged + resp.Total.WriteSectors += stat.WriteSectors + resp.Total.WriteTimeMs += stat.WriteTimeMs + resp.Total.IoInProgress += stat.IoInProgress + resp.Total.IoTimeMs += stat.IoTimeMs + resp.Total.IoTimeWeightedMs += stat.IoTimeWeightedMs + resp.Total.DiscardCompleted += stat.DiscardCompleted + resp.Total.DiscardMerged += stat.DiscardMerged + resp.Total.DiscardSectors += stat.DiscardSectors + resp.Total.DiscardTimeMs += stat.DiscardTimeMs + } + + if err = scanner.Err(); err != nil { + return nil, err + } + + reply := &machine.DiskStatsResponse{ + Messages: []*machine.DiskStats{ + &resp, + }, + } + + return reply, nil +} diff --git a/internal/app/machined/internal/server/v1alpha1/v1alpha1_server.go b/internal/app/machined/internal/server/v1alpha1/v1alpha1_server.go new file mode 100644 index 0000000..fbf0dc0 --- /dev/null +++ b/internal/app/machined/internal/server/v1alpha1/v1alpha1_server.go @@ -0,0 +1,2383 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "archive/tar" + "bufio" + "bytes" + "compress/gzip" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "net" + "os" + "path/filepath" + "slices" + "strings" + "syscall" + "time" + + criconstants "github.com/containerd/containerd/pkg/cri/constants" + cosiv1alpha1 "github.com/cosi-project/runtime/api/v1alpha1" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/protobuf/server" + "github.com/google/go-cmp/cmp" + "github.com/google/uuid" + "github.com/gopacket/gopacket/afpacket" + multierror "github.com/hashicorp/go-multierror" + "github.com/nberlee/go-netstat/netstat" + "github.com/prometheus/procfs" + "github.com/rs/xid" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/go-blockdevice/blockdevice/partition/gpt" + bddisk "github.com/siderolabs/go-blockdevice/blockdevice/util/disk" + "github.com/siderolabs/go-kmsg" + "github.com/siderolabs/go-pointer" + "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/client/v3/concurrency" + "golang.org/x/net/bpf" + "golang.org/x/sys/unix" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" + + installer "github.com/siderolabs/talos/cmd/installer/pkg/install" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/bootloader" + "github.com/aenix-io/talm/internal/app/machined/pkg/system" + "github.com/aenix-io/talm/internal/app/resources" + storaged "github.com/aenix-io/talm/internal/app/storaged" + "github.com/aenix-io/talm/internal/pkg/configuration" + "github.com/aenix-io/talm/internal/pkg/containers" + taloscontainerd "github.com/aenix-io/talm/internal/pkg/containers/containerd" + "github.com/aenix-io/talm/internal/pkg/containers/cri" + "github.com/aenix-io/talm/internal/pkg/etcd" + "github.com/aenix-io/talm/internal/pkg/install" + "github.com/aenix-io/talm/internal/pkg/meta" + "github.com/aenix-io/talm/internal/pkg/miniprocfs" + "github.com/aenix-io/talm/internal/pkg/pcap" + "github.com/siderolabs/talos/pkg/archiver" + "github.com/siderolabs/talos/pkg/chunker" + "github.com/siderolabs/talos/pkg/chunker/stream" + "github.com/siderolabs/talos/pkg/kubeconfig" + "github.com/siderolabs/talos/pkg/machinery/api/cluster" + "github.com/siderolabs/talos/pkg/machinery/api/common" + "github.com/siderolabs/talos/pkg/machinery/api/inspect" + "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/api/storage" + timeapi "github.com/siderolabs/talos/pkg/machinery/api/time" + clientconfig "github.com/siderolabs/talos/pkg/machinery/client/config" + "github.com/siderolabs/talos/pkg/machinery/config/configloader" + "github.com/siderolabs/talos/pkg/machinery/config/generate/secrets" + machinetype "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + etcdresource "github.com/siderolabs/talos/pkg/machinery/resources/etcd" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + timeresource "github.com/siderolabs/talos/pkg/machinery/resources/time" + "github.com/siderolabs/talos/pkg/machinery/role" + "github.com/siderolabs/talos/pkg/machinery/version" +) + +// MinimumEtcdUpgradeLeaseLockSeconds indicates the minimum number of seconds for which we open a lease lock for upgrading Etcd nodes. +// This is not intended to lock for the duration of an upgrade. +// Rather, it is intended to make sure only one node processes the various pre-upgrade checks at a time. +// Thus, this timeout should be reflective of the expected time for the pre-upgrade checks, NOT the time to perform the upgrade itself. +const MinimumEtcdUpgradeLeaseLockSeconds = 60 + +// OSPathSeparator is the string version of the os.PathSeparator. +const OSPathSeparator = string(os.PathSeparator) + +// Server implements ClusterService and MachineService APIs +// and is also responsible for registering ResourceServer and InspectServer. +type Server struct { + cluster.UnimplementedClusterServiceServer + machine.UnimplementedMachineServiceServer + + Controller runtime.Controller + // breaking the import loop cycle between services/ package and v1alpha1_server.go + EtcdBootstrapper func(context.Context, runtime.Runtime, *machine.BootstrapRequest) error + + // ShutdownCtx signals that the server is shutting down. + ShutdownCtx context.Context //nolint:containedctx + + server *grpc.Server +} + +func (s *Server) checkSupported(feature runtime.ModeCapability) error { + mode := s.Controller.Runtime().State().Platform().Mode() + + if !mode.Supports(feature) { + return status.Errorf(codes.FailedPrecondition, "method is not supported in %s mode", mode.String()) + } + + return nil +} + +func (s *Server) checkControlplane(apiName string) error { + switch s.Controller.Runtime().Config().Machine().Type() { //nolint:exhaustive + case machinetype.TypeControlPlane: + fallthrough + case machinetype.TypeInit: + return nil + } + + return status.Errorf(codes.Unimplemented, "%s is only available on control plane nodes", apiName) +} + +// Register implements the factory.Registrator interface. +func (s *Server) Register(obj *grpc.Server) { + s.server = obj + + // wrap resources with access filter + resourceState := s.Controller.Runtime().State().V1Alpha2().Resources() + resourceState = state.WrapCore(state.Filter(resourceState, resources.AccessPolicy(resourceState))) + + machine.RegisterMachineServiceServer(obj, s) + cluster.RegisterClusterServiceServer(obj, s) + cosiv1alpha1.RegisterStateServer(obj, server.NewState(resourceState)) + inspect.RegisterInspectServiceServer(obj, &InspectServer{server: s}) + storage.RegisterStorageServiceServer(obj, &storaged.Server{Controller: s.Controller}) + timeapi.RegisterTimeServiceServer(obj, &TimeServer{ConfigProvider: s.Controller.Runtime()}) +} + +// modeWrapper overrides RequiresInstall() based on actual installed status. +type modeWrapper struct { + runtime.Mode + installed bool +} + +func (m modeWrapper) RequiresInstall() bool { + return m.Mode.RequiresInstall() && !m.installed +} + +// ApplyConfiguration implements machine.MachineService. +// +//nolint:gocyclo,cyclop +func (s *Server) ApplyConfiguration(ctx context.Context, in *machine.ApplyConfigurationRequest) (*machine.ApplyConfigurationResponse, error) { + mode := in.Mode.String() + modeDetails := "Applied configuration with a reboot" + modeErr := "" + + if in.Mode != machine.ApplyConfigurationRequest_TRY { + s.Controller.Runtime().CancelConfigRollbackTimeout() + } + + cfgProvider, err := configloader.NewFromBytes(in.GetData()) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + + warnings, err := cfgProvider.Validate( + modeWrapper{ + Mode: s.Controller.Runtime().State().Platform().Mode(), + installed: s.Controller.Runtime().State().Machine().Installed(), + }, + ) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + + //nolint:exhaustive + switch in.Mode { + // --mode=try + case machine.ApplyConfigurationRequest_TRY: + fallthrough + // --mode=no-reboot + case machine.ApplyConfigurationRequest_NO_REBOOT: + if err = s.Controller.Runtime().CanApplyImmediate(cfgProvider); err != nil { + return nil, status.Errorf(codes.InvalidArgument, err.Error()) + } + + modeDetails = "Applied configuration without a reboot" + // --mode=staged + case machine.ApplyConfigurationRequest_STAGED: + modeDetails = "Staged configuration to be applied after the next reboot" + // --mode=auto detect actual update mode + case machine.ApplyConfigurationRequest_AUTO: + if err = s.Controller.Runtime().CanApplyImmediate(cfgProvider); err != nil { + in.Mode = machine.ApplyConfigurationRequest_REBOOT + modeDetails = "Applied configuration with a reboot" + modeErr = ": " + err.Error() + } else { + in.Mode = machine.ApplyConfigurationRequest_NO_REBOOT + modeDetails = "Applied configuration without a reboot" + } + + mode = fmt.Sprintf("%s(%s)", mode, in.Mode) + } + + if in.DryRun { + var config interface{} + if s.Controller.Runtime().Config() != nil { + config = s.Controller.Runtime().ConfigContainer().RawV1Alpha1() + } + + diff := cmp.Diff(config, cfgProvider.RawV1Alpha1(), cmp.AllowUnexported(v1alpha1.InstallDiskSizeMatcher{})) + if diff == "" { + diff = "No changes." + } + + return &machine.ApplyConfigurationResponse{ + Messages: []*machine.ApplyConfiguration{ + { + Mode: in.Mode, + ModeDetails: fmt.Sprintf(`Dry run summary: +%s (skipped in dry-run). +Config diff: +%s`, modeDetails, diff), + }, + }, + }, nil + } + + log.Printf("apply config request: mode %s", strings.ToLower(mode)) + + cfg, err := cfgProvider.Bytes() + if err != nil { + return nil, err + } + + if in.Mode != machine.ApplyConfigurationRequest_TRY { + if err := os.WriteFile(constants.ConfigPath, cfg, 0o600); err != nil { + return nil, err + } + } + + //nolint:exhaustive + switch in.Mode { + // --mode=try + case machine.ApplyConfigurationRequest_TRY: + timeout := constants.ConfigTryTimeout + if in.TryModeTimeout != nil { + timeout = in.TryModeTimeout.AsDuration() + } + + modeDetails += fmt.Sprintf("\nThe config is applied in 'try' mode and will be automatically reverted back in %s", timeout.String()) + + if err := s.Controller.Runtime().RollbackToConfigAfter(timeout); err != nil { + return nil, err + } + + fallthrough + // --mode=no-reboot + case machine.ApplyConfigurationRequest_NO_REBOOT: + if err := s.Controller.Runtime().SetConfig(cfgProvider); err != nil { + return nil, err + } + // --mode=staged + case machine.ApplyConfigurationRequest_STAGED: + // --mode=reboot + case machine.ApplyConfigurationRequest_REBOOT: + go func() { + if err := s.Controller.Run(context.Background(), runtime.SequenceReboot, nil, runtime.WithTakeover()); err != nil { + if !runtime.IsRebootError(err) { + log.Println("apply configuration failed:", err) + } + } + }() + default: + return nil, fmt.Errorf("incorrect mode '%s' specified for the apply config call", in.Mode.String()) + } + + return &machine.ApplyConfigurationResponse{ + Messages: []*machine.ApplyConfiguration{ + { + Mode: in.Mode, + Warnings: warnings, + ModeDetails: modeDetails + modeErr, + }, + }, + }, nil +} + +// GenerateConfiguration implements the machine.MachineServer interface. +func (s *Server) GenerateConfiguration(ctx context.Context, in *machine.GenerateConfigurationRequest) (reply *machine.GenerateConfigurationResponse, err error) { + if s.Controller.Runtime().Config().Machine().Type() == machinetype.TypeWorker { + return nil, errors.New("config can't be generated on worker nodes") + } + + return configuration.Generate(ctx, in) +} + +// Reboot implements the machine.MachineServer interface. +func (s *Server) Reboot(ctx context.Context, in *machine.RebootRequest) (reply *machine.RebootResponse, err error) { + actorID := uuid.New().String() + + log.Printf("reboot via API received. actor id: %s", actorID) + + if err := s.checkSupported(runtime.Reboot); err != nil { + return nil, err + } + + rebootCtx := context.WithValue(context.Background(), runtime.ActorIDCtxKey{}, actorID) + + go func() { + if err := s.Controller.Run(rebootCtx, runtime.SequenceReboot, in); err != nil { + if !runtime.IsRebootError(err) { + log.Println("reboot failed:", err) + } + } + }() + + reply = &machine.RebootResponse{ + Messages: []*machine.Reboot{ + { + ActorId: actorID, + }, + }, + } + + return reply, nil +} + +// Rollback implements the machine.MachineServer interface. +func (s *Server) Rollback(ctx context.Context, in *machine.RollbackRequest) (*machine.RollbackResponse, error) { + log.Printf("rollback via API received") + + if err := s.checkSupported(runtime.Rollback); err != nil { + return nil, err + } + + systemDisk := s.Controller.Runtime().State().Machine().Disk() + if systemDisk == nil { + return nil, status.Errorf(codes.FailedPrecondition, "system disk not found") + } + + if err := func() error { + config, err := bootloader.Probe(ctx, systemDisk.Device().Name()) + if err != nil { + return err + } + + return config.Revert(ctx) + }(); err != nil { + return nil, err + } + + go func() { + if err := s.Controller.Run(context.Background(), runtime.SequenceReboot, in, runtime.WithTakeover()); err != nil { + if !runtime.IsRebootError(err) { + log.Println("reboot failed:", err) + } + } + }() + + return &machine.RollbackResponse{ + Messages: []*machine.Rollback{ + {}, + }, + }, nil +} + +// Bootstrap implements the machine.MachineServer interface. +func (s *Server) Bootstrap(ctx context.Context, in *machine.BootstrapRequest) (reply *machine.BootstrapResponse, err error) { + log.Printf("bootstrap request received") + + if !s.Controller.Runtime().IsBootstrapAllowed() { + return nil, status.Error(codes.FailedPrecondition, "bootstrap is not available yet") + } + + if s.Controller.Runtime().Config().Machine().Type() == machinetype.TypeWorker { + return nil, status.Error(codes.FailedPrecondition, "bootstrap can only be performed on a control plane node") + } + + timeCtx, timeCtxCancel := context.WithTimeout(ctx, 5*time.Second) + defer timeCtxCancel() + + if err := timeresource.NewSyncCondition(s.Controller.Runtime().State().V1Alpha2().Resources()).Wait(timeCtx); err != nil { + return nil, status.Error(codes.FailedPrecondition, "time is not in sync yet") + } + + if entries, _ := os.ReadDir(constants.EtcdDataPath); len(entries) > 0 { //nolint:errcheck + return nil, status.Error(codes.AlreadyExists, "etcd data directory is not empty") + } + + if err := s.EtcdBootstrapper(ctx, s.Controller.Runtime(), in); err != nil { + return nil, err + } + + reply = &machine.BootstrapResponse{ + Messages: []*machine.Bootstrap{ + {}, + }, + } + + return reply, nil +} + +// Shutdown implements the machine.MachineServer interface. +func (s *Server) Shutdown(ctx context.Context, in *machine.ShutdownRequest) (reply *machine.ShutdownResponse, err error) { + actorID := uuid.New().String() + + log.Printf("shutdown via API received. actor id: %s", actorID) + + if err = s.checkSupported(runtime.Shutdown); err != nil { + return nil, err + } + + shutdownCtx := context.WithValue(context.Background(), runtime.ActorIDCtxKey{}, actorID) + + go func() { + if err := s.Controller.Run(shutdownCtx, runtime.SequenceShutdown, in, runtime.WithTakeover()); err != nil { + if !runtime.IsRebootError(err) { + log.Println("shutdown failed:", err) + } + } + }() + + reply = &machine.ShutdownResponse{ + Messages: []*machine.Shutdown{ + { + ActorId: actorID, + }, + }, + } + + return reply, nil +} + +// Upgrade initiates an upgrade. +// +//nolint:gocyclo +func (s *Server) Upgrade(ctx context.Context, in *machine.UpgradeRequest) (*machine.UpgradeResponse, error) { + actorID := uuid.New().String() + + ctx = context.WithValue(ctx, runtime.ActorIDCtxKey{}, actorID) + + if err := s.checkSupported(runtime.Upgrade); err != nil { + return nil, err + } + + log.Printf("upgrade request received: preserve %v, staged %v, force %v, reboot mode %v", in.GetPreserve(), in.GetStage(), in.GetForce(), in.GetRebootMode().String()) + + log.Printf("validating %q", in.GetImage()) + + if err := install.PullAndValidateInstallerImage(ctx, s.Controller.Runtime().Config().Machine().Registries(), in.GetImage()); err != nil { + return nil, fmt.Errorf("error validating installer image %q: %w", in.GetImage(), err) + } + + if s.Controller.Runtime().Config().Machine().Type() != machinetype.TypeWorker && !in.GetForce() { + etcdClient, err := etcd.NewClientFromControlPlaneIPs(ctx, s.Controller.Runtime().State().V1Alpha2().Resources()) + if err != nil { + return nil, fmt.Errorf("failed to create etcd client: %w", err) + } + + // acquire the upgrade mutex + unlocker, err := tryLockUpgradeMutex(ctx, etcdClient) + if err != nil { + return nil, fmt.Errorf("failed to acquire upgrade mutex: %w", err) + } + + // unlock the mutex once the API call is done, as it protects only pre-upgrade checks + defer unlocker() + + if err = etcdClient.ValidateForUpgrade(ctx, s.Controller.Runtime().Config(), in.GetPreserve()); err != nil { + return nil, fmt.Errorf("error validating etcd for upgrade: %w", err) + } + } + + runCtx := context.WithValue(context.Background(), runtime.ActorIDCtxKey{}, actorID) + + if in.GetStage() { + if ok, err := s.Controller.Runtime().State().Machine().Meta().SetTag(ctx, meta.StagedUpgradeImageRef, in.GetImage()); !ok || err != nil { + return nil, fmt.Errorf("error adding staged upgrade image ref tag: %w", err) + } + + opts := install.DefaultInstallOptions() + if err := opts.Apply(install.OptionsFromUpgradeRequest(s.Controller.Runtime(), in)...); err != nil { + return nil, fmt.Errorf("error applying install options: %w", err) + } + + serialized, err := json.Marshal(opts) + if err != nil { + return nil, fmt.Errorf("error serializing install options: %s", err) + } + + var ok bool + + if ok, err = s.Controller.Runtime().State().Machine().Meta().SetTag(ctx, meta.StagedUpgradeInstallOptions, string(serialized)); !ok || err != nil { + return nil, fmt.Errorf("error adding staged upgrade install options tag: %w", err) + } + + if err = s.Controller.Runtime().State().Machine().Meta().Flush(); err != nil { + return nil, fmt.Errorf("error writing meta: %w", err) + } + + go func() { + if err := s.Controller.Run(runCtx, runtime.SequenceStageUpgrade, in); err != nil { + if !runtime.IsRebootError(err) { + log.Println("reboot for staged upgrade failed:", err) + } + } + }() + } else { + go func() { + if err := s.Controller.Run(runCtx, runtime.SequenceUpgrade, in); err != nil { + if !runtime.IsRebootError(err) { + log.Println("upgrade failed:", err) + } + } + }() + } + + return &machine.UpgradeResponse{ + Messages: []*machine.Upgrade{ + { + Ack: "Upgrade request received", + ActorId: actorID, + }, + }, + }, nil +} + +// ResetOptions implements runtime.ResetOptions interface. +type ResetOptions struct { + *machine.ResetRequest + + systemDiskTargets []*installer.Target +} + +// GetSystemDiskTargets implements runtime.ResetOptions interface. +func (opt *ResetOptions) GetSystemDiskTargets() []runtime.PartitionTarget { + if opt.systemDiskTargets == nil { + return nil + } + + return xslices.Map(opt.systemDiskTargets, func(t *installer.Target) runtime.PartitionTarget { return t }) +} + +// Reset resets the node. +// +//nolint:gocyclo +func (s *Server) Reset(ctx context.Context, in *machine.ResetRequest) (reply *machine.ResetResponse, err error) { + actorID := uuid.New().String() + + log.Printf("reset request received. actorID: %s", actorID) + + opts := ResetOptions{ + ResetRequest: in, + } + + if len(in.GetUserDisksToWipe()) > 0 { + if in.Mode == machine.ResetRequest_SYSTEM_DISK { + return nil, errors.New("reset failed: invalid input, wipe mode SYSTEM_DISK doesn't support UserDisksToWipe parameter") + } + + var diskList []*bddisk.Disk + + diskList, err = bddisk.List() + if err != nil { + return nil, err + } + + disks := xslices.ToMap(diskList, func(disk *bddisk.Disk) (string, *bddisk.Disk) { + return disk.DeviceName, disk + }) + + systemDisk := s.Controller.Runtime().State().Machine().Disk() + + // validate input + for _, deviceName := range in.GetUserDisksToWipe() { + disk, ok := disks[deviceName] + if !ok { + return nil, fmt.Errorf("reset user disk failed: device %s wasn't found", deviceName) + } + + if disk.ReadOnly { + return nil, fmt.Errorf("reset user disk failed: device %s is readonly", deviceName) + } + + if systemDisk != nil && deviceName == systemDisk.Device().Name() { + return nil, fmt.Errorf("reset user disk failed: device %s is the system disk", deviceName) + } + } + } + + if len(in.GetSystemPartitionsToWipe()) > 0 { + if in.Mode == machine.ResetRequest_USER_DISKS { + return nil, errors.New("reset failed: invalid input, wipe mode USER_DISKS doesn't support SystemPartitionsToWipe parameter") + } + + bd := s.Controller.Runtime().State().Machine().Disk().BlockDevice + + var pt *gpt.GPT + + pt, err = bd.PartitionTable() + if err != nil { + return nil, fmt.Errorf("error reading partition table: %w", err) + } + + for _, spec := range in.GetSystemPartitionsToWipe() { + target, err := installer.ParseTarget(spec.Label, bd.Device().Name()) + if err != nil { + return nil, err + } + + _, err = target.Locate(pt) + if err != nil { + return nil, fmt.Errorf("failed location partition with label %q: %w", spec.Label, err) + } + + if spec.Wipe { + opts.systemDiskTargets = append(opts.systemDiskTargets, target) + } + } + } + + resetCtx := context.WithValue(context.Background(), runtime.ActorIDCtxKey{}, actorID) + + go func() { + if err := s.Controller.Run(resetCtx, runtime.SequenceReset, &opts); err != nil { + if !runtime.IsRebootError(err) { + log.Println("reset failed:", err) + } + } + }() + + reply = &machine.ResetResponse{ + Messages: []*machine.Reset{ + { + ActorId: actorID, + }, + }, + } + + return reply, nil +} + +// ServiceList returns list of the registered services and their status. +func (s *Server) ServiceList(ctx context.Context, in *emptypb.Empty) (result *machine.ServiceListResponse, err error) { + services := system.Services(s.Controller.Runtime()).List() + + result = &machine.ServiceListResponse{ + Messages: []*machine.ServiceList{ + { + Services: xslices.Map(services, (*system.ServiceRunner).AsProto), + }, + }, + } + + return result, nil +} + +// ServiceStart implements the machine.MachineServer interface and starts a +// service running on Talos. +func (s *Server) ServiceStart(ctx context.Context, in *machine.ServiceStartRequest) (reply *machine.ServiceStartResponse, err error) { + if err = system.Services(s.Controller.Runtime()).APIStart(ctx, in.Id); err != nil { + return &machine.ServiceStartResponse{}, err + } + + reply = &machine.ServiceStartResponse{ + Messages: []*machine.ServiceStart{ + { + Resp: fmt.Sprintf("Service %q started", in.Id), + }, + }, + } + + return reply, err +} + +// ServiceStop implements the machine.MachineServer interface and stops a +// service running on Talos. +func (s *Server) ServiceStop(ctx context.Context, in *machine.ServiceStopRequest) (reply *machine.ServiceStopResponse, err error) { + if err = system.Services(s.Controller.Runtime()).APIStop(ctx, in.Id); err != nil { + return &machine.ServiceStopResponse{}, err + } + + reply = &machine.ServiceStopResponse{ + Messages: []*machine.ServiceStop{ + { + Resp: fmt.Sprintf("Service %q stopped", in.Id), + }, + }, + } + + return reply, err +} + +// ServiceRestart implements the machine.MachineServer interface and stops a +// service running on Talos. +func (s *Server) ServiceRestart(ctx context.Context, in *machine.ServiceRestartRequest) (reply *machine.ServiceRestartResponse, err error) { + if err = system.Services(s.Controller.Runtime()).APIRestart(ctx, in.Id); err != nil { + return &machine.ServiceRestartResponse{}, err + } + + reply = &machine.ServiceRestartResponse{ + Messages: []*machine.ServiceRestart{ + { + Resp: fmt.Sprintf("Service %q restarted", in.Id), + }, + }, + } + + return reply, err +} + +// Copy implements the machine.MachineServer interface and copies data out of Talos node. +func (s *Server) Copy(req *machine.CopyRequest, obj machine.MachineService_CopyServer) error { + path := req.RootPath + path = filepath.Clean(path) + + if !filepath.IsAbs(path) { + return fmt.Errorf("path is not absolute %v", path) + } + + pr, pw := io.Pipe() + + errCh := make(chan error, 1) + + ctx, ctxCancel := context.WithCancel(obj.Context()) + defer ctxCancel() + + go func() { + //nolint:errcheck + defer pw.Close() + errCh <- archiver.TarGz(ctx, path, pw) + }() + + chunker := stream.NewChunker(ctx, pr) + chunkCh := chunker.Read() + + for data := range chunkCh { + err := obj.SendMsg(&common.Data{Bytes: data}) + if err != nil { + ctxCancel() + } + } + + archiveErr := <-errCh + if archiveErr != nil { + return obj.SendMsg(&common.Data{ + Metadata: &common.Metadata{ + Error: archiveErr.Error(), + }, + }) + } + + return nil +} + +// List implements the machine.MachineServer interface. +// +//nolint:gocyclo +func (s *Server) List(req *machine.ListRequest, obj machine.MachineService_ListServer) error { + if req == nil { + req = new(machine.ListRequest) + } + + if !strings.HasPrefix(req.Root, OSPathSeparator) { + // Make sure we use complete paths + req.Root = OSPathSeparator + req.Root + } + + req.Root = strings.TrimSuffix(req.Root, OSPathSeparator) + if req.Root == "" { + req.Root = "/" + } + + var recursionDepth int + + if req.Recurse { + if req.RecursionDepth == 0 { + recursionDepth = -1 + } else { + recursionDepth = int(req.RecursionDepth) + } + } + + opts := []archiver.WalkerOption{ + archiver.WithMaxRecurseDepth(recursionDepth), + } + + if len(req.Types) > 0 { + types := make([]archiver.FileType, 0, len(req.Types)) + + for _, t := range req.Types { + switch t { + case machine.ListRequest_REGULAR: + types = append(types, archiver.RegularFileType) + case machine.ListRequest_DIRECTORY: + types = append(types, archiver.DirectoryFileType) + case machine.ListRequest_SYMLINK: + types = append(types, archiver.SymlinkFileType) + } + } + + opts = append(opts, archiver.WithFileTypes(types...)) + } + + files, err := archiver.Walker(obj.Context(), req.Root, opts...) + if err != nil { + return err + } + + for fi := range files { + if fi.Error != nil { + err = obj.Send(&machine.FileInfo{ + Name: fi.FullPath, + RelativeName: fi.RelPath, + Error: fi.Error.Error(), + }) + } else { + err = obj.Send(&machine.FileInfo{ + Name: fi.FullPath, + RelativeName: fi.RelPath, + Size: fi.FileInfo.Size(), + Mode: uint32(fi.FileInfo.Mode()), + Modified: fi.FileInfo.ModTime().Unix(), + IsDir: fi.FileInfo.IsDir(), + Link: fi.Link, + Uid: fi.FileInfo.Sys().(*syscall.Stat_t).Uid, + Gid: fi.FileInfo.Sys().(*syscall.Stat_t).Gid, + }) + } + + if err != nil { + return err + } + } + + return nil +} + +// DiskUsage implements the machine.MachineServer interface. +// +//nolint:cyclop +func (s *Server) DiskUsage(req *machine.DiskUsageRequest, obj machine.MachineService_DiskUsageServer) error { //nolint:gocyclo + if req == nil { + req = new(machine.DiskUsageRequest) + } + + for _, path := range req.Paths { + if !strings.HasPrefix(path, OSPathSeparator) { + // Make sure we use complete paths + path = OSPathSeparator + path + } + + path = strings.TrimSuffix(path, OSPathSeparator) + if path == "" { + path = "/" + } + + _, err := os.Stat(path) + if err == os.ErrNotExist { + err = obj.Send( + &machine.DiskUsageInfo{ + Name: path, + RelativeName: path, + Error: err.Error(), + }, + ) + if err != nil { + return err + } + + continue + } + + files, err := archiver.Walker(obj.Context(), path, archiver.WithMaxRecurseDepth(-1)) + if err != nil { + err = obj.Send( + &machine.DiskUsageInfo{ + Name: path, + RelativeName: path, + Error: err.Error(), + }, + ) + if err != nil { + return err + } + + continue + } + + folders := map[string]*machine.DiskUsageInfo{} + + // send a record back to client if the message shouldn't be skipped + // at the same time use record information for folder size estimation + sendSize := func(info *machine.DiskUsageInfo, depth int32, isDir bool) error { + prefix := strings.TrimRight(filepath.Dir(info.Name), "/") + if folder, ok := folders[prefix]; ok { + folder.Size += info.Size + } + + // recursion depth check + skip := depth >= req.RecursionDepth && req.RecursionDepth > 0 + // skip files check + skip = skip || !isDir && !req.All + // threshold check + skip = skip || req.Threshold > 0 && info.Size < req.Threshold + skip = skip || req.Threshold < 0 && info.Size > -req.Threshold + + if skip { + return nil + } + + return obj.Send(info) + } + + var ( + depth int32 + prefix = path + rootDepth = int32(strings.Count(path, archiver.OSPathSeparator)) + ) + + // flush all folder sizes until we get to the common prefix + flushFolders := func(prefix, nextPrefix string) error { + for !strings.HasPrefix(nextPrefix, prefix) { + currentDepth := int32(strings.Count(prefix, archiver.OSPathSeparator)) - rootDepth + + if folder, ok := folders[prefix]; ok { + err = sendSize(folder, currentDepth, true) + if err != nil { + return err + } + + delete(folders, prefix) + } + + prefix = strings.TrimRight(filepath.Dir(prefix), "/") + } + + return nil + } + + for fi := range files { + if fi.Error != nil { + err = obj.Send( + &machine.DiskUsageInfo{ + Name: fi.FullPath, + RelativeName: fi.RelPath, + Error: fi.Error.Error(), + }, + ) + } else { + currentDepth := int32(strings.Count(fi.FullPath, archiver.OSPathSeparator)) - rootDepth + + size := fi.FileInfo.Size() + if size < 0 { + size = 0 + } + + // kcore file size gives wrong value, this code should be smarter when it reads it + // TODO: figure out better way to skip such file + if fi.FullPath == "/proc/kcore" { + size = 0 + } + + if fi.FileInfo.IsDir() { + folders[strings.TrimRight(fi.FullPath, "/")] = &machine.DiskUsageInfo{ + Name: fi.FullPath, + RelativeName: fi.RelPath, + Size: size, + } + } else { + err = sendSize(&machine.DiskUsageInfo{ + Name: fi.FullPath, + RelativeName: fi.RelPath, + Size: size, + }, currentDepth, false) + if err != nil { + return err + } + } + + // depth goes down when walker gets to the next sibling folder + if currentDepth < depth { + nextPrefix := fi.FullPath + + if err = flushFolders(prefix, nextPrefix); err != nil { + return err + } + + prefix = nextPrefix + } + + if fi.FileInfo.IsDir() { + prefix = fi.FullPath + } + + depth = currentDepth + } + } + + if path != "" { + p := strings.TrimRight(path, "/") + if folder, ok := folders[p]; ok { + err = flushFolders(prefix, p) + if err != nil { + return err + } + + err = sendSize(folder, 0, true) + if err != nil { + return err + } + } + } + + return nil + } + + return nil +} + +// Mounts implements the machine.MachineServer interface. +func (s *Server) Mounts(ctx context.Context, in *emptypb.Empty) (reply *machine.MountsResponse, err error) { + file, err := os.Open("/proc/mounts") + if err != nil { + return nil, err + } + //nolint:errcheck + defer file.Close() + + var ( + stat unix.Statfs_t + multiErr *multierror.Error + ) + + stats := []*machine.MountStat{} + scanner := bufio.NewScanner(file) + + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + + if len(fields) < 2 { + continue + } + + filesystem := fields[0] + mountpoint := fields[1] + + var ( + totalSize uint64 + totalAvail uint64 + ) + + if statInfo, err := os.Stat(mountpoint); err == nil && statInfo.Mode().IsDir() { + if err := unix.Statfs(mountpoint, &stat); err != nil { + multiErr = multierror.Append(multiErr, err) + } else { + totalSize = uint64(stat.Bsize) * stat.Blocks + totalAvail = uint64(stat.Bsize) * stat.Bavail + } + } + + stat := &machine.MountStat{ + Filesystem: filesystem, + Size: totalSize, + Available: totalAvail, + MountedOn: mountpoint, + } + + stats = append(stats, stat) + } + + if err := scanner.Err(); err != nil { + multiErr = multierror.Append(multiErr, err) + } + + reply = &machine.MountsResponse{ + Messages: []*machine.Mounts{ + { + Stats: stats, + }, + }, + } + + return reply, multiErr.ErrorOrNil() +} + +// Version implements the machine.MachineServer interface. +func (s *Server) Version(ctx context.Context, in *emptypb.Empty) (reply *machine.VersionResponse, err error) { + var platform *machine.PlatformInfo + + if s.Controller.Runtime().State().Platform() != nil { + platform = &machine.PlatformInfo{ + Name: s.Controller.Runtime().State().Platform().Name(), + Mode: s.Controller.Runtime().State().Platform().Mode().String(), + } + } + + var features *machine.FeaturesInfo + + config := s.Controller.Runtime().Config() + if config != nil && config.Machine() != nil { + features = &machine.FeaturesInfo{ + Rbac: config.Machine().Features().RBACEnabled(), + } + } + + return &machine.VersionResponse{ + Messages: []*machine.Version{ + { + Version: version.NewVersion(), + Platform: platform, + Features: features, + }, + }, + }, nil +} + +// Kubeconfig implements the machine.MachineServer interface. +func (s *Server) Kubeconfig(empty *emptypb.Empty, obj machine.MachineService_KubeconfigServer) error { + if err := s.checkControlplane("kubeconfig"); err != nil { + return err + } + + var b bytes.Buffer + + if err := kubeconfig.GenerateAdmin(s.Controller.Runtime().Config().Cluster(), &b); err != nil { + return err + } + + // wrap in .tar.gz to match Copy protocol + var buf bytes.Buffer + + zw := gzip.NewWriter(&buf) + + tarW := tar.NewWriter(zw) + + err := tarW.WriteHeader(&tar.Header{ + Typeflag: tar.TypeReg, + Name: "kubeconfig", + Size: int64(b.Len()), + ModTime: time.Now(), + Mode: 0o600, + }) + if err != nil { + return err + } + + _, err = io.Copy(tarW, &b) + if err != nil { + return err + } + + if err = zw.Close(); err != nil { + return err + } + + return obj.Send(&common.Data{ + Bytes: buf.Bytes(), + }) +} + +// Logs provides a service or container logs can be requested and the contents of the +// log file are streamed in chunks. +func (s *Server) Logs(req *machine.LogsRequest, l machine.MachineService_LogsServer) (err error) { + var chunk chunker.Chunker + + switch { + case req.Namespace == constants.SystemContainerdNamespace || req.Id == "kubelet": + var options []runtime.LogOption + + if req.Follow { + options = append(options, runtime.WithFollow()) + } + + if req.TailLines >= 0 { + options = append(options, runtime.WithTailLines(int(req.TailLines))) + } + + var logR io.ReadCloser + + logR, err = s.Controller.Runtime().Logging().ServiceLog(req.Id).Reader(options...) + if err != nil { + return + } + + //nolint:errcheck + defer logR.Close() + + chunk = stream.NewChunker(l.Context(), logR) + default: + var file io.Closer + + if chunk, file, err = k8slogs(l.Context(), req); err != nil { + return err + } + //nolint:errcheck + defer file.Close() + } + + for data := range chunk.Read() { + if err = l.Send(&common.Data{Bytes: data}); err != nil { + return + } + } + + return nil +} + +// LogsContainers provide a list of registered log containers. +func (s *Server) LogsContainers(context.Context, *emptypb.Empty) (*machine.LogsContainersResponse, error) { + return &machine.LogsContainersResponse{ + Messages: []*machine.LogsContainer{ + { + Ids: s.Controller.Runtime().Logging().RegisteredLogs(), + }, + }, + }, nil +} + +func k8slogs(ctx context.Context, req *machine.LogsRequest) (chunker.Chunker, io.Closer, error) { + inspector, err := getContainerInspector(ctx, req.Namespace, req.Driver) + if err != nil { + return nil, nil, err + } + //nolint:errcheck + defer inspector.Close() + + container, err := inspector.Container(req.Id) + if err != nil { + return nil, nil, err + } + + if container == nil { + return nil, nil, fmt.Errorf("container %q not found", req.Id) + } + + return container.GetLogChunker(ctx, req.Follow, int(req.TailLines)) +} + +func getContainerInspector(ctx context.Context, namespace string, driver common.ContainerDriver) (containers.Inspector, error) { + switch driver { + case common.ContainerDriver_CRI: + if namespace != criconstants.K8sContainerdNamespace { + return nil, errors.New("CRI inspector is supported only for K8s namespace") + } + + return cri.NewInspector(ctx) + case common.ContainerDriver_CONTAINERD: + addr := constants.CRIContainerdAddress + if namespace == constants.SystemContainerdNamespace { + addr = constants.SystemContainerdAddress + } + + return taloscontainerd.NewInspector(ctx, namespace, taloscontainerd.WithContainerdAddress(addr)) + default: + return nil, fmt.Errorf("unsupported driver %q", driver) + } +} + +// Read implements the read API. +func (s *Server) Read(in *machine.ReadRequest, srv machine.MachineService_ReadServer) (err error) { + stat, err := os.Stat(in.Path) + if err != nil { + return err + } + + switch mode := stat.Mode(); { + case mode.IsRegular(): + f, err := os.OpenFile(in.Path, os.O_RDONLY, 0) + if err != nil { + return err + } + + defer f.Close() //nolint:errcheck + + ctx, cancel := context.WithCancel(srv.Context()) + defer cancel() + + chunker := stream.NewChunker(ctx, f) + chunkCh := chunker.Read() + + for data := range chunkCh { + err := srv.SendMsg(&common.Data{Bytes: data}) + if err != nil { + cancel() + } + } + + return nil + default: + return errors.New("path must be a regular file") + } +} + +// Events streams runtime events. +// +//nolint:gocyclo +func (s *Server) Events(req *machine.EventsRequest, l machine.MachineService_EventsServer) error { + // send an empty (hello) event to indicate to client that streaming has started + err := sendEmptyEvent(req, l) + if err != nil { + return err + } + + errCh := make(chan error) + + var opts []runtime.WatchOptionFunc + + if req.TailEvents != 0 { + opts = append(opts, runtime.WithTailEvents(int(req.TailEvents))) + } + + if req.TailId != "" { + tailID, err := xid.FromString(req.TailId) + if err != nil { + return fmt.Errorf("error parsing tail_id: %w", err) + } + + opts = append(opts, runtime.WithTailID(tailID)) + } + + if req.TailSeconds != 0 { + opts = append(opts, runtime.WithTailDuration(time.Duration(req.TailSeconds)*time.Second)) + } + + if req.WithActorId != "" { + opts = append(opts, runtime.WithActorID(req.WithActorId)) + } + + if err := s.Controller.Runtime().Events().Watch(func(events <-chan runtime.EventInfo) { + errCh <- func() error { + for { + select { + case <-s.ShutdownCtx.Done(): + return nil + case <-l.Context().Done(): + return l.Context().Err() + case event, ok := <-events: + if !ok { + return nil + } + + msg, err := event.ToMachineEvent() + if err != nil { + return err + } + + if err = l.Send(msg); err != nil { + return err + } + } + } + }() + }, opts...); err != nil { + return err + } + + return <-errCh +} + +func sendEmptyEvent(req *machine.EventsRequest, l machine.MachineService_EventsServer) error { + emptyEvent, err := pointer.To(runtime.NewEvent(nil, req.WithActorId)).ToMachineEvent() + if err != nil { + return err + } + + return l.Send(emptyEvent) +} + +// Containers implements the machine.MachineServer interface. +func (s *Server) Containers(ctx context.Context, in *machine.ContainersRequest) (reply *machine.ContainersResponse, err error) { + inspector, err := getContainerInspector(ctx, in.Namespace, in.Driver) + if err != nil { + return nil, err + } + //nolint:errcheck + defer inspector.Close() + + pods, err := inspector.Pods() + if err != nil { + // fatal error + if pods == nil { + return nil, err + } + // TODO: only some failed, need to handle it better via client + log.Println(err.Error()) + } + + containers := []*machine.ContainerInfo{} + + for _, pod := range pods { + for _, container := range pod.Containers { + container := &machine.ContainerInfo{ + Namespace: in.Namespace, + Id: container.Display, + PodId: pod.Name, + Name: container.Name, + Image: container.Image, + Pid: container.Pid, + Status: container.Status, + NetworkNamespace: container.NetworkNamespace, + } + containers = append(containers, container) + } + } + + reply = &machine.ContainersResponse{ + Messages: []*machine.Container{ + { + Containers: containers, + }, + }, + } + + return reply, nil +} + +// Stats implements the machine.MachineServer interface. +func (s *Server) Stats(ctx context.Context, in *machine.StatsRequest) (reply *machine.StatsResponse, err error) { + inspector, err := getContainerInspector(ctx, in.Namespace, in.Driver) + if err != nil { + return nil, err + } + //nolint:errcheck + defer inspector.Close() + + pods, err := inspector.Pods() + if err != nil { + // fatal error + if pods == nil { + return nil, err + } + // TODO: only some failed, need to handle it better via client + log.Println(err.Error()) + } + + stats := []*machine.Stat{} + + for _, pod := range pods { + for _, container := range pod.Containers { + if container.Metrics == nil { + continue + } + + stat := &machine.Stat{ + Namespace: in.Namespace, + Id: container.Display, + PodId: pod.Name, + Name: container.Name, + MemoryUsage: container.Metrics.MemoryUsage, + CpuUsage: container.Metrics.CPUUsage, + } + + stats = append(stats, stat) + } + } + + reply = &machine.StatsResponse{ + Messages: []*machine.Stats{ + { + Stats: stats, + }, + }, + } + + return reply, nil +} + +// Restart implements the machine.MachineServer interface. +func (s *Server) Restart(ctx context.Context, in *machine.RestartRequest) (*machine.RestartResponse, error) { + inspector, err := getContainerInspector(ctx, in.Namespace, in.Driver) + if err != nil { + return nil, err + } + //nolint:errcheck + defer inspector.Close() + + container, err := inspector.Container(in.Id) + if err != nil { + return nil, err + } + + if container == nil { + return nil, fmt.Errorf("container %q not found", in.Id) + } + + err = container.Kill(syscall.SIGTERM) + if err != nil { + return nil, err + } + + return &machine.RestartResponse{ + Messages: []*machine.Restart{ + {}, + }, + }, nil +} + +// Dmesg implements the machine.MachineServer interface. +// +//nolint:gocyclo +func (s *Server) Dmesg(req *machine.DmesgRequest, srv machine.MachineService_DmesgServer) error { + ctx := srv.Context() + + var options []kmsg.Option + + if req.Follow { + options = append(options, kmsg.Follow()) + } + + if req.Tail { + options = append(options, kmsg.FromTail()) + } + + reader, err := kmsg.NewReader(options...) + if err != nil { + return fmt.Errorf("error opening /dev/kmsg reader: %w", err) + } + defer reader.Close() //nolint:errcheck + + ch := reader.Scan(ctx) + + for { + select { + case <-s.ShutdownCtx.Done(): + if err = reader.Close(); err != nil { + return err + } + case <-ctx.Done(): + if err = reader.Close(); err != nil { + return err + } + case packet, ok := <-ch: + if !ok { + return nil + } + + if packet.Err != nil { + err = srv.Send(&common.Data{ + Metadata: &common.Metadata{ + Error: packet.Err.Error(), + }, + }) + } else { + msg := packet.Message + err = srv.Send(&common.Data{ + Bytes: []byte(fmt.Sprintf("%s: %7s: [%s]: %s", msg.Facility, msg.Priority, msg.Timestamp.Format(time.RFC3339Nano), msg.Message)), + }) + } + + if err != nil { + return err + } + } + } +} + +// Processes implements the machine.MachineServer interface. +func (s *Server) Processes(ctx context.Context, in *emptypb.Empty) (reply *machine.ProcessesResponse, err error) { + var processes []*machine.ProcessInfo + + procs, err := miniprocfs.NewProcesses() + if err != nil { + return nil, err + } + + for { + info, err := procs.Next() + if err != nil { + return nil, err + } + + if info == nil { + break + } + + processes = append(processes, info) + } + + reply = &machine.ProcessesResponse{ + Messages: []*machine.Process{ + { + Processes: processes, + }, + }, + } + + return reply, nil +} + +// Memory implements the machine.MachineServer interface. +func (s *Server) Memory(ctx context.Context, in *emptypb.Empty) (reply *machine.MemoryResponse, err error) { + proc, err := procfs.NewDefaultFS() + if err != nil { + return nil, err + } + + info, err := proc.Meminfo() + if err != nil { + return nil, err + } + + meminfo := &machine.MemInfo{ + Memtotal: pointer.SafeDeref(info.MemTotal), + Memfree: pointer.SafeDeref(info.MemFree), + Memavailable: pointer.SafeDeref(info.MemAvailable), + Buffers: pointer.SafeDeref(info.Buffers), + Cached: pointer.SafeDeref(info.Cached), + Swapcached: pointer.SafeDeref(info.SwapCached), + Active: pointer.SafeDeref(info.Active), + Inactive: pointer.SafeDeref(info.Inactive), + Activeanon: pointer.SafeDeref(info.ActiveAnon), + Inactiveanon: pointer.SafeDeref(info.InactiveAnon), + Activefile: pointer.SafeDeref(info.ActiveFile), + Inactivefile: pointer.SafeDeref(info.InactiveFile), + Unevictable: pointer.SafeDeref(info.Unevictable), + Mlocked: pointer.SafeDeref(info.Mlocked), + Swaptotal: pointer.SafeDeref(info.SwapTotal), + Swapfree: pointer.SafeDeref(info.SwapFree), + Dirty: pointer.SafeDeref(info.Dirty), + Writeback: pointer.SafeDeref(info.Writeback), + Anonpages: pointer.SafeDeref(info.AnonPages), + Mapped: pointer.SafeDeref(info.Mapped), + Shmem: pointer.SafeDeref(info.Shmem), + Slab: pointer.SafeDeref(info.Slab), + Sreclaimable: pointer.SafeDeref(info.SReclaimable), + Sunreclaim: pointer.SafeDeref(info.SUnreclaim), + Kernelstack: pointer.SafeDeref(info.KernelStack), + Pagetables: pointer.SafeDeref(info.PageTables), + Nfsunstable: pointer.SafeDeref(info.NFSUnstable), + Bounce: pointer.SafeDeref(info.Bounce), + Writebacktmp: pointer.SafeDeref(info.WritebackTmp), + Commitlimit: pointer.SafeDeref(info.CommitLimit), + Committedas: pointer.SafeDeref(info.CommittedAS), + Vmalloctotal: pointer.SafeDeref(info.VmallocTotal), + Vmallocused: pointer.SafeDeref(info.VmallocUsed), + Vmallocchunk: pointer.SafeDeref(info.VmallocChunk), + Hardwarecorrupted: pointer.SafeDeref(info.HardwareCorrupted), + Anonhugepages: pointer.SafeDeref(info.AnonHugePages), + Shmemhugepages: pointer.SafeDeref(info.ShmemHugePages), + Shmempmdmapped: pointer.SafeDeref(info.ShmemPmdMapped), + Cmatotal: pointer.SafeDeref(info.CmaTotal), + Cmafree: pointer.SafeDeref(info.CmaFree), + Hugepagestotal: pointer.SafeDeref(info.HugePagesTotal), + Hugepagesfree: pointer.SafeDeref(info.HugePagesFree), + Hugepagesrsvd: pointer.SafeDeref(info.HugePagesRsvd), + Hugepagessurp: pointer.SafeDeref(info.HugePagesSurp), + Hugepagesize: pointer.SafeDeref(info.Hugepagesize), + Directmap4K: pointer.SafeDeref(info.DirectMap4k), + Directmap2M: pointer.SafeDeref(info.DirectMap2M), + Directmap1G: pointer.SafeDeref(info.DirectMap1G), + } + + reply = &machine.MemoryResponse{ + Messages: []*machine.Memory{ + { + Meminfo: meminfo, + }, + }, + } + + return reply, err +} + +// EtcdMemberList implements the machine.MachineServer interface. +func (s *Server) EtcdMemberList(ctx context.Context, in *machine.EtcdMemberListRequest) (*machine.EtcdMemberListResponse, error) { + if err := s.checkControlplane("member list"); err != nil { + return nil, err + } + + var ( + client *etcd.Client + err error + ) + + if in.QueryLocal { + client, err = etcd.NewLocalClient(ctx) + } else { + client, err = etcd.NewClientFromControlPlaneIPs(ctx, s.Controller.Runtime().State().V1Alpha2().Resources()) + } + + if err != nil { + return nil, err + } + + //nolint:errcheck + defer client.Close() + + ctx = clientv3.WithRequireLeader(ctx) + + resp, err := client.MemberList(ctx) + if err != nil { + return nil, err + } + + return &machine.EtcdMemberListResponse{ + Messages: []*machine.EtcdMembers{ + { + LegacyMembers: xslices.Map(resp.Members, (*etcdserverpb.Member).GetName), + Members: xslices.Map(resp.Members, func(member *etcdserverpb.Member) *machine.EtcdMember { + return &machine.EtcdMember{ + Id: member.GetID(), + Hostname: member.GetName(), + PeerUrls: member.GetPeerURLs(), + ClientUrls: member.GetClientURLs(), + IsLearner: member.GetIsLearner(), + } + }), + }, + }, + }, nil +} + +// EtcdRemoveMemberByID implements the machine.MachineServer interface. +func (s *Server) EtcdRemoveMemberByID(ctx context.Context, in *machine.EtcdRemoveMemberByIDRequest) (*machine.EtcdRemoveMemberByIDResponse, error) { + if err := s.checkControlplane("etcd remove member"); err != nil { + return nil, err + } + + client, err := etcd.NewClientFromControlPlaneIPs(ctx, s.Controller.Runtime().State().V1Alpha2().Resources()) + if err != nil { + return nil, fmt.Errorf("failed to create etcd client: %w", err) + } + + defer client.Close() //nolint:errcheck + + ctx = clientv3.WithRequireLeader(ctx) + + if err = client.RemoveMemberByMemberID(ctx, in.MemberId); err != nil { + if errors.Is(err, rpctypes.ErrMemberNotFound) { + return nil, status.Errorf(codes.NotFound, err.Error()) + } + + return nil, fmt.Errorf("failed to remove member: %w", err) + } + + return &machine.EtcdRemoveMemberByIDResponse{ + Messages: []*machine.EtcdRemoveMemberByID{ + {}, + }, + }, nil +} + +// EtcdLeaveCluster implements the machine.MachineServer interface. +func (s *Server) EtcdLeaveCluster(ctx context.Context, in *machine.EtcdLeaveClusterRequest) (*machine.EtcdLeaveClusterResponse, error) { + if err := s.checkControlplane("etcd leave"); err != nil { + return nil, err + } + + client, err := etcd.NewClientFromControlPlaneIPs(ctx, s.Controller.Runtime().State().V1Alpha2().Resources()) + if err != nil { + return nil, fmt.Errorf("failed to create etcd client: %w", err) + } + + defer client.Close() //nolint:errcheck + + ctx = clientv3.WithRequireLeader(ctx) + + if err = client.LeaveCluster(ctx, s.Controller.Runtime().State().V1Alpha2().Resources()); err != nil { + return nil, fmt.Errorf("failed to leave cluster: %w", err) + } + + return &machine.EtcdLeaveClusterResponse{ + Messages: []*machine.EtcdLeaveCluster{ + {}, + }, + }, nil +} + +// EtcdForfeitLeadership implements the machine.MachineServer interface. +func (s *Server) EtcdForfeitLeadership(ctx context.Context, in *machine.EtcdForfeitLeadershipRequest) (*machine.EtcdForfeitLeadershipResponse, error) { + if err := s.checkControlplane("etcd forfeit leadership"); err != nil { + return nil, err + } + + client, err := etcd.NewClientFromControlPlaneIPs(ctx, s.Controller.Runtime().State().V1Alpha2().Resources()) + if err != nil { + return nil, fmt.Errorf("failed to create etcd client: %w", err) + } + + defer client.Close() //nolint:errcheck + + ctx = clientv3.WithRequireLeader(ctx) + + memberID, err := etcd.GetLocalMemberID(ctx, s.Controller.Runtime().State().V1Alpha2().Resources()) + if err != nil { + return nil, err + } + + leader, err := client.ForfeitLeadership(ctx, etcdresource.FormatMemberID(memberID)) + if err != nil { + return nil, fmt.Errorf("failed to forfeit leadership: %w", err) + } + + return &machine.EtcdForfeitLeadershipResponse{ + Messages: []*machine.EtcdForfeitLeadership{ + { + Member: leader, + }, + }, + }, nil +} + +// EtcdSnapshot implements the machine.MachineServer interface. +func (s *Server) EtcdSnapshot(in *machine.EtcdSnapshotRequest, srv machine.MachineService_EtcdSnapshotServer) error { + if err := s.checkControlplane("etcd snapshot"); err != nil { + return err + } + + ctx, cancel := context.WithCancel(srv.Context()) + defer cancel() + + client, err := etcd.NewLocalClient(ctx) + if err != nil { + return fmt.Errorf("failed to create etcd client: %w", err) + } + + //nolint:errcheck + defer client.Close() + + rd, err := client.Snapshot(srv.Context()) + if err != nil { + return fmt.Errorf("failed reading etcd snapshot: %w", err) + } + + chunker := stream.NewChunker(ctx, rd) + chunkCh := chunker.Read() + + for data := range chunkCh { + err := srv.SendMsg(&common.Data{Bytes: data}) + if err != nil { + cancel() + + return err + } + } + + return nil +} + +// EtcdRecover implements the machine.MachineServer interface. +// +//nolint:gocyclo +func (s *Server) EtcdRecover(srv machine.MachineService_EtcdRecoverServer) error { + if _, err := os.Stat(filepath.Dir(constants.EtcdRecoverySnapshotPath)); err != nil { + if os.IsNotExist(err) { + return status.Error(codes.FailedPrecondition, "etcd service is not ready for recovery yet") + } + + return err + } + + if err := s.checkControlplane("etcd recover"); err != nil { + return err + } + + snapshot, err := os.OpenFile(constants.EtcdRecoverySnapshotPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o700) + if err != nil { + return fmt.Errorf("error creating etcd recovery snapshot: %w", err) + } + + defer snapshot.Close() //nolint:errcheck + + successfulUpload := false + + defer func() { + if !successfulUpload { + os.Remove(snapshot.Name()) //nolint:errcheck + } + }() + + for { + var msg *common.Data + + msg, err = srv.Recv() + if err != nil { + if err == io.EOF { + break + } + + return err + } + + _, err = snapshot.Write(msg.Bytes) + if err != nil { + return fmt.Errorf("error writing snapshot: %w", err) + } + } + + if err = snapshot.Sync(); err != nil { + return fmt.Errorf("error fsyncing snapshot: %w", err) + } + + if err = snapshot.Close(); err != nil { + return fmt.Errorf("error closing snapshot: %w", err) + } + + successfulUpload = true + + return srv.SendAndClose(&machine.EtcdRecoverResponse{ + Messages: []*machine.EtcdRecover{ + {}, + }, + }) +} + +func mapAlarms(alarms []*etcdserverpb.AlarmMember) []*machine.EtcdMemberAlarm { + mapAlarmType := func(alarmType etcdserverpb.AlarmType) machine.EtcdMemberAlarm_AlarmType { + switch alarmType { + case etcdserverpb.AlarmType_NOSPACE: + return machine.EtcdMemberAlarm_NOSPACE + case etcdserverpb.AlarmType_CORRUPT: + return machine.EtcdMemberAlarm_CORRUPT + case etcdserverpb.AlarmType_NONE: + return machine.EtcdMemberAlarm_NONE + default: + return machine.EtcdMemberAlarm_NONE + } + } + + return xslices.Map(alarms, func(alarm *etcdserverpb.AlarmMember) *machine.EtcdMemberAlarm { + return &machine.EtcdMemberAlarm{ + MemberId: alarm.MemberID, + Alarm: mapAlarmType(alarm.Alarm), + } + }) +} + +// EtcdAlarmList lists etcd alarms for the current node. +// +// This method is available only on control plane nodes (which run etcd). +func (s *Server) EtcdAlarmList(ctx context.Context, in *emptypb.Empty) (*machine.EtcdAlarmListResponse, error) { + if err := s.checkControlplane("etcd alarm list"); err != nil { + return nil, err + } + + client, err := etcd.NewLocalClient(ctx) + if err != nil { + return nil, fmt.Errorf("failed to create etcd client: %w", err) + } + + //nolint:errcheck + defer client.Close() + + resp, err := client.AlarmList(ctx) + if err != nil { + return nil, fmt.Errorf("failed to list etcd alarms: %w", err) + } + + return &machine.EtcdAlarmListResponse{ + Messages: []*machine.EtcdAlarm{ + { + MemberAlarms: mapAlarms(resp.Alarms), + }, + }, + }, nil +} + +// EtcdAlarmDisarm disarms etcd alarms for the current node. +// +// This method is available only on control plane nodes (which run etcd). +func (s *Server) EtcdAlarmDisarm(ctx context.Context, in *emptypb.Empty) (*machine.EtcdAlarmDisarmResponse, error) { + if err := s.checkControlplane("etcd alarm list"); err != nil { + return nil, err + } + + client, err := etcd.NewLocalClient(ctx) + if err != nil { + return nil, fmt.Errorf("failed to create etcd client: %w", err) + } + + //nolint:errcheck + defer client.Close() + + resp, err := client.AlarmDisarm(ctx, &clientv3.AlarmMember{}) + if err != nil { + return nil, fmt.Errorf("failed to disarm etcd alarm: %w", err) + } + + return &machine.EtcdAlarmDisarmResponse{ + Messages: []*machine.EtcdAlarmDisarm{ + { + MemberAlarms: mapAlarms(resp.Alarms), + }, + }, + }, nil +} + +// EtcdDefragment defragments etcd data directory for the current node. +// +// Defragmentation is a resource-heavy operation, so it should only run on a specific +// node. +// +// This method is available only on control plane nodes (which run etcd). +func (s *Server) EtcdDefragment(ctx context.Context, in *emptypb.Empty) (*machine.EtcdDefragmentResponse, error) { + if err := s.checkControlplane("etcd defragment"); err != nil { + return nil, err + } + + client, err := etcd.NewLocalClient(ctx) + if err != nil { + return nil, fmt.Errorf("failed to create etcd client: %w", err) + } + + //nolint:errcheck + defer client.Close() + + _, err = client.Defragment(ctx, nethelpers.JoinHostPort("localhost", constants.EtcdClientPort)) + if err != nil { + return nil, fmt.Errorf("failed to defragment etcd: %w", err) + } + + return &machine.EtcdDefragmentResponse{ + Messages: []*machine.EtcdDefragment{ + {}, + }, + }, nil +} + +// EtcdStatus returns etcd status for the member of the cluster. +// +// This method is available only on control plane nodes (which run etcd). +func (s *Server) EtcdStatus(ctx context.Context, in *emptypb.Empty) (*machine.EtcdStatusResponse, error) { + if err := s.checkControlplane("etcd status"); err != nil { + return nil, err + } + + client, err := etcd.NewLocalClient(ctx) + if err != nil { + return nil, fmt.Errorf("failed to create etcd client: %w", err) + } + + //nolint:errcheck + defer client.Close() + + resp, err := client.Status(ctx, nethelpers.JoinHostPort("localhost", constants.EtcdClientPort)) + if err != nil { + return nil, fmt.Errorf("failed to query etcd status: %w", err) + } + + return &machine.EtcdStatusResponse{ + Messages: []*machine.EtcdStatus{ + { + MemberStatus: &machine.EtcdMemberStatus{ + MemberId: resp.Header.MemberId, + ProtocolVersion: resp.Version, + DbSize: resp.DbSize, + DbSizeInUse: resp.DbSizeInUse, + Leader: resp.Leader, + RaftIndex: resp.RaftIndex, + RaftTerm: resp.RaftTerm, + RaftAppliedIndex: resp.RaftAppliedIndex, + Errors: resp.Errors, + IsLearner: resp.IsLearner, + }, + }, + }, + }, nil +} + +// GenerateClientConfiguration implements the machine.MachineServer interface. +func (s *Server) GenerateClientConfiguration(ctx context.Context, in *machine.GenerateClientConfigurationRequest) (*machine.GenerateClientConfigurationResponse, error) { + if s.Controller.Runtime().Config().Machine().Type() == machinetype.TypeWorker { + return nil, status.Error(codes.FailedPrecondition, "client configuration (talosconfig) can't be generated on worker nodes") + } + + crtTTL := in.CrtTtl.AsDuration() + if crtTTL <= 0 { + return nil, status.Error(codes.InvalidArgument, "crt_ttl should be positive") + } + + roles, _ := role.Parse(in.Roles) + + secretsBundle := secrets.NewBundleFromConfig(secrets.NewFixedClock(time.Now()), s.Controller.Runtime().Config()) + + cert, err := secretsBundle.GenerateTalosAPIClientCertificateWithTTL(roles, crtTTL) + if err != nil { + return nil, err + } + + // make a nice context name + contextName := s.Controller.Runtime().Config().Cluster().Name() + if r := roles.Strings(); len(r) == 1 { + contextName = strings.TrimPrefix(r[0], role.Prefix) + "@" + contextName + } + + talosconfig := clientconfig.NewConfig(contextName, nil, secretsBundle.Certs.OS.Crt, cert) + + b, err := talosconfig.Bytes() + if err != nil { + return nil, err + } + + reply := &machine.GenerateClientConfigurationResponse{ + Messages: []*machine.GenerateClientConfiguration{ + { + Ca: secretsBundle.Certs.OS.Crt, + Crt: cert.Crt, + Key: cert.Key, + Talosconfig: b, + }, + }, + } + + return reply, nil +} + +type packetStreamWriter struct { + stream machine.MachineService_PacketCaptureServer +} + +func (w *packetStreamWriter) Write(data []byte) (int, error) { + // copy the data as the stream may not send it immediately + data = slices.Clone(data) + + err := w.stream.Send(&common.Data{Bytes: data}) + if err != nil { + return 0, err + } + + return len(data), nil +} + +// PacketCapture performs packet capture and streams the pcap file. +// +//nolint:gocyclo +func (s *Server) PacketCapture(in *machine.PacketCaptureRequest, srv machine.MachineService_PacketCaptureServer) error { + linkInfo, err := safe.StateGetResource(srv.Context(), s.Controller.Runtime().State().V1Alpha2().Resources(), network.NewLinkStatus(network.NamespaceName, in.Interface)) + if err != nil { + if state.IsNotFoundError(err) { + return status.Errorf(codes.NotFound, "interface %q not found", in.Interface) + } + + return err + } + + var linkType pcap.LinkType + + switch linkInfo.TypedSpec().Type { //nolint:exhaustive + case nethelpers.LinkEther, nethelpers.LinkLoopbck: + linkType = pcap.LinkTypeEthernet + case nethelpers.LinkNone: + linkType = pcap.LinkTypeRaw + default: + return status.Errorf(codes.InvalidArgument, "unsupported link type %s", linkInfo.TypedSpec().Type) + } + + if in.SnapLen == 0 { + in.SnapLen = afpacket.DefaultFrameSize + } + + filter := make([]bpf.RawInstruction, 0, len(in.BpfFilter)) + + for _, f := range in.BpfFilter { + filter = append(filter, bpf.RawInstruction{ + Op: uint16(f.Op), + Jt: uint8(f.Jt), + Jf: uint8(f.Jf), + K: f.K, + }) + } + + handle, err := afpacket.NewTPacket( + afpacket.OptInterface(in.Interface), + afpacket.OptPollTimeout(100*time.Millisecond), + ) + if err != nil { + return fmt.Errorf("error creating afpacket handle: %w", err) + } + + if len(filter) > 0 { + if err = handle.SetBPF(filter); err != nil { + handle.Close() + + return fmt.Errorf("error setting BPF filter: %w", err) + } + } + + if err = handle.SetPromiscuous(in.Promiscuous); err != nil { + handle.Close() + + return fmt.Errorf("error setting promiscuous mode %v: %w", in.Promiscuous, err) + } + + return capturePackets(srv.Context(), &packetStreamWriter{srv}, handle, in.SnapLen, linkType) +} + +//nolint:gocyclo,cyclop +func capturePackets(ctx context.Context, w io.Writer, handle *afpacket.TPacket, snapLen uint32, linkType pcap.LinkType) error { + defer handle.Close() + + pcapw := pcap.NewWriter(w) + + if err := pcapw.WriteFileHeader(snapLen, linkType); err != nil { + return err + } + + defer func() { + infoMessage := "pcap: " + + stats, errStats := handle.Stats() + if errStats == nil { + infoMessage += fmt.Sprintf("packets captured %d, polls %d", stats.Packets, stats.Polls) + } + + _, socketStatsV3, socketStatsErr := handle.SocketStats() + if socketStatsErr == nil { + infoMessage += fmt.Sprintf(", socket stats: drops %d, packets %d, queue freezes %d", socketStatsV3.Drops(), socketStatsV3.Packets(), socketStatsV3.QueueFreezes()) + } + + log.Print(infoMessage) + }() + + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + data, captureData, err := handle.ZeroCopyReadPacketData() + if err == nil { + if err = pcapw.WritePacket(captureData, data); err != nil { + return err + } + + continue + } + + // Immediately retry for temporary network errors + if nerr, ok := err.(net.Error); ok && nerr.Temporary() { //nolint:staticcheck + continue + } + + // Immediately retry for EAGAIN and poll timeout + if errors.Is(err, syscall.EAGAIN) || errors.Is(err, afpacket.ErrTimeout) { + continue + } + + // Immediately break for known unrecoverable errors + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || + errors.Is(err, io.ErrNoProgress) || errors.Is(err, io.ErrClosedPipe) || errors.Is(err, io.ErrShortBuffer) || + errors.Is(err, syscall.EBADF) || errors.Is(err, afpacket.ErrPoll) || + strings.Contains(err.Error(), "use of closed file") { + return err + } + + time.Sleep(5 * time.Millisecond) // short sleep before retrying some errors + } +} + +func tryLockUpgradeMutex(ctx context.Context, etcdClient *etcd.Client) (unlock func(), err error) { + sess, err := concurrency.NewSession(etcdClient.Client, + concurrency.WithContext(ctx), + concurrency.WithTTL(MinimumEtcdUpgradeLeaseLockSeconds), + ) + if err != nil { + return nil, fmt.Errorf("error establishing etcd concurrency session: %w", err) + } + + mu := concurrency.NewMutex(sess, constants.EtcdTalosEtcdUpgradeMutex) + + if err = mu.TryLock(ctx); err != nil { + return nil, fmt.Errorf("error trying to lock etcd upgrade mutex: %w", err) + } + + log.Printf("etcd upgrade mutex locked with session ID %08x", sess.Lease()) + + return func() { + unlockCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if err := mu.Unlock(unlockCtx); err != nil { + log.Printf("error unlocking etcd upgrade mutex: %v", err) + } + + if err := sess.Close(); err != nil { + log.Printf("error closing etcd upgrade mutex session: %v", err) + } + + log.Printf("etcd upgrade mutex unlocked and session closed") + }, nil +} + +// Netstat implements the machine.MachineServer interface. +func (s *Server) Netstat(ctx context.Context, req *machine.NetstatRequest) (*machine.NetstatResponse, error) { + if req == nil { + req = new(machine.NetstatRequest) + } + + features := netstat.EnableFeatures{ + TCP: req.L4Proto.Tcp, + TCP6: req.L4Proto.Tcp6, + UDP: req.L4Proto.Udp, + UDP6: req.L4Proto.Udp6, + UDPLite: req.L4Proto.Udplite, + UDPLite6: req.L4Proto.Udplite6, + Raw: req.L4Proto.Raw, + Raw6: req.L4Proto.Raw6, + PID: req.Feature.Pid, + NoHostNetwork: !req.Netns.Hostnetwork, + AllNetNs: req.Netns.Allnetns, + NetNsName: req.Netns.Netns, + } + + var fn netstat.AcceptFn + + switch req.Filter { + case machine.NetstatRequest_ALL: + fn = func(*netstat.SockTabEntry) bool { return true } + case machine.NetstatRequest_LISTENING: + fn = func(s *netstat.SockTabEntry) bool { + return s.RemoteEndpoint.IP.IsUnspecified() && s.RemoteEndpoint.Port == 0 + } + case machine.NetstatRequest_CONNECTED: + fn = func(s *netstat.SockTabEntry) bool { + return !s.RemoteEndpoint.IP.IsUnspecified() && s.RemoteEndpoint.Port != 0 + } + } + + netstatResp, err := netstat.Netstat(ctx, features, fn) + if err != nil { + return nil, err + } + + records := make([]*machine.ConnectRecord, len(netstatResp)) + + for i, entry := range netstatResp { + records[i] = &machine.ConnectRecord{ + L4Proto: entry.Transport, + Localip: entry.LocalEndpoint.IP.String(), + Localport: uint32(entry.LocalEndpoint.Port), + Remoteip: entry.RemoteEndpoint.IP.String(), + Remoteport: uint32(entry.RemoteEndpoint.Port), + State: machine.ConnectRecord_State(entry.State), + Txqueue: entry.TxQueue, + Rxqueue: entry.RxQueue, + Tr: machine.ConnectRecord_TimerActive(entry.Tr), + Timerwhen: entry.TimerWhen, + Retrnsmt: entry.Retrnsmt, + Uid: entry.UID, + Timeout: entry.Timeout, + Inode: entry.Inode, + Ref: entry.Ref, + Pointer: entry.Pointer, + Process: &machine.ConnectRecord_Process{}, + Netns: entry.NetNS, + } + if entry.Process != nil { + records[i].Process = &machine.ConnectRecord_Process{ + Pid: uint32(entry.Process.Pid), + Name: entry.Process.Name, + } + } + } + + reply := &machine.NetstatResponse{ + Messages: []*machine.Netstat{ + { + Connectrecord: records, + }, + }, + } + + return reply, err +} diff --git a/internal/app/machined/internal/server/v1alpha1/v1alpha1_time.go b/internal/app/machined/internal/server/v1alpha1/v1alpha1_time.go new file mode 100644 index 0000000..313976b --- /dev/null +++ b/internal/app/machined/internal/server/v1alpha1/v1alpha1_time.go @@ -0,0 +1,72 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + "fmt" + "time" + + "github.com/beevik/ntp" + "google.golang.org/grpc" + "google.golang.org/protobuf/types/known/emptypb" + "google.golang.org/protobuf/types/known/timestamppb" + + timeapi "github.com/siderolabs/talos/pkg/machinery/api/time" + "github.com/siderolabs/talos/pkg/machinery/config" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// ConfigProvider defines an interface sufficient for the TimeServer. +type ConfigProvider interface { + Config() config.Config +} + +// TimeServer implements TimeService API. +type TimeServer struct { + timeapi.UnimplementedTimeServiceServer + + ConfigProvider ConfigProvider +} + +// Register implements the factory.Registrator interface. +func (r *TimeServer) Register(s *grpc.Server) { + timeapi.RegisterTimeServiceServer(s, r) +} + +// Time issues a query to the configured ntp server and displays the results. +func (r *TimeServer) Time(ctx context.Context, in *emptypb.Empty) (reply *timeapi.TimeResponse, err error) { + timeServers := r.ConfigProvider.Config().Machine().Time().Servers() + + if len(timeServers) == 0 { + timeServers = []string{constants.DefaultNTPServer} + } + + return r.TimeCheck(ctx, &timeapi.TimeRequest{ + Server: timeServers[0], + }) +} + +// TimeCheck issues a query to the specified ntp server and displays the results. +func (r *TimeServer) TimeCheck(ctx context.Context, in *timeapi.TimeRequest) (reply *timeapi.TimeResponse, err error) { + rt, err := ntp.Query(in.Server) + if err != nil { + return nil, fmt.Errorf("error querying NTP server %q: %w", in.Server, err) + } + + if err = rt.Validate(); err != nil { + return nil, fmt.Errorf("error validating NTP response: %w", err) + } + + return &timeapi.TimeResponse{ + Messages: []*timeapi.Time{ + { + Server: in.Server, + Localtime: timestamppb.New(time.Now()), + Remotetime: timestamppb.New(rt.Time), + }, + }, + }, nil +} diff --git a/internal/app/machined/internal/server/v1alpha1/v1alpha1_time_test.go b/internal/app/machined/internal/server/v1alpha1/v1alpha1_time_test.go new file mode 100644 index 0000000..1c08766 --- /dev/null +++ b/internal/app/machined/internal/server/v1alpha1/v1alpha1_time_test.go @@ -0,0 +1,128 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime_test + +import ( + "context" + "fmt" + "net" + "os" + "testing" + + "github.com/stretchr/testify/suite" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/protobuf/types/known/emptypb" + + runtime "github.com/aenix-io/talm/internal/app/machined/internal/server/v1alpha1" + "github.com/siderolabs/talos/pkg/grpc/dialer" + "github.com/siderolabs/talos/pkg/grpc/factory" + timeapi "github.com/siderolabs/talos/pkg/machinery/api/time" + "github.com/siderolabs/talos/pkg/machinery/config" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" +) + +type TimedSuite struct { + suite.Suite +} + +func TestTimedSuite(t *testing.T) { + // Hide all our state transition messages + // log.SetOutput(ioutil.Discard) + suite.Run(t, new(TimedSuite)) +} + +type mockConfigProvider struct { + timeServer string +} + +func (provider *mockConfigProvider) Config() config.Config { + return container.NewV1Alpha1(&v1alpha1.Config{ + MachineConfig: &v1alpha1.MachineConfig{ + MachineTime: &v1alpha1.TimeConfig{ + TimeServers: []string{provider.timeServer}, + }, + }, + }) +} + +func (suite *TimedSuite) TestTime() { + testServer := "time.cloudflare.com" + + // Create gRPC server + api := &runtime.TimeServer{ + ConfigProvider: &mockConfigProvider{timeServer: testServer}, + } + server := factory.NewServer(api) + listener, err := fakeTimedRPC() + suite.Assert().NoError(err) + + defer server.Stop() + + //nolint:errcheck + defer os.Remove(listener.Addr().String()) + + //nolint:errcheck + go server.Serve(listener) + + conn, err := grpc.Dial( + fmt.Sprintf("%s://%s", "unix", listener.Addr().String()), + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer(dialer.DialUnix()), + ) + suite.Require().NoError(err) + + nClient := timeapi.NewTimeServiceClient(conn) + reply, err := nClient.Time(context.Background(), &emptypb.Empty{}) + suite.Require().NoError(err) + suite.Assert().Equal(reply.Messages[0].Server, testServer) +} + +func (suite *TimedSuite) TestTimeCheck() { + testServer := "time.cloudflare.com" + + // Create ntp client with bogus server + // so we can check that we explicitly check the time of the + // specified server ( testserver ) + + // Create gRPC server + api := &runtime.TimeServer{} + server := factory.NewServer(api) + listener, err := fakeTimedRPC() + suite.Assert().NoError(err) + + defer server.Stop() + + //nolint:errcheck + defer os.Remove(listener.Addr().String()) + + //nolint:errcheck + go server.Serve(listener) + + conn, err := grpc.Dial( + fmt.Sprintf("%s://%s", "unix", listener.Addr().String()), + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer(dialer.DialUnix()), + ) + suite.Require().NoError(err) + + nClient := timeapi.NewTimeServiceClient(conn) + reply, err := nClient.TimeCheck(context.Background(), &timeapi.TimeRequest{Server: testServer}) + suite.Require().NoError(err) + suite.Assert().Equal(reply.Messages[0].Server, testServer) +} + +func fakeTimedRPC() (net.Listener, error) { + tmpfile, err := os.CreateTemp("", "timed") + if err != nil { + return nil, err + } + + return factory.NewListener( + factory.Network("unix"), + factory.SocketPath(tmpfile.Name()), + ) +} diff --git a/internal/app/machined/main.go b/internal/app/machined/main.go new file mode 100644 index 0000000..fd5e367 --- /dev/null +++ b/internal/app/machined/main.go @@ -0,0 +1,326 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package machined provides machined implementation. +package main + +import ( + "context" + "errors" + "fmt" + "log" + "net/http" + "os" + "os/signal" + "path/filepath" + "syscall" + "time" + + "github.com/hashicorp/go-cleanhttp" + "github.com/siderolabs/go-cmd/pkg/cmd/proc" + "github.com/siderolabs/go-cmd/pkg/cmd/proc/reaper" + debug "github.com/siderolabs/go-debug" + "github.com/siderolabs/go-procfs/procfs" + "golang.org/x/sys/unix" + + "github.com/aenix-io/talm/internal/app/apid" + "github.com/aenix-io/talm/internal/app/dashboard" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/emergency" + v1alpha1runtime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1" + "github.com/aenix-io/talm/internal/app/machined/pkg/system" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/services" + "github.com/aenix-io/talm/internal/app/maintenance" + "github.com/aenix-io/talm/internal/app/poweroff" + "github.com/aenix-io/talm/internal/app/trustd" + "github.com/aenix-io/talm/internal/app/wrapperd" + "github.com/aenix-io/talm/internal/pkg/mount" + "github.com/siderolabs/talos/pkg/httpdefaults" + "github.com/siderolabs/talos/pkg/machinery/api/common" + "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/startup" +) + +func init() { + // Patch a default HTTP client with updated transport to handle cases when default client is being used. + http.DefaultClient.Transport = httpdefaults.PatchTransport(cleanhttp.DefaultPooledTransport()) +} + +func recovery(ctx context.Context) { + if r := recover(); r != nil { + var ( + err error + ok bool + ) + + err, ok = r.(error) + if ok { + handle(ctx, err) + } + } +} + +// syncNonVolatileStorageBuffers invokes unix.Sync and waits up to 30 seconds +// for it to finish. +// +// See http://man7.org/linux/man-pages/man2/reboot.2.html. +func syncNonVolatileStorageBuffers() { + syncdone := make(chan struct{}) + + go func() { + defer close(syncdone) + + unix.Sync() + }() + + log.Printf("waiting for sync...") + + for i := 29; i >= 0; i-- { + select { + case <-syncdone: + log.Printf("sync done") + + return + case <-time.After(time.Second): + } + + if i != 0 { + log.Printf("waiting %d more seconds for sync to finish", i) + } + } + + log.Printf("sync hasn't completed in time, aborting...") +} + +//nolint:gocyclo +func handle(ctx context.Context, err error) { + rebootCmd := int(emergency.RebootCmd.Load()) + + var rebootErr runtime.RebootError + + if errors.As(err, &rebootErr) { + // not a failure, but wrapped reboot command + rebootCmd = rebootErr.Cmd + + err = nil + } + + if err != nil { + log.Print(err) + revertBootloader(ctx) + + if p := procfs.ProcCmdline().Get(constants.KernelParamPanic).First(); p != nil { + if *p == "0" { + log.Printf("panic=0 kernel flag found, sleeping forever") + + rebootCmd = 0 + } + } + } + + if rebootCmd == unix.LINUX_REBOOT_CMD_RESTART { + for i := 10; i >= 0; i-- { + log.Printf("rebooting in %d seconds\n", i) + time.Sleep(1 * time.Second) + } + } + + if err = proc.KillAll(); err != nil { + log.Printf("error killing all procs: %s", err) + } + + if err = mount.UnmountAll(); err != nil { + log.Printf("error unmounting: %s", err) + } + + syncNonVolatileStorageBuffers() + + if rebootCmd == 0 { + exitSignal := make(chan os.Signal, 1) + + signal.Notify(exitSignal, syscall.SIGINT, syscall.SIGTERM) + + <-exitSignal + } else if unix.Reboot(rebootCmd) == nil { + // Wait forever. + select {} + } +} + +func runDebugServer(ctx context.Context) { + const debugAddr = ":9982" + + debugLogFunc := func(msg string) { + log.Print(msg) + } + + if err := debug.ListenAndServe(ctx, debugAddr, debugLogFunc); err != nil { + log.Fatalf("failed to start debug server: %s", err) + } +} + +//nolint:gocyclo +func run() error { + errCh := make(chan error) + + // Limit GOMAXPROCS. + startup.LimitMaxProcs(constants.MachinedMaxProcs) + + // Set the PATH env var. + if err := os.Setenv("PATH", constants.PATH); err != nil { + return errors.New("error setting PATH") + } + + // Initialize the controller without a config. + c, err := v1alpha1runtime.NewController() + if err != nil { + return err + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + drainer := runtime.NewDrainer() + defer func() { + drainCtx, drainCtxCancel := context.WithTimeout(context.Background(), time.Second*10) + defer drainCtxCancel() + + if e := drainer.Drain(drainCtx); e != nil { + log.Printf("WARNING: failed to drain controllers: %s", e) + } + }() + + go runDebugServer(ctx) + + // Schedule service shutdown on any return. + defer system.Services(c.Runtime()).Shutdown(ctx) + + // Start signal and ACPI listeners. + go func() { + if e := c.ListenForEvents(ctx); e != nil { + log.Printf("WARNING: signals and ACPI events will be ignored: %s", e) + } + }() + + // Start v2 controller runtime. + go func() { + if e := c.V1Alpha2().Run(ctx, drainer); e != nil { + ctrlErr := fmt.Errorf("fatal controller runtime error: %s", e) + + log.Printf("controller runtime goroutine error: %s", ctrlErr) + + errCh <- ctrlErr + } + + log.Printf("controller runtime finished") + }() + + // Inject controller into maintenance service. + maintenance.InjectController(c) + + // Load machined service. + system.Services(c.Runtime()).Load( + &services.Machined{Controller: c}, + ) + + initializeCanceled := false + + // Initialize the machine. + if err = c.Run(ctx, runtime.SequenceInitialize, nil); err != nil { + if errors.Is(err, context.Canceled) { + initializeCanceled = true + } else { + return err + } + } + + // If Initialize sequence was canceled, don't run any other sequence. + if !initializeCanceled { + // Perform an installation if required. + if err = c.Run(ctx, runtime.SequenceInstall, nil); err != nil { + return err + } + + // Start the machine API. + system.Services(c.Runtime()).LoadAndStart( + &services.APID{}, + ) + + // Boot the machine. + if err = c.Run(ctx, runtime.SequenceBoot, nil); err != nil && !errors.Is(err, context.Canceled) { + return err + } + } + + // Watch and handle runtime events. + //nolint:errcheck + _ = c.Runtime().Events().Watch( + func(events <-chan runtime.EventInfo) { + for { + for event := range events { + switch msg := event.Payload.(type) { + case *machine.SequenceEvent: + if msg.Error != nil { + if msg.Error.GetCode() == common.Code_LOCKED || + msg.Error.GetCode() == common.Code_CANCELED { + // ignore sequence lock and canceled errors, they're not fatal + continue + } + + errCh <- fmt.Errorf( + "fatal sequencer error in %q sequence: %v", + msg.GetSequence(), + msg.GetError().String(), + ) + } + case *machine.RestartEvent: + errCh <- runtime.RebootError{Cmd: int(msg.Cmd)} + } + } + } + }, + ) + + return <-errCh +} + +func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + switch filepath.Base(os.Args[0]) { + case "apid": + apid.Main() + + return + case "trustd": + trustd.Main() + + return + // Azure uses the hv_utils kernel module to shutdown the node in hyper-v by calling perform_shutdown which will call orderly_poweroff which will call /sbin/poweroff. + case "poweroff", "shutdown": + poweroff.Main(os.Args) + + return + case "wrapperd": + wrapperd.Main() + + return + case "dashboard": + dashboard.Main() + + return + default: + } + + // Setup panic handler. + defer recovery(ctx) + + // Initialize the process reaper. + reaper.Run() + defer reaper.Shutdown() + + handle(ctx, run()) +} diff --git a/internal/app/machined/pkg/adapters/cluster/cluster.go b/internal/app/machined/pkg/adapters/cluster/cluster.go new file mode 100644 index 0000000..dbd420e --- /dev/null +++ b/internal/app/machined/pkg/adapters/cluster/cluster.go @@ -0,0 +1,6 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package cluster implements adapters wrapping resources/cluster to provide additional functionality. +package cluster diff --git a/internal/app/machined/pkg/adapters/cluster/identity.go b/internal/app/machined/pkg/adapters/cluster/identity.go new file mode 100644 index 0000000..ca0ce23 --- /dev/null +++ b/internal/app/machined/pkg/adapters/cluster/identity.go @@ -0,0 +1,55 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cluster + +import ( + "crypto/rand" + "encoding/hex" + "io" + + "github.com/jxskiss/base62" + + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" +) + +// IdentitySpec adapter provides identity generation. +// +//nolint:revive,golint +func IdentitySpec(r *cluster.IdentitySpec) identity { + return identity{ + IdentitySpec: r, + } +} + +type identity struct { + *cluster.IdentitySpec +} + +// Generate new identity. +func (a identity) Generate() error { + buf := make([]byte, constants.DefaultNodeIdentitySize) + + if _, err := io.ReadFull(rand.Reader, buf); err != nil { + return err + } + + a.IdentitySpec.NodeID = base62.EncodeToString(buf) + + return nil +} + +// ConvertMachineID returns /etc/machine-id compatible representation. +func (a identity) ConvertMachineID() ([]byte, error) { + raw, err := base62.DecodeString(a.IdentitySpec.NodeID) + if err != nil { + return nil, err + } + + buf := make([]byte, 32) + hex.Encode(buf, raw[:16]) + + return buf, nil +} diff --git a/internal/app/machined/pkg/adapters/cluster/identity_test.go b/internal/app/machined/pkg/adapters/cluster/identity_test.go new file mode 100644 index 0000000..66b70f3 --- /dev/null +++ b/internal/app/machined/pkg/adapters/cluster/identity_test.go @@ -0,0 +1,40 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cluster_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + clusteradapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/cluster" + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" +) + +func TestIdentityGenerate(t *testing.T) { + var spec1, spec2 cluster.IdentitySpec + + require.NoError(t, clusteradapter.IdentitySpec(&spec1).Generate()) + require.NoError(t, clusteradapter.IdentitySpec(&spec2).Generate()) + + assert.NotEqual(t, spec1, spec2) + + length := len(spec1.NodeID) + + assert.GreaterOrEqual(t, length, 43) + assert.LessOrEqual(t, length, 45) +} + +func TestIdentityConvertMachineID(t *testing.T) { + spec := cluster.IdentitySpec{ + NodeID: "sou7yy34ykX3n373Zw1DXKb8zD7UnyKT6HT3QDsGH6L", + } + + machineID, err := clusteradapter.IdentitySpec(&spec).ConvertMachineID() + require.NoError(t, err) + + assert.Equal(t, "be871ac0d0dd31fa4caca753b0f3f1b2", string(machineID)) +} diff --git a/internal/app/machined/pkg/adapters/hardware/hardware.go b/internal/app/machined/pkg/adapters/hardware/hardware.go new file mode 100644 index 0000000..268abf9 --- /dev/null +++ b/internal/app/machined/pkg/adapters/hardware/hardware.go @@ -0,0 +1,6 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package hardware implements adapters wrapping resources/hardware to provide additional functionality. +package hardware diff --git a/internal/app/machined/pkg/adapters/hardware/memorymodule.go b/internal/app/machined/pkg/adapters/hardware/memorymodule.go new file mode 100644 index 0000000..4d64620 --- /dev/null +++ b/internal/app/machined/pkg/adapters/hardware/memorymodule.go @@ -0,0 +1,54 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package hardware + +import ( + "github.com/siderolabs/go-smbios/smbios" + + "github.com/siderolabs/talos/pkg/machinery/resources/hardware" +) + +// MemoryModule adapter provider conversion from smbios.SMBIOS. +// +//nolint:revive,golint +func MemoryModule(m *hardware.MemoryModule) memoryModule { + return memoryModule{ + MemoryModule: m, + } +} + +type memoryModule struct { + *hardware.MemoryModule +} + +// Update current processor info. +func (m memoryModule) Update(memory *smbios.MemoryDevice) { + translateMemoryModuleInfo := func(in *smbios.MemoryDevice) hardware.MemoryModuleSpec { + var memoryModuleSpec hardware.MemoryModuleSpec + + if in.Size != 0 && in.Size != 0xFFFF { + var size uint32 + + if in.Size == 0x7FFF { + size = uint32(in.ExtendedSize) + } else { + size = uint32(in.Size) + } + + memoryModuleSpec.AssetTag = in.AssetTag + memoryModuleSpec.BankLocator = in.BankLocator + memoryModuleSpec.DeviceLocator = in.DeviceLocator + memoryModuleSpec.Manufacturer = in.Manufacturer + memoryModuleSpec.ProductName = in.PartNumber + memoryModuleSpec.SerialNumber = in.SerialNumber + memoryModuleSpec.Size = size + memoryModuleSpec.Speed = uint32(in.Speed) + } + + return memoryModuleSpec + } + + *m.MemoryModule.TypedSpec() = translateMemoryModuleInfo(memory) +} diff --git a/internal/app/machined/pkg/adapters/hardware/processor.go b/internal/app/machined/pkg/adapters/hardware/processor.go new file mode 100644 index 0000000..7ae64ac --- /dev/null +++ b/internal/app/machined/pkg/adapters/hardware/processor.go @@ -0,0 +1,50 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package hardware + +import ( + "github.com/siderolabs/go-smbios/smbios" + + "github.com/siderolabs/talos/pkg/machinery/resources/hardware" +) + +// Processor adapter provider conversion from smbios.SMBIOS. +// +//nolint:revive,golint +func Processor(p *hardware.Processor) processor { + return processor{ + Processor: p, + } +} + +type processor struct { + *hardware.Processor +} + +// Update current processor info. +func (p processor) Update(processor *smbios.ProcessorInformation) { + translateProcessorInfo := func(in *smbios.ProcessorInformation) hardware.ProcessorSpec { + var processorSpec hardware.ProcessorSpec + + if in.Status.SocketPopulated() { + processorSpec.Socket = in.SocketDesignation + processorSpec.Manufacturer = in.ProcessorManufacturer + processorSpec.ProductName = in.ProcessorVersion + processorSpec.MaxSpeed = uint32(in.MaxSpeed) + processorSpec.BootSpeed = uint32(in.CurrentSpeed) + processorSpec.Status = uint32(in.Status) + processorSpec.SerialNumber = in.SerialNumber + processorSpec.AssetTag = in.AssetTag + processorSpec.PartNumber = in.PartNumber + processorSpec.CoreCount = uint32(in.CoreCount) + processorSpec.CoreEnabled = uint32(in.CoreEnabled) + processorSpec.ThreadCount = uint32(in.ThreadCount) + } + + return processorSpec + } + + *p.Processor.TypedSpec() = translateProcessorInfo(processor) +} diff --git a/internal/app/machined/pkg/adapters/hardware/system_information.go b/internal/app/machined/pkg/adapters/hardware/system_information.go new file mode 100644 index 0000000..497b5b3 --- /dev/null +++ b/internal/app/machined/pkg/adapters/hardware/system_information.go @@ -0,0 +1,41 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package hardware + +import ( + "github.com/siderolabs/go-smbios/smbios" + + "github.com/siderolabs/talos/pkg/machinery/resources/hardware" +) + +// SystemInformation adapter provider conversion from smbios.SMBIOS. +// +//nolint:revive,golint +func SystemInformation(p *hardware.SystemInformation) systemInformation { + return systemInformation{ + SystemInformation: p, + } +} + +type systemInformation struct { + *hardware.SystemInformation +} + +// Update current systemInformation info. +func (p systemInformation) Update(systemInformation *smbios.SystemInformation, uuidRewrite string) { + if uuidRewrite == "" { + uuidRewrite = systemInformation.UUID + } + + *p.SystemInformation.TypedSpec() = hardware.SystemInformationSpec{ + Manufacturer: systemInformation.Manufacturer, + ProductName: systemInformation.ProductName, + Version: systemInformation.Version, + SerialNumber: systemInformation.SerialNumber, + UUID: uuidRewrite, + WakeUpType: systemInformation.WakeUpType.String(), + SKUNumber: systemInformation.SKUNumber, + } +} diff --git a/internal/app/machined/pkg/adapters/k8s/k8s.go b/internal/app/machined/pkg/adapters/k8s/k8s.go new file mode 100644 index 0000000..8e9789a --- /dev/null +++ b/internal/app/machined/pkg/adapters/k8s/k8s.go @@ -0,0 +1,6 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package k8s implements adapters wrapping resources/k8s to provide additional functionality. +package k8s diff --git a/internal/app/machined/pkg/adapters/k8s/manifest.go b/internal/app/machined/pkg/adapters/k8s/manifest.go new file mode 100644 index 0000000..993f30b --- /dev/null +++ b/internal/app/machined/pkg/adapters/k8s/manifest.go @@ -0,0 +1,101 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "io" + + "github.com/siderolabs/gen/xslices" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/yaml" + + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +// Manifest adapter provides conversion from procfs. +// +//nolint:revive,golint +func Manifest(r *k8s.Manifest) manifest { + return manifest{ + Manifest: r, + } +} + +type manifest struct { + *k8s.Manifest +} + +// SetYAML parses manifest from YAML. +// +//nolint:gocyclo +func (a manifest) SetYAML(yamlBytes []byte) error { + a.Manifest.TypedSpec().Items = nil + reader := yaml.NewYAMLReader(bufio.NewReader(bytes.NewReader(yamlBytes))) + + for { + yamlManifest, err := reader.Read() + if err != nil { + if err == io.EOF { + break + } + + return err + } + + yamlManifest = bytes.TrimSpace(yamlManifest) + + if len(yamlManifest) == 0 { + continue + } + + jsonManifest, err := yaml.ToJSON(yamlManifest) + if err != nil { + return fmt.Errorf("error converting manifest to JSON: %w", err) + } + + if bytes.Equal(jsonManifest, []byte("null")) || bytes.Equal(jsonManifest, []byte("{}")) { + // skip YAML docs which contain only comments + continue + } + + var obj unstructured.Unstructured + + if err = json.Unmarshal(jsonManifest, &obj); err != nil { + return fmt.Errorf("error loading JSON manifest into unstructured: %w", err) + } + + // if the manifest is a list, we will unwrap it + if obj.IsList() { + if err = obj.EachListItem(func(item runtime.Object) error { + obj, ok := item.(*unstructured.Unstructured) + if !ok { + return fmt.Errorf("list item is not Unstructured: %T", item) + } + + a.Manifest.TypedSpec().Items = append(a.Manifest.TypedSpec().Items, k8s.SingleManifest{Object: obj.Object}) + + return nil + }); err != nil { + return fmt.Errorf("error unwrapping a List: %w", err) + } + } else { + a.Manifest.TypedSpec().Items = append(a.Manifest.TypedSpec().Items, k8s.SingleManifest{Object: obj.Object}) + } + } + + return nil +} + +// Objects returns list of unstructured object. +func (a manifest) Objects() []*unstructured.Unstructured { + return xslices.Map(a.Manifest.TypedSpec().Items, func(item k8s.SingleManifest) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: item.Object} + }) +} diff --git a/internal/app/machined/pkg/adapters/k8s/manifest_test.go b/internal/app/machined/pkg/adapters/k8s/manifest_test.go new file mode 100644 index 0000000..dfd4bce --- /dev/null +++ b/internal/app/machined/pkg/adapters/k8s/manifest_test.go @@ -0,0 +1,69 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s_test + +import ( + _ "embed" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + k8sadapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +func TestManifestSetYAML(t *testing.T) { + manifest := k8s.NewManifest(k8s.ControlPlaneNamespaceName, "test") + adapter := k8sadapter.Manifest(manifest) + + require.NoError(t, adapter.SetYAML([]byte(strings.TrimSpace(` +--- +apiVersion: audit.k8s.io/v1 +kind: Policy +rules: +- level: Metadata +--- +`)))) + + assert.Len(t, adapter.Objects(), 1) + assert.Equal(t, adapter.Objects()[0].GetKind(), "Policy") +} + +func TestManifestSetYAMLEmptyComments(t *testing.T) { + manifest := k8s.NewManifest(k8s.ControlPlaneNamespaceName, "test") + adapter := k8sadapter.Manifest(manifest) + + require.NoError(t, adapter.SetYAML([]byte(strings.TrimSpace(` +--- +apiVersion: audit.k8s.io/v1 +kind: Policy +rules: +- level: Metadata +--- +# Left empty +--- +`)))) + + assert.Len(t, adapter.Objects(), 1) + assert.Equal(t, adapter.Objects()[0].GetKind(), "Policy") +} + +//go:embed testdata/list.yaml +var listManifest []byte + +func TestManifestSetYAMLList(t *testing.T) { + manifest := k8s.NewManifest(k8s.ControlPlaneNamespaceName, "test") + adapter := k8sadapter.Manifest(manifest) + + require.NoError(t, adapter.SetYAML(listManifest)) + + assert.Len(t, adapter.Objects(), 2) + assert.Equal(t, "ClusterRoleBinding", adapter.Objects()[0].GetKind()) + assert.Equal(t, "system:cloud-node-controller", adapter.Objects()[0].GetName()) + assert.Equal(t, "ClusterRoleBinding", adapter.Objects()[1].GetKind()) + assert.Equal(t, "system:cloud-controller-manager", adapter.Objects()[1].GetName()) +} diff --git a/internal/app/machined/pkg/adapters/k8s/static_pod.go b/internal/app/machined/pkg/adapters/k8s/static_pod.go new file mode 100644 index 0000000..c545733 --- /dev/null +++ b/internal/app/machined/pkg/adapters/k8s/static_pod.go @@ -0,0 +1,52 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import ( + "encoding/json" + + v1 "k8s.io/api/core/v1" + + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +// StaticPod adapter provides conversion from *v1.Pod. +// +//nolint:revive,golint +func StaticPod(r *k8s.StaticPod) staticPod { + return staticPod{ + StaticPod: r, + } +} + +type staticPod struct { + *k8s.StaticPod +} + +// Pod returns native Kubernetes resource. +func (a staticPod) Pod() (*v1.Pod, error) { + var spec v1.Pod + + jsonSerialized, err := json.Marshal(a.StaticPod.TypedSpec().Pod) + if err != nil { + return nil, err + } + + err = json.Unmarshal(jsonSerialized, &spec) + + return &spec, err +} + +// SetPod sets spec from native Kubernetes resource. +func (a staticPod) SetPod(podSpec *v1.Pod) error { + jsonSerialized, err := json.Marshal(podSpec) + if err != nil { + return err + } + + a.StaticPod.TypedSpec().Pod = map[string]interface{}{} + + return json.Unmarshal(jsonSerialized, &a.StaticPod.TypedSpec().Pod) +} diff --git a/internal/app/machined/pkg/adapters/k8s/static_pod_status.go b/internal/app/machined/pkg/adapters/k8s/static_pod_status.go new file mode 100644 index 0000000..bdcea57 --- /dev/null +++ b/internal/app/machined/pkg/adapters/k8s/static_pod_status.go @@ -0,0 +1,52 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import ( + "encoding/json" + + v1 "k8s.io/api/core/v1" + + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +// StaticPodStatus adapter provides conversion from *v1.PodStatus. +// +//nolint:revive,golint +func StaticPodStatus(r *k8s.StaticPodStatus) staticPodStatus { + return staticPodStatus{ + StaticPodStatus: r, + } +} + +type staticPodStatus struct { + *k8s.StaticPodStatus +} + +// SetStatus sets status from native Kubernetes resource. +func (a staticPodStatus) SetStatus(status *v1.PodStatus) error { + jsonSerialized, err := json.Marshal(status) + if err != nil { + return err + } + + a.StaticPodStatus.TypedSpec().PodStatus = map[string]interface{}{} + + return json.Unmarshal(jsonSerialized, &a.StaticPodStatus.TypedSpec().PodStatus) +} + +// Status gets status from native Kubernetes resource. +func (a staticPodStatus) Status() (*v1.PodStatus, error) { + var spec v1.PodStatus + + jsonSerialized, err := json.Marshal(a.StaticPodStatus.TypedSpec().PodStatus) + if err != nil { + return nil, err + } + + err = json.Unmarshal(jsonSerialized, &spec) + + return &spec, err +} diff --git a/internal/app/machined/pkg/adapters/k8s/testdata/list.yaml b/internal/app/machined/pkg/adapters/k8s/testdata/list.yaml new file mode 100644 index 0000000..ae5cf06 --- /dev/null +++ b/internal/app/machined/pkg/adapters/k8s/testdata/list.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +items: +- apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: system:cloud-node-controller + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-node-controller + subjects: + - kind: ServiceAccount + name: cloud-node-controller + namespace: kube-system +- apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: List +metadata: {} diff --git a/internal/app/machined/pkg/adapters/kubespan/identity.go b/internal/app/machined/pkg/adapters/kubespan/identity.go new file mode 100644 index 0000000..c31670a --- /dev/null +++ b/internal/app/machined/pkg/adapters/kubespan/identity.go @@ -0,0 +1,75 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package kubespan + +import ( + "errors" + "fmt" + "net" + "net/netip" + + "github.com/mdlayher/netx/eui64" + "github.com/siderolabs/gen/value" + "go4.org/netipx" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + "github.com/siderolabs/talos/pkg/machinery/resources/kubespan" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// IdentitySpec adapter provides identity generation. +// +//nolint:revive,golint +func IdentitySpec(r *kubespan.IdentitySpec) identity { + return identity{ + IdentitySpec: r, + } +} + +type identity struct { + *kubespan.IdentitySpec +} + +// GenerateKey generates new Wireguard key. +func (a identity) GenerateKey() error { + key, err := wgtypes.GeneratePrivateKey() + if err != nil { + return err + } + + a.IdentitySpec.PrivateKey = key.String() + a.IdentitySpec.PublicKey = key.PublicKey().String() + + return nil +} + +// UpdateAddress re-calculates node address based on input data. +func (a identity) UpdateAddress(clusterID string, mac net.HardwareAddr) error { + a.IdentitySpec.Subnet = network.ULAPrefix(clusterID, network.ULAKubeSpan) + + var err error + + a.IdentitySpec.Address, err = wgEUI64(a.IdentitySpec.Subnet, mac) + + return err +} + +func wgEUI64(prefix netip.Prefix, mac net.HardwareAddr) (out netip.Prefix, err error) { + if value.IsZero(prefix) { + return out, errors.New("cannot calculate IP from zero prefix") + } + + stdIP, err := eui64.ParseMAC(netipx.PrefixIPNet(prefix).IP, mac) + if err != nil { + return out, fmt.Errorf("failed to parse MAC into EUI-64 address: %w", err) + } + + ip, ok := netipx.FromStdIP(stdIP) + if !ok { + return out, fmt.Errorf("failed to parse intermediate standard IP %q: %w", stdIP.String(), err) + } + + return netip.PrefixFrom(ip, ip.BitLen()), nil +} diff --git a/internal/app/machined/pkg/adapters/kubespan/identity_test.go b/internal/app/machined/pkg/adapters/kubespan/identity_test.go new file mode 100644 index 0000000..ec92a48 --- /dev/null +++ b/internal/app/machined/pkg/adapters/kubespan/identity_test.go @@ -0,0 +1,34 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package kubespan_test + +import ( + "net" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + kubespanadapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/kubespan" + "github.com/siderolabs/talos/pkg/machinery/resources/kubespan" +) + +func TestIdentityGenerateKey(t *testing.T) { + var spec kubespan.IdentitySpec + + assert.NoError(t, kubespanadapter.IdentitySpec(&spec).GenerateKey()) +} + +func TestIdentityUpdateAddress(t *testing.T) { + var spec kubespan.IdentitySpec + + mac, err := net.ParseMAC("2e:1a:b6:53:81:69") + require.NoError(t, err) + + assert.NoError(t, kubespanadapter.IdentitySpec(&spec).UpdateAddress("8XuV9TZHW08DOk3bVxQjH9ih_TBKjnh-j44tsCLSBzo=", mac)) + + assert.Equal(t, "fd7f:175a:b97c:5602:2c1a:b6ff:fe53:8169/128", spec.Address.String()) + assert.Equal(t, "fd7f:175a:b97c:5602::/64", spec.Subnet.String()) +} diff --git a/internal/app/machined/pkg/adapters/kubespan/kubespan.go b/internal/app/machined/pkg/adapters/kubespan/kubespan.go new file mode 100644 index 0000000..5b5995c --- /dev/null +++ b/internal/app/machined/pkg/adapters/kubespan/kubespan.go @@ -0,0 +1,6 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package kubespan implements adapters wrapping resources/kubespan to provide additional functionality. +package kubespan diff --git a/internal/app/machined/pkg/adapters/kubespan/peer_status.go b/internal/app/machined/pkg/adapters/kubespan/peer_status.go new file mode 100644 index 0000000..01a66d0 --- /dev/null +++ b/internal/app/machined/pkg/adapters/kubespan/peer_status.go @@ -0,0 +1,153 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package kubespan + +import ( + "net/netip" + "time" + + "github.com/siderolabs/gen/value" + "go4.org/netipx" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/wireguard" + "github.com/siderolabs/talos/pkg/machinery/resources/kubespan" +) + +// PeerStatusSpec adapter provides Wireguard integration and state management. +// +//nolint:revive,golint +func PeerStatusSpec(r *kubespan.PeerStatusSpec) peerStatus { + return peerStatus{ + PeerStatusSpec: r, + } +} + +type peerStatus struct { + *kubespan.PeerStatusSpec +} + +// EndpointConnectionTimeout is time to wait for initial handshake when the endpoint is just set. +const EndpointConnectionTimeout = 15 * time.Second + +// CalculateState updates connection state based on other fields values. +// +// Goal: endpoint is ultimately down if we haven't seen handshake for more than peerDownInterval, +// but as the endpoints get updated we want faster feedback, so we start checking more aggressively +// that the handshake happened within endpointConnectionTimeout since last endpoint change. +// +// Timeline: +// +// ----------------------------------------------------------------------> +// ^ ^ ^ +// | | | +// T0 T0+endpointConnectionTimeout T0+peerDownInterval +// +// Where T0 = LastEndpointChange +// +// The question is where is LastHandshakeTimeout vs. those points above: +// +// - if we're past (T0+peerDownInterval), simply check that time since last handshake < peerDownInterval +// - if we're between (T0+endpointConnectionTimeout) and (T0+peerDownInterval), and there's no handshake +// after the endpoint change, assume that the endpoint is down +// - if we're between (T0) and (T0+endpointConnectionTimeout), and there's no handshake since the endpoint change, +// consider the state to be unknown +func (a peerStatus) CalculateState() { + sinceLastHandshake := time.Since(a.PeerStatusSpec.LastHandshakeTime) + sinceEndpointChange := time.Since(a.PeerStatusSpec.LastEndpointChange) + + a.CalculateStateWithDurations(sinceLastHandshake, sinceEndpointChange) +} + +// CalculateStateWithDurations calculates the state based on the time since events. +func (a peerStatus) CalculateStateWithDurations(sinceLastHandshake, sinceEndpointChange time.Duration) { + switch { + case sinceEndpointChange > wireguard.PeerDownInterval: // past T0+peerDownInterval + // if we got handshake in the last peerDownInterval, endpoint is up + if sinceLastHandshake < wireguard.PeerDownInterval { + a.PeerStatusSpec.State = kubespan.PeerStateUp + } else { + a.PeerStatusSpec.State = kubespan.PeerStateDown + } + case sinceEndpointChange < EndpointConnectionTimeout: // between (T0) and (T0+endpointConnectionTimeout) + // endpoint got recently updated, consider no handshake as 'unknown' + if a.PeerStatusSpec.LastHandshakeTime.After(a.PeerStatusSpec.LastEndpointChange) { + a.PeerStatusSpec.State = kubespan.PeerStateUp + } else { + a.PeerStatusSpec.State = kubespan.PeerStateUnknown + } + + default: // otherwise, we're between (T0+endpointConnectionTimeout) and (T0+peerDownInterval) + // if we haven't had the handshake yet, consider the endpoint to be down + if a.PeerStatusSpec.LastHandshakeTime.After(a.PeerStatusSpec.LastEndpointChange) { + a.PeerStatusSpec.State = kubespan.PeerStateUp + } else { + a.PeerStatusSpec.State = kubespan.PeerStateDown + } + } + + if a.PeerStatusSpec.State == kubespan.PeerStateDown && value.IsZero(a.PeerStatusSpec.LastUsedEndpoint) { + // no endpoint, so unknown + a.PeerStatusSpec.State = kubespan.PeerStateUnknown + } +} + +// UpdateFromWireguard updates fields from wgtypes information. +func (a peerStatus) UpdateFromWireguard(peer wgtypes.Peer) { + if peer.Endpoint != nil { + a.PeerStatusSpec.Endpoint, _ = netipx.FromStdAddr(peer.Endpoint.IP, peer.Endpoint.Port, "") + } else { + a.PeerStatusSpec.Endpoint = netip.AddrPort{} + } + + a.PeerStatusSpec.LastHandshakeTime = peer.LastHandshakeTime + a.PeerStatusSpec.TransmitBytes = peer.TransmitBytes + a.PeerStatusSpec.ReceiveBytes = peer.ReceiveBytes +} + +// UpdateEndpoint updates the endpoint information and last update timestamp. +func (a peerStatus) UpdateEndpoint(endpoint netip.AddrPort) { + a.PeerStatusSpec.Endpoint = endpoint + a.PeerStatusSpec.LastUsedEndpoint = endpoint + a.PeerStatusSpec.LastEndpointChange = time.Now() + a.PeerStatusSpec.State = kubespan.PeerStateUnknown +} + +// ShouldChangeEndpoint tells whether endpoint should be updated. +func (a peerStatus) ShouldChangeEndpoint() bool { + return a.PeerStatusSpec.State == kubespan.PeerStateDown || value.IsZero(a.PeerStatusSpec.LastUsedEndpoint) +} + +// PickNewEndpoint picks new endpoint given the state and list of available endpoints. +// +// If returned newEndpoint is zero value, no new endpoint is available. +func (a peerStatus) PickNewEndpoint(endpoints []netip.AddrPort) (newEndpoint netip.AddrPort) { + if len(endpoints) == 0 { + return + } + + if value.IsZero(a.PeerStatusSpec.LastUsedEndpoint) { + // first time setting the endpoint + newEndpoint = endpoints[0] + } else { + // find the next endpoint after LastUsedEndpoint and use it + idx := -1 + + for i := range endpoints { + if endpoints[i] == a.PeerStatusSpec.LastUsedEndpoint { + idx = i + + break + } + } + + // special case: if the peer has just a single endpoint, we can't rotate + if !(len(endpoints) == 1 && idx == 0 && a.PeerStatusSpec.Endpoint == a.PeerStatusSpec.LastUsedEndpoint) { + newEndpoint = endpoints[(idx+1)%len(endpoints)] + } + } + + return +} diff --git a/internal/app/machined/pkg/adapters/kubespan/peer_status_test.go b/internal/app/machined/pkg/adapters/kubespan/peer_status_test.go new file mode 100644 index 0000000..ca29cb8 --- /dev/null +++ b/internal/app/machined/pkg/adapters/kubespan/peer_status_test.go @@ -0,0 +1,134 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package kubespan_test + +import ( + "net/netip" + "testing" + "time" + + "github.com/siderolabs/gen/value" + "github.com/stretchr/testify/assert" + + kubespanadapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/kubespan" + "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/wireguard" + "github.com/siderolabs/talos/pkg/machinery/resources/kubespan" +) + +func TestPeerStatus_PickNewEndpoint(t *testing.T) { + // zero status + peerStatus := kubespan.PeerStatusSpec{} + + // no endpoint => no way to pick new one + assert.True(t, value.IsZero(kubespanadapter.PeerStatusSpec(&peerStatus).PickNewEndpoint(nil))) + + endpoints := []netip.AddrPort{ + netip.MustParseAddrPort("10.3.4.5:10500"), + netip.MustParseAddrPort("192.168.3.8:457"), + } + + // initial choice should be the first endpoint + newEndpoint := kubespanadapter.PeerStatusSpec(&peerStatus).PickNewEndpoint(endpoints) + assert.Equal(t, endpoints[0], newEndpoint) + kubespanadapter.PeerStatusSpec(&peerStatus).UpdateEndpoint(newEndpoint) + + // next choice should be 2nd endpoint + newEndpoint = kubespanadapter.PeerStatusSpec(&peerStatus).PickNewEndpoint(endpoints) + assert.Equal(t, endpoints[1], newEndpoint) + kubespanadapter.PeerStatusSpec(&peerStatus).UpdateEndpoint(newEndpoint) + + // back to the first endpoint + newEndpoint = kubespanadapter.PeerStatusSpec(&peerStatus).PickNewEndpoint(endpoints) + assert.Equal(t, endpoints[0], newEndpoint) + kubespanadapter.PeerStatusSpec(&peerStatus).UpdateEndpoint(newEndpoint) + + // can't rotate a single endpoint + assert.True(t, value.IsZero(kubespanadapter.PeerStatusSpec(&peerStatus).PickNewEndpoint(endpoints[:1]))) + + // can rotate if the endpoint is different + newEndpoint = kubespanadapter.PeerStatusSpec(&peerStatus).PickNewEndpoint(endpoints[1:]) + assert.Equal(t, endpoints[1], newEndpoint) + kubespanadapter.PeerStatusSpec(&peerStatus).UpdateEndpoint(newEndpoint) + + // if totally new list of endpoints is given, pick the first one + endpoints = []netip.AddrPort{ + netip.MustParseAddrPort("10.3.4.5:10501"), + netip.MustParseAddrPort("192.168.3.8:458"), + } + newEndpoint = kubespanadapter.PeerStatusSpec(&peerStatus).PickNewEndpoint(endpoints) + assert.Equal(t, endpoints[0], newEndpoint) + kubespanadapter.PeerStatusSpec(&peerStatus).UpdateEndpoint(newEndpoint) +} + +func TestPeerStatus_CalculateState(t *testing.T) { + for _, tt := range []struct { + name string + + sinceLastHandshake, sinceEndpointChange time.Duration + + lastUsedEndpointZero bool + + expectedState kubespan.PeerState + }{ + { + name: "no endpoint set", + sinceLastHandshake: time.Hour, + sinceEndpointChange: time.Hour, + lastUsedEndpointZero: true, + expectedState: kubespan.PeerStateUnknown, + }, + { + name: "peer is down", + sinceLastHandshake: 2 * wireguard.PeerDownInterval, + sinceEndpointChange: 2 * wireguard.PeerDownInterval, + expectedState: kubespan.PeerStateDown, + }, + { + name: "fresh peer, no handshake", + sinceLastHandshake: 2 * wireguard.PeerDownInterval, + sinceEndpointChange: kubespanadapter.EndpointConnectionTimeout / 2, + expectedState: kubespan.PeerStateUnknown, + }, + { + name: "fresh peer, with handshake", + sinceLastHandshake: 0, + sinceEndpointChange: kubespanadapter.EndpointConnectionTimeout / 2, + expectedState: kubespan.PeerStateUp, + }, + { + name: "peer after initial timeout, with handshake", + sinceLastHandshake: 0, + sinceEndpointChange: kubespanadapter.EndpointConnectionTimeout + 1, + expectedState: kubespan.PeerStateUp, + }, + { + name: "peer after initial timeout, no handshake", + sinceLastHandshake: 2 * kubespanadapter.EndpointConnectionTimeout, + sinceEndpointChange: kubespanadapter.EndpointConnectionTimeout + 1, + expectedState: kubespan.PeerStateDown, + }, + { + name: "established peer, up", + sinceLastHandshake: wireguard.PeerDownInterval / 2, + sinceEndpointChange: wireguard.PeerDownInterval + 1, + expectedState: kubespan.PeerStateUp, + }, + } { + t.Run(tt.name, func(t *testing.T) { + peerStatus := kubespan.PeerStatusSpec{ + LastHandshakeTime: time.Now().Add(-tt.sinceLastHandshake), + LastEndpointChange: time.Now().Add(-tt.sinceEndpointChange), + } + + if !tt.lastUsedEndpointZero { + peerStatus.LastUsedEndpoint = netip.MustParseAddrPort("192.168.1.1:10000") + } + + kubespanadapter.PeerStatusSpec(&peerStatus).CalculateStateWithDurations(tt.sinceLastHandshake, tt.sinceEndpointChange) + + assert.Equal(t, tt.expectedState, peerStatus.State) + }) + } +} diff --git a/internal/app/machined/pkg/adapters/network/bond_master_spec.go b/internal/app/machined/pkg/adapters/network/bond_master_spec.go new file mode 100644 index 0000000..5cd4798 --- /dev/null +++ b/internal/app/machined/pkg/adapters/network/bond_master_spec.go @@ -0,0 +1,197 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "github.com/mdlayher/netlink" + "golang.org/x/sys/unix" + + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// BondMasterSpec adapter provides encoding/decoding to netlink structures. +// +//nolint:revive,golint +func BondMasterSpec(r *network.BondMasterSpec) bondMaster { + return bondMaster{ + BondMasterSpec: r, + } +} + +type bondMaster struct { + *network.BondMasterSpec +} + +// FillDefaults fills zero values with proper default values. +func (a bondMaster) FillDefaults() { + bond := a.BondMasterSpec + + if bond.ResendIGMP == 0 { + bond.ResendIGMP = 1 + } + + if bond.LPInterval == 0 { + bond.LPInterval = 1 + } + + if bond.PacketsPerSlave == 0 { + bond.PacketsPerSlave = 1 + } + + if bond.NumPeerNotif == 0 { + bond.NumPeerNotif = 1 + } + + if bond.Mode != nethelpers.BondModeALB && bond.Mode != nethelpers.BondModeTLB { + bond.TLBDynamicLB = 1 + } + + if bond.Mode == nethelpers.BondMode8023AD { + bond.ADActorSysPrio = 65535 + } +} + +// Encode the BondMasterSpec into netlink attributes. +// +//nolint:gocyclo +func (a bondMaster) Encode() ([]byte, error) { + bond := a.BondMasterSpec + + encoder := netlink.NewAttributeEncoder() + + encoder.Uint8(unix.IFLA_BOND_MODE, uint8(bond.Mode)) + encoder.Uint8(unix.IFLA_BOND_XMIT_HASH_POLICY, uint8(bond.HashPolicy)) + + if bond.Mode == nethelpers.BondMode8023AD { + encoder.Uint8(unix.IFLA_BOND_AD_LACP_RATE, uint8(bond.LACPRate)) + } + + if bond.Mode != nethelpers.BondMode8023AD && bond.Mode != nethelpers.BondModeALB && bond.Mode != nethelpers.BondModeTLB { + encoder.Uint32(unix.IFLA_BOND_ARP_VALIDATE, uint32(bond.ARPValidate)) + } + + encoder.Uint32(unix.IFLA_BOND_ARP_ALL_TARGETS, uint32(bond.ARPAllTargets)) + + if bond.Mode == nethelpers.BondModeActiveBackup || bond.Mode == nethelpers.BondModeALB || bond.Mode == nethelpers.BondModeTLB { + encoder.Uint32(unix.IFLA_BOND_PRIMARY, bond.PrimaryIndex) + } + + encoder.Uint8(unix.IFLA_BOND_PRIMARY_RESELECT, uint8(bond.PrimaryReselect)) + encoder.Uint8(unix.IFLA_BOND_FAIL_OVER_MAC, uint8(bond.FailOverMac)) + encoder.Uint8(unix.IFLA_BOND_AD_SELECT, uint8(bond.ADSelect)) + encoder.Uint32(unix.IFLA_BOND_MIIMON, bond.MIIMon) + + if bond.MIIMon != 0 { + encoder.Uint32(unix.IFLA_BOND_UPDELAY, bond.UpDelay) + encoder.Uint32(unix.IFLA_BOND_DOWNDELAY, bond.DownDelay) + } + + if bond.Mode != nethelpers.BondMode8023AD && bond.Mode != nethelpers.BondModeALB && bond.Mode != nethelpers.BondModeTLB { + encoder.Uint32(unix.IFLA_BOND_ARP_INTERVAL, bond.ARPInterval) + } + + encoder.Uint32(unix.IFLA_BOND_RESEND_IGMP, bond.ResendIGMP) + encoder.Uint32(unix.IFLA_BOND_MIN_LINKS, bond.MinLinks) + encoder.Uint32(unix.IFLA_BOND_LP_INTERVAL, bond.LPInterval) + + if bond.Mode == nethelpers.BondModeRoundrobin { + encoder.Uint32(unix.IFLA_BOND_PACKETS_PER_SLAVE, bond.PacketsPerSlave) + } + + encoder.Uint8(unix.IFLA_BOND_NUM_PEER_NOTIF, bond.NumPeerNotif) + + if bond.Mode == nethelpers.BondModeALB || bond.Mode == nethelpers.BondModeTLB { + encoder.Uint8(unix.IFLA_BOND_TLB_DYNAMIC_LB, bond.TLBDynamicLB) + } + + encoder.Uint8(unix.IFLA_BOND_ALL_SLAVES_ACTIVE, bond.AllSlavesActive) + + var useCarrier uint8 + + if bond.UseCarrier { + useCarrier = 1 + } + + encoder.Uint8(unix.IFLA_BOND_USE_CARRIER, useCarrier) + + if bond.Mode == nethelpers.BondMode8023AD { + encoder.Uint16(unix.IFLA_BOND_AD_ACTOR_SYS_PRIO, bond.ADActorSysPrio) + encoder.Uint16(unix.IFLA_BOND_AD_USER_PORT_KEY, bond.ADUserPortKey) + } + + if bond.MIIMon != 0 { + encoder.Uint32(unix.IFLA_BOND_PEER_NOTIF_DELAY, bond.PeerNotifyDelay) + } + + return encoder.Encode() +} + +// Decode the BondMasterSpec from netlink attributes. +// +//nolint:gocyclo,cyclop +func (a bondMaster) Decode(data []byte) error { + bond := a.BondMasterSpec + + decoder, err := netlink.NewAttributeDecoder(data) + if err != nil { + return err + } + + for decoder.Next() { + switch decoder.Type() { + case unix.IFLA_BOND_MODE: + bond.Mode = nethelpers.BondMode(decoder.Uint8()) + case unix.IFLA_BOND_XMIT_HASH_POLICY: + bond.HashPolicy = nethelpers.BondXmitHashPolicy(decoder.Uint8()) + case unix.IFLA_BOND_AD_LACP_RATE: + bond.LACPRate = nethelpers.LACPRate(decoder.Uint8()) + case unix.IFLA_BOND_ARP_VALIDATE: + bond.ARPValidate = nethelpers.ARPValidate(decoder.Uint32()) + case unix.IFLA_BOND_ARP_ALL_TARGETS: + bond.ARPAllTargets = nethelpers.ARPAllTargets(decoder.Uint32()) + case unix.IFLA_BOND_PRIMARY: + bond.PrimaryIndex = decoder.Uint32() + case unix.IFLA_BOND_PRIMARY_RESELECT: + bond.PrimaryReselect = nethelpers.PrimaryReselect(decoder.Uint8()) + case unix.IFLA_BOND_FAIL_OVER_MAC: + bond.FailOverMac = nethelpers.FailOverMAC(decoder.Uint8()) + case unix.IFLA_BOND_AD_SELECT: + bond.ADSelect = nethelpers.ADSelect(decoder.Uint8()) + case unix.IFLA_BOND_MIIMON: + bond.MIIMon = decoder.Uint32() + case unix.IFLA_BOND_UPDELAY: + bond.UpDelay = decoder.Uint32() + case unix.IFLA_BOND_DOWNDELAY: + bond.DownDelay = decoder.Uint32() + case unix.IFLA_BOND_ARP_INTERVAL: + bond.ARPInterval = decoder.Uint32() + case unix.IFLA_BOND_RESEND_IGMP: + bond.ResendIGMP = decoder.Uint32() + case unix.IFLA_BOND_MIN_LINKS: + bond.MinLinks = decoder.Uint32() + case unix.IFLA_BOND_LP_INTERVAL: + bond.LPInterval = decoder.Uint32() + case unix.IFLA_BOND_PACKETS_PER_SLAVE: + bond.PacketsPerSlave = decoder.Uint32() + case unix.IFLA_BOND_NUM_PEER_NOTIF: + bond.NumPeerNotif = decoder.Uint8() + case unix.IFLA_BOND_TLB_DYNAMIC_LB: + bond.TLBDynamicLB = decoder.Uint8() + case unix.IFLA_BOND_ALL_SLAVES_ACTIVE: + bond.AllSlavesActive = decoder.Uint8() + case unix.IFLA_BOND_USE_CARRIER: + bond.UseCarrier = decoder.Uint8() == 1 + case unix.IFLA_BOND_AD_ACTOR_SYS_PRIO: + bond.ADActorSysPrio = decoder.Uint16() + case unix.IFLA_BOND_AD_USER_PORT_KEY: + bond.ADUserPortKey = decoder.Uint16() + case unix.IFLA_BOND_PEER_NOTIF_DELAY: + bond.PeerNotifyDelay = decoder.Uint32() + } + } + + return decoder.Err() +} diff --git a/internal/app/machined/pkg/adapters/network/bond_master_spec_test.go b/internal/app/machined/pkg/adapters/network/bond_master_spec_test.go new file mode 100644 index 0000000..1fed6f9 --- /dev/null +++ b/internal/app/machined/pkg/adapters/network/bond_master_spec_test.go @@ -0,0 +1,33 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + networkadapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/network" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +func TestBondMasterSpec(t *testing.T) { + spec := network.BondMasterSpec{ + Mode: nethelpers.BondModeActiveBackup, + MIIMon: 100, + UpDelay: 200, + DownDelay: 300, + } + + b, err := networkadapter.BondMasterSpec(&spec).Encode() + require.NoError(t, err) + + var decodedSpec network.BondMasterSpec + + require.NoError(t, networkadapter.BondMasterSpec(&decodedSpec).Decode(b)) + + require.Equal(t, spec, decodedSpec) +} diff --git a/internal/app/machined/pkg/adapters/network/bridge_master_spec.go b/internal/app/machined/pkg/adapters/network/bridge_master_spec.go new file mode 100644 index 0000000..35ca5d4 --- /dev/null +++ b/internal/app/machined/pkg/adapters/network/bridge_master_spec.go @@ -0,0 +1,60 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "github.com/mdlayher/netlink" + "golang.org/x/sys/unix" + + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// BridgeMasterSpec adapter provides encoding/decoding to netlink structures. +// +//nolint:revive +func BridgeMasterSpec(r *network.BridgeMasterSpec) bridgeMaster { + return bridgeMaster{ + BridgeMasterSpec: r, + } +} + +// bridgeMaster contains the bridge master spec and provides methods for encoding/decoding it to netlink structures. +type bridgeMaster struct { + *network.BridgeMasterSpec +} + +// Encode the BridgeMasterSpec into netlink attributes. +func (a bridgeMaster) Encode() ([]byte, error) { + bridge := a.BridgeMasterSpec + + encoder := netlink.NewAttributeEncoder() + + stpEnabled := 0 + if bridge.STP.Enabled { + stpEnabled = 1 + } + + encoder.Uint32(unix.IFLA_BR_STP_STATE, uint32(stpEnabled)) + + return encoder.Encode() +} + +// Decode the BridgeMasterSpec from netlink attributes. +func (a bridgeMaster) Decode(data []byte) error { + bridge := a.BridgeMasterSpec + + decoder, err := netlink.NewAttributeDecoder(data) + if err != nil { + return err + } + + for decoder.Next() { + if decoder.Type() == unix.IFLA_BR_STP_STATE { + bridge.STP.Enabled = decoder.Uint32() == 1 + } + } + + return decoder.Err() +} diff --git a/internal/app/machined/pkg/adapters/network/ipset.go b/internal/app/machined/pkg/adapters/network/ipset.go new file mode 100644 index 0000000..326ffde --- /dev/null +++ b/internal/app/machined/pkg/adapters/network/ipset.go @@ -0,0 +1,39 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "net/netip" + + "go4.org/netipx" +) + +// BuildIPSet builds an IPSet from the given include and exclude prefixes. +func BuildIPSet(include, exclude []netip.Prefix) (*netipx.IPSet, error) { + var builder netipx.IPSetBuilder + + for _, pfx := range include { + builder.AddPrefix(pfx) + } + + for _, pfx := range exclude { + builder.RemovePrefix(pfx) + } + + return builder.IPSet() +} + +// SplitIPSet splits the given IPSet into IPv4 and IPv6 ranges. +func SplitIPSet(set *netipx.IPSet) (ipv4, ipv6 []netipx.IPRange) { + for _, rng := range set.Ranges() { + if rng.From().Is4() { + ipv4 = append(ipv4, rng) + } else { + ipv6 = append(ipv6, rng) + } + } + + return +} diff --git a/internal/app/machined/pkg/adapters/network/ipset_test.go b/internal/app/machined/pkg/adapters/network/ipset_test.go new file mode 100644 index 0000000..69b826d --- /dev/null +++ b/internal/app/machined/pkg/adapters/network/ipset_test.go @@ -0,0 +1,58 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network_test + +import ( + "net/netip" + "testing" + + "github.com/siderolabs/gen/xslices" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go4.org/netipx" + + "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/network" +) + +func TestBuildIPSet(t *testing.T) { + ipset, err := network.BuildIPSet( + []netip.Prefix{ + netip.MustParsePrefix("10.0.0.0/8"), + netip.MustParsePrefix("2001:db8::/32"), + }, + []netip.Prefix{ + netip.MustParsePrefix("10.4.0.0/16"), + }) + require.NoError(t, err) + + assert.Equal(t, + []string{"10.0.0.0-10.3.255.255", "10.5.0.0-10.255.255.255", "2001:db8::-2001:db8:ffff:ffff:ffff:ffff:ffff:ffff"}, + xslices.Map(ipset.Ranges(), netipx.IPRange.String), + ) +} + +func TestSplitIPSet(t *testing.T) { + ipset, err := network.BuildIPSet( + []netip.Prefix{ + netip.MustParsePrefix("10.0.0.0/8"), + netip.MustParsePrefix("2001:db8::/32"), + }, + []netip.Prefix{ + netip.MustParsePrefix("10.4.0.0/16"), + }) + require.NoError(t, err) + + v4, v6 := network.SplitIPSet(ipset) + + assert.Equal(t, + []string{"10.0.0.0-10.3.255.255", "10.5.0.0-10.255.255.255"}, + xslices.Map(v4, netipx.IPRange.String), + ) + + assert.Equal(t, + []string{"2001:db8::-2001:db8:ffff:ffff:ffff:ffff:ffff:ffff"}, + xslices.Map(v6, netipx.IPRange.String), + ) +} diff --git a/internal/app/machined/pkg/adapters/network/network.go b/internal/app/machined/pkg/adapters/network/network.go new file mode 100644 index 0000000..1381663 --- /dev/null +++ b/internal/app/machined/pkg/adapters/network/network.go @@ -0,0 +1,6 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package network implements adapters wrapping resources/network to provide additional functionality. +package network diff --git a/internal/app/machined/pkg/adapters/network/nftables_rule.go b/internal/app/machined/pkg/adapters/network/nftables_rule.go new file mode 100644 index 0000000..618b8d1 --- /dev/null +++ b/internal/app/machined/pkg/adapters/network/nftables_rule.go @@ -0,0 +1,697 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "fmt" + "net/netip" + "os" + "slices" + + "github.com/google/nftables" + "github.com/google/nftables/binaryutil" + "github.com/google/nftables/expr" + "github.com/siderolabs/gen/xslices" + "go4.org/netipx" + "golang.org/x/sys/unix" + + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// NfTablesRule adapter provides encoding to nftables instructions. +// +//nolint:revive,golint +func NfTablesRule(r *network.NfTablesRule) nftablesRule { + return nftablesRule{ + NfTablesRule: r, + } +} + +type nftablesRule struct { + *network.NfTablesRule +} + +// SetKind is the type of the nftables Set. +type SetKind uint8 + +// SetKind constants. +const ( + SetKindIPv4 SetKind = iota + SetKindIPv6 + SetKindPort + SetKindIfName + SetKindConntrackState +) + +// NfTablesSet is a compiled representation of the set. +type NfTablesSet struct { + Kind SetKind + Addresses []netipx.IPRange + Ports [][2]uint16 + Strings [][]byte + ConntrackStates []nethelpers.ConntrackState +} + +// IsInterval returns true if the set is an interval set. +func (set NfTablesSet) IsInterval() bool { + switch set.Kind { + case SetKindIPv4, SetKindIPv6, SetKindPort: + return true + case SetKindIfName, SetKindConntrackState: + return false + default: + panic(fmt.Sprintf("unknown set kind: %d", set.Kind)) + } +} + +// KeyType returns the type of the set. +func (set NfTablesSet) KeyType() nftables.SetDatatype { + switch set.Kind { + case SetKindIPv4: + return nftables.TypeIPAddr + case SetKindIPv6: + return nftables.TypeIP6Addr + case SetKindPort: + return nftables.TypeInetService + case SetKindIfName: + return nftables.TypeIFName + case SetKindConntrackState: + return nftables.TypeCTState + default: + panic(fmt.Sprintf("unknown set kind: %d", set.Kind)) + } +} + +// SetElements returns the set elements. +func (set NfTablesSet) SetElements() []nftables.SetElement { + switch set.Kind { + case SetKindIPv4, SetKindIPv6: + elements := make([]nftables.SetElement, 0, len(set.Addresses)*2) + + for _, r := range set.Addresses { + fromBin, _ := r.From().MarshalBinary() //nolint:errcheck // doesn't fail + toBin, _ := r.To().Next().MarshalBinary() //nolint:errcheck // doesn't fail + + elements = append(elements, + nftables.SetElement{ + Key: fromBin, + IntervalEnd: false, + }, + nftables.SetElement{ + Key: toBin, + IntervalEnd: true, + }, + ) + } + + return elements + case SetKindPort: + elements := make([]nftables.SetElement, 0, len(set.Ports)) + + for _, p := range set.Ports { + from := binaryutil.BigEndian.PutUint16(p[0]) + to := binaryutil.BigEndian.PutUint16(p[1] + 1) + + elements = append(elements, + nftables.SetElement{ + Key: from, + IntervalEnd: false, + }, + nftables.SetElement{ + Key: to, + IntervalEnd: true, + }, + ) + } + + return elements + case SetKindIfName: + elements := make([]nftables.SetElement, 0, len(set.Strings)) + + for _, s := range set.Strings { + elements = append(elements, + nftables.SetElement{ + Key: s, + }, + ) + } + + return elements + case SetKindConntrackState: + elements := make([]nftables.SetElement, 0, len(set.ConntrackStates)) + + for _, s := range set.ConntrackStates { + elements = append(elements, + nftables.SetElement{ + Key: binaryutil.NativeEndian.PutUint32(uint32(s)), + }, + ) + } + + return elements + default: + panic(fmt.Sprintf("unknown set kind: %d", set.Kind)) + } +} + +// NfTablesCompiled is a compiled representation of the rule. +type NfTablesCompiled struct { + Rules [][]expr.Any + Sets []NfTablesSet +} + +var ( + matchV4 = []expr.Any{ + // Store protocol type to register 1 + &expr.Meta{ + Key: expr.MetaKeyNFPROTO, + Register: 1, + }, + // Match IP Family + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{byte(nftables.TableFamilyIPv4)}, + }, + } + + matchV6 = []expr.Any{ + // Store protocol type to register 1 + &expr.Meta{ + Key: expr.MetaKeyNFPROTO, + Register: 1, + }, + // Match IP Family + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{byte(nftables.TableFamilyIPv6)}, + }, + } + + firstIPv4 = netip.MustParseAddr("0.0.0.0") + lastIPv4 = netip.MustParseAddr("255.255.255.255") + + firstIPv6 = netip.MustParseAddr("::") + lastIPv6 = netip.MustParseAddr("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff") +) + +// Compile translates the rule into the set of nftables instructions. +// +//nolint:gocyclo,cyclop +func (a nftablesRule) Compile() (*NfTablesCompiled, error) { + var ( + // common for ipv4 & ipv6 expression, pre & post + rulePre []expr.Any + rulePost []expr.Any + // speficic for ipv4 & ipv6 expression + rule4, rule6 []expr.Any + + result NfTablesCompiled + ) + + matchIfNames := func(operator nethelpers.MatchOperator, ifnames []string) { + if len(ifnames) == 1 { + rulePre = append(rulePre, + // [ cmp eq/neq reg 1 ] + &expr.Cmp{ + Op: expr.CmpOp(operator), + Register: 1, + Data: ifname(ifnames[0]), + }, + ) + } else { + result.Sets = append(result.Sets, + NfTablesSet{ + Kind: SetKindIfName, + Strings: xslices.Map(ifnames, ifname), + }) + + rulePre = append(rulePre, + // Match from target set + &expr.Lookup{ + SourceRegister: 1, + SetID: uint32(len(result.Sets) - 1), // reference will be fixed up by the controller + Invert: operator == nethelpers.OperatorNotEqual, + }, + ) + } + } + + if a.NfTablesRule.MatchIIfName != nil { + match := a.NfTablesRule.MatchIIfName + + rulePre = append(rulePre, + // [ meta load iifname => reg 1 ] + &expr.Meta{ + Key: expr.MetaKeyIIFNAME, + Register: 1, + }, + ) + + matchIfNames(match.Operator, match.InterfaceNames) + } + + if a.NfTablesRule.MatchOIfName != nil { + match := a.NfTablesRule.MatchOIfName + + rulePre = append(rulePre, + // [ meta load oifname => reg 1 ] + &expr.Meta{ + Key: expr.MetaKeyOIFNAME, + Register: 1, + }, + ) + + matchIfNames(match.Operator, match.InterfaceNames) + } + + if a.NfTablesRule.MatchMark != nil { + match := a.NfTablesRule.MatchMark + + rulePre = append(rulePre, + // [ meta load mark => reg 1 ] + &expr.Meta{ + Key: expr.MetaKeyMARK, + Register: 1, + }, + // Mask the mark with the configured mask: + // R1 = R1 & mask ^ xor + &expr.Bitwise{ + SourceRegister: 1, + DestRegister: 1, + Len: 4, + Xor: binaryutil.NativeEndian.PutUint32(match.Xor), + Mask: binaryutil.NativeEndian.PutUint32(match.Mask), + }, + // Compare the masked firewall mark with expected value + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: binaryutil.NativeEndian.PutUint32(match.Value), + }, + ) + } + + if a.NfTablesRule.MatchConntrackState != nil { + match := a.NfTablesRule.MatchConntrackState + + if len(match.States) == 1 { + rulePre = append(rulePre, + // [ ct load state => reg 1 ] + &expr.Ct{ + Key: expr.CtKeySTATE, + Register: 1, + }, + // [ bitwise reg 1 = ( reg 1 & state ) ^ 0x00000000 ] + &expr.Bitwise{ + SourceRegister: 1, + DestRegister: 1, + Len: 4, + Mask: binaryutil.NativeEndian.PutUint32(uint32(match.States[0])), + Xor: []byte{0x0, 0x0, 0x0, 0x0}, + }, + // [ cmp neq reg 1 0x00000000 ] + &expr.Cmp{ + Op: expr.CmpOpNeq, + Register: 1, + Data: []byte{0x0, 0x0, 0x0, 0x0}, + }, + ) + } else { + result.Sets = append(result.Sets, + NfTablesSet{ + Kind: SetKindConntrackState, + ConntrackStates: match.States, + }) + + rulePre = append(rulePre, + // [ ct load state => reg 1 ] + &expr.Ct{ + Key: expr.CtKeySTATE, + Register: 1, + }, + // [ lookup reg 1 set ] + &expr.Lookup{ + SourceRegister: 1, + SetID: uint32(len(result.Sets) - 1), // reference will be fixed up by the controller + }, + ) + } + } + + addressMatchExpression := func(match *network.NfTablesAddressMatch, label string, offV4, offV6 uint32) error { + ipSet, err := BuildIPSet(match.IncludeSubnets, match.ExcludeSubnets) + if err != nil { + return fmt.Errorf("failed to build IPSet for %s address match: %w", label, err) + } + + v4Set, v6Set := SplitIPSet(ipSet) + + if v4Set == nil && v6Set == nil && !match.Invert { + // this rule doesn't match anything + return os.ErrNotExist + } + + v4SetCoversAll := len(v4Set) == 1 && v4Set[0].From() == firstIPv4 && v4Set[0].To() == lastIPv4 + v6SetCoversAll := len(v6Set) == 1 && v6Set[0].From() == firstIPv6 && v6Set[0].To() == lastIPv6 + + if v4SetCoversAll && v6SetCoversAll && match.Invert { + // this rule doesn't match anything + return os.ErrNotExist + } + + switch { //nolint:dupl + case v4SetCoversAll && !match.Invert, match.Invert && v4Set == nil: + // match any v4 IP + if rule4 == nil { + rule4 = []expr.Any{} + } + case !v4SetCoversAll && match.Invert, !match.Invert && v4Set != nil: + // match specific v4 IPs + result.Sets = append(result.Sets, + NfTablesSet{ + Kind: SetKindIPv4, + Addresses: v4Set, + }, + ) + + rule4 = append(rule4, + // Store the destination IP address to register 1 + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseNetworkHeader, + Offset: offV4, + Len: 4, + }, + // Match from target set + &expr.Lookup{ + SourceRegister: 1, + SetID: uint32(len(result.Sets) - 1), // reference will be fixed up by the controller + Invert: match.Invert, + }, + ) + default: // otherwise skip generating v4 rule, as it doesn't match anything + } + + switch { //nolint:dupl + case v6SetCoversAll && !match.Invert, match.Invert && v6Set == nil: + // match any v6 IP + if rule6 == nil { + rule6 = []expr.Any{} + } + case !v6SetCoversAll && match.Invert, !match.Invert && v6Set != nil: + // match specific v6 IPs + result.Sets = append(result.Sets, + NfTablesSet{ + Kind: SetKindIPv6, + Addresses: v6Set, + }) + + rule6 = append(rule6, + // Store the destination IP address to register 1 + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseNetworkHeader, + Offset: offV6, + Len: 16, + }, + // Match from target set + &expr.Lookup{ + SourceRegister: 1, + SetID: uint32(len(result.Sets) - 1), // reference will be fixed up by the controller + Invert: match.Invert, + }, + ) + default: // otherwise skip generating v6 rule, as it doesn't match anything + } + + return nil + } + + if a.NfTablesRule.MatchSourceAddress != nil { + match := a.NfTablesRule.MatchSourceAddress + + if err := addressMatchExpression(match, "source", 12, 8); err != nil { + if os.IsNotExist(err) { + return &NfTablesCompiled{}, nil + } + + return nil, err + } + } + + if a.NfTablesRule.MatchDestinationAddress != nil { + match := a.NfTablesRule.MatchDestinationAddress + + if err := addressMatchExpression(match, "destination", 16, 24); err != nil { + if os.IsNotExist(err) { + return &NfTablesCompiled{}, nil + } + + return nil, err + } + } + + if a.NfTablesRule.MatchLayer4 != nil { + match := a.NfTablesRule.MatchLayer4 + + rulePre = append(rulePre, + // [ meta load l4proto => reg 1 ] + &expr.Meta{ + Key: expr.MetaKeyL4PROTO, + Register: 1, + }, + // [ cmp eq reg 1 ] + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{byte(match.Protocol)}, + }, + ) + + portMatch := func(off uint32, ports []network.PortRange) { + result.Sets = append(result.Sets, + NfTablesSet{ + Kind: SetKindPort, + Ports: xslices.Map(ports, func(r network.PortRange) [2]uint16 { return [2]uint16{r.Lo, r.Hi} }), + }, + ) + + rulePost = append(rulePost, + // [ payload load 2b @ transport header + => reg 1 ] + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseTransportHeader, + Offset: off, + Len: 2, + }, + // [ lookup reg 1 set ] + &expr.Lookup{ + SourceRegister: 1, + SetID: uint32(len(result.Sets) - 1), // reference will be fixed up by the controller + }, + ) + } + + if match.MatchSourcePort != nil { + portMatch(0, match.MatchSourcePort.Ranges) + } + + if match.MatchDestinationPort != nil { + portMatch(2, match.MatchDestinationPort.Ranges) + } + } + + if a.NfTablesRule.MatchLimit != nil { + match := a.NfTablesRule.MatchLimit + + rulePost = append(rulePost, + // [ limit rate ] + &expr.Limit{ + Type: expr.LimitTypePkts, + Rate: match.PacketRatePerSecond, + Burst: uint32(match.PacketRatePerSecond), + Unit: expr.LimitTimeSecond, + }, + ) + } + + clampMSS := func(family nftables.TableFamily, mtu uint16) []expr.Any { + var mss uint16 + + switch family { //nolint:exhaustive + case nftables.TableFamilyIPv4: + mss = mtu - 40 // TCP + IPv4 overhead + case nftables.TableFamilyIPv6: + mss = mtu - 60 // TCP + IPv6 overhead + default: + panic("unexpected IP family") + } + + return []expr.Any{ + // Load the L4 protocol into register 1 + &expr.Meta{ + Key: expr.MetaKeyL4PROTO, + Register: 1, + }, + // Match TCP Family + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{unix.IPPROTO_TCP}, + }, + // [ payload load 1b @ transport header + 13 => reg 1 ] + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseTransportHeader, + Offset: 13, + Len: 1, + }, + // [ bitwise reg 1 = ( reg 1 & 0x00000006 ) ^ 0x00000000 ] + &expr.Bitwise{ + DestRegister: 1, + SourceRegister: 1, + Len: 1, + Mask: []byte{0x02 | 0x04}, + Xor: []byte{0x00}, + }, + // [ cmp eq reg 1 0x00000002 ] + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{0x02}, + }, + // [ exthdr load tcpopt 2b @ 2 + 2 => reg 1 ] + &expr.Exthdr{ + DestRegister: 1, + Type: 2, + Offset: 2, + Len: 2, + Op: expr.ExthdrOpTcpopt, + }, + // [ cmp gte reg 1 MTU ] + &expr.Cmp{ + Op: expr.CmpOpGt, + Register: 1, + Data: binaryutil.BigEndian.PutUint16(mss), + }, + // [ immediate reg 1 MTU ] + &expr.Immediate{ + Register: 1, + Data: binaryutil.BigEndian.PutUint16(mss), + }, + // [ exthdr write tcpopt reg 1 => 2b @ 2 + 2 ] + &expr.Exthdr{ + SourceRegister: 1, + Type: 2, + Offset: 2, + Len: 2, + Op: expr.ExthdrOpTcpopt, + }, + } + } + + if a.NfTablesRule.ClampMSS != nil { + rule4 = append(rule4, clampMSS(nftables.TableFamilyIPv4, a.NfTablesRule.ClampMSS.MTU)...) + rule6 = append(rule6, clampMSS(nftables.TableFamilyIPv6, a.NfTablesRule.ClampMSS.MTU)...) + } + + if a.NfTablesRule.SetMark != nil { + set := a.NfTablesRule.SetMark + + rulePost = append(rulePost, + // Load the current packet mark into register 1 + &expr.Meta{ + Key: expr.MetaKeyMARK, + Register: 1, + }, + // Calculate the new mark value in register 1 + &expr.Bitwise{ + SourceRegister: 1, + DestRegister: 1, + Len: 4, + Xor: binaryutil.NativeEndian.PutUint32(set.Xor), + Mask: binaryutil.NativeEndian.PutUint32(set.Mask), + }, + // Set firewall mark to the value computed in register 1 + &expr.Meta{ + Key: expr.MetaKeyMARK, + SourceRegister: true, + Register: 1, + }, + ) + } + + if a.NfTablesRule.AnonCounter { + rulePost = append(rulePost, + // [ counter ] + &expr.Counter{}, + ) + } + + if a.NfTablesRule.Verdict != nil { + rulePost = append(rulePost, + // [ verdict accept|drop ] + &expr.Verdict{ + Kind: expr.VerdictKind(*a.NfTablesRule.Verdict), + }, + ) + } + + // Build v4/v6 rules as requested. + // + // If there's no IPv4/IPv6 part, generate a single rule. + // If there's a specific IPv4/IPv6 part, generate a rule per IP version. + switch { + case rule4 == nil && rule6 == nil && rulePre == nil && rulePost == nil: + // nothing + case rule4 == nil && rule6 == nil: + result.Rules = [][]expr.Any{append(rulePre, rulePost...)} + case rule4 != nil && rule6 == nil: + result.Rules = [][]expr.Any{ + append(rulePre, + append( + append(matchV4, rule4...), + rulePost..., + )..., + ), + } + case rule4 == nil && rule6 != nil: + result.Rules = [][]expr.Any{ + append(rulePre, + append( + append(matchV6, rule6...), + rulePost..., + )..., + ), + } + case rule4 != nil && rule6 != nil: + result.Rules = [][]expr.Any{ + append(slices.Clone(rulePre), + append( + append(matchV4, rule4...), + rulePost..., + )..., + ), + append(slices.Clone(rulePre), + append( + append(matchV6, rule6...), + rulePost..., + )..., + ), + } + } + + return &result, nil +} + +func ifname(name string) []byte { + b := make([]byte, 16) + copy(b, []byte(name)) + + return b +} diff --git a/internal/app/machined/pkg/adapters/network/nftables_rule_test.go b/internal/app/machined/pkg/adapters/network/nftables_rule_test.go new file mode 100644 index 0000000..2c873c0 --- /dev/null +++ b/internal/app/machined/pkg/adapters/network/nftables_rule_test.go @@ -0,0 +1,715 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network_test + +import ( + "net/netip" + "testing" + + "github.com/google/nftables" + "github.com/google/nftables/expr" + "github.com/siderolabs/go-pointer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go4.org/netipx" + + "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/network" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + networkres "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +func TestNfTablesRuleCompile(t *testing.T) { //nolint:tparallel + t.Parallel() + + for _, test := range []struct { + name string + + spec networkres.NfTablesRule + + expectedRules [][]expr.Any + expectedSets []network.NfTablesSet + }{ + { + name: "empty", + }, + { + name: "match oifname", + spec: networkres.NfTablesRule{ + MatchOIfName: &networkres.NfTablesIfNameMatch{ + InterfaceNames: []string{"eth0"}, + Operator: nethelpers.OperatorEqual, + }, + }, + expectedRules: [][]expr.Any{ + { + &expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte("eth0\000\000\000\000\000\000\000\000\000\000\000\000"), + }, + }, + }, + }, + { + name: "match iifname", + spec: networkres.NfTablesRule{ + MatchIIfName: &networkres.NfTablesIfNameMatch{ + InterfaceNames: []string{"lo"}, + Operator: nethelpers.OperatorNotEqual, + }, + }, + expectedRules: [][]expr.Any{ + { + &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpNeq, + Register: 1, + Data: []byte("lo\000\000\000\000\000\000\000\000\000\000\000\000\000\000"), + }, + }, + }, + }, + { + name: "match multiple iifname", + spec: networkres.NfTablesRule{ + MatchIIfName: &networkres.NfTablesIfNameMatch{ + InterfaceNames: []string{"siderolink", "kubespan"}, + Operator: nethelpers.OperatorEqual, + }, + }, + expectedRules: [][]expr.Any{ + { + &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, + &expr.Lookup{ + SourceRegister: 1, + SetID: 0, + }, + }, + }, + expectedSets: []network.NfTablesSet{ + { + Kind: network.SetKindIfName, + Strings: [][]byte{ + []byte("siderolink\000\000\000\000\000\000"), + []byte("kubespan\000\000\000\000\000\000\000\000"), + }, + }, + }, + }, + { + name: "verdict accept", + spec: networkres.NfTablesRule{ + MatchOIfName: &networkres.NfTablesIfNameMatch{ + InterfaceNames: []string{"eth0"}, + Operator: nethelpers.OperatorNotEqual, + }, + Verdict: pointer.To(nethelpers.VerdictAccept), + }, + expectedRules: [][]expr.Any{ + { + &expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpNeq, + Register: 1, + Data: []byte("eth0\000\000\000\000\000\000\000\000\000\000\000\000"), + }, + &expr.Verdict{Kind: expr.VerdictAccept}, + }, + }, + }, + { + name: "match and set mark", + spec: networkres.NfTablesRule{ + MatchMark: &networkres.NfTablesMark{ + Mask: 0xff00ffff, + Xor: 0x00ff0000, + Value: 0x00ee0000, + }, + SetMark: &networkres.NfTablesMark{ + Mask: 0x0000ffff, + Xor: 0xffff0000, + }, + }, + expectedRules: [][]expr.Any{ + { + &expr.Meta{Key: expr.MetaKeyMARK, Register: 1}, + &expr.Bitwise{ + SourceRegister: 1, + DestRegister: 1, + Len: 4, + Xor: []byte{0x00, 0x00, 0xff, 0x00}, + Mask: []byte{0xff, 0xff, 0x00, 0xff}, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{0x00, 0x00, 0xee, 0x00}, + }, + &expr.Meta{Key: expr.MetaKeyMARK, Register: 1}, + &expr.Bitwise{ + SourceRegister: 1, + DestRegister: 1, + Len: 4, + Xor: []byte{0x00, 0x00, 0xff, 0xff}, + Mask: []byte{0xff, 0xff, 0x00, 0x00}, + }, + &expr.Meta{Key: expr.MetaKeyMARK, SourceRegister: true, Register: 1}, + }, + }, + }, + { + name: "match on empty source address", + spec: networkres.NfTablesRule{ + MatchSourceAddress: &networkres.NfTablesAddressMatch{}, + Verdict: pointer.To(nethelpers.VerdictDrop), + }, + }, + { + name: "match on v4 source address", + spec: networkres.NfTablesRule{ + MatchSourceAddress: &networkres.NfTablesAddressMatch{ + IncludeSubnets: []netip.Prefix{ + netip.MustParsePrefix("192.168.0.0/16"), + }, + ExcludeSubnets: []netip.Prefix{ + netip.MustParsePrefix("192.168.4.0/24"), + }, + }, + Verdict: pointer.To(nethelpers.VerdictDrop), + }, + expectedRules: [][]expr.Any{ + { + &expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{byte(nftables.TableFamilyIPv4)}, + }, + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseNetworkHeader, + Offset: 12, + Len: 4, + }, + &expr.Lookup{ + SourceRegister: 1, + SetID: 0, + }, + &expr.Verdict{ + Kind: expr.VerdictDrop, + }, + }, + }, + expectedSets: []network.NfTablesSet{ + { + Kind: network.SetKindIPv4, + Addresses: []netipx.IPRange{netipx.MustParseIPRange("192.168.0.0-192.168.3.255"), netipx.MustParseIPRange("192.168.5.0-192.168.255.255")}, + }, + }, + }, + { + name: "match on v6 source and destination addresses", + spec: networkres.NfTablesRule{ + MatchSourceAddress: &networkres.NfTablesAddressMatch{ + IncludeSubnets: []netip.Prefix{ + netip.MustParsePrefix("2001::/16"), + }, + }, + MatchDestinationAddress: &networkres.NfTablesAddressMatch{ + IncludeSubnets: []netip.Prefix{ + netip.MustParsePrefix("20fe::/16"), + }, + Invert: true, + }, + Verdict: pointer.To(nethelpers.VerdictDrop), + }, + expectedRules: [][]expr.Any{ + { + &expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{byte(nftables.TableFamilyIPv4)}, + }, + &expr.Verdict{ + Kind: expr.VerdictDrop, + }, + }, + { + &expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{byte(nftables.TableFamilyIPv6)}, + }, + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseNetworkHeader, + Offset: 8, + Len: 16, + }, + &expr.Lookup{ + SourceRegister: 1, + SetID: 0, + }, + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseNetworkHeader, + Offset: 24, + Len: 16, + }, + &expr.Lookup{ + SourceRegister: 1, + SetID: 1, + Invert: true, + }, + &expr.Verdict{ + Kind: expr.VerdictDrop, + }, + }, + }, + expectedSets: []network.NfTablesSet{ + { + Kind: network.SetKindIPv6, + Addresses: []netipx.IPRange{netipx.MustParseIPRange("2001::-2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff")}, + }, + { + Kind: network.SetKindIPv6, + Addresses: []netipx.IPRange{netipx.MustParseIPRange("20fe::-20fe:ffff:ffff:ffff:ffff:ffff:ffff:ffff")}, + }, + }, + }, + { + name: "match on v6 destination addresses", + spec: networkres.NfTablesRule{ + MatchDestinationAddress: &networkres.NfTablesAddressMatch{ + IncludeSubnets: []netip.Prefix{ + netip.MustParsePrefix("20fe::/16"), + }, + }, + Verdict: pointer.To(nethelpers.VerdictDrop), + }, + expectedRules: [][]expr.Any{ + { + &expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{byte(nftables.TableFamilyIPv6)}, + }, + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseNetworkHeader, + Offset: 24, + Len: 16, + }, + &expr.Lookup{ + SourceRegister: 1, + SetID: 0, + }, + &expr.Verdict{ + Kind: expr.VerdictDrop, + }, + }, + }, + expectedSets: []network.NfTablesSet{ + { + Kind: network.SetKindIPv6, + Addresses: []netipx.IPRange{netipx.MustParseIPRange("20fe::-20fe:ffff:ffff:ffff:ffff:ffff:ffff:ffff")}, + }, + }, + }, + { + name: "match on any v6 address", + spec: networkres.NfTablesRule{ + MatchSourceAddress: &networkres.NfTablesAddressMatch{ + IncludeSubnets: []netip.Prefix{ + netip.MustParsePrefix("192.168.37.45/32"), + }, + Invert: true, + }, + Verdict: pointer.To(nethelpers.VerdictDrop), + }, + expectedRules: [][]expr.Any{ + { + &expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{byte(nftables.TableFamilyIPv4)}, + }, + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseNetworkHeader, + Offset: 12, + Len: 4, + }, + &expr.Lookup{ + SourceRegister: 1, + SetID: 0, + Invert: true, + }, + &expr.Verdict{ + Kind: expr.VerdictDrop, + }, + }, + { + &expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{byte(nftables.TableFamilyIPv6)}, + }, + &expr.Verdict{ + Kind: expr.VerdictDrop, + }, + }, + }, + expectedSets: []network.NfTablesSet{ + { + Kind: network.SetKindIPv4, + Addresses: []netipx.IPRange{netipx.MustParseIPRange("192.168.37.45-192.168.37.45")}, + }, + }, + }, + { + name: "clamp MSS", + spec: networkres.NfTablesRule{ + ClampMSS: &networkres.NfTablesClampMSS{ + MTU: 1280, + }, + }, + expectedRules: [][]expr.Any{ + { //nolint:dupl + &expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{byte(nftables.TableFamilyIPv4)}, + }, + &expr.Meta{ + Key: expr.MetaKeyL4PROTO, + Register: 1, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{6}, + }, + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseTransportHeader, + Offset: 13, + Len: 1, + }, + &expr.Bitwise{ + DestRegister: 1, + SourceRegister: 1, + Len: 1, + Mask: []byte{0x02 | 0x04}, + Xor: []byte{0x00}, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{0x02}, + }, + &expr.Exthdr{ + DestRegister: 1, + Type: 2, + Offset: 2, + Len: 2, + Op: expr.ExthdrOpTcpopt, + }, + &expr.Cmp{ + Op: expr.CmpOpGt, + Register: 1, + Data: []byte{0x04, 0xd8}, + }, + &expr.Immediate{ + Register: 1, + Data: []byte{0x04, 0xd8}, + }, + &expr.Exthdr{ + SourceRegister: 1, + Type: 2, + Offset: 2, + Len: 2, + Op: expr.ExthdrOpTcpopt, + }, + }, + { //nolint:dupl + &expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{byte(nftables.TableFamilyIPv6)}, + }, + &expr.Meta{ + Key: expr.MetaKeyL4PROTO, + Register: 1, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{6}, + }, + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseTransportHeader, + Offset: 13, + Len: 1, + }, + &expr.Bitwise{ + DestRegister: 1, + SourceRegister: 1, + Len: 1, + Mask: []byte{0x02 | 0x04}, + Xor: []byte{0x00}, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{0x02}, + }, + &expr.Exthdr{ + DestRegister: 1, + Type: 2, + Offset: 2, + Len: 2, + Op: expr.ExthdrOpTcpopt, + }, + &expr.Cmp{ + Op: expr.CmpOpGt, + Register: 1, + Data: []byte{0x04, 0xc4}, + }, + &expr.Immediate{ + Register: 1, + Data: []byte{0x04, 0xc4}, + }, + &expr.Exthdr{ + SourceRegister: 1, + Type: 2, + Offset: 2, + Len: 2, + Op: expr.ExthdrOpTcpopt, + }, + }, + }, + }, + { + name: "match L4 proto", + spec: networkres.NfTablesRule{ + MatchLayer4: &networkres.NfTablesLayer4Match{ + Protocol: nethelpers.ProtocolUDP, + }, + }, + expectedRules: [][]expr.Any{ + { + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{0x11}, + }, + }, + }, + }, + { + name: "match L4 proto and src port", + spec: networkres.NfTablesRule{ + MatchLayer4: &networkres.NfTablesLayer4Match{ + Protocol: nethelpers.ProtocolTCP, + MatchSourcePort: &networkres.NfTablesPortMatch{ + Ranges: []networkres.PortRange{ + { + Lo: 1000, + Hi: 1025, + }, + { + Lo: 2000, + Hi: 2000, + }, + }, + }, + }, + }, + expectedRules: [][]expr.Any{ + { + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{0x6}, + }, + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseTransportHeader, + Offset: 0, + Len: 2, + }, + &expr.Lookup{ + SourceRegister: 1, + SetID: 0, + }, + }, + }, + expectedSets: []network.NfTablesSet{ + { + Kind: network.SetKindPort, + Ports: [][2]uint16{ + {1000, 1025}, + {2000, 2000}, + }, + }, + }, + }, + { + name: "match L4 proto and dst port", + spec: networkres.NfTablesRule{ + MatchLayer4: &networkres.NfTablesLayer4Match{ + Protocol: nethelpers.ProtocolTCP, + MatchDestinationPort: &networkres.NfTablesPortMatch{ + Ranges: []networkres.PortRange{ + { + Lo: 2000, + Hi: 2000, + }, + }, + }, + }, + }, + expectedRules: [][]expr.Any{ + { + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{0x6}, + }, + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseTransportHeader, + Offset: 2, + Len: 2, + }, + &expr.Lookup{ + SourceRegister: 1, + SetID: 0, + }, + }, + }, + expectedSets: []network.NfTablesSet{ + { + Kind: network.SetKindPort, + Ports: [][2]uint16{ + {2000, 2000}, + }, + }, + }, + }, + { + name: "limit", + spec: networkres.NfTablesRule{ + MatchLimit: &networkres.NfTablesLimitMatch{ + PacketRatePerSecond: 5, + }, + }, + expectedRules: [][]expr.Any{ + { + &expr.Limit{ + Type: expr.LimitTypePkts, + Rate: 5, + Burst: 5, + Unit: expr.LimitTimeSecond, + }, + }, + }, + }, + { + name: "counter", + spec: networkres.NfTablesRule{ + AnonCounter: true, + }, + expectedRules: [][]expr.Any{ + { + &expr.Counter{}, + }, + }, + }, + { + name: "ct state", + spec: networkres.NfTablesRule{ + MatchConntrackState: &networkres.NfTablesConntrackStateMatch{ + States: []nethelpers.ConntrackState{ + nethelpers.ConntrackStateInvalid, + }, + }, + }, + expectedRules: [][]expr.Any{ + { + &expr.Ct{ + Key: expr.CtKeySTATE, + Register: 1, + }, + &expr.Bitwise{ + DestRegister: 1, + SourceRegister: 1, + Len: 4, + Mask: []byte{0x01, 0x00, 0x00, 0x00}, + Xor: []byte{0x00, 0x00, 0x00, 0x00}, + }, + &expr.Cmp{ + Op: expr.CmpOpNeq, + Register: 1, + Data: []byte{0x00, 0x00, 0x00, 0x00}, + }, + }, + }, + }, + { + name: "ct states", + spec: networkres.NfTablesRule{ + MatchConntrackState: &networkres.NfTablesConntrackStateMatch{ + States: []nethelpers.ConntrackState{ + nethelpers.ConntrackStateRelated, + nethelpers.ConntrackStateEstablished, + }, + }, + }, + expectedRules: [][]expr.Any{ + { + &expr.Ct{ + Key: expr.CtKeySTATE, + Register: 1, + }, + &expr.Lookup{ + SourceRegister: 1, + SetID: 0, + }, + }, + }, + expectedSets: []network.NfTablesSet{ + { + Kind: network.SetKindConntrackState, + ConntrackStates: []nethelpers.ConntrackState{ + nethelpers.ConntrackStateRelated, + nethelpers.ConntrackStateEstablished, + }, + }, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + result, err := network.NfTablesRule(&test.spec).Compile() + require.NoError(t, err) + + assert.Equal(t, test.expectedRules, result.Rules) + assert.Equal(t, test.expectedSets, result.Sets) + }) + } +} diff --git a/internal/app/machined/pkg/adapters/network/vlan_spec.go b/internal/app/machined/pkg/adapters/network/vlan_spec.go new file mode 100644 index 0000000..220a4f1 --- /dev/null +++ b/internal/app/machined/pkg/adapters/network/vlan_spec.go @@ -0,0 +1,64 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "encoding/binary" + + "github.com/mdlayher/netlink" + "golang.org/x/sys/unix" + + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// VLANSpec adapter provides encoding/decoding to netlink structures. +// +//nolint:revive,golint +func VLANSpec(r *network.VLANSpec) vlanSpec { + return vlanSpec{ + VLANSpec: r, + } +} + +type vlanSpec struct { + *network.VLANSpec +} + +// Encode the VLANSpec into netlink attributes. +func (a vlanSpec) Encode() ([]byte, error) { + vlan := a.VLANSpec + + encoder := netlink.NewAttributeEncoder() + + encoder.Uint16(unix.IFLA_VLAN_ID, vlan.VID) + + buf := make([]byte, 2) + binary.BigEndian.PutUint16(buf, uint16(vlan.Protocol)) + encoder.Bytes(unix.IFLA_VLAN_PROTOCOL, buf) + + return encoder.Encode() +} + +// Decode the VLANSpec from netlink attributes. +func (a vlanSpec) Decode(data []byte) error { + vlan := a.VLANSpec + + decoder, err := netlink.NewAttributeDecoder(data) + if err != nil { + return err + } + + for decoder.Next() { + switch decoder.Type() { + case unix.IFLA_VLAN_ID: + vlan.VID = decoder.Uint16() + case unix.IFLA_VLAN_PROTOCOL: + vlan.Protocol = nethelpers.VLANProtocol(binary.BigEndian.Uint16(decoder.Bytes())) + } + } + + return decoder.Err() +} diff --git a/internal/app/machined/pkg/adapters/network/vlan_spec_test.go b/internal/app/machined/pkg/adapters/network/vlan_spec_test.go new file mode 100644 index 0000000..b6358b1 --- /dev/null +++ b/internal/app/machined/pkg/adapters/network/vlan_spec_test.go @@ -0,0 +1,31 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + networkadapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/network" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +func TestVLANSpec(t *testing.T) { + spec := network.VLANSpec{ + VID: 25, + Protocol: nethelpers.VLANProtocol8021AD, + } + + b, err := networkadapter.VLANSpec(&spec).Encode() + require.NoError(t, err) + + var decodedSpec network.VLANSpec + + require.NoError(t, networkadapter.VLANSpec(&decodedSpec).Decode(b)) + + require.Equal(t, spec, decodedSpec) +} diff --git a/internal/app/machined/pkg/adapters/network/wireguard_spec.go b/internal/app/machined/pkg/adapters/network/wireguard_spec.go new file mode 100644 index 0000000..84759b0 --- /dev/null +++ b/internal/app/machined/pkg/adapters/network/wireguard_spec.go @@ -0,0 +1,199 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "net" + "net/netip" + + "github.com/siderolabs/gen/xslices" + "go4.org/netipx" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// WireguardSpec adapter provides encoding/decoding to netlink structures. +// +//nolint:revive,golint +func WireguardSpec(r *network.WireguardSpec) wireguardSpec { + return wireguardSpec{ + WireguardSpec: r, + } +} + +type wireguardSpec struct { + *network.WireguardSpec +} + +// Encode converts WireguardSpec to wgctrl.Config "patch" to adjust the config to match the spec. +// +// Both specs should be sorted. +// +// Encode produces a "diff" as *wgtypes.Config which when applied transitions `existing` configuration into +// configuration `spec`. +// +//nolint:gocyclo,cyclop +func (a wireguardSpec) Encode(existing *network.WireguardSpec) (*wgtypes.Config, error) { + spec := a.WireguardSpec + + cfg := &wgtypes.Config{} + + if existing.PrivateKey != spec.PrivateKey { + key, err := wgtypes.ParseKey(spec.PrivateKey) + if err != nil { + return nil, err + } + + cfg.PrivateKey = &key + } + + if existing.ListenPort != spec.ListenPort { + cfg.ListenPort = &spec.ListenPort + } + + if existing.FirewallMark != spec.FirewallMark { + cfg.FirewallMark = &spec.FirewallMark + } + + // perform a merge of two sorted list of peers producing diff + l, r := 0, 0 + + for l < len(existing.Peers) || r < len(spec.Peers) { + addPeer := func(peer *network.WireguardPeer) error { + pubKey, err := wgtypes.ParseKey(peer.PublicKey) + if err != nil { + return err + } + + var presharedKey *wgtypes.Key + + if peer.PresharedKey != "" { + var parsedKey wgtypes.Key + + parsedKey, err = wgtypes.ParseKey(peer.PresharedKey) + if err != nil { + return err + } + + presharedKey = &parsedKey + } + + var endpoint *net.UDPAddr + + if peer.Endpoint != "" { + endpoint, err = net.ResolveUDPAddr("", peer.Endpoint) + if err != nil { + return err + } + } + + cfg.Peers = append(cfg.Peers, wgtypes.PeerConfig{ + PublicKey: pubKey, + Endpoint: endpoint, + PresharedKey: presharedKey, + PersistentKeepaliveInterval: &peer.PersistentKeepaliveInterval, + ReplaceAllowedIPs: true, + AllowedIPs: xslices.Map(peer.AllowedIPs, func(peerIP netip.Prefix) net.IPNet { + return *netipx.PrefixIPNet(peerIP) + }), + }) + + return nil + } + + deletePeer := func(peer *network.WireguardPeer) error { + pubKey, err := wgtypes.ParseKey(peer.PublicKey) + if err != nil { + return err + } + + cfg.Peers = append(cfg.Peers, wgtypes.PeerConfig{ + PublicKey: pubKey, + Remove: true, + }) + + return nil + } + + var left, right *network.WireguardPeer + + if l < len(existing.Peers) { + left = &existing.Peers[l] + } + + if r < len(spec.Peers) { + right = &spec.Peers[r] + } + + switch { + // peer from the "right" (new spec) is missing in "existing" (left), add it + case left == nil || (right != nil && left.PublicKey > right.PublicKey): + if err := addPeer(right); err != nil { + return nil, err + } + + r++ + // peer from the "left" (existing) is missing in new spec (right), so it should be removed + case right == nil || (left != nil && left.PublicKey < right.PublicKey): + // deleting peers from the existing + if err := deletePeer(left); err != nil { + return nil, err + } + + l++ + // peer public keys are equal, so either they are identical or peer should be replaced + case left.PublicKey == right.PublicKey: + if !left.Equal(right) { + // replace peer + if err := addPeer(right); err != nil { + return nil, err + } + } + + l++ + r++ + } + } + + return cfg, nil +} + +// Decode spec from the device state. +func (a wireguardSpec) Decode(dev *wgtypes.Device, isStatus bool) { + spec := a.WireguardSpec + + if isStatus { + spec.PublicKey = dev.PublicKey.String() + } else { + spec.PrivateKey = dev.PrivateKey.String() + } + + spec.ListenPort = dev.ListenPort + spec.FirewallMark = dev.FirewallMark + + spec.Peers = make([]network.WireguardPeer, len(dev.Peers)) + + for i := range spec.Peers { + spec.Peers[i].PublicKey = dev.Peers[i].PublicKey.String() + + if dev.Peers[i].Endpoint != nil { + spec.Peers[i].Endpoint = dev.Peers[i].Endpoint.String() + } + + var zeroKey wgtypes.Key + + if dev.Peers[i].PresharedKey != zeroKey { + spec.Peers[i].PresharedKey = dev.Peers[i].PresharedKey.String() + } + + spec.Peers[i].PersistentKeepaliveInterval = dev.Peers[i].PersistentKeepaliveInterval + spec.Peers[i].AllowedIPs = xslices.Map(dev.Peers[i].AllowedIPs, func(peerIP net.IPNet) netip.Prefix { + res, _ := netipx.FromStdIPNet(&peerIP) + + return res + }) + } +} diff --git a/internal/app/machined/pkg/adapters/network/wireguard_spec_test.go b/internal/app/machined/pkg/adapters/network/wireguard_spec_test.go new file mode 100644 index 0000000..c23a768 --- /dev/null +++ b/internal/app/machined/pkg/adapters/network/wireguard_spec_test.go @@ -0,0 +1,280 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network_test + +import ( + "net" + "net/netip" + "testing" + "time" + + "github.com/siderolabs/go-pointer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + networkadapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/network" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +func TestWireguardSpecDecode(t *testing.T) { + priv, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err) + + pub1, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err) + + pub2, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err) + + var spec network.WireguardSpec + + // decode in spec mode + networkadapter.WireguardSpec(&spec).Decode(&wgtypes.Device{ + PrivateKey: priv, + ListenPort: 30000, + FirewallMark: 1, + Peers: []wgtypes.Peer{ + { + PublicKey: pub1.PublicKey(), + PresharedKey: priv, + Endpoint: &net.UDPAddr{ + IP: net.ParseIP("10.2.0.3"), + Port: 20000, + }, + AllowedIPs: []net.IPNet{ + { + IP: net.ParseIP("172.24.0.0"), + Mask: net.IPv4Mask(255, 255, 0, 0), + }, + }, + }, + { + PublicKey: pub2.PublicKey(), + AllowedIPs: []net.IPNet{ + { + IP: net.ParseIP("172.25.0.0"), + Mask: net.IPv4Mask(255, 255, 255, 0), + }, + }, + }, + }, + }, false) + + expected := network.WireguardSpec{ + PrivateKey: priv.String(), + ListenPort: 30000, + FirewallMark: 1, + Peers: []network.WireguardPeer{ + { + PublicKey: pub1.PublicKey().String(), + PresharedKey: priv.String(), + Endpoint: "10.2.0.3:20000", + AllowedIPs: []netip.Prefix{ + netip.MustParsePrefix("172.24.0.0/16"), + }, + }, + { + PublicKey: pub2.PublicKey().String(), + AllowedIPs: []netip.Prefix{ + netip.MustParsePrefix("172.25.0.0/24"), + }, + }, + }, + } + + assert.Equal(t, expected, spec) + assert.True(t, expected.Equal(&spec)) + + // zeroed out listen port is still acceptable on the right side + spec.ListenPort = 0 + assert.True(t, expected.Equal(&spec)) + + // ... but not on the left side + expected.ListenPort = 0 + spec.ListenPort = 30000 + assert.False(t, expected.Equal(&spec)) + + var zeroSpec network.WireguardSpec + + assert.False(t, zeroSpec.Equal(&spec)) +} + +func TestWireguardSpecDecodeStatus(t *testing.T) { + priv, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err) + + var spec network.WireguardSpec + + // decode in status mode + networkadapter.WireguardSpec(&spec).Decode(&wgtypes.Device{ + PrivateKey: priv, + PublicKey: priv.PublicKey(), + ListenPort: 30000, + FirewallMark: 1, + }, true) + + expected := network.WireguardSpec{ + PublicKey: priv.PublicKey().String(), + ListenPort: 30000, + FirewallMark: 1, + Peers: []network.WireguardPeer{}, + } + + assert.Equal(t, expected, spec) +} + +func TestWireguardSpecEncode(t *testing.T) { + priv, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err) + + pub1, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err) + + pub2, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err) + + // make sure pub1 < pub2 + if pub1.PublicKey().String() > pub2.PublicKey().String() { + pub1, pub2 = pub2, pub1 + } + + specV1 := network.WireguardSpec{ + PrivateKey: priv.String(), + ListenPort: 30000, + FirewallMark: 1, + Peers: []network.WireguardPeer{ + { + PublicKey: pub1.PublicKey().String(), + Endpoint: "10.2.0.3:20000", + AllowedIPs: []netip.Prefix{ + netip.MustParsePrefix("172.24.0.0/16"), + }, + }, + { + PublicKey: pub2.PublicKey().String(), + AllowedIPs: []netip.Prefix{ + netip.MustParsePrefix("172.25.0.0/24"), + }, + }, + }, + } + + specV1.Sort() + + var zero network.WireguardSpec + + networkadapter.WireguardSpec(&zero).Decode(&wgtypes.Device{}, false) + zero.Sort() + + // from zero (empty) config to config with two peers + delta, err := networkadapter.WireguardSpec(&specV1).Encode(&zero) + require.NoError(t, err) + + assert.Equal(t, &wgtypes.Config{ + PrivateKey: &priv, + ListenPort: pointer.To(30000), + FirewallMark: pointer.To(1), + Peers: []wgtypes.PeerConfig{ + { + PublicKey: pub1.PublicKey(), + Endpoint: &net.UDPAddr{ + IP: net.ParseIP("10.2.0.3"), + Port: 20000, + }, + PersistentKeepaliveInterval: pointer.To[time.Duration](0), + ReplaceAllowedIPs: true, + AllowedIPs: []net.IPNet{ + { + IP: net.ParseIP("172.24.0.0").To4(), + Mask: net.IPv4Mask(255, 255, 0, 0), + }, + }, + }, + { + PublicKey: pub2.PublicKey(), + PersistentKeepaliveInterval: pointer.To[time.Duration](0), + ReplaceAllowedIPs: true, + AllowedIPs: []net.IPNet{ + { + IP: net.ParseIP("172.25.0.0").To4(), + Mask: net.IPv4Mask(255, 255, 255, 0), + }, + }, + }, + }, + }, delta) + + // noop + delta, err = networkadapter.WireguardSpec(&specV1).Encode(&specV1) + require.NoError(t, err) + + assert.Equal(t, &wgtypes.Config{}, delta) + + // delete peer2 + specV2 := network.WireguardSpec{ + PrivateKey: priv.String(), + ListenPort: 30000, + FirewallMark: 1, + Peers: []network.WireguardPeer{ + { + PublicKey: pub1.PublicKey().String(), + Endpoint: "10.2.0.3:20000", + AllowedIPs: []netip.Prefix{ + netip.MustParsePrefix("172.24.0.0/16"), + }, + }, + }, + } + + delta, err = networkadapter.WireguardSpec(&specV2).Encode(&specV1) + require.NoError(t, err) + + assert.Equal(t, &wgtypes.Config{ + Peers: []wgtypes.PeerConfig{ + { + PublicKey: pub2.PublicKey(), + Remove: true, + }, + }, + }, delta) + + // update peer1, firewallMark + specV3 := network.WireguardSpec{ + PrivateKey: priv.String(), + ListenPort: 30000, + FirewallMark: 2, + Peers: []network.WireguardPeer{ + { + PublicKey: pub1.PublicKey().String(), + PresharedKey: priv.String(), + AllowedIPs: []netip.Prefix{ + netip.MustParsePrefix("172.24.0.0/16"), + }, + }, + }, + } + + delta, err = networkadapter.WireguardSpec(&specV3).Encode(&specV2) + require.NoError(t, err) + + assert.Equal(t, &wgtypes.Config{ + FirewallMark: pointer.To(2), + Peers: []wgtypes.PeerConfig{ + { + PublicKey: pub1.PublicKey(), + PresharedKey: &priv, + PersistentKeepaliveInterval: pointer.To[time.Duration](0), + ReplaceAllowedIPs: true, + AllowedIPs: []net.IPNet{ + { + IP: net.ParseIP("172.24.0.0").To4(), + Mask: net.IPv4Mask(255, 255, 0, 0), + }, + }, + }, + }, + }, delta) +} diff --git a/internal/app/machined/pkg/adapters/perf/cpu.go b/internal/app/machined/pkg/adapters/perf/cpu.go new file mode 100644 index 0000000..886d329 --- /dev/null +++ b/internal/app/machined/pkg/adapters/perf/cpu.go @@ -0,0 +1,71 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package perf + +import ( + "github.com/prometheus/procfs" + + "github.com/siderolabs/talos/pkg/machinery/resources/perf" +) + +// CPU adapter provides conversion from procfs. +// +//nolint:revive,golint +func CPU(r *perf.CPU) cpu { + return cpu{ + CPU: r, + } +} + +type cpu struct { + *perf.CPU +} + +// Update current CPU snapshot. +func (a cpu) Update(stat *procfs.Stat) { + translateCPUStat := func(in procfs.CPUStat) perf.CPUStat { + return perf.CPUStat{ + User: in.User, + Nice: in.Nice, + System: in.System, + Idle: in.Idle, + Iowait: in.Iowait, + Irq: in.IRQ, + SoftIrq: in.SoftIRQ, + Steal: in.Steal, + Guest: in.Guest, + GuestNice: in.GuestNice, + } + } + + translateListOfCPUStat := func(in map[int64]procfs.CPUStat) []perf.CPUStat { + maxCore := int64(-1) + + for core := range in { + if core > maxCore { + maxCore = core + } + } + + slc := make([]perf.CPUStat, maxCore+1) + + for core, stat := range in { + slc[core] = translateCPUStat(stat) + } + + return slc + } + + *a.CPU.TypedSpec() = perf.CPUSpec{ + CPUTotal: translateCPUStat(stat.CPUTotal), + CPU: translateListOfCPUStat(stat.CPU), + IRQTotal: stat.IRQTotal, + ContextSwitches: stat.ContextSwitches, + ProcessCreated: stat.ProcessCreated, + ProcessRunning: stat.ProcessesRunning, + ProcessBlocked: stat.ProcessesBlocked, + SoftIrqTotal: stat.SoftIRQTotal, + } +} diff --git a/internal/app/machined/pkg/adapters/perf/mem.go b/internal/app/machined/pkg/adapters/perf/mem.go new file mode 100644 index 0000000..d7f480b --- /dev/null +++ b/internal/app/machined/pkg/adapters/perf/mem.go @@ -0,0 +1,79 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package perf + +import ( + "github.com/prometheus/procfs" + "github.com/siderolabs/go-pointer" + + "github.com/siderolabs/talos/pkg/machinery/resources/perf" +) + +// Memory adapter provides conversion from procfs. +// +//nolint:revive,golint +func Memory(r *perf.Memory) memory { + return memory{ + Memory: r, + } +} + +type memory struct { + *perf.Memory +} + +// Update current Mem snapshot. +func (a memory) Update(info *procfs.Meminfo) { + *a.Memory.TypedSpec() = perf.MemorySpec{ + MemTotal: pointer.SafeDeref(info.MemTotal), + MemUsed: pointer.SafeDeref(info.MemTotal) - pointer.SafeDeref(info.MemFree), + MemAvailable: pointer.SafeDeref(info.MemAvailable), + Buffers: pointer.SafeDeref(info.Buffers), + Cached: pointer.SafeDeref(info.Cached), + SwapCached: pointer.SafeDeref(info.SwapCached), + Active: pointer.SafeDeref(info.Active), + Inactive: pointer.SafeDeref(info.Inactive), + ActiveAnon: pointer.SafeDeref(info.ActiveAnon), + InactiveAnon: pointer.SafeDeref(info.InactiveAnon), + ActiveFile: pointer.SafeDeref(info.ActiveFile), + InactiveFile: pointer.SafeDeref(info.InactiveFile), + Unevictable: pointer.SafeDeref(info.Unevictable), + Mlocked: pointer.SafeDeref(info.Mlocked), + SwapTotal: pointer.SafeDeref(info.SwapTotal), + SwapFree: pointer.SafeDeref(info.SwapFree), + Dirty: pointer.SafeDeref(info.Dirty), + Writeback: pointer.SafeDeref(info.Writeback), + AnonPages: pointer.SafeDeref(info.AnonPages), + Mapped: pointer.SafeDeref(info.Mapped), + Shmem: pointer.SafeDeref(info.Shmem), + Slab: pointer.SafeDeref(info.Slab), + SReclaimable: pointer.SafeDeref(info.SReclaimable), + SUnreclaim: pointer.SafeDeref(info.SUnreclaim), + KernelStack: pointer.SafeDeref(info.KernelStack), + PageTables: pointer.SafeDeref(info.PageTables), + NFSunstable: pointer.SafeDeref(info.NFSUnstable), + Bounce: pointer.SafeDeref(info.Bounce), + WritebackTmp: pointer.SafeDeref(info.WritebackTmp), + CommitLimit: pointer.SafeDeref(info.CommitLimit), + CommittedAS: pointer.SafeDeref(info.CommittedAS), + VmallocTotal: pointer.SafeDeref(info.VmallocTotal), + VmallocUsed: pointer.SafeDeref(info.VmallocUsed), + VmallocChunk: pointer.SafeDeref(info.VmallocChunk), + HardwareCorrupted: pointer.SafeDeref(info.HardwareCorrupted), + AnonHugePages: pointer.SafeDeref(info.AnonHugePages), + ShmemHugePages: pointer.SafeDeref(info.ShmemHugePages), + ShmemPmdMapped: pointer.SafeDeref(info.ShmemPmdMapped), + CmaTotal: pointer.SafeDeref(info.CmaTotal), + CmaFree: pointer.SafeDeref(info.CmaFree), + HugePagesTotal: pointer.SafeDeref(info.HugePagesTotal), + HugePagesFree: pointer.SafeDeref(info.HugePagesFree), + HugePagesRsvd: pointer.SafeDeref(info.HugePagesRsvd), + HugePagesSurp: pointer.SafeDeref(info.HugePagesSurp), + Hugepagesize: pointer.SafeDeref(info.Hugepagesize), + DirectMap4k: pointer.SafeDeref(info.DirectMap4k), + DirectMap2m: pointer.SafeDeref(info.DirectMap2M), + DirectMap1g: pointer.SafeDeref(info.DirectMap1G), + } +} diff --git a/internal/app/machined/pkg/adapters/perf/perf.go b/internal/app/machined/pkg/adapters/perf/perf.go new file mode 100644 index 0000000..52fab87 --- /dev/null +++ b/internal/app/machined/pkg/adapters/perf/perf.go @@ -0,0 +1,6 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package perf implements adapters wrapping resources/perf to provide additional functionality. +package perf diff --git a/internal/app/machined/pkg/adapters/wireguard/wireguard.go b/internal/app/machined/pkg/adapters/wireguard/wireguard.go new file mode 100644 index 0000000..4d2adc2 --- /dev/null +++ b/internal/app/machined/pkg/adapters/wireguard/wireguard.go @@ -0,0 +1,16 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package wireguard implements common wireguard functionality. +package wireguard + +import "time" + +// PeerDownInterval is the time since last handshake when established peer is considered to be down. +// +// WG whitepaper defines a downed peer as being: +// Handshake Timeout (180s) + Rekey Timeout (5s) + Rekey Attempt Timeout (90s) +// +// This interval is applied when the link is already established. +const PeerDownInterval = (180 + 5 + 90) * time.Second diff --git a/internal/app/machined/pkg/controllers/block/block.go b/internal/app/machined/pkg/controllers/block/block.go new file mode 100644 index 0000000..6144740 --- /dev/null +++ b/internal/app/machined/pkg/controllers/block/block.go @@ -0,0 +1,6 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package block provides the controllers related to blockdevices, mounts, etc. +package block diff --git a/internal/app/machined/pkg/controllers/block/devices.go b/internal/app/machined/pkg/controllers/block/devices.go new file mode 100644 index 0000000..d41b336 --- /dev/null +++ b/internal/app/machined/pkg/controllers/block/devices.go @@ -0,0 +1,246 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package block + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strconv" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "go.uber.org/zap" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/block/internal/inotify" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/block/internal/kobject" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/block/internal/sysblock" + machineruntime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/block" +) + +// DevicesController provides a view of available block devices with information about pending updates. +type DevicesController struct { + V1Alpha1Mode machineruntime.Mode +} + +// Name implements controller.Controller interface. +func (ctrl *DevicesController) Name() string { + return "block.DevicesController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *DevicesController) Inputs() []controller.Input { + return nil +} + +// Outputs implements controller.Controller interface. +func (ctrl *DevicesController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: block.DeviceType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *DevicesController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + // in container mode, no devices + if ctrl.V1Alpha1Mode == machineruntime.ModeContainer { + return nil + } + + // start the watcher first + watcher, err := kobject.NewWatcher() + if err != nil { + return fmt.Errorf("failed to create kobject watcher: %w", err) + } + + defer watcher.Close() //nolint:errcheck + + watchCh := watcher.Run(logger) + + // start the inotify watcher + inotifyWatcher, err := inotify.NewWatcher() + if err != nil { + return fmt.Errorf("failed to create inotify watcher: %w", err) + } + + defer inotifyWatcher.Close() //nolint:errcheck + + inotifyCh, inotifyErrCh := inotifyWatcher.Run() + + // reconcile the initial list of devices while the watcher is running + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + if err = ctrl.resync(ctx, r, logger, inotifyWatcher); err != nil { + return fmt.Errorf("failed to resync: %w", err) + } + + for { + select { + case ev := <-watchCh: + if ev.Subsystem != "block" { + continue + } + + ev.DevicePath = filepath.Join("/sys", ev.DevicePath) + + if err = ctrl.processEvent(ctx, r, logger, inotifyWatcher, ev); err != nil { + return err + } + case err = <-inotifyErrCh: + return fmt.Errorf("inotify watcher failed: %w", err) + case updatedPath := <-inotifyCh: + id := filepath.Base(updatedPath) + + if err = ctrl.bumpGeneration(ctx, r, logger, id); err != nil { + return err + } + case <-ctx.Done(): + return nil + } + } +} + +func (ctrl *DevicesController) bumpGeneration(ctx context.Context, r controller.Runtime, logger *zap.Logger, id string) error { + _, err := safe.ReaderGetByID[*block.Device](ctx, r, id) + if err != nil { + if state.IsNotFoundError(err) { + // skip it + return nil + } + + return err + } + + logger.Debug("bumping generation for device, inotify update", zap.String("id", id)) + + return safe.WriterModify(ctx, r, block.NewDevice(block.NamespaceName, id), func(dev *block.Device) error { + dev.TypedSpec().Generation++ + + return nil + }) +} + +func (ctrl *DevicesController) resync(ctx context.Context, r controller.Runtime, logger *zap.Logger, inotifyWatcher *inotify.Watcher) error { + events, err := sysblock.Walk("/sys/block") + if err != nil { + return fmt.Errorf("failed to walk /sys/block: %w", err) + } + + touchedIDs := make(map[string]struct{}, len(events)) + + for _, ev := range events { + if err = ctrl.processEvent(ctx, r, logger, inotifyWatcher, ev); err != nil { + return err + } + + touchedIDs[ev.Values["DEVNAME"]] = struct{}{} + } + + // remove devices that were not touched + devices, err := safe.ReaderListAll[*block.Device](ctx, r) + if err != nil { + return fmt.Errorf("failed to list devices: %w", err) + } + + for iterator := devices.Iterator(); iterator.Next(); { + dev := iterator.Value() + + if _, ok := touchedIDs[dev.Metadata().ID()]; ok { + continue + } + + if err = r.Destroy(ctx, dev.Metadata()); err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("failed to remove device: %w", err) + } + } + + return nil +} + +//nolint:gocyclo +func (ctrl *DevicesController) processEvent(ctx context.Context, r controller.Runtime, logger *zap.Logger, inotifyWatcher *inotify.Watcher, ev *kobject.Event) error { + logger = logger.With( + zap.String("action", string(ev.Action)), + zap.String("path", ev.DevicePath), + zap.String("id", ev.Values["DEVNAME"]), + ) + + logger.Debug("processing event") + + id := ev.Values["DEVNAME"] + devPath := filepath.Join("/dev", id) + + // re-stat the sysfs entry to make sure we are not out of sync with events + _, reStatErr := os.Stat(ev.DevicePath) + + switch ev.Action { + case kobject.ActionAdd, kobject.ActionBind, kobject.ActionOnline, kobject.ActionChange, kobject.ActionMove, kobject.ActionUnbind, kobject.ActionOffline: + if reStatErr != nil { + logger.Debug("skipped, as device path doesn't exist") + + return nil //nolint:nilerr // entry doesn't exist now, so skip the event + } + + if err := safe.WriterModify(ctx, r, block.NewDevice(block.NamespaceName, id), func(dev *block.Device) error { + dev.TypedSpec().Type = ev.Values["DEVTYPE"] + dev.TypedSpec().Major = atoiOrZero(ev.Values["MAJOR"]) + dev.TypedSpec().Minor = atoiOrZero(ev.Values["MINOR"]) + dev.TypedSpec().PartitionName = ev.Values["PARTNAME"] + dev.TypedSpec().PartitionNumber = atoiOrZero(ev.Values["PARTN"]) + + dev.TypedSpec().DevicePath = ev.DevicePath + + if dev.TypedSpec().Type == "partition" { + dev.TypedSpec().Parent = filepath.Base(filepath.Dir(dev.TypedSpec().DevicePath)) + } + + dev.TypedSpec().Generation++ + + return nil + }); err != nil { + return fmt.Errorf("failed to modify device %q: %w", id, err) + } + + if err := inotifyWatcher.Add(devPath); err != nil { + return fmt.Errorf("failed to add inotify watch for %q: %w", devPath, err) + } + case kobject.ActionRemove: + if reStatErr == nil { // entry still exists, skip removing + logger.Debug("skipped, as device path still exists") + + return nil + } + + if err := r.Destroy(ctx, block.NewDevice(block.NamespaceName, id).Metadata()); err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("failed to remove device %q: %w", id, err) + } + + if err := inotifyWatcher.Remove(devPath); err != nil { + logger.Debug("failed to remove inotify watch", zap.String("device", devPath), zap.Error(err)) + } + default: + logger.Debug("skipped, as action is not supported") + } + + return nil +} + +func atoiOrZero(s string) int { + i, _ := strconv.Atoi(s) //nolint:errcheck + + return i +} diff --git a/internal/app/machined/pkg/controllers/block/devices_test.go b/internal/app/machined/pkg/controllers/block/devices_test.go new file mode 100644 index 0000000..d394403 --- /dev/null +++ b/internal/app/machined/pkg/controllers/block/devices_test.go @@ -0,0 +1,39 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package block_test + +import ( + "os" + "testing" + + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + blockctrls "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/block" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + "github.com/siderolabs/talos/pkg/machinery/resources/block" +) + +type DevicesSuite struct { + ctest.DefaultSuite +} + +func TestDevicesSuite(t *testing.T) { + suite.Run(t, new(DevicesSuite)) +} + +func (suite *DevicesSuite) TestDiscover() { + if os.Geteuid() != 0 { + suite.T().Skip("skipping test; must be root to use inotify") + } + + suite.Require().NoError(suite.Runtime().RegisterController(&blockctrls.DevicesController{})) + + // these devices should always exist on Linux + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []string{"loop0", "loop1"}, func(r *block.Device, assertions *assert.Assertions) { + assertions.Equal("disk", r.TypedSpec().Type) + }) +} diff --git a/internal/app/machined/pkg/controllers/block/discovery.go b/internal/app/machined/pkg/controllers/block/discovery.go new file mode 100644 index 0000000..9a791da --- /dev/null +++ b/internal/app/machined/pkg/controllers/block/discovery.go @@ -0,0 +1,277 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package block + +import ( + "context" + "fmt" + "path/filepath" + "strconv" + "time" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/maps" + "github.com/siderolabs/go-blockdevice/v2/blkid" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/block" +) + +// DiscoveryController provides a filesystem/partition discovery for blockdevices. +type DiscoveryController struct{} + +// Name implements controller.Controller interface. +func (ctrl *DiscoveryController) Name() string { + return "block.DiscoveryController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *DiscoveryController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: block.NamespaceName, + Type: block.DeviceType, + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *DiscoveryController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: block.DiscoveredVolumeType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *DiscoveryController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + // lastObservedGenerations holds the last observed generation of each device. + // + // when the generation of a device changes, the device might have changed and might need to be re-probed. + lastObservedGenerations := map[string]int{} + + // nextRescan holds the pool of devices to be rescanned in the next batch. + nextRescan := map[string]int{} + + rescanTicker := time.NewTicker(100 * time.Millisecond) + defer rescanTicker.Stop() + + for { + select { + case <-ctx.Done(): + return nil + case <-rescanTicker.C: + if len(nextRescan) == 0 { + continue + } + + if err := ctrl.rescan(ctx, r, logger, maps.Keys(nextRescan)); err != nil { + return fmt.Errorf("failed to rescan devices: %w", err) + } + + nextRescan = map[string]int{} + case <-r.EventCh(): + devices, err := safe.ReaderListAll[*block.Device](ctx, r) + if err != nil { + return fmt.Errorf("failed to list devices: %w", err) + } + + parents := map[string]string{} + allDevices := map[string]struct{}{} + + for iterator := devices.Iterator(); iterator.Next(); { + device := iterator.Value() + + allDevices[device.Metadata().ID()] = struct{}{} + + if device.TypedSpec().Parent != "" { + parents[device.Metadata().ID()] = device.TypedSpec().Parent + } + + if device.TypedSpec().Generation == lastObservedGenerations[device.Metadata().ID()] { + continue + } + + nextRescan[device.Metadata().ID()] = device.TypedSpec().Generation + lastObservedGenerations[device.Metadata().ID()] = device.TypedSpec().Generation + } + + // remove child devices if the parent is marked for rescan + for id := range nextRescan { + if parent, ok := parents[id]; ok { + if _, ok := nextRescan[parent]; ok { + delete(nextRescan, id) + } + } + } + + // if the device is removed, add it to the nextRescan, and remove from lastObservedGenerations + for id := range lastObservedGenerations { + if _, ok := allDevices[id]; !ok { + nextRescan[id] = lastObservedGenerations[id] + delete(lastObservedGenerations, id) + } + } + } + } +} + +//nolint:gocyclo +func (ctrl *DiscoveryController) rescan(ctx context.Context, r controller.Runtime, logger *zap.Logger, ids []string) error { + failedIDs := map[string]struct{}{} + touchedIDs := map[string]struct{}{} + + for _, id := range ids { + device, err := safe.ReaderGetByID[*block.Device](ctx, r, id) + if err != nil { + if state.IsNotFoundError(err) { + failedIDs[id] = struct{}{} + + continue + } + + return fmt.Errorf("failed to get device: %w", err) + } + + info, err := blkid.ProbePath(filepath.Join("/dev", id)) + if err != nil { + logger.Debug("failed to probe device", zap.String("id", id), zap.Error(err)) + + failedIDs[id] = struct{}{} + + continue + } + + if err = safe.WriterModify(ctx, r, block.NewDiscoveredVolume(block.NamespaceName, id), func(dv *block.DiscoveredVolume) error { + dv.TypedSpec().Type = device.TypedSpec().Type + dv.TypedSpec().DevicePath = device.TypedSpec().DevicePath + dv.TypedSpec().Parent = device.TypedSpec().Parent + + dv.TypedSpec().Size = info.Size + dv.TypedSpec().SectorSize = info.SectorSize + dv.TypedSpec().IOSize = info.IOSize + + ctrl.fillDiscoveredVolumeFromInfo(dv, info.ProbeResult) + + return nil + }); err != nil { + return fmt.Errorf("failed to write discovered volume: %w", err) + } + + touchedIDs[id] = struct{}{} + + for _, nested := range info.Parts { + partID := partitionID(id, nested.PartitionIndex) + + if err = safe.WriterModify(ctx, r, block.NewDiscoveredVolume(block.NamespaceName, partID), func(dv *block.DiscoveredVolume) error { + dv.TypedSpec().Type = "partition" + dv.TypedSpec().DevicePath = filepath.Join(device.TypedSpec().DevicePath, partID) + dv.TypedSpec().Parent = id + + dv.TypedSpec().Size = nested.ProbedSize + dv.TypedSpec().SectorSize = info.SectorSize + dv.TypedSpec().IOSize = info.IOSize + + ctrl.fillDiscoveredVolumeFromInfo(dv, nested.ProbeResult) + + if nested.PartitionUUID != nil { + dv.TypedSpec().PartitionUUID = nested.PartitionUUID.String() + } else { + dv.TypedSpec().PartitionUUID = "" + } + + if nested.PartitionType != nil { + dv.TypedSpec().PartitionType = nested.PartitionType.String() + } else { + dv.TypedSpec().PartitionType = "" + } + + if nested.PartitionLabel != nil { + dv.TypedSpec().PartitionLabel = *nested.PartitionLabel + } else { + dv.TypedSpec().PartitionLabel = "" + } + + dv.TypedSpec().PartitionIndex = nested.PartitionIndex + + return nil + }); err != nil { + return fmt.Errorf("failed to write discovered volume: %w", err) + } + + touchedIDs[partID] = struct{}{} + } + } + + // clean up discovered volumes + discoveredVolumes, err := safe.ReaderListAll[*block.DiscoveredVolume](ctx, r) + if err != nil { + return fmt.Errorf("failed to list discovered volumes: %w", err) + } + + for iterator := discoveredVolumes.Iterator(); iterator.Next(); { + dv := iterator.Value() + + if _, ok := touchedIDs[dv.Metadata().ID()]; ok { + continue + } + + _, isFailed := failedIDs[dv.Metadata().ID()] + + parentTouched := false + + if dv.TypedSpec().Parent != "" { + if _, ok := touchedIDs[dv.TypedSpec().Parent]; ok { + parentTouched = true + } + } + + if isFailed || parentTouched { + // if the probe failed, or if the parent was touched, while this device was not, remove it + if err = r.Destroy(ctx, dv.Metadata()); err != nil { + return fmt.Errorf("failed to destroy discovered volume: %w", err) + } + } + } + + return nil +} + +func (ctrl *DiscoveryController) fillDiscoveredVolumeFromInfo(dv *block.DiscoveredVolume, info blkid.ProbeResult) { + dv.TypedSpec().Name = info.Name + + if info.UUID != nil { + dv.TypedSpec().UUID = info.UUID.String() + } else { + dv.TypedSpec().UUID = "" + } + + if info.Label != nil { + dv.TypedSpec().Label = *info.Label + } else { + dv.TypedSpec().Label = "" + } + + dv.TypedSpec().BlockSize = info.BlockSize + dv.TypedSpec().FilesystemBlockSize = info.FilesystemBlockSize + dv.TypedSpec().ProbedSize = info.ProbedSize +} + +func partitionID(devname string, part uint) string { + result := devname + + if len(result) > 0 && result[len(result)-1] >= '0' && result[len(result)-1] <= '9' { + result += "p" + } + + return result + strconv.FormatUint(uint64(part), 10) +} diff --git a/internal/app/machined/pkg/controllers/block/internal/inotify/inotify.go b/internal/app/machined/pkg/controllers/block/internal/inotify/inotify.go new file mode 100644 index 0000000..d02d605 --- /dev/null +++ b/internal/app/machined/pkg/controllers/block/internal/inotify/inotify.go @@ -0,0 +1,278 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package inotify implements a specialized inotify watcher for block devices. +package inotify + +import ( + "errors" + "os" + "strings" + "sync" + "unsafe" + + "golang.org/x/sys/unix" +) + +type ( + watches struct { + mu sync.RWMutex + wd map[uint32]*watch // wd → watch + path map[string]uint32 // pathname → wd + } + watch struct { + wd uint32 // Watch descriptor (as returned by the inotify_add_watch() syscall) + flags uint32 // inotify flags of this watch (see inotify(7) for the list of valid flags) + path string // Watch path. + } +) + +func newWatches() *watches { + return &watches{ + wd: make(map[uint32]*watch), + path: make(map[string]uint32), + } +} + +func (w *watches) remove(wd uint32) { + w.mu.Lock() + defer w.mu.Unlock() + + if _, ok := w.wd[wd]; ok { + delete(w.path, w.wd[wd].path) + } + + delete(w.wd, wd) +} + +func (w *watches) removePath(path string) (uint32, bool) { + w.mu.Lock() + defer w.mu.Unlock() + + wd, ok := w.path[path] + if !ok { + return 0, false + } + + delete(w.path, path) + delete(w.wd, wd) + + return wd, true +} + +func (w *watches) byWd(wd uint32) *watch { + w.mu.RLock() + defer w.mu.RUnlock() + + return w.wd[wd] +} + +func (w *watches) updatePath(path string, f func(*watch) (*watch, error)) error { + w.mu.Lock() + defer w.mu.Unlock() + + var existing *watch + + wd, ok := w.path[path] + if ok { + existing = w.wd[wd] + } + + upd, err := f(existing) + if err != nil { + return err + } + + if upd != nil { + w.wd[upd.wd] = upd + w.path[upd.path] = upd.wd + + if upd.wd != wd { + delete(w.wd, wd) + } + } + + return nil +} + +// Watcher implements inotify-based file watching. +type Watcher struct { + wg sync.WaitGroup + fd int + inotifyFile *os.File + watches *watches +} + +// NewWatcher creates a new inotify Watcher. +func NewWatcher() (*Watcher, error) { + // Need to set nonblocking mode for SetDeadline to work, otherwise blocking + // I/O operations won't terminate on close. + fd, errno := unix.InotifyInit1(unix.IN_CLOEXEC | unix.IN_NONBLOCK) + if fd == -1 { + return nil, errno + } + + return &Watcher{ + fd: fd, + inotifyFile: os.NewFile(uintptr(fd), ""), + watches: newWatches(), + }, nil +} + +// Close the inotify watcher. +func (w *Watcher) Close() error { + // Causes any blocking reads to return with an error, provided the file + // still supports deadline operations. + err := w.inotifyFile.Close() + if err != nil { + return err + } + + // Wait for goroutine to close + w.wg.Wait() + + return nil +} + +// Run the watcher, returns two channels for errors and events (paths changed). +// +//nolint:gocyclo +func (w *Watcher) Run() (<-chan string, <-chan error) { + errCh := make(chan error, 1) + eventCh := make(chan string, 128) + + w.wg.Add(1) + + var buf [unix.SizeofInotifyEvent * 4096]byte // Buffer for a maximum of 4096 raw events + + go func() { + defer w.wg.Done() + + for { + n, err := w.inotifyFile.Read(buf[:]) + + switch { + case errors.Is(err, os.ErrClosed): + return + case err != nil: + errCh <- err + + return + } + + if n < unix.SizeofInotifyEvent { + errCh <- errors.New("short read from inotify") + + return + } + + var offset uint32 + + // We don't know how many events we just read into the buffer + // While the offset points to at least one whole event... + for offset <= uint32(n-unix.SizeofInotifyEvent) { + var ( + // Point "raw" to the event in the buffer + raw = (*unix.InotifyEvent)(unsafe.Pointer(&buf[offset])) + mask = raw.Mask + nameLen = raw.Len + ) + + if mask&unix.IN_Q_OVERFLOW != 0 { + errCh <- errors.New("inotify queue overflow") + + return + } + + // If the event happened to the watched directory or the watched file, the kernel + // doesn't append the filename to the event, but we would like to always fill the + // the "Name" field with a valid filename. We retrieve the path of the watch from + // the "paths" map. + watch := w.watches.byWd(uint32(raw.Wd)) + + // inotify will automatically remove the watch on deletes; just need + // to clean our state here. + if watch != nil && mask&unix.IN_DELETE_SELF == unix.IN_DELETE_SELF { + w.watches.remove(watch.wd) + } + + var name string + if watch != nil { + name = watch.path + } + + if nameLen > 0 { + // Point "bytes" at the first byte of the filename + bytes := (*[unix.PathMax]byte)(unsafe.Pointer(&buf[offset+unix.SizeofInotifyEvent]))[:nameLen:nameLen] + // The filename is padded with NULL bytes. TrimRight() gets rid of those. + name += "/" + strings.TrimRight(string(bytes[0:nameLen]), "\000") + } + + // Send the events that are not ignored on the events channel + if mask&unix.IN_IGNORED == 0 && mask&unix.IN_CLOSE_WRITE != 0 { + eventCh <- name + } + + // Move to the next event in the buffer + offset += unix.SizeofInotifyEvent + nameLen + } + } + }() + + return eventCh, errCh +} + +// Add a watch to the inotify watcher. +func (w *Watcher) Add(name string) error { + var flags uint32 = unix.IN_CLOSE_WRITE | unix.IN_DELETE_SELF + + return w.watches.updatePath(name, func(existing *watch) (*watch, error) { + if existing != nil { + flags |= existing.flags | unix.IN_MASK_ADD + } + + wd, err := unix.InotifyAddWatch(w.fd, name, flags) + if wd == -1 { + return nil, err + } + + if existing == nil { + return &watch{ + wd: uint32(wd), + path: name, + flags: flags, + }, nil + } + + existing.wd = uint32(wd) + existing.flags = flags + + return existing, nil + }) +} + +// Remove a watch from the inotify watcher. +func (w *Watcher) Remove(name string) error { + wd, ok := w.watches.removePath(name) + if !ok { + return nil + } + + success, errno := unix.InotifyRmWatch(w.fd, wd) + if success == -1 { + // TODO: Perhaps it's not helpful to return an error here in every case; + // The only two possible errors are: + // + // - EBADF, which happens when w.fd is not a valid file descriptor + // of any kind. + // - EINVAL, which is when fd is not an inotify descriptor or wd + // is not a valid watch descriptor. Watch descriptors are + // invalidated when they are removed explicitly or implicitly; + // explicitly by inotify_rm_watch, implicitly when the file they + // are watching is deleted. + return errno + } + + return nil +} diff --git a/internal/app/machined/pkg/controllers/block/internal/inotify/inotify_test.go b/internal/app/machined/pkg/controllers/block/internal/inotify/inotify_test.go new file mode 100644 index 0000000..d5de851 --- /dev/null +++ b/internal/app/machined/pkg/controllers/block/internal/inotify/inotify_test.go @@ -0,0 +1,85 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package inotify_test + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/block/internal/inotify" +) + +//nolint:gocyclo +func TestWatcher(t *testing.T) { + watcher, err := inotify.NewWatcher() + require.NoError(t, err) + + d := t.TempDir() + + require.NoError(t, os.WriteFile(filepath.Join(d, "file1"), []byte("test1"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(d, "file2"), []byte("test2"), 0o644)) + + require.NoError(t, watcher.Add(filepath.Join(d, "file1"))) + + watchCh, errCh := watcher.Run() + + require.NoError(t, watcher.Add(filepath.Join(d, "file2"))) + + select { + case path := <-watchCh: + require.FailNow(t, "unexpected path", "%s", path) + case err = <-errCh: + require.FailNow(t, "unexpected error", "%s", err) + case <-time.After(100 * time.Millisecond): + } + + // open file1 for writing, should get inotify event + f1, err := os.OpenFile(filepath.Join(d, "file1"), os.O_WRONLY, 0) + require.NoError(t, err) + + require.NoError(t, f1.Close()) + + select { + case path := <-watchCh: + require.Equal(t, filepath.Join(d, "file1"), path) + case err = <-errCh: + require.FailNow(t, "unexpected error", "%s", err) + case <-time.After(time.Second): + require.FailNow(t, "timeout") + } + + // open file2 for reading, should not get inotify event + f2, err := os.OpenFile(filepath.Join(d, "file2"), os.O_RDONLY, 0) + require.NoError(t, err) + + require.NoError(t, f2.Close()) + + select { + case path := <-watchCh: + require.FailNow(t, "unexpected path", "%s", path) + case err = <-errCh: + require.FailNow(t, "unexpected error", "%s", err) + case <-time.After(100 * time.Millisecond): + } + + // remove file2 + require.NoError(t, os.Remove(filepath.Join(d, "file2"))) + + select { + case path := <-watchCh: + require.FailNow(t, "unexpected path", "%s", path) + case err = <-errCh: + require.FailNow(t, "unexpected error", "%s", err) + case <-time.After(100 * time.Millisecond): + } + + require.NoError(t, watcher.Remove(filepath.Join(d, "file2"))) + + require.NoError(t, watcher.Close()) +} diff --git a/internal/app/machined/pkg/controllers/block/internal/kobject/kobject.go b/internal/app/machined/pkg/controllers/block/internal/kobject/kobject.go new file mode 100644 index 0000000..43ab621 --- /dev/null +++ b/internal/app/machined/pkg/controllers/block/internal/kobject/kobject.go @@ -0,0 +1,92 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package kobject implements Linux kernel kobject uvent watcher. +package kobject + +import ( + "fmt" + "sync" + + "github.com/mdlayher/kobject" + "go.uber.org/zap" +) + +const readBufferSize = 64 * 1024 * 1024 + +// Event is exported. +type Event = kobject.Event + +// Re-export action constants. +const ( + ActionAdd = kobject.Add + ActionRemove = kobject.Remove + ActionChange = kobject.Change + ActionMove = kobject.Move + ActionOnline = kobject.Online + ActionOffline = kobject.Offline + ActionBind = kobject.Bind + ActionUnbind = kobject.Unbind +) + +// Watcher is a kobject uvent watcher. +type Watcher struct { + wg sync.WaitGroup + + cli *kobject.Client +} + +// NewWatcher creates a new kobject watcher. +func NewWatcher() (*Watcher, error) { + cli, err := kobject.New() + if err != nil { + return nil, fmt.Errorf("failed to create kobject client: %w", err) + } + + if err = cli.SetReadBuffer(readBufferSize); err != nil { + return nil, err + } + + return &Watcher{ + cli: cli, + }, nil +} + +// Close the watcher. +func (w *Watcher) Close() error { + if err := w.cli.Close(); err != nil { + return err + } + + w.wg.Wait() + + return nil +} + +// Run the watcher, returns the channel of events. +func (w *Watcher) Run(logger *zap.Logger) <-chan *Event { + ch := make(chan *kobject.Event, 128) + + w.wg.Add(1) + + go func() { + defer w.wg.Done() + defer close(ch) + + for { + ev, err := w.cli.Receive() + if err != nil { + if err.Error() != "use of closed file" { // unfortunately not an exported error, just errors.New() + logger.Error("failed to receive kobject event", zap.Error(err)) + } + + return + } + + ch <- ev + } + }() + + return ch +} diff --git a/internal/app/machined/pkg/controllers/block/internal/kobject/kobject_test.go b/internal/app/machined/pkg/controllers/block/internal/kobject/kobject_test.go new file mode 100644 index 0000000..284d198 --- /dev/null +++ b/internal/app/machined/pkg/controllers/block/internal/kobject/kobject_test.go @@ -0,0 +1,27 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package kobject_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/block/internal/kobject" +) + +func TestWatcher(t *testing.T) { + watcher, err := kobject.NewWatcher() + require.NoError(t, err) + + evCh := watcher.Run(zaptest.NewLogger(t)) + + require.NoError(t, watcher.Close()) + + // the evCh should be closed + for range evCh { //nolint:revive + } +} diff --git a/internal/app/machined/pkg/controllers/block/internal/sysblock/sysblock.go b/internal/app/machined/pkg/controllers/block/internal/sysblock/sysblock.go new file mode 100644 index 0000000..954aa4d --- /dev/null +++ b/internal/app/machined/pkg/controllers/block/internal/sysblock/sysblock.go @@ -0,0 +1,144 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package sysblock implements gathering block device information from /sys/block filesystem. +package sysblock + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + + "github.com/mdlayher/kobject" +) + +// Walk the /sys/block filesystem and gather block device information. +// +//nolint:gocyclo +func Walk(root string) ([]*kobject.Event, error) { + entries, err := os.ReadDir(root) + if err != nil { + return nil, fmt.Errorf("failed to read %q: %w", root, err) + } + + result := make([]*kobject.Event, 0, len(entries)) + + for _, entry := range entries { + fi, err := entry.Info() + if err != nil { + if os.IsNotExist(err) { + continue + } + + return nil, fmt.Errorf("failed to stat %s: %w", entry.Name(), err) + } + + if fi.Mode()&os.ModeSymlink == 0 { + continue + } + + path, err := filepath.EvalSymlinks(filepath.Join(root, entry.Name())) + if err != nil { + if os.IsNotExist(err) { + continue + } + + return nil, fmt.Errorf("failed to resolve symlink %s: %w", entry.Name(), err) + } + + uevent, err := readUevent(path) + if err != nil { + if os.IsNotExist(err) { + continue + } + + return nil, err + } + + result = append(result, &kobject.Event{ + Action: kobject.Add, + DevicePath: path, + Subsystem: "block", + Values: uevent, + }) + + partitions, err := readPartitions(path) + if err != nil { + return nil, err + } + + result = append(result, partitions...) + } + + return result, nil +} + +// readUevent reads the /sys/block//uevent file and returns the content. +func readUevent(path string) (map[string]string, error) { + path = filepath.Join(path, "uevent") + + content, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read %q: %w", path, err) + } + + result := map[string]string{} + + for _, kv := range bytes.Split(content, []byte("\n")) { + key, value, ok := bytes.Cut(kv, []byte("=")) + if !ok { + continue + } + + result[string(key)] = string(value) + } + + return result, nil +} + +// readPartitions reads partitions for a given device and returns the list of events. +func readPartitions(path string) ([]*kobject.Event, error) { + entries, err := os.ReadDir(path) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + + return nil, fmt.Errorf("failed to read %s: %w", path, err) + } + + var result []*kobject.Event //nolint:prealloc + + for _, entry := range entries { + if !entry.IsDir() { + continue + } + + partitionPath := filepath.Join(path, entry.Name()) + + _, err = os.Stat(filepath.Join(partitionPath, "partition")) + if err != nil { + continue + } + + uevent, err := readUevent(partitionPath) + if err != nil { + if os.IsNotExist(err) { + continue + } + + return nil, err + } + + result = append(result, &kobject.Event{ + Action: kobject.Add, + DevicePath: partitionPath, + Subsystem: "block", + Values: uevent, + }) + } + + return result, nil +} diff --git a/internal/app/machined/pkg/controllers/block/internal/sysblock/sysblock_test.go b/internal/app/machined/pkg/controllers/block/internal/sysblock/sysblock_test.go new file mode 100644 index 0000000..35821f6 --- /dev/null +++ b/internal/app/machined/pkg/controllers/block/internal/sysblock/sysblock_test.go @@ -0,0 +1,42 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package sysblock_test + +import ( + "testing" + + "github.com/mdlayher/kobject" + "github.com/stretchr/testify/require" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/block/internal/sysblock" +) + +func TestWalk(t *testing.T) { + events, err := sysblock.Walk("/sys/block") + require.NoError(t, err) + + require.NotEmpty(t, events) + + // there should be at least a single blockdevice and a partition + partitions, disks := 0, 0 + + for _, event := range events { + require.Equal(t, "block", event.Subsystem) + require.EqualValues(t, kobject.Add, event.Action) + + require.NotEmpty(t, event.DevicePath) + require.NotEmpty(t, event.Action) + + switch event.Values["DEVTYPE"] { + case "partition": + partitions++ + case "disk": + disks++ + } + } + + require.Greater(t, partitions, 0) + require.Greater(t, disks, 0) +} diff --git a/internal/app/machined/pkg/controllers/cluster/affiliate_merge.go b/internal/app/machined/pkg/controllers/cluster/affiliate_merge.go new file mode 100644 index 0000000..8d8f71d --- /dev/null +++ b/internal/app/machined/pkg/controllers/cluster/affiliate_merge.go @@ -0,0 +1,114 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cluster + +import ( + "context" + "errors" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" +) + +// AffiliateMergeController merges raw Affiliates from the RawNamespaceName into final representation in the NamespaceName. +type AffiliateMergeController struct{} + +// Name implements controller.Controller interface. +func (ctrl *AffiliateMergeController) Name() string { + return "cluster.AffiliateMergeController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *AffiliateMergeController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: cluster.RawNamespaceName, + Type: cluster.AffiliateType, + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *AffiliateMergeController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: cluster.AffiliateType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *AffiliateMergeController) Run(ctx context.Context, r controller.Runtime, _ *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + rawAffiliates, err := safe.ReaderList[*cluster.Affiliate](ctx, r, resource.NewMetadata(cluster.RawNamespaceName, cluster.AffiliateType, "", resource.VersionUndefined)) + if err != nil { + return errors.New("error listing affiliates") + } + + mergedAffiliates := make(map[resource.ID]*cluster.AffiliateSpec, rawAffiliates.Len()) + + for it := rawAffiliates.Iterator(); it.Next(); { + affiliateSpec := it.Value().TypedSpec() + id := affiliateSpec.NodeID + + if affiliate, ok := mergedAffiliates[id]; ok { + affiliate.Merge(affiliateSpec) + } else { + mergedAffiliates[id] = affiliateSpec + } + } + + touchedIDs := make(map[resource.ID]struct{}, len(mergedAffiliates)) + + for id, affiliateSpec := range mergedAffiliates { + if err = safe.WriterModify(ctx, r, cluster.NewAffiliate(cluster.NamespaceName, id), func(res *cluster.Affiliate) error { + *res.TypedSpec() = *affiliateSpec + + return nil + }); err != nil { + return err + } + + touchedIDs[id] = struct{}{} + } + + // list keys for cleanup + list, err := safe.ReaderListAll[*cluster.Affiliate](ctx, r) + if err != nil { + return fmt.Errorf("error listing resources: %w", err) + } + + for it := list.Iterator(); it.Next(); { + res := it.Value() + + if res.Metadata().Owner() != ctrl.Name() { + continue + } + + if _, ok := touchedIDs[res.Metadata().ID()]; !ok { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return fmt.Errorf("error cleaning up specs: %w", err) + } + } + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/cluster/affiliate_merge_test.go b/internal/app/machined/pkg/controllers/cluster/affiliate_merge_test.go new file mode 100644 index 0000000..7043e29 --- /dev/null +++ b/internal/app/machined/pkg/controllers/cluster/affiliate_merge_test.go @@ -0,0 +1,132 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cluster_test + +import ( + "net/netip" + "testing" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + clusterctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/cluster" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" +) + +type AffiliateMergeSuite struct { + ClusterSuite +} + +func (suite *AffiliateMergeSuite) TestReconcileDefault() { + suite.startRuntime() + + suite.Require().NoError(suite.runtime.RegisterController(&clusterctrl.AffiliateMergeController{})) + + affiliate1 := cluster.NewAffiliate(cluster.RawNamespaceName, "k8s/7x1SuC8Ege5BGXdAfTEff5iQnlWZLfv9h1LGMxA2pYkC") + *affiliate1.TypedSpec() = cluster.AffiliateSpec{ + NodeID: "7x1SuC8Ege5BGXdAfTEff5iQnlWZLfv9h1LGMxA2pYkC", + Hostname: "foo.com", + Nodename: "bar", + MachineType: machine.TypeControlPlane, + Addresses: []netip.Addr{netip.MustParseAddr("192.168.3.4")}, + KubeSpan: cluster.KubeSpanAffiliateSpec{ + PublicKey: "PLPNBddmTgHJhtw0vxltq1ZBdPP9RNOEUd5JjJZzBRY=", + Address: netip.MustParseAddr("fd50:8d60:4238:6302:f857:23ff:fe21:d1e0"), + AdditionalAddresses: []netip.Prefix{netip.MustParsePrefix("10.244.3.1/24")}, + Endpoints: []netip.AddrPort{netip.MustParseAddrPort("10.0.0.2:51820"), netip.MustParseAddrPort("192.168.3.4:51820")}, + }, + ControlPlane: &cluster.ControlPlane{APIServerPort: 6443}, + } + + affiliate2 := cluster.NewAffiliate(cluster.RawNamespaceName, "service/7x1SuC8Ege5BGXdAfTEff5iQnlWZLfv9h1LGMxA2pYkC") + *affiliate2.TypedSpec() = cluster.AffiliateSpec{ + NodeID: "7x1SuC8Ege5BGXdAfTEff5iQnlWZLfv9h1LGMxA2pYkC", + Hostname: "foo.com", + Nodename: "bar", + MachineType: machine.TypeControlPlane, + Addresses: []netip.Addr{netip.MustParseAddr("192.168.3.4"), netip.MustParseAddr("10.5.0.2")}, + } + + affiliate3 := cluster.NewAffiliate(cluster.RawNamespaceName, "service/9dwHNUViZlPlIervqX9Qo256RUhrfhgO0xBBnKcKl4F") + *affiliate3.TypedSpec() = cluster.AffiliateSpec{ + NodeID: "9dwHNUViZlPlIervqX9Qo256RUhrfhgO0xBBnKcKl4F", + Hostname: "worker-1", + Nodename: "worker-1", + MachineType: machine.TypeWorker, + Addresses: []netip.Addr{netip.MustParseAddr("192.168.3.5")}, + } + + for _, r := range []resource.Resource{affiliate1, affiliate2, affiliate3} { + suite.Require().NoError(suite.state.Create(suite.ctx, r)) + } + + // there should be two merged affiliates: one from affiliate1+affiliate2, and another from affiliate3 + ctest.AssertResource( + suite, + affiliate1.TypedSpec().NodeID, + func(r *cluster.Affiliate, asrt *assert.Assertions) { + spec := r.TypedSpec() + + asrt.Equal(affiliate1.TypedSpec().NodeID, spec.NodeID) + asrt.Equal([]netip.Addr{netip.MustParseAddr("192.168.3.4"), netip.MustParseAddr("10.5.0.2")}, spec.Addresses) + asrt.Equal("foo.com", spec.Hostname) + asrt.Equal("bar", spec.Nodename) + asrt.Equal(machine.TypeControlPlane, spec.MachineType) + asrt.Equal(netip.MustParseAddr("fd50:8d60:4238:6302:f857:23ff:fe21:d1e0"), spec.KubeSpan.Address) + asrt.Equal("PLPNBddmTgHJhtw0vxltq1ZBdPP9RNOEUd5JjJZzBRY=", spec.KubeSpan.PublicKey) + asrt.Equal([]netip.Prefix{netip.MustParsePrefix("10.244.3.1/24")}, spec.KubeSpan.AdditionalAddresses) + asrt.Equal([]netip.AddrPort{netip.MustParseAddrPort("10.0.0.2:51820"), netip.MustParseAddrPort("192.168.3.4:51820")}, spec.KubeSpan.Endpoints) + asrt.Equal(&cluster.ControlPlane{APIServerPort: 6443}, spec.ControlPlane) + }, + ) + + ctest.AssertResource( + suite, + affiliate3.TypedSpec().NodeID, + func(r *cluster.Affiliate, asrt *assert.Assertions) { + spec := r.TypedSpec() + + asrt.Equal(affiliate3.TypedSpec().NodeID, spec.NodeID) + asrt.Equal([]netip.Addr{netip.MustParseAddr("192.168.3.5")}, spec.Addresses) + asrt.Equal("worker-1", spec.Hostname) + asrt.Equal("worker-1", spec.Nodename) + asrt.Equal(machine.TypeWorker, spec.MachineType) + asrt.Zero(spec.KubeSpan.PublicKey) + asrt.Nil(spec.ControlPlane) + }, + ) + + // remove affiliate2, KubeSpan information should eventually go away + suite.Require().NoError(suite.state.Destroy(suite.ctx, affiliate1.Metadata())) + + ctest.AssertResource( + suite, + affiliate1.TypedSpec().NodeID, + func(r *cluster.Affiliate, asrt *assert.Assertions) { + spec := r.TypedSpec() + + asrt.Equal(affiliate1.TypedSpec().NodeID, spec.NodeID) + asrt.Zero(spec.KubeSpan.Address) + asrt.Zero(spec.KubeSpan.PublicKey) + asrt.Zero(spec.KubeSpan.AdditionalAddresses) + asrt.Zero(spec.KubeSpan.Endpoints) + asrt.Nil(spec.ControlPlane) + }, + ) + + // remove affiliate3, merged affiliate should be removed + suite.Require().NoError(suite.state.Destroy(suite.ctx, affiliate3.Metadata())) + + ctest.AssertNoResource[*cluster.Affiliate](suite, affiliate3.TypedSpec().NodeID) +} + +func TestAffiliateMergeSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, new(AffiliateMergeSuite)) +} diff --git a/internal/app/machined/pkg/controllers/cluster/cluster.go b/internal/app/machined/pkg/controllers/cluster/cluster.go new file mode 100644 index 0000000..4303d4f --- /dev/null +++ b/internal/app/machined/pkg/controllers/cluster/cluster.go @@ -0,0 +1,44 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package cluster provides controllers which manage Talos cluster resources. +package cluster + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" +) + +func cleanupAffiliates(ctx context.Context, ctrl controller.Controller, r controller.Runtime, touchedIDs map[resource.ID]struct{}) error { + // list keys for cleanup + list, err := safe.ReaderList[*cluster.Affiliate]( + ctx, + r, + resource.NewMetadata(cluster.RawNamespaceName, cluster.AffiliateType, "", resource.VersionUndefined), + ) + if err != nil { + return fmt.Errorf("error listing resources: %w", err) + } + + for it := list.Iterator(); it.Next(); { + res := it.Value() + if res.Metadata().Owner() != ctrl.Name() { + continue + } + + if _, ok := touchedIDs[res.Metadata().ID()]; !ok { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return fmt.Errorf("error cleaning up specs: %w", err) + } + } + } + + return nil +} diff --git a/internal/app/machined/pkg/controllers/cluster/cluster_test.go b/internal/app/machined/pkg/controllers/cluster/cluster_test.go new file mode 100644 index 0000000..75ffecc --- /dev/null +++ b/internal/app/machined/pkg/controllers/cluster/cluster_test.go @@ -0,0 +1,84 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cluster_test + +import ( + "context" + "log" + "sync" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + "github.com/siderolabs/talos/pkg/logging" +) + +type ClusterSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *ClusterSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + logger := logging.Wrap(log.Writer()) + + suite.runtime, err = runtime.NewRuntime(suite.state, logger) + suite.Require().NoError(err) +} + +func (suite *ClusterSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *ClusterSuite) assertResource(md resource.Metadata, check func(res resource.Resource) error) func() error { + return func() error { + r, err := suite.state.Get(suite.ctx, md) + if err != nil { + if state.IsNotFoundError(err) { + return retry.ExpectedError(err) + } + + return err + } + + return check(r) + } +} + +func (suite *ClusterSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func (suite *ClusterSuite) State() state.State { return suite.state } + +func (suite *ClusterSuite) Ctx() context.Context { return suite.ctx } diff --git a/internal/app/machined/pkg/controllers/cluster/config.go b/internal/app/machined/pkg/controllers/cluster/config.go new file mode 100644 index 0000000..1f50b70 --- /dev/null +++ b/internal/app/machined/pkg/controllers/cluster/config.go @@ -0,0 +1,93 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cluster + +import ( + "context" + "encoding/base64" + "net" + "net/url" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/controller/generic/transform" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" + "github.com/siderolabs/talos/pkg/machinery/resources/config" +) + +// ConfigController watches v1alpha1.Config, updates discovery config. +type ConfigController = transform.Controller[*config.MachineConfig, *cluster.Config] + +// NewConfigController instanciates the config controller. +func NewConfigController() *ConfigController { + return transform.NewController( + transform.Settings[*config.MachineConfig, *cluster.Config]{ + Name: "cluster.ConfigController", + MapMetadataOptionalFunc: func(cfg *config.MachineConfig) optional.Optional[*cluster.Config] { + if cfg.Metadata().ID() != config.V1Alpha1ID { + return optional.None[*cluster.Config]() + } + + if cfg.Config().Cluster() == nil { + return optional.None[*cluster.Config]() + } + + return optional.Some(cluster.NewConfig(config.NamespaceName, cluster.ConfigID)) + }, + TransformFunc: func(ctx context.Context, r controller.Reader, logger *zap.Logger, cfg *config.MachineConfig, res *cluster.Config) error { + c := cfg.Config() + + res.TypedSpec().DiscoveryEnabled = c.Cluster().Discovery().Enabled() + + if c.Cluster().Discovery().Enabled() { + res.TypedSpec().RegistryKubernetesEnabled = c.Cluster().Discovery().Registries().Kubernetes().Enabled() + res.TypedSpec().RegistryServiceEnabled = c.Cluster().Discovery().Registries().Service().Enabled() + + if c.Cluster().Discovery().Registries().Service().Enabled() { + var u *url.URL + + u, err := url.ParseRequestURI(c.Cluster().Discovery().Registries().Service().Endpoint()) + if err != nil { + return err + } + + host := u.Hostname() + port := u.Port() + + if port == "" { + if u.Scheme == "http" { + port = "80" + } else { + port = "443" // use default https port for everything else + } + } + + res.TypedSpec().ServiceEndpoint = net.JoinHostPort(host, port) + res.TypedSpec().ServiceEndpointInsecure = u.Scheme == "http" + + res.TypedSpec().ServiceEncryptionKey, err = base64.StdEncoding.DecodeString(c.Cluster().Secret()) + if err != nil { + return err + } + + res.TypedSpec().ServiceClusterID = c.Cluster().ID() + } else { + res.TypedSpec().ServiceEndpoint = "" + res.TypedSpec().ServiceEndpointInsecure = false + res.TypedSpec().ServiceEncryptionKey = nil + res.TypedSpec().ServiceClusterID = "" + } + } else { + res.TypedSpec().RegistryKubernetesEnabled = false + res.TypedSpec().RegistryServiceEnabled = false + } + + return nil + }, + }, + ) +} diff --git a/internal/app/machined/pkg/controllers/cluster/config_test.go b/internal/app/machined/pkg/controllers/cluster/config_test.go new file mode 100644 index 0000000..bb56acd --- /dev/null +++ b/internal/app/machined/pkg/controllers/cluster/config_test.go @@ -0,0 +1,199 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:dupl +package cluster_test + +import ( + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/siderolabs/go-pointer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + clusterctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/cluster" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" + "github.com/siderolabs/talos/pkg/machinery/resources/config" +) + +type ConfigSuite struct { + ctest.DefaultSuite +} + +func (suite *ConfigSuite) TestReconcileConfig() { + cfg := config.NewMachineConfig(container.NewV1Alpha1(&v1alpha1.Config{ + ConfigVersion: "v1alpha1", + ClusterConfig: &v1alpha1.ClusterConfig{ + ClusterID: "cluster1", + ClusterSecret: "kCQsKr4B28VUl7qw1sVkTDNF9fFH++ViIuKsss+C6kc=", + ClusterDiscoveryConfig: &v1alpha1.ClusterDiscoveryConfig{ + DiscoveryEnabled: pointer.To(true), + }, + }, + })) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{cluster.ConfigID}, + func(res *cluster.Config, asrt *assert.Assertions) { + spec := res.TypedSpec() + + asrt.True(spec.DiscoveryEnabled) + asrt.True(spec.RegistryKubernetesEnabled) + asrt.True(spec.RegistryServiceEnabled) + asrt.Equal("discovery.talos.dev:443", spec.ServiceEndpoint) + asrt.False(spec.ServiceEndpointInsecure) + asrt.Equal("cluster1", spec.ServiceClusterID) + asrt.Equal( + []byte("\x90\x24\x2c\x2a\xbe\x01\xdb\xc5\x54\x97\xba\xb0\xd6\xc5\x64\x4c\x33\x45\xf5\xf1\x47\xfb\xe5\x62\x22\xe2\xac\xb2\xcf\x82\xea\x47"), + spec.ServiceEncryptionKey, + ) + }) + + suite.Require().NoError(suite.State().Destroy(suite.Ctx(), cfg.Metadata())) + + rtestutils.AssertNoResource[*cluster.Config](suite.Ctx(), suite.T(), suite.State(), cluster.ConfigID) +} + +func (suite *ConfigSuite) TestReconcileConfigCustom() { + cfg := config.NewMachineConfig(container.NewV1Alpha1(&v1alpha1.Config{ + ConfigVersion: "v1alpha1", + ClusterConfig: &v1alpha1.ClusterConfig{ + ClusterID: "cluster1", + ClusterSecret: "kCQsKr4B28VUl7qw1sVkTDNF9fFH++ViIuKsss+C6kc=", + ClusterDiscoveryConfig: &v1alpha1.ClusterDiscoveryConfig{ + DiscoveryEnabled: pointer.To(true), + DiscoveryRegistries: v1alpha1.DiscoveryRegistriesConfig{ + RegistryKubernetes: v1alpha1.RegistryKubernetesConfig{ + RegistryDisabled: pointer.To(true), + }, + RegistryService: v1alpha1.RegistryServiceConfig{ + RegistryEndpoint: "https://[2001:470:6d:30e:565d:e162:e2a0:cf5a]:3456/", + }, + }, + }, + }, + })) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{cluster.ConfigID}, + func(res *cluster.Config, asrt *assert.Assertions) { + spec := res.TypedSpec() + + asrt.True(spec.DiscoveryEnabled) + asrt.False(spec.RegistryKubernetesEnabled) + asrt.True(spec.RegistryServiceEnabled) + asrt.Equal("[2001:470:6d:30e:565d:e162:e2a0:cf5a]:3456", spec.ServiceEndpoint) + asrt.False(spec.ServiceEndpointInsecure) + }, + ) +} + +func (suite *ConfigSuite) TestReconcileConfigCustomInsecure() { + cfg := config.NewMachineConfig(container.NewV1Alpha1(&v1alpha1.Config{ + ConfigVersion: "v1alpha1", + ClusterConfig: &v1alpha1.ClusterConfig{ + ClusterID: "cluster1", + ClusterSecret: "kCQsKr4B28VUl7qw1sVkTDNF9fFH++ViIuKsss+C6kc=", + ClusterDiscoveryConfig: &v1alpha1.ClusterDiscoveryConfig{ + DiscoveryEnabled: pointer.To(true), + DiscoveryRegistries: v1alpha1.DiscoveryRegistriesConfig{ + RegistryKubernetes: v1alpha1.RegistryKubernetesConfig{ + RegistryDisabled: pointer.To(true), + }, + RegistryService: v1alpha1.RegistryServiceConfig{ + RegistryEndpoint: "http://localhost:3000", + }, + }, + }, + }, + })) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{cluster.ConfigID}, + func(res *cluster.Config, asrt *assert.Assertions) { + spec := res.TypedSpec() + + asrt.True(spec.DiscoveryEnabled) + asrt.False(spec.RegistryKubernetesEnabled) + asrt.True(spec.RegistryServiceEnabled) + asrt.Equal("localhost:3000", spec.ServiceEndpoint) + asrt.True(spec.ServiceEndpointInsecure) + }, + ) +} + +func (suite *ConfigSuite) TestReconcileDisabled() { + cfg := config.NewMachineConfig(container.NewV1Alpha1(&v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{}, + ClusterConfig: &v1alpha1.ClusterConfig{}, + })) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{cluster.ConfigID}, + func(res *cluster.Config, asrt *assert.Assertions) { + spec := res.TypedSpec() + + asrt.False(spec.DiscoveryEnabled) + asrt.False(spec.RegistryKubernetesEnabled) + }, + ) +} + +func (suite *ConfigSuite) TestReconcilePartial() { + cfg := config.NewMachineConfig(container.NewV1Alpha1(&v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{}, + ClusterConfig: &v1alpha1.ClusterConfig{}, + })) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{cluster.ConfigID}, + func(res *cluster.Config, asrt *assert.Assertions) { + spec := res.TypedSpec() + + asrt.False(spec.DiscoveryEnabled) + asrt.False(spec.RegistryKubernetesEnabled) + }, + ) + + newCfg := config.NewMachineConfig(must(container.New())) + newCfg.Metadata().SetVersion(cfg.Metadata().Version()) + + suite.Require().NoError(suite.State().Update(suite.Ctx(), newCfg)) + + rtestutils.AssertNoResource[*cluster.Config](suite.Ctx(), suite.T(), suite.State(), cluster.ConfigID) +} + +func TestConfigSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, &ConfigSuite{ + DefaultSuite: ctest.DefaultSuite{ + Timeout: 5 * time.Second, + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(clusterctrl.NewConfigController())) + }, + }, + }) +} + +func must[T any](t T, err error) T { + if err != nil { + panic(err) + } + + return t +} diff --git a/internal/app/machined/pkg/controllers/cluster/discovery_service.go b/internal/app/machined/pkg/controllers/cluster/discovery_service.go new file mode 100644 index 0000000..9db2a7b --- /dev/null +++ b/internal/app/machined/pkg/controllers/cluster/discovery_service.go @@ -0,0 +1,502 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cluster + +import ( + "context" + "crypto/aes" + "crypto/cipher" + "errors" + "fmt" + "net/netip" + "time" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/discovery-api/api/v1alpha1/client/pb" + discoveryclient "github.com/siderolabs/discovery-client/pkg/client" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/xslices" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/proto" + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/kubespan" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" + "github.com/siderolabs/talos/pkg/machinery/version" +) + +const defaultDiscoveryTTL = 30 * time.Minute + +// DiscoveryServiceController pushes Affiliate resource to the Kubernetes registry. +type DiscoveryServiceController struct { + localAffiliateID resource.ID + discoveryConfigVersion resource.Version +} + +// Name implements controller.Controller interface. +func (ctrl *DiscoveryServiceController) Name() string { + return "cluster.DiscoveryServiceController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *DiscoveryServiceController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: cluster.ConfigType, + ID: optional.Some(cluster.ConfigID), + Kind: controller.InputWeak, + }, + { + Namespace: cluster.NamespaceName, + Type: cluster.IdentityType, + ID: optional.Some(cluster.LocalIdentity), + Kind: controller.InputWeak, + }, + { + Namespace: kubespan.NamespaceName, + Type: kubespan.EndpointType, + Kind: controller.InputWeak, + }, + { + Namespace: runtime.NamespaceName, + Type: runtime.MachineResetSignalType, + ID: optional.Some(runtime.MachineResetSignalID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *DiscoveryServiceController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: cluster.AffiliateType, + Kind: controller.OutputShared, + }, + { + Type: network.AddressStatusType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *DiscoveryServiceController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + var ( + client *discoveryclient.Client + clientCtxCancel context.CancelFunc + ) + + clientErrCh := make(chan error, 1) + + defer func() { + if clientCtxCancel != nil { + clientCtxCancel() + + <-clientErrCh + } + }() + + notifyCh := make(chan struct{}, 1) + + var ( + prevLocalData *pb.Affiliate + prevLocalEndpoints []*pb.Endpoint + prevOtherEndpoints []discoveryclient.Endpoint + ) + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + case <-notifyCh: + case err := <-clientErrCh: + if clientCtxCancel != nil { + clientCtxCancel() + } + + clientCtxCancel = nil + + if err != nil && !errors.Is(err, context.Canceled) { + return fmt.Errorf("error from discovery client: %w", err) + } + } + + cleanupClient := func() { + if clientCtxCancel != nil { + clientCtxCancel() + + <-clientErrCh + + clientCtxCancel = nil + client = nil + + prevLocalData = nil + prevLocalEndpoints = nil + prevOtherEndpoints = nil + } + } + + discoveryConfig, err := safe.ReaderGetByID[*cluster.Config](ctx, r, cluster.ConfigID) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting discovery config: %w", err) + } + + continue + } + + if !discoveryConfig.TypedSpec().RegistryServiceEnabled { + // if discovery is disabled cleanup existing resources + if err = cleanupAffiliates(ctx, ctrl, r, nil); err != nil { + return err + } + + cleanupClient() + + continue + } + + if !discoveryConfig.Metadata().Version().Equal(ctrl.discoveryConfigVersion) { + // force reconnect on config change + cleanupClient() + } + + identity, err := safe.ReaderGetByID[*cluster.Identity](ctx, r, cluster.LocalIdentity) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting local identity: %w", err) + } + + continue + } + + localAffiliateID := identity.TypedSpec().NodeID + + if ctrl.localAffiliateID != localAffiliateID { + ctrl.localAffiliateID = localAffiliateID + + if err = r.UpdateInputs(append(ctrl.Inputs(), + controller.Input{ + Namespace: cluster.NamespaceName, + Type: cluster.AffiliateType, + ID: optional.Some(ctrl.localAffiliateID), + Kind: controller.InputWeak, + }, + )); err != nil { + return err + } + + cleanupClient() + } + + affiliate, err := safe.ReaderGetByID[*cluster.Affiliate](ctx, r, ctrl.localAffiliateID) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting local affiliate: %w", err) + } + + continue + } + + affiliateSpec := affiliate.TypedSpec() + + otherEndpointsList, err := safe.ReaderListAll[*kubespan.Endpoint](ctx, r) + if err != nil { + return fmt.Errorf("error listing endpoints: %w", err) + } + + machineResetSginal, err := safe.ReaderGetByID[*runtime.MachineResetSignal](ctx, r, runtime.MachineResetSignalID) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error getting machine reset signal: %w", err) + } + + if client == nil { + var cipherBlock cipher.Block + + cipherBlock, err = aes.NewCipher(discoveryConfig.TypedSpec().ServiceEncryptionKey) + if err != nil { + return fmt.Errorf("error initializing AES cipher: %w", err) + } + + client, err = discoveryclient.NewClient(discoveryclient.Options{ + Cipher: cipherBlock, + Endpoint: discoveryConfig.TypedSpec().ServiceEndpoint, + ClusterID: discoveryConfig.TypedSpec().ServiceClusterID, + AffiliateID: localAffiliateID, + TTL: defaultDiscoveryTTL, + Insecure: discoveryConfig.TypedSpec().ServiceEndpointInsecure, + ClientVersion: version.Tag, + }) + if err != nil { + return fmt.Errorf("error initializing discovery client: %w", err) + } + + var clientCtx context.Context + + clientCtx, clientCtxCancel = context.WithCancel(ctx) //nolint:govet + + ctrl.discoveryConfigVersion = discoveryConfig.Metadata().Version() + + go func() { + clientErrCh <- client.Run(clientCtx, logger, notifyCh) + }() + } + + // delete/update local affiliate + // + // if the node enters final resetting stage, cleanup the local affiliate + // otherwise, update local affiliate data + if machineResetSginal != nil { + client.DeleteLocalAffiliate() + } else { + localData := pbAffiliate(affiliateSpec) + localEndpoints := pbEndpoints(affiliateSpec) + otherEndpoints := pbOtherEndpoints(otherEndpointsList) + + // don't send updates on localData if it hasn't changed: this introduces positive feedback loop, + // as the watch loop will notify on self update + if !proto.Equal(localData, prevLocalData) || !equalEndpoints(localEndpoints, prevLocalEndpoints) || !equalOtherEndpoints(otherEndpoints, prevOtherEndpoints) { + if err = client.SetLocalData(&discoveryclient.Affiliate{ + Affiliate: localData, + Endpoints: localEndpoints, + }, otherEndpoints); err != nil { + return fmt.Errorf("error setting local affiliate data: %w", err) + } + + prevLocalData = localData + prevLocalEndpoints = localEndpoints + prevOtherEndpoints = otherEndpoints + } + } + + // discover public IP + if publicIP := client.GetPublicIP(); len(publicIP) > 0 { + if err = safe.WriterModify(ctx, r, network.NewAddressStatus(cluster.NamespaceName, "service"), func(address *network.AddressStatus) error { + var addr netip.Addr + + if err = addr.UnmarshalBinary(publicIP); err != nil { + return fmt.Errorf("error unmarshaling public IP: %w", err) + } + + address.TypedSpec().Address = netip.PrefixFrom(addr, addr.BitLen()) + + return nil + }); err != nil { + return err //nolint:govet + } + } + + // discover other nodes (affiliates) + touchedIDs := make(map[resource.ID]struct{}) + + for _, discoveredAffiliate := range client.GetAffiliates() { + id := fmt.Sprintf("service/%s", discoveredAffiliate.Affiliate.NodeId) + + if err = safe.WriterModify(ctx, r, cluster.NewAffiliate(cluster.RawNamespaceName, id), func(res *cluster.Affiliate) error { + *res.TypedSpec() = specAffiliate(discoveredAffiliate.Affiliate, discoveredAffiliate.Endpoints) + + return nil + }); err != nil { + return err + } + + touchedIDs[id] = struct{}{} + } + + if err := cleanupAffiliates(ctx, ctrl, r, touchedIDs); err != nil { + return err + } + + r.ResetRestartBackoff() + } +} + +func pbAffiliate(affiliate *cluster.AffiliateSpec) *pb.Affiliate { + addresses := xslices.Map(affiliate.Addresses, func(address netip.Addr) []byte { + return takeResult(address.MarshalBinary()) + }) + + var kubeSpan *pb.KubeSpan + + if affiliate.KubeSpan.PublicKey != "" { + kubeSpan = &pb.KubeSpan{ + PublicKey: affiliate.KubeSpan.PublicKey, + Address: takeResult(affiliate.KubeSpan.Address.MarshalBinary()), + AdditionalAddresses: xslices.Map(affiliate.KubeSpan.AdditionalAddresses, func(address netip.Prefix) *pb.IPPrefix { + return &pb.IPPrefix{ + Bits: uint32(address.Bits()), + Ip: takeResult(address.Addr().MarshalBinary()), + } + }), + } + } + + return &pb.Affiliate{ + NodeId: affiliate.NodeID, + Addresses: addresses, + Hostname: affiliate.Hostname, + Nodename: affiliate.Nodename, + MachineType: affiliate.MachineType.String(), + OperatingSystem: affiliate.OperatingSystem, + Kubespan: kubeSpan, + ControlPlane: toPlane(affiliate.ControlPlane), + } +} + +func toPlane(data *cluster.ControlPlane) *pb.ControlPlane { + if data == nil { + return nil + } + + return &pb.ControlPlane{ApiServerPort: uint32(data.APIServerPort)} +} + +func pbEndpoints(affiliate *cluster.AffiliateSpec) []*pb.Endpoint { + if affiliate.KubeSpan.PublicKey == "" || len(affiliate.KubeSpan.Endpoints) == 0 { + return nil + } + + return xslices.Map(affiliate.KubeSpan.Endpoints, func(endpoint netip.AddrPort) *pb.Endpoint { + return &pb.Endpoint{ + Port: uint32(endpoint.Port()), + Ip: takeResult(endpoint.Addr().MarshalBinary()), + } + }) +} + +func pbOtherEndpoints(otherEndpointsList safe.List[*kubespan.Endpoint]) []discoveryclient.Endpoint { + if otherEndpointsList.Len() == 0 { + return nil + } + + result := make([]discoveryclient.Endpoint, 0, otherEndpointsList.Len()) + + for it := otherEndpointsList.Iterator(); it.Next(); { + endpoint := it.Value().TypedSpec() + + result = append(result, discoveryclient.Endpoint{ + AffiliateID: endpoint.AffiliateID, + Endpoints: []*pb.Endpoint{ + { + Port: uint32(endpoint.Endpoint.Port()), + Ip: takeResult(endpoint.Endpoint.Addr().MarshalBinary()), + }, + }, + }) + } + + return result +} + +func equalEndpoints(a, b []*pb.Endpoint) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + + if len(a) != len(b) { + return false + } + + for i := range a { + if !proto.Equal(a[i], b[i]) { + return false + } + } + + return true +} + +func equalOtherEndpoints(a, b []discoveryclient.Endpoint) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + + if len(a) != len(b) { + return false + } + + for i := range a { + if a[i].AffiliateID != b[i].AffiliateID { + return false + } + + if !equalEndpoints(a[i].Endpoints, b[i].Endpoints) { + return false + } + } + + return true +} + +func specAffiliate(affiliate *pb.Affiliate, endpoints []*pb.Endpoint) cluster.AffiliateSpec { + result := cluster.AffiliateSpec{ + NodeID: affiliate.NodeId, + Hostname: affiliate.Hostname, + Nodename: affiliate.Nodename, + OperatingSystem: affiliate.OperatingSystem, + MachineType: takeResult(machine.ParseType(affiliate.MachineType)), // ignore parse error (machine.TypeUnknown) + ControlPlane: fromControlPlane(affiliate.ControlPlane), + } + + result.Addresses = make([]netip.Addr, 0, len(affiliate.Addresses)) + + for i := range affiliate.Addresses { + var ip netip.Addr + + if err := ip.UnmarshalBinary(affiliate.Addresses[i]); err == nil { + result.Addresses = append(result.Addresses, ip) + } + } + + if affiliate.Kubespan != nil { + result.KubeSpan.PublicKey = affiliate.Kubespan.PublicKey + result.KubeSpan.Address.UnmarshalBinary(affiliate.Kubespan.Address) //nolint:errcheck // ignore error, address will be zero + + result.KubeSpan.AdditionalAddresses = make([]netip.Prefix, 0, len(affiliate.Kubespan.AdditionalAddresses)) + + for i := range affiliate.Kubespan.AdditionalAddresses { + var ip netip.Addr + + if err := ip.UnmarshalBinary(affiliate.Kubespan.AdditionalAddresses[i].Ip); err == nil { + result.KubeSpan.AdditionalAddresses = append(result.KubeSpan.AdditionalAddresses, netip.PrefixFrom(ip, int(affiliate.Kubespan.AdditionalAddresses[i].Bits))) + } + } + + result.KubeSpan.Endpoints = make([]netip.AddrPort, 0, len(endpoints)) + + for i := range endpoints { + var ip netip.Addr + + if err := ip.UnmarshalBinary(endpoints[i].Ip); err == nil { + result.KubeSpan.Endpoints = append(result.KubeSpan.Endpoints, netip.AddrPortFrom(ip, uint16(endpoints[i].Port))) + } + } + } + + return result +} + +func fromControlPlane(plane *pb.ControlPlane) *cluster.ControlPlane { + if plane == nil { + return nil + } + + return &cluster.ControlPlane{APIServerPort: int(plane.ApiServerPort)} +} + +func takeResult[T any](arg1 T, _ error) T { + return arg1 +} diff --git a/internal/app/machined/pkg/controllers/cluster/discovery_service_test.go b/internal/app/machined/pkg/controllers/cluster/discovery_service_test.go new file mode 100644 index 0000000..5036fb2 --- /dev/null +++ b/internal/app/machined/pkg/controllers/cluster/discovery_service_test.go @@ -0,0 +1,371 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cluster_test + +import ( + "context" + "crypto/aes" + "crypto/rand" + "encoding/base64" + "io" + "log" + "net/netip" + "net/url" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/siderolabs/discovery-api/api/v1alpha1/client/pb" + "github.com/siderolabs/discovery-client/pkg/client" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + clusteradapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/cluster" + clusterctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/cluster" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/proto" + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/kubespan" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +type DiscoveryServiceSuite struct { + ClusterSuite +} + +func (suite *DiscoveryServiceSuite) TestReconcile() { + suite.startRuntime() + + suite.Require().NoError(suite.runtime.RegisterController(&clusterctrl.DiscoveryServiceController{})) + + serviceEndpoint, err := url.Parse(constants.DefaultDiscoveryServiceEndpoint) + suite.Require().NoError(err) + + if serviceEndpoint.Port() == "" { + serviceEndpoint.Host += ":443" + } + + clusterIDRaw := make([]byte, constants.DefaultClusterIDSize) + _, err = io.ReadFull(rand.Reader, clusterIDRaw) + suite.Require().NoError(err) + + clusterID := base64.StdEncoding.EncodeToString(clusterIDRaw) + + encryptionKey := make([]byte, constants.DefaultClusterSecretSize) + _, err = io.ReadFull(rand.Reader, encryptionKey) + suite.Require().NoError(err) + + // regular discovery affiliate + discoveryConfig := cluster.NewConfig(config.NamespaceName, cluster.ConfigID) + discoveryConfig.TypedSpec().DiscoveryEnabled = true + discoveryConfig.TypedSpec().RegistryServiceEnabled = true + discoveryConfig.TypedSpec().ServiceEndpoint = serviceEndpoint.Host + discoveryConfig.TypedSpec().ServiceClusterID = clusterID + discoveryConfig.TypedSpec().ServiceEncryptionKey = encryptionKey + suite.Require().NoError(suite.state.Create(suite.ctx, discoveryConfig)) + + nodeIdentity := cluster.NewIdentity(cluster.NamespaceName, cluster.LocalIdentity) + suite.Require().NoError(clusteradapter.IdentitySpec(nodeIdentity.TypedSpec()).Generate()) + suite.Require().NoError(suite.state.Create(suite.ctx, nodeIdentity)) + + localAffiliate := cluster.NewAffiliate(cluster.NamespaceName, nodeIdentity.TypedSpec().NodeID) + *localAffiliate.TypedSpec() = cluster.AffiliateSpec{ + NodeID: nodeIdentity.TypedSpec().NodeID, + Hostname: "foo.com", + Nodename: "bar", + MachineType: machine.TypeControlPlane, + Addresses: []netip.Addr{netip.MustParseAddr("192.168.3.4")}, + KubeSpan: cluster.KubeSpanAffiliateSpec{ + PublicKey: "PLPNBddmTgHJhtw0vxltq1ZBdPP9RNOEUd5JjJZzBRY=", + Address: netip.MustParseAddr("fd50:8d60:4238:6302:f857:23ff:fe21:d1e0"), + AdditionalAddresses: []netip.Prefix{netip.MustParsePrefix("10.244.3.1/24")}, + Endpoints: []netip.AddrPort{netip.MustParseAddrPort("10.0.0.2:51820"), netip.MustParseAddrPort("192.168.3.4:51820")}, + }, + ControlPlane: &cluster.ControlPlane{APIServerPort: 6443}, + } + suite.Require().NoError(suite.state.Create(suite.ctx, localAffiliate)) + + // create a test client connected to the same cluster but under different affiliate ID + cipher, err := aes.NewCipher(discoveryConfig.TypedSpec().ServiceEncryptionKey) + suite.Require().NoError(err) + + cli, err := client.NewClient(client.Options{ + Cipher: cipher, + Endpoint: serviceEndpoint.Host, + ClusterID: discoveryConfig.TypedSpec().ServiceClusterID, + AffiliateID: "7x1SuC8Ege5BGXdAfTEff5iQnlWZLfv9h1LGMxA2pYkC", + TTL: 5 * time.Minute, + }) + suite.Require().NoError(err) + + errCh := make(chan error, 1) + notifyCh := make(chan struct{}, 1) + + cliCtx, cliCtxCancel := context.WithCancel(suite.ctx) + defer cliCtxCancel() + + go func() { + errCh <- cli.Run(cliCtx, logging.Wrap(log.Writer()), notifyCh) + }() + + suite.Assert().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + // controller should register its local affiliate, and we should see it being discovered + affiliates := cli.GetAffiliates() + + if len(affiliates) != 1 { + return retry.ExpectedErrorf("affiliates len %d != 1", len(affiliates)) + } + + suite.Require().Len(affiliates[0].Endpoints, 2) + suite.Assert().True(proto.Equal(&pb.Affiliate{ + NodeId: nodeIdentity.TypedSpec().NodeID, + Addresses: [][]byte{[]byte("\xc0\xa8\x03\x04")}, + Hostname: "foo.com", + Nodename: "bar", + MachineType: "controlplane", + OperatingSystem: "", + Kubespan: &pb.KubeSpan{ + PublicKey: "PLPNBddmTgHJhtw0vxltq1ZBdPP9RNOEUd5JjJZzBRY=", + Address: []byte("\xfd\x50\x8d\x60\x42\x38\x63\x02\xf8\x57\x23\xff\xfe\x21\xd1\xe0"), + AdditionalAddresses: []*pb.IPPrefix{ + { + Ip: []byte("\x0a\xf4\x03\x01"), + Bits: 24, + }, + }, + }, + ControlPlane: &pb.ControlPlane{ApiServerPort: 6443}, + }, affiliates[0].Affiliate)) + suite.Assert().True(proto.Equal( + &pb.Endpoint{ + Ip: []byte("\n\x00\x00\x02"), + Port: 51820, + }, + affiliates[0].Endpoints[0]), "expected %v", affiliates[0].Endpoints[0]) + suite.Assert().True(proto.Equal( + &pb.Endpoint{ + Ip: []byte("\xc0\xa8\x03\x04"), + Port: 51820, + }, + affiliates[0].Endpoints[1]), "expected %v", affiliates[0].Endpoints[1]) + + return nil + }, + )) + + // inject some affiliate via our client, controller should publish it as an affiliate + suite.Require().NoError(cli.SetLocalData(&client.Affiliate{ + Affiliate: &pb.Affiliate{ + NodeId: "7x1SuC8Ege5BGXdAfTEff5iQnlWZLfv9h1LGMxA2pYkC", + Addresses: [][]byte{[]byte("\xc0\xa8\x03\x05")}, + Hostname: "some.com", + Nodename: "some", + MachineType: "worker", + OperatingSystem: "test OS", + Kubespan: &pb.KubeSpan{ + PublicKey: "1CXkdhWBm58c36kTpchR8iGlXHG1ruHa5W8gsFqD8Qs=", + Address: []byte("\xfd\x50\x8d\x60\x42\x38\x63\x02\xf8\x57\x23\xff\xfe\x21\xd1\xe1"), + AdditionalAddresses: []*pb.IPPrefix{ + { + Ip: []byte("\x0a\xf4\x04\x01"), + Bits: 24, + }, + }, + }, + }, + Endpoints: []*pb.Endpoint{ + { + Ip: []byte("\xc0\xa8\x03\x05"), + Port: 51820, + }, + }, + }, nil)) + + ctest.AssertResource( + suite, + "service/7x1SuC8Ege5BGXdAfTEff5iQnlWZLfv9h1LGMxA2pYkC", + func(r *cluster.Affiliate, asrt *assert.Assertions) { + spec := r.TypedSpec() + + suite.Assert().Equal("7x1SuC8Ege5BGXdAfTEff5iQnlWZLfv9h1LGMxA2pYkC", spec.NodeID) + suite.Assert().Equal([]netip.Addr{netip.MustParseAddr("192.168.3.5")}, spec.Addresses) + suite.Assert().Equal("some.com", spec.Hostname) + suite.Assert().Equal("some", spec.Nodename) + suite.Assert().Equal(machine.TypeWorker, spec.MachineType) + suite.Assert().Equal("test OS", spec.OperatingSystem) + suite.Assert().Equal(netip.MustParseAddr("fd50:8d60:4238:6302:f857:23ff:fe21:d1e1"), spec.KubeSpan.Address) + suite.Assert().Equal("1CXkdhWBm58c36kTpchR8iGlXHG1ruHa5W8gsFqD8Qs=", spec.KubeSpan.PublicKey) + suite.Assert().Equal([]netip.Prefix{netip.MustParsePrefix("10.244.4.1/24")}, spec.KubeSpan.AdditionalAddresses) + suite.Assert().Equal([]netip.AddrPort{netip.MustParseAddrPort("192.168.3.5:51820")}, spec.KubeSpan.Endpoints) + suite.Assert().Zero(spec.ControlPlane) + }, + rtestutils.WithNamespace(cluster.RawNamespaceName), + ) + + // controller should publish public IP + ctest.AssertResource(suite, "service", func(r *network.AddressStatus, assertions *assert.Assertions) { + spec := r.TypedSpec() + + assertions.True(spec.Address.IsValid()) + assertions.True(spec.Address.IsSingleIP()) + }, rtestutils.WithNamespace(cluster.NamespaceName)) + + // make controller inject additional endpoint via kubespan.Endpoint + endpoint := kubespan.NewEndpoint(kubespan.NamespaceName, "1CXkdhWBm58c36kTpchR8iGlXHG1ruHa5W8gsFqD8Qs=") + *endpoint.TypedSpec() = kubespan.EndpointSpec{ + AffiliateID: "7x1SuC8Ege5BGXdAfTEff5iQnlWZLfv9h1LGMxA2pYkC", + Endpoint: netip.MustParseAddrPort("1.1.1.1:343"), + } + suite.Require().NoError(suite.state.Create(suite.ctx, endpoint)) + + ctest.AssertResource(suite, + "service/7x1SuC8Ege5BGXdAfTEff5iQnlWZLfv9h1LGMxA2pYkC", + func(r *cluster.Affiliate, assertions *assert.Assertions) { + spec := r.TypedSpec() + + assertions.Len(spec.KubeSpan.Endpoints, 2) + assertions.Equal([]netip.AddrPort{ + netip.MustParseAddrPort("192.168.3.5:51820"), + netip.MustParseAddrPort("1.1.1.1:343"), + }, spec.KubeSpan.Endpoints) + }, + rtestutils.WithNamespace(cluster.RawNamespaceName), + ) + + // pretend that machine is being reset + machineResetSignal := runtime.NewMachineResetSignal() + suite.Require().NoError(suite.state.Create(suite.ctx, machineResetSignal)) + + // client should see the affiliate being deleted + suite.Assert().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + // controller should delete its local affiliate + affiliates := cli.GetAffiliates() + + if len(affiliates) != 0 { + return retry.ExpectedErrorf("affiliates len %d != 0", len(affiliates)) + } + + return nil + }, + )) + + cliCtxCancel() + suite.Assert().NoError(<-errCh) +} + +func (suite *DiscoveryServiceSuite) TestDisable() { + suite.startRuntime() + + suite.Require().NoError(suite.runtime.RegisterController(&clusterctrl.DiscoveryServiceController{})) + + serviceEndpoint, err := url.Parse(constants.DefaultDiscoveryServiceEndpoint) + suite.Require().NoError(err) + + if serviceEndpoint.Port() == "" { + serviceEndpoint.Host += ":443" + } + + clusterIDRaw := make([]byte, constants.DefaultClusterIDSize) + _, err = io.ReadFull(rand.Reader, clusterIDRaw) + suite.Require().NoError(err) + + clusterID := base64.StdEncoding.EncodeToString(clusterIDRaw) + + encryptionKey := make([]byte, constants.DefaultClusterSecretSize) + _, err = io.ReadFull(rand.Reader, encryptionKey) + suite.Require().NoError(err) + + // regular discovery affiliate + discoveryConfig := cluster.NewConfig(config.NamespaceName, cluster.ConfigID) + discoveryConfig.TypedSpec().DiscoveryEnabled = true + discoveryConfig.TypedSpec().RegistryServiceEnabled = true + discoveryConfig.TypedSpec().ServiceEndpoint = serviceEndpoint.Host + discoveryConfig.TypedSpec().ServiceClusterID = clusterID + discoveryConfig.TypedSpec().ServiceEncryptionKey = encryptionKey + suite.Require().NoError(suite.state.Create(suite.ctx, discoveryConfig)) + + nodeIdentity := cluster.NewIdentity(cluster.NamespaceName, cluster.LocalIdentity) + suite.Require().NoError(clusteradapter.IdentitySpec(nodeIdentity.TypedSpec()).Generate()) + suite.Require().NoError(suite.state.Create(suite.ctx, nodeIdentity)) + + localAffiliate := cluster.NewAffiliate(cluster.NamespaceName, nodeIdentity.TypedSpec().NodeID) + *localAffiliate.TypedSpec() = cluster.AffiliateSpec{ + NodeID: nodeIdentity.TypedSpec().NodeID, + Hostname: "foo.com", + Nodename: "bar", + MachineType: machine.TypeControlPlane, + Addresses: []netip.Addr{netip.MustParseAddr("192.168.3.4")}, + } + suite.Require().NoError(suite.state.Create(suite.ctx, localAffiliate)) + + // create a test client connected to the same cluster but under different affiliate ID + cipher, err := aes.NewCipher(discoveryConfig.TypedSpec().ServiceEncryptionKey) + suite.Require().NoError(err) + + cli, err := client.NewClient(client.Options{ + Cipher: cipher, + Endpoint: serviceEndpoint.Host, + ClusterID: discoveryConfig.TypedSpec().ServiceClusterID, + AffiliateID: "7x1SuC8Ege5BGXdAfTEff5iQnlWZLfv9h1LGMxA2pYkC", + TTL: 5 * time.Minute, + }) + suite.Require().NoError(err) + + errCh := make(chan error, 1) + notifyCh := make(chan struct{}, 1) + + cliCtx, cliCtxCancel := context.WithCancel(suite.ctx) + defer cliCtxCancel() + + go func() { + errCh <- cli.Run(cliCtx, logging.Wrap(log.Writer()), notifyCh) + }() + + // inject some affiliate via our client, controller should publish it as an affiliate + suite.Require().NoError(cli.SetLocalData(&client.Affiliate{ + Affiliate: &pb.Affiliate{ + NodeId: "7x1SuC8Ege5BGXdAfTEff5iQnlWZLfv9h1LGMxA2pYkC", + }, + }, nil)) + + ctest.AssertResource( + suite, + "service/7x1SuC8Ege5BGXdAfTEff5iQnlWZLfv9h1LGMxA2pYkC", + func(r *cluster.Affiliate, asrt *assert.Assertions) { + suite.Assert().Equal("7x1SuC8Ege5BGXdAfTEff5iQnlWZLfv9h1LGMxA2pYkC", r.TypedSpec().NodeID) + }, + rtestutils.WithNamespace(cluster.RawNamespaceName), + ) + + // now disable the service registry + ctest.UpdateWithConflicts(suite, discoveryConfig, func(r *cluster.Config) error { + r.TypedSpec().RegistryServiceEnabled = false + + return nil + }) + + ctest.AssertNoResource[*cluster.Affiliate]( + suite, + "service/7x1SuC8Ege5BGXdAfTEff5iQnlWZLfv9h1LGMxA2pYkC", + rtestutils.WithNamespace(cluster.RawNamespaceName), + ) + + cliCtxCancel() + suite.Assert().NoError(<-errCh) +} + +func TestDiscoveryServiceSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, new(DiscoveryServiceSuite)) +} diff --git a/internal/app/machined/pkg/controllers/cluster/endpoint.go b/internal/app/machined/pkg/controllers/cluster/endpoint.go new file mode 100644 index 0000000..3ef2287 --- /dev/null +++ b/internal/app/machined/pkg/controllers/cluster/endpoint.go @@ -0,0 +1,99 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cluster + +import ( + "context" + "fmt" + "net/netip" + "reflect" + "sort" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +// EndpointController looks up control plane endpoints. +type EndpointController struct{} + +// Name implements controller.Controller interface. +func (ctrl *EndpointController) Name() string { + return "cluster.EndpointController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *EndpointController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: cluster.NamespaceName, + Type: cluster.MemberType, + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *EndpointController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: k8s.EndpointType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +func (ctrl *EndpointController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + memberList, err := safe.ReaderListAll[*cluster.Member](ctx, r) + if err != nil { + return fmt.Errorf("error listing members: %w", err) + } + + var endpoints []netip.Addr + + for it := memberList.Iterator(); it.Next(); { + member := it.Value().TypedSpec() + + if !(member.MachineType == machine.TypeControlPlane || member.MachineType == machine.TypeInit) { + continue + } + + endpoints = append(endpoints, member.Addresses...) + } + + sort.Slice(endpoints, func(i, j int) bool { return endpoints[i].Compare(endpoints[j]) < 0 }) + + if err := safe.WriterModify( + ctx, + r, + k8s.NewEndpoint(k8s.ControlPlaneNamespaceName, k8s.ControlPlaneDiscoveredEndpointsID), + func(r *k8s.Endpoint) error { + if !reflect.DeepEqual(r.TypedSpec().Addresses, endpoints) { + logger.Debug("updated controlplane endpoints", zap.Any("endpoints", endpoints)) + } + + r.TypedSpec().Addresses = endpoints + + return nil + }, + ); err != nil { + return fmt.Errorf("error updating endpoints: %w", err) + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/cluster/endpoint_test.go b/internal/app/machined/pkg/controllers/cluster/endpoint_test.go new file mode 100644 index 0000000..eb2fd2a --- /dev/null +++ b/internal/app/machined/pkg/controllers/cluster/endpoint_test.go @@ -0,0 +1,83 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cluster_test + +import ( + "net/netip" + "testing" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/siderolabs/gen/xslices" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + clusterctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/cluster" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +type EndpointSuite struct { + ClusterSuite +} + +func (suite *EndpointSuite) TestReconcileDefault() { + suite.startRuntime() + + suite.Require().NoError(suite.runtime.RegisterController(&clusterctrl.EndpointController{})) + + member1 := cluster.NewMember(cluster.NamespaceName, "talos-default-controlplane-1") + *member1.TypedSpec() = cluster.MemberSpec{ + NodeID: "7x1SuC8Ege5BGXdAfTEff5iQnlWZLfv9h1LGMxA2pYkC", + Addresses: []netip.Addr{netip.MustParseAddr("172.20.0.2"), netip.MustParseAddr("fd50:8d60:4238:6302:f857:23ff:fe21:d1e0")}, + Hostname: "talos-default-controlplane-1", + MachineType: machine.TypeControlPlane, + OperatingSystem: "Talos (v1.0.0)", + } + + member2 := cluster.NewMember(cluster.NamespaceName, "talos-default-controlplane-2") + *member2.TypedSpec() = cluster.MemberSpec{ + NodeID: "9dwHNUViZlPlIervqX9Qo256RUhrfhgO0xBBnKcKl4F", + Addresses: []netip.Addr{netip.MustParseAddr("172.20.0.3"), netip.MustParseAddr("fd50:8d60:4238:6302:f857:23ff:fe21:d1e1")}, + Hostname: "talos-default-controlplane-2", + MachineType: machine.TypeControlPlane, + OperatingSystem: "Talos (v1.0.0)", + } + + member3 := cluster.NewMember(cluster.NamespaceName, "talos-default-worker-1") + *member3.TypedSpec() = cluster.MemberSpec{ + NodeID: "xCnFFfxylOf9i5ynhAkt6ZbfcqaLDGKfIa3gwpuaxe7F", + Addresses: []netip.Addr{netip.MustParseAddr("172.20.0.4")}, + Hostname: "talos-default-worker-1", + MachineType: machine.TypeWorker, + OperatingSystem: "Talos (v1.0.0)", + } + + for _, r := range []resource.Resource{member1, member2, member3} { + suite.Require().NoError(suite.state.Create(suite.ctx, r)) + } + + // control plane members should be translated to Endpoints + ctest.AssertResource(suite, k8s.ControlPlaneDiscoveredEndpointsID, func(r *k8s.Endpoint, asrt *assert.Assertions) { + spec := r.TypedSpec() + + asrt.Equal( + []string{ + "172.20.0.2", + "172.20.0.3", + "fd50:8d60:4238:6302:f857:23ff:fe21:d1e0", + "fd50:8d60:4238:6302:f857:23ff:fe21:d1e1", + }, + xslices.Map(spec.Addresses, netip.Addr.String), + ) + }) +} + +func TestEndpointSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, new(EndpointSuite)) +} diff --git a/internal/app/machined/pkg/controllers/cluster/info.go b/internal/app/machined/pkg/controllers/cluster/info.go new file mode 100644 index 0000000..18420fd --- /dev/null +++ b/internal/app/machined/pkg/controllers/cluster/info.go @@ -0,0 +1,46 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cluster + +import ( + "context" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/controller/generic/transform" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" + "github.com/siderolabs/talos/pkg/machinery/resources/config" +) + +// InfoController looks up control plane infos. +type InfoController = transform.Controller[*config.MachineConfig, *cluster.Info] + +// NewInfoController instanciates the cluster info controller. +func NewInfoController() *InfoController { + return transform.NewController( + transform.Settings[*config.MachineConfig, *cluster.Info]{ + Name: "cluster.InfoController", + MapMetadataOptionalFunc: func(cfg *config.MachineConfig) optional.Optional[*cluster.Info] { + if cfg.Metadata().ID() != config.V1Alpha1ID { + return optional.None[*cluster.Info]() + } + + if cfg.Config().Cluster() == nil { + return optional.None[*cluster.Info]() + } + + return optional.Some(cluster.NewInfo()) + }, + TransformFunc: func(ctx context.Context, r controller.Reader, logger *zap.Logger, cfg *config.MachineConfig, info *cluster.Info) error { + info.TypedSpec().ClusterID = cfg.Config().Cluster().ID() + info.TypedSpec().ClusterName = cfg.Config().Cluster().Name() + + return nil + }, + }, + ) +} diff --git a/internal/app/machined/pkg/controllers/cluster/info_test.go b/internal/app/machined/pkg/controllers/cluster/info_test.go new file mode 100644 index 0000000..5738b58 --- /dev/null +++ b/internal/app/machined/pkg/controllers/cluster/info_test.go @@ -0,0 +1,63 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cluster_test + +import ( + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + clusterctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/cluster" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" + "github.com/siderolabs/talos/pkg/machinery/resources/config" +) + +type InfoSuite struct { + ctest.DefaultSuite +} + +func (suite *InfoSuite) TestReconcile() { + cfg := config.NewMachineConfig(container.NewV1Alpha1(&v1alpha1.Config{ + ConfigVersion: "v1alpha1", + ClusterConfig: &v1alpha1.ClusterConfig{ + ClusterID: "cluster1", + ClusterName: "foo", + }, + })) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{cluster.InfoID}, + func(res *cluster.Info, asrt *assert.Assertions) { + spec := res.TypedSpec() + + asrt.Equal("cluster1", spec.ClusterID) + asrt.Equal("foo", spec.ClusterName) + }) + + suite.Require().NoError(suite.State().Destroy(suite.Ctx(), cfg.Metadata())) + + rtestutils.AssertNoResource[*cluster.Config](suite.Ctx(), suite.T(), suite.State(), cluster.ConfigID) +} + +func TestInfoSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, &InfoSuite{ + DefaultSuite: ctest.DefaultSuite{ + Timeout: 5 * time.Second, + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(clusterctrl.NewInfoController())) + }, + }, + }) +} diff --git a/internal/app/machined/pkg/controllers/cluster/kubernetes_pull.go b/internal/app/machined/pkg/controllers/cluster/kubernetes_pull.go new file mode 100644 index 0000000..8b819de --- /dev/null +++ b/internal/app/machined/pkg/controllers/cluster/kubernetes_pull.go @@ -0,0 +1,176 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cluster + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/aenix-io/talm/internal/pkg/discovery/registry" + "github.com/siderolabs/talos/pkg/conditions" + "github.com/siderolabs/talos/pkg/kubernetes" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +// KubernetesPullController pulls list of Affiliate resource from the Kubernetes registry. +type KubernetesPullController struct{} + +// Name implements controller.Controller interface. +func (ctrl *KubernetesPullController) Name() string { + return "cluster.KubernetesPullController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *KubernetesPullController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: cluster.ConfigType, + ID: optional.Some(cluster.ConfigID), + Kind: controller.InputWeak, + }, + { + Namespace: k8s.NamespaceName, + Type: k8s.NodenameType, + ID: optional.Some(k8s.NodenameID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *KubernetesPullController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: cluster.AffiliateType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *KubernetesPullController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + var ( + kubernetesClient *kubernetes.Client + kubernetesRegistry *registry.Kubernetes + watchCtxCancel context.CancelFunc + notifyCh <-chan struct{} + notifyCloser func() + ) + + defer func() { + if watchCtxCancel != nil { + watchCtxCancel() + } + + if notifyCloser != nil { + notifyCloser() + } + + if kubernetesClient != nil { + kubernetesClient.Close() //nolint:errcheck + } + }() + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + case <-notifyCh: + } + + discoveryConfig, err := safe.ReaderGetByID[*cluster.Config](ctx, r, cluster.ConfigID) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting discovery config: %w", err) + } + + continue + } + + if !discoveryConfig.TypedSpec().RegistryKubernetesEnabled { + // if discovery is disabled cleanup existing resources + if err = cleanupAffiliates(ctx, ctrl, r, nil); err != nil { + return err + } + + continue + } + + if err = conditions.WaitForKubeconfigReady(constants.KubeletKubeconfig).Wait(ctx); err != nil { + return err + } + + nodename, err := safe.ReaderGetByID[*k8s.Nodename](ctx, r, k8s.NodenameID) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting nodename: %w", err) + } + + continue + } + + if kubernetesClient == nil { + kubernetesClient, err = kubernetes.NewClientFromKubeletKubeconfig() + if err != nil { + return fmt.Errorf("error building kubernetes client: %w", err) + } + } + + if kubernetesRegistry == nil { + kubernetesRegistry = registry.NewKubernetes(kubernetesClient) + } + + if notifyCh == nil { + var watchCtx context.Context + watchCtx, watchCtxCancel = context.WithCancel(ctx) //nolint:govet + + notifyCh, notifyCloser, err = kubernetesRegistry.Watch(watchCtx, logger) + if err != nil { + return fmt.Errorf("error setting up registry watcher: %w", err) //nolint:govet + } + } + + affiliateSpecs, err := kubernetesRegistry.List(nodename.TypedSpec().Nodename) + if err != nil { + return fmt.Errorf("error listing affiliates: %w", err) + } + + touchedIDs := make(map[resource.ID]struct{}) + + for _, affilateSpec := range affiliateSpecs { + id := fmt.Sprintf("k8s/%s", affilateSpec.NodeID) + + if err = safe.WriterModify(ctx, r, cluster.NewAffiliate(cluster.RawNamespaceName, id), func(res *cluster.Affiliate) error { + *res.TypedSpec() = *affilateSpec + + return nil + }); err != nil { + return err + } + + touchedIDs[id] = struct{}{} + } + + if err := cleanupAffiliates(ctx, ctrl, r, touchedIDs); err != nil { + return err + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/cluster/kubernetes_push.go b/internal/app/machined/pkg/controllers/cluster/kubernetes_push.go new file mode 100644 index 0000000..1f3c759 --- /dev/null +++ b/internal/app/machined/pkg/controllers/cluster/kubernetes_push.go @@ -0,0 +1,147 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cluster + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/aenix-io/talm/internal/pkg/discovery/registry" + "github.com/siderolabs/talos/pkg/conditions" + "github.com/siderolabs/talos/pkg/kubernetes" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" + "github.com/siderolabs/talos/pkg/machinery/resources/config" +) + +// KubernetesPushController pushes Affiliate resource to the Kubernetes registry. +type KubernetesPushController struct { + localAffiliateID resource.ID + kubernetesClient *kubernetes.Client +} + +// Name implements controller.Controller interface. +func (ctrl *KubernetesPushController) Name() string { + return "cluster.KubernetesPushController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *KubernetesPushController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: cluster.ConfigType, + ID: optional.Some(cluster.ConfigID), + Kind: controller.InputWeak, + }, + { + Namespace: cluster.NamespaceName, + Type: cluster.IdentityType, + ID: optional.Some(cluster.LocalIdentity), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *KubernetesPushController) Outputs() []controller.Output { + return nil +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *KubernetesPushController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + defer func() { + if ctrl.kubernetesClient != nil { + ctrl.kubernetesClient.Close() //nolint:errcheck + } + + ctrl.kubernetesClient = nil + }() + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + discoveryConfig, err := safe.ReaderGetByID[*cluster.Config](ctx, r, cluster.ConfigID) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting discovery config: %w", err) + } + + continue + } + + if !discoveryConfig.TypedSpec().RegistryKubernetesEnabled { + continue + } + + if err = conditions.WaitForKubeconfigReady(constants.KubeletKubeconfig).Wait(ctx); err != nil { + return err + } + + identity, err := safe.ReaderGetByID[*cluster.Identity](ctx, r, cluster.LocalIdentity) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting local identity: %w", err) + } + + continue + } + + localAffiliateID := identity.TypedSpec().NodeID + + if ctrl.localAffiliateID != localAffiliateID { + ctrl.localAffiliateID = localAffiliateID + + if err = r.UpdateInputs(append(ctrl.Inputs(), + controller.Input{ + Namespace: cluster.NamespaceName, + Type: cluster.AffiliateType, + ID: optional.Some(ctrl.localAffiliateID), + Kind: controller.InputWeak, + }, + )); err != nil { + return err + } + } + + affiliate, err := safe.ReaderGetByID[*cluster.Affiliate](ctx, r, ctrl.localAffiliateID) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting local affiliate: %w", err) + } + + continue + } + + if ctrl.kubernetesClient == nil { + ctrl.kubernetesClient, err = kubernetes.NewClientFromKubeletKubeconfig() + if err != nil { + return fmt.Errorf("error building kubernetes client: %w", err) + } + } + + if err = registry.NewKubernetes(ctrl.kubernetesClient).Push(ctx, affiliate); err != nil { + // reset client connection + ctrl.kubernetesClient.Close() //nolint:errcheck + ctrl.kubernetesClient = nil + + return fmt.Errorf("error pushing to Kubernetes registry: %w", err) + } + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/cluster/local_affiliate.go b/internal/app/machined/pkg/controllers/cluster/local_affiliate.go new file mode 100644 index 0000000..d79ef48 --- /dev/null +++ b/internal/app/machined/pkg/controllers/cluster/local_affiliate.go @@ -0,0 +1,323 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cluster + +import ( + "context" + "fmt" + "net/netip" + "slices" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/net" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/kubespan" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/version" +) + +// LocalAffiliateController builds Affiliate resource for the local node. +type LocalAffiliateController struct{} + +// Name implements controller.Controller interface. +func (ctrl *LocalAffiliateController) Name() string { + return "cluster.LocalAffiliateController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *LocalAffiliateController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: cluster.ConfigType, + ID: optional.Some(cluster.ConfigID), + Kind: controller.InputWeak, + }, + { + Namespace: cluster.NamespaceName, + Type: cluster.IdentityType, + ID: optional.Some(cluster.LocalIdentity), + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.HostnameStatusType, + ID: optional.Some(network.HostnameID), + Kind: controller.InputWeak, + }, + { + Namespace: k8s.NamespaceName, + Type: k8s.NodenameType, + ID: optional.Some(k8s.NodenameID), + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.NodeAddressType, + Kind: controller.InputWeak, + }, + { + Namespace: kubespan.NamespaceName, + Type: kubespan.IdentityType, + ID: optional.Some(kubespan.LocalIdentity), + Kind: controller.InputWeak, + }, + { + Namespace: config.NamespaceName, + Type: kubespan.ConfigType, + ID: optional.Some(kubespan.ConfigID), + Kind: controller.InputWeak, + }, + { + Namespace: config.NamespaceName, + Type: config.MachineTypeType, + ID: optional.Some(config.MachineTypeID), + Kind: controller.InputWeak, + }, + { + Namespace: cluster.NamespaceName, + Type: network.AddressStatusType, + Kind: controller.InputWeak, + }, + { + Namespace: k8s.ControlPlaneNamespaceName, + Type: k8s.APIServerConfigType, + ID: optional.Some(k8s.APIServerConfigID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *LocalAffiliateController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: cluster.AffiliateType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *LocalAffiliateController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + // mandatory resources to be fetched + discoveryConfig, err := safe.ReaderGetByID[*cluster.Config](ctx, r, cluster.ConfigID) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting discovery config: %w", err) + } + + continue + } + + identity, err := safe.ReaderGetByID[*cluster.Identity](ctx, r, cluster.LocalIdentity) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting local identity: %w", err) + } + + continue + } + + hostname, err := safe.ReaderGetByID[*network.HostnameStatus](ctx, r, network.HostnameID) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting hostname: %w", err) + } + + continue + } + + nodename, err := safe.ReaderGetByID[*k8s.Nodename](ctx, r, k8s.NodenameID) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting nodename: %w", err) + } + + continue + } + + routedAddresses, err := safe.ReaderGetByID[*network.NodeAddress](ctx, r, network.FilteredNodeAddressID(network.NodeAddressRoutedID, k8s.NodeAddressFilterNoK8s)) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting addresses: %w", err) + } + + continue + } + + currentAddresses, err := safe.ReaderGetByID[*network.NodeAddress](ctx, r, network.FilteredNodeAddressID(network.NodeAddressCurrentID, k8s.NodeAddressFilterNoK8s)) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting addresses: %w", err) + } + + continue + } + + machineType, err := safe.ReaderGetByID[*config.MachineType](ctx, r, config.MachineTypeID) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting machine type: %w", err) + } + + continue + } + + // optional resources (kubespan) + kubespanIdentity, err := safe.ReaderGetByID[*kubespan.Identity](ctx, r, kubespan.LocalIdentity) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error getting kubespan identity: %w", err) + } + + kubespanConfig, err := safe.ReaderGetByID[*kubespan.Config](ctx, r, kubespan.ConfigID) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error getting kubespan config: %w", err) + } + + ksAdditionalAddresses, err := safe.ReaderGetByID[*network.NodeAddress](ctx, r, network.FilteredNodeAddressID(network.NodeAddressCurrentID, k8s.NodeAddressFilterOnlyK8s)) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error getting kubespan additional addresses: %w", err) + } + + discoveredPublicIPs, err := safe.ReaderList[*network.AddressStatus](ctx, r, resource.NewMetadata(cluster.NamespaceName, network.AddressStatusType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error getting discovered public IP: %w", err) + } + + // optional resources (kubernetes) + apiServerConfig, err := safe.ReaderGetByID[*k8s.APIServerConfig](ctx, r, k8s.APIServerConfigID) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error getting API server config: %w", err) + } + + localID := identity.TypedSpec().NodeID + + touchedIDs := map[resource.ID]struct{}{} + + if discoveryConfig.TypedSpec().DiscoveryEnabled { + if err = safe.WriterModify(ctx, r, cluster.NewAffiliate(cluster.NamespaceName, localID), func(res *cluster.Affiliate) error { + spec := res.TypedSpec() + + spec.NodeID = localID + spec.Hostname = hostname.TypedSpec().FQDN() + spec.Nodename = nodename.TypedSpec().Nodename + spec.MachineType = machineType.MachineType() + spec.OperatingSystem = fmt.Sprintf("%s (%s)", version.Name, version.Tag) + + if machineType.MachineType().IsControlPlane() && apiServerConfig != nil { + spec.ControlPlane = &cluster.ControlPlane{ + APIServerPort: apiServerConfig.TypedSpec().LocalPort, + } + } else { + spec.ControlPlane = nil + } + + routedNodeIPs := routedAddresses.TypedSpec().IPs() + currentNodeIPs := currentAddresses.TypedSpec().IPs() + + spec.Addresses = routedNodeIPs + + spec.KubeSpan = cluster.KubeSpanAffiliateSpec{} + + if kubespanIdentity != nil && kubespanConfig != nil { + spec.KubeSpan.Address = kubespanIdentity.TypedSpec().Address.Addr() + spec.KubeSpan.PublicKey = kubespanIdentity.TypedSpec().PublicKey + + if kubespanConfig.TypedSpec().AdvertiseKubernetesNetworks && ksAdditionalAddresses != nil { + spec.KubeSpan.AdditionalAddresses = slices.Clone(ksAdditionalAddresses.TypedSpec().Addresses) + } else { + spec.KubeSpan.AdditionalAddresses = nil + } + + endpointIPs := xslices.Filter(currentNodeIPs, func(ip netip.Addr) bool { + if ip == spec.KubeSpan.Address { + // skip kubespan local address + return false + } + + if network.IsULA(ip, network.ULASideroLink) { + // ignore SideroLink addresses, as they are point-to-point addresses + return false + } + + return true + }) + + // mix in discovered public IPs + for iter := discoveredPublicIPs.Iterator(); iter.Next(); { + addr := iter.Value().TypedSpec().Address.Addr() + + if slices.ContainsFunc(endpointIPs, func(a netip.Addr) bool { return addr == a }) { + // this address is already published + continue + } + + endpointIPs = append(endpointIPs, addr) + } + + // filter endpoints if configured + if kubespanConfig.TypedSpec().EndpointFilters != nil { + endpointIPs, err = net.FilterIPs(endpointIPs, kubespanConfig.TypedSpec().EndpointFilters) + if err != nil { + return fmt.Errorf("error filtering KubeSpan endpoints: %w", err) + } + } + + spec.KubeSpan.Endpoints = xslices.Map(endpointIPs, func(addr netip.Addr) netip.AddrPort { + return netip.AddrPortFrom(addr, constants.KubeSpanDefaultPort) + }) + } + + return nil + }); err != nil { + return err + } + + touchedIDs[localID] = struct{}{} + } + + // list keys for cleanup + list, err := safe.ReaderListAll[*cluster.Affiliate](ctx, r) + if err != nil { + return fmt.Errorf("error listing resources: %w", err) + } + + for it := list.Iterator(); it.Next(); { + res := it.Value() + + if res.Metadata().Owner() != ctrl.Name() { + continue + } + + if _, ok := touchedIDs[res.Metadata().ID()]; !ok { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return fmt.Errorf("error cleaning up specs: %w", err) + } + } + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/cluster/local_affiliate_test.go b/internal/app/machined/pkg/controllers/cluster/local_affiliate_test.go new file mode 100644 index 0000000..c5ee0cc --- /dev/null +++ b/internal/app/machined/pkg/controllers/cluster/local_affiliate_test.go @@ -0,0 +1,228 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cluster_test + +import ( + "net" + "net/netip" + "testing" + + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/go-pointer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + clusteradapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/cluster" + kubespanadapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/kubespan" + clusterctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/cluster" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/kubespan" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/version" +) + +type LocalAffiliateSuite struct { + ClusterSuite +} + +func (suite *LocalAffiliateSuite) TestGeneration() { + suite.startRuntime() + + suite.Require().NoError(suite.runtime.RegisterController(&clusterctrl.LocalAffiliateController{})) + + nodeIdentity, nonK8sRoutedAddresses, discoveryConfig := suite.createResources() + + machineType := config.NewMachineType() + machineType.SetMachineType(machine.TypeWorker) + suite.Require().NoError(suite.state.Create(suite.ctx, machineType)) + + ctest.AssertResource(suite, nodeIdentity.TypedSpec().NodeID, func(r *cluster.Affiliate, asrt *assert.Assertions) { + spec := r.TypedSpec() + + asrt.Equal([]string{ + "172.20.0.2", + "10.5.0.1", + "192.168.192.168", + "2001:123:4567::1", + }, xslices.Map(spec.Addresses, netip.Addr.String)) + asrt.Equal("example1", spec.Hostname) + asrt.Equal("example1.com", spec.Nodename) + asrt.Equal(machine.TypeWorker, spec.MachineType) + asrt.Equal("Talos ("+version.Tag+")", spec.OperatingSystem) + asrt.Equal(cluster.KubeSpanAffiliateSpec{}, spec.KubeSpan) + }) + + // enable kubespan + mac, err := net.ParseMAC("ea:71:1b:b2:cc:ee") + suite.Require().NoError(err) + + ksIdentity := kubespan.NewIdentity(kubespan.NamespaceName, kubespan.LocalIdentity) + suite.Require().NoError(kubespanadapter.IdentitySpec(ksIdentity.TypedSpec()).GenerateKey()) + suite.Require().NoError(kubespanadapter.IdentitySpec(ksIdentity.TypedSpec()).UpdateAddress("8XuV9TZHW08DOk3bVxQjH9ih_TBKjnh-j44tsCLSBzo=", mac)) + suite.Require().NoError(suite.state.Create(suite.ctx, ksIdentity)) + + ksConfig := kubespan.NewConfig(config.NamespaceName, kubespan.ConfigID) + ksConfig.TypedSpec().EndpointFilters = []string{"0.0.0.0/0", "!192.168.0.0/16", "2001::/16"} + ksConfig.TypedSpec().AdvertiseKubernetesNetworks = true + suite.Require().NoError(suite.state.Create(suite.ctx, ksConfig)) + + // add KS address to the list of node addresses, it should be ignored in the endpoints + nonK8sRoutedAddresses.TypedSpec().Addresses = append(nonK8sRoutedAddresses.TypedSpec().Addresses, ksIdentity.TypedSpec().Address) + suite.Require().NoError(suite.state.Update(suite.ctx, nonK8sRoutedAddresses)) + + onlyK8sAddresses := network.NewNodeAddress(network.NamespaceName, network.FilteredNodeAddressID(network.NodeAddressCurrentID, k8s.NodeAddressFilterOnlyK8s)) + onlyK8sAddresses.TypedSpec().Addresses = []netip.Prefix{netip.MustParsePrefix("10.244.1.0/24")} + suite.Require().NoError(suite.state.Create(suite.ctx, onlyK8sAddresses)) + + // add discovered public IPs + for _, addr := range []netip.Addr{ + netip.MustParseAddr("1.1.1.1"), + netip.MustParseAddr("2001:123:4567::1"), // duplicate, will be ignored + } { + discoveredAddr := network.NewAddressStatus(cluster.NamespaceName, addr.String()) + discoveredAddr.TypedSpec().Address = netip.PrefixFrom(addr, addr.BitLen()) + suite.Require().NoError(suite.state.Create(suite.ctx, discoveredAddr)) + } + + ctest.AssertResource(suite, nodeIdentity.TypedSpec().NodeID, func(r *cluster.Affiliate, asrt *assert.Assertions) { + spec := r.TypedSpec() + + asrt.False(len(spec.Addresses) < 5) + + asrt.Equal([]netip.Addr{ + netip.MustParseAddr("172.20.0.2"), + netip.MustParseAddr("10.5.0.1"), + netip.MustParseAddr("192.168.192.168"), + netip.MustParseAddr("2001:123:4567::1"), + ksIdentity.TypedSpec().Address.Addr(), + }, spec.Addresses) + + asrt.Equal("example1", spec.Hostname) + asrt.Equal("example1.com", spec.Nodename) + asrt.Equal(machine.TypeWorker, spec.MachineType) + + asrt.NotZero(spec.KubeSpan.PublicKey) + asrt.NotZero(spec.KubeSpan.AdditionalAddresses) + asrt.Len(spec.KubeSpan.Endpoints, 4) + + asrt.Equal(ksIdentity.TypedSpec().Address.Addr(), spec.KubeSpan.Address) + asrt.Equal(ksIdentity.TypedSpec().PublicKey, spec.KubeSpan.PublicKey) + asrt.Equal([]netip.Prefix{netip.MustParsePrefix("10.244.1.0/24")}, spec.KubeSpan.AdditionalAddresses) + asrt.Equal( + []string{ + "172.20.0.2:51820", + "10.5.0.1:51820", + "1.1.1.1:51820", + "[2001:123:4567::1]:51820", + }, + xslices.Map(spec.KubeSpan.Endpoints, netip.AddrPort.String), + ) + }) + + // disable advertising K8s addresses + ksConfig.TypedSpec().AdvertiseKubernetesNetworks = false + suite.Require().NoError(suite.state.Update(suite.ctx, ksConfig)) + + ctest.AssertResource(suite, nodeIdentity.TypedSpec().NodeID, func(r *cluster.Affiliate, asrt *assert.Assertions) { + asrt.Empty(r.TypedSpec().KubeSpan.AdditionalAddresses) + }) + + // disable discovery, local affiliate should be removed + discoveryConfig.TypedSpec().DiscoveryEnabled = false + suite.Require().NoError(suite.state.Update(suite.ctx, discoveryConfig)) + + ctest.AssertNoResource[*cluster.Affiliate](suite, nodeIdentity.TypedSpec().NodeID) +} + +func (suite *LocalAffiliateSuite) TestCPGeneration() { + suite.startRuntime() + + suite.Require().NoError(suite.runtime.RegisterController(&clusterctrl.LocalAffiliateController{})) + + nodeIdentity, _, discoveryConfig := suite.createResources() + + machineType := config.NewMachineType() + machineType.SetMachineType(machine.TypeControlPlane) + suite.Require().NoError(suite.state.Create(suite.ctx, machineType)) + + apiServerConfig := k8s.NewAPIServerConfig() + apiServerConfig.TypedSpec().LocalPort = 6445 + suite.Require().NoError(suite.state.Create(suite.ctx, apiServerConfig)) + + ctest.AssertResource(suite, nodeIdentity.TypedSpec().NodeID, func(r *cluster.Affiliate, asrt *assert.Assertions) { + spec := r.TypedSpec() + + asrt.Equal([]string{ + "172.20.0.2", + "10.5.0.1", + "192.168.192.168", + "2001:123:4567::1", + }, xslices.Map(spec.Addresses, netip.Addr.String)) + asrt.Equal("example1", spec.Hostname) + asrt.Equal("example1.com", spec.Nodename) + asrt.Equal(machine.TypeControlPlane, spec.MachineType) + asrt.Equal("Talos ("+version.Tag+")", spec.OperatingSystem) + asrt.Equal(cluster.KubeSpanAffiliateSpec{}, spec.KubeSpan) + asrt.NotNil(spec.ControlPlane) + asrt.Equal(6445, pointer.SafeDeref(spec.ControlPlane).APIServerPort) + }) + + discoveryConfig.TypedSpec().DiscoveryEnabled = false + suite.Require().NoError(suite.state.Update(suite.ctx, discoveryConfig)) + + ctest.AssertNoResource[*cluster.Affiliate](suite, nodeIdentity.TypedSpec().NodeID) +} + +func (suite *LocalAffiliateSuite) createResources() (*cluster.Identity, *network.NodeAddress, *cluster.Config) { + // regular discovery affiliate + discoveryConfig := cluster.NewConfig(config.NamespaceName, cluster.ConfigID) + discoveryConfig.TypedSpec().DiscoveryEnabled = true + suite.Require().NoError(suite.state.Create(suite.ctx, discoveryConfig)) + + nodeIdentity := cluster.NewIdentity(cluster.NamespaceName, cluster.LocalIdentity) + suite.Require().NoError(clusteradapter.IdentitySpec(nodeIdentity.TypedSpec()).Generate()) + suite.Require().NoError(suite.state.Create(suite.ctx, nodeIdentity)) + + hostnameStatus := network.NewHostnameStatus(network.NamespaceName, network.HostnameID) + hostnameStatus.TypedSpec().Hostname = "example1" + suite.Require().NoError(suite.state.Create(suite.ctx, hostnameStatus)) + + nodename := k8s.NewNodename(k8s.NamespaceName, k8s.NodenameID) + nodename.TypedSpec().Nodename = "example1.com" + suite.Require().NoError(suite.state.Create(suite.ctx, nodename)) + + nonK8sCurrentAddresses := network.NewNodeAddress(network.NamespaceName, network.FilteredNodeAddressID(network.NodeAddressCurrentID, k8s.NodeAddressFilterNoK8s)) + nonK8sCurrentAddresses.TypedSpec().Addresses = []netip.Prefix{ + netip.MustParsePrefix("172.20.0.2/24"), + netip.MustParsePrefix("10.5.0.1/32"), + netip.MustParsePrefix("192.168.192.168/24"), + netip.MustParsePrefix("2001:123:4567::1/64"), + netip.MustParsePrefix("2001:123:4567::1/128"), + netip.MustParsePrefix("fdae:41e4:649b:9303:60be:7e36:c270:3238/128"), // SideroLink, should be ignored + } + suite.Require().NoError(suite.state.Create(suite.ctx, nonK8sCurrentAddresses)) + + nonK8sRoutedAddresses := network.NewNodeAddress(network.NamespaceName, network.FilteredNodeAddressID(network.NodeAddressRoutedID, k8s.NodeAddressFilterNoK8s)) + nonK8sRoutedAddresses.TypedSpec().Addresses = []netip.Prefix{ // routed node addresses don't contain SideroLink addresses + netip.MustParsePrefix("172.20.0.2/24"), + netip.MustParsePrefix("10.5.0.1/32"), + netip.MustParsePrefix("192.168.192.168/24"), + netip.MustParsePrefix("2001:123:4567::1/64"), + netip.MustParsePrefix("2001:123:4567::1/128"), + } + suite.Require().NoError(suite.state.Create(suite.ctx, nonK8sRoutedAddresses)) + + return nodeIdentity, nonK8sRoutedAddresses, discoveryConfig +} + +func TestLocalAffiliateSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, new(LocalAffiliateSuite)) +} diff --git a/internal/app/machined/pkg/controllers/cluster/member.go b/internal/app/machined/pkg/controllers/cluster/member.go new file mode 100644 index 0000000..c0a0c58 --- /dev/null +++ b/internal/app/machined/pkg/controllers/cluster/member.go @@ -0,0 +1,115 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cluster + +import ( + "context" + "errors" + "fmt" + "slices" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" +) + +// MemberController converts Affiliates which have Nodename set into Members. +type MemberController struct{} + +// Name implements controller.Controller interface. +func (ctrl *MemberController) Name() string { + return "cluster.MemberController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *MemberController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: cluster.NamespaceName, + Type: cluster.AffiliateType, + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *MemberController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: cluster.MemberType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *MemberController) Run(ctx context.Context, r controller.Runtime, _ *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + affiliates, err := safe.ReaderListAll[*cluster.Affiliate](ctx, r) + if err != nil { + return errors.New("error listing affiliates") + } + + touchedIDs := make(map[resource.ID]struct{}) + + for it := affiliates.Iterator(); it.Next(); { + affiliateSpec := it.Value().TypedSpec() + if affiliateSpec.Nodename == "" { + // not a cluster member + continue + } + + if err = safe.WriterModify(ctx, r, cluster.NewMember(cluster.NamespaceName, affiliateSpec.Nodename), func(res *cluster.Member) error { + spec := res.TypedSpec() + + spec.Addresses = slices.Clone(affiliateSpec.Addresses) + spec.Hostname = affiliateSpec.Hostname + spec.MachineType = affiliateSpec.MachineType + spec.OperatingSystem = affiliateSpec.OperatingSystem + spec.NodeID = affiliateSpec.NodeID + spec.ControlPlane = affiliateSpec.ControlPlane + + return nil + }); err != nil { + return err + } + + touchedIDs[affiliateSpec.Nodename] = struct{}{} + } + + // list keys for cleanup + list, err := safe.ReaderListAll[*cluster.Member](ctx, r) + if err != nil { + return fmt.Errorf("error listing resources: %w", err) + } + + for it := list.Iterator(); it.Next(); { + res := it.Value() + + if res.Metadata().Owner() != ctrl.Name() { + continue + } + + if _, ok := touchedIDs[res.Metadata().ID()]; !ok { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return fmt.Errorf("error cleaning up specs: %w", err) + } + } + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/cluster/member_test.go b/internal/app/machined/pkg/controllers/cluster/member_test.go new file mode 100644 index 0000000..03e6372 --- /dev/null +++ b/internal/app/machined/pkg/controllers/cluster/member_test.go @@ -0,0 +1,104 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cluster_test + +import ( + "net/netip" + "testing" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + clusterctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/cluster" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" +) + +type MemberSuite struct { + ClusterSuite +} + +func (suite *MemberSuite) TestReconcileDefault() { + suite.startRuntime() + + suite.Require().NoError(suite.runtime.RegisterController(&clusterctrl.MemberController{})) + + affiliate1 := cluster.NewAffiliate(cluster.NamespaceName, "7x1SuC8Ege5BGXdAfTEff5iQnlWZLfv9h1LGMxA2pYkC") + *affiliate1.TypedSpec() = cluster.AffiliateSpec{ + NodeID: "7x1SuC8Ege5BGXdAfTEff5iQnlWZLfv9h1LGMxA2pYkC", + Hostname: "foo.com", + Nodename: "bar", + MachineType: machine.TypeControlPlane, + OperatingSystem: "Talos (v1.0.0)", + Addresses: []netip.Addr{netip.MustParseAddr("192.168.3.4")}, + KubeSpan: cluster.KubeSpanAffiliateSpec{ + PublicKey: "PLPNBddmTgHJhtw0vxltq1ZBdPP9RNOEUd5JjJZzBRY=", + Address: netip.MustParseAddr("fd50:8d60:4238:6302:f857:23ff:fe21:d1e0"), + AdditionalAddresses: []netip.Prefix{netip.MustParsePrefix("10.244.3.1/24")}, + Endpoints: []netip.AddrPort{netip.MustParseAddrPort("10.0.0.2:51820"), netip.MustParseAddrPort("192.168.3.4:51820")}, + }, + ControlPlane: &cluster.ControlPlane{APIServerPort: 6443}, + } + + affiliate2 := cluster.NewAffiliate(cluster.NamespaceName, "9dwHNUViZlPlIervqX9Qo256RUhrfhgO0xBBnKcKl4F") + *affiliate2.TypedSpec() = cluster.AffiliateSpec{ + NodeID: "9dwHNUViZlPlIervqX9Qo256RUhrfhgO0xBBnKcKl4F", + Hostname: "worker-1", + Nodename: "worker-1", + MachineType: machine.TypeWorker, + Addresses: []netip.Addr{netip.MustParseAddr("192.168.3.5")}, + } + + affiliate3 := cluster.NewAffiliate(cluster.NamespaceName, "xCnFFfxylOf9i5ynhAkt6ZbfcqaLDGKfIa3gwpuaxe7F") + *affiliate3.TypedSpec() = cluster.AffiliateSpec{ + NodeID: "xCnFFfxylOf9i5ynhAkt6ZbfcqaLDGKfIa3gwpuaxe7F", + MachineType: machine.TypeWorker, + Addresses: []netip.Addr{netip.MustParseAddr("192.168.3.6")}, + } + + for _, r := range []resource.Resource{affiliate1, affiliate2, affiliate3} { + suite.Require().NoError(suite.state.Create(suite.ctx, r)) + } + + // affiliates with non-empty Nodename should be translated to Members + ctest.AssertResource( + suite, + affiliate1.TypedSpec().Nodename, + func(r *cluster.Member, asrt *assert.Assertions) { + spec := r.TypedSpec() + + asrt.Equal(affiliate1.TypedSpec().NodeID, spec.NodeID) + asrt.Equal([]netip.Addr{netip.MustParseAddr("192.168.3.4")}, spec.Addresses) + asrt.Equal("foo.com", spec.Hostname) + asrt.Equal(machine.TypeControlPlane, spec.MachineType) + asrt.Equal("Talos (v1.0.0)", spec.OperatingSystem) + asrt.Equal(6443, spec.ControlPlane.APIServerPort) + }, + ) + + ctest.AssertResource( + suite, + affiliate2.TypedSpec().Nodename, + func(r *cluster.Member, asrt *assert.Assertions) { + spec := r.TypedSpec() + + asrt.Equal(affiliate2.TypedSpec().NodeID, spec.NodeID) + asrt.Equal([]netip.Addr{netip.MustParseAddr("192.168.3.5")}, spec.Addresses) + asrt.Equal("worker-1", spec.Hostname) + asrt.Equal(machine.TypeWorker, spec.MachineType) + }, + ) + + // remove affiliate2, member information should eventually go away + suite.Require().NoError(suite.state.Destroy(suite.ctx, affiliate2.Metadata())) + + ctest.AssertNoResource[*cluster.Member](suite, affiliate2.TypedSpec().Nodename) +} + +func TestMemberSuite(t *testing.T) { + suite.Run(t, new(MemberSuite)) +} diff --git a/internal/app/machined/pkg/controllers/cluster/node_identity.go b/internal/app/machined/pkg/controllers/cluster/node_identity.go new file mode 100644 index 0000000..c41b1fd --- /dev/null +++ b/internal/app/machined/pkg/controllers/cluster/node_identity.go @@ -0,0 +1,131 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cluster + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + clusteradapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/cluster" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" + "github.com/siderolabs/talos/pkg/machinery/resources/files" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +// NodeIdentityController manages runtime.Identity caching identity in the STATE. +type NodeIdentityController struct { + V1Alpha1Mode runtime.Mode + StatePath string + + identityEstablished bool +} + +// Name implements controller.Controller interface. +func (ctrl *NodeIdentityController) Name() string { + return "cluster.NodeIdentityController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *NodeIdentityController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: v1alpha1.NamespaceName, + Type: runtimeres.MountStatusType, + ID: optional.Some(constants.StatePartitionLabel), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *NodeIdentityController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: cluster.IdentityType, + Kind: controller.OutputShared, + }, + { + Type: files.EtcFileSpecType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *NodeIdentityController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + if ctrl.StatePath == "" { + ctrl.StatePath = constants.StateMountPoint + } + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + if _, err := r.Get(ctx, resource.NewMetadata(v1alpha1.NamespaceName, runtimeres.MountStatusType, constants.StatePartitionLabel, resource.VersionUndefined)); err != nil { + if state.IsNotFoundError(err) { + // in container mode STATE is always mounted + if ctrl.V1Alpha1Mode != runtime.ModeContainer { + // wait for the STATE to be mounted + continue + } + } else { + return fmt.Errorf("error reading mount status: %w", err) + } + } + + var localIdentity cluster.IdentitySpec + + if err := controllers.LoadOrNewFromFile(filepath.Join(ctrl.StatePath, constants.NodeIdentityFilename), &localIdentity, func(v interface{}) error { + return clusteradapter.IdentitySpec(v.(*cluster.IdentitySpec)).Generate() + }); err != nil { + return fmt.Errorf("error caching node identity: %w", err) + } + + if err := r.Modify(ctx, cluster.NewIdentity(cluster.NamespaceName, cluster.LocalIdentity), func(r resource.Resource) error { + *r.(*cluster.Identity).TypedSpec() = localIdentity + + return nil + }); err != nil { + return fmt.Errorf("error modifying resource: %w", err) + } + + // generate `/etc/machine-id` from node identity + if err := r.Modify(ctx, files.NewEtcFileSpec(files.NamespaceName, "machine-id"), + func(r resource.Resource) error { + var err error + + r.(*files.EtcFileSpec).TypedSpec().Contents, err = clusteradapter.IdentitySpec(&localIdentity).ConvertMachineID() + r.(*files.EtcFileSpec).TypedSpec().Mode = 0o444 + + return err + }); err != nil { + return fmt.Errorf("error modifying resolv.conf: %w", err) + } + + if !ctrl.identityEstablished { + logger.Info("node identity established", zap.String("node_id", localIdentity.NodeID)) + + ctrl.identityEstablished = true + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/cluster/node_identity_test.go b/internal/app/machined/pkg/controllers/cluster/node_identity_test.go new file mode 100644 index 0000000..0b175f4 --- /dev/null +++ b/internal/app/machined/pkg/controllers/cluster/node_identity_test.go @@ -0,0 +1,117 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cluster_test + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + clusterctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/cluster" + v1alpha1runtime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" + "github.com/siderolabs/talos/pkg/machinery/resources/files" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +type NodeIdentitySuite struct { + ClusterSuite + + statePath string +} + +func (suite *NodeIdentitySuite) TestContainerMode() { + suite.statePath = suite.T().TempDir() + suite.startRuntime() + + suite.Require().NoError(suite.runtime.RegisterController(&clusterctrl.NodeIdentityController{ + StatePath: suite.statePath, + V1Alpha1Mode: v1alpha1runtime.ModeContainer, + })) + + suite.Assert().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertResource(*cluster.NewIdentity(cluster.NamespaceName, cluster.LocalIdentity).Metadata(), func(_ resource.Resource) error { + return nil + }), + )) +} + +func (suite *NodeIdentitySuite) TestDefault() { + suite.statePath = suite.T().TempDir() + suite.startRuntime() + + suite.Require().NoError(suite.runtime.RegisterController(&clusterctrl.NodeIdentityController{ + StatePath: suite.statePath, + V1Alpha1Mode: v1alpha1runtime.ModeMetal, + })) + + time.Sleep(500 * time.Millisecond) + + _, err := suite.state.Get(suite.ctx, cluster.NewIdentity(cluster.NamespaceName, cluster.LocalIdentity).Metadata()) + suite.Assert().True(state.IsNotFoundError(err)) + + stateMount := runtimeres.NewMountStatus(v1alpha1.NamespaceName, constants.StatePartitionLabel) + + suite.Assert().NoError(suite.state.Create(suite.ctx, stateMount)) + + suite.Assert().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertResource(*cluster.NewIdentity(cluster.NamespaceName, cluster.LocalIdentity).Metadata(), func(_ resource.Resource) error { + return nil + }), + )) + + suite.Assert().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertResource(*files.NewEtcFileSpec(files.NamespaceName, "machine-id").Metadata(), func(_ resource.Resource) error { + return nil + }), + )) +} + +func (suite *NodeIdentitySuite) TestLoad() { + suite.statePath = suite.T().TempDir() + suite.startRuntime() + + suite.Require().NoError(suite.runtime.RegisterController(&clusterctrl.NodeIdentityController{ + StatePath: suite.statePath, + V1Alpha1Mode: v1alpha1runtime.ModeMetal, + })) + + // using verbatim data here to make sure nodeId representation is supported in future version fo Talos + suite.Require().NoError(os.WriteFile(filepath.Join(suite.statePath, constants.NodeIdentityFilename), []byte("nodeId: gvqfS27LxD58lPlASmpaueeRVzuof16iXoieRgEvBWaE\n"), 0o600)) + + stateMount := runtimeres.NewMountStatus(v1alpha1.NamespaceName, constants.StatePartitionLabel) + + suite.Assert().NoError(suite.state.Create(suite.ctx, stateMount)) + + suite.Assert().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertResource(*cluster.NewIdentity(cluster.NamespaceName, cluster.LocalIdentity).Metadata(), func(r resource.Resource) error { + suite.Assert().Equal("gvqfS27LxD58lPlASmpaueeRVzuof16iXoieRgEvBWaE", r.(*cluster.Identity).TypedSpec().NodeID) + + return nil + }), + )) + + suite.Assert().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertResource(*files.NewEtcFileSpec(files.NamespaceName, "machine-id").Metadata(), func(r resource.Resource) error { + suite.Assert().Equal("8d2c0de2408fa2a178bad7f45d9aa8fb", string(r.(*files.EtcFileSpec).TypedSpec().Contents)) + + return nil + }), + )) +} + +func TestNodeIdentitySuite(t *testing.T) { + t.Parallel() + + suite.Run(t, new(NodeIdentitySuite)) +} diff --git a/internal/app/machined/pkg/controllers/config/acquire.go b/internal/app/machined/pkg/controllers/config/acquire.go new file mode 100644 index 0000000..e396477 --- /dev/null +++ b/internal/app/machined/pkg/controllers/config/acquire.go @@ -0,0 +1,465 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package config + +import ( + "bytes" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "net/http" + "os" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + talosruntime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform" + platformerrors "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/errors" + machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/config" + "github.com/siderolabs/talos/pkg/machinery/config/configloader" + "github.com/siderolabs/talos/pkg/machinery/config/validation" + "github.com/siderolabs/talos/pkg/machinery/constants" + configresource "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +// PlatformConfigurator is a reduced interface of runtime.Platform. +type PlatformConfigurator interface { + Name() string + Configuration(context.Context) ([]byte, error) +} + +// PlatformEventer sends events based on the config process via platform-specific interface. +type PlatformEventer interface { + FireEvent(context.Context, platform.Event) +} + +// Setter sets the current machine config. +type Setter interface { + SetConfig(config.Provider) error +} + +// AcquireController loads the machine configuration from multiple sources. +type AcquireController struct { + PlatformConfiguration PlatformConfigurator + PlatformEvent PlatformEventer + ConfigSetter Setter + EventPublisher talosruntime.Publisher + ValidationMode validation.RuntimeMode + ConfigPath string + + configSourcesUsed []string +} + +// Name implements controller.Controller interface. +func (ctrl *AcquireController) Name() string { + return "config.AcquireController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *AcquireController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: v1alpha1.NamespaceName, + Type: v1alpha1.AcquireConfigSpecType, + Kind: controller.InputWeak, + }, + { + Namespace: configresource.NamespaceName, + Type: configresource.MachineConfigType, + ID: optional.Some(configresource.MaintenanceID), + Kind: controller.InputWeak, + }, + { + Namespace: runtime.NamespaceName, + Type: runtime.MaintenanceServiceRequestType, + Kind: controller.InputDestroyReady, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *AcquireController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: v1alpha1.AcquireConfigStatusType, + Kind: controller.OutputExclusive, + }, + { + Type: runtime.MaintenanceServiceRequestType, + Kind: controller.OutputExclusive, + }, + } +} + +// stateMachineFunc represents the state machine of config.AcquireController. +type stateMachineFunc func(context.Context, controller.Runtime, *zap.Logger) (stateMachineFunc, config.Provider, error) + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *AcquireController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + if ctrl.ConfigPath == "" { + ctrl.ConfigPath = constants.ConfigPath + } + + // start always with loading config from disk + var currentState stateMachineFunc = ctrl.stateDisk + + // initialize with empty sources + ctrl.configSourcesUsed = []string{} + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + // check the spec first + _, err := safe.ReaderGet[*v1alpha1.AcquireConfigSpec](ctx, r, v1alpha1.NewAcquireConfigSpec().Metadata()) + if err != nil { + if state.IsNotFoundError(err) { + // spec is not found, wait for it + continue + } + + return fmt.Errorf("failed to get spec: %w", err) + } + + // run the state machine + for { + newState, cfg, err := currentState(ctx, r, logger) + if err != nil { + ctrl.EventPublisher.Publish(ctx, &machineapi.ConfigLoadErrorEvent{ + Error: err.Error(), + }) + + ctrl.PlatformEvent.FireEvent( + ctx, + platform.Event{ + Type: platform.EventTypeFailure, + Message: "Error loading and validating Talos machine config.", + Error: err, + }, + ) + + return err + } + + if cfg != nil { + // apply config + if err = ctrl.ConfigSetter.SetConfig(cfg); err != nil { + return fmt.Errorf("failed to set config: %w", err) + } + } + + if newState == nil { + // wait for reconcile event, keep running in the same state + break + } + + currentState = newState + } + + r.ResetRestartBackoff() + } +} + +// stateDisk acquires machine configuration from disk (STATE partition). +// +// Transitions: +// +// --> platform: no config found on disk, proceed to platform +// --> maintenanceEnter: config found on disk, but it's incomplete, proceed to maintenance +// --> done: config found on disk, and it's complete +func (ctrl *AcquireController) stateDisk(ctx context.Context, r controller.Runtime, logger *zap.Logger) (stateMachineFunc, config.Provider, error) { + cfg, err := ctrl.loadFromDisk(logger) + if err != nil { + return nil, nil, err + } + + if cfg != nil { + ctrl.configSourcesUsed = append(ctrl.configSourcesUsed, "state") + } + + switch { + case cfg == nil: + // no config loaded, proceed to platform + return ctrl.statePlatform, nil, nil + case cfg.CompleteForBoot(): + // complete config, we are done + return ctrl.stateDone, cfg, nil + default: + // incomplete config, proceed to maintenance + return ctrl.stateMaintenanceEnter, cfg, nil + } +} + +// validationModeDiskConfig is a "fake" validation mode for config loaded from disk. +type validationModeDiskConfig struct{} + +// RequiresInstall implements validation.RuntimeMode interface. +func (validationModeDiskConfig) RequiresInstall() bool { + return false +} + +// InContainer implements validation.RuntimeMode interface. +func (validationModeDiskConfig) InContainer() bool { + // containers don't persist config to disk + return false +} + +// String implements validation.RuntimeMode interface. +func (validationModeDiskConfig) String() string { + return "diskConfig" +} + +// loadFromDisk is a helper function for stateDisk. +func (ctrl *AcquireController) loadFromDisk(logger *zap.Logger) (config.Provider, error) { + logger.Debug("loading config from STATE", zap.String("path", ctrl.ConfigPath)) + + _, err := os.Stat(ctrl.ConfigPath) + if err != nil { + if os.IsNotExist(err) { + // no saved machine config + return nil, nil + } + + return nil, fmt.Errorf("failed to stat %s: %w", ctrl.ConfigPath, err) + } + + cfg, err := configloader.NewFromFile(ctrl.ConfigPath) + if err != nil { + return nil, fmt.Errorf("failed to load config from STATE: %w", err) + } + + // if the STATE partition is present & contains machine config, Talos is already installed + warnings, err := cfg.Validate(validationModeDiskConfig{}) + if err != nil { + return nil, fmt.Errorf("failed to validate on-disk config: %w", err) + } + + for _, warning := range warnings { + logger.Warn("config validation warning", zap.String("warning", warning)) + } + + return cfg, nil +} + +// statePlatform acquires machine configuration from the platform source. +// +// Transitions: +// +// --> maintenanceEnter: config loaded from platform, but it's incomplete, or no config from platform: proceed to maintenance +// --> done: config loaded from platform, and it's complete +func (ctrl *AcquireController) statePlatform(ctx context.Context, r controller.Runtime, logger *zap.Logger) (stateMachineFunc, config.Provider, error) { + cfg, err := ctrl.loadFromPlatform(ctx, logger) + if err != nil { + return nil, nil, err + } + + if cfg != nil { + ctrl.configSourcesUsed = append(ctrl.configSourcesUsed, ctrl.PlatformConfiguration.Name()) + } + + switch { + case cfg == nil: + fallthrough + case !cfg.CompleteForBoot(): + // incomplete or missing config, proceed to maintenance + return ctrl.stateMaintenanceEnter, cfg, nil + default: + // complete config, we are done + return ctrl.stateDone, cfg, nil + } +} + +// loadFromPlatform is a helper function for statePlatform. +func (ctrl *AcquireController) loadFromPlatform(ctx context.Context, logger *zap.Logger) (config.Provider, error) { + platformName := ctrl.PlatformConfiguration.Name() + + logger.Info("downloading config", zap.String("platform", platformName)) + + cfgBytes, err := ctrl.PlatformConfiguration.Configuration(ctx) + if err != nil { + if errors.Is(err, platformerrors.ErrNoConfigSource) { + // no config in the platform + return nil, nil + } + + return nil, fmt.Errorf("error acquiring via platform %s: %w", platformName, err) + } + + // Detect if config is a gzip archive and unzip it if so + contentType := http.DetectContentType(cfgBytes) + if contentType == "application/x-gzip" { + var gzipReader *gzip.Reader + + gzipReader, err = gzip.NewReader(bytes.NewReader(cfgBytes)) + if err != nil { + return nil, fmt.Errorf("error creating gzip reader: %w", err) + } + + //nolint:errcheck + defer gzipReader.Close() + + var unzippedData []byte + + unzippedData, err = io.ReadAll(gzipReader) + if err != nil { + return nil, fmt.Errorf("error unzipping machine config: %w", err) + } + + cfgBytes = unzippedData + } + + cfg, err := configloader.NewFromBytes(cfgBytes) + if err != nil { + return nil, fmt.Errorf("failed to load config via platform %s: %w", platformName, err) + } + + warnings, err := cfg.Validate(ctrl.ValidationMode) + if err != nil { + return nil, fmt.Errorf("failed to validate config acquired via platform %s: %w", platformName, err) + } + + for _, warning := range warnings { + logger.Warn("config validation warning", zap.String("platform", platformName), zap.String("warning", warning)) + } + + return cfg, nil +} + +// stateMaintenanceEnter initializes maintenance service. +// +// Transitions: +// +// --> maintenance: run the maintenance service +func (ctrl *AcquireController) stateMaintenanceEnter(ctx context.Context, r controller.Runtime, logger *zap.Logger) (stateMachineFunc, config.Provider, error) { + logger.Info("entering maintenance service") + + // nb: we treat maintenance mode as an "activate" + // event b/c the user is expected to be able to + // interact with the system at this point. + ctrl.PlatformEvent.FireEvent( + ctx, + platform.Event{ + Type: platform.EventTypeActivate, + Message: "Talos booted into maintenance mode. Ready for user interaction.", + }, + ) + + // add "fake" events to signal when Talos enters and leaves maintenance mode + ctrl.EventPublisher.Publish(ctx, &machineapi.TaskEvent{ + Action: machineapi.TaskEvent_START, + Task: "runningMaintenance", + }) + + return ctrl.stateMaintenance, nil, nil +} + +// stateMaintenance acquires machine configuration from the maintenance service. +// +// Transitions: +// +// --> maintenanceLeave: config loaded from maintenance service, and it's complete +func (ctrl *AcquireController) stateMaintenance(ctx context.Context, r controller.Runtime, logger *zap.Logger) (stateMachineFunc, config.Provider, error) { + // init maintenance + if err := safe.WriterModify(ctx, r, runtime.NewMaintenanceServiceRequest(), func(*runtime.MaintenanceServiceRequest) error { + return nil + }); err != nil { + return nil, nil, fmt.Errorf("failed creating maintenance service request: %w", err) + } + + // check current maintenance config + cfgResource, err := safe.ReaderGetByID[*configresource.MachineConfig](ctx, r, configresource.MaintenanceID) + if err != nil { + if state.IsNotFoundError(err) { + // no config loaded, wait for it + return nil, nil, nil + } + + return nil, nil, fmt.Errorf("failed to get maintenance config: %w", err) + } + + cfg := cfgResource.Provider() + + if cfg.CompleteForBoot() { + // complete config, we are done + ctrl.configSourcesUsed = append(ctrl.configSourcesUsed, "maintenance") + + return ctrl.stateMaintenanceLeave, cfg, nil + } + + // incomplete config, keep waiting, but apply new config + return nil, cfg, nil +} + +// stateMaintenanceLeave leaves the maintenance service. +// +// Transitions: +// +// --> done: proceed to done state +func (ctrl *AcquireController) stateMaintenanceLeave(ctx context.Context, r controller.Runtime, logger *zap.Logger) (stateMachineFunc, config.Provider, error) { + // stop the maintenance service + ready, err := r.Teardown(ctx, runtime.NewMaintenanceServiceRequest().Metadata()) + + switch { + case err != nil && !state.IsNotFoundError(err): + return nil, nil, fmt.Errorf("failed to tear down maintenance service: %w", err) + case err == nil && !ready: + // wait for the maintenance service to be torn down + return nil, nil, nil + case err == nil && ready: + if err = r.Destroy(ctx, runtime.NewMaintenanceServiceRequest().Metadata()); err != nil { + return nil, nil, fmt.Errorf("failed cleaning up maintenance service request: %w", err) + } + } + + ctrl.EventPublisher.Publish(ctx, &machineapi.TaskEvent{ + Action: machineapi.TaskEvent_STOP, + Task: "runningMaintenance", + }) + + logger.Info("leaving maintenance service") + + return ctrl.stateDone, nil, nil +} + +// stateDone is the final state of the controller. +func (ctrl *AcquireController) stateDone(ctx context.Context, r controller.Runtime, logger *zap.Logger) (stateMachineFunc, config.Provider, error) { + if err := safe.WriterModify(ctx, r, v1alpha1.NewAcquireConfigStatus(), func(_ *v1alpha1.AcquireConfigStatus) error { + return nil + }); err != nil { + return nil, nil, fmt.Errorf("failed to write status: %w", err) + } + + ctrl.PlatformEvent.FireEvent( + ctx, + platform.Event{ + Type: platform.EventTypeConfigLoaded, + Message: "Talos machine config loaded successfully.", + }, + ) + + logger.Info("machine config loaded successfully", zap.Strings("sources", ctrl.configSourcesUsed)) + + // fall through to the controller loop + return ctrl.stateFinal, nil, nil +} + +// stateFinal just makes the controller do nothing. +func (ctrl *AcquireController) stateFinal(ctx context.Context, r controller.Runtime, logger *zap.Logger) (stateMachineFunc, config.Provider, error) { + return nil, nil, nil +} diff --git a/internal/app/machined/pkg/controllers/config/acquire_test.go b/internal/app/machined/pkg/controllers/config/acquire_test.go new file mode 100644 index 0000000..0f12569 --- /dev/null +++ b/internal/app/machined/pkg/controllers/config/acquire_test.go @@ -0,0 +1,474 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package config_test + +import ( + "bytes" + "compress/gzip" + "context" + stderrors "errors" + "fmt" + "math/rand" + "net/url" + "os" + "path/filepath" + "slices" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + configctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/config" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/errors" + machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/config" + "github.com/siderolabs/talos/pkg/machinery/config/configloader" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/generate" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/config/types/siderolink" + "github.com/siderolabs/talos/pkg/machinery/proto" + configresource "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +type AcquireSuite struct { + ctest.DefaultSuite + + configPath string + platformConfig *platformConfigMock + platformEvent *platformEventMock + configSetter *configSetterMock + eventPublisher *eventPublisherMock + + clusterName string + completeMachineConfig []byte + partialMachineConfig []byte +} + +type platformConfigMock struct { + configuration []byte + err error +} + +func (p *platformConfigMock) Configuration(context.Context) ([]byte, error) { + return p.configuration, p.err +} + +func (p *platformConfigMock) Name() string { + return "mock" +} + +type platformEventMock struct { + mu sync.Mutex + events []platform.Event +} + +func (p *platformEventMock) FireEvent(_ context.Context, ev platform.Event) { + p.mu.Lock() + defer p.mu.Unlock() + + p.events = append(p.events, ev) +} + +func (p *platformEventMock) getEvents() []platform.Event { + p.mu.Lock() + defer p.mu.Unlock() + + return slices.Clone(p.events) +} + +type configSetterMock struct { + cfgCh chan config.Provider +} + +func (c *configSetterMock) SetConfig(cfg config.Provider) error { + c.cfgCh <- cfg + + return nil +} + +type eventPublisherMock struct { + mu sync.Mutex + events []proto.Message +} + +func (e *eventPublisherMock) Publish(_ context.Context, ev proto.Message) { + e.mu.Lock() + defer e.mu.Unlock() + + e.events = append(e.events, ev) +} + +func (e *eventPublisherMock) getEvents() []proto.Message { + e.mu.Lock() + defer e.mu.Unlock() + + return slices.Clone(e.events) +} + +type validationModeMock struct{} + +func (v validationModeMock) String() string { + return "mock" +} + +func (v validationModeMock) RequiresInstall() bool { + return false +} + +func (v validationModeMock) InContainer() bool { + return false +} + +func TestAcquireSuite(t *testing.T) { + t.Parallel() + + s := &AcquireSuite{ + DefaultSuite: ctest.DefaultSuite{ + Timeout: 15 * time.Second, + }, + } + + s.DefaultSuite.AfterSetup = func(*ctest.DefaultSuite) { + tmpDir := s.T().TempDir() + s.configPath = filepath.Join(tmpDir, "config.yaml") + s.platformConfig = &platformConfigMock{ + err: errors.ErrNoConfigSource, + } + s.platformEvent = &platformEventMock{} + s.configSetter = &configSetterMock{ + cfgCh: make(chan config.Provider, 1), + } + s.eventPublisher = &eventPublisherMock{} + + s.clusterName = fmt.Sprintf("cluster-%d", rand.Int31()) + input, err := generate.NewInput(s.clusterName, "https://localhost:6443", "") + s.Require().NoError(err) + + cfg, err := input.Config(machine.TypeControlPlane) + s.Require().NoError(err) + + s.completeMachineConfig, err = cfg.Bytes() + s.Require().NoError(err) + + sideroLinkCfg := siderolink.NewConfigV1Alpha1() + sideroLinkCfg.APIUrlConfig.URL = must(url.Parse("https://siderolink.api/?jointoken=secret&user=alice")) + + pCfg, err := container.New(sideroLinkCfg) + s.Require().NoError(err) + + s.partialMachineConfig, err = pCfg.Bytes() + s.Require().NoError(err) + + s.Require().NoError(s.Runtime().RegisterController(&configctrl.AcquireController{ + PlatformConfiguration: s.platformConfig, + PlatformEvent: s.platformEvent, + ConfigSetter: s.configSetter, + EventPublisher: s.eventPublisher, + ValidationMode: validationModeMock{}, + ConfigPath: s.configPath, + })) + } + + suite.Run(t, s) +} + +func (suite *AcquireSuite) triggerAcquire() { + suite.Require().NoError(suite.State().Create(suite.Ctx(), v1alpha1.NewAcquireConfigSpec())) +} + +func (suite *AcquireSuite) waitForConfig() config.Provider { + var cfg config.Provider + + select { + case cfg = <-suite.configSetter.cfgCh: + case <-suite.Ctx().Done(): + suite.Require().Fail("timed out waiting for config") + } + + status := v1alpha1.NewAcquireConfigStatus() + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{status.Metadata().ID()}, func(*v1alpha1.AcquireConfigStatus, *assert.Assertions) {}) + + return cfg +} + +func (suite *AcquireSuite) injectViaMaintenance(cfg []byte) { + _, err := suite.State().WatchFor(suite.Ctx(), runtime.NewMaintenanceServiceRequest().Metadata(), state.WithEventTypes(state.Created)) + suite.Require().NoError(err) + + mCfg, err := configloader.NewFromBytes(cfg) + suite.Require().NoError(err) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), configresource.NewMachineConfigWithID(mCfg, configresource.MaintenanceID))) + + _, err = suite.State().WatchFor(suite.Ctx(), runtime.NewMaintenanceServiceRequest().Metadata(), state.WithEventTypes(state.Destroyed)) + suite.Require().NoError(err) +} + +func (suite *AcquireSuite) TestFromDisk() { + suite.Require().NoError(os.WriteFile(suite.configPath, suite.completeMachineConfig, 0o644)) + + suite.triggerAcquire() + + cfg := suite.waitForConfig() + suite.Require().Equal(cfg.Cluster().Name(), suite.clusterName) + + suite.Assert().Empty(suite.eventPublisher.getEvents()) + suite.Assert().Equal( + []platform.Event{ + { + Type: platform.EventTypeConfigLoaded, + Message: "Talos machine config loaded successfully.", + }, + }, + suite.platformEvent.getEvents(), + ) +} + +func (suite *AcquireSuite) TestFromDiskFailure() { + suite.Require().NoError(os.WriteFile(suite.configPath, append([]byte("aaa"), suite.completeMachineConfig...), 0o644)) + + suite.triggerAcquire() + + suite.AssertWithin(time.Second, 10*time.Millisecond, func() error { + if len(suite.platformEvent.getEvents()) == 0 || len(suite.eventPublisher.getEvents()) == 0 { + return retry.ExpectedErrorf("no events received") + } + + return nil + }) + + ev := suite.platformEvent.getEvents()[0] + suite.Assert().Equal(platform.EventTypeFailure, ev.Type) + suite.Assert().Equal("Error loading and validating Talos machine config.", ev.Message) + suite.Assert().Equal("failed to load config from STATE: unknown keys found during decoding:\naaaversion: v1alpha1 # Indicates the schema used to decode the contents.\n", ev.Error.Error()) + + suite.Assert().Equal(&machineapi.ConfigLoadErrorEvent{ + Error: "failed to load config from STATE: unknown keys found during decoding:\naaaversion: v1alpha1 # Indicates the schema used to decode the contents.\n", + }, suite.eventPublisher.getEvents()[0]) +} + +func (suite *AcquireSuite) TestFromDiskToMaintenance() { + suite.Require().NoError(os.WriteFile(suite.configPath, suite.partialMachineConfig, 0o644)) + + suite.triggerAcquire() + + var cfg config.Provider + + select { + case cfg = <-suite.configSetter.cfgCh: + case <-suite.Ctx().Done(): + suite.Require().Fail("timed out waiting for config") + } + + suite.Require().Equal(cfg.SideroLink().APIUrl().Host, "siderolink.api") + + suite.injectViaMaintenance(suite.completeMachineConfig) + + cfg = suite.waitForConfig() + suite.Require().Equal(cfg.Cluster().Name(), suite.clusterName) + + suite.Assert().Equal( + []proto.Message{ + &machineapi.TaskEvent{ + Action: machineapi.TaskEvent_START, + Task: "runningMaintenance", + }, + &machineapi.TaskEvent{ + Action: machineapi.TaskEvent_STOP, + Task: "runningMaintenance", + }, + }, + suite.eventPublisher.getEvents(), + ) + suite.Assert().Equal( + []platform.Event{ + { + Type: platform.EventTypeActivate, + Message: "Talos booted into maintenance mode. Ready for user interaction.", + }, + { + Type: platform.EventTypeConfigLoaded, + Message: "Talos machine config loaded successfully.", + }, + }, + suite.platformEvent.getEvents(), + ) +} + +func (suite *AcquireSuite) TestFromPlatform() { + suite.platformConfig.configuration = suite.completeMachineConfig + suite.platformConfig.err = nil + + suite.triggerAcquire() + + cfg := suite.waitForConfig() + suite.Require().Equal(cfg.Cluster().Name(), suite.clusterName) + + suite.Assert().Empty(suite.eventPublisher.getEvents()) + suite.Assert().Equal( + []platform.Event{ + { + Type: platform.EventTypeConfigLoaded, + Message: "Talos machine config loaded successfully.", + }, + }, + suite.platformEvent.getEvents(), + ) +} + +func (suite *AcquireSuite) TestFromPlatformFailure() { + suite.platformConfig.err = stderrors.New("mock error") + + suite.triggerAcquire() + + suite.AssertWithin(time.Second, 10*time.Millisecond, func() error { + if len(suite.platformEvent.getEvents()) == 0 || len(suite.eventPublisher.getEvents()) == 0 { + return retry.ExpectedErrorf("no events received") + } + + return nil + }) + + ev := suite.platformEvent.getEvents()[0] + suite.Assert().Equal(platform.EventTypeFailure, ev.Type) + suite.Assert().Equal("Error loading and validating Talos machine config.", ev.Message) + suite.Assert().Equal("error acquiring via platform mock: mock error", ev.Error.Error()) + + suite.Assert().Equal(&machineapi.ConfigLoadErrorEvent{ + Error: "error acquiring via platform mock: mock error", + }, suite.eventPublisher.getEvents()[0]) +} + +func (suite *AcquireSuite) TestFromPlatformGzip() { + var buf bytes.Buffer + + gz := gzip.NewWriter(&buf) + _, err := gz.Write(suite.completeMachineConfig) + suite.Require().NoError(err) + suite.Require().NoError(gz.Close()) + + suite.platformConfig.configuration = buf.Bytes() + suite.platformConfig.err = nil + + suite.triggerAcquire() + + cfg := suite.waitForConfig() + suite.Require().Equal(cfg.Cluster().Name(), suite.clusterName) + + suite.Assert().Empty(suite.eventPublisher.getEvents()) + suite.Assert().Equal( + []platform.Event{ + { + Type: platform.EventTypeConfigLoaded, + Message: "Talos machine config loaded successfully.", + }, + }, + suite.platformEvent.getEvents(), + ) +} + +func (suite *AcquireSuite) TestFromPlatformToMaintenance() { + suite.platformConfig.configuration = suite.partialMachineConfig + suite.platformConfig.err = nil + + suite.triggerAcquire() + + var cfg config.Provider + + select { + case cfg = <-suite.configSetter.cfgCh: + case <-suite.Ctx().Done(): + suite.Require().Fail("timed out waiting for config") + } + + suite.Require().Equal(cfg.SideroLink().APIUrl().Host, "siderolink.api") + + suite.injectViaMaintenance(suite.completeMachineConfig) + + cfg = suite.waitForConfig() + suite.Require().Equal(cfg.Cluster().Name(), suite.clusterName) + + suite.Assert().Equal( + []proto.Message{ + &machineapi.TaskEvent{ + Action: machineapi.TaskEvent_START, + Task: "runningMaintenance", + }, + &machineapi.TaskEvent{ + Action: machineapi.TaskEvent_STOP, + Task: "runningMaintenance", + }, + }, + suite.eventPublisher.getEvents(), + ) + suite.Assert().Equal( + []platform.Event{ + { + Type: platform.EventTypeActivate, + Message: "Talos booted into maintenance mode. Ready for user interaction.", + }, + { + Type: platform.EventTypeConfigLoaded, + Message: "Talos machine config loaded successfully.", + }, + }, + suite.platformEvent.getEvents(), + ) +} + +func (suite *AcquireSuite) TestFromMaintenance() { + suite.triggerAcquire() + + suite.injectViaMaintenance(suite.completeMachineConfig) + + cfg := suite.waitForConfig() + suite.Require().Equal(cfg.Cluster().Name(), suite.clusterName) + + suite.Assert().Equal( + []proto.Message{ + &machineapi.TaskEvent{ + Action: machineapi.TaskEvent_START, + Task: "runningMaintenance", + }, + &machineapi.TaskEvent{ + Action: machineapi.TaskEvent_STOP, + Task: "runningMaintenance", + }, + }, + suite.eventPublisher.getEvents(), + ) + suite.Assert().Equal( + []platform.Event{ + { + Type: platform.EventTypeActivate, + Message: "Talos booted into maintenance mode. Ready for user interaction.", + }, + { + Type: platform.EventTypeConfigLoaded, + Message: "Talos machine config loaded successfully.", + }, + }, + suite.platformEvent.getEvents(), + ) +} + +func must[T any](t T, err error) T { + if err != nil { + panic(err) + } + + return t +} diff --git a/internal/app/machined/pkg/controllers/config/config.go b/internal/app/machined/pkg/controllers/config/config.go new file mode 100644 index 0000000..230c5b5 --- /dev/null +++ b/internal/app/machined/pkg/controllers/config/config.go @@ -0,0 +1,6 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package config provides controllers which manage config resources. +package config diff --git a/internal/app/machined/pkg/controllers/config/machine_type.go b/internal/app/machined/pkg/controllers/config/machine_type.go new file mode 100644 index 0000000..e3321d3 --- /dev/null +++ b/internal/app/machined/pkg/controllers/config/machine_type.go @@ -0,0 +1,82 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package config + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/resources/config" +) + +// MachineTypeController manages config.MachineType based on configuration. +type MachineTypeController struct{} + +// Name implements controller.Controller interface. +func (ctrl *MachineTypeController) Name() string { + return "config.MachineTypeController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *MachineTypeController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: config.MachineConfigType, + ID: optional.Some(config.V1Alpha1ID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *MachineTypeController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: config.MachineTypeType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +func (ctrl *MachineTypeController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + var machineType machine.Type + + cfg, err := safe.ReaderGetByID[*config.MachineConfig](ctx, r, config.V1Alpha1ID) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting config: %w", err) + } + } else if cfg.Config().Machine() != nil { + machineType = cfg.Config().Machine().Type() + } + + if err = r.Modify(ctx, config.NewMachineType(), func(r resource.Resource) error { + r.(*config.MachineType).SetMachineType(machineType) + + return nil + }); err != nil { + return fmt.Errorf("error updating objects: %w", err) + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/cri/cri.go b/internal/app/machined/pkg/controllers/cri/cri.go new file mode 100644 index 0000000..6efca64 --- /dev/null +++ b/internal/app/machined/pkg/controllers/cri/cri.go @@ -0,0 +1,5 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cri diff --git a/internal/app/machined/pkg/controllers/cri/cri_test.go b/internal/app/machined/pkg/controllers/cri/cri_test.go new file mode 100644 index 0000000..cd0c7d6 --- /dev/null +++ b/internal/app/machined/pkg/controllers/cri/cri_test.go @@ -0,0 +1,5 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cri_test diff --git a/internal/app/machined/pkg/controllers/cri/seccomp_profile.go b/internal/app/machined/pkg/controllers/cri/seccomp_profile.go new file mode 100644 index 0000000..d41213d --- /dev/null +++ b/internal/app/machined/pkg/controllers/cri/seccomp_profile.go @@ -0,0 +1,88 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cri + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/cri" +) + +// SeccompProfileController manages SeccompProfiles. +type SeccompProfileController struct{} + +// Name implements controller.StatsController interface. +func (ctrl *SeccompProfileController) Name() string { + return "cri.SeccompProfileController" +} + +// Inputs implements controller.StatsController interface. +func (ctrl *SeccompProfileController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: config.MachineConfigType, + ID: optional.Some(config.V1Alpha1ID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.StatsController interface. +func (ctrl *SeccompProfileController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: cri.SeccompProfileType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.StatsController interface. +func (ctrl *SeccompProfileController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + cfg, err := safe.ReaderGetByID[*config.MachineConfig](ctx, r, config.V1Alpha1ID) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting config: %w", err) + } + + r.StartTrackingOutputs() + + if cfg.Config().Machine() != nil { + for _, profile := range cfg.Config().Machine().SeccompProfiles() { + if err = safe.WriterModify(ctx, r, cri.NewSeccompProfile(profile.Name()), func(cri *cri.SeccompProfile) error { + cri.TypedSpec().Name = profile.Name() + cri.TypedSpec().Value = profile.Value() + + return nil + }); err != nil { + return err + } + } + } + + if err = safe.CleanupOutputs[*cri.SeccompProfile](ctx, r); err != nil { + return err + } + } +} diff --git a/internal/app/machined/pkg/controllers/cri/seccomp_profile_file.go b/internal/app/machined/pkg/controllers/cri/seccomp_profile_file.go new file mode 100644 index 0000000..61dabcb --- /dev/null +++ b/internal/app/machined/pkg/controllers/cri/seccomp_profile_file.go @@ -0,0 +1,179 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cri + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + runtimetalos "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/cri" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// SeccompProfileFileController manages the Seccomp Profiles on the host. +type SeccompProfileFileController struct { + V1Alpha1Mode runtimetalos.Mode + SeccompProfilesDirectory string +} + +// Name implements controller.StatsController interface. +func (ctrl *SeccompProfileFileController) Name() string { + return "cri.SeccompProfileFileController" +} + +// Inputs implements controller.StatsController interface. +func (ctrl *SeccompProfileFileController) Inputs() []controller.Input { + return nil +} + +// Outputs implements controller.StatsController interface. +func (ctrl *SeccompProfileFileController) Outputs() []controller.Output { + return nil +} + +// Run implements controller.StatsController interface. +// +//nolint:gocyclo,cyclop +func (ctrl *SeccompProfileFileController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + // initially, wait for /var to be mounted + if err := r.UpdateInputs([]controller.Input{ + { + Namespace: runtimeres.NamespaceName, + Type: runtimeres.MountStatusType, + ID: optional.Some(constants.EphemeralPartitionLabel), + Kind: controller.InputWeak, + }, + }); err != nil { + return err + } + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + _, err := safe.ReaderGet[*runtimeres.MountStatus](ctx, r, resource.NewMetadata(runtimeres.NamespaceName, runtimeres.MountStatusType, constants.EphemeralPartitionLabel, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + // in container mode EPHEMERAL is always mounted + if ctrl.V1Alpha1Mode != runtimetalos.ModeContainer { + // wait for the EPHEMERAL to be mounted + continue + } + } else { + return fmt.Errorf("error getting ephemeral mount status: %w", err) + } + } + + break + } + + // normal reconcile loop + if err := r.UpdateInputs([]controller.Input{ + { + Namespace: cri.NamespaceName, + Type: cri.SeccompProfileType, + Kind: controller.InputWeak, + }, + }); err != nil { + return err + } + + r.QueueReconcile() + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + list, err := safe.ReaderListAll[*cri.SeccompProfile](ctx, r) + if err != nil { + return fmt.Errorf("error listing seccomp profiles: %w", err) + } + + touchedIDs := make(map[string]struct{}, list.Len()) + + for iter := list.Iterator(); iter.Next(); { + profile := iter.Value() + + profileName := profile.TypedSpec().Name + profilePath := filepath.Join(ctrl.SeccompProfilesDirectory, profileName) + + profileContent, err := json.Marshal(profile.TypedSpec().Value) + if err != nil { + return fmt.Errorf("error marshaling seccomp profile: %w", err) + } + + existingProfileContent, err := os.ReadFile(profilePath) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("error reading existing seccomp profile at %s: %w", profilePath, err) + } + + if err := writeSeccompFile(profilePath, profileContent); err != nil { + return err + } + } else { + if val := bytes.Compare(existingProfileContent, profileContent); val != 0 { + if err := writeSeccompFile(profilePath, profileContent); err != nil { + return err + } + } + } + + touchedIDs[profileName] = struct{}{} + } + + // cleanup + if err := filepath.WalkDir(ctrl.SeccompProfilesDirectory, func(path string, d fs.DirEntry, err error) error { + fileName, errRel := filepath.Rel(ctrl.SeccompProfilesDirectory, path) + if errRel != nil { + return errRel + } + + // ignore current folder + if fileName != "." { + if _, ok := touchedIDs[fileName]; !ok { + if err := os.RemoveAll(path); err != nil { + return err + } + } + } + + return nil + }); err != nil { + return err + } + + r.ResetRestartBackoff() + } +} + +func writeSeccompFile(path string, content []byte) error { + if err := os.WriteFile(path, content, 0o644); err != nil { + return fmt.Errorf("error writing seccomp profile at %s: %w", path, err) + } + + return nil +} diff --git a/internal/app/machined/pkg/controllers/cri/seccomp_profile_file_test.go b/internal/app/machined/pkg/controllers/cri/seccomp_profile_file_test.go new file mode 100644 index 0000000..cc2c382 --- /dev/null +++ b/internal/app/machined/pkg/controllers/cri/seccomp_profile_file_test.go @@ -0,0 +1,130 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cri_test + +import ( + "encoding/json" + "errors" + "os" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/cri" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + criseccompresource "github.com/siderolabs/talos/pkg/machinery/resources/cri" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +func (suite *CRISeccompProfileFileSuite) TestReconcileSeccompProfileFile() { + // need to mock mountStatus so that the controller moves ahead with the actual code + mountStatus := runtimeres.NewMountStatus(runtimeres.NamespaceName, "EPHEMERAL") + suite.Require().NoError(suite.State().Create(suite.Ctx(), mountStatus)) + + for _, tt := range []struct { + seccompProfileName string + seccompProfileValue map[string]interface{} + }{ + { + seccompProfileName: "audit.json", + seccompProfileValue: map[string]interface{}{ + "defaultAction": "SCMP_ACT_LOG", + }, + }, + { + seccompProfileName: "deny.json", + seccompProfileValue: map[string]interface{}{ + "defaultAction": "SCMP_ACT_ERRNO", + }, + }, + } { + seccompProfiles := criseccompresource.NewSeccompProfile(tt.seccompProfileName) + seccompProfiles.TypedSpec().Name = tt.seccompProfileName + seccompProfiles.TypedSpec().Value = tt.seccompProfileValue + suite.Require().NoError(suite.State().Create(suite.Ctx(), seccompProfiles)) + + suite.AssertWithin(1*time.Second, 100*time.Millisecond, func() error { + if _, err := os.Stat(suite.seccompProfilesDirectory + "/" + tt.seccompProfileName); err != nil { + if errors.Is(err, os.ErrNotExist) { + return retry.ExpectedError(err) + } + + return err + } + + seccompProfileContent, err := os.ReadFile(suite.seccompProfilesDirectory + "/" + tt.seccompProfileName) + suite.Assert().NoError(err) + + expectedSeccompProfileContent, err := json.Marshal(tt.seccompProfileValue) + suite.Assert().NoError(err) + + suite.Assert().Equal(seccompProfileContent, expectedSeccompProfileContent) + + return nil + }) + } + + // create a directory and file manually in the seccomp profile directory + // ensure that the controller deletes the manually created directory/file + // also ensure that an update doesn't update existing files timestamp + suite.Assert().NoError(os.Mkdir(suite.seccompProfilesDirectory+"/test", 0o755)) + suite.Assert().NoError(os.WriteFile(suite.seccompProfilesDirectory+"/test.json", []byte("{}"), 0o644)) + + auditJSONSeccompProfile, err := os.Stat(suite.seccompProfilesDirectory + "/audit.json") + suite.Assert().NoError(err) + + // delete deny.json resource + suite.Assert().NoError(suite.State().Destroy(suite.Ctx(), resource.NewMetadata(criseccompresource.NamespaceName, criseccompresource.SeccompProfileType, "deny.json", resource.VersionUndefined))) + + suite.AssertWithin(1*time.Second, 100*time.Millisecond, func() error { + auditJSONSeccompProfileAfterUpdate, err := os.Stat(suite.seccompProfilesDirectory + "/audit.json") + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return retry.ExpectedError(err) + } + + return err + } + + suite.Eventually(func() bool { + return suite.NoFileExists(suite.seccompProfilesDirectory + "/deny.json") + }, 1*time.Second, 100*time.Millisecond) + + suite.Eventually(func() bool { + return suite.NoFileExists(suite.seccompProfilesDirectory + "/test.json") + }, 1*time.Second, 100*time.Millisecond) + + suite.Eventually(func() bool { + return suite.NoDirExists(suite.seccompProfilesDirectory + "/test") + }, 1*time.Second, 100*time.Millisecond) + + suite.Assert().Equal(auditJSONSeccompProfile.ModTime(), auditJSONSeccompProfileAfterUpdate.ModTime()) + + return nil + }) +} + +func TestSeccompProfileFileSuite(t *testing.T) { + seccompProfiesDirectory := t.TempDir() + + suite.Run(t, &CRISeccompProfileFileSuite{ + DefaultSuite: ctest.DefaultSuite{ + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(&cri.SeccompProfileFileController{ + SeccompProfilesDirectory: seccompProfiesDirectory, + })) + }, + }, + seccompProfilesDirectory: seccompProfiesDirectory, + }) +} + +type CRISeccompProfileFileSuite struct { + ctest.DefaultSuite + seccompProfilesDirectory string +} diff --git a/internal/app/machined/pkg/controllers/cri/seccomp_profile_test.go b/internal/app/machined/pkg/controllers/cri/seccomp_profile_test.go new file mode 100644 index 0000000..ba815d4 --- /dev/null +++ b/internal/app/machined/pkg/controllers/cri/seccomp_profile_test.go @@ -0,0 +1,135 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cri_test + +import ( + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/cri" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + criseccompresource "github.com/siderolabs/talos/pkg/machinery/resources/cri" +) + +func (suite *CRISeccompProfileSuite) TestReconcileSeccompProfile() { + cfg := config.NewMachineConfig(container.NewV1Alpha1(&v1alpha1.Config{ + MachineConfig: &v1alpha1.MachineConfig{ + MachineSeccompProfiles: []*v1alpha1.MachineSeccompProfile{ + { + MachineSeccompProfileName: "audit.json", + MachineSeccompProfileValue: v1alpha1.Unstructured{ + Object: map[string]interface{}{ + "defaultAction": "SCMP_ACT_LOG", + }, + }, + }, + { + MachineSeccompProfileName: "deny.json", + MachineSeccompProfileValue: v1alpha1.Unstructured{ + Object: map[string]interface{}{ + "defaultAction": "SCMP_ACT_ERRNO", + }, + }, + }, + }, + }, + })) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + for _, tt := range []struct { + name string + value map[string]interface{} + }{ + { + name: "audit.json", + value: map[string]interface{}{ + "defaultAction": "SCMP_ACT_LOG", + }, + }, + { + name: "deny.json", + value: map[string]interface{}{ + "defaultAction": "SCMP_ACT_ERRNO", + }, + }, + } { + suite.AssertWithin(1*time.Second, 100*time.Millisecond, func() error { + seccompProfile, err := ctest.Get[*criseccompresource.SeccompProfile]( + suite, + criseccompresource.NewSeccompProfile(tt.name).Metadata(), + ) + if err != nil { + if state.IsNotFoundError(err) { + return retry.ExpectedError(err) + } + + return err + } + + spec := seccompProfile.TypedSpec() + + suite.Assert().Equal(tt.name, spec.Name) + suite.Assert().Equal(tt.value, spec.Value) + + return nil + }) + } + + // test deletion + cfg = config.NewMachineConfig(container.NewV1Alpha1(&v1alpha1.Config{ + MachineConfig: &v1alpha1.MachineConfig{ + MachineSeccompProfiles: []*v1alpha1.MachineSeccompProfile{ + { + MachineSeccompProfileName: "audit.json", + MachineSeccompProfileValue: v1alpha1.Unstructured{ + Object: map[string]interface{}{ + "defaultAction": "SCMP_ACT_LOG", + }, + }, + }, + }, + }, + })) + + ctest.UpdateWithConflicts(suite, cfg, func(mc *config.MachineConfig) error { return nil }) + + suite.AssertWithin(1*time.Second, 100*time.Millisecond, func() error { + _, err := ctest.Get[*criseccompresource.SeccompProfile]( + suite, + criseccompresource.NewSeccompProfile("deny.json").Metadata(), + ) + if err != nil { + if !state.IsNotFoundError(err) { + return err + } + + return err + } + + return nil + }) +} + +func TestSeccompProfileSuite(t *testing.T) { + suite.Run(t, &CRISeccompProfileSuite{ + DefaultSuite: ctest.DefaultSuite{ + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(&cri.SeccompProfileController{})) + }, + }, + }) +} + +type CRISeccompProfileSuite struct { + ctest.DefaultSuite +} diff --git a/internal/app/machined/pkg/controllers/ctest/assert.go b/internal/app/machined/pkg/controllers/ctest/assert.go new file mode 100644 index 0000000..7b1b4cc --- /dev/null +++ b/internal/app/machined/pkg/controllers/ctest/assert.go @@ -0,0 +1,67 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package ctest + +import ( + "fmt" + "sort" + "strings" + + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type assertionAggregator struct { + errors map[string]struct{} + failNow bool + hadErrors bool +} + +func (agg *assertionAggregator) Errorf(format string, args ...any) { + errorString := fmt.Sprintf(format, args...) + + if agg.errors == nil { + agg.errors = make(map[string]struct{}) + } + + agg.errors[errorString] = struct{}{} + agg.hadErrors = true +} + +func (agg *assertionAggregator) FailNow() { + agg.failNow = true +} + +func (agg *assertionAggregator) Error() error { + if !agg.hadErrors { + return nil + } + + lines := make([]string, 0, len(agg.errors)) + + for errorString := range agg.errors { + lines = append(lines, " * "+errorString) + } + + sort.Strings(lines) + + return fmt.Errorf("%s", strings.Join(lines, "\n")) +} + +// WrapRetry wraps the function with assertions and requires to return retry-compatible errors. +func WrapRetry(f func(*assert.Assertions, *require.Assertions)) func() error { + return func() error { + var errs assertionAggregator + + f(assert.New(&errs), require.New(&errs)) + + if errs.failNow { + return errs.Error() + } + + return retry.ExpectedError(errs.Error()) + } +} diff --git a/internal/app/machined/pkg/controllers/ctest/ctest.go b/internal/app/machined/pkg/controllers/ctest/ctest.go new file mode 100644 index 0000000..059dc5d --- /dev/null +++ b/internal/app/machined/pkg/controllers/ctest/ctest.go @@ -0,0 +1,189 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package ctest provides basic types and functions for controller testing. +package ctest + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "go.uber.org/zap/zaptest" +) + +// DefaultSuite is a base suite for controller testing. +type DefaultSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc + + AfterSetup func(suite *DefaultSuite) + AfterTearDown func(suite *DefaultSuite) + Timeout time.Duration +} + +// SetupTest is a function for setting up a test. +func (suite *DefaultSuite) SetupTest() { + if suite.Timeout == 0 { + suite.Timeout = 3 * time.Minute + } + + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), suite.Timeout) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, zaptest.NewLogger(suite.T())) + suite.Require().NoError(err) + + suite.startRuntime() + + if suite.AfterSetup != nil { + suite.AfterSetup(suite) + } +} + +func (suite *DefaultSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +// Runtime returns the runtime of the suite. +func (suite *DefaultSuite) Runtime() *runtime.Runtime { + return suite.runtime +} + +// State returns the state of the suite. +func (suite *DefaultSuite) State() state.State { + return suite.state +} + +// Ctx returns the context of the suite. +func (suite *DefaultSuite) Ctx() context.Context { + return suite.ctx +} + +// AssertWithin asserts that fn returns within the given duration without an error. +func (suite *DefaultSuite) AssertWithin(d time.Duration, rate time.Duration, fn func() error) { + retryer := retry.Constant(d, retry.WithUnits(rate)) + suite.Assert().NoError(retryer.Retry(fn)) +} + +// TearDownTest is a function for tearing down a test. +func (suite *DefaultSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() + + if suite.AfterTearDown != nil { + suite.AfterTearDown(suite) + } +} + +// Create creates a new resource in the state of the suite. +func (suite *DefaultSuite) Create(res resource.Resource, opts ...state.CreateOption) { + suite.Require().NoError(suite.State().Create(suite.Ctx(), res, opts...)) +} + +// Suite is a type which describes the suite type. +type Suite interface { + T() *testing.T + Require() *require.Assertions + State() state.State + Ctx() context.Context +} + +// UpdateWithConflicts is a type safe wrapper around state.UpdateWithConflicts which uses the provided suite. +func UpdateWithConflicts[T resource.Resource](suite Suite, res T, updateFn func(T) error, options ...state.UpdateOption) T { //nolint:ireturn + suite.T().Helper() + result, err := safe.StateUpdateWithConflicts(suite.Ctx(), suite.State(), res.Metadata(), updateFn, options...) + suite.Require().NoError(err) + + return result +} + +// GetUsingResource is a type safe wrapper around state.StateGetResource which uses the provided suite. +func GetUsingResource[T resource.Resource](suite Suite, res T, options ...state.GetOption) (T, error) { //nolint:ireturn + return safe.StateGetResource(suite.Ctx(), suite.State(), res, options...) +} + +// Get is a type safe wrapper around state.Get which uses the provided suite. +func Get[T resource.Resource](suite Suite, ptr resource.Pointer, options ...state.GetOption) (T, error) { //nolint:ireturn + return safe.StateGet[T](suite.Ctx(), suite.State(), ptr, options...) +} + +// Suiter is like Suite but do not require Require() method. +type Suiter interface { + T() *testing.T + State() state.State + Ctx() context.Context +} + +// AssertResources asserts on a resource list. +func AssertResources[R rtestutils.ResourceWithRD]( + suiter Suiter, + requiredIDs []resource.ID, + check func(R, *assert.Assertions), + opts ...rtestutils.Option, +) { + ctx, cancel := context.WithTimeout(suiter.Ctx(), 10*time.Second) + defer cancel() + + rtestutils.AssertResources(ctx, suiter.T(), suiter.State(), requiredIDs, check, opts...) +} + +// AssertResource asserts on a single resource. +func AssertResource[R rtestutils.ResourceWithRD]( + suiter Suiter, + requiredIDs resource.ID, + check func(R, *assert.Assertions), + opts ...rtestutils.Option, +) { + AssertResources(suiter, []resource.ID{requiredIDs}, check, opts...) +} + +// AssertNoResource asserts that a resource no longer exists. +func AssertNoResource[R rtestutils.ResourceWithRD]( + suiter Suiter, + id string, + opts ...rtestutils.Option, +) { + ctx, cancel := context.WithTimeout(suiter.Ctx(), 10*time.Second) + defer cancel() + + rtestutils.AssertNoResource[R]( + ctx, + suiter.T(), + suiter.State(), + id, + opts..., + ) +} diff --git a/internal/app/machined/pkg/controllers/etcd/advertised_peer.go b/internal/app/machined/pkg/controllers/etcd/advertised_peer.go new file mode 100644 index 0000000..9dd1672 --- /dev/null +++ b/internal/app/machined/pkg/controllers/etcd/advertised_peer.go @@ -0,0 +1,177 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package etcd + +import ( + "context" + "errors" + "fmt" + "net/netip" + "reflect" + "time" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/xslices" + "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.uber.org/zap" + + etcdcli "github.com/aenix-io/talm/internal/pkg/etcd" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/etcd" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +// AdvertisedPeerController updates advertised peer list for this instance of etcd. +type AdvertisedPeerController struct{} + +// Name implements controller.Controller interface. +func (ctrl *AdvertisedPeerController) Name() string { + return "etcd.AdvertisedPeerController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *AdvertisedPeerController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: etcd.NamespaceName, + Type: etcd.SpecType, + ID: optional.Some(etcd.SpecID), + Kind: controller.InputWeak, + }, + { + Namespace: etcd.NamespaceName, + Type: etcd.PKIStatusType, + ID: optional.Some(etcd.PKIID), + Kind: controller.InputWeak, + }, + { + Namespace: v1alpha1.NamespaceName, + Type: v1alpha1.ServiceType, + ID: optional.Some("etcd"), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *AdvertisedPeerController) Outputs() []controller.Output { + return nil +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *AdvertisedPeerController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + etcdService, err := safe.ReaderGet[*v1alpha1.Service](ctx, r, resource.NewMetadata(v1alpha1.NamespaceName, v1alpha1.ServiceType, "etcd", resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting etcd service: %w", err) + } + + if !(etcdService.TypedSpec().Healthy && etcdService.TypedSpec().Running) { + continue + } + + etcdSpec, err := safe.ReaderGet[*etcd.Spec](ctx, r, resource.NewMetadata(etcd.NamespaceName, etcd.SpecType, etcd.SpecID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting etcd spec: %w", err) + } + + _, err = safe.ReaderGet[*etcd.PKIStatus](ctx, r, resource.NewMetadata(etcd.NamespaceName, etcd.PKIStatusType, etcd.PKIID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting etcd PKI status: %w", err) + } + + if err = ctrl.updateAdvertisedPeers(ctx, logger, etcdSpec.TypedSpec().AdvertisedAddresses); err != nil { + return fmt.Errorf("error updating advertised peers: %w", err) + } + + r.ResetRestartBackoff() + } +} + +func (ctrl *AdvertisedPeerController) updateAdvertisedPeers(ctx context.Context, logger *zap.Logger, advertisedAddresses []netip.Addr) error { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + client, err := etcdcli.NewLocalClient(ctx) + if err != nil { + return fmt.Errorf("error creating etcd client: %w", err) + } + + defer client.Close() //nolint:errcheck + + // figure out local member ID + resp, err := client.MemberList(ctx) + if err != nil { + return fmt.Errorf("error getting member list: %w", err) + } + + localMemberID := resp.Header.MemberId + + var localMember *etcdserverpb.Member + + for _, member := range resp.Members { + if member.ID == localMemberID { + localMember = member + + break + } + } + + if localMember == nil { + return errors.New("local member not found in member list") + } + + newPeerURLs := xslices.Map(advertisedAddresses, func(addr netip.Addr) string { + return fmt.Sprintf("https://%s", nethelpers.JoinHostPort(addr.String(), constants.EtcdPeerPort)) + }) + currentPeerURLs := localMember.PeerURLs + + if reflect.DeepEqual(newPeerURLs, currentPeerURLs) { + return nil + } + + logger.Debug("updating etcd peer URLs", + zap.Strings("current_peer_urls", currentPeerURLs), + zap.Strings("new_peer_urls", newPeerURLs), + zap.Uint64("member_id", localMemberID), + ) + + _, err = client.MemberUpdate(ctx, localMemberID, newPeerURLs) + if err != nil { + return fmt.Errorf("error updating member: %w", err) + } + + logger.Info("updated etcd peer URLs", + zap.Strings("new_peer_urls", newPeerURLs), + zap.Uint64("member_id", localMemberID), + ) + + return nil +} diff --git a/internal/app/machined/pkg/controllers/etcd/config.go b/internal/app/machined/pkg/controllers/etcd/config.go new file mode 100644 index 0000000..3b76989 --- /dev/null +++ b/internal/app/machined/pkg/controllers/etcd/config.go @@ -0,0 +1,69 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package etcd + +import ( + "context" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/controller/generic/transform" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/etcd" +) + +// ConfigController watches v1alpha1.Config, updates etcd config. +type ConfigController = transform.Controller[*config.MachineConfig, *etcd.Config] + +// NewConfigController instanciates the config controller. +func NewConfigController() *ConfigController { + return transform.NewController( + transform.Settings[*config.MachineConfig, *etcd.Config]{ + Name: "etcd.ConfigController", + MapMetadataOptionalFunc: func(cfg *config.MachineConfig) optional.Optional[*etcd.Config] { + if cfg.Metadata().ID() != config.V1Alpha1ID { + return optional.None[*etcd.Config]() + } + + if cfg.Config().Machine() == nil || cfg.Config().Cluster() == nil { + return optional.None[*etcd.Config]() + } + + if !cfg.Config().Machine().Type().IsControlPlane() { + // etcd only runs on controlplane nodes + return optional.None[*etcd.Config]() + } + + return optional.Some(etcd.NewConfig(etcd.NamespaceName, etcd.ConfigID)) + }, + TransformFunc: func(ctx context.Context, r controller.Reader, logger *zap.Logger, machineConfig *config.MachineConfig, cfg *etcd.Config) error { + cfg.TypedSpec().AdvertiseValidSubnets = machineConfig.Config().Cluster().Etcd().AdvertisedSubnets() + cfg.TypedSpec().AdvertiseExcludeSubnets = nil + cfg.TypedSpec().ListenValidSubnets = machineConfig.Config().Cluster().Etcd().ListenSubnets() + cfg.TypedSpec().ListenExcludeSubnets = nil + + // filter out any virtual IPs, they can't be node IPs either + for _, device := range machineConfig.Config().Machine().Network().Devices() { + if device.VIPConfig() != nil { + cfg.TypedSpec().AdvertiseExcludeSubnets = append(cfg.TypedSpec().AdvertiseExcludeSubnets, device.VIPConfig().IP()) + } + + for _, vlan := range device.Vlans() { + if vlan.VIPConfig() != nil { + cfg.TypedSpec().AdvertiseExcludeSubnets = append(cfg.TypedSpec().AdvertiseExcludeSubnets, vlan.VIPConfig().IP()) + } + } + } + + cfg.TypedSpec().Image = machineConfig.Config().Cluster().Etcd().Image() + cfg.TypedSpec().ExtraArgs = machineConfig.Config().Cluster().Etcd().ExtraArgs() + + return nil + }, + }, + ) +} diff --git a/internal/app/machined/pkg/controllers/etcd/config_test.go b/internal/app/machined/pkg/controllers/etcd/config_test.go new file mode 100644 index 0000000..fd3fadd --- /dev/null +++ b/internal/app/machined/pkg/controllers/etcd/config_test.go @@ -0,0 +1,178 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package etcd_test + +import ( + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/safe" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + etcdctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/etcd" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/etcd" +) + +func TestConfigSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, &ConfigSuite{ + DefaultSuite: ctest.DefaultSuite{ + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(etcdctrl.NewConfigController())) + }, + }, + }) +} + +type ConfigSuite struct { + ctest.DefaultSuite +} + +func (suite *ConfigSuite) TestReconcile() { + for _, tt := range []struct { + name string + etcdConfig *v1alpha1.EtcdConfig + networkConfig v1alpha1.NetworkDeviceList + expectedConfig etcd.ConfigSpec + }{ + { + name: "default config", + etcdConfig: &v1alpha1.EtcdConfig{ + ContainerImage: "foo/bar:v1.0.0", + }, + expectedConfig: etcd.ConfigSpec{ + Image: "foo/bar:v1.0.0", + ExtraArgs: map[string]string{}, + AdvertiseValidSubnets: nil, + ListenValidSubnets: nil, + }, + }, + { + name: "legacy subnet", + etcdConfig: &v1alpha1.EtcdConfig{ + ContainerImage: "foo/bar:v1.0.0", + EtcdExtraArgs: map[string]string{ + "arg": "value", + }, + EtcdSubnet: "10.0.0.0/8", + }, + expectedConfig: etcd.ConfigSpec{ + Image: "foo/bar:v1.0.0", + ExtraArgs: map[string]string{ + "arg": "value", + }, + AdvertiseValidSubnets: []string{"10.0.0.0/8"}, + ListenValidSubnets: nil, + }, + }, + { + name: "advertised subnets", + etcdConfig: &v1alpha1.EtcdConfig{ + ContainerImage: "foo/bar:v1.0.0", + EtcdAdvertisedSubnets: []string{"10.0.0.0/8", "192.168.0.0/24"}, + }, + expectedConfig: etcd.ConfigSpec{ + Image: "foo/bar:v1.0.0", + ExtraArgs: map[string]string{}, + AdvertiseValidSubnets: []string{"10.0.0.0/8", "192.168.0.0/24"}, + ListenValidSubnets: []string{"10.0.0.0/8", "192.168.0.0/24"}, + }, + }, + { + name: "advertised and listen subnets", + etcdConfig: &v1alpha1.EtcdConfig{ + ContainerImage: "foo/bar:v1.0.0", + EtcdAdvertisedSubnets: []string{"10.0.0.0/8", "192.168.0.0/24"}, + EtcdListenSubnets: []string{"10.0.0.0/8"}, + }, + expectedConfig: etcd.ConfigSpec{ + Image: "foo/bar:v1.0.0", + ExtraArgs: map[string]string{}, + AdvertiseValidSubnets: []string{"10.0.0.0/8", "192.168.0.0/24"}, + ListenValidSubnets: []string{"10.0.0.0/8"}, + }, + }, + { + name: "default with vip", + etcdConfig: &v1alpha1.EtcdConfig{ + ContainerImage: "foo/bar:v1.0.0", + }, + networkConfig: v1alpha1.NetworkDeviceList{ + { + DeviceInterface: "eth0", + DeviceVIPConfig: &v1alpha1.DeviceVIPConfig{ + SharedIP: "10.0.0.4", + }, + }, + }, + expectedConfig: etcd.ConfigSpec{ + Image: "foo/bar:v1.0.0", + ExtraArgs: map[string]string{}, + AdvertiseValidSubnets: nil, + AdvertiseExcludeSubnets: []string{"10.0.0.4"}, + ListenValidSubnets: nil, + }, + }, + { + name: "advertised with vip", + etcdConfig: &v1alpha1.EtcdConfig{ + ContainerImage: "foo/bar:v1.0.0", + EtcdAdvertisedSubnets: []string{"10.0.0.0/8", "192.168.0.0/24"}, + }, + networkConfig: v1alpha1.NetworkDeviceList{ + { + DeviceInterface: "eth0", + DeviceVIPConfig: &v1alpha1.DeviceVIPConfig{ + SharedIP: "10.0.0.4", + }, + }, + }, + expectedConfig: etcd.ConfigSpec{ + Image: "foo/bar:v1.0.0", + ExtraArgs: map[string]string{}, + AdvertiseValidSubnets: []string{"10.0.0.0/8", "192.168.0.0/24"}, + AdvertiseExcludeSubnets: []string{"10.0.0.4"}, + ListenValidSubnets: []string{"10.0.0.0/8", "192.168.0.0/24"}, + }, + }, + } { + suite.Run(tt.name, func() { + cfg := container.NewV1Alpha1(&v1alpha1.Config{ + ClusterConfig: &v1alpha1.ClusterConfig{ + EtcdConfig: tt.etcdConfig, + }, + MachineConfig: &v1alpha1.MachineConfig{ + MachineType: "controlplane", + MachineNetwork: &v1alpha1.NetworkConfig{ + NetworkInterfaces: tt.networkConfig, + }, + }, + }) + + machineConfig := config.NewMachineConfig(cfg) + suite.Require().NoError(suite.State().Create(suite.Ctx(), machineConfig)) + + suite.AssertWithin(3*time.Second, 100*time.Millisecond, ctest.WrapRetry(func(assert *assert.Assertions, require *require.Assertions) { + etcdConfig, err := safe.StateGet[*etcd.Config](suite.Ctx(), suite.State(), etcd.NewConfig(etcd.NamespaceName, etcd.ConfigID).Metadata()) + if err != nil { + assert.NoError(err) + + return + } + + assert.Equal(tt.expectedConfig, *etcdConfig.TypedSpec()) + })) + + suite.Require().NoError(suite.State().Destroy(suite.Ctx(), machineConfig.Metadata())) + }) + } +} diff --git a/internal/app/machined/pkg/controllers/etcd/etcd.go b/internal/app/machined/pkg/controllers/etcd/etcd.go new file mode 100644 index 0000000..0c3a69e --- /dev/null +++ b/internal/app/machined/pkg/controllers/etcd/etcd.go @@ -0,0 +1,6 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package etcd provides controllers which manage etcd resources. +package etcd diff --git a/internal/app/machined/pkg/controllers/etcd/member.go b/internal/app/machined/pkg/controllers/etcd/member.go new file mode 100644 index 0000000..c2065bb --- /dev/null +++ b/internal/app/machined/pkg/controllers/etcd/member.go @@ -0,0 +1,116 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package etcd + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + pkgetcd "github.com/aenix-io/talm/internal/pkg/etcd" + "github.com/siderolabs/talos/pkg/machinery/resources/etcd" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +// MemberController updates information about the local etcd member. +type MemberController struct { + GetLocalMemberIDFunc func(ctx context.Context) (uint64, error) +} + +// Name implements controller.Controller interface. +func (ctrl *MemberController) Name() string { + return "etcd.MemberController" +} + +const etcdServiceID = "etcd" + +// Inputs implements controller.Controller interface. +func (ctrl *MemberController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: v1alpha1.NamespaceName, + Type: v1alpha1.ServiceType, + ID: optional.Some(etcdServiceID), + Kind: controller.InputStrong, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *MemberController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: etcd.MemberType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *MemberController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + m := etcd.NewMember(etcd.NamespaceName, etcd.LocalMemberID) + + etcdService, err := safe.ReaderGet[*v1alpha1.Service](ctx, r, v1alpha1.NewService(etcdServiceID).Metadata()) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error getting etcd service resource: %w", err) + } + + updateMemberID := etcdService != nil && etcdService.Metadata().Phase() == resource.PhaseRunning && etcdService.TypedSpec().Healthy + + if updateMemberID { + var memberID uint64 + + memberID, err = ctrl.getLocalMemberID(ctx) + if err != nil { + return fmt.Errorf("error getting etcd local member ID: %w", err) + } + + if err = safe.WriterModify(ctx, r, m, func(status *etcd.Member) error { + status.TypedSpec().MemberID = etcd.FormatMemberID(memberID) + + return nil + }); err != nil { + return fmt.Errorf("error updating etcd member resource: %w", err) + } + } else { + if err = r.Destroy(ctx, m.Metadata()); err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error destroying etcd member resource: %w", err) + } + } + + r.ResetRestartBackoff() + } +} + +// getLocalMemberID gets the etcd member ID of the local node. +func (ctrl *MemberController) getLocalMemberID(ctx context.Context) (uint64, error) { + if ctrl.GetLocalMemberIDFunc != nil { + return ctrl.GetLocalMemberIDFunc(ctx) + } + + client, err := pkgetcd.NewLocalClient(ctx) + if err != nil { + return 0, err + } + + defer client.Close() //nolint:errcheck + + return client.GetMemberID(ctx) +} diff --git a/internal/app/machined/pkg/controllers/etcd/member_test.go b/internal/app/machined/pkg/controllers/etcd/member_test.go new file mode 100644 index 0000000..6d60ec5 --- /dev/null +++ b/internal/app/machined/pkg/controllers/etcd/member_test.go @@ -0,0 +1,149 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package etcd_test + +import ( + "context" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + etcdctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/etcd" + "github.com/siderolabs/talos/pkg/machinery/resources/etcd" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +func TestMemberSuite(t *testing.T) { + t.Parallel() + + ctrl := &etcdctrl.MemberController{} + + suite.Run(t, &MemberSuite{ + ctrl: ctrl, + DefaultSuite: ctest.DefaultSuite{ + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(ctrl)) + }, + }, + }) +} + +type MemberSuite struct { + ctest.DefaultSuite + + ctrl *etcdctrl.MemberController +} + +func (suite *MemberSuite) assertEtcdMember(member *etcd.Member) func() error { + return func() error { + r, err := suite.State().Get(suite.Ctx(), member.Metadata()) + if err != nil { + if state.IsNotFoundError(err) { + return retry.ExpectedError(err) + } + + return err + } + + spec := r.(*etcd.Member).TypedSpec() + expectedSpec := member.TypedSpec() + + suite.Require().Equal(expectedSpec.MemberID, spec.MemberID) + + return nil + } +} + +func (suite *MemberSuite) assertInexistentEtcdMember(member *etcd.Member) func() error { + return func() error { + _, err := suite.State().Get(suite.Ctx(), member.Metadata()) + if err != nil { + if state.IsNotFoundError(err) { + return nil + } + + return retry.ExpectedError(err) + } + + return retry.ExpectedErrorf("should not exist") + } +} + +func (suite *MemberSuite) TestEtcdRunning() { + // given + suite.ctrl.GetLocalMemberIDFunc = func(ctx context.Context) (uint64, error) { + return 123, nil + } + etcdService := v1alpha1.NewService("etcd") + etcdService.TypedSpec().Running = true + etcdService.TypedSpec().Healthy = true + + // when + suite.Require().NoError(suite.State().Create(suite.Ctx(), etcdService)) + + // then + expectedMember := etcd.NewMember(etcd.NamespaceName, etcd.LocalMemberID) + expectedMember.TypedSpec().MemberID = "000000000000007b" + + suite.Assert().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertEtcdMember(expectedMember), + ), + ) +} + +func (suite *MemberSuite) TestEtcdNotRunning() { + // given + suite.ctrl.GetLocalMemberIDFunc = func(ctx context.Context) (uint64, error) { + return 123, nil + } + etcdService := v1alpha1.NewService("etcd") + etcdService.TypedSpec().Running = false + + // when + suite.Require().NoError(suite.State().Create(suite.Ctx(), etcdService)) + + // then + expectedMember := etcd.NewMember(etcd.NamespaceName, etcd.LocalMemberID) + expectedMember.TypedSpec().MemberID = "" + + suite.Assert().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertInexistentEtcdMember(expectedMember), + ), + ) +} + +func (suite *MemberSuite) TestCleanup() { + // given + suite.ctrl.GetLocalMemberIDFunc = func(ctx context.Context) (uint64, error) { + return 123, nil + } + etcdService := v1alpha1.NewService("etcd") + etcdService.TypedSpec().Running = true + etcdService.TypedSpec().Healthy = true + + expectedMember := etcd.NewMember(etcd.NamespaceName, etcd.LocalMemberID) + expectedMember.TypedSpec().MemberID = "000000000000007b" + + suite.Require().NoError(suite.State().Create(suite.Ctx(), etcdService)) + + suite.Assert().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertEtcdMember(expectedMember), + ), + ) + + // when + okToDestroy, err := suite.State().Teardown(suite.Ctx(), etcdService.Metadata()) + suite.Require().NoError(err) + suite.Require().True(okToDestroy) + + // then + suite.Assert().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertInexistentEtcdMember(expectedMember), + )) +} diff --git a/internal/app/machined/pkg/controllers/etcd/pki.go b/internal/app/machined/pkg/controllers/etcd/pki.go new file mode 100644 index 0000000..98f240f --- /dev/null +++ b/internal/app/machined/pkg/controllers/etcd/pki.go @@ -0,0 +1,150 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package etcd + +import ( + "context" + "fmt" + "os" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/crypto/x509" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/filetree" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/etcd" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" +) + +// PKIController renders manifests based on templates and config/secrets. +type PKIController struct{} + +// Name implements controller.Controller interface. +func (ctrl *PKIController) Name() string { + return "etcd.PKIController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *PKIController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: secrets.NamespaceName, + Type: secrets.EtcdRootType, + ID: optional.Some(secrets.EtcdRootID), + Kind: controller.InputWeak, + }, + { + Namespace: secrets.NamespaceName, + Type: secrets.EtcdType, + ID: optional.Some(secrets.EtcdID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *PKIController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: etcd.PKIStatusType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *PKIController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + rootScrts, err := safe.ReaderGet[*secrets.EtcdRoot](ctx, r, resource.NewMetadata(secrets.NamespaceName, secrets.EtcdRootType, secrets.EtcdRootID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting root secrets: %w", err) + } + + scrts, err := safe.ReaderGet[*secrets.Etcd](ctx, r, resource.NewMetadata(secrets.NamespaceName, secrets.EtcdType, secrets.EtcdID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting secrets: %w", err) + } + + if err = os.MkdirAll(constants.EtcdPKIPath, 0o700); err != nil { + return err + } + + if err = os.WriteFile(constants.EtcdCACert, rootScrts.TypedSpec().EtcdCA.Crt, 0o400); err != nil { + return fmt.Errorf("failed to write CA certificate: %w", err) + } + + if err = os.WriteFile(constants.EtcdCAKey, rootScrts.TypedSpec().EtcdCA.Key, 0o400); err != nil { + return fmt.Errorf("failed to write CA key: %w", err) + } + + etcdCerts := scrts.TypedSpec() + + for _, keypair := range []struct { + getter func() *x509.PEMEncodedCertificateAndKey + keyPath string + certPath string + }{ + { + getter: func() *x509.PEMEncodedCertificateAndKey { return etcdCerts.Etcd }, + keyPath: constants.EtcdKey, + certPath: constants.EtcdCert, + }, + { + getter: func() *x509.PEMEncodedCertificateAndKey { return etcdCerts.EtcdPeer }, + keyPath: constants.EtcdPeerKey, + certPath: constants.EtcdPeerCert, + }, + { + getter: func() *x509.PEMEncodedCertificateAndKey { return etcdCerts.EtcdAdmin }, + keyPath: constants.EtcdAdminKey, + certPath: constants.EtcdAdminCert, + }, + } { + if err = os.WriteFile(keypair.keyPath, keypair.getter().Key, 0o400); err != nil { + return err + } + + if err = os.WriteFile(keypair.certPath, keypair.getter().Crt, 0o400); err != nil { + return err + } + } + + if err = filetree.ChownRecursive(constants.EtcdPKIPath, constants.EtcdUserID, constants.EtcdUserID); err != nil { + return err + } + + if err = safe.WriterModify(ctx, r, etcd.NewPKIStatus(etcd.NamespaceName, etcd.PKIID), func(status *etcd.PKIStatus) error { + status.TypedSpec().Ready = true + status.TypedSpec().Version = scrts.Metadata().Version().String() + + return nil + }); err != nil { + return fmt.Errorf("error updating PKI status: %w", err) + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/etcd/spec.go b/internal/app/machined/pkg/controllers/etcd/spec.go new file mode 100644 index 0000000..bde9b88 --- /dev/null +++ b/internal/app/machined/pkg/controllers/etcd/spec.go @@ -0,0 +1,221 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package etcd + +import ( + "context" + "fmt" + "net/netip" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/net" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/etcd" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// SpecController renders manifests based on templates and Spec/secrets. +type SpecController struct{} + +// Name implements controller.Controller interface. +func (ctrl *SpecController) Name() string { + return "etcd.SpecController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *SpecController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: etcd.NamespaceName, + Type: etcd.ConfigType, + ID: optional.Some(etcd.ConfigID), + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.HostnameStatusType, + ID: optional.Some(network.HostnameID), + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.NodeAddressType, + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *SpecController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: etcd.SpecType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *SpecController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + etcdConfig, err := safe.ReaderGet[*etcd.Config](ctx, r, resource.NewMetadata(etcd.NamespaceName, etcd.ConfigType, etcd.ConfigID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting etcd config: %w", err) + } + + hostnameStatus, err := safe.ReaderGet[*network.HostnameStatus](ctx, r, resource.NewMetadata(network.NamespaceName, network.HostnameStatusType, network.HostnameID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting hostname status: %w", err) + } + + nodeRoutedAddrs, err := safe.ReaderGet[*network.NodeAddress]( + ctx, + r, + resource.NewMetadata( + network.NamespaceName, + network.NodeAddressType, + network.FilteredNodeAddressID(network.NodeAddressRoutedID, k8s.NodeAddressFilterNoK8s), + resource.VersionUndefined, + ), + ) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting addresses: %w", err) + } + + nodeCurrentAddrs, err := safe.ReaderGet[*network.NodeAddress]( + ctx, + r, + resource.NewMetadata( + network.NamespaceName, + network.NodeAddressType, + network.FilteredNodeAddressID(network.NodeAddressCurrentID, k8s.NodeAddressFilterNoK8s), + resource.VersionUndefined, + ), + ) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting addresses: %w", err) + } + + routedAddrs := nodeRoutedAddrs.TypedSpec().IPs() + currentAddrs := nodeCurrentAddrs.TypedSpec().IPs() + + // need at least a single address + if len(routedAddrs) == 0 { + continue + } + + advertiseValidSubnets := etcdConfig.TypedSpec().AdvertiseValidSubnets + + if len(advertiseValidSubnets) == 0 { + // not specified, advertise all addresses + advertiseValidSubnets = []string{"0.0.0.0/0", "::/0"} + } + + advertisedCIDRs := make([]string, 0, len(advertiseValidSubnets)+len(etcdConfig.TypedSpec().AdvertiseExcludeSubnets)) + advertisedCIDRs = append(advertisedCIDRs, advertiseValidSubnets...) + advertisedCIDRs = append(advertisedCIDRs, xslices.Map(etcdConfig.TypedSpec().AdvertiseExcludeSubnets, func(cidr string) string { return "!" + cidr })...) + + listenCIDRs := make([]string, 0, len(etcdConfig.TypedSpec().ListenValidSubnets)+len(etcdConfig.TypedSpec().ListenExcludeSubnets)) + listenCIDRs = append(listenCIDRs, etcdConfig.TypedSpec().ListenValidSubnets...) + listenCIDRs = append(listenCIDRs, xslices.Map(etcdConfig.TypedSpec().ListenExcludeSubnets, func(cidr string) string { return "!" + cidr })...) + + defaultListenAddress := netip.AddrFrom4([4]byte{0, 0, 0, 0}) + loopbackAddress := netip.AddrFrom4([4]byte{127, 0, 0, 1}) + + for _, ip := range routedAddrs { + if ip.Is6() { + defaultListenAddress = netip.IPv6Unspecified() + loopbackAddress = netip.MustParseAddr("::1") + + break + } + } + + var ( + advertisedIPs []netip.Addr + listenPeerIPs []netip.Addr + listenClientIPs []netip.Addr + ) + + if len(etcdConfig.TypedSpec().AdvertiseValidSubnets) == 0 { + advertisedIPs, err = net.FilterIPs(routedAddrs, advertisedCIDRs) + if err != nil { + return fmt.Errorf("error filtering IPs: %w", err) + } + + // if advertise subnet is not set, advertise the first address + if len(advertisedIPs) > 0 { + advertisedIPs = advertisedIPs[:1] + } + } else { + advertisedIPs, err = net.FilterIPs(currentAddrs, advertisedCIDRs) + if err != nil { + return fmt.Errorf("error filtering IPs: %w", err) + } + } + + if len(listenCIDRs) > 0 { + listenPeerIPs, err = net.FilterIPs(routedAddrs, listenCIDRs) + if err != nil { + return fmt.Errorf("error filtering IPs: %w", err) + } + + listenClientIPs = append([]netip.Addr{loopbackAddress}, listenPeerIPs...) + } else { + listenPeerIPs = []netip.Addr{defaultListenAddress} + listenClientIPs = []netip.Addr{defaultListenAddress} + } + + if len(advertisedIPs) == 0 || len(listenPeerIPs) == 0 { + continue + } + + if err = safe.WriterModify(ctx, r, etcd.NewSpec(etcd.NamespaceName, etcd.SpecID), func(status *etcd.Spec) error { + status.TypedSpec().AdvertisedAddresses = advertisedIPs + status.TypedSpec().ListenClientAddresses = listenClientIPs + status.TypedSpec().ListenPeerAddresses = listenPeerIPs + status.TypedSpec().Name = hostnameStatus.TypedSpec().Hostname + status.TypedSpec().Image = etcdConfig.TypedSpec().Image + status.TypedSpec().ExtraArgs = etcdConfig.TypedSpec().ExtraArgs + + return nil + }); err != nil { + return fmt.Errorf("error updating Spec status: %w", err) + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/etcd/spec_test.go b/internal/app/machined/pkg/controllers/etcd/spec_test.go new file mode 100644 index 0000000..0ec5bad --- /dev/null +++ b/internal/app/machined/pkg/controllers/etcd/spec_test.go @@ -0,0 +1,220 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package etcd_test + +import ( + "net/netip" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + etcdctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/etcd" + "github.com/siderolabs/talos/pkg/machinery/resources/etcd" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +func TestSpecSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, &SpecSuite{ + DefaultSuite: ctest.DefaultSuite{ + Timeout: 3 * time.Second, + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(&etcdctrl.SpecController{})) + }, + }, + }) +} + +type SpecSuite struct { + ctest.DefaultSuite +} + +func (suite *SpecSuite) TestReconcile() { + hostnameStatus := network.NewHostnameStatus(network.NamespaceName, network.HostnameID) + hostnameStatus.TypedSpec().Hostname = "worker1" + hostnameStatus.TypedSpec().Domainname = "some.domain" + suite.Require().NoError(suite.State().Create(suite.Ctx(), hostnameStatus)) + + routedAddresses := network.NewNodeAddress( + network.NamespaceName, + network.FilteredNodeAddressID(network.NodeAddressRoutedID, k8s.NodeAddressFilterNoK8s), + ) + + routedAddresses.TypedSpec().Addresses = []netip.Prefix{ + netip.MustParsePrefix("10.0.0.5/24"), + netip.MustParsePrefix("192.168.1.1/24"), + netip.MustParsePrefix("192.168.1.50/32"), + netip.MustParsePrefix("2001:0db8:85a3:0000:0000:8a2e:0370:7334/64"), + netip.MustParsePrefix("2002:0db8:85a3:0000:0000:8a2e:0370:7335/64"), + } + + suite.Require().NoError(suite.State().Create(suite.Ctx(), routedAddresses)) + + currentAddrs := network.NewNodeAddress( + network.NamespaceName, + network.FilteredNodeAddressID(network.NodeAddressCurrentID, k8s.NodeAddressFilterNoK8s), + ) + + currentAddrs.TypedSpec().Addresses = append( + []netip.Prefix{netip.MustParsePrefix("1.3.5.7/32")}, + routedAddresses.TypedSpec().Addresses..., + ) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), currentAddrs)) + + for _, tt := range []struct { + name string + cfg etcd.ConfigSpec + expected etcd.SpecSpec + }{ + { + name: "defaults", + cfg: etcd.ConfigSpec{ + Image: "foo/bar:v1.0.0", + ExtraArgs: map[string]string{ + "arg": "value", + }, + }, + expected: etcd.SpecSpec{ + Name: "worker1", + Image: "foo/bar:v1.0.0", + ExtraArgs: map[string]string{ + "arg": "value", + }, + AdvertisedAddresses: []netip.Addr{ + netip.MustParseAddr("10.0.0.5"), + }, + ListenPeerAddresses: []netip.Addr{ + netip.IPv6Unspecified(), + }, + ListenClientAddresses: []netip.Addr{ + netip.IPv6Unspecified(), + }, + }, + }, + { + name: "defaults with exclude", + cfg: etcd.ConfigSpec{ + Image: "foo/bar:v1.0.0", + AdvertiseExcludeSubnets: []string{ + "10.0.0.5", + }, + }, + expected: etcd.SpecSpec{ + Name: "worker1", + Image: "foo/bar:v1.0.0", + AdvertisedAddresses: []netip.Addr{ + netip.MustParseAddr("192.168.1.1"), + }, + ListenPeerAddresses: []netip.Addr{ + netip.IPv6Unspecified(), + }, + ListenClientAddresses: []netip.Addr{ + netip.IPv6Unspecified(), + }, + }, + }, + { + name: "only advertised", + cfg: etcd.ConfigSpec{ + Image: "foo/bar:v1.0.0", + AdvertiseValidSubnets: []string{ + "192.168.0.0/16", + "1.3.5.7/32", + }, + }, + expected: etcd.SpecSpec{ + Name: "worker1", + Image: "foo/bar:v1.0.0", + AdvertisedAddresses: []netip.Addr{ + netip.MustParseAddr("192.168.1.1"), + netip.MustParseAddr("192.168.1.50"), + netip.MustParseAddr("1.3.5.7"), + }, + ListenPeerAddresses: []netip.Addr{ + netip.IPv6Unspecified(), + }, + ListenClientAddresses: []netip.Addr{ + netip.IPv6Unspecified(), + }, + }, + }, + { + name: "only advertised with exclude", + cfg: etcd.ConfigSpec{ + Image: "foo/bar:v1.0.0", + AdvertiseValidSubnets: []string{ + "192.168.0.0/16", + }, + AdvertiseExcludeSubnets: []string{ + "10.0.0.5", + "192.168.1.50", + }, + }, + expected: etcd.SpecSpec{ + Name: "worker1", + Image: "foo/bar:v1.0.0", + AdvertisedAddresses: []netip.Addr{ + netip.MustParseAddr("192.168.1.1"), + }, + ListenPeerAddresses: []netip.Addr{ + netip.IPv6Unspecified(), + }, + ListenClientAddresses: []netip.Addr{ + netip.IPv6Unspecified(), + }, + }, + }, + { + name: "advertised and listen", + cfg: etcd.ConfigSpec{ + Image: "foo/bar:v1.0.0", + AdvertiseValidSubnets: []string{ + "192.168.0.0/16", + "2001::/16", + }, + ListenValidSubnets: []string{ + "192.168.0.0/16", + }, + }, + expected: etcd.SpecSpec{ + Name: "worker1", + Image: "foo/bar:v1.0.0", + AdvertisedAddresses: []netip.Addr{ + netip.MustParseAddr("192.168.1.1"), + netip.MustParseAddr("192.168.1.50"), + netip.MustParseAddr("2001:0db8:85a3:0000:0000:8a2e:0370:7334"), + }, + ListenPeerAddresses: []netip.Addr{ + netip.MustParseAddr("192.168.1.1"), + netip.MustParseAddr("192.168.1.50"), + }, + ListenClientAddresses: []netip.Addr{ + netip.MustParseAddr("::1"), + netip.MustParseAddr("192.168.1.1"), + netip.MustParseAddr("192.168.1.50"), + }, + }, + }, + } { + suite.Run(tt.name, func() { + etcdConfig := etcd.NewConfig(etcd.NamespaceName, etcd.ConfigID) + *etcdConfig.TypedSpec() = tt.cfg + + suite.Require().NoError(suite.State().Create(suite.Ctx(), etcdConfig)) + + ctest.AssertResource(suite, etcd.SpecID, func(etcdSpec *etcd.Spec, asrt *assert.Assertions) { + asrt.Equal(tt.expected, *etcdSpec.TypedSpec(), "spec %v", *etcdSpec.TypedSpec()) + }) + + suite.Require().NoError(suite.State().Destroy(suite.Ctx(), etcdConfig.Metadata())) + }) + } +} diff --git a/internal/app/machined/pkg/controllers/files/cri_config_parts.go b/internal/app/machined/pkg/controllers/files/cri_config_parts.go new file mode 100644 index 0000000..d04ebf9 --- /dev/null +++ b/internal/app/machined/pkg/controllers/files/cri_config_parts.go @@ -0,0 +1,94 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package files + +import ( + "context" + "fmt" + "path/filepath" + "sort" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "go.uber.org/zap" + + "github.com/aenix-io/talm/internal/pkg/toml" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/files" +) + +// CRIConfigPartsController merges parts of the CRI config from /etc/cri/conf.d/*.part into final /etc/cri/conf.d/cri.toml. +type CRIConfigPartsController struct { + // Path to /etc/cri/conf.d directory. + CRIConfdPath string +} + +// Name implements controller.Controller interface. +func (ctrl *CRIConfigPartsController) Name() string { + return "files.CRIConfigPartsController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *CRIConfigPartsController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: files.NamespaceName, + Type: files.EtcFileStatusType, + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *CRIConfigPartsController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: files.EtcFileSpecType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +func (ctrl *CRIConfigPartsController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + if ctrl.CRIConfdPath == "" { + ctrl.CRIConfdPath = constants.CRIConfdPath + } + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + // scan conf.d directory for config parts and merge them together into final configuration + parts, err := filepath.Glob(filepath.Join(ctrl.CRIConfdPath, "*.part")) + if err != nil { + return err + } + + sort.Strings(parts) + + out, err := toml.Merge(parts) + if err != nil { + return err + } + + if err := r.Modify(ctx, files.NewEtcFileSpec(files.NamespaceName, constants.CRIConfig), + func(r resource.Resource) error { + spec := r.(*files.EtcFileSpec).TypedSpec() + + spec.Contents = out + spec.Mode = 0o600 + + return nil + }); err != nil { + return fmt.Errorf("error modifying resource: %w", err) + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/files/cri_registry_config.go b/internal/app/machined/pkg/controllers/files/cri_registry_config.go new file mode 100644 index 0000000..cd135ff --- /dev/null +++ b/internal/app/machined/pkg/controllers/files/cri_registry_config.go @@ -0,0 +1,201 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package files + +import ( + "bytes" + "context" + "fmt" + "io/fs" + "os" + "path/filepath" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/xslices" + "go.uber.org/zap" + "golang.org/x/sys/unix" + + "github.com/aenix-io/talm/internal/pkg/containers/cri/containerd" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/files" +) + +// CRIRegistryConfigController generates parts of the CRI config for registry configuration. +type CRIRegistryConfigController struct { + bindMountCreated bool +} + +// Name implements controller.Controller interface. +func (ctrl *CRIRegistryConfigController) Name() string { + return "files.CRIRegistryConfigController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *CRIRegistryConfigController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: config.MachineConfigType, + ID: optional.Some(config.V1Alpha1ID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *CRIRegistryConfigController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: files.EtcFileSpecType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *CRIRegistryConfigController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + basePath := filepath.Join(constants.CRIConfdPath, "hosts") + shadowPath := filepath.Join(constants.SystemPath, basePath) + + // bind mount shadow path over to base path + // shadow path is writeable, controller is going to update it + // base path is read-only, containerd will read from it + if !ctrl.bindMountCreated { + // create shadow path + if err := os.MkdirAll(shadowPath, 0o700); err != nil { + return err + } + + if err := unix.Mount(shadowPath, basePath, "", unix.MS_BIND|unix.MS_RDONLY, ""); err != nil { + return fmt.Errorf("failed to create bind mount for %s -> %s: %w", shadowPath, basePath, err) + } + + ctrl.bindMountCreated = true + } + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + cfg, err := safe.ReaderGetByID[*config.MachineConfig](ctx, r, config.V1Alpha1ID) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error getting config: %w", err) + } + + var ( + criRegistryContents []byte + criHosts *containerd.HostsConfig + ) + + if cfg != nil && cfg.Config().Machine() != nil { + criRegistryContents, err = containerd.GenerateCRIConfig(cfg.Config().Machine().Registries()) + if err != nil { + return err + } + + criHosts, err = containerd.GenerateHosts(cfg.Config().Machine().Registries(), basePath) + if err != nil { + return err + } + } else { + criHosts = &containerd.HostsConfig{} + } + + if err := r.Modify(ctx, files.NewEtcFileSpec(files.NamespaceName, constants.CRIRegistryConfigPart), + func(r resource.Resource) error { + spec := r.(*files.EtcFileSpec).TypedSpec() + + spec.Contents = criRegistryContents + spec.Mode = 0o600 + + return nil + }); err != nil { + return fmt.Errorf("error modifying resource: %w", err) + } + + if err := ctrl.syncHosts(shadowPath, criHosts); err != nil { + return fmt.Errorf("error syncing hosts: %w", err) + } + + r.ResetRestartBackoff() + } +} + +//nolint:gocyclo +func (ctrl *CRIRegistryConfigController) syncHosts(shadowPath string, criHosts *containerd.HostsConfig) error { + // 1. create/update all files and directories + for dirName, directory := range criHosts.Directories { + path := filepath.Join(shadowPath, dirName) + + if err := os.MkdirAll(path, 0o700); err != nil { + return err + } + + for _, file := range directory.Files { + // match contents to see if the update can be skipped + contents, err := os.ReadFile(filepath.Join(path, file.Name)) + if err == nil && bytes.Equal(contents, file.Contents) { + continue + } + + // write file + if err = os.WriteFile(filepath.Join(path, file.Name), file.Contents, file.Mode); err != nil { + return err + } + } + + // remove any files which shouldn't be present + fileList, err := os.ReadDir(path) + if err != nil { + return err + } + + fileListMap := xslices.ToSetFunc(fileList, fs.DirEntry.Name) + + for _, file := range directory.Files { + delete(fileListMap, file.Name) + } + + for file := range fileListMap { + if err = os.Remove(filepath.Join(path, file)); err != nil { + return err + } + } + } + + // 2. remove any directories which shouldn't be present + directoryList, err := os.ReadDir(shadowPath) + if err != nil { + return err + } + + directoryListMap := make(map[string]struct{}, len(directoryList)) + + for _, dir := range directoryList { + directoryListMap[dir.Name()] = struct{}{} + } + + for dirName := range criHosts.Directories { + delete(directoryListMap, dirName) + } + + for dirName := range directoryListMap { + if err = os.RemoveAll(filepath.Join(shadowPath, dirName)); err != nil { + return err + } + } + + return nil +} diff --git a/internal/app/machined/pkg/controllers/files/etcfile.go b/internal/app/machined/pkg/controllers/files/etcfile.go new file mode 100644 index 0000000..5a11f10 --- /dev/null +++ b/internal/app/machined/pkg/controllers/files/etcfile.go @@ -0,0 +1,204 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package files + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "go.uber.org/zap" + "golang.org/x/sys/unix" + + "github.com/siderolabs/talos/pkg/machinery/resources/files" +) + +// EtcFileController watches EtcFileSpecs, creates/updates files. +type EtcFileController struct { + // Path to /etc directory, read-only filesystem. + EtcPath string + // Shadow path where actual file will be created and bind mounted into EtcdPath. + ShadowPath string + + // Cache of bind mounts created. + bindMounts map[string]interface{} +} + +// Name implements controller.Controller interface. +func (ctrl *EtcFileController) Name() string { + return "files.EtcFileController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *EtcFileController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: files.NamespaceName, + Type: files.EtcFileSpecType, + Kind: controller.InputStrong, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *EtcFileController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: files.EtcFileStatusType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *EtcFileController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + if ctrl.bindMounts == nil { + ctrl.bindMounts = make(map[string]interface{}) + } + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + list, err := r.List(ctx, resource.NewMetadata(files.NamespaceName, files.EtcFileSpecType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing specs: %w", err) + } + + // add finalizers for all live resources + for _, res := range list.Items { + if res.Metadata().Phase() != resource.PhaseRunning { + continue + } + + if err = r.AddFinalizer(ctx, res.Metadata(), ctrl.Name()); err != nil { + return fmt.Errorf("error adding finalizer: %w", err) + } + } + + touchedIDs := make(map[resource.ID]struct{}) + + for _, item := range list.Items { + spec := item.(*files.EtcFileSpec) //nolint:errcheck,forcetypeassert + filename := spec.Metadata().ID() + _, mountExists := ctrl.bindMounts[filename] + + src := filepath.Join(ctrl.ShadowPath, filename) + dst := filepath.Join(ctrl.EtcPath, filename) + + switch spec.Metadata().Phase() { + case resource.PhaseTearingDown: + if mountExists { + logger.Debug("removing bind mount", zap.String("src", src), zap.String("dst", dst)) + + if err = unix.Unmount(dst, 0); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("failed to unmount bind mount %q: %w", dst, err) + } + + delete(ctrl.bindMounts, filename) + } + + logger.Debug("removing file", zap.String("src", src)) + + if err = os.Remove(src); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("failed to remove %q: %w", src, err) + } + + // now remove finalizer as the link was deleted + if err = r.RemoveFinalizer(ctx, spec.Metadata(), ctrl.Name()); err != nil { + return fmt.Errorf("error removing finalizer: %w", err) + } + case resource.PhaseRunning: + if !mountExists { + logger.Debug("creating bind mount", zap.String("src", src), zap.String("dst", dst)) + + if err = createBindMount(src, dst, spec.TypedSpec().Mode); err != nil { + return fmt.Errorf("failed to create shadow bind mount %q -> %q: %w", src, dst, err) + } + + ctrl.bindMounts[filename] = struct{}{} + } + + logger.Debug("writing file contents", zap.String("dst", dst), zap.Stringer("version", spec.Metadata().Version())) + + if err = UpdateFile(dst, spec.TypedSpec().Contents, spec.TypedSpec().Mode); err != nil { + return fmt.Errorf("error updating %q: %w", dst, err) + } + + if err = r.Modify(ctx, files.NewEtcFileStatus(files.NamespaceName, filename), func(r resource.Resource) error { + r.(*files.EtcFileStatus).TypedSpec().SpecVersion = spec.Metadata().Version().String() + + return nil + }); err != nil { + return fmt.Errorf("error updating status: %w", err) + } + + touchedIDs[filename] = struct{}{} + } + } + + // list statuses for cleanup + list, err = r.List(ctx, resource.NewMetadata(files.NamespaceName, files.EtcFileStatusType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing resources: %w", err) + } + + for _, res := range list.Items { + if _, ok := touchedIDs[res.Metadata().ID()]; !ok { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return fmt.Errorf("error cleaning up specs: %w", err) + } + } + } + + r.ResetRestartBackoff() + } +} + +// createBindMount creates a common way to create a writable source file with a +// bind mounted destination. This is most commonly used for well known files +// under /etc that need to be adjusted during startup. +func createBindMount(src, dst string, mode os.FileMode) (err error) { + if err = os.MkdirAll(filepath.Dir(src), 0o755); err != nil { + return err + } + + var f *os.File + + if f, err = os.OpenFile(src, os.O_WRONLY|os.O_CREATE, mode); err != nil { + return err + } + + if err = f.Close(); err != nil { + return err + } + + if err = unix.Mount(src, dst, "", unix.MS_BIND|unix.MS_RDONLY, ""); err != nil { + return fmt.Errorf("failed to create bind mount for %s: %w", dst, err) + } + + return nil +} + +// UpdateFile is like `os.WriteFile`, but it will only update the file if the +// contents have changed. +func UpdateFile(filename string, contents []byte, mode os.FileMode) error { + oldContents, err := os.ReadFile(filename) + if err == nil && bytes.Equal(oldContents, contents) { + return nil + } + + return os.WriteFile(filename, contents, mode) +} diff --git a/internal/app/machined/pkg/controllers/files/etcfile_test.go b/internal/app/machined/pkg/controllers/files/etcfile_test.go new file mode 100644 index 0000000..5ed6fec --- /dev/null +++ b/internal/app/machined/pkg/controllers/files/etcfile_test.go @@ -0,0 +1,162 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package files_test + +import ( + "context" + "log" + "os" + "path/filepath" + "strconv" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + filesctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/files" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/resources/files" +) + +type EtcFileSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc + + etcPath string + shadowPath string +} + +func (suite *EtcFileSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.startRuntime() + + suite.etcPath = suite.T().TempDir() + suite.shadowPath = suite.T().TempDir() + + suite.Require().NoError( + suite.runtime.RegisterController( + &filesctrl.EtcFileController{ + EtcPath: suite.etcPath, + ShadowPath: suite.shadowPath, + }, + ), + ) +} + +func (suite *EtcFileSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *EtcFileSuite) assertEtcFile(filename, contents string, expectedVersion resource.Version) error { + b, err := os.ReadFile(filepath.Join(suite.etcPath, filename)) + if err != nil { + return retry.ExpectedError(err) + } + + if string(b) != contents { + return retry.ExpectedErrorf("contents don't match %q != %q", string(b), contents) + } + + r, err := safe.ReaderGet[*files.EtcFileStatus](suite.ctx, suite.state, resource.NewMetadata(files.NamespaceName, files.EtcFileStatusType, filename, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + return retry.ExpectedError(err) + } + + return err + } + + version := r.TypedSpec().SpecVersion + + expected, err := strconv.Atoi(expectedVersion.String()) + suite.Require().NoError(err) + + ver, err := strconv.Atoi(version) + suite.Require().NoError(err) + + if ver < expected { + return retry.ExpectedErrorf("version mismatch %s > %s", expectedVersion, version) + } + + return nil +} + +func (suite *EtcFileSuite) TestFiles() { + etcFileSpec := files.NewEtcFileSpec(files.NamespaceName, "test1") + etcFileSpec.TypedSpec().Contents = []byte("foo") + etcFileSpec.TypedSpec().Mode = 0o644 + + // create "read-only" mock (in Talos it's part of rootfs) + suite.T().Logf("mock created %q", filepath.Join(suite.etcPath, etcFileSpec.Metadata().ID())) + suite.Require().NoError(os.WriteFile(filepath.Join(suite.etcPath, etcFileSpec.Metadata().ID()), nil, 0o644)) + + suite.Require().NoError(suite.state.Create(suite.ctx, etcFileSpec)) + + suite.Assert().NoError( + retry.Constant(5*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertEtcFile("test1", "foo", etcFileSpec.Metadata().Version()) + }, + ), + ) + + for _, r := range []resource.Resource{etcFileSpec} { + for { + ready, err := suite.state.Teardown(suite.ctx, r.Metadata()) + suite.Require().NoError(err) + + if ready { + break + } + + time.Sleep(100 * time.Millisecond) + } + } +} + +func (suite *EtcFileSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestEtcFileSuite(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("requires root") + } + + suite.Run(t, new(EtcFileSuite)) +} diff --git a/internal/app/machined/pkg/controllers/files/files.go b/internal/app/machined/pkg/controllers/files/files.go new file mode 100644 index 0000000..998970e --- /dev/null +++ b/internal/app/machined/pkg/controllers/files/files.go @@ -0,0 +1,6 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package files provides controllers which manage file resources. +package files diff --git a/internal/app/machined/pkg/controllers/hardware/hardware.go b/internal/app/machined/pkg/controllers/hardware/hardware.go new file mode 100644 index 0000000..435d173 --- /dev/null +++ b/internal/app/machined/pkg/controllers/hardware/hardware.go @@ -0,0 +1,6 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package hardware provides the hardware controller implementation. +package hardware diff --git a/internal/app/machined/pkg/controllers/hardware/hardware_test.go b/internal/app/machined/pkg/controllers/hardware/hardware_test.go new file mode 100644 index 0000000..515e9bc --- /dev/null +++ b/internal/app/machined/pkg/controllers/hardware/hardware_test.go @@ -0,0 +1,78 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package hardware_test + +import ( + "context" + "log" + "sync" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + "github.com/siderolabs/talos/pkg/logging" +) + +type HardwareSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *HardwareSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + logger := logging.Wrap(log.Writer()) + + suite.runtime, err = runtime.NewRuntime(suite.state, logger) + suite.Require().NoError(err) +} + +func (suite *HardwareSuite) assertNoResource(md resource.Metadata) func() error { + return func() error { + _, err := suite.state.Get(suite.ctx, md) + if err == nil { + return retry.ExpectedErrorf("resource %s still exists", md) + } + + if state.IsNotFoundError(err) { + return nil + } + + return err + } +} + +func (suite *HardwareSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func (suite *HardwareSuite) State() state.State { + return suite.state +} + +func (suite *HardwareSuite) Ctx() context.Context { + return suite.ctx +} diff --git a/internal/app/machined/pkg/controllers/hardware/system.go b/internal/app/machined/pkg/controllers/hardware/system.go new file mode 100644 index 0000000..a503e6f --- /dev/null +++ b/internal/app/machined/pkg/controllers/hardware/system.go @@ -0,0 +1,156 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package hardware + +import ( + "context" + "fmt" + "strings" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/go-smbios/smbios" + "go.uber.org/zap" + + hwadapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/hardware" + runtimetalos "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/pkg/meta" + pkgSMBIOS "github.com/aenix-io/talm/internal/pkg/smbios" + "github.com/siderolabs/talos/pkg/machinery/resources/hardware" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// SystemInfoController populates CPU information of the underlying hardware. +type SystemInfoController struct { + V1Alpha1Mode runtimetalos.Mode + SMBIOS *smbios.SMBIOS +} + +// Name implements controller.Controller interface. +func (ctrl *SystemInfoController) Name() string { + return "hardware.SystemInfoController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *SystemInfoController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: runtime.NamespaceName, + Type: runtime.MetaKeyType, + Kind: controller.InputWeak, + }, + { + Namespace: runtime.NamespaceName, + Type: runtime.MetaLoadedType, + ID: optional.Some(runtime.MetaLoadedID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *SystemInfoController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: hardware.ProcessorType, + Kind: controller.OutputExclusive, + }, + { + Type: hardware.MemoryModuleType, + Kind: controller.OutputExclusive, + }, + { + Type: hardware.SystemInformationType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *SystemInfoController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + // smbios info is not available inside container, so skip the controller + if ctrl.V1Alpha1Mode == runtimetalos.ModeContainer { + return nil + } + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + _, err := safe.ReaderGetByID[*runtime.MetaLoaded](ctx, r, runtime.MetaLoadedID) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting meta loaded resource: %w", err) + } + + if ctrl.SMBIOS == nil { + var s *smbios.SMBIOS + + s, err = pkgSMBIOS.GetSMBIOSInfo() + if err != nil { + return err + } + + ctrl.SMBIOS = s + } + + uuidRewriteRes, err := safe.ReaderGetByID[*runtime.MetaKey](ctx, r, runtime.MetaKeyTagToID(meta.UUIDOverride)) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error getting meta key resource: %w", err) + } + + var uuidRewrite string + + if uuidRewriteRes != nil && uuidRewriteRes.TypedSpec().Value != "" { + uuidRewrite = uuidRewriteRes.TypedSpec().Value + + logger.Info("using UUID rewrite", zap.String("uuid", uuidRewrite)) + } + + if err := safe.WriterModify(ctx, r, hardware.NewSystemInformation(hardware.SystemInformationID), func(res *hardware.SystemInformation) error { + hwadapter.SystemInformation(res).Update(&ctrl.SMBIOS.SystemInformation, uuidRewrite) + + return nil + }); err != nil { + return fmt.Errorf("error updating objects: %w", err) + } + + for _, p := range ctrl.SMBIOS.ProcessorInformation { + // replaces `CPU 0` with `CPU-0` + id := strings.ReplaceAll(p.SocketDesignation, " ", "-") + + if err := safe.WriterModify(ctx, r, hardware.NewProcessorInfo(id), func(res *hardware.Processor) error { + hwadapter.Processor(res).Update(&p) + + return nil + }); err != nil { + return fmt.Errorf("error updating objects: %w", err) + } + } + + for _, m := range ctrl.SMBIOS.MemoryDevices { + // replaces `SIMM 0` with `SIMM-0` + id := strings.ReplaceAll(m.DeviceLocator, " ", "-") + + if err := safe.WriterModify(ctx, r, hardware.NewMemoryModuleInfo(id), func(res *hardware.MemoryModule) error { + hwadapter.MemoryModule(res).Update(&m) + + return nil + }); err != nil { + return fmt.Errorf("error updating objects: %w", err) + } + } + } +} diff --git a/internal/app/machined/pkg/controllers/hardware/system_test.go b/internal/app/machined/pkg/controllers/hardware/system_test.go new file mode 100644 index 0000000..a67e971 --- /dev/null +++ b/internal/app/machined/pkg/controllers/hardware/system_test.go @@ -0,0 +1,173 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package hardware_test + +import ( + "os" + "testing" + "time" + + "github.com/siderolabs/go-retry/retry" + "github.com/siderolabs/go-smbios/smbios" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + hardwarectrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/hardware" + runtimetalos "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/pkg/meta" + "github.com/siderolabs/talos/pkg/machinery/resources/hardware" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +type SystemInfoSuite struct { + HardwareSuite +} + +func (suite *SystemInfoSuite) TestPopulateSystemInformation() { + stream, err := os.Open("testdata/SuperMicro-Dual-Xeon.dmi") + suite.Require().NoError(err) + + suite.T().Cleanup(func() { suite.NoError(stream.Close()) }) + + version := smbios.Version{Major: 3, Minor: 3, Revision: 0} // dummy version + s, err := smbios.Decode(stream, version) + suite.Require().NoError(err) + + suite.Require().NoError( + suite.runtime.RegisterController( + &hardwarectrl.SystemInfoController{ + SMBIOS: s, + }, + ), + ) + + suite.startRuntime() + + suite.Require().NoError(suite.state.Create(suite.ctx, runtime.NewMetaLoaded())) + + cpuSpecs := map[string]hardware.ProcessorSpec{ + "CPU-1": { + Socket: "CPU 1", + Manufacturer: "Intel", + ProductName: "Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz", + MaxSpeed: 4000, + BootSpeed: 2600, + Status: 65, + AssetTag: "3A65E8E29D76BF8D", + CoreCount: 8, + CoreEnabled: 8, + ThreadCount: 16, + }, + "CPU-2": { + Socket: "CPU 2", + Manufacturer: "Intel", + ProductName: "Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz", + MaxSpeed: 4000, + BootSpeed: 2600, + Status: 65, + CoreCount: 8, + CoreEnabled: 8, + ThreadCount: 16, + }, + } + + memorySpecs := map[string]hardware.MemoryModuleSpec{ + "P1-DIMMA1": { + Size: 4096, + DeviceLocator: "P1-DIMMA1", + BankLocator: "P0_Node0_Channel0_Dimm0", + Speed: 1333, + Manufacturer: "Micron", + SerialNumber: "346C4A12", + AssetTag: "Dimm0_AssetTag", + ProductName: "18KSF51272PZ-1G4K", + }, + "P1-DIMMA2": { + Size: 4096, + DeviceLocator: "P1-DIMMA2", + BankLocator: "P0_Node0_Channel0_Dimm1", + Speed: 1333, + Manufacturer: "Kingston", + SerialNumber: "D2166C8B", + AssetTag: "Dimm1_AssetTag", + ProductName: "HP647647-071-HYE", + }, + } + + for k, v := range cpuSpecs { + ctest.AssertResource(suite, k, func(r *hardware.Processor, assertions *assert.Assertions) { + assertions.Equal(v, *r.TypedSpec()) + }) + } + + for k, v := range memorySpecs { + ctest.AssertResource(suite, k, func(r *hardware.MemoryModule, assertions *assert.Assertions) { + assertions.Equal(v, *r.TypedSpec()) + }) + } +} + +func (suite *SystemInfoSuite) TestUUIDOverwrite() { + stream, err := os.Open("testdata/SuperMicro-Dual-Xeon.dmi") + suite.Require().NoError(err) + + suite.T().Cleanup(func() { suite.NoError(stream.Close()) }) + + version := smbios.Version{Major: 3, Minor: 3, Revision: 0} // dummy version + s, err := smbios.Decode(stream, version) + suite.Require().NoError(err) + + suite.Require().NoError( + suite.runtime.RegisterController( + &hardwarectrl.SystemInfoController{ + SMBIOS: s, + }, + ), + ) + + suite.startRuntime() + + suite.Require().NoError(suite.state.Create(suite.ctx, runtime.NewMetaLoaded())) + + key := runtime.NewMetaKey(runtime.NamespaceName, runtime.MetaKeyTagToID(meta.UUIDOverride)) + key.TypedSpec().Value = "00000000-0000-0000-0000-000000000001" + + suite.Require().NoError(suite.state.Create(suite.ctx, key)) + + ctest.AssertResource(suite, hardware.SystemInformationID, func(r *hardware.SystemInformation, assertions *assert.Assertions) { + assertions.Equal("00000000-0000-0000-0000-000000000001", r.TypedSpec().UUID) + }) +} + +func (suite *SystemInfoSuite) TestPopulateSystemInformationIsDisabledInContainerMode() { + suite.Require().NoError( + suite.runtime.RegisterController( + &hardwarectrl.SystemInfoController{ + V1Alpha1Mode: runtimetalos.ModeContainer, + }, + ), + ) + + suite.startRuntime() + + suite.Require().NoError(suite.state.Create(suite.ctx, runtime.NewMetaLoaded())) + + suite.Assert().NoError(retry.Constant(1*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(suite.assertNoResource(*hardware.NewSystemInformation("systeminformation").Metadata()))) +} + +func TestSystemInfoSyncSuite(t *testing.T) { + suite.Run(t, new(SystemInfoSuite)) +} + +func (suite *SystemInfoSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} diff --git a/internal/app/machined/pkg/controllers/hardware/testdata/SuperMicro-Dual-Xeon.dmi b/internal/app/machined/pkg/controllers/hardware/testdata/SuperMicro-Dual-Xeon.dmi new file mode 100644 index 0000000000000000000000000000000000000000..cb515d3e8b8aae671b97bcb85e8ffba5823b178c GIT binary patch literal 6852 zcmc(k&vzQv702()fP{o)Kz6WW$<0)D9NR6@{08czG7L!Q2NnUzKa!?45@Q)x0tX?- zavh1-b=$;gn{Ik~y6S27ZO`ed-N=7Ra!z;Ibl;v#@0$kV3128Q7n>%d${>3Z<09=6cn`UC$sFgR06+UZh7V9;mQrh7cDjU-vPYW9$$TN~M zBMG8{M^7-QzF;tUPSgrEw27kA_wPlp}^*w%NIz648b_)X=V6C%zM}{~Ug^n}cjHmK9 zI?r@HD=KP15L8K$-Sk|}4qTW6hV+ud&FUsv|1p81C=vzpz8*f4)BC8YTE+;NocFf)|Eh7{CKQU+}*$26}Fd7h$1NH@4{S zi}_3ZO`}>N-=s0EMkGxY_=ggIm6xV9VLtO1SISRZ z?C6N}tviBfBfYpsdvW{SxqTzY0L=&-UlbD}*bPBCD9Oxjt3WIR9Gt{ zMG_l78k0!u#L<{cVkdza2M8{TA|K)LU}?q*zt##%G_1_1Bs<&+%QUQ!;3;5C0CRCQ zmn9bi)F9y&1ngpRo4N(EYi@HH&?sy=%2jws$SGv+Z51AHI?A zw0AASH`=?FVLaU}Bnra5b@X@J0cAf*j-P4;Rucl*KWFbF%zo%RFn{%XJPZzOELUXO zhvgmCxB=vJLiq3N}JhY%tjdi%^wOnqq^=EwKI2G*$6qN{;%D zBu9Opp(@8{3Snuc*en9|eauQ>$g~_^PGLo z@mXA-tB-h`iGWKqE(ZL9j72|Pzh~4cMje+3$1qMzzc&snJ)}^ql=hOfa%t0;$rbB6 z>$|nhnS6D7%h>B!RQc^HQMC$R04-YT@>^ zYRM38>-UP4im@f!PL;Q}1;~>7j|%Uyfvjk{k`N`*4%QL36FWObeYLm=Vr;1}r;1Wk z%Dop6=anTM;N-a)n1!O~3n$6Fn(v$#hI@ zb1VJ0=a3s$P;-_^yK~PE$ldKXPqc1s z{RsCWayO9skXdl&UK)_Q+i%L<-)}_~bbK6W`yK7ay^P#1k=taJ+_;0MvTl|C8XgTS z9?fN)YG`O`1+u*>zhqVZG+rsus{G=smA~JWN~*DRO4Hgs7W(x#*JPe$SKyixHL3jX z4#3JCsipbq%uO}R%>SDk^3roeZao! z&RrOcd#>B>0V{th9#v!UltB7q&FH?Be-*hK$bHBb+_~2V&X3*jBR$!o%_aM-1FUjw<`ZYcQyj9{!PC-D!=nedy|ZB4l6%*!=3xqfZW}F zw<>?16-!aXcp|1aSANGkoA(g=Dsmrix7@kkAB;QG?RTs4_whTeiD@mS+jnFgmEYO# zA0c-GxevMb-MK#)kh|OOR^{*Gw{<5O?YNWdto)9C{}{PnBDcwX;Kt3Lyp2h(Zd3j{ zz<5ZMTh2yf5qvy96E2cU+Gu}BYlE~V0L};3v5^IQ=mzrk3*el81DHE-2a0a*)nGDQ z>ZNFODc&WD8&GmpoCs`U>o{T(Hb$>jxA5kiPZxviRPX_GULSl$JQM!V;S}WhB*{r? z_(l?64O%bsj$t>>`FDVM01x0pSLie0dN0LYe6x8R0uY1{j9ZEiiQ*2_T@@$8AN5jv zgpEhX&>m5I2p_pBP6QrftK$_%PtFH_+9mmfNbbR7SINon$Gsr;yA%(I;uHATRdFKF zz*cw2Pr4+Z63GJ`6mE~|XT#4NAX904CO9AbS=WGiPV}Dk)cd)E-h5_(=>4Kg@0UdH zc@Moa;ZK3Nj|r_04v==Wm<)gB-Z&9>0iVI9%N#x%{@meEYa+cB{uO=o?;RS3#-Bo; z!VCD^%|I7l{|{lf 0 { + gcMemLimit := memoryLimit.Value() * GoGCMemLimitPercentage / 100 + envVar = v1.EnvVar{ + Name: "GOMEMLIMIT", + Value: strconv.FormatInt(gcMemLimit, 10), + } + } + + return envVar +} + +func (ctrl *ControlPlaneStaticPodController) manageAPIServer(ctx context.Context, r controller.Runtime, _ *zap.Logger, + configResource resource.Resource, secretsVersion, configVersion string, +) (string, error) { + cfg := configResource.(*k8s.APIServerConfig).TypedSpec() + + enabledAdmissionPlugins := []string{"NodeRestriction"} + + if cfg.PodSecurityPolicyEnabled { + enabledAdmissionPlugins = append(enabledAdmissionPlugins, "PodSecurityPolicy") + } + + args := []string{ + "/usr/local/bin/kube-apiserver", + } + + builder := argsbuilder.Args{ + "admission-control-config-file": filepath.Join(constants.KubernetesAPIServerConfigDir, "admission-control-config.yaml"), + "allow-privileged": "true", + // Do not accept anonymous requests by default. Otherwise the kube-apiserver will set the request's group to system:unauthenticated exposing endpoints like /version etc. + "anonymous-auth": "false", + "api-audiences": cfg.ControlPlaneEndpoint, + "authorization-mode": "Node,RBAC", + "bind-address": "0.0.0.0", + "client-ca-file": filepath.Join(constants.KubernetesAPIServerSecretsDir, "ca.crt"), + "enable-admission-plugins": strings.Join(enabledAdmissionPlugins, ","), + "requestheader-client-ca-file": filepath.Join(constants.KubernetesAPIServerSecretsDir, "aggregator-ca.crt"), + "requestheader-allowed-names": "front-proxy-client", + "requestheader-extra-headers-prefix": "X-Remote-Extra-", + "requestheader-group-headers": "X-Remote-Group", + "requestheader-username-headers": "X-Remote-User", + "proxy-client-cert-file": filepath.Join(constants.KubernetesAPIServerSecretsDir, "front-proxy-client.crt"), + "proxy-client-key-file": filepath.Join(constants.KubernetesAPIServerSecretsDir, "front-proxy-client.key"), + "enable-bootstrap-token-auth": "true", + // NB: using TLS 1.2 instead of 1.3 here for interoperability, since this is an externally-facing service. + "tls-min-version": "VersionTLS12", + "tls-cipher-suites": "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256", //nolint:lll + "encryption-provider-config": filepath.Join(constants.KubernetesAPIServerSecretsDir, "encryptionconfig.yaml"), + "audit-policy-file": filepath.Join(constants.KubernetesAPIServerConfigDir, "auditpolicy.yaml"), + "audit-log-path": filepath.Join(constants.KubernetesAuditLogDir, "kube-apiserver.log"), + "audit-log-maxage": "30", + "audit-log-maxbackup": "10", + "audit-log-maxsize": "100", + "profiling": "false", + "etcd-cafile": filepath.Join(constants.KubernetesAPIServerSecretsDir, "etcd-client-ca.crt"), + "etcd-certfile": filepath.Join(constants.KubernetesAPIServerSecretsDir, "etcd-client.crt"), + "etcd-keyfile": filepath.Join(constants.KubernetesAPIServerSecretsDir, "etcd-client.key"), + "etcd-servers": strings.Join(cfg.EtcdServers, ","), + "kubelet-client-certificate": filepath.Join(constants.KubernetesAPIServerSecretsDir, "apiserver-kubelet-client.crt"), + "kubelet-client-key": filepath.Join(constants.KubernetesAPIServerSecretsDir, "apiserver-kubelet-client.key"), + "secure-port": strconv.FormatInt(int64(cfg.LocalPort), 10), + "service-account-issuer": cfg.ControlPlaneEndpoint, + "service-account-key-file": filepath.Join(constants.KubernetesAPIServerSecretsDir, "service-account.pub"), + "service-account-signing-key-file": filepath.Join(constants.KubernetesAPIServerSecretsDir, "service-account.key"), + "service-cluster-ip-range": strings.Join(cfg.ServiceCIDRs, ","), + "tls-cert-file": filepath.Join(constants.KubernetesAPIServerSecretsDir, "apiserver.crt"), + "tls-private-key-file": filepath.Join(constants.KubernetesAPIServerSecretsDir, "apiserver.key"), + "kubelet-preferred-address-types": "InternalIP,ExternalIP,Hostname", + } + + if cfg.AdvertisedAddress != "" { + builder.Set("advertise-address", cfg.AdvertisedAddress) + } + + if cfg.CloudProvider != "" { + builder.Set("cloud-provider", cfg.CloudProvider) + } + + mergePolicies := argsbuilder.MergePolicies{ + "enable-admission-plugins": argsbuilder.MergeAdditive, + "feature-gates": argsbuilder.MergeAdditive, + "authorization-mode": argsbuilder.MergeAdditive, + "tls-cipher-suites": argsbuilder.MergeAdditive, + + "etcd-servers": argsbuilder.MergeDenied, + "client-ca-file": argsbuilder.MergeDenied, + "requestheader-client-ca-file": argsbuilder.MergeDenied, + "proxy-client-cert-file": argsbuilder.MergeDenied, + "proxy-client-key-file": argsbuilder.MergeDenied, + "encryption-provider-config": argsbuilder.MergeDenied, + "etcd-cafile": argsbuilder.MergeDenied, + "etcd-certfile": argsbuilder.MergeDenied, + "etcd-keyfile": argsbuilder.MergeDenied, + "kubelet-client-certificate": argsbuilder.MergeDenied, + "kubelet-client-key": argsbuilder.MergeDenied, + "service-account-key-file": argsbuilder.MergeDenied, + "service-account-signing-key-file": argsbuilder.MergeDenied, + "tls-cert-file": argsbuilder.MergeDenied, + "tls-private-key-file": argsbuilder.MergeDenied, + } + + if err := builder.Merge(cfg.ExtraArgs, argsbuilder.WithMergePolicies(mergePolicies)); err != nil { + return "", err + } + + args = append(args, builder.Args()...) + + resources, err := resources(cfg.Resources, "200m", "512Mi") + if err != nil { + return "", err + } + + env := envVars(cfg.EnvironmentVariables) + if goGCEnv := goGCEnvFromResources(resources); goGCEnv.Name != "" { + env = append(env, goGCEnv) + } + + return k8s.APIServerID, safe.WriterModify(ctx, r, k8s.NewStaticPod(k8s.NamespaceName, k8s.APIServerID), func(r *k8s.StaticPod) error { + return k8sadapter.StaticPod(r).SetPod(&v1.Pod{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Pod", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: k8s.APIServerID, + Namespace: "kube-system", + Annotations: map[string]string{ + constants.AnnotationStaticPodSecretsVersion: secretsVersion, + constants.AnnotationStaticPodConfigFileVersion: configVersion, + constants.AnnotationStaticPodConfigVersion: configResource.Metadata().Version().String(), + }, + Labels: map[string]string{ + "tier": "control-plane", + "k8s-app": k8s.APIServerID, + }, + }, + Spec: v1.PodSpec{ + Priority: pointer.To(systemCriticalPriority), + PriorityClassName: "system-cluster-critical", + Containers: []v1.Container{ + { + Name: k8s.APIServerID, + Image: cfg.Image, + Command: args, + Env: append( + []v1.EnvVar{ + { + Name: "POD_IP", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "status.podIP", + }, + }, + }, + }, + env...), + VolumeMounts: append([]v1.VolumeMount{ + { + Name: "secrets", + MountPath: constants.KubernetesAPIServerSecretsDir, + ReadOnly: true, + }, + { + Name: "config", + MountPath: constants.KubernetesAPIServerConfigDir, + ReadOnly: true, + }, + { + Name: "audit", + MountPath: constants.KubernetesAuditLogDir, + ReadOnly: false, + }, + }, volumeMounts(cfg.ExtraVolumes)...), + Resources: resources, + SecurityContext: &v1.SecurityContext{ + AllowPrivilegeEscalation: pointer.To(false), + Capabilities: &v1.Capabilities{ + Drop: []v1.Capability{"ALL"}, + // kube-apiserver binary has cap_net_bind_service=+ep set. + // It does not matter if ports < 1024 are configured, the setcap flag causes a capability dependency. + // https://github.com/kubernetes/kubernetes/blob/5b92e46b2238b4d84358451013e634361084ff7d/build/server-image/kube-apiserver/Dockerfile#L26 + Add: []v1.Capability{"NET_BIND_SERVICE"}, + }, + SeccompProfile: &v1.SeccompProfile{ + Type: v1.SeccompProfileTypeRuntimeDefault, + }, + }, + }, + }, + HostNetwork: true, + SecurityContext: &v1.PodSecurityContext{ + RunAsNonRoot: pointer.To(true), + RunAsUser: pointer.To[int64](constants.KubernetesAPIServerRunUser), + RunAsGroup: pointer.To[int64](constants.KubernetesAPIServerRunGroup), + }, + Volumes: append([]v1.Volume{ + { + Name: "secrets", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: constants.KubernetesAPIServerSecretsDir, + }, + }, + }, + { + Name: "config", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: constants.KubernetesAPIServerConfigDir, + }, + }, + }, + { + Name: "audit", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: constants.KubernetesAuditLogDir, + }, + }, + }, + }, volumes(cfg.ExtraVolumes)...), + }, + }) + }) +} + +func (ctrl *ControlPlaneStaticPodController) manageControllerManager(ctx context.Context, r controller.Runtime, + _ *zap.Logger, configResource resource.Resource, secretsVersion, _ string, +) (string, error) { + cfg := configResource.(*k8s.ControllerManagerConfig).TypedSpec() + + if !cfg.Enabled { + return "", nil + } + + args := []string{ + "/usr/local/bin/kube-controller-manager", + "--use-service-account-credentials", + } + + builder := argsbuilder.Args{ + "allocate-node-cidrs": "true", + "bind-address": "127.0.0.1", + "cluster-cidr": strings.Join(cfg.PodCIDRs, ","), + "service-cluster-ip-range": strings.Join(cfg.ServiceCIDRs, ","), + "cluster-signing-cert-file": filepath.Join(constants.KubernetesControllerManagerSecretsDir, "ca.crt"), + "cluster-signing-key-file": filepath.Join(constants.KubernetesControllerManagerSecretsDir, "ca.key"), + "controllers": "*,tokencleaner", + "configure-cloud-routes": "false", + "kubeconfig": filepath.Join(constants.KubernetesControllerManagerSecretsDir, "kubeconfig"), + "authentication-kubeconfig": filepath.Join(constants.KubernetesControllerManagerSecretsDir, "kubeconfig"), + "authorization-kubeconfig": filepath.Join(constants.KubernetesControllerManagerSecretsDir, "kubeconfig"), + "leader-elect": "true", + "root-ca-file": filepath.Join(constants.KubernetesControllerManagerSecretsDir, "ca.crt"), + "service-account-private-key-file": filepath.Join(constants.KubernetesControllerManagerSecretsDir, "service-account.key"), + "profiling": "false", + "tls-min-version": "VersionTLS13", + } + + if cfg.CloudProvider != "" { + builder.Set("cloud-provider", cfg.CloudProvider) + } + + mergePolicies := argsbuilder.MergePolicies{ + "service-cluster-ip-range": argsbuilder.MergeAdditive, + "controllers": argsbuilder.MergeAdditive, + + "cluster-signing-cert-file": argsbuilder.MergeDenied, + "cluster-signing-key-file": argsbuilder.MergeDenied, + "authentication-kubeconfig": argsbuilder.MergeDenied, + "authorization-kubeconfig": argsbuilder.MergeDenied, + "root-ca-file": argsbuilder.MergeDenied, + "service-account-private-key-file": argsbuilder.MergeDenied, + } + + if err := builder.Merge(cfg.ExtraArgs, argsbuilder.WithMergePolicies(mergePolicies)); err != nil { + return "", err + } + + args = append(args, builder.Args()...) + + resources, err := resources(cfg.Resources, "50m", "256Mi") + if err != nil { + return "", err + } + + env := envVars(cfg.EnvironmentVariables) + if goGCEnv := goGCEnvFromResources(resources); goGCEnv.Name != "" { + env = append(env, goGCEnv) + } + + return k8s.ControllerManagerID, safe.WriterModify(ctx, r, k8s.NewStaticPod(k8s.NamespaceName, k8s.ControllerManagerID), func(r *k8s.StaticPod) error { + return k8sadapter.StaticPod(r).SetPod(&v1.Pod{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Pod", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: k8s.ControllerManagerID, + Namespace: "kube-system", + Annotations: map[string]string{ + constants.AnnotationStaticPodSecretsVersion: secretsVersion, + constants.AnnotationStaticPodConfigVersion: configResource.Metadata().Version().String(), + }, + Labels: map[string]string{ + "tier": "control-plane", + "k8s-app": k8s.ControllerManagerID, + }, + }, + Spec: v1.PodSpec{ + Priority: pointer.To(systemCriticalPriority), + PriorityClassName: "system-cluster-critical", + Containers: []v1.Container{ + { + Name: k8s.ControllerManagerID, + Image: cfg.Image, + Command: args, + Env: env, + VolumeMounts: append([]v1.VolumeMount{ + { + Name: "secrets", + MountPath: constants.KubernetesControllerManagerSecretsDir, + ReadOnly: true, + }, + }, volumeMounts(cfg.ExtraVolumes)...), + StartupProbe: &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/healthz", + Host: "localhost", + Port: intstr.FromInt(10257), + Scheme: v1.URISchemeHTTPS, + }, + }, + // Give 60 seconds for the container to start up + PeriodSeconds: 5, + FailureThreshold: 12, + TerminationGracePeriodSeconds: nil, + }, + LivenessProbe: &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/healthz", + Host: "localhost", + Port: intstr.FromInt(10257), + Scheme: v1.URISchemeHTTPS, + }, + }, + TimeoutSeconds: 15, + }, + Resources: resources, + SecurityContext: &v1.SecurityContext{ + AllowPrivilegeEscalation: pointer.To(false), + Capabilities: &v1.Capabilities{ + Drop: []v1.Capability{"ALL"}, + }, + SeccompProfile: &v1.SeccompProfile{ + Type: v1.SeccompProfileTypeRuntimeDefault, + }, + }, + }, + }, + HostNetwork: true, + SecurityContext: &v1.PodSecurityContext{ + RunAsNonRoot: pointer.To(true), + RunAsUser: pointer.To[int64](constants.KubernetesControllerManagerRunUser), + RunAsGroup: pointer.To[int64](constants.KubernetesControllerManagerRunGroup), + }, + Volumes: append([]v1.Volume{ + { + Name: "secrets", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: constants.KubernetesControllerManagerSecretsDir, + }, + }, + }, + }, volumes(cfg.ExtraVolumes)...), + }, + }) + }) +} + +func (ctrl *ControlPlaneStaticPodController) manageScheduler(ctx context.Context, r controller.Runtime, + _ *zap.Logger, configResource resource.Resource, secretsVersion, _ string, +) (string, error) { + cfg := configResource.(*k8s.SchedulerConfig).TypedSpec() + + if !cfg.Enabled { + return "", nil + } + + args := []string{ + "/usr/local/bin/kube-scheduler", + } + + builder := argsbuilder.Args{ + "config": filepath.Join(constants.KubernetesSchedulerConfigDir, "scheduler-config.yaml"), + "authentication-tolerate-lookup-failure": "false", + "authentication-kubeconfig": filepath.Join(constants.KubernetesSchedulerSecretsDir, "kubeconfig"), + "authorization-kubeconfig": filepath.Join(constants.KubernetesSchedulerSecretsDir, "kubeconfig"), + "bind-address": "127.0.0.1", + "leader-elect": "true", + "profiling": "false", + "tls-min-version": "VersionTLS13", + } + + mergePolicies := argsbuilder.MergePolicies{ + "kubeconfig": argsbuilder.MergeDenied, + "authentication-kubeconfig": argsbuilder.MergeDenied, + "authorization-kubeconfig": argsbuilder.MergeDenied, + "config": argsbuilder.MergeDenied, + } + + if err := builder.Merge(cfg.ExtraArgs, argsbuilder.WithMergePolicies(mergePolicies)); err != nil { + return "", err + } + + args = append(args, builder.Args()...) + + resources, err := resources(cfg.Resources, "10m", "64Mi") + if err != nil { + return "", err + } + + env := envVars(cfg.EnvironmentVariables) + if goGCEnv := goGCEnvFromResources(resources); goGCEnv.Name != "" { + env = append(env, goGCEnv) + } + + return k8s.SchedulerID, safe.WriterModify(ctx, r, k8s.NewStaticPod(k8s.NamespaceName, k8s.SchedulerID), func(r *k8s.StaticPod) error { + return k8sadapter.StaticPod(r).SetPod(&v1.Pod{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Pod", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: k8s.SchedulerID, + Namespace: "kube-system", + Annotations: map[string]string{ + constants.AnnotationStaticPodSecretsVersion: secretsVersion, + constants.AnnotationStaticPodConfigVersion: configResource.Metadata().Version().String(), + }, + Labels: map[string]string{ + "tier": "control-plane", + "k8s-app": k8s.SchedulerID, + }, + }, + Spec: v1.PodSpec{ + Priority: pointer.To(systemCriticalPriority), + PriorityClassName: "system-cluster-critical", + Containers: []v1.Container{ + { + Name: k8s.SchedulerID, + Image: cfg.Image, + Command: args, + Env: env, + VolumeMounts: append([]v1.VolumeMount{ + { + Name: "secrets", + MountPath: constants.KubernetesSchedulerSecretsDir, + ReadOnly: true, + }, + { + Name: "config", + MountPath: constants.KubernetesSchedulerConfigDir, + ReadOnly: true, + }, + }, volumeMounts(cfg.ExtraVolumes)...), + StartupProbe: &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/healthz", + Host: "localhost", + Port: intstr.FromInt(10259), + Scheme: v1.URISchemeHTTPS, + }, + }, + // Give 60 seconds for the container to start up + PeriodSeconds: 5, + FailureThreshold: 12, + TerminationGracePeriodSeconds: nil, + }, + LivenessProbe: &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/healthz", + Host: "localhost", + Port: intstr.FromInt(10259), + Scheme: v1.URISchemeHTTPS, + }, + }, + TimeoutSeconds: 15, + }, + Resources: resources, + SecurityContext: &v1.SecurityContext{ + AllowPrivilegeEscalation: pointer.To(false), + Capabilities: &v1.Capabilities{ + Drop: []v1.Capability{"ALL"}, + }, + SeccompProfile: &v1.SeccompProfile{ + Type: v1.SeccompProfileTypeRuntimeDefault, + }, + }, + }, + }, + HostNetwork: true, + SecurityContext: &v1.PodSecurityContext{ + RunAsNonRoot: pointer.To(true), + RunAsUser: pointer.To[int64](constants.KubernetesSchedulerRunUser), + RunAsGroup: pointer.To[int64](constants.KubernetesSchedulerRunGroup), + }, + Volumes: append([]v1.Volume{ + { + Name: "secrets", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: constants.KubernetesSchedulerSecretsDir, + }, + }, + }, + { + Name: "config", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: constants.KubernetesSchedulerConfigDir, + }, + }, + }, + }, volumes(cfg.ExtraVolumes)...), + }, + }) + }) +} diff --git a/internal/app/machined/pkg/controllers/k8s/control_plane_static_pod_test.go b/internal/app/machined/pkg/controllers/k8s/control_plane_static_pod_test.go new file mode 100644 index 0000000..9ff8e3b --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/control_plane_static_pod_test.go @@ -0,0 +1,789 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s_test + +import ( + "context" + "fmt" + "log" + "reflect" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + v1 "k8s.io/api/core/v1" + apiresource "k8s.io/apimachinery/pkg/api/resource" + + k8sadapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/k8s" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + k8sctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/k8s" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +type ControlPlaneStaticPodSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *ControlPlaneStaticPodSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.Require().NoError(suite.runtime.RegisterController(&k8sctrl.ControlPlaneStaticPodController{})) + + etcdService := v1alpha1.NewService("etcd") + etcdService.TypedSpec().Running = true + etcdService.TypedSpec().Healthy = true + + suite.Require().NoError(suite.state.Create(suite.ctx, etcdService)) + + suite.startRuntime() +} + +func (suite *ControlPlaneStaticPodSuite) State() state.State { return suite.state } + +func (suite *ControlPlaneStaticPodSuite) Ctx() context.Context { return suite.ctx } + +func (suite *ControlPlaneStaticPodSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *ControlPlaneStaticPodSuite) assertControlPlaneStaticPods(manifests []string) error { + resources, err := suite.state.List( + suite.ctx, + resource.NewMetadata(k8s.NamespaceName, k8s.StaticPodType, "", resource.VersionUndefined), + ) + if err != nil { + return err + } + + ids := xslices.Map(resources.Items, func(r resource.Resource) string { return r.Metadata().ID() }) + + if !reflect.DeepEqual(manifests, ids) { + return retry.ExpectedErrorf("expected %q, got %q", manifests, ids) + } + + return nil +} + +func (suite *ControlPlaneStaticPodSuite) TestReconcileDefaults() { + secretStatus := k8s.NewSecretsStatus(k8s.ControlPlaneNamespaceName, k8s.StaticPodSecretsStaticPodID) + configStatus := k8s.NewConfigStatus(k8s.ControlPlaneNamespaceName, k8s.ConfigStatusStaticPodID) + configAPIServer := k8s.NewAPIServerConfig() + configControllerManager := k8s.NewControllerManagerConfig() + configControllerManager.TypedSpec().Enabled = true + configScheduler := k8s.NewSchedulerConfig() + configScheduler.TypedSpec().Enabled = true + + suite.Require().NoError(suite.state.Create(suite.ctx, configStatus)) + suite.Require().NoError(suite.state.Create(suite.ctx, secretStatus)) + suite.Require().NoError(suite.state.Create(suite.ctx, configAPIServer)) + suite.Require().NoError(suite.state.Create(suite.ctx, configControllerManager)) + suite.Require().NoError(suite.state.Create(suite.ctx, configScheduler)) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertControlPlaneStaticPods( + []string{ + "kube-apiserver", + "kube-controller-manager", + "kube-scheduler", + }, + ) + }, + ), + ) + + // tear down etcd service + suite.Require().NoError(suite.state.Destroy(suite.ctx, v1alpha1.NewService("etcd").Metadata())) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + list, err := suite.state.List( + suite.ctx, + resource.NewMetadata(k8s.NamespaceName, k8s.StaticPodType, "", resource.VersionUndefined), + ) + if err != nil { + return err + } + + if len(list.Items) > 0 { + return retry.ExpectedErrorf("expected no pods, got %d", len(list.Items)) + } + + return nil + }, + ), + ) +} + +func (suite *ControlPlaneStaticPodSuite) TestReconcileExtraMounts() { + secretStatus := k8s.NewSecretsStatus(k8s.ControlPlaneNamespaceName, k8s.StaticPodSecretsStaticPodID) + configStatus := k8s.NewConfigStatus(k8s.ControlPlaneNamespaceName, k8s.ConfigStatusStaticPodID) + configAPIServer := k8s.NewAPIServerConfig() + *configAPIServer.TypedSpec() = k8s.APIServerConfigSpec{ + ExtraVolumes: []k8s.ExtraVolume{ + { + Name: "foo", + HostPath: "/var/lib", + MountPath: "/var/foo", + ReadOnly: true, + }, + }, + } + + configControllerManager := k8s.NewControllerManagerConfig() + configControllerManager.TypedSpec().Enabled = true + configScheduler := k8s.NewSchedulerConfig() + configScheduler.TypedSpec().Enabled = true + + suite.Require().NoError(suite.state.Create(suite.ctx, configStatus)) + suite.Require().NoError(suite.state.Create(suite.ctx, secretStatus)) + suite.Require().NoError(suite.state.Create(suite.ctx, configAPIServer)) + suite.Require().NoError(suite.state.Create(suite.ctx, configControllerManager)) + suite.Require().NoError(suite.state.Create(suite.ctx, configScheduler)) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertControlPlaneStaticPods( + []string{ + "kube-apiserver", + "kube-controller-manager", + "kube-scheduler", + }, + ) + }, + ), + ) + + r, err := suite.state.Get( + suite.ctx, + resource.NewMetadata(k8s.NamespaceName, k8s.StaticPodType, "kube-apiserver", resource.VersionUndefined), + ) + suite.Require().NoError(err) + + apiServerPod, err := k8sadapter.StaticPod(r.(*k8s.StaticPod)).Pod() + suite.Require().NoError(err) + + suite.Assert().Len(apiServerPod.Spec.Volumes, 4) + suite.Assert().Len(apiServerPod.Spec.Containers[0].VolumeMounts, 4) + + suite.Assert().Equal( + v1.Volume{ + Name: "secrets", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: constants.KubernetesAPIServerSecretsDir, + }, + }, + }, apiServerPod.Spec.Volumes[0], + ) + + suite.Assert().Equal( + v1.Volume{ + Name: "config", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: constants.KubernetesAPIServerConfigDir, + }, + }, + }, apiServerPod.Spec.Volumes[1], + ) + + suite.Assert().Equal( + v1.Volume{ + Name: "audit", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: constants.KubernetesAuditLogDir, + }, + }, + }, apiServerPod.Spec.Volumes[2], + ) + + suite.Assert().Equal( + v1.Volume{ + Name: "foo", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/var/lib", + }, + }, + }, apiServerPod.Spec.Volumes[3], + ) + + suite.Assert().Equal( + v1.VolumeMount{ + Name: "secrets", + MountPath: constants.KubernetesAPIServerSecretsDir, + ReadOnly: true, + }, apiServerPod.Spec.Containers[0].VolumeMounts[0], + ) + + suite.Assert().Equal( + v1.VolumeMount{ + Name: "config", + MountPath: constants.KubernetesAPIServerConfigDir, + ReadOnly: true, + }, apiServerPod.Spec.Containers[0].VolumeMounts[1], + ) + + suite.Assert().Equal( + v1.VolumeMount{ + Name: "audit", + MountPath: constants.KubernetesAuditLogDir, + ReadOnly: false, + }, apiServerPod.Spec.Containers[0].VolumeMounts[2], + ) + + suite.Assert().Equal( + v1.VolumeMount{ + Name: "foo", + MountPath: "/var/foo", + ReadOnly: true, + }, apiServerPod.Spec.Containers[0].VolumeMounts[3], + ) +} + +func (suite *ControlPlaneStaticPodSuite) TestReconcileExtraArgs() { + tests := []struct { + args map[string]string + expected map[string]string + expectError bool + }{ + { + args: map[string]string{ + "enable-admission-plugins": "NodeRestriction,PodNodeSelector", + "authorization-mode": "Webhook", + "bind-address": "127.0.0.1", + "audit-log-batch-max-size": "2", + }, + expected: map[string]string{ + "enable-admission-plugins": "NodeRestriction,PodNodeSelector", + "authorization-mode": "Node,RBAC,Webhook", + "bind-address": "127.0.0.1", + "audit-log-batch-max-size": "2", + }, + }, + { + args: map[string]string{ + "proxy-client-key-file": "front-proxy-client.key", + }, + expectError: true, + }, + } + for _, test := range tests { + configStatus := k8s.NewConfigStatus(k8s.ControlPlaneNamespaceName, k8s.ConfigStatusStaticPodID) + secretStatus := k8s.NewSecretsStatus(k8s.ControlPlaneNamespaceName, k8s.StaticPodSecretsStaticPodID) + configAPIServer := k8s.NewAPIServerConfig() + + *configAPIServer.TypedSpec() = k8s.APIServerConfigSpec{ + ExtraArgs: test.args, + } + + suite.Require().NoError(suite.state.Create(suite.ctx, configStatus)) + suite.Require().NoError(suite.state.Create(suite.ctx, secretStatus)) + suite.Require().NoError(suite.state.Create(suite.ctx, configAPIServer)) + + if test.expectError { + // wait for some time to ensure that controller has picked the input + time.Sleep(500 * time.Millisecond) + + _, err := suite.state.Get( + suite.ctx, + resource.NewMetadata(k8s.NamespaceName, k8s.StaticPodType, "kube-apiserver", resource.VersionUndefined), + ) + suite.Require().Error(err) + + continue + } + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertControlPlaneStaticPods( + []string{ + "kube-apiserver", + }, + ) + }, + ), + ) + + r, err := suite.state.Get( + suite.ctx, + resource.NewMetadata(k8s.NamespaceName, k8s.StaticPodType, "kube-apiserver", resource.VersionUndefined), + ) + suite.Require().NoError(err) + + apiServerPod, err := k8sadapter.StaticPod(r.(*k8s.StaticPod)).Pod() + suite.Require().NoError(err) + + suite.Require().NotEmpty(apiServerPod.Spec.Containers) + + assertArg := func(arg, equals string) { + for _, param := range apiServerPod.Spec.Containers[0].Command { + if strings.HasPrefix(param, fmt.Sprintf("--%s", arg)) { + parts := strings.Split(param, "=") + + suite.Require().Equal(equals, parts[1]) + } + } + } + + for k, v := range test.expected { + assertArg(k, v) + } + + suite.Require().NoError(suite.state.Destroy(suite.ctx, configStatus.Metadata())) + suite.Require().NoError(suite.state.Destroy(suite.ctx, secretStatus.Metadata())) + suite.Require().NoError(suite.state.Destroy(suite.ctx, configAPIServer.Metadata())) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + list, err := suite.state.List( + suite.ctx, + resource.NewMetadata(k8s.NamespaceName, k8s.StaticPodType, "", resource.VersionUndefined), + ) + if err != nil { + return err + } + + if len(list.Items) > 0 { + return retry.ExpectedErrorf("expected no pods, got %d", len(list.Items)) + } + + return nil + }, + ), + ) + } +} + +func (suite *ControlPlaneStaticPodSuite) TestReconcileEnvironmentVariables() { + configStatus := k8s.NewConfigStatus(k8s.ControlPlaneNamespaceName, k8s.ConfigStatusStaticPodID) + secretStatus := k8s.NewSecretsStatus(k8s.ControlPlaneNamespaceName, k8s.StaticPodSecretsStaticPodID) + + suite.Require().NoError(suite.state.Create(suite.ctx, configStatus)) + suite.Require().NoError(suite.state.Create(suite.ctx, secretStatus)) + + tests := []struct { + env map[string]string + expected []v1.EnvVar + }{ + { + env: nil, + expected: []v1.EnvVar{ + { + Name: "POD_IP", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "status.podIP", + }, + }, + }, + }, + }, + { + env: map[string]string{ + "foo": "bar", + "baz": "$(foo)", + }, + expected: []v1.EnvVar{ + { + Name: "POD_IP", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "status.podIP", + }, + }, + }, + { + Name: "baz", + Value: "$$(foo)", + }, + { + Name: "foo", + Value: "bar", + }, + }, + }, + } + for _, test := range tests { + configAPIServer := k8s.NewAPIServerConfig() + + *configAPIServer.TypedSpec() = k8s.APIServerConfigSpec{ + EnvironmentVariables: test.env, + } + + suite.Require().NoError(suite.state.Create(suite.ctx, configAPIServer)) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertControlPlaneStaticPods( + []string{ + "kube-apiserver", + }, + ) + }, + ), + ) + + r, err := suite.state.Get( + suite.ctx, + resource.NewMetadata(k8s.NamespaceName, k8s.StaticPodType, "kube-apiserver", resource.VersionUndefined), + ) + suite.Require().NoError(err) + + apiServerPod, err := k8sadapter.StaticPod(r.(*k8s.StaticPod)).Pod() + suite.Require().NoError(err) + + suite.Require().NotEmpty(apiServerPod.Spec.Containers) + + suite.Assert().Equal(test.expected, apiServerPod.Spec.Containers[0].Env) + + suite.Require().NoError(suite.state.Destroy(suite.ctx, configAPIServer.Metadata())) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + list, err := suite.state.List( + suite.ctx, + resource.NewMetadata(k8s.NamespaceName, k8s.StaticPodType, "", resource.VersionUndefined), + ) + if err != nil { + return err + } + + if len(list.Items) > 0 { + return retry.ExpectedErrorf("expected no pods, got %d", len(list.Items)) + } + + return nil + }, + ), + ) + } +} + +func (suite *ControlPlaneStaticPodSuite) TestReconcileAdvertisedAddressArg() { + configStatus := k8s.NewConfigStatus(k8s.ControlPlaneNamespaceName, k8s.ConfigStatusStaticPodID) + secretStatus := k8s.NewSecretsStatus(k8s.ControlPlaneNamespaceName, k8s.StaticPodSecretsStaticPodID) + + suite.Require().NoError(suite.state.Create(suite.ctx, configStatus)) + suite.Require().NoError(suite.state.Create(suite.ctx, secretStatus)) + + configAPIServer := k8s.NewAPIServerConfig() + + *configAPIServer.TypedSpec() = k8s.APIServerConfigSpec{ + AdvertisedAddress: "$(POD_IP)", + } + + suite.Require().NoError(suite.state.Create(suite.ctx, configAPIServer)) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertControlPlaneStaticPods( + []string{ + "kube-apiserver", + }, + ) + }, + ), + ) + + r, err := suite.state.Get( + suite.ctx, + resource.NewMetadata(k8s.NamespaceName, k8s.StaticPodType, "kube-apiserver", resource.VersionUndefined), + ) + suite.Require().NoError(err) + + apiServerPod, err := k8sadapter.StaticPod(r.(*k8s.StaticPod)).Pod() + suite.Require().NoError(err) + + suite.Require().NotEmpty(apiServerPod.Spec.Containers) + + suite.Assert().Contains(apiServerPod.Spec.Containers[0].Command, "--advertise-address=$(POD_IP)") + + configAPIServer.TypedSpec().AdvertisedAddress = "" + + suite.Assert().NoError(suite.state.Update(suite.ctx, configAPIServer)) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + r, err = suite.state.Get( + suite.ctx, + resource.NewMetadata(k8s.NamespaceName, k8s.StaticPodType, "kube-apiserver", resource.VersionUndefined), + ) + suite.Require().NoError(err) + + apiServerPod, err = k8sadapter.StaticPod(r.(*k8s.StaticPod)).Pod() + suite.Require().NoError(err) + + for _, arg := range apiServerPod.Spec.Containers[0].Command { + if strings.Contains(arg, "--advertise-address=") { + return retry.ExpectedErrorf("expected no advertise-address, got %s", arg) + } + } + + return nil + }, + ), + ) + + suite.Require().NoError(suite.state.Destroy(suite.ctx, configAPIServer.Metadata())) +} + +func (suite *ControlPlaneStaticPodSuite) TestControlPlaneStaticPodsExceptScheduler() { + configStatus := k8s.NewConfigStatus(k8s.ControlPlaneNamespaceName, k8s.ConfigStatusStaticPodID) + secretStatus := k8s.NewSecretsStatus(k8s.ControlPlaneNamespaceName, k8s.StaticPodSecretsStaticPodID) + configAPIServer := k8s.NewAPIServerConfig() + configControllerManager := k8s.NewControllerManagerConfig() + configControllerManager.TypedSpec().Enabled = true + configScheduler := k8s.NewSchedulerConfig() + configScheduler.TypedSpec().Enabled = true + + suite.Require().NoError(suite.state.Create(suite.ctx, configStatus)) + suite.Require().NoError(suite.state.Create(suite.ctx, secretStatus)) + suite.Require().NoError(suite.state.Create(suite.ctx, configAPIServer)) + suite.Require().NoError(suite.state.Create(suite.ctx, configControllerManager)) + suite.Require().NoError(suite.state.Create(suite.ctx, configScheduler)) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertControlPlaneStaticPods( + []string{ + "kube-apiserver", + "kube-controller-manager", + "kube-scheduler", + }, + ) + }, + ), + ) + + // flip enabled to disable scheduler + ctest.UpdateWithConflicts(suite, configScheduler, func(r *k8s.SchedulerConfig) error { + r.TypedSpec().Enabled = false + + return nil + }) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertControlPlaneStaticPods( + []string{ + "kube-apiserver", + "kube-controller-manager", + }, + ) + }, + ), + ) +} + +func (suite *ControlPlaneStaticPodSuite) TestReconcileStaticPodResources() { + configStatus := k8s.NewConfigStatus(k8s.ControlPlaneNamespaceName, k8s.ConfigStatusStaticPodID) + secretStatus := k8s.NewSecretsStatus(k8s.ControlPlaneNamespaceName, k8s.StaticPodSecretsStaticPodID) + + suite.Require().NoError(suite.state.Create(suite.ctx, configStatus)) + suite.Require().NoError(suite.state.Create(suite.ctx, secretStatus)) + + tests := []struct { + resources k8s.Resources + expected v1.ResourceRequirements + expectedEnv v1.EnvVar + }{ + { + resources: k8s.Resources{ + Requests: map[string]string{ + string(v1.ResourceCPU): "100m", + string(v1.ResourceMemory): "256Mi", + }, + }, + expected: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]apiresource.Quantity{ + v1.ResourceCPU: apiresource.MustParse("100m"), + v1.ResourceMemory: apiresource.MustParse("256Mi"), + }, + }, + }, + { + resources: k8s.Resources{ + Requests: map[string]string{ + string(v1.ResourceCPU): "100m", + string(v1.ResourceMemory): "256Mi", + }, + Limits: map[string]string{ + string(v1.ResourceCPU): "1", + string(v1.ResourceMemory): "1Gi", + }, + }, + expected: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]apiresource.Quantity{ + v1.ResourceCPU: apiresource.MustParse("100m"), + v1.ResourceMemory: apiresource.MustParse("256Mi"), + }, + Limits: map[v1.ResourceName]apiresource.Quantity{ + v1.ResourceCPU: apiresource.MustParse("1"), + v1.ResourceMemory: apiresource.MustParse("1Gi"), + }, + }, + expectedEnv: v1.EnvVar{ + Name: "GOMEMLIMIT", + Value: strconv.FormatInt(1024*1024*1024*k8sctrl.GoGCMemLimitPercentage/100, 10), + }, + }, + } + for _, test := range tests { + configAPIServer := k8s.NewAPIServerConfig() + configControllerManager := k8s.NewControllerManagerConfig() + configControllerManager.TypedSpec().Enabled = true + configScheduler := k8s.NewSchedulerConfig() + configScheduler.TypedSpec().Enabled = true + + configAPIServer.TypedSpec().Resources = test.resources + configControllerManager.TypedSpec().Resources = test.resources + configScheduler.TypedSpec().Resources = test.resources + + suite.Require().NoError(suite.state.Create(suite.ctx, configAPIServer)) + suite.Require().NoError(suite.state.Create(suite.ctx, configControllerManager)) + suite.Require().NoError(suite.state.Create(suite.ctx, configScheduler)) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertControlPlaneStaticPods( + []string{ + "kube-apiserver", + "kube-controller-manager", + "kube-scheduler", + }, + ) + }, + ), + ) + + r, err := suite.state.Get( + suite.ctx, + resource.NewMetadata(k8s.NamespaceName, k8s.StaticPodType, "kube-apiserver", resource.VersionUndefined), + ) + suite.Require().NoError(err) + + apiServerPod, err := k8sadapter.StaticPod(r.(*k8s.StaticPod)).Pod() + suite.Require().NoError(err) + + r, err = suite.state.Get( + suite.ctx, + resource.NewMetadata(k8s.NamespaceName, k8s.StaticPodType, "kube-controller-manager", resource.VersionUndefined), + ) + suite.Require().NoError(err) + + controllerManagerPod, err := k8sadapter.StaticPod(r.(*k8s.StaticPod)).Pod() + suite.Require().NoError(err) + + r, err = suite.state.Get( + suite.ctx, + resource.NewMetadata(k8s.NamespaceName, k8s.StaticPodType, "kube-scheduler", resource.VersionUndefined), + ) + suite.Require().NoError(err) + + schedulerPod, err := k8sadapter.StaticPod(r.(*k8s.StaticPod)).Pod() + suite.Require().NoError(err) + + suite.Require().NotEmpty(apiServerPod.Spec.Containers) + suite.Require().NotEmpty(controllerManagerPod.Spec.Containers) + suite.Require().NotEmpty(schedulerPod.Spec.Containers) + + suite.Assert().Equal(test.expected, apiServerPod.Spec.Containers[0].Resources) + suite.Assert().Equal(test.expected, controllerManagerPod.Spec.Containers[0].Resources) + suite.Assert().Equal(test.expected, schedulerPod.Spec.Containers[0].Resources) + + if test.expectedEnv.Name != "" { + suite.Assert().Contains(apiServerPod.Spec.Containers[0].Env, test.expectedEnv) + suite.Assert().Contains(controllerManagerPod.Spec.Containers[0].Env, test.expectedEnv) + suite.Assert().Contains(schedulerPod.Spec.Containers[0].Env, test.expectedEnv) + } + + suite.Require().NoError(suite.state.Destroy(suite.ctx, configAPIServer.Metadata())) + suite.Require().NoError(suite.state.Destroy(suite.ctx, configControllerManager.Metadata())) + suite.Require().NoError(suite.state.Destroy(suite.ctx, configScheduler.Metadata())) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + list, err := suite.state.List( + suite.ctx, + resource.NewMetadata(k8s.NamespaceName, k8s.StaticPodType, "", resource.VersionUndefined), + ) + if err != nil { + return err + } + + if len(list.Items) > 0 { + return retry.ExpectedErrorf("expected no pods, got %d", len(list.Items)) + } + + return nil + }, + ), + ) + } +} + +func (suite *ControlPlaneStaticPodSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestControlPlaneStaticPodSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, new(ControlPlaneStaticPodSuite)) +} diff --git a/internal/app/machined/pkg/controllers/k8s/control_plane_test.go b/internal/app/machined/pkg/controllers/k8s/control_plane_test.go new file mode 100644 index 0000000..fc619c4 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/control_plane_test.go @@ -0,0 +1,569 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s_test + +import ( + "net/url" + "strings" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/siderolabs/go-pointer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + k8sctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/k8s" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +type K8sControlPlaneSuite struct { + ctest.DefaultSuite +} + +// setupMachine creates a machine with given configuration, waits for it to become ready, +// and returns API server's spec. +func (suite *K8sControlPlaneSuite) setupMachine(cfg *config.MachineConfig) { + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.AdmissionControlConfigID}, func(*k8s.AdmissionControlConfig, *assert.Assertions) {}) + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.AuditPolicyConfigID}, func(*k8s.AuditPolicyConfig, *assert.Assertions) {}) + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.APIServerConfigID}, func(*k8s.APIServerConfig, *assert.Assertions) {}) + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.ControllerManagerConfigID}, func(*k8s.ControllerManagerConfig, *assert.Assertions) {}) + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.SchedulerConfigID}, func(*k8s.SchedulerConfig, *assert.Assertions) {}) + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.BootstrapManifestsConfigID}, func(*k8s.BootstrapManifestsConfig, *assert.Assertions) {}) + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.ExtraManifestsConfigID}, func(*k8s.ExtraManifestsConfig, *assert.Assertions) {}) +} + +func (suite *K8sControlPlaneSuite) TestReconcileDefaults() { + u, err := url.Parse("https://foo:6443") + suite.Require().NoError(err) + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineType: "controlplane", + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: u, + }, + }, + }, + }, + ), + ) + + suite.setupMachine(cfg) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.APIServerConfigID}, + func(apiServer *k8s.APIServerConfig, assert *assert.Assertions) { + apiServerCfg := apiServer.TypedSpec() + + assert.Empty(apiServerCfg.CloudProvider) + }, + ) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.ControllerManagerConfigID}, + func(controllerManager *k8s.ControllerManagerConfig, assert *assert.Assertions) { + assert.Empty(controllerManager.TypedSpec().CloudProvider) + }, + ) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.BootstrapManifestsConfigID}, + func(bootstrapConfig *k8s.BootstrapManifestsConfig, assert *assert.Assertions) { + assert.Equal("10.96.0.10", bootstrapConfig.TypedSpec().DNSServiceIP) + assert.Equal("", bootstrapConfig.TypedSpec().DNSServiceIPv6) + }, + ) +} + +func (suite *K8sControlPlaneSuite) TestReconcileTransitionWorker() { + u, err := url.Parse("https://foo:6443") + suite.Require().NoError(err) + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineType: "controlplane", + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: u, + }, + }, + }, + }, + ), + ) + + suite.setupMachine(cfg) + + cfg.Container().RawV1Alpha1().MachineConfig.MachineType = "worker" + suite.Require().NoError(suite.State().Update(suite.Ctx(), cfg)) + + rtestutils.AssertNoResource[*k8s.AdmissionControlConfig](suite.Ctx(), suite.T(), suite.State(), k8s.AdmissionControlConfigID) + rtestutils.AssertNoResource[*k8s.AuditPolicyConfig](suite.Ctx(), suite.T(), suite.State(), k8s.AuditPolicyConfigID) + rtestutils.AssertNoResource[*k8s.APIServerConfig](suite.Ctx(), suite.T(), suite.State(), k8s.APIServerConfigID) + rtestutils.AssertNoResource[*k8s.ControllerManagerConfig](suite.Ctx(), suite.T(), suite.State(), k8s.ControllerManagerConfigID) + rtestutils.AssertNoResource[*k8s.SchedulerConfig](suite.Ctx(), suite.T(), suite.State(), k8s.SchedulerConfigID) + rtestutils.AssertNoResource[*k8s.BootstrapManifestsConfig](suite.Ctx(), suite.T(), suite.State(), k8s.BootstrapManifestsConfigID) + rtestutils.AssertNoResource[*k8s.ExtraManifestsConfig](suite.Ctx(), suite.T(), suite.State(), k8s.ExtraManifestsConfigID) +} + +func (suite *K8sControlPlaneSuite) TestReconcileIPv6() { + u, err := url.Parse("https://foo:6443") + suite.Require().NoError(err) + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineType: "controlplane", + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: u, + }, + }, + ClusterNetwork: &v1alpha1.ClusterNetworkConfig{ + PodSubnet: []string{constants.DefaultIPv6PodNet}, + ServiceSubnet: []string{constants.DefaultIPv6ServiceNet}, + }, + }, + }, + ), + ) + + suite.setupMachine(cfg) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.BootstrapManifestsConfigID}, + func(bootstrapConfig *k8s.BootstrapManifestsConfig, assert *assert.Assertions) { + assert.Equal("", bootstrapConfig.TypedSpec().DNSServiceIP) + assert.Equal("fc00:db8:20::a", bootstrapConfig.TypedSpec().DNSServiceIPv6) + }, + ) +} + +func (suite *K8sControlPlaneSuite) TestReconcileDualStack() { + u, err := url.Parse("https://foo:6443") + suite.Require().NoError(err) + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineType: "controlplane", + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: u, + }, + }, + ClusterNetwork: &v1alpha1.ClusterNetworkConfig{ + PodSubnet: []string{constants.DefaultIPv4PodNet, constants.DefaultIPv6PodNet}, + ServiceSubnet: []string{constants.DefaultIPv4ServiceNet, constants.DefaultIPv6ServiceNet}, + }, + }, + }, + ), + ) + + suite.setupMachine(cfg) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.BootstrapManifestsConfigID}, + func(bootstrapConfig *k8s.BootstrapManifestsConfig, assert *assert.Assertions) { + assert.Equal("10.96.0.10", bootstrapConfig.TypedSpec().DNSServiceIP) + assert.Equal("fc00:db8:20::a", bootstrapConfig.TypedSpec().DNSServiceIPv6) + }, + ) +} + +func (suite *K8sControlPlaneSuite) TestReconcileExtraVolumes() { + u, err := url.Parse("https://foo:6443") + suite.Require().NoError(err) + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineType: "controlplane", + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: u, + }, + }, + APIServerConfig: &v1alpha1.APIServerConfig{ + ExtraVolumesConfig: []v1alpha1.VolumeMountConfig{ + { + VolumeHostPath: "/var/lib", + VolumeMountPath: "/var/foo/", + }, + { + VolumeHostPath: "/var/lib/a.foo", + VolumeMountPath: "/var/foo/b.foo", + }, + }, + }, + }, + }, + ), + ) + + suite.setupMachine(cfg) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.APIServerConfigID}, + func(apiServer *k8s.APIServerConfig, assert *assert.Assertions) { + apiServerCfg := apiServer.TypedSpec() + + assert.Equal( + []k8s.ExtraVolume{ + { + Name: "var-foo", + HostPath: "/var/lib", + MountPath: "/var/foo/", + ReadOnly: false, + }, + { + Name: "var-foo-b-foo", + HostPath: "/var/lib/a.foo", + MountPath: "/var/foo/b.foo", + ReadOnly: false, + }, + }, apiServerCfg.ExtraVolumes, + ) + }, + ) +} + +func (suite *K8sControlPlaneSuite) TestReconcileEnvironment() { + u, err := url.Parse("https://foo:6443") + suite.Require().NoError(err) + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineType: "controlplane", + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: u, + }, + }, + APIServerConfig: &v1alpha1.APIServerConfig{ + EnvConfig: v1alpha1.Env{ + "HTTP_PROXY": "foo", + }, + }, + }, + }, + ), + ) + + suite.setupMachine(cfg) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.APIServerConfigID}, + func(apiServer *k8s.APIServerConfig, assert *assert.Assertions) { + apiServerCfg := apiServer.TypedSpec() + + assert.Equal( + map[string]string{ + "HTTP_PROXY": "foo", + }, apiServerCfg.EnvironmentVariables, + ) + }, + ) +} + +func (suite *K8sControlPlaneSuite) TestReconcileResources() { + u, err := url.Parse("https://foo:6443") + suite.Require().NoError(err) + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineType: "controlplane", + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: u, + }, + }, + APIServerConfig: &v1alpha1.APIServerConfig{ + ResourcesConfig: &v1alpha1.ResourcesConfig{ + Requests: v1alpha1.Unstructured{ + Object: map[string]any{ + "cpu": "100m", + "memory": "1Gi", + }, + }, + Limits: v1alpha1.Unstructured{ + Object: map[string]any{ + "cpu": 2, + "memory": "1500Mi", + }, + }, + }, + }, + ControllerManagerConfig: &v1alpha1.ControllerManagerConfig{ + ResourcesConfig: &v1alpha1.ResourcesConfig{ + Requests: v1alpha1.Unstructured{ + Object: map[string]any{ + "cpu": "50m", + "memory": "500Mi", + }, + }, + Limits: v1alpha1.Unstructured{ + Object: map[string]any{ + "cpu": 1, + "memory": "1000Mi", + }, + }, + }, + }, + SchedulerConfig: &v1alpha1.SchedulerConfig{ + ResourcesConfig: &v1alpha1.ResourcesConfig{ + Requests: v1alpha1.Unstructured{ + Object: map[string]any{ + "cpu": "150m", + "memory": "2Gi", + }, + }, + Limits: v1alpha1.Unstructured{ + Object: map[string]any{ + "cpu": 3, + "memory": "2000Mi", + }, + }, + }, + }, + }, + }, + ), + ) + + suite.setupMachine(cfg) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.APIServerConfigID}, + func(apiServer *k8s.APIServerConfig, assert *assert.Assertions) { + apiServerCfg := apiServer.TypedSpec() + + assert.Equal( + k8s.Resources{ + Requests: map[string]string{ + "cpu": "100m", + "memory": "1Gi", + }, + Limits: map[string]string{ + "cpu": "2", + "memory": "1500Mi", + }, + }, apiServerCfg.Resources, + ) + }, + ) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.ControllerManagerConfigID}, + func(controllerManager *k8s.ControllerManagerConfig, assert *assert.Assertions) { + controllerManagerCfg := controllerManager.TypedSpec() + + assert.Equal( + k8s.Resources{ + Requests: map[string]string{ + "cpu": "50m", + "memory": "500Mi", + }, + Limits: map[string]string{ + "cpu": "1", + "memory": "1000Mi", + }, + }, controllerManagerCfg.Resources, + ) + }, + ) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.SchedulerConfigID}, + func(scheduler *k8s.SchedulerConfig, assert *assert.Assertions) { + schedulerCfg := scheduler.TypedSpec() + + assert.Equal( + k8s.Resources{ + Requests: map[string]string{ + "cpu": "150m", + "memory": "2Gi", + }, + Limits: map[string]string{ + "cpu": "3", + "memory": "2000Mi", + }, + }, schedulerCfg.Resources, + ) + }, + ) +} + +func (suite *K8sControlPlaneSuite) TestReconcileExternalCloudProvider() { + u, err := url.Parse("https://foo:6443") + suite.Require().NoError(err) + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineType: "controlplane", + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: u, + }, + }, + ExternalCloudProviderConfig: &v1alpha1.ExternalCloudProviderConfig{ + ExternalEnabled: pointer.To(true), + ExternalManifests: []string{ + "https://raw.githubusercontent.com/kubernetes/cloud-provider-aws/v1.20.0-alpha.0/manifests/rbac.yaml", + "https://raw.githubusercontent.com/kubernetes/cloud-provider-aws/v1.20.0-alpha.0/manifests/aws-cloud-controller-manager-daemonset.yaml", + }, + }, + }, + }, + ), + ) + + suite.setupMachine(cfg) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.APIServerConfigID}, + func(apiServer *k8s.APIServerConfig, assert *assert.Assertions) { + apiServerCfg := apiServer.TypedSpec() + + assert.Equal("external", apiServerCfg.CloudProvider) + }, + ) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.ControllerManagerConfigID}, + func(controllerManager *k8s.ControllerManagerConfig, assert *assert.Assertions) { + assert.Equal("external", controllerManager.TypedSpec().CloudProvider) + }, + ) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.ExtraManifestsConfigID}, + func(extraManifests *k8s.ExtraManifestsConfig, assert *assert.Assertions) { + assert.Equal( + &k8s.ExtraManifestsConfigSpec{ + ExtraManifests: []k8s.ExtraManifest{ + { + Name: "https://raw.githubusercontent.com/kubernetes/cloud-provider-aws/v1.20.0-alpha.0/manifests/rbac.yaml", + URL: "https://raw.githubusercontent.com/kubernetes/cloud-provider-aws/v1.20.0-alpha.0/manifests/rbac.yaml", + Priority: "30", + }, + { + Name: "https://raw.githubusercontent.com/kubernetes/cloud-provider-aws/v1.20.0-alpha.0/manifests/aws-cloud-controller-manager-daemonset.yaml", + URL: "https://raw.githubusercontent.com/kubernetes/cloud-provider-aws/v1.20.0-alpha.0/manifests/aws-cloud-controller-manager-daemonset.yaml", + Priority: "30", + }, + }, + }, extraManifests.TypedSpec()) + }, + ) +} + +func (suite *K8sControlPlaneSuite) TestReconcileInlineManifests() { + u, err := url.Parse("https://foo:6443") + suite.Require().NoError(err) + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineType: "controlplane", + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: u, + }, + }, + ClusterInlineManifests: v1alpha1.ClusterInlineManifests{ + { + InlineManifestName: "namespace-ci", + InlineManifestContents: strings.TrimSpace( + ` +apiVersion: v1 +kind: Namespace +metadata: + name: ci +`, + ), + }, + }, + }, + }, + ), + ) + + suite.setupMachine(cfg) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.ExtraManifestsConfigID}, + func(extraManifests *k8s.ExtraManifestsConfig, assert *assert.Assertions) { + assert.Equal( + &k8s.ExtraManifestsConfigSpec{ + ExtraManifests: []k8s.ExtraManifest{ + { + Name: "namespace-ci", + Priority: "99", + InlineManifest: "apiVersion: v1\nkind: Namespace\nmetadata:\n\tname: ci", + }, + }, + }, + extraManifests.TypedSpec()) + }, + ) +} + +func TestK8sControlPlaneSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, &K8sControlPlaneSuite{ + DefaultSuite: ctest.DefaultSuite{ + Timeout: 10 * time.Second, + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(k8sctrl.NewControlPlaneAPIServerController())) + suite.Require().NoError(suite.Runtime().RegisterController(k8sctrl.NewControlPlaneAdmissionControlController())) + suite.Require().NoError(suite.Runtime().RegisterController(k8sctrl.NewControlPlaneAuditPolicyController())) + suite.Require().NoError(suite.Runtime().RegisterController(k8sctrl.NewControlPlaneBootstrapManifestsController())) + suite.Require().NoError(suite.Runtime().RegisterController(k8sctrl.NewControlPlaneControllerManagerController())) + suite.Require().NoError(suite.Runtime().RegisterController(k8sctrl.NewControlPlaneExtraManifestsController())) + suite.Require().NoError(suite.Runtime().RegisterController(k8sctrl.NewControlPlaneSchedulerController())) + }, + }, + }) +} diff --git a/internal/app/machined/pkg/controllers/k8s/endpoint.go b/internal/app/machined/pkg/controllers/k8s/endpoint.go new file mode 100644 index 0000000..0b29b2e --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/endpoint.go @@ -0,0 +1,301 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import ( + "context" + "fmt" + "net/netip" + "reflect" + "slices" + "time" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/client-go/informers" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + + "github.com/siderolabs/talos/pkg/conditions" + "github.com/siderolabs/talos/pkg/kubernetes" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" +) + +// EndpointController looks up control plane endpoints. +type EndpointController struct{} + +// Name implements controller.Controller interface. +func (ctrl *EndpointController) Name() string { + return "k8s.EndpointController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *EndpointController) Inputs() []controller.Input { + return nil +} + +// Outputs implements controller.Controller interface. +func (ctrl *EndpointController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: k8s.EndpointType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *EndpointController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + if err := r.UpdateInputs([]controller.Input{ + { + Namespace: config.NamespaceName, + Type: config.MachineTypeType, + ID: optional.Some(config.MachineTypeID), + Kind: controller.InputWeak, + }, + }); err != nil { + return err + } + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + machineTypeRes, err := safe.ReaderGetByID[*config.MachineType](ctx, r, config.MachineTypeID) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting machine type: %w", err) + } + + machineType := machineTypeRes.MachineType() + + switch machineType { //nolint:exhaustive + case machine.TypeWorker: + if err = ctrl.watchEndpointsOnWorker(ctx, r, logger); err != nil { + return err + } + case machine.TypeControlPlane, machine.TypeInit: + if err = ctrl.watchEndpointsOnControlPlane(ctx, r, logger); err != nil { + return err + } + } + + r.ResetRestartBackoff() + } +} + +func (ctrl *EndpointController) watchEndpointsOnWorker(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + logger.Debug("waiting for kubelet client config", zap.String("file", constants.KubeletKubeconfig)) + + if err := conditions.WaitForKubeconfigReady(constants.KubeletKubeconfig).Wait(ctx); err != nil { + return err + } + + client, err := kubernetes.NewClientFromKubeletKubeconfig() + if err != nil { + return fmt.Errorf("error building Kubernetes client: %w", err) + } + + defer client.Close() //nolint:errcheck + + ticker := time.NewTicker(10 * time.Minute) + defer ticker.Stop() + + for { + // unfortunately we can't use Watch or CachedInformer here as system:node role is only allowed verb 'Get' + endpoints, err := client.CoreV1().Endpoints(corev1.NamespaceDefault).Get(ctx, "kubernetes", v1.GetOptions{}) + if err != nil { + return fmt.Errorf("error getting endpoints: %w", err) + } + + if err = ctrl.updateEndpointsResource(ctx, r, logger, endpoints); err != nil { + return err + } + + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + case <-r.EventCh(): + } + } +} + +func (ctrl *EndpointController) watchEndpointsOnControlPlane(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + if err := r.UpdateInputs([]controller.Input{ + { + Namespace: config.NamespaceName, + Type: config.MachineTypeType, + ID: optional.Some(config.MachineTypeID), + Kind: controller.InputWeak, + }, + { + Namespace: secrets.NamespaceName, + Type: secrets.KubernetesType, + ID: optional.Some(secrets.KubernetesID), + Kind: controller.InputWeak, + }, + }); err != nil { + return err + } + + r.QueueReconcile() + + for { + select { + case <-r.EventCh(): + case <-ctx.Done(): + return nil + } + + secretsResources, err := safe.ReaderGetByID[*secrets.Kubernetes](ctx, r, secrets.KubernetesID) + if err != nil { + if state.IsNotFoundError(err) { + return nil + } + + return err + } + + secrets := secretsResources.TypedSpec() + + kubeconfig, err := clientcmd.BuildConfigFromKubeconfigGetter("", func() (*clientcmdapi.Config, error) { + // using here kubeconfig with cluster control plane endpoint, as endpoint discovery should work before local API server is ready + return clientcmd.Load([]byte(secrets.AdminKubeconfig)) + }) + if err != nil { + return fmt.Errorf("error loading kubeconfig: %w", err) + } + + if err = ctrl.watchKubernetesEndpoint(ctx, r, logger, kubeconfig); err != nil { + return err + } + } +} + +func (ctrl *EndpointController) updateEndpointsResource(ctx context.Context, r controller.Runtime, logger *zap.Logger, endpoints *corev1.Endpoints) error { + var addrs []netip.Addr + + for _, endpoint := range endpoints.Subsets { + for _, addr := range endpoint.Addresses { + ip, err := netip.ParseAddr(addr.IP) + if err == nil { + addrs = append(addrs, ip) + } + } + } + + slices.SortFunc(addrs, func(a, b netip.Addr) int { return a.Compare(b) }) + + if err := safe.WriterModify(ctx, + r, + k8s.NewEndpoint(k8s.ControlPlaneNamespaceName, k8s.ControlPlaneAPIServerEndpointsID), + func(r *k8s.Endpoint) error { + if !reflect.DeepEqual(r.TypedSpec().Addresses, addrs) { + logger.Debug("updated controlplane endpoints", zap.Any("endpoints", addrs)) + } + + r.TypedSpec().Addresses = addrs + + return nil + }, + ); err != nil { + return fmt.Errorf("error updating endpoints: %w", err) + } + + return nil +} + +func (ctrl *EndpointController) watchKubernetesEndpoint(ctx context.Context, r controller.Runtime, logger *zap.Logger, kubeconfig *rest.Config) error { + client, err := kubernetes.NewForConfig(kubeconfig) + if err != nil { + return fmt.Errorf("error building Kubernetes client: %w", err) + } + + defer client.Close() //nolint:errcheck + + // abort the watch on any return from this function + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + notifyCh, watchCloser, err := kubernetesEndpointWatcher(ctx, logger, client) + if err != nil { + return fmt.Errorf("error watching Kubernetes endpoint: %w", err) + } + + defer func() { + cancel() // cancel the context before stopping the watcher + + watchCloser() + }() + + for { + select { + case endpoints := <-notifyCh: + if err = ctrl.updateEndpointsResource(ctx, r, logger, endpoints); err != nil { + return err + } + case <-ctx.Done(): + return nil + case <-r.EventCh(): + // something got updated, probably kubeconfig, restart the watch + r.QueueReconcile() + + return nil + } + } +} + +func kubernetesEndpointWatcher(ctx context.Context, logger *zap.Logger, client *kubernetes.Client) (chan *corev1.Endpoints, func(), error) { + informerFactory := informers.NewSharedInformerFactoryWithOptions( + client.Clientset, 30*time.Second, + informers.WithNamespace(corev1.NamespaceDefault), + informers.WithTweakListOptions(func(options *v1.ListOptions) { + options.FieldSelector = fields.OneTermEqualSelector("metadata.name", "kubernetes").String() + }), + ) + + notifyCh := make(chan *corev1.Endpoints, 1) + + informer := informerFactory.Core().V1().Endpoints().Informer() + + if err := informer.SetWatchErrorHandler(func(r *cache.Reflector, err error) { + logger.Error("kubernetes endpoint watch error", zap.Error(err)) + }); err != nil { + return nil, nil, fmt.Errorf("error setting watch error handler: %w", err) + } + + if _, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj any) { notifyCh <- obj.(*corev1.Endpoints) }, + DeleteFunc: func(_ any) { notifyCh <- &corev1.Endpoints{} }, + UpdateFunc: func(_, obj any) { notifyCh <- obj.(*corev1.Endpoints) }, + }); err != nil { + return nil, nil, fmt.Errorf("error adding watch event handler: %w", err) + } + + informerFactory.Start(ctx.Done()) + + return notifyCh, informerFactory.Shutdown, nil +} diff --git a/internal/app/machined/pkg/controllers/k8s/extra_manifest.go b/internal/app/machined/pkg/controllers/k8s/extra_manifest.go new file mode 100644 index 0000000..a8813fa --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/extra_manifest.go @@ -0,0 +1,261 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import ( + "context" + "fmt" + "net/http" + "os" + "path/filepath" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/hashicorp/go-cleanhttp" + "github.com/hashicorp/go-getter/v2" + "github.com/hashicorp/go-multierror" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + k8sadapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/k8s" + "github.com/siderolabs/talos/pkg/httpdefaults" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// ExtraManifestController renders manifests based on templates and config/secrets. +type ExtraManifestController struct{} + +// Name implements controller.Controller interface. +func (ctrl *ExtraManifestController) Name() string { + return "k8s.ExtraManifestController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *ExtraManifestController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: k8s.ControlPlaneNamespaceName, + Type: k8s.ExtraManifestsConfigType, + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.StatusType, + ID: optional.Some(network.StatusID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *ExtraManifestController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: k8s.ManifestType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *ExtraManifestController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + // wait for network to be ready as networking is required to download extra manifests + networkResource, err := safe.ReaderGetByID[*network.Status](ctx, r, network.StatusID) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + networkStatus := networkResource.TypedSpec() + + if !(networkStatus.AddressReady && networkStatus.ConnectivityReady) { + continue + } + + configResource, err := safe.ReaderGetByID[*k8s.ExtraManifestsConfig](ctx, r, k8s.ExtraManifestsConfigID) + if err != nil { + if state.IsNotFoundError(err) { + if err = ctrl.teardownAll(ctx, r); err != nil { + return fmt.Errorf("error tearing down: %w", err) + } + + continue + } + + return err + } + + config := *configResource.TypedSpec() + + var multiErr *multierror.Error + + presentManifests := map[resource.ID]struct{}{} + + for _, manifest := range config.ExtraManifests { + var id resource.ID + + id, err = ctrl.process(ctx, r, logger, manifest) + if err != nil { + multiErr = multierror.Append(multiErr, err) + } + + presentManifests[id] = struct{}{} + } + + if multiErr.ErrorOrNil() != nil { + return multiErr.ErrorOrNil() + } + + allManifests, err := r.List(ctx, resource.NewMetadata(k8s.ControlPlaneNamespaceName, k8s.ManifestType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing extra manifests: %w", err) + } + + for _, manifest := range allManifests.Items { + if manifest.Metadata().Owner() != ctrl.Name() { + continue + } + + if _, exists := presentManifests[manifest.Metadata().ID()]; !exists { + if err = r.Destroy(ctx, manifest.Metadata()); err != nil { + return fmt.Errorf("error cleaning up extra manifest: %w", err) + } + } + } + + r.ResetRestartBackoff() + } +} + +func (ctrl *ExtraManifestController) process(ctx context.Context, r controller.Runtime, logger *zap.Logger, manifest k8s.ExtraManifest) (id resource.ID, err error) { + id = fmt.Sprintf("%s-%s", manifest.Priority, manifest.Name) + + // inline manifests don't require download + if manifest.InlineManifest != "" { + return id, ctrl.processInline(ctx, r, manifest, id) + } + + return id, ctrl.processURL(ctx, r, logger, manifest, id) +} + +func (ctrl *ExtraManifestController) processURL(ctx context.Context, r controller.Runtime, logger *zap.Logger, manifest k8s.ExtraManifest, id resource.ID) (err error) { + var tmpDir string + + tmpDir, err = os.MkdirTemp("", "talos") + if err != nil { + return + } + + defer os.RemoveAll(tmpDir) //nolint:errcheck + + // I wish we never used go-getter package, as it doesn't allow downloading into memory. + // But there's not much we can do about it right now, as it supports lots of magic + // users might rely upon. + + // Disable netrc since we don't have getent installed, and most likely + // never will. + httpGetter := &getter.HttpGetter{ + Netrc: false, + Client: &http.Client{ + Transport: httpdefaults.PatchTransport(cleanhttp.DefaultTransport()), + }, + } + + httpGetter.Header = make(http.Header) + + for k, v := range manifest.ExtraHeaders { + httpGetter.Header.Add(k, v) + } + + client := &getter.Client{ + Getters: []getter.Getter{ + httpGetter, + }, + } + + dst := filepath.Join(tmpDir, "manifest.yaml") + + if _, err = client.Get(ctx, &getter.Request{ + Src: manifest.URL, + Dst: dst, + Pwd: tmpDir, + GetMode: getter.ModeFile, + }); err != nil { + err = fmt.Errorf("error downloading %q: %w", manifest.URL, err) + + return + } + + logger.Sugar().Infof("downloaded manifest %q", manifest.URL) + + var contents []byte + + contents, err = os.ReadFile(dst) + if err != nil { + return + } + + if err = safe.WriterModify(ctx, r, k8s.NewManifest(k8s.ControlPlaneNamespaceName, id), + func(r *k8s.Manifest) error { + return k8sadapter.Manifest(r).SetYAML(contents) + }); err != nil { + err = fmt.Errorf("error updating manifests: %w", err) + + return + } + + return nil +} + +func (ctrl *ExtraManifestController) processInline(ctx context.Context, r controller.Runtime, manifest k8s.ExtraManifest, id resource.ID) error { + err := safe.WriterModify( + ctx, + r, + k8s.NewManifest(k8s.ControlPlaneNamespaceName, id), + func(r *k8s.Manifest) error { + return k8sadapter.Manifest(r).SetYAML([]byte(manifest.InlineManifest)) + }, + ) + if err != nil { + return fmt.Errorf("error updating manifests: %w", err) + } + + return nil +} + +//nolint:dupl +func (ctrl *ExtraManifestController) teardownAll(ctx context.Context, r controller.Runtime) error { + manifests, err := r.List(ctx, resource.NewMetadata(k8s.ControlPlaneNamespaceName, k8s.ManifestType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing extra manifests: %w", err) + } + + for _, manifest := range manifests.Items { + if manifest.Metadata().Owner() != ctrl.Name() { + continue + } + + if err = r.Destroy(ctx, manifest.Metadata()); err != nil { + return fmt.Errorf("error destroying extra manifest: %w", err) + } + } + + return nil +} diff --git a/internal/app/machined/pkg/controllers/k8s/extra_manifest_test.go b/internal/app/machined/pkg/controllers/k8s/extra_manifest_test.go new file mode 100644 index 0000000..bbb5720 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/extra_manifest_test.go @@ -0,0 +1,162 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:dupl +package k8s_test + +import ( + "context" + "log" + "reflect" + "strings" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + k8sadapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/k8s" + k8sctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/k8s" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type ExtraManifestSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *ExtraManifestSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.Require().NoError(suite.runtime.RegisterController(&k8sctrl.ExtraManifestController{})) + + suite.startRuntime() +} + +func (suite *ExtraManifestSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +//nolint:dupl +func (suite *ExtraManifestSuite) assertExtraManifests(manifests []string) error { + resources, err := suite.state.List( + suite.ctx, + resource.NewMetadata(k8s.ControlPlaneNamespaceName, k8s.ManifestType, "", resource.VersionUndefined), + ) + if err != nil { + return err + } + + ids := xslices.Map(resources.Items, func(r resource.Resource) string { return r.Metadata().ID() }) + + if !reflect.DeepEqual(manifests, ids) { + return retry.ExpectedErrorf("expected %q, got %q", manifests, ids) + } + + return nil +} + +func (suite *ExtraManifestSuite) TestReconcileInlineManifests() { + configExtraManifests := k8s.NewExtraManifestsConfig() + *configExtraManifests.TypedSpec() = k8s.ExtraManifestsConfigSpec{ + ExtraManifests: []k8s.ExtraManifest{ + { + Name: "namespaces", + Priority: "99", + InlineManifest: strings.TrimSpace( + ` +apiVersion: v1 +kind: Namespace +metadata: + name: ci +--- +apiVersion: v1 +kind: Namespace +metadata: + name: build +`, + ), + }, + }, + } + + statusNetwork := network.NewStatus(network.NamespaceName, network.StatusID) + statusNetwork.TypedSpec().AddressReady = true + statusNetwork.TypedSpec().ConnectivityReady = true + + suite.Require().NoError(suite.state.Create(suite.ctx, configExtraManifests)) + suite.Require().NoError(suite.state.Create(suite.ctx, statusNetwork)) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertExtraManifests( + []string{ + "99-namespaces", + }, + ) + }, + ), + ) + + r, err := suite.state.Get( + suite.ctx, + resource.NewMetadata( + k8s.ControlPlaneNamespaceName, + k8s.ManifestType, + "99-namespaces", + resource.VersionUndefined, + ), + ) + suite.Require().NoError(err) + + manifest := r.(*k8s.Manifest) //nolint:errcheck,forcetypeassert + + suite.Assert().Len(k8sadapter.Manifest(manifest).Objects(), 2) + suite.Assert().Equal("ci", k8sadapter.Manifest(manifest).Objects()[0].GetName()) + suite.Assert().Equal("build", k8sadapter.Manifest(manifest).Objects()[1].GetName()) +} + +func (suite *ExtraManifestSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestExtraManifestSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, new(ExtraManifestSuite)) +} diff --git a/internal/app/machined/pkg/controllers/k8s/internal/nodename/nodename.go b/internal/app/machined/pkg/controllers/k8s/internal/nodename/nodename.go new file mode 100644 index 0000000..bd1d416 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/internal/nodename/nodename.go @@ -0,0 +1,56 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package nodename provides utility functions to generate nodenames. +package nodename + +import ( + "fmt" + "strings" +) + +// FromHostname converts a hostname to Kubernetes Node name. +// +// UNIX hostname has almost no restrictions, but Kubernetes Node name has +// to be RFC 1123 compliant. This function converts a hostname to a valid +// Kubernetes Node name (if possible). +// +// The allowed format is: +// +// [a-z0-9]([-a-z0-9]*[a-z0-9])? +// +//nolint:gocyclo +func FromHostname(hostname string) (string, error) { + nodename := strings.Map(func(r rune) rune { + switch { + case r >= 'a' && r <= 'z': + // allow lowercase + return r + case r >= 'A' && r <= 'Z': + // lowercase uppercase letters + return r - 'A' + 'a' + case r >= '0' && r <= '9': + // allow digits + return r + case r == '-' || r == '_': + // allow dash, convert underscore to dash + return '-' + case r == '.': + // allow dot + return '.' + default: + // drop anything else + return -1 + } + }, hostname) + + // now drop any dashes/dots at the beginning or end + nodename = strings.Trim(nodename, "-.") + + if len(nodename) == 0 { + return "", fmt.Errorf("could not convert hostname %q to a valid Kubernetes Node name", hostname) + } + + return nodename, nil +} diff --git a/internal/app/machined/pkg/controllers/k8s/internal/nodename/nodename_test.go b/internal/app/machined/pkg/controllers/k8s/internal/nodename/nodename_test.go new file mode 100644 index 0000000..74f5410 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/internal/nodename/nodename_test.go @@ -0,0 +1,73 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package nodename_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/k8s/internal/nodename" +) + +func TestFromHostname(t *testing.T) { + for _, test := range []struct { + hostname string + + expectedNodeName string + expectedError string + }{ + { + hostname: "foo", + + expectedNodeName: "foo", + }, + { + hostname: "foo_ია", + + expectedNodeName: "foo", + }, + { + hostname: "Node1", + + expectedNodeName: "node1", + }, + { + hostname: "MY_test_server_", + + expectedNodeName: "my-test-server", + }, + { + hostname: "123", + + expectedNodeName: "123", + }, + { + hostname: "-my-server-", + + expectedNodeName: "my-server", + }, + { + hostname: "კომპიუტერი", + + expectedError: "could not convert hostname \"კომპიუტერი\" to a valid Kubernetes Node name", + }, + { + hostname: "foo.bar.tld.", + + expectedNodeName: "foo.bar.tld", + }, + } { + t.Run(test.hostname, func(t *testing.T) { + nodename, err := nodename.FromHostname(test.hostname) + if test.expectedError != "" { + require.EqualError(t, err, test.expectedError) + } else { + require.NoError(t, err) + require.Equal(t, test.expectedNodeName, nodename) + } + }) + } +} diff --git a/internal/app/machined/pkg/controllers/k8s/internal/nodewatch/nodewatch.go b/internal/app/machined/pkg/controllers/k8s/internal/nodewatch/nodewatch.go new file mode 100644 index 0000000..7167274 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/internal/nodewatch/nodewatch.go @@ -0,0 +1,94 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package nodewatch implements Kubernetes node watcher. +package nodewatch + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/client-go/informers" + informersv1 "k8s.io/client-go/informers/core/v1" + "k8s.io/client-go/tools/cache" + + "github.com/siderolabs/talos/pkg/kubernetes" +) + +// NodeWatcher defines a NodeWatcher-based node watcher. +type NodeWatcher struct { + client *kubernetes.Client + + nodename string + nodes informersv1.NodeInformer +} + +// NewNodeWatcher creates new Kubernetes node watcher. +func NewNodeWatcher(client *kubernetes.Client, nodename string) *NodeWatcher { + return &NodeWatcher{ + nodename: nodename, + client: client, + } +} + +// Nodename returns the watched nodename. +func (r *NodeWatcher) Nodename() string { + return r.nodename +} + +// Get returns the Node resource. +func (r *NodeWatcher) Get() (*corev1.Node, error) { + return r.nodes.Lister().Get(r.nodename) +} + +// Watch starts watching Node state and notifies on updates via notify channel. +func (r *NodeWatcher) Watch(ctx context.Context) (<-chan struct{}, <-chan error, func(), error) { + informerFactory := informers.NewSharedInformerFactoryWithOptions( + r.client.Clientset, + 0, + informers.WithTweakListOptions( + func(opts *metav1.ListOptions) { + opts.FieldSelector = fields.OneTermEqualSelector(metav1.ObjectNameField, r.nodename).String() + }, + ), + ) + + notifyCh := make(chan struct{}, 1) + watchErrCh := make(chan error, 1) + + notify := func(_ any) { + select { + case notifyCh <- struct{}{}: + default: + } + } + + r.nodes = informerFactory.Core().V1().Nodes() + + if err := r.nodes.Informer().SetWatchErrorHandler(func(r *cache.Reflector, err error) { + select { + case watchErrCh <- err: + default: + } + }); err != nil { + return nil, nil, nil, fmt.Errorf("failed to set watch error handler: %w", err) + } + + if _, err := r.nodes.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: notify, + DeleteFunc: notify, + UpdateFunc: func(_, _ any) { notify(nil) }, + }); err != nil { + return nil, nil, nil, fmt.Errorf("failed to add event handler: %w", err) + } + + informerFactory.Start(ctx.Done()) + + informerFactory.WaitForCacheSync(ctx.Done()) + + return notifyCh, watchErrCh, informerFactory.Shutdown, nil +} diff --git a/internal/app/machined/pkg/controllers/k8s/k8s.go b/internal/app/machined/pkg/controllers/k8s/k8s.go new file mode 100644 index 0000000..0b4fa5a --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/k8s.go @@ -0,0 +1,16 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package k8s provides controllers which manage Kubernetes resources. +package k8s + +import ( + utilruntime "k8s.io/apimachinery/pkg/util/runtime" +) + +func init() { + // ugly hack, but it doesn't look like there's better API + // cut out error handler which logs error to standard logger + utilruntime.ErrorHandlers = utilruntime.ErrorHandlers[len(utilruntime.ErrorHandlers)-1:] //nolint:reassign +} diff --git a/internal/app/machined/pkg/controllers/k8s/kubelet_config.go b/internal/app/machined/pkg/controllers/k8s/kubelet_config.go new file mode 100644 index 0000000..9689075 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/kubelet_config.go @@ -0,0 +1,94 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import ( + "context" + "fmt" + "net/netip" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/controller/generic/transform" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/xerrors" + "github.com/siderolabs/gen/xslices" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +// KubeletConfigController renders kubelet configuration based on machine config. +type KubeletConfigController = transform.Controller[*config.MachineConfig, *k8s.KubeletConfig] + +// NewKubeletConfigController instanciates the config controller. +func NewKubeletConfigController() *KubeletConfigController { + return transform.NewController( + transform.Settings[*config.MachineConfig, *k8s.KubeletConfig]{ + Name: "k8s.KubeletConfigController", + MapMetadataOptionalFunc: func(cfg *config.MachineConfig) optional.Optional[*k8s.KubeletConfig] { + if cfg.Metadata().ID() != config.V1Alpha1ID { + return optional.None[*k8s.KubeletConfig]() + } + + if cfg.Config().Cluster() == nil || cfg.Config().Machine() == nil { + return optional.None[*k8s.KubeletConfig]() + } + + return optional.Some(k8s.NewKubeletConfig(k8s.NamespaceName, k8s.KubeletID)) + }, + TransformFunc: func(ctx context.Context, r controller.Reader, logger *zap.Logger, cfg *config.MachineConfig, res *k8s.KubeletConfig) error { + staticPodURL, err := safe.ReaderGetByID[*k8s.StaticPodServerStatus](ctx, r, k8s.StaticPodServerStatusResourceID) + if err != nil { + if state.IsNotFoundError(err) { + return xerrors.NewTaggedf[transform.SkipReconcileTag]("static pod server status resource not found; not creating kubelet config") + } + + return err + } + + kubeletConfig := res.TypedSpec() + cfgProvider := cfg.Config() + + kubeletConfig.Image = cfgProvider.Machine().Kubelet().Image() + + kubeletConfig.ClusterDNS = cfgProvider.Machine().Kubelet().ClusterDNS() + + if len(kubeletConfig.ClusterDNS) == 0 { + addrs, err := cfgProvider.Cluster().Network().DNSServiceIPs() + if err != nil { + return fmt.Errorf("error building DNS service IPs: %w", err) + } + + kubeletConfig.ClusterDNS = xslices.Map(addrs, netip.Addr.String) + } + + kubeletConfig.ClusterDomain = cfgProvider.Cluster().Network().DNSDomain() + kubeletConfig.ExtraArgs = cfgProvider.Machine().Kubelet().ExtraArgs() + kubeletConfig.ExtraMounts = cfgProvider.Machine().Kubelet().ExtraMounts() + kubeletConfig.ExtraConfig = cfgProvider.Machine().Kubelet().ExtraConfig() + kubeletConfig.CloudProviderExternal = cfgProvider.Cluster().ExternalCloudProvider().Enabled() + kubeletConfig.DefaultRuntimeSeccompEnabled = cfgProvider.Machine().Kubelet().DefaultRuntimeSeccompProfileEnabled() + kubeletConfig.SkipNodeRegistration = cfgProvider.Machine().Kubelet().SkipNodeRegistration() + kubeletConfig.StaticPodListURL = staticPodURL.TypedSpec().URL + kubeletConfig.DisableManifestsDirectory = cfgProvider.Machine().Kubelet().DisableManifestsDirectory() + kubeletConfig.EnableFSQuotaMonitoring = cfgProvider.Machine().Features().DiskQuotaSupportEnabled() + kubeletConfig.CredentialProviderConfig = cfgProvider.Machine().Kubelet().CredentialProviderConfig() + + return nil + }, + }, + transform.WithExtraInputs( + controller.Input{ + Namespace: k8s.NamespaceName, + Type: k8s.StaticPodServerStatusType, + ID: optional.Some(k8s.StaticPodServerStatusResourceID), + Kind: controller.InputWeak, + }, + ), + ) +} diff --git a/internal/app/machined/pkg/controllers/k8s/kubelet_config_test.go b/internal/app/machined/pkg/controllers/k8s/kubelet_config_test.go new file mode 100644 index 0000000..46af6bc --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/kubelet_config_test.go @@ -0,0 +1,265 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:dupl +package k8s_test + +import ( + "context" + "log" + "net/url" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/opencontainers/runtime-spec/specs-go" + "github.com/siderolabs/go-pointer" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + k8sctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/k8s" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +type KubeletConfigSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *KubeletConfigSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.Require().NoError(suite.runtime.RegisterController(k8sctrl.NewKubeletConfigController())) + + suite.startRuntime() +} + +func (suite *KubeletConfigSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *KubeletConfigSuite) createStaticPodServerStatus() { + staticPodServerStatus := k8s.NewStaticPodServerStatus(k8s.NamespaceName, k8s.StaticPodServerStatusResourceID) + + staticPodServerStatus.TypedSpec().URL = "http://127.0.0.1:12345" + + suite.Require().NoError(suite.state.Create(suite.ctx, staticPodServerStatus)) +} + +func (suite *KubeletConfigSuite) TestReconcile() { + u, err := url.Parse("https://foo:6443") + suite.Require().NoError(err) + + suite.createStaticPodServerStatus() + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineKubelet: &v1alpha1.KubeletConfig{ + KubeletImage: "kubelet", + KubeletClusterDNS: []string{"10.0.0.1"}, + KubeletExtraArgs: map[string]string{ + "enable-feature": "foo", + }, + KubeletExtraMounts: []v1alpha1.ExtraMount{ + { + Destination: "/tmp", + Source: "/var", + Type: "tmpfs", + }, + }, + KubeletExtraConfig: v1alpha1.Unstructured{ + Object: map[string]any{ + "serverTLSBootstrap": true, + }, + }, + KubeletDefaultRuntimeSeccompProfileEnabled: pointer.To(true), + }, + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: u, + }, + }, + ExternalCloudProviderConfig: &v1alpha1.ExternalCloudProviderConfig{ + ExternalEnabled: pointer.To(true), + }, + ClusterNetwork: &v1alpha1.ClusterNetworkConfig{ + DNSDomain: "service.svc", + }, + }, + }, + ), + ) + + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + kubeletConfig, err := suite.state.Get( + suite.ctx, + resource.NewMetadata( + k8s.NamespaceName, + k8s.KubeletConfigType, + k8s.KubeletID, + resource.VersionUndefined, + ), + ) + if err != nil { + if state.IsNotFoundError(err) { + return retry.ExpectedError(err) + } + + return err + } + + spec := kubeletConfig.(*k8s.KubeletConfig).TypedSpec() + + suite.Assert().Equal("kubelet", spec.Image) + suite.Assert().Equal([]string{"10.0.0.1"}, spec.ClusterDNS) + suite.Assert().Equal("service.svc", spec.ClusterDomain) + suite.Assert().Equal( + map[string]string{ + "enable-feature": "foo", + }, + spec.ExtraArgs, + ) + suite.Assert().Equal( + []specs.Mount{ + { + Destination: "/tmp", + Source: "/var", + Type: "tmpfs", + }, + }, + spec.ExtraMounts, + ) + suite.Assert().Equal( + map[string]any{ + "serverTLSBootstrap": true, + }, + spec.ExtraConfig, + ) + suite.Assert().True(spec.CloudProviderExternal) + suite.Assert().True(spec.DefaultRuntimeSeccompEnabled) + + return nil + }, + ), + ) +} + +func (suite *KubeletConfigSuite) TestReconcileDefaults() { + u, err := url.Parse("https://foo:6443") + suite.Require().NoError(err) + + suite.createStaticPodServerStatus() + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineKubelet: &v1alpha1.KubeletConfig{ + KubeletImage: "kubelet", + }, + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: u, + }, + }, + ClusterNetwork: &v1alpha1.ClusterNetworkConfig{ + ServiceSubnet: []string{constants.DefaultIPv4ServiceNet}, + }, + }, + }, + ), + ) + + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + kubeletConfig, err := suite.state.Get( + suite.ctx, + resource.NewMetadata( + k8s.NamespaceName, + k8s.KubeletConfigType, + k8s.KubeletID, + resource.VersionUndefined, + ), + ) + if err != nil { + if state.IsNotFoundError(err) { + return retry.ExpectedError(err) + } + + return err + } + + spec := kubeletConfig.(*k8s.KubeletConfig).TypedSpec() + + suite.Assert().Equal("kubelet", spec.Image) + suite.Assert().Equal([]string{"10.96.0.10"}, spec.ClusterDNS) + suite.Assert().Equal(constants.DefaultDNSDomain, spec.ClusterDomain) + suite.Assert().Empty(spec.ExtraArgs) + suite.Assert().Empty(spec.ExtraMounts) + suite.Assert().False(spec.CloudProviderExternal) + + return nil + }, + ), + ) +} + +func (suite *KubeletConfigSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestKubeletConfigSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, new(KubeletConfigSuite)) +} diff --git a/internal/app/machined/pkg/controllers/k8s/kubelet_service.go b/internal/app/machined/pkg/controllers/k8s/kubelet_service.go new file mode 100644 index 0000000..5259b6d --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/kubelet_service.go @@ -0,0 +1,565 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import ( + "bytes" + "context" + "crypto/x509" + "encoding/base64" + stdjson "encoding/json" + "encoding/pem" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "text/template" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + talosx509 "github.com/siderolabs/crypto/x509" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/xslices" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer/json" + "k8s.io/client-go/tools/clientcmd" + kubeletconfig "k8s.io/kubelet/config/v1beta1" + + runtimetalos "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/system" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/services" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/files" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" +) + +// ServiceManager is the interface to the v1alpha1 services subsystems. +type ServiceManager interface { + IsRunning(id string) (system.Service, bool, error) + Load(services ...system.Service) []string + Stop(ctx context.Context, serviceIDs ...string) (err error) + Start(serviceIDs ...string) error +} + +// KubeletServiceController renders kubelet configuration files and controls kubelet service lifecycle. +type KubeletServiceController struct { + V1Alpha1Services ServiceManager + V1Alpha1Mode runtimetalos.Mode +} + +// Name implements controller.Controller interface. +func (ctrl *KubeletServiceController) Name() string { + return "k8s.KubeletServiceController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *KubeletServiceController) Inputs() []controller.Input { + return nil +} + +// Outputs implements controller.Controller interface. +func (ctrl *KubeletServiceController) Outputs() []controller.Output { + return nil +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *KubeletServiceController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + // initially, wait for the machine-id to be generated and /var to be mounted + if err := r.UpdateInputs([]controller.Input{ + { + Namespace: files.NamespaceName, + Type: files.EtcFileStatusType, + ID: optional.Some("machine-id"), + Kind: controller.InputWeak, + }, + { + Namespace: runtimeres.NamespaceName, + Type: runtimeres.MountStatusType, + ID: optional.Some(constants.EphemeralPartitionLabel), + Kind: controller.InputWeak, + }, + }); err != nil { + return err + } + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + _, err := r.Get(ctx, resource.NewMetadata(files.NamespaceName, files.EtcFileStatusType, "machine-id", resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting etc file status: %w", err) + } + + _, err = r.Get(ctx, resource.NewMetadata(runtimeres.NamespaceName, runtimeres.MountStatusType, constants.EphemeralPartitionLabel, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + // in container mode EPHEMERAL is always mounted + if ctrl.V1Alpha1Mode != runtimetalos.ModeContainer { + // wait for the EPHEMERAL to be mounted + continue + } + } else { + return fmt.Errorf("error getting ephemeral mount status: %w", err) + } + } + + break + } + + // normal reconcile loop + if err := r.UpdateInputs([]controller.Input{ + { + Namespace: k8s.NamespaceName, + Type: k8s.KubeletSpecType, + ID: optional.Some(k8s.KubeletID), + Kind: controller.InputWeak, + }, + { + Namespace: secrets.NamespaceName, + Type: secrets.KubeletType, + ID: optional.Some(secrets.KubeletID), + Kind: controller.InputWeak, + }, + }); err != nil { + return err + } + + r.QueueReconcile() + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + cfg, err := safe.ReaderGetByID[*k8s.KubeletSpec](ctx, r, k8s.KubeletID) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting config: %w", err) + } + + cfgSpec := cfg.TypedSpec() + + secret, err := safe.ReaderGetByID[*secrets.Kubelet](ctx, r, secrets.KubeletID) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting secrets: %w", err) + } + + secretSpec := secret.TypedSpec() + + if err = ctrl.writePKI(secretSpec); err != nil { + return fmt.Errorf("error writing kubelet PKI: %w", err) + } + + if err = ctrl.writeConfig(cfgSpec); err != nil { + return fmt.Errorf("error writing kubelet configuration: %w", err) + } + + if err = ctrl.writeKubeletCredentialProviderConfig(cfgSpec); err != nil { + return fmt.Errorf("error writing kubelet credential provider configuration: %w", err) + } + + _, running, err := ctrl.V1Alpha1Services.IsRunning("kubelet") + if err != nil { + ctrl.V1Alpha1Services.Load(&services.Kubelet{}) + } + + if running { + if err = ctrl.V1Alpha1Services.Stop(ctx, "kubelet"); err != nil { + return fmt.Errorf("error stopping kubelet service: %w", err) + } + } + + if err = ctrl.refreshKubeletCerts(cfgSpec.ExpectedNodename, secretSpec.AcceptedCAs, logger); err != nil { + return err + } + + if err = ctrl.handlePolicyChange(cfgSpec, logger); err != nil { + return err + } + + if err = ctrl.refreshSelfServingCert(); err != nil { + return err + } + + if err = ctrl.updateKubeconfig(secretSpec.Endpoint, secretSpec.AcceptedCAs, logger); err != nil { + return err + } + + if err = ctrl.V1Alpha1Services.Start("kubelet"); err != nil { + return fmt.Errorf("error starting kubelet service: %w", err) + } + + r.ResetRestartBackoff() + } +} + +// handlePolicyChange handles the cpuManagerPolicy change. +func (ctrl *KubeletServiceController) handlePolicyChange(cfgSpec *k8s.KubeletSpecSpec, logger *zap.Logger) error { + const managerFilename = "/var/lib/kubelet/cpu_manager_state" + + oldPolicy, err := loadPolicyFromFile(managerFilename) + + switch { + case errors.Is(err, os.ErrNotExist): + return nil // no cpu_manager_state file, nothing to do + case err != nil: + return fmt.Errorf("error loading cpu_manager_state file: %w", err) + } + + policy, err := getFromMap[string](cfgSpec.Config, "cpuManagerPolicy") + if err != nil { + return err + } + + newPolicy := policy.ValueOrZero() + if equalPolicy(oldPolicy, newPolicy) { + return nil + } + + logger.Info("cpuManagerPolicy changed", zap.String("old", oldPolicy), zap.String("new", newPolicy)) + + err = os.Remove(managerFilename) + if err != nil { + return fmt.Errorf("error removing cpu_manager_state file: %w", err) + } + + return nil +} + +func loadPolicyFromFile(filename string) (string, error) { + raw, err := os.ReadFile(filename) + if err != nil { + return "", err + } + + cpuManagerState := struct { + Policy string `json:"policyName"` + }{} + + if err = stdjson.Unmarshal(raw, &cpuManagerState); err != nil { + return "", err + } + + return cpuManagerState.Policy, nil +} + +func equalPolicy(current, newOne string) bool { + if current == "none" { + current = "" + } + + if newOne == "none" { + newOne = "" + } + + return current == newOne +} + +func getFromMap[T any](m map[string]any, key string) (optional.Optional[T], error) { + var zero optional.Optional[T] + + res, ok := m[key] + if !ok { + return zero, nil + } + + if res, ok := res.(T); ok { + return optional.Some(res), nil + } + + return zero, fmt.Errorf("unexpected type for key %q: found %T, expected %T", key, res, *new(T)) +} + +func (ctrl *KubeletServiceController) writePKI(secretSpec *secrets.KubeletSpec) error { + acceptedCAs := bytes.Join(xslices.Map(secretSpec.AcceptedCAs, func(ca *talosx509.PEMEncodedCertificate) []byte { return ca.Crt }), nil) + + cfg := struct { + Server string + CACert string + BootstrapTokenID string + BootstrapTokenSecret string + }{ + Server: secretSpec.Endpoint.String(), + CACert: base64.StdEncoding.EncodeToString(acceptedCAs), + BootstrapTokenID: secretSpec.BootstrapTokenID, + BootstrapTokenSecret: secretSpec.BootstrapTokenSecret, + } + + templ := template.Must(template.New("tmpl").Parse(string(kubeletKubeConfigTemplate))) + + var buf bytes.Buffer + + if err := templ.Execute(&buf, cfg); err != nil { + return err + } + + if err := os.WriteFile(constants.KubeletBootstrapKubeconfig, buf.Bytes(), 0o600); err != nil { + return err + } + + if err := os.MkdirAll(filepath.Dir(constants.KubernetesCACert), 0o700); err != nil { + return err + } + + return os.WriteFile(constants.KubernetesCACert, acceptedCAs, 0o400) +} + +var kubeletKubeConfigTemplate = []byte(`apiVersion: v1 +kind: Config +clusters: +- name: local + cluster: + server: {{ .Server }} + certificate-authority-data: {{ .CACert }} +users: +- name: kubelet + user: + token: {{ .BootstrapTokenID }}.{{ .BootstrapTokenSecret }} +contexts: +- context: + cluster: local + user: kubelet +`) + +func (ctrl *KubeletServiceController) writeConfig(cfgSpec *k8s.KubeletSpecSpec) error { + var kubeletConfiguration kubeletconfig.KubeletConfiguration + + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(cfgSpec.Config, &kubeletConfiguration); err != nil { + return fmt.Errorf("error converting kubelet configuration from unstructured: %w", err) + } + + serializer := json.NewSerializerWithOptions( + json.DefaultMetaFactory, + nil, + nil, + json.SerializerOptions{ + Yaml: true, + }, + ) + + var buf bytes.Buffer + + if err := serializer.Encode(&kubeletConfiguration, &buf); err != nil { + return err + } + + return os.WriteFile("/etc/kubernetes/kubelet.yaml", buf.Bytes(), 0o600) +} + +func (ctrl *KubeletServiceController) writeKubeletCredentialProviderConfig(cfgSpec *k8s.KubeletSpecSpec) error { + if cfgSpec.CredentialProviderConfig == nil { + return os.RemoveAll(constants.KubeletCredentialProviderConfig) + } + + var kubeletCredentialProviderConfig kubeletconfig.CredentialProviderConfig + + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(cfgSpec.CredentialProviderConfig, &kubeletCredentialProviderConfig); err != nil { + return fmt.Errorf("error converting kubelet credentialprovider configuration from unstructured: %w", err) + } + + serializer := json.NewSerializerWithOptions( + json.DefaultMetaFactory, + nil, + nil, + json.SerializerOptions{ + Yaml: true, + }, + ) + + var buf bytes.Buffer + + if err := serializer.Encode(&kubeletCredentialProviderConfig, &buf); err != nil { + return err + } + + return os.WriteFile(constants.KubeletCredentialProviderConfig, buf.Bytes(), 0o600) +} + +// updateKubeconfig updates the kubeconfig of kubelet with the given endpoint if it exists. +func (ctrl *KubeletServiceController) updateKubeconfig(newEndpoint *url.URL, acceptedCAs []*talosx509.PEMEncodedCertificate, logger *zap.Logger) error { + config, err := clientcmd.LoadFromFile(constants.KubeletKubeconfig) + if errors.Is(err, os.ErrNotExist) { + return nil + } + + if err != nil { + return err + } + + context := config.Contexts[config.CurrentContext] + if context == nil { + // this should never happen, but we can't fix kubeconfig if it is malformed + logger.Error("kubeconfig is missing current context", zap.String("context", config.CurrentContext)) + + return nil + } + + cluster := config.Clusters[context.Cluster] + + if cluster == nil { + // this should never happen, but we can't fix kubeconfig if it is malformed + logger.Error("kubeconfig is missing cluster", zap.String("context", config.CurrentContext), zap.String("cluster", context.Cluster)) + + return nil + } + + cluster.Server = newEndpoint.String() + cluster.CertificateAuthorityData = bytes.Join(xslices.Map(acceptedCAs, func(ca *talosx509.PEMEncodedCertificate) []byte { return ca.Crt }), nil) + + return clientcmd.WriteToFile(*config, constants.KubeletKubeconfig) +} + +// refreshKubeletCerts checks if the existing kubelet certificates match the node hostname and expected CA. +// If they don't match, it clears the certificate directory and the removes kubelet's kubeconfig so that +// they can be regenerated next time kubelet is started. +// +//nolint:gocyclo +func (ctrl *KubeletServiceController) refreshKubeletCerts(expectedNodename string, acceptedCAs []*talosx509.PEMEncodedCertificate, logger *zap.Logger) error { + cert, err := ctrl.readKubeletClientCertificate() + if err != nil { + return err + } + + if cert == nil { + return nil + } + + valid := true + + // refresh certs only if we are managing the node name (not overridden by the user) + if expectedNodename != "" { + expectedCommonName := fmt.Sprintf("system:node:%s", expectedNodename) + + valid = valid && expectedCommonName == cert.Subject.CommonName + + if !valid { + logger.Info("kubelet client certificate does not match expected nodename, removing", + zap.String("expected", expectedCommonName), + zap.String("actual", cert.Subject.CommonName), + ) + } + } + + // check against CAs + if valid { + rootCAs := x509.NewCertPool() + + for _, ca := range acceptedCAs { + if !rootCAs.AppendCertsFromPEM(ca.Crt) { + return fmt.Errorf("error adding CA to root pool: %w", err) + } + } + + _, verifyErr := cert.Verify(x509.VerifyOptions{ + Roots: rootCAs, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}, + }) + + valid = valid && verifyErr == nil + + if !valid { + logger.Info("kubelet client certificate does not match any accepted CAs, removing", zap.NamedError("verify_error", verifyErr)) + } + } + + if valid { + // certificate looks good, no need to refresh + return nil + } + + // remove the pki directory + err = os.RemoveAll(constants.KubeletPKIDir) + if err != nil { + return err + } + + // clear the kubelet kubeconfig + err = os.Remove(constants.KubeletKubeconfig) + if errors.Is(err, os.ErrNotExist) { + return nil + } + + return err +} + +// refreshSelfServingCert removes the self-signed serving certificate (if exists) to force the kubelet to renew it. +func (ctrl *KubeletServiceController) refreshSelfServingCert() error { + for _, filename := range []string{ + "kubelet.crt", + "kubelet.key", + } { + path := filepath.Join(constants.KubeletPKIDir, filename) + + _, err := os.Stat(path) + if err == nil { + err = os.Remove(path) + if err != nil { + return fmt.Errorf("error removing self-signed certificate: %w", err) + } + } + } + + return nil +} + +func (ctrl *KubeletServiceController) readKubeletClientCertificate() (*x509.Certificate, error) { + raw, err := os.ReadFile(filepath.Join(constants.KubeletPKIDir, "kubelet-client-current.pem")) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + + if err != nil { + return nil, err + } + + for { + block, rest := pem.Decode(raw) + if block == nil { + return nil, nil + } + + raw = rest + + if block.Type != "CERTIFICATE" { + continue + } + + var cert *x509.Certificate + + cert, err = x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, err + } + + if !cert.IsCA { + return cert, nil + } + } +} diff --git a/internal/app/machined/pkg/controllers/k8s/kubelet_spec.go b/internal/app/machined/pkg/controllers/k8s/kubelet_spec.go new file mode 100644 index 0000000..14a23a9 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/kubelet_spec.go @@ -0,0 +1,367 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import ( + "context" + "fmt" + "net/netip" + "strings" + "time" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/hashicorp/go-multierror" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/go-kubernetes/kubernetes/compatibility" + "github.com/siderolabs/go-pointer" + "go.uber.org/zap" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + kubeletconfig "k8s.io/kubelet/config/v1beta1" + + v1alpha1runtime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/pkg/cgroup" + "github.com/siderolabs/talos/pkg/argsbuilder" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/kubelet" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +// KubeletSpecController renders manifests based on templates and config/secrets. +type KubeletSpecController struct { + V1Alpha1Mode v1alpha1runtime.Mode +} + +// Name implements controller.Controller interface. +func (ctrl *KubeletSpecController) Name() string { + return "k8s.KubeletSpecController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *KubeletSpecController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: k8s.NamespaceName, + Type: k8s.KubeletConfigType, + ID: optional.Some(k8s.KubeletID), + Kind: controller.InputWeak, + }, + { + Namespace: k8s.NamespaceName, + Type: k8s.NodenameType, + ID: optional.Some(k8s.NodenameID), + Kind: controller.InputWeak, + }, + { + Namespace: k8s.NamespaceName, + Type: k8s.NodeIPType, + ID: optional.Some(k8s.KubeletID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *KubeletSpecController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: k8s.KubeletSpecType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *KubeletSpecController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + cfg, err := safe.ReaderGetByID[*k8s.KubeletConfig](ctx, r, k8s.KubeletID) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting config: %w", err) + } + + cfgSpec := cfg.TypedSpec() + + kubeletVersion := compatibility.VersionFromImageRef(cfgSpec.Image) + + nodename, err := safe.ReaderGetByID[*k8s.Nodename](ctx, r, k8s.NodenameID) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting nodename: %w", err) + } + + expectedNodename := nodename.TypedSpec().Nodename + + args := argsbuilder.Args{ + "config": "/etc/kubernetes/kubelet.yaml", + + "cert-dir": constants.KubeletPKIDir, + + "hostname-override": expectedNodename, + } + + if !cfgSpec.SkipNodeRegistration { + args["bootstrap-kubeconfig"] = constants.KubeletBootstrapKubeconfig + args["kubeconfig"] = constants.KubeletKubeconfig + } + + if cfgSpec.CloudProviderExternal { + args["cloud-provider"] = "external" + } + + if !kubeletVersion.SupportsKubeletConfigContainerRuntimeEndpoint() { + args["container-runtime-endpoint"] = constants.CRIContainerdAddress + } + + extraArgs := argsbuilder.Args(cfgSpec.ExtraArgs) + + // if the user supplied a hostname override, we do not manage it anymore + if extraArgs.Contains("hostname-override") { + expectedNodename = "" + } + + // if the user supplied node-ip via extra args, no need to pick automatically + if !extraArgs.Contains("node-ip") { + nodeIP, nodeErr := safe.ReaderGetByID[*k8s.NodeIP](ctx, r, k8s.KubeletID) + if nodeErr != nil { + if state.IsNotFoundError(nodeErr) { + continue + } + + return fmt.Errorf("error getting node IPs: %w", nodeErr) + } + + nodeIPsString := xslices.Map(nodeIP.TypedSpec().Addresses, netip.Addr.String) + args["node-ip"] = strings.Join(nodeIPsString, ",") + } + + if err = args.Merge(extraArgs, argsbuilder.WithMergePolicies( + argsbuilder.MergePolicies{ + "bootstrap-kubeconfig": argsbuilder.MergeDenied, + "kubeconfig": argsbuilder.MergeDenied, + "container-runtime": argsbuilder.MergeDenied, + "container-runtime-endpoint": argsbuilder.MergeDenied, + "config": argsbuilder.MergeDenied, + "cert-dir": argsbuilder.MergeDenied, + }, + )); err != nil { + return fmt.Errorf("error merging arguments: %w", err) + } + + // these flags are present from v1.24 + if cfgSpec.CredentialProviderConfig != nil { + args["image-credential-provider-bin-dir"] = constants.KubeletCredentialProviderBinDir + args["image-credential-provider-config"] = constants.KubeletCredentialProviderConfig + } + + kubeletConfig, err := NewKubeletConfiguration(cfgSpec, kubeletVersion) + if err != nil { + return fmt.Errorf("error creating kubelet configuration: %w", err) + } + + // If our platform is container, we cannot rely on the ability to change kernel parameters. + // Therefore, we need to NOT attempt to enforce the kernel parameter checking done by the kubelet + // when the `ProtectKernelDefaults` setting is enabled. + if ctrl.V1Alpha1Mode == v1alpha1runtime.ModeContainer { + kubeletConfig.ProtectKernelDefaults = false + } + + unstructuredConfig, err := runtime.DefaultUnstructuredConverter.ToUnstructured(kubeletConfig) + if err != nil { + return fmt.Errorf("error converting to unstructured: %w", err) + } + + if err = safe.WriterModify( + ctx, + r, + k8s.NewKubeletSpec(k8s.NamespaceName, k8s.KubeletID), + func(r *k8s.KubeletSpec) error { + kubeletSpec := r.TypedSpec() + + kubeletSpec.Image = cfgSpec.Image + kubeletSpec.ExtraMounts = cfgSpec.ExtraMounts + kubeletSpec.Args = args.Args() + kubeletSpec.Config = unstructuredConfig + kubeletSpec.ExpectedNodename = expectedNodename + kubeletSpec.CredentialProviderConfig = cfgSpec.CredentialProviderConfig + + return nil + }, + ); err != nil { + return fmt.Errorf("error modifying KubeletSpec resource: %w", err) + } + + r.ResetRestartBackoff() + } +} + +func prepareExtraConfig(extraConfig map[string]any) (*kubeletconfig.KubeletConfiguration, error) { + // check for fields that can't be overridden via extraConfig + var multiErr *multierror.Error + + for _, field := range kubelet.ProtectedConfigurationFields { + if _, exists := extraConfig[field]; exists { + multiErr = multierror.Append(multiErr, fmt.Errorf("field %q can't be overridden", field)) + } + } + + if err := multiErr.ErrorOrNil(); err != nil { + return nil, err + } + + var config kubeletconfig.KubeletConfiguration + + // unmarshal extra config into the config structure + // as unmarshalling zeroes the missing fields, we can't do that after setting the defaults + if err := runtime.DefaultUnstructuredConverter.FromUnstructuredWithValidation(extraConfig, &config, true); err != nil { + return nil, fmt.Errorf("error unmarshalling extra kubelet configuration: %w", err) + } + + return &config, nil +} + +// NewKubeletConfiguration builds kubelet configuration with defaults and overrides from extraConfig. +// +//nolint:gocyclo,cyclop +func NewKubeletConfiguration(cfgSpec *k8s.KubeletConfigSpec, kubeletVersion compatibility.Version) (*kubeletconfig.KubeletConfiguration, error) { + config, err := prepareExtraConfig(cfgSpec.ExtraConfig) + if err != nil { + return nil, err + } + + // required fields (always set) + config.TypeMeta = metav1.TypeMeta{ + APIVersion: kubeletconfig.SchemeGroupVersion.String(), + Kind: "KubeletConfiguration", + } + + if cfgSpec.DisableManifestsDirectory { + config.StaticPodPath = "" + } else { + config.StaticPodPath = constants.ManifestsDirectory + } + + config.StaticPodURL = cfgSpec.StaticPodListURL + config.Port = constants.KubeletPort + config.Authentication = kubeletconfig.KubeletAuthentication{ + X509: kubeletconfig.KubeletX509Authentication{ + ClientCAFile: constants.KubernetesCACert, + }, + Webhook: kubeletconfig.KubeletWebhookAuthentication{ + Enabled: pointer.To(true), + }, + Anonymous: kubeletconfig.KubeletAnonymousAuthentication{ + Enabled: pointer.To(false), + }, + } + config.Authorization = kubeletconfig.KubeletAuthorization{ + Mode: kubeletconfig.KubeletAuthorizationModeWebhook, + } + config.CgroupRoot = cgroup.Root() + config.SystemCgroups = cgroup.Path(constants.CgroupSystem) + config.KubeletCgroups = cgroup.Path(constants.CgroupKubelet) + config.RotateCertificates = true + config.ProtectKernelDefaults = true + + if kubeletVersion.SupportsKubeletConfigContainerRuntimeEndpoint() { + config.ContainerRuntimeEndpoint = "unix://" + constants.CRIContainerdAddress + } + + if cfgSpec.DefaultRuntimeSeccompEnabled { + config.SeccompDefault = pointer.To(true) + } + + if cfgSpec.EnableFSQuotaMonitoring { + if _, overridden := config.FeatureGates["LocalStorageCapacityIsolationFSQuotaMonitoring"]; !overridden { + if config.FeatureGates == nil { + config.FeatureGates = map[string]bool{} + } + + config.FeatureGates["LocalStorageCapacityIsolationFSQuotaMonitoring"] = true + } + } + + if cfgSpec.SkipNodeRegistration { + config.Authentication.Webhook.Enabled = pointer.To(false) + config.Authorization.Mode = kubeletconfig.KubeletAuthorizationModeAlwaysAllow + } + + // fields which can be overridden + if config.Address == "" { + config.Address = "0.0.0.0" + } + + if config.OOMScoreAdj == nil { + config.OOMScoreAdj = pointer.To[int32](constants.KubeletOOMScoreAdj) + } + + if config.ClusterDomain == "" { + config.ClusterDomain = cfgSpec.ClusterDomain + } + + if len(config.ClusterDNS) == 0 { + config.ClusterDNS = cfgSpec.ClusterDNS + } + + if config.SerializeImagePulls == nil { + config.SerializeImagePulls = pointer.To(false) + } + + if config.FailSwapOn == nil { + config.FailSwapOn = pointer.To(false) + } + + if len(config.SystemReserved) == 0 { + config.SystemReserved = map[string]string{ + "cpu": constants.KubeletSystemReservedCPU, + "memory": constants.KubeletSystemReservedMemory, + "pid": constants.KubeletSystemReservedPid, + "ephemeral-storage": constants.KubeletSystemReservedEphemeralStorage, + } + } + + if config.Logging.Format == "" { + config.Logging.Format = "json" + } + + extraConfig := cfgSpec.ExtraConfig + + if _, overridden := extraConfig["shutdownGracePeriod"]; !overridden && config.ShutdownGracePeriod.Duration == 0 { + config.ShutdownGracePeriod = metav1.Duration{Duration: constants.KubeletShutdownGracePeriod} + } + + if _, overridden := extraConfig["shutdownGracePeriodCriticalPods"]; !overridden && config.ShutdownGracePeriodCriticalPods.Duration == 0 { + config.ShutdownGracePeriodCriticalPods = metav1.Duration{Duration: constants.KubeletShutdownGracePeriodCriticalPods} + } + + if config.StreamingConnectionIdleTimeout.Duration == 0 { + config.StreamingConnectionIdleTimeout = metav1.Duration{Duration: 5 * time.Minute} + } + + if config.TLSMinVersion == "" { + config.TLSMinVersion = "VersionTLS13" + } + + config.ResolverConfig = pointer.To(constants.PodResolvConfPath) + + return config, nil +} diff --git a/internal/app/machined/pkg/controllers/k8s/kubelet_spec_test.go b/internal/app/machined/pkg/controllers/k8s/kubelet_spec_test.go new file mode 100644 index 0000000..4ac2a38 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/kubelet_spec_test.go @@ -0,0 +1,474 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:goconst +package k8s_test + +import ( + "net/netip" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/opencontainers/runtime-spec/specs-go" + "github.com/siderolabs/go-kubernetes/kubernetes/compatibility" + "github.com/siderolabs/go-pointer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sruntime "k8s.io/apimachinery/pkg/runtime" + v1 "k8s.io/component-base/logs/api/v1" + kubeletconfig "k8s.io/kubelet/config/v1beta1" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + k8sctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/k8s" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +type KubeletSpecSuite struct { + ctest.DefaultSuite +} + +func (suite *KubeletSpecSuite) TestReconcileDefault() { + cfg := k8s.NewKubeletConfig(k8s.NamespaceName, k8s.KubeletID) + cfg.TypedSpec().Image = "kubelet:v1.29.0" + cfg.TypedSpec().ClusterDNS = []string{"10.96.0.10"} + cfg.TypedSpec().ClusterDomain = "cluster.local" + cfg.TypedSpec().ExtraArgs = map[string]string{"foo": "bar"} + cfg.TypedSpec().ExtraMounts = []specs.Mount{ + { + Destination: "/tmp", + Source: "/var", + Type: "tmpfs", + }, + } + cfg.TypedSpec().CloudProviderExternal = true + + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + nodeIP := k8s.NewNodeIP(k8s.NamespaceName, k8s.KubeletID) + nodeIP.TypedSpec().Addresses = []netip.Addr{netip.MustParseAddr("172.20.0.2")} + + suite.Require().NoError(suite.State().Create(suite.Ctx(), nodeIP)) + + nodename := k8s.NewNodename(k8s.NamespaceName, k8s.NodenameID) + nodename.TypedSpec().Nodename = "example.com" + + suite.Require().NoError(suite.State().Create(suite.Ctx(), nodename)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.KubeletID}, func(kubeletSpec *k8s.KubeletSpec, asrt *assert.Assertions) { + spec := kubeletSpec.TypedSpec() + + asrt.Equal(cfg.TypedSpec().Image, spec.Image) + asrt.Equal( + []string{ + "--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubeconfig", + "--cert-dir=/var/lib/kubelet/pki", + "--cloud-provider=external", + "--config=/etc/kubernetes/kubelet.yaml", + "--foo=bar", + "--hostname-override=example.com", + "--kubeconfig=/etc/kubernetes/kubeconfig-kubelet", + "--node-ip=172.20.0.2", + }, spec.Args, + ) + asrt.Equal(cfg.TypedSpec().ExtraMounts, spec.ExtraMounts) + + asrt.Equal([]any{"10.96.0.10"}, spec.Config["clusterDNS"]) + asrt.Equal("cluster.local", spec.Config["clusterDomain"]) + }) +} + +func (suite *KubeletSpecSuite) TestReconcileWithExplicitNodeIP() { + cfg := k8s.NewKubeletConfig(k8s.NamespaceName, k8s.KubeletID) + cfg.TypedSpec().Image = "kubelet:v1.29.0" + cfg.TypedSpec().ClusterDNS = []string{"10.96.0.10"} + cfg.TypedSpec().ClusterDomain = "cluster.local" + cfg.TypedSpec().ExtraArgs = map[string]string{"node-ip": "10.0.0.1"} + + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + nodename := k8s.NewNodename(k8s.NamespaceName, k8s.NodenameID) + nodename.TypedSpec().Nodename = "example.com" + + suite.Require().NoError(suite.State().Create(suite.Ctx(), nodename)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.KubeletID}, func(kubeletSpec *k8s.KubeletSpec, asrt *assert.Assertions) { + spec := kubeletSpec.TypedSpec() + + asrt.Equal(cfg.TypedSpec().Image, spec.Image) + asrt.Equal( + []string{ + "--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubeconfig", + "--cert-dir=/var/lib/kubelet/pki", + "--config=/etc/kubernetes/kubelet.yaml", + "--hostname-override=example.com", + "--kubeconfig=/etc/kubernetes/kubeconfig-kubelet", + "--node-ip=10.0.0.1", + }, spec.Args, + ) + }) +} + +func (suite *KubeletSpecSuite) TestReconcileWithContainerRuntimeEnpointFlag() { + cfg := k8s.NewKubeletConfig(k8s.NamespaceName, k8s.KubeletID) + cfg.TypedSpec().Image = "kubelet:v1.25.0" + cfg.TypedSpec().ClusterDNS = []string{"10.96.0.10"} + cfg.TypedSpec().ClusterDomain = "cluster.local" + cfg.TypedSpec().ExtraArgs = map[string]string{"node-ip": "10.0.0.1"} + + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + nodename := k8s.NewNodename(k8s.NamespaceName, k8s.NodenameID) + nodename.TypedSpec().Nodename = "example.com" + + suite.Require().NoError(suite.State().Create(suite.Ctx(), nodename)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.KubeletID}, func(kubeletSpec *k8s.KubeletSpec, asrt *assert.Assertions) { + spec := kubeletSpec.TypedSpec() + + asrt.Equal(cfg.TypedSpec().Image, spec.Image) + asrt.Equal( + []string{ + "--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubeconfig", + "--cert-dir=/var/lib/kubelet/pki", + "--config=/etc/kubernetes/kubelet.yaml", + "--container-runtime-endpoint=/run/containerd/containerd.sock", + "--hostname-override=example.com", + "--kubeconfig=/etc/kubernetes/kubeconfig-kubelet", + "--node-ip=10.0.0.1", + }, spec.Args, + ) + + var kubeletConfiguration kubeletconfig.KubeletConfiguration + + if err := k8sruntime.DefaultUnstructuredConverter.FromUnstructured( + spec.Config, + &kubeletConfiguration, + ); err != nil { + asrt.NoError(err) + + return + } + + asrt.Empty(kubeletConfiguration.ContainerRuntimeEndpoint) + }) +} + +func (suite *KubeletSpecSuite) TestReconcileWithExtraConfig() { + cfg := k8s.NewKubeletConfig(k8s.NamespaceName, k8s.KubeletID) + cfg.TypedSpec().Image = "kubelet:v2.0.0" + cfg.TypedSpec().ClusterDNS = []string{"10.96.0.11"} + cfg.TypedSpec().ClusterDomain = "some.local" + cfg.TypedSpec().ExtraConfig = map[string]any{ + "serverTLSBootstrap": true, + } + + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + nodename := k8s.NewNodename(k8s.NamespaceName, k8s.NodenameID) + nodename.TypedSpec().Nodename = "foo.com" + + suite.Require().NoError(suite.State().Create(suite.Ctx(), nodename)) + + nodeIP := k8s.NewNodeIP(k8s.NamespaceName, k8s.KubeletID) + nodeIP.TypedSpec().Addresses = []netip.Addr{netip.MustParseAddr("172.20.0.3")} + + suite.Require().NoError(suite.State().Create(suite.Ctx(), nodeIP)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.KubeletID}, func(kubeletSpec *k8s.KubeletSpec, asrt *assert.Assertions) { + spec := kubeletSpec.TypedSpec() + + var kubeletConfiguration kubeletconfig.KubeletConfiguration + + if err := k8sruntime.DefaultUnstructuredConverter.FromUnstructured( + spec.Config, + &kubeletConfiguration, + ); err != nil { + asrt.NoError(err) + + return + } + + asrt.Equal("/", kubeletConfiguration.CgroupRoot) + asrt.Equal(cfg.TypedSpec().ClusterDomain, kubeletConfiguration.ClusterDomain) + asrt.True(kubeletConfiguration.ServerTLSBootstrap) + }) +} + +func (suite *KubeletSpecSuite) TestReconcileWithSkipNodeRegistration() { + cfg := k8s.NewKubeletConfig(k8s.NamespaceName, k8s.KubeletID) + cfg.TypedSpec().Image = "kubelet:v2.0.0" + cfg.TypedSpec().ClusterDNS = []string{"10.96.0.11"} + cfg.TypedSpec().ClusterDomain = "some.local" + cfg.TypedSpec().SkipNodeRegistration = true + + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + nodename := k8s.NewNodename(k8s.NamespaceName, k8s.NodenameID) + nodename.TypedSpec().Nodename = "foo.com" + + suite.Require().NoError(suite.State().Create(suite.Ctx(), nodename)) + + nodeIP := k8s.NewNodeIP(k8s.NamespaceName, k8s.KubeletID) + nodeIP.TypedSpec().Addresses = []netip.Addr{netip.MustParseAddr("172.20.0.3")} + + suite.Require().NoError(suite.State().Create(suite.Ctx(), nodeIP)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.KubeletID}, func(kubeletSpec *k8s.KubeletSpec, asrt *assert.Assertions) { + spec := kubeletSpec.TypedSpec() + + var kubeletConfiguration kubeletconfig.KubeletConfiguration + + if err := k8sruntime.DefaultUnstructuredConverter.FromUnstructured( + spec.Config, + &kubeletConfiguration, + ); err != nil { + asrt.NoError(err) + + return + } + + asrt.Equal("/", kubeletConfiguration.CgroupRoot) + asrt.Equal(cfg.TypedSpec().ClusterDomain, kubeletConfiguration.ClusterDomain) + asrt.Equal([]string{ + "--cert-dir=/var/lib/kubelet/pki", + "--config=/etc/kubernetes/kubelet.yaml", + "--hostname-override=foo.com", + "--node-ip=172.20.0.3", + }, spec.Args) + }) +} + +func TestKubeletSpecSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, &KubeletSpecSuite{ + DefaultSuite: ctest.DefaultSuite{ + Timeout: 3 * time.Second, + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(&k8sctrl.KubeletSpecController{})) + }, + }, + }) +} + +func TestNewKubeletConfigurationFail(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + name string + cfgSpec *k8s.KubeletConfigSpec + expectedErr string + }{ + { + name: "wrong fields", + cfgSpec: &k8s.KubeletConfigSpec{ + ClusterDNS: []string{"10.96.0.10"}, + ClusterDomain: "cluster.svc", + ExtraConfig: map[string]any{ + "API": "v1", + "foo": "bar", + "Port": "xyz", + }, + }, + expectedErr: "error unmarshalling extra kubelet configuration: strict decoding error: unknown field \"API\", unknown field \"Port\", unknown field \"foo\"", + }, + { + name: "wrong field type", + cfgSpec: &k8s.KubeletConfigSpec{ + ClusterDNS: []string{"10.96.0.10"}, + ClusterDomain: "cluster.svc", + ExtraConfig: map[string]any{ + "oomScoreAdj": "v1", + }, + }, + expectedErr: "error unmarshalling extra kubelet configuration: unrecognized type: int32", + }, + { + name: "not overridable", + cfgSpec: &k8s.KubeletConfigSpec{ + ClusterDNS: []string{"10.96.0.10"}, + ClusterDomain: "cluster.svc", + ExtraConfig: map[string]any{ + "oomScoreAdj": -300, + "port": 81, + "authentication": nil, + }, + }, + expectedErr: "2 errors occurred:\n\t* field \"authentication\" can't be overridden\n\t* field \"port\" can't be overridden\n\n", + }, + } { + t.Run( + tt.name, func(t *testing.T) { + t.Parallel() + + _, err := k8sctrl.NewKubeletConfiguration(tt.cfgSpec, compatibility.VersionFromImageRef("")) + require.Error(t, err) + + assert.EqualError(t, err, tt.expectedErr) + }, + ) + } +} + +func TestNewKubeletConfigurationMerge(t *testing.T) { + t.Parallel() + + defaultKubeletConfig := kubeletconfig.KubeletConfiguration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: kubeletconfig.SchemeGroupVersion.String(), + Kind: "KubeletConfiguration", + }, + Port: constants.KubeletPort, + Authentication: kubeletconfig.KubeletAuthentication{ + X509: kubeletconfig.KubeletX509Authentication{ + ClientCAFile: constants.KubernetesCACert, + }, + Webhook: kubeletconfig.KubeletWebhookAuthentication{ + Enabled: pointer.To(true), + }, + Anonymous: kubeletconfig.KubeletAnonymousAuthentication{ + Enabled: pointer.To(false), + }, + }, + Authorization: kubeletconfig.KubeletAuthorization{ + Mode: kubeletconfig.KubeletAuthorizationModeWebhook, + }, + CgroupRoot: "/", + SystemCgroups: constants.CgroupSystem, + KubeletCgroups: constants.CgroupKubelet, + RotateCertificates: true, + ProtectKernelDefaults: true, + Address: "0.0.0.0", + OOMScoreAdj: pointer.To[int32](constants.KubeletOOMScoreAdj), + ClusterDomain: "cluster.local", + ClusterDNS: []string{"10.0.0.5"}, + SerializeImagePulls: pointer.To(false), + FailSwapOn: pointer.To(false), + SystemReserved: map[string]string{ + "cpu": constants.KubeletSystemReservedCPU, + "memory": constants.KubeletSystemReservedMemory, + "pid": constants.KubeletSystemReservedPid, + "ephemeral-storage": constants.KubeletSystemReservedEphemeralStorage, + }, + Logging: v1.LoggingConfiguration{ + Format: "json", + }, + ShutdownGracePeriod: metav1.Duration{Duration: constants.KubeletShutdownGracePeriod}, + ShutdownGracePeriodCriticalPods: metav1.Duration{Duration: constants.KubeletShutdownGracePeriodCriticalPods}, + StreamingConnectionIdleTimeout: metav1.Duration{Duration: 5 * time.Minute}, + TLSMinVersion: "VersionTLS13", + StaticPodPath: constants.ManifestsDirectory, + ContainerRuntimeEndpoint: "unix://" + constants.CRIContainerdAddress, + ResolverConfig: pointer.To(constants.PodResolvConfPath), + } + + for _, tt := range []struct { + name string + cfgSpec *k8s.KubeletConfigSpec + kubeletVersion compatibility.Version + expectedOverrides func(*kubeletconfig.KubeletConfiguration) + }{ + { + name: "override some", + cfgSpec: &k8s.KubeletConfigSpec{ + ClusterDNS: []string{"10.0.0.5"}, + ClusterDomain: "cluster.local", + ExtraConfig: map[string]any{ + "oomScoreAdj": -300, + "enableDebuggingHandlers": true, + }, + }, + kubeletVersion: compatibility.VersionFromImageRef("ghcr.io/siderolabs/kubelet:v1.29.0"), + expectedOverrides: func(kc *kubeletconfig.KubeletConfiguration) { + kc.OOMScoreAdj = pointer.To[int32](-300) + kc.EnableDebuggingHandlers = pointer.To(true) + }, + }, + { + name: "disable graceful shutdown", + cfgSpec: &k8s.KubeletConfigSpec{ + ClusterDNS: []string{"10.0.0.5"}, + ClusterDomain: "cluster.local", + ExtraConfig: map[string]any{ + "shutdownGracePeriod": "0s", + "shutdownGracePeriodCriticalPods": "0s", + }, + }, + kubeletVersion: compatibility.VersionFromImageRef("ghcr.io/siderolabs/kubelet:v1.29.0"), + expectedOverrides: func(kc *kubeletconfig.KubeletConfiguration) { + kc.ShutdownGracePeriod = metav1.Duration{} + kc.ShutdownGracePeriodCriticalPods = metav1.Duration{} + }, + }, + { + name: "enable seccomp default", + cfgSpec: &k8s.KubeletConfigSpec{ + ClusterDNS: []string{"10.0.0.5"}, + ClusterDomain: "cluster.local", + DefaultRuntimeSeccompEnabled: true, + }, + kubeletVersion: compatibility.VersionFromImageRef("ghcr.io/siderolabs/kubelet:v1.29.0"), + expectedOverrides: func(kc *kubeletconfig.KubeletConfiguration) { + kc.SeccompDefault = pointer.To(true) + }, + }, + { + name: "enable skipNodeRegistration", + cfgSpec: &k8s.KubeletConfigSpec{ + ClusterDNS: []string{"10.0.0.5"}, + ClusterDomain: "cluster.local", + SkipNodeRegistration: true, + }, + kubeletVersion: compatibility.VersionFromImageRef("ghcr.io/siderolabs/kubelet:v1.29.0"), + expectedOverrides: func(kc *kubeletconfig.KubeletConfiguration) { + kc.Authentication.Webhook.Enabled = pointer.To(false) + kc.Authorization.Mode = kubeletconfig.KubeletAuthorizationModeAlwaysAllow + }, + }, + { + name: "disable manifests directory", + cfgSpec: &k8s.KubeletConfigSpec{ + ClusterDNS: []string{"10.0.0.5"}, + ClusterDomain: "cluster.local", + DisableManifestsDirectory: true, + }, + kubeletVersion: compatibility.VersionFromImageRef("ghcr.io/siderolabs/kubelet:v1.29.0"), + expectedOverrides: func(kc *kubeletconfig.KubeletConfiguration) { + kc.StaticPodPath = "" + }, + }, + { + name: "enable local FS quota monitoring", + cfgSpec: &k8s.KubeletConfigSpec{ + ClusterDNS: []string{"10.0.0.5"}, + ClusterDomain: "cluster.local", + EnableFSQuotaMonitoring: true, + }, + kubeletVersion: compatibility.VersionFromImageRef("ghcr.io/siderolabs/kubelet:v1.29.0"), + expectedOverrides: func(kc *kubeletconfig.KubeletConfiguration) { + kc.FeatureGates = map[string]bool{ + "LocalStorageCapacityIsolationFSQuotaMonitoring": true, + } + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + expected := defaultKubeletConfig + tt.expectedOverrides(&expected) + + config, err := k8sctrl.NewKubeletConfiguration(tt.cfgSpec, tt.kubeletVersion) + + require.NoError(t, err) + + assert.Equal(t, &expected, config) + }) + } +} diff --git a/internal/app/machined/pkg/controllers/k8s/kubelet_static_pod.go b/internal/app/machined/pkg/controllers/k8s/kubelet_static_pod.go new file mode 100644 index 0000000..670cd0d --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/kubelet_static_pod.go @@ -0,0 +1,233 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import ( + "context" + "fmt" + "time" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + k8sadapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/k8s" + "github.com/siderolabs/talos/pkg/kubernetes/kubelet" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +// KubeletStaticPodController renders static pod definitions and manages k8s.StaticPodStatus. +type KubeletStaticPodController struct{} + +// Name implements controller.Controller interface. +func (ctrl *KubeletStaticPodController) Name() string { + return "k8s.KubeletStaticPodController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *KubeletStaticPodController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: k8s.NamespaceName, + Type: k8s.NodenameType, + ID: optional.Some(k8s.NodenameID), + Kind: controller.InputWeak, + }, + { + Namespace: v1alpha1.NamespaceName, + Type: v1alpha1.ServiceType, + ID: optional.Some("kubelet"), + Kind: controller.InputWeak, + }, + { + Namespace: secrets.NamespaceName, + Type: secrets.KubernetesDynamicCertsType, + ID: optional.Some(secrets.KubernetesDynamicCertsID), + Kind: controller.InputWeak, + }, + { + Namespace: secrets.NamespaceName, + Type: secrets.KubernetesRootType, + ID: optional.Some(secrets.KubernetesRootID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *KubeletStaticPodController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: k8s.StaticPodStatusType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *KubeletStaticPodController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + var kubeletClient *kubelet.Client + + refreshTicker := time.NewTicker(15 * time.Second) // refresh kubelet pods status every 15 seconds + defer refreshTicker.Stop() + + for { + select { + case <-ctx.Done(): + return nil + case <-refreshTicker.C: + if kubeletClient != nil { + if err := ctrl.refreshPodStatus(ctx, r, kubeletClient); err != nil { + return fmt.Errorf("error refreshing pod status: %w", err) + } + } + + continue + case <-r.EventCh(): + } + + kubeletService, err := safe.ReaderGet[*v1alpha1.Service](ctx, r, resource.NewMetadata(v1alpha1.NamespaceName, v1alpha1.ServiceType, "kubelet", resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + kubeletClient = nil + + if err = ctrl.teardownStatuses(ctx, r); err != nil { + return fmt.Errorf("error tearing down: %w", err) + } + + continue + } + + return err + } + + if !kubeletService.TypedSpec().Running { + kubeletClient = nil + + if err = ctrl.teardownStatuses(ctx, r); err != nil { + return fmt.Errorf("error tearing down: %w", err) + } + + continue + } + + // on worker nodes, there's no way to connect to the kubelet to fetch the pod status (only API server can do that) + // on control plane nodes, use API servers' client kubelet certificate to fetch statuses + rootSecrets, err := safe.ReaderGet[*secrets.KubernetesRoot](ctx, r, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesRootType, secrets.KubernetesRootID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + kubeletClient = nil + + continue + } + + return err + } + + certsResource, err := safe.ReaderGet[*secrets.KubernetesDynamicCerts]( + ctx, r, + resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesDynamicCertsType, secrets.KubernetesDynamicCertsID, resource.VersionUndefined), + ) + if err != nil { + if state.IsNotFoundError(err) { + kubeletClient = nil + + continue + } + + return err + } + + certs := certsResource.TypedSpec() + + nodename, err := safe.ReaderGet[*k8s.Nodename](ctx, r, resource.NewMetadata(k8s.NamespaceName, k8s.NodenameType, k8s.NodenameID, resource.VersionUndefined)) + if err != nil { + // nodename should exist if the kubelet is running + return err + } + + kubeletClient, err = kubelet.NewClient( + nodename.TypedSpec().Nodename, + certs.APIServerKubeletClient.Crt, + certs.APIServerKubeletClient.Key, + rootSecrets.TypedSpec().IssuingCA.Crt, + ) + if err != nil { + return fmt.Errorf("error building kubelet client: %w", err) + } + + r.ResetRestartBackoff() + } +} + +func (ctrl *KubeletStaticPodController) teardownStatuses(ctx context.Context, r controller.Runtime) error { + statuses, err := r.List(ctx, resource.NewMetadata(k8s.NamespaceName, k8s.StaticPodStatusType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing pod statuses: %w", err) + } + + for _, status := range statuses.Items { + // TODO: proper teardown sequence? + if err = r.Destroy(ctx, status.Metadata()); err != nil { + return fmt.Errorf("error destroying stale pod status: %w", err) + } + } + + return nil +} + +func (ctrl *KubeletStaticPodController) refreshPodStatus(ctx context.Context, r controller.Runtime, kubeletClient *kubelet.Client) error { + podList, err := kubeletClient.Pods(ctx) + if err != nil { + return fmt.Errorf("error fetching pod status: %w", err) + } + + podsSeen := map[string]struct{}{} + + for _, pod := range podList.Items { + switch pod.Metadata.Annotations.ConfigSource { + case "file": + // static pod from a file source + case "http": + // static pod from an HTTP source + default: + // anything else is not a static pod, skip it + continue + } + + statusID := fmt.Sprintf("%s/%s", pod.Metadata.Namespace, pod.Metadata.Name) + + podsSeen[statusID] = struct{}{} + + if err = safe.WriterModify(ctx, r, k8s.NewStaticPodStatus(k8s.NamespaceName, statusID), func(r *k8s.StaticPodStatus) error { + return k8sadapter.StaticPodStatus(r).SetStatus(&pod.Status) + }); err != nil { + return fmt.Errorf("error updating pod status: %w", err) + } + } + + statuses, err := r.List(ctx, resource.NewMetadata(k8s.NamespaceName, k8s.StaticPodStatusType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing pod statuses: %w", err) + } + + for _, status := range statuses.Items { + if _, exists := podsSeen[status.Metadata().ID()]; !exists { + // TODO: proper teardown sequence? + if err = r.Destroy(ctx, status.Metadata()); err != nil { + return fmt.Errorf("error destroying stale pod status: %w", err) + } + } + } + + return nil +} diff --git a/internal/app/machined/pkg/controllers/k8s/kubeprism.go b/internal/app/machined/pkg/controllers/k8s/kubeprism.go new file mode 100644 index 0000000..7f24592 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/kubeprism.go @@ -0,0 +1,284 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import ( + "context" + "fmt" + "net" + "strconv" + "time" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/go-loadbalancer/controlplane" + "github.com/siderolabs/go-loadbalancer/upstream" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +// KubePrismController creates KubePrism load balancer based on KubePrismEndpointsType resource. +type KubePrismController struct { + balancerHost string + balancerPort int + lb *controlplane.LoadBalancer + ticker *time.Ticker + upstreamCh chan []string +} + +// Name implements controller.Controller interface. +func (ctrl *KubePrismController) Name() string { + return "k8s.KubePrismController" +} + +// Inputs implements controller.Controll +// er interface. +func (ctrl *KubePrismController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: k8s.NamespaceName, + Type: k8s.KubePrismConfigType, + ID: optional.Some(k8s.KubePrismConfigID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *KubePrismController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: k8s.KubePrismStatusesType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *KubePrismController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + logger = logger.Named("kubeprism") + + defer func() { + if ctrl.lb == nil { + return + } + + ctrl.stopKubePrism(logger) //nolint:errcheck + }() + + for { + select { + case <-ctx.Done(): + return nil + case <-ctrl.takeTickerC(): + err := ctrl.writeKubePrismStatus(ctx, r) + if err != nil { + return err + } + + continue + case <-r.EventCh(): + } + + lbCfg, err := safe.ReaderGetByID[*k8s.KubePrismConfig](ctx, r, k8s.KubePrismConfigID) + if err != nil && !state.IsNotFoundError(err) { + return err + } + + switch { + case ctrl.lb == nil && lbCfg != nil: + err = ctrl.startKubePrism(lbCfg, logger) + if err != nil { + return err + } + case ctrl.lb != nil && lbCfg == nil: + err = ctrl.stopKubePrism(logger) + if err != nil { + return err + } + case ctrl.lb != nil && lbCfg != nil: + if lbCfg.TypedSpec().Host != ctrl.balancerHost || lbCfg.TypedSpec().Port != ctrl.balancerPort { + err = ctrl.stopKubePrism(logger) + if err != nil { + return err + } + + err = ctrl.startKubePrism(lbCfg, logger) + if err != nil { + return err + } + } else { + ctrl.upstreamChan() <- makeEndpoints(lbCfg.TypedSpec()) + } + } + + err = ctrl.writeKubePrismStatus(ctx, r) + if err != nil { + return err + } + + r.ResetRestartBackoff() + } +} + +//nolint:gocyclo +func (ctrl *KubePrismController) writeKubePrismStatus( + ctx context.Context, + r controller.Runtime, +) error { + if ctrl.lb != nil && ctrl.endpoint() != "" { + healthy, err := ctrl.lb.Healthy() + if err != nil { + return fmt.Errorf("failed to check KubePrism health: %w", err) + } + + got, err := safe.ReaderGetByID[*k8s.KubePrismStatuses](ctx, r, k8s.KubePrismStatusesID) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("failed to get KubePrism status: %w", err) + } + + if got != nil && got.TypedSpec().Healthy == healthy { + return nil + } + + err = safe.WriterModify( + ctx, + r, + k8s.NewKubePrismStatuses(k8s.NamespaceName, k8s.KubePrismStatusesID), + func(res *k8s.KubePrismStatuses) error { + res.TypedSpec().Host = ctrl.endpoint() + res.TypedSpec().Healthy = healthy + + return nil + }, + ) + if err != nil { + return fmt.Errorf("failed to write KubePrism status: %w", err) + } + } + + // list keys for cleanup + list, err := safe.ReaderListAll[*k8s.KubePrismStatuses](ctx, r) + if err != nil { + return fmt.Errorf("error listing KubePrism resources: %w", err) + } + + for it := list.Iterator(); it.Next(); { + res := it.Value() + + if ctrl.lb == nil || res.Metadata().ID() != k8s.KubePrismStatusesID { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return fmt.Errorf("error cleaning up KubePrism specs: %w", err) + } + } + } + + return nil +} + +func (ctrl *KubePrismController) startKubePrism(lbCfg *k8s.KubePrismConfig, logger *zap.Logger) error { + spec := lbCfg.TypedSpec() + ctrl.balancerHost = spec.Host + ctrl.balancerPort = spec.Port + + lb, err := controlplane.NewLoadBalancer(ctrl.balancerHost, ctrl.balancerPort, + logger.WithOptions(zap.IncreaseLevel(zap.ErrorLevel)), // silence the load balancer logs + controlplane.WithDialTimeout(constants.KubePrismDialTimeout), + controlplane.WithKeepAlivePeriod(constants.KubePrismKeepAlivePeriod), + controlplane.WithTCPUserTimeout(constants.KubePrismTCPUserTimeout), + controlplane.WithHealthCheckOptions( + upstream.WithHealthcheckInterval(constants.KubePrismHealthCheckInterval), + upstream.WithHealthcheckTimeout(constants.KubePrismHealthCheckTimeout), + ), + ) + if err != nil { + return fmt.Errorf("failed to create KubePrism: %w", err) + } + + err = lb.Start(ctrl.upstreamChan()) + if err != nil { + return fmt.Errorf("failed to start KubePrism: %w", err) + } + + logger.Info("KubePrism is enabled", zap.String("endpoint", ctrl.endpoint())) + + ctrl.upstreamChan() <- makeEndpoints(spec) + + ctrl.lb = lb + + return nil +} + +func makeEndpoints(spec *k8s.KubePrismConfigSpec) []string { + return xslices.Map(spec.Endpoints, func(e k8s.KubePrismEndpoint) string { + return net.JoinHostPort(e.Host, strconv.FormatUint(uint64(e.Port), 10)) + }) +} + +func (ctrl *KubePrismController) takeTickerC() <-chan time.Time { + switch { + case ctrl.lb == nil && ctrl.ticker == nil: + return nil + case ctrl.lb != nil && ctrl.ticker == nil: + ctrl.ticker = time.NewTicker(5 * time.Second) + + return ctrl.ticker.C + case ctrl.lb == nil: + ticker := replaceWithZero(&ctrl.ticker) + if ticker != nil { + ticker.Stop() + } + + return nil + default: + return ctrl.ticker.C + } +} + +func (ctrl *KubePrismController) endpoint() string { + return net.JoinHostPort(ctrl.balancerHost, strconv.FormatUint(uint64(ctrl.balancerPort), 10)) +} + +func (ctrl *KubePrismController) upstreamChan() chan []string { + if ctrl.upstreamCh == nil { + ctrl.upstreamCh = make(chan []string) + } + + return ctrl.upstreamCh +} + +func (ctrl *KubePrismController) stopKubePrism(logger *zap.Logger) error { + replaceWithZero(&ctrl.upstreamCh) + + lb := replaceWithZero(&ctrl.lb) + + err := lb.Shutdown() + if err != nil { + logger.Error("failed to shutdown KubePrism", zap.Error(err)) + + return err + } + + logger.Info("KubePrism is disabled") + + return nil +} + +func replaceWithZero[T any](v *T) T { + var zero T + + result := *v + + *v = zero + + return result +} diff --git a/internal/app/machined/pkg/controllers/k8s/kubeprism_config.go b/internal/app/machined/pkg/controllers/k8s/kubeprism_config.go new file mode 100644 index 0000000..0bca535 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/kubeprism_config.go @@ -0,0 +1,81 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import ( + "context" + "strconv" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/controller/generic/transform" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/xerrors" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +// KubePrismConfigController creates config for KubePrism. +type KubePrismConfigController = transform.Controller[*config.MachineConfig, *k8s.KubePrismConfig] + +// NewKubePrismConfigController instanciates the controller. +func NewKubePrismConfigController() *KubePrismConfigController { + return transform.NewController( + transform.Settings[*config.MachineConfig, *k8s.KubePrismConfig]{ + Name: "k8s.KubePrismConfigController", + MapMetadataOptionalFunc: func(cfg *config.MachineConfig) optional.Optional[*k8s.KubePrismConfig] { + if cfg.Metadata().ID() != config.V1Alpha1ID { + return optional.None[*k8s.KubePrismConfig]() + } + + if cfg.Config().Machine() == nil { + return optional.None[*k8s.KubePrismConfig]() + } + + if !cfg.Config().Machine().Features().KubePrism().Enabled() { + return optional.None[*k8s.KubePrismConfig]() + } + + return optional.Some(k8s.NewKubePrismConfig(k8s.NamespaceName, k8s.KubePrismConfigID)) + }, + TransformFunc: func(ctx context.Context, r controller.Reader, logger *zap.Logger, cfg *config.MachineConfig, res *k8s.KubePrismConfig) error { + endpt, err := safe.ReaderGetByID[*k8s.KubePrismEndpoints](ctx, r, k8s.KubePrismEndpointsID) + if err != nil { + if state.IsNotFoundError(err) { + return xerrors.NewTaggedf[transform.SkipReconcileTag]("KubePrism endpoints resource not found; not creating KubePrism config") + } + + return err + } + + spec := res.TypedSpec() + spec.Endpoints = endpt.TypedSpec().Endpoints + spec.Host = "127.0.0.1" + spec.Port = cfg.Config().Machine().Features().KubePrism().Port() + + return nil + }, + }, + transform.WithExtraInputs( + safe.Input[*k8s.KubePrismEndpoints](controller.InputWeak), + ), + ) +} + +func toPort(port string) uint32 { + if port == "" { + return 443 + } + + p, err := strconv.ParseUint(port, 10, 32) + if err != nil { + return 443 + } + + return uint32(p) +} diff --git a/internal/app/machined/pkg/controllers/k8s/kubeprism_config_test.go b/internal/app/machined/pkg/controllers/k8s/kubeprism_config_test.go new file mode 100644 index 0000000..8468938 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/kubeprism_config_test.go @@ -0,0 +1,142 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s_test + +import ( + "net/url" + "testing" + + "github.com/siderolabs/go-pointer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + clusterctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/k8s" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +type KubePrismConfigControllerSuite struct { + ctest.DefaultSuite +} + +func (suite *KubePrismConfigControllerSuite) TestGeneration() { + cfg := &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineFeatures: &v1alpha1.FeaturesConfig{ + KubePrismSupport: &v1alpha1.KubePrism{ + ServerEnabled: pointer.To(true), + ServerPort: 7445, + }, + }, + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: must(url.Parse("https://example.com"))(suite.Require()), + }, + LocalAPIServerPort: 6445, + }, + }, + } + + mc := config.NewMachineConfig(container.NewV1Alpha1(cfg)) + suite.Create(mc) + + endpoints := k8s.NewKubePrismEndpoints(k8s.NamespaceName, k8s.KubePrismEndpointsID) + endpoints.TypedSpec().Endpoints = []k8s.KubePrismEndpoint{ + {Host: "example.com", Port: 443}, + {Host: "localhost", Port: 6445}, + {Host: "192.168.3.4", Port: 6446}, + {Host: "192.168.3.6", Port: 6443}, + } + + suite.Create(endpoints) + + ctest.AssertResource(suite, k8s.KubePrismConfigID, func(e *k8s.KubePrismConfig, asrt *assert.Assertions) { + asrt.Equal( + &k8s.KubePrismConfigSpec{ + Host: "127.0.0.1", + Port: 7445, + Endpoints: []k8s.KubePrismEndpoint{ + {Host: "example.com", Port: 443}, + {Host: "localhost", Port: 6445}, + {Host: "192.168.3.4", Port: 6446}, + {Host: "192.168.3.6", Port: 6443}, + }, + }, + e.TypedSpec(), + ) + }) + + ctest.UpdateWithConflicts(suite, mc, func(cfg *config.MachineConfig) error { + balancer := cfg.Config().Machine().Features().KubePrism().(*v1alpha1.KubePrism) //nolint:errcheck + balancer.ServerEnabled = pointer.To(false) + + return nil + }) + + ctest.AssertNoResource[*k8s.KubePrismConfig](suite, k8s.KubePrismConfigID) + + ctest.UpdateWithConflicts(suite, mc, func(cfg *config.MachineConfig) error { + balancer := cfg.Config().Machine().Features().KubePrism().(*v1alpha1.KubePrism) //nolint:errcheck + balancer.ServerEnabled = pointer.To(true) + balancer.ServerPort = 7446 + + return nil + }) + + ctest.AssertResource(suite, k8s.KubePrismConfigID, func(e *k8s.KubePrismConfig, asrt *assert.Assertions) { + asrt.Equal( + &k8s.KubePrismConfigSpec{ + Host: "127.0.0.1", + Port: 7446, + Endpoints: []k8s.KubePrismEndpoint{ + {Host: "example.com", Port: 443}, + {Host: "localhost", Port: 6445}, + {Host: "192.168.3.4", Port: 6446}, + {Host: "192.168.3.6", Port: 6443}, + }, + }, + e.TypedSpec(), + ) + }) + suite.Require().NoError(suite.State().Destroy(suite.Ctx(), mc.Metadata())) + + ctest.AssertNoResource[*k8s.KubePrismConfig](suite, k8s.KubePrismConfigID) + + suite.Create(mc) + + ctest.AssertResource(suite, k8s.KubePrismConfigID, func(e *k8s.KubePrismConfig, asrt *assert.Assertions) { + asrt.Equal( + &k8s.KubePrismConfigSpec{ + Host: "127.0.0.1", + Port: 7445, + Endpoints: []k8s.KubePrismEndpoint{ + {Host: "example.com", Port: 443}, + {Host: "localhost", Port: 6445}, + {Host: "192.168.3.4", Port: 6446}, + {Host: "192.168.3.6", Port: 6443}, + }, + }, + e.TypedSpec(), + ) + }) +} + +func TestEndpointsBalancerConfigControllerSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, &KubePrismConfigControllerSuite{ + DefaultSuite: ctest.DefaultSuite{ + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(clusterctrl.NewKubePrismConfigController())) + }, + }, + }) +} diff --git a/internal/app/machined/pkg/controllers/k8s/kubeprism_endpoints.go b/internal/app/machined/pkg/controllers/k8s/kubeprism_endpoints.go new file mode 100644 index 0000000..2468c41 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/kubeprism_endpoints.go @@ -0,0 +1,88 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/controller/generic/transform" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +// KubePrismEndpointsController creates a list of API server endpoints. +type KubePrismEndpointsController = transform.Controller[*config.MachineConfig, *k8s.KubePrismEndpoints] + +// NewKubePrismEndpointsController instanciates the controller. +// +//nolint:gocyclo +func NewKubePrismEndpointsController() *KubePrismEndpointsController { + return transform.NewController( + transform.Settings[*config.MachineConfig, *k8s.KubePrismEndpoints]{ + Name: "k8s.KubePrismEndpointsController", + MapMetadataOptionalFunc: func(cfg *config.MachineConfig) optional.Optional[*k8s.KubePrismEndpoints] { + if cfg.Metadata().ID() != config.V1Alpha1ID { + return optional.None[*k8s.KubePrismEndpoints]() + } + + if cfg.Config().Cluster() == nil || cfg.Config().Machine() == nil { + return optional.None[*k8s.KubePrismEndpoints]() + } + + return optional.Some(k8s.NewKubePrismEndpoints(k8s.NamespaceName, k8s.KubePrismEndpointsID)) + }, + TransformFunc: func(ctx context.Context, r controller.Reader, logger *zap.Logger, machineConfig *config.MachineConfig, res *k8s.KubePrismEndpoints) error { + members, err := safe.ReaderListAll[*cluster.Member](ctx, r) + if err != nil { + return fmt.Errorf("error listing affiliates: %w", err) + } + + var endpoints []k8s.KubePrismEndpoint + + ce := machineConfig.Config().Cluster().Endpoint() + if ce != nil { + endpoints = append(endpoints, k8s.KubePrismEndpoint{ + Host: ce.Hostname(), + Port: toPort(ce.Port()), + }) + } + + if machineConfig.Config().Machine().Type().IsControlPlane() { + endpoints = append(endpoints, k8s.KubePrismEndpoint{ + Host: "localhost", + Port: uint32(machineConfig.Config().Cluster().LocalAPIServerPort()), + }) + } + + for it := members.Iterator(); it.Next(); { + memberSpec := it.Value().TypedSpec() + + if len(memberSpec.Addresses) > 0 && memberSpec.ControlPlane != nil { + for _, addr := range memberSpec.Addresses { + endpoints = append(endpoints, k8s.KubePrismEndpoint{ + Host: addr.String(), + Port: uint32(memberSpec.ControlPlane.APIServerPort), + }) + } + } + } + + res.TypedSpec().Endpoints = endpoints + + return nil + }, + }, + transform.WithExtraInputs( + safe.Input[*cluster.Member](controller.InputWeak), + ), + ) +} diff --git a/internal/app/machined/pkg/controllers/k8s/kubeprism_endpoints_test.go b/internal/app/machined/pkg/controllers/k8s/kubeprism_endpoints_test.go new file mode 100644 index 0000000..3a91fdd --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/kubeprism_endpoints_test.go @@ -0,0 +1,130 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s_test + +import ( + "net/netip" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + clusteradapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/cluster" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + clusterctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/k8s" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +type KubePrismControllerSuite struct { + ctest.DefaultSuite +} + +func (suite *KubePrismControllerSuite) TestGeneration() { + nodeIdentity := cluster.NewIdentity(cluster.NamespaceName, cluster.LocalIdentity) + suite.Require().NoError(clusteradapter.IdentitySpec(nodeIdentity.TypedSpec()).Generate()) + suite.Create(nodeIdentity) + + mc := config.NewMachineConfig(container.NewV1Alpha1(&v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineType: "controlplane", + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: must(url.Parse("https://example.com"))(suite.Require()), + }, + LocalAPIServerPort: 6445, + }, + }, + })) + + suite.Create(mc) + + member1 := cluster.NewMember(cluster.NamespaceName, "service/7x1SuC8Ege5BGXdAfTEff5iQnlWZLfv9h1LGMxA2pYkC") + *member1.TypedSpec() = cluster.MemberSpec{ + NodeID: "7x1SuC8Ege5BGXdAfTEff5iQnlWZLfv9h1LGMxA2pYkC", + Hostname: "foo.com", + MachineType: machine.TypeControlPlane, + Addresses: []netip.Addr{netip.MustParseAddr("192.168.3.4")}, + ControlPlane: &cluster.ControlPlane{APIServerPort: 6446}, + } + + suite.Create(member1) + + member2 := cluster.NewMember(cluster.NamespaceName, "service/xCnFFfxylOf9i5ynhAkt6ZbfcqaLDGKfIa3gwpuaxe7F") + *member2.TypedSpec() = cluster.MemberSpec{ + NodeID: nodeIdentity.TypedSpec().NodeID, + Hostname: "foo2.com", + MachineType: machine.TypeControlPlane, + Addresses: []netip.Addr{netip.MustParseAddr("192.168.3.6")}, + ControlPlane: &cluster.ControlPlane{APIServerPort: 6443}, + } + + suite.Create(member2) + + member3 := cluster.NewMember(cluster.NamespaceName, "service/9dwHNUViZlPlIervqX9Qo256RUhrfhgO0xBBnKcKl4F") + *member3.TypedSpec() = cluster.MemberSpec{ + NodeID: "9dwHNUViZlPlIervqX9Qo256RUhrfhgO0xBBnKcKl4F", + Hostname: "worker-1", + MachineType: machine.TypeWorker, + Addresses: []netip.Addr{netip.MustParseAddr("192.168.3.5")}, + } + + suite.Create(member3) + + ctest.AssertResource(suite, k8s.KubePrismEndpointsID, func(e *k8s.KubePrismEndpoints, asrt *assert.Assertions) { + asrt.Equal( + &k8s.KubePrismEndpointsSpec{ + Endpoints: []k8s.KubePrismEndpoint{ + { + Host: "example.com", + Port: 443, + }, + { + Host: "localhost", + Port: 6445, + }, + { + Host: "192.168.3.4", + Port: 6446, + }, + { + Host: "192.168.3.6", + Port: 6443, + }, + }, + }, + e.TypedSpec(), + ) + }) +} + +func must[T any](res T, err error) func(t *require.Assertions) T { + return func(t *require.Assertions) T { + t.NoError(err) + + return res + } +} + +func TestEndpointsBalancerControllerSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, &KubePrismControllerSuite{ + DefaultSuite: ctest.DefaultSuite{ + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(clusterctrl.NewKubePrismEndpointsController())) + }, + }, + }) +} diff --git a/internal/app/machined/pkg/controllers/k8s/manifest.go b/internal/app/machined/pkg/controllers/k8s/manifest.go new file mode 100644 index 0000000..67d545e --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/manifest.go @@ -0,0 +1,302 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "slices" + "strings" + "text/template" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + k8sadapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/k8s" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" +) + +// ManifestController renders manifests based on templates and config/secrets. +type ManifestController struct{} + +// Name implements controller.Controller interface. +func (ctrl *ManifestController) Name() string { + return "k8s.ManifestController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *ManifestController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: k8s.ControlPlaneNamespaceName, + Type: k8s.BootstrapManifestsConfigType, + Kind: controller.InputWeak, + }, + { + Namespace: secrets.NamespaceName, + Type: secrets.KubernetesRootType, + ID: optional.Some(secrets.KubernetesRootID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *ManifestController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: k8s.ManifestType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *ManifestController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + configResource, err := safe.ReaderGetByID[*k8s.BootstrapManifestsConfig](ctx, r, k8s.BootstrapManifestsConfigID) + if err != nil { + if state.IsNotFoundError(err) { + if err = ctrl.teardownAll(ctx, r); err != nil { + return fmt.Errorf("error tearing down: %w", err) + } + + continue + } + + return err + } + + config := *configResource.TypedSpec() + + secretsResources, err := safe.ReaderGetByID[*secrets.KubernetesRoot](ctx, r, secrets.KubernetesRootID) + if err != nil { + if state.IsNotFoundError(err) { + if err = ctrl.teardownAll(ctx, r); err != nil { + return fmt.Errorf("error tearing down: %w", err) + } + + continue + } + + return err + } + + secrets := secretsResources.TypedSpec() + + renderedManifests, err := ctrl.render(config, secrets) + if err != nil { + return err + } + + for _, renderedManifest := range renderedManifests { + if err = safe.WriterModify(ctx, r, k8s.NewManifest(k8s.ControlPlaneNamespaceName, renderedManifest.name), + func(r *k8s.Manifest) error { + return k8sadapter.Manifest(r).SetYAML(renderedManifest.data) + }); err != nil { + return fmt.Errorf("error updating manifests: %w", err) + } + } + + // remove any manifests which weren't rendered + manifests, err := r.List(ctx, resource.NewMetadata(k8s.ControlPlaneNamespaceName, k8s.ManifestType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing manifests: %w", err) + } + + manifestsToDelete := map[string]struct{}{} + + for _, manifest := range manifests.Items { + if manifest.Metadata().Owner() != ctrl.Name() { + continue + } + + manifestsToDelete[manifest.Metadata().ID()] = struct{}{} + } + + for _, renderedManifest := range renderedManifests { + delete(manifestsToDelete, renderedManifest.name) + } + + for id := range manifestsToDelete { + if err = r.Destroy(ctx, resource.NewMetadata(k8s.ControlPlaneNamespaceName, k8s.ManifestType, id, resource.VersionUndefined)); err != nil { + return fmt.Errorf("error cleaning up manifests: %w", err) + } + } + + r.ResetRestartBackoff() + } +} + +type renderedManifest struct { + name string + data []byte +} + +func jsonify(input string) (string, error) { + out, err := json.Marshal(input) + + return string(out), err +} + +func (ctrl *ManifestController) render(cfg k8s.BootstrapManifestsConfigSpec, scrt *secrets.KubernetesRootSpec) ([]renderedManifest, error) { + templateConfig := struct { + k8s.BootstrapManifestsConfigSpec + + Secrets *secrets.KubernetesRootSpec + + KubernetesTalosAPIServiceName string + KubernetesTalosAPIServiceNamespace string + + ApidPort int + + TalosServiceAccount TalosServiceAccount + + HostDNSAddr string + }{ + BootstrapManifestsConfigSpec: cfg, + Secrets: scrt, + + KubernetesTalosAPIServiceName: constants.KubernetesTalosAPIServiceName, + KubernetesTalosAPIServiceNamespace: constants.KubernetesTalosAPIServiceNamespace, + + ApidPort: constants.ApidPort, + + TalosServiceAccount: TalosServiceAccount{ + Group: constants.ServiceAccountResourceGroup, + Version: constants.ServiceAccountResourceVersion, + Kind: constants.ServiceAccountResourceKind, + ResourceSingular: constants.ServiceAccountResourceSingular, + ResourcePlural: constants.ServiceAccountResourcePlural, + ShortName: constants.ServiceAccountResourceShortName, + }, + } + + type manifestDesc struct { + name string + template []byte + } + + defaultManifests := []manifestDesc{ + {"00-kubelet-bootstrapping-token", kubeletBootstrappingToken}, + {"01-csr-node-bootstrap", csrNodeBootstrapTemplate}, + {"01-csr-approver-role-binding", csrApproverRoleBindingTemplate}, + {"01-csr-renewal-role-binding", csrRenewalRoleBindingTemplate}, + {"11-kube-config-in-cluster", kubeConfigInClusterTemplate}, + } + + if cfg.CoreDNSEnabled { + defaultManifests = slices.Concat(defaultManifests, + []manifestDesc{ + {"11-core-dns", coreDNSTemplate}, + {"11-core-dns-svc", coreDNSSvcTemplate}, + }, + ) + } + + if cfg.FlannelEnabled { + defaultManifests = append(defaultManifests, + manifestDesc{"05-flannel", flannelTemplate}) + } + + if cfg.ProxyEnabled { + defaultManifests = append(defaultManifests, + manifestDesc{"10-kube-proxy", kubeProxyTemplate}) + } + + if cfg.PodSecurityPolicyEnabled { + defaultManifests = append(defaultManifests, + manifestDesc{"03-default-pod-security-policy", podSecurityPolicy}, + ) + } + + if cfg.TalosAPIServiceEnabled { + defaultManifests = slices.Concat(defaultManifests, + []manifestDesc{ + {"12-talos-api-service", talosAPIService}, + {"13-talos-service-account-crd", talosServiceAccountCRDTemplate}, + }, + ) + } + + if cfg.ServiceHostDNSAddress != "" { + defaultManifests = append(defaultManifests, + manifestDesc{"15-host-dns-service", talosHostDNSSvcTemplate}, + ) + } + + manifests := make([]renderedManifest, len(defaultManifests)) + + for i := range defaultManifests { + tmpl, err := template.New(defaultManifests[i].name). + Funcs(template.FuncMap{ + "json": jsonify, + "join": strings.Join, + "contains": strings.Contains, + }). + Parse(string(defaultManifests[i].template)) + if err != nil { + return nil, fmt.Errorf("error parsing manifest template %q: %w", defaultManifests[i].name, err) + } + + var buf bytes.Buffer + + if err = tmpl.Execute(&buf, &templateConfig); err != nil { + return nil, fmt.Errorf("error executing template %q: %w", defaultManifests[i].name, err) + } + + manifests[i].name = defaultManifests[i].name + manifests[i].data = buf.Bytes() + } + + return manifests, nil +} + +//nolint:dupl +func (ctrl *ManifestController) teardownAll(ctx context.Context, r controller.Runtime) error { + manifests, err := r.List(ctx, resource.NewMetadata(k8s.ControlPlaneNamespaceName, k8s.ManifestType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing manifests: %w", err) + } + + for _, manifest := range manifests.Items { + if manifest.Metadata().Owner() != ctrl.Name() { + continue + } + + if err = r.Destroy(ctx, manifest.Metadata()); err != nil { + return fmt.Errorf("error destroying manifest: %w", err) + } + } + + return nil +} + +// TalosServiceAccount is a struct used by the template engine which contains the needed variables to +// be able to construct the Talos Service Account CRD. +type TalosServiceAccount struct { + Group string + Version string + Kind string + ResourceSingular string + ResourcePlural string + ShortName string +} diff --git a/internal/app/machined/pkg/controllers/k8s/manifest_apply.go b/internal/app/machined/pkg/controllers/k8s/manifest_apply.go new file mode 100644 index 0000000..ca81fed --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/manifest_apply.go @@ -0,0 +1,308 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import ( + "cmp" + "context" + "fmt" + "slices" + "sort" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/hashicorp/go-multierror" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/xslices" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/discovery" + memory "k8s.io/client-go/discovery/cached" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + + k8sadapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/k8s" + "github.com/aenix-io/talm/internal/pkg/etcd" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +// ManifestApplyController applies manifests via control plane endpoint. +type ManifestApplyController struct{} + +// Name implements controller.Controller interface. +func (ctrl *ManifestApplyController) Name() string { + return "k8s.ManifestApplyController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *ManifestApplyController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: secrets.NamespaceName, + Type: secrets.KubernetesType, + ID: optional.Some(secrets.KubernetesID), + Kind: controller.InputWeak, + }, + { + Namespace: k8s.ControlPlaneNamespaceName, + Type: k8s.ManifestType, + Kind: controller.InputWeak, + }, + { + Namespace: v1alpha1.NamespaceName, + Type: v1alpha1.ServiceType, + ID: optional.Some("etcd"), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *ManifestApplyController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: k8s.ManifestStatusType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *ManifestApplyController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + secretsResources, err := safe.ReaderGetByID[*secrets.Kubernetes](ctx, r, secrets.KubernetesID) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + secrets := secretsResources.TypedSpec() + + // wait for etcd to be healthy as controller relies on etcd for locking + etcdResource, err := safe.ReaderGetByID[*v1alpha1.Service](ctx, r, "etcd") + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + if !etcdResource.TypedSpec().Healthy { + continue + } + + manifests, err := r.List(ctx, resource.NewMetadata(k8s.ControlPlaneNamespaceName, k8s.ManifestType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing manifests: %w", err) + } + + slices.SortFunc(manifests.Items, func(a, b resource.Resource) int { + return cmp.Compare(a.Metadata().ID(), b.Metadata().ID()) + }) + + if len(manifests.Items) > 0 { + var ( + kubeconfig *rest.Config + dc *discovery.DiscoveryClient + dyn dynamic.Interface + ) + + kubeconfig, err = clientcmd.BuildConfigFromKubeconfigGetter("", func() (*clientcmdapi.Config, error) { + return clientcmd.Load([]byte(secrets.LocalhostAdminKubeconfig)) + }) + if err != nil { + return fmt.Errorf("error loading kubeconfig: %w", err) + } + + kubeconfig.WarningHandler = rest.NewWarningWriter(logging.NewWriter(logger, zapcore.WarnLevel), rest.WarningWriterOptions{ + Deduplicate: true, + }) + + dc, err = discovery.NewDiscoveryClientForConfig(kubeconfig) + if err != nil { + return fmt.Errorf("error building discovery client: %w", err) + } + + mapper := restmapper.NewDeferredDiscoveryRESTMapper(memory.NewMemCacheClient(dc)) + + dyn, err = dynamic.NewForConfig(kubeconfig) + if err != nil { + return fmt.Errorf("error building dynamic client: %w", err) + } + + if err = etcd.WithLock(ctx, constants.EtcdTalosManifestApplyMutex, logger, func() error { + return ctrl.apply(ctx, logger, mapper, dyn, manifests) + }); err != nil { + return err + } + } + + if err = safe.WriterModify(ctx, r, k8s.NewManifestStatus(k8s.ControlPlaneNamespaceName), func(r *k8s.ManifestStatus) error { + status := r.TypedSpec() + + status.ManifestsApplied = xslices.Map(manifests.Items, func(m resource.Resource) string { + return m.Metadata().ID() + }) + + return nil + }); err != nil { + return fmt.Errorf("error updating manifest status: %w", err) + } + + r.ResetRestartBackoff() + } +} + +//nolint:gocyclo,cyclop +func (ctrl *ManifestApplyController) apply(ctx context.Context, logger *zap.Logger, mapper *restmapper.DeferredDiscoveryRESTMapper, dyn dynamic.Interface, manifests resource.List) error { + // flatten list of objects to be applied + objects := xslices.FlatMap(manifests.Items, func(m resource.Resource) []*unstructured.Unstructured { + return k8sadapter.Manifest(m.(*k8s.Manifest)).Objects() + }) + + // sort the list so that namespaces come first, followed by CRDs and everything else after that + sort.SliceStable(objects, func(i, j int) bool { + objL := objects[i] + objR := objects[j] + + gvkL := objL.GroupVersionKind() + gvkR := objR.GroupVersionKind() + + if isNamespace(gvkL) { + if !isNamespace(gvkR) { + return true + } + + return objL.GetName() < objR.GetName() + } + + if isNamespace(gvkR) { + return false + } + + if isCRD(gvkL) { + if !isCRD(gvkR) { + return true + } + + return objL.GetName() < objR.GetName() + } + + if isCRD(gvkR) { + return false + } + + return false + }) + + var multiErr *multierror.Error + + for _, obj := range objects { + gvk := obj.GroupVersionKind() + objName := fmt.Sprintf("%s/%s/%s/%s", gvk.Group, gvk.Version, gvk.Kind, obj.GetName()) + + mapping, err := mapper.RESTMapping(obj.GroupVersionKind().GroupKind(), obj.GroupVersionKind().Version) + if err != nil { + switch { + case apierrors.IsNotFound(err): + fallthrough + case apierrors.IsInvalid(err): + fallthrough + case meta.IsNoMatchError(err): + // most probably a problem with the manifest, so we should continue with other manifests + multiErr = multierror.Append(multiErr, fmt.Errorf("error creating mapping for object %s: %w", objName, err)) + + continue + default: + // connection errors, etc.; it makes no sense to continue with other manifests + return fmt.Errorf("error creating mapping for object %s: %w", objName, err) + } + } + + var dr dynamic.ResourceInterface + + if mapping.Scope.Name() == meta.RESTScopeNameNamespace { + // default the namespace if it's not set in the manifest + if obj.GetNamespace() == "" { + obj.SetNamespace(corev1.NamespaceDefault) + } + + // namespaced resources should specify the namespace + dr = dyn.Resource(mapping.Resource).Namespace(obj.GetNamespace()) + } else { + // for cluster-wide resources + dr = dyn.Resource(mapping.Resource) + } + + _, err = dr.Get(ctx, obj.GetName(), metav1.GetOptions{}) + if err == nil { + // already exists + continue + } + + if !apierrors.IsNotFound(err) { + return fmt.Errorf("error checking resource existence: %w", err) + } + + _, err = dr.Create(ctx, obj, metav1.CreateOptions{ + FieldManager: "talos", + }) + if err != nil { + switch { + case apierrors.IsAlreadyExists(err): + // later on we might want to do something here, e.g. do server-side apply, for now do nothing + case apierrors.IsMethodNotSupported(err): + fallthrough + case apierrors.IsBadRequest(err): + fallthrough + case apierrors.IsInvalid(err): + // resource is malformed, continue with other manifests + multiErr = multierror.Append(multiErr, fmt.Errorf("error creating %s: %w", objName, err)) + default: + // connection errors, etc.; it makes no sense to continue with other manifests + return fmt.Errorf("error creating %s: %w", objName, err) + } + } else { + logger.Sugar().Infof("created %s", objName) + } + } + + return multiErr.ErrorOrNil() +} + +func isNamespace(gvk schema.GroupVersionKind) bool { + return gvk.Kind == "Namespace" && gvk.Version == "v1" +} + +func isCRD(gvk schema.GroupVersionKind) bool { + return gvk.Kind == "CustomResourceDefinition" && gvk.Group == "apiextensions.k8s.io" +} diff --git a/internal/app/machined/pkg/controllers/k8s/manifest_test.go b/internal/app/machined/pkg/controllers/k8s/manifest_test.go new file mode 100644 index 0000000..3c8aa35 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/manifest_test.go @@ -0,0 +1,362 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:dupl +package k8s_test + +import ( + "context" + "fmt" + "log" + "reflect" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + k8sadapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/k8s" + k8sctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/k8s" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" +) + +type ManifestSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *ManifestSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.Require().NoError(suite.runtime.RegisterController(&k8sctrl.ManifestController{})) + + suite.startRuntime() +} + +func (suite *ManifestSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +//nolint:dupl +func (suite *ManifestSuite) assertManifests(manifests []string) error { + resources, err := suite.state.List( + suite.ctx, + resource.NewMetadata(k8s.ControlPlaneNamespaceName, k8s.ManifestType, "", resource.VersionUndefined), + ) + if err != nil { + return err + } + + ids := xslices.Map(resources.Items, func(r resource.Resource) string { return r.Metadata().ID() }) + + if !reflect.DeepEqual(manifests, ids) { + return retry.ExpectedErrorf("expected %q, got %q", manifests, ids) + } + + return nil +} + +var defaultManifestSpec = k8s.BootstrapManifestsConfigSpec{ + Server: "127.0.0.1", + ClusterDomain: "cluster.", + + PodCIDRs: []string{constants.DefaultIPv4PodNet}, + + ProxyEnabled: true, + ProxyImage: "foo/bar", + ProxyArgs: []string{ + fmt.Sprintf("--cluster-cidr=%s", constants.DefaultIPv4PodNet), + "--hostname-override=$(NODE_NAME)", + "--kubeconfig=/etc/kubernetes/kubeconfig", + "--proxy-mode=iptables", + "--conntrack-max-per-core=0", + }, + + CoreDNSEnabled: true, + CoreDNSImage: "foo/bar", + + DNSServiceIP: "192.168.0.1", + + FlannelEnabled: true, + FlannelImage: "foo/bar", + FlannelCNIImage: "foo/bar", + + PodSecurityPolicyEnabled: true, +} + +func (suite *ManifestSuite) TestReconcileDefaults() { + rootSecrets := secrets.NewKubernetesRoot(secrets.KubernetesRootID) + manifestConfig := k8s.NewBootstrapManifestsConfig() + *manifestConfig.TypedSpec() = defaultManifestSpec + + suite.Require().NoError(suite.state.Create(suite.ctx, rootSecrets)) + suite.Require().NoError(suite.state.Create(suite.ctx, manifestConfig)) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertManifests( + []string{ + "00-kubelet-bootstrapping-token", + "01-csr-approver-role-binding", + "01-csr-node-bootstrap", + "01-csr-renewal-role-binding", + "03-default-pod-security-policy", + "05-flannel", + "10-kube-proxy", + "11-core-dns", + "11-core-dns-svc", + "11-kube-config-in-cluster", + }, + ) + }, + ), + ) +} + +func (suite *ManifestSuite) TestReconcileDisableKubeProxy() { + rootSecrets := secrets.NewKubernetesRoot(secrets.KubernetesRootID) + manifestConfig := k8s.NewBootstrapManifestsConfig() + spec := defaultManifestSpec + spec.ProxyEnabled = false + *manifestConfig.TypedSpec() = spec + + suite.Require().NoError(suite.state.Create(suite.ctx, rootSecrets)) + suite.Require().NoError(suite.state.Create(suite.ctx, manifestConfig)) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertManifests( + []string{ + "00-kubelet-bootstrapping-token", + "01-csr-approver-role-binding", + "01-csr-node-bootstrap", + "01-csr-renewal-role-binding", + "03-default-pod-security-policy", + "05-flannel", + "11-core-dns", + "11-core-dns-svc", + "11-kube-config-in-cluster", + }, + ) + }, + ), + ) +} + +func (suite *ManifestSuite) TestReconcileKubeProxyExtraArgs() { + rootSecrets := secrets.NewKubernetesRoot(secrets.KubernetesRootID) + manifestConfig := k8s.NewBootstrapManifestsConfig() + spec := defaultManifestSpec + spec.ProxyArgs = append(spec.ProxyArgs, "--bind-address=\"::\"") + *manifestConfig.TypedSpec() = spec + + suite.Require().NoError(suite.state.Create(suite.ctx, rootSecrets)) + suite.Require().NoError(suite.state.Create(suite.ctx, manifestConfig)) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertManifests( + []string{ + "00-kubelet-bootstrapping-token", + "01-csr-approver-role-binding", + "01-csr-node-bootstrap", + "01-csr-renewal-role-binding", + "03-default-pod-security-policy", + "05-flannel", + "10-kube-proxy", + "11-core-dns", + "11-core-dns-svc", + "11-kube-config-in-cluster", + }, + ) + }, + ), + ) + + r, err := suite.state.Get( + suite.ctx, + resource.NewMetadata( + k8s.ControlPlaneNamespaceName, + k8s.ManifestType, + "10-kube-proxy", + resource.VersionUndefined, + ), + ) + suite.Require().NoError(err) + + manifest := r.(*k8s.Manifest) //nolint:errcheck,forcetypeassert + suite.Assert().Len(k8sadapter.Manifest(manifest).Objects(), 3) + + suite.Assert().Equal("DaemonSet", k8sadapter.Manifest(manifest).Objects()[0].GetKind()) + + ds := k8sadapter.Manifest(manifest).Objects()[0].Object + containerSpec := ds["spec"].(map[string]any)["template"].(map[string]any)["spec"].(map[string]any)["containers"].([]any)[0] + args := containerSpec.(map[string]any)["command"].([]any) //nolint:errcheck,forcetypeassert + + suite.Assert().Equal("--bind-address=\"::\"", args[len(args)-1]) +} + +func (suite *ManifestSuite) TestReconcileIPv6() { + rootSecrets := secrets.NewKubernetesRoot(secrets.KubernetesRootID) + manifestConfig := k8s.NewBootstrapManifestsConfig() + spec := defaultManifestSpec + spec.PodCIDRs = []string{constants.DefaultIPv6PodNet} + spec.DNSServiceIP = "" + spec.DNSServiceIPv6 = "fc00:db8:10::10" + *manifestConfig.TypedSpec() = spec + + suite.Require().NoError(suite.state.Create(suite.ctx, rootSecrets)) + suite.Require().NoError(suite.state.Create(suite.ctx, manifestConfig)) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertManifests( + []string{ + "00-kubelet-bootstrapping-token", + "01-csr-approver-role-binding", + "01-csr-node-bootstrap", + "01-csr-renewal-role-binding", + "03-default-pod-security-policy", + "05-flannel", + "10-kube-proxy", + "11-core-dns", + "11-core-dns-svc", + "11-kube-config-in-cluster", + }, + ) + }, + ), + ) + + r, err := suite.state.Get( + suite.ctx, + resource.NewMetadata( + k8s.ControlPlaneNamespaceName, + k8s.ManifestType, + "11-core-dns-svc", + resource.VersionUndefined, + ), + ) + suite.Require().NoError(err) + + manifest := r.(*k8s.Manifest) //nolint:errcheck,forcetypeassert + suite.Assert().Len(k8sadapter.Manifest(manifest).Objects(), 1) + + service := k8sadapter.Manifest(manifest).Objects()[0] + suite.Assert().Equal("Service", service.GetKind()) + + v, _, _ := unstructured.NestedString(service.Object, "spec", "clusterIP") //nolint:errcheck + suite.Assert().Equal(spec.DNSServiceIPv6, v) + + vv, _, _ := unstructured.NestedStringSlice(service.Object, "spec", "clusterIPs") //nolint:errcheck + suite.Assert().Equal([]string{spec.DNSServiceIPv6}, vv) + + vv, _, _ = unstructured.NestedStringSlice(service.Object, "spec", "ipFamilies") //nolint:errcheck + suite.Assert().Equal([]string{"IPv6"}, vv) + + v, _, _ = unstructured.NestedString(service.Object, "spec", "ipFamilyPolicy") //nolint:errcheck + suite.Assert().Equal("SingleStack", v) + + r, err = suite.state.Get( + suite.ctx, + resource.NewMetadata( + k8s.ControlPlaneNamespaceName, + k8s.ManifestType, + "05-flannel", + resource.VersionUndefined, + ), + ) + suite.Require().NoError(err) + + manifest = r.(*k8s.Manifest) //nolint:errcheck,forcetypeassert + suite.Assert().Len(k8sadapter.Manifest(manifest).Objects(), 5) + + configmap := k8sadapter.Manifest(manifest).Objects()[3] + suite.Assert().Equal("ConfigMap", configmap.GetKind()) + + v, _, _ = unstructured.NestedString(configmap.Object, "data", "net-conf.json") //nolint:errcheck + suite.Assert().Contains(v, `"EnableIPv4": false`) + suite.Assert().Contains(v, `"EnableIPv6": true`) + suite.Assert().Contains(v, fmt.Sprintf(`"IPv6Network": "%s"`, constants.DefaultIPv6PodNet)) +} + +func (suite *ManifestSuite) TestReconcileDisablePSP() { + rootSecrets := secrets.NewKubernetesRoot(secrets.KubernetesRootID) + manifestConfig := k8s.NewBootstrapManifestsConfig() + spec := defaultManifestSpec + spec.PodSecurityPolicyEnabled = false + *manifestConfig.TypedSpec() = spec + + suite.Require().NoError(suite.state.Create(suite.ctx, rootSecrets)) + suite.Require().NoError(suite.state.Create(suite.ctx, manifestConfig)) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertManifests( + []string{ + "00-kubelet-bootstrapping-token", + "01-csr-approver-role-binding", + "01-csr-node-bootstrap", + "01-csr-renewal-role-binding", + "05-flannel", + "10-kube-proxy", + "11-core-dns", + "11-core-dns-svc", + "11-kube-config-in-cluster", + }, + ) + }, + ), + ) +} + +func (suite *ManifestSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestManifestSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, new(ManifestSuite)) +} diff --git a/internal/app/machined/pkg/controllers/k8s/node_apply.go b/internal/app/machined/pkg/controllers/k8s/node_apply.go new file mode 100644 index 0000000..d6ee03f --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/node_apply.go @@ -0,0 +1,443 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "time" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/maps" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/go-retry/retry" + "go.uber.org/zap" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/siderolabs/talos/pkg/conditions" + "github.com/siderolabs/talos/pkg/kubernetes" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" +) + +// NodeApplyController watches k8s.NodeLabelSpecs, k8s.NodeTaintSpecs and applies them to the k8s Node object. +type NodeApplyController struct{} + +// Name implements controller.Controller interface. +func (ctrl *NodeApplyController) Name() string { + return "k8s.NodeApplyController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *NodeApplyController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: k8s.NamespaceName, + Type: k8s.NodeLabelSpecType, + Kind: controller.InputWeak, + }, + { + Namespace: k8s.NamespaceName, + Type: k8s.NodeTaintSpecType, + Kind: controller.InputWeak, + }, + { + Namespace: k8s.NamespaceName, + Type: k8s.NodeCordonedSpecType, + Kind: controller.InputWeak, + }, + { + // NodeStatus is used to trigger the controller on node status updates. + Namespace: k8s.NamespaceName, + Type: k8s.NodeStatusType, + Kind: controller.InputWeak, + }, + { + Namespace: secrets.NamespaceName, + Type: secrets.KubernetesRootType, + ID: optional.Some(secrets.KubernetesRootID), + Kind: controller.InputWeak, + }, + { + Namespace: k8s.NamespaceName, + Type: k8s.NodenameType, + ID: optional.Some(k8s.NodenameID), + Kind: controller.InputWeak, + }, + { + Namespace: config.NamespaceName, + Type: config.MachineTypeType, + ID: optional.Some(config.MachineTypeID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *NodeApplyController) Outputs() []controller.Output { + return nil +} + +// Run implements controller.Controller interface. +func (ctrl *NodeApplyController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + if err := ctrl.reconcileWithK8s(ctx, r, logger); err != nil { + return err + } + + r.ResetRestartBackoff() + } +} + +func (ctrl *NodeApplyController) getNodeLabelSpecs(ctx context.Context, r controller.Runtime) (map[string]string, error) { + items, err := safe.ReaderListAll[*k8s.NodeLabelSpec](ctx, r) + if err != nil { + return nil, fmt.Errorf("error listing node label spec resources: %w", err) + } + + result := make(map[string]string, items.Len()) + + for iter := items.Iterator(); iter.Next(); { + result[iter.Value().TypedSpec().Key] = iter.Value().TypedSpec().Value + } + + return result, nil +} + +func (ctrl *NodeApplyController) getNodeTaintSpecs(ctx context.Context, r controller.Runtime) ([]k8s.NodeTaintSpecSpec, error) { + items, err := safe.ReaderListAll[*k8s.NodeTaintSpec](ctx, r) + if err != nil { + return nil, fmt.Errorf("error listing node taint spec resources: %w", err) + } + + result := make([]k8s.NodeTaintSpecSpec, 0, items.Len()) + + for iter := items.Iterator(); iter.Next(); { + result = append(result, *iter.Value().TypedSpec()) + } + + return result, nil +} + +func (ctrl *NodeApplyController) getNodeCordoned(ctx context.Context, r controller.Runtime) (bool, error) { + items, err := safe.ReaderListAll[*k8s.NodeCordonedSpec](ctx, r) + if err != nil { + return false, fmt.Errorf("error listing node cordoned spec resources: %w", err) + } + + return items.Len() > 0, nil +} + +func (ctrl *NodeApplyController) getK8sClient(ctx context.Context, r controller.Runtime, logger *zap.Logger) (*kubernetes.Client, error) { + machineType, err := safe.ReaderGet[*config.MachineType](ctx, r, resource.NewMetadata(config.NamespaceName, config.MachineTypeType, config.MachineTypeID, resource.VersionUndefined)) + if err != nil { + return nil, fmt.Errorf("error getting machine type: %w", err) + } + + if machineType.MachineType().IsControlPlane() { + return kubernetes.NewTemporaryClientControlPlane(ctx, r) + } + + logger.Debug("waiting for kubelet client config", zap.String("file", constants.KubeletKubeconfig)) + + if err := conditions.WaitForKubeconfigReady(constants.KubeletKubeconfig).Wait(ctx); err != nil { + return nil, err + } + + return kubernetes.NewClientFromKubeletKubeconfig() +} + +func (ctrl *NodeApplyController) reconcileWithK8s( + ctx context.Context, + r controller.Runtime, + logger *zap.Logger, +) error { + nodenameResource, err := safe.ReaderGet[*k8s.Nodename](ctx, r, resource.NewMetadata(k8s.NamespaceName, k8s.NodenameType, k8s.NodenameID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + return nil + } + + return err + } + + if nodenameResource.TypedSpec().SkipNodeRegistration { + // if the node registration is skipped, we don't need to do anything + return nil + } + + nodename := nodenameResource.TypedSpec().Nodename + + k8sClient, err := ctrl.getK8sClient(ctx, r, logger) + if err != nil { + return fmt.Errorf("error building kubernetes client: %w", err) + } + + if k8sClient == nil { + // not ready yet + return nil + } + + defer k8sClient.Close() //nolint:errcheck + + nodeLabelSpecs, err := ctrl.getNodeLabelSpecs(ctx, r) + if err != nil { + return err + } + + nodeTaintSpecs, err := ctrl.getNodeTaintSpecs(ctx, r) + if err != nil { + return err + } + + nodeShouldCordon, err := ctrl.getNodeCordoned(ctx, r) + if err != nil { + return err + } + + return ctrl.sync(ctx, logger, k8sClient, nodename, nodeLabelSpecs, nodeTaintSpecs, nodeShouldCordon) +} + +func (ctrl *NodeApplyController) sync( + ctx context.Context, + logger *zap.Logger, + k8sClient *kubernetes.Client, + nodeName string, + nodeLabelSpecs map[string]string, + nodeTaintSpecs []k8s.NodeTaintSpecSpec, + nodeShouldCordon bool, +) error { + // run several attempts retrying conflict errors + return retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).RetryWithContext(ctx, func(ctx context.Context) error { + err := ctrl.syncOnce(ctx, logger, k8sClient, nodeName, nodeLabelSpecs, nodeTaintSpecs, nodeShouldCordon) + + if err != nil && (apierrors.IsConflict(err) || apierrors.IsForbidden(err)) { + return retry.ExpectedError(err) + } + + return err + }) +} + +func umarshalOwnedAnnotation(node *v1.Node, annotation string) (map[string]struct{}, error) { + ownedJSON := []byte(node.Annotations[annotation]) + + var owned []string + + if len(ownedJSON) > 0 { + if err := json.Unmarshal(ownedJSON, &owned); err != nil { + return nil, err + } + } + + ownedMap := xslices.ToSet(owned) + if ownedMap == nil { + ownedMap = map[string]struct{}{} + } + + return ownedMap, nil +} + +func marshalOwnedAnnotation(node *v1.Node, annotation string, ownedMap map[string]struct{}) error { + owned := maps.Keys(ownedMap) + slices.Sort(owned) + + if len(owned) > 0 { + ownedJSON, err := json.Marshal(owned) + if err != nil { + return err + } + + node.Annotations[annotation] = string(ownedJSON) + } else { + delete(node.Annotations, annotation) + } + + return nil +} + +func (ctrl *NodeApplyController) syncOnce( + ctx context.Context, + logger *zap.Logger, + k8sClient *kubernetes.Client, + nodeName string, + nodeLabelSpecs map[string]string, + nodeTaintSpecs []k8s.NodeTaintSpecSpec, + nodeShouldCordon bool, +) error { + node, err := k8sClient.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("error getting node: %w", err) + } + + if node.Labels == nil { + node.Labels = make(map[string]string) + } + + ownedLabelsMap, err := umarshalOwnedAnnotation(node, constants.AnnotationOwnedLabels) + if err != nil { + return fmt.Errorf("error unmarshaling owned labels: %w", err) + } + + ownedTaintsMap, err := umarshalOwnedAnnotation(node, constants.AnnotationOwnedTaints) + if err != nil { + return fmt.Errorf("error unmarshaling owned taints: %w", err) + } + + ctrl.ApplyLabels(logger, node, ownedLabelsMap, nodeLabelSpecs) + ctrl.ApplyTaints(logger, node, ownedTaintsMap, nodeTaintSpecs) + ctrl.ApplyCordoned(logger, node, nodeShouldCordon) + + if err = marshalOwnedAnnotation(node, constants.AnnotationOwnedLabels, ownedLabelsMap); err != nil { + return fmt.Errorf("error marshaling owned labels: %w", err) + } + + if err = marshalOwnedAnnotation(node, constants.AnnotationOwnedTaints, ownedTaintsMap); err != nil { + return fmt.Errorf("error marshaling owned taints: %w", err) + } + + _, err = k8sClient.CoreV1().Nodes().Update(ctx, node, metav1.UpdateOptions{}) + + return err +} + +// ApplyLabels performs the inner loop of the node label reconciliation. +// +// This method is exported for testing purposes. +func (ctrl *NodeApplyController) ApplyLabels(logger *zap.Logger, node *v1.Node, ownedLabels map[string]struct{}, nodeLabelSpecs map[string]string) { + // set labels from the spec + for key, value := range nodeLabelSpecs { + currentValue, exists := node.Labels[key] + + // label is not set on the node yet, so take it over + if !exists { + node.Labels[key] = value + ownedLabels[key] = struct{}{} + + continue + } + + // no change to the label, skip it + if currentValue == value { + ownedLabels[key] = struct{}{} + + continue + } + + if _, owned := ownedLabels[key]; !owned { + logger.Debug("skipping label update, label is not owned", zap.String("key", key), zap.String("value", value)) + + continue + } + + node.Labels[key] = value + } + + // remove labels which are owned but are not in the spec + for key := range ownedLabels { + if _, exists := nodeLabelSpecs[key]; !exists { + delete(node.Labels, key) + delete(ownedLabels, key) + } + } +} + +// ApplyTaints performs the inner loop of the node taints reconciliation. +// +// This method is exported for testing purposes. +// +//nolint:gocyclo +func (ctrl *NodeApplyController) ApplyTaints(logger *zap.Logger, node *v1.Node, ownedTaints map[string]struct{}, nodeTaints []k8s.NodeTaintSpecSpec) { + // set taints from the spec + for _, taint := range nodeTaints { + var currentValue *v1.Taint + + for i, nodeTaint := range node.Spec.Taints { + if nodeTaint.Key == taint.Key { + currentValue = &node.Spec.Taints[i] + } + } + + if currentValue == nil { + // taint is not set on the node yet, so take it over + node.Spec.Taints = append(node.Spec.Taints, v1.Taint{ + Key: taint.Key, + Value: taint.Value, + Effect: v1.TaintEffect(taint.Effect), + }) + ownedTaints[taint.Key] = struct{}{} + } else { + // taint with the same key exists, check if it is owned + if _, owned := ownedTaints[taint.Key]; owned { + // taint is owned, so update it + currentValue.Value = taint.Value + currentValue.Effect = v1.TaintEffect(taint.Effect) + } else if currentValue.Value == taint.Value && currentValue.Effect == v1.TaintEffect(taint.Effect) { + // no change to the taint, skip it, but mark it as owned + ownedTaints[taint.Key] = struct{}{} + } else { + logger.Debug("skipping taint update, taint is not owned", zap.String("key", taint.Key), zap.String("value", taint.Value), zap.String("effect", taint.Effect)) + } + } + } + + // remove taints which are owned but are not in the spec + node.Spec.Taints = xslices.FilterInPlace(node.Spec.Taints, + func(nodeTaint v1.Taint) bool { + if _, owned := ownedTaints[nodeTaint.Key]; !owned { + return true + } + + for _, taint := range nodeTaints { + if nodeTaint.Key == taint.Key { + return true + } + } + + delete(ownedTaints, nodeTaint.Key) + + return false + }) +} + +// ApplyCordoned marks the node as unschedulable if it is cordoned. +// +// This method is exported for testing purposes. +func (ctrl *NodeApplyController) ApplyCordoned(logger *zap.Logger, node *v1.Node, shouldCordon bool) { + switch { + case shouldCordon && !node.Spec.Unschedulable: + node.Spec.Unschedulable = true + + if node.Annotations == nil { + node.Annotations = map[string]string{} + } + + node.Annotations[constants.AnnotationCordonedKey] = constants.AnnotationCordonedValue + case !shouldCordon && node.Spec.Unschedulable: + if _, exists := node.Annotations[constants.AnnotationCordonedKey]; !exists { + // not cordoned by Talos, skip + return + } + + node.Spec.Unschedulable = false + delete(node.Annotations, constants.AnnotationCordonedKey) + } +} diff --git a/internal/app/machined/pkg/controllers/k8s/node_apply_test.go b/internal/app/machined/pkg/controllers/k8s/node_apply_test.go new file mode 100644 index 0000000..1dccc4b --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/node_apply_test.go @@ -0,0 +1,407 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s_test + +import ( + "slices" + "testing" + + "github.com/siderolabs/gen/maps" + "github.com/siderolabs/gen/xslices" + "github.com/stretchr/testify/assert" + "go.uber.org/zap/zaptest" + v1 "k8s.io/api/core/v1" + + k8sctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/k8s" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +func TestApplyLabels(t *testing.T) { + t.Parallel() + + ctrl := &k8sctrl.NodeApplyController{} + logger := zaptest.NewLogger(t) + + for _, tt := range []struct { + name string + inputLabels map[string]string + ownedLabels []string + labelSpec map[string]string + + expectedLabels map[string]string + expectedOwnedLabels []string + }{ + { + name: "empty", + inputLabels: map[string]string{}, + ownedLabels: []string{}, + labelSpec: map[string]string{}, + + expectedLabels: map[string]string{}, + expectedOwnedLabels: []string{}, + }, + { + name: "initial set labels", + inputLabels: map[string]string{ + "hostname": "foo", + }, + ownedLabels: []string{}, + labelSpec: map[string]string{ + "label1": "value1", + "label2": "value2", + }, + + expectedLabels: map[string]string{ + "hostname": "foo", + "label1": "value1", + "label2": "value2", + }, + expectedOwnedLabels: []string{ + "label1", + "label2", + }, + }, + { + name: "update owned labels", + inputLabels: map[string]string{ + "hostname": "foo", + "label1": "value1", + "label2": "value2", + }, + ownedLabels: []string{ + "label1", + "label2", + }, + labelSpec: map[string]string{ + "label1": "value3", + }, + + expectedLabels: map[string]string{ + "hostname": "foo", + "label1": "value3", + }, + expectedOwnedLabels: []string{ + "label1", + }, + }, + { + name: "ignore not owned labels", + inputLabels: map[string]string{ + "hostname": "foo", + "label1": "value1", + "label2": "value2", + "label3": "value3", + }, + ownedLabels: []string{}, + labelSpec: map[string]string{ + "label1": "value3", + "label2": "value2", + }, + + expectedLabels: map[string]string{ + "hostname": "foo", + "label1": "value1", + "label2": "value2", + "label3": "value3", + }, + expectedOwnedLabels: []string{ + "label2", + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + node := &v1.Node{} + node.Labels = tt.inputLabels + + ownedLabels := xslices.ToSet(tt.ownedLabels) + if ownedLabels == nil { + ownedLabels = map[string]struct{}{} + } + + ctrl.ApplyLabels(logger, node, ownedLabels, tt.labelSpec) + + newOwnedLabels := maps.Keys(ownedLabels) + if newOwnedLabels == nil { + newOwnedLabels = []string{} + } + + slices.Sort(newOwnedLabels) + + assert.Equal(t, tt.expectedLabels, node.Labels) + assert.Equal(t, tt.expectedOwnedLabels, newOwnedLabels) + }) + } +} + +func TestApplyTaints(t *testing.T) { + t.Parallel() + + ctrl := &k8sctrl.NodeApplyController{} + logger := zaptest.NewLogger(t) + + for _, tt := range []struct { + name string + inputTaints []v1.Taint + ownedTaints []string + taintSpec []k8s.NodeTaintSpecSpec + + expectedTaints []v1.Taint + expectedOwnedTaints []string + }{ + { + name: "empty", + inputTaints: nil, + ownedTaints: []string{}, + taintSpec: nil, + + expectedTaints: nil, + expectedOwnedTaints: []string{}, + }, + { + name: "initial set taints", + inputTaints: []v1.Taint{ + { + Key: "foo", + Value: "bar", + }, + }, + ownedTaints: []string{}, + taintSpec: []k8s.NodeTaintSpecSpec{ + { + Key: "taint1", + Value: "value1", + Effect: "NoSchedule", + }, + { + Key: "taint2", + Value: "value2", + }, + }, + + expectedTaints: []v1.Taint{ + { + Key: "foo", + Value: "bar", + }, + { + Key: "taint1", + Value: "value1", + Effect: "NoSchedule", + }, + { + Key: "taint2", + Value: "value2", + }, + }, + expectedOwnedTaints: []string{ + "taint1", + "taint2", + }, + }, + { + name: "update owned taints", + inputTaints: []v1.Taint{ + { + Key: "foo", + Value: "bar", + }, + { + Key: "taint1", + Value: "value1", + Effect: "NoSchedule", + }, + { + Key: "taint2", + Value: "value2", + }, + }, + ownedTaints: []string{ + "taint1", + "taint2", + }, + taintSpec: []k8s.NodeTaintSpecSpec{ + { + Key: "taint1", + Value: "value3", + }, + }, + + expectedTaints: []v1.Taint{ + { + Key: "foo", + Value: "bar", + }, + { + Key: "taint1", + Value: "value3", + }, + }, + expectedOwnedTaints: []string{ + "taint1", + }, + }, + { + name: "ignore not owned taints", + inputTaints: []v1.Taint{ + { + Key: "foo", + Value: "bar", + }, + { + Key: "taint1", + Value: "value1", + Effect: "NoSchedule", + }, + { + Key: "taint2", + Value: "value2", + }, + }, + ownedTaints: []string{}, + taintSpec: []k8s.NodeTaintSpecSpec{ + { + Key: "taint1", + Value: "value1", + Effect: "NoSchedule", + }, + { + Key: "taint2", + Value: "value3", + }, + }, + + expectedTaints: []v1.Taint{ + { + Key: "foo", + Value: "bar", + }, + { + Key: "taint1", + Value: "value1", + Effect: "NoSchedule", + }, + { + Key: "taint2", + Value: "value2", + }, + }, + expectedOwnedTaints: []string{ + "taint1", + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + node := &v1.Node{} + node.Spec.Taints = tt.inputTaints + + ownedTaints := xslices.ToSet(tt.ownedTaints) + if ownedTaints == nil { + ownedTaints = map[string]struct{}{} + } + + ctrl.ApplyTaints(logger, node, ownedTaints, tt.taintSpec) + + newOwnedTaints := maps.Keys(ownedTaints) + if newOwnedTaints == nil { + newOwnedTaints = []string{} + } + + slices.Sort(newOwnedTaints) + + assert.Equal(t, tt.expectedTaints, node.Spec.Taints) + assert.Equal(t, tt.expectedOwnedTaints, newOwnedTaints) + }) + } +} + +func TestApplyCordoned(t *testing.T) { + t.Parallel() + + ctrl := &k8sctrl.NodeApplyController{} + logger := zaptest.NewLogger(t) + + for _, tt := range []struct { + name string + inputAnnotations map[string]string + inputUnschedulable bool + shouldCordon bool + + expectedUnschedulable bool + expectedAnnotations map[string]string + }{ + { + name: "not cordoned - uncordon", + inputAnnotations: nil, + inputUnschedulable: false, + shouldCordon: false, + + expectedUnschedulable: false, + expectedAnnotations: nil, + }, + { + name: "not cordoned - cordon", + inputAnnotations: nil, + inputUnschedulable: false, + shouldCordon: true, + + expectedUnschedulable: true, + expectedAnnotations: map[string]string{constants.AnnotationCordonedKey: constants.AnnotationCordonedValue}, + }, + { + name: "cordoned - no annotation - cordon", + inputAnnotations: nil, + inputUnschedulable: true, + shouldCordon: true, + + expectedUnschedulable: true, + expectedAnnotations: nil, + }, + { + name: "cordoned - with annotation - cordon", + inputAnnotations: map[string]string{constants.AnnotationCordonedKey: constants.AnnotationCordonedValue}, + inputUnschedulable: true, + shouldCordon: true, + + expectedUnschedulable: true, + expectedAnnotations: map[string]string{constants.AnnotationCordonedKey: constants.AnnotationCordonedValue}, + }, + { + name: "cordoned - with annotation - uncordon", + inputAnnotations: map[string]string{constants.AnnotationCordonedKey: constants.AnnotationCordonedValue}, + inputUnschedulable: true, + shouldCordon: false, + + expectedUnschedulable: false, + expectedAnnotations: map[string]string{}, + }, + { + name: "cordoned - no annotation - uncordon", + inputAnnotations: map[string]string{"foo": "bar"}, + inputUnschedulable: true, + shouldCordon: false, + + expectedUnschedulable: true, + expectedAnnotations: map[string]string{"foo": "bar"}, + }, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + node := &v1.Node{} + node.Annotations = tt.inputAnnotations + node.Spec.Unschedulable = tt.inputUnschedulable + + ctrl.ApplyCordoned(logger, node, tt.shouldCordon) + + assert.Equal(t, tt.expectedUnschedulable, node.Spec.Unschedulable) + assert.Equal(t, tt.expectedAnnotations, node.Annotations) + }) + } +} diff --git a/internal/app/machined/pkg/controllers/k8s/node_cordoned_spec.go b/internal/app/machined/pkg/controllers/k8s/node_cordoned_spec.go new file mode 100644 index 0000000..ab932a8 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/node_cordoned_spec.go @@ -0,0 +1,105 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// NodeCordonedSpecController manages node cordoned status based on configuration. +type NodeCordonedSpecController struct{} + +// Name implements controller.Controller interface. +func (ctrl *NodeCordonedSpecController) Name() string { + return "k8s.NodeCordonedSpecController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *NodeCordonedSpecController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: runtime.NamespaceName, + Type: runtime.MachineStatusType, + ID: optional.Some(runtime.MachineStatusID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *NodeCordonedSpecController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: k8s.NodeCordonedSpecType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *NodeCordonedSpecController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + status, err := safe.ReaderGetByID[*runtime.MachineStatus](ctx, r, runtime.MachineStatusID) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting config: %w", err) + } + + var shouldCordon bool + + switch status.TypedSpec().Stage { //nolint:exhaustive + case runtime.MachineStageShuttingDown, runtime.MachineStageUpgrading, runtime.MachineStageResetting: + shouldCordon = true + case runtime.MachineStageBooting, runtime.MachineStageRunning: + shouldCordon = false + default: + // don't change cordoned status + continue + } + + if shouldCordon { + if err = safe.WriterModify(ctx, r, k8s.NewNodeCordonedSpec(k8s.NodeCordonedID), + func(k *k8s.NodeCordonedSpec) error { + return nil + }); err != nil { + return fmt.Errorf("error updating node cordoned spec: %w", err) + } + } else { + nodeCordoned, err := safe.ReaderListAll[*k8s.NodeCordonedSpec](ctx, r) + if err != nil { + return fmt.Errorf("error getting node cordoned specs: %w", err) + } + + for iter := nodeCordoned.Iterator(); iter.Next(); { + if err = r.Destroy(ctx, iter.Value().Metadata()); err != nil { + return fmt.Errorf("error destroying node cordoned spec: %w", err) + } + } + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/k8s/node_cordoned_spec_test.go b/internal/app/machined/pkg/controllers/k8s/node_cordoned_spec_test.go new file mode 100644 index 0000000..d3b7a60 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/node_cordoned_spec_test.go @@ -0,0 +1,76 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s_test + +import ( + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + k8sctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +type NodeCordonedSuite struct { + ctest.DefaultSuite +} + +func TestNodeCordonedSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, &NodeCordonedSuite{ + DefaultSuite: ctest.DefaultSuite{ + Timeout: 5 * time.Second, + AfterSetup: func(s *ctest.DefaultSuite) { + s.Require().NoError(s.Runtime().RegisterController(&k8sctrl.NodeCordonedSpecController{})) + }, + }, + }) +} + +func (suite *NodeCordonedSuite) updateMachineStage(stage runtime.MachineStage) { + status, err := safe.StateGetByID[*runtime.MachineStatus](suite.Ctx(), suite.State(), runtime.MachineStatusID) + if err != nil && !state.IsNotFoundError(err) { + suite.Require().NoError(err) + } + + if status == nil { + status = runtime.NewMachineStatus() + status.TypedSpec().Stage = stage + + suite.Require().NoError(suite.State().Create(suite.Ctx(), status)) + } else { + status.TypedSpec().Stage = stage + suite.Require().NoError(suite.State().Update(suite.Ctx(), status)) + } +} + +func (suite *NodeCordonedSuite) TestBootingRunning() { + suite.updateMachineStage(runtime.MachineStageBooting) + + rtestutils.AssertNoResource[*k8s.NodeCordonedSpec](suite.Ctx(), suite.T(), suite.State(), k8s.NodeCordonedID) + + suite.updateMachineStage(runtime.MachineStageRunning) + + rtestutils.AssertNoResource[*k8s.NodeCordonedSpec](suite.Ctx(), suite.T(), suite.State(), k8s.NodeCordonedID) +} + +func (suite *NodeCordonedSuite) TestResetting() { + suite.updateMachineStage(runtime.MachineStageRunning) + + rtestutils.AssertNoResource[*k8s.NodeCordonedSpec](suite.Ctx(), suite.T(), suite.State(), k8s.NodeCordonedID) + + suite.updateMachineStage(runtime.MachineStageResetting) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []string{k8s.NodeCordonedID}, + func(*k8s.NodeCordonedSpec, *assert.Assertions) {}) +} diff --git a/internal/app/machined/pkg/controllers/k8s/node_label_spec.go b/internal/app/machined/pkg/controllers/k8s/node_label_spec.go new file mode 100644 index 0000000..9392e82 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/node_label_spec.go @@ -0,0 +1,99 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +// NodeLabelSpecController manages k8s.NodeLabelsConfig based on configuration. +type NodeLabelSpecController struct{} + +// Name implements controller.Controller interface. +func (ctrl *NodeLabelSpecController) Name() string { + return "k8s.NodeLabelSpecController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *NodeLabelSpecController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: config.MachineConfigType, + ID: optional.Some(config.V1Alpha1ID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *NodeLabelSpecController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: k8s.NodeLabelSpecType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *NodeLabelSpecController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + cfg, err := safe.ReaderGetByID[*config.MachineConfig](ctx, r, config.V1Alpha1ID) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error getting config: %w", err) + } + + r.StartTrackingOutputs() + + var nodeLabels map[string]string + + if cfg != nil && cfg.Config().Machine() != nil { + nodeLabels = cfg.Config().Machine().NodeLabels() + + if cfg.Config().Machine().Type().IsControlPlane() { + if nodeLabels == nil { + nodeLabels = map[string]string{} + } + + nodeLabels[constants.LabelNodeRoleControlPlane] = "" + } + } + + for key, value := range nodeLabels { + if err = safe.WriterModify(ctx, r, k8s.NewNodeLabelSpec(key), func(k *k8s.NodeLabelSpec) error { + k.TypedSpec().Key = key + k.TypedSpec().Value = value + + return nil + }); err != nil { + return fmt.Errorf("error updating node label spec: %w", err) + } + } + + if err = safe.CleanupOutputs[*k8s.NodeLabelSpec](ctx, r); err != nil { + return err + } + } +} diff --git a/internal/app/machined/pkg/controllers/k8s/node_label_spec_test.go b/internal/app/machined/pkg/controllers/k8s/node_label_spec_test.go new file mode 100644 index 0000000..f412dac --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/node_label_spec_test.go @@ -0,0 +1,135 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s_test + +import ( + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + k8sctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/k8s" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +type NodeLabelsSuite struct { + ctest.DefaultSuite +} + +func TestNodeLabelsSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, &NodeLabelsSuite{ + DefaultSuite: ctest.DefaultSuite{ + Timeout: 5 * time.Second, + AfterSetup: func(s *ctest.DefaultSuite) { + s.Require().NoError(s.Runtime().RegisterController(&k8sctrl.NodeLabelSpecController{})) + }, + }, + }) +} + +func (suite *NodeLabelsSuite) updateMachineConfig(machineType machine.Type, labels map[string]string) { + cfg, err := safe.StateGetByID[*config.MachineConfig](suite.Ctx(), suite.State(), config.V1Alpha1ID) + if err != nil && !state.IsNotFoundError(err) { + suite.Require().NoError(err) + } + + if cfg == nil { + cfg = config.NewMachineConfig(container.NewV1Alpha1(&v1alpha1.Config{ + MachineConfig: &v1alpha1.MachineConfig{ + MachineType: machineType.String(), + MachineNodeLabels: labels, + }, + })) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + } else { + cfg.Container().RawV1Alpha1().MachineConfig.MachineNodeLabels = labels + cfg.Container().RawV1Alpha1().MachineConfig.MachineType = machineType.String() + suite.Require().NoError(suite.State().Update(suite.Ctx(), cfg)) + } +} + +func (suite *NodeLabelsSuite) TestAddLabel() { + // given + expectedLabel := "expectedLabel" + expectedValue := "expectedValue" + + // when + suite.updateMachineConfig(machine.TypeWorker, map[string]string{ + expectedLabel: expectedValue, + }) + + // then + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []string{expectedLabel}, + func(labelSpec *k8s.NodeLabelSpec, asrt *assert.Assertions) { + asrt.Equal(expectedValue, labelSpec.TypedSpec().Value) + }) + rtestutils.AssertNoResource[*k8s.NodeLabelSpec](suite.Ctx(), suite.T(), suite.State(), constants.LabelNodeRoleControlPlane) +} + +func (suite *NodeLabelsSuite) TestChangeLabel() { + // given + expectedLabel := "someLabel" + oldValue := "oldValue" + expectedValue := "newValue" + + // when + suite.updateMachineConfig(machine.TypeControlPlane, map[string]string{ + expectedLabel: oldValue, + }) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []string{expectedLabel}, + func(labelSpec *k8s.NodeLabelSpec, asrt *assert.Assertions) { + asrt.Equal(oldValue, labelSpec.TypedSpec().Value) + }) + + suite.updateMachineConfig(machine.TypeControlPlane, map[string]string{ + expectedLabel: expectedValue, + }) + + // then + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []string{expectedLabel}, + func(labelSpec *k8s.NodeLabelSpec, asrt *assert.Assertions) { + asrt.Equal(expectedValue, labelSpec.TypedSpec().Value) + }) + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []string{constants.LabelNodeRoleControlPlane}, + func(labelSpec *k8s.NodeLabelSpec, asrt *assert.Assertions) { + asrt.Empty(labelSpec.TypedSpec().Value) + }) +} + +func (suite *NodeLabelsSuite) TestDeleteLabel() { + // given + expectedLabel := "label" + expectedValue := "labelValue" + + // when + suite.updateMachineConfig(machine.TypeWorker, map[string]string{ + expectedLabel: expectedValue, + }) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []string{expectedLabel}, + func(labelSpec *k8s.NodeLabelSpec, asrt *assert.Assertions) { + asrt.Equal(expectedValue, labelSpec.TypedSpec().Value) + }) + + suite.updateMachineConfig(machine.TypeWorker, map[string]string{}) + + // then + rtestutils.AssertNoResource[*k8s.NodeLabelSpec](suite.Ctx(), suite.T(), suite.State(), expectedLabel) + rtestutils.AssertNoResource[*k8s.NodeLabelSpec](suite.Ctx(), suite.T(), suite.State(), constants.LabelNodeRoleControlPlane) +} diff --git a/internal/app/machined/pkg/controllers/k8s/node_status.go b/internal/app/machined/pkg/controllers/k8s/node_status.go new file mode 100644 index 0000000..18dec9f --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/node_status.go @@ -0,0 +1,215 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/k8s/internal/nodewatch" + "github.com/siderolabs/talos/pkg/conditions" + "github.com/siderolabs/talos/pkg/kubernetes" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +// watchErrorsThreshold is the number of consecutive watch errors before the controller stops watching. +const watchErrorsThreshold = 5 + +// NodeStatusController pulls list of Affiliate resource from the Kubernetes registry. +type NodeStatusController struct{} + +// Name implements controller.Controller interface. +func (ctrl *NodeStatusController) Name() string { + return "k8s.NodeStatusController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *NodeStatusController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: k8s.NamespaceName, + Type: k8s.NodenameType, + ID: optional.Some(k8s.NodenameID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *NodeStatusController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: k8s.NodeStatusType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *NodeStatusController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + var ( + kubernetesClient *kubernetes.Client + nodewatcher *nodewatch.NodeWatcher + watchCtxCancel context.CancelFunc + notifyCh <-chan struct{} + watchErrCh <-chan error + notifyCloser func() + watchErrors int + ) + + closeWatcher := func() { + if watchCtxCancel != nil { + watchCtxCancel() + watchCtxCancel = nil + } + + if notifyCloser != nil { + notifyCloser() + notifyCloser = nil + notifyCh = nil + watchErrCh = nil + watchErrors = 0 + } + + if kubernetesClient != nil { + kubernetesClient.Close() //nolint:errcheck + + kubernetesClient = nil + } + + nodewatcher = nil + } + + defer closeWatcher() + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + case <-notifyCh: + watchErrors = 0 + case watchErr := <-watchErrCh: + logger.Error("node watch error", zap.Error(watchErr), zap.Int("error_count", watchErrors)) + + watchErrors++ + + if watchErrors >= watchErrorsThreshold { + closeWatcher() + + watchErrors = 0 + } else { + // keep waiting + continue + } + } + + nodename, err := safe.ReaderGetByID[*k8s.Nodename](ctx, r, k8s.NodenameID) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting nodename: %w", err) + } + + continue + } + + if nodename.TypedSpec().SkipNodeRegistration { + // node is not registered with Kubernetes, so we can't pull the status + closeWatcher() + + continue + } + + if err = conditions.WaitForKubeconfigReady(constants.KubeletKubeconfig).Wait(ctx); err != nil { + return err + } + + if nodewatcher != nil && nodewatcher.Nodename() != nodename.TypedSpec().Nodename { + // nodename changed, so we need to reinitialize the watcher + closeWatcher() + } + + if kubernetesClient == nil { + kubernetesClient, err = kubernetes.NewClientFromKubeletKubeconfig() + if err != nil { + return fmt.Errorf("error building kubernetes client: %w", err) + } + } + + if nodewatcher == nil { + nodewatcher = nodewatch.NewNodeWatcher(kubernetesClient, nodename.TypedSpec().Nodename) + } + + if notifyCh == nil { + var watchCtx context.Context + watchCtx, watchCtxCancel = context.WithCancel(ctx) //nolint:govet + + notifyCh, watchErrCh, notifyCloser, err = nodewatcher.Watch(watchCtx) + if err != nil { + return fmt.Errorf("error setting up registry watcher: %w", err) //nolint:govet + } + } + + touchedIDs := make(map[resource.ID]struct{}) + + node, err := nodewatcher.Get() + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("error getting node: %w", err) + } + + if node != nil { + if err = safe.WriterModify[*k8s.NodeStatus](ctx, r, k8s.NewNodeStatus(k8s.NamespaceName, node.Name), + func(res *k8s.NodeStatus) error { + res.TypedSpec().Nodename = node.Name + res.TypedSpec().Unschedulable = node.Spec.Unschedulable + res.TypedSpec().Labels = node.Labels + res.TypedSpec().Annotations = node.Annotations + res.TypedSpec().NodeReady = false + + for _, condition := range node.Status.Conditions { + if condition.Type == v1.NodeReady { + res.TypedSpec().NodeReady = condition.Status == v1.ConditionTrue + } + } + + return nil + }, + ); err != nil { + return err + } + + touchedIDs[node.Name] = struct{}{} + } + + items, err := safe.ReaderListAll[*k8s.NodeStatus](ctx, r) + if err != nil { + return fmt.Errorf("error listing node statuses: %w", err) + } + + for iter := items.Iterator(); iter.Next(); { + if _, touched := touchedIDs[iter.Value().Metadata().ID()]; touched { + continue + } + + if err = r.Destroy(ctx, iter.Value().Metadata()); err != nil { + return fmt.Errorf("error destroying node status: %w", err) + } + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/k8s/node_taint_spec.go b/internal/app/machined/pkg/controllers/k8s/node_taint_spec.go new file mode 100644 index 0000000..e509d94 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/node_taint_spec.go @@ -0,0 +1,112 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import ( + "context" + "fmt" + "strings" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + v1 "k8s.io/api/core/v1" + + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +// NodeTaintSpecController manages k8s.NodeTaintSpec based on configuration. +type NodeTaintSpecController struct{} + +// Name implements controller.Controller interface. +func (ctrl *NodeTaintSpecController) Name() string { + return "k8s.NodeTaintSpecController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *NodeTaintSpecController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: config.MachineConfigType, + ID: optional.Some(config.V1Alpha1ID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *NodeTaintSpecController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: k8s.NodeTaintSpecType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *NodeTaintSpecController) Run(ctx context.Context, r controller.Runtime, _ *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + cfg, err := safe.ReaderGetByID[*config.MachineConfig](ctx, r, config.V1Alpha1ID) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error getting config: %w", err) + } + + r.StartTrackingOutputs() + + if cfg != nil && cfg.Config().Machine() != nil { + if cfg.Config().Cluster() != nil { + if cfg.Config().Machine().Type().IsControlPlane() && !cfg.Config().Cluster().ScheduleOnControlPlanes() { + if err = createTaint(ctx, r, constants.LabelNodeRoleControlPlane, "", string(v1.TaintEffectNoSchedule)); err != nil { + return err + } + } + } + + for key, val := range cfg.Config().Machine().NodeTaints() { + value, effect, found := strings.Cut(val, ":") + if !found { + effect = value + value = "" + } + + if err = createTaint(ctx, r, key, value, effect); err != nil { + return err + } + } + } + + if err = safe.CleanupOutputs[*k8s.NodeTaintSpec](ctx, r); err != nil { + return err + } + } +} + +func createTaint(ctx context.Context, r controller.Runtime, key string, val string, effect string) error { + if err := safe.WriterModify(ctx, r, k8s.NewNodeTaintSpec(key), func(k *k8s.NodeTaintSpec) error { + k.TypedSpec().Key = key + k.TypedSpec().Value = val + k.TypedSpec().Effect = effect + + return nil + }); err != nil { + return fmt.Errorf("error updating node taint spec: %w", err) + } + + return nil +} diff --git a/internal/app/machined/pkg/controllers/k8s/node_taint_spec_test.go b/internal/app/machined/pkg/controllers/k8s/node_taint_spec_test.go new file mode 100644 index 0000000..cfc508f --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/node_taint_spec_test.go @@ -0,0 +1,118 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s_test + +import ( + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/go-pointer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + v1 "k8s.io/api/core/v1" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + k8sctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/k8s" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +type NodeTaintsSuite struct { + ctest.DefaultSuite +} + +func TestNodeTaintsSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, &NodeTaintsSuite{ + DefaultSuite: ctest.DefaultSuite{ + Timeout: 5 * time.Second, + AfterSetup: func(s *ctest.DefaultSuite) { + s.Require().NoError(s.Runtime().RegisterController(&k8sctrl.NodeTaintSpecController{})) + }, + }, + }) +} + +func (suite *NodeTaintsSuite) updateMachineConfig(machineType machine.Type, allowScheduling bool, taints ...customTaint) { + cfg, err := safe.StateGetByID[*config.MachineConfig](suite.Ctx(), suite.State(), config.V1Alpha1ID) + if err != nil && !state.IsNotFoundError(err) { + suite.Require().NoError(err) + } + + nodeTaints := xslices.ToMap(taints, func(t customTaint) (string, string) { return t.key, t.value }) + + if cfg == nil { + cfg = config.NewMachineConfig(container.NewV1Alpha1(&v1alpha1.Config{ + MachineConfig: &v1alpha1.MachineConfig{ + MachineType: machineType.String(), + MachineNodeTaints: nodeTaints, + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + AllowSchedulingOnControlPlanes: pointer.To(allowScheduling), + }, + })) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + } else { + cfg.Container().RawV1Alpha1().ClusterConfig.AllowSchedulingOnControlPlanes = pointer.To(allowScheduling) + cfg.Container().RawV1Alpha1().MachineConfig.MachineType = machineType.String() + cfg.Container().RawV1Alpha1().MachineConfig.MachineNodeTaints = nodeTaints + suite.Require().NoError(suite.State().Update(suite.Ctx(), cfg)) + } +} + +func (suite *NodeTaintsSuite) TestWorker() { + suite.updateMachineConfig(machine.TypeWorker, false) + + rtestutils.AssertNoResource[*k8s.NodeTaintSpec](suite.Ctx(), suite.T(), suite.State(), constants.LabelNodeRoleControlPlane) +} + +func (suite *NodeTaintsSuite) TestControlplane() { + suite.updateMachineConfig(machine.TypeControlPlane, false) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []string{constants.LabelNodeRoleControlPlane}, + func(labelSpec *k8s.NodeTaintSpec, asrt *assert.Assertions) { + asrt.Empty(labelSpec.TypedSpec().Value) + asrt.Equal(string(v1.TaintEffectNoSchedule), labelSpec.TypedSpec().Effect) + }) + + suite.updateMachineConfig(machine.TypeControlPlane, true) + + rtestutils.AssertNoResource[*k8s.NodeTaintSpec](suite.Ctx(), suite.T(), suite.State(), constants.LabelNodeRoleControlPlane) +} + +func (suite *NodeTaintsSuite) TestCustomTaints() { + const customTaintKey = "key1" + + suite.updateMachineConfig(machine.TypeControlPlane, false, customTaint{ + key: customTaintKey, + value: "value1:NoSchedule", + }) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []string{customTaintKey}, + func(labelSpec *k8s.NodeTaintSpec, asrt *assert.Assertions) { + asrt.Equal(customTaintKey, labelSpec.TypedSpec().Key) + asrt.Equal("value1", labelSpec.TypedSpec().Value) + asrt.Equal(string(v1.TaintEffectNoSchedule), labelSpec.TypedSpec().Effect) + }) + + suite.updateMachineConfig(machine.TypeControlPlane, false) + + rtestutils.AssertNoResource[*k8s.NodeTaintSpec](suite.Ctx(), suite.T(), suite.State(), customTaintKey) +} + +type customTaint struct { + key string + value string +} diff --git a/internal/app/machined/pkg/controllers/k8s/nodeip.go b/internal/app/machined/pkg/controllers/k8s/nodeip.go new file mode 100644 index 0000000..20c9050 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/nodeip.go @@ -0,0 +1,159 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import ( + "context" + "fmt" + "net/netip" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/net" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// NodeIPController renders manifests based on templates and config/secrets. +type NodeIPController struct{} + +// Name implements controller.Controller interface. +func (ctrl *NodeIPController) Name() string { + return "k8s.NodeIPController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *NodeIPController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: k8s.NamespaceName, + Type: k8s.NodeIPConfigType, + ID: optional.Some(k8s.KubeletID), + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.NodeAddressType, + ID: optional.Some(network.FilteredNodeAddressID(network.NodeAddressRoutedID, k8s.NodeAddressFilterNoK8s)), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *NodeIPController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: k8s.NodeIPType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *NodeIPController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + cfg, err := safe.ReaderGet[*k8s.NodeIPConfig](ctx, r, resource.NewMetadata(k8s.NamespaceName, k8s.NodeIPConfigType, k8s.KubeletID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting config: %w", err) + } + + cfgSpec := cfg.TypedSpec() + + nodeAddrs, err := safe.ReaderGet[*network.NodeAddress]( + ctx, + r, + resource.NewMetadata( + network.NamespaceName, + network.NodeAddressType, + network.FilteredNodeAddressID(network.NodeAddressRoutedID, k8s.NodeAddressFilterNoK8s), + resource.VersionUndefined, + ), + ) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting addresses: %w", err) + } + + addrs := nodeAddrs.TypedSpec().IPs() + + cidrs := make([]string, 0, len(cfgSpec.ValidSubnets)+len(cfgSpec.ExcludeSubnets)) + cidrs = append(cidrs, cfgSpec.ValidSubnets...) + cidrs = append(cidrs, xslices.Map(cfgSpec.ExcludeSubnets, func(cidr string) string { return "!" + cidr })...) + + ips, err := net.FilterIPs(addrs, cidrs) + if err != nil { + return fmt.Errorf("error filtering IPs: %w", err) + } + + if len(ips) == 0 { + logger.Warn("no suitable node IP found, please make sure .machine.kubelet.nodeIP filters and pod/service subnets are set up correctly") + + continue + } + + // filter down to make sure only one IPv4 and one IPv6 address stays + var hasIPv4, hasIPv6 bool + + nodeIPs := make([]netip.Addr, 0, 2) + + for _, ip := range ips { + switch { + case ip.Is4(): + if !hasIPv4 { + nodeIPs = append(nodeIPs, ip) + hasIPv4 = true + } else { + logger.Warn("node IP skipped, please use .machine.kubelet.nodeIP to provide explicit subnet for the node IP", zap.Stringer("address", ip)) + } + case ip.Is6(): + if !hasIPv6 { + nodeIPs = append(nodeIPs, ip) + hasIPv6 = true + } else { + logger.Warn("node IP skipped, please use .machine.kubelet.nodeIP to provide explicit subnet for the node IP", zap.Stringer("address", ip)) + } + } + } + + if err = safe.WriterModify( + ctx, + r, + k8s.NewNodeIP(k8s.NamespaceName, k8s.KubeletID), + func(r *k8s.NodeIP) error { + spec := r.TypedSpec() + + spec.Addresses = nodeIPs + + return nil + }, + ); err != nil { + return fmt.Errorf("error modifying NodeIP resource: %w", err) + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/k8s/nodeip_config.go b/internal/app/machined/pkg/controllers/k8s/nodeip_config.go new file mode 100644 index 0000000..afec989 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/nodeip_config.go @@ -0,0 +1,105 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import ( + "context" + "fmt" + "net/netip" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/controller/generic/transform" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +// NodeIPConfigController configures k8s.NodeIP based on machine config. +type NodeIPConfigController = transform.Controller[*config.MachineConfig, *k8s.NodeIPConfig] + +// NewNodeIPConfigController instanciates the controller. +func NewNodeIPConfigController() *NodeIPConfigController { + return transform.NewController( + transform.Settings[*config.MachineConfig, *k8s.NodeIPConfig]{ + Name: "k8s.NodeIPConfigController", + MapMetadataOptionalFunc: func(cfg *config.MachineConfig) optional.Optional[*k8s.NodeIPConfig] { + if cfg.Metadata().ID() != config.V1Alpha1ID { + return optional.None[*k8s.NodeIPConfig]() + } + + if cfg.Config().Machine() == nil || cfg.Config().Cluster() == nil { + return optional.None[*k8s.NodeIPConfig]() + } + + return optional.Some(k8s.NewNodeIPConfig(k8s.NamespaceName, k8s.KubeletID)) + }, + TransformFunc: func(ctx context.Context, r controller.Reader, logger *zap.Logger, cfg *config.MachineConfig, res *k8s.NodeIPConfig) error { + spec := res.TypedSpec() + cfgProvider := cfg.Config() + + spec.ValidSubnets = cfgProvider.Machine().Kubelet().NodeIP().ValidSubnets() + + if len(spec.ValidSubnets) == 0 { + // automatically deduce validsubnets from ServiceCIDRs + var err error + + spec.ValidSubnets, err = ipSubnetsFromServiceCIDRs(cfgProvider.Cluster().Network().ServiceCIDRs()) + if err != nil { + return fmt.Errorf("error building valid subnets: %w", err) + } + } + + spec.ExcludeSubnets = nil + + // filter out Pod & Service CIDRs, they can't be kubelet IPs + spec.ExcludeSubnets = append( + append( + spec.ExcludeSubnets, + cfgProvider.Cluster().Network().PodCIDRs()..., + ), + cfgProvider.Cluster().Network().ServiceCIDRs()..., + ) + + // filter out any virtual IPs, they can't be node IPs either + for _, device := range cfgProvider.Machine().Network().Devices() { + if device.VIPConfig() != nil { + spec.ExcludeSubnets = append(spec.ExcludeSubnets, device.VIPConfig().IP()) + } + + for _, vlan := range device.Vlans() { + if vlan.VIPConfig() != nil { + spec.ExcludeSubnets = append(spec.ExcludeSubnets, vlan.VIPConfig().IP()) + } + } + } + + return nil + }, + }, + ) +} + +func ipSubnetsFromServiceCIDRs(serviceCIDRs []string) ([]string, error) { + // automatically configure valid IP subnets based on service CIDRs + // if the primary service CIDR is IPv4, primary kubelet node IP should be IPv4 as well, and so on + result := make([]string, 0, len(serviceCIDRs)) + + for _, cidr := range serviceCIDRs { + network, err := netip.ParsePrefix(cidr) + if err != nil { + return nil, fmt.Errorf("failed to parse subnet: %w", err) + } + + if network.Addr().Is6() { + result = append(result, "::/0") + } else { + result = append(result, "0.0.0.0/0") + } + } + + return result, nil +} diff --git a/internal/app/machined/pkg/controllers/k8s/nodeip_config_test.go b/internal/app/machined/pkg/controllers/k8s/nodeip_config_test.go new file mode 100644 index 0000000..ad17cb1 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/nodeip_config_test.go @@ -0,0 +1,225 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:dupl +package k8s_test + +import ( + "context" + "log" + "net/url" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + k8sctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/k8s" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +type NodeIPConfigSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *NodeIPConfigSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.Require().NoError(suite.runtime.RegisterController(k8sctrl.NewNodeIPConfigController())) + + suite.startRuntime() +} + +func (suite *NodeIPConfigSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *NodeIPConfigSuite) TestReconcileWithSubnets() { + u, err := url.Parse("https://foo:6443") + suite.Require().NoError(err) + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineKubelet: &v1alpha1.KubeletConfig{ + KubeletNodeIP: &v1alpha1.KubeletNodeIPConfig{ + KubeletNodeIPValidSubnets: []string{"10.0.0.0/24"}, + }, + }, + MachineNetwork: &v1alpha1.NetworkConfig{ + NetworkInterfaces: []*v1alpha1.Device{ + { + DeviceVIPConfig: &v1alpha1.DeviceVIPConfig{ + SharedIP: "1.2.3.4", + }, + DeviceVlans: []*v1alpha1.Vlan{ + { + VlanID: 100, + VlanVIP: &v1alpha1.DeviceVIPConfig{ + SharedIP: "5.6.7.8", + }, + }, + }, + }, + }, + }, + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: u, + }, + }, + ClusterNetwork: &v1alpha1.ClusterNetworkConfig{ + ServiceSubnet: []string{constants.DefaultIPv4ServiceNet}, + PodSubnet: []string{constants.DefaultIPv4PodNet}, + }, + }, + }, + ), + ) + + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + NodeIPConfig, err := suite.state.Get( + suite.ctx, + resource.NewMetadata( + k8s.NamespaceName, + k8s.NodeIPConfigType, + k8s.KubeletID, + resource.VersionUndefined, + ), + ) + if err != nil { + if state.IsNotFoundError(err) { + return retry.ExpectedError(err) + } + + return err + } + + spec := NodeIPConfig.(*k8s.NodeIPConfig).TypedSpec() + + suite.Assert().Equal([]string{"10.0.0.0/24"}, spec.ValidSubnets) + suite.Assert().Equal( + []string{"10.244.0.0/16", "10.96.0.0/12", "1.2.3.4", "5.6.7.8"}, + spec.ExcludeSubnets, + ) + + return nil + }, + ), + ) +} + +func (suite *NodeIPConfigSuite) TestReconcileDefaults() { + u, err := url.Parse("https://foo:6443") + suite.Require().NoError(err) + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{}, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: u, + }, + }, + ClusterNetwork: &v1alpha1.ClusterNetworkConfig{ + ServiceSubnet: []string{constants.DefaultIPv4ServiceNet, constants.DefaultIPv6ServiceNet}, + PodSubnet: []string{constants.DefaultIPv4PodNet, constants.DefaultIPv6PodNet}, + }, + }, + }, + ), + ) + + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + NodeIPConfig, err := suite.state.Get( + suite.ctx, + resource.NewMetadata( + k8s.NamespaceName, + k8s.NodeIPConfigType, + k8s.KubeletID, + resource.VersionUndefined, + ), + ) + if err != nil { + if state.IsNotFoundError(err) { + return retry.ExpectedError(err) + } + + return err + } + + spec := NodeIPConfig.(*k8s.NodeIPConfig).TypedSpec() + + suite.Assert().Equal([]string{"0.0.0.0/0", "::/0"}, spec.ValidSubnets) + suite.Assert().Equal( + []string{"10.244.0.0/16", "fc00:db8:10::/56", "10.96.0.0/12", "fc00:db8:20::/112"}, + spec.ExcludeSubnets, + ) + + return nil + }, + ), + ) +} + +func (suite *NodeIPConfigSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestNodeIPConfigSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, new(NodeIPConfigSuite)) +} diff --git a/internal/app/machined/pkg/controllers/k8s/nodeip_test.go b/internal/app/machined/pkg/controllers/k8s/nodeip_test.go new file mode 100644 index 0000000..c676d61 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/nodeip_test.go @@ -0,0 +1,113 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s_test + +import ( + "fmt" + "net/netip" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + k8sctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type NodeIPSuite struct { + ctest.DefaultSuite +} + +func (suite *NodeIPSuite) TestReconcileIPv4() { + cfg := k8s.NewNodeIPConfig(k8s.NamespaceName, k8s.KubeletID) + cfg.TypedSpec().ValidSubnets = []string{"10.0.0.0/24", "::/0"} + cfg.TypedSpec().ExcludeSubnets = []string{"10.0.0.2"} + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + addresses := network.NewNodeAddress( + network.NamespaceName, + network.FilteredNodeAddressID(network.NodeAddressRoutedID, k8s.NodeAddressFilterNoK8s), + ) + + addresses.TypedSpec().Addresses = []netip.Prefix{ + netip.MustParsePrefix("10.0.0.2/32"), // excluded explicitly + netip.MustParsePrefix("10.0.0.5/24"), + } + + suite.Require().NoError(suite.State().Create(suite.Ctx(), addresses)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.KubeletID}, func(nodeIP *k8s.NodeIP, asrt *assert.Assertions) { + asrt.Equal("[10.0.0.5]", fmt.Sprintf("%s", nodeIP.TypedSpec().Addresses)) + }) +} + +func (suite *NodeIPSuite) TestReconcileDefaultSubnets() { + cfg := k8s.NewNodeIPConfig(k8s.NamespaceName, k8s.KubeletID) + cfg.TypedSpec().ValidSubnets = []string{"0.0.0.0/0", "::/0"} + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + addresses := network.NewNodeAddress( + network.NamespaceName, + network.FilteredNodeAddressID(network.NodeAddressRoutedID, k8s.NodeAddressFilterNoK8s), + ) + addresses.TypedSpec().Addresses = []netip.Prefix{ + netip.MustParsePrefix("10.0.0.5/24"), + netip.MustParsePrefix("192.168.1.1/24"), + netip.MustParsePrefix("2001:0db8:85a3:0000:0000:8a2e:0370:7334/64"), + netip.MustParsePrefix("2001:0db8:85a3:0000:0000:8a2e:0370:7335/64"), + } + suite.Require().NoError(suite.State().Create(suite.Ctx(), addresses)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.KubeletID}, func(nodeIP *k8s.NodeIP, asrt *assert.Assertions) { + asrt.Equal("[10.0.0.5 2001:db8:85a3::8a2e:370:7334]", fmt.Sprintf("%s", nodeIP.TypedSpec().Addresses)) + }) +} + +func (suite *NodeIPSuite) TestReconcileNoMatch() { + cfg := k8s.NewNodeIPConfig(k8s.NamespaceName, k8s.KubeletID) + cfg.TypedSpec().ValidSubnets = []string{"0.0.0.0/0"} + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + addresses := network.NewNodeAddress( + network.NamespaceName, + network.FilteredNodeAddressID(network.NodeAddressRoutedID, k8s.NodeAddressFilterNoK8s), + ) + addresses.TypedSpec().Addresses = []netip.Prefix{ + netip.MustParsePrefix("10.0.0.2/32"), + netip.MustParsePrefix("10.0.0.5/24"), + } + suite.Require().NoError(suite.State().Create(suite.Ctx(), addresses)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.KubeletID}, func(nodeIP *k8s.NodeIP, asrt *assert.Assertions) { + asrt.Equal("[10.0.0.2]", fmt.Sprintf("%s", nodeIP.TypedSpec().Addresses)) + }) + + cfg.TypedSpec().ValidSubnets = nil + cfg.TypedSpec().ExcludeSubnets = []string{"10.0.0.2"} + suite.Require().NoError(suite.State().Update(suite.Ctx(), cfg)) + + // the node IP doesn't change, as there's no match for the filter + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.KubeletID}, func(nodeIP *k8s.NodeIP, asrt *assert.Assertions) { + asrt.Equal("[10.0.0.2]", fmt.Sprintf("%s", nodeIP.TypedSpec().Addresses)) + }) +} + +func TestNodeIPSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, &NodeIPSuite{ + DefaultSuite: ctest.DefaultSuite{ + Timeout: 5 * time.Second, + AfterSetup: func(s *ctest.DefaultSuite) { + s.Require().NoError(s.Runtime().RegisterController(&k8sctrl.NodeIPController{})) + }, + }, + }) +} diff --git a/internal/app/machined/pkg/controllers/k8s/nodename.go b/internal/app/machined/pkg/controllers/k8s/nodename.go new file mode 100644 index 0000000..592b3d3 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/nodename.go @@ -0,0 +1,123 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/k8s/internal/nodename" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// NodenameController renders manifests based on templates and config/secrets. +type NodenameController struct{} + +// Name implements controller.Controller interface. +func (ctrl *NodenameController) Name() string { + return "k8s.NodenameController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *NodenameController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: config.MachineConfigType, + ID: optional.Some(config.V1Alpha1ID), + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.HostnameStatusType, + ID: optional.Some(network.HostnameID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *NodenameController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: k8s.NodenameType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *NodenameController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + cfg, err := safe.ReaderGetByID[*config.MachineConfig](ctx, r, config.V1Alpha1ID) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting config: %w", err) + } + + cfgProvider := cfg.Config() + + if cfgProvider.Machine() == nil { + continue + } + + hostnameStatus, err := safe.ReaderGetByID[*network.HostnameStatus](ctx, r, network.HostnameID) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + if err = safe.WriterModify( + ctx, + r, + k8s.NewNodename(k8s.NamespaceName, k8s.NodenameID), + func(res *k8s.Nodename) error { + var hostname string + + if cfgProvider.Machine().Kubelet().RegisterWithFQDN() { + hostname = hostnameStatus.TypedSpec().FQDN() + } else { + hostname = hostnameStatus.TypedSpec().Hostname + } + + res.TypedSpec().Nodename, err = nodename.FromHostname(hostname) + if err != nil { + return err + } + + res.TypedSpec().HostnameVersion = hostnameStatus.Metadata().Version().String() + res.TypedSpec().SkipNodeRegistration = cfgProvider.Machine().Kubelet().SkipNodeRegistration() + + return nil + }, + ); err != nil { + return fmt.Errorf("error modifying nodename resource: %w", err) + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/k8s/nodename_test.go b/internal/app/machined/pkg/controllers/k8s/nodename_test.go new file mode 100644 index 0000000..d6f462c --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/nodename_test.go @@ -0,0 +1,114 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s_test + +import ( + "net/url" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/siderolabs/go-pointer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + k8sctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/k8s" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type NodenameSuite struct { + ctest.DefaultSuite +} + +func (suite *NodenameSuite) assertNodename(expected string) { + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.NodenameID}, func(nodename *k8s.Nodename, asrt *assert.Assertions) { + asrt.Equal(expected, nodename.TypedSpec().Nodename) + }) +} + +func (suite *NodenameSuite) TestDefault() { + u, err := url.Parse("https://foo:6443") + suite.Require().NoError(err) + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{}, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: u, + }, + }, + }, + }, + ), + ) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + hostnameStatus := network.NewHostnameStatus(network.NamespaceName, network.HostnameID) + hostnameStatus.TypedSpec().Hostname = "Foo-" + hostnameStatus.TypedSpec().Domainname = "bar.ltd" + + suite.Require().NoError(suite.State().Create(suite.Ctx(), hostnameStatus)) + + suite.assertNodename("foo") +} + +func (suite *NodenameSuite) TestFQDN() { + u, err := url.Parse("https://foo:6443") + suite.Require().NoError(err) + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineKubelet: &v1alpha1.KubeletConfig{ + KubeletRegisterWithFQDN: pointer.To(true), + }, + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: u, + }, + }, + }, + }, + ), + ) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + hostnameStatus := network.NewHostnameStatus(network.NamespaceName, network.HostnameID) + hostnameStatus.TypedSpec().Hostname = "foo" + hostnameStatus.TypedSpec().Domainname = "bar.ltd" + + suite.Require().NoError(suite.State().Create(suite.Ctx(), hostnameStatus)) + + suite.assertNodename("foo.bar.ltd") +} + +func TestNodenameSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, &NodenameSuite{ + DefaultSuite: ctest.DefaultSuite{ + Timeout: 3 * time.Second, + AfterSetup: func(s *ctest.DefaultSuite) { + s.Require().NoError(s.Runtime().RegisterController(&k8sctrl.NodenameController{})) + }, + }, + }) +} diff --git a/internal/app/machined/pkg/controllers/k8s/render_config_static_pods.go b/internal/app/machined/pkg/controllers/k8s/render_config_static_pods.go new file mode 100644 index 0000000..639e2e9 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/render_config_static_pods.go @@ -0,0 +1,257 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/runtime" + k8sjson "k8s.io/apimachinery/pkg/runtime/serializer/json" + apiserverv1 "k8s.io/apiserver/pkg/apis/apiserver/v1" + auditv1 "k8s.io/apiserver/pkg/apis/audit/v1" + schedulerv1 "k8s.io/kube-scheduler/config/v1" + + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +// RenderConfigsStaticPodController manages k8s.ConfigsReady and renders configs for the control plane. +type RenderConfigsStaticPodController struct{} + +// Name implements controller.Controller interface. +func (ctrl *RenderConfigsStaticPodController) Name() string { + return "k8s.RenderConfigsStaticPodController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *RenderConfigsStaticPodController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: k8s.ControlPlaneNamespaceName, + Type: k8s.AdmissionControlConfigType, + Kind: controller.InputWeak, + }, + { + Namespace: k8s.ControlPlaneNamespaceName, + Type: k8s.AuditPolicyConfigType, + Kind: controller.InputWeak, + }, + { + Namespace: k8s.ControlPlaneNamespaceName, + Type: k8s.SchedulerConfigType, + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *RenderConfigsStaticPodController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: k8s.ConfigStatusType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *RenderConfigsStaticPodController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + admissionRes, err := safe.ReaderGetByID[*k8s.AdmissionControlConfig](ctx, r, k8s.AdmissionControlConfigID) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting admission config resource: %w", err) + } + + admissionConfig := admissionRes.TypedSpec() + + auditRes, err := safe.ReaderGetByID[*k8s.AuditPolicyConfig](ctx, r, k8s.AuditPolicyConfigID) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting audit config resource: %w", err) + } + + auditConfig := auditRes.TypedSpec() + + kubeSchedulerRes, err := safe.ReaderGetByID[*k8s.SchedulerConfig](ctx, r, k8s.SchedulerConfigID) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting scheduler config resource: %w", err) + } + + kubeSchedulerConfig := kubeSchedulerRes.TypedSpec() + + type configFile struct { + filename string + f func() (runtime.Object, error) + } + + serializer := k8sjson.NewSerializerWithOptions( + k8sjson.DefaultMetaFactory, nil, nil, + k8sjson.SerializerOptions{ + Yaml: true, + Pretty: true, + Strict: true, + }, + ) + + for _, pod := range []struct { + name string + directory string + uid int + gid int + configs []configFile + }{ + { + name: "kube-apiserver", + directory: constants.KubernetesAPIServerConfigDir, + uid: constants.KubernetesAPIServerRunUser, + gid: constants.KubernetesAPIServerRunGroup, + configs: []configFile{ + { + filename: "admission-control-config.yaml", + f: admissionControlConfig(admissionConfig), + }, + { + filename: "auditpolicy.yaml", + f: auditPolicyConfig(auditConfig), + }, + }, + }, + { + name: "kube-scheduler", + directory: constants.KubernetesSchedulerConfigDir, + uid: constants.KubernetesSchedulerRunUser, + gid: constants.KubernetesSchedulerRunGroup, + configs: []configFile{ + { + filename: "scheduler-config.yaml", + f: schedulerConfig(kubeSchedulerConfig), + }, + }, + }, + } { + if err = os.MkdirAll(pod.directory, 0o755); err != nil { + return fmt.Errorf("error creating config directory for %q: %w", pod.name, err) + } + + for _, configFile := range pod.configs { + var obj runtime.Object + + obj, err = configFile.f() + if err != nil { + return fmt.Errorf("error generating configuration %q for %q: %w", configFile.filename, pod.name, err) + } + + var buf bytes.Buffer + + if err = serializer.Encode(obj, &buf); err != nil { + return fmt.Errorf("error marshaling configuration %q for %q: %w", configFile.filename, pod.name, err) + } + + if err = os.WriteFile(filepath.Join(pod.directory, configFile.filename), buf.Bytes(), 0o400); err != nil { + return fmt.Errorf("error writing configuration %q for %q: %w", configFile.filename, pod.name, err) + } + + if err = os.Chown(filepath.Join(pod.directory, configFile.filename), pod.uid, pod.gid); err != nil { + return fmt.Errorf("error chowning %q for %q: %w", configFile.filename, pod.name, err) + } + } + } + + if err = safe.WriterModify(ctx, r, k8s.NewConfigStatus(k8s.ControlPlaneNamespaceName, k8s.ConfigStatusStaticPodID), func(r *k8s.ConfigStatus) error { + r.TypedSpec().Ready = true + r.TypedSpec().Version = admissionRes.Metadata().Version().String() + auditRes.Metadata().Version().String() + kubeSchedulerRes.Metadata().Version().String() + + return nil + }); err != nil { + return err + } + + r.ResetRestartBackoff() + } +} + +func admissionControlConfig(spec *k8s.AdmissionControlConfigSpec) func() (runtime.Object, error) { + return func() (runtime.Object, error) { + var cfg apiserverv1.AdmissionConfiguration + + cfg.APIVersion = apiserverv1.SchemeGroupVersion.String() + cfg.Kind = "AdmissionConfiguration" + cfg.Plugins = []apiserverv1.AdmissionPluginConfiguration{} + + for _, plugin := range spec.Config { + raw, err := json.Marshal(plugin.Configuration) + if err != nil { + return nil, fmt.Errorf("error marshaling configuration for plugin %q: %w", plugin.Name, err) + } + + cfg.Plugins = append(cfg.Plugins, + apiserverv1.AdmissionPluginConfiguration{ + Name: plugin.Name, + Configuration: &runtime.Unknown{ + Raw: raw, + }, + }, + ) + } + + return &cfg, nil + } +} + +func auditPolicyConfig(spec *k8s.AuditPolicyConfigSpec) func() (runtime.Object, error) { + return func() (runtime.Object, error) { + var cfg auditv1.Policy + + if err := runtime.DefaultUnstructuredConverter.FromUnstructuredWithValidation(spec.Config, &cfg, true); err != nil { + return nil, fmt.Errorf("error unmarshaling audit policy configuration: %w", err) + } + + return &cfg, nil + } +} + +func schedulerConfig(spec *k8s.SchedulerConfigSpec) func() (runtime.Object, error) { + return func() (runtime.Object, error) { + var cfg schedulerv1.KubeSchedulerConfiguration + + if err := runtime.DefaultUnstructuredConverter.FromUnstructuredWithValidation(spec.Config, &cfg, false); err != nil { + return nil, fmt.Errorf("error unmarshaling scheduler configuration: %w", err) + } + + cfg.APIVersion = "kubescheduler.config.k8s.io/v1" + cfg.Kind = "KubeSchedulerConfiguration" + cfg.ClientConnection.Kubeconfig = filepath.Join(constants.KubernetesSchedulerSecretsDir, "kubeconfig") + + return &cfg, nil + } +} diff --git a/internal/app/machined/pkg/controllers/k8s/render_secrets_static_pod.go b/internal/app/machined/pkg/controllers/k8s/render_secrets_static_pod.go new file mode 100644 index 0000000..61bbca7 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/render_secrets_static_pod.go @@ -0,0 +1,346 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + stdlibtemplate "text/template" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/crypto/x509" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/xslices" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" +) + +// RenderSecretsStaticPodController manages k8s.SecretsReady and renders secrets from secrets.Kubernetes. +type RenderSecretsStaticPodController struct{} + +// Name implements controller.Controller interface. +func (ctrl *RenderSecretsStaticPodController) Name() string { + return "k8s.RenderSecretsStaticPodController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *RenderSecretsStaticPodController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: secrets.NamespaceName, + Type: secrets.KubernetesRootType, + ID: optional.Some(secrets.KubernetesRootID), + Kind: controller.InputWeak, + }, + { + Namespace: secrets.NamespaceName, + Type: secrets.EtcdRootType, + ID: optional.Some(secrets.EtcdRootID), + Kind: controller.InputWeak, + }, + { + Namespace: secrets.NamespaceName, + Type: secrets.KubernetesType, + ID: optional.Some(secrets.KubernetesID), + Kind: controller.InputWeak, + }, + { + Namespace: secrets.NamespaceName, + Type: secrets.KubernetesDynamicCertsType, + ID: optional.Some(secrets.KubernetesDynamicCertsID), + Kind: controller.InputWeak, + }, + { + Namespace: secrets.NamespaceName, + Type: secrets.EtcdType, + ID: optional.Some(secrets.EtcdID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *RenderSecretsStaticPodController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: k8s.SecretsStatusType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + secretsRes, err := safe.ReaderGet[*secrets.Kubernetes](ctx, r, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesType, secrets.KubernetesID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting secrets resource: %w", err) + } + + certsRes, err := safe.ReaderGet[*secrets.KubernetesDynamicCerts]( + ctx, r, + resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesDynamicCertsType, secrets.KubernetesDynamicCertsID, resource.VersionUndefined), + ) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting certificates resource: %w", err) + } + + etcdRes, err := safe.ReaderGet[*secrets.Etcd](ctx, r, resource.NewMetadata(secrets.NamespaceName, secrets.EtcdType, secrets.EtcdID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting secrets resource: %w", err) + } + + rootEtcdRes, err := safe.ReaderGet[*secrets.EtcdRoot](ctx, r, resource.NewMetadata(secrets.NamespaceName, secrets.EtcdRootType, secrets.EtcdRootID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting secrets resource: %w", err) + } + + rootK8sRes, err := safe.ReaderGet[*secrets.KubernetesRoot](ctx, r, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesRootType, secrets.KubernetesRootID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting secrets resource: %w", err) + } + + rootEtcdSecrets := rootEtcdRes.TypedSpec() + rootK8sSecrets := rootK8sRes.TypedSpec() + etcdSecrets := etcdRes.TypedSpec() + k8sSecrets := secretsRes.TypedSpec() + k8sCerts := certsRes.TypedSpec() + + serviceAccountKey, err := rootK8sSecrets.ServiceAccount.GetKey() + if err != nil { + return fmt.Errorf("error parsing service account key: %w", err) + } + + type secret struct { + getter func() *x509.PEMEncodedCertificateAndKey + certFilename string + keyFilename string + } + + type template struct { + filename string + template []byte + } + + for _, pod := range []struct { + name string + directory string + uid int + gid int + secrets []secret + templates []template + }{ + { + name: "kube-apiserver", + directory: constants.KubernetesAPIServerSecretsDir, + uid: constants.KubernetesAPIServerRunUser, + gid: constants.KubernetesAPIServerRunGroup, + secrets: []secret{ + { + getter: func() *x509.PEMEncodedCertificateAndKey { return rootEtcdSecrets.EtcdCA }, + certFilename: "etcd-client-ca.crt", + }, + { + getter: func() *x509.PEMEncodedCertificateAndKey { return etcdSecrets.EtcdAPIServer }, + certFilename: "etcd-client.crt", + keyFilename: "etcd-client.key", + }, + { + getter: func() *x509.PEMEncodedCertificateAndKey { + return &x509.PEMEncodedCertificateAndKey{ + Crt: bytes.Join(xslices.Map(rootK8sSecrets.AcceptedCAs, func(ca *x509.PEMEncodedCertificate) []byte { return ca.Crt }), nil), + } + }, + certFilename: "ca.crt", + }, + { + getter: func() *x509.PEMEncodedCertificateAndKey { return k8sCerts.APIServer }, + certFilename: "apiserver.crt", + keyFilename: "apiserver.key", + }, + { + getter: func() *x509.PEMEncodedCertificateAndKey { return k8sCerts.APIServerKubeletClient }, + certFilename: "apiserver-kubelet-client.crt", + keyFilename: "apiserver-kubelet-client.key", + }, + { + getter: func() *x509.PEMEncodedCertificateAndKey { + return &x509.PEMEncodedCertificateAndKey{ + Crt: serviceAccountKey.GetPublicKeyPEM(), + Key: serviceAccountKey.GetPrivateKeyPEM(), + } + }, + certFilename: "service-account.pub", + keyFilename: "service-account.key", + }, + { + getter: func() *x509.PEMEncodedCertificateAndKey { return rootK8sSecrets.AggregatorCA }, + certFilename: "aggregator-ca.crt", + }, + { + getter: func() *x509.PEMEncodedCertificateAndKey { return k8sCerts.FrontProxy }, + certFilename: "front-proxy-client.crt", + keyFilename: "front-proxy-client.key", + }, + }, + templates: []template{ + { + filename: "encryptionconfig.yaml", + template: kubeSystemEncryptionConfigTemplate, + }, + }, + }, + { + name: "kube-controller-manager", + directory: constants.KubernetesControllerManagerSecretsDir, + uid: constants.KubernetesControllerManagerRunUser, + gid: constants.KubernetesControllerManagerRunGroup, + secrets: []secret{ + { + getter: func() *x509.PEMEncodedCertificateAndKey { return rootK8sSecrets.IssuingCA }, + certFilename: "ca.crt", + keyFilename: "ca.key", + }, + { + getter: func() *x509.PEMEncodedCertificateAndKey { + return &x509.PEMEncodedCertificateAndKey{ + Crt: serviceAccountKey.GetPublicKeyPEM(), + Key: serviceAccountKey.GetPrivateKeyPEM(), + } + }, + keyFilename: "service-account.key", + }, + }, + templates: []template{ + { + filename: "kubeconfig", + template: []byte("{{ .Secrets.ControllerManagerKubeconfig }}"), + }, + }, + }, + { + name: "kube-scheduler", + directory: constants.KubernetesSchedulerSecretsDir, + uid: constants.KubernetesSchedulerRunUser, + gid: constants.KubernetesSchedulerRunGroup, + templates: []template{ + { + filename: "kubeconfig", + template: []byte("{{ .Secrets.SchedulerKubeconfig }}"), + }, + }, + }, + } { + if err = os.MkdirAll(pod.directory, 0o755); err != nil { + return fmt.Errorf("error creating secrets directory for %q: %w", pod.name, err) + } + + for _, secret := range pod.secrets { + certAndKey := secret.getter() + + if secret.certFilename != "" { + if err = os.WriteFile(filepath.Join(pod.directory, secret.certFilename), certAndKey.Crt, 0o400); err != nil { + return fmt.Errorf("error writing certificate %q for %q: %w", secret.certFilename, pod.name, err) + } + + if err = os.Chown(filepath.Join(pod.directory, secret.certFilename), pod.uid, pod.gid); err != nil { + return fmt.Errorf("error chowning %q for %q: %w", secret.certFilename, pod.name, err) + } + } + + if secret.keyFilename != "" { + if err = os.WriteFile(filepath.Join(pod.directory, secret.keyFilename), certAndKey.Key, 0o400); err != nil { + return fmt.Errorf("error writing key %q for %q: %w", secret.keyFilename, pod.name, err) + } + + if err = os.Chown(filepath.Join(pod.directory, secret.keyFilename), pod.uid, pod.gid); err != nil { + return fmt.Errorf("error chowning %q for %q: %w", secret.keyFilename, pod.name, err) + } + } + } + + type templateParams struct { + Root *secrets.KubernetesRootSpec + Secrets *secrets.KubernetesCertsSpec + } + + params := templateParams{ + Root: rootK8sSecrets, + Secrets: k8sSecrets, + } + + for _, templ := range pod.templates { + var t *stdlibtemplate.Template + + t, err = stdlibtemplate.New(templ.filename).Parse(string(templ.template)) + if err != nil { + return fmt.Errorf("error parsing template %q: %w", templ.filename, err) + } + + var buf bytes.Buffer + + if err = t.Execute(&buf, ¶ms); err != nil { + return fmt.Errorf("error executing template %q: %w", templ.filename, err) + } + + if err = os.WriteFile(filepath.Join(pod.directory, templ.filename), buf.Bytes(), 0o400); err != nil { + return fmt.Errorf("error writing template %q for %q: %w", templ.filename, pod.name, err) + } + + if err = os.Chown(filepath.Join(pod.directory, templ.filename), pod.uid, pod.gid); err != nil { + return fmt.Errorf("error chowning %q for %q: %w", templ.filename, pod.name, err) + } + } + } + + if err = safe.WriterModify(ctx, r, k8s.NewSecretsStatus(k8s.ControlPlaneNamespaceName, k8s.StaticPodSecretsStaticPodID), func(r *k8s.SecretsStatus) error { + r.TypedSpec().Ready = true + r.TypedSpec().Version = secretsRes.Metadata().Version().String() + + return nil + }); err != nil { + return err + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/k8s/static_endpoint.go b/internal/app/machined/pkg/controllers/k8s/static_endpoint.go new file mode 100644 index 0000000..8497d19 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/static_endpoint.go @@ -0,0 +1,100 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import ( + "context" + "fmt" + "net" + "net/netip" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/xslices" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +// StaticEndpointController injects endpoints based on machine configuration. +type StaticEndpointController struct{} + +// Name implements controller.Controller interface. +func (ctrl *StaticEndpointController) Name() string { + return "k8s.StaticEndpointController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *StaticEndpointController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: config.MachineConfigType, + ID: optional.Some(config.V1Alpha1ID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *StaticEndpointController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: k8s.EndpointType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *StaticEndpointController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + machineConfig, err := safe.ReaderGetByID[*config.MachineConfig](ctx, r, config.V1Alpha1ID) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error getting machine config: %w", err) + } + + r.StartTrackingOutputs() + + if machineConfig != nil && machineConfig.Config().Cluster() != nil { + cpHostname := machineConfig.Config().Cluster().Endpoint().Hostname() + + var ( + resolver net.Resolver + addrs []netip.Addr + ) + + addrs, err = resolver.LookupNetIP(ctx, "ip", cpHostname) + if err != nil { + return fmt.Errorf("error resolving %q: %w", cpHostname, err) + } + + addrs = xslices.Map(addrs, netip.Addr.Unmap) + + if err = safe.WriterModify(ctx, r, k8s.NewEndpoint(k8s.ControlPlaneNamespaceName, k8s.ControlPlaneKubernetesEndpointsID), func(endpoint *k8s.Endpoint) error { + endpoint.TypedSpec().Addresses = addrs + + return nil + }); err != nil { + return fmt.Errorf("error modifying endpoint: %w", err) + } + } + + if err = safe.CleanupOutputs[*k8s.Endpoint](ctx, r); err != nil { + return err + } + } +} diff --git a/internal/app/machined/pkg/controllers/k8s/static_endpoint_test.go b/internal/app/machined/pkg/controllers/k8s/static_endpoint_test.go new file mode 100644 index 0000000..ead546b --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/static_endpoint_test.go @@ -0,0 +1,71 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s_test + +import ( + "net/netip" + "net/url" + "testing" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + k8sctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/k8s" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +type StaticEndpointControllerSuite struct { + ctest.DefaultSuite +} + +func (suite *StaticEndpointControllerSuite) TestReconcile() { + u, err := url.Parse("https://[2001:db8::1]:6443/") + suite.Require().NoError(err) + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{}, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: u, + }, + }, + }, + }, + ), + ) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{k8s.ControlPlaneKubernetesEndpointsID}, + func(endpoint *k8s.Endpoint, assert *assert.Assertions) { + assert.Equal([]netip.Addr{netip.MustParseAddr("2001:db8::1")}, endpoint.TypedSpec().Addresses) + }) + + suite.Require().NoError(suite.State().Destroy(suite.Ctx(), cfg.Metadata())) + + rtestutils.AssertNoResource[*k8s.Endpoint](suite.Ctx(), suite.T(), suite.State(), k8s.ControlPlaneKubernetesEndpointsID) +} + +func TestStaticEndpointControllerSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, &StaticEndpointControllerSuite{ + DefaultSuite: ctest.DefaultSuite{ + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(&k8sctrl.StaticEndpointController{})) + }, + }, + }) +} diff --git a/internal/app/machined/pkg/controllers/k8s/static_pod_config.go b/internal/app/machined/pkg/controllers/k8s/static_pod_config.go new file mode 100644 index 0000000..1b8a219 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/static_pod_config.go @@ -0,0 +1,118 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import ( + "context" + "errors" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +// StaticPodConfigController manages k8s.StaticPod based on machine configuration. +type StaticPodConfigController struct{} + +// Name implements controller.Controller interface. +func (ctrl *StaticPodConfigController) Name() string { + return "k8s.StaticPodConfigController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *StaticPodConfigController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: config.MachineConfigType, + ID: optional.Some(config.V1Alpha1ID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *StaticPodConfigController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: k8s.StaticPodType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *StaticPodConfigController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + cfg, err := safe.ReaderGetByID[*config.MachineConfig](ctx, r, config.V1Alpha1ID) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting config: %w", err) + } + } + + r.StartTrackingOutputs() + + if cfg != nil && cfg.Config().Machine() != nil { + cfgProvider := cfg.Config() + + for _, pod := range cfgProvider.Machine().Pods() { + var ( + name, namespace string + ok bool + ) + + name, ok, err = unstructured.NestedString(pod, "metadata", "name") + if err != nil { + return fmt.Errorf("error getting name from static pod: %w", err) + } + + if !ok { + return errors.New("name is missing in static pod metadata") + } + + namespace, ok, err = unstructured.NestedString(pod, "metadata", "namespace") + if err != nil { + return fmt.Errorf("error getting namespace from static pod: %w", err) + } + + if !ok { + namespace = corev1.NamespaceDefault + } + + id := fmt.Sprintf("%s-%s", namespace, name) + + if err = safe.WriterModify(ctx, r, k8s.NewStaticPod(k8s.NamespaceName, id), func(r *k8s.StaticPod) error { + r.TypedSpec().Pod = pod + + return nil + }); err != nil { + return fmt.Errorf("error modifying resource: %w", err) + } + } + } + + // clean up static pods which haven't been touched + if err = safe.CleanupOutputs[*k8s.StaticPod](ctx, r); err != nil { + return err + } + } +} diff --git a/internal/app/machined/pkg/controllers/k8s/static_pod_config_test.go b/internal/app/machined/pkg/controllers/k8s/static_pod_config_test.go new file mode 100644 index 0000000..ad7db84 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/static_pod_config_test.go @@ -0,0 +1,206 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:dupl +package k8s_test + +import ( + "context" + "log" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + k8sctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/k8s" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +type StaticPodConfigSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *StaticPodConfigSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.Require().NoError(suite.runtime.RegisterController(&k8sctrl.StaticPodConfigController{})) + + suite.startRuntime() +} + +func (suite *StaticPodConfigSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *StaticPodConfigSuite) assertResource( + md resource.Metadata, + check func(res resource.Resource) error, +) func() error { + return func() error { + r, err := suite.state.Get(suite.ctx, md) + if err != nil { + if state.IsNotFoundError(err) { + return retry.ExpectedError(err) + } + + return err + } + + return check(r) + } +} + +func (suite *StaticPodConfigSuite) assertNoResource(md resource.Metadata) func() error { + return func() error { + _, err := suite.state.Get(suite.ctx, md) + if err == nil { + return retry.ExpectedErrorf("resource %s still exists", md) + } + + if state.IsNotFoundError(err) { + return nil + } + + return err + } +} + +func (suite *StaticPodConfigSuite) TestReconcile() { + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachinePods: []v1alpha1.Unstructured{ + { + Object: map[string]any{ + "apiVersion": "v1", + "kind": "pod", + "metadata": map[string]any{ + "name": "nginx", + }, + "spec": map[string]any{ + "containers": []any{ + map[string]any{ + "name": "nginx", + "image": "nginx", + }, + }, + }, + }, + }, + }, + }, + ClusterConfig: &v1alpha1.ClusterConfig{}, + }, + )) + + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertResource( + *k8s.NewStaticPod(k8s.NamespaceName, "default-nginx").Metadata(), + func(res resource.Resource) error { + v, ok, err := unstructured.NestedString(res.(*k8s.StaticPod).TypedSpec().Pod, "kind") + suite.Require().NoError(err) + suite.Assert().True(ok) + suite.Assert().Equal("pod", v) + + return nil + }, + ), + ), + ) + + // update the pod changing the namespace + cfg.Container().RawV1Alpha1().MachineConfig.MachinePods[0].Object["metadata"].(map[string]any)["namespace"] = "custom" + suite.Require().NoError(suite.state.Update(suite.ctx, cfg)) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertNoResource( + *k8s.NewStaticPod(k8s.NamespaceName, "default-nginx").Metadata(), + ), + ), + ) + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertResource( + *k8s.NewStaticPod(k8s.NamespaceName, "custom-nginx").Metadata(), + func(res resource.Resource) error { + v, ok, err := unstructured.NestedString( + res.(*k8s.StaticPod).TypedSpec().Pod, + "metadata", + "namespace", + ) + suite.Require().NoError(err) + suite.Assert().True(ok) + suite.Assert().Equal("custom", v) + + return nil + }, + ), + ), + ) + + // remove all pods + cfg.Container().RawV1Alpha1().MachineConfig.MachinePods = nil + suite.Require().NoError(suite.state.Update(suite.ctx, cfg)) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertNoResource( + *k8s.NewStaticPod(k8s.NamespaceName, "custom-nginx").Metadata(), + ), + ), + ) +} + +func (suite *StaticPodConfigSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestStaticPodConfigSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, new(StaticPodConfigSuite)) +} diff --git a/internal/app/machined/pkg/controllers/k8s/static_pod_server.go b/internal/app/machined/pkg/controllers/k8s/static_pod_server.go new file mode 100644 index 0000000..cc4c610 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/static_pod_server.go @@ -0,0 +1,210 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import ( + "context" + "fmt" + "net" + "net/http" + "sync" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "go.uber.org/zap" + "gopkg.in/yaml.v3" + + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +// StaticPodServerController renders all static pod definitions as a PodList and serves it as YAML via HTTP. +type StaticPodServerController struct { + podList []byte + podListMu sync.Mutex + + staticPodVersions map[string]string +} + +// Name implements controller.Controller interface. +func (ctrl *StaticPodServerController) Name() string { + return "k8s.StaticPodServerController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *StaticPodServerController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: k8s.NamespaceName, + Type: k8s.StaticPodType, + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *StaticPodServerController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: k8s.StaticPodServerStatusType, + Kind: controller.OutputExclusive, + }, + } +} + +type pod map[string]any + +type podList struct { + Kind string `json:"kind,omitempty" protobuf:"bytes,1,opt,name=kind"` + + Items []pod `json:"items" protobuf:"bytes,2,rep,name=items"` + + APIVersion string `json:"apiVersion,omitempty" protobuf:"bytes,3,opt,name=apiVersion"` +} + +// Run implements controller.Controller interface. +func (ctrl *StaticPodServerController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + ctrl.staticPodVersions = map[string]string{} + + shutdownServer, serverError, err := ctrl.createServer(ctx, r, logger) + if err != nil { + return fmt.Errorf("failed to start http server to serve static pod list: %w", err) + } + + defer shutdownServer() + + for { + select { + case <-ctx.Done(): + return nil + case err := <-serverError: + return fmt.Errorf("http server closed unexpectedly: %w", err) + case <-r.EventCh(): + staticPodList, err := ctrl.buildPodList(ctx, r, logger) + if err != nil { + logger.Error("error building static pod list", zap.Error(err)) + } + + ctrl.podListMu.Lock() + ctrl.podList = staticPodList + ctrl.podListMu.Unlock() + } + + r.ResetRestartBackoff() + } +} + +func (ctrl *StaticPodServerController) buildPodList(ctx context.Context, r controller.Runtime, logger *zap.Logger) ([]byte, error) { + staticPods, err := safe.ReaderListAll[*k8s.StaticPod](ctx, r) + if err != nil { + return nil, fmt.Errorf("error listing static pods: %w", err) + } + + pl := podList{ + Kind: "PodList", + APIVersion: "v1", + } + + touchedPodIDs := map[string]struct{}{} + + for iter := staticPods.Iterator(); iter.Next(); { + id := iter.Value().Metadata().ID() + version := iter.Value().Metadata().Version().String() + + if oldVersion, exists := ctrl.staticPodVersions[id]; !exists || oldVersion != version { + ctrl.staticPodVersions[id] = version + + if !exists { + logger.Info("rendered new static pod", zap.String("id", id)) + } else { + logger.Info("rendered updated static pod", zap.String("id", id), zap.String("old_version", oldVersion), zap.String("new_version", version)) + } + } + + staticPodSpec := iter.Value().TypedSpec() + + pl.Items = append(pl.Items, staticPodSpec.Pod) + + touchedPodIDs[id] = struct{}{} + } + + for id := range ctrl.staticPodVersions { + if _, exists := touchedPodIDs[id]; exists { + continue + } + + logger.Info("removed static pod", zap.String("id", id)) + + delete(ctrl.staticPodVersions, id) + } + + manifestContent, err := yaml.Marshal(pl) + if err != nil { + return nil, fmt.Errorf("error rendering list of static pods as yaml: %w", err) + } + + return manifestContent, nil +} + +func (ctrl *StaticPodServerController) createServer(ctx context.Context, r controller.Runtime, logger *zap.Logger) (func(), <-chan error, error) { + mux := http.NewServeMux() + + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + ctrl.podListMu.Lock() + staticPodList := ctrl.podList + ctrl.podListMu.Unlock() + + logger.Debug("serving static pod manifests", zap.Int("size", len(staticPodList))) + + if staticPodList == nil { + http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + + return + } + + _, err := w.Write(staticPodList) + if err != nil { + logger.Error("failed to serve static pod manifests", zap.Error(err)) + } + }) + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return nil, nil, fmt.Errorf("failed to create listener for serving static pod manifests: %w", err) + } + + httpServer := &http.Server{ + Handler: mux, + } + + shutdownServer := func() { + if err := httpServer.Shutdown(ctx); err != nil { + logger.Error("failed to shut down HTTP server, serving static pod manifests", zap.Error(err)) + } + } + + go func() { + <-ctx.Done() + + shutdownServer() + }() + + if err := safe.WriterModify(ctx, r, k8s.NewStaticPodServerStatus(k8s.NamespaceName, k8s.StaticPodServerStatusResourceID), func(r *k8s.StaticPodServerStatus) error { + url := fmt.Sprintf("http://%s", listener.Addr().String()) + + r.TypedSpec().URL = url + + return nil + }); err != nil { + return nil, nil, fmt.Errorf("error modifying StaticPodListURL resource: %w", err) + } + + serverError := make(chan error, 1) + + go func() { + serverError <- httpServer.Serve(listener) + }() + + return shutdownServer, serverError, nil +} diff --git a/internal/app/machined/pkg/controllers/k8s/static_pod_server_test.go b/internal/app/machined/pkg/controllers/k8s/static_pod_server_test.go new file mode 100644 index 0000000..99cda10 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/static_pod_server_test.go @@ -0,0 +1,192 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:dupl +package k8s_test + +import ( + "context" + "io" + "log" + "net/http" + "strings" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + k8sctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/k8s" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +type StaticPodListSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *StaticPodListSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.Require().NoError(suite.runtime.RegisterController(&k8sctrl.StaticPodServerController{})) + + suite.startRuntime() +} + +func (suite *StaticPodListSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *StaticPodListSuite) assertResource( + md resource.Metadata, + check func(res resource.Resource) error, +) func() error { + return func() error { + r, err := suite.state.Get(suite.ctx, md) + if err != nil { + if state.IsNotFoundError(err) { + return retry.ExpectedError(err) + } + + return err + } + + return check(r) + } +} + +func (suite *StaticPodListSuite) getResource( + md resource.Metadata, +) resource.Resource { + var ret resource.Resource + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(func() error { + r, err := suite.state.Get(suite.ctx, md) + if err != nil { + if state.IsNotFoundError(err) { + return retry.ExpectedError(err) + } + + return err + } + + ret = r + + return nil + })) + + return ret +} + +func newTestPod(name string) *k8s.StaticPod { + testPod := k8s.NewStaticPod(k8s.NamespaceName, name) + + testPod.TypedSpec().Pod = map[string]any{ + "metadata": name, + "spec": "testSpec", + } + + return testPod +} + +func (suite *StaticPodListSuite) TestCreatesStaticPodServerStatus() { + // given + testPod := newTestPod("testPod") + + // when + suite.Require().NoError(suite.state.Create(suite.ctx, testPod)) + + // then + expectedPodListURL := k8s.NewStaticPodServerStatus(k8s.NamespaceName, k8s.StaticPodServerStatusResourceID) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertResource(*expectedPodListURL.Metadata(), func(res resource.Resource) error { + suite.Require().True(strings.HasPrefix( + res.(*k8s.StaticPodServerStatus).TypedSpec().URL, + "http://127.0.0.1:", + ), + ) + + return nil + }, + ), + ), + ) +} + +func (suite *StaticPodListSuite) TestServesStaticPodList() { + // given + testPod1 := newTestPod("testPod1") + testPod2 := newTestPod("testPod2") + + // when + suite.Require().NoError(suite.state.Create(suite.ctx, testPod1)) + suite.Require().NoError(suite.state.Create(suite.ctx, testPod2)) + + // then + expectedPodListURL := k8s.NewStaticPodServerStatus(k8s.NamespaceName, k8s.StaticPodServerStatusResourceID) + + podListURL := suite.getResource(*expectedPodListURL.Metadata()) + + suite.Require().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(func() error { + resp, err := http.Get(podListURL.(*k8s.StaticPodServerStatus).TypedSpec().URL) //nolint:noctx + if err != nil { + return retry.ExpectedError(err) + } + + defer resp.Body.Close() //nolint:errcheck + + content, err := io.ReadAll(resp.Body) + suite.Assert().NoError(err) + + suite.Require().Equal("kind: PodList\nitems:\n - metadata: testPod1\n spec: testSpec\n - metadata: testPod2\n spec: testSpec\napiversion: v1\n", string(content)) + + return nil + }), + ) +} + +func (suite *StaticPodListSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestStaticPodListSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, new(StaticPodListSuite)) +} diff --git a/internal/app/machined/pkg/controllers/k8s/templates.go b/internal/app/machined/pkg/controllers/k8s/templates.go new file mode 100644 index 0000000..27158d8 --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/templates.go @@ -0,0 +1,608 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import "github.com/siderolabs/talos/pkg/flannel" + +// kube-apiserver configuration: + +var kubeSystemEncryptionConfigTemplate = []byte(`apiVersion: v1 +kind: EncryptionConfig +resources: +- resources: + - secrets + providers: + {{if .Root.SecretboxEncryptionSecret}} + - secretbox: + keys: + - name: key2 + secret: {{ .Root.SecretboxEncryptionSecret }} + {{end}} + {{if .Root.AESCBCEncryptionSecret}} + - aescbc: + keys: + - name: key1 + secret: {{ .Root.AESCBCEncryptionSecret }} + {{end}} + - identity: {} +`) + +// manifests injected into kube-apiserver + +var kubeletBootstrappingToken = []byte(`apiVersion: v1 +kind: Secret +metadata: + name: bootstrap-token-{{ .Secrets.BootstrapTokenID }} + namespace: kube-system +type: bootstrap.kubernetes.io/token +stringData: + token-id: "{{ .Secrets.BootstrapTokenID }}" + token-secret: "{{ .Secrets.BootstrapTokenSecret }}" + usage-bootstrap-authentication: "true" + + # Extra groups to authenticate the token as. Must start with "system:bootstrappers:" + auth-extra-groups: system:bootstrappers:nodes +`) + +// csrNodeBootstrapTemplate lets bootstrapping tokens and nodes request CSRs. +var csrNodeBootstrapTemplate = []byte(`apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system-bootstrap-node-bootstrapper +subjects: +- kind: Group + name: system:bootstrappers:nodes + apiGroup: rbac.authorization.k8s.io +- kind: Group + name: system:nodes + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: ClusterRole + name: system:node-bootstrapper + apiGroup: rbac.authorization.k8s.io +`) + +// csrApproverRoleBindingTemplate instructs the csrapprover controller to +// automatically approve CSRs made by bootstrapping tokens for client +// credentials. +// +// This binding should be removed to disable CSR auto-approval. +var csrApproverRoleBindingTemplate = []byte(`apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system-bootstrap-approve-node-client-csr +subjects: +- kind: Group + name: system:bootstrappers:nodes + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: ClusterRole + name: system:certificates.k8s.io:certificatesigningrequests:nodeclient + apiGroup: rbac.authorization.k8s.io +`) + +// csrRenewalRoleBindingTemplate instructs the csrapprover controller to +// automatically approve all CSRs made by nodes to renew their client +// certificates. +// +// This binding should be altered in the future to hold a list of node +// names instead of targeting `system:nodes` so we can revoke individual +// node's ability to renew its certs. +var csrRenewalRoleBindingTemplate = []byte(`apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system-bootstrap-node-renewal +subjects: +- kind: Group + name: system:nodes + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: ClusterRole + name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient + apiGroup: rbac.authorization.k8s.io +`) + +var kubeProxyTemplate = []byte(`apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kube-proxy + namespace: kube-system + labels: + tier: node + k8s-app: kube-proxy +spec: + selector: + matchLabels: + tier: node + k8s-app: kube-proxy + template: + metadata: + labels: + tier: node + k8s-app: kube-proxy + spec: + containers: + - name: kube-proxy + image: {{ .ProxyImage }} + command: + - /usr/local/bin/kube-proxy + {{- range $arg := .ProxyArgs }} + - {{ $arg | json }} + {{- end }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + securityContext: + privileged: true + volumeMounts: + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /etc/ssl/certs + name: ssl-certs-host + readOnly: true + - name: kubeconfig + mountPath: /etc/kubernetes + readOnly: true + hostNetwork: true + priorityClassName: system-cluster-critical + serviceAccountName: kube-proxy + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + volumes: + - name: lib-modules + hostPath: + path: /lib/modules + - name: ssl-certs-host + hostPath: + path: /etc/ssl/certs + - name: kubeconfig + configMap: + name: kubeconfig-in-cluster + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + namespace: kube-system + name: kube-proxy +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kube-proxy +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:node-proxier # Automatically created system role. +subjects: +- kind: ServiceAccount + name: kube-proxy + namespace: kube-system +`) + +// kubeConfigInCluster instructs clients to use their service account token, +// but unlike an in-cluster client doesn't rely on the `KUBERNETES_SERVICE_PORT` +// and `KUBERNETES_PORT` to determine the API servers address. +// +// This kubeconfig is used by bootstrapping pods that might not have access to +// these env vars, such as kube-proxy, which sets up the API server endpoint +// (chicken and egg), and the checkpointer, which needs to run as a static pod +// even if the API server isn't available. +var kubeConfigInClusterTemplate = []byte(`apiVersion: v1 +kind: ConfigMap +metadata: + name: kubeconfig-in-cluster + namespace: kube-system +data: + kubeconfig: | + apiVersion: v1 + clusters: + - name: local + cluster: + server: {{ .Server }} + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + users: + - name: service-account + user: + # Use service account token + tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + contexts: + - context: + cluster: local + user: service-account +`) + +var coreDNSTemplate = []byte(`apiVersion: v1 +kind: ServiceAccount +metadata: + name: coredns + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:coredns + labels: + kubernetes.io/bootstrapping: rbac-defaults + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:coredns +subjects: + - kind: ServiceAccount + name: coredns + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:coredns + labels: + kubernetes.io/bootstrapping: rbac-defaults +rules: + - apiGroups: [""] + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch + - apiGroups: ["discovery.k8s.io"] + resources: + - endpointslices + verbs: + - list + - watch +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns + namespace: kube-system +data: + Corefile: | + .:53 { + errors + health { + lameduck 5s + } + ready + log . { + class error + } + prometheus :9153 + + kubernetes {{ .ClusterDomain }} in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + } + forward . /etc/resolv.conf + cache 30 + loop + reload + loadbalance + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coredns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/name: "CoreDNS" +spec: + replicas: 2 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: kube-dns + template: + metadata: + labels: + k8s-app: kube-dns + spec: + nodeSelector: + kubernetes.io/os: linux + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: k8s-app + operator: In + values: + - kube-dns + topologyKey: kubernetes.io/hostname + serviceAccountName: coredns + priorityClassName: system-cluster-critical + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: node.cloudprovider.kubernetes.io/uninitialized + operator: Exists + effect: NoSchedule + containers: + - name: coredns + image: {{ .CoreDNSImage }} + imagePullPolicy: IfNotPresent + resources: + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + env: + - name: GOMEMLIMIT + value: "161MiB" + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + readOnly: true + ports: + - name: dns + protocol: UDP + containerPort: 53 + - name: dns-tcp + protocol: TCP + containerPort: 53 + - name: metrics + protocol: TCP + containerPort: 9153 + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /ready + port: 8181 + scheme: HTTP + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + readOnlyRootFilesystem: true + dnsPolicy: Default + volumes: + - name: config-volume + configMap: + name: coredns + items: + - key: Corefile + path: Corefile +`) + +var coreDNSSvcTemplate = []byte(`apiVersion: v1 +kind: Service +metadata: + name: kube-dns + namespace: kube-system + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9153" + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" +spec: + selector: + k8s-app: kube-dns + clusterIP: {{ or .DNSServiceIP .DNSServiceIPv6 }} + clusterIPs: + {{- if .DNSServiceIP }} + - {{ .DNSServiceIP }} + {{- end }} + {{- if .DNSServiceIPv6 }} + - {{ .DNSServiceIPv6 }} + {{- end }} + ipFamilies: + {{- if .DNSServiceIP }} + - IPv4 + {{- end }} + {{- if .DNSServiceIPv6 }} + - IPv6 + {{- end }} + {{- if and .DNSServiceIP .DNSServiceIPv6 }} + ipFamilyPolicy: RequireDualStack + {{- else }} + ipFamilyPolicy: SingleStack + {{- end }} + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP + - name: metrics + port: 9153 + protocol: TCP +`) + +// podSecurityPolicy is the default PSP. +var podSecurityPolicy = []byte(`kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: psp:privileged +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - privileged +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: psp:privileged +roleRef: + kind: ClusterRole + name: psp:privileged + apiGroup: rbac.authorization.k8s.io +subjects: +# Authorize all service accounts in a namespace: +- kind: Group + apiGroup: rbac.authorization.k8s.io + name: system:serviceaccounts +# Authorize all authenticated users in a namespace: +- kind: Group + apiGroup: rbac.authorization.k8s.io + name: system:authenticated +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: privileged + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' +spec: + fsGroup: + rule: RunAsAny + privileged: true + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - '*' + allowedCapabilities: + - '*' + hostPID: true + hostIPC: true + hostNetwork: true + hostPorts: + - min: 1 + max: 65536 +`) + +// talosAPIService is the service to access Talos API from Kubernetes. +// Service exposes the Endpoints which are managed by controllers. +var talosAPIService = []byte(`apiVersion: v1 +kind: Service +metadata: + labels: + component: apid + provider: talos + name: {{ .KubernetesTalosAPIServiceName }} + namespace: {{ .KubernetesTalosAPIServiceNamespace }} +spec: + ports: + - name: apid + port: {{ .ApidPort }} + protocol: TCP + targetPort: {{ .ApidPort }} +`) + +var flannelTemplate = flannel.Template + +// talosServiceAccountCRDTemplate is the template of the CRD which +// allows injecting Talos with credentials into the Kubernetes cluster. +var talosServiceAccountCRDTemplate = []byte(`apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: {{ .TalosServiceAccount.ResourcePlural }}.{{ .TalosServiceAccount.Group }} +spec: + conversion: + strategy: None + group: {{ .TalosServiceAccount.Group }} + names: + kind: {{ .TalosServiceAccount.Kind }} + listKind: {{ .TalosServiceAccount.Kind }}List + plural: {{ .TalosServiceAccount.ResourcePlural }} + singular: {{ .TalosServiceAccount.ResourceSingular }} + shortNames: + - {{ .TalosServiceAccount.ShortName }} + scope: Namespaced + versions: + - name: {{ .TalosServiceAccount.Version }} + schema: + openAPIV3Schema: + properties: + spec: + type: object + properties: + roles: + type: array + items: + type: string + status: + type: object + properties: + failureReason: + type: string + type: object + served: true + storage: true +`) + +var talosHostDNSSvcTemplate = []byte(`apiVersion: v1 +kind: Service +metadata: + name: host-dns + namespace: kube-system +spec: + clusterIP: {{ .ServiceHostDNSAddress }} + ports: + - name: dns + port: 53 + protocol: UDP + targetPort: 53 + - name: dns-tcp + port: 53 + protocol: TCP + targetPort: 53 + type: ClusterIP +--- +apiVersion: v1 +kind: Endpoints +metadata: + name: host-dns + namespace: kube-system +subsets: + - addresses: + - ip: {{ .ServiceHostDNSAddress }} + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP +`) diff --git a/internal/app/machined/pkg/controllers/kubeaccess/config.go b/internal/app/machined/pkg/controllers/kubeaccess/config.go new file mode 100644 index 0000000..26f98c0 --- /dev/null +++ b/internal/app/machined/pkg/controllers/kubeaccess/config.go @@ -0,0 +1,59 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package kubeaccess + +import ( + "context" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/controller/generic/transform" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/kubeaccess" +) + +// ConfigController watches v1alpha1.Config, updates Talos API access config. +type ConfigController = transform.Controller[*config.MachineConfig, *kubeaccess.Config] + +// NewConfigController instanciates the config controller. +func NewConfigController() *ConfigController { + return transform.NewController( + transform.Settings[*config.MachineConfig, *kubeaccess.Config]{ + Name: "kubeaccess.ConfigController", + MapMetadataOptionalFunc: func(cfg *config.MachineConfig) optional.Optional[*kubeaccess.Config] { + if cfg.Metadata().ID() != config.V1Alpha1ID { + return optional.None[*kubeaccess.Config]() + } + + if cfg.Config().Machine() == nil { + return optional.None[*kubeaccess.Config]() + } + + if !cfg.Config().Machine().Type().IsControlPlane() { + return optional.None[*kubeaccess.Config]() + } + + return optional.Some(kubeaccess.NewConfig(config.NamespaceName, kubeaccess.ConfigID)) + }, + TransformFunc: func(ctx context.Context, r controller.Reader, logger *zap.Logger, cfg *config.MachineConfig, res *kubeaccess.Config) error { + spec := res.TypedSpec() + + *spec = kubeaccess.ConfigSpec{} + + if cfg != nil && cfg.Config().Machine() != nil { + c := cfg.Config() + + spec.Enabled = c.Machine().Features().KubernetesTalosAPIAccess().Enabled() + spec.AllowedAPIRoles = c.Machine().Features().KubernetesTalosAPIAccess().AllowedRoles() + spec.AllowedKubernetesNamespaces = c.Machine().Features().KubernetesTalosAPIAccess().AllowedKubernetesNamespaces() + } + + return nil + }, + }, + ) +} diff --git a/internal/app/machined/pkg/controllers/kubeaccess/config_test.go b/internal/app/machined/pkg/controllers/kubeaccess/config_test.go new file mode 100644 index 0000000..d60e525 --- /dev/null +++ b/internal/app/machined/pkg/controllers/kubeaccess/config_test.go @@ -0,0 +1,106 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package kubeaccess_test + +import ( + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/siderolabs/go-pointer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + kubeaccessctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/kubeaccess" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/kubeaccess" +) + +type ConfigSuite struct { + ctest.DefaultSuite +} + +func (suite *ConfigSuite) TestReconcileConfig() { + cfg := config.NewMachineConfig(container.NewV1Alpha1(&v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineType: "controlplane", + MachineFeatures: &v1alpha1.FeaturesConfig{ + KubernetesTalosAPIAccessConfig: &v1alpha1.KubernetesTalosAPIAccessConfig{ + AccessEnabled: pointer.To(true), + AccessAllowedRoles: []string{"os:admin"}, + AccessAllowedKubernetesNamespaces: []string{"kube-system"}, + }, + }, + }, + })) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{kubeaccess.ConfigID}, func(r *kubeaccess.Config, asrt *assert.Assertions) { + spec := r.TypedSpec() + + asrt.True(spec.Enabled) + asrt.Equal([]string{"os:admin"}, spec.AllowedAPIRoles) + asrt.Equal([]string{"kube-system"}, spec.AllowedKubernetesNamespaces) + }) +} + +func (suite *ConfigSuite) TestReconcileDisabled() { + cfg := config.NewMachineConfig(container.NewV1Alpha1(&v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineType: "init", + }, + })) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{kubeaccess.ConfigID}, func(r *kubeaccess.Config, asrt *assert.Assertions) { + spec := r.TypedSpec() + + asrt.False(spec.Enabled) + asrt.Empty(spec.AllowedAPIRoles) + asrt.Empty(spec.AllowedKubernetesNamespaces) + }) +} + +func (suite *ConfigSuite) TestReconcileWorker() { + cfg := config.NewMachineConfig(container.NewV1Alpha1(&v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineType: "worker", + MachineFeatures: &v1alpha1.FeaturesConfig{ + KubernetesTalosAPIAccessConfig: &v1alpha1.KubernetesTalosAPIAccessConfig{ + AccessEnabled: pointer.To(true), + AccessAllowedRoles: []string{"os:admin"}, + AccessAllowedKubernetesNamespaces: []string{"kube-system"}, + }, + }, + }, + })) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + // worker should have feature disabled even if it is enabled in the config + rtestutils.AssertNoResource[*kubeaccess.Config](suite.Ctx(), suite.T(), suite.State(), kubeaccess.ConfigID) +} + +func TestConfigSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, &ConfigSuite{ + DefaultSuite: ctest.DefaultSuite{ + Timeout: 5 * time.Second, + AfterSetup: func(s *ctest.DefaultSuite) { + s.Require().NoError(s.Runtime().RegisterController(kubeaccessctrl.NewConfigController())) + }, + }, + }) +} diff --git a/internal/app/machined/pkg/controllers/kubeaccess/endpoint.go b/internal/app/machined/pkg/controllers/kubeaccess/endpoint.go new file mode 100644 index 0000000..0bc011e --- /dev/null +++ b/internal/app/machined/pkg/controllers/kubeaccess/endpoint.go @@ -0,0 +1,210 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package kubeaccess + +import ( + "context" + "fmt" + "reflect" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + + "github.com/siderolabs/talos/pkg/kubernetes" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/kubeaccess" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" +) + +// EndpointController manages Kubernetes endpoints resource for Talos API endpoints. +type EndpointController struct{} + +// Name implements controller.Controller interface. +func (ctrl *EndpointController) Name() string { + return "kubeaccess.EndpointController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *EndpointController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: kubeaccess.ConfigType, + ID: optional.Some(kubeaccess.ConfigID), + Kind: controller.InputWeak, + }, + { + Namespace: secrets.NamespaceName, + Type: secrets.KubernetesType, + ID: optional.Some(secrets.KubernetesID), + Kind: controller.InputWeak, + }, + { + Namespace: k8s.ControlPlaneNamespaceName, + Type: k8s.EndpointType, + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *EndpointController) Outputs() []controller.Output { + return nil +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *EndpointController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-r.EventCh(): + case <-ctx.Done(): + return nil + } + + kubeaccessConfig, err := safe.ReaderGet[*kubeaccess.Config](ctx, r, kubeaccess.NewConfig(config.NamespaceName, kubeaccess.ConfigID).Metadata()) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error fetching kubeaccess config: %w", err) + } + } + + if kubeaccessConfig == nil || !kubeaccessConfig.TypedSpec().Enabled { + // disabled, do not do anything + continue + } + + // use only api-server endpoints to leave only kubelet node IPs + endpointResource, err := safe.ReaderGet[*k8s.Endpoint](ctx, r, resource.NewMetadata(k8s.ControlPlaneNamespaceName, k8s.EndpointType, k8s.ControlPlaneAPIServerEndpointsID, resource.VersionUndefined)) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting endpoints resources: %w", err) + } + } + + var endpointAddrs k8s.EndpointList + + if endpointResource != nil { + endpointAddrs = endpointAddrs.Merge(endpointResource) + } + + if len(endpointAddrs) == 0 { + continue + } + + secretsResources, err := safe.ReaderGet[*secrets.Kubernetes](ctx, r, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesType, secrets.KubernetesID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + secrets := secretsResources.TypedSpec() + + kubeconfig, err := clientcmd.BuildConfigFromKubeconfigGetter("", func() (*clientcmdapi.Config, error) { + return clientcmd.Load([]byte(secrets.LocalhostAdminKubeconfig)) + }) + if err != nil { + return fmt.Errorf("error loading kubeconfig: %w", err) + } + + if err = ctrl.updateTalosEndpoints(ctx, logger, kubeconfig, endpointAddrs); err != nil { + return err + } + + r.ResetRestartBackoff() + } +} + +//nolint:gocyclo +func (ctrl *EndpointController) updateTalosEndpoints(ctx context.Context, logger *zap.Logger, kubeconfig *rest.Config, endpointAddrs k8s.EndpointList) error { + client, err := kubernetes.NewForConfig(kubeconfig) + if err != nil { + return fmt.Errorf("error building Kubernetes client: %w", err) + } + + defer client.Close() //nolint:errcheck + + for { + oldEndpoints, err := client.CoreV1().Endpoints(constants.KubernetesTalosAPIServiceNamespace).Get(ctx, constants.KubernetesTalosAPIServiceName, metav1.GetOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("error getting endpoints: %w", err) + } + + var newEndpoints *corev1.Endpoints + + if apierrors.IsNotFound(err) { + newEndpoints = &corev1.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.KubernetesTalosAPIServiceName, + Namespace: constants.KubernetesTalosAPIServiceNamespace, + Labels: map[string]string{ + "provider": constants.KubernetesTalosProvider, + "component": "apid", + }, + }, + } + } else { + newEndpoints = oldEndpoints.DeepCopy() + } + + newEndpoints.Subsets = []corev1.EndpointSubset{ + { + Ports: []corev1.EndpointPort{ + { + Name: "apid", + Port: constants.ApidPort, + Protocol: "TCP", + }, + }, + }, + } + + for _, addr := range endpointAddrs { + newEndpoints.Subsets[0].Addresses = append(newEndpoints.Subsets[0].Addresses, + corev1.EndpointAddress{ + IP: addr.String(), + }, + ) + } + + if oldEndpoints != nil && reflect.DeepEqual(oldEndpoints.Subsets, newEndpoints.Subsets) { + // no change, bail out + return nil + } + + if oldEndpoints == nil { + _, err = client.CoreV1().Endpoints(constants.KubernetesTalosAPIServiceNamespace).Create(ctx, newEndpoints, metav1.CreateOptions{}) + } else { + _, err = client.CoreV1().Endpoints(constants.KubernetesTalosAPIServiceNamespace).Update(ctx, newEndpoints, metav1.UpdateOptions{}) + } + + switch { + case err == nil: + logger.Info("updated Talos API endpoints in Kubernetes", zap.Strings("endpoints", endpointAddrs.Strings())) + + return nil + case apierrors.IsConflict(err) || apierrors.IsAlreadyExists(err): + // retry + default: + return fmt.Errorf("error updating Kubernetes Talos API endpoints: %w", err) + } + } +} diff --git a/internal/app/machined/pkg/controllers/kubeaccess/kubeaccess.go b/internal/app/machined/pkg/controllers/kubeaccess/kubeaccess.go new file mode 100644 index 0000000..2e711bb --- /dev/null +++ b/internal/app/machined/pkg/controllers/kubeaccess/kubeaccess.go @@ -0,0 +1,6 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package kubeaccess provides controllers which manage Talos API access from Kubernetes workloads. +package kubeaccess diff --git a/internal/app/machined/pkg/controllers/kubeaccess/serviceaccount.go b/internal/app/machined/pkg/controllers/kubeaccess/serviceaccount.go new file mode 100644 index 0000000..267d46c --- /dev/null +++ b/internal/app/machined/pkg/controllers/kubeaccess/serviceaccount.go @@ -0,0 +1,206 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package kubeaccess + +import ( + "context" + "errors" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/crypto/x509" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/kubeaccess/serviceaccount" + "github.com/aenix-io/talm/internal/pkg/etcd" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/kubeaccess" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" +) + +// CRDController manages Kubernetes endpoints resource for Talos API endpoints. +type CRDController struct{} + +// Name implements controller.Controller interface. +func (ctrl *CRDController) Name() string { + return "kubeaccess.CRDController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *CRDController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: kubeaccess.ConfigType, + ID: optional.Some(kubeaccess.ConfigID), + Kind: controller.InputWeak, + }, + { + Namespace: secrets.NamespaceName, + Type: secrets.KubernetesType, + ID: optional.Some(secrets.KubernetesID), + Kind: controller.InputWeak, + }, + { + Namespace: secrets.NamespaceName, + Type: secrets.OSRootType, + ID: optional.Some(secrets.OSRootID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *CRDController) Outputs() []controller.Output { + return nil +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *CRDController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + var crdControllerCtxCancel context.CancelFunc + + crdControllerErrCh := make(chan error, 1) + + stopCRDController := func() { + if crdControllerCtxCancel != nil { + crdControllerCtxCancel() + + <-crdControllerErrCh + + crdControllerCtxCancel = nil + } + } + + defer stopCRDController() + + for { + select { + case <-ctx.Done(): + return nil //nolint:govet + case <-r.EventCh(): + case err := <-crdControllerErrCh: + if crdControllerCtxCancel != nil { + crdControllerCtxCancel() + } + + crdControllerCtxCancel = nil + + if err != nil && !errors.Is(err, context.Canceled) { + return fmt.Errorf("error from crd controller: %w", err) + } + } + + kubeaccessConfig, err := safe.ReaderGet[*kubeaccess.Config](ctx, r, kubeaccess.NewConfig(config.NamespaceName, kubeaccess.ConfigID).Metadata()) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error fetching kubeaccess config: %w", err) + } + + continue + } + + var kubeaccessConfigSpec *kubeaccess.ConfigSpec + + if kubeaccessConfig != nil { + kubeaccessConfigSpec = kubeaccessConfig.TypedSpec() + } + + if kubeaccessConfig == nil || kubeaccessConfigSpec == nil || !kubeaccessConfigSpec.Enabled { + stopCRDController() + + continue + } + + kubeSecretsResources, err := safe.ReaderGet[*secrets.Kubernetes](ctx, r, resource.NewMetadata( + secrets.NamespaceName, + secrets.KubernetesType, + secrets.KubernetesID, + resource.VersionUndefined, + )) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error fetching kubernetes secrets: %w", err) + } + + continue + } + + kubeSecretsSpec := kubeSecretsResources.TypedSpec() + + osSecretsResource, err := safe.ReaderGet[*secrets.OSRoot](ctx, r, resource.NewMetadata( + secrets.NamespaceName, + secrets.OSRootType, + secrets.OSRootID, + resource.VersionUndefined, + )) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error fetching os secrets: %w", err) + } + + continue + } + + osSecretsSpec := osSecretsResource.TypedSpec() + + kubeconfig, err := clientcmd.BuildConfigFromKubeconfigGetter("", func() (*clientcmdapi.Config, error) { + return clientcmd.Load([]byte(kubeSecretsSpec.LocalhostAdminKubeconfig)) + }) + if err != nil { + return fmt.Errorf("error loading kubeconfig: %w", err) + } + + stopCRDController() + + var crdControllerCtx context.Context + + crdControllerCtx, crdControllerCtxCancel = context.WithCancel(ctx) //nolint:govet + + go func() { + crdControllerErrCh <- ctrl.runCRDController( + crdControllerCtx, + osSecretsSpec.IssuingCA, + kubeconfig, + kubeaccessConfigSpec, + logger, + ) + }() + + r.ResetRestartBackoff() + } +} + +func (ctrl *CRDController) runCRDController( + ctx context.Context, + talosCA *x509.PEMEncodedCertificateAndKey, + kubeconfig *rest.Config, + kubeaccessCfgSpec *kubeaccess.ConfigSpec, + logger *zap.Logger, +) error { + return etcd.WithLock(ctx, constants.EtcdTalosServiceAccountCRDControllerMutex, logger, func() error { + crdCtrl, err := serviceaccount.NewCRDController( + talosCA, + kubeconfig, + kubeaccessCfgSpec.AllowedKubernetesNamespaces, + kubeaccessCfgSpec.AllowedAPIRoles, + logger, + ) + if err != nil { + return err + } + + return crdCtrl.Run(ctx, 1) + }) +} diff --git a/internal/app/machined/pkg/controllers/kubeaccess/serviceaccount/crd_controller.go b/internal/app/machined/pkg/controllers/kubeaccess/serviceaccount/crd_controller.go new file mode 100644 index 0000000..0ecd5a4 --- /dev/null +++ b/internal/app/machined/pkg/controllers/kubeaccess/serviceaccount/crd_controller.go @@ -0,0 +1,651 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package serviceaccount + +import ( + "bytes" + "context" + stdlibx509 "crypto/x509" + "encoding/base64" + "encoding/pem" + "errors" + "fmt" + "reflect" + "slices" + "sort" + "sync" + "time" + + "github.com/siderolabs/crypto/x509" + "github.com/siderolabs/gen/xslices" + taloskubernetes "github.com/siderolabs/go-kubernetes/kubernetes" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + kubeerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/dynamic/dynamicinformer" + "k8s.io/client-go/dynamic/dynamiclister" + kubeinformers "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" + corelisters "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/connrotation" + "k8s.io/client-go/util/workqueue" + + clientconfig "github.com/siderolabs/talos/pkg/machinery/client/config" + "github.com/siderolabs/talos/pkg/machinery/config/generate/secrets" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/role" +) + +const ( + certTTL = time.Hour * 6 + certRenewThreshold = time.Hour * 1 + + successResourceSynced = "Synced" + messageResourceSynced = "Synced successfully" + + errResourceExists = "ErrResourceExists" + messageResourceExists = "%s already exists and is not managed by controller: %s" + + errRolesNotFound = "ErrRolesNotFound" + messageRolesNotFound = "Roles not found" + + errNamespaceNotAllowed = "ErrNamespaceNotAllowed" + messageNamespaceNotAllowed = "Namespace is not allowed: %s" + + errRolesNotAllowed = "ErrRolesNotAllowed" + messageRolesNotAllowed = "Roles not allowed: %v" + + controllerAgentName = "talos-sa-controller" + informerResyncPeriod = time.Minute * 1 + + talosconfigContextName = "default" + endpoint = constants.KubernetesTalosAPIServiceName + "." + constants.KubernetesTalosAPIServiceNamespace + + kindSecret = "Secret" +) + +var ( + talosSAGV = schema.GroupVersion{ + Group: constants.ServiceAccountResourceGroup, + Version: constants.ServiceAccountResourceVersion, + } + + talosSAGVR = talosSAGV.WithResource(constants.ServiceAccountResourcePlural) + talosSAGVK = talosSAGV.WithKind(constants.ServiceAccountResourceKind) +) + +// CRDController is the controller implementation for TalosServiceAccount resources. +type CRDController struct { + talosCA *x509.PEMEncodedCertificateAndKey + + allowedNamespaces []string + allowedRoles map[string]struct{} + + queue workqueue.RateLimitingInterface + + kubeInformerFactory kubeinformers.SharedInformerFactory + dynamicInformerFactory dynamicinformer.DynamicSharedInformerFactory + + kubeClient kubernetes.Interface + dynamicClient dynamic.Interface + dialer *connrotation.Dialer + + secretsSynced cache.InformerSynced + talosSAsSynced cache.InformerSynced + + secretsLister corelisters.SecretLister + dynamicLister dynamiclister.Lister + + eventRecorder record.EventRecorder + + logger *zap.Logger +} + +// NewCRDController creates a new CRD controller. +func NewCRDController( + talosCA *x509.PEMEncodedCertificateAndKey, + kubeconfig *rest.Config, + allowedNamespaces []string, + allowedRoles []string, + logger *zap.Logger, +) (*CRDController, error) { + dialer := taloskubernetes.NewDialer() + kubeconfig.Dial = dialer.DialContext + + kubeCli, err := kubernetes.NewForConfig(kubeconfig) + if err != nil { + return nil, err + } + + dynCli, err := dynamic.NewForConfig(kubeconfig) + if err != nil { + return nil, err + } + + dynamicInformerFactory := dynamicinformer.NewDynamicSharedInformerFactory(dynCli, informerResyncPeriod) + resourceInformer := dynamicInformerFactory.ForResource(talosSAGVR) + informer := resourceInformer.Informer() + + indexer := informer.GetIndexer() + lister := dynamiclister.New(indexer, talosSAGVR) + + kubeInformerFactory := kubeinformers.NewSharedInformerFactory(kubeCli, informerResyncPeriod) + secrets := kubeInformerFactory.Core().V1().Secrets() + + logger.Debug("creating event broadcaster") + + eventBroadcaster := record.NewBroadcaster() + eventBroadcaster.StartStructuredLogging(0) + eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: kubeCli.CoreV1().Events("")}) + + recorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: controllerAgentName}) + + controller := CRDController{ + talosCA: talosCA, + allowedNamespaces: allowedNamespaces, + allowedRoles: xslices.ToSet(allowedRoles), + dynamicInformerFactory: dynamicInformerFactory, + kubeInformerFactory: kubeInformerFactory, + kubeClient: kubeCli, + dynamicClient: dynCli, + dialer: dialer, + dynamicLister: lister, + queue: workqueue.NewNamedRateLimitingQueue( + workqueue.DefaultControllerRateLimiter(), + constants.ServiceAccountResourceKind, + ), + logger: logger, + secretsSynced: secrets.Informer().HasSynced, + talosSAsSynced: informer.HasSynced, + eventRecorder: recorder, + secretsLister: secrets.Lister(), + } + + if _, err = informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: controller.enqueueTalosSA, + UpdateFunc: func(oldTalosSA, newTalosSA interface{}) { + controller.enqueueTalosSA(newTalosSA) + }, + }); err != nil { + return nil, err + } + + if _, err = secrets.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: controller.handleSecret, + UpdateFunc: func(oldSec, newSec interface{}) { + newSecret := newSec.(*corev1.Secret) //nolint:errcheck + oldSecret := oldSec.(*corev1.Secret) //nolint:errcheck + + if newSecret.ResourceVersion == oldSecret.ResourceVersion { + return + } + + controller.handleSecret(newSec) + }, + DeleteFunc: controller.handleSecret, + }); err != nil { + return nil, err + } + + return &controller, nil +} + +// Run starts the CRD controller. +func (t *CRDController) Run(ctx context.Context, workers int) error { + var wg sync.WaitGroup + + defer func() { + t.queue.ShutDown() + t.dialer.CloseAll() + + wg.Wait() + t.logger.Debug("all workers have shut down") + }() + + t.kubeInformerFactory.Start(ctx.Done()) + t.dynamicInformerFactory.Start(ctx.Done()) + + t.logger.Sugar().Debugf("starting %s controller", constants.ServiceAccountResourceKind) + + t.logger.Debug("waiting for informer caches to sync") + + if ok := cache.WaitForCacheSync(ctx.Done(), t.secretsSynced, t.talosSAsSynced); !ok { + return errors.New("failed to wait for caches to sync") + } + + t.logger.Debug("starting workers") + + wg.Add(workers) + + for range workers { + go func() { + wait.Until(func() { t.runWorker(ctx) }, time.Second, ctx.Done()) + wg.Done() + }() + } + + t.logger.Debug("started workers") + + <-ctx.Done() + + t.logger.Debug("shutting down workers") + + t.kubeInformerFactory.Shutdown() + + return nil +} + +func (t *CRDController) runWorker(ctx context.Context) { + for t.processNextWorkItem(ctx) { + } +} + +func (t *CRDController) processNextWorkItem(ctx context.Context) bool { + obj, shutdown := t.queue.Get() + + if shutdown { + return false + } + + err := func(obj interface{}) error { + defer t.queue.Done(obj) + + var key string + + var ok bool + + if key, ok = obj.(string); !ok { + t.queue.Forget(obj) + utilruntime.HandleError(fmt.Errorf("expected string in workqueue but got %#v", obj)) + + return nil + } + + if err := t.syncHandler(ctx, key); err != nil { + t.queue.AddRateLimited(key) + + return fmt.Errorf("error syncing '%s': %s, requeuing", key, err.Error()) + } + + t.queue.Forget(obj) + t.logger.Sugar().Debugf("successfully synced '%s'", key) + + return nil + }(obj) + if err != nil { + utilruntime.HandleError(err) + + return true + } + + return true +} + +//nolint:gocyclo,cyclop +func (t *CRDController) syncHandler(ctx context.Context, key string) error { + namespace, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) + + return nil //nolint:nilerr + } + + talosSA, err := t.dynamicLister.Namespace(namespace).Get(name) + if err != nil { + if kubeerrors.IsNotFound(err) { + utilruntime.HandleError(fmt.Errorf("talosSA '%s' in work queue no longer exists", key)) + + return nil + } + + return err + } + + secret, err := t.secretsLister.Secrets(namespace).Get(name) + secretNotFound := kubeerrors.IsNotFound(err) + + if err != nil && !secretNotFound { + return err + } + + if !secretNotFound && !metav1.IsControlledBy(secret, talosSA) { + msg := fmt.Sprintf(messageResourceExists, kindSecret, key) + + err = t.updateTalosSAStatus(ctx, talosSA, msg) + if err != nil { + return err + } + + t.eventRecorder.Event(talosSA, corev1.EventTypeWarning, errResourceExists, msg) + + return errors.New(msg) + } + + desiredRoles, found, err := unstructured.NestedStringSlice(talosSA.UnstructuredContent(), "spec", "roles") + if err != nil || !found { + msg := messageRolesNotFound + + updateErr := t.updateTalosSAStatus(ctx, talosSA, msg) + if updateErr != nil { + return updateErr + } + + t.eventRecorder.Event(talosSA, corev1.EventTypeWarning, errRolesNotFound, messageRolesNotFound) + + if err != nil { + return fmt.Errorf("%s: %w", msg, err) + } + + return errors.New(msg) + } + + desiredRoleSet, _ := role.Parse(desiredRoles) + + if !slices.ContainsFunc(t.allowedNamespaces, func(allowedNS string) bool { + return allowedNS == namespace + }) { + msg := fmt.Sprintf(messageNamespaceNotAllowed, namespace) + + err = t.updateTalosSAStatus(ctx, talosSA, msg) + if err != nil { + return err + } + + t.eventRecorder.Event(talosSA, corev1.EventTypeWarning, errNamespaceNotAllowed, msg) + + return nil + } + + var unallowedRoles []string + + for _, desiredRole := range desiredRoles { + _, allowed := t.allowedRoles[desiredRole] + if !allowed { + unallowedRoles = append(unallowedRoles, desiredRole) + } + } + + if len(unallowedRoles) > 0 { + msg := fmt.Sprintf(messageRolesNotAllowed, unallowedRoles) + + err = t.updateTalosSAStatus(ctx, talosSA, msg) + if err != nil { + return err + } + + t.eventRecorder.Event(talosSA, corev1.EventTypeWarning, errRolesNotAllowed, msg) + + return nil + } + + if secretNotFound { + var newSecret *corev1.Secret + + newSecret, err = t.newSecret(talosSA, desiredRoleSet) + if err != nil { + return err + } + + _, err = t.kubeClient.CoreV1().Secrets(namespace).Create(ctx, newSecret, metav1.CreateOptions{}) + if err != nil { + return err + } + } else if t.needsUpdate(secret, desiredRoleSet.Strings()) { + var newTalosconfigBytes []byte + + newTalosconfigBytes, err = t.generateTalosconfig(desiredRoleSet) + if err != nil { + return err + } + + secret.Data[constants.TalosconfigFilename] = newTalosconfigBytes + + _, err = t.kubeClient.CoreV1().Secrets(namespace).Update(ctx, secret, metav1.UpdateOptions{}) + if err != nil { + return err + } + } + + err = t.updateTalosSAStatus(ctx, talosSA, "") + if err != nil { + return err + } + + t.eventRecorder.Event(talosSA, corev1.EventTypeNormal, successResourceSynced, messageResourceSynced) + + return nil +} + +func (t *CRDController) enqueueTalosSA(obj interface{}) { + key, err := cache.MetaNamespaceKeyFunc(obj) + if err != nil { + utilruntime.HandleError(err) + + return + } + + t.queue.Add(key) +} + +func (t *CRDController) handleSecret(obj interface{}) { + var object metav1.Object + + var ok bool + + if object, ok = obj.(metav1.Object); !ok { + tombstone, tombstoneOK := obj.(cache.DeletedFinalStateUnknown) + if !tombstoneOK { + utilruntime.HandleError(errors.New("error decoding object, invalid type")) + + return + } + + object, tombstoneOK = tombstone.Obj.(metav1.Object) + if !tombstoneOK { + utilruntime.HandleError(errors.New("error decoding object tombstone, invalid type")) + + return + } + + t.logger.Sugar().Debugf("recovered deleted object '%s' from tombstone", object.GetName()) + } + + t.logger.Sugar().Debugf("processing object: %s", object.GetName()) + + if ownerRef := metav1.GetControllerOf(object); ownerRef != nil { + if ownerRef.Kind != constants.ServiceAccountResourceKind { + return + } + + talosSA, err := t.dynamicLister.Namespace(object.GetNamespace()).Get(ownerRef.Name) + if err != nil { + t.logger.Sugar().Debugf("ignoring orphaned object '%s/%s' of %s '%s'", + object.GetNamespace(), object.GetName(), constants.ServiceAccountResourceKind, ownerRef.Name) + + return + } + + t.enqueueTalosSA(talosSA) + + return + } +} + +func (t *CRDController) updateTalosSAStatus( + ctx context.Context, + talosSA *unstructured.Unstructured, + failureReason string, +) error { + var err error + + talosSACopy := talosSA.DeepCopy() + + if err != nil { + return err + } + + if failureReason == "" { + unstructured.RemoveNestedField(talosSACopy.UnstructuredContent(), "status", "failureReason") + + if err != nil { + return err + } + } else { + err = unstructured.SetNestedField(talosSACopy.UnstructuredContent(), failureReason, "status", "failureReason") + if err != nil { + return err + } + } + + _, err = t.dynamicClient.Resource(talosSAGVR). + Namespace(talosSACopy.GetNamespace()). + Update(ctx, talosSACopy, metav1.UpdateOptions{}) + + return err +} + +//nolint:gocyclo +func (t *CRDController) needsUpdate(secret *corev1.Secret, desiredRoles []string) bool { + talosconfigInSecret, ok := secret.Data[constants.TalosconfigFilename] + if !ok { + t.logger.Debug("talosconfig not found in secret", zap.String("key", constants.TalosconfigFilename)) + + return true + } + + parsedTalosconfigInSecret, err := clientconfig.ReadFrom(bytes.NewReader(talosconfigInSecret)) + if err != nil { + t.logger.Debug("error parsing talosconfig in secret", zap.Error(err)) + + return true + } + + talosconfigCtx := parsedTalosconfigInSecret.Contexts[parsedTalosconfigInSecret.Context] + + talosconfigCA, err := base64.StdEncoding.DecodeString(talosconfigCtx.CA) + if err != nil { + t.logger.Debug("error decoding talosconfig CA", zap.Error(err)) + + return true + } + + if !reflect.DeepEqual(t.talosCA.Crt, talosconfigCA) { + t.logger.Debug("ca mismatch detected") + + return true + } + + if len(talosconfigCtx.Endpoints) != 1 || talosconfigCtx.Endpoints[0] != endpoint { + t.logger.Debug( + "endpoint mismatch detected", + zap.Strings("actual", talosconfigCtx.Endpoints), + zap.Strings("expected", []string{endpoint}), + ) + + return true + } + + talosconfigCRT, err := base64.StdEncoding.DecodeString(talosconfigCtx.Crt) + if err != nil { + t.logger.Debug("error decoding talosconfig CRT", zap.Error(err)) + + return true + } + + block, _ := pem.Decode(talosconfigCRT) + if block == nil { + t.logger.Debug("could not decode talosconfig CRT") + + return true + } + + certificate, err := stdlibx509.ParseCertificate(block.Bytes) + if err != nil { + t.logger.Debug("error parsing certificate in talosconfig of secret", zap.Error(err)) + + return true + } + + if certificate.NotAfter.IsZero() { + t.logger.Debug("certificate in talosconfig of secret has no expiration date", zap.Error(err)) + + return true + } + + if time.Now().Add(certTTL).Before(certificate.NotAfter) { + t.logger.Debug( + "certificate in talosconfig has expiration date too far in the future", + zap.Time("expiration", certificate.NotAfter), + ) + + return true + } + + if time.Now().Add(certRenewThreshold).After(certificate.NotAfter) { + t.logger.Debug( + "certificate in talosconfig needs renewal", + zap.Time("expiration", certificate.NotAfter), + ) + + return true + } + + actualRoles := certificate.Subject.Organization + + sort.Strings(actualRoles) + sort.Strings(desiredRoles) + + if !reflect.DeepEqual(actualRoles, desiredRoles) { + t.logger.Debug("roles in certificate do not match desired roles", + zap.Strings("actual", actualRoles), zap.Strings("desired", desiredRoles)) + + return true + } + + return false +} + +func (t *CRDController) newSecret(talosSA *unstructured.Unstructured, roles role.Set) (*corev1.Secret, error) { + config, err := t.generateTalosconfig(roles) + if err != nil { + return nil, err + } + + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: talosSA.GetName(), + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(talosSA, talosSAGVK), + }, + }, + Data: map[string][]byte{ + constants.TalosconfigFilename: config, + }, + }, nil +} + +func (t *CRDController) generateTalosconfig(roles role.Set) ([]byte, error) { + var newCert *x509.PEMEncodedCertificateAndKey + + newCert, err := secrets.NewAdminCertificateAndKey(time.Now(), t.talosCA, roles, certTTL) + if err != nil { + return nil, err + } + + newTalosconfig := clientconfig.NewConfig(talosconfigContextName, []string{endpoint}, t.talosCA.Crt, newCert) + + return newTalosconfig.Bytes() +} diff --git a/internal/app/machined/pkg/controllers/kubespan/config.go b/internal/app/machined/pkg/controllers/kubespan/config.go new file mode 100644 index 0000000..5d7f299 --- /dev/null +++ b/internal/app/machined/pkg/controllers/kubespan/config.go @@ -0,0 +1,60 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package kubespan + +import ( + "context" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/controller/generic/transform" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/kubespan" +) + +// ConfigController watches v1alpha1.Config, updates KubeSpan config. +type ConfigController = transform.Controller[*config.MachineConfig, *kubespan.Config] + +// NewConfigController instanciates the config controller. +func NewConfigController() *ConfigController { + return transform.NewController( + transform.Settings[*config.MachineConfig, *kubespan.Config]{ + Name: "kubespan.ConfigController", + MapMetadataOptionalFunc: func(cfg *config.MachineConfig) optional.Optional[*kubespan.Config] { + if cfg.Metadata().ID() != config.V1Alpha1ID { + return optional.None[*kubespan.Config]() + } + + if cfg.Config().Machine() == nil || cfg.Config().Cluster() == nil { + return optional.None[*kubespan.Config]() + } + + return optional.Some(kubespan.NewConfig(config.NamespaceName, kubespan.ConfigID)) + }, + TransformFunc: func(ctx context.Context, r controller.Reader, logger *zap.Logger, cfg *config.MachineConfig, res *kubespan.Config) error { + spec := res.TypedSpec() + + *spec = kubespan.ConfigSpec{} + + if cfg != nil && cfg.Config().Machine() != nil { + c := cfg.Config() + + res.TypedSpec().Enabled = c.Machine().Network().KubeSpan().Enabled() + res.TypedSpec().ClusterID = c.Cluster().ID() + res.TypedSpec().SharedSecret = c.Cluster().Secret() + res.TypedSpec().ForceRouting = c.Machine().Network().KubeSpan().ForceRouting() + res.TypedSpec().AdvertiseKubernetesNetworks = c.Machine().Network().KubeSpan().AdvertiseKubernetesNetworks() + res.TypedSpec().HarvestExtraEndpoints = c.Machine().Network().KubeSpan().HarvestExtraEndpoints() + res.TypedSpec().MTU = c.Machine().Network().KubeSpan().MTU() + res.TypedSpec().EndpointFilters = c.Machine().Network().KubeSpan().Filters().Endpoints() + } + + return nil + }, + }, + ) +} diff --git a/internal/app/machined/pkg/controllers/kubespan/config_test.go b/internal/app/machined/pkg/controllers/kubespan/config_test.go new file mode 100644 index 0000000..55558db --- /dev/null +++ b/internal/app/machined/pkg/controllers/kubespan/config_test.go @@ -0,0 +1,106 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. +package kubespan_test + +import ( + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/siderolabs/go-pointer" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + kubespanctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/kubespan" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/kubespan" +) + +type ConfigSuite struct { + KubeSpanSuite +} + +func (suite *ConfigSuite) TestReconcileConfig() { + suite.Require().NoError(suite.runtime.RegisterController(kubespanctrl.NewConfigController())) + + suite.startRuntime() + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineNetwork: &v1alpha1.NetworkConfig{ + NetworkKubeSpan: &v1alpha1.NetworkKubeSpan{ + KubeSpanEnabled: pointer.To(true), + }, + }, + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + ClusterID: "8XuV9TZHW08DOk3bVxQjH9ih_TBKjnh-j44tsCLSBzo=", + ClusterSecret: "I+1In7fLnpcRIjUmEoeugZnSyFoTF6MztLxICL5Yu0s=", + }, + })) + + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + specMD := resource.NewMetadata(config.NamespaceName, kubespan.ConfigType, kubespan.ConfigID, resource.VersionUndefined) + + suite.Assert().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertResource( + specMD, + func(res resource.Resource) error { + spec := res.(*kubespan.Config).TypedSpec() + + suite.Assert().True(spec.Enabled) + suite.Assert().Equal("8XuV9TZHW08DOk3bVxQjH9ih_TBKjnh-j44tsCLSBzo=", spec.ClusterID) + suite.Assert().Equal("I+1In7fLnpcRIjUmEoeugZnSyFoTF6MztLxICL5Yu0s=", spec.SharedSecret) + suite.Assert().True(spec.ForceRouting) + suite.Assert().False(spec.AdvertiseKubernetesNetworks) + suite.Assert().False(spec.HarvestExtraEndpoints) + + return nil + }, + ), + )) +} + +func (suite *ConfigSuite) TestReconcileDisabled() { + suite.Require().NoError(suite.runtime.RegisterController(kubespanctrl.NewConfigController())) + + suite.startRuntime() + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{}, + ClusterConfig: &v1alpha1.ClusterConfig{}, + })) + + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + specMD := resource.NewMetadata(config.NamespaceName, kubespan.ConfigType, kubespan.ConfigID, resource.VersionUndefined) + + suite.Assert().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertResource( + specMD, + func(res resource.Resource) error { + spec := res.(*kubespan.Config).TypedSpec() + + suite.Assert().False(spec.Enabled) + + return nil + }, + ), + )) +} + +func TestConfigSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, new(ConfigSuite)) +} diff --git a/internal/app/machined/pkg/controllers/kubespan/endpoint.go b/internal/app/machined/pkg/controllers/kubespan/endpoint.go new file mode 100644 index 0000000..69e3dfa --- /dev/null +++ b/internal/app/machined/pkg/controllers/kubespan/endpoint.go @@ -0,0 +1,145 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package kubespan + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/value" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/kubespan" +) + +// EndpointController watches KubeSpanPeerStatuses, Affiliates and harvests additional endpoints for the peers. +type EndpointController struct{} + +// Name implements controller.Controller interface. +func (ctrl *EndpointController) Name() string { + return "kubespan.EndpointController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *EndpointController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: kubespan.ConfigType, + ID: optional.Some(kubespan.ConfigID), + Kind: controller.InputWeak, + }, + { + Namespace: cluster.NamespaceName, + Type: cluster.AffiliateType, + Kind: controller.InputWeak, + }, + { + Namespace: kubespan.NamespaceName, + Type: kubespan.PeerStatusType, + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *EndpointController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: kubespan.EndpointType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *EndpointController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + cfg, err := safe.ReaderGetByID[*kubespan.Config](ctx, r, kubespan.ConfigID) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error getting kubespan configuration: %w", err) + } + + r.StartTrackingOutputs() + + if cfg == nil || !cfg.TypedSpec().HarvestExtraEndpoints { + // not enabled, short-circuit early + if err = safe.CleanupOutputs[*kubespan.Endpoint](ctx, r); err != nil { + return err + } + + continue + } + + // for every kubespan peer, if it's up and has endpoint, harvest that endpoint + peerStatuses, err := safe.ReaderListAll[*kubespan.PeerStatus](ctx, r) + if err != nil { + return fmt.Errorf("error listing cluster affiliates: %w", err) + } + + affiliates, err := safe.ReaderListAll[*cluster.Affiliate](ctx, r) + if err != nil { + return fmt.Errorf("error listing cluster affiliates: %w", err) + } + + // build lookup table of affiliate's kubespan public key back to affiliate ID + affiliateLookup := make(map[string]string) + + for it := affiliates.Iterator(); it.Next(); { + affiliate := it.Value().TypedSpec() + + if affiliate.KubeSpan.PublicKey != "" { + affiliateLookup[affiliate.KubeSpan.PublicKey] = affiliate.NodeID + } + } + + for it := peerStatuses.Iterator(); it.Next(); { + res := it.Value() + peerStatus := res.TypedSpec() + + if peerStatus.State != kubespan.PeerStateUp { + continue + } + + if value.IsZero(peerStatus.Endpoint) { + continue + } + + affiliateID, ok := affiliateLookup[res.Metadata().ID()] + if !ok { + continue + } + + if err = safe.WriterModify(ctx, r, kubespan.NewEndpoint(kubespan.NamespaceName, res.Metadata().ID()), func(res *kubespan.Endpoint) error { + *res.TypedSpec() = kubespan.EndpointSpec{ + AffiliateID: affiliateID, + Endpoint: peerStatus.Endpoint, + } + + return nil + }); err != nil { + return err + } + } + + if err = safe.CleanupOutputs[*kubespan.Endpoint](ctx, r); err != nil { + return err + } + } +} diff --git a/internal/app/machined/pkg/controllers/kubespan/endpoint_test.go b/internal/app/machined/pkg/controllers/kubespan/endpoint_test.go new file mode 100644 index 0000000..dba5504 --- /dev/null +++ b/internal/app/machined/pkg/controllers/kubespan/endpoint_test.go @@ -0,0 +1,136 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. +package kubespan_test + +import ( + "net/netip" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + kubespanctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/kubespan" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/kubespan" +) + +type EndpointSuite struct { + KubeSpanSuite +} + +func (suite *EndpointSuite) TestReconcile() { + suite.Require().NoError(suite.runtime.RegisterController(&kubespanctrl.EndpointController{})) + + suite.startRuntime() + + cfg := kubespan.NewConfig(config.NamespaceName, kubespan.ConfigID) + cfg.TypedSpec().HarvestExtraEndpoints = true + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + // create some affiliates and peer statuses + affiliate1 := cluster.NewAffiliate(cluster.NamespaceName, "7x1SuC8Ege5BGXdAfTEff5iQnlWZLfv9h1LGMxA2pYkC") + *affiliate1.TypedSpec() = cluster.AffiliateSpec{ + NodeID: "7x1SuC8Ege5BGXdAfTEff5iQnlWZLfv9h1LGMxA2pYkC", + Hostname: "foo.com", + Nodename: "bar", + MachineType: machine.TypeControlPlane, + Addresses: []netip.Addr{netip.MustParseAddr("192.168.3.4")}, + KubeSpan: cluster.KubeSpanAffiliateSpec{ + PublicKey: "PLPNBddmTgHJhtw0vxltq1ZBdPP9RNOEUd5JjJZzBRY=", + Address: netip.MustParseAddr("fd50:8d60:4238:6302:f857:23ff:fe21:d1e0"), + AdditionalAddresses: []netip.Prefix{netip.MustParsePrefix("10.244.3.1/24")}, + Endpoints: []netip.AddrPort{netip.MustParseAddrPort("10.0.0.2:51820"), netip.MustParseAddrPort("192.168.3.4:51820")}, + }, + } + + affiliate2 := cluster.NewAffiliate(cluster.NamespaceName, "roLng5hmP0Gv9S5Pbfzaa93JSZjsdpXNAn7vzuCfsc8") + *affiliate2.TypedSpec() = cluster.AffiliateSpec{ + NodeID: "roLng5hmP0Gv9S5Pbfzaa93JSZjsdpXNAn7vzuCfsc8", + MachineType: machine.TypeControlPlane, + Addresses: []netip.Addr{netip.MustParseAddr("192.168.3.5")}, + KubeSpan: cluster.KubeSpanAffiliateSpec{ + PublicKey: "1CXkdhWBm58c36kTpchR8iGlXHG1ruHa5W8gsFqD8Qs=", + Address: netip.MustParseAddr("fd50:8d60:4238:6302:f857:23ff:fe21:d1e1"), + }, + } + + suite.Require().NoError(suite.state.Create(suite.ctx, affiliate1)) + suite.Require().NoError(suite.state.Create(suite.ctx, affiliate2)) + + peerStatus1 := kubespan.NewPeerStatus(kubespan.NamespaceName, affiliate1.TypedSpec().KubeSpan.PublicKey) + *peerStatus1.TypedSpec() = kubespan.PeerStatusSpec{ + Endpoint: netip.MustParseAddrPort("10.3.4.8:278"), + State: kubespan.PeerStateUp, + } + + peerStatus2 := kubespan.NewPeerStatus(kubespan.NamespaceName, affiliate2.TypedSpec().KubeSpan.PublicKey) + *peerStatus2.TypedSpec() = kubespan.PeerStatusSpec{ + Endpoint: netip.MustParseAddrPort("10.3.4.9:279"), + State: kubespan.PeerStateUnknown, + } + + peerStatus3 := kubespan.NewPeerStatus(kubespan.NamespaceName, "LoXPyyYh3kZwyKyWfCcf9VvgVv588cKhSKXavuUZqDg=") + *peerStatus3.TypedSpec() = kubespan.PeerStatusSpec{ + Endpoint: netip.MustParseAddrPort("10.3.4.10:270"), + State: kubespan.PeerStateUp, + } + + suite.Require().NoError(suite.state.Create(suite.ctx, peerStatus1)) + suite.Require().NoError(suite.state.Create(suite.ctx, peerStatus2)) + suite.Require().NoError(suite.state.Create(suite.ctx, peerStatus3)) + + // peer1 is up and has matching affiliate + suite.Assert().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertResource( + resource.NewMetadata( + kubespan.NamespaceName, + kubespan.EndpointType, + peerStatus1.Metadata().ID(), + resource.VersionUndefined, + ), + func(res resource.Resource) error { + spec := res.(*kubespan.Endpoint).TypedSpec() + + suite.Assert().Equal(peerStatus1.TypedSpec().Endpoint, spec.Endpoint) + suite.Assert().Equal(affiliate1.TypedSpec().NodeID, spec.AffiliateID) + + return nil + }, + ), + )) + + // peer2 is not up, it shouldn't be published as an endpoint + suite.Assert().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertNoResource( + resource.NewMetadata( + kubespan.NamespaceName, + kubespan.EndpointType, + peerStatus2.Metadata().ID(), + resource.VersionUndefined, + ), + ), + )) + + // peer3 is up, but has not matching affiliate + suite.Assert().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertNoResource( + resource.NewMetadata( + kubespan.NamespaceName, + kubespan.EndpointType, + peerStatus3.Metadata().ID(), + resource.VersionUndefined, + ), + ), + )) +} + +func TestEndpointSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, new(EndpointSuite)) +} diff --git a/internal/app/machined/pkg/controllers/kubespan/identity.go b/internal/app/machined/pkg/controllers/kubespan/identity.go new file mode 100644 index 0000000..dfa8249 --- /dev/null +++ b/internal/app/machined/pkg/controllers/kubespan/identity.go @@ -0,0 +1,155 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package kubespan + +import ( + "context" + "fmt" + "net" + "path/filepath" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + kubespanadapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/kubespan" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/kubespan" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +// IdentityController watches KubeSpan configuration, updates KubeSpan Identity. +type IdentityController struct { + StatePath string +} + +// Name implements controller.Controller interface. +func (ctrl *IdentityController) Name() string { + return "kubespan.IdentityController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *IdentityController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: kubespan.ConfigType, + ID: optional.Some(kubespan.ConfigID), + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.HardwareAddrType, + ID: optional.Some(network.FirstHardwareAddr), + Kind: controller.InputWeak, + }, + { + Namespace: v1alpha1.NamespaceName, + Type: runtimeres.MountStatusType, + ID: optional.Some(constants.StatePartitionLabel), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *IdentityController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: kubespan.IdentityType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *IdentityController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + if ctrl.StatePath == "" { + ctrl.StatePath = constants.StateMountPoint + } + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + if _, err := r.Get(ctx, resource.NewMetadata(v1alpha1.NamespaceName, runtimeres.MountStatusType, constants.StatePartitionLabel, resource.VersionUndefined)); err != nil { + if state.IsNotFoundError(err) { + // wait for STATE to be mounted + continue + } + + return fmt.Errorf("error reading mount status: %w", err) + } + + cfg, err := r.Get(ctx, resource.NewMetadata(config.NamespaceName, kubespan.ConfigType, kubespan.ConfigID, resource.VersionUndefined)) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error getting kubespan configuration: %w", err) + } + + firstMAC, err := r.Get(ctx, resource.NewMetadata(network.NamespaceName, network.HardwareAddrType, network.FirstHardwareAddr, resource.VersionUndefined)) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error getting first MAC address: %w", err) + } + + touchedIDs := make(map[resource.ID]struct{}) + + if cfg != nil && firstMAC != nil && cfg.(*kubespan.Config).TypedSpec().Enabled { + var localIdentity kubespan.IdentitySpec + + if err = controllers.LoadOrNewFromFile(filepath.Join(ctrl.StatePath, constants.KubeSpanIdentityFilename), &localIdentity, func(v interface{}) error { + return kubespanadapter.IdentitySpec(v.(*kubespan.IdentitySpec)).GenerateKey() + }); err != nil { + return fmt.Errorf("error caching kubespan identity: %w", err) + } + + kubespanCfg := cfg.(*kubespan.Config).TypedSpec() + mac := firstMAC.(*network.HardwareAddr).TypedSpec() + + if err = kubespanadapter.IdentitySpec(&localIdentity).UpdateAddress(kubespanCfg.ClusterID, net.HardwareAddr(mac.HardwareAddr)); err != nil { + return fmt.Errorf("error updating KubeSpan address: %w", err) + } + + if err = r.Modify(ctx, kubespan.NewIdentity(kubespan.NamespaceName, kubespan.LocalIdentity), func(res resource.Resource) error { + *res.(*kubespan.Identity).TypedSpec() = localIdentity + + return nil + }); err != nil { + return err + } + + touchedIDs[kubespan.LocalIdentity] = struct{}{} + } + + // list keys for cleanup + list, err := r.List(ctx, resource.NewMetadata(kubespan.NamespaceName, kubespan.IdentityType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing resources: %w", err) + } + + for _, res := range list.Items { + if res.Metadata().Owner() != ctrl.Name() { + continue + } + + if _, ok := touchedIDs[res.Metadata().ID()]; !ok { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return fmt.Errorf("error cleaning up specs: %w", err) + } + } + } + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/kubespan/identity_test.go b/internal/app/machined/pkg/controllers/kubespan/identity_test.go new file mode 100644 index 0000000..b2669e3 --- /dev/null +++ b/internal/app/machined/pkg/controllers/kubespan/identity_test.go @@ -0,0 +1,143 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. +package kubespan_test + +import ( + "net" + "os" + "path/filepath" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + kubespanctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/kubespan" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/kubespan" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +type IdentitySuite struct { + KubeSpanSuite + + statePath string +} + +func (suite *IdentitySuite) TestGenerate() { + suite.statePath = suite.T().TempDir() + + suite.Require().NoError(suite.runtime.RegisterController(&kubespanctrl.IdentityController{ + StatePath: suite.statePath, + })) + + suite.startRuntime() + + stateMount := runtimeres.NewMountStatus(v1alpha1.NamespaceName, constants.StatePartitionLabel) + + suite.Assert().NoError(suite.state.Create(suite.ctx, stateMount)) + + cfg := kubespan.NewConfig(config.NamespaceName, kubespan.ConfigID) + cfg.TypedSpec().Enabled = true + cfg.TypedSpec().ClusterID = "8XuV9TZHW08DOk3bVxQjH9ih_TBKjnh-j44tsCLSBzo=" + + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + firstMac := network.NewHardwareAddr(network.NamespaceName, network.FirstHardwareAddr) + mac, err := net.ParseMAC("ea:71:1b:b2:cc:ee") + suite.Require().NoError(err) + + firstMac.TypedSpec().HardwareAddr = nethelpers.HardwareAddr(mac) + + suite.Require().NoError(suite.state.Create(suite.ctx, firstMac)) + + specMD := resource.NewMetadata(kubespan.NamespaceName, kubespan.IdentityType, kubespan.LocalIdentity, resource.VersionUndefined) + + suite.Assert().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertResource( + specMD, + func(res resource.Resource) error { + spec := res.(*kubespan.Identity).TypedSpec() + + _, err := wgtypes.ParseKey(spec.PrivateKey) + suite.Assert().NoError(err) + + _, err = wgtypes.ParseKey(spec.PublicKey) + suite.Assert().NoError(err) + + suite.Assert().Equal("fd7f:175a:b97c:5602:e871:1bff:feb2:ccee/128", spec.Address.String()) + suite.Assert().Equal("fd7f:175a:b97c:5602::/64", spec.Subnet.String()) + + return nil + }, + ), + )) +} + +func (suite *IdentitySuite) TestLoad() { + // using verbatim data here to make sure nodeId representation is supported in future version of Talos + const identityYaml = `address: "" +subnet: "" +privateKey: sF45u5ePau58WeeCUY3T8D9foEKaQ8Opx4cGC8g4XE4= +publicKey: Oak2fBEWngBhwslBxDVgnRNHXs88OAp4kjroSX0uqUE= +` + + suite.statePath = suite.T().TempDir() + + suite.Require().NoError(suite.runtime.RegisterController(&kubespanctrl.IdentityController{ + StatePath: suite.statePath, + })) + + suite.startRuntime() + + suite.Require().NoError(os.WriteFile(filepath.Join(suite.statePath, constants.KubeSpanIdentityFilename), []byte(identityYaml), 0o600)) + + stateMount := runtimeres.NewMountStatus(v1alpha1.NamespaceName, constants.StatePartitionLabel) + + suite.Assert().NoError(suite.state.Create(suite.ctx, stateMount)) + + cfg := kubespan.NewConfig(config.NamespaceName, kubespan.ConfigID) + cfg.TypedSpec().Enabled = true + cfg.TypedSpec().ClusterID = "8XuV9TZHW08DOk3bVxQjH9ih_TBKjnh-j44tsCLSBzo=" + + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + firstMac := network.NewHardwareAddr(network.NamespaceName, network.FirstHardwareAddr) + mac, err := net.ParseMAC("ea:71:1b:b2:cc:ee") + suite.Require().NoError(err) + + firstMac.TypedSpec().HardwareAddr = nethelpers.HardwareAddr(mac) + + suite.Require().NoError(suite.state.Create(suite.ctx, firstMac)) + + specMD := resource.NewMetadata(kubespan.NamespaceName, kubespan.IdentityType, kubespan.LocalIdentity, resource.VersionUndefined) + + suite.Assert().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertResource( + specMD, + func(res resource.Resource) error { + spec := res.(*kubespan.Identity).TypedSpec() + + suite.Assert().Equal("sF45u5ePau58WeeCUY3T8D9foEKaQ8Opx4cGC8g4XE4=", spec.PrivateKey) + suite.Assert().Equal("Oak2fBEWngBhwslBxDVgnRNHXs88OAp4kjroSX0uqUE=", spec.PublicKey) + suite.Assert().Equal("fd7f:175a:b97c:5602:e871:1bff:feb2:ccee/128", spec.Address.String()) + suite.Assert().Equal("fd7f:175a:b97c:5602::/64", spec.Subnet.String()) + + return nil + }, + ), + )) +} + +func TestIdentitySuite(t *testing.T) { + t.Parallel() + + suite.Run(t, new(IdentitySuite)) +} diff --git a/internal/app/machined/pkg/controllers/kubespan/kubespan.go b/internal/app/machined/pkg/controllers/kubespan/kubespan.go new file mode 100644 index 0000000..cb59c1e --- /dev/null +++ b/internal/app/machined/pkg/controllers/kubespan/kubespan.go @@ -0,0 +1,6 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package kubespan provides controllers which manage Talos KubeSpan feature. +package kubespan diff --git a/internal/app/machined/pkg/controllers/kubespan/kubespan_test.go b/internal/app/machined/pkg/controllers/kubespan/kubespan_test.go new file mode 100644 index 0000000..b7b1412 --- /dev/null +++ b/internal/app/machined/pkg/controllers/kubespan/kubespan_test.go @@ -0,0 +1,117 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package kubespan_test + +import ( + "context" + "log" + "reflect" + "sort" + "sync" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + "github.com/siderolabs/talos/pkg/logging" +) + +type KubeSpanSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *KubeSpanSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + logger := logging.Wrap(log.Writer()) + + suite.runtime, err = runtime.NewRuntime(suite.state, logger) + suite.Require().NoError(err) +} + +func (suite *KubeSpanSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *KubeSpanSuite) assertResourceIDs(md resource.Metadata, expectedIDs []resource.ID) func() error { + return func() error { + l, err := suite.state.List(suite.ctx, md) + if err != nil { + return err + } + + actualIDs := xslices.Map(l.Items, func(r resource.Resource) string { return r.Metadata().ID() }) + + sort.Strings(expectedIDs) + + if !reflect.DeepEqual(actualIDs, expectedIDs) { + return retry.ExpectedErrorf("ids do no match expected %v != actual %v", expectedIDs, actualIDs) + } + + return nil + } +} + +func (suite *KubeSpanSuite) assertNoResource(md resource.Metadata) func() error { + return func() error { + _, err := suite.state.Get(suite.ctx, md) + if err == nil { + return retry.ExpectedErrorf("resource %s still exists", md) + } + + if state.IsNotFoundError(err) { + return nil + } + + return err + } +} + +func (suite *KubeSpanSuite) assertResource(md resource.Metadata, check func(res resource.Resource) error) func() error { + return func() error { + r, err := suite.state.Get(suite.ctx, md) + if err != nil { + if state.IsNotFoundError(err) { + return retry.ExpectedError(err) + } + + return err + } + + return check(r) + } +} + +func (suite *KubeSpanSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} diff --git a/internal/app/machined/pkg/controllers/kubespan/manager.go b/internal/app/machined/pkg/controllers/kubespan/manager.go new file mode 100644 index 0000000..51743b7 --- /dev/null +++ b/internal/app/machined/pkg/controllers/kubespan/manager.go @@ -0,0 +1,613 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package kubespan + +import ( + "context" + "errors" + "fmt" + "net/netip" + "os" + "slices" + "time" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/value" + "github.com/siderolabs/go-pointer" + "go.uber.org/zap" + "go4.org/netipx" + "golang.zx2c4.com/wireguard/wgctrl" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + kubespanadapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/kubespan" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/kubespan" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// DefaultPeerReconcileInterval is interval between peer status reconciliation on timer. +// +// Peers might be reconciled more often e.g. when peerSpecs are updated. +const DefaultPeerReconcileInterval = 30 * time.Second + +// ManagerController sets up Wireguard networking based on KubeSpan configuration, watches and updates peer statuses. +type ManagerController struct { + WireguardClientFactory WireguardClientFactory + RulesManagerFactory RulesManagerFactory + PeerReconcileInterval time.Duration +} + +// Name implements controller.Controller interface. +func (ctrl *ManagerController) Name() string { + return "kubespan.ManagerController" +} + +// WireguardClientFactory allows mocking Wireguard client. +type WireguardClientFactory func() (WireguardClient, error) + +// WireguardClient allows mocking Wireguard client. +type WireguardClient interface { + Device(string) (*wgtypes.Device, error) + Close() error +} + +// RulesManagerFactory allows mocking RulesManager. +type RulesManagerFactory func(targetTable, internalMark, markMask int) RulesManager + +// Inputs implements controller.Controller interface. +func (ctrl *ManagerController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: kubespan.ConfigType, + ID: optional.Some(kubespan.ConfigID), + Kind: controller.InputWeak, + }, + { + Namespace: kubespan.NamespaceName, + Type: kubespan.PeerSpecType, + Kind: controller.InputWeak, + }, + { + Namespace: kubespan.NamespaceName, + Type: kubespan.IdentityType, + ID: optional.Some(kubespan.LocalIdentity), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *ManagerController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.LinkSpecType, + Kind: controller.OutputShared, + }, + { + Type: network.AddressSpecType, + Kind: controller.OutputShared, + }, + { + Type: network.RouteSpecType, + Kind: controller.OutputShared, + }, + { + Type: network.NfTablesChainType, + Kind: controller.OutputShared, + }, + { + Type: kubespan.PeerStatusType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *ManagerController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + var ( + tickerC <-chan time.Time + ticker *time.Ticker + ) + + if ctrl.WireguardClientFactory == nil { + ctrl.WireguardClientFactory = func() (WireguardClient, error) { + return wgctrl.New() + } + } + + if ctrl.RulesManagerFactory == nil { + ctrl.RulesManagerFactory = NewRulesManager + } + + if ctrl.PeerReconcileInterval == 0 { + ctrl.PeerReconcileInterval = DefaultPeerReconcileInterval + } + + var wgClient WireguardClient + + defer func() { + if wgClient != nil { + wgClient.Close() //nolint:errcheck + } + }() + + var rulesMgr RulesManager + + defer func() { + if rulesMgr != nil { + if err := rulesMgr.Cleanup(); err != nil { + logger.Error("failed cleaning up routing rules", zap.Error(err)) + } + } + }() + + for { + var updateSpecs bool + + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + updateSpecs = true + case <-tickerC: + } + + cfg, err := r.Get(ctx, resource.NewMetadata(config.NamespaceName, kubespan.ConfigType, kubespan.ConfigID, resource.VersionUndefined)) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error getting kubespan configuration: %w", err) + } + + if cfg == nil || !cfg.(*kubespan.Config).TypedSpec().Enabled { + if ticker != nil { + ticker.Stop() + + tickerC = nil + } + + // KubeSpan is not enabled, cleanup everything + if err = ctrl.cleanup(ctx, r); err != nil { + return err + } + + if rulesMgr != nil { + if err = rulesMgr.Cleanup(); err != nil { + logger.Error("failed cleaning up routing rules", zap.Error(err)) + } + + rulesMgr = nil + } + + continue + } + + if wgClient == nil { + wgClient, err = ctrl.WireguardClientFactory() + if err != nil { + return fmt.Errorf("error creating wireguard client: %w", err) + } + } + + if ticker == nil { + ticker = time.NewTicker(ctrl.PeerReconcileInterval) + tickerC = ticker.C + } + + cfgSpec := cfg.(*kubespan.Config).TypedSpec() + + localIdentity, err := r.Get(ctx, resource.NewMetadata(kubespan.NamespaceName, kubespan.IdentityType, kubespan.LocalIdentity, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting local KubeSpan identity: %w", err) + } + + localSpec := localIdentity.(*kubespan.Identity).TypedSpec() + + // fetch PeerSpecs and PeerStatuses and sync them + peerSpecList, err := r.List(ctx, resource.NewMetadata(kubespan.NamespaceName, kubespan.PeerSpecType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing peer specs: %w", err) + } + + peerSpecs := make(map[string]*kubespan.PeerSpecSpec, len(peerSpecList.Items)) + + for _, res := range peerSpecList.Items { + peerSpecs[res.Metadata().ID()] = res.(*kubespan.PeerSpec).TypedSpec() + } + + peerStatusList, err := r.List(ctx, resource.NewMetadata(kubespan.NamespaceName, kubespan.PeerStatusType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing peer status: %w", err) + } + + peerStatuses := make(map[string]*kubespan.PeerStatusSpec, len(peerStatusList.Items)) + + for _, res := range peerStatusList.Items { + // drop any peer statuses which are not in the peer specs + if _, ok := peerSpecs[res.Metadata().ID()]; !ok { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return fmt.Errorf("error destroying peer status: %w", err) + } + + continue + } + + peerStatuses[res.Metadata().ID()] = res.(*kubespan.PeerStatus).TypedSpec() + } + + // create missing peer statuses + for pubKey, peerSpec := range peerSpecs { + if _, ok := peerStatuses[pubKey]; !ok { + peerStatuses[pubKey] = &kubespan.PeerStatusSpec{ + Label: peerSpec.Label, + } + } + } + + // update peer status from Wireguard data + wgDevice, err := wgClient.Device(constants.KubeSpanLinkName) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("error fetching wireguard link status: %w", err) + } + + if wgDevice != nil { // wgDevice might be nil if the link is not created yet + for _, peerInfo := range wgDevice.Peers { + if peerStatus, ok := peerStatuses[peerInfo.PublicKey.String()]; ok { + kubespanadapter.PeerStatusSpec(peerStatus).UpdateFromWireguard(peerInfo) + } + } + } + + // calculate peer status connection state + for _, peerStatus := range peerStatuses { + kubespanadapter.PeerStatusSpec(peerStatus).CalculateState() + } + + // build wireguard peer configuration + wgPeers := make([]network.WireguardPeer, 0, len(peerSpecs)) + + for pubKey, peerSpec := range peerSpecs { + // list of statuses and specs should be in sync at this point + peerStatus := peerStatuses[pubKey] + + var endpoint string + + // check if the endpoint should be updated + if kubespanadapter.PeerStatusSpec(peerStatus).ShouldChangeEndpoint() { + newEndpoint := kubespanadapter.PeerStatusSpec(peerStatus).PickNewEndpoint(peerSpec.Endpoints) + + if !value.IsZero(newEndpoint) { + logger.Debug("updating endpoint for the peer", zap.String("peer", pubKey), zap.String("label", peerSpec.Label), zap.Stringer("endpoint", newEndpoint)) + + endpoint = newEndpoint.String() + kubespanadapter.PeerStatusSpec(peerStatus).UpdateEndpoint(newEndpoint) + + updateSpecs = true + } + } + + // re-establish the endpoint if it wasn't applied to the Wireguard config completely + if !value.IsZero(peerStatus.LastUsedEndpoint) && (value.IsZero(peerStatus.Endpoint) || peerStatus.Endpoint == peerStatus.LastUsedEndpoint) { + endpoint = peerStatus.LastUsedEndpoint.String() + peerStatus.Endpoint = peerStatus.LastUsedEndpoint + + updateSpecs = true + } + + wgPeers = append(wgPeers, network.WireguardPeer{ + PublicKey: pubKey, + PresharedKey: cfgSpec.SharedSecret, + Endpoint: endpoint, + PersistentKeepaliveInterval: constants.KubeSpanDefaultPeerKeepalive, + AllowedIPs: slices.Clone(peerSpec.AllowedIPs), + }) + } + + // build full allowedIPs set + var allowedIPsBuilder netipx.IPSetBuilder + + for pubKey, peerSpec := range peerSpecs { + // list of statuses and specs should be in sync at this point + peerStatus := peerStatuses[pubKey] + + // add allowedIPs to the nftables set if either routing is forced (for any peer state) + // or if the peer connection state is up. + if cfgSpec.ForceRouting || peerStatus.State == kubespan.PeerStateUp { + for _, prefix := range peerSpec.AllowedIPs { + allowedIPsBuilder.AddPrefix(prefix) + } + } + } + + allowedIPsSet, err := allowedIPsBuilder.IPSet() + if err != nil { + return fmt.Errorf("failed building allowed IPs set: %w", err) + } + + // update peer statuses + for pubKey, peerStatus := range peerStatuses { + if err = safe.WriterModify(ctx, r, + kubespan.NewPeerStatus( + kubespan.NamespaceName, + pubKey, + ), + func(r *kubespan.PeerStatus) error { + *r.TypedSpec() = *peerStatus + + return nil + }, + ); err != nil { + return fmt.Errorf("error modifying peer status: %w", err) + } + } + + mtu := cfgSpec.MTU + + // always update the firewall rules, as allowedIPsSet might change at any moment due to peer up/down events + if err = safe.WriterModify(ctx, r, + network.NewNfTablesChain( + network.NamespaceName, + "kubespan_prerouting", + ), + func(r *network.NfTablesChain) error { + spec := r.TypedSpec() + + spec.Type = nethelpers.ChainTypeFilter + spec.Hook = nethelpers.ChainHookPrerouting + spec.Priority = nethelpers.ChainPriorityFilter + spec.Policy = nethelpers.VerdictAccept + + spec.Rules = []network.NfTablesRule{ + { + MatchMark: &network.NfTablesMark{ + Mask: constants.KubeSpanDefaultFirewallMask, + Value: constants.KubeSpanDefaultFirewallMark, + }, + Verdict: pointer.To(nethelpers.VerdictAccept), + }, + { + MatchDestinationAddress: &network.NfTablesAddressMatch{ + IncludeSubnets: allowedIPsSet.Prefixes(), + }, + SetMark: &network.NfTablesMark{ + Mask: ^uint32(constants.KubeSpanDefaultFirewallMask), + Xor: constants.KubeSpanDefaultForceFirewallMark, + }, + Verdict: pointer.To(nethelpers.VerdictAccept), + }, + } + + return nil + }, + ); err != nil { + return fmt.Errorf("error modifying nftables chain: %w", err) + } + + if err = safe.WriterModify(ctx, r, + network.NewNfTablesChain( + network.NamespaceName, + "kubespan_outgoing", + ), + func(r *network.NfTablesChain) error { + spec := r.TypedSpec() + + spec.Type = nethelpers.ChainTypeRoute + spec.Hook = nethelpers.ChainHookOutput + spec.Priority = nethelpers.ChainPriorityFilter + spec.Policy = nethelpers.VerdictAccept + + spec.Rules = []network.NfTablesRule{ + { + MatchMark: &network.NfTablesMark{ + Mask: constants.KubeSpanDefaultFirewallMask, + Value: constants.KubeSpanDefaultFirewallMark, + }, + Verdict: pointer.To(nethelpers.VerdictAccept), + }, + { + MatchOIfName: &network.NfTablesIfNameMatch{ + InterfaceNames: []string{"lo"}, + }, + Verdict: pointer.To(nethelpers.VerdictAccept), + }, + { + MatchDestinationAddress: &network.NfTablesAddressMatch{ + IncludeSubnets: allowedIPsSet.Prefixes(), + }, + ClampMSS: &network.NfTablesClampMSS{ + MTU: uint16(mtu), + }, + }, + { + MatchDestinationAddress: &network.NfTablesAddressMatch{ + IncludeSubnets: allowedIPsSet.Prefixes(), + }, + SetMark: &network.NfTablesMark{ + Mask: ^uint32(constants.KubeSpanDefaultFirewallMask), + Xor: constants.KubeSpanDefaultForceFirewallMark, + }, + Verdict: pointer.To(nethelpers.VerdictAccept), + }, + } + + return nil + }, + ); err != nil { + return fmt.Errorf("error modifying nftables chain: %w", err) + } + + if !updateSpecs { + // micro-optimization: skip updating specs if there are no changes to the incoming resources and no endpoint changes + r.ResetRestartBackoff() + + continue + } + + if err = safe.WriterModify(ctx, r, + network.NewAddressSpec( + network.ConfigNamespaceName, + network.LayeredID(network.ConfigOperator, network.AddressID(constants.KubeSpanLinkName, localSpec.Address)), + ), + func(r *network.AddressSpec) error { + spec := r.TypedSpec() + + spec.Address = netip.PrefixFrom(localSpec.Address.Addr(), localSpec.Subnet.Bits()) + spec.ConfigLayer = network.ConfigOperator + spec.Family = nethelpers.FamilyInet6 + spec.Flags = nethelpers.AddressFlags(nethelpers.AddressPermanent) + spec.LinkName = constants.KubeSpanLinkName + spec.Scope = nethelpers.ScopeGlobal + + return nil + }, + ); err != nil { + return fmt.Errorf("error modifying address: %w", err) + } + + for _, spec := range []network.RouteSpecSpec{ + { + Family: nethelpers.FamilyInet4, + Destination: netip.Prefix{}, + Source: netip.Addr{}, + Gateway: netip.Addr{}, + MTU: mtu, + OutLinkName: constants.KubeSpanLinkName, + Table: nethelpers.RoutingTable(constants.KubeSpanDefaultRoutingTable), + Priority: 1, + Scope: nethelpers.ScopeGlobal, + Type: nethelpers.TypeUnicast, + Flags: 0, + Protocol: nethelpers.ProtocolStatic, + ConfigLayer: network.ConfigOperator, + }, + { + Family: nethelpers.FamilyInet6, + Destination: netip.Prefix{}, + Source: netip.Addr{}, + Gateway: netip.Addr{}, + MTU: mtu, + OutLinkName: constants.KubeSpanLinkName, + Table: nethelpers.RoutingTable(constants.KubeSpanDefaultRoutingTable), + Priority: 1, + Scope: nethelpers.ScopeGlobal, + Type: nethelpers.TypeUnicast, + Flags: 0, + Protocol: nethelpers.ProtocolStatic, + ConfigLayer: network.ConfigOperator, + }, + } { + if err = safe.WriterModify(ctx, r, + network.NewRouteSpec( + network.ConfigNamespaceName, + network.LayeredID(network.ConfigOperator, network.RouteID(spec.Table, spec.Family, spec.Destination, spec.Gateway, spec.Priority, spec.OutLinkName)), + ), + func(r *network.RouteSpec) error { + *r.TypedSpec() = spec + + return nil + }, + ); err != nil { + return fmt.Errorf("error modifying route spec: %w", err) + } + } + + if err = safe.WriterModify(ctx, r, + network.NewLinkSpec( + network.ConfigNamespaceName, + network.LayeredID(network.ConfigOperator, network.LinkID(constants.KubeSpanLinkName)), + ), + func(r *network.LinkSpec) error { + spec := r.TypedSpec() + + spec.ConfigLayer = network.ConfigOperator + spec.Name = constants.KubeSpanLinkName + spec.Type = nethelpers.LinkNone + spec.Kind = "wireguard" + spec.Up = true + spec.Logical = true + spec.MTU = mtu + + spec.Wireguard = network.WireguardSpec{ + PrivateKey: localSpec.PrivateKey, + ListenPort: constants.KubeSpanDefaultPort, + FirewallMark: constants.KubeSpanDefaultFirewallMark, + Peers: wgPeers, + } + spec.Wireguard.Sort() + + return nil + }, + ); err != nil { + return fmt.Errorf("error modifying link spec: %w", err) + } + + if rulesMgr == nil { + rulesMgr = ctrl.RulesManagerFactory(constants.KubeSpanDefaultRoutingTable, constants.KubeSpanDefaultForceFirewallMark, constants.KubeSpanDefaultFirewallMask) + + if err = rulesMgr.Install(); err != nil { + return fmt.Errorf("failed setting up routing rules: %w", err) + } + } + + r.ResetRestartBackoff() + } +} + +func (ctrl *ManagerController) cleanup(ctx context.Context, r controller.Runtime) error { + for _, item := range []struct { + namespace resource.Namespace + typ resource.Type + }{ + { + namespace: network.ConfigNamespaceName, + typ: network.LinkSpecType, + }, + { + namespace: network.ConfigNamespaceName, + typ: network.AddressSpecType, + }, + { + namespace: network.ConfigNamespaceName, + typ: network.RouteSpecType, + }, + { + namespace: network.NamespaceName, + typ: network.NfTablesChainType, + }, + { + namespace: kubespan.NamespaceName, + typ: kubespan.PeerStatusType, + }, + } { + // list keys for cleanup + list, err := r.List(ctx, resource.NewMetadata(item.namespace, item.typ, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing resources: %w", err) + } + + for _, res := range list.Items { + if res.Metadata().Owner() != ctrl.Name() { + continue + } + + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return fmt.Errorf("error cleaning up resource %s: %w", res, err) + } + } + } + + return nil +} diff --git a/internal/app/machined/pkg/controllers/kubespan/manager_test.go b/internal/app/machined/pkg/controllers/kubespan/manager_test.go new file mode 100644 index 0000000..a28aac6 --- /dev/null +++ b/internal/app/machined/pkg/controllers/kubespan/manager_test.go @@ -0,0 +1,408 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. +package kubespan_test + +import ( + "net" + "net/netip" + "os" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/siderolabs/go-pointer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + kubespanadapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/kubespan" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + kubespanctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/kubespan" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/kubespan" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type ManagerSuite struct { + ctest.DefaultSuite + + mockWireguard *mockWireguardClient +} + +func (suite *ManagerSuite) TestDisabled() { + cfg := kubespan.NewConfig(config.NamespaceName, kubespan.ConfigID) + cfg.TypedSpec().Enabled = false + + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + ctest.AssertNoResource[*network.NfTablesChain](suite, "kubespan_outgoing") +} + +type mockWireguardClient struct { + deviceStateMu sync.Mutex + deviceState *wgtypes.Device +} + +func (mock *mockWireguardClient) update(newState *wgtypes.Device) { + mock.deviceStateMu.Lock() + defer mock.deviceStateMu.Unlock() + + mock.deviceState = newState +} + +func (mock *mockWireguardClient) Device(name string) (*wgtypes.Device, error) { + mock.deviceStateMu.Lock() + defer mock.deviceStateMu.Unlock() + + if mock.deviceState != nil { + return mock.deviceState, nil + } + + return nil, os.ErrNotExist +} + +func (mock *mockWireguardClient) Close() error { + return nil +} + +type mockRulesManager struct{} + +func (mock mockRulesManager) Install() error { + return nil +} + +func (mock mockRulesManager) Cleanup() error { + return nil +} + +func (suite *ManagerSuite) TestReconcile() { + cfg := kubespan.NewConfig(config.NamespaceName, kubespan.ConfigID) + cfg.TypedSpec().Enabled = true + cfg.TypedSpec().SharedSecret = "TPbGXrYlvuXgAl8dERpwjlA5tnEMoihPDPxlovcLtVg=" + cfg.TypedSpec().ForceRouting = true + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + mac, err := net.ParseMAC("ea:71:1b:b2:cc:ee") + suite.Require().NoError(err) + + localIdentity := kubespan.NewIdentity(kubespan.NamespaceName, kubespan.LocalIdentity) + suite.Require().NoError(kubespanadapter.IdentitySpec(localIdentity.TypedSpec()).GenerateKey()) + suite.Require().NoError( + kubespanadapter.IdentitySpec(localIdentity.TypedSpec()).UpdateAddress( + "v16UCWpO2iOm82n6F8dGCJ41ZXXBvDrjRDs2su7C_zs=", + mac, + ), + ) + suite.Require().NoError(suite.State().Create(suite.Ctx(), localIdentity)) + + // initial setup: link should be created without any peers + ctest.AssertResource(suite, + network.LayeredID(network.ConfigOperator, network.LinkID(constants.KubeSpanLinkName)), + func(res *network.LinkSpec, asrt *assert.Assertions) { + spec := res.TypedSpec() + + asrt.Equal(network.ConfigOperator, spec.ConfigLayer) + asrt.Equal(constants.KubeSpanLinkName, spec.Name) + asrt.Equal(nethelpers.LinkNone, spec.Type) + asrt.Equal("wireguard", spec.Kind) + asrt.True(spec.Up) + asrt.True(spec.Logical) + + asrt.Equal(localIdentity.TypedSpec().PrivateKey, spec.Wireguard.PrivateKey) + asrt.Equal(constants.KubeSpanDefaultPort, spec.Wireguard.ListenPort) + asrt.Equal(constants.KubeSpanDefaultFirewallMark, spec.Wireguard.FirewallMark) + asrt.Len(spec.Wireguard.Peers, 0) + }, + rtestutils.WithNamespace(network.ConfigNamespaceName), + ) + + ctest.AssertResource(suite, + network.LayeredID( + network.ConfigOperator, + network.AddressID(constants.KubeSpanLinkName, localIdentity.TypedSpec().Address), + ), func(res *network.AddressSpec, asrt *assert.Assertions) { + spec := res.TypedSpec() + + asrt.Equal(localIdentity.TypedSpec().Address.Addr(), spec.Address.Addr()) + asrt.Equal(localIdentity.TypedSpec().Subnet.Bits(), spec.Address.Bits()) + asrt.Equal(network.ConfigOperator, spec.ConfigLayer) + asrt.Equal(nethelpers.FamilyInet6, spec.Family) + asrt.Equal(nethelpers.AddressFlags(nethelpers.AddressPermanent), spec.Flags) + asrt.Equal(constants.KubeSpanLinkName, spec.LinkName) + asrt.Equal(nethelpers.ScopeGlobal, spec.Scope) + }, rtestutils.WithNamespace(network.ConfigNamespaceName)) + + ctest.AssertResource(suite, + network.LayeredID( + network.ConfigOperator, + network.RouteID( + constants.KubeSpanDefaultRoutingTable, + nethelpers.FamilyInet4, + netip.Prefix{}, + netip.Addr{}, + 1, + "kubespan", + ), + ), + func(res *network.RouteSpec, asrt *assert.Assertions) {}, + rtestutils.WithNamespace(network.ConfigNamespaceName), + ) + + ctest.AssertResource(suite, + network.LayeredID( + network.ConfigOperator, + network.RouteID( + constants.KubeSpanDefaultRoutingTable, + nethelpers.FamilyInet6, + netip.Prefix{}, + netip.Addr{}, + 1, + "kubespan", + ), + ), + func(res *network.RouteSpec, asrt *assert.Assertions) {}, + rtestutils.WithNamespace(network.ConfigNamespaceName), + ) + + // add two peers, they should be added to the wireguard link spec and should be tracked in peer statuses + peer1 := kubespan.NewPeerSpec(kubespan.NamespaceName, "3FxU7UuwektMjbyuJBs7i1hDj2rQA6tHnbNB6WrQxww=") + peer1.TypedSpec().Address = netip.MustParseAddr("fd8a:4396:731e:e702:145e:c4ff:fe41:1ef9") + peer1.TypedSpec().Label = "worker-1" + peer1.TypedSpec().AllowedIPs = []netip.Prefix{ + netip.MustParsePrefix("10.244.1.0/24"), + } + peer1.TypedSpec().Endpoints = []netip.AddrPort{ + netip.MustParseAddrPort("172.20.0.3:51280"), + } + suite.Require().NoError(suite.State().Create(suite.Ctx(), peer1)) + + key1, err := wgtypes.ParseKey(peer1.Metadata().ID()) + suite.Require().NoError(err) + + peer2 := kubespan.NewPeerSpec(kubespan.NamespaceName, "tQuicRD0tqCu48M+zrySTe4slT15JxWhWIboZOB4tWs=") + peer2.TypedSpec().Address = netip.MustParseAddr("fd8a:4396:731e:e702:9c83:cbff:fed0:f94b") + peer2.TypedSpec().Label = "worker-2" + peer2.TypedSpec().AllowedIPs = []netip.Prefix{ + netip.MustParsePrefix("10.244.2.0/24"), + } + peer2.TypedSpec().Endpoints = []netip.AddrPort{ + netip.MustParseAddrPort("172.20.0.4:51280"), + } + suite.Require().NoError(suite.State().Create(suite.Ctx(), peer2)) + + key2, err := wgtypes.ParseKey(peer2.Metadata().ID()) + suite.Require().NoError(err) + + ctest.AssertResource(suite, + network.LayeredID(network.ConfigOperator, network.LinkID(constants.KubeSpanLinkName)), + func(res *network.LinkSpec, asrt *assert.Assertions) { + spec := res.TypedSpec() + + asrt.Len(spec.Wireguard.Peers, 2) + + if len(spec.Wireguard.Peers) != 2 { + return + } + + for i, peer := range []*kubespan.PeerSpec{peer1, peer2} { + asrt.Equal(peer.Metadata().ID(), spec.Wireguard.Peers[i].PublicKey) + asrt.Equal(cfg.TypedSpec().SharedSecret, spec.Wireguard.Peers[i].PresharedKey) + asrt.Equal(peer.TypedSpec().AllowedIPs, spec.Wireguard.Peers[i].AllowedIPs) + asrt.Equal(peer.TypedSpec().Endpoints[0].String(), spec.Wireguard.Peers[i].Endpoint) + } + }, + rtestutils.WithNamespace(network.ConfigNamespaceName), + ) + + for _, peer := range []*kubespan.PeerSpec{peer1, peer2} { + ctest.AssertResource(suite, + peer.Metadata().ID(), + func(res *kubespan.PeerStatus, asrt *assert.Assertions) { + spec := res.TypedSpec() + + asrt.Equal(peer.TypedSpec().Label, spec.Label) + asrt.Equal(kubespan.PeerStateUnknown, spec.State) + asrt.Equal(peer.TypedSpec().Endpoints[0], spec.Endpoint) + asrt.Equal(peer.TypedSpec().Endpoints[0], spec.LastUsedEndpoint) + asrt.WithinDuration(time.Now(), spec.LastEndpointChange, 3*time.Second) + }, + ) + } + + // check firewall rules + ctest.AssertResource(suite, + "kubespan_prerouting", + func(res *network.NfTablesChain, asrt *assert.Assertions) { + spec := res.TypedSpec() + + asrt.Equal(nethelpers.ChainTypeFilter, spec.Type) + asrt.Equal(nethelpers.ChainHookPrerouting, spec.Hook) + asrt.Equal(nethelpers.ChainPriorityFilter, spec.Priority) + asrt.Equal(nethelpers.VerdictAccept, spec.Policy) + + asrt.Len(spec.Rules, 2) + + if len(spec.Rules) != 2 { + return + } + + asrt.Equal( + network.NfTablesRule{ + MatchMark: &network.NfTablesMark{ + Mask: constants.KubeSpanDefaultFirewallMask, + Value: constants.KubeSpanDefaultFirewallMark, + }, + Verdict: pointer.To(nethelpers.VerdictAccept), + }, + spec.Rules[0], + ) + + asrt.Equal( + network.NfTablesRule{ + MatchDestinationAddress: &network.NfTablesAddressMatch{ + IncludeSubnets: []netip.Prefix{ + netip.MustParsePrefix("10.244.1.0/24"), + netip.MustParsePrefix("10.244.2.0/24"), + }, + }, + SetMark: &network.NfTablesMark{ + Mask: ^uint32(constants.KubeSpanDefaultFirewallMask), + Xor: constants.KubeSpanDefaultForceFirewallMark, + }, + Verdict: pointer.To(nethelpers.VerdictAccept), + }, + spec.Rules[1], + ) + }, + ) + + // update config and disable force routing, nothing should be routed + cfg.TypedSpec().ForceRouting = false + suite.Require().NoError(suite.State().Update(suite.Ctx(), cfg)) + + ctest.AssertResource(suite, + "kubespan_prerouting", + func(res *network.NfTablesChain, asrt *assert.Assertions) { + spec := res.TypedSpec() + + asrt.Equal( + network.NfTablesRule{ + MatchDestinationAddress: &network.NfTablesAddressMatch{ + IncludeSubnets: []netip.Prefix{}, + }, + SetMark: &network.NfTablesMark{ + Mask: ^uint32(constants.KubeSpanDefaultFirewallMask), + Xor: constants.KubeSpanDefaultForceFirewallMark, + }, + Verdict: pointer.To(nethelpers.VerdictAccept), + }, + spec.Rules[1], + ) + }, + ) + + // report up status via wireguard mock + suite.mockWireguard.update( + &wgtypes.Device{ + Peers: []wgtypes.Peer{ + { + PublicKey: key1, + Endpoint: asUDP(peer1.TypedSpec().Endpoints[0]), + LastHandshakeTime: time.Now(), + }, + { + PublicKey: key2, + Endpoint: asUDP(peer2.TypedSpec().Endpoints[0]), + LastHandshakeTime: time.Now(), + }, + }, + }, + ) + + for _, peer := range []*kubespan.PeerSpec{peer1, peer2} { + ctest.AssertResource(suite, + peer.Metadata().ID(), + func(res *kubespan.PeerStatus, asrt *assert.Assertions) { + spec := res.TypedSpec() + + asrt.Equal(kubespan.PeerStateUp, spec.State) + }, + ) + } + + ctest.AssertResource(suite, + "kubespan_prerouting", + func(res *network.NfTablesChain, asrt *assert.Assertions) { + spec := res.TypedSpec() + + asrt.Equal( + network.NfTablesRule{ + MatchDestinationAddress: &network.NfTablesAddressMatch{ + IncludeSubnets: []netip.Prefix{ + netip.MustParsePrefix("10.244.1.0/24"), + netip.MustParsePrefix("10.244.2.0/24"), + }, + }, + SetMark: &network.NfTablesMark{ + Mask: ^uint32(constants.KubeSpanDefaultFirewallMask), + Xor: constants.KubeSpanDefaultForceFirewallMark, + }, + Verdict: pointer.To(nethelpers.VerdictAccept), + }, + spec.Rules[1], + ) + }, + ) + + // update config and disable wireguard, everything should be cleaned up + cfg.TypedSpec().Enabled = false + suite.Require().NoError(suite.State().Update(suite.Ctx(), cfg)) + + ctest.AssertNoResource[*network.LinkSpec]( + suite, + network.LayeredID(network.ConfigOperator, network.LinkID(constants.KubeSpanLinkName)), + rtestutils.WithNamespace(network.ConfigNamespaceName), + ) + ctest.AssertNoResource[*network.NfTablesChain]( + suite, + "kubespan_prerouting", + ) +} + +func asUDP(addr netip.AddrPort) *net.UDPAddr { + return &net.UDPAddr{ + IP: addr.Addr().AsSlice(), + Port: int(addr.Port()), + Zone: addr.Addr().Zone(), + } +} + +func TestManagerSuite(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("requires root") + } + + mockWireguard := &mockWireguardClient{} + + suite.Run(t, &ManagerSuite{ + mockWireguard: mockWireguard, + DefaultSuite: ctest.DefaultSuite{ + Timeout: 5 * time.Second, + AfterSetup: func(s *ctest.DefaultSuite) { + s.Require().NoError(s.Runtime().RegisterController(&kubespanctrl.ManagerController{ + WireguardClientFactory: func() (kubespanctrl.WireguardClient, error) { + return mockWireguard, nil + }, + RulesManagerFactory: func(_, _, _ int) kubespanctrl.RulesManager { + return mockRulesManager{} + }, + PeerReconcileInterval: time.Second, + })) + }, + }, + }) +} diff --git a/internal/app/machined/pkg/controllers/kubespan/peer_spec.go b/internal/app/machined/pkg/controllers/kubespan/peer_spec.go new file mode 100644 index 0000000..07d5960 --- /dev/null +++ b/internal/app/machined/pkg/controllers/kubespan/peer_spec.go @@ -0,0 +1,209 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package kubespan + +import ( + "context" + "fmt" + "slices" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/xslices" + "go.uber.org/zap" + "go4.org/netipx" + + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/kubespan" +) + +// PeerSpecController watches cluster.Affiliates updates PeerSpec. +type PeerSpecController struct{} + +// Name implements controller.Controller interface. +func (ctrl *PeerSpecController) Name() string { + return "kubespan.PeerSpecController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *PeerSpecController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: kubespan.ConfigType, + ID: optional.Some(kubespan.ConfigID), + Kind: controller.InputWeak, + }, + { + Namespace: cluster.NamespaceName, + Type: cluster.AffiliateType, + Kind: controller.InputWeak, + }, + { + Namespace: cluster.NamespaceName, + Type: cluster.IdentityType, + ID: optional.Some(cluster.LocalIdentity), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *PeerSpecController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: kubespan.PeerSpecType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *PeerSpecController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + cfg, err := safe.ReaderGetByID[*kubespan.Config](ctx, r, kubespan.ConfigID) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error getting kubespan configuration: %w", err) + } + + localIdentity, err := safe.ReaderGetByID[*cluster.Identity](ctx, r, cluster.LocalIdentity) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error getting first MAC address: %w", err) + } + + affiliates, err := safe.ReaderListAll[*cluster.Affiliate](ctx, r) + if err != nil { + return fmt.Errorf("error listing cluster affiliates: %w", err) + } + + touchedIDs := map[resource.ID]struct{}{} + + if cfg != nil && localIdentity != nil && cfg.TypedSpec().Enabled { + localAffiliateID := localIdentity.TypedSpec().NodeID + + peerIPSets := make(map[string]*netipx.IPSet, affiliates.Len()) + + affiliateLoop: + for it := affiliates.Iterator(); it.Next(); { + affiliate := it.Value() + + if affiliate.Metadata().ID() == localAffiliateID { + // skip local affiliate, it's not a peer + continue + } + + spec := affiliate.TypedSpec() + + if spec.KubeSpan.PublicKey == "" { + // no kubespan information, skip it + continue + } + + var builder netipx.IPSetBuilder + + for _, ipPrefix := range spec.KubeSpan.AdditionalAddresses { + builder.AddPrefix(ipPrefix) + } + + for _, ip := range spec.Addresses { + builder.Add(ip) + } + + builder.Add(spec.KubeSpan.Address) + + var ipSet *netipx.IPSet + + ipSet, err = builder.IPSet() + if err != nil { + logger.Warn("failed building list of IP ranges for the peer", zap.String("ignored_peer", spec.KubeSpan.PublicKey), zap.String("label", spec.Nodename), zap.Error(err)) + + continue + } + + for otherPublicKey, otherIPSet := range peerIPSets { + if otherIPSet.Overlaps(ipSet) { + logger.Warn("peer address overlap", zap.String("this_peer", spec.KubeSpan.PublicKey), zap.String("other_peer", otherPublicKey), + zap.Strings("this_ips", dumpSet(ipSet)), zap.Strings("other_ips", dumpSet(otherIPSet))) + + // exclude overlapping IPs from the ipSet + var bldr netipx.IPSetBuilder + + // ipSet = ipSet & ~otherIPSet + bldr.AddSet(otherIPSet) + bldr.Complement() + bldr.Intersect(ipSet) + + ipSet, err = bldr.IPSet() + if err != nil { + logger.Warn("failed building list of IP ranges for the peer", zap.String("ignored_peer", spec.KubeSpan.PublicKey), zap.String("label", spec.Nodename), zap.Error(err)) + + continue affiliateLoop + } + + if len(ipSet.Ranges()) == 0 { + logger.Warn("conflict resolution removed all ranges", zap.String("this_peer", spec.KubeSpan.PublicKey), zap.String("other_peer", otherPublicKey)) + } + } + } + + peerIPSets[spec.KubeSpan.PublicKey] = ipSet + + if err = safe.WriterModify(ctx, r, kubespan.NewPeerSpec(kubespan.NamespaceName, spec.KubeSpan.PublicKey), func(res *kubespan.PeerSpec) error { + *res.TypedSpec() = kubespan.PeerSpecSpec{ + Address: spec.KubeSpan.Address, + AllowedIPs: ipSet.Prefixes(), + Endpoints: slices.Clone(spec.KubeSpan.Endpoints), + Label: spec.Nodename, + } + + return nil + }); err != nil { + return err + } + + touchedIDs[spec.KubeSpan.PublicKey] = struct{}{} + } + } + + // list keys for cleanup + list, err := safe.ReaderListAll[*kubespan.PeerSpec](ctx, r) + if err != nil { + return fmt.Errorf("error listing resources: %w", err) + } + + for it := list.Iterator(); it.Next(); { + res := it.Value() + + if res.Metadata().Owner() != ctrl.Name() { + continue + } + + if _, ok := touchedIDs[res.Metadata().ID()]; !ok { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return fmt.Errorf("error cleaning up specs: %w", err) + } + } + } + + r.ResetRestartBackoff() + } +} + +// dumpSet converts IPSet to a form suitable for logging. +func dumpSet(set *netipx.IPSet) []string { + return xslices.Map(set.Ranges(), netipx.IPRange.String) +} diff --git a/internal/app/machined/pkg/controllers/kubespan/peer_spec_test.go b/internal/app/machined/pkg/controllers/kubespan/peer_spec_test.go new file mode 100644 index 0000000..91662d8 --- /dev/null +++ b/internal/app/machined/pkg/controllers/kubespan/peer_spec_test.go @@ -0,0 +1,257 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. +package kubespan_test + +import ( + "fmt" + "net/netip" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + clusteradapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/cluster" + kubespanctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/kubespan" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/kubespan" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +type PeerSpecSuite struct { + KubeSpanSuite + + statePath string +} + +func (suite *PeerSpecSuite) TestReconcile() { + suite.statePath = suite.T().TempDir() + + suite.Require().NoError(suite.runtime.RegisterController(&kubespanctrl.PeerSpecController{})) + + suite.startRuntime() + + stateMount := runtimeres.NewMountStatus(v1alpha1.NamespaceName, constants.StatePartitionLabel) + + suite.Assert().NoError(suite.state.Create(suite.ctx, stateMount)) + + cfg := kubespan.NewConfig(config.NamespaceName, kubespan.ConfigID) + cfg.TypedSpec().Enabled = true + + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + nodeIdentity := cluster.NewIdentity(cluster.NamespaceName, cluster.LocalIdentity) + suite.Require().NoError(clusteradapter.IdentitySpec(nodeIdentity.TypedSpec()).Generate()) + suite.Require().NoError(suite.state.Create(suite.ctx, nodeIdentity)) + + affiliate1 := cluster.NewAffiliate(cluster.NamespaceName, "7x1SuC8Ege5BGXdAfTEff5iQnlWZLfv9h1LGMxA2pYkC") + *affiliate1.TypedSpec() = cluster.AffiliateSpec{ + NodeID: "7x1SuC8Ege5BGXdAfTEff5iQnlWZLfv9h1LGMxA2pYkC", + Hostname: "foo.com", + Nodename: "bar", + MachineType: machine.TypeControlPlane, + Addresses: []netip.Addr{netip.MustParseAddr("192.168.3.4")}, + KubeSpan: cluster.KubeSpanAffiliateSpec{ + PublicKey: "PLPNBddmTgHJhtw0vxltq1ZBdPP9RNOEUd5JjJZzBRY=", + Address: netip.MustParseAddr("fd50:8d60:4238:6302:f857:23ff:fe21:d1e0"), + AdditionalAddresses: []netip.Prefix{netip.MustParsePrefix("10.244.3.1/24"), netip.MustParsePrefix("10.244.3.0/32")}, + Endpoints: []netip.AddrPort{netip.MustParseAddrPort("10.0.0.2:51820"), netip.MustParseAddrPort("192.168.3.4:51820")}, + }, + } + + affiliate2 := cluster.NewAffiliate(cluster.NamespaceName, "9dwHNUViZlPlIervqX9Qo256RUhrfhgO0xBBnKcKl4F") + *affiliate2.TypedSpec() = cluster.AffiliateSpec{ + NodeID: "9dwHNUViZlPlIervqX9Qo256RUhrfhgO0xBBnKcKl4F", + Hostname: "worker-1", + Nodename: "worker-1", + MachineType: machine.TypeWorker, + Addresses: []netip.Addr{netip.MustParseAddr("192.168.3.5")}, + } + + affiliate3 := cluster.NewAffiliate(cluster.NamespaceName, "xCnFFfxylOf9i5ynhAkt6ZbfcqaLDGKfIa3gwpuaxe7F") + *affiliate3.TypedSpec() = cluster.AffiliateSpec{ + NodeID: "xCnFFfxylOf9i5ynhAkt6ZbfcqaLDGKfIa3gwpuaxe7F", + MachineType: machine.TypeWorker, + Nodename: "worker-2", + Addresses: []netip.Addr{netip.MustParseAddr("192.168.3.6")}, + KubeSpan: cluster.KubeSpanAffiliateSpec{ + PublicKey: "mB6WlFOR66Jx5rtPMIpxJ3s4XHyer9NCzqWPP7idGRo", + Address: netip.MustParseAddr("fdc8:8aee:4e2d:1202:f073:9cff:fe6c:4d67"), + AdditionalAddresses: []netip.Prefix{netip.MustParsePrefix("10.244.4.1/24")}, + Endpoints: []netip.AddrPort{netip.MustParseAddrPort("192.168.3.6:51820")}, + }, + } + + // local node affiliate, should be skipped as a peer + affiliate4 := cluster.NewAffiliate(cluster.NamespaceName, nodeIdentity.TypedSpec().NodeID) + *affiliate4.TypedSpec() = cluster.AffiliateSpec{ + NodeID: nodeIdentity.TypedSpec().NodeID, + MachineType: machine.TypeWorker, + Addresses: []netip.Addr{netip.MustParseAddr("192.168.3.7")}, + KubeSpan: cluster.KubeSpanAffiliateSpec{ + PublicKey: "27E8I+ekrqT21cq2iW6+fDe+H7WBw6q9J7vqLCeswiM=", + Address: netip.MustParseAddr("fdc8:8aee:4e2d:1202:f073:9cff:fe6c:4d67"), + AdditionalAddresses: []netip.Prefix{netip.MustParsePrefix("10.244.5.1/24")}, + Endpoints: []netip.AddrPort{netip.MustParseAddrPort("192.168.3.7:51820")}, + }, + } + + for _, r := range []resource.Resource{affiliate1, affiliate2, affiliate3, affiliate4} { + suite.Require().NoError(suite.state.Create(suite.ctx, r)) + } + + // affiliate2 shouldn't be rendered as a peer, as it doesn't have kubespan data + suite.Assert().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertResourceIDs(resource.NewMetadata(kubespan.NamespaceName, kubespan.PeerSpecType, "", resource.VersionUndefined), + []resource.ID{ + affiliate1.TypedSpec().KubeSpan.PublicKey, + affiliate3.TypedSpec().KubeSpan.PublicKey, + }, + ), + )) + + suite.Assert().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertResource( + resource.NewMetadata(kubespan.NamespaceName, kubespan.PeerSpecType, affiliate1.TypedSpec().KubeSpan.PublicKey, resource.VersionUndefined), + func(res resource.Resource) error { + spec := res.(*kubespan.PeerSpec).TypedSpec() + + suite.Assert().Equal("fd50:8d60:4238:6302:f857:23ff:fe21:d1e0", spec.Address.String()) + suite.Assert().Equal("[10.244.3.0/24 192.168.3.4/32 fd50:8d60:4238:6302:f857:23ff:fe21:d1e0/128]", fmt.Sprintf("%v", spec.AllowedIPs)) + suite.Assert().Equal([]netip.AddrPort{netip.MustParseAddrPort("10.0.0.2:51820"), netip.MustParseAddrPort("192.168.3.4:51820")}, spec.Endpoints) + suite.Assert().Equal("bar", spec.Label) + + return nil + }, + ), + )) + + suite.Assert().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertResource( + resource.NewMetadata(kubespan.NamespaceName, kubespan.PeerSpecType, affiliate3.TypedSpec().KubeSpan.PublicKey, resource.VersionUndefined), + func(res resource.Resource) error { + spec := res.(*kubespan.PeerSpec).TypedSpec() + + suite.Assert().Equal("fdc8:8aee:4e2d:1202:f073:9cff:fe6c:4d67", spec.Address.String()) + suite.Assert().Equal("[10.244.4.0/24 192.168.3.6/32 fdc8:8aee:4e2d:1202:f073:9cff:fe6c:4d67/128]", fmt.Sprintf("%v", spec.AllowedIPs)) + suite.Assert().Equal([]netip.AddrPort{netip.MustParseAddrPort("192.168.3.6:51820")}, spec.Endpoints) + suite.Assert().Equal("worker-2", spec.Label) + + return nil + }, + ), + )) + + // disabling kubespan should remove all peers + cfg.TypedSpec().Enabled = false + suite.Require().NoError(suite.state.Update(suite.ctx, cfg)) + + suite.Assert().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertNoResource( + resource.NewMetadata(kubespan.NamespaceName, kubespan.PeerSpecType, affiliate1.TypedSpec().KubeSpan.PublicKey, resource.VersionUndefined), + ), + )) + suite.Assert().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertNoResource( + resource.NewMetadata(kubespan.NamespaceName, kubespan.PeerSpecType, affiliate3.TypedSpec().KubeSpan.PublicKey, resource.VersionUndefined), + ), + )) +} + +func (suite *PeerSpecSuite) TestIPOverlap() { + suite.statePath = suite.T().TempDir() + + suite.Require().NoError(suite.runtime.RegisterController(&kubespanctrl.PeerSpecController{})) + + suite.startRuntime() + + stateMount := runtimeres.NewMountStatus(v1alpha1.NamespaceName, constants.StatePartitionLabel) + + suite.Assert().NoError(suite.state.Create(suite.ctx, stateMount)) + + cfg := kubespan.NewConfig(config.NamespaceName, kubespan.ConfigID) + cfg.TypedSpec().Enabled = true + + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + nodeIdentity := cluster.NewIdentity(cluster.NamespaceName, cluster.LocalIdentity) + suite.Require().NoError(clusteradapter.IdentitySpec(nodeIdentity.TypedSpec()).Generate()) + suite.Require().NoError(suite.state.Create(suite.ctx, nodeIdentity)) + + affiliate1 := cluster.NewAffiliate(cluster.NamespaceName, "7x1SuC8Ege5BGXdAfTEff5iQnlWZLfv9h1LGMxA2pYkC") + *affiliate1.TypedSpec() = cluster.AffiliateSpec{ + NodeID: "7x1SuC8Ege5BGXdAfTEff5iQnlWZLfv9h1LGMxA2pYkC", + Nodename: "bar", + MachineType: machine.TypeControlPlane, + KubeSpan: cluster.KubeSpanAffiliateSpec{ + PublicKey: "PLPNBddmTgHJhtw0vxltq1ZBdPP9RNOEUd5JjJZzBRY=", + Address: netip.MustParseAddr("fd50:8d60:4238:6302:f857:23ff:fe21:d1e0"), + AdditionalAddresses: []netip.Prefix{netip.MustParsePrefix("10.244.3.1/24"), netip.MustParsePrefix("10.244.3.0/32")}, + Endpoints: []netip.AddrPort{netip.MustParseAddrPort("10.0.0.2:51820"), netip.MustParseAddrPort("192.168.3.4:51820")}, + }, + } + + affiliate2 := cluster.NewAffiliate(cluster.NamespaceName, "9dwHNUViZlPlIervqX9Qo256RUhrfhgO0xBBnKcKl4F") + *affiliate2.TypedSpec() = cluster.AffiliateSpec{ + NodeID: "9dwHNUViZlPlIervqX9Qo256RUhrfhgO0xBBnKcKl4F", + Hostname: "worker-1", + Nodename: "worker-1", + MachineType: machine.TypeWorker, + KubeSpan: cluster.KubeSpanAffiliateSpec{ + PublicKey: "Zr5ewpUm2Ywo1c+/59WFKIBjZ3c/nVbIWsT5elbjwCU=", + Address: netip.MustParseAddr("fd50:8d60:4238:6302:f857:23ff:fe21:d1e1"), + AdditionalAddresses: []netip.Prefix{netip.MustParsePrefix("10.244.2.0/23"), netip.MustParsePrefix("192.168.3.0/24")}, + Endpoints: []netip.AddrPort{netip.MustParseAddrPort("10.0.0.2:51820"), netip.MustParseAddrPort("192.168.3.4:51820")}, + }, + } + + for _, r := range []resource.Resource{affiliate1, affiliate2} { + suite.Require().NoError(suite.state.Create(suite.ctx, r)) + } + + // affiliate2 should be rendered as a peer, but with reduced address as its AdditionalAddresses overlap with affiliate1 addresses + suite.Assert().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertResourceIDs(resource.NewMetadata(kubespan.NamespaceName, kubespan.PeerSpecType, "", resource.VersionUndefined), + []resource.ID{ + affiliate1.TypedSpec().KubeSpan.PublicKey, + affiliate2.TypedSpec().KubeSpan.PublicKey, + }, + ), + )) + + suite.Assert().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertResource(resource.NewMetadata(kubespan.NamespaceName, kubespan.PeerSpecType, affiliate1.TypedSpec().KubeSpan.PublicKey, resource.VersionUndefined), + func(res resource.Resource) error { + spec := res.(*kubespan.PeerSpec).TypedSpec() + + suite.Assert().Equal(`["10.244.3.0/24" "fd50:8d60:4238:6302:f857:23ff:fe21:d1e0/128"]`, fmt.Sprintf("%q", spec.AllowedIPs)) + + return nil + }, + ), + )) + + suite.Assert().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertResource(resource.NewMetadata(kubespan.NamespaceName, kubespan.PeerSpecType, affiliate2.TypedSpec().KubeSpan.PublicKey, resource.VersionUndefined), + func(res resource.Resource) error { + spec := res.(*kubespan.PeerSpec).TypedSpec() + + suite.Assert().Equal(`["10.244.2.0/24" "192.168.3.0/24" "fd50:8d60:4238:6302:f857:23ff:fe21:d1e1/128"]`, fmt.Sprintf("%q", spec.AllowedIPs)) + + return nil + }, + ), + )) +} + +func TestPeerSpecSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, new(PeerSpecSuite)) +} diff --git a/internal/app/machined/pkg/controllers/kubespan/routing_rules.go b/internal/app/machined/pkg/controllers/kubespan/routing_rules.go new file mode 100644 index 0000000..09d0d24 --- /dev/null +++ b/internal/app/machined/pkg/controllers/kubespan/routing_rules.go @@ -0,0 +1,155 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package kubespan + +import ( + "errors" + "fmt" + "os" + + "github.com/hashicorp/go-multierror" + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" +) + +// RulesManager manages routing rules outside of controllers/resources scope. +// +// TODO: this might be refactored later to support routing rules in the native network resources. +type RulesManager interface { + Install() error + Cleanup() error +} + +// NewRulesManager initializes new RulesManager. +func NewRulesManager(targetTable, internalMark, markMask int) RulesManager { + return &rulesManager{ + TargetTable: targetTable, + InternalMark: internalMark, + MarkMask: markMask, + } +} + +type rulesManager struct { + TargetTable int + InternalMark int + MarkMask int +} + +// Install routing rules. +func (m *rulesManager) Install() error { + nc, err := netlink.NewHandle() + if err != nil { + return fmt.Errorf("failed to get netlink handle: %w", err) + } + + defer nc.Close() + + if err := nc.RuleAdd(&netlink.Rule{ + Priority: nextRuleNumber(nc, unix.AF_INET), + Family: unix.AF_INET, + Table: m.TargetTable, + Mark: m.InternalMark, + Mask: m.MarkMask, + Goto: -1, + Flow: -1, + SuppressIfgroup: -1, + SuppressPrefixlen: -1, + }); err != nil { + if !errors.Is(err, os.ErrExist) { + return fmt.Errorf("failed to add IPv4 table-mark rule: %w", err) + } + } + + if err := nc.RuleAdd(&netlink.Rule{ + Priority: nextRuleNumber(nc, unix.AF_INET6), + Family: unix.AF_INET6, + Table: m.TargetTable, + Mark: m.InternalMark, + Mask: m.MarkMask, + Goto: -1, + Flow: -1, + SuppressIfgroup: -1, + SuppressPrefixlen: -1, + }); err != nil { + if !errors.Is(err, os.ErrExist) { + return fmt.Errorf("failed to add IPv6 table-mark rule: %w", err) + } + } + + return nil +} + +func (m *rulesManager) deleteRulesFamily(nc *netlink.Handle, family int) error { + var merr *multierror.Error + + list, err := nc.RuleList(family) + if err != nil { + merr = multierror.Append(merr, fmt.Errorf("failed to get route rules: %w", err)) + } + + for _, r := range list { + if r.Table == m.TargetTable && + r.Mark == m.InternalMark { + thisRule := r + + if err := nc.RuleDel(&thisRule); err != nil { + if !errors.Is(err, os.ErrNotExist) { + merr = multierror.Append(merr, err) + } + } + + break + } + } + + return merr.ErrorOrNil() +} + +// Cleanup the installed routing rules. +func (m *rulesManager) Cleanup() error { + var merr *multierror.Error + + nc, err := netlink.NewHandle() + if err != nil { + return fmt.Errorf("failed to get netlink handle: %w", err) + } + + defer nc.Close() + + if err = m.deleteRulesFamily(nc, unix.AF_INET); err != nil { + merr = multierror.Append(merr, fmt.Errorf("failed to delete all IPv4 route rules: %w", err)) + } + + if err = m.deleteRulesFamily(nc, unix.AF_INET6); err != nil { + merr = multierror.Append(merr, fmt.Errorf("failed to delete all IPv6 route rules: %w", err)) + } + + return merr.ErrorOrNil() +} + +func nextRuleNumber(nc *netlink.Handle, family int) int { + list, err := nc.RuleList(family) + if err != nil { + return 0 + } + + for i := 32500; i > 0; i-- { + var found bool + + for _, r := range list { + if r.Priority == i { + found = true + + break + } + } + + if !found { + return i + } + } + + return 0 +} diff --git a/internal/app/machined/pkg/controllers/kubespan/routing_rules_test.go b/internal/app/machined/pkg/controllers/kubespan/routing_rules_test.go new file mode 100644 index 0000000..eda6877 --- /dev/null +++ b/internal/app/machined/pkg/controllers/kubespan/routing_rules_test.go @@ -0,0 +1,31 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. +package kubespan_test + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/kubespan" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +func TestRoutingRules(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("requires root") + } + + // use a different table/mark to avoid conflicts with running kubespan + mgr := kubespan.NewRulesManager(constants.KubeSpanDefaultRoutingTable+10, constants.KubeSpanDefaultForceFirewallMark<<1, constants.KubeSpanDefaultFirewallMask<<1) + + // cleanup should be fine if nothing is installed + assert.NoError(t, mgr.Cleanup()) + + defer mgr.Cleanup() //nolint:errcheck + + assert.NoError(t, mgr.Install()) + assert.NoError(t, mgr.Cleanup()) +} diff --git a/internal/app/machined/pkg/controllers/network/address_config.go b/internal/app/machined/pkg/controllers/network/address_config.go new file mode 100644 index 0000000..f9d67a0 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/address_config.go @@ -0,0 +1,312 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "context" + "fmt" + "net/netip" + "strings" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/value" + "github.com/siderolabs/go-procfs/procfs" + "go.uber.org/zap" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/config/config" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// AddressConfigController manages network.AddressSpec based on machine configuration, kernel cmdline and some built-in defaults. +type AddressConfigController struct { + Cmdline *procfs.Cmdline + V1Alpha1Mode runtime.Mode +} + +// Name implements controller.Controller interface. +func (ctrl *AddressConfigController) Name() string { + return "network.AddressConfigController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *AddressConfigController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: network.NamespaceName, + Type: network.DeviceConfigSpecType, + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *AddressConfigController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.AddressSpecType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *AddressConfigController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + // apply defaults for the loopback interface once + defaultTouchedIDs, err := ctrl.apply(ctx, r, ctrl.loopbackDefaults()) + if err != nil { + return fmt.Errorf("error generating loopback interface defaults: %w", err) + } + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + touchedIDs := make(map[resource.ID]struct{}) + + for _, id := range defaultTouchedIDs { + touchedIDs[id] = struct{}{} + } + + items, err := r.List(ctx, resource.NewMetadata(network.NamespaceName, network.DeviceConfigSpecType, "", resource.VersionUndefined)) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting config: %w", err) + } + } + + ignoredInterfaces := map[string]struct{}{} + + devices := make([]config.Device, len(items.Items)) + + for i, item := range items.Items { + device := item.(*network.DeviceConfigSpec).TypedSpec().Device + + devices[i] = device + + if device.Ignore() { + ignoredInterfaces[device.Interface()] = struct{}{} + } + } + + // parse kernel cmdline for the address + cmdlineAddresses := ctrl.parseCmdline(logger) + for _, cmdlineAddress := range cmdlineAddresses { + if _, ignored := ignoredInterfaces[cmdlineAddress.LinkName]; !ignored { + var ids []string + + ids, err = ctrl.apply(ctx, r, []network.AddressSpecSpec{cmdlineAddress}) + if err != nil { + return fmt.Errorf("error applying cmdline address: %w", err) + } + + for _, id := range ids { + touchedIDs[id] = struct{}{} + } + } + } + + // parse machine configuration for static addresses + if len(devices) > 0 { + addresses := ctrl.processDevicesConfiguration(logger, devices) + + var ids []string + + ids, err = ctrl.apply(ctx, r, addresses) + if err != nil { + return fmt.Errorf("error applying machine configuration address: %w", err) + } + + for _, id := range ids { + touchedIDs[id] = struct{}{} + } + } + + // list addresses for cleanup + list, err := r.List(ctx, resource.NewMetadata(network.ConfigNamespaceName, network.AddressSpecType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing resources: %w", err) + } + + for _, res := range list.Items { + if res.Metadata().Owner() != ctrl.Name() { + // skip specs created by other controllers + continue + } + + if _, ok := touchedIDs[res.Metadata().ID()]; !ok { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return fmt.Errorf("error cleaning up addresses: %w", err) + } + } + } + + r.ResetRestartBackoff() + } +} + +//nolint:dupl +func (ctrl *AddressConfigController) apply(ctx context.Context, r controller.Runtime, addresses []network.AddressSpecSpec) ([]resource.ID, error) { + ids := make([]string, 0, len(addresses)) + + for _, address := range addresses { + id := network.LayeredID(address.ConfigLayer, network.AddressID(address.LinkName, address.Address)) + + if err := r.Modify( + ctx, + network.NewAddressSpec(network.ConfigNamespaceName, id), + func(r resource.Resource) error { + *r.(*network.AddressSpec).TypedSpec() = address + + return nil + }, + ); err != nil { + return ids, err + } + + ids = append(ids, id) + } + + return ids, nil +} + +func (ctrl *AddressConfigController) loopbackDefaults() []network.AddressSpecSpec { + if ctrl.V1Alpha1Mode == runtime.ModeContainer { + // skip configuring lo addresses in container mode + return nil + } + + return []network.AddressSpecSpec{ + { + Address: netip.PrefixFrom(netip.AddrFrom4([4]byte{127, 0, 0, 1}), 8), + Family: nethelpers.FamilyInet4, + Scope: nethelpers.ScopeHost, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + LinkName: "lo", + ConfigLayer: network.ConfigDefault, + }, + } +} + +func (ctrl *AddressConfigController) parseCmdline(logger *zap.Logger) (addresses []network.AddressSpecSpec) { + if ctrl.Cmdline == nil { + return + } + + settings, err := ParseCmdlineNetwork(ctrl.Cmdline) + if err != nil { + logger.Info("ignoring cmdline parse failure", zap.Error(err)) + + return + } + + for _, linkConfig := range settings.LinkConfigs { + if value.IsZero(linkConfig.Address) { + continue + } + + var address network.AddressSpecSpec + + address.Address = linkConfig.Address + if address.Address.Addr().Is6() { + address.Family = nethelpers.FamilyInet6 + } else { + address.Family = nethelpers.FamilyInet4 + } + + address.Scope = nethelpers.ScopeGlobal + address.Flags = nethelpers.AddressFlags(nethelpers.AddressPermanent) + address.ConfigLayer = network.ConfigCmdline + address.LinkName = linkConfig.LinkName + + addresses = append(addresses, address) + } + + return addresses +} + +func parseIPOrIPPrefix(address string) (netip.Prefix, error) { + if strings.IndexByte(address, '/') >= 0 { + return netip.ParsePrefix(address) + } + + // parse as IP address and assume netmask of all ones + ip, err := netip.ParseAddr(address) + if err != nil { + return netip.Prefix{}, err + } + + return netip.PrefixFrom(ip, ip.BitLen()), nil +} + +func (ctrl *AddressConfigController) processDevicesConfiguration(logger *zap.Logger, devices []config.Device) (addresses []network.AddressSpecSpec) { + for _, device := range devices { + if device.Ignore() { + continue + } + + for _, cidr := range device.Addresses() { + ipPrefix, err := parseIPOrIPPrefix(cidr) + if err != nil { + logger.Info(fmt.Sprintf("skipping address %q on interface %q", cidr, device.Interface()), zap.Error(err)) + + continue + } + + address := network.AddressSpecSpec{ + Address: ipPrefix, + Scope: nethelpers.ScopeGlobal, + LinkName: device.Interface(), + ConfigLayer: network.ConfigMachineConfiguration, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + } + + if address.Address.Addr().Is6() { + address.Family = nethelpers.FamilyInet6 + } else { + address.Family = nethelpers.FamilyInet4 + } + + addresses = append(addresses, address) + } + + for _, vlan := range device.Vlans() { + for _, cidr := range vlan.Addresses() { + ipPrefix, err := netip.ParsePrefix(cidr) + if err != nil { + logger.Info(fmt.Sprintf("skipping address %q on interface %q vlan %d", cidr, device.Interface(), vlan.ID()), zap.Error(err)) + + continue + } + + address := network.AddressSpecSpec{ + Address: ipPrefix, + Scope: nethelpers.ScopeGlobal, + LinkName: nethelpers.VLANLinkName(device.Interface(), vlan.ID()), + ConfigLayer: network.ConfigMachineConfiguration, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + } + + if address.Address.Addr().Is6() { + address.Family = nethelpers.FamilyInet6 + } else { + address.Family = nethelpers.FamilyInet4 + } + + addresses = append(addresses, address) + } + } + } + + return addresses +} diff --git a/internal/app/machined/pkg/controllers/network/address_config_test.go b/internal/app/machined/pkg/controllers/network/address_config_test.go new file mode 100644 index 0000000..a0b2c94 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/address_config_test.go @@ -0,0 +1,249 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network_test + +import ( + "context" + "fmt" + "log" + "net" + "net/url" + "sort" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-pointer" + "github.com/siderolabs/go-procfs/procfs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type AddressConfigSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *AddressConfigSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.DeviceConfigController{})) +} + +func (suite *AddressConfigSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *AddressConfigSuite) assertAddresses(requiredIDs []string, check func(*network.AddressSpec, *assert.Assertions)) { + assertResources(suite.ctx, suite.T(), suite.state, requiredIDs, check, rtestutils.WithNamespace(network.ConfigNamespaceName)) +} + +func (suite *AddressConfigSuite) TestLoopback() { + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.AddressConfigController{})) + + suite.startRuntime() + + suite.assertAddresses( + []string{ + "default/lo/127.0.0.1/8", + }, func(r *network.AddressSpec, asrt *assert.Assertions) { + asrt.Equal("lo", r.TypedSpec().LinkName) + asrt.Equal(nethelpers.ScopeHost, r.TypedSpec().Scope) + }, + ) +} + +func (suite *AddressConfigSuite) TestCmdline() { + suite.Require().NoError( + suite.runtime.RegisterController( + &netctrl.AddressConfigController{ + Cmdline: procfs.NewCmdline("ip=172.20.0.2::172.20.0.1:255.255.255.0::eth1::::: ip=eth3:dhcp ip=10.3.5.7::10.3.5.1:255.255.255.0::eth4"), + }, + ), + ) + + suite.startRuntime() + + suite.assertAddresses( + []string{ + "cmdline/eth1/172.20.0.2/24", + "cmdline/eth4/10.3.5.7/24", + }, func(r *network.AddressSpec, asrt *assert.Assertions) { + switch r.Metadata().ID() { + case "cmdline/eth1/172.20.0.2/24": + asrt.Equal("eth1", r.TypedSpec().LinkName) + case "cmdline/eth4/10.3.5.7/24": + asrt.Equal("eth4", r.TypedSpec().LinkName) + } + }, + ) +} + +func (suite *AddressConfigSuite) TestCmdlineNoNetmask() { + suite.Require().NoError( + suite.runtime.RegisterController( + &netctrl.AddressConfigController{ + Cmdline: procfs.NewCmdline("ip=172.20.0.2::172.20.0.1"), + }, + ), + ) + + suite.startRuntime() + + ifaces, _ := net.Interfaces() //nolint:errcheck // ignoring error here as ifaces will be empty + + sort.Slice(ifaces, func(i, j int) bool { return ifaces[i].Name < ifaces[j].Name }) + + ifaceName := "" + + for _, iface := range ifaces { + if iface.Flags&net.FlagLoopback != 0 { + continue + } + + ifaceName = iface.Name + + break + } + + suite.Assert().NotEmpty(ifaceName) + + suite.assertAddresses( + []string{ + fmt.Sprintf("cmdline/%s/172.20.0.2/32", ifaceName), + }, func(r *network.AddressSpec, asrt *assert.Assertions) { + asrt.Equal(ifaceName, r.TypedSpec().LinkName) + asrt.Equal(network.ConfigCmdline, r.TypedSpec().ConfigLayer) + }, + ) +} + +func (suite *AddressConfigSuite) TestMachineConfiguration() { + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.AddressConfigController{})) + + suite.startRuntime() + + u, err := url.Parse("https://foo:6443") + suite.Require().NoError(err) + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineNetwork: &v1alpha1.NetworkConfig{ + NetworkInterfaces: []*v1alpha1.Device{ + { + DeviceInterface: "eth3", + DeviceCIDR: "192.168.0.24/28", + }, + { + DeviceIgnore: pointer.To(true), + DeviceInterface: "eth4", + DeviceCIDR: "192.168.0.24/28", + }, + { + DeviceInterface: "eth2", + DeviceCIDR: "2001:470:6d:30e:8ed2:b60c:9d2f:803a/64", + }, + { + DeviceInterface: "eth5", + DeviceCIDR: "10.5.0.7", + }, + { + DeviceInterface: "eth6", + DeviceAddresses: []string{ + "10.5.0.8", + "2001:470:6d:30e:8ed2:b60c:9d2f:803b/64", + }, + }, + { + DeviceInterface: "eth0", + DeviceVlans: []*v1alpha1.Vlan{ + { + VlanID: 24, + VlanCIDR: "10.0.0.1/8", + }, + { + VlanID: 25, + VlanAddresses: []string{ + "11.0.0.1/8", + }, + }, + }, + }, + }, + }, + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: u, + }, + }, + }, + }, + ), + ) + + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + suite.assertAddresses( + []string{ + "configuration/eth2/2001:470:6d:30e:8ed2:b60c:9d2f:803a/64", + "configuration/eth3/192.168.0.24/28", + "configuration/eth5/10.5.0.7/32", + "configuration/eth6/10.5.0.8/32", + "configuration/eth6/2001:470:6d:30e:8ed2:b60c:9d2f:803b/64", + "configuration/eth0.24/10.0.0.1/8", + "configuration/eth0.25/11.0.0.1/8", + }, func(r *network.AddressSpec, asrt *assert.Assertions) {}, + ) +} + +func (suite *AddressConfigSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestAddressConfigSuite(t *testing.T) { + suite.Run(t, new(AddressConfigSuite)) +} diff --git a/internal/app/machined/pkg/controllers/network/address_event.go b/internal/app/machined/pkg/controllers/network/address_event.go new file mode 100644 index 0000000..47d0165 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/address_event.go @@ -0,0 +1,116 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "context" + "time" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// AddressEventController reports aggregated enpoints state from hostname statuses and k8s endpoints +// to the events stream. +type AddressEventController struct { + V1Alpha1Events runtime.Publisher +} + +// Name implements controller.Controller interface. +func (ctrl *AddressEventController) Name() string { + return "network.AddressEventController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *AddressEventController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: network.NamespaceName, + Type: network.NodeAddressType, + Kind: controller.InputWeak, + ID: optional.Some(network.FilteredNodeAddressID( + network.NodeAddressCurrentID, + k8s.NodeAddressFilterNoK8s)), + }, + { + Namespace: network.NamespaceName, + Type: network.HostnameStatusType, + Kind: controller.InputWeak, + ID: optional.Some(network.HostnameID), + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *AddressEventController) Outputs() []controller.Output { + return nil +} + +// Run implements controller.Controller interface. +func (ctrl *AddressEventController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + ticker := time.NewTicker(time.Minute * 10) + + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + case <-r.EventCh(): + } + + addresses := []string{} + + nodeAddr, err := r.Get( + ctx, + resource.NewMetadata( + network.NamespaceName, + network.NodeAddressType, + network.FilteredNodeAddressID( + network.NodeAddressCurrentID, + k8s.NodeAddressFilterNoK8s), + resource.VersionUndefined), + ) + if err != nil { + if !state.IsNotFoundError(err) { + return err + } + } else { + for _, addr := range nodeAddr.(*network.NodeAddress).TypedSpec().Addresses { + addresses = append( + addresses, + addr.Addr().String(), + ) + } + } + + var hostname string + + hostnameStatus, err := r.Get(ctx, resource.NewMetadata(network.NamespaceName, network.HostnameStatusType, network.HostnameID, resource.VersionUndefined)) + if err != nil { + if !state.IsNotFoundError(err) { + return err + } + } else { + hostname = hostnameStatus.(*network.HostnameStatus).TypedSpec().Hostname + } + + ctrl.V1Alpha1Events.Publish(ctx, &machine.AddressEvent{ + Hostname: hostname, + Addresses: addresses, + }) + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/network/address_event_test.go b/internal/app/machined/pkg/controllers/network/address_event_test.go new file mode 100644 index 0000000..5d993dc --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/address_event_test.go @@ -0,0 +1,191 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network_test + +import ( + "context" + "errors" + "log" + "net/netip" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/proto" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + networkresource "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type mockEventsStream struct { + messagesMu sync.Mutex + messages []proto.Message +} + +func (s *mockEventsStream) Publish(_ context.Context, m proto.Message) { + s.messagesMu.Lock() + defer s.messagesMu.Unlock() + + s.messages = append(s.messages, m) +} + +type AddressEventsSuite struct { + suite.Suite + + events *mockEventsStream + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *AddressEventsSuite) SetupTest() { + suite.events = &mockEventsStream{ + messages: []proto.Message{}, + } + + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.Require().NoError( + suite.runtime.RegisterController( + &network.AddressEventController{ + V1Alpha1Events: suite.events, + }, + ), + ) + + suite.startRuntime() +} + +func (suite *AddressEventsSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *AddressEventsSuite) TestReconcile() { + hostname := networkresource.NewHostnameStatus(networkresource.NamespaceName, networkresource.HostnameID) + hostname.TypedSpec().Hostname = "localhost" + + suite.Require().NoError(suite.state.Create(suite.ctx, hostname)) + + var event *machine.AddressEvent + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + suite.events.messagesMu.Lock() + defer suite.events.messagesMu.Unlock() + + if len(suite.events.messages) == 0 { + return retry.ExpectedErrorf("no events created") + } + + m := suite.events.messages[len(suite.events.messages)-1] + + var ok bool + + event, ok = m.(*machine.AddressEvent) + if !ok { + return errors.New("not an endpoint event") + } + + if event.Hostname == "" { + return retry.ExpectedErrorf("expected hostname to be set") + } + + return nil + }, + ), + ) + + suite.Require().Equal(hostname.TypedSpec().Hostname, event.Hostname) + suite.Require().Empty(event.Addresses) + + nodeAddress := networkresource.NewNodeAddress( + networkresource.NamespaceName, networkresource.FilteredNodeAddressID( + networkresource.NodeAddressCurrentID, + k8s.NodeAddressFilterNoK8s, + ), + ) + + addrs := []string{ + "10.5.0.2", + "127.0.0.2", + } + + nodeAddress.TypedSpec().Addresses = append( + nodeAddress.TypedSpec().Addresses, + netip.PrefixFrom(netip.MustParseAddr(addrs[0]), 32), + netip.PrefixFrom(netip.MustParseAddr(addrs[1]), 32), + ) + + suite.Require().NoError(suite.state.Create(suite.ctx, nodeAddress)) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + suite.events.messagesMu.Lock() + defer suite.events.messagesMu.Unlock() + + if len(suite.events.messages) == 0 { + return retry.ExpectedErrorf("no events created") + } + + m := suite.events.messages[len(suite.events.messages)-1] + + var ok bool + + event, ok = m.(*machine.AddressEvent) + if !ok { + return errors.New("not an address event") + } + + if len(event.Addresses) == 0 { + return retry.ExpectedErrorf("expected addresses to be set") + } + + return nil + }, + ), + ) + + suite.Require().Equal(addrs, event.Addresses) +} + +func (suite *AddressEventsSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestAddressEventsSuite(t *testing.T) { + suite.Run(t, new(AddressEventsSuite)) +} diff --git a/internal/app/machined/pkg/controllers/network/address_merge.go b/internal/app/machined/pkg/controllers/network/address_merge.go new file mode 100644 index 0000000..5d986a0 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/address_merge.go @@ -0,0 +1,140 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package network provides controllers which manage network resources. +// +//nolint:dupl +package network + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// AddressMergeController merges network.AddressSpec in network.ConfigNamespace and produces final network.AddressSpec in network.Namespace. +type AddressMergeController struct{} + +// Name implements controller.Controller interface. +func (ctrl *AddressMergeController) Name() string { + return "network.AddressMergeController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *AddressMergeController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: network.ConfigNamespaceName, + Type: network.AddressSpecType, + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.AddressSpecType, + Kind: controller.InputDestroyReady, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *AddressMergeController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.AddressSpecType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *AddressMergeController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + // list source network configuration resources + list, err := r.List(ctx, resource.NewMetadata(network.ConfigNamespaceName, network.AddressSpecType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing source network addresses: %w", err) + } + + // address is allowed as long as it's not duplicate, for duplicate higher layer takes precedence + addresses := map[string]*network.AddressSpec{} + + for _, res := range list.Items { + address := res.(*network.AddressSpec) //nolint:errcheck,forcetypeassert + id := network.AddressID(address.TypedSpec().LinkName, address.TypedSpec().Address) + + existing, ok := addresses[id] + if ok && existing.TypedSpec().ConfigLayer > address.TypedSpec().ConfigLayer { + // skip this address, as existing one is higher layer + continue + } + + addresses[id] = address + } + + conflictsDetected := 0 + + for id, address := range addresses { + if err = r.Modify(ctx, network.NewAddressSpec(network.NamespaceName, id), func(res resource.Resource) error { + addr := res.(*network.AddressSpec) //nolint:errcheck,forcetypeassert + + *addr.TypedSpec() = *address.TypedSpec() + + return nil + }); err != nil { + if state.IsPhaseConflictError(err) { + // phase conflict, resource is being torn down, skip updating it and trigger reconcile + // later by failing the + conflictsDetected++ + + delete(addresses, id) + } else { + return fmt.Errorf("error updating resource: %w", err) + } + } + } + + // list addresses for cleanup + list, err = r.List(ctx, resource.NewMetadata(network.NamespaceName, network.AddressSpecType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing resources: %w", err) + } + + for _, res := range list.Items { + if _, ok := addresses[res.Metadata().ID()]; !ok { + var okToDestroy bool + + okToDestroy, err = r.Teardown(ctx, res.Metadata()) + if err != nil { + return fmt.Errorf("error cleaning up addresses: %w", err) + } + + if okToDestroy { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return fmt.Errorf("error cleaning up addresses: %w", err) + } + } + } + } + + if conflictsDetected > 0 { + return fmt.Errorf("%d conflict(s) detected", conflictsDetected) + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/network/address_merge_test.go b/internal/app/machined/pkg/controllers/network/address_merge_test.go new file mode 100644 index 0000000..f6034bc --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/address_merge_test.go @@ -0,0 +1,315 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:dupl +package network_test + +import ( + "context" + "log" + "net/netip" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "golang.org/x/sync/errgroup" + + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type AddressMergeSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *AddressMergeSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.AddressMergeController{})) + + suite.startRuntime() +} + +func (suite *AddressMergeSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *AddressMergeSuite) assertAddresses(requiredIDs []string, check func(*network.AddressSpec, *assert.Assertions)) { + assertResources(suite.ctx, suite.T(), suite.state, requiredIDs, check) +} + +func (suite *AddressMergeSuite) assertNoAddress(id string) error { + resources, err := suite.state.List( + suite.ctx, + resource.NewMetadata(network.NamespaceName, network.AddressSpecType, "", resource.VersionUndefined), + ) + if err != nil { + return err + } + + for _, res := range resources.Items { + if res.Metadata().ID() == id { + return retry.ExpectedErrorf("address %q is still there", id) + } + } + + return nil +} + +func (suite *AddressMergeSuite) TestMerge() { + loopback := network.NewAddressSpec(network.ConfigNamespaceName, "default/lo/127.0.0.1/8") + *loopback.TypedSpec() = network.AddressSpecSpec{ + Address: netip.MustParsePrefix("127.0.0.1/8"), + LinkName: "lo", + Family: nethelpers.FamilyInet4, + Scope: nethelpers.ScopeHost, + ConfigLayer: network.ConfigDefault, + } + + dhcp := network.NewAddressSpec(network.ConfigNamespaceName, "dhcp/eth0/10.0.0.1/8") + *dhcp.TypedSpec() = network.AddressSpecSpec{ + Address: netip.MustParsePrefix("10.0.0.1/8"), + LinkName: "eth0", + Family: nethelpers.FamilyInet4, + Scope: nethelpers.ScopeGlobal, + ConfigLayer: network.ConfigOperator, + } + + static := network.NewAddressSpec(network.ConfigNamespaceName, "configuration/eth0/10.0.0.35/32") + *static.TypedSpec() = network.AddressSpecSpec{ + Address: netip.MustParsePrefix("10.0.0.35/32"), + LinkName: "eth0", + Family: nethelpers.FamilyInet4, + Scope: nethelpers.ScopeGlobal, + ConfigLayer: network.ConfigMachineConfiguration, + } + + override := network.NewAddressSpec(network.ConfigNamespaceName, "configuration/eth0/10.0.0.1/8") + *override.TypedSpec() = network.AddressSpecSpec{ + Address: netip.MustParsePrefix("10.0.0.1/8"), + LinkName: "eth0", + Family: nethelpers.FamilyInet4, + Scope: nethelpers.ScopeHost, + ConfigLayer: network.ConfigMachineConfiguration, + } + + for _, res := range []resource.Resource{loopback, dhcp, static, override} { + suite.Require().NoError(suite.state.Create(suite.ctx, res), "%v", res.Spec()) + } + + suite.assertAddresses( + []string{ + "lo/127.0.0.1/8", + "eth0/10.0.0.1/8", + "eth0/10.0.0.35/32", + }, func(r *network.AddressSpec, asrt *assert.Assertions) { + switch r.Metadata().ID() { + case "lo/127.0.0.1/8": + asrt.Equal(*loopback.TypedSpec(), *r.TypedSpec()) + case "eth0/10.0.0.1/8": + asrt.Equal(*override.TypedSpec(), *r.TypedSpec()) + case "eth0/10.0.0.35/32": + asrt.Equal(*static.TypedSpec(), *r.TypedSpec()) + } + }, + ) + + suite.Require().NoError(suite.state.Destroy(suite.ctx, static.Metadata())) + + suite.assertAddresses( + []string{ + "lo/127.0.0.1/8", + "eth0/10.0.0.35/32", + }, func(*network.AddressSpec, *assert.Assertions) {}, + ) + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertNoAddress("eth0/10.0.0.35/32") + }, + ), + ) +} + +func (suite *AddressMergeSuite) TestMergeFlapping() { + // simulate two conflicting address definitions which are getting removed/added constantly + dhcp := network.NewAddressSpec(network.ConfigNamespaceName, "dhcp/eth0/10.0.0.1/8") + *dhcp.TypedSpec() = network.AddressSpecSpec{ + Address: netip.MustParsePrefix("10.0.0.1/8"), + LinkName: "eth0", + Family: nethelpers.FamilyInet4, + Scope: nethelpers.ScopeGlobal, + ConfigLayer: network.ConfigOperator, + } + + override := network.NewAddressSpec(network.ConfigNamespaceName, "configuration/eth0/10.0.0.1/8") + *override.TypedSpec() = network.AddressSpecSpec{ + Address: netip.MustParsePrefix("10.0.0.1/8"), + LinkName: "eth0", + Family: nethelpers.FamilyInet4, + Scope: nethelpers.ScopeHost, + ConfigLayer: network.ConfigMachineConfiguration, + } + + resources := []resource.Resource{dhcp, override} + + flipflop := func(idx int) func() error { + return func() error { + for range 500 { + if err := suite.state.Create(suite.ctx, resources[idx]); err != nil { + return err + } + + if err := suite.state.Destroy(suite.ctx, resources[idx].Metadata()); err != nil { + return err + } + + time.Sleep(time.Millisecond) + } + + return suite.state.Create(suite.ctx, resources[idx]) + } + } + + var eg errgroup.Group + + eg.Go(flipflop(0)) + eg.Go(flipflop(1)) + eg.Go( + func() error { + // add/remove finalizer to the merged resource + for range 1000 { + if err := suite.state.AddFinalizer( + suite.ctx, + resource.NewMetadata( + network.NamespaceName, + network.AddressSpecType, + "eth0/10.0.0.1/8", + resource.VersionUndefined, + ), + "foo", + ); err != nil { + if !state.IsNotFoundError(err) { + return err + } + + continue + } + + suite.T().Log("finalizer added") + + time.Sleep(10 * time.Millisecond) + + if err := suite.state.RemoveFinalizer( + suite.ctx, + resource.NewMetadata( + network.NamespaceName, + network.AddressSpecType, + "eth0/10.0.0.1/8", + resource.VersionUndefined, + ), + "foo", + ); err != nil && !state.IsNotFoundError(err) { + return err + } + } + + return nil + }, + ) + + suite.Require().NoError(eg.Wait()) + + suite.assertAddresses( + []string{ + "eth0/10.0.0.1/8", + }, func(r *network.AddressSpec, asrt *assert.Assertions) { + asrt.Equal(r.Metadata().Phase(), resource.PhaseRunning, "resource phase is %s", r.Metadata().Phase()) + asrt.Equal(*override.TypedSpec(), *r.TypedSpec()) + }, + ) +} + +func (suite *AddressMergeSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestAddressMergeSuite(t *testing.T) { + suite.Run(t, new(AddressMergeSuite)) +} + +func assertResources[R rtestutils.ResourceWithRD]( + ctx context.Context, + t *testing.T, + state state.State, + requiredIDs []string, + check func(R, *assert.Assertions), + opts ...rtestutils.Option, +) { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + rtestutils.AssertResources( + ctx, + t, + state, + xslices.Map(requiredIDs, func(id string) resource.ID { return id }), + check, + opts..., + ) +} + +func assertNoResource[R rtestutils.ResourceWithRD]( + ctx context.Context, + t *testing.T, + state state.State, + id string, +) { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + rtestutils.AssertNoResource[R]( + ctx, + t, + state, + id, + ) +} diff --git a/internal/app/machined/pkg/controllers/network/address_spec.go b/internal/app/machined/pkg/controllers/network/address_spec.go new file mode 100644 index 0000000..cf977d4 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/address_spec.go @@ -0,0 +1,310 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "context" + "errors" + "fmt" + "net" + "net/netip" + "os" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/jsimonetti/rtnetlink" + "github.com/mdlayher/arp" + "go.uber.org/zap" + "go4.org/netipx" + "golang.org/x/sys/unix" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network/watch" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// AddressSpecController applies network.AddressSpec to the actual interfaces. +type AddressSpecController struct{} + +// Name implements controller.Controller interface. +func (ctrl *AddressSpecController) Name() string { + return "network.AddressSpecController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *AddressSpecController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: network.NamespaceName, + Type: network.AddressSpecType, + Kind: controller.InputStrong, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *AddressSpecController) Outputs() []controller.Output { + return nil +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *AddressSpecController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + // watch link changes as some address might need to be re-applied if the link appears + watcher, err := watch.NewRtNetlink(watch.NewDefaultRateLimitedTrigger(ctx, r), unix.RTMGRP_LINK) + if err != nil { + return err + } + + defer watcher.Done() + + conn, err := rtnetlink.Dial(nil) + if err != nil { + return fmt.Errorf("error dialing rtnetlink socket: %w", err) + } + + defer conn.Close() //nolint:errcheck + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + // list source network configuration resources + list, err := r.List(ctx, resource.NewMetadata(network.NamespaceName, network.AddressSpecType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing source network addresses: %w", err) + } + + // add finalizers for all live resources + for _, res := range list.Items { + if res.Metadata().Phase() != resource.PhaseRunning { + continue + } + + if err = r.AddFinalizer(ctx, res.Metadata(), ctrl.Name()); err != nil { + return fmt.Errorf("error adding finalizer: %w", err) + } + } + + // list rtnetlink links (interfaces) + links, err := conn.Link.List() + if err != nil { + return fmt.Errorf("error listing links: %w", err) + } + + // list rtnetlink addresses + addrs, err := conn.Address.List() + if err != nil { + return fmt.Errorf("error listing addresses: %w", err) + } + + // loop over addresses and make reconcile decision + for _, res := range list.Items { + address := res.(*network.AddressSpec) //nolint:forcetypeassert,errcheck + + if err = ctrl.syncAddress(ctx, r, logger, conn, links, addrs, address); err != nil { + return err + } + } + + r.ResetRestartBackoff() + } +} + +func resolveLinkName(links []rtnetlink.LinkMessage, linkName string) uint32 { + for _, link := range links { + if link.Attributes.Name == linkName { + return link.Index + } + } + + return 0 +} + +func findAddress(addrs []rtnetlink.AddressMessage, linkIndex uint32, ipPrefix netip.Prefix) *rtnetlink.AddressMessage { + for i, addr := range addrs { + if addr.Index != linkIndex { + continue + } + + if int(addr.PrefixLength) != ipPrefix.Bits() { + continue + } + + if !addr.Attributes.Address.Equal(ipPrefix.Addr().AsSlice()) { + continue + } + + return &addrs[i] + } + + return nil +} + +//nolint:gocyclo +func (ctrl *AddressSpecController) syncAddress(ctx context.Context, r controller.Runtime, logger *zap.Logger, conn *rtnetlink.Conn, + links []rtnetlink.LinkMessage, addrs []rtnetlink.AddressMessage, address *network.AddressSpec, +) error { + linkIndex := resolveLinkName(links, address.TypedSpec().LinkName) + + switch address.Metadata().Phase() { + case resource.PhaseTearingDown: + if linkIndex == 0 { + // address should be deleted, but link is gone, so assume address is gone + if err := r.RemoveFinalizer(ctx, address.Metadata(), ctrl.Name()); err != nil { + return fmt.Errorf("error removing finalizer: %w", err) + } + + return nil + } + + if existing := findAddress(addrs, linkIndex, address.TypedSpec().Address); existing != nil { + // delete address + if err := conn.Address.Delete(existing); err != nil { + return fmt.Errorf("error removing address: %w", err) + } + + logger.Sugar().Infof("removed address %s from %q", address.TypedSpec().Address, address.TypedSpec().LinkName) + } + + // now remove finalizer as address was deleted + if err := r.RemoveFinalizer(ctx, address.Metadata(), ctrl.Name()); err != nil { + return fmt.Errorf("error removing finalizer: %w", err) + } + case resource.PhaseRunning: + if linkIndex == 0 { + // address can't be assigned as link doesn't exist (yet), skip it + return nil + } + + if existing := findAddress(addrs, linkIndex, address.TypedSpec().Address); existing != nil { + // clear out tentative flag, it is set by the kernel, we shouldn't try to enforce it + existing.Flags &= ^uint8(nethelpers.AddressTentative) + existing.Attributes.Flags &= ^uint32(nethelpers.AddressTentative) + + // check if existing matches the spec: if it does, skip update + if existing.Scope == uint8(address.TypedSpec().Scope) && existing.Flags == uint8(address.TypedSpec().Flags) && + existing.Attributes.Flags == uint32(address.TypedSpec().Flags) { + return nil + } + + logger.Debug("replacing address", + zap.Stringer("address", address.TypedSpec().Address), + zap.String("link", address.TypedSpec().LinkName), + zap.Stringer("old_scope", nethelpers.Scope(existing.Scope)), + zap.Stringer("new_scope", address.TypedSpec().Scope), + zap.Stringer("old_flags", nethelpers.AddressFlags(existing.Attributes.Flags)), + zap.Stringer("new_flags", address.TypedSpec().Flags), + ) + + // delete address to get new one assigned below + if err := conn.Address.Delete(existing); err != nil { + return fmt.Errorf("error removing address: %w", err) + } + + logger.Info("removed address", zap.Stringer("address", address.TypedSpec().Address), zap.String("link", address.TypedSpec().LinkName)) + } + + // add address + if err := conn.Address.New(&rtnetlink.AddressMessage{ + Family: uint8(address.TypedSpec().Family), + PrefixLength: uint8(address.TypedSpec().Address.Bits()), + Flags: uint8(address.TypedSpec().Flags), + Scope: uint8(address.TypedSpec().Scope), + Index: linkIndex, + Attributes: &rtnetlink.AddressAttributes{ + Address: address.TypedSpec().Address.Addr().AsSlice(), + Local: address.TypedSpec().Address.Addr().AsSlice(), + Broadcast: broadcastAddr(address.TypedSpec().Address), + Flags: uint32(address.TypedSpec().Flags), + }, + }); err != nil { + // ignore EEXIST error + if !errors.Is(err, os.ErrExist) { + return fmt.Errorf("error adding address %s to %q: %w", address.TypedSpec().Address, address.TypedSpec().LinkName, err) + } + } + + logger.Info("assigned address", zap.Stringer("address", address.TypedSpec().Address), zap.String("link", address.TypedSpec().LinkName)) + + if address.TypedSpec().AnnounceWithARP { + if err := ctrl.gratuitousARP(logger, linkIndex, address.TypedSpec().Address.Addr()); err != nil { + logger.Warn("failure sending gratuitous ARP", zap.Stringer("address", address.TypedSpec().Address), zap.String("link", address.TypedSpec().LinkName), zap.Error(err)) + } + } + } + + return nil +} + +func (ctrl *AddressSpecController) gratuitousARP(logger *zap.Logger, linkIndex uint32, ip netip.Addr) error { + etherBroadcast := net.HardwareAddr{0xff, 0xff, 0xff, 0xff, 0xff, 0xff} + + if !ip.Is4() { + return nil + } + + iface, err := net.InterfaceByIndex(int(linkIndex)) + if err != nil { + return err + } + + if len(iface.HardwareAddr) != 6 { + // not ethernet + return nil + } + + cli, err := arp.Dial(iface) + if err != nil { + return fmt.Errorf("error creating arp client: %w", err) + } + + defer cli.Close() //nolint:errcheck + + packet, err := arp.NewPacket(arp.OperationRequest, cli.HardwareAddr(), ip, cli.HardwareAddr(), ip) + if err != nil { + return fmt.Errorf("error building packet: %w", err) + } + + if err = cli.WriteTo(packet, etherBroadcast); err != nil { + return fmt.Errorf("error sending gratuitous ARP: %w", err) + } + + logger.Info("sent gratuitous ARP", zap.Stringer("address", ip), zap.String("link", iface.Name)) + + return nil +} + +func broadcastAddr(addr netip.Prefix) net.IP { + if !addr.Addr().Is4() { + return nil + } + + ipnet := netipx.PrefixIPNet(addr) + + ip := ipnet.IP.To4() + if ip == nil { + return nil + } + + mask := net.IP(ipnet.Mask).To4() + + n := len(ip) + if n != len(mask) { + return nil + } + + out := make(net.IP, n) + + for i := range n { + out[i] = ip[i] | ^mask[i] + } + + return out +} diff --git a/internal/app/machined/pkg/controllers/network/address_spec_test.go b/internal/app/machined/pkg/controllers/network/address_spec_test.go new file mode 100644 index 0000000..56f62e1 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/address_spec_test.go @@ -0,0 +1,256 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:dupl +package network_test + +import ( + "context" + "fmt" + "log" + "math/rand" + "net" + "net/netip" + "os" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/jsimonetti/rtnetlink" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + "golang.org/x/sys/unix" + + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type AddressSpecSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *AddressSpecSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.AddressSpecController{})) + + suite.startRuntime() +} + +func (suite *AddressSpecSuite) uniqueDummyInterface() string { + return fmt.Sprintf("dummy%02x%02x%02x", rand.Int31()&0xff, rand.Int31()&0xff, rand.Int31()&0xff) +} + +func (suite *AddressSpecSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *AddressSpecSuite) assertLinkAddress(linkName, address string) error { + addr := netip.MustParsePrefix(address) + + iface, err := net.InterfaceByName(linkName) + suite.Require().NoError(err) + + conn, err := rtnetlink.Dial(nil) + suite.Require().NoError(err) + + defer conn.Close() //nolint:errcheck + + linkAddresses, err := conn.Address.List() + suite.Require().NoError(err) + + for _, linkAddress := range linkAddresses { + if linkAddress.Index != uint32(iface.Index) { + continue + } + + if int(linkAddress.PrefixLength) != addr.Bits() { + continue + } + + if !linkAddress.Attributes.Address.Equal(addr.Addr().AsSlice()) { + continue + } + + return nil + } + + return retry.ExpectedErrorf("address %s not found on %q", addr, linkName) +} + +func (suite *AddressSpecSuite) assertNoLinkAddress(linkName, address string) error { + addr := netip.MustParsePrefix(address) + + iface, err := net.InterfaceByName(linkName) + suite.Require().NoError(err) + + conn, err := rtnetlink.Dial(nil) + suite.Require().NoError(err) + + defer conn.Close() //nolint:errcheck + + linkAddresses, err := conn.Address.List() + suite.Require().NoError(err) + + for _, linkAddress := range linkAddresses { + if linkAddress.Index == uint32(iface.Index) && int(linkAddress.PrefixLength) == addr.Bits() && linkAddress.Attributes.Address.Equal(addr.Addr().AsSlice()) { + return retry.ExpectedErrorf("address %s is assigned to %q", addr, linkName) + } + } + + return nil +} + +func (suite *AddressSpecSuite) TestLoopback() { + loopback := network.NewAddressSpec(network.NamespaceName, "lo/127.0.0.1/8") + *loopback.TypedSpec() = network.AddressSpecSpec{ + Address: netip.MustParsePrefix("127.11.0.1/32"), + LinkName: "lo", + Family: nethelpers.FamilyInet4, + Scope: nethelpers.ScopeHost, + ConfigLayer: network.ConfigDefault, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + } + + for _, res := range []resource.Resource{loopback} { + suite.Require().NoError(suite.state.Create(suite.ctx, res), "%v", res.Spec()) + } + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertLinkAddress("lo", "127.11.0.1/32") + }, + ), + ) + + // teardown the address + for { + ready, err := suite.state.Teardown(suite.ctx, loopback.Metadata()) + suite.Require().NoError(err) + + if ready { + break + } + + time.Sleep(100 * time.Millisecond) + } + + // torn down address should be removed immediately + suite.Assert().NoError(suite.assertNoLinkAddress("lo", "127.11.0.1/32")) + + suite.Require().NoError(suite.state.Destroy(suite.ctx, loopback.Metadata())) +} + +func (suite *AddressSpecSuite) TestDummy() { + dummyInterface := suite.uniqueDummyInterface() + + conn, err := rtnetlink.Dial(nil) + suite.Require().NoError(err) + + defer conn.Close() //nolint:errcheck + + dummy := network.NewAddressSpec(network.NamespaceName, "dummy/10.0.0.1/8") + *dummy.TypedSpec() = network.AddressSpecSpec{ + Address: netip.MustParsePrefix("10.0.0.1/8"), + LinkName: dummyInterface, + Family: nethelpers.FamilyInet4, + Scope: nethelpers.ScopeGlobal, + ConfigLayer: network.ConfigDefault, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + } + + // it's fine to create the address before the interface is actually created + for _, res := range []resource.Resource{dummy} { + suite.Require().NoError(suite.state.Create(suite.ctx, res), "%v", res.Spec()) + } + + // create dummy interface + suite.Require().NoError( + conn.Link.New( + &rtnetlink.LinkMessage{ + Type: unix.ARPHRD_ETHER, + Attributes: &rtnetlink.LinkAttributes{ + Name: dummyInterface, + MTU: 1400, + Info: &rtnetlink.LinkInfo{ + Kind: "dummy", + }, + }, + }, + ), + ) + + iface, err := net.InterfaceByName(dummyInterface) + suite.Require().NoError(err) + + defer conn.Link.Delete(uint32(iface.Index)) //nolint:errcheck + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertLinkAddress(dummyInterface, "10.0.0.1/8") + }, + ), + ) + + // delete dummy interface, address should be unassigned automatically + suite.Require().NoError(conn.Link.Delete(uint32(iface.Index))) + + // teardown the address + for { + ready, err := suite.state.Teardown(suite.ctx, dummy.Metadata()) + suite.Require().NoError(err) + + if ready { + break + } + + time.Sleep(100 * time.Millisecond) + } +} + +func (suite *AddressSpecSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestAddressSpecSuite(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("requires root") + } + + suite.Run(t, new(AddressSpecSuite)) +} diff --git a/internal/app/machined/pkg/controllers/network/address_status.go b/internal/app/machined/pkg/controllers/network/address_status.go new file mode 100644 index 0000000..6e8934e --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/address_status.go @@ -0,0 +1,145 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "context" + "fmt" + "net/netip" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/jsimonetti/rtnetlink" + "go.uber.org/zap" + "golang.org/x/sys/unix" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network/watch" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// AddressStatusController manages secrets.Etcd based on configuration. +type AddressStatusController struct{} + +// Name implements controller.Controller interface. +func (ctrl *AddressStatusController) Name() string { + return "network.AddressStatusController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *AddressStatusController) Inputs() []controller.Input { + return nil +} + +// Outputs implements controller.Controller interface. +func (ctrl *AddressStatusController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.AddressStatusType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *AddressStatusController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + watcher, err := watch.NewRtNetlink(watch.NewDefaultRateLimitedTrigger(ctx, r), unix.RTMGRP_LINK|unix.RTMGRP_IPV4_IFADDR|unix.RTMGRP_IPV6_IFADDR) + if err != nil { + return err + } + + defer watcher.Done() + + conn, err := rtnetlink.Dial(nil) + if err != nil { + return fmt.Errorf("error dialing rtnetlink socket: %w", err) + } + + defer conn.Close() //nolint:errcheck + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + // build links lookup table + links, err := conn.Link.List() + if err != nil { + return fmt.Errorf("error listing links: %w", err) + } + + linkLookup := make(map[uint32]string, len(links)) + + for _, link := range links { + linkLookup[link.Index] = link.Attributes.Name + } + + touchedIDs := map[resource.ID]struct{}{} + + addrs, err := conn.Address.List() + if err != nil { + return fmt.Errorf("error listing addresses: %w", err) + } + + for _, addr := range addrs { + // TODO: should we use local address actually? + // from if_addr.h: + // IFA_ADDRESS is prefix address, rather than local interface address. + // * It makes no difference for normally configured broadcast interfaces, + // * but for point-to-point IFA_ADDRESS is DESTINATION address, + // * local address is supplied in IFA_LOCAL attribute. + ipAddr, _ := netip.AddrFromSlice(addr.Attributes.Address) + ipPrefix := netip.PrefixFrom(ipAddr, int(addr.PrefixLength)) + id := network.AddressID(linkLookup[addr.Index], ipPrefix) + + if err = r.Modify(ctx, network.NewAddressStatus(network.NamespaceName, id), func(r resource.Resource) error { + status := r.(*network.AddressStatus).TypedSpec() + + status.Address = ipPrefix + status.Local, _ = netip.AddrFromSlice(addr.Attributes.Local) + status.Broadcast, _ = netip.AddrFromSlice(addr.Attributes.Broadcast) + status.Anycast, _ = netip.AddrFromSlice(addr.Attributes.Anycast) + status.Multicast, _ = netip.AddrFromSlice(addr.Attributes.Multicast) + status.LinkIndex = addr.Index + status.LinkName = linkLookup[addr.Index] + status.Family = nethelpers.Family(addr.Family) + status.Scope = nethelpers.Scope(addr.Scope) + status.Flags = nethelpers.AddressFlags(addr.Attributes.Flags) + + return nil + }); err != nil { + return fmt.Errorf("error modifying resource: %w", err) + } + + touchedIDs[id] = struct{}{} + } + + // list resources for cleanup + list, err := r.List(ctx, resource.NewMetadata(network.NamespaceName, network.AddressStatusType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing resources: %w", err) + } + + for _, res := range list.Items { + if res.Metadata().Owner() != ctrl.Name() { + continue + } + + if _, ok := touchedIDs[res.Metadata().ID()]; ok { + continue + } + + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return fmt.Errorf("error deleting address status %s: %w", res, err) + } + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/network/address_status_test.go b/internal/app/machined/pkg/controllers/network/address_status_test.go new file mode 100644 index 0000000..b4f8a05 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/address_status_test.go @@ -0,0 +1,84 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:dupl +package network_test + +import ( + "context" + "log" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type AddressStatusSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *AddressStatusSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.AddressStatusController{})) + + suite.startRuntime() +} + +func (suite *AddressStatusSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *AddressStatusSuite) assertAddresses(requiredIDs []string, check func(*network.AddressStatus, *assert.Assertions)) { + assertResources(suite.ctx, suite.T(), suite.state, requiredIDs, check) +} + +func (suite *AddressStatusSuite) TestLoopback() { + suite.assertAddresses( + []string{"lo/127.0.0.1/8"}, func(r *network.AddressStatus, asrt *assert.Assertions) {}, + ) +} + +func (suite *AddressStatusSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestAddressStatusSuite(t *testing.T) { + suite.Run(t, new(AddressStatusSuite)) +} diff --git a/internal/app/machined/pkg/controllers/network/cmdline.go b/internal/app/machined/pkg/controllers/network/cmdline.go new file mode 100644 index 0000000..b16b2ff --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/cmdline.go @@ -0,0 +1,513 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "errors" + "fmt" + "net" + "net/netip" + "sort" + "strconv" + "strings" + + "github.com/siderolabs/gen/pair/ordered" + "github.com/siderolabs/go-procfs/procfs" + + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// CmdlineNetworking contains parsed cmdline networking settings. +type CmdlineNetworking struct { + LinkConfigs []CmdlineLinkConfig + Hostname string + DNSAddresses []netip.Addr + NTPAddresses []netip.Addr + IgnoreInterfaces []string + NetworkLinkSpecs []network.LinkSpecSpec +} + +// CmdlineLinkConfig contains parsed cmdline networking settings for a single link. +type CmdlineLinkConfig struct { + LinkName string + Address netip.Prefix + Gateway netip.Addr + DHCP bool +} + +func (linkConfig *CmdlineLinkConfig) resolveLinkName() error { + if !strings.HasPrefix(linkConfig.LinkName, "enx") { + return nil + } + + ifaces, _ := net.Interfaces() //nolint:errcheck // ignoring error here as ifaces will be empty + mac := strings.ToLower(strings.TrimPrefix(linkConfig.LinkName, "enx")) + + for _, iface := range ifaces { + ifaceMAC := strings.ReplaceAll(iface.HardwareAddr.String(), ":", "") + if ifaceMAC == mac { + linkConfig.LinkName = iface.Name + + return nil + } + } + + if strings.HasPrefix(linkConfig.LinkName, "enx") { + return fmt.Errorf("cmdline device parse failure: interface by MAC not found %s", linkConfig.LinkName) + } + + return nil +} + +// splitIPArgument splits the `ip=` kernel argument honoring the IPv6 addresses in square brackets. +func splitIPArgument(val string) []string { + var ( + squared, prev int + parts []string + ) + + for i, c := range val { + switch c { + case '[': + squared++ + case ']': + squared-- + case ':': + if squared != 0 { + continue + } + + parts = append(parts, strings.Trim(val[prev:i], "[]")) + prev = i + 1 + } + } + + parts = append(parts, strings.Trim(val[prev:], "[]")) + + return parts +} + +const autoconfDHCP = "dhcp" + +// ParseCmdlineNetwork parses `ip=` and Talos specific kernel cmdline argument producing all the available configuration options. +// +//nolint:gocyclo,cyclop +func ParseCmdlineNetwork(cmdline *procfs.Cmdline) (CmdlineNetworking, error) { + var ( + settings CmdlineNetworking + err error + linkSpecSpecs []network.LinkSpecSpec + ) + + // process Talos specific kernel params + cmdlineHostname := cmdline.Get(constants.KernelParamHostname).First() + if cmdlineHostname != nil { + settings.Hostname = *cmdlineHostname + } + + ignoreInterfaces := cmdline.Get(constants.KernelParamNetworkInterfaceIgnore) + for i := 0; ignoreInterfaces.Get(i) != nil; i++ { + settings.IgnoreInterfaces = append(settings.IgnoreInterfaces, *ignoreInterfaces.Get(i)) + } + + // standard ip= + ipSettings := cmdline.Get("ip") + + for idx := 0; ipSettings.Get(idx) != nil; idx++ { + // https://www.kernel.org/doc/Documentation/filesystems/nfs/nfsroot.txt + // https://man7.org/linux/man-pages/man7/dracut.cmdline.7.html + // + // supported formats: + // ip=::::::::: + // ip=dhcp (ignored) + // ip=:dhcp + fields := splitIPArgument(*ipSettings.Get(idx)) + + switch { + case len(fields) == 1 && fields[0] == autoconfDHCP: + // ignore + case len(fields) == 2 && fields[1] == autoconfDHCP: + // ip=:dhcp + linkConfig := CmdlineLinkConfig{ + LinkName: fields[0], + DHCP: true, + } + + if err = linkConfig.resolveLinkName(); err != nil { + return settings, err + } + + linkSpecSpecs = append(linkSpecSpecs, network.LinkSpecSpec{ + Name: linkConfig.LinkName, + Up: true, + ConfigLayer: network.ConfigCmdline, + }) + + settings.LinkConfigs = append(settings.LinkConfigs, linkConfig) + default: + linkConfig := CmdlineLinkConfig{} + + for i := range fields { + if fields[i] == "" { + continue + } + + switch i { + case 0: + var ip netip.Addr + + ip, err = netip.ParseAddr(fields[0]) + if err != nil { + return settings, fmt.Errorf("cmdline address parse failure: %s", err) + } + + // default is to have complete address masked + linkConfig.Address = netip.PrefixFrom(ip, ip.BitLen()) + case 2: + linkConfig.Gateway, err = netip.ParseAddr(fields[2]) + if err != nil { + return settings, fmt.Errorf("cmdline gateway parse failure: %s", err) + } + case 3: + var ( + netmask netip.Addr + ones int + ) + + ones, err = strconv.Atoi(fields[3]) + if err != nil { + netmask, err = netip.ParseAddr(fields[3]) + if err != nil { + return settings, fmt.Errorf("cmdline netmask parse failure: %s", err) + } + + ones, _ = net.IPMask(netmask.AsSlice()).Size() + } + + linkConfig.Address = netip.PrefixFrom(linkConfig.Address.Addr(), ones) + case 4: + if settings.Hostname == "" { + settings.Hostname = fields[4] + } + case 5: + linkConfig.LinkName = fields[5] + case 6: + if fields[6] == autoconfDHCP { + linkConfig.DHCP = true + } + case 7, 8: + var dnsIP netip.Addr + + dnsIP, err = netip.ParseAddr(fields[i]) + if err != nil { + return settings, fmt.Errorf("error parsing DNS IP: %w", err) + } + + settings.DNSAddresses = append(settings.DNSAddresses, dnsIP) + case 9: + var ntpIP netip.Addr + + ntpIP, err = netip.ParseAddr(fields[i]) + if err != nil { + return settings, fmt.Errorf("error parsing DNS IP: %w", err) + } + + settings.NTPAddresses = append(settings.NTPAddresses, ntpIP) + } + } + + // resolve enx* (with MAC address) to the actual interface name + if err = linkConfig.resolveLinkName(); err != nil { + return settings, err + } + + // if interface name is not set, pick the first non-loopback interface + if linkConfig.LinkName == "" { + ifaces, _ := net.Interfaces() //nolint:errcheck // ignoring error here as ifaces will be empty + + sort.Slice(ifaces, func(i, j int) bool { return ifaces[i].Name < ifaces[j].Name }) + + for _, iface := range ifaces { + if iface.Flags&net.FlagLoopback != 0 { + continue + } + + linkConfig.LinkName = iface.Name + + break + } + } + + linkSpecSpecs = append(linkSpecSpecs, network.LinkSpecSpec{ + Name: linkConfig.LinkName, + Up: true, + ConfigLayer: network.ConfigCmdline, + }) + + settings.LinkConfigs = append(settings.LinkConfigs, linkConfig) + } + } + + // dracut bond= + // ref: https://man7.org/linux/man-pages/man7/dracut.cmdline.7.html + bondSettings := cmdline.Get(constants.KernelParamBonding).First() + + if bondSettings != nil { + var ( + bondName, bondMTU string + bondSlaves []string + bondOptions v1alpha1.Bond + ) + + // bond=[::[:[:]]] + fields := strings.Split(*bondSettings, ":") + + for i := range fields { + if fields[i] == "" { + continue + } + + switch i { + case 0: + bondName = fields[0] + case 1: + bondSlaves = strings.Split(fields[1], ",") + case 2: + bondOptions, err = parseBondOptions(fields[2]) + if err != nil { + return settings, err + } + case 3: + bondMTU = fields[3] + } + } + + // set defaults as per https://man7.org/linux/man-pages/man7/dracut.cmdline.7.html + // Talos by default sets bond mode to balance-rr + if bondSlaves == nil { + bondSlaves = []string{ + "eth0", + "eth1", + } + } + + bondLinkSpec := network.LinkSpecSpec{ + Name: bondName, + Up: true, + ConfigLayer: network.ConfigCmdline, + } + + if bondMTU != "" { + mtu, err := strconv.Atoi(bondMTU) + if err != nil { + return settings, fmt.Errorf("error parsing bond MTU: %w", err) + } + + bondLinkSpec.MTU = uint32(mtu) + } + + if err := SetBondMaster(&bondLinkSpec, &bondOptions); err != nil { + return settings, fmt.Errorf("error setting bond master: %w", err) + } + + linkSpecSpecs = append(linkSpecSpecs, bondLinkSpec) + + for idx, slave := range bondSlaves { + slaveLinkSpec := network.LinkSpecSpec{ + Name: slave, + Up: true, + ConfigLayer: network.ConfigCmdline, + } + SetBondSlave(&slaveLinkSpec, ordered.MakePair(bondName, idx)) + linkSpecSpecs = append(linkSpecSpecs, slaveLinkSpec) + } + } + // dracut vlan=: + vlanSettings := cmdline.Get(constants.KernelParamVlan).First() + if vlanSettings != nil { + vlanName, phyDevice, ok := strings.Cut(*vlanSettings, ":") + if !ok { + return settings, fmt.Errorf("malformed vlan commandline argument: %s", *vlanSettings) + } + + _, vlanNumberString, ok := strings.Cut(vlanName, ".") + if !ok { + return settings, fmt.Errorf("malformed vlan commandline argument: %s", *vlanSettings) + } + + vlanID, err := strconv.Atoi(vlanNumberString) + + if err != nil || vlanNumberString == "" { + return settings, errors.New("unable to parse vlan") + } + + vlanSpec := network.VLANSpec{ + VID: uint16(vlanID), + Protocol: nethelpers.VLANProtocol8021Q, + } + + vlanName = nethelpers.VLANLinkName(phyDevice, uint16(vlanID)) + + linkSpecUpdated := false + + for i, linkSpec := range linkSpecSpecs { + if linkSpec.Name == vlanName { + vlanLink(&linkSpecSpecs[i], phyDevice, vlanSpec) + + linkSpecUpdated = true + + break + } + } + + if !linkSpecUpdated { + linkSpec := network.LinkSpecSpec{ + Name: vlanName, + Up: true, + ConfigLayer: network.ConfigCmdline, + } + + vlanLink(&linkSpec, phyDevice, vlanSpec) + + linkSpecSpecs = append(linkSpecSpecs, linkSpec) + } + } + + settings.NetworkLinkSpecs = linkSpecSpecs + + return settings, nil +} + +// parseBondOptions parses the options string into v1alpha1.Bond +// v1alpha1.Bond was chosen to re-use the `SetBondMaster` and `SetBondSlave` functions +// ref: modinfo bonding +// +//nolint:gocyclo,cyclop,dupword +func parseBondOptions(options string) (v1alpha1.Bond, error) { + var bond v1alpha1.Bond + + bondOptions := strings.Split(options, ",") + + for _, opt := range bondOptions { + optionPair := strings.Split(opt, "=") + + switch optionPair[0] { + case "arp_ip_target": + bond.BondARPIPTarget = strings.Split(optionPair[1], ";") + case "mode": + bond.BondMode = optionPair[1] + case "xmit_hash_policy": + bond.BondHashPolicy = optionPair[1] + case "lacp_rate": + bond.BondLACPRate = optionPair[1] + case "arp_validate": + bond.BondARPValidate = optionPair[1] + case "arp_all_targets": + bond.BondARPAllTargets = optionPair[1] + case "primary": + bond.BondPrimary = optionPair[1] + case "primary_reselect": + bond.BondPrimaryReselect = optionPair[1] + case "fail_over_mac": + bond.BondFailOverMac = optionPair[1] + case "ad_select": + bond.BondADSelect = optionPair[1] + case "miimon": + miimon, err := strconv.Atoi(optionPair[1]) + if err != nil { + return bond, fmt.Errorf("error parsing bond option miimon: %w", err) + } + + bond.BondMIIMon = uint32(miimon) + case "updelay": + updelay, err := strconv.Atoi(optionPair[1]) + if err != nil { + return bond, fmt.Errorf("error parsing bond option updelay: %w", err) + } + + bond.BondUpDelay = uint32(updelay) + case "downdelay": + downdelay, err := strconv.Atoi(optionPair[1]) + if err != nil { + return bond, fmt.Errorf("error parsing bond option downdelay: %w", err) + } + + bond.BondDownDelay = uint32(downdelay) + case "arp_interval": + arpInterval, err := strconv.Atoi(optionPair[1]) + if err != nil { + return bond, fmt.Errorf("error parsing bond option arp_interval: %w", err) + } + + bond.BondARPInterval = uint32(arpInterval) + case "resend_igmp": + resendIGMP, err := strconv.Atoi(optionPair[1]) + if err != nil { + return bond, fmt.Errorf("error parsing bond option resend_igmp: %w", err) + } + + bond.BondResendIGMP = uint32(resendIGMP) + case "min_links": + minLinks, err := strconv.Atoi(optionPair[1]) + if err != nil { + return bond, fmt.Errorf("error parsing bond option min_links: %w", err) + } + + bond.BondMinLinks = uint32(minLinks) + case "lp_interval": + lpInterval, err := strconv.Atoi(optionPair[1]) + if err != nil { + return bond, fmt.Errorf("error parsing bond option lp_interval: %w", err) + } + + bond.BondLPInterval = uint32(lpInterval) + case "packets_per_slave": + packetsPerSlave, err := strconv.Atoi(optionPair[1]) + if err != nil { + return bond, fmt.Errorf("error parsing bond option packets_per_slave: %w", err) + } + + bond.BondPacketsPerSlave = uint32(packetsPerSlave) + case "num_grat_arp": + numGratArp, err := strconv.Atoi(optionPair[1]) + if err != nil { + return bond, fmt.Errorf("error parsing bond option num_grat_arp: %w", err) + } + + bond.BondNumPeerNotif = uint8(numGratArp) + case "num_unsol_na": + numGratArp, err := strconv.Atoi(optionPair[1]) + if err != nil { + return bond, fmt.Errorf("error parsing bond option num_unsol_na: %w", err) + } + + bond.BondNumPeerNotif = uint8(numGratArp) + case "all_slaves_active": + allSlavesActive, err := strconv.Atoi(optionPair[1]) + if err != nil { + return bond, fmt.Errorf("error parsing bond option all_slaves_active: %w", err) + } + + bond.BondAllSlavesActive = uint8(allSlavesActive) + case "use_carrier": + useCarrier, err := strconv.Atoi(optionPair[1]) + if err != nil { + return bond, fmt.Errorf("error parsing bond option use_carrier: %w", err) + } + + if useCarrier == 1 { + val := []bool{true} + bond.BondUseCarrier = &val[0] + } + default: + return bond, fmt.Errorf("unknown bond option: %s", optionPair[0]) + } + } + + return bond, nil +} diff --git a/internal/app/machined/pkg/controllers/network/cmdline_test.go b/internal/app/machined/pkg/controllers/network/cmdline_test.go new file mode 100644 index 0000000..f28dc49 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/cmdline_test.go @@ -0,0 +1,492 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network_test + +import ( + "net" + "net/netip" + "sort" + "testing" + + "github.com/siderolabs/go-procfs/procfs" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + netconfig "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type CmdlineSuite struct { + suite.Suite +} + +func (suite *CmdlineSuite) TestParse() { + ifaces, _ := net.Interfaces() //nolint:errcheck // ignoring error here as ifaces will be empty + + sort.Slice(ifaces, func(i, j int) bool { return ifaces[i].Name < ifaces[j].Name }) + + defaultIfaceName := "" + + for _, iface := range ifaces { + if iface.Flags&net.FlagLoopback != 0 { + continue + } + + defaultIfaceName = iface.Name + + break + } + + defaultBondSettings := network.CmdlineNetworking{ + NetworkLinkSpecs: []netconfig.LinkSpecSpec{ + { + Name: "bond0", + Kind: "bond", + Type: nethelpers.LinkEther, + Logical: true, + Up: true, + ConfigLayer: netconfig.ConfigCmdline, + BondMaster: netconfig.BondMasterSpec{ + Mode: nethelpers.BondModeRoundrobin, + ResendIGMP: 1, + LPInterval: 1, + PacketsPerSlave: 1, + NumPeerNotif: 1, + TLBDynamicLB: 1, + UseCarrier: true, + }, + }, + { + Name: "eth0", + Up: true, + Logical: false, + ConfigLayer: netconfig.ConfigCmdline, + BondSlave: netconfig.BondSlave{ + MasterName: "bond0", + SlaveIndex: 0, + }, + }, + { + Name: "eth1", + Up: true, + Logical: false, + ConfigLayer: netconfig.ConfigCmdline, + BondSlave: netconfig.BondSlave{ + MasterName: "bond0", + SlaveIndex: 1, + }, + }, + }, + } + + for _, test := range []struct { + name string + cmdline string + + expectedSettings network.CmdlineNetworking + expectedError string + }{ + { + name: "zero", + cmdline: "", + }, + { + name: "static IP", + cmdline: "ip=172.20.0.2::172.20.0.1:255.255.255.0::eth1:::::", + + expectedSettings: network.CmdlineNetworking{ + LinkConfigs: []network.CmdlineLinkConfig{ + { + Address: netip.MustParsePrefix("172.20.0.2/24"), + Gateway: netip.MustParseAddr("172.20.0.1"), + LinkName: "eth1", + }, + }, + NetworkLinkSpecs: []netconfig.LinkSpecSpec{ + { + Name: "eth1", + Up: true, + ConfigLayer: netconfig.ConfigCmdline, + }, + }, + }, + }, + { + name: "no iface", + cmdline: "ip=172.20.0.2::172.20.0.1", + + expectedSettings: network.CmdlineNetworking{ + LinkConfigs: []network.CmdlineLinkConfig{ + { + Address: netip.MustParsePrefix("172.20.0.2/32"), + Gateway: netip.MustParseAddr("172.20.0.1"), + LinkName: defaultIfaceName, + }, + }, + NetworkLinkSpecs: []netconfig.LinkSpecSpec{ + { + Name: defaultIfaceName, + Up: true, + ConfigLayer: netconfig.ConfigCmdline, + }, + }, + }, + }, + { + name: "no iface by mac address", + cmdline: "ip=172.20.0.2::172.20.0.1:255.255.255.0::enx001122aabbcc", + + expectedError: "cmdline device parse failure: interface by MAC not found enx001122aabbcc", + }, + { + name: "complete", + cmdline: "ip=172.20.0.2:172.21.0.1:172.20.0.1:255.255.255.0:master1:eth1::10.0.0.1:10.0.0.2:10.0.0.1", + + expectedSettings: network.CmdlineNetworking{ + LinkConfigs: []network.CmdlineLinkConfig{ + { + Address: netip.MustParsePrefix("172.20.0.2/24"), + Gateway: netip.MustParseAddr("172.20.0.1"), + LinkName: "eth1", + }, + }, + Hostname: "master1", + DNSAddresses: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")}, + NTPAddresses: []netip.Addr{netip.MustParseAddr("10.0.0.1")}, + NetworkLinkSpecs: []netconfig.LinkSpecSpec{ + { + Name: "eth1", + Up: true, + ConfigLayer: netconfig.ConfigCmdline, + }, + }, + }, + }, + { + name: "ipv6", + cmdline: "ip=[2001:db8::a]:[2001:db8::b]:[fe80::1]::master1:eth1::[2001:4860:4860::6464]:[2001:4860:4860::64]:[2001:4860:4806::]", + expectedSettings: network.CmdlineNetworking{ + LinkConfigs: []network.CmdlineLinkConfig{ + { + Address: netip.MustParsePrefix("2001:db8::a/128"), + Gateway: netip.MustParseAddr("fe80::1"), + LinkName: "eth1", + }, + }, + Hostname: "master1", + DNSAddresses: []netip.Addr{netip.MustParseAddr("2001:4860:4860::6464"), netip.MustParseAddr("2001:4860:4860::64")}, + NTPAddresses: []netip.Addr{netip.MustParseAddr("2001:4860:4806::")}, + NetworkLinkSpecs: []netconfig.LinkSpecSpec{ + { + Name: "eth1", + Up: true, + ConfigLayer: netconfig.ConfigCmdline, + }, + }, + }, + }, + { + name: "ipv6-mask", + cmdline: "ip=[2a03:1:2::12]::[2a03:1:2::11]:[ffff:ffff:ffff:ffff:ffff:ffff:ffff:fff8]:master:eth0:off:[2001:4860:4860::8888]:[2606:4700::1111]:[2606:4700:f1::1]", + expectedSettings: network.CmdlineNetworking{ + LinkConfigs: []network.CmdlineLinkConfig{ + { + Address: netip.MustParsePrefix("2a03:1:2::12/125"), + Gateway: netip.MustParseAddr("2a03:1:2::11"), + LinkName: "eth0", + }, + }, + Hostname: "master", + DNSAddresses: []netip.Addr{netip.MustParseAddr("2001:4860:4860::8888"), netip.MustParseAddr("2606:4700::1111")}, + NTPAddresses: []netip.Addr{netip.MustParseAddr("2606:4700:f1::1")}, + NetworkLinkSpecs: []netconfig.LinkSpecSpec{ + { + Name: "eth0", + Up: true, + ConfigLayer: netconfig.ConfigCmdline, + }, + }, + }, + }, + { + name: "ipv6-mask-number", + cmdline: "ip=[2a03:1:2::12]::[2a03:1:2::11]:125:master:eth0:off:[2001:4860:4860::8888]:[2606:4700::1111]:[2606:4700:f1::1]", + expectedSettings: network.CmdlineNetworking{ + LinkConfigs: []network.CmdlineLinkConfig{ + { + Address: netip.MustParsePrefix("2a03:1:2::12/125"), + Gateway: netip.MustParseAddr("2a03:1:2::11"), + LinkName: "eth0", + }, + }, + Hostname: "master", + DNSAddresses: []netip.Addr{netip.MustParseAddr("2001:4860:4860::8888"), netip.MustParseAddr("2606:4700::1111")}, + NTPAddresses: []netip.Addr{netip.MustParseAddr("2606:4700:f1::1")}, + NetworkLinkSpecs: []netconfig.LinkSpecSpec{ + { + Name: "eth0", + Up: true, + ConfigLayer: netconfig.ConfigCmdline, + }, + }, + }, + }, + { + name: "unparseable IP", + cmdline: "ip=xyz:", + + expectedError: "cmdline address parse failure: ParseAddr(\"xyz\"): unable to parse IP", + }, + { + name: "hostname override", + cmdline: "ip=::::master1:eth1 talos.hostname=master2", + + expectedSettings: network.CmdlineNetworking{ + LinkConfigs: []network.CmdlineLinkConfig{ + { + LinkName: "eth1", + }, + }, + Hostname: "master2", + NetworkLinkSpecs: []netconfig.LinkSpecSpec{ + { + Name: "eth1", + Up: true, + ConfigLayer: netconfig.ConfigCmdline, + }, + }, + }, + }, + { + name: "only hostname", + cmdline: "talos.hostname=master2", + + expectedSettings: network.CmdlineNetworking{ + Hostname: "master2", + }, + }, + { + name: "ignore interfaces", + cmdline: "talos.network.interface.ignore=eth2 talos.network.interface.ignore=eth3", + + expectedSettings: network.CmdlineNetworking{ + IgnoreInterfaces: []string{"eth2", "eth3"}, + }, + }, + { + name: "bond with no interfaces and no options set", + cmdline: "bond=bond0", + expectedSettings: defaultBondSettings, + }, + { + name: "bond with no interfaces and empty options set", + cmdline: "bond=bond0:::", + expectedSettings: defaultBondSettings, + }, + { + name: "bond with interfaces and no options set", + cmdline: "bond=bond1:eth3,eth4", + expectedSettings: network.CmdlineNetworking{ + NetworkLinkSpecs: []netconfig.LinkSpecSpec{ + { + Name: "bond1", + Kind: "bond", + Type: nethelpers.LinkEther, + Logical: true, + Up: true, + ConfigLayer: netconfig.ConfigCmdline, + BondMaster: netconfig.BondMasterSpec{ + ResendIGMP: 1, + LPInterval: 1, + PacketsPerSlave: 1, + NumPeerNotif: 1, + TLBDynamicLB: 1, + UseCarrier: true, + }, + }, + { + Name: "eth3", + Up: true, + Logical: false, + ConfigLayer: netconfig.ConfigCmdline, + BondSlave: netconfig.BondSlave{ + MasterName: "bond1", + SlaveIndex: 0, + }, + }, + { + Name: "eth4", + Up: true, + Logical: false, + ConfigLayer: netconfig.ConfigCmdline, + BondSlave: netconfig.BondSlave{ + MasterName: "bond1", + SlaveIndex: 1, + }, + }, + }, + }, + }, + { + name: "bond with interfaces, options and mtu set", + cmdline: "bond=bond1:eth3,eth4:mode=802.3ad,xmit_hash_policy=layer2+3:1450", + expectedSettings: network.CmdlineNetworking{ + NetworkLinkSpecs: []netconfig.LinkSpecSpec{ + { + Name: "bond1", + Kind: "bond", + Type: nethelpers.LinkEther, + Logical: true, + Up: true, + MTU: 1450, + ConfigLayer: netconfig.ConfigCmdline, + BondMaster: netconfig.BondMasterSpec{ + Mode: nethelpers.BondMode8023AD, + HashPolicy: nethelpers.BondXmitPolicyLayer23, + ADActorSysPrio: 65535, + ResendIGMP: 1, + LPInterval: 1, + PacketsPerSlave: 1, + NumPeerNotif: 1, + TLBDynamicLB: 1, + UseCarrier: true, + }, + }, + { + Name: "eth3", + Up: true, + Logical: false, + ConfigLayer: netconfig.ConfigCmdline, + BondSlave: netconfig.BondSlave{ + MasterName: "bond1", + SlaveIndex: 0, + }, + }, + { + Name: "eth4", + Up: true, + Logical: false, + ConfigLayer: netconfig.ConfigCmdline, + BondSlave: netconfig.BondSlave{ + MasterName: "bond1", + SlaveIndex: 1, + }, + }, + }, + }, + }, + { + name: "unparseable bond options", + cmdline: "bond=bond0:eth1,eth2:mod=balance-rr", + + expectedError: "unknown bond option: mod", + }, + { + name: "vlan configuration", + cmdline: "vlan=eth1.169:eth1 ip=172.20.0.2::172.20.0.1:255.255.255.0::eth1.169:::::", + expectedSettings: network.CmdlineNetworking{ + LinkConfigs: []network.CmdlineLinkConfig{ + { + Address: netip.MustParsePrefix("172.20.0.2/24"), + Gateway: netip.MustParseAddr("172.20.0.1"), + LinkName: "eth1.169", + }, + }, + NetworkLinkSpecs: []netconfig.LinkSpecSpec{ + { + Name: "eth1.169", + Logical: true, + Up: true, + Kind: netconfig.LinkKindVLAN, + Type: nethelpers.LinkEther, + ParentName: "eth1", + ConfigLayer: netconfig.ConfigCmdline, + VLAN: netconfig.VLANSpec{ + VID: 169, + Protocol: nethelpers.VLANProtocol8021Q, + }, + }, + }, + }, + }, + { + name: "vlan configuration without ip configuration", + cmdline: "vlan=eth1.5:eth1", + expectedSettings: network.CmdlineNetworking{ + NetworkLinkSpecs: []netconfig.LinkSpecSpec{ + { + Name: "eth1.5", + Logical: true, + Up: true, + Kind: netconfig.LinkKindVLAN, + Type: nethelpers.LinkEther, + ParentName: "eth1", + ConfigLayer: netconfig.ConfigCmdline, + VLAN: netconfig.VLANSpec{ + VID: 5, + Protocol: nethelpers.VLANProtocol8021Q, + }, + }, + }, + }, + }, + { + name: "multiple ip configurations", + cmdline: "ip=172.20.0.2::172.20.0.1:255.255.255.0::eth1::::: ip=eth3:dhcp ip=:::::eth4:dhcp::::", + + expectedSettings: network.CmdlineNetworking{ + LinkConfigs: []network.CmdlineLinkConfig{ + { + Address: netip.MustParsePrefix("172.20.0.2/24"), + Gateway: netip.MustParseAddr("172.20.0.1"), + LinkName: "eth1", + }, + { + LinkName: "eth3", + DHCP: true, + }, + { + LinkName: "eth4", + DHCP: true, + }, + }, + NetworkLinkSpecs: []netconfig.LinkSpecSpec{ + { + Name: "eth1", + Up: true, + ConfigLayer: netconfig.ConfigCmdline, + }, + { + Name: "eth3", + Up: true, + ConfigLayer: netconfig.ConfigCmdline, + }, + { + Name: "eth4", + Up: true, + ConfigLayer: netconfig.ConfigCmdline, + }, + }, + }, + }, + } { + suite.Run(test.name, func() { + cmdline := procfs.NewCmdline(test.cmdline) + + settings, err := network.ParseCmdlineNetwork(cmdline) + + if test.expectedError != "" { + suite.Assert().EqualError(err, test.expectedError) + } else { + suite.Assert().NoError(err) + suite.Assert().Equal(test.expectedSettings, settings) + } + }) + } +} + +func TestCmdlineSuite(t *testing.T) { + suite.Run(t, new(CmdlineSuite)) +} diff --git a/internal/app/machined/pkg/controllers/network/device_config.go b/internal/app/machined/pkg/controllers/network/device_config.go new file mode 100644 index 0000000..f40cb53 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/device_config.go @@ -0,0 +1,244 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + glob "github.com/ryanuber/go-glob" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/xslices" + "go.uber.org/zap" + + talosconfig "github.com/siderolabs/talos/pkg/machinery/config/config" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// DeviceConfigController manages network.DeviceConfig based on configuration. +type DeviceConfigController struct { + devices map[string]networkDevice +} + +//nolint:unused +type networkDevice struct { + hardwareAddress string + busPrefix string + driver string + pciID string +} + +// Name implements controller.Controller interface. +func (ctrl *DeviceConfigController) Name() string { + return "network.DeviceConfigController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *DeviceConfigController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: config.MachineConfigType, + ID: optional.Some(config.V1Alpha1ID), + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.LinkStatusType, + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *DeviceConfigController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.DeviceConfigSpecType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *DeviceConfigController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + ctrl.devices = map[string]networkDevice{} + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + links, err := safe.ReaderListAll[*network.LinkStatus](ctx, r) + if err != nil { + return err + } + + var cfgProvider talosconfig.Config + + cfg, err := safe.ReaderGetByID[*config.MachineConfig](ctx, r, config.V1Alpha1ID) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting config: %w", err) + } + } else { + cfgProvider = cfg.Config() + } + + r.StartTrackingOutputs() + + if cfgProvider != nil && cfgProvider.Machine() != nil { + for index, device := range cfgProvider.Machine().Network().Devices() { + out := []talosconfig.Device{device} + + if device.Selector() != nil { + var matched []*v1alpha1.Device + + matched, err = ctrl.getDevicesBySelector(device, links) + if err != nil { + logger.Warn("failed to select an interface for a device", zap.Error(err)) + + continue + } + + out = xslices.Map(matched, func(device *v1alpha1.Device) talosconfig.Device { return device }) + } else if device.Bond() != nil && len(device.Bond().Selectors()) > 0 { + dev := device.(*v1alpha1.Device).DeepCopy() + device = dev + + err = ctrl.expandBondSelector(dev, links) + if err != nil { + logger.Warn("failed to select interfaces for a bond device", zap.Error(err)) + + continue + } + + out = []talosconfig.Device{device} + } + + for j, outDevice := range out { + id := fmt.Sprintf("%s/%03d", outDevice.Interface(), index) + + if len(out) > 1 { + id = fmt.Sprintf("%s/%03d", id, j) + } + + if err = safe.WriterModify( + ctx, + r, + network.NewDeviceConfig(id, outDevice), + func(r *network.DeviceConfigSpec) error { + r.TypedSpec().Device = outDevice + + return nil + }, + ); err != nil { + return err + } + } + } + } + + if err = safe.CleanupOutputs[*network.DeviceConfigSpec](ctx, r); err != nil { + return err + } + } +} + +func (ctrl *DeviceConfigController) getDevicesBySelector(device talosconfig.Device, links safe.List[*network.LinkStatus]) ([]*v1alpha1.Device, error) { + selector := device.Selector() + + matches := ctrl.selectDevices(selector, links) + if len(matches) == 0 { + return nil, fmt.Errorf("no matching network device for defined selector: %+v", selector) + } + + out := make([]*v1alpha1.Device, len(matches)) + + for i, link := range matches { + out[i] = device.(*v1alpha1.Device).DeepCopy() + out[i].DeviceInterface = link.Metadata().ID() + } + + return out, nil +} + +func (ctrl *DeviceConfigController) expandBondSelector(device *v1alpha1.Device, links safe.List[*network.LinkStatus]) error { + var matches []*network.LinkStatus + + for _, selector := range device.Bond().Selectors() { + matches = append(matches, + // filter out bond device itself, as it will inherit the MAC address of the first link + xslices.Filter( + ctrl.selectDevices(selector, links), + func(link *network.LinkStatus) bool { + return link.Metadata().ID() != device.Interface() + })...) + } + + device.DeviceBond.BondInterfaces = xslices.Map(matches, func(link *network.LinkStatus) string { return link.Metadata().ID() }) + + if len(device.DeviceBond.BondInterfaces) == 0 { + return fmt.Errorf("no matching network device for defined bond selectors: %v", + xslices.Map(device.Bond().Selectors(), + func(selector talosconfig.NetworkDeviceSelector) string { + return fmt.Sprintf("%+v", selector) + }, + ), + ) + } + + device.DeviceBond.BondDeviceSelectors = nil + + return nil +} + +func (ctrl *DeviceConfigController) selectDevices(selector talosconfig.NetworkDeviceSelector, links safe.List[*network.LinkStatus]) []*network.LinkStatus { + var result []*network.LinkStatus + + for iter := links.Iterator(); iter.Next(); { + linkStatus := iter.Value().TypedSpec() + + var match optional.Optional[bool] + + for _, pair := range [][]string{ + {selector.HardwareAddress(), linkStatus.HardwareAddr.String()}, + {selector.PCIID(), linkStatus.PCIID}, + {selector.KernelDriver(), linkStatus.Driver}, + {selector.Bus(), linkStatus.BusPath}, + } { + if pair[0] == "" { + continue + } + + if !glob.Glob(pair[0], pair[1]) { + match = optional.Some(false) + + break + } + + match = optional.Some(true) + } + + if selector.Physical() != nil && match.ValueOr(true) { + match = optional.Some(*selector.Physical() == linkStatus.Physical()) + } + + if match.ValueOrZero() { + result = append(result, iter.Value()) + } + } + + return result +} diff --git a/internal/app/machined/pkg/controllers/network/device_config_test.go b/internal/app/machined/pkg/controllers/network/device_config_test.go new file mode 100644 index 0000000..475fe67 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/device_config_test.go @@ -0,0 +1,251 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network_test + +import ( + "fmt" + "net" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/siderolabs/gen/maps" + "github.com/siderolabs/go-pointer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + configs "github.com/siderolabs/talos/pkg/machinery/config/config" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type DeviceConfigSpecSuite struct { + ctest.DefaultSuite +} + +func (suite *DeviceConfigSpecSuite) TestDeviceConfigs() { + cfgProvider := container.NewV1Alpha1(&v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineNetwork: &v1alpha1.NetworkConfig{ + NetworkInterfaces: []*v1alpha1.Device{ + { + DeviceInterface: "eth0", + DeviceAddresses: []string{"192.168.2.0/24"}, + DeviceMTU: 1500, + }, + { + DeviceInterface: "bond0", + DeviceAddresses: []string{"192.168.2.0/24"}, + DeviceBond: &v1alpha1.Bond{ + BondMode: "balance-rr", + BondInterfaces: []string{"eth1", "eth2"}, + }, + }, + { + DeviceInterface: "eth0", + DeviceAddresses: []string{"192.168.3.0/24"}, + }, + }, + }, + }, + }) + + cfg := config.NewMachineConfig(cfgProvider) + + devices := map[string]configs.Device{} + for index, item := range cfgProvider.Machine().Network().Devices() { + devices[fmt.Sprintf("%s/%03d", item.Interface(), index)] = item + } + + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), maps.Keys(devices), + func(r *network.DeviceConfigSpec, assert *assert.Assertions) { + assert.Equal(r.TypedSpec().Device, devices[r.Metadata().ID()]) + }, + ) +} + +func (suite *DeviceConfigSpecSuite) TestSelectors() { + kernelDriver := "thedriver" + + cfgProvider := container.NewV1Alpha1(&v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineNetwork: &v1alpha1.NetworkConfig{ + NetworkInterfaces: []*v1alpha1.Device{ + // device selector selecing a single interface + { + DeviceSelector: &v1alpha1.NetworkDeviceSelector{ + NetworkDeviceKernelDriver: kernelDriver, + }, + DeviceAddresses: []string{"192.168.2.0/24"}, + DeviceMTU: 1500, + }, + // no device selector (explicit name) + { + DeviceInterface: "eth0", + DeviceAddresses: []string{"192.168.3.0/24"}, + }, + // device selector which doesn't match anything + { + DeviceSelector: &v1alpha1.NetworkDeviceSelector{ + NetworkDeviceKernelDriver: "no-match", + }, + DeviceAddresses: []string{"192.168.4.0/24"}, + }, + // device selector which matches multiple interfaces + { + DeviceSelector: &v1alpha1.NetworkDeviceSelector{ + NetworkDeviceBus: "0000:01*", + }, + DeviceAddresses: []string{"192.168.5.0/24"}, + }, + // device selector which matches physical interfaces + { + DeviceSelector: &v1alpha1.NetworkDeviceSelector{ + NetworkDevicePhysical: pointer.To(true), + }, + DeviceAddresses: []string{"192.168.6.0/24"}, + }, + }, + }, + }, + }) + + cfg := config.NewMachineConfig(cfgProvider) + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + status := network.NewLinkStatus(network.NamespaceName, "eth0") + status.TypedSpec().Driver = kernelDriver + status.TypedSpec().BusPath = "0000:01:00.0" + status.TypedSpec().Type = nethelpers.LinkEther // physical + suite.Require().NoError(suite.State().Create(suite.Ctx(), status)) + + status = network.NewLinkStatus(network.NamespaceName, "eth1") + status.TypedSpec().BusPath = "0000:01:01.0" + suite.Require().NoError(suite.State().Create(suite.Ctx(), status)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []string{"eth0/000"}, + func(r *network.DeviceConfigSpec, assert *assert.Assertions) { + assert.Equal(1500, r.TypedSpec().Device.MTU()) + assert.Equal([]string{"192.168.2.0/24"}, r.TypedSpec().Device.Addresses()) + }, + ) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []string{"eth0/001"}, + func(r *network.DeviceConfigSpec, assert *assert.Assertions) { + assert.Equal([]string{"192.168.3.0/24"}, r.TypedSpec().Device.Addresses()) + }, + ) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []string{"eth0/003/000", "eth1/003/001"}, + func(r *network.DeviceConfigSpec, assert *assert.Assertions) { + assert.Equal([]string{"192.168.5.0/24"}, r.TypedSpec().Device.Addresses()) + }, + ) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []string{"eth0/004"}, + func(r *network.DeviceConfigSpec, assert *assert.Assertions) { + assert.Equal([]string{"192.168.6.0/24"}, r.TypedSpec().Device.Addresses()) + }, + ) +} + +func (suite *DeviceConfigSpecSuite) TestBondSelectors() { + cfgProvider := container.NewV1Alpha1(&v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineNetwork: &v1alpha1.NetworkConfig{ + NetworkInterfaces: []*v1alpha1.Device{ + { + DeviceInterface: "bond0", + DeviceAddresses: []string{"192.168.2.0/24"}, + DeviceMTU: 1500, + DeviceBond: &v1alpha1.Bond{ + BondMode: "balance-rr", + BondDeviceSelectors: []v1alpha1.NetworkDeviceSelector{ + { + NetworkDeviceHardwareAddress: "00:*", + }, + { + NetworkDeviceHardwareAddress: "01:*", + }, + }, + }, + }, + }, + }, + }, + }) + + cfg := config.NewMachineConfig(cfgProvider) + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + for _, link := range []string{"eth0", "eth1"} { + status := network.NewLinkStatus(network.NamespaceName, link) + suite.Require().NoError(suite.State().Create(suite.Ctx(), status)) + } + + rtestutils.AssertNoResource[*network.DeviceConfigSpec](suite.Ctx(), suite.T(), suite.State(), "bond0/000") + + for _, link := range []struct { + name string + hwaddr string + }{ + { + name: "bond0", + hwaddr: "00:11:22:33:44:55", // bond0 will inherit MAC of the first link + }, + { + name: "eth3", + hwaddr: "00:11:22:33:44:55", + }, + { + name: "eth4", + hwaddr: "01:11:22:33:44:55", + }, + { + name: "eth5", + hwaddr: "01:11:22:33:44:ef", + }, + { + name: "eth6", + hwaddr: "02:11:22:33:44:55", + }, + } { + hwaddr, err := net.ParseMAC(link.hwaddr) + suite.Require().NoError(err) + + status := network.NewLinkStatus(network.NamespaceName, link.name) + status.TypedSpec().HardwareAddr = nethelpers.HardwareAddr(hwaddr) + suite.Require().NoError(suite.State().Create(suite.Ctx(), status)) + } + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []string{"bond0/000"}, + func(r *network.DeviceConfigSpec, assert *assert.Assertions) { + assert.Equal(1500, r.TypedSpec().Device.MTU()) + assert.Equal([]string{"192.168.2.0/24"}, r.TypedSpec().Device.Addresses()) + assert.Equal([]string{"eth3", "eth4", "eth5"}, r.TypedSpec().Device.Bond().Interfaces()) + }, + ) +} + +func TestDeviceConfigSpecSuite(t *testing.T) { + suite.Run(t, &DeviceConfigSpecSuite{ + DefaultSuite: ctest.DefaultSuite{ + Timeout: 3 * time.Second, + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(&netctrl.DeviceConfigController{})) + }, + }, + }) +} diff --git a/internal/app/machined/pkg/controllers/network/dns_resolve_cache.go b/internal/app/machined/pkg/controllers/network/dns_resolve_cache.go new file mode 100644 index 0000000..c9e0170 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/dns_resolve_cache.go @@ -0,0 +1,345 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "context" + "errors" + "fmt" + "net" + "net/netip" + "slices" + "strings" + "sync" + "time" + + "github.com/coredns/coredns/plugin/pkg/proxy" + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + dnssrv "github.com/miekg/dns" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/pair" + "go.uber.org/zap" + + "github.com/aenix-io/talm/internal/pkg/dns" + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// DNSResolveCacheController starts dns server on both udp and tcp ports based on finalized network configuration. +type DNSResolveCacheController struct { + State state.State + Logger *zap.Logger + + mx sync.Mutex + handler *dns.Handler + nodeHandler *dns.NodeHandler + cache *dns.Cache + runners map[runnerConfig]pair.Pair[func(), <-chan struct{}] + reconcile chan struct{} + originalCtx context.Context //nolint:containedctx +} + +// Name implements controller.Controller interface. +func (ctrl *DNSResolveCacheController) Name() string { + return "network.DNSResolveCacheController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *DNSResolveCacheController) Inputs() []controller.Input { + return []controller.Input{ + safe.Input[*network.DNSUpstream](controller.InputWeak), + { + Namespace: network.NamespaceName, + Type: network.HostDNSConfigType, + ID: optional.Some(network.HostDNSConfigID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *DNSResolveCacheController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.DNSResolveCacheType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *DNSResolveCacheController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + ctrl.init(ctx) + + ctrl.mx.Lock() + defer ctrl.mx.Unlock() + + defer ctrl.stopRunners(ctx, false) + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + case <-ctrl.reconcile: + for cfg, stop := range ctrl.runners { + select { + default: + continue + case <-stop.F2: + } + + stop.F1() + delete(ctrl.runners, cfg) + } + } + + cfg, err := safe.ReaderGetByID[*network.HostDNSConfig](ctx, r, network.HostDNSConfigID) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting host dns config: %w", err) + } + + r.StartTrackingOutputs() + + if !cfg.TypedSpec().Enabled { + ctrl.stopRunners(ctx, true) + + if err = safe.CleanupOutputs[*network.DNSResolveCache](ctx, r); err != nil { + return fmt.Errorf("error cleaning up dns status on disable: %w", err) + } + + continue + } + + ctrl.nodeHandler.SetEnabled(cfg.TypedSpec().ResolveMemberNames) + + touchedRunners := make(map[runnerConfig]struct{}, len(ctrl.runners)) + + for _, addr := range cfg.TypedSpec().ListenAddresses { + for _, netwk := range []string{"udp", "tcp"} { + runnerCfg := runnerConfig{net: netwk, addr: addr} + + if _, ok := ctrl.runners[runnerCfg]; !ok { + runner, rErr := newDNSRunner(runnerCfg, ctrl.cache, ctrl.Logger) + if rErr != nil { + return fmt.Errorf("error creating dns runner: %w", rErr) + } + + ctrl.runners[runnerCfg] = pair.MakePair(runner.Start(ctrl.handleDone(ctx, logger))) + } + + if err = ctrl.writeDNSStatus(ctx, r, runnerCfg); err != nil { + return fmt.Errorf("error writing dns status: %w", err) + } + + touchedRunners[runnerCfg] = struct{}{} + } + } + + for runnerCfg, stop := range ctrl.runners { + if _, ok := touchedRunners[runnerCfg]; !ok { + stop.F1() + delete(ctrl.runners, runnerCfg) + + continue + } + } + + upstreams, err := safe.ReaderListAll[*network.DNSUpstream](ctx, r) + if err != nil { + return fmt.Errorf("error getting resolver status: %w", err) + } + + addrs, prxs := make([]string, 0, upstreams.Len()), make([]*proxy.Proxy, 0, upstreams.Len()) + + for it := upstreams.Iterator(); it.Next(); { + prx := it.Value().TypedSpec().Value.Prx + + addrs = append(addrs, prx.Addr()) + prxs = append(prxs, prx.(*proxy.Proxy)) //nolint:forcetypeassert + } + + if ctrl.handler.SetProxy(prxs) { + ctrl.Logger.Info("updated dns server nameservers", zap.Strings("addrs", addrs)) + } + + if err = safe.CleanupOutputs[*network.DNSResolveCache](ctx, r); err != nil { + return fmt.Errorf("error cleaning up dns status: %w", err) + } + } +} + +func (ctrl *DNSResolveCacheController) writeDNSStatus(ctx context.Context, r controller.Runtime, config runnerConfig) error { + return safe.WriterModify(ctx, r, network.NewDNSResolveCache(fmt.Sprintf("%s-%s", config.net, config.addr)), func(drc *network.DNSResolveCache) error { + drc.TypedSpec().Status = "running" + + return nil + }) +} + +func (ctrl *DNSResolveCacheController) init(ctx context.Context) { + if ctrl.runners != nil { + if ctrl.originalCtx != ctx { + // This should not happen, but if it does, it's a bug. + panic("DNSResolveCacheController is called with a different context") + } + + return + } + + ctrl.originalCtx = ctx + ctrl.handler = dns.NewHandler(ctrl.Logger) + ctrl.nodeHandler = dns.NewNodeHandler(ctrl.handler, &stateMapper{state: ctrl.State}, ctrl.Logger) + ctrl.cache = dns.NewCache(ctrl.nodeHandler, ctrl.Logger) + ctrl.runners = map[runnerConfig]pair.Pair[func(), <-chan struct{}]{} + ctrl.reconcile = make(chan struct{}, 1) + + // Ensure we stop all runners when the context is canceled, no matter where we are currently. + // For example if we are in Controller runtime sleeping after error and ctx is canceled, we should stop all runners + // but, we will never call Run method again, so we need to ensure this happens regardless of the current state. + context.AfterFunc(ctx, func() { + ctrl.mx.Lock() + defer ctrl.mx.Unlock() + + ctrl.stopRunners(ctx, true) + }) +} + +func (ctrl *DNSResolveCacheController) stopRunners(ctx context.Context, ignoreCtx bool) { + if !ignoreCtx && ctx.Err() == nil { + // context not yet canceled, preserve runners, cache and handler + return + } + + for _, stop := range ctrl.runners { + stop.F1() + } + + clear(ctrl.runners) + + ctrl.handler.Stop() +} + +func (ctrl *DNSResolveCacheController) handleDone(ctx context.Context, logger *zap.Logger) func(err error) { + return func(err error) { + if ctx.Err() != nil { + if err != nil && !errors.Is(err, net.ErrClosed) { + logger.Error("controller is closing, but error running dns server", zap.Error(err)) + } + + return + } + + if err != nil { + logger.Error("error running dns server", zap.Error(err)) + } + + select { + case ctrl.reconcile <- struct{}{}: + default: + } + } +} + +type runnerConfig struct { + net string + addr netip.AddrPort +} + +func newDNSRunner(cfg runnerConfig, cache *dns.Cache, logger *zap.Logger) (*dns.Server, error) { + if cfg.addr.Addr().Is6() { + cfg.net += "6" + } + + logger = logger.With(zap.String("net", cfg.net), zap.Stringer("addr", cfg.addr)) + + var serverOpts dns.ServerOptions + + switch cfg.net { + case "udp", "udp6": + packetConn, err := dns.NewUDPPacketConn(cfg.net, cfg.addr.String()) + if err != nil { + return nil, fmt.Errorf("error creating %q packet conn: %w", cfg.net, err) + } + + serverOpts = dns.ServerOptions{ + PacketConn: packetConn, + Handler: cache, + Logger: logger, + } + + case "tcp", "tcp6": + listener, err := dns.NewTCPListener(cfg.net, cfg.addr.String()) + if err != nil { + return nil, fmt.Errorf("error creating %q listener: %w", cfg.net, err) + } + + serverOpts = dns.ServerOptions{ + Listener: listener, + Handler: cache, + ReadTimeout: 3 * time.Second, + WriteTimeout: 5 * time.Second, + IdleTimeout: func() time.Duration { return 10 * time.Second }, + MaxTCPQueries: -1, + Logger: logger, + } + } + + return dns.NewServer(serverOpts), nil +} + +type stateMapper struct { + state state.State +} + +func (s *stateMapper) ResolveAddr(ctx context.Context, qType uint16, name string) []netip.Addr { + name = strings.TrimRight(name, ".") + + list, err := safe.ReaderListAll[*cluster.Member](ctx, s.state) + if err != nil { + return nil + } + + elem, ok := list.Find(func(res *cluster.Member) bool { + return fqdnMatch(name, res.TypedSpec().Hostname) || fqdnMatch(name, res.Metadata().ID()) + }) + if !ok { + return nil + } + + result := slices.DeleteFunc(slices.Clone(elem.TypedSpec().Addresses), func(addr netip.Addr) bool { + return !((qType == dnssrv.TypeA && addr.Is4()) || (qType == dnssrv.TypeAAAA && addr.Is6())) + }) + + if len(result) == 0 { + return nil + } + + return result +} + +func fqdnMatch(what, where string) bool { + what = strings.TrimRight(what, ".") + where = strings.TrimRight(where, ".") + + if what == where { + return true + } + + first, _, found := strings.Cut(where, ".") + if !found { + return false + } + + return what == first +} diff --git a/internal/app/machined/pkg/controllers/network/dns_resolve_cache_test.go b/internal/app/machined/pkg/controllers/network/dns_resolve_cache_test.go new file mode 100644 index 0000000..bab96fb --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/dns_resolve_cache_test.go @@ -0,0 +1,286 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network_test + +import ( + "errors" + "net" + "net/netip" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/miekg/dns" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/gen/xtesting/must" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "go.uber.org/zap/zaptest" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type DNSServer struct { + ctest.DefaultSuite +} + +func expectedDNSRunners(port string) []resource.ID { + return []resource.ID{ + "tcp-127.0.0.53:" + port, + "udp-127.0.0.53:" + port, + // our dns server makes no promises about actually starting on IPv6, so we don't check it here either + } +} + +func (suite *DNSServer) TestResolving() { + dnsSlice := []string{"8.8.8.8", "1.1.1.1"} + port := must.Value(getDynamicPort())(suite.T()) + + cfg := network.NewHostDNSConfig(network.HostDNSConfigID) + cfg.TypedSpec().Enabled = true + cfg.TypedSpec().ListenAddresses = makeAddrs(port) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + resolverSpec := network.NewResolverStatus(network.NamespaceName, network.ResolverID) + resolverSpec.TypedSpec().DNSServers = xslices.Map(dnsSlice, netip.MustParseAddr) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), resolverSpec)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), + expectedDNSRunners(port), + func(r *network.DNSResolveCache, assert *assert.Assertions) { + assert.Equal("running", r.TypedSpec().Status) + }, + ) + + rtestutils.AssertLength[*network.DNSUpstream](suite.Ctx(), suite.T(), suite.State(), len(dnsSlice)) + + msg := &dns.Msg{ + MsgHdr: dns.MsgHdr{ + Id: dns.Id(), + RecursionDesired: true, + }, + Question: []dns.Question{ + { + Name: dns.Fqdn("google.com"), + Qtype: dns.TypeA, + Qclass: dns.ClassINET, + }, + }, + } + + var res *dns.Msg + + err := retry.Constant(5*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(func() error { + r, err := dns.Exchange(msg, "127.0.0.53:"+port) + if err != nil { + return retry.ExpectedError(err) + } + + if r.Rcode != dns.RcodeSuccess { + return retry.ExpectedErrorf("expected rcode %d, got %d", dns.RcodeSuccess, r.Rcode) + } + + res = r + + return nil + }) + suite.Require().NoError(err) + suite.Require().Equal(dns.RcodeSuccess, res.Rcode, res) +} + +func (suite *DNSServer) TestSetupStartStop() { + dnsSlice := []string{"8.8.8.8", "1.1.1.1"} + port := must.Value(getDynamicPort())(suite.T()) + + resolverSpec := network.NewResolverStatus(network.NamespaceName, network.ResolverID) + resolverSpec.TypedSpec().DNSServers = xslices.Map(dnsSlice, netip.MustParseAddr) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), resolverSpec)) + + cfg := network.NewHostDNSConfig(network.HostDNSConfigID) + cfg.TypedSpec().Enabled = true + cfg.TypedSpec().ListenAddresses = makeAddrs(port) + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), + expectedDNSRunners(port), + func(r *network.DNSResolveCache, assert *assert.Assertions) { + assert.Equal("running", r.TypedSpec().Status) + }) + + rtestutils.AssertLength[*network.DNSUpstream](suite.Ctx(), suite.T(), suite.State(), len(dnsSlice)) + // stop dns resolver + + cfg.TypedSpec().Enabled = false + suite.Require().NoError(suite.State().Update(suite.Ctx(), cfg)) + + for _, runner := range expectedDNSRunners(port) { + ctest.AssertNoResource[*network.DNSResolveCache](suite, runner) + } + + for _, d := range dnsSlice { + ctest.AssertNoResource[*network.DNSUpstream](suite, d) + } + + // start dns resolver again + cfg.TypedSpec().Enabled = true + suite.Require().NoError(suite.State().Update(suite.Ctx(), cfg)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), expectedDNSRunners(port), func(r *network.DNSResolveCache, assert *assert.Assertions) { + assert.Equal("running", r.TypedSpec().Status) + }) + + rtestutils.AssertLength[*network.DNSUpstream](suite.Ctx(), suite.T(), suite.State(), len(dnsSlice)) +} + +func (suite *DNSServer) TestResolveMembers() { + port := must.Value(getDynamicPort())(suite.T()) + + const ( + id = "talos-default-controlplane-1" + id2 = "foo.example.com." + ) + + member := cluster.NewMember(cluster.NamespaceName, id) + *member.TypedSpec() = cluster.MemberSpec{ + NodeID: id, + Addresses: []netip.Addr{ + netip.MustParseAddr("172.20.0.2"), + }, + Hostname: id, + MachineType: machine.TypeControlPlane, + OperatingSystem: "Talos dev", + ControlPlane: nil, + } + + suite.Require().NoError(suite.State().Create(suite.Ctx(), member)) + + member = cluster.NewMember(cluster.NamespaceName, id2) + *member.TypedSpec() = cluster.MemberSpec{ + NodeID: id2, + Addresses: []netip.Addr{ + netip.MustParseAddr("172.20.0.3"), + }, + Hostname: id2, + MachineType: machine.TypeWorker, + OperatingSystem: "Talos dev", + ControlPlane: nil, + } + + suite.Require().NoError(suite.State().Create(suite.Ctx(), member)) + + cfg := network.NewHostDNSConfig(network.HostDNSConfigID) + cfg.TypedSpec().Enabled = true + cfg.TypedSpec().ListenAddresses = makeAddrs(port) + cfg.TypedSpec().ResolveMemberNames = true + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), + expectedDNSRunners(port), + func(r *network.DNSResolveCache, assert *assert.Assertions) { + assert.Equal("running", r.TypedSpec().Status) + }, + ) + + suite.Require().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(func() error { + exchange, err := dns.Exchange( + &dns.Msg{ + MsgHdr: dns.MsgHdr{Id: dns.Id(), RecursionDesired: true}, + Question: []dns.Question{ + {Name: dns.Fqdn(id), Qtype: dns.TypeA, Qclass: dns.ClassINET}, + }, + }, + "127.0.0.53:"+port, + ) + if err != nil { + return retry.ExpectedError(err) + } + + if exchange.Rcode != dns.RcodeSuccess { + return retry.ExpectedErrorf("expected rcode %d, got %d for %q", dns.RcodeSuccess, exchange.Rcode, id) + } + + proper := dns.Fqdn(id) + + if exchange.Answer[0].Header().Name != proper { + return retry.ExpectedErrorf("expected answer name %q, got %q", proper, exchange.Answer[0].Header().Name) + } + + return nil + })) + + suite.Require().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(func() error { + exchange, err := dns.Exchange( + &dns.Msg{ + MsgHdr: dns.MsgHdr{Id: dns.Id(), RecursionDesired: true}, + Question: []dns.Question{ + {Name: dns.Fqdn("foo"), Qtype: dns.TypeA, Qclass: dns.ClassINET}, + }, + }, + "127.0.0.53:"+port, + ) + if err != nil { + return retry.ExpectedError(err) + } + + if exchange.Rcode != dns.RcodeSuccess { + return retry.ExpectedErrorf("expected rcode %d, got %d for %q", dns.RcodeSuccess, exchange.Rcode, id2) + } + + if !exchange.Answer[0].(*dns.A).A.Equal(net.ParseIP("172.20.0.3")) { + return retry.ExpectedError(errors.New("unexpected ip")) + } + + return nil + })) +} + +func TestDNSServer(t *testing.T) { + suite.Run(t, &DNSServer{ + DefaultSuite: ctest.DefaultSuite{ + Timeout: 10 * time.Second, + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(&netctrl.DNSUpstreamController{})) + suite.Require().NoError(suite.Runtime().RegisterController(&netctrl.DNSResolveCacheController{ + Logger: zaptest.NewLogger(t), + State: suite.State(), + })) + }, + }, + }) +} + +func getDynamicPort() (string, error) { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return "", err + } + + closeOnce := sync.OnceValue(l.Close) + + defer closeOnce() //nolint:errcheck + + _, port, err := net.SplitHostPort(l.Addr().String()) + if err != nil { + return "", err + } + + return port, closeOnce() +} + +func makeAddrs(port string) []netip.AddrPort { + return []netip.AddrPort{ + netip.MustParseAddrPort("127.0.0.53:" + port), + } +} diff --git a/internal/app/machined/pkg/controllers/network/dns_upstream.go b/internal/app/machined/pkg/controllers/network/dns_upstream.go new file mode 100644 index 0000000..9831233 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/dns_upstream.go @@ -0,0 +1,162 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "context" + "net" + "time" + + "github.com/coredns/coredns/plugin/pkg/proxy" + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// DNSUpstreamController is a controller that manages DNS upstreams. +type DNSUpstreamController struct{} + +// Name implements controller.Controller interface. +func (ctrl *DNSUpstreamController) Name() string { + return "network.DNSUpstreamController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *DNSUpstreamController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: network.NamespaceName, + Type: network.HostDNSConfigType, + ID: optional.Some(network.HostDNSConfigID), + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.ResolverStatusType, + ID: optional.Some(network.ResolverID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *DNSUpstreamController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.DNSUpstreamType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +func (ctrl *DNSUpstreamController) Run(ctx context.Context, r controller.Runtime, l *zap.Logger) error { + defer ctrl.cleanupUpstream(context.Background(), r, nil, l) + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + if err := ctrl.run(ctx, r, l); err != nil { + return err + } + + r.ResetRestartBackoff() + } +} + +func (ctrl *DNSUpstreamController) run(ctx context.Context, r controller.Runtime, l *zap.Logger) error { + touchedIDs := map[resource.ID]struct{}{} + + defer ctrl.cleanupUpstream(ctx, r, touchedIDs, l) + + cfg, err := safe.ReaderGetByID[*network.HostDNSConfig](ctx, r, network.HostDNSConfigID) + if err != nil { + if state.IsNotFoundError(err) { + return nil + } + + return err + } + + if !cfg.TypedSpec().Enabled { + // host DNS is disabled, cleanup all upstreams + return nil + } + + rs, err := safe.ReaderGetByID[*network.ResolverStatus](ctx, r, network.ResolverID) + if err != nil { + if state.IsNotFoundError(err) { + return nil + } + + return err + } + + for _, s := range rs.TypedSpec().DNSServers { + remoteAddr := s.String() + + if err = safe.WriterModify[*network.DNSUpstream]( + ctx, + r, + network.NewDNSUpstream(remoteAddr), + func(u *network.DNSUpstream) error { + touchedIDs[u.Metadata().ID()] = struct{}{} + + if u.TypedSpec().Value.Prx != nil { + return nil + } + + prx := proxy.NewProxy(remoteAddr, net.JoinHostPort(remoteAddr, "53"), "dns") + + prx.Start(500 * time.Millisecond) + + u.TypedSpec().Value.Prx = prx + + l.Info("created dns upstream", zap.String("addr", remoteAddr)) + + return nil + }, + ); err != nil { + return err + } + } + + return nil +} + +func (ctrl *DNSUpstreamController) cleanupUpstream(ctx context.Context, r controller.Runtime, touchedIDs map[resource.ID]struct{}, l *zap.Logger) { + list, err := safe.ReaderListAll[*network.DNSUpstream](ctx, r) + if err != nil { + l.Error("error listing upstreams", zap.Error(err)) + + return + } + + for it := list.Iterator(); it.Next(); { + val := it.Value() + md := val.Metadata() + + if _, ok := touchedIDs[md.ID()]; !ok { + val.TypedSpec().Value.Prx.Stop() + + if err = r.Destroy(ctx, md); err != nil { + l.Error("error destroying upstream", zap.Error(err), zap.String("id", md.ID())) + + return + } + + l.Info("destroyed dns upstream", zap.String("addr", md.ID())) + } + } +} diff --git a/internal/app/machined/pkg/controllers/network/etcfile.go b/internal/app/machined/pkg/controllers/network/etcfile.go new file mode 100644 index 0000000..bf2ecec --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/etcfile.go @@ -0,0 +1,261 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "bytes" + "context" + "fmt" + "net/netip" + "os" + "path/filepath" + "strings" + "text/tabwriter" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/value" + "go.uber.org/zap" + + efiles "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/files" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + talosconfig "github.com/siderolabs/talos/pkg/machinery/config" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/files" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// EtcFileController creates /etc/hostname and /etc/resolv.conf files based on finalized network configuration. +type EtcFileController struct { + PodResolvConfPath string + V1Alpha1Mode runtime.Mode +} + +// Name implements controller.Controller interface. +func (ctrl *EtcFileController) Name() string { + return "network.EtcFileController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *EtcFileController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: config.MachineConfigType, + ID: optional.Some(config.V1Alpha1ID), + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.HostnameStatusType, + ID: optional.Some(network.HostnameID), + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.ResolverStatusType, + ID: optional.Some(network.ResolverID), + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.NodeAddressType, + ID: optional.Some(network.NodeAddressDefaultID), + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.HostDNSConfigType, + ID: optional.Some(network.HostDNSConfigID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *EtcFileController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: files.EtcFileSpecType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *EtcFileController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + var cfgProvider talosconfig.Config + + cfg, err := safe.ReaderGetByID[*config.MachineConfig](ctx, r, config.V1Alpha1ID) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting config: %w", err) + } + } else { + cfgProvider = cfg.Config() + } + + hostnameStatus, err := safe.ReaderGetByID[*network.HostnameStatus](ctx, r, network.HostnameID) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting hostname status: %w", err) + } + } + + nodeAddressStatus, err := safe.ReaderGetByID[*network.NodeAddress](ctx, r, network.NodeAddressDefaultID) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting network address status: %w", err) + } + } + + resolverStatus, err := safe.ReaderGetByID[*network.ResolverStatus](ctx, r, network.ResolverID) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error resolver status: %w", err) + } + } + + hostDNSCfg, err := safe.ReaderGetByID[*network.HostDNSConfig](ctx, r, network.HostDNSConfigID) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting host dns config: %w", err) + } + } + + var hostnameStatusSpec *network.HostnameStatusSpec + if hostnameStatus != nil { + hostnameStatusSpec = hostnameStatus.TypedSpec() + } + + if resolverStatus != nil && hostDNSCfg != nil && !ctrl.V1Alpha1Mode.InContainer() { + // in container mode, keep the original resolv.conf to use the resolvers supplied by the container runtime + if err = safe.WriterModify(ctx, r, files.NewEtcFileSpec(files.NamespaceName, "resolv.conf"), + func(r *files.EtcFileSpec) error { + r.TypedSpec().Contents = renderResolvConf(pickNameservers(hostDNSCfg, resolverStatus), hostnameStatusSpec, cfgProvider) + r.TypedSpec().Mode = 0o644 + + return nil + }); err != nil { + return fmt.Errorf("error modifying resolv.conf: %w", err) + } + } + + if resolverStatus != nil && hostDNSCfg != nil { + dnsServers := resolverStatus.TypedSpec().DNSServers + + if !value.IsZero(hostDNSCfg.TypedSpec().ServiceHostDNSAddress) { + dnsServers = []netip.Addr{hostDNSCfg.TypedSpec().ServiceHostDNSAddress} + } + + conf := renderResolvConf(dnsServers, hostnameStatusSpec, cfgProvider) + + if err = os.MkdirAll(filepath.Dir(ctrl.PodResolvConfPath), 0o755); err != nil { + return fmt.Errorf("error creating pod resolv.conf dir: %w", err) + } + + err = efiles.UpdateFile(ctrl.PodResolvConfPath, conf, 0o644) + if err != nil { + return fmt.Errorf("error writing pod resolv.conf: %w", err) + } + } + + if hostnameStatus != nil && nodeAddressStatus != nil { + if err = safe.WriterModify(ctx, r, files.NewEtcFileSpec(files.NamespaceName, "hosts"), + func(r *files.EtcFileSpec) error { + r.TypedSpec().Contents, err = ctrl.renderHosts(hostnameStatus.TypedSpec(), nodeAddressStatus.TypedSpec(), cfgProvider) + r.TypedSpec().Mode = 0o644 + + return err + }); err != nil { + return fmt.Errorf("error modifying hosts: %w", err) + } + } + + r.ResetRestartBackoff() + } +} + +var localDNS = []netip.Addr{netip.MustParseAddr("127.0.0.53")} + +func pickNameservers(hostDNSCfg *network.HostDNSConfig, resolverStatus *network.ResolverStatus) []netip.Addr { + if hostDNSCfg.TypedSpec().Enabled { + // local dns resolve cache enabled, route host dns requests to 127.0.0.1 + return localDNS + } + + return resolverStatus.TypedSpec().DNSServers +} + +func renderResolvConf(nameservers []netip.Addr, hostnameStatus *network.HostnameStatusSpec, cfgProvider talosconfig.Config) []byte { + var buf bytes.Buffer + + for i, ns := range nameservers { + if i >= 3 { + // only use first 3 nameservers, see MAXNS in https://linux.die.net/man/5/resolv.conf + break + } + + fmt.Fprintf(&buf, "nameserver %s\n", ns) + } + + var disableSearchDomain bool + if cfgProvider != nil && cfgProvider.Machine() != nil { + disableSearchDomain = cfgProvider.Machine().Network().DisableSearchDomain() + } + + if !disableSearchDomain && hostnameStatus != nil && hostnameStatus.Domainname != "" { + fmt.Fprintf(&buf, "\nsearch %s\n", hostnameStatus.Domainname) + } + + return buf.Bytes() +} + +func (ctrl *EtcFileController) renderHosts(hostnameStatus *network.HostnameStatusSpec, nodeAddressStatus *network.NodeAddressSpec, cfgProvider talosconfig.Config) ([]byte, error) { + var buf bytes.Buffer + + tabW := tabwriter.NewWriter(&buf, 0, 0, 1, ' ', 0) + + write := func(s string) { + tabW.Write([]byte(s)) //nolint:errcheck + } + + write("127.0.0.1\tlocalhost\n") + + write(fmt.Sprintf("%s\t%s", nodeAddressStatus.Addresses[0].Addr(), hostnameStatus.FQDN())) + + if hostnameStatus.Hostname != hostnameStatus.FQDN() { + write(" " + hostnameStatus.Hostname) + } + + write("\n") + + write("::1\tlocalhost ip6-localhost ip6-loopback\n") + write("ff02::1\tip6-allnodes\n") + write("ff02::2\tip6-allrouters\n") + + if cfgProvider != nil && cfgProvider.Machine() != nil { + for _, extraHost := range cfgProvider.Machine().Network().ExtraHosts() { + write(fmt.Sprintf("%s\t%s\n", extraHost.IP(), strings.Join(extraHost.Aliases(), " "))) + } + } + + if err := tabW.Flush(); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} diff --git a/internal/app/machined/pkg/controllers/network/etcfile_test.go b/internal/app/machined/pkg/controllers/network/etcfile_test.go new file mode 100644 index 0000000..d776e86 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/etcfile_test.go @@ -0,0 +1,320 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network_test + +import ( + "context" + "errors" + "log" + "net/netip" + "net/url" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-pointer" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + v1alpha1runtime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/files" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type EtcFileConfigSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc + + cfg *config.MachineConfig + defaultAddress *network.NodeAddress + hostnameStatus *network.HostnameStatus + resolverStatus *network.ResolverStatus + hostDNSConfig *network.HostDNSConfig + + podResolvConfPath string +} + +func (suite *EtcFileConfigSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.startRuntime() + + suite.podResolvConfPath = filepath.Join(suite.T().TempDir(), "resolv.conf") + + suite.Assert().NoFileExists(suite.podResolvConfPath) + + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.EtcFileController{ + PodResolvConfPath: suite.podResolvConfPath, + V1Alpha1Mode: v1alpha1runtime.ModeMetal, + })) + + u, err := url.Parse("https://foo:6443") + suite.Require().NoError(err) + + suite.cfg = config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineNetwork: &v1alpha1.NetworkConfig{ + ExtraHostEntries: []*v1alpha1.ExtraHost{ + { + HostIP: "10.0.0.1", + HostAliases: []string{"a", "b"}, + }, + { + HostIP: "10.0.0.2", + HostAliases: []string{"c", "d"}, + }, + }, + }, + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: u, + }, + }, + }, + }, + ), + ) + + suite.defaultAddress = network.NewNodeAddress(network.NamespaceName, network.NodeAddressDefaultID) + suite.defaultAddress.TypedSpec().Addresses = []netip.Prefix{netip.MustParsePrefix("33.11.22.44/32")} + + suite.hostnameStatus = network.NewHostnameStatus(network.NamespaceName, network.HostnameID) + suite.hostnameStatus.TypedSpec().Hostname = "foo" + suite.hostnameStatus.TypedSpec().Domainname = "example.com" + + suite.resolverStatus = network.NewResolverStatus(network.NamespaceName, network.ResolverID) + suite.resolverStatus.TypedSpec().DNSServers = []netip.Addr{ + netip.MustParseAddr("1.1.1.1"), + netip.MustParseAddr("2.2.2.2"), + netip.MustParseAddr("3.3.3.3"), + netip.MustParseAddr("4.4.4.4"), + } + + suite.hostDNSConfig = network.NewHostDNSConfig(network.HostDNSConfigID) + suite.hostDNSConfig.TypedSpec().Enabled = true + suite.hostDNSConfig.TypedSpec().ListenAddresses = []netip.AddrPort{ + netip.MustParseAddrPort("127.0.0.53:53"), + netip.MustParseAddrPort("10.96.0.9:53"), + } + suite.hostDNSConfig.TypedSpec().ServiceHostDNSAddress = netip.MustParseAddr("10.96.0.9") +} + +func (suite *EtcFileConfigSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +type etcFileContents struct { + hosts string + resolvConf string + resolvGlobalConf string +} + +//nolint:gocyclo +func (suite *EtcFileConfigSuite) testFiles(resources []resource.Resource, contents etcFileContents) { + for _, r := range resources { + suite.Require().NoError(suite.state.Create(suite.ctx, r)) + } + + var ( + expectedIDs []string + unexpectedIDs []string + ) + + if contents.resolvConf != "" { + expectedIDs = append(expectedIDs, "resolv.conf") + } else { + unexpectedIDs = append(unexpectedIDs, "resolv.conf") + } + + if contents.hosts != "" { + expectedIDs = append(expectedIDs, "hosts") + } else { + unexpectedIDs = append(unexpectedIDs, "hosts") + } + + assertResources( + suite.ctx, + suite.T(), + suite.state, + expectedIDs, + func(r *files.EtcFileSpec, asrt *assert.Assertions) { + switch r.Metadata().ID() { + case "hosts": + asrt.Equal(contents.hosts, string(r.TypedSpec().Contents)) + case "resolv.conf": + asrt.Equal(contents.resolvConf, string(r.TypedSpec().Contents)) + } + }, + ) + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(func() error { + if contents.resolvGlobalConf == "" { + _, err := os.Lstat(suite.podResolvConfPath) + + switch { + case err == nil: + return retry.ExpectedErrorf("unexpected pod %s", suite.podResolvConfPath) + case errors.Is(err, os.ErrNotExist): + return nil + default: + return err + } + } + + file, err := os.ReadFile(suite.podResolvConfPath) + + switch { + case errors.Is(err, os.ErrNotExist): + return retry.ExpectedErrorf("missing pod %s", suite.podResolvConfPath) + case err != nil: + return err + case len(file) == 0: + return retry.ExpectedErrorf("empty pod %s", suite.podResolvConfPath) + default: + suite.Assert().Equal(contents.resolvGlobalConf, string(file)) + + return nil + } + }), + ) + + for _, id := range unexpectedIDs { + assertNoResource[*files.EtcFileSpec](suite.ctx, suite.T(), suite.state, id) + } +} + +func (suite *EtcFileConfigSuite) TestComplete() { + suite.testFiles( + []resource.Resource{suite.cfg, suite.defaultAddress, suite.hostnameStatus, suite.resolverStatus, suite.hostDNSConfig}, + etcFileContents{ + hosts: "127.0.0.1 localhost\n33.11.22.44 foo.example.com foo\n::1 localhost ip6-localhost ip6-loopback\nff02::1 ip6-allnodes\nff02::2 ip6-allrouters\n10.0.0.1 a b\n10.0.0.2 c d\n", //nolint:lll + resolvConf: "nameserver 127.0.0.53\n\nsearch example.com\n", + resolvGlobalConf: "nameserver 10.96.0.9\n\nsearch example.com\n", + }, + ) +} + +func (suite *EtcFileConfigSuite) TestNoExtraHosts() { + suite.testFiles( + []resource.Resource{suite.defaultAddress, suite.hostnameStatus, suite.resolverStatus, suite.hostDNSConfig}, + etcFileContents{ + hosts: "127.0.0.1 localhost\n33.11.22.44 foo.example.com foo\n::1 localhost ip6-localhost ip6-loopback\nff02::1 ip6-allnodes\nff02::2 ip6-allrouters\n", + resolvConf: "nameserver 127.0.0.53\n\nsearch example.com\n", + resolvGlobalConf: "nameserver 10.96.0.9\n\nsearch example.com\n", + }, + ) +} + +func (suite *EtcFileConfigSuite) TestNoSearchDomain() { + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineNetwork: &v1alpha1.NetworkConfig{ + NetworkDisableSearchDomain: pointer.To(true), + }, + }, + }, + ), + ) + suite.testFiles( + []resource.Resource{cfg, suite.defaultAddress, suite.hostnameStatus, suite.resolverStatus, suite.hostDNSConfig}, + etcFileContents{ + hosts: "127.0.0.1 localhost\n33.11.22.44 foo.example.com foo\n::1 localhost ip6-localhost ip6-loopback\nff02::1 ip6-allnodes\nff02::2 ip6-allrouters\n", + resolvConf: "nameserver 127.0.0.53\n", + resolvGlobalConf: "nameserver 10.96.0.9\n", + }, + ) +} + +func (suite *EtcFileConfigSuite) TestNoDomainname() { + suite.hostnameStatus.TypedSpec().Domainname = "" + + suite.testFiles( + []resource.Resource{suite.defaultAddress, suite.hostnameStatus, suite.resolverStatus, suite.hostDNSConfig}, + etcFileContents{ + hosts: "127.0.0.1 localhost\n33.11.22.44 foo\n::1 localhost ip6-localhost ip6-loopback\nff02::1 ip6-allnodes\nff02::2 ip6-allrouters\n", + resolvConf: "nameserver 127.0.0.53\n", + resolvGlobalConf: "nameserver 10.96.0.9\n", + }, + ) +} + +func (suite *EtcFileConfigSuite) TestOnlyResolvers() { + suite.testFiles( + []resource.Resource{suite.resolverStatus, suite.hostDNSConfig}, + etcFileContents{ + hosts: "", + resolvConf: "nameserver 127.0.0.53\n", + resolvGlobalConf: "nameserver 10.96.0.9\n", + }, + ) +} + +func (suite *EtcFileConfigSuite) TestOnlyHostname() { + suite.testFiles( + []resource.Resource{suite.defaultAddress, suite.hostnameStatus}, + etcFileContents{ + hosts: "127.0.0.1 localhost\n33.11.22.44 foo.example.com foo\n::1 localhost ip6-localhost ip6-loopback\nff02::1 ip6-allnodes\nff02::2 ip6-allrouters\n", + resolvConf: "", + resolvGlobalConf: "", + }, + ) +} + +func (suite *EtcFileConfigSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + if _, err := os.Lstat(suite.podResolvConfPath); err == nil { + suite.Require().NoError(os.Remove(suite.podResolvConfPath)) + } + + suite.wg.Wait() +} + +func TestEtcFileConfigSuite(t *testing.T) { + suite.Run(t, new(EtcFileConfigSuite)) +} diff --git a/internal/app/machined/pkg/controllers/network/hardware_addr.go b/internal/app/machined/pkg/controllers/network/hardware_addr.go new file mode 100644 index 0000000..d2ecf28 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/hardware_addr.go @@ -0,0 +1,108 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// HardwareAddrController manages secrets.Etcd based on configuration. +type HardwareAddrController struct{} + +// Name implements controller.Controller interface. +func (ctrl *HardwareAddrController) Name() string { + return "network.HardwareAddrController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *HardwareAddrController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: network.NamespaceName, + Type: network.LinkStatusType, + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *HardwareAddrController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.HardwareAddrType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *HardwareAddrController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + // list the existing HardwareAddr resources and mark them all to be deleted, as the actual link is discovered via netlink, resource ID is removed from the list + list, err := r.List(ctx, resource.NewMetadata(network.NamespaceName, network.HardwareAddrType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing resources: %w", err) + } + + itemsToDelete := map[resource.ID]struct{}{} + + for _, r := range list.Items { + itemsToDelete[r.Metadata().ID()] = struct{}{} + } + + // list links and find the first physical link + links, err := r.List(ctx, resource.NewMetadata(network.NamespaceName, network.LinkStatusType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing resources: %w", err) + } + + for _, res := range links.Items { + link := res.(*network.LinkStatus) //nolint:errcheck,forcetypeassert + + if !link.TypedSpec().Physical() { + continue + } + + if err = r.Modify(ctx, network.NewHardwareAddr(network.NamespaceName, network.FirstHardwareAddr), func(r resource.Resource) error { + spec := r.(*network.HardwareAddr).TypedSpec() + + spec.HardwareAddr = link.TypedSpec().HardwareAddr + spec.Name = link.Metadata().ID() + + return nil + }); err != nil { + return fmt.Errorf("error modifying resource: %w", err) + } + + delete(itemsToDelete, network.FirstHardwareAddr) + + // as link status are listed in sorted order, first physical link in the list is the one we need + break + } + + for id := range itemsToDelete { + if err = r.Destroy(ctx, resource.NewMetadata(network.NamespaceName, network.HardwareAddrType, id, resource.VersionUndefined)); err != nil { + return fmt.Errorf("error deleting resource %q: %w", id, err) + } + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/network/hardware_addr_test.go b/internal/app/machined/pkg/controllers/network/hardware_addr_test.go new file mode 100644 index 0000000..bea444b --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/hardware_addr_test.go @@ -0,0 +1,152 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:dupl +package network_test + +import ( + "context" + "log" + "net" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type HardwareAddrSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *HardwareAddrSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.HardwareAddrController{})) + + suite.startRuntime() +} + +func (suite *HardwareAddrSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *HardwareAddrSuite) assertHWAddr(requiredIDs []string, check func(*network.HardwareAddr, *assert.Assertions)) { + assertResources(suite.ctx, suite.T(), suite.state, requiredIDs, check) +} + +func (suite *HardwareAddrSuite) assertNoHWAddr(id string) error { + resources, err := suite.state.List( + suite.ctx, + resource.NewMetadata(network.NamespaceName, network.HardwareAddrType, "", resource.VersionUndefined), + ) + if err != nil { + return err + } + + for _, res := range resources.Items { + if res.Metadata().ID() == id { + return retry.ExpectedErrorf("interface %q is still there", id) + } + } + + return nil +} + +func (suite *HardwareAddrSuite) TestFirst() { + mustParseMAC := func(addr string) nethelpers.HardwareAddr { + mac, err := net.ParseMAC(addr) + suite.Require().NoError(err) + + return nethelpers.HardwareAddr(mac) + } + + eth0 := network.NewLinkStatus(network.NamespaceName, "eth0") + eth0.TypedSpec().Type = nethelpers.LinkEther + eth0.TypedSpec().HardwareAddr = mustParseMAC("56:a0:a0:87:1c:fa") + + eth1 := network.NewLinkStatus(network.NamespaceName, "eth1") + eth1.TypedSpec().Type = nethelpers.LinkEther + eth1.TypedSpec().HardwareAddr = mustParseMAC("6a:2b:bd:b2:fc:e0") + + bond0 := network.NewLinkStatus(network.NamespaceName, "bond0") + bond0.TypedSpec().Type = nethelpers.LinkEther + bond0.TypedSpec().Kind = "bond" + bond0.TypedSpec().HardwareAddr = mustParseMAC("56:a0:a0:87:1c:fb") + + suite.Require().NoError(suite.state.Create(suite.ctx, bond0)) + suite.Require().NoError(suite.state.Create(suite.ctx, eth1)) + + suite.assertHWAddr( + []string{network.FirstHardwareAddr}, func(r *network.HardwareAddr, asrt *assert.Assertions) { + asrt.Equal(eth1.Metadata().ID(), r.TypedSpec().Name) + asrt.Equal("6a:2b:bd:b2:fc:e0", net.HardwareAddr(r.TypedSpec().HardwareAddr).String()) + }, + ) + + suite.Require().NoError(suite.state.Create(suite.ctx, eth0)) + + suite.assertHWAddr( + []string{network.FirstHardwareAddr}, func(r *network.HardwareAddr, asrt *assert.Assertions) { + asrt.Equal(eth0.Metadata().ID(), r.TypedSpec().Name) + asrt.Equal("56:a0:a0:87:1c:fa", net.HardwareAddr(r.TypedSpec().HardwareAddr).String()) + }, + ) + + suite.Require().NoError(suite.state.Destroy(suite.ctx, eth0.Metadata())) + suite.Require().NoError(suite.state.Destroy(suite.ctx, eth1.Metadata())) + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertNoHWAddr(network.FirstHardwareAddr) + }, + ), + ) +} + +func (suite *HardwareAddrSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestHardwareAddrSuite(t *testing.T) { + suite.Run(t, new(HardwareAddrSuite)) +} diff --git a/internal/app/machined/pkg/controllers/network/hostdns_config.go b/internal/app/machined/pkg/controllers/network/hostdns_config.go new file mode 100644 index 0000000..77e869b --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/hostdns_config.go @@ -0,0 +1,216 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "context" + "fmt" + "net/netip" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/value" + "github.com/siderolabs/go-procfs/procfs" + "go.uber.org/zap" + + talosconfig "github.com/siderolabs/talos/pkg/machinery/config" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// HostDNSConfigController manages network.HostDNSConfig based on machine configuration. +type HostDNSConfigController struct { + Cmdline *procfs.Cmdline +} + +// Name implements controller.Controller interface. +func (ctrl *HostDNSConfigController) Name() string { + return "network.HostDNSConfigController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *HostDNSConfigController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: config.MachineConfigType, + ID: optional.Some(config.V1Alpha1ID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *HostDNSConfigController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.HostDNSConfigType, + Kind: controller.OutputExclusive, + }, + { + Type: network.AddressSpecType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *HostDNSConfigController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + var cfgProvider talosconfig.Config + + cfg, err := safe.ReaderGetByID[*config.MachineConfig](ctx, r, config.V1Alpha1ID) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting config: %w", err) + } + } else if cfg.Config().Machine() != nil { + cfgProvider = cfg.Config() + } + + var newServiceAddr netip.Addr + + if err := safe.WriterModify(ctx, r, network.NewHostDNSConfig(network.HostDNSConfigID), func(res *network.HostDNSConfig) error { + res.TypedSpec().ListenAddresses = []netip.AddrPort{ + netip.MustParseAddrPort("127.0.0.53:53"), + } + + res.TypedSpec().ServiceHostDNSAddress = netip.Addr{} + + if cfgProvider == nil { + res.TypedSpec().Enabled = false + + return nil + } + + res.TypedSpec().Enabled = cfgProvider.Machine().Features().HostDNS().Enabled() + res.TypedSpec().ResolveMemberNames = cfgProvider.Machine().Features().HostDNS().ResolveMemberNames() + + if cfgProvider.Machine().Features().HostDNS().ForwardKubeDNSToHost() { + serviceCIDRStr := cfgProvider.Cluster().Network().ServiceCIDRs()[0] + + serviceCIDR, err := netip.ParsePrefix(serviceCIDRStr) + if err != nil { + return fmt.Errorf("error parsing service CIDR: %w", err) + } + + newServiceAddr = serviceCIDR.Addr() + for range 9 { + newServiceAddr = newServiceAddr.Next() + } + + res.TypedSpec().ListenAddresses = append(res.TypedSpec().ListenAddresses, netip.AddrPortFrom(newServiceAddr, 53)) + res.TypedSpec().ServiceHostDNSAddress = newServiceAddr + } + + return nil + }); err != nil { + return fmt.Errorf("error writing host dns config: %w", err) + } + + var touched *network.AddressSpec + + if !value.IsZero(newServiceAddr) { + touched, err = updateSpec(ctx, r, newServiceAddr, logger) + if err != nil { + return err + } + } + + if err = ctrl.cleanupLinkSpecs( + ctx, + r, + func(id resource.ID) bool { + if touched == nil { + return false + } + + return id == touched.Metadata().ID() + }, + logger, + ); err != nil { + return err + } + + r.ResetRestartBackoff() + } +} + +func (ctrl *HostDNSConfigController) cleanupLinkSpecs(ctx context.Context, r controller.Runtime, checkResource func(id resource.ID) bool, logger *zap.Logger) error { + list, err := safe.ReaderList[*network.AddressSpec](ctx, r, network.NewAddressSpec(network.ConfigNamespaceName, "").Metadata()) + if err != nil { + return err + } + + for iter := list.Iterator(); iter.Next(); { + link := iter.Value() + + if link.Metadata().Owner() != ctrl.Name() { + continue + } + + if checkResource(link.Metadata().ID()) { + continue + } + + if err = r.Destroy(ctx, link.Metadata()); err != nil && !state.IsNotFoundError(err) { + return err + } + + logger.Info("destroyed link spec", zap.String("link_id", link.Metadata().ID())) + } + + return nil +} + +func updateSpec(ctx context.Context, r controller.Runtime, newServiceAddr netip.Addr, logger *zap.Logger) (*network.AddressSpec, error) { + newDNSAddrPrefix := netip.PrefixFrom(newServiceAddr, newServiceAddr.BitLen()) + + logger.Debug("creating new host dns address spec", zap.String("address", newServiceAddr.String())) + + res, err := safe.WriterModifyWithResult( + ctx, + r, + network.NewAddressSpec( + network.ConfigNamespaceName, + network.LayeredID(network.ConfigOperator, network.AddressID("lo", newDNSAddrPrefix)), + ), + func(r *network.AddressSpec) error { + spec := r.TypedSpec() + + spec.Address = newDNSAddrPrefix + spec.ConfigLayer = network.ConfigOperator + + if newServiceAddr.Is4() { + spec.Family = nethelpers.FamilyInet4 + } else { + spec.Family = nethelpers.FamilyInet6 + } + + spec.Flags = nethelpers.AddressFlags(nethelpers.AddressPermanent) + spec.LinkName = "lo" + spec.Scope = nethelpers.ScopeHost + + return nil + }, + ) + if err != nil { + return nil, fmt.Errorf("error modifying address: %w", err) + } + + return res, nil +} diff --git a/internal/app/machined/pkg/controllers/network/hostname_config.go b/internal/app/machined/pkg/controllers/network/hostname_config.go new file mode 100644 index 0000000..c757ec4 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/hostname_config.go @@ -0,0 +1,270 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "context" + "crypto/sha256" + "fmt" + "strings" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/martinlindhe/base36" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/go-procfs/procfs" + "go.uber.org/zap" + + talosconfig "github.com/siderolabs/talos/pkg/machinery/config" + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// HostnameConfigController manages network.HostnameSpec based on machine configuration, kernel cmdline. +type HostnameConfigController struct { + Cmdline *procfs.Cmdline +} + +// Name implements controller.Controller interface. +func (ctrl *HostnameConfigController) Name() string { + return "network.HostnameConfigController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *HostnameConfigController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: config.MachineConfigType, + ID: optional.Some(config.V1Alpha1ID), + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.NodeAddressType, + ID: optional.Some(network.NodeAddressDefaultID), + Kind: controller.InputWeak, + }, + { + Namespace: cluster.NamespaceName, + Type: cluster.IdentityType, + ID: optional.Some(cluster.LocalIdentity), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *HostnameConfigController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.HostnameSpecType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *HostnameConfigController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + touchedIDs := make(map[resource.ID]struct{}) + + var cfgProvider talosconfig.Config + + cfg, err := safe.ReaderGetByID[*config.MachineConfig](ctx, r, config.V1Alpha1ID) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting config: %w", err) + } + } else if cfg.Config().Machine() != nil { + cfgProvider = cfg.Config() + } + + var specs []network.HostnameSpecSpec + + // defaults + var defaultAddr *network.NodeAddress + + addrs, err := r.Get(ctx, resource.NewMetadata(network.NamespaceName, network.NodeAddressType, network.NodeAddressDefaultID, resource.VersionUndefined)) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting config: %w", err) + } + } else { + defaultAddr = addrs.(*network.NodeAddress) //nolint:errcheck,forcetypeassert + } + + // parse kernel cmdline for the default gateway + cmdlineHostname := ctrl.parseCmdline(logger) + if cmdlineHostname.Hostname != "" { + specs = append(specs, cmdlineHostname) + } + + // parse machine configuration for specs + if cfgProvider != nil { + configHostname := ctrl.parseMachineConfiguration(logger, cfgProvider) + + if configHostname.Hostname != "" { + specs = append(specs, configHostname) + } + + if cfgProvider.Machine().Features().StableHostnameEnabled() { + var identity resource.Resource + + identity, err = r.Get(ctx, resource.NewMetadata(cluster.NamespaceName, cluster.IdentityType, cluster.LocalIdentity, resource.VersionUndefined)) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting local identity: %w", err) + } + + continue + } + + nodeID := identity.(*cluster.Identity).TypedSpec().NodeID + + stableHostname := ctrl.getStableDefault(nodeID) + specs = append(specs, *stableHostname) + } else { + specs = append(specs, ctrl.getDefault(defaultAddr)) + } + } + + var ids []string + + ids, err = ctrl.apply(ctx, r, specs) + if err != nil { + return fmt.Errorf("error applying specs: %w", err) + } + + for _, id := range ids { + touchedIDs[id] = struct{}{} + } + + // list specs for cleanup + list, err := r.List(ctx, resource.NewMetadata(network.ConfigNamespaceName, network.HostnameSpecType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing resources: %w", err) + } + + for _, res := range list.Items { + if res.Metadata().Owner() != ctrl.Name() { + // skip specs created by other controllers + continue + } + + if _, ok := touchedIDs[res.Metadata().ID()]; !ok { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return fmt.Errorf("error cleaning up specs: %w", err) + } + } + } + + r.ResetRestartBackoff() + } +} + +//nolint:dupl +func (ctrl *HostnameConfigController) apply(ctx context.Context, r controller.Runtime, specs []network.HostnameSpecSpec) ([]resource.ID, error) { + ids := make([]string, 0, len(specs)) + + for _, spec := range specs { + id := network.LayeredID(spec.ConfigLayer, network.HostnameID) + + if err := r.Modify( + ctx, + network.NewHostnameSpec(network.ConfigNamespaceName, id), + func(r resource.Resource) error { + *r.(*network.HostnameSpec).TypedSpec() = spec + + return nil + }, + ); err != nil { + return ids, err + } + + ids = append(ids, id) + } + + return ids, nil +} + +func (ctrl *HostnameConfigController) getStableDefault(nodeID string) *network.HostnameSpecSpec { + hashBytes := sha256.Sum256([]byte(nodeID)) + b36 := strings.ToLower(base36.EncodeBytes(hashBytes[:8])) + + hostname := fmt.Sprintf("talos-%s-%s", b36[1:4], b36[4:7]) + + return &network.HostnameSpecSpec{ + Hostname: hostname, + ConfigLayer: network.ConfigDefault, + } +} + +func (ctrl *HostnameConfigController) getDefault(defaultAddr *network.NodeAddress) (spec network.HostnameSpecSpec) { + if defaultAddr == nil || len(defaultAddr.TypedSpec().Addresses) != 1 { + return + } + + spec.Hostname = fmt.Sprintf("talos-%s", strings.ReplaceAll(strings.ReplaceAll(defaultAddr.TypedSpec().Addresses[0].Addr().String(), ":", ""), ".", "-")) + spec.ConfigLayer = network.ConfigDefault + + return spec +} + +func (ctrl *HostnameConfigController) parseCmdline(logger *zap.Logger) (spec network.HostnameSpecSpec) { + if ctrl.Cmdline == nil { + return + } + + settings, err := ParseCmdlineNetwork(ctrl.Cmdline) + if err != nil { + logger.Warn("ignoring error", zap.Error(err)) + + return + } + + if settings.Hostname == "" { + return + } + + if err = spec.ParseFQDN(settings.Hostname); err != nil { + logger.Warn("ignoring error", zap.Error(err)) + + return network.HostnameSpecSpec{} + } + + spec.ConfigLayer = network.ConfigCmdline + + return spec +} + +func (ctrl *HostnameConfigController) parseMachineConfiguration(logger *zap.Logger, cfgProvider talosconfig.Config) (spec network.HostnameSpecSpec) { + hostname := cfgProvider.Machine().Network().Hostname() + + if hostname == "" { + return + } + + if err := spec.ParseFQDN(hostname); err != nil { + logger.Warn("ignoring error", zap.Error(err)) + + return network.HostnameSpecSpec{} + } + + spec.ConfigLayer = network.ConfigMachineConfiguration + + return spec +} diff --git a/internal/app/machined/pkg/controllers/network/hostname_config_test.go b/internal/app/machined/pkg/controllers/network/hostname_config_test.go new file mode 100644 index 0000000..5a075ed --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/hostname_config_test.go @@ -0,0 +1,259 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network_test + +import ( + "context" + "log" + "net/netip" + "net/url" + "strings" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-pointer" + "github.com/siderolabs/go-procfs/procfs" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type HostnameConfigSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *HostnameConfigSuite) State() state.State { return suite.state } + +func (suite *HostnameConfigSuite) Ctx() context.Context { return suite.ctx } + +func (suite *HostnameConfigSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) +} + +func (suite *HostnameConfigSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *HostnameConfigSuite) assertHostnames(requiredIDs []string, check func(*network.HostnameSpec, *assert.Assertions)) { + assertResources(suite.ctx, suite.T(), suite.state, requiredIDs, check, rtestutils.WithNamespace(network.ConfigNamespaceName)) +} + +func (suite *HostnameConfigSuite) assertNoHostname(id string) error { + resources, err := suite.state.List( + suite.ctx, + resource.NewMetadata(network.ConfigNamespaceName, network.HostnameSpecType, "", resource.VersionUndefined), + ) + if err != nil { + return err + } + + for _, res := range resources.Items { + if res.Metadata().ID() == id { + return retry.ExpectedErrorf("spec %q is still there", id) + } + } + + return nil +} + +func (suite *HostnameConfigSuite) TestNoDefaultWithoutMachineConfig() { + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.HostnameConfigController{})) + + suite.startRuntime() + + defaultAddress := network.NewNodeAddress(network.NamespaceName, network.NodeAddressDefaultID) + defaultAddress.TypedSpec().Addresses = []netip.Prefix{netip.MustParsePrefix("33.11.22.44/32")} + + suite.Require().NoError(suite.state.Create(suite.ctx, defaultAddress)) + + suite.assertHostnames(nil, func(r *network.HostnameSpec, asrt *assert.Assertions) { + asrt.NotEqual("default/hostname", r.Metadata().ID(), "default hostname is still there") + }) +} + +func (suite *HostnameConfigSuite) TestDefaultIPBasedHostname() { + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.HostnameConfigController{})) + + suite.startRuntime() + + cfg := config.NewMachineConfig(container.NewV1Alpha1(&v1alpha1.Config{ConfigVersion: "v1alpha1"})) + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + defaultAddress := network.NewNodeAddress(network.NamespaceName, network.NodeAddressDefaultID) + defaultAddress.TypedSpec().Addresses = []netip.Prefix{netip.MustParsePrefix("33.11.22.44/32")} + + suite.Require().NoError(suite.state.Create(suite.ctx, defaultAddress)) + + suite.assertHostnames( + []string{ + "default/hostname", + }, func(r *network.HostnameSpec, asrt *assert.Assertions) { + asrt.Equal("talos-33-11-22-44", r.TypedSpec().Hostname) + asrt.Equal("", r.TypedSpec().Domainname) + asrt.Equal(network.ConfigDefault, r.TypedSpec().ConfigLayer) + }, + ) +} + +func (suite *HostnameConfigSuite) TestDefaultStableHostname() { + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.HostnameConfigController{})) + + suite.startRuntime() + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineFeatures: &v1alpha1.FeaturesConfig{ + StableHostname: pointer.To(true), + }, + }, + }, + ), + ) + + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + id := cluster.NewIdentity(cluster.NamespaceName, cluster.LocalIdentity) + id.TypedSpec().NodeID = "fGdOI05hVrx3YMagLo0Bwxa2Nm9BAswWm8XLeEj0aS4" + + suite.Require().NoError(suite.state.Create(suite.ctx, id)) + + suite.assertHostnames( + []string{ + "default/hostname", + }, func(r *network.HostnameSpec, asrt *assert.Assertions) { + asrt.Equal("talos-hwz-sw5", r.TypedSpec().Hostname) + }, + ) +} + +func (suite *HostnameConfigSuite) TestCmdline() { + suite.Require().NoError( + suite.runtime.RegisterController( + &netctrl.HostnameConfigController{ + Cmdline: procfs.NewCmdline("ip=172.20.0.2:172.21.0.1:172.20.0.1:255.255.255.0:master1.domain.tld:eth1::10.0.0.1:10.0.0.2:10.0.0.1"), + }, + ), + ) + + suite.startRuntime() + + suite.assertHostnames( + []string{ + "cmdline/hostname", + }, func(r *network.HostnameSpec, asrt *assert.Assertions) { + asrt.Equal("master1", r.TypedSpec().Hostname) + asrt.Equal("domain.tld", r.TypedSpec().Domainname) + asrt.Equal(network.ConfigCmdline, r.TypedSpec().ConfigLayer) + }, + ) +} + +func (suite *HostnameConfigSuite) TestMachineConfiguration() { + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.HostnameConfigController{})) + + suite.startRuntime() + + u, err := url.Parse("https://foo:6443") + suite.Require().NoError(err) + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineNetwork: &v1alpha1.NetworkConfig{ + NetworkHostname: "foo", + }, + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: u, + }, + }, + }, + }, + ), + ) + + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + suite.assertHostnames( + []string{ + "configuration/hostname", + }, func(r *network.HostnameSpec, asrt *assert.Assertions) { + asrt.Equal("foo", r.TypedSpec().Hostname) + asrt.Equal("", r.TypedSpec().Domainname) + asrt.Equal(network.ConfigMachineConfiguration, r.TypedSpec().ConfigLayer) + }, + ) + + ctest.UpdateWithConflicts(suite, cfg, func(r *config.MachineConfig) error { + r.Container().RawV1Alpha1().MachineConfig.MachineNetwork.NetworkHostname = strings.Repeat("a", 128) + + return nil + }) + suite.Require().NoError(err) + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertNoHostname("configuration/hostname") + }, + ), + ) +} + +func (suite *HostnameConfigSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestHostnameConfigSuite(t *testing.T) { + suite.Run(t, new(HostnameConfigSuite)) +} diff --git a/internal/app/machined/pkg/controllers/network/hostname_merge.go b/internal/app/machined/pkg/controllers/network/hostname_merge.go new file mode 100644 index 0000000..eb84947 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/hostname_merge.go @@ -0,0 +1,124 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package network provides controllers which manage network resources. +package network + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// HostnameMergeController merges network.HostnameSpec in network.ConfigNamespace and produces final network.HostnameSpec in network.Namespace. +type HostnameMergeController struct{} + +// Name implements controller.Controller interface. +func (ctrl *HostnameMergeController) Name() string { + return "network.HostnameMergeController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *HostnameMergeController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: network.ConfigNamespaceName, + Type: network.HostnameSpecType, + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.HostnameSpecType, + Kind: controller.InputDestroyReady, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *HostnameMergeController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.HostnameSpecType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *HostnameMergeController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + // list source network configuration resources + list, err := r.List(ctx, resource.NewMetadata(network.ConfigNamespaceName, network.HostnameSpecType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing source network addresses: %w", err) + } + + // simply merge by layers, overriding with the next configuration layer + var final network.HostnameSpecSpec + + for _, res := range list.Items { + spec := res.(*network.HostnameSpec) //nolint:errcheck,forcetypeassert + + if final.Hostname != "" && spec.TypedSpec().ConfigLayer <= final.ConfigLayer { + // skip this spec, as existing one is higher layer + continue + } + + final = *spec.TypedSpec() + } + + if final.Hostname != "" { + if err = r.Modify(ctx, network.NewHostnameSpec(network.NamespaceName, network.HostnameID), func(res resource.Resource) error { + spec := res.(*network.HostnameSpec) //nolint:errcheck,forcetypeassert + + *spec.TypedSpec() = final + + return nil + }); err != nil { + if state.IsPhaseConflictError(err) { + // conflict + final.Hostname = "" + + r.QueueReconcile() + } else { + return fmt.Errorf("error updating resource: %w", err) + } + } + } + + if final.Hostname == "" { + // remove existing + var okToDestroy bool + + md := resource.NewMetadata(network.NamespaceName, network.HostnameSpecType, network.HostnameID, resource.VersionUndefined) + + okToDestroy, err = r.Teardown(ctx, md) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error cleaning up specs: %w", err) + } + + if okToDestroy { + if err = r.Destroy(ctx, md); err != nil { + return fmt.Errorf("error cleaning up specs: %w", err) + } + } + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/network/hostname_merge_test.go b/internal/app/machined/pkg/controllers/network/hostname_merge_test.go new file mode 100644 index 0000000..d04d174 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/hostname_merge_test.go @@ -0,0 +1,131 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:dupl +package network_test + +import ( + "context" + "log" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type HostnameMergeSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *HostnameMergeSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.HostnameMergeController{})) + + suite.startRuntime() +} + +func (suite *HostnameMergeSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *HostnameMergeSuite) assertHostnames(requiredIDs []string, check func(*network.HostnameSpec, *assert.Assertions)) { + assertResources(suite.ctx, suite.T(), suite.state, requiredIDs, check) +} + +func (suite *HostnameMergeSuite) TestMerge() { + def := network.NewHostnameSpec(network.ConfigNamespaceName, "default/hostname") + *def.TypedSpec() = network.HostnameSpecSpec{ + Hostname: "foo", + Domainname: "tld", + ConfigLayer: network.ConfigDefault, + } + + dhcp1 := network.NewHostnameSpec(network.ConfigNamespaceName, "dhcp/eth0") + *dhcp1.TypedSpec() = network.HostnameSpecSpec{ + Hostname: "eth-0", + ConfigLayer: network.ConfigOperator, + } + + dhcp2 := network.NewHostnameSpec(network.ConfigNamespaceName, "dhcp/eth1") + *dhcp2.TypedSpec() = network.HostnameSpecSpec{ + Hostname: "eth-1", + ConfigLayer: network.ConfigOperator, + } + + static := network.NewHostnameSpec(network.ConfigNamespaceName, "configuration/hostname") + *static.TypedSpec() = network.HostnameSpecSpec{ + Hostname: "bar", + Domainname: "com", + ConfigLayer: network.ConfigMachineConfiguration, + } + + for _, res := range []resource.Resource{def, dhcp1, dhcp2, static} { + suite.Require().NoError(suite.state.Create(suite.ctx, res), "%v", res.Spec()) + } + + suite.assertHostnames( + []string{ + "hostname", + }, func(r *network.HostnameSpec, asrt *assert.Assertions) { + asrt.Equal("bar.com", r.TypedSpec().FQDN()) + asrt.Equal("bar", r.TypedSpec().Hostname) + asrt.Equal("com", r.TypedSpec().Domainname) + }, + ) + + suite.Require().NoError(suite.state.Destroy(suite.ctx, static.Metadata())) + + suite.assertHostnames( + []string{ + "hostname", + }, func(r *network.HostnameSpec, asrt *assert.Assertions) { + asrt.Equal("eth-0", r.TypedSpec().FQDN()) + }, + ) +} + +func (suite *HostnameMergeSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestHostnameMergeSuite(t *testing.T) { + suite.Run(t, new(HostnameMergeSuite)) +} diff --git a/internal/app/machined/pkg/controllers/network/hostname_spec.go b/internal/app/machined/pkg/controllers/network/hostname_spec.go new file mode 100644 index 0000000..572dee8 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/hostname_spec.go @@ -0,0 +1,122 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "go.uber.org/zap" + "golang.org/x/sys/unix" + + v1alpha1runtime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// HostnameSpecController applies network.HostnameSpec to the actual interfaces. +type HostnameSpecController struct { + V1Alpha1Mode v1alpha1runtime.Mode +} + +// Name implements controller.Controller interface. +func (ctrl *HostnameSpecController) Name() string { + return "network.HostnameSpecController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *HostnameSpecController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: network.NamespaceName, + Type: network.HostnameSpecType, + Kind: controller.InputStrong, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *HostnameSpecController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.HostnameStatusType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *HostnameSpecController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + // list source network configuration resources + list, err := r.List(ctx, resource.NewMetadata(network.NamespaceName, network.HostnameSpecType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing source network addresses: %w", err) + } + + // add finalizers for all live resources + for _, res := range list.Items { + if res.Metadata().Phase() != resource.PhaseRunning { + continue + } + + if err = r.AddFinalizer(ctx, res.Metadata(), ctrl.Name()); err != nil { + return fmt.Errorf("error adding finalizer: %w", err) + } + } + + // loop over specs and sync to statuses + for _, res := range list.Items { + spec := res.(*network.HostnameSpec) //nolint:forcetypeassert,errcheck + + switch spec.Metadata().Phase() { + case resource.PhaseTearingDown: + if err = r.Destroy(ctx, resource.NewMetadata(network.NamespaceName, network.HostnameStatusType, spec.Metadata().ID(), resource.VersionUndefined)); err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error destroying status: %w", err) + } + + if err = r.RemoveFinalizer(ctx, spec.Metadata(), ctrl.Name()); err != nil { + return fmt.Errorf("error removing finalizer: %w", err) + } + case resource.PhaseRunning: + if err = r.Modify(ctx, network.NewHostnameStatus(network.NamespaceName, spec.Metadata().ID()), func(r resource.Resource) error { + status := r.(*network.HostnameStatus) //nolint:forcetypeassert,errcheck + + status.TypedSpec().Hostname = spec.TypedSpec().Hostname + status.TypedSpec().Domainname = spec.TypedSpec().Domainname + + return nil + }); err != nil { + return fmt.Errorf("error modifying status: %w", err) + } + + // apply hostname unless running in container mode + if ctrl.V1Alpha1Mode != v1alpha1runtime.ModeContainer { + logger.Info("setting hostname", zap.String("hostname", spec.TypedSpec().Hostname), zap.String("domainname", spec.TypedSpec().Domainname)) + + if err = unix.Sethostname([]byte(spec.TypedSpec().Hostname)); err != nil { + return fmt.Errorf("error setting hostname: %w", err) + } + + if err = unix.Setdomainname([]byte(spec.TypedSpec().Domainname)); err != nil { + return fmt.Errorf("error setting domainname: %w", err) + } + } + } + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/network/hostname_spec_test.go b/internal/app/machined/pkg/controllers/network/hostname_spec_test.go new file mode 100644 index 0000000..6ecb851 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/hostname_spec_test.go @@ -0,0 +1,124 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network_test + +import ( + "context" + "log" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + v1alpha1runtime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type HostnameSpecSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *HostnameSpecSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.Require().NoError( + suite.runtime.RegisterController( + &netctrl.HostnameSpecController{ + V1Alpha1Mode: v1alpha1runtime.ModeContainer, // run in container mode to skip _actually_ setting hostname + }, + ), + ) + + suite.startRuntime() +} + +func (suite *HostnameSpecSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *HostnameSpecSuite) assertStatus(id string, fqdn string) error { + r, err := suite.state.Get( + suite.ctx, + resource.NewMetadata(network.NamespaceName, network.HostnameStatusType, id, resource.VersionUndefined), + ) + if err != nil { + if state.IsNotFoundError(err) { + return retry.ExpectedError(err) + } + + return err + } + + status := r.(*network.HostnameStatus) //nolint:errcheck,forcetypeassert + + if status.TypedSpec().FQDN() != fqdn { + return retry.ExpectedErrorf("fqdn mismatch: %q != %q", status.TypedSpec().FQDN(), fqdn) + } + + return nil +} + +func (suite *HostnameSpecSuite) TestSpec() { + spec := network.NewHostnameSpec(network.NamespaceName, "hostname") + *spec.TypedSpec() = network.HostnameSpecSpec{ + Hostname: "foo", + Domainname: "bar", + ConfigLayer: network.ConfigDefault, + } + + for _, res := range []resource.Resource{spec} { + suite.Require().NoError(suite.state.Create(suite.ctx, res), "%v", res.Spec()) + } + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertStatus("hostname", "foo.bar") + }, + ), + ) +} + +func (suite *HostnameSpecSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestHostnameSpecSuite(t *testing.T) { + suite.Run(t, new(HostnameSpecSuite)) +} diff --git a/internal/app/machined/pkg/controllers/network/internal/probe/probe.go b/internal/app/machined/pkg/controllers/network/internal/probe/probe.go new file mode 100644 index 0000000..406313f --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/internal/probe/probe.go @@ -0,0 +1,159 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package probe contains implementation of the network probe runners. +package probe + +import ( + "context" + "errors" + "net" + "sync" + "syscall" + + "github.com/benbjohnson/clock" + "github.com/siderolabs/gen/channel" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// Runner describes a state of running probe. +type Runner struct { + ID string + Spec network.ProbeSpecSpec + Clock clock.Clock + + cancel context.CancelFunc + wg sync.WaitGroup +} + +// Notification of a runner status. +type Notification struct { + ID string + Status network.ProbeStatusSpec +} + +// Start a runner with a given context. +func (runner *Runner) Start(ctx context.Context, notifyCh chan<- Notification, logger *zap.Logger) { + runner.wg.Add(1) + + ctx, runner.cancel = context.WithCancel(ctx) + + go func() { + defer runner.wg.Done() + + runner.run(ctx, notifyCh, logger) + }() +} + +// Stop a runner. +func (runner *Runner) Stop() { + runner.cancel() + + runner.wg.Wait() +} + +// run a probe. +// +//nolint:gocyclo +func (runner *Runner) run(ctx context.Context, notifyCh chan<- Notification, logger *zap.Logger) { + logger = logger.With(zap.String("probe", runner.ID)) + + if runner.Clock == nil { + runner.Clock = clock.New() + } + + ticker := runner.Clock.Ticker(runner.Spec.Interval) + defer ticker.Stop() + + consecutiveFailures := 0 + firstIteration := true + + for { + if !firstIteration { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } else { + firstIteration = false + } + + err := runner.probe(ctx) + if err == nil { + if consecutiveFailures > 0 { + logger.Info("probe succeeded") + } + + consecutiveFailures = 0 + + if !channel.SendWithContext(ctx, notifyCh, Notification{ + ID: runner.ID, + Status: network.ProbeStatusSpec{ + Success: true, + }, + }) { + return + } + + continue + } + + if consecutiveFailures == runner.Spec.FailureThreshold { + logger.Error("probe failed", zap.Error(err)) + } + + consecutiveFailures++ + + if consecutiveFailures < runner.Spec.FailureThreshold { + continue + } + + if !channel.SendWithContext(ctx, notifyCh, Notification{ + ID: runner.ID, + Status: network.ProbeStatusSpec{ + Success: false, + LastError: err.Error(), + }, + }) { + return + } + } +} + +// probe runs a probe. +func (runner *Runner) probe(ctx context.Context) error { + var zeroTCP network.TCPProbeSpec + + switch { + case runner.Spec.TCP != zeroTCP: + return runner.probeTCP(ctx) + default: + return errors.New("no probe type specified") + } +} + +// probeTCP runs a TCP probe. +func (runner *Runner) probeTCP(ctx context.Context) error { + dialer := &net.Dialer{ + // The dialer reduces the TIME-WAIT period to 1 seconds instead of the OS default of 60 seconds. + Control: func(network, address string, c syscall.RawConn) error { + return c.Control(func(fd uintptr) { + syscall.SetsockoptLinger(int(fd), syscall.SOL_SOCKET, syscall.SO_LINGER, &syscall.Linger{Onoff: 1, Linger: 1}) //nolint: errcheck + }) + }, + } + + ctx, cancel := context.WithTimeout(ctx, runner.Spec.TCP.Timeout) + defer cancel() + + conn, err := dialer.DialContext(ctx, "tcp", runner.Spec.TCP.Endpoint) + if err != nil { + return err + } + + return conn.Close() +} diff --git a/internal/app/machined/pkg/controllers/network/internal/probe/probe_test.go b/internal/app/machined/pkg/controllers/network/internal/probe/probe_test.go new file mode 100644 index 0000000..7b1632a --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/internal/probe/probe_test.go @@ -0,0 +1,152 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package probe_test + +import ( + "context" + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + "github.com/benbjohnson/clock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network/internal/probe" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +func TestProbeHTTP(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(server.Close) + + u, err := url.Parse(server.URL) + require.NoError(t, err) + + p := probe.Runner{ + ID: "test", + Spec: network.ProbeSpecSpec{ + Interval: 10 * time.Millisecond, + TCP: network.TCPProbeSpec{ + Endpoint: u.Host, + Timeout: time.Second, + }, + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + t.Cleanup(cancel) + + notifyCh := make(chan probe.Notification) + + p.Start(ctx, notifyCh, zaptest.NewLogger(t)) + t.Cleanup(p.Stop) + + // probe should always succeed + for range 3 { + assert.Equal(t, probe.Notification{ + ID: "test", + Status: network.ProbeStatusSpec{ + Success: true, + }, + }, <-notifyCh) + } + + // stop the test server, probe should fail + server.Close() + + for { + notification := <-notifyCh + + if notification.Status.Success { + continue + } + + assert.Equal(t, "test", notification.ID) + assert.False(t, notification.Status.Success) + assert.Contains(t, notification.Status.LastError, "connection refused") + + break + } +} + +func TestProbeConsecutiveFailures(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(server.Close) + + u, err := url.Parse(server.URL) + require.NoError(t, err) + + mockClock := clock.NewMock() + + p := probe.Runner{ + ID: "consecutive-failures", + Spec: network.ProbeSpecSpec{ + Interval: 10 * time.Millisecond, + FailureThreshold: 3, + TCP: network.TCPProbeSpec{ + Endpoint: u.Host, + Timeout: time.Second, + }, + }, + Clock: mockClock, + } + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + t.Cleanup(cancel) + + notifyCh := make(chan probe.Notification) + + p.Start(ctx, notifyCh, zaptest.NewLogger(t)) + t.Cleanup(p.Stop) + + // first iteration should succeed + assert.Equal(t, probe.Notification{ + ID: "consecutive-failures", + Status: network.ProbeStatusSpec{ + Success: true, + }, + }, <-notifyCh) + + // stop the test server, probe should fail + server.Close() + + for range p.Spec.FailureThreshold - 1 { + // probe should fail, but no notification should be sent yet (failure threshold not reached) + mockClock.Add(p.Spec.Interval) + + select { + case ev := <-notifyCh: + require.Fail(t, "unexpected notification", "got: %v", ev) + case <-time.After(100 * time.Millisecond): + } + } + + // advance clock to trigger another failure(s) + mockClock.Add(p.Spec.Interval) + + notify := <-notifyCh + assert.Equal(t, "consecutive-failures", notify.ID) + assert.False(t, notify.Status.Success) + assert.Contains(t, notify.Status.LastError, "connection refused") + + // advance clock to trigger another failure(s) + mockClock.Add(p.Spec.Interval) + + notify = <-notifyCh + assert.Equal(t, "consecutive-failures", notify.ID) + assert.False(t, notify.Status.Success) +} diff --git a/internal/app/machined/pkg/controllers/network/link_config.go b/internal/app/machined/pkg/controllers/network/link_config.go new file mode 100644 index 0000000..7ac4025 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/link_config.go @@ -0,0 +1,453 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "context" + "fmt" + "net/netip" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/maps" + "github.com/siderolabs/gen/pair/ordered" + "github.com/siderolabs/go-procfs/procfs" + "go.uber.org/zap" + + talosconfig "github.com/siderolabs/talos/pkg/machinery/config/config" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// LinkConfigController manages network.LinkSpec based on machine configuration, kernel cmdline. +type LinkConfigController struct { + Cmdline *procfs.Cmdline +} + +// Name implements controller.Controller interface. +func (ctrl *LinkConfigController) Name() string { + return "network.LinkConfigController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *LinkConfigController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: network.NamespaceName, + Type: network.DeviceConfigSpecType, + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.LinkStatusType, + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *LinkConfigController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.LinkSpecType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *LinkConfigController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + touchedIDs := make(map[resource.ID]struct{}) + + items, err := r.List(ctx, resource.NewMetadata(network.NamespaceName, network.DeviceConfigSpecType, "", resource.VersionUndefined)) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting config: %w", err) + } + } + + ignoredInterfaces := map[string]struct{}{} + + devices := make([]talosconfig.Device, len(items.Items)) + + for i, item := range items.Items { + device := item.(*network.DeviceConfigSpec).TypedSpec().Device + devices[i] = device + + if device.Ignore() { + ignoredInterfaces[device.Interface()] = struct{}{} + } + } + + // bring up loopback interface + { + var ids []string + + ids, err = ctrl.apply(ctx, r, []network.LinkSpecSpec{ + { + Name: "lo", + Up: true, + ConfigLayer: network.ConfigDefault, + }, + }) + if err != nil { + return fmt.Errorf("error applying cmdline route: %w", err) + } + + for _, id := range ids { + touchedIDs[id] = struct{}{} + } + } + + // parse kernel cmdline for the interface name + cmdlineLinks, cmdlineIgnored := ctrl.parseCmdline(logger) + for _, cmdlineLink := range cmdlineLinks { + if cmdlineLink.Name != "" { + if _, ignored := ignoredInterfaces[cmdlineLink.Name]; !ignored { + var ids []string + + ids, err = ctrl.apply(ctx, r, []network.LinkSpecSpec{cmdlineLink}) + if err != nil { + return fmt.Errorf("error applying cmdline route: %w", err) + } + + for _, id := range ids { + touchedIDs[id] = struct{}{} + } + } + } + } + + // parse machine configuration for link specs + if len(devices) > 0 { + links := ctrl.processDevicesConfiguration(logger, devices) + + var ids []string + + ids, err = ctrl.apply(ctx, r, links) + if err != nil { + return fmt.Errorf("error applying machine configuration address: %w", err) + } + + for _, id := range ids { + touchedIDs[id] = struct{}{} + } + } + + // bring up any physical link not mentioned explicitly in the machine configuration + configuredLinks := map[string]struct{}{} + + for _, linkName := range cmdlineIgnored { + configuredLinks[linkName] = struct{}{} + } + + for _, cmdlineLink := range cmdlineLinks { + if cmdlineLink.Name != "" { + configuredLinks[cmdlineLink.Name] = struct{}{} + } + } + + if len(devices) > 0 { + for _, device := range devices { + configuredLinks[device.Interface()] = struct{}{} + + if device.Bond() != nil { + for _, link := range device.Bond().Interfaces() { + configuredLinks[link] = struct{}{} + } + } + + if device.Bridge() != nil { + for _, link := range device.Bridge().Interfaces() { + configuredLinks[link] = struct{}{} + } + } + } + } + + list, err := r.List(ctx, resource.NewMetadata(network.NamespaceName, network.LinkStatusType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing link statuses: %w", err) + } + + for _, item := range list.Items { + linkStatus := item.(*network.LinkStatus) //nolint:errcheck,forcetypeassert + + if _, configured := configuredLinks[linkStatus.Metadata().ID()]; !configured { + if linkStatus.TypedSpec().Physical() { + var ids []string + + ids, err = ctrl.apply(ctx, r, []network.LinkSpecSpec{ + { + Name: linkStatus.Metadata().ID(), + Up: true, + ConfigLayer: network.ConfigDefault, + }, + }) + if err != nil { + return fmt.Errorf("error applying default link up: %w", err) + } + + for _, id := range ids { + touchedIDs[id] = struct{}{} + } + } + } + } + + // list links for cleanup + list, err = r.List(ctx, resource.NewMetadata(network.ConfigNamespaceName, network.LinkSpecType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing resources: %w", err) + } + + for _, res := range list.Items { + if res.Metadata().Owner() != ctrl.Name() { + // skip specs created by other controllers + continue + } + + if _, ok := touchedIDs[res.Metadata().ID()]; !ok { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return fmt.Errorf("error cleaning up routes: %w", err) + } + } + } + + r.ResetRestartBackoff() + } +} + +func (ctrl *LinkConfigController) apply(ctx context.Context, r controller.Runtime, links []network.LinkSpecSpec) ([]resource.ID, error) { + ids := make([]string, 0, len(links)) + + for _, link := range links { + id := network.LayeredID(link.ConfigLayer, network.LinkID(link.Name)) + + if err := r.Modify( + ctx, + network.NewLinkSpec(network.ConfigNamespaceName, id), + func(r resource.Resource) error { + *r.(*network.LinkSpec).TypedSpec() = link + + return nil + }, + ); err != nil { + return ids, err + } + + ids = append(ids, id) + } + + return ids, nil +} + +func (ctrl *LinkConfigController) parseCmdline(logger *zap.Logger) ([]network.LinkSpecSpec, []string) { + if ctrl.Cmdline == nil { + return []network.LinkSpecSpec{}, nil + } + + settings, err := ParseCmdlineNetwork(ctrl.Cmdline) + if err != nil { + logger.Info("ignoring error", zap.Error(err)) + + return []network.LinkSpecSpec{}, nil + } + + return settings.NetworkLinkSpecs, settings.IgnoreInterfaces +} + +//nolint:gocyclo,cyclop +func (ctrl *LinkConfigController) processDevicesConfiguration(logger *zap.Logger, devices []talosconfig.Device) []network.LinkSpecSpec { + // scan for the bonds or bridges + bondedLinks := map[string]ordered.Pair[string, int]{} // mapping physical interface -> bond interface + bridgedLinks := map[string]string{} // mapping physical interface -> bridge interface + + for _, device := range devices { + if device.Ignore() { + continue + } + + if device.Bond() == nil && device.Bridge() == nil { + continue + } + + if device.Bond() != nil { + for idx, linkName := range device.Bond().Interfaces() { + if bondData, exists := bondedLinks[linkName]; exists && bondData.F1 != device.Interface() { + logger.Sugar().Warnf("link %q is included into more than two bonds", linkName) + } + + if bridgeIface, exists := bridgedLinks[linkName]; exists && bridgeIface != device.Interface() { + logger.Sugar().Warnf("link %q is included into both a bond and a bridge", linkName) + } + + bondedLinks[linkName] = ordered.MakePair(device.Interface(), idx) + } + } + + if device.Bridge() != nil { + for _, linkName := range device.Bridge().Interfaces() { + if iface, exists := bridgedLinks[linkName]; exists && iface != device.Interface() { + logger.Sugar().Warnf("link %q is included into more than two bridges", linkName) + } + + if bondData, exists := bondedLinks[linkName]; exists && bondData.F1 != device.Interface() { + logger.Sugar().Warnf("link %q is included into both a bond and a bridge", linkName) + } + + bridgedLinks[linkName] = device.Interface() + } + } + } + + linkMap := map[string]*network.LinkSpecSpec{} + + for _, device := range devices { + if device.Ignore() { + continue + } + + if _, exists := linkMap[device.Interface()]; !exists { + linkMap[device.Interface()] = &network.LinkSpecSpec{ + Name: device.Interface(), + Up: true, + ConfigLayer: network.ConfigMachineConfiguration, + } + } + + if device.MTU() != 0 { + linkMap[device.Interface()].MTU = uint32(device.MTU()) + } + + if device.Bond() != nil { + if err := SetBondMaster(linkMap[device.Interface()], device.Bond()); err != nil { + logger.Error("error parsing bond config", zap.Error(err)) + } + } + + if device.Bridge() != nil { + if err := SetBridgeMaster(linkMap[device.Interface()], device.Bridge()); err != nil { + logger.Error("error parsing bridge config", zap.Error(err)) + } + } + + if device.WireguardConfig() != nil { + if err := wireguardLink(linkMap[device.Interface()], device.WireguardConfig()); err != nil { + logger.Error("error parsing wireguard config", zap.Error(err)) + } + } + + if device.Dummy() { + dummyLink(linkMap[device.Interface()]) + } + + for _, vlan := range device.Vlans() { + vlanName := nethelpers.VLANLinkName(device.Interface(), vlan.ID()) + + linkMap[vlanName] = &network.LinkSpecSpec{ + Name: device.Interface(), + Up: true, + ConfigLayer: network.ConfigMachineConfiguration, + } + + vlanLink(linkMap[vlanName], device.Interface(), vlan) + } + } + + for slaveName, bondData := range bondedLinks { + if _, exists := linkMap[slaveName]; !exists { + linkMap[slaveName] = &network.LinkSpecSpec{ + Name: slaveName, + Up: true, + ConfigLayer: network.ConfigMachineConfiguration, + } + } + + SetBondSlave(linkMap[slaveName], bondData) + } + + for slaveName, bridgeIface := range bridgedLinks { + if _, exists := linkMap[slaveName]; !exists { + linkMap[slaveName] = &network.LinkSpecSpec{ + Name: slaveName, + Up: true, + ConfigLayer: network.ConfigMachineConfiguration, + } + } + + SetBridgeSlave(linkMap[slaveName], bridgeIface) + } + + return maps.ValuesFunc(linkMap, func(link *network.LinkSpecSpec) network.LinkSpecSpec { return *link }) +} + +type vlaner interface { + ID() uint16 + MTU() uint32 +} + +func vlanLink(link *network.LinkSpecSpec, linkName string, vlan vlaner) { + link.Name = nethelpers.VLANLinkName(linkName, vlan.ID()) + link.Logical = true + link.Up = true + link.MTU = vlan.MTU() + link.Kind = network.LinkKindVLAN + link.Type = nethelpers.LinkEther + link.ParentName = linkName + link.VLAN = network.VLANSpec{ + VID: vlan.ID(), + Protocol: nethelpers.VLANProtocol8021Q, + } +} + +func wireguardLink(link *network.LinkSpecSpec, config talosconfig.WireguardConfig) error { + link.Logical = true + link.Kind = network.LinkKindWireguard + link.Type = nethelpers.LinkNone + link.Wireguard = network.WireguardSpec{ + PrivateKey: config.PrivateKey(), + ListenPort: config.ListenPort(), + FirewallMark: config.FirewallMark(), + } + + for _, peer := range config.Peers() { + allowedIPs := make([]netip.Prefix, 0, len(peer.AllowedIPs())) + + for _, allowedIP := range peer.AllowedIPs() { + ip, err := netip.ParsePrefix(allowedIP) + if err != nil { + return err + } + + allowedIPs = append(allowedIPs, ip) + } + + link.Wireguard.Peers = append(link.Wireguard.Peers, network.WireguardPeer{ + PublicKey: peer.PublicKey(), + Endpoint: peer.Endpoint(), + PersistentKeepaliveInterval: peer.PersistentKeepaliveInterval(), + AllowedIPs: allowedIPs, + }) + } + + return nil +} + +func dummyLink(link *network.LinkSpecSpec) { + link.Logical = true + link.Kind = "dummy" + link.Type = nethelpers.LinkEther +} diff --git a/internal/app/machined/pkg/controllers/network/link_config_test.go b/internal/app/machined/pkg/controllers/network/link_config_test.go new file mode 100644 index 0000000..a1f37f7 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/link_config_test.go @@ -0,0 +1,484 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:dupl,goconst +package network_test + +import ( + "context" + "log" + "net/netip" + "net/url" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-pointer" + "github.com/siderolabs/go-procfs/procfs" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type LinkConfigSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *LinkConfigSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.DeviceConfigController{})) +} + +func (suite *LinkConfigSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *LinkConfigSuite) assertLinks(requiredIDs []string, check func(*network.LinkSpec, *assert.Assertions)) { + assertResources(suite.ctx, suite.T(), suite.state, requiredIDs, check, rtestutils.WithNamespace(network.ConfigNamespaceName)) +} + +func (suite *LinkConfigSuite) assertNoLinks(unexpectedIDs []string) error { + unexpIDs := make(map[string]struct{}, len(unexpectedIDs)) + + for _, id := range unexpectedIDs { + unexpIDs[id] = struct{}{} + } + + resources, err := suite.state.List( + suite.ctx, + resource.NewMetadata(network.ConfigNamespaceName, network.LinkSpecType, "", resource.VersionUndefined), + ) + if err != nil { + return err + } + + for _, res := range resources.Items { + _, unexpected := unexpIDs[res.Metadata().ID()] + if unexpected { + return retry.ExpectedErrorf("unexpected ID %q", res.Metadata().ID()) + } + } + + return nil +} + +func (suite *LinkConfigSuite) TestLoopback() { + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.LinkConfigController{})) + + suite.startRuntime() + + suite.assertLinks( + []string{ + "default/lo", + }, func(r *network.LinkSpec, asrt *assert.Assertions) { + asrt.Equal("lo", r.TypedSpec().Name) + asrt.True(r.TypedSpec().Up) + asrt.False(r.TypedSpec().Logical) + asrt.Equal(network.ConfigDefault, r.TypedSpec().ConfigLayer) + }, + ) +} + +func (suite *LinkConfigSuite) TestCmdline() { + suite.Require().NoError( + suite.runtime.RegisterController( + &netctrl.LinkConfigController{ + Cmdline: procfs.NewCmdline("ip=172.20.0.2::172.20.0.1:255.255.255.0::eth1:::::"), + }, + ), + ) + + suite.startRuntime() + + suite.assertLinks( + []string{ + "cmdline/eth1", + }, func(r *network.LinkSpec, asrt *assert.Assertions) { + asrt.Equal("eth1", r.TypedSpec().Name) + asrt.True(r.TypedSpec().Up) + asrt.False(r.TypedSpec().Logical) + asrt.Equal(network.ConfigCmdline, r.TypedSpec().ConfigLayer) + }, + ) +} + +//nolint:gocyclo +func (suite *LinkConfigSuite) TestMachineConfiguration() { + const kernelDriver = "somekerneldriver" + + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.LinkConfigController{})) + + suite.startRuntime() + + u, err := url.Parse("https://foo:6443") + suite.Require().NoError(err) + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineNetwork: &v1alpha1.NetworkConfig{ + NetworkInterfaces: []*v1alpha1.Device{ + { + DeviceInterface: "eth0", + DeviceVlans: []*v1alpha1.Vlan{ + { + VlanID: 24, + VlanMTU: 1000, + VlanAddresses: []string{ + "10.0.0.1/8", + }, + }, + { + VlanID: 48, + VlanAddresses: []string{ + "10.0.0.2/8", + }, + }, + }, + }, + { + DeviceInterface: "eth1", + DeviceAddresses: []string{"192.168.0.24/28"}, + }, + { + DeviceInterface: "eth1", + DeviceMTU: 9001, + }, + { + DeviceIgnore: pointer.To(true), + DeviceInterface: "eth2", + DeviceAddresses: []string{"192.168.0.24/28"}, + }, + { + DeviceInterface: "eth2", + }, + { + DeviceInterface: "bond0", + DeviceBond: &v1alpha1.Bond{ + BondInterfaces: []string{"eth2", "eth3"}, + BondMode: "balance-xor", + }, + }, + { + DeviceInterface: "bond1", + DeviceBond: &v1alpha1.Bond{ + BondDeviceSelectors: []v1alpha1.NetworkDeviceSelector{{ + NetworkDeviceKernelDriver: kernelDriver, + }}, + BondMode: "balance-xor", + }, + }, + { + DeviceInterface: "eth4", + DeviceAddresses: []string{"192.168.0.42/24"}, + }, + { + DeviceInterface: "eth5", + DeviceAddresses: []string{"192.168.0.43/24"}, + }, + { + DeviceInterface: "br0", + DeviceBridge: &v1alpha1.Bridge{ + BridgedInterfaces: []string{"eth4", "eth5"}, + BridgeSTP: &v1alpha1.STP{ + STPEnabled: pointer.To(false), + }, + }, + }, + { + DeviceInterface: "br0", + DeviceBridge: &v1alpha1.Bridge{ + BridgeSTP: &v1alpha1.STP{ + STPEnabled: pointer.To(true), + }, + }, + }, + { + DeviceInterface: "dummy0", + DeviceDummy: pointer.To(true), + }, + { + DeviceInterface: "wireguard0", + DeviceWireguardConfig: &v1alpha1.DeviceWireguardConfig{ + WireguardPrivateKey: "ABC", + WireguardPeers: []*v1alpha1.DeviceWireguardPeer{ + { + WireguardPublicKey: "DEF", + WireguardEndpoint: "10.0.0.1:3000", + WireguardAllowedIPs: []string{ + "10.2.3.0/24", + "10.2.4.0/24", + }, + }, + }, + }, + }, + }, + }, + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: u, + }, + }, + }, + }, + ), + ) + + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + for _, name := range []string{"eth6", "eth7"} { + status := network.NewLinkStatus(network.NamespaceName, name) + status.TypedSpec().Driver = kernelDriver + + suite.Require().NoError(suite.state.Create(suite.ctx, status)) + } + + suite.assertLinks( + []string{ + "configuration/eth0", + "configuration/eth0.24", + "configuration/eth0.48", + "configuration/eth1", + "configuration/eth2", + "configuration/eth3", + "configuration/eth6", + "configuration/eth7", + "configuration/bond0", + "configuration/bond1", + "configuration/br0", + "configuration/dummy0", + "configuration/wireguard0", + }, func(r *network.LinkSpec, asrt *assert.Assertions) { + asrt.Equal(network.ConfigMachineConfiguration, r.TypedSpec().ConfigLayer) + + switch r.TypedSpec().Name { + case "eth0", "eth1": + asrt.True(r.TypedSpec().Up) + asrt.False(r.TypedSpec().Logical) + + if r.TypedSpec().Name == "eth0" { + asrt.EqualValues(0, r.TypedSpec().MTU) + } else { + asrt.EqualValues(9001, r.TypedSpec().MTU) + } + case "eth0.24", "eth0.48": + asrt.True(r.TypedSpec().Up) + asrt.True(r.TypedSpec().Logical) + asrt.Equal(nethelpers.LinkEther, r.TypedSpec().Type) + asrt.Equal(network.LinkKindVLAN, r.TypedSpec().Kind) + asrt.Equal("eth0", r.TypedSpec().ParentName) + asrt.Equal(nethelpers.VLANProtocol8021Q, r.TypedSpec().VLAN.Protocol) + + if r.TypedSpec().Name == "eth0.24" { + asrt.EqualValues(24, r.TypedSpec().VLAN.VID) + asrt.EqualValues(1000, r.TypedSpec().MTU) + } else { + asrt.EqualValues(48, r.TypedSpec().VLAN.VID) + asrt.EqualValues(0, r.TypedSpec().MTU) + } + case "eth2", "eth3": + asrt.True(r.TypedSpec().Up) + asrt.False(r.TypedSpec().Logical) + asrt.Equal("bond0", r.TypedSpec().BondSlave.MasterName) + case "eth6", "eth7": + asrt.True(r.TypedSpec().Up) + asrt.False(r.TypedSpec().Logical) + asrt.Equal("bond1", r.TypedSpec().BondSlave.MasterName) + case "bond0": + asrt.True(r.TypedSpec().Up) + asrt.True(r.TypedSpec().Logical) + asrt.Equal(nethelpers.LinkEther, r.TypedSpec().Type) + asrt.Equal(network.LinkKindBond, r.TypedSpec().Kind) + asrt.Equal(nethelpers.BondModeXOR, r.TypedSpec().BondMaster.Mode) + asrt.True(r.TypedSpec().BondMaster.UseCarrier) + case "bond1": + asrt.True(r.TypedSpec().Up) + asrt.True(r.TypedSpec().Logical) + asrt.Equal(nethelpers.LinkEther, r.TypedSpec().Type) + asrt.Equal(network.LinkKindBond, r.TypedSpec().Kind) + asrt.Equal(nethelpers.BondModeXOR, r.TypedSpec().BondMaster.Mode) + asrt.True(r.TypedSpec().BondMaster.UseCarrier) + case "eth4", "eth5": + asrt.True(r.TypedSpec().Up) + asrt.False(r.TypedSpec().Logical) + asrt.Equal("br0", r.TypedSpec().BridgeSlave.MasterName) + case "br0": + asrt.True(r.TypedSpec().Up) + asrt.True(r.TypedSpec().Logical) + asrt.Equal(nethelpers.LinkEther, r.TypedSpec().Type) + asrt.Equal(network.LinkKindBridge, r.TypedSpec().Kind) + asrt.Equal(true, r.TypedSpec().BridgeMaster.STP.Enabled) + case "wireguard0": + asrt.True(r.TypedSpec().Up) + asrt.True(r.TypedSpec().Logical) + asrt.Equal(nethelpers.LinkNone, r.TypedSpec().Type) + asrt.Equal(network.LinkKindWireguard, r.TypedSpec().Kind) + asrt.Equal( + network.WireguardSpec{ + PrivateKey: "ABC", + Peers: []network.WireguardPeer{ + { + PublicKey: "DEF", + Endpoint: "10.0.0.1:3000", + AllowedIPs: []netip.Prefix{ + netip.MustParsePrefix("10.2.3.0/24"), + netip.MustParsePrefix("10.2.4.0/24"), + }, + }, + }, + }, r.TypedSpec().Wireguard, + ) + } + }, + ) +} + +func (suite *LinkConfigSuite) TestDefaultUp() { + suite.Require().NoError( + suite.runtime.RegisterController( + &netctrl.LinkConfigController{ + Cmdline: procfs.NewCmdline("talos.network.interface.ignore=eth2"), + }, + ), + ) + + for _, link := range []string{"eth0", "eth1", "eth2", "eth3", "eth4"} { + linkStatus := network.NewLinkStatus(network.NamespaceName, link) + linkStatus.TypedSpec().Type = nethelpers.LinkEther + linkStatus.TypedSpec().LinkState = true + + suite.Require().NoError(suite.state.Create(suite.ctx, linkStatus)) + } + + u, err := url.Parse("https://foo:6443") + suite.Require().NoError(err) + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineNetwork: &v1alpha1.NetworkConfig{ + NetworkInterfaces: []*v1alpha1.Device{ + { + DeviceInterface: "eth0", + DeviceVlans: []*v1alpha1.Vlan{ + { + VlanID: 24, + VlanAddresses: []string{ + "10.0.0.1/8", + }, + }, + { + VlanID: 48, + VlanAddresses: []string{ + "10.0.0.2/8", + }, + }, + }, + }, + { + DeviceInterface: "bond0", + DeviceBond: &v1alpha1.Bond{ + BondInterfaces: []string{ + "eth3", + "eth4", + }, + }, + }, + }, + }, + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: u, + }, + }, + }, + }, + ), + ) + + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + suite.startRuntime() + + suite.assertLinks( + []string{ + "default/eth1", + }, func(r *network.LinkSpec, asrt *assert.Assertions) { + asrt.Equal(network.ConfigDefault, r.TypedSpec().ConfigLayer) + asrt.True(r.TypedSpec().Up) + }, + ) + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertNoLinks( + []string{ + "default/eth0", + "default/eth2", + "default/eth3", + "default/eth4", + }, + ) + }, + ), + ) +} + +func (suite *LinkConfigSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestLinkConfigSuite(t *testing.T) { + suite.Run(t, new(LinkConfigSuite)) +} diff --git a/internal/app/machined/pkg/controllers/network/link_merge.go b/internal/app/machined/pkg/controllers/network/link_merge.go new file mode 100644 index 0000000..0b0d1ad --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/link_merge.go @@ -0,0 +1,156 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package network provides controllers which manage network resources. +package network + +import ( + "context" + "fmt" + "sort" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// LinkMergeController merges network.LinkSpec in network.ConfigNamespace and produces final network.LinkSpec in network.Namespace. +type LinkMergeController struct{} + +// Name implements controller.Controller interface. +func (ctrl *LinkMergeController) Name() string { + return "network.LinkMergeController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *LinkMergeController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: network.ConfigNamespaceName, + Type: network.LinkSpecType, + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.LinkSpecType, + Kind: controller.InputDestroyReady, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *LinkMergeController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.LinkSpecType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *LinkMergeController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + // list source network configuration resources + list, err := r.List(ctx, resource.NewMetadata(network.ConfigNamespaceName, network.LinkSpecType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing source network routes: %w", err) + } + + // sort by link name, configuration layer + sort.Slice(list.Items, func(i, j int) bool { + left := list.Items[i].(*network.LinkSpec) //nolint:errcheck,forcetypeassert + right := list.Items[j].(*network.LinkSpec) //nolint:errcheck,forcetypeassert + + if left.TypedSpec().Name < right.TypedSpec().Name { + return false + } + + if left.TypedSpec().Name == right.TypedSpec().Name { + return left.TypedSpec().ConfigLayer < right.TypedSpec().ConfigLayer + } + + return true + }) + + // build final link definition merging multiple layers + links := map[string]*network.LinkSpecSpec{} + + for _, res := range list.Items { + link := res.(*network.LinkSpec) //nolint:errcheck,forcetypeassert + id := network.LinkID(link.TypedSpec().Name) + + existing, ok := links[id] + if !ok { + links[id] = link.TypedSpec() + } else if err = existing.Merge(link.TypedSpec()); err != nil { + logger.Warn("error merging links", zap.Error(err)) + } + } + + conflictsDetected := 0 + + for id, link := range links { + if err = r.Modify(ctx, network.NewLinkSpec(network.NamespaceName, id), func(res resource.Resource) error { + l := res.(*network.LinkSpec) //nolint:errcheck,forcetypeassert + + *l.TypedSpec() = *link + + return nil + }); err != nil { + if state.IsPhaseConflictError(err) { + // phase conflict, resource is being torn down, skip updating it and trigger reconcile + // later by failing the + conflictsDetected++ + + delete(links, id) + } else { + return fmt.Errorf("error updating resource: %w", err) + } + } + + logger.Debug("merged link spec", zap.String("id", id), zap.Any("spec", link)) + } + + // list link for cleanup + list, err = r.List(ctx, resource.NewMetadata(network.NamespaceName, network.LinkSpecType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing resources: %w", err) + } + + for _, res := range list.Items { + if _, ok := links[res.Metadata().ID()]; !ok { + var okToDestroy bool + + okToDestroy, err = r.Teardown(ctx, res.Metadata()) + if err != nil { + return fmt.Errorf("error cleaning up addresses: %w", err) + } + + if okToDestroy { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return fmt.Errorf("error cleaning up addresses: %w", err) + } + } + } + } + + if conflictsDetected > 0 { + return fmt.Errorf("%d conflict(s) detected", conflictsDetected) + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/network/link_merge_test.go b/internal/app/machined/pkg/controllers/network/link_merge_test.go new file mode 100644 index 0000000..a25d900 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/link_merge_test.go @@ -0,0 +1,377 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:dupl +package network_test + +import ( + "context" + "log" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "golang.org/x/sync/errgroup" + + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type LinkMergeSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *LinkMergeSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.LinkMergeController{})) + + suite.startRuntime() +} + +func (suite *LinkMergeSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *LinkMergeSuite) assertLinks(requiredIDs []string, check func(*network.LinkSpec, *assert.Assertions)) { + assertResources(suite.ctx, suite.T(), suite.state, requiredIDs, check) +} + +func (suite *LinkMergeSuite) assertNoLinks(id string) error { + resources, err := suite.state.List( + suite.ctx, + resource.NewMetadata(network.NamespaceName, network.AddressStatusType, "", resource.VersionUndefined), + ) + if err != nil { + return err + } + + for _, res := range resources.Items { + if res.Metadata().ID() == id { + return retry.ExpectedErrorf("link %q is still there", id) + } + } + + return nil +} + +func (suite *LinkMergeSuite) TestMerge() { + loopback := network.NewLinkSpec(network.ConfigNamespaceName, "default/lo") + *loopback.TypedSpec() = network.LinkSpecSpec{ + Name: "lo", + Up: true, + ConfigLayer: network.ConfigDefault, + } + + dhcp := network.NewLinkSpec(network.ConfigNamespaceName, "dhcp/eth0") + *dhcp.TypedSpec() = network.LinkSpecSpec{ + Name: "eth0", + Up: true, + MTU: 1450, + ConfigLayer: network.ConfigOperator, + } + + static := network.NewLinkSpec(network.ConfigNamespaceName, "configuration/eth0") + *static.TypedSpec() = network.LinkSpecSpec{ + Name: "eth0", + Up: true, + MTU: 1500, + ConfigLayer: network.ConfigMachineConfiguration, + } + + for _, res := range []resource.Resource{loopback, dhcp, static} { + suite.Require().NoError(suite.state.Create(suite.ctx, res), "%v", res.Spec()) + } + + suite.assertLinks( + []string{ + "lo", + "eth0", + }, func(r *network.LinkSpec, asrt *assert.Assertions) { + switch r.Metadata().ID() { + case "lo": + asrt.Equal(*loopback.TypedSpec(), *r.TypedSpec()) + case "eth0": + asrt.EqualValues(1500, r.TypedSpec().MTU) // static should override dhcp + } + }, + ) + + suite.Require().NoError(suite.state.Destroy(suite.ctx, static.Metadata())) + + suite.assertLinks( + []string{ + "lo", + "eth0", + }, func(r *network.LinkSpec, asrt *assert.Assertions) { + switch r.Metadata().ID() { + case "lo": + asrt.Equal(*loopback.TypedSpec(), *r.TypedSpec()) + case "eth0": + // reconcile happens eventually, so give it some time + asrt.EqualValues(1450, r.TypedSpec().MTU) + } + }, + ) + + suite.Require().NoError(suite.state.Destroy(suite.ctx, loopback.Metadata())) + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertNoLinks("lo") + }, + ), + ) +} + +func (suite *LinkMergeSuite) TestMergeLogicalLink() { + bondPlatform := network.NewLinkSpec(network.ConfigNamespaceName, "platform/bond0") + *bondPlatform.TypedSpec() = network.LinkSpecSpec{ + Name: "bond0", + Logical: true, + Up: true, + BondMaster: network.BondMasterSpec{ + Mode: nethelpers.BondMode8023AD, + }, + ConfigLayer: network.ConfigPlatform, + } + + bondMachineConfig := network.NewLinkSpec(network.ConfigNamespaceName, "config/bond0") + *bondMachineConfig.TypedSpec() = network.LinkSpecSpec{ + Name: "bond0", + MTU: 1450, + Up: true, + ConfigLayer: network.ConfigMachineConfiguration, + } + + for _, res := range []resource.Resource{bondPlatform, bondMachineConfig} { + suite.Require().NoError(suite.state.Create(suite.ctx, res), "%v", res.Spec()) + } + + suite.assertLinks( + []string{ + "bond0", + }, func(r *network.LinkSpec, asrt *assert.Assertions) { + asrt.True(r.TypedSpec().Logical) + asrt.EqualValues(1450, r.TypedSpec().MTU) + }, + ) +} + +func (suite *LinkMergeSuite) TestMergeFlapping() { + // simulate two conflicting link definitions which are getting removed/added constantly + dhcp := network.NewLinkSpec(network.ConfigNamespaceName, "dhcp/eth0") + *dhcp.TypedSpec() = network.LinkSpecSpec{ + Name: "eth0", + Up: true, + MTU: 1450, + ConfigLayer: network.ConfigOperator, + } + + static := network.NewLinkSpec(network.ConfigNamespaceName, "configuration/eth0") + *static.TypedSpec() = network.LinkSpecSpec{ + Name: "eth0", + Up: true, + MTU: 1500, + ConfigLayer: network.ConfigMachineConfiguration, + } + + resources := []resource.Resource{dhcp, static} + + flipflop := func(idx int) func() error { + return func() error { + for range 500 { + if err := suite.state.Create(suite.ctx, resources[idx]); err != nil { + return err + } + + if err := suite.state.Destroy(suite.ctx, resources[idx].Metadata()); err != nil { + return err + } + + time.Sleep(time.Millisecond) + } + + return suite.state.Create(suite.ctx, resources[idx]) + } + } + + var eg errgroup.Group + + eg.Go(flipflop(0)) + eg.Go(flipflop(1)) + eg.Go( + func() error { + // add/remove finalizer to the merged resource + for range 1000 { + if err := suite.state.AddFinalizer( + suite.ctx, + resource.NewMetadata( + network.NamespaceName, + network.LinkSpecType, + "eth0", + resource.VersionUndefined, + ), + "foo", + ); err != nil { + if !state.IsNotFoundError(err) { + return err + } + + continue + } + + suite.T().Log("finalizer added") + + time.Sleep(10 * time.Millisecond) + + if err := suite.state.RemoveFinalizer( + suite.ctx, + resource.NewMetadata( + network.NamespaceName, + network.LinkSpecType, + "eth0", + resource.VersionUndefined, + ), + "foo", + ); err != nil && !state.IsNotFoundError(err) { + return err + } + } + + return nil + }, + ) + + suite.Require().NoError(eg.Wait()) + + suite.assertLinks( + []string{ + "eth0", + }, func(r *network.LinkSpec, asrt *assert.Assertions) { + asrt.EqualValues(1500, r.TypedSpec().MTU) + asrt.EqualValues(resource.PhaseRunning, r.Metadata().Phase()) + }, + ) +} + +func (suite *LinkMergeSuite) TestMergeWireguard() { + static := network.NewLinkSpec(network.ConfigNamespaceName, "configuration/kubespan") + *static.TypedSpec() = network.LinkSpecSpec{ + Name: "kubespan", + Wireguard: network.WireguardSpec{ + ListenPort: 1234, + Peers: []network.WireguardPeer{ + { + PublicKey: "bGsc2rOpl6JHd/Pm4fYrIkEABL0ZxW7IlaSyh77IMhw=", + Endpoint: "127.0.0.1:9999", + }, + }, + }, + ConfigLayer: network.ConfigMachineConfiguration, + } + + kubespanOperator := network.NewLinkSpec(network.ConfigNamespaceName, "kubespan/kubespan") + *kubespanOperator.TypedSpec() = network.LinkSpecSpec{ + Name: "kubespan", + Wireguard: network.WireguardSpec{ + PrivateKey: "IG9MqCII7z54Ysof1fQ9a7WcMNG+qNJRMyRCQz3JTUY=", + ListenPort: 3456, + Peers: []network.WireguardPeer{ + { + PublicKey: "RXdQkMTD1Jcxd/Wizr9k8syw8ANs57l5jTormDVHAVs=", + Endpoint: "127.0.0.1:1234", + }, + }, + }, + ConfigLayer: network.ConfigOperator, + } + + for _, res := range []resource.Resource{static, kubespanOperator} { + suite.Require().NoError(suite.state.Create(suite.ctx, res), "%v", res.Spec()) + } + + suite.assertLinks( + []string{ + "kubespan", + }, func(r *network.LinkSpec, asrt *assert.Assertions) { + asrt.Equal( + "IG9MqCII7z54Ysof1fQ9a7WcMNG+qNJRMyRCQz3JTUY=", + r.TypedSpec().Wireguard.PrivateKey, + ) + asrt.Equal(1234, r.TypedSpec().Wireguard.ListenPort) + asrt.Len(r.TypedSpec().Wireguard.Peers, 2) + + asrt.Equal( + network.WireguardPeer{ + PublicKey: "RXdQkMTD1Jcxd/Wizr9k8syw8ANs57l5jTormDVHAVs=", + Endpoint: "127.0.0.1:1234", + }, + r.TypedSpec().Wireguard.Peers[0], + ) + + asrt.Equal( + network.WireguardPeer{ + PublicKey: "bGsc2rOpl6JHd/Pm4fYrIkEABL0ZxW7IlaSyh77IMhw=", + Endpoint: "127.0.0.1:9999", + }, + r.TypedSpec().Wireguard.Peers[1], + ) + }, + ) + + suite.Require().NoError(suite.state.Destroy(suite.ctx, kubespanOperator.Metadata())) + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertNoLinks("kubespan") + }, + ), + ) +} + +func (suite *LinkMergeSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestLinkMergeSuite(t *testing.T) { + suite.Run(t, new(LinkMergeSuite)) +} diff --git a/internal/app/machined/pkg/controllers/network/link_spec.go b/internal/app/machined/pkg/controllers/network/link_spec.go new file mode 100644 index 0000000..9c2403e --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/link_spec.go @@ -0,0 +1,615 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "context" + "errors" + "fmt" + "sort" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/hashicorp/go-multierror" + "github.com/jsimonetti/rtnetlink" + "github.com/siderolabs/gen/pair/ordered" + "github.com/siderolabs/go-pointer" + "go.uber.org/zap" + "golang.org/x/sys/unix" + "golang.zx2c4.com/wireguard/wgctrl" + + networkadapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/network" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network/watch" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/runtime" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// LinkSpecController applies network.LinkSpec to the actual interfaces. +type LinkSpecController struct{} + +// Name implements controller.Controller interface. +func (ctrl *LinkSpecController) Name() string { + return "network.LinkSpecController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *LinkSpecController) Inputs() []controller.Input { + return nil +} + +// Outputs implements controller.Controller interface. +func (ctrl *LinkSpecController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.LinkRefreshType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *LinkSpecController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + // wait for udevd to be healthy, which implies that all link renames are done + if err := runtime.WaitForDevicesReady(ctx, r, + []controller.Input{ + { + Namespace: network.NamespaceName, + Type: network.LinkSpecType, + Kind: controller.InputStrong, + }, + }, + ); err != nil { + return err + } + + // watch link changes as some routes might need to be re-applied if the link appears + watcher, err := watch.NewRtNetlink(watch.NewDefaultRateLimitedTrigger(ctx, r), unix.RTMGRP_LINK) + if err != nil { + return err + } + + defer watcher.Done() + + conn, err := rtnetlink.Dial(nil) + if err != nil { + return fmt.Errorf("error dialing rtnetlink socket: %w", err) + } + + defer conn.Close() //nolint:errcheck + + wgClient, err := wgctrl.New() + if err != nil { + logger.Warn("error creating wireguard client", zap.Error(err)) + } else { + defer wgClient.Close() //nolint:errcheck + } + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + // list source network configuration resources + list, err := r.List(ctx, resource.NewMetadata(network.NamespaceName, network.LinkSpecType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing source network addresses: %w", err) + } + + // add finalizers for all live resources + for _, res := range list.Items { + if res.Metadata().Phase() != resource.PhaseRunning { + continue + } + + if err = r.AddFinalizer(ctx, res.Metadata(), ctrl.Name()); err != nil { + return fmt.Errorf("error adding finalizer: %w", err) + } + } + + // list rtnetlink links (interfaces) + links, err := conn.Link.List() + if err != nil { + return fmt.Errorf("error listing links: %w", err) + } + + // loop over links and make reconcile decision + var multiErr *multierror.Error + + SortBonds(list.Items) + + for _, res := range list.Items { + link := res.(*network.LinkSpec) //nolint:forcetypeassert,errcheck + + if err = ctrl.syncLink(ctx, r, logger, conn, wgClient, &links, link); err != nil { + multiErr = multierror.Append(multiErr, err) + } + } + + if err = multiErr.ErrorOrNil(); err != nil { + return err + } + + r.ResetRestartBackoff() + } +} + +// SortBonds sort resources in increasing order, except it places slave interfaces right after the bond +// in proper order. +func SortBonds(items []resource.Resource) { + sort.Slice(items, func(i, j int) bool { + left := items[i].(*network.LinkSpec).TypedSpec() + right := items[j].(*network.LinkSpec).TypedSpec() + + l := ordered.MakeTriple(left.Name, 0, "") + if left.BondSlave.MasterName != "" { + l = ordered.MakeTriple(left.BondSlave.MasterName, left.BondSlave.SlaveIndex, left.Name) + } + + r := ordered.MakeTriple(right.Name, 0, "") + if right.BondSlave.MasterName != "" { + r = ordered.MakeTriple(right.BondSlave.MasterName, right.BondSlave.SlaveIndex, right.Name) + } + + return l.LessThan(r) + }) +} + +func findLink(links []rtnetlink.LinkMessage, name string) *rtnetlink.LinkMessage { + for i, link := range links { + if link.Attributes.Name == name { + return &links[i] + } + } + + return nil +} + +// syncLink syncs kernel state with the LinkSpec link. +// +// This method is really long, but it's hard to break it down in multiple pieces, are those pieces and steps are inter-dependent, so, instead, +// I'm going to provide high-level flow of the method here to help understand it: +// +// First of all, if the spec is being torn down - remove the link from the kernel, done. +// If the link spec is not being torn down, start the sync process: +// +// - for physical links, there's not much we can sync - only MTU and 'UP' flag +// - for logical links, controller handles creation and sync of the settings depending on the interface type +// +// If the logical link kind or type got changed (for example, "link0" was a bond, and now it's wireguard interface), the link +// is dropped and replaced with the new one. +// Same replace flow is used for VLAN links, as VLAN settings can't be changed on the fly. +// +// For bonded links, there are two sync steps applied: +// +// - bond slave interfaces are enslaved to be part of the bond (by changing MasterIndex) +// - bond master link settings are synced with the spec: some settings can't be applied on UP bond and a bond which has slaves, +// so slaves are removed and bond is brought down (these settings are going to be reconciled back in the next sync cycle) +// +// For wireguard links, only settings are synced with the diff generated by the WireguardSpec. +// +//nolint:gocyclo,cyclop,dupl +func (ctrl *LinkSpecController) syncLink(ctx context.Context, r controller.Runtime, logger *zap.Logger, conn *rtnetlink.Conn, wgClient *wgctrl.Client, + links *[]rtnetlink.LinkMessage, link *network.LinkSpec, +) error { + logger = logger.With(zap.String("link", link.TypedSpec().Name)) + + switch link.Metadata().Phase() { + case resource.PhaseTearingDown: + // TODO: should we bring link down if it's physical and the spec was torn down? + if link.TypedSpec().Logical { + existing := findLink(*links, link.TypedSpec().Name) + + if existing != nil { + if err := conn.Link.Delete(existing.Index); err != nil { + return fmt.Errorf("error deleting link %q: %w", link.TypedSpec().Name, err) + } + + logger.Info("deleted link") + + // refresh links as the link list got changed + var err error + + *links, err = conn.Link.List() + if err != nil { + return fmt.Errorf("error listing links: %w", err) + } + } + } + + // now remove finalizer as link was deleted + if err := r.RemoveFinalizer(ctx, link.Metadata(), ctrl.Name()); err != nil { + return fmt.Errorf("error removing finalizer: %w", err) + } + case resource.PhaseRunning: + existing := findLink(*links, link.TypedSpec().Name) + + // check if type/kind matches for the existing logical link + if existing != nil && link.TypedSpec().Logical { + replace := false + + if existing.Attributes.Info == nil { + logger.Warn("requested logical link has no info, skipping sync", + zap.String("name", existing.Attributes.Name), + zap.Stringer("type", nethelpers.LinkType(existing.Type)), + zap.Uint32("index", existing.Index), + ) + + return nil + } + + // if type/kind doesn't match, recreate the link to change it + if existing.Type != uint16(link.TypedSpec().Type) || existing.Attributes.Info.Kind != link.TypedSpec().Kind { + logger.Info("replacing logical link", + zap.String("old_kind", existing.Attributes.Info.Kind), + zap.String("new_kind", link.TypedSpec().Kind), + zap.Stringer("old_type", nethelpers.LinkType(existing.Type)), + zap.Stringer("new_type", link.TypedSpec().Type), + ) + + replace = true + } + + // sync VLAN spec, as it can't be modified on the fly + if !replace && link.TypedSpec().Kind == network.LinkKindVLAN { + var existingVLAN network.VLANSpec + + if err := networkadapter.VLANSpec(&existingVLAN).Decode(existing.Attributes.Info.Data); err != nil { + return fmt.Errorf("error decoding VLAN properties on %q: %w", link.TypedSpec().Name, err) + } + + if existingVLAN != link.TypedSpec().VLAN { + logger.Info("replacing VLAN link", + zap.Uint16("old_id", existingVLAN.VID), + zap.Uint16("new_id", link.TypedSpec().VLAN.VID), + zap.Stringer("old_protocol", existingVLAN.Protocol), + zap.Stringer("new_protocol", link.TypedSpec().VLAN.Protocol), + ) + + replace = true + } + } + + if replace { + if err := conn.Link.Delete(existing.Index); err != nil { + return fmt.Errorf("error deleting link %q: %w", link.TypedSpec().Name, err) + } + + // not refreshing links, as the link is set to be re-created + + existing = nil + } + } + + if existing == nil { + if !link.TypedSpec().Logical { + // physical interface doesn't exist yet, nothing to be done + return nil + } + + // create logical interface + var ( + parentIndex uint32 + data []byte + err error + ) + + // VLAN settings should be set on interface creation (parent + VLAN settings) + if link.TypedSpec().ParentName != "" { + parent := findLink(*links, link.TypedSpec().ParentName) + if parent == nil { + // parent doesn't exist yet, skip it + return nil + } + + parentIndex = parent.Index + } + + if link.TypedSpec().Kind == network.LinkKindVLAN { + data, err = networkadapter.VLANSpec(&link.TypedSpec().VLAN).Encode() + if err != nil { + return fmt.Errorf("error encoding VLAN attributes for link %q: %w", link.TypedSpec().Name, err) + } + } + + if err = conn.Link.New(&rtnetlink.LinkMessage{ + Type: uint16(link.TypedSpec().Type), + Attributes: &rtnetlink.LinkAttributes{ + Name: link.TypedSpec().Name, + Type: parentIndex, + Info: &rtnetlink.LinkInfo{ + Kind: link.TypedSpec().Kind, + Data: data, + }, + }, + }); err != nil { + return fmt.Errorf("error creating logical link %q: %w", link.TypedSpec().Name, err) + } + + logger.Info("created new link", zap.String("kind", link.TypedSpec().Kind)) + + // refresh links as the link list got changed + *links, err = conn.Link.List() + if err != nil { + return fmt.Errorf("error listing links: %w", err) + } + + existing = findLink(*links, link.TypedSpec().Name) + if existing == nil { + return fmt.Errorf("created link %q not found in the link list", link.TypedSpec().Name) + } + } + + // sync bond settings + if link.TypedSpec().Kind == network.LinkKindBond { + var existingBond network.BondMasterSpec + + if err := networkadapter.BondMasterSpec(&existingBond).Decode(existing.Attributes.Info.Data); err != nil { + return fmt.Errorf("error parsing bond attributes for %q: %w", link.TypedSpec().Name, err) + } + + if existingBond != link.TypedSpec().BondMaster { + logger.Debug("updating bond settings", + zap.String("old", fmt.Sprintf("%+v", existingBond)), + zap.String("new", fmt.Sprintf("%+v", link.TypedSpec().BondMaster)), + ) + + data, err := networkadapter.BondMasterSpec(&link.TypedSpec().BondMaster).Encode() + if err != nil { + return fmt.Errorf("error encoding bond attributes for %q: %w", link.TypedSpec().Name, err) + } + + // bring bond down + if err = conn.Link.Set(&rtnetlink.LinkMessage{ + Family: existing.Family, + Type: existing.Type, + Index: existing.Index, + Flags: 0, + Change: unix.IFF_UP, + }); err != nil { + return fmt.Errorf("error changing flags for %q: %w", link.TypedSpec().Name, err) + } + + // unslave all slaves + for i, slave := range *links { + if slave.Attributes.Master != nil && *slave.Attributes.Master == existing.Index { + if err = conn.Link.Set(&rtnetlink.LinkMessage{ + Family: slave.Family, + Type: slave.Type, + Index: slave.Index, + Attributes: &rtnetlink.LinkAttributes{ + Master: pointer.To[uint32](0), + }, + }); err != nil { + return fmt.Errorf("error unslaving link %q under %q: %w", slave.Attributes.Name, link.TypedSpec().BondSlave.MasterName, err) + } + + (*links)[i].Attributes.Master = nil + } + } + + // update settings + if err = conn.Link.Set(&rtnetlink.LinkMessage{ + Family: existing.Family, + Type: existing.Type, + Index: existing.Index, + Attributes: &rtnetlink.LinkAttributes{ + Info: &rtnetlink.LinkInfo{ + Kind: existing.Attributes.Info.Kind, + Data: data, + }, + }, + }); err != nil { + return fmt.Errorf("error updating bond settings for %q: %w", link.TypedSpec().Name, err) + } + + logger.Info("updated bond settings") + } + } + + // sync bridge settings + if link.TypedSpec().Kind == network.LinkKindBridge { + var existingBridge network.BridgeMasterSpec + + if err := networkadapter.BridgeMasterSpec(&existingBridge).Decode(existing.Attributes.Info.Data); err != nil { + return fmt.Errorf("error parsing bridge attributes for %q: %w", link.TypedSpec().Name, err) + } + + if existingBridge != link.TypedSpec().BridgeMaster { + logger.Debug("updating bridge settings", + zap.String("old", fmt.Sprintf("%+v", existingBridge)), + zap.String("new", fmt.Sprintf("%+v", link.TypedSpec().BridgeMaster)), + ) + + data, err := networkadapter.BridgeMasterSpec(&link.TypedSpec().BridgeMaster).Encode() + if err != nil { + return fmt.Errorf("error encoding bridge attributes for %q: %w", link.TypedSpec().Name, err) + } + + // bring bridge down + if err = conn.Link.Set(&rtnetlink.LinkMessage{ + Family: existing.Family, + Type: existing.Type, + Index: existing.Index, + Flags: 0, + Change: unix.IFF_UP, + }); err != nil { + return fmt.Errorf("error changing flags for %q: %w", link.TypedSpec().Name, err) + } + + // unslave all slaves + for i, slave := range *links { + if slave.Attributes.Master != nil && *slave.Attributes.Master == existing.Index { + if err = conn.Link.Set(&rtnetlink.LinkMessage{ + Family: slave.Family, + Type: slave.Type, + Index: slave.Index, + Attributes: &rtnetlink.LinkAttributes{ + Master: pointer.To[uint32](0), + }, + }); err != nil { + return fmt.Errorf("error unslaving link %q under %q: %w", slave.Attributes.Name, link.TypedSpec().BridgeSlave.MasterName, err) + } + + (*links)[i].Attributes.Master = nil + } + } + + // update settings + if err = conn.Link.Set(&rtnetlink.LinkMessage{ + Family: existing.Family, + Type: existing.Type, + Index: existing.Index, + Attributes: &rtnetlink.LinkAttributes{ + Info: &rtnetlink.LinkInfo{ + Kind: existing.Attributes.Info.Kind, + Data: data, + }, + }, + }); err != nil { + return fmt.Errorf("error updating bridge settings for %q: %w", link.TypedSpec().Name, err) + } + + logger.Info("updated bridge settings") + } + } + + // sync wireguard settings + if link.TypedSpec().Kind == network.LinkKindWireguard { + if wgClient == nil { + return fmt.Errorf("wireguard client not available, cannot configure wireguard link %q", link.TypedSpec().Name) + } + + wgDev, err := wgClient.Device(link.TypedSpec().Name) + if err != nil { + return fmt.Errorf("error getting wireguard settings for %q: %w", link.TypedSpec().Name, err) + } + + var existingSpec network.WireguardSpec + + networkadapter.WireguardSpec(&existingSpec).Decode(wgDev, false) + existingSpec.Sort() + + link.TypedSpec().Wireguard.Sort() + + // order here is important: we allow listenPort to be zero in the configuration + if !existingSpec.Equal(&link.TypedSpec().Wireguard) { + config, err := networkadapter.WireguardSpec(&link.TypedSpec().Wireguard).Encode(&existingSpec) + if err != nil { + return fmt.Errorf("error creating wireguard config patch for %q: %w", link.TypedSpec().Name, err) + } + + if err = wgClient.ConfigureDevice(link.TypedSpec().Name, *config); err != nil { + return fmt.Errorf("error configuring wireguard device %q: %w", link.TypedSpec().Name, err) + } + + logger.Info("reconfigured wireguard link", zap.Int("peers", len(link.TypedSpec().Wireguard.Peers))) + + // notify link status controller, as wireguard updates can't be watched via netlink API + if err = safe.WriterModify[*network.LinkRefresh](ctx, r, network.NewLinkRefresh(network.NamespaceName, network.LinkKindWireguard), func(r *network.LinkRefresh) error { + r.TypedSpec().Bump() + + return nil + }); err != nil { + return errors.New("error bumping link refresh") + } + } + } + + // sync UP flag + existingUp := existing.Flags&unix.IFF_UP == unix.IFF_UP + if existingUp != link.TypedSpec().Up { + flags := uint32(0) + + if link.TypedSpec().Up { + flags = unix.IFF_UP + } + + if err := conn.Link.Set(&rtnetlink.LinkMessage{ + Family: existing.Family, + Type: existing.Type, + Index: existing.Index, + Flags: flags, + Change: unix.IFF_UP, + }); err != nil { + return fmt.Errorf("error changing flags for %q: %w", link.TypedSpec().Name, err) + } + + logger.Debug("brought link up/down", zap.Bool("up", link.TypedSpec().Up)) + } + + // sync MTU if it's set in the spec + if link.TypedSpec().MTU != 0 && existing.Attributes.MTU != link.TypedSpec().MTU { + if err := conn.Link.Set(&rtnetlink.LinkMessage{ + Family: existing.Family, + Type: existing.Type, + Index: existing.Index, + Attributes: &rtnetlink.LinkAttributes{ + MTU: link.TypedSpec().MTU, + }, + }); err != nil { + return fmt.Errorf("error setting MTU for %q: %w", link.TypedSpec().Name, err) + } + + existing.Attributes.MTU = link.TypedSpec().MTU + + logger.Info("changed MTU for the link", zap.Uint32("mtu", link.TypedSpec().MTU)) + } + + // sync master index (for links which are bridge or bond slaves) + var masterIndex uint32 + + var masterName string + + bondMasterName := link.TypedSpec().BondSlave.MasterName + if bondMasterName != "" { + if master := findLink(*links, bondMasterName); master != nil { + masterName = bondMasterName + masterIndex = master.Index + } + } + + bridgeMasterName := link.TypedSpec().BridgeSlave.MasterName + if bridgeMasterName != "" { + if master := findLink(*links, bridgeMasterName); master != nil { + masterName = bridgeMasterName + masterIndex = master.Index + } + } + + if (existing.Attributes.Master == nil && masterIndex != 0) || (existing.Attributes.Master != nil && *existing.Attributes.Master != masterIndex) { + if err := conn.Link.Set(&rtnetlink.LinkMessage{ + Family: existing.Family, + Type: existing.Type, + Index: existing.Index, + Change: unix.IFF_UP, + }); err != nil { + return fmt.Errorf("error bring down link %q before enslaving under %q: %w", link.TypedSpec().Name, masterName, err) + } + + if err := conn.Link.Set(&rtnetlink.LinkMessage{ + Family: existing.Family, + Type: existing.Type, + Index: existing.Index, + Attributes: &rtnetlink.LinkAttributes{ + Master: pointer.To(masterIndex), + }, + }); err != nil { + return fmt.Errorf("error enslaving/unslaving link %q under %q: %w", link.TypedSpec().Name, masterName, err) + } + + existing.Attributes.Master = pointer.To(masterIndex) + + logger.Info("enslaved/unslaved link", zap.String("parent", masterName)) + } + } + + return nil +} diff --git a/internal/app/machined/pkg/controllers/network/link_spec_test.go b/internal/app/machined/pkg/controllers/network/link_spec_test.go new file mode 100644 index 0000000..45bec80 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/link_spec_test.go @@ -0,0 +1,974 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:dupl +package network_test + +import ( + "context" + "fmt" + "log" + "math/rand" + "net/netip" + "os" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + networkadapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/network" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +type LinkSpecSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *LinkSpecSuite) State() state.State { return suite.state } + +func (suite *LinkSpecSuite) Ctx() context.Context { return suite.ctx } + +func (suite *LinkSpecSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + // create fake device ready status + deviceStatus := runtimeres.NewDevicesStatus(runtimeres.NamespaceName, runtimeres.DevicesID) + deviceStatus.TypedSpec().Ready = true + suite.Require().NoError(suite.state.Create(suite.ctx, deviceStatus)) + + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.LinkSpecController{})) + + // register status controller to assert on the created links + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.LinkStatusController{})) + + suite.startRuntime() +} + +func (suite *LinkSpecSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *LinkSpecSuite) assertInterfaces(requiredIDs []string, check func(*network.LinkStatus) error) error { + missingIDs := make(map[string]struct{}, len(requiredIDs)) + + for _, id := range requiredIDs { + missingIDs[id] = struct{}{} + } + + resources, err := suite.state.List( + suite.ctx, + resource.NewMetadata(network.NamespaceName, network.LinkStatusType, "", resource.VersionUndefined), + ) + if err != nil { + return err + } + + for _, res := range resources.Items { + _, required := missingIDs[res.Metadata().ID()] + if !required { + continue + } + + delete(missingIDs, res.Metadata().ID()) + + if err = check(res.(*network.LinkStatus)); err != nil { + return retry.ExpectedError(err) + } + } + + if len(missingIDs) > 0 { + return retry.ExpectedErrorf("some resources are missing: %q", missingIDs) + } + + return nil +} + +func (suite *LinkSpecSuite) assertNoInterface(id string) error { + resources, err := suite.state.List( + suite.ctx, + resource.NewMetadata(network.NamespaceName, network.LinkStatusType, "", resource.VersionUndefined), + ) + if err != nil { + return err + } + + for _, res := range resources.Items { + if res.Metadata().ID() == id { + return retry.ExpectedErrorf("interface %q is still there", id) + } + } + + return nil +} + +func (suite *LinkSpecSuite) uniqueDummyInterface() string { + return fmt.Sprintf("dummy%02x%02x%02x", rand.Int31()&0xff, rand.Int31()&0xff, rand.Int31()&0xff) +} + +func (suite *LinkSpecSuite) TestLoopback() { + loopback := network.NewLinkSpec(network.NamespaceName, "lo") + *loopback.TypedSpec() = network.LinkSpecSpec{ + Name: "lo", + Up: true, + ConfigLayer: network.ConfigDefault, + } + + for _, res := range []resource.Resource{loopback} { + suite.Require().NoError(suite.state.Create(suite.ctx, res), "%v", res.Spec()) + } + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertInterfaces( + []string{"lo"}, func(r *network.LinkStatus) error { + return nil + }, + ) + }, + ), + ) +} + +func (suite *LinkSpecSuite) TestDummy() { + dummyInterface := suite.uniqueDummyInterface() + + dummy := network.NewLinkSpec(network.NamespaceName, dummyInterface) + *dummy.TypedSpec() = network.LinkSpecSpec{ + Name: dummyInterface, + Type: nethelpers.LinkEther, + Kind: "dummy", + MTU: 1400, + Up: true, + Logical: true, + ConfigLayer: network.ConfigDefault, + } + + for _, res := range []resource.Resource{dummy} { + suite.Require().NoError(suite.state.Create(suite.ctx, res), "%v", res.Spec()) + } + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertInterfaces( + []string{dummyInterface}, func(r *network.LinkStatus) error { + suite.Assert().Equal("dummy", r.TypedSpec().Kind) + + if r.TypedSpec().OperationalState != nethelpers.OperStateUnknown && r.TypedSpec().OperationalState != nethelpers.OperStateUp { + return retry.ExpectedErrorf("link is not up") + } + + if r.TypedSpec().MTU != 1400 { + return retry.ExpectedErrorf("unexpected MTU %d", r.TypedSpec().MTU) + } + + return nil + }, + ) + }, + ), + ) + + // teardown the link + for { + ready, err := suite.state.Teardown(suite.ctx, dummy.Metadata()) + suite.Require().NoError(err) + + if ready { + break + } + + time.Sleep(100 * time.Millisecond) + } + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertNoInterface(dummyInterface) + }, + ), + ) +} + +//nolint:gocyclo +func (suite *LinkSpecSuite) TestVLAN() { + dummyInterface := suite.uniqueDummyInterface() + + dummy := network.NewLinkSpec(network.NamespaceName, dummyInterface) + *dummy.TypedSpec() = network.LinkSpecSpec{ + Name: dummyInterface, + Type: nethelpers.LinkEther, + Kind: "dummy", + Up: true, + Logical: true, + ConfigLayer: network.ConfigDefault, + } + + vlanName1 := fmt.Sprintf("%s.%d", dummyInterface, 2) + vlan1 := network.NewLinkSpec(network.NamespaceName, vlanName1) + *vlan1.TypedSpec() = network.LinkSpecSpec{ + Name: vlanName1, + Type: nethelpers.LinkEther, + Kind: network.LinkKindVLAN, + Up: true, + Logical: true, + ParentName: dummyInterface, + ConfigLayer: network.ConfigDefault, + VLAN: network.VLANSpec{ + VID: 2, + Protocol: nethelpers.VLANProtocol8021Q, + }, + } + + vlanName2 := fmt.Sprintf("%s.%d", dummyInterface, 4) + vlan2 := network.NewLinkSpec(network.NamespaceName, vlanName2) + *vlan2.TypedSpec() = network.LinkSpecSpec{ + Name: vlanName2, + Type: nethelpers.LinkEther, + Kind: network.LinkKindVLAN, + Up: true, + Logical: true, + ParentName: dummyInterface, + ConfigLayer: network.ConfigDefault, + VLAN: network.VLANSpec{ + VID: 4, + Protocol: nethelpers.VLANProtocol8021Q, + }, + } + + for _, res := range []resource.Resource{dummy, vlan1, vlan2} { + suite.Require().NoError(suite.state.Create(suite.ctx, res), "%v", res.Spec()) + } + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertInterfaces( + []string{dummyInterface, vlanName1, vlanName2}, func(r *network.LinkStatus) error { + switch r.Metadata().ID() { + case dummyInterface: + suite.Assert().Equal("dummy", r.TypedSpec().Kind) + case vlanName1, vlanName2: + suite.Assert().Equal(network.LinkKindVLAN, r.TypedSpec().Kind) + suite.Assert().Equal(nethelpers.VLANProtocol8021Q, r.TypedSpec().VLAN.Protocol) + + if r.Metadata().ID() == vlanName1 { + suite.Assert().EqualValues(2, r.TypedSpec().VLAN.VID) + } else { + suite.Assert().EqualValues(4, r.TypedSpec().VLAN.VID) + } + } + + if r.TypedSpec().OperationalState != nethelpers.OperStateUnknown && r.TypedSpec().OperationalState != nethelpers.OperStateUp { + return retry.ExpectedErrorf("link is not up") + } + + return nil + }, + ) + }, + ), + ) + + // attempt to change VLAN ID + ctest.UpdateWithConflicts(suite, vlan1, func(r *network.LinkSpec) error { + r.TypedSpec().VLAN.VID = 42 + + return nil + }) + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertInterfaces( + []string{vlanName1}, func(r *network.LinkStatus) error { + if r.TypedSpec().VLAN.VID != 42 { + return retry.ExpectedErrorf("vlan ID is not 42: %d", r.TypedSpec().VLAN.VID) + } + + return nil + }, + ) + }, + ), + ) + + // teardown the links + for _, r := range []resource.Resource{vlan1, vlan2, dummy} { + for { + ready, err := suite.state.Teardown(suite.ctx, r.Metadata()) + suite.Require().NoError(err) + + if ready { + break + } + + time.Sleep(100 * time.Millisecond) + } + } + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertNoInterface(dummyInterface) + }, + ), + ) +} + +//nolint:gocyclo +func (suite *LinkSpecSuite) TestBond() { + bondName := suite.uniqueDummyInterface() + bond := network.NewLinkSpec(network.NamespaceName, bondName) + *bond.TypedSpec() = network.LinkSpecSpec{ + Name: bondName, + Type: nethelpers.LinkEther, + Kind: network.LinkKindBond, + Up: true, + Logical: true, + BondMaster: network.BondMasterSpec{ + Mode: nethelpers.BondModeActiveBackup, + ARPAllTargets: nethelpers.ARPAllTargetsAll, + PrimaryReselect: nethelpers.PrimaryReselectBetter, + FailOverMac: nethelpers.FailOverMACFollow, + ADSelect: nethelpers.ADSelectBandwidth, + MIIMon: 100, + DownDelay: 100, + ResendIGMP: 2, + UseCarrier: true, + }, + ConfigLayer: network.ConfigDefault, + } + networkadapter.BondMasterSpec(&bond.TypedSpec().BondMaster).FillDefaults() + + dummy0Name := suite.uniqueDummyInterface() + dummy0 := network.NewLinkSpec(network.NamespaceName, dummy0Name) + *dummy0.TypedSpec() = network.LinkSpecSpec{ + Name: dummy0Name, + Type: nethelpers.LinkEther, + Kind: "dummy", + Up: true, + Logical: true, + BondSlave: network.BondSlave{ + MasterName: bondName, + SlaveIndex: 0, + }, + ConfigLayer: network.ConfigDefault, + } + + dummy1Name := suite.uniqueDummyInterface() + dummy1 := network.NewLinkSpec(network.NamespaceName, dummy1Name) + *dummy1.TypedSpec() = network.LinkSpecSpec{ + Name: dummy1Name, + Type: nethelpers.LinkEther, + Kind: "dummy", + Up: true, + Logical: true, + BondSlave: network.BondSlave{ + MasterName: bondName, + SlaveIndex: 1, + }, + ConfigLayer: network.ConfigDefault, + } + + for _, res := range []resource.Resource{dummy0, dummy1, bond} { + suite.Require().NoError(suite.state.Create(suite.ctx, res), "%v", res.Spec()) + } + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertInterfaces( + []string{dummy0Name, dummy1Name, bondName}, func(r *network.LinkStatus) error { + switch r.Metadata().ID() { + case bondName: + suite.Assert().Equal(network.LinkKindBond, r.TypedSpec().Kind) + + if r.TypedSpec().OperationalState != nethelpers.OperStateUnknown && r.TypedSpec().OperationalState != nethelpers.OperStateUp { + return retry.ExpectedErrorf("link is not up: %s", r.TypedSpec().OperationalState) + } + case dummy0Name, dummy1Name: + suite.Assert().Equal("dummy", r.TypedSpec().Kind) + + if r.TypedSpec().OperationalState != nethelpers.OperStateUnknown { + return retry.ExpectedErrorf("link is not up: %s", r.TypedSpec().OperationalState) + } + + if r.TypedSpec().MasterIndex == 0 { + return retry.ExpectedErrorf("masterIndex should be non-zero") + } + } + + return nil + }, + ) + }, + ), + ) + + // attempt to change bond type + ctest.UpdateWithConflicts(suite, bond, func(r *network.LinkSpec) error { + r.TypedSpec().BondMaster.Mode = nethelpers.BondModeRoundrobin + + return nil + }) + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertInterfaces( + []string{bondName}, func(r *network.LinkStatus) error { + if r.TypedSpec().BondMaster.Mode != nethelpers.BondModeRoundrobin { + return retry.ExpectedErrorf( + "bond mode is not %s: %s", + nethelpers.BondModeRoundrobin, + r.TypedSpec().BondMaster.Mode, + ) + } + + return nil + }, + ) + }, + ), + ) + + // unslave one of the interfaces + ctest.UpdateWithConflicts(suite, dummy0, func(r *network.LinkSpec) error { + r.TypedSpec().BondSlave.MasterName = "" + + return nil + }) + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertInterfaces( + []string{dummy0Name}, func(r *network.LinkStatus) error { + if r.TypedSpec().MasterIndex != 0 { + return retry.ExpectedErrorf("iface not unslaved yet") + } + + return nil + }, + ) + }, + ), + ) + + // teardown the links + for _, r := range []resource.Resource{dummy0, dummy1, bond} { + for { + ready, err := suite.state.Teardown(suite.ctx, r.Metadata()) + suite.Require().NoError(err) + + if ready { + break + } + + time.Sleep(100 * time.Millisecond) + } + } + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertNoInterface(bondName) + }, + ), + ) +} + +//nolint:gocyclo +func (suite *LinkSpecSuite) TestBond8023ad() { + bondName := suite.uniqueDummyInterface() + bond := network.NewLinkSpec(network.NamespaceName, bondName) + *bond.TypedSpec() = network.LinkSpecSpec{ + Name: bondName, + Type: nethelpers.LinkEther, + Kind: network.LinkKindBond, + MTU: 9000, + Up: true, + Logical: true, + BondMaster: network.BondMasterSpec{ + Mode: nethelpers.BondMode8023AD, + LACPRate: nethelpers.LACPRateFast, + UseCarrier: true, + }, + ConfigLayer: network.ConfigDefault, + } + networkadapter.BondMasterSpec(&bond.TypedSpec().BondMaster).FillDefaults() + + //nolint:prealloc + var ( + dummies []resource.Resource + dummyNames []string + ) + + for range 4 { + dummyName := suite.uniqueDummyInterface() + dummy := network.NewLinkSpec(network.NamespaceName, dummyName) + *dummy.TypedSpec() = network.LinkSpecSpec{ + Name: dummyName, + Type: nethelpers.LinkEther, + Kind: "dummy", + Up: true, + Logical: true, + BondSlave: network.BondSlave{ + MasterName: bondName, + SlaveIndex: 0, + }, + ConfigLayer: network.ConfigDefault, + } + + dummies = append(dummies, dummy) + dummyNames = append(dummyNames, dummyName) + } + + for _, res := range append(dummies, bond) { + suite.Require().NoError(suite.state.Create(suite.ctx, res), "%v", res.Spec()) + } + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertInterfaces( + append(dummyNames, bondName), func(r *network.LinkStatus) error { + if r.Metadata().ID() == bondName { + // master + suite.Assert().Equal(network.LinkKindBond, r.TypedSpec().Kind) + + if r.TypedSpec().OperationalState != nethelpers.OperStateUnknown && r.TypedSpec().OperationalState != nethelpers.OperStateUp { + return retry.ExpectedErrorf("link is not up: %s", r.TypedSpec().OperationalState) + } + } else { + // slaves + suite.Assert().Equal("dummy", r.TypedSpec().Kind) + + if r.TypedSpec().OperationalState != nethelpers.OperStateUnknown { + return retry.ExpectedErrorf("link is not up: %s", r.TypedSpec().OperationalState) + } + + if r.TypedSpec().MasterIndex == 0 { + return retry.ExpectedErrorf("masterIndex should be non-zero") + } + } + + return nil + }, + ) + }, + ), + ) + + // teardown the links + for _, r := range append(dummies, bond) { + for { + ready, err := suite.state.Teardown(suite.ctx, r.Metadata()) + suite.Require().NoError(err) + + if ready { + break + } + + time.Sleep(100 * time.Millisecond) + } + } + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertNoInterface(bondName) + }, + ), + ) +} + +//nolint:gocyclo +func (suite *LinkSpecSuite) TestBridge() { + bridgeName := suite.uniqueDummyInterface() + bridge := network.NewLinkSpec(network.NamespaceName, bridgeName) + *bridge.TypedSpec() = network.LinkSpecSpec{ + Name: bridgeName, + Type: nethelpers.LinkEther, + Kind: network.LinkKindBridge, + Up: true, + Logical: true, + BridgeMaster: network.BridgeMasterSpec{ + STP: network.STPSpec{ + Enabled: false, + }, + }, + ConfigLayer: network.ConfigDefault, + } + + dummy0Name := suite.uniqueDummyInterface() + dummy0 := network.NewLinkSpec(network.NamespaceName, dummy0Name) + *dummy0.TypedSpec() = network.LinkSpecSpec{ + Name: dummy0Name, + Type: nethelpers.LinkEther, + Kind: "dummy", + Up: true, + Logical: true, + BridgeSlave: network.BridgeSlave{ + MasterName: bridgeName, + }, + ConfigLayer: network.ConfigDefault, + } + + dummy1Name := suite.uniqueDummyInterface() + dummy1 := network.NewLinkSpec(network.NamespaceName, dummy1Name) + *dummy1.TypedSpec() = network.LinkSpecSpec{ + Name: dummy1Name, + Type: nethelpers.LinkEther, + Kind: "dummy", + Up: true, + Logical: true, + BridgeSlave: network.BridgeSlave{ + MasterName: bridgeName, + }, + ConfigLayer: network.ConfigDefault, + } + + for _, res := range []resource.Resource{dummy0, dummy1, bridge} { + suite.Require().NoError(suite.state.Create(suite.ctx, res), "%v", res.Spec()) + } + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertInterfaces( + []string{dummy0Name, dummy1Name, bridgeName}, func(r *network.LinkStatus) error { + switch r.Metadata().ID() { + case bridgeName: + suite.Assert().Equal(network.LinkKindBridge, r.TypedSpec().Kind) + + if r.TypedSpec().OperationalState != nethelpers.OperStateUnknown && r.TypedSpec().OperationalState != nethelpers.OperStateUp { + return retry.ExpectedErrorf("link is not up: %s", r.TypedSpec().OperationalState) + } + case dummy0Name, dummy1Name: + suite.Assert().Equal("dummy", r.TypedSpec().Kind) + + if r.TypedSpec().OperationalState != nethelpers.OperStateUnknown { + return retry.ExpectedErrorf("link is not up: %s", r.TypedSpec().OperationalState) + } + + if r.TypedSpec().MasterIndex == 0 { + return retry.ExpectedErrorf("masterIndex should be non-zero") + } + } + + return nil + }, + ) + }, + ), + ) + + // attempt to enable STP + ctest.UpdateWithConflicts(suite, bridge, func(r *network.LinkSpec) error { + r.TypedSpec().BridgeMaster.STP.Enabled = true + + return nil + }) + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertInterfaces( + []string{bridgeName}, func(r *network.LinkStatus) error { + if !r.TypedSpec().BridgeMaster.STP.Enabled { + return retry.ExpectedErrorf( + "stp is not enabled on bridge %s", r.Metadata().ID(), + ) + } + + return nil + }, + ) + }, + ), + ) + + // unslave one of the interfaces + ctest.UpdateWithConflicts(suite, dummy0, func(r *network.LinkSpec) error { + r.TypedSpec().BridgeSlave.MasterName = "" + + return nil + }) + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertInterfaces( + []string{dummy0Name}, func(r *network.LinkStatus) error { + if r.TypedSpec().MasterIndex != 0 { + return retry.ExpectedErrorf("iface not unslaved yet") + } + + return nil + }, + ) + }, + ), + ) + + // teardown the links + for _, r := range []resource.Resource{dummy0, dummy1, bridge} { + for { + ready, err := suite.state.Teardown(suite.ctx, r.Metadata()) + suite.Require().NoError(err) + + if ready { + break + } + + time.Sleep(100 * time.Millisecond) + } + } + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertNoInterface(bridgeName) + }, + ), + ) +} + +func (suite *LinkSpecSuite) TestWireguard() { + priv, err := wgtypes.GeneratePrivateKey() + suite.Require().NoError(err) + + pub1, err := wgtypes.GeneratePrivateKey() + suite.Require().NoError(err) + + pub2, err := wgtypes.GeneratePrivateKey() + suite.Require().NoError(err) + + wgInterface := suite.uniqueDummyInterface() + + wg := network.NewLinkSpec(network.NamespaceName, wgInterface) + *wg.TypedSpec() = network.LinkSpecSpec{ + Name: wgInterface, + Type: nethelpers.LinkNone, + Kind: "wireguard", + Up: true, + Logical: true, + Wireguard: network.WireguardSpec{ + PrivateKey: priv.String(), + ListenPort: 30000, + FirewallMark: 1, + Peers: []network.WireguardPeer{ + { + PublicKey: pub1.PublicKey().String(), + Endpoint: "10.2.0.3:20000", + AllowedIPs: []netip.Prefix{ + netip.MustParsePrefix("172.24.0.0/16"), + }, + }, + { + PublicKey: pub2.PublicKey().String(), + AllowedIPs: []netip.Prefix{ + netip.MustParsePrefix("172.25.0.0/24"), + }, + }, + }, + }, + ConfigLayer: network.ConfigDefault, + } + + for _, res := range []resource.Resource{wg} { + suite.Require().NoError(suite.state.Create(suite.ctx, res), "%v", res.Spec()) + } + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertInterfaces( + []string{wgInterface}, func(r *network.LinkStatus) error { + suite.Assert().Equal("wireguard", r.TypedSpec().Kind) + + if r.TypedSpec().Wireguard.PublicKey != priv.PublicKey().String() { + return retry.ExpectedErrorf("private key not set") + } + + if len(r.TypedSpec().Wireguard.Peers) != 2 { + return retry.ExpectedErrorf("peers are not set up") + } + + if r.TypedSpec().OperationalState != nethelpers.OperStateUnknown && r.TypedSpec().OperationalState != nethelpers.OperStateUp { + return retry.ExpectedErrorf("link is not up") + } + + return nil + }, + ) + }, + ), + ) + + // attempt to change wireguard private key + priv2, err := wgtypes.GeneratePrivateKey() + suite.Require().NoError(err) + + ctest.UpdateWithConflicts(suite, wg, func(r *network.LinkSpec) error { + r.TypedSpec().Wireguard.PrivateKey = priv2.String() + + return nil + }) + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertInterfaces( + []string{wgInterface}, func(r *network.LinkStatus) error { + if r.TypedSpec().Wireguard.PublicKey != priv2.PublicKey().String() { + return retry.ExpectedErrorf("private key was not updated") + } + + return nil + }, + ) + }, + ), + ) + + // teardown the links + for _, r := range []resource.Resource{wg} { + for { + ready, err := suite.state.Teardown(suite.ctx, r.Metadata()) + suite.Require().NoError(err) + + if ready { + break + } + + time.Sleep(100 * time.Millisecond) + } + } + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertNoInterface(wgInterface) + }, + ), + ) +} + +func (suite *LinkSpecSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestLinkSpecSuite(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("requires root") + } + + suite.Run(t, new(LinkSpecSuite)) +} + +func TestSortBonds(t *testing.T) { + expectedSlice := []network.LinkSpecSpec{ + { + Name: "A", + }, { + Name: "G", + BondSlave: network.BondSlave{ + MasterName: "A", + SlaveIndex: 0, + }, + }, { + Name: "C", + }, { + Name: "E", + BondSlave: network.BondSlave{ + MasterName: "C", + SlaveIndex: 0, + }, + }, { + Name: "F", + BondSlave: network.BondSlave{ + MasterName: "C", + SlaveIndex: 1, + }, + }, { + Name: "B", + BondSlave: network.BondSlave{ + MasterName: "C", + SlaveIndex: 2, + }, + }, + } + + seed := time.Now().Unix() + rnd := rand.New(rand.NewSource(seed)) + + for i := range 100 { + res := toResources(expectedSlice) + rnd.Shuffle(len(res), func(i, j int) { res[i], res[j] = res[j], res[i] }) + netctrl.SortBonds(res) + sorted := toSpecs(res) + require.Equal(t, expectedSlice, sorted, "failed with seed %d iteration %d", seed, i) + } +} + +func toResources(slice []network.LinkSpecSpec) []resource.Resource { + return xslices.Map(slice, func(spec network.LinkSpecSpec) resource.Resource { + link := network.NewLinkSpec(network.NamespaceName, "bar") + *link.TypedSpec() = spec + + return link + }) +} + +func toSpecs(slice []resource.Resource) []network.LinkSpecSpec { + return xslices.Map(slice, func(r resource.Resource) network.LinkSpecSpec { + v := r.Spec().(*network.LinkSpecSpec) //nolint:errcheck + + return *v + }) +} diff --git a/internal/app/machined/pkg/controllers/network/link_status.go b/internal/app/machined/pkg/controllers/network/link_status.go new file mode 100644 index 0000000..e636841 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/link_status.go @@ -0,0 +1,343 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "context" + "errors" + "fmt" + "net" + "os" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/jsimonetti/rtnetlink" + "github.com/mdlayher/ethtool" + ethtoolioctl "github.com/safchain/ethtool" + "go.uber.org/zap" + "golang.org/x/sys/unix" + "golang.zx2c4.com/wireguard/wgctrl" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + networkadapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/network" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network/watch" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/runtime" + "github.com/aenix-io/talm/internal/pkg/pci" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// LinkStatusController manages secrets.Etcd based on configuration. +type LinkStatusController struct{} + +// Name implements controller.Controller interface. +func (ctrl *LinkStatusController) Name() string { + return "network.LinkStatusController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *LinkStatusController) Inputs() []controller.Input { + return nil +} + +// Outputs implements controller.Controller interface. +func (ctrl *LinkStatusController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.LinkStatusType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *LinkStatusController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + // wait for udevd to be healthy, which implies that all link renames are done + if err := runtime.WaitForDevicesReady(ctx, r, + []controller.Input{ + { + Namespace: network.NamespaceName, + Type: network.LinkSpecType, + Kind: controller.InputStrong, + }, + }, + ); err != nil { + return err + } + + // create watch connections to rtnetlink and ethtool via genetlink + // these connections are used only to join multicast groups and receive notifications on changes + // other connections are used to send requests and receive responses, as we can't mix the notifications and request/responses + rtnetlinkWatcher, err := watch.NewRtNetlink(watch.NewDefaultRateLimitedTrigger(ctx, r), unix.RTMGRP_LINK) + if err != nil { + return err + } + + defer rtnetlinkWatcher.Done() + + ethtoolWatcher, err := watch.NewEthtool(watch.NewDefaultRateLimitedTrigger(ctx, r)) + if err != nil { + logger.Warn("ethtool watcher failed to start", zap.Error(err)) + } else { + defer ethtoolWatcher.Done() + } + + conn, err := rtnetlink.Dial(nil) + if err != nil { + return fmt.Errorf("error dialing rtnetlink socket: %w", err) + } + + defer conn.Close() //nolint:errcheck + + ethClient, err := ethtool.New() + if err != nil { + logger.Warn("error dialing ethtool socket", zap.Error(err)) + } else { + defer ethClient.Close() //nolint:errcheck + } + + ethIoctlClient, err := ethtoolioctl.NewEthtool() + if err != nil { + logger.Warn("error dialing ethtool ioctl socket", zap.Error(err)) + } else { + defer ethIoctlClient.Close() + } + + wgClient, err := wgctrl.New() + if err != nil { + logger.Warn("error creating wireguard client", zap.Error(err)) + } else { + defer wgClient.Close() //nolint:errcheck + } + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + if err = ctrl.reconcile(ctx, r, logger, conn, ethClient, ethIoctlClient, wgClient); err != nil { + return err + } + + r.ResetRestartBackoff() + } +} + +// reconcile function runs for every reconciliation loop querying the netlink state and updating resources. +// +//nolint:gocyclo,cyclop +func (ctrl *LinkStatusController) reconcile( + ctx context.Context, + r controller.Runtime, + logger *zap.Logger, + conn *rtnetlink.Conn, + ethClient *ethtool.Client, + ethtoolIoctlClient *ethtoolioctl.Ethtool, + wgClient *wgctrl.Client, +) error { + // list the existing LinkStatus resources and mark them all to be deleted, as the actual link is discovered via netlink, resource ID is removed from the list + list, err := r.List(ctx, resource.NewMetadata(network.NamespaceName, network.LinkStatusType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing resources: %w", err) + } + + itemsToDelete := map[resource.ID]struct{}{} + + for _, r := range list.Items { + itemsToDelete[r.Metadata().ID()] = struct{}{} + } + + links, err := conn.Link.List() + if err != nil { + return fmt.Errorf("error listing links: %w", err) + } + + // for every rtnetlink discovered link + for _, link := range links { + var ( + ethState *ethtool.LinkState + ethInfo *ethtool.LinkInfo + ethMode *ethtool.LinkMode + driverInfo ethtoolioctl.DrvInfo + permanentAddr net.HardwareAddr + ) + + if ethClient != nil { + // query additional information via ethtool (if supported) + ethState, err = ethClient.LinkState(ethtool.Interface{ + Index: int(link.Index), + }) + if err != nil && !errors.Is(err, os.ErrNotExist) { + logger.Warn("error querying ethtool link state", zap.String("link", link.Attributes.Name), zap.Error(err)) + } + + // skip if previous call failed (e.g. not supported) + if err == nil { + ethInfo, err = ethClient.LinkInfo(ethtool.Interface{ + Index: int(link.Index), + }) + if err != nil && !errors.Is(err, os.ErrNotExist) { + logger.Warn("error querying ethtool link info", zap.String("link", link.Attributes.Name), zap.Error(err)) + } + } + + // skip if previous call failed (e.g. not supported) + if err == nil { + ethMode, err = ethClient.LinkMode(ethtool.Interface{ + Index: int(link.Index), + }) + if err != nil && !errors.Is(err, os.ErrNotExist) { + logger.Warn("error querying ethtool link mode", zap.String("link", link.Attributes.Name), zap.Error(err)) + } + } + } + + if ethtoolIoctlClient != nil { + driverInfo, _ = ethtoolIoctlClient.DriverInfo(link.Attributes.Name) //nolint:errcheck + + var permAddr string + + permAddr, err = ethtoolIoctlClient.PermAddr(link.Attributes.Name) + if err == nil && permAddr != "" { + permanentAddr, _ = net.ParseMAC(permAddr) //nolint:errcheck + } + } + + if err = r.Modify(ctx, network.NewLinkStatus(network.NamespaceName, link.Attributes.Name), func(r resource.Resource) error { + status := r.(*network.LinkStatus).TypedSpec() + + status.Index = link.Index + status.HardwareAddr = nethelpers.HardwareAddr(link.Attributes.Address) + status.PermanentAddr = nethelpers.HardwareAddr(permanentAddr) + status.BroadcastAddr = nethelpers.HardwareAddr(link.Attributes.Broadcast) + status.LinkIndex = link.Attributes.Type + status.Flags = nethelpers.LinkFlags(link.Flags) + status.Type = nethelpers.LinkType(link.Type) + status.QueueDisc = link.Attributes.QueueDisc + status.MTU = link.Attributes.MTU + if link.Attributes.Master != nil { + status.MasterIndex = *link.Attributes.Master + } else { + status.MasterIndex = 0 + } + status.OperationalState = nethelpers.OperationalState(link.Attributes.OperationalState) + if link.Attributes.Info != nil { + status.Kind = link.Attributes.Info.Kind + status.SlaveKind = link.Attributes.Info.SlaveKind + } else { + status.Kind = "" + status.SlaveKind = "" + } + + if ethState != nil { + status.LinkState = ethState.Link + } else { + status.LinkState = false + } + + if ethInfo != nil { + status.Port = nethelpers.Port(ethInfo.Port) + } else { + status.Port = nethelpers.Port(ethtool.Other) + } + + if ethMode != nil { + status.SpeedMegabits = ethMode.SpeedMegabits + status.Duplex = nethelpers.Duplex(ethMode.Duplex) + } else { + status.SpeedMegabits = 0 + status.Duplex = nethelpers.Duplex(ethtool.Unknown) + } + + var deviceInfo *nethelpers.DeviceInfo + + deviceInfo, err = nethelpers.GetDeviceInfo(link.Attributes.Name) + if err != nil { + logger.Warn("failure getting device information from /sys/class/net/*", zap.Error(err), zap.String("link", link.Attributes.Name)) + } + + if deviceInfo != nil { + status.BusPath = deviceInfo.BusPath + status.Driver = deviceInfo.Driver + status.PCIID = deviceInfo.PCIID + } + + if status.Driver == "" { + status.Driver = driverInfo.Driver + } + + if status.BusPath == "" { + status.BusPath = driverInfo.BusInfo + } + + var pciDev *pci.Device + + pciDev, err = pci.SysfsDeviceInfo(driverInfo.BusInfo) + if err != nil { + logger.Warn("failure looking up sysfs PCI info", zap.Error(err), zap.String("link", link.Attributes.Name)) + } + + if pciDev != nil { + pciDev.LookupDB() + + status.VendorID = fmt.Sprintf("0x%04x", pciDev.VendorID) + status.ProductID = fmt.Sprintf("0x%04x", pciDev.ProductID) + + status.Vendor = pciDev.Vendor + status.Product = pciDev.Product + } + + status.DriverVersion = driverInfo.Version + status.FirmwareVersion = driverInfo.FwVersion + + // link.Attributes.Info will be non-nil, because we set status.Kind above using link.Attributes.Info.Kind + switch status.Kind { + case network.LinkKindVLAN: + if err = networkadapter.VLANSpec(&status.VLAN).Decode(link.Attributes.Info.Data); err != nil { + logger.Warn("failure decoding VLAN attributes", zap.Error(err), zap.String("link", link.Attributes.Name)) + } + case network.LinkKindBond: + if err = networkadapter.BondMasterSpec(&status.BondMaster).Decode(link.Attributes.Info.Data); err != nil { + logger.Warn("failure decoding bond attributes", zap.Error(err), zap.String("link", link.Attributes.Name)) + } + case network.LinkKindBridge: + if err = networkadapter.BridgeMasterSpec(&status.BridgeMaster).Decode(link.Attributes.Info.Data); err != nil { + logger.Warn("failure decoding bridge attributes", zap.Error(err), zap.String("link", link.Attributes.Name)) + } + case network.LinkKindWireguard: + if wgClient == nil { + return fmt.Errorf("wireguard client not available, but wireguard interface was discovered: %q", link.Attributes.Name) + } + + var wgDev *wgtypes.Device + + wgDev, err = wgClient.Device(link.Attributes.Name) + if err != nil { + logger.Warn("failure getting wireguard attributes", zap.Error(err), zap.String("link", link.Attributes.Name)) + } else { + networkadapter.WireguardSpec(&status.Wireguard).Decode(wgDev, true) + } + } + + return nil + }); err != nil { + return fmt.Errorf("error modifying resource: %w", err) + } + + delete(itemsToDelete, link.Attributes.Name) + } + + for id := range itemsToDelete { + if err = r.Destroy(ctx, resource.NewMetadata(network.NamespaceName, network.LinkStatusType, id, resource.VersionUndefined)); err != nil { + return fmt.Errorf("error deleting link status %q: %w", id, err) + } + } + + return nil +} diff --git a/internal/app/machined/pkg/controllers/network/link_status_test.go b/internal/app/machined/pkg/controllers/network/link_status_test.go new file mode 100644 index 0000000..a12115a --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/link_status_test.go @@ -0,0 +1,372 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:dupl +package network_test + +import ( + "context" + "errors" + "fmt" + "log" + "math/rand" + "net" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/jsimonetti/rtnetlink" + "github.com/mdlayher/netlink" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + "golang.org/x/sys/unix" + + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +type LinkStatusSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *LinkStatusSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + // create fake device ready status + deviceStatus := runtimeres.NewDevicesStatus(runtimeres.NamespaceName, runtimeres.DevicesID) + deviceStatus.TypedSpec().Ready = true + suite.Require().NoError(suite.state.Create(suite.ctx, deviceStatus)) + + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.LinkStatusController{})) + + suite.startRuntime() +} + +func (suite *LinkStatusSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *LinkStatusSuite) uniqueDummyInterface() string { + return fmt.Sprintf("dummy%02x%02x%02x", rand.Int31()&0xff, rand.Int31()&0xff, rand.Int31()&0xff) +} + +func (suite *LinkStatusSuite) assertInterfaces(requiredIDs []string, check func(*network.LinkStatus) error) error { + missingIDs := make(map[string]struct{}, len(requiredIDs)) + + for _, id := range requiredIDs { + missingIDs[id] = struct{}{} + } + + resources, err := suite.state.List( + suite.ctx, + resource.NewMetadata(network.NamespaceName, network.LinkStatusType, "", resource.VersionUndefined), + ) + if err != nil { + return err + } + + for _, res := range resources.Items { + _, required := missingIDs[res.Metadata().ID()] + if !required { + continue + } + + delete(missingIDs, res.Metadata().ID()) + + if err = check(res.(*network.LinkStatus)); err != nil { + return retry.ExpectedError(err) + } + } + + if len(missingIDs) > 0 { + return retry.ExpectedErrorf("some resources are missing: %q", missingIDs) + } + + return nil +} + +func (suite *LinkStatusSuite) assertNoInterface(id string) error { + resources, err := suite.state.List( + suite.ctx, + resource.NewMetadata(network.NamespaceName, network.LinkStatusType, "", resource.VersionUndefined), + ) + if err != nil { + return err + } + + for _, res := range resources.Items { + if res.Metadata().ID() == id { + return retry.ExpectedErrorf("interface %q is still there", id) + } + } + + return nil +} + +func (suite *LinkStatusSuite) TestInterfaceHwInfo() { + errNoInterfaces := errors.New("no suitable interfaces found") + + err := retry.Constant(5*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + resources, err := suite.state.List( + suite.ctx, + resource.NewMetadata(network.NamespaceName, network.LinkStatusType, "", resource.VersionUndefined), + ) + suite.Require().NoError(err) + + for _, res := range resources.Items { + spec := res.(*network.LinkStatus).TypedSpec() //nolint:forcetypeassert + + if !spec.Physical() { + continue + } + + if spec.Type != nethelpers.LinkEther { + continue + } + + emptyFields := []string{} + + for key, value := range map[string]string{ + "hw addr": spec.HardwareAddr.String(), + "perm addr": spec.PermanentAddr.String(), + "driver": spec.Driver, + "bus path": spec.BusPath, + "PCI id": spec.PCIID, + } { + if value == "" { + emptyFields = append(emptyFields, key) + } + } + + if len(emptyFields) > 0 { + return fmt.Errorf("the interface %s has the following fields empty: %s", res.Metadata().ID(), strings.Join(emptyFields, ", ")) + } + + return nil + } + + return retry.ExpectedError(errNoInterfaces) + }, + ) + + if errors.Is(err, errNoInterfaces) { + suite.T().Skip(err.Error()) + } + + suite.Require().NoError(err) +} + +func (suite *LinkStatusSuite) TestLoopbackInterface() { + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertInterfaces( + []string{"lo"}, func(r *network.LinkStatus) error { + suite.Assert().Equal("loopback", r.TypedSpec().Type.String()) + suite.Assert().EqualValues(65536, r.TypedSpec().MTU) + + return nil + }, + ) + }, + ), + ) +} + +func (suite *LinkStatusSuite) TestDummyInterface() { + if os.Geteuid() != 0 { + suite.T().Skip("requires root") + } + + dummyInterface := suite.uniqueDummyInterface() + + conn, err := rtnetlink.Dial(nil) + suite.Require().NoError(err) + + defer conn.Close() //nolint:errcheck + + suite.Require().NoError( + conn.Link.New( + &rtnetlink.LinkMessage{ + Type: unix.ARPHRD_ETHER, + Attributes: &rtnetlink.LinkAttributes{ + Name: dummyInterface, + MTU: 1400, + Info: &rtnetlink.LinkInfo{ + Kind: "dummy", + }, + }, + }, + ), + ) + + iface, err := net.InterfaceByName(dummyInterface) + suite.Require().NoError(err) + + defer conn.Link.Delete(uint32(iface.Index)) //nolint:errcheck + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertInterfaces( + []string{dummyInterface}, func(r *network.LinkStatus) error { + suite.Assert().Equal("ether", r.TypedSpec().Type.String()) + suite.Assert().EqualValues(1400, r.TypedSpec().MTU) + suite.Assert().Equal(nethelpers.OperStateDown, r.TypedSpec().OperationalState) + + return nil + }, + ) + }, + ), + ) + + suite.Require().NoError( + conn.Link.Set( + &rtnetlink.LinkMessage{ + Type: unix.ARPHRD_ETHER, + Index: uint32(iface.Index), + Flags: unix.IFF_UP, + Change: unix.IFF_UP, + }, + ), + ) + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertInterfaces( + []string{dummyInterface}, func(r *network.LinkStatus) error { + if r.TypedSpec().OperationalState != nethelpers.OperStateUp && r.TypedSpec().OperationalState != nethelpers.OperStateUnknown { + return retry.ExpectedErrorf( + "operational state is not up: %s", + r.TypedSpec().OperationalState, + ) + } + + return nil + }, + ) + }, + ), + ) + + suite.Require().NoError(conn.Link.Delete(uint32(iface.Index))) + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertNoInterface(dummyInterface) + }, + ), + ) +} + +func (suite *LinkStatusSuite) TestBridgeInterface() { + if os.Geteuid() != 0 { + suite.T().Skip("requires root") + } + + bridgeInterface := suite.uniqueDummyInterface() + + conn, err := rtnetlink.Dial(nil) + suite.Require().NoError(err) + + defer conn.Close() //nolint:errcheck + + bridgeData, err := encodeBridgeData(true) + suite.Require().NoError(err) + + suite.Require().NoError( + conn.Link.New( + &rtnetlink.LinkMessage{ + Type: unix.ARPHRD_ETHER, + Attributes: &rtnetlink.LinkAttributes{ + Name: bridgeInterface, + Info: &rtnetlink.LinkInfo{ + Kind: "bridge", + Data: bridgeData, + }, + }, + }, + ), + ) + + bridgeIface, err := net.InterfaceByName(bridgeInterface) + suite.Require().NoError(err) + + defer conn.Link.Delete(uint32(bridgeIface.Index)) //nolint:errcheck + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertInterfaces( + []string{bridgeInterface}, func(r *network.LinkStatus) error { + suite.Assert().Equal("ether", r.TypedSpec().Type.String()) + suite.Assert().True(r.TypedSpec().BridgeMaster.STP.Enabled) + + return nil + }, + ) + }, + ), + ) +} + +func encodeBridgeData(stpEnabled bool) ([]byte, error) { + encoder := netlink.NewAttributeEncoder() + + var stpState uint32 + if stpEnabled { + stpState = 1 + } + + encoder.Uint32(unix.IFLA_BR_STP_STATE, stpState) + + return encoder.Encode() +} + +func (suite *LinkStatusSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestLinkStatusSuite(t *testing.T) { + suite.Run(t, new(LinkStatusSuite)) +} diff --git a/internal/app/machined/pkg/controllers/network/network.go b/internal/app/machined/pkg/controllers/network/network.go new file mode 100644 index 0000000..4a65ee0 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/network.go @@ -0,0 +1,138 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package network provides controllers which manage network resources. +package network + +import ( + "net" + + "github.com/siderolabs/gen/pair/ordered" + + networkadapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/network" + talosconfig "github.com/siderolabs/talos/pkg/machinery/config/config" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// SetBondSlave sets the bond slave spec. +func SetBondSlave(link *network.LinkSpecSpec, bond ordered.Pair[string, int]) { + link.BondSlave = network.BondSlave{ + MasterName: bond.F1, + SlaveIndex: bond.F2, + } +} + +// SetBondMaster sets the bond master spec. +// +//nolint:gocyclo +func SetBondMaster(link *network.LinkSpecSpec, bond talosconfig.Bond) error { + link.Logical = true + link.Kind = network.LinkKindBond + link.Type = nethelpers.LinkEther + + bondMode, err := nethelpers.BondModeByName(bond.Mode()) + if err != nil { + return err + } + + hashPolicy, err := nethelpers.BondXmitHashPolicyByName(bond.HashPolicy()) + if err != nil { + return err + } + + lacpRate, err := nethelpers.LACPRateByName(bond.LACPRate()) + if err != nil { + return err + } + + arpValidate, err := nethelpers.ARPValidateByName(bond.ARPValidate()) + if err != nil { + return err + } + + arpAllTargets, err := nethelpers.ARPAllTargetsByName(bond.ARPAllTargets()) + if err != nil { + return err + } + + var primary uint32 + + if bond.Primary() != "" { + var iface *net.Interface + + iface, err = net.InterfaceByName(bond.Primary()) + if err != nil { + return err + } + + primary = uint32(iface.Index) + } + + primaryReselect, err := nethelpers.PrimaryReselectByName(bond.PrimaryReselect()) + if err != nil { + return err + } + + failOverMAC, err := nethelpers.FailOverMACByName(bond.FailOverMac()) + if err != nil { + return err + } + + adSelect, err := nethelpers.ADSelectByName(bond.ADSelect()) + if err != nil { + return err + } + + link.BondMaster = network.BondMasterSpec{ + Mode: bondMode, + HashPolicy: hashPolicy, + LACPRate: lacpRate, + ARPValidate: arpValidate, + ARPAllTargets: arpAllTargets, + PrimaryIndex: primary, + PrimaryReselect: primaryReselect, + FailOverMac: failOverMAC, + ADSelect: adSelect, + MIIMon: bond.MIIMon(), + UpDelay: bond.UpDelay(), + DownDelay: bond.DownDelay(), + ARPInterval: bond.ARPInterval(), + ResendIGMP: bond.ResendIGMP(), + MinLinks: bond.MinLinks(), + LPInterval: bond.LPInterval(), + PacketsPerSlave: bond.PacketsPerSlave(), + NumPeerNotif: bond.NumPeerNotif(), + TLBDynamicLB: bond.TLBDynamicLB(), + AllSlavesActive: bond.AllSlavesActive(), + UseCarrier: bond.UseCarrier(), + ADActorSysPrio: bond.ADActorSysPrio(), + ADUserPortKey: bond.ADUserPortKey(), + PeerNotifyDelay: bond.PeerNotifyDelay(), + } + networkadapter.BondMasterSpec(&link.BondMaster).FillDefaults() + + return nil +} + +// SetBridgeSlave sets the bridge slave spec. +func SetBridgeSlave(link *network.LinkSpecSpec, bridge string) { + link.BridgeSlave = network.BridgeSlave{ + MasterName: bridge, + } +} + +// SetBridgeMaster sets the bridge master spec. +func SetBridgeMaster(link *network.LinkSpecSpec, bridge talosconfig.Bridge) error { + link.Logical = true + link.Kind = network.LinkKindBridge + link.Type = nethelpers.LinkEther + link.BridgeMaster = network.BridgeMasterSpec{ + STP: network.STPSpec{ + Enabled: bridge.STP().Enabled(), + }, + } + + return nil +} diff --git a/internal/app/machined/pkg/controllers/network/nftables_chain.go b/internal/app/machined/pkg/controllers/network/nftables_chain.go new file mode 100644 index 0000000..2d5510c --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/nftables_chain.go @@ -0,0 +1,213 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "context" + "fmt" + "slices" + "strconv" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/google/nftables" + "github.com/google/nftables/expr" + "github.com/siderolabs/go-pointer" + "go.uber.org/zap" + + networkadapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/network" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// NfTablesChainController applies network.NfTablesChain to the Linux nftables interface. +type NfTablesChainController struct { + TableName string +} + +// Name implements controller.Controller interface. +func (ctrl *NfTablesChainController) Name() string { + return "network.NfTablesChainController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *NfTablesChainController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: network.NamespaceName, + Type: network.NfTablesChainType, + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *NfTablesChainController) Outputs() []controller.Output { + return nil +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *NfTablesChainController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + if ctrl.TableName == "" { + ctrl.TableName = constants.DefaultNfTablesTableName + } + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + var conn nftables.Conn + + if err := ctrl.preCreateIptablesNFTable(logger, &conn); err != nil { + return fmt.Errorf("error pre-creating iptables-nft table: %w", err) + } + + list, err := safe.ReaderListAll[*network.NfTablesChain](ctx, r) + if err != nil { + return fmt.Errorf("error listing nftables chains: %w", err) + } + + existingTables, err := conn.ListTablesOfFamily(nftables.TableFamilyINet) + if err != nil { + return fmt.Errorf("error listing existing nftables tables: %w", err) + } + + var talosTable *nftables.Table + + if idx := slices.IndexFunc(existingTables, func(t *nftables.Table) bool { return t.Name == ctrl.TableName }); idx != -1 { + talosTable = existingTables[idx] + } + + if talosTable == nil { + talosTable = &nftables.Table{ + Family: nftables.TableFamilyINet, + Name: ctrl.TableName, + } + + conn.AddTable(talosTable) + } + + // drop all chains, they will be re-created + existingChains, err := conn.ListChains() + if err != nil { + return fmt.Errorf("error listing existing nftables chains: %w", err) + } + + for _, chain := range existingChains { + if chain.Table.Name != ctrl.TableName { // not our chain + continue + } + + conn.DelChain(chain) + } + + setID := uint32(0) + + for iter := list.Iterator(); iter.Next(); { + chain := iter.Value() + + nfChain := conn.AddChain(&nftables.Chain{ + Name: chain.Metadata().ID(), + Table: talosTable, + Hooknum: pointer.To(nftables.ChainHook(chain.TypedSpec().Hook)), + Priority: pointer.To(nftables.ChainPriority(chain.TypedSpec().Priority)), + Type: nftables.ChainType(chain.TypedSpec().Type), + Policy: pointer.To(nftables.ChainPolicy(chain.TypedSpec().Policy)), + }) + + for _, rule := range chain.TypedSpec().Rules { + compiled, err := networkadapter.NfTablesRule(&rule).Compile() + if err != nil { + return fmt.Errorf("error compiling nftables rule for chain %s: %w", nfChain.Name, err) + } + + for _, compiledRule := range compiled.Rules { + // check for lookup rules and add/fix up the set ID if needed + for i := range compiledRule { + if lookup, ok := compiledRule[i].(*expr.Lookup); ok { + if lookup.SetID >= uint32(len(compiled.Sets)) { + return fmt.Errorf("invalid set ID %d in lookup", lookup.SetID) + } + + set := compiled.Sets[lookup.SetID] + setName := "__set" + strconv.Itoa(int(setID)) + + if err = conn.AddSet(&nftables.Set{ + Table: talosTable, + ID: setID, + Name: setName, + Anonymous: true, + Constant: true, + Interval: set.IsInterval(), + KeyType: set.KeyType(), + }, set.SetElements()); err != nil { + return fmt.Errorf("error adding nftables set for chain %s: %w", nfChain.Name, err) + } + + lookupOp := *lookup + lookupOp.SetID = setID + lookupOp.SetName = setName + + compiledRule[i] = &lookupOp + + setID++ + } + } + + conn.AddRule(&nftables.Rule{ + Table: talosTable, + Chain: nfChain, + Exprs: compiledRule, + }) + } + } + } + + if err := conn.Flush(); err != nil { + return fmt.Errorf("error flushing nftables: %w", err) + } + + chainNames, _ := safe.Map(list, func(chain *network.NfTablesChain) (string, error) { return chain.Metadata().ID(), nil }) //nolint:errcheck // doesn't fail + logger.Info("nftables chains updated", zap.Strings("chains", chainNames)) + + r.ResetRestartBackoff() + } +} + +func (ctrl *NfTablesChainController) preCreateIptablesNFTable(logger *zap.Logger, conn *nftables.Conn) error { + // Pre-create the iptables-nft table, if it doesn't exist. + // This is required to ensure that the iptables universal binary prefers iptables-nft over + // iptables-legacy can be used to manage the nftables rules. + tables, err := conn.ListTablesOfFamily(nftables.TableFamilyIPv4) + if err != nil { + return fmt.Errorf("error listing existing nftables tables: %w", err) + } + + if slices.IndexFunc(tables, func(t *nftables.Table) bool { return t.Name == "mangle" }) != -1 { + return nil + } + + table := &nftables.Table{ + Family: nftables.TableFamilyIPv4, + Name: "mangle", + } + conn.AddTable(table) + + chain := &nftables.Chain{ + Name: "KUBE-IPTABLES-HINT", + Table: table, + Type: nftables.ChainTypeNAT, + } + conn.AddChain(chain) + + logger.Info("pre-created iptables-nft table 'mangle'/'KUBE-IPTABLES-HINT'") + + return nil +} diff --git a/internal/app/machined/pkg/controllers/network/nftables_chain_config.go b/internal/app/machined/pkg/controllers/network/nftables_chain_config.go new file mode 100644 index 0000000..4075db3 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/nftables_chain_config.go @@ -0,0 +1,224 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "cmp" + "context" + "fmt" + "net/netip" + "slices" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/go-pointer" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// IngressChainName is the name of the ingress chain. +const IngressChainName = "ingress" + +// NfTablesChainConfigController generates nftables rules based on machine configuration. +type NfTablesChainConfigController struct{} + +// Name implements controller.Controller interface. +func (ctrl *NfTablesChainConfigController) Name() string { + return "network.NfTablesChainConfigController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *NfTablesChainConfigController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: config.MachineConfigType, + ID: optional.Some(config.V1Alpha1ID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *NfTablesChainConfigController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.NfTablesChainType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *NfTablesChainConfigController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) (err error) { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + cfg, err := safe.ReaderGetByID[*config.MachineConfig](ctx, r, config.V1Alpha1ID) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error getting machine config: %w", err) + } + + r.StartTrackingOutputs() + + if cfg != nil && !(cfg.Config().NetworkRules().DefaultAction() == nethelpers.DefaultActionAccept && cfg.Config().NetworkRules().Rules() == nil) { + if err = safe.WriterModify(ctx, r, network.NewNfTablesChain(network.NamespaceName, IngressChainName), + func(chain *network.NfTablesChain) error { + spec := chain.TypedSpec() + + spec.Type = nethelpers.ChainTypeFilter + spec.Hook = nethelpers.ChainHookInput + spec.Priority = nethelpers.ChainPriorityFilter + spec.Policy = nethelpers.VerdictAccept + + // preamble + spec.Rules = []network.NfTablesRule{ + // trusted interfaces: loopback, siderolink and kubespan + { + MatchIIfName: &network.NfTablesIfNameMatch{ + InterfaceNames: []string{ + "lo", + constants.SideroLinkName, + constants.KubeSpanLinkName, + }, + Operator: nethelpers.OperatorEqual, + }, + AnonCounter: true, + Verdict: pointer.To(nethelpers.VerdictAccept), + }, + } + + defaultAction := cfg.Config().NetworkRules().DefaultAction() + + if defaultAction == nethelpers.DefaultActionBlock { + spec.Policy = nethelpers.VerdictDrop + + spec.Rules = append(spec.Rules, + // conntrack + network.NfTablesRule{ + MatchConntrackState: &network.NfTablesConntrackStateMatch{ + States: []nethelpers.ConntrackState{ + nethelpers.ConntrackStateEstablished, + nethelpers.ConntrackStateRelated, + }, + }, + AnonCounter: true, + Verdict: pointer.To(nethelpers.VerdictAccept), + }, + network.NfTablesRule{ + MatchConntrackState: &network.NfTablesConntrackStateMatch{ + States: []nethelpers.ConntrackState{ + nethelpers.ConntrackStateInvalid, + }, + }, + AnonCounter: true, + Verdict: pointer.To(nethelpers.VerdictDrop), + }, + // allow ICMP and ICMPv6 explicitly + network.NfTablesRule{ + MatchLayer4: &network.NfTablesLayer4Match{ + Protocol: nethelpers.ProtocolICMP, + }, + MatchLimit: &network.NfTablesLimitMatch{ + PacketRatePerSecond: 5, + }, + AnonCounter: true, + Verdict: pointer.To(nethelpers.VerdictAccept), + }, + network.NfTablesRule{ + MatchLayer4: &network.NfTablesLayer4Match{ + Protocol: nethelpers.ProtocolICMPv6, + }, + MatchLimit: &network.NfTablesLimitMatch{ + PacketRatePerSecond: 5, + }, + AnonCounter: true, + Verdict: pointer.To(nethelpers.VerdictAccept), + }, + ) + + if cfg.Config().Cluster() != nil { + spec.Rules = append(spec.Rules, + // allow Kubernetes pod/service traffic + network.NfTablesRule{ + MatchSourceAddress: &network.NfTablesAddressMatch{ + IncludeSubnets: xslices.Map( + append(slices.Clone(cfg.Config().Cluster().Network().PodCIDRs()), cfg.Config().Cluster().Network().ServiceCIDRs()...), + netip.MustParsePrefix, + ), + }, + MatchDestinationAddress: &network.NfTablesAddressMatch{ + IncludeSubnets: xslices.Map( + append(slices.Clone(cfg.Config().Cluster().Network().PodCIDRs()), cfg.Config().Cluster().Network().ServiceCIDRs()...), + netip.MustParsePrefix, + ), + }, + AnonCounter: true, + Verdict: pointer.To(nethelpers.VerdictAccept), + }, + ) + } + } + + for _, rule := range cfg.Config().NetworkRules().Rules() { + portRanges := rule.PortRanges() + + // sort port ranges, machine config validation ensures that there are no overlaps + slices.SortFunc(portRanges, func(a, b [2]uint16) int { + return cmp.Compare(a[0], b[0]) + }) + + // if default accept, drop anything that doesn't match the rule + verdict := nethelpers.VerdictDrop + + if defaultAction == nethelpers.DefaultActionBlock { + verdict = nethelpers.VerdictAccept + } + + spec.Rules = append(spec.Rules, + network.NfTablesRule{ + MatchSourceAddress: &network.NfTablesAddressMatch{ + IncludeSubnets: rule.Subnets(), + ExcludeSubnets: rule.ExceptSubnets(), + Invert: defaultAction == nethelpers.DefaultActionAccept, + }, + MatchLayer4: &network.NfTablesLayer4Match{ + Protocol: rule.Protocol(), + MatchDestinationPort: &network.NfTablesPortMatch{ + Ranges: xslices.Map(portRanges, func(pr [2]uint16) network.PortRange { + return network.PortRange{Lo: pr[0], Hi: pr[1]} + }), + }, + }, + AnonCounter: true, + Verdict: pointer.To(verdict), + }, + ) + } + + return nil + }); err != nil { + return err + } + } + + if err = safe.CleanupOutputs[*network.NfTablesChain](ctx, r); err != nil { + return err + } + } +} diff --git a/internal/app/machined/pkg/controllers/network/nftables_chain_config_test.go b/internal/app/machined/pkg/controllers/network/nftables_chain_config_test.go new file mode 100644 index 0000000..729d02e --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/nftables_chain_config_test.go @@ -0,0 +1,283 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network_test + +import ( + "net/netip" + "testing" + "time" + + "github.com/siderolabs/go-pointer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + configtypes "github.com/siderolabs/talos/pkg/machinery/config/config" + "github.com/siderolabs/talos/pkg/machinery/config/container" + networkcfg "github.com/siderolabs/talos/pkg/machinery/config/types/network" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type NfTablesChainConfigTestSuite struct { + ctest.DefaultSuite +} + +func (suite *NfTablesChainConfigTestSuite) injectConfig(block bool) { + kubeletIngressCfg := networkcfg.NewRuleConfigV1Alpha1() + kubeletIngressCfg.MetaName = "kubelet-ingress" + kubeletIngressCfg.PortSelector.Ports = []networkcfg.PortRange{ + { + Lo: 10250, + Hi: 10250, + }, + } + kubeletIngressCfg.PortSelector.Protocol = nethelpers.ProtocolTCP + kubeletIngressCfg.Ingress = []networkcfg.IngressRule{ + { + Subnet: netip.MustParsePrefix("10.0.0.0/8"), + Except: networkcfg.Prefix{Prefix: netip.MustParsePrefix("10.3.0.0/16")}, + }, + { + Subnet: netip.MustParsePrefix("192.168.0.0/16"), + }, + } + + apidIngressCfg := networkcfg.NewRuleConfigV1Alpha1() + apidIngressCfg.MetaName = "apid-ingress" + apidIngressCfg.PortSelector.Ports = []networkcfg.PortRange{ + { + Lo: 50000, + Hi: 50000, + }, + } + apidIngressCfg.PortSelector.Protocol = nethelpers.ProtocolTCP + apidIngressCfg.Ingress = []networkcfg.IngressRule{ + { + Subnet: netip.MustParsePrefix("0.0.0.0/0"), + }, + } + + configs := []configtypes.Document{kubeletIngressCfg, apidIngressCfg} + + if block { + defaultActionCfg := networkcfg.NewDefaultActionConfigV1Alpha1() + defaultActionCfg.Ingress = nethelpers.DefaultActionBlock + + configs = append(configs, defaultActionCfg) + } + + cfg, err := container.New(configs...) + suite.Require().NoError(err) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), config.NewMachineConfig(cfg))) +} + +func (suite *NfTablesChainConfigTestSuite) TestDefaultAccept() { + ctest.AssertNoResource[*network.NfTablesChain](suite, netctrl.IngressChainName) + + suite.injectConfig(false) + + ctest.AssertResource(suite, netctrl.IngressChainName, func(chain *network.NfTablesChain, asrt *assert.Assertions) { + spec := chain.TypedSpec() + + asrt.Equal(nethelpers.ChainTypeFilter, spec.Type) + asrt.Equal(nethelpers.ChainPriorityFilter, spec.Priority) + asrt.Equal(nethelpers.ChainHookInput, spec.Hook) + asrt.Equal(nethelpers.VerdictAccept, spec.Policy) + + asrt.Equal( + []network.NfTablesRule{ + { + MatchIIfName: &network.NfTablesIfNameMatch{ + InterfaceNames: []string{ + "lo", + constants.SideroLinkName, + constants.KubeSpanLinkName, + }, + Operator: nethelpers.OperatorEqual, + }, + AnonCounter: true, + Verdict: pointer.To(nethelpers.VerdictAccept), + }, + { + MatchSourceAddress: &network.NfTablesAddressMatch{ + IncludeSubnets: []netip.Prefix{ + netip.MustParsePrefix("10.0.0.0/8"), + netip.MustParsePrefix("192.168.0.0/16"), + }, + ExcludeSubnets: []netip.Prefix{ + netip.MustParsePrefix("10.3.0.0/16"), + }, + Invert: true, + }, + MatchLayer4: &network.NfTablesLayer4Match{ + Protocol: nethelpers.ProtocolTCP, + MatchDestinationPort: &network.NfTablesPortMatch{ + Ranges: []network.PortRange{ + { + Lo: 10250, + Hi: 10250, + }, + }, + }, + }, + AnonCounter: true, + Verdict: pointer.To(nethelpers.VerdictDrop), + }, + { + MatchSourceAddress: &network.NfTablesAddressMatch{ + IncludeSubnets: []netip.Prefix{ + netip.MustParsePrefix("0.0.0.0/0"), + }, + Invert: true, + }, + MatchLayer4: &network.NfTablesLayer4Match{ + Protocol: nethelpers.ProtocolTCP, + MatchDestinationPort: &network.NfTablesPortMatch{ + Ranges: []network.PortRange{ + { + Lo: 50000, + Hi: 50000, + }, + }, + }, + }, + AnonCounter: true, + Verdict: pointer.To(nethelpers.VerdictDrop), + }, + }, + spec.Rules) + }) +} + +func (suite *NfTablesChainConfigTestSuite) TestDefaultBlock() { + ctest.AssertNoResource[*network.NfTablesChain](suite, netctrl.IngressChainName) + + suite.injectConfig(true) + + ctest.AssertResource(suite, netctrl.IngressChainName, func(chain *network.NfTablesChain, asrt *assert.Assertions) { + spec := chain.TypedSpec() + + asrt.Equal(nethelpers.ChainTypeFilter, spec.Type) + asrt.Equal(nethelpers.ChainPriorityFilter, spec.Priority) + asrt.Equal(nethelpers.ChainHookInput, spec.Hook) + asrt.Equal(nethelpers.VerdictDrop, spec.Policy) + + asrt.Equal( + []network.NfTablesRule{ + { + MatchIIfName: &network.NfTablesIfNameMatch{ + InterfaceNames: []string{ + "lo", + constants.SideroLinkName, + constants.KubeSpanLinkName, + }, + Operator: nethelpers.OperatorEqual, + }, + AnonCounter: true, + Verdict: pointer.To(nethelpers.VerdictAccept), + }, + { + MatchConntrackState: &network.NfTablesConntrackStateMatch{ + States: []nethelpers.ConntrackState{ + nethelpers.ConntrackStateEstablished, + nethelpers.ConntrackStateRelated, + }, + }, + AnonCounter: true, + Verdict: pointer.To(nethelpers.VerdictAccept), + }, + { + MatchConntrackState: &network.NfTablesConntrackStateMatch{ + States: []nethelpers.ConntrackState{ + nethelpers.ConntrackStateInvalid, + }, + }, + AnonCounter: true, + Verdict: pointer.To(nethelpers.VerdictDrop), + }, + { + MatchLayer4: &network.NfTablesLayer4Match{ + Protocol: nethelpers.ProtocolICMP, + }, + MatchLimit: &network.NfTablesLimitMatch{ + PacketRatePerSecond: 5, + }, + AnonCounter: true, + Verdict: pointer.To(nethelpers.VerdictAccept), + }, + { + MatchLayer4: &network.NfTablesLayer4Match{ + Protocol: nethelpers.ProtocolICMPv6, + }, + MatchLimit: &network.NfTablesLimitMatch{ + PacketRatePerSecond: 5, + }, + AnonCounter: true, + Verdict: pointer.To(nethelpers.VerdictAccept), + }, + { + MatchSourceAddress: &network.NfTablesAddressMatch{ + IncludeSubnets: []netip.Prefix{ + netip.MustParsePrefix("10.0.0.0/8"), + netip.MustParsePrefix("192.168.0.0/16"), + }, + ExcludeSubnets: []netip.Prefix{ + netip.MustParsePrefix("10.3.0.0/16"), + }, + }, + MatchLayer4: &network.NfTablesLayer4Match{ + Protocol: nethelpers.ProtocolTCP, + MatchDestinationPort: &network.NfTablesPortMatch{ + Ranges: []network.PortRange{ + { + Lo: 10250, + Hi: 10250, + }, + }, + }, + }, + AnonCounter: true, + Verdict: pointer.To(nethelpers.VerdictAccept), + }, + { + MatchSourceAddress: &network.NfTablesAddressMatch{ + IncludeSubnets: []netip.Prefix{ + netip.MustParsePrefix("0.0.0.0/0"), + }, + }, + MatchLayer4: &network.NfTablesLayer4Match{ + Protocol: nethelpers.ProtocolTCP, + MatchDestinationPort: &network.NfTablesPortMatch{ + Ranges: []network.PortRange{ + { + Lo: 50000, + Hi: 50000, + }, + }, + }, + }, + AnonCounter: true, + Verdict: pointer.To(nethelpers.VerdictAccept), + }, + }, + spec.Rules) + }) +} + +func TestNfTablesChainConfig(t *testing.T) { + suite.Run(t, &NfTablesChainConfigTestSuite{ + DefaultSuite: ctest.DefaultSuite{ + Timeout: 5 * time.Second, + AfterSetup: func(s *ctest.DefaultSuite) { + s.Require().NoError(s.Runtime().RegisterController(&netctrl.NfTablesChainConfigController{})) + }, + }, + }) +} diff --git a/internal/app/machined/pkg/controllers/network/nftables_chain_test.go b/internal/app/machined/pkg/controllers/network/nftables_chain_test.go new file mode 100644 index 0000000..a22cb7e --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/nftables_chain_test.go @@ -0,0 +1,514 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network_test + +import ( + "net/netip" + "os" + "os/exec" + "strings" + "testing" + "time" + + "github.com/siderolabs/go-pointer" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type NfTablesChainSuite struct { + ctest.DefaultSuite +} + +func (s *NfTablesChainSuite) nftOutput() string { + out, err := exec.Command("nft", "list", "table", "inet", "talos-test").CombinedOutput() + s.Require().NoError(err, "nft list table inet talos-test failed: %s", string(out)) + + return string(out) +} + +func (s *NfTablesChainSuite) checkNftOutput(expected string) { + s.T().Helper() + + var prevOutput string + + s.Eventually(func() bool { + output := s.nftOutput() + + if output != prevOutput { + if strings.TrimSpace(output) != expected { + s.T().Logf("nft list table inet talos-test:\n%s", output) + } + + prevOutput = output + } + + return strings.TrimSpace(output) == expected + }, 5*time.Second, 100*time.Millisecond) +} + +func (s *NfTablesChainSuite) TestEmpty() { + s.checkNftOutput(`table inet talos-test { +}`) +} + +func (s *NfTablesChainSuite) TestAcceptLo() { + chain := network.NewNfTablesChain(network.NamespaceName, "test1") + chain.TypedSpec().Type = nethelpers.ChainTypeFilter + chain.TypedSpec().Hook = nethelpers.ChainHookInput + chain.TypedSpec().Priority = nethelpers.ChainPrioritySecurity + chain.TypedSpec().Policy = nethelpers.VerdictAccept + chain.TypedSpec().Rules = []network.NfTablesRule{ + { + MatchOIfName: &network.NfTablesIfNameMatch{ + InterfaceNames: []string{"lo"}, + }, + Verdict: pointer.To(nethelpers.VerdictAccept), + }, + } + + s.Require().NoError(s.State().Create(s.Ctx(), chain)) + + s.checkNftOutput(`table inet talos-test { + chain test1 { + type filter hook input priority security; policy accept; + oifname "lo" accept + } +}`) +} + +func (s *NfTablesChainSuite) TestAcceptMultipleIfnames() { + chain := network.NewNfTablesChain(network.NamespaceName, "test1") + chain.TypedSpec().Type = nethelpers.ChainTypeFilter + chain.TypedSpec().Hook = nethelpers.ChainHookInput + chain.TypedSpec().Priority = nethelpers.ChainPrioritySecurity + chain.TypedSpec().Policy = nethelpers.VerdictAccept + chain.TypedSpec().Rules = []network.NfTablesRule{ + { + MatchIIfName: &network.NfTablesIfNameMatch{ + InterfaceNames: []string{"eth0", "eth1"}, + }, + Verdict: pointer.To(nethelpers.VerdictAccept), + }, + } + + s.Require().NoError(s.State().Create(s.Ctx(), chain)) + + // this seems to be a bug in the nft cli, it doesn't decoded the ifname anonymous set correctly + // it might be that google/nftables doesn't set some magic on the anonymous set for the nft CLI to pick it up (?) + s.checkNftOutput(`table inet talos-test { + chain test1 { + type filter hook input priority security; policy accept; + iifname { "", "" } accept + } +}`) +} + +func (s *NfTablesChainSuite) TestPolicyDrop() { + chain := network.NewNfTablesChain(network.NamespaceName, "test1") + chain.TypedSpec().Type = nethelpers.ChainTypeFilter + chain.TypedSpec().Hook = nethelpers.ChainHookInput + chain.TypedSpec().Priority = nethelpers.ChainPrioritySecurity + chain.TypedSpec().Policy = nethelpers.VerdictDrop + chain.TypedSpec().Rules = []network.NfTablesRule{ + { + Verdict: pointer.To(nethelpers.VerdictAccept), + }, + } + + s.Require().NoError(s.State().Create(s.Ctx(), chain)) + + s.checkNftOutput(`table inet talos-test { + chain test1 { + type filter hook input priority security; policy drop; + accept + } +}`) +} + +func (s *NfTablesChainSuite) TestICMPLimit() { + chain := network.NewNfTablesChain(network.NamespaceName, "test1") + chain.TypedSpec().Type = nethelpers.ChainTypeFilter + chain.TypedSpec().Hook = nethelpers.ChainHookInput + chain.TypedSpec().Priority = nethelpers.ChainPrioritySecurity + chain.TypedSpec().Policy = nethelpers.VerdictAccept + chain.TypedSpec().Rules = []network.NfTablesRule{ + { + MatchLayer4: &network.NfTablesLayer4Match{ + Protocol: nethelpers.ProtocolICMP, + }, + MatchLimit: &network.NfTablesLimitMatch{ + PacketRatePerSecond: 5, + }, + Verdict: pointer.To(nethelpers.VerdictAccept), + }, + } + + s.Require().NoError(s.State().Create(s.Ctx(), chain)) + + s.checkNftOutput(`table inet talos-test { + chain test1 { + type filter hook input priority security; policy accept; + meta l4proto icmp limit rate 5/second accept + } +}`) +} + +func (s *NfTablesChainSuite) TestConntrackCounter() { + chain := network.NewNfTablesChain(network.NamespaceName, "test1") + chain.TypedSpec().Type = nethelpers.ChainTypeFilter + chain.TypedSpec().Hook = nethelpers.ChainHookInput + chain.TypedSpec().Priority = nethelpers.ChainPrioritySecurity + chain.TypedSpec().Policy = nethelpers.VerdictAccept + chain.TypedSpec().Rules = []network.NfTablesRule{ + { + MatchConntrackState: &network.NfTablesConntrackStateMatch{ + States: []nethelpers.ConntrackState{ + nethelpers.ConntrackStateEstablished, + nethelpers.ConntrackStateRelated, + }, + }, + Verdict: pointer.To(nethelpers.VerdictAccept), + }, + { + MatchConntrackState: &network.NfTablesConntrackStateMatch{ + States: []nethelpers.ConntrackState{ + nethelpers.ConntrackStateInvalid, + }, + }, + AnonCounter: true, + Verdict: pointer.To(nethelpers.VerdictDrop), + }, + } + + s.Require().NoError(s.State().Create(s.Ctx(), chain)) + + s.checkNftOutput(`table inet talos-test { + chain test1 { + type filter hook input priority security; policy accept; + ct state { established, related } accept + ct state invalid counter packets 0 bytes 0 drop + } +}`) +} + +func (s *NfTablesChainSuite) TestMatchMarksSubnets() { + chain1 := network.NewNfTablesChain(network.NamespaceName, "test1") + chain1.TypedSpec().Type = nethelpers.ChainTypeFilter + chain1.TypedSpec().Hook = nethelpers.ChainHookInput + chain1.TypedSpec().Priority = nethelpers.ChainPriorityFilter + chain1.TypedSpec().Policy = nethelpers.VerdictAccept + chain1.TypedSpec().Rules = []network.NfTablesRule{ + { + MatchMark: &network.NfTablesMark{ + Mask: constants.KubeSpanDefaultFirewallMask, + Value: constants.KubeSpanDefaultFirewallMark, + }, + MatchSourceAddress: &network.NfTablesAddressMatch{ + IncludeSubnets: []netip.Prefix{ + netip.MustParsePrefix("10.0.0.0/8"), + netip.MustParsePrefix("0::/0"), + }, + ExcludeSubnets: []netip.Prefix{ + netip.MustParsePrefix("10.3.0.0/16"), + }, + Invert: true, + }, + MatchDestinationAddress: &network.NfTablesAddressMatch{ + IncludeSubnets: []netip.Prefix{ + netip.MustParsePrefix("192.168.0.0/24"), + }, + }, + Verdict: pointer.To(nethelpers.VerdictAccept), + }, + } + + s.Require().NoError(s.State().Create(s.Ctx(), chain1)) + + chain2 := network.NewNfTablesChain(network.NamespaceName, "test2") + chain2.TypedSpec().Type = nethelpers.ChainTypeFilter + chain2.TypedSpec().Hook = nethelpers.ChainHookInput + chain2.TypedSpec().Priority = nethelpers.ChainPriorityFilter + chain2.TypedSpec().Policy = nethelpers.VerdictAccept + chain2.TypedSpec().Rules = []network.NfTablesRule{ + { + MatchDestinationAddress: &network.NfTablesAddressMatch{ + IncludeSubnets: []netip.Prefix{ + netip.MustParsePrefix("192.168.3.5/32"), + }, + }, + SetMark: &network.NfTablesMark{ + Mask: ^uint32(constants.KubeSpanDefaultFirewallMask), + Xor: constants.KubeSpanDefaultFirewallMark, + }, + }, + } + + s.Require().NoError(s.State().Create(s.Ctx(), chain2)) + + s.checkNftOutput(`table inet talos-test { + chain test1 { + type filter hook input priority filter; policy accept; + meta mark & 0x00000060 == 0x00000020 ip saddr != { 10.0.0.0-10.2.255.255, 10.4.0.0-10.255.255.255 } ip daddr { 192.168.0.0/24 } accept + } + + chain test2 { + type filter hook input priority filter; policy accept; + ip daddr { 192.168.3.5 } meta mark set meta mark & 0xffffffbf | 0x00000020 + } +}`) +} + +func (s *NfTablesChainSuite) TestUpdateChains() { + chain := network.NewNfTablesChain(network.NamespaceName, "test1") + chain.TypedSpec().Type = nethelpers.ChainTypeFilter + chain.TypedSpec().Hook = nethelpers.ChainHookInput + chain.TypedSpec().Priority = nethelpers.ChainPriorityFilter + chain.TypedSpec().Policy = nethelpers.VerdictAccept + chain.TypedSpec().Rules = []network.NfTablesRule{ + { + MatchSourceAddress: &network.NfTablesAddressMatch{ + IncludeSubnets: []netip.Prefix{ + netip.MustParsePrefix("10.0.0.0/8"), + }, + ExcludeSubnets: []netip.Prefix{ + netip.MustParsePrefix("10.3.0.0/16"), + }, + Invert: true, + }, + MatchDestinationAddress: &network.NfTablesAddressMatch{ + IncludeSubnets: []netip.Prefix{ + netip.MustParsePrefix("192.168.0.0/24"), + }, + }, + Verdict: pointer.To(nethelpers.VerdictAccept), + }, + } + + s.Require().NoError(s.State().Create(s.Ctx(), chain)) + + s.checkNftOutput(`table inet talos-test { + chain test1 { + type filter hook input priority filter; policy accept; + ip saddr != { 10.0.0.0-10.2.255.255, 10.4.0.0-10.255.255.255 } ip daddr { 192.168.0.0/24 } accept + meta nfproto ipv6 accept + } +}`) + + chain.TypedSpec().Rules = []network.NfTablesRule{ + { + MatchSourceAddress: &network.NfTablesAddressMatch{ + IncludeSubnets: []netip.Prefix{ + netip.MustParsePrefix("10.0.0.0/8"), + }, + ExcludeSubnets: []netip.Prefix{ + netip.MustParsePrefix("10.4.0.0/16"), + }, + Invert: true, + }, + SetMark: &network.NfTablesMark{ + Mask: ^uint32(constants.KubeSpanDefaultFirewallMask), + Xor: constants.KubeSpanDefaultFirewallMark, + }, + }, + } + + s.Require().NoError(s.State().Update(s.Ctx(), chain)) + + s.checkNftOutput(`table inet talos-test { + chain test1 { + type filter hook input priority filter; policy accept; + ip saddr != { 10.0.0.0/14, 10.5.0.0-10.255.255.255 } meta mark set meta mark & 0xffffffbf | 0x00000020 + meta nfproto ipv6 meta mark set meta mark & 0xffffffbf | 0x00000020 + } +}`) + + s.Require().NoError(s.State().Destroy(s.Ctx(), chain.Metadata())) + + s.checkNftOutput(`table inet talos-test { +}`) +} + +func (s *NfTablesChainSuite) TestClampMSS() { + chain := network.NewNfTablesChain(network.NamespaceName, "test1") + chain.TypedSpec().Type = nethelpers.ChainTypeFilter + chain.TypedSpec().Hook = nethelpers.ChainHookInput + chain.TypedSpec().Priority = nethelpers.ChainPriorityFilter + chain.TypedSpec().Policy = nethelpers.VerdictAccept + chain.TypedSpec().Rules = []network.NfTablesRule{ + { + ClampMSS: &network.NfTablesClampMSS{ + MTU: constants.KubeSpanLinkMTU, + }, + }, + } + + s.Require().NoError(s.State().Create(s.Ctx(), chain)) + + s.checkNftOutput(`table inet talos-test { + chain test1 { + type filter hook input priority filter; policy accept; + meta nfproto ipv4 tcp flags syn / syn,rst tcp option maxseg size > 1380 tcp option maxseg size set 1380 + meta nfproto ipv6 tcp flags syn / syn,rst tcp option maxseg size > 1360 tcp option maxseg size set 1360 + } +}`) +} + +func (s *NfTablesChainSuite) TestL4Match() { + chain := network.NewNfTablesChain(network.NamespaceName, "test-tcp") + chain.TypedSpec().Type = nethelpers.ChainTypeFilter + chain.TypedSpec().Hook = nethelpers.ChainHookInput + chain.TypedSpec().Priority = nethelpers.ChainPriorityFilter + chain.TypedSpec().Policy = nethelpers.VerdictAccept + chain.TypedSpec().Rules = []network.NfTablesRule{ + { + MatchDestinationAddress: &network.NfTablesAddressMatch{ + IncludeSubnets: []netip.Prefix{ + netip.MustParsePrefix("10.0.0.0/8"), + netip.MustParsePrefix("2001::/16"), + }, + }, + MatchLayer4: &network.NfTablesLayer4Match{ + Protocol: nethelpers.ProtocolTCP, + MatchDestinationPort: &network.NfTablesPortMatch{ + Ranges: []network.PortRange{ + { + Lo: 1023, + Hi: 1025, + }, + { + Lo: 1027, + Hi: 1029, + }, + }, + }, + }, + Verdict: pointer.To(nethelpers.VerdictDrop), + }, + } + + s.Require().NoError(s.State().Create(s.Ctx(), chain)) + + s.checkNftOutput(`table inet talos-test { + chain test-tcp { + type filter hook input priority filter; policy accept; + ip daddr { 10.0.0.0/8 } tcp dport { 1023-1025, 1027-1029 } drop + ip6 daddr { 2001::/16 } tcp dport { 1023-1025, 1027-1029 } drop + } +}`) +} + +func (s *NfTablesChainSuite) TestL4Match2() { + chain := network.NewNfTablesChain(network.NamespaceName, "test-tcp") + chain.TypedSpec().Type = nethelpers.ChainTypeFilter + chain.TypedSpec().Hook = nethelpers.ChainHookInput + chain.TypedSpec().Priority = nethelpers.ChainPriorityFilter + chain.TypedSpec().Policy = nethelpers.VerdictAccept + chain.TypedSpec().Rules = []network.NfTablesRule{ + { + MatchSourceAddress: &network.NfTablesAddressMatch{ + IncludeSubnets: []netip.Prefix{ + netip.MustParsePrefix("10.0.0.0/8"), + }, + Invert: true, + }, + MatchLayer4: &network.NfTablesLayer4Match{ + Protocol: nethelpers.ProtocolTCP, + MatchDestinationPort: &network.NfTablesPortMatch{ + Ranges: []network.PortRange{ + { + Lo: 1023, + Hi: 1023, + }, + { + Lo: 1024, + Hi: 1024, + }, + }, + }, + }, + Verdict: pointer.To(nethelpers.VerdictDrop), + }, + } + + s.Require().NoError(s.State().Create(s.Ctx(), chain)) + + s.checkNftOutput(`table inet talos-test { + chain test-tcp { + type filter hook input priority filter; policy accept; + ip saddr != { 10.0.0.0/8 } tcp dport { 1023, 1024 } drop + meta nfproto ipv6 tcp dport { 1023, 1024 } drop + } +}`) +} + +func (s *NfTablesChainSuite) TestL4MatchAny() { + chain := network.NewNfTablesChain(network.NamespaceName, "test-tcp") + chain.TypedSpec().Type = nethelpers.ChainTypeFilter + chain.TypedSpec().Hook = nethelpers.ChainHookInput + chain.TypedSpec().Priority = nethelpers.ChainPriorityFilter + chain.TypedSpec().Policy = nethelpers.VerdictAccept + chain.TypedSpec().Rules = []network.NfTablesRule{ + { + MatchSourceAddress: &network.NfTablesAddressMatch{ + IncludeSubnets: []netip.Prefix{ + netip.MustParsePrefix("0.0.0.0/0"), + }, + }, + MatchLayer4: &network.NfTablesLayer4Match{ + Protocol: nethelpers.ProtocolTCP, + MatchDestinationPort: &network.NfTablesPortMatch{ + Ranges: []network.PortRange{ + { + Lo: 1023, + Hi: 1023, + }, + }, + }, + }, + Verdict: pointer.To(nethelpers.VerdictAccept), + }, + } + + s.Require().NoError(s.State().Create(s.Ctx(), chain)) + + s.checkNftOutput(`table inet talos-test { + chain test-tcp { + type filter hook input priority filter; policy accept; + meta nfproto ipv4 tcp dport { 1023 } accept + } +}`) +} + +func TestNftablesChainSuite(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("requires root") + } + + if exec.Command("nft", "list", "tables").Run() != nil { + t.Skip("requires nftables CLI to be installed") + } + + suite.Run(t, &NfTablesChainSuite{ + DefaultSuite: ctest.DefaultSuite{ + Timeout: 5 * time.Second, + AfterSetup: func(s *ctest.DefaultSuite) { + // try to see if the table is there + if exec.Command("nft", "list", "table", "inet", "talos-test").Run() == nil { + s.Require().NoError(exec.Command("nft", "delete", "table", "inet", "talos-test").Run()) + } + + s.Require().NoError(s.Runtime().RegisterController(&netctrl.NfTablesChainController{TableName: "talos-test"})) + }, + AfterTearDown: func(s *ctest.DefaultSuite) { + s.Require().NoError(exec.Command("nft", "delete", "table", "inet", "talos-test").Run()) + }, + }, + }) +} diff --git a/internal/app/machined/pkg/controllers/network/node_address.go b/internal/app/machined/pkg/controllers/network/node_address.go new file mode 100644 index 0000000..a37f7af --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/node_address.go @@ -0,0 +1,342 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "context" + "fmt" + "net/netip" + "slices" + "sort" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/siderolabs/gen/value" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// NodeAddressController manages secrets.Etcd based on configuration. +type NodeAddressController struct{} + +// Name implements controller.Controller interface. +func (ctrl *NodeAddressController) Name() string { + return "network.NodeAddressController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *NodeAddressController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: network.NamespaceName, + Type: network.AddressStatusType, + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.LinkStatusType, + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.NodeAddressFilterType, + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *NodeAddressController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.NodeAddressType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *NodeAddressController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + var addressStatusController AddressStatusController + + addressStatusControllerName := addressStatusController.Name() + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + // fetch link and address status resources + links, err := r.List(ctx, resource.NewMetadata(network.NamespaceName, network.LinkStatusType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing links: %w", err) + } + + // build "link up" lookup table + linksUp := make(map[uint32]struct{}) + + for _, r := range links.Items { + link := r.(*network.LinkStatus) //nolint:errcheck,forcetypeassert + + if link.TypedSpec().OperationalState == nethelpers.OperStateUp || link.TypedSpec().OperationalState == nethelpers.OperStateUnknown { + // skip physical interfaces without carrier + if !link.TypedSpec().Physical() || link.TypedSpec().LinkState { + linksUp[link.TypedSpec().Index] = struct{}{} + } + } + } + + // fetch list of filters + filters, err := r.List(ctx, resource.NewMetadata(network.NamespaceName, network.NodeAddressFilterType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing address filters: %w", err) + } + + addresses, err := r.List(ctx, resource.NewMetadata(network.NamespaceName, network.AddressStatusType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing links: %w", err) + } + + var ( + defaultAddress netip.Prefix + defaultAddrLinkName string + current []netip.Prefix + routed []netip.Prefix + accumulative []netip.Prefix + ) + + for _, r := range addresses.Items { + addr := r.(*network.AddressStatus) //nolint:errcheck,forcetypeassert + + if addr.TypedSpec().Scope >= nethelpers.ScopeLink { + continue + } + + ip := addr.TypedSpec().Address + + if ip.Addr().IsLoopback() || ip.Addr().IsMulticast() || ip.Addr().IsLinkLocalUnicast() { + continue + } + + // set defaultAddress to the smallest IP from the alphabetically first link + if addr.Metadata().Owner() == addressStatusControllerName { + if value.IsZero(defaultAddress) || addr.TypedSpec().LinkName < defaultAddrLinkName || (addr.TypedSpec().LinkName == defaultAddrLinkName && ip.Addr().Compare(defaultAddress.Addr()) < 0) { + defaultAddress = ip + defaultAddrLinkName = addr.TypedSpec().LinkName + } + } + + // assume addresses from external IPs to be always up + if _, up := linksUp[addr.TypedSpec().LinkIndex]; up || addr.TypedSpec().LinkName == externalLink { + current = append(current, ip) + } + + // routed: filter out external addresses and addresses from SideroLink + if _, up := linksUp[addr.TypedSpec().LinkIndex]; up && addr.TypedSpec().LinkName != externalLink { + if network.NotSideroLinkIP(ip.Addr()) { + routed = append(routed, ip) + } + } + + accumulative = append(accumulative, ip) + } + + // sort current addresses + sort.Slice(current, func(i, j int) bool { return current[i].Addr().Compare(current[j].Addr()) < 0 }) + sort.Slice(routed, func(i, j int) bool { return routed[i].Addr().Compare(routed[j].Addr()) < 0 }) + + // remove duplicates from current addresses + current = deduplicateIPPrefixes(current) + routed = deduplicateIPPrefixes(routed) + + touchedIDs := make(map[resource.ID]struct{}) + + // update output resources + if !value.IsZero(defaultAddress) { + if err = r.Modify(ctx, network.NewNodeAddress(network.NamespaceName, network.NodeAddressDefaultID), func(r resource.Resource) error { + spec := r.(*network.NodeAddress).TypedSpec() + + // never overwrite default address if it's already set + // we should start handing default address updates, but for now we're not ready + // + // at the same time check that recorded default address is still on the host, if it's not => replace it + if len(spec.Addresses) > 0 && slices.ContainsFunc(current, func(addr netip.Prefix) bool { return spec.Addresses[0] == addr }) { + return nil + } + + spec.Addresses = []netip.Prefix{defaultAddress} + + return nil + }); err != nil { + return fmt.Errorf("error updating output resource: %w", err) + } + + touchedIDs[network.NodeAddressDefaultID] = struct{}{} + } + + if err = updateCurrentAddresses(ctx, r, network.NodeAddressCurrentID, current); err != nil { + return err + } + + touchedIDs[network.NodeAddressCurrentID] = struct{}{} + + if err = updateCurrentAddresses(ctx, r, network.NodeAddressRoutedID, routed); err != nil { + return err + } + + touchedIDs[network.NodeAddressRoutedID] = struct{}{} + + if err = updateAccumulativeAddresses(ctx, r, network.NodeAddressAccumulativeID, accumulative); err != nil { + return err + } + + touchedIDs[network.NodeAddressAccumulativeID] = struct{}{} + + // update filtered resources + for _, res := range filters.Items { + filterID := res.Metadata().ID() + filter := res.(*network.NodeAddressFilter).TypedSpec() + + filteredCurrent := filterIPs(current, filter.IncludeSubnets, filter.ExcludeSubnets) + filteredRouted := filterIPs(routed, filter.IncludeSubnets, filter.ExcludeSubnets) + filteredAccumulative := filterIPs(accumulative, filter.IncludeSubnets, filter.ExcludeSubnets) + + if err = updateCurrentAddresses(ctx, r, network.FilteredNodeAddressID(network.NodeAddressCurrentID, filterID), filteredCurrent); err != nil { + return err + } + + if err = updateCurrentAddresses(ctx, r, network.FilteredNodeAddressID(network.NodeAddressRoutedID, filterID), filteredRouted); err != nil { + return err + } + + if err = updateAccumulativeAddresses(ctx, r, network.FilteredNodeAddressID(network.NodeAddressAccumulativeID, filterID), filteredAccumulative); err != nil { + return err + } + + touchedIDs[network.FilteredNodeAddressID(network.NodeAddressCurrentID, filterID)] = struct{}{} + touchedIDs[network.FilteredNodeAddressID(network.NodeAddressRoutedID, filterID)] = struct{}{} + touchedIDs[network.FilteredNodeAddressID(network.NodeAddressAccumulativeID, filterID)] = struct{}{} + } + + // list keys for cleanup + list, err := r.List(ctx, resource.NewMetadata(network.NamespaceName, network.NodeAddressType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing resources: %w", err) + } + + for _, res := range list.Items { + if res.Metadata().Owner() != ctrl.Name() { + continue + } + + if _, ok := touchedIDs[res.Metadata().ID()]; !ok { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return fmt.Errorf("error cleaning up specs: %w", err) + } + } + } + + r.ResetRestartBackoff() + } +} + +func deduplicateIPPrefixes(current []netip.Prefix) []netip.Prefix { + // assumes that current is sorted + n := 0 + + var prev netip.Prefix + + for _, x := range current { + if prev != x { + current[n] = x + n++ + } + + prev = x + } + + return current[:n] +} + +func filterIPs(addrs []netip.Prefix, includeSubnets, excludeSubnets []netip.Prefix) []netip.Prefix { + result := make([]netip.Prefix, 0, len(addrs)) + +outer: + for _, ip := range addrs { + if len(includeSubnets) > 0 { + matchesAny := false + + for _, subnet := range includeSubnets { + if subnet.Contains(ip.Addr()) { + matchesAny = true + + break + } + } + + if !matchesAny { + continue outer + } + } + + for _, subnet := range excludeSubnets { + if subnet.Contains(ip.Addr()) { + continue outer + } + } + + result = append(result, ip) + } + + return result +} + +func updateCurrentAddresses(ctx context.Context, r controller.Runtime, id resource.ID, current []netip.Prefix) error { + if err := r.Modify(ctx, network.NewNodeAddress(network.NamespaceName, id), func(r resource.Resource) error { + spec := r.(*network.NodeAddress).TypedSpec() + + spec.Addresses = current + + return nil + }); err != nil { + return fmt.Errorf("error updating output resource: %w", err) + } + + return nil +} + +func updateAccumulativeAddresses(ctx context.Context, r controller.Runtime, id resource.ID, accumulative []netip.Prefix) error { + if err := r.Modify(ctx, network.NewNodeAddress(network.NamespaceName, id), func(r resource.Resource) error { + spec := r.(*network.NodeAddress).TypedSpec() + + for _, ip := range accumulative { + // find insert position using binary search + i := sort.Search(len(spec.Addresses), func(j int) bool { + return !spec.Addresses[j].Addr().Less(ip.Addr()) + }) + + if i < len(spec.Addresses) && spec.Addresses[i].Addr().Compare(ip.Addr()) == 0 { + continue + } + + // insert at position i + spec.Addresses = append(spec.Addresses, netip.Prefix{}) + copy(spec.Addresses[i+1:], spec.Addresses[i:]) + spec.Addresses[i] = ip + } + + return nil + }); err != nil { + return fmt.Errorf("error updating output resource: %w", err) + } + + return nil +} diff --git a/internal/app/machined/pkg/controllers/network/node_address_test.go b/internal/app/machined/pkg/controllers/network/node_address_test.go new file mode 100644 index 0000000..cf982f3 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/node_address_test.go @@ -0,0 +1,418 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:dupl +package network_test + +import ( + "net/netip" + "sort" + "strings" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/cosi-project/runtime/pkg/state" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +type NodeAddressSuite struct { + ctest.DefaultSuite +} + +func (suite *NodeAddressSuite) TestDefaults() { + // create fake device ready status + deviceStatus := runtimeres.NewDevicesStatus(runtimeres.NamespaceName, runtimeres.DevicesID) + deviceStatus.TypedSpec().Ready = true + suite.Require().NoError(suite.State().Create(suite.Ctx(), deviceStatus)) + + suite.Require().NoError(suite.Runtime().RegisterController(&netctrl.AddressStatusController{})) + suite.Require().NoError(suite.Runtime().RegisterController(&netctrl.LinkStatusController{})) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), + []resource.ID{ + network.NodeAddressDefaultID, + network.NodeAddressCurrentID, + network.NodeAddressRoutedID, + network.NodeAddressAccumulativeID, + }, + func(r *network.NodeAddress, asrt *assert.Assertions) { + addrs := r.TypedSpec().Addresses + + suite.T().Logf("id %q val %s", r.Metadata().ID(), addrs) + + asrt.True( + sort.SliceIsSorted( + addrs, func(i, j int) bool { + return addrs[i].Addr().Compare(addrs[j].Addr()) < 0 + }, + ), "addresses %s", addrs, + ) + + if r.Metadata().ID() == network.NodeAddressDefaultID { + asrt.Len(addrs, 1) + } else { + asrt.NotEmpty(addrs) + } + }, + ) +} + +//nolint:gocyclo +func (suite *NodeAddressSuite) TestFilters() { + var ( + addressStatusController netctrl.AddressStatusController + platformConfigController netctrl.PlatformConfigController + ) + + linkUp := network.NewLinkStatus(network.NamespaceName, "eth0") + linkUp.TypedSpec().Type = nethelpers.LinkEther + linkUp.TypedSpec().LinkState = true + linkUp.TypedSpec().Index = 1 + suite.Require().NoError(suite.State().Create(suite.Ctx(), linkUp)) + + linkDown := network.NewLinkStatus(network.NamespaceName, "eth1") + linkDown.TypedSpec().Type = nethelpers.LinkEther + linkDown.TypedSpec().LinkState = false + linkDown.TypedSpec().Index = 2 + suite.Require().NoError(suite.State().Create(suite.Ctx(), linkDown)) + + newAddress := func(addr netip.Prefix, link *network.LinkStatus) { + addressStatus := network.NewAddressStatus(network.NamespaceName, network.AddressID(link.Metadata().ID(), addr)) + addressStatus.TypedSpec().Address = addr + addressStatus.TypedSpec().LinkName = link.Metadata().ID() + addressStatus.TypedSpec().LinkIndex = link.TypedSpec().Index + suite.Require().NoError( + suite.State().Create( + suite.Ctx(), + addressStatus, + state.WithCreateOwner(addressStatusController.Name()), + ), + ) + } + + newExternalAddress := func(addr netip.Prefix) { + addressStatus := network.NewAddressStatus(network.NamespaceName, network.AddressID("external", addr)) + addressStatus.TypedSpec().Address = addr + addressStatus.TypedSpec().LinkName = "external" + suite.Require().NoError( + suite.State().Create( + suite.Ctx(), + addressStatus, + state.WithCreateOwner(platformConfigController.Name()), + ), + ) + } + + for _, addr := range []string{ + "10.0.0.1/8", + "25.3.7.9/32", + "2001:470:6d:30e:4a62:b3ba:180b:b5b8/64", + "127.0.0.1/8", + "fdae:41e4:649b:9303:7886:731d:1ce9:4d4/128", + } { + newAddress(netip.MustParsePrefix(addr), linkUp) + } + + for _, addr := range []string{"10.0.0.2/8", "192.168.3.7/24"} { + newAddress(netip.MustParsePrefix(addr), linkDown) + } + + for _, addr := range []string{"1.2.3.4/32", "25.3.7.9/32"} { // duplicate with link address: 25.3.7.9 + newExternalAddress(netip.MustParsePrefix(addr)) + } + + filter1 := network.NewNodeAddressFilter(network.NamespaceName, "no-k8s") + filter1.TypedSpec().ExcludeSubnets = []netip.Prefix{netip.MustParsePrefix("10.0.0.0/8")} + suite.Require().NoError(suite.State().Create(suite.Ctx(), filter1)) + + filter2 := network.NewNodeAddressFilter(network.NamespaceName, "only-k8s") + filter2.TypedSpec().IncludeSubnets = []netip.Prefix{ + netip.MustParsePrefix("10.0.0.0/8"), + netip.MustParsePrefix("192.168.0.0/16"), + } + suite.Require().NoError(suite.State().Create(suite.Ctx(), filter2)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), + []resource.ID{ + network.NodeAddressDefaultID, + network.NodeAddressCurrentID, + network.NodeAddressRoutedID, + network.NodeAddressAccumulativeID, + network.FilteredNodeAddressID(network.NodeAddressCurrentID, filter1.Metadata().ID()), + network.FilteredNodeAddressID(network.NodeAddressRoutedID, filter1.Metadata().ID()), + network.FilteredNodeAddressID(network.NodeAddressAccumulativeID, filter1.Metadata().ID()), + network.FilteredNodeAddressID(network.NodeAddressCurrentID, filter2.Metadata().ID()), + network.FilteredNodeAddressID(network.NodeAddressRoutedID, filter2.Metadata().ID()), + network.FilteredNodeAddressID(network.NodeAddressAccumulativeID, filter2.Metadata().ID()), + }, + func(r *network.NodeAddress, asrt *assert.Assertions) { + addrs := r.TypedSpec().Addresses + + switch r.Metadata().ID() { + case network.NodeAddressDefaultID: + asrt.Equal(addrs, ipList("10.0.0.1/8")) + case network.NodeAddressCurrentID: + asrt.Equal( + ipList("1.2.3.4/32 10.0.0.1/8 25.3.7.9/32 2001:470:6d:30e:4a62:b3ba:180b:b5b8/64 fdae:41e4:649b:9303:7886:731d:1ce9:4d4/128"), + addrs, + ) + case network.NodeAddressRoutedID: + asrt.Equal( + ipList("10.0.0.1/8 25.3.7.9/32 2001:470:6d:30e:4a62:b3ba:180b:b5b8/64"), + addrs, + ) + case network.NodeAddressAccumulativeID: + asrt.Equal( + ipList("1.2.3.4/32 10.0.0.1/8 10.0.0.2/8 25.3.7.9/32 192.168.3.7/24 2001:470:6d:30e:4a62:b3ba:180b:b5b8/64 fdae:41e4:649b:9303:7886:731d:1ce9:4d4/128"), + addrs, + ) + case network.FilteredNodeAddressID(network.NodeAddressCurrentID, filter1.Metadata().ID()): + asrt.Equal( + ipList("1.2.3.4/32 25.3.7.9/32 2001:470:6d:30e:4a62:b3ba:180b:b5b8/64 fdae:41e4:649b:9303:7886:731d:1ce9:4d4/128"), + addrs, + ) + case network.FilteredNodeAddressID(network.NodeAddressRoutedID, filter1.Metadata().ID()): + asrt.Equal( + ipList("25.3.7.9/32 2001:470:6d:30e:4a62:b3ba:180b:b5b8/64"), + addrs, + ) + case network.FilteredNodeAddressID(network.NodeAddressAccumulativeID, filter1.Metadata().ID()): + asrt.Equal( + ipList("1.2.3.4/32 25.3.7.9/32 192.168.3.7/24 2001:470:6d:30e:4a62:b3ba:180b:b5b8/64 fdae:41e4:649b:9303:7886:731d:1ce9:4d4/128"), + addrs, + ) + case network.FilteredNodeAddressID(network.NodeAddressCurrentID, filter2.Metadata().ID()), + network.FilteredNodeAddressID(network.NodeAddressRoutedID, filter2.Metadata().ID()): + asrt.Equal(addrs, ipList("10.0.0.1/8")) + case network.FilteredNodeAddressID(network.NodeAddressAccumulativeID, filter2.Metadata().ID()): + asrt.Equal(addrs, ipList("10.0.0.1/8 10.0.0.2/8 192.168.3.7/24")) + } + }, + ) +} + +func (suite *NodeAddressSuite) TestFilterOverlappingSubnets() { + linkUp := network.NewLinkStatus(network.NamespaceName, "eth0") + linkUp.TypedSpec().Type = nethelpers.LinkEther + linkUp.TypedSpec().LinkState = true + linkUp.TypedSpec().Index = 1 + suite.Require().NoError(suite.State().Create(suite.Ctx(), linkUp)) + + newAddress := func(addr netip.Prefix, link *network.LinkStatus) { + addressStatus := network.NewAddressStatus(network.NamespaceName, network.AddressID(link.Metadata().ID(), addr)) + addressStatus.TypedSpec().Address = addr + addressStatus.TypedSpec().LinkName = link.Metadata().ID() + addressStatus.TypedSpec().LinkIndex = link.TypedSpec().Index + suite.Require().NoError( + suite.State().Create( + suite.Ctx(), + addressStatus, + ), + ) + } + + for _, addr := range []string{ + "10.0.0.1/8", + "10.96.0.2/32", + "25.3.7.9/32", + } { + newAddress(netip.MustParsePrefix(addr), linkUp) + } + + filter1 := network.NewNodeAddressFilter(network.NamespaceName, "no-k8s") + filter1.TypedSpec().ExcludeSubnets = []netip.Prefix{netip.MustParsePrefix("10.96.0.0/12")} + suite.Require().NoError(suite.State().Create(suite.Ctx(), filter1)) + + filter2 := network.NewNodeAddressFilter(network.NamespaceName, "only-k8s") + filter2.TypedSpec().IncludeSubnets = []netip.Prefix{netip.MustParsePrefix("10.96.0.0/12")} + suite.Require().NoError(suite.State().Create(suite.Ctx(), filter2)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), + []resource.ID{ + network.NodeAddressCurrentID, + network.NodeAddressRoutedID, + network.NodeAddressAccumulativeID, + network.FilteredNodeAddressID(network.NodeAddressCurrentID, filter1.Metadata().ID()), + network.FilteredNodeAddressID(network.NodeAddressRoutedID, filter1.Metadata().ID()), + network.FilteredNodeAddressID(network.NodeAddressAccumulativeID, filter1.Metadata().ID()), + network.FilteredNodeAddressID(network.NodeAddressCurrentID, filter2.Metadata().ID()), + network.FilteredNodeAddressID(network.NodeAddressRoutedID, filter2.Metadata().ID()), + network.FilteredNodeAddressID(network.NodeAddressAccumulativeID, filter2.Metadata().ID()), + }, + func(r *network.NodeAddress, asrt *assert.Assertions) { + addrs := r.TypedSpec().Addresses + + switch r.Metadata().ID() { + case network.NodeAddressCurrentID, network.NodeAddressRoutedID, network.NodeAddressAccumulativeID: + asrt.Equal( + ipList("10.0.0.1/8 10.96.0.2/32 25.3.7.9/32"), + addrs, + ) + case network.FilteredNodeAddressID(network.NodeAddressCurrentID, filter1.Metadata().ID()), + network.FilteredNodeAddressID(network.NodeAddressRoutedID, filter1.Metadata().ID()), + network.FilteredNodeAddressID(network.NodeAddressAccumulativeID, filter1.Metadata().ID()): + asrt.Equal( + ipList("10.0.0.1/8 25.3.7.9/32"), + addrs, + ) + case network.FilteredNodeAddressID(network.NodeAddressCurrentID, filter2.Metadata().ID()), + network.FilteredNodeAddressID(network.NodeAddressRoutedID, filter2.Metadata().ID()), + network.FilteredNodeAddressID(network.NodeAddressAccumulativeID, filter2.Metadata().ID()): + asrt.Equal( + ipList("10.96.0.2/32"), + addrs, + ) + } + }, + ) +} + +//nolint:gocyclo +func (suite *NodeAddressSuite) TestDefaultAddressChange() { + var addressStatusController netctrl.AddressStatusController + + linkUp := network.NewLinkStatus(network.NamespaceName, "eth0") + linkUp.TypedSpec().Type = nethelpers.LinkEther + linkUp.TypedSpec().LinkState = true + linkUp.TypedSpec().Index = 1 + suite.Require().NoError(suite.State().Create(suite.Ctx(), linkUp)) + + newAddress := func(addr netip.Prefix, link *network.LinkStatus) { + addressStatus := network.NewAddressStatus(network.NamespaceName, network.AddressID(link.Metadata().ID(), addr)) + addressStatus.TypedSpec().Address = addr + addressStatus.TypedSpec().LinkName = link.Metadata().ID() + addressStatus.TypedSpec().LinkIndex = link.TypedSpec().Index + suite.Require().NoError( + suite.State().Create( + suite.Ctx(), + addressStatus, + state.WithCreateOwner(addressStatusController.Name()), + ), + ) + } + + for _, addr := range []string{ + "10.0.0.5/8", + "25.3.7.9/32", + "127.0.0.1/8", + } { + newAddress(netip.MustParsePrefix(addr), linkUp) + } + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), + []resource.ID{ + network.NodeAddressDefaultID, + network.NodeAddressCurrentID, + network.NodeAddressAccumulativeID, + }, func(r *network.NodeAddress, asrt *assert.Assertions) { + addrs := r.TypedSpec().Addresses + + switch r.Metadata().ID() { + case network.NodeAddressDefaultID: + asrt.Equal(addrs, ipList("10.0.0.5/8")) + case network.NodeAddressCurrentID: + asrt.Equal( + addrs, + ipList("10.0.0.5/8 25.3.7.9/32"), + ) + case network.NodeAddressAccumulativeID: + asrt.Equal( + addrs, + ipList("10.0.0.5/8 25.3.7.9/32"), + ) + } + }, + ) + + // add another address which is "smaller", but default address shouldn't change + newAddress(netip.MustParsePrefix("1.1.1.1/32"), linkUp) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), + []resource.ID{ + network.NodeAddressDefaultID, + network.NodeAddressCurrentID, + network.NodeAddressAccumulativeID, + }, func(r *network.NodeAddress, asrt *assert.Assertions) { + addrs := r.TypedSpec().Addresses + + switch r.Metadata().ID() { + case network.NodeAddressDefaultID: + asrt.Equal(addrs, ipList("10.0.0.5/8")) + case network.NodeAddressCurrentID: + asrt.Equal( + addrs, + ipList("1.1.1.1/32 10.0.0.5/8 25.3.7.9/32"), + ) + case network.NodeAddressAccumulativeID: + asrt.Equal( + addrs, + ipList("1.1.1.1/32 10.0.0.5/8 25.3.7.9/32"), + ) + } + }, + ) + + // remove the previous default address, now default address should change + suite.Require().NoError(suite.State().Destroy(suite.Ctx(), + network.NewAddressStatus(network.NamespaceName, network.AddressID(linkUp.Metadata().ID(), netip.MustParsePrefix("10.0.0.5/8"))).Metadata(), + state.WithDestroyOwner(addressStatusController.Name()), + )) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), + []resource.ID{ + network.NodeAddressDefaultID, + network.NodeAddressCurrentID, + network.NodeAddressAccumulativeID, + }, func(r *network.NodeAddress, asrt *assert.Assertions) { + addrs := r.TypedSpec().Addresses + + switch r.Metadata().ID() { + case network.NodeAddressDefaultID: + asrt.Equal(addrs, ipList("1.1.1.1/32")) + case network.NodeAddressCurrentID: + asrt.Equal( + addrs, + ipList("1.1.1.1/32 25.3.7.9/32"), + ) + case network.NodeAddressAccumulativeID: + asrt.Equal( + addrs, + ipList("1.1.1.1/32 10.0.0.5/8 25.3.7.9/32"), + ) + } + }, + ) +} + +func TestNodeAddressSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, &NodeAddressSuite{ + DefaultSuite: ctest.DefaultSuite{ + Timeout: 5 * time.Second, + AfterSetup: func(s *ctest.DefaultSuite) { + s.Require().NoError(s.Runtime().RegisterController(&netctrl.NodeAddressController{})) + }, + }, + }) +} + +func ipList(ips string) []netip.Prefix { + var result []netip.Prefix //nolint:prealloc + + for _, ip := range strings.Split(ips, " ") { + result = append(result, netip.MustParsePrefix(ip)) + } + + return result +} diff --git a/internal/app/machined/pkg/controllers/network/operator/dhcp4.go b/internal/app/machined/pkg/controllers/network/operator/dhcp4.go new file mode 100644 index 0000000..ff91fd7 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/operator/dhcp4.go @@ -0,0 +1,575 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package operator + +import ( + "context" + "errors" + "fmt" + "net" + "net/netip" + "strings" + "sync" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/insomniacslk/dhcp/dhcpv4" + "github.com/insomniacslk/dhcp/dhcpv4/nclient4" + "github.com/siderolabs/gen/channel" + "github.com/siderolabs/gen/xslices" + "go.uber.org/zap" + "go4.org/netipx" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// DHCP4 implements the DHCPv4 network operator. +type DHCP4 struct { + logger *zap.Logger + state state.State + + linkName string + routeMetric uint32 + skipHostnameRequest bool + requestMTU bool + + lease *nclient4.Lease + + mu sync.Mutex + addresses []network.AddressSpecSpec + links []network.LinkSpecSpec + routes []network.RouteSpecSpec + hostname []network.HostnameSpecSpec + resolvers []network.ResolverSpecSpec + timeservers []network.TimeServerSpecSpec +} + +// NewDHCP4 creates DHCPv4 operator. +func NewDHCP4(logger *zap.Logger, linkName string, config network.DHCP4OperatorSpec, platform runtime.Platform, state state.State) *DHCP4 { + return &DHCP4{ + logger: logger, + state: state, + linkName: linkName, + routeMetric: config.RouteMetric, + skipHostnameRequest: config.SkipHostnameRequest, + // <3 azure + // When including dhcp.OptionInterfaceMTU we don't get a dhcp offer back on azure. + // So we'll need to explicitly exclude adding this option for azure. + requestMTU: platform.Name() != "azure", + } +} + +// Prefix returns unique operator prefix which gets prepended to each spec. +func (d *DHCP4) Prefix() string { + return fmt.Sprintf("dhcp4/%s", d.linkName) +} + +// extractHostname extracts a hostname from the given resource if it is a valid network.HostnameStatus. +func extractHostname(res resource.Resource) network.HostnameStatusSpec { + if res, ok := res.(*network.HostnameStatus); ok { + return *res.TypedSpec() + } + + return network.HostnameStatusSpec{} +} + +// setupHostnameWatch returns the initial hostname and a channel that outputs all events related to hostname changes. +func (d *DHCP4) setupHostnameWatch(ctx context.Context) (<-chan state.Event, error) { + hostnameWatchCh := make(chan state.Event) + if err := d.state.Watch(ctx, resource.NewMetadata( + network.NamespaceName, + network.HostnameStatusType, + network.HostnameID, + resource.VersionUndefined, + ), hostnameWatchCh); err != nil { + return nil, err + } + + return hostnameWatchCh, nil +} + +// knownHostname checks if the given hostname has been defined by this operator. +func (d *DHCP4) knownHostname(hostname network.HostnameStatusSpec) bool { + d.mu.Lock() + defer d.mu.Unlock() + + for i := range d.hostname { + if d.hostname[i].FQDN() == hostname.FQDN() { + return true + } + } + + return false +} + +// waitForNetworkReady waits for the network to be ready and the leased address to +// be assigned to the associated so that unicast operations can bind successfully. +func (d *DHCP4) waitForNetworkReady(ctx context.Context) error { + // If an IP address has been registered, wait for the address association to be ready + if len(d.addresses) > 0 { + _, err := d.state.WatchFor(ctx, + resource.NewMetadata( + network.NamespaceName, + network.AddressStatusType, + network.AddressID(d.linkName, d.addresses[0].Address), + resource.VersionUndefined, + ), + state.WithPhases(resource.PhaseRunning), + ) + if err != nil { + return fmt.Errorf("failed to wait for the address association to be ready: %w", err) + } + } + + // Wait for the network (address and connectivity) to be ready + if err := network.NewReadyCondition(d.state, network.AddressReady, network.ConnectivityReady).Wait(ctx); err != nil { + return fmt.Errorf("failed to wait for the network address and connectivity to be ready: %w", err) + } + + return nil +} + +// Run the operator loop. +// +//nolint:gocyclo,cyclop +func (d *DHCP4) Run(ctx context.Context, notifyCh chan<- struct{}) { + const minRenewDuration = 5 * time.Second // Protect from renewing too often + + renewInterval := minRenewDuration + + // Never send the hostname on the first iteration, to have a chance to query the hostname from the DHCP server. + // If the DHCP server doesn't provide a hostname, or if the hostname is overridden e.g. via machine config. + // we'll restart the sequence and send the hostname. + var hostname network.HostnameStatusSpec + + hostnameWatchCh, err := d.setupHostnameWatch(ctx) + if err != nil && !errors.Is(err, context.Canceled) { + d.logger.Warn("failed to watch for hostname changes", zap.Error(err)) + } + + for { + // Track if we need to acquire a new lease + newLease := d.lease == nil + + // Perform a lease request or renewal + leaseTime, err := d.requestRenew(ctx, hostname) + if err != nil && !errors.Is(err, context.Canceled) { + d.logger.Warn("request/renew failed", zap.Error(err), zap.String("link", d.linkName)) + } + + if err == nil { + // Notify the underlying controller about the new lease + if !channel.SendWithContext(ctx, notifyCh, struct{}{}) { + return + } + + if newLease { + // Wait for networking to be established before transitioning to unicast operations + if err = d.waitForNetworkReady(ctx); err != nil && !errors.Is(err, context.Canceled) { + d.logger.Warn("failed to wait for networking to become ready", zap.Error(err)) + } + } + } + + if leaseTime > 0 { + renewInterval = leaseTime / 2 + } else { + renewInterval /= 2 + } + + if renewInterval < minRenewDuration { + renewInterval = minRenewDuration + } + + for { + select { + case <-ctx.Done(): + return + case <-time.After(renewInterval): + case event := <-hostnameWatchCh: + // Attempt to drain the hostname watch channel coalescing multiple events into a single + // change to the DHCP. + drainLoop: + for { + select { + case event = <-hostnameWatchCh: + case <-ctx.Done(): + return + case <-time.After(time.Second): + break drainLoop + } + } + + // If the hostname resource was deleted entirely, we must still inform the DHCP + // server that the node has no hostname anymore. `extractHostname` will return a + // blank hostname for a Tombstone resource generated by a deletion event. + oldHostname := hostname + hostname = extractHostname(event.Resource) + + d.logger.Debug("detected hostname change", + zap.String("old", oldHostname.FQDN()), + zap.String("new", hostname.FQDN()), + ) + + // If, on first invocation, the DHCP server has given a new hostname for the node, + // and the `network.HostnameSpecController` decides to apply it as a preferred + // hostname, this operator would unnecessarily drop the lease and restart DHCP + // discovery. Thus, if the selected hostname has been sourced from this operator, + // we don't need to do anything. + if (oldHostname == network.HostnameStatusSpec{} && d.knownHostname(hostname)) || oldHostname == hostname { + continue + } + + // While updating the hostname together with a RENEW request works with dnsmasq, it + // doesn't work with the Windows Server DHCP + DNS. A hostname update via an + // INIT-REBOOT request also gets ignored. Thus, the only reliable way to update the + // hostname seems to be to forget the old release and initiate a new DISCOVER flow + // with the new hostname. RFC 2131 doesn't define any better way to do this, and, + // as a DISCOVER request cannot be targeted at the previous lessor according to the + // spec, the node may switch DHCP servers on hostname change. However, this is not + // a major concern, since a single network should not host multiple competing DHCP + // servers in the first place. + d.lease = nil + + d.logger.Debug("restarting DHCP sequence due to hostname change", + zap.Strings("dhcp_hostname", xslices.Map(d.hostname, func(spec network.HostnameSpecSpec) string { + return spec.Hostname + })), + ) + } + + break + } + } +} + +// AddressSpecs implements Operator interface. +func (d *DHCP4) AddressSpecs() []network.AddressSpecSpec { + d.mu.Lock() + defer d.mu.Unlock() + + return d.addresses +} + +// LinkSpecs implements Operator interface. +func (d *DHCP4) LinkSpecs() []network.LinkSpecSpec { + d.mu.Lock() + defer d.mu.Unlock() + + return d.links +} + +// RouteSpecs implements Operator interface. +func (d *DHCP4) RouteSpecs() []network.RouteSpecSpec { + d.mu.Lock() + defer d.mu.Unlock() + + return d.routes +} + +// HostnameSpecs implements Operator interface. +func (d *DHCP4) HostnameSpecs() []network.HostnameSpecSpec { + d.mu.Lock() + defer d.mu.Unlock() + + return d.hostname +} + +// ResolverSpecs implements Operator interface. +func (d *DHCP4) ResolverSpecs() []network.ResolverSpecSpec { + d.mu.Lock() + defer d.mu.Unlock() + + return d.resolvers +} + +// TimeServerSpecs implements Operator interface. +func (d *DHCP4) TimeServerSpecs() []network.TimeServerSpecSpec { + d.mu.Lock() + defer d.mu.Unlock() + + return d.timeservers +} + +//nolint:gocyclo +func (d *DHCP4) parseNetworkConfigFromAck(ack *dhcpv4.DHCPv4, useHostname bool) { + d.mu.Lock() + defer d.mu.Unlock() + + addr, _ := netipx.FromStdIPNet(&net.IPNet{ + IP: ack.YourIPAddr, + Mask: ack.SubnetMask(), + }) + + d.addresses = []network.AddressSpecSpec{ + { + Address: addr, + LinkName: d.linkName, + Family: nethelpers.FamilyInet4, + Scope: nethelpers.ScopeGlobal, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + ConfigLayer: network.ConfigOperator, + }, + } + + mtu, err := dhcpv4.GetUint16(dhcpv4.OptionInterfaceMTU, ack.Options) + if err == nil { + d.links = []network.LinkSpecSpec{ + { + Name: d.linkName, + MTU: uint32(mtu), + Up: true, + }, + } + } else { + d.links = nil + } + + // rfc3442: + // If the DHCP server returns both a Classless Static Routes option and + // a Router option, the DHCP client MUST ignore the Router option. + d.routes = nil + + if len(ack.ClasslessStaticRoute()) > 0 { + for _, route := range ack.ClasslessStaticRoute() { + gw, _ := netipx.FromStdIP(route.Router) + dst, _ := netipx.FromStdIPNet(route.Dest) + + d.routes = append(d.routes, network.RouteSpecSpec{ + Family: nethelpers.FamilyInet4, + Destination: dst, + Source: addr.Addr(), + Gateway: gw, + OutLinkName: d.linkName, + Table: nethelpers.TableMain, + Priority: d.routeMetric, + Scope: nethelpers.ScopeGlobal, + Type: nethelpers.TypeUnicast, + Protocol: nethelpers.ProtocolBoot, + ConfigLayer: network.ConfigOperator, + }) + } + } else { + for _, router := range ack.Router() { + gw, _ := netipx.FromStdIP(router) + + d.routes = append(d.routes, network.RouteSpecSpec{ + Family: nethelpers.FamilyInet4, + Gateway: gw, + Source: addr.Addr(), + OutLinkName: d.linkName, + Table: nethelpers.TableMain, + Priority: d.routeMetric, + Scope: nethelpers.ScopeGlobal, + Type: nethelpers.TypeUnicast, + Protocol: nethelpers.ProtocolBoot, + ConfigLayer: network.ConfigOperator, + }) + + if !addr.Contains(gw) { + // Add an interface route for the gateway if it's not in the same network + d.routes = append(d.routes, network.RouteSpecSpec{ + Family: nethelpers.FamilyInet4, + Destination: netip.PrefixFrom(gw, gw.BitLen()), + Source: addr.Addr(), + OutLinkName: d.linkName, + Table: nethelpers.TableMain, + Priority: d.routeMetric, + Scope: nethelpers.ScopeLink, + Type: nethelpers.TypeUnicast, + Protocol: nethelpers.ProtocolBoot, + ConfigLayer: network.ConfigOperator, + }) + } + } + } + + for i := range d.routes { + d.routes[i].Normalize() + } + + if useHostname { + d.hostname = nil + + if ack.HostName() != "" { + spec := network.HostnameSpecSpec{ + ConfigLayer: network.ConfigOperator, + } + + if err := spec.ParseFQDN(ack.HostName()); err == nil { + if ack.DomainName() != "" { + spec.Domainname = ack.DomainName() + } + + d.hostname = []network.HostnameSpecSpec{ + spec, + } + } + } + } + + if len(ack.DNS()) > 0 { + convertIP := func(ip net.IP) netip.Addr { + result, _ := netipx.FromStdIP(ip) + + return result + } + + d.resolvers = []network.ResolverSpecSpec{ + { + DNSServers: xslices.Map(ack.DNS(), convertIP), + ConfigLayer: network.ConfigOperator, + }, + } + } else { + d.resolvers = nil + } + + if len(ack.NTPServers()) > 0 { + convertIP := func(ip net.IP) string { + result, _ := netipx.FromStdIP(ip) + + return result.String() + } + + d.timeservers = []network.TimeServerSpecSpec{ + { + NTPServers: xslices.Map(ack.NTPServers(), convertIP), + ConfigLayer: network.ConfigOperator, + }, + } + } else { + d.timeservers = nil + } +} + +func (d *DHCP4) newClient() (*nclient4.Client, error) { + var clientOpts []nclient4.ClientOpt + + // We have an existing lease, target the server with unicast + if d.lease != nil && !d.lease.ACK.ServerIPAddr.IsUnspecified() { + // RFC 2131, section 4.3.2: + // DHCPREQUEST generated during RENEWING state: + // ... This message will be unicast, so no relay + // agents will be involved in its transmission. + clientOpts = append(clientOpts, + nclient4.WithServerAddr(&net.UDPAddr{ + IP: d.lease.ACK.ServerIPAddr, + Port: nclient4.ServerPort, + }), + // WithUnicast must be specified manually, WithServerAddr is not enough + nclient4.WithUnicast(&net.UDPAddr{ + IP: d.lease.ACK.YourIPAddr, + Port: nclient4.ClientPort, + }), + ) + } + + // Create a new client, the caller is responsible for closing it + return nclient4.New(d.linkName, clientOpts...) +} + +//nolint:gocyclo +func (d *DHCP4) requestRenew(ctx context.Context, hostname network.HostnameStatusSpec) (time.Duration, error) { + opts := []dhcpv4.OptionCode{ + dhcpv4.OptionClasslessStaticRoute, + dhcpv4.OptionDomainNameServer, + // TODO(twelho): This is unused until network.ResolverSpec supports search domains + dhcpv4.OptionDNSDomainSearchList, + dhcpv4.OptionNTPServers, + } + + if d.requestMTU { + opts = append(opts, dhcpv4.OptionInterfaceMTU) + } + + sendHostnameRequest := !d.skipHostnameRequest + if hostname.Hostname != "" && !d.knownHostname(hostname) { + // If we are supposed to publish a hostname, don't request one from the DHCP server. + // + // DHCP hostname parroting protection: if, e.g., `dnsmasq` receives a request that both + // sends a hostname and requests one, it will "parrot" the sent hostname back if no other + // name has been defined for the requesting host. This causes update anomalies, since + // removing a hostname defined previously by, e.g., the configuration layer, causes a copy + // of that hostname to live on in a spec defined by this operator, even though it isn't + // sourced from DHCP. + // + // To avoid this issue, never send and request a hostname in the same operation. When + // negotiating a new lease, first send the current hostname when acquiring the lease, and + // then follow up with a dedicated INFORM request asking the server for a DHCP-defined + // hostname. When renewing a lease, we're free to always request a hostname with an INFORM + // (to detect server-side changes), since any changes to the node hostname will cause a + // lease invalidation and re-start the negotiation process. More details below. + sendHostnameRequest = false + } + + if sendHostnameRequest { + opts = append(opts, dhcpv4.OptionHostName, dhcpv4.OptionDomainName) + } + + mods := []dhcpv4.Modifier{dhcpv4.WithRequestedOptions(opts...)} + + if !sendHostnameRequest { + // If the node has a hostname, always send it to the DHCP + // server with option 12 during lease acquisition and renewal + if len(hostname.Hostname) > 0 { + mods = append(mods, dhcpv4.WithOption(dhcpv4.OptHostName(hostname.Hostname))) + } + + if len(hostname.Domainname) > 0 { + mods = append(mods, dhcpv4.WithOption(dhcpv4.OptDomainName(hostname.Domainname))) + } + } + + client, err := d.newClient() + if err != nil { + return 0, err + } + + //nolint:errcheck + defer client.Close() + + switch { + case d.lease != nil && !d.lease.ACK.ServerIPAddr.IsUnspecified(): + d.logger.Debug("DHCP RENEW", zap.String("link", d.linkName)) + d.lease, err = client.Renew(ctx, d.lease, mods...) + case d.lease != nil && d.lease.Offer != nil: + d.logger.Debug("DHCP REQUEST FROM OFFER", zap.String("link", d.linkName)) + d.lease, err = client.RequestFromOffer(ctx, d.lease.Offer, mods...) + default: + d.logger.Debug("DHCP REQUEST", zap.String("link", d.linkName)) + d.lease, err = client.Request(ctx, mods...) + } + + if err != nil { + // explicitly clear the lease on failure to start with the discovery sequence next time + d.lease = nil + + return 0, err + } + + d.logger.Debug("DHCP ACK", zap.String("link", d.linkName), zap.String("dhcp", collapseSummary(d.lease.ACK.Summary()))) + + d.parseNetworkConfigFromAck(d.lease.ACK, sendHostnameRequest) + + return d.lease.ACK.IPAddressLeaseTime(time.Minute * 30), nil +} + +func collapseSummary(summary string) string { + lines := strings.Split(summary, "\n")[1:] + + for i := range lines { + lines[i] = strings.TrimSpace(lines[i]) + } + + if len(lines) > 0 && lines[len(lines)-1] == "" { + lines = lines[:len(lines)-1] + } + + return strings.Join(lines, ", ") +} diff --git a/internal/app/machined/pkg/controllers/network/operator/dhcp6.go b/internal/app/machined/pkg/controllers/network/operator/dhcp6.go new file mode 100644 index 0000000..9be946a --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/operator/dhcp6.go @@ -0,0 +1,305 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package operator + +import ( + "context" + "encoding/hex" + "errors" + "fmt" + "net" + "net/netip" + "strings" + "sync" + "time" + + "github.com/insomniacslk/dhcp/dhcpv6" + "github.com/insomniacslk/dhcp/dhcpv6/nclient6" + "github.com/jsimonetti/rtnetlink" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/go-retry/retry" + "go.uber.org/zap" + "go4.org/netipx" + "golang.org/x/sys/unix" + + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// DHCP6 implements the DHCPv6 network operator. +type DHCP6 struct { + logger *zap.Logger + + linkName string + duid []byte + skipHostnameRequest bool + + mu sync.Mutex + addresses []network.AddressSpecSpec + hostname []network.HostnameSpecSpec + resolvers []network.ResolverSpecSpec + timeservers []network.TimeServerSpecSpec +} + +// NewDHCP6 creates DHCPv6 operator. +func NewDHCP6(logger *zap.Logger, linkName string, config network.DHCP6OperatorSpec) *DHCP6 { + duidBin, _ := hex.DecodeString(config.DUID) //nolint:errcheck + + return &DHCP6{ + logger: logger, + linkName: linkName, + duid: duidBin, + skipHostnameRequest: config.SkipHostnameRequest, + } +} + +// Prefix returns unique operator prefix which gets prepended to each spec. +func (d *DHCP6) Prefix() string { + return fmt.Sprintf("dhcp6/%s", d.linkName) +} + +// Run the operator loop. +// +//nolint:gocyclo +func (d *DHCP6) Run(ctx context.Context, notifyCh chan<- struct{}) { + iface, err := net.InterfaceByName(d.linkName) + if err != nil { + d.logger.Warn("link not found", zap.String("link", d.linkName)) + } else if err = d.waitIPv6LinkReady(ctx, iface); err != nil { + d.logger.Warn("error waiting for IPv6 ready", zap.Error(err), zap.String("link", d.linkName)) + } + + const minRenewDuration = 5 * time.Second // protect from renewing too often + + renewInterval := minRenewDuration + + for { + leaseTime, err := d.renew(ctx) + if err != nil && !errors.Is(err, context.Canceled) { + d.logger.Warn("renew failed", zap.Error(err), zap.String("link", d.linkName)) + } + + if err == nil { + select { + case notifyCh <- struct{}{}: + case <-ctx.Done(): + return + } + } + + if leaseTime > 0 { + renewInterval = leaseTime / 2 + } else { + renewInterval /= 2 + } + + if renewInterval < minRenewDuration { + renewInterval = minRenewDuration + } + + select { + case <-ctx.Done(): + return + case <-time.After(renewInterval): + } + } +} + +// AddressSpecs implements Operator interface. +func (d *DHCP6) AddressSpecs() []network.AddressSpecSpec { + d.mu.Lock() + defer d.mu.Unlock() + + return d.addresses +} + +// LinkSpecs implements Operator interface. +func (d *DHCP6) LinkSpecs() []network.LinkSpecSpec { + return nil +} + +// RouteSpecs implements Operator interface. +func (d *DHCP6) RouteSpecs() []network.RouteSpecSpec { + return nil +} + +// HostnameSpecs implements Operator interface. +func (d *DHCP6) HostnameSpecs() []network.HostnameSpecSpec { + d.mu.Lock() + defer d.mu.Unlock() + + return d.hostname +} + +// ResolverSpecs implements Operator interface. +func (d *DHCP6) ResolverSpecs() []network.ResolverSpecSpec { + d.mu.Lock() + defer d.mu.Unlock() + + return d.resolvers +} + +// TimeServerSpecs implements Operator interface. +func (d *DHCP6) TimeServerSpecs() []network.TimeServerSpecSpec { + d.mu.Lock() + defer d.mu.Unlock() + + return d.timeservers +} + +func (d *DHCP6) parseReply(reply *dhcpv6.Message) (leaseTime time.Duration) { + d.mu.Lock() + defer d.mu.Unlock() + + if reply.Options.OneIANA() != nil && reply.Options.OneIANA().Options.OneAddress() != nil { + addr, _ := netipx.FromStdIPNet(&net.IPNet{ + IP: reply.Options.OneIANA().Options.OneAddress().IPv6Addr, + Mask: net.CIDRMask(128, 128), + }) + + d.addresses = []network.AddressSpecSpec{ + { + Address: addr, + LinkName: d.linkName, + Family: nethelpers.FamilyInet6, + Scope: nethelpers.ScopeGlobal, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + ConfigLayer: network.ConfigOperator, + }, + } + + leaseTime = reply.Options.OneIANA().Options.OneAddress().ValidLifetime + } else { + d.addresses = nil + } + + if len(reply.Options.DNS()) > 0 { + convertIP := func(ip net.IP) netip.Addr { + result, _ := netipx.FromStdIP(ip) + + return result + } + + d.resolvers = []network.ResolverSpecSpec{ + { + DNSServers: xslices.Map(reply.Options.DNS(), convertIP), + ConfigLayer: network.ConfigOperator, + }, + } + } else { + d.resolvers = nil + } + + if reply.Options.FQDN() != nil && len(reply.Options.FQDN().DomainName.Labels) > 0 && !d.skipHostnameRequest { + d.hostname = []network.HostnameSpecSpec{ + { + Hostname: reply.Options.FQDN().DomainName.Labels[0], + Domainname: strings.Join(reply.Options.FQDN().DomainName.Labels[1:], "."), + ConfigLayer: network.ConfigOperator, + }, + } + } else { + d.hostname = nil + } + + if len(reply.Options.NTPServers()) > 0 { + convertIP := func(ip net.IP) string { + result, _ := netipx.FromStdIP(ip) + + return result.String() + } + + d.timeservers = []network.TimeServerSpecSpec{ + { + NTPServers: xslices.Map(reply.Options.NTPServers(), convertIP), + ConfigLayer: network.ConfigOperator, + }, + } + } else { + d.timeservers = nil + } + + return leaseTime +} + +func (d *DHCP6) renew(ctx context.Context) (time.Duration, error) { + cli, err := nclient6.New(d.linkName) + if err != nil { + return 0, err + } + + defer cli.Close() //nolint:errcheck + + var modifiers []dhcpv6.Modifier + + if len(d.duid) > 0 { + duid, derr := dhcpv6.DUIDFromBytes(d.duid) + if derr != nil { + d.logger.Error("failed to parse DUID, ignored", zap.String("link", d.linkName)) + } else { + modifiers = []dhcpv6.Modifier{dhcpv6.WithClientID(duid)} + } + } + + reply, err := cli.RapidSolicit(ctx, modifiers...) + if err != nil { + return 0, err + } + + d.logger.Debug("DHCP6 REPLY", zap.String("link", d.linkName), zap.String("dhcp", collapseSummary(reply.Summary()))) + + return d.parseReply(reply), nil +} + +func (d *DHCP6) waitIPv6LinkReady(ctx context.Context, iface *net.Interface) error { + conn, err := rtnetlink.Dial(nil) + if err != nil { + return err + } + + defer conn.Close() //nolint:errcheck + + return retry.Constant(30*time.Second, retry.WithUnits(100*time.Millisecond)).RetryWithContext(ctx, func(ctx context.Context) error { + ready, err := d.isIPv6LinkReady(iface, conn) + if err != nil { + return err + } + + if !ready { + return retry.ExpectedErrorf("IPv6 address is still tentative") + } + + return nil + }) +} + +// isIPv6LinkReady returns true if the interface has a link-local address +// which is not tentative. +func (d *DHCP6) isIPv6LinkReady(iface *net.Interface, conn *rtnetlink.Conn) (bool, error) { + addrs, err := conn.Address.List() + if err != nil { + return false, err + } + + for _, addr := range addrs { + if addr.Index != uint32(iface.Index) { + continue + } + + if addr.Family != unix.AF_INET6 { + continue + } + + if addr.Attributes.Address.IsLinkLocalUnicast() && (addr.Flags&unix.IFA_F_TENTATIVE == 0) { + if addr.Flags&unix.IFA_F_DADFAILED != 0 { + d.logger.Warn("DADFAILED for %v, continuing anyhow", zap.Stringer("address", addr.Attributes.Address), zap.String("link", d.linkName)) + } + + return true, nil + } + } + + return false, nil +} diff --git a/internal/app/machined/pkg/controllers/network/operator/operator.go b/internal/app/machined/pkg/controllers/network/operator/operator.go new file mode 100644 index 0000000..ad7f18b --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/operator/operator.go @@ -0,0 +1,27 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package operator implements network operators. +package operator + +import ( + "context" + + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// Operator describes common interface of the operators. +type Operator interface { + Run(ctx context.Context, notifyCh chan<- struct{}) + + Prefix() string + + AddressSpecs() []network.AddressSpecSpec + RouteSpecs() []network.RouteSpecSpec + LinkSpecs() []network.LinkSpecSpec + + HostnameSpecs() []network.HostnameSpecSpec + ResolverSpecs() []network.ResolverSpecSpec + TimeServerSpecs() []network.TimeServerSpecSpec +} diff --git a/internal/app/machined/pkg/controllers/network/operator/vip.go b/internal/app/machined/pkg/controllers/network/operator/vip.go new file mode 100644 index 0000000..735c1d9 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/operator/vip.go @@ -0,0 +1,392 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package operator + +import ( + "context" + "errors" + "fmt" + "net/netip" + "os" + "strings" + "sync" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "go.etcd.io/etcd/client/v3/concurrency" + "go.uber.org/zap" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network/operator/vip" + "github.com/aenix-io/talm/internal/pkg/etcd" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +const campaignRetryInterval = time.Second + +// VIP implements the Virtual (Shared) IP network operator. +type VIP struct { + logger *zap.Logger + + linkName string + sharedIP netip.Addr + gratuitousARP bool + + state state.State + + mu sync.Mutex + leader bool + + handler vip.Handler +} + +// NewVIP creates Virtual IP operator. +func NewVIP(logger *zap.Logger, linkName string, spec network.VIPOperatorSpec, state state.State) *VIP { + var handler vip.Handler + + switch { + case spec.EquinixMetal != network.VIPEquinixMetalSpec{}: + handler = vip.NewEquinixMetalHandler(logger, spec.IP.String(), spec.EquinixMetal) + case spec.HCloud != network.VIPHCloudSpec{}: + handler = vip.NewHCloudHandler(logger, spec.IP.String(), spec.HCloud) + default: + handler = vip.NopHandler{} + } + + return &VIP{ + logger: logger, + linkName: linkName, + sharedIP: spec.IP, + gratuitousARP: spec.GratuitousARP, + state: state, + handler: handler, + } +} + +// Prefix returns unique operator prefix which gets prepended to each spec. +func (vip *VIP) Prefix() string { + return fmt.Sprintf("vip/%s", vip.linkName) +} + +// Run the operator loop. +func (vip *VIP) Run(ctx context.Context, notifyCh chan<- struct{}) { + for { + err := vip.campaign(ctx, notifyCh) + if err != nil { + if !errors.Is(err, context.Canceled) { + vip.logger.Warn("campaign failure", zap.Error(err), zap.String("link", vip.linkName), zap.Stringer("ip", vip.sharedIP)) + } + + select { + case <-time.After(campaignRetryInterval): + case <-ctx.Done(): + return + } + } + } +} + +// AddressSpecs implements Operator interface. +func (vip *VIP) AddressSpecs() []network.AddressSpecSpec { + vip.mu.Lock() + defer vip.mu.Unlock() + + if !vip.leader { + return nil + } + + family := nethelpers.FamilyInet6 + gratuitousARP := false + + if vip.sharedIP.Is4() { + family = nethelpers.FamilyInet4 + gratuitousARP = vip.gratuitousARP + } + + return []network.AddressSpecSpec{ + { + Address: netip.PrefixFrom(vip.sharedIP, vip.sharedIP.BitLen()), + LinkName: vip.linkName, + Family: family, + Scope: nethelpers.ScopeGlobal, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + AnnounceWithARP: gratuitousARP, + ConfigLayer: network.ConfigOperator, + }, + } +} + +// LinkSpecs implements Operator interface. +func (vip *VIP) LinkSpecs() []network.LinkSpecSpec { + return nil +} + +// RouteSpecs implements Operator interface. +func (vip *VIP) RouteSpecs() []network.RouteSpecSpec { + return nil +} + +// HostnameSpecs implements Operator interface. +func (vip *VIP) HostnameSpecs() []network.HostnameSpecSpec { + return nil +} + +// ResolverSpecs implements Operator interface. +func (vip *VIP) ResolverSpecs() []network.ResolverSpecSpec { + return nil +} + +// TimeServerSpecs implements Operator interface. +func (vip *VIP) TimeServerSpecs() []network.TimeServerSpecSpec { + return nil +} + +func (vip *VIP) etcdElectionKey() string { + return fmt.Sprintf("%s:vip:election:%s", constants.EtcdRootTalosKey, vip.sharedIP.String()) +} + +func (vip *VIP) waitForPreconditions(ctx context.Context) error { + // wait for the etcd to be up + _, err := vip.state.WatchFor(ctx, resource.NewMetadata(v1alpha1.NamespaceName, v1alpha1.ServiceType, "etcd", resource.VersionUndefined), + state.WithCondition(func(r resource.Resource) (bool, error) { + if resource.IsTombstone(r) { + return false, nil + } + + svc := r.(*v1alpha1.Service) //nolint:errcheck,forcetypeassert + + return svc.TypedSpec().Running && svc.TypedSpec().Healthy, nil + })) + if err != nil { + return fmt.Errorf("etcd health wait failure: %w", err) + } + + // wait for the kubelet lifecycle to be up, and not being torn down + _, err = vip.state.WatchFor(ctx, resource.NewMetadata(k8s.NamespaceName, k8s.KubeletLifecycleType, k8s.KubeletLifecycleID, resource.VersionUndefined), + state.WithCondition(func(r resource.Resource) (bool, error) { + if resource.IsTombstone(r) { + return false, nil + } + + if r.Metadata().Phase() == resource.PhaseTearingDown { + return false, nil + } + + return true, nil + })) + if err != nil { + return fmt.Errorf("kubelet lifecycle wait failure: %w", err) + } + + return nil +} + +//nolint:gocyclo,cyclop +func (vip *VIP) campaign(ctx context.Context, notifyCh chan<- struct{}) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + if err := vip.waitForPreconditions(ctx); err != nil { + return fmt.Errorf("error waiting for preconditions: %w", err) + } + + // put a finalizer on the kubelet lifecycle and remove once the campaign is done + kubeletLifecycle := resource.NewMetadata(k8s.NamespaceName, k8s.KubeletLifecycleType, k8s.KubeletLifecycleID, resource.VersionUndefined) + if err := vip.state.AddFinalizer(ctx, kubeletLifecycle, vip.Prefix()); err != nil { + return fmt.Errorf("error adding kubelet lifecycle finalizer: %w", err) + } + + defer func() { + vip.state.RemoveFinalizer(ctx, kubeletLifecycle, vip.Prefix()) //nolint:errcheck + }() + + hostname, err := os.Hostname() // TODO: this should be etcd nodename + if err != nil { + return errors.New("refusing to join election without a hostname") + } + + ec, err := etcd.NewLocalClient(ctx) + if err != nil { + return fmt.Errorf("failed to create local etcd client: %w", err) + } + + defer ec.Close() //nolint:errcheck + + sess, err := concurrency.NewSession(ec.Client) + if err != nil { + return fmt.Errorf("failed to create concurrency session: %w", err) + } + defer sess.Close() //nolint:errcheck + + election := concurrency.NewElection(sess, vip.etcdElectionKey()) + + node, err := election.Leader(ctx) + if err != nil { + if err != concurrency.ErrElectionNoLeader { + return fmt.Errorf("failed getting current leader: %w", err) + } + } else if string(node.Kvs[0].Value) == hostname { + vip.logger.Info("resigning from previous election") + + // we are still leader from the previous election, attempt to resign to force new election + resumedElection := concurrency.ResumeElection(sess, vip.etcdElectionKey(), string(node.Kvs[0].Key), node.Kvs[0].CreateRevision) + + if err = resumedElection.Resign(ctx); err != nil { + return fmt.Errorf("failed resigning from previous elections: %w", err) + } + } + + campaignErrCh := make(chan error) + + go func() { + campaignErrCh <- election.Campaign(ctx, hostname) + }() + + watchCh := make(chan state.Event) + + if err = vip.state.Watch(ctx, resource.NewMetadata(v1alpha1.NamespaceName, v1alpha1.ServiceType, "etcd", resource.VersionUndefined), watchCh); err != nil { + return fmt.Errorf("error setting up etcd watch: %w", err) + } + + if err = vip.state.Watch(ctx, kubeletLifecycle, watchCh); err != nil { + return fmt.Errorf("error setting up etcd watch: %w", err) + } + + err = vip.state.WatchKind(ctx, resource.NewMetadata(k8s.NamespaceName, k8s.StaticPodStatusType, "", resource.VersionUndefined), watchCh) + if err != nil { + return fmt.Errorf("kube-apiserver health wait failure: %w", err) + } + + // wait for the etcd election campaign to be complete + // while waiting, also observe the kubelet lifecycle object (if the node is shutting down) and etcd status +campaignLoop: + for { + select { + case err = <-campaignErrCh: + if err != nil { + return fmt.Errorf("failed to conduct campaign: %w", err) + } + + // node won the election campaign! + break campaignLoop + case <-sess.Done(): + vip.logger.Info("etcd session closed") + + return nil + case <-ctx.Done(): + return nil + case event := <-watchCh: + // note: here we don't wait for kube-apiserver, as it might not be up on cluster bootstrap, but VIP should be still assigned + + // break the loop when etcd is stopped + if event.Type == state.Destroyed && event.Resource.Metadata().ID() == "etcd" { + return nil + } + + // break the loop if the kubelet lifecycle is entering teardown phase + if event.Resource != nil { + if event.Resource.Metadata().Type() == kubeletLifecycle.Type() && event.Resource.Metadata().ID() == kubeletLifecycle.ID() && event.Resource.Metadata().Phase() == resource.PhaseTearingDown { + return nil + } + } + } + } + + defer func() { + // use a new context to resign, as `ctx` might be canceled + resignCtx, resignCancel := context.WithTimeout(context.Background(), 10*time.Second) + defer resignCancel() + + election.Resign(resignCtx) //nolint:errcheck + }() + + if err = vip.markAsLeader(ctx, notifyCh, true); err != nil { + return err + } + + defer func() { + if err = vip.markAsLeader(ctx, notifyCh, false); err != nil && !errors.Is(err, context.Canceled) { + vip.logger.Info("failed disabling shared IP", zap.String("link", vip.linkName), zap.Stringer("ip", vip.sharedIP), zap.Error(err)) + } + + vip.logger.Info("removing shared IP", zap.String("link", vip.linkName), zap.Stringer("ip", vip.sharedIP)) + }() + + vip.logger.Info("enabled shared IP", zap.String("link", vip.linkName), zap.Stringer("ip", vip.sharedIP)) + + observe := election.Observe(ctx) + +observeLoop: + for { + select { + case <-sess.Done(): + vip.logger.Info("etcd session closed") + + break observeLoop + case <-ctx.Done(): + break observeLoop + case resp, ok := <-observe: + if !ok { + break observeLoop + } + + if string(resp.Kvs[0].Value) != hostname { + vip.logger.Info("detected new leader", zap.ByteString("leader", resp.Kvs[0].Value)) + + break observeLoop + } + case event := <-watchCh: + // break the loop when etcd is stopped or kube-apiserver is stopped + if event.Type == state.Destroyed { + if event.Resource.Metadata().ID() == "etcd" || strings.HasPrefix(event.Resource.Metadata().ID(), "kube-system/kube-apiserver-") { + break observeLoop + } + } + + // break the loop if the kubelet lifecycle is entering teardown phase + if event.Resource != nil { + if event.Resource.Metadata().Type() == kubeletLifecycle.Type() && event.Resource.Metadata().ID() == kubeletLifecycle.ID() && event.Resource.Metadata().Phase() == resource.PhaseTearingDown { + break observeLoop + } + } + } + } + + return nil +} + +func (vip *VIP) markAsLeader(ctx context.Context, notifyCh chan<- struct{}, leader bool) error { + var handlerErr error + + if leader { + handlerErr = vip.handler.Acquire(ctx) + + if handlerErr != nil { + // if failed to acquire, we are not a leader, we will resign from the election + // so don't mark as leader, so that Talos doesn't announce IPs on the host + leader = false + } + } else { + handlerErr = vip.handler.Release(ctx) + } + + func() { + vip.mu.Lock() + defer vip.mu.Unlock() + + vip.leader = leader + }() + + select { + case <-ctx.Done(): + return ctx.Err() + case notifyCh <- struct{}{}: + return handlerErr + } +} diff --git a/internal/app/machined/pkg/controllers/network/operator/vip/equinix_metal.go b/internal/app/machined/pkg/controllers/network/operator/vip/equinix_metal.go new file mode 100644 index 0000000..d1b8962 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/operator/vip/equinix_metal.go @@ -0,0 +1,133 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package vip + +import ( + "context" + "encoding/json" + "fmt" + "path" + + "github.com/packethost/packngo" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/download" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// EquinixMetalHandler implements assignment and release of Virtual IPs using API. +type EquinixMetalHandler struct { + client *packngo.Client + + logger *zap.Logger + + vip string + projectID string + deviceID string + + assignmentID string +} + +// NewEquinixMetalHandler creates new EquinixMetalHandler. +func NewEquinixMetalHandler(logger *zap.Logger, vip string, spec network.VIPEquinixMetalSpec) *EquinixMetalHandler { + return &EquinixMetalHandler{ + client: packngo.NewClientWithAuth("talos", spec.APIToken, nil), + + logger: logger, + + vip: vip, + projectID: spec.ProjectID, + deviceID: spec.DeviceID, + } +} + +// Acquire implements Handler interface. +func (handler *EquinixMetalHandler) Acquire(ctx context.Context) error { + ips, _, err := handler.client.ProjectIPs.List(handler.projectID, &packngo.ListOptions{}) + if err != nil { + return fmt.Errorf("error listing project IPs: %w", err) + } + + // look up assignments for the VIP and unassign it + for _, ip := range ips { + if ip.Address != handler.vip { + continue + } + + for _, assignment := range ip.Assignments { + assignmentID := path.Base(assignment.Href) + + if _, err = handler.client.DeviceIPs.Unassign(assignmentID); err != nil { + return fmt.Errorf("error removing assignment %s: %w", assignment.String(), err) + } + + handler.logger.Info("cleared previous Equinix Metal IP assignment", zap.String("assignment", assignmentID), zap.String("vip", handler.vip)) + } + } + + // assign the VIP to this device + assignment, _, err := handler.client.DeviceIPs.Assign(handler.deviceID, &packngo.AddressStruct{ + Address: handler.vip, + }) + if err != nil { + return fmt.Errorf("error assigning %q to %q: %w", handler.vip, handler.deviceID, err) + } + + handler.logger.Info("assigned Equinix Metal IP", zap.String("vip", handler.vip), zap.String("device_id", handler.deviceID), zap.String("assignment", assignment.ID)) + handler.assignmentID = assignment.ID + + return nil +} + +// Release implements Handler interface. +func (handler *EquinixMetalHandler) Release(ctx context.Context) error { + if handler.assignmentID == "" { + return nil + } + + _, err := handler.client.DeviceIPs.Unassign(handler.assignmentID) + if err != nil { + return fmt.Errorf("error removing assignment %s: %w", handler.assignmentID, err) + } + + handler.logger.Info("unassigned Equinix Metal IP", zap.String("assignment", handler.assignmentID), zap.String("vip", handler.vip)) + + return nil +} + +// EquinixMetalMetaDataEndpoint is the local endpoint for machine info like networking. +const EquinixMetalMetaDataEndpoint = "https://metadata.platformequinix.com/metadata" + +// GetProjectAndDeviceIDs fills in parts of the spec based on the API token and instance metadata. +func GetProjectAndDeviceIDs(ctx context.Context, spec *network.VIPEquinixMetalSpec) error { + metadataConfig, err := download.Download(ctx, EquinixMetalMetaDataEndpoint) + if err != nil { + return fmt.Errorf("error downloading metadata: %w", err) + } + + type Metadata struct { + ID string `json:"id"` + } + + var unmarshalledMetadataConfig Metadata + if err = json.Unmarshal(metadataConfig, &unmarshalledMetadataConfig); err != nil { + return fmt.Errorf("error unmarshaling metadata: %w", err) + } + + spec.DeviceID = unmarshalledMetadataConfig.ID + + client := packngo.NewClientWithAuth("talos", spec.APIToken, nil) + + device, _, err := client.Devices.Get(spec.DeviceID, &packngo.GetOptions{ + Includes: []string{"project"}, + }) + if err != nil { + return fmt.Errorf("error getting device: %w", err) + } + + spec.ProjectID = device.Project.ID + + return nil +} diff --git a/internal/app/machined/pkg/controllers/network/operator/vip/equinix_metal_test.go b/internal/app/machined/pkg/controllers/network/operator/vip/equinix_metal_test.go new file mode 100644 index 0000000..556ec35 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/operator/vip/equinix_metal_test.go @@ -0,0 +1,69 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package vip_test + +import ( + "context" + "log" + "os" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network/operator/vip" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +func TestEquinixMetalHandler(t *testing.T) { + // WARNING: this test requires interaction with Equinix Metal API with real device IDs and API token + // it is skipped by default unless following variables are set: + // TALOS_EM_API_TOKEN + // TALOS_EM_PROJECT_ID + // TALOS_EM_DEVICE_ID_1 + // TALOS_EM_DEVICE_ID_2 + // TALOS_EM_VIP + settings := map[string]string{} + + for _, variable := range []string{ + "TALOS_EM_API_TOKEN", + "TALOS_EM_PROJECT_ID", + "TALOS_EM_DEVICE_ID_1", + "TALOS_EM_DEVICE_ID_2", + "TALOS_EM_VIP", + } { + var ok bool + + settings[variable], ok = os.LookupEnv(variable) + + if !ok { + t.Skip("skipping the test as the environment variable is not set", variable) + } + } + + logger := logging.Wrap(log.Writer()) + + handler1 := vip.NewEquinixMetalHandler(logger, settings["TALOS_EM_VIP"], network.VIPEquinixMetalSpec{ + ProjectID: settings["TALOS_EM_PROJECT_ID"], + DeviceID: settings["TALOS_EM_DEVICE_ID_1"], + APIToken: settings["TALOS_EM_API_TOKEN"], + }) + + handler2 := vip.NewEquinixMetalHandler(logger, settings["TALOS_EM_VIP"], network.VIPEquinixMetalSpec{ + ProjectID: settings["TALOS_EM_PROJECT_ID"], + DeviceID: settings["TALOS_EM_DEVICE_ID_2"], + APIToken: settings["TALOS_EM_API_TOKEN"], + }) + + // not graceful + require.NoError(t, handler1.Acquire(context.Background())) + require.NoError(t, handler2.Acquire(context.Background())) + + // graceful + require.NoError(t, handler1.Acquire(context.Background())) + require.NoError(t, handler1.Release(context.Background())) + require.NoError(t, handler2.Acquire(context.Background())) + require.NoError(t, handler2.Release(context.Background())) +} diff --git a/internal/app/machined/pkg/controllers/network/operator/vip/hcloud.go b/internal/app/machined/pkg/controllers/network/operator/vip/hcloud.go new file mode 100644 index 0000000..c9d1fbe --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/operator/vip/hcloud.go @@ -0,0 +1,208 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package vip + +import ( + "context" + "fmt" + "net" + "net/netip" + "strconv" + + "github.com/hetznercloud/hcloud-go/v2/hcloud" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/download" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// HCloudHandler implements assignment and release of Virtual IPs using API. +type HCloudHandler struct { + client *hcloud.Client + + logger *zap.Logger + + vip string + deviceID int64 + floatingID int64 + networkID int64 +} + +// NewHCloudHandler creates new NewEHCloudHandler. +func NewHCloudHandler(logger *zap.Logger, vip string, spec network.VIPHCloudSpec) *HCloudHandler { + return &HCloudHandler{ + client: hcloud.NewClient(hcloud.WithToken(spec.APIToken)), + + logger: logger, + + vip: vip, + deviceID: spec.DeviceID, + networkID: spec.NetworkID, + } +} + +// Acquire implements Handler interface. +func (handler *HCloudHandler) Acquire(ctx context.Context) error { + if handler.networkID > 0 { + var action *hcloud.Action + + alias := hcloud.ServerChangeAliasIPsOpts{ + Network: &hcloud.Network{ID: handler.networkID}, + AliasIPs: []net.IP{}, + } + + // trying to find the old active server + // and remove alias IP from it + serverList, err := handler.client.Server.All(ctx) + if err != nil { + return fmt.Errorf("error getting server list: %w", err) + } + + oldDeviceID := findServerByAlias(serverList, handler.networkID, handler.vip) + if oldDeviceID != 0 { + action, _, err = handler.client.Server.ChangeAliasIPs(ctx, + &hcloud.Server{ID: oldDeviceID}, + hcloud.ServerChangeAliasIPsOpts{ + Network: &hcloud.Network{ID: handler.networkID}, + AliasIPs: []net.IP{}, + }) + if err != nil { + return fmt.Errorf("error remove alias IPs %q on server %d: %w", handler.vip, oldDeviceID, err) + } + + handler.logger.Info("cleared previous Hetzner Cloud IP alias", zap.String("vip", handler.vip), + zap.Int64("device_id", oldDeviceID), zap.String("status", string(action.Status))) + } + + netIP := net.ParseIP(handler.vip) + alias.AliasIPs = []net.IP{netIP} + + action, _, err = handler.client.Server.ChangeAliasIPs(ctx, + &hcloud.Server{ID: handler.deviceID}, + alias) + if err != nil { + return fmt.Errorf("error change alias IPs %q to server %d: %w", handler.vip, handler.deviceID, err) + } + + handler.logger.Info("assigned Hetzner Cloud alias IP", zap.String("vip", handler.vip), zap.Int64("device_id", handler.deviceID), + zap.Int64("network_id", handler.networkID), zap.String("status", string(action.Status))) + + return nil + } + + floatips, err := handler.client.FloatingIP.All(ctx) + if err != nil { + return fmt.Errorf("error getting floatingIPs list: %w", err) + } + + for _, floatip := range floatips { + if floatip.IP.String() == handler.vip { + action, _, err := handler.client.FloatingIP.Assign(ctx, floatip, &hcloud.Server{ID: handler.deviceID}) + if err != nil { + return fmt.Errorf("error assigning %q on server %d: %w", handler.vip, handler.deviceID, err) + } + + handler.logger.Info("assigned Hetzner Cloud floating IP", zap.String("vip", handler.vip), zap.Int64("device_id", handler.deviceID), zap.String("status", string(action.Status))) + handler.floatingID = floatip.ID + + return nil + } + } + + return fmt.Errorf("error assigning %q to server %d: floating IP is not found", handler.vip, handler.deviceID) +} + +// Release implements Handler interface. +func (handler *HCloudHandler) Release(ctx context.Context) error { + if handler.networkID > 0 { + alias := hcloud.ServerChangeAliasIPsOpts{ + Network: &hcloud.Network{ID: handler.networkID}, + AliasIPs: []net.IP{}, + } + + action, _, err := handler.client.Server.ChangeAliasIPs(ctx, + &hcloud.Server{ID: handler.deviceID}, + alias) + if err != nil { + return fmt.Errorf("error remove alias IPs %q on server %d: %w", handler.vip, handler.deviceID, err) + } + + handler.logger.Info("unassigned Hetzner Cloud alias IP", zap.String("vip", handler.vip), zap.Int64("device_id", handler.deviceID), + zap.Int64("network_id", handler.networkID), zap.String("status", string(action.Status))) + + return nil + } + + if handler.floatingID > 0 { + floatip, _, err := handler.client.FloatingIP.GetByID(ctx, handler.floatingID) + if err != nil { + return fmt.Errorf("error getting floatingIP info: %w", err) + } + + if floatip.Server == nil || floatip.Server.ID != handler.deviceID { + handler.logger.Info("unassigned Hetzner Cloud floating IP", zap.String("vip", handler.vip), zap.Int64("device_id", handler.deviceID)) + } + + handler.floatingID = 0 + } + + return nil +} + +// HCloudMetaDataEndpoint is the local endpoint for machine info like networking. +const HCloudMetaDataEndpoint = "http://169.254.169.254/hetzner/v1/metadata/instance-id" + +// GetNetworkAndDeviceIDs fills in parts of the spec based on the API token and instance metadata. +func GetNetworkAndDeviceIDs(ctx context.Context, spec *network.VIPHCloudSpec, vip netip.Addr) error { + metadataInstanceID, err := download.Download(ctx, HCloudMetaDataEndpoint) + if err != nil { + return fmt.Errorf("error downloading instance-id: %w", err) + } + + spec.DeviceID, err = strconv.ParseInt(string(metadataInstanceID), 10, 64) + if err != nil { + return fmt.Errorf("error getting instance-id id: %w", err) + } + + client := hcloud.NewClient(hcloud.WithToken(spec.APIToken)) + + server, _, err := client.Server.GetByID(ctx, spec.DeviceID) + if err != nil { + return fmt.Errorf("error getting server info: %w", err) + } + + spec.NetworkID = 0 + + for _, privnet := range server.PrivateNet { + network, _, err := client.Network.GetByID(ctx, privnet.Network.ID) + if err != nil { + return fmt.Errorf("error getting network info: %w", err) + } + + if network.IPRange.Contains(vip.AsSlice()) { + spec.NetworkID = privnet.Network.ID + + break + } + } + + return nil +} + +func findServerByAlias(serverList []*hcloud.Server, networkID int64, vip string) (deviceID int64) { + for _, server := range serverList { + for _, network := range server.PrivateNet { + if network.Network.ID == networkID { + for _, alias := range network.Aliases { + if alias.String() == vip { + return server.ID + } + } + } + } + } + + return 0 +} diff --git a/internal/app/machined/pkg/controllers/network/operator/vip/nop.go b/internal/app/machined/pkg/controllers/network/operator/vip/nop.go new file mode 100644 index 0000000..92fa0e9 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/operator/vip/nop.go @@ -0,0 +1,22 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package vip + +import ( + "context" +) + +// NopHandler does nothing. +type NopHandler struct{} + +// Acquire implements Handler interface. +func (handler NopHandler) Acquire(ctx context.Context) error { + return nil +} + +// Release implements Handler interface. +func (handler NopHandler) Release(ctx context.Context) error { + return nil +} diff --git a/internal/app/machined/pkg/controllers/network/operator/vip/vip.go b/internal/app/machined/pkg/controllers/network/operator/vip/vip.go new file mode 100644 index 0000000..ed3df06 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/operator/vip/vip.go @@ -0,0 +1,14 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package vip contains implementations of specific methods to acquire/release virtual IPs. +package vip + +import "context" + +// Handler implements custom actions to manage virtual IP assignment. +type Handler interface { + Acquire(ctx context.Context) error + Release(ctx context.Context) error +} diff --git a/internal/app/machined/pkg/controllers/network/operator_config.go b/internal/app/machined/pkg/controllers/network/operator_config.go new file mode 100644 index 0000000..0354680 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/operator_config.go @@ -0,0 +1,322 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/hashicorp/go-multierror" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/go-procfs/procfs" + "go.uber.org/zap" + + talosconfig "github.com/siderolabs/talos/pkg/machinery/config/config" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// OperatorConfigController manages network.OperatorSpec based on machine configuration, kernel cmdline. +type OperatorConfigController struct { + Cmdline *procfs.Cmdline +} + +// Name implements controller.Controller interface. +func (ctrl *OperatorConfigController) Name() string { + return "network.OperatorConfigController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *OperatorConfigController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: network.NamespaceName, + Type: network.DeviceConfigSpecType, + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.LinkStatusType, + Kind: controller.InputWeak, + }, + { + Namespace: network.ConfigNamespaceName, + Type: network.LinkSpecType, + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *OperatorConfigController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.OperatorSpecType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *OperatorConfigController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + touchedIDs := make(map[resource.ID]struct{}) + + items, err := r.List(ctx, resource.NewMetadata(network.NamespaceName, network.DeviceConfigSpecType, "", resource.VersionUndefined)) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting config: %w", err) + } + } + + var ( + specs []network.OperatorSpecSpec + specErrors *multierror.Error + ) + + ignoredInterfaces := map[string]struct{}{} + + if ctrl.Cmdline != nil { + var settings CmdlineNetworking + + settings, err = ParseCmdlineNetwork(ctrl.Cmdline) + if err != nil { + logger.Warn("ignored cmdline parse failure", zap.Error(err)) + } + + for _, link := range settings.IgnoreInterfaces { + ignoredInterfaces[link] = struct{}{} + } + + for _, linkConfig := range settings.LinkConfigs { + if !linkConfig.DHCP { + continue + } + + specs = append(specs, network.OperatorSpecSpec{ + Operator: network.OperatorDHCP4, + LinkName: linkConfig.LinkName, + RequireUp: true, + DHCP4: network.DHCP4OperatorSpec{ + RouteMetric: network.DefaultRouteMetric, + }, + ConfigLayer: network.ConfigCmdline, + }) + } + } + + devices := xslices.Map(items.Items, func(item resource.Resource) talosconfig.Device { + return item.(*network.DeviceConfigSpec).TypedSpec().Device + }) + + // operators from the config + if len(devices) > 0 { + for _, device := range devices { + if device.Ignore() { + ignoredInterfaces[device.Interface()] = struct{}{} + } + + if _, ignore := ignoredInterfaces[device.Interface()]; ignore { + continue + } + + if device.DHCP() && device.DHCPOptions().IPv4() { + routeMetric := device.DHCPOptions().RouteMetric() + if routeMetric == 0 { + routeMetric = network.DefaultRouteMetric + } + + specs = append(specs, network.OperatorSpecSpec{ + Operator: network.OperatorDHCP4, + LinkName: device.Interface(), + RequireUp: true, + DHCP4: network.DHCP4OperatorSpec{ + RouteMetric: routeMetric, + }, + ConfigLayer: network.ConfigMachineConfiguration, + }) + } + + if device.DHCP() && device.DHCPOptions().IPv6() { + routeMetric := device.DHCPOptions().RouteMetric() + if routeMetric == 0 { + routeMetric = network.DefaultRouteMetric + } + + specs = append(specs, network.OperatorSpecSpec{ + Operator: network.OperatorDHCP6, + LinkName: device.Interface(), + RequireUp: true, + DHCP6: network.DHCP6OperatorSpec{ + RouteMetric: routeMetric, + DUID: device.DHCPOptions().DUIDv6(), + }, + ConfigLayer: network.ConfigMachineConfiguration, + }) + } + + for _, vlan := range device.Vlans() { + if vlan.DHCP() && vlan.DHCPOptions().IPv4() { + routeMetric := vlan.DHCPOptions().RouteMetric() + if routeMetric == 0 { + routeMetric = network.DefaultRouteMetric + } + + specs = append(specs, network.OperatorSpecSpec{ + Operator: network.OperatorDHCP4, + LinkName: nethelpers.VLANLinkName(device.Interface(), vlan.ID()), + RequireUp: true, + DHCP4: network.DHCP4OperatorSpec{ + RouteMetric: routeMetric, + }, + ConfigLayer: network.ConfigMachineConfiguration, + }) + } + + if vlan.DHCP() && vlan.DHCPOptions().IPv6() { + routeMetric := vlan.DHCPOptions().RouteMetric() + if routeMetric == 0 { + routeMetric = network.DefaultRouteMetric + } + + specs = append(specs, network.OperatorSpecSpec{ + Operator: network.OperatorDHCP6, + LinkName: nethelpers.VLANLinkName(device.Interface(), vlan.ID()), + RequireUp: true, + DHCP6: network.DHCP6OperatorSpec{ + RouteMetric: routeMetric, + DUID: vlan.DHCPOptions().DUIDv6(), + }, + ConfigLayer: network.ConfigMachineConfiguration, + }) + } + } + } + } + + // build configuredInterfaces from linkSpecs in `network-config` namespace + // any link which has any configuration derived from the machine configuration or platform configuration should be ignored + configuredInterfaces := map[string]struct{}{} + + list, err := r.List(ctx, resource.NewMetadata(network.ConfigNamespaceName, network.LinkSpecType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing link specs: %w", err) + } + + for _, item := range list.Items { + linkSpec := item.(*network.LinkSpec).TypedSpec() + + switch linkSpec.ConfigLayer { + case network.ConfigDefault: + // ignore default link specs + case network.ConfigOperator: + // specs produced by operators, ignore + case network.ConfigCmdline, network.ConfigMachineConfiguration, network.ConfigPlatform: + // interface is configured explicitly, don't run default dhcp4 + configuredInterfaces[linkSpec.Name] = struct{}{} + } + } + + // operators from defaults + list, err = r.List(ctx, resource.NewMetadata(network.NamespaceName, network.LinkStatusType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing link statuses: %w", err) + } + + for _, item := range list.Items { + linkStatus := item.(*network.LinkStatus) //nolint:errcheck,forcetypeassert + + if linkStatus.TypedSpec().Physical() { + if _, configured := configuredInterfaces[linkStatus.Metadata().ID()]; !configured { + if _, ignored := ignoredInterfaces[linkStatus.Metadata().ID()]; !ignored { + // enable DHCPv4 operator on physical interfaces which don't have any explicit configuration and are not ignored + specs = append(specs, network.OperatorSpecSpec{ + Operator: network.OperatorDHCP4, + LinkName: linkStatus.Metadata().ID(), + RequireUp: true, + DHCP4: network.DHCP4OperatorSpec{ + RouteMetric: network.DefaultRouteMetric, + }, + ConfigLayer: network.ConfigDefault, + }) + } + } + } + } + + var ids []string + + ids, err = ctrl.apply(ctx, r, specs) + if err != nil { + return fmt.Errorf("error applying operator specs: %w", err) + } + + for _, id := range ids { + touchedIDs[id] = struct{}{} + } + + // list specs for cleanup + list, err = r.List(ctx, resource.NewMetadata(network.ConfigNamespaceName, network.OperatorSpecType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing resources: %w", err) + } + + for _, res := range list.Items { + if res.Metadata().Owner() != ctrl.Name() { + // skip specs created by other controllers + continue + } + + if _, ok := touchedIDs[res.Metadata().ID()]; !ok { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return fmt.Errorf("error cleaning up routes: %w", err) + } + } + } + + // last, check if some specs failed to build; fail last so that other operator specs are applied successfully + if err = specErrors.ErrorOrNil(); err != nil { + return err + } + + r.ResetRestartBackoff() + } +} + +//nolint:dupl +func (ctrl *OperatorConfigController) apply(ctx context.Context, r controller.Runtime, specs []network.OperatorSpecSpec) ([]resource.ID, error) { + ids := make([]string, 0, len(specs)) + + for _, spec := range specs { + id := network.LayeredID(spec.ConfigLayer, network.OperatorID(spec.Operator, spec.LinkName)) + + if err := r.Modify( + ctx, + network.NewOperatorSpec(network.ConfigNamespaceName, id), + func(r resource.Resource) error { + *r.(*network.OperatorSpec).TypedSpec() = spec + + return nil + }, + ); err != nil { + return ids, err + } + + ids = append(ids, id) + } + + return ids, nil +} diff --git a/internal/app/machined/pkg/controllers/network/operator_config_test.go b/internal/app/machined/pkg/controllers/network/operator_config_test.go new file mode 100644 index 0000000..141e56c --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/operator_config_test.go @@ -0,0 +1,436 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:dupl +package network_test + +import ( + "context" + "log" + "net/url" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-pointer" + "github.com/siderolabs/go-procfs/procfs" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type OperatorConfigSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *OperatorConfigSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.DeviceConfigController{})) +} + +func (suite *OperatorConfigSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *OperatorConfigSuite) assertOperators(requiredIDs []string, check func(*network.OperatorSpec, *assert.Assertions)) { + assertResources(suite.ctx, suite.T(), suite.state, requiredIDs, check, rtestutils.WithNamespace(network.ConfigNamespaceName)) +} + +func (suite *OperatorConfigSuite) assertNoOperators(unexpectedIDs []string) error { + unexpIDs := make(map[string]struct{}, len(unexpectedIDs)) + + for _, id := range unexpectedIDs { + unexpIDs[id] = struct{}{} + } + + resources, err := suite.state.List( + suite.ctx, + resource.NewMetadata(network.ConfigNamespaceName, network.OperatorSpecType, "", resource.VersionUndefined), + ) + if err != nil { + return err + } + + for _, res := range resources.Items { + _, unexpected := unexpIDs[res.Metadata().ID()] + if unexpected { + return retry.ExpectedErrorf("unexpected ID %q", res.Metadata().ID()) + } + } + + return nil +} + +func (suite *OperatorConfigSuite) TestDefaultDHCP() { + suite.Require().NoError( + suite.runtime.RegisterController( + &netctrl.OperatorConfigController{ + Cmdline: procfs.NewCmdline("talos.network.interface.ignore=eth2"), + }, + ), + ) + + suite.startRuntime() + + for _, link := range []string{"eth0", "eth1", "eth2"} { + linkStatus := network.NewLinkStatus(network.NamespaceName, link) + linkStatus.TypedSpec().Type = nethelpers.LinkEther + linkStatus.TypedSpec().LinkState = true + + suite.Require().NoError(suite.state.Create(suite.ctx, linkStatus)) + } + + suite.assertOperators( + []string{ + "default/dhcp4/eth0", + "default/dhcp4/eth1", + }, func(r *network.OperatorSpec, asrt *assert.Assertions) { + asrt.Equal(network.OperatorDHCP4, r.TypedSpec().Operator) + asrt.True(r.TypedSpec().RequireUp) + asrt.EqualValues(network.DefaultRouteMetric, r.TypedSpec().DHCP4.RouteMetric) + + switch r.Metadata().ID() { + case "default/dhcp4/eth0": + asrt.Equal("eth0", r.TypedSpec().LinkName) + case "default/dhcp4/eth1": + asrt.Equal("eth1", r.TypedSpec().LinkName) + } + }, + ) +} + +func (suite *OperatorConfigSuite) TestDefaultDHCPCmdline() { + suite.Require().NoError( + suite.runtime.RegisterController( + &netctrl.OperatorConfigController{ + Cmdline: procfs.NewCmdline("ip=172.20.0.2::172.20.0.1:255.255.255.0::eth1::::: ip=eth3:dhcp"), + }, + ), + ) + + suite.startRuntime() + + for _, link := range []string{"eth0", "eth1", "eth2"} { + linkStatus := network.NewLinkStatus(network.NamespaceName, link) + linkStatus.TypedSpec().Type = nethelpers.LinkEther + linkStatus.TypedSpec().LinkState = true + + suite.Require().NoError(suite.state.Create(suite.ctx, linkStatus)) + } + + suite.assertOperators( + []string{ + "default/dhcp4/eth0", + "default/dhcp4/eth2", + "cmdline/dhcp4/eth3", + }, func(r *network.OperatorSpec, asrt *assert.Assertions) { + asrt.Equal(network.OperatorDHCP4, r.TypedSpec().Operator) + asrt.True(r.TypedSpec().RequireUp) + asrt.EqualValues(network.DefaultRouteMetric, r.TypedSpec().DHCP4.RouteMetric) + + switch r.Metadata().ID() { + case "default/dhcp4/eth0": + asrt.Equal("eth0", r.TypedSpec().LinkName) + case "default/dhcp4/eth2": + asrt.Equal("eth2", r.TypedSpec().LinkName) + case "cmdline/dhcp4/eth3": + asrt.Equal("eth3", r.TypedSpec().LinkName) + } + }, + ) + + // remove link + suite.Require().NoError( + suite.state.Destroy( + suite.ctx, + resource.NewMetadata(network.NamespaceName, network.LinkStatusType, "eth2", resource.VersionUndefined), + ), + ) + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertNoOperators( + []string{ + "default/dhcp4/eth2", + }, + ) + }, + ), + ) +} + +func (suite *OperatorConfigSuite) TestMachineConfigurationDHCP4() { + suite.Require().NoError( + suite.runtime.RegisterController( + &netctrl.OperatorConfigController{ + Cmdline: procfs.NewCmdline("talos.network.interface.ignore=eth5"), + }, + ), + ) + // add LinkConfig controller to produce link specs based on machine configuration + suite.Require().NoError( + suite.runtime.RegisterController( + &netctrl.LinkConfigController{ + Cmdline: procfs.NewCmdline("talos.network.interface.ignore=eth5"), + }, + ), + ) + + suite.startRuntime() + + for _, link := range []string{"eth0", "eth1", "eth2"} { + linkStatus := network.NewLinkStatus(network.NamespaceName, link) + linkStatus.TypedSpec().Type = nethelpers.LinkEther + linkStatus.TypedSpec().LinkState = true + + suite.Require().NoError(suite.state.Create(suite.ctx, linkStatus)) + } + + u, err := url.Parse("https://foo:6443") + suite.Require().NoError(err) + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineNetwork: &v1alpha1.NetworkConfig{ + NetworkInterfaces: []*v1alpha1.Device{ + { + DeviceInterface: "eth0", + }, + { + DeviceInterface: "eth1", + DeviceDHCP: pointer.To(true), + }, + { + DeviceIgnore: pointer.To(true), + DeviceInterface: "eth2", + DeviceDHCP: pointer.To(true), + }, + { + DeviceInterface: "eth3", + DeviceDHCP: pointer.To(true), + DeviceDHCPOptions: &v1alpha1.DHCPOptions{ + DHCPIPv4: pointer.To(true), + DHCPRouteMetric: 256, + }, + }, + { + DeviceInterface: "eth4", + DeviceVlans: []*v1alpha1.Vlan{ + { + VlanID: 25, + VlanDHCP: pointer.To(true), + }, + { + VlanID: 26, + }, + { + VlanID: 27, + VlanDHCPOptions: &v1alpha1.DHCPOptions{ + DHCPRouteMetric: 256, + }, + }, + }, + }, + { + DeviceInterface: "eth5", + DeviceDHCP: pointer.To(true), + }, + }, + }, + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: u, + }, + }, + }, + }, + ), + ) + + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + suite.assertOperators( + []string{ + "configuration/dhcp4/eth1", + "configuration/dhcp4/eth3", + "configuration/dhcp4/eth4.25", + }, func(r *network.OperatorSpec, asrt *assert.Assertions) { + asrt.Equal(network.OperatorDHCP4, r.TypedSpec().Operator) + asrt.True(r.TypedSpec().RequireUp) + + switch r.Metadata().ID() { + case "configuration/dhcp4/eth1": + asrt.Equal("eth1", r.TypedSpec().LinkName) + asrt.EqualValues(network.DefaultRouteMetric, r.TypedSpec().DHCP4.RouteMetric) + case "configuration/dhcp4/eth3": + asrt.Equal("eth3", r.TypedSpec().LinkName) + asrt.EqualValues(256, r.TypedSpec().DHCP4.RouteMetric) + case "configuration/dhcp4/eth4.25": + asrt.Equal("eth4.25", r.TypedSpec().LinkName) + asrt.EqualValues(network.DefaultRouteMetric, r.TypedSpec().DHCP4.RouteMetric) + case "configuration/dhcp4/eth4.26": + asrt.Equal("eth4.26", r.TypedSpec().LinkName) + asrt.EqualValues(network.DefaultRouteMetric, r.TypedSpec().DHCP4.RouteMetric) + case "configuration/dhcp4/eth4.27": + asrt.Equal("eth4.27", r.TypedSpec().LinkName) + asrt.EqualValues(256, r.TypedSpec().DHCP4.RouteMetric) + } + }, + ) + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertNoOperators( + []string{ + "configuration/dhcp4/eth0", + "default/dhcp4/eth0", + "configuration/dhcp4/eth2", + "default/dhcp4/eth2", + "configuration/dhcp4/eth4.26", + }, + ) + }, + ), + ) +} + +func (suite *OperatorConfigSuite) TestMachineConfigurationDHCP6() { + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.OperatorConfigController{})) + + suite.startRuntime() + + u, err := url.Parse("https://foo:6443") + suite.Require().NoError(err) + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineNetwork: &v1alpha1.NetworkConfig{ + NetworkInterfaces: []*v1alpha1.Device{ + { + DeviceInterface: "eth1", + DeviceDHCP: pointer.To(true), + DeviceDHCPOptions: &v1alpha1.DHCPOptions{ + DHCPIPv4: pointer.To(true), + }, + }, + { + DeviceInterface: "eth2", + DeviceDHCP: pointer.To(true), + DeviceDHCPOptions: &v1alpha1.DHCPOptions{ + DHCPIPv6: pointer.To(true), + }, + }, + { + DeviceInterface: "eth3", + DeviceDHCP: pointer.To(true), + DeviceDHCPOptions: &v1alpha1.DHCPOptions{ + DHCPIPv6: pointer.To(true), + DHCPRouteMetric: 512, + }, + }, + }, + }, + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: u, + }, + }, + }, + }, + ), + ) + + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + suite.assertOperators( + []string{ + "configuration/dhcp6/eth2", + "configuration/dhcp6/eth3", + }, func(r *network.OperatorSpec, asrt *assert.Assertions) { + asrt.Equal(network.OperatorDHCP6, r.TypedSpec().Operator) + asrt.True(r.TypedSpec().RequireUp) + + switch r.Metadata().ID() { + case "configuration/dhcp6/eth2": + asrt.Equal("eth2", r.TypedSpec().LinkName) + asrt.EqualValues(network.DefaultRouteMetric, r.TypedSpec().DHCP6.RouteMetric) + case "configuration/dhcp6/eth3": + asrt.Equal("eth3", r.TypedSpec().LinkName) + asrt.EqualValues(512, r.TypedSpec().DHCP6.RouteMetric) + } + }, + ) + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertNoOperators( + []string{ + "configuration/dhcp6/eth1", + }, + ) + }, + ), + ) +} + +func (suite *OperatorConfigSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestOperatorConfigSuite(t *testing.T) { + suite.Run(t, new(OperatorConfigSuite)) +} diff --git a/internal/app/machined/pkg/controllers/network/operator_merge.go b/internal/app/machined/pkg/controllers/network/operator_merge.go new file mode 100644 index 0000000..1df70e4 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/operator_merge.go @@ -0,0 +1,140 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package network provides controllers which manage network resources. +// +//nolint:dupl +package network + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// OperatorMergeController merges network.OperatorSpec in network.ConfigNamespace and produces final network.OperatorSpec in network.Namespace. +type OperatorMergeController struct{} + +// Name implements controller.Controller interface. +func (ctrl *OperatorMergeController) Name() string { + return "network.OperatorMergeController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *OperatorMergeController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: network.ConfigNamespaceName, + Type: network.OperatorSpecType, + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.OperatorSpecType, + Kind: controller.InputDestroyReady, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *OperatorMergeController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.OperatorSpecType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *OperatorMergeController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + // list source network configuration resources + list, err := r.List(ctx, resource.NewMetadata(network.ConfigNamespaceName, network.OperatorSpecType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing source network operators: %w", err) + } + + // operator is allowed as long as it's not duplicate, for duplicate higher layer takes precedence + operators := map[string]*network.OperatorSpec{} + + for _, res := range list.Items { + operator := res.(*network.OperatorSpec) //nolint:errcheck,forcetypeassert + id := network.OperatorID(operator.TypedSpec().Operator, operator.TypedSpec().LinkName) + + existing, ok := operators[id] + if ok && existing.TypedSpec().ConfigLayer > operator.TypedSpec().ConfigLayer { + // skip this operator, as existing one is higher layer + continue + } + + operators[id] = operator + } + + conflictsDetected := 0 + + for id, operator := range operators { + if err = r.Modify(ctx, network.NewOperatorSpec(network.NamespaceName, id), func(res resource.Resource) error { + op := res.(*network.OperatorSpec) //nolint:errcheck,forcetypeassert + + *op.TypedSpec() = *operator.TypedSpec() + + return nil + }); err != nil { + if state.IsPhaseConflictError(err) { + // phase conflict, resource is being torn down, skip updating it and trigger reconcile + // later by failing the loop after all processing is done + conflictsDetected++ + + delete(operators, id) + } else { + return fmt.Errorf("error updating resource: %w", err) + } + } + } + + // list operators for cleanup + list, err = r.List(ctx, resource.NewMetadata(network.NamespaceName, network.OperatorSpecType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing resources: %w", err) + } + + for _, res := range list.Items { + if _, ok := operators[res.Metadata().ID()]; !ok { + var okToDestroy bool + + okToDestroy, err = r.Teardown(ctx, res.Metadata()) + if err != nil { + return fmt.Errorf("error cleaning up operators: %w", err) + } + + if okToDestroy { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return fmt.Errorf("error cleaning up operators: %w", err) + } + } + } + } + + if conflictsDetected > 0 { + return fmt.Errorf("%d conflict(s) detected", conflictsDetected) + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/network/operator_merge_test.go b/internal/app/machined/pkg/controllers/network/operator_merge_test.go new file mode 100644 index 0000000..bd4142a --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/operator_merge_test.go @@ -0,0 +1,314 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:dupl +package network_test + +import ( + "context" + "log" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + "golang.org/x/sync/errgroup" + + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type OperatorMergeSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *OperatorMergeSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.OperatorMergeController{})) + + suite.startRuntime() +} + +func (suite *OperatorMergeSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *OperatorMergeSuite) assertOperators(requiredIDs []string, check func(*network.OperatorSpec) error) error { + missingIDs := make(map[string]struct{}, len(requiredIDs)) + + for _, id := range requiredIDs { + missingIDs[id] = struct{}{} + } + + resources, err := suite.state.List( + suite.ctx, + resource.NewMetadata(network.NamespaceName, network.OperatorSpecType, "", resource.VersionUndefined), + ) + if err != nil { + return err + } + + for _, res := range resources.Items { + _, required := missingIDs[res.Metadata().ID()] + if !required { + continue + } + + delete(missingIDs, res.Metadata().ID()) + + if err = check(res.(*network.OperatorSpec)); err != nil { + return retry.ExpectedError(err) + } + } + + if len(missingIDs) > 0 { + return retry.ExpectedErrorf("some resources are missing: %q", missingIDs) + } + + return nil +} + +func (suite *OperatorMergeSuite) assertNoOperator(id string) error { + resources, err := suite.state.List( + suite.ctx, + resource.NewMetadata(network.NamespaceName, network.OperatorSpecType, "", resource.VersionUndefined), + ) + if err != nil { + return err + } + + for _, res := range resources.Items { + if res.Metadata().ID() == id { + return retry.ExpectedErrorf("operator %q is still there", id) + } + } + + return nil +} + +func (suite *OperatorMergeSuite) TestMerge() { + dhcp1 := network.NewOperatorSpec(network.ConfigNamespaceName, "default/dhcp4/eth0") + *dhcp1.TypedSpec() = network.OperatorSpecSpec{ + Operator: network.OperatorDHCP4, + LinkName: "eth0", + ConfigLayer: network.ConfigDefault, + } + + dhcp2 := network.NewOperatorSpec(network.ConfigNamespaceName, "configuration/dhcp4/eth0") + *dhcp2.TypedSpec() = network.OperatorSpecSpec{ + Operator: network.OperatorDHCP4, + LinkName: "eth0", + RequireUp: true, + ConfigLayer: network.ConfigMachineConfiguration, + } + + dhcp6 := network.NewOperatorSpec(network.ConfigNamespaceName, "configuration/dhcp6/eth0") + *dhcp6.TypedSpec() = network.OperatorSpecSpec{ + Operator: network.OperatorDHCP6, + LinkName: "eth0", + RequireUp: true, + ConfigLayer: network.ConfigMachineConfiguration, + } + + for _, res := range []resource.Resource{dhcp1, dhcp2, dhcp6} { + suite.Require().NoError(suite.state.Create(suite.ctx, res), "%v", res.Spec()) + } + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertOperators( + []string{ + "dhcp4/eth0", + "dhcp6/eth0", + }, func(r *network.OperatorSpec) error { + switch r.Metadata().ID() { + case "dhcp4/eth0": + suite.Assert().Equal(*dhcp2.TypedSpec(), *r.TypedSpec()) + case "dhcp6/eth0": + suite.Assert().Equal(*dhcp6.TypedSpec(), *r.TypedSpec()) + } + + return nil + }, + ) + }, + ), + ) + + suite.Require().NoError(suite.state.Destroy(suite.ctx, dhcp6.Metadata())) + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertOperators( + []string{ + "dhcp4/eth0", + }, func(r *network.OperatorSpec) error { + return nil + }, + ) + }, + ), + ) + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertNoOperator("dhcp6/eth0") + }, + ), + ) +} + +//nolint:gocyclo +func (suite *OperatorMergeSuite) TestMergeFlapping() { + // simulate two conflicting operator definitions which are getting removed/added constantly + dhcp := network.NewOperatorSpec(network.ConfigNamespaceName, "default/dhcp4/eth0") + *dhcp.TypedSpec() = network.OperatorSpecSpec{ + Operator: network.OperatorDHCP4, + LinkName: "eth0", + ConfigLayer: network.ConfigDefault, + } + + override := network.NewOperatorSpec(network.ConfigNamespaceName, "configuration/dhcp4/eth0") + *override.TypedSpec() = network.OperatorSpecSpec{ + Operator: network.OperatorDHCP4, + LinkName: "eth0", + RequireUp: true, + ConfigLayer: network.ConfigMachineConfiguration, + } + + resources := []resource.Resource{dhcp, override} + + flipflop := func(idx int) func() error { + return func() error { + for range 500 { + if err := suite.state.Create(suite.ctx, resources[idx]); err != nil { + return err + } + + if err := suite.state.Destroy(suite.ctx, resources[idx].Metadata()); err != nil { + return err + } + + time.Sleep(time.Millisecond) + } + + return suite.state.Create(suite.ctx, resources[idx]) + } + } + + var eg errgroup.Group + + eg.Go(flipflop(0)) + eg.Go(flipflop(1)) + eg.Go( + func() error { + // add/remove finalizer to the merged resource + for range 1000 { + if err := suite.state.AddFinalizer( + suite.ctx, + resource.NewMetadata( + network.NamespaceName, + network.OperatorSpecType, + "dhcp4/eth0", + resource.VersionUndefined, + ), + "foo", + ); err != nil { + if !state.IsNotFoundError(err) { + return err + } + + continue + } + + suite.T().Log("finalizer added") + + time.Sleep(10 * time.Millisecond) + + if err := suite.state.RemoveFinalizer( + suite.ctx, + resource.NewMetadata( + network.NamespaceName, + network.OperatorSpecType, + "dhcp4/eth0", + resource.VersionUndefined, + ), + "foo", + ); err != nil && !state.IsNotFoundError(err) { + return err + } + } + + return nil + }, + ) + + suite.Require().NoError(eg.Wait()) + + suite.Assert().NoError( + retry.Constant(15*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertOperators( + []string{ + "dhcp4/eth0", + }, func(r *network.OperatorSpec) error { + if r.Metadata().Phase() != resource.PhaseRunning { + return retry.ExpectedErrorf("resource phase is %s", r.Metadata().Phase()) + } + + if *override.TypedSpec() != *r.TypedSpec() { + // using retry here, as it might not be reconciled immediately + return retry.ExpectedErrorf("not equal yet") + } + + return nil + }, + ) + }, + ), + ) +} + +func (suite *OperatorMergeSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestOperatorMergeSuite(t *testing.T) { + suite.Run(t, new(OperatorMergeSuite)) +} diff --git a/internal/app/machined/pkg/controllers/network/operator_spec.go b/internal/app/machined/pkg/controllers/network/operator_spec.go new file mode 100644 index 0000000..305732c --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/operator_spec.go @@ -0,0 +1,429 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/cenkalti/backoff/v4" + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "go.uber.org/zap" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network/operator" + v1alpha1runtime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// OperatorSpecController applies network.OperatorSpec to the actual interfaces. +type OperatorSpecController struct { + V1alpha1Platform v1alpha1runtime.Platform + State state.State + + // Factory can be overridden for unit-testing. + Factory OperatorFactory + + operators map[string]*operatorRunState +} + +// Name implements controller.Controller interface. +func (ctrl *OperatorSpecController) Name() string { + return "network.OperatorSpecController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *OperatorSpecController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: network.NamespaceName, + Type: network.OperatorSpecType, + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.LinkStatusType, + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *OperatorSpecController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.AddressSpecType, + Kind: controller.OutputShared, + }, + { + Type: network.LinkSpecType, + Kind: controller.OutputShared, + }, + { + Type: network.RouteSpecType, + Kind: controller.OutputShared, + }, + { + Type: network.HostnameSpecType, + Kind: controller.OutputShared, + }, + { + Type: network.ResolverSpecType, + Kind: controller.OutputShared, + }, + { + Type: network.TimeServerSpecType, + Kind: controller.OutputShared, + }, + } +} + +// operatorRunState describes a state of running operator. +type operatorRunState struct { + Operator operator.Operator + Spec network.OperatorSpecSpec + + cancel context.CancelFunc + wg sync.WaitGroup +} + +func (state *operatorRunState) Start(ctx context.Context, notifyCh chan<- struct{}, logger *zap.Logger, id string) { + state.wg.Add(1) + + ctx, state.cancel = context.WithCancel(ctx) + + go func() { + defer state.wg.Done() + + state.runWithRestarts(ctx, notifyCh, logger, id) + }() +} + +func (state *operatorRunState) runWithRestarts(ctx context.Context, notifyCh chan<- struct{}, logger *zap.Logger, id string) { + backoff := backoff.NewExponentialBackOff() + + // disable number of retries limit + backoff.MaxElapsedTime = 0 + + for ctx.Err() == nil { + if err := state.runWithPanicHandler(ctx, notifyCh, logger, id); err == nil { + // operator finished without an error + return + } + + interval := backoff.NextBackOff() + + logger.Debug("restarting operator", zap.Duration("interval", interval), zap.String("operator", id)) + + select { + case <-ctx.Done(): + return + case <-time.After(interval): + } + } +} + +func (state *operatorRunState) runWithPanicHandler(ctx context.Context, notifyCh chan<- struct{}, logger *zap.Logger, id string) (err error) { + defer func() { + if p := recover(); p != nil { + err = fmt.Errorf("panic: %v", p) + + logger.Error("operator panicked", zap.Stack("stack"), zap.Error(err), zap.String("operator", id)) + } + }() + + state.Operator.Run(ctx, notifyCh) + + return nil +} + +func (state *operatorRunState) Stop() { + state.cancel() + + state.wg.Wait() +} + +// Run implements controller.Controller interface. +func (ctrl *OperatorSpecController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + notifyCh := make(chan struct{}) + + ctrl.operators = make(map[string]*operatorRunState) + + defer func() { + for _, operator := range ctrl.operators { + operator.Stop() + } + }() + + if ctrl.Factory == nil { + ctrl.Factory = ctrl.newOperator + } + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + if err := ctrl.reconcileOperators(ctx, r, logger, notifyCh); err != nil { + return err + } + case <-notifyCh: + if err := ctrl.reconcileOperatorOutputs(ctx, r); err != nil { + return err + } + } + + r.ResetRestartBackoff() + } +} + +//nolint:gocyclo +func (ctrl *OperatorSpecController) reconcileOperators(ctx context.Context, r controller.Runtime, logger *zap.Logger, notifyCh chan<- struct{}) error { + // build link up statuses + linkStatuses := make(map[string]bool) + + list, err := r.List(ctx, resource.NewMetadata(network.NamespaceName, network.LinkStatusType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing source network addresses: %w", err) + } + + for _, item := range list.Items { + linkStatus := item.(*network.LinkStatus) //nolint:errcheck,forcetypeassert + + linkStatuses[linkStatus.Metadata().ID()] = linkStatus.TypedSpec().OperationalState == nethelpers.OperStateUnknown || linkStatus.TypedSpec().OperationalState == nethelpers.OperStateUp + } + + // list operator specs + list, err = r.List(ctx, resource.NewMetadata(network.NamespaceName, network.OperatorSpecType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing source network addresses: %w", err) + } + + // figure out which operators should run + shouldRun := make(map[string]*network.OperatorSpecSpec) + + for _, item := range list.Items { + operatorSpec := item.(*network.OperatorSpec) //nolint:errcheck,forcetypeassert + + up, exists := linkStatuses[operatorSpec.TypedSpec().LinkName] + + // link doesn't exist, skip operator + if !exists { + continue + } + + // link is down and operator requires link to be up, skip it + if operatorSpec.TypedSpec().RequireUp && !up { + continue + } + + shouldRun[operatorSpec.Metadata().ID()] = operatorSpec.TypedSpec() + } + + // stop running operators which shouldn't run + for id := range ctrl.operators { + if _, exists := shouldRun[id]; !exists { + logger.Debug("stopping operator", zap.String("operator", id)) + + // stop operator + ctrl.operators[id].Stop() + delete(ctrl.operators, id) + } else if !ctrl.operators[id].Spec.Equal(*shouldRun[id]) { + logger.Debug("replacing operator", zap.String("operator", id)) + + // stop operator + ctrl.operators[id].Stop() + delete(ctrl.operators, id) + } + } + + // start operators which aren't running + for id := range shouldRun { + if _, exists := ctrl.operators[id]; !exists { + ctrl.operators[id] = &operatorRunState{ + Operator: ctrl.Factory(logger, shouldRun[id]), + Spec: *shouldRun[id], + } + + logger.Debug("starting operator", zap.String("operator", id)) + ctrl.operators[id].Start(ctx, notifyCh, logger, id) + } + } + + // now reconcile outputs as the operators might have changed + return ctrl.reconcileOperatorOutputs(ctx, r) +} + +//nolint:gocyclo,cyclop +func (ctrl *OperatorSpecController) reconcileOperatorOutputs(ctx context.Context, r controller.Runtime) error { + // query specs from all operators and update outputs + touchedIDs := map[string]map[string]struct{}{} + + apply := func(res resource.Resource, fn func(resource.Resource)) error { + if touchedIDs[res.Metadata().Type()] == nil { + touchedIDs[res.Metadata().Type()] = map[string]struct{}{} + } + + touchedIDs[res.Metadata().Type()][res.Metadata().ID()] = struct{}{} + + return r.Modify(ctx, res, func(r resource.Resource) error { + fn(r) + + return nil + }) + } + + for _, op := range ctrl.operators { + for _, addressSpec := range op.Operator.AddressSpecs() { + if err := apply( + network.NewAddressSpec( + network.ConfigNamespaceName, + fmt.Sprintf("%s/%s", op.Operator.Prefix(), network.AddressID(addressSpec.LinkName, addressSpec.Address)), + ), + func(r resource.Resource) { + *r.(*network.AddressSpec).TypedSpec() = addressSpec + }, + ); err != nil { + return fmt.Errorf("error applying spec: %w", err) + } + } + + for _, routeSpec := range op.Operator.RouteSpecs() { + if err := apply( + network.NewRouteSpec( + network.ConfigNamespaceName, + fmt.Sprintf("%s/%s", + op.Operator.Prefix(), + network.RouteID(routeSpec.Table, routeSpec.Family, routeSpec.Destination, routeSpec.Gateway, routeSpec.Priority, routeSpec.OutLinkName), + ), + ), + func(r resource.Resource) { + *r.(*network.RouteSpec).TypedSpec() = routeSpec + }, + ); err != nil { + return fmt.Errorf("error applying spec: %w", err) + } + } + + for _, linkSpec := range op.Operator.LinkSpecs() { + if err := apply( + network.NewLinkSpec( + network.ConfigNamespaceName, + fmt.Sprintf("%s/%s", op.Operator.Prefix(), network.LinkID(linkSpec.Name)), + ), + func(r resource.Resource) { + *r.(*network.LinkSpec).TypedSpec() = linkSpec + }, + ); err != nil { + return fmt.Errorf("error applying spec: %w", err) + } + } + + for _, hostnameSpec := range op.Operator.HostnameSpecs() { + if err := apply( + network.NewHostnameSpec( + network.ConfigNamespaceName, + fmt.Sprintf("%s/%s", op.Operator.Prefix(), network.HostnameID), + ), + func(r resource.Resource) { + *r.(*network.HostnameSpec).TypedSpec() = hostnameSpec + }, + ); err != nil { + return fmt.Errorf("error applying spec: %w", err) + } + } + + for _, resolverSpec := range op.Operator.ResolverSpecs() { + if err := apply( + network.NewResolverSpec( + network.ConfigNamespaceName, + fmt.Sprintf("%s/%s", op.Operator.Prefix(), network.ResolverID), + ), + func(r resource.Resource) { + *r.(*network.ResolverSpec).TypedSpec() = resolverSpec + }, + ); err != nil { + return fmt.Errorf("error applying spec: %w", err) + } + } + + for _, timeserverSpec := range op.Operator.TimeServerSpecs() { + if err := apply( + network.NewTimeServerSpec( + network.ConfigNamespaceName, + fmt.Sprintf("%s/%s", op.Operator.Prefix(), network.TimeServerID), + ), + func(r resource.Resource) { + *r.(*network.TimeServerSpec).TypedSpec() = timeserverSpec + }, + ); err != nil { + return fmt.Errorf("error applying spec: %w", err) + } + } + } + + // clean up not touched specs + for _, resourceType := range []resource.Type{ + network.AddressSpecType, + network.LinkSpecType, + network.RouteSpecType, + network.HostnameSpecType, + network.ResolverSpecType, + network.TimeServerSpecType, + } { + list, err := r.List(ctx, resource.NewMetadata(network.ConfigNamespaceName, resourceType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing specs: %w", err) + } + + for _, item := range list.Items { + if item.Metadata().Owner() != ctrl.Name() { + continue + } + + touched := false + + if touchedIDs[resourceType] != nil { + if _, exists := touchedIDs[resourceType][item.Metadata().ID()]; exists { + touched = true + } + } + + if !touched { + if err = r.Destroy(ctx, item.Metadata()); err != nil { + return fmt.Errorf("error cleaning up untouched spec: %w", err) + } + } + } + } + + return nil +} + +// OperatorFactory creates operator based on the spec. +type OperatorFactory func(*zap.Logger, *network.OperatorSpecSpec) operator.Operator + +func (ctrl *OperatorSpecController) newOperator(logger *zap.Logger, spec *network.OperatorSpecSpec) operator.Operator { + switch spec.Operator { + case network.OperatorDHCP4: + logger = logger.With(zap.String("operator", "dhcp4")) + + return operator.NewDHCP4(logger, spec.LinkName, spec.DHCP4, ctrl.V1alpha1Platform, ctrl.State) + case network.OperatorDHCP6: + logger = logger.With(zap.String("operator", "dhcp6")) + + return operator.NewDHCP6(logger, spec.LinkName, spec.DHCP6) + case network.OperatorVIP: + logger = logger.With(zap.String("operator", "vip")) + + return operator.NewVIP(logger, spec.LinkName, spec.VIP, ctrl.State) + default: + panic(fmt.Sprintf("unexpected operator %s", spec.Operator)) + } +} diff --git a/internal/app/machined/pkg/controllers/network/operator_spec_test.go b/internal/app/machined/pkg/controllers/network/operator_spec_test.go new file mode 100644 index 0000000..60cfd08 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/operator_spec_test.go @@ -0,0 +1,553 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network_test + +import ( + "context" + "fmt" + "log" + "net/netip" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + "go.uber.org/zap" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network/operator" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type OperatorSpecSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *OperatorSpecSuite) State() state.State { return suite.state } + +func (suite *OperatorSpecSuite) Ctx() context.Context { return suite.ctx } + +type mockOperator struct { + spec network.OperatorSpecSpec + notifyCh chan<- struct{} + panicked bool + + mu sync.Mutex + addresses []network.AddressSpecSpec + links []network.LinkSpecSpec + routes []network.RouteSpecSpec + hostname []network.HostnameSpecSpec + resolvers []network.ResolverSpecSpec + timeservers []network.TimeServerSpecSpec +} + +var ( + runningOperators = map[string]*mockOperator{} + runningOperatorsMu sync.Mutex +) + +func (mock *mockOperator) Prefix() string { + return fmt.Sprintf("%s/%s", mock.spec.Operator, mock.spec.LinkName) +} + +func (mock *mockOperator) Run(ctx context.Context, notifyCh chan<- struct{}) { + mock.notifyCh = notifyCh + + { + runningOperatorsMu.Lock() + runningOperators[mock.Prefix()] = mock + runningOperatorsMu.Unlock() + } + + defer func() { + runningOperatorsMu.Lock() + delete(runningOperators, mock.Prefix()) + runningOperatorsMu.Unlock() + }() + + if mock.spec.Operator == network.OperatorDHCP6 { + // DHCP6 operator panics on odd run + if !mock.panicked { + mock.panicked = true + + panic("oh no, IPv6!!!") + } + } + + <-ctx.Done() +} + +func (mock *mockOperator) notify() { + mock.notifyCh <- struct{}{} +} + +func (mock *mockOperator) AddressSpecs() []network.AddressSpecSpec { + mock.mu.Lock() + defer mock.mu.Unlock() + + return mock.addresses +} + +func (mock *mockOperator) LinkSpecs() []network.LinkSpecSpec { + mock.mu.Lock() + defer mock.mu.Unlock() + + return mock.links +} + +func (mock *mockOperator) RouteSpecs() []network.RouteSpecSpec { + mock.mu.Lock() + defer mock.mu.Unlock() + + return mock.routes +} + +func (mock *mockOperator) HostnameSpecs() []network.HostnameSpecSpec { + mock.mu.Lock() + defer mock.mu.Unlock() + + return mock.hostname +} + +func (mock *mockOperator) ResolverSpecs() []network.ResolverSpecSpec { + mock.mu.Lock() + defer mock.mu.Unlock() + + return mock.resolvers +} + +func (mock *mockOperator) TimeServerSpecs() []network.TimeServerSpecSpec { + mock.mu.Lock() + defer mock.mu.Unlock() + + return mock.timeservers +} + +func (suite *OperatorSpecSuite) newOperator(logger *zap.Logger, spec *network.OperatorSpecSpec) operator.Operator { + return &mockOperator{ + spec: *spec, + } +} + +func (suite *OperatorSpecSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + runningOperators = map[string]*mockOperator{} + + suite.Require().NoError( + suite.runtime.RegisterController( + &netctrl.OperatorSpecController{ + Factory: suite.newOperator, + }, + ), + ) + + suite.startRuntime() +} + +func (suite *OperatorSpecSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *OperatorSpecSuite) assertRunning(runningIDs []string, assertFunc func(*mockOperator) error) error { + runningOperatorsMu.Lock() + defer runningOperatorsMu.Unlock() + + for _, id := range runningIDs { + op, exists := runningOperators[id] + + if !exists { + return retry.ExpectedErrorf("operator %q is not running", id) + } + + if err := assertFunc(op); err != nil { + return retry.ExpectedError(err) + } + } + + for id := range runningOperators { + found := false + + for _, expectedID := range runningIDs { + if expectedID == id { + found = true + + break + } + } + + if !found { + return retry.ExpectedErrorf("operator %s should not be running", id) + } + } + + return nil +} + +func (suite *OperatorSpecSuite) assertResources(resourceType resource.Type, requiredIDs []string) error { + missingIDs := make(map[string]struct{}, len(requiredIDs)) + + for _, id := range requiredIDs { + missingIDs[id] = struct{}{} + } + + resources, err := suite.state.List( + suite.ctx, + resource.NewMetadata(network.ConfigNamespaceName, resourceType, "", resource.VersionUndefined), + ) + if err != nil { + return err + } + + for _, res := range resources.Items { + delete(missingIDs, res.Metadata().ID()) + } + + if len(missingIDs) > 0 { + return retry.ExpectedErrorf("some resources are missing: %q", missingIDs) + } + + return nil +} + +func (suite *OperatorSpecSuite) TestScheduling() { + specDHCP := network.NewOperatorSpec(network.NamespaceName, "dhcp4/eth0") + *specDHCP.TypedSpec() = network.OperatorSpecSpec{ + Operator: network.OperatorDHCP4, + LinkName: "eth0", + RequireUp: true, + DHCP4: network.DHCP4OperatorSpec{ + RouteMetric: 1024, + }, + } + + specVIP := network.NewOperatorSpec(network.NamespaceName, "vip/eth0") + *specVIP.TypedSpec() = network.OperatorSpecSpec{ + Operator: network.OperatorVIP, + LinkName: "eth0", + RequireUp: false, + VIP: network.VIPOperatorSpec{ + IP: netip.MustParseAddr("1.2.3.4"), + }, + } + + suite.Require().NoError(suite.state.Create(suite.ctx, specDHCP)) + suite.Require().NoError(suite.state.Create(suite.ctx, specVIP)) + + // operators shouldn't be running yet, as link state is not known yet + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertRunning( + nil, func(op *mockOperator) error { + return nil + }, + ) + }, + ), + ) + + linkState := network.NewLinkStatus(network.NamespaceName, "eth0") + *linkState.TypedSpec() = network.LinkStatusSpec{ + OperationalState: nethelpers.OperStateDown, + } + + suite.Require().NoError(suite.state.Create(suite.ctx, linkState)) + + // vip operator should be scheduled now, as VIP operator doesn't require link to be up + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertRunning( + []string{"vip/eth0"}, func(op *mockOperator) error { + suite.Assert().Equal(netip.MustParseAddr("1.2.3.4"), op.spec.VIP.IP) + + return nil + }, + ) + }, + ), + ) + + ctest.UpdateWithConflicts(suite, linkState, func(r *network.LinkStatus) error { + r.TypedSpec().OperationalState = nethelpers.OperStateUp + + return nil + }) + + // now all operators should be scheduled + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertRunning( + []string{"dhcp4/eth0", "vip/eth0"}, func(op *mockOperator) error { + switch op.spec.Operator { //nolint:exhaustive + case network.OperatorDHCP4: + suite.Assert().EqualValues(1024, op.spec.DHCP4.RouteMetric) + case network.OperatorVIP: + suite.Assert().Equal(netip.MustParseAddr("1.2.3.4"), op.spec.VIP.IP) + default: + panic("unreachable") + } + + return nil + }, + ) + }, + ), + ) + + // change the spec, operator should be rescheduled + ctest.UpdateWithConflicts(suite, specVIP, func(r *network.OperatorSpec) error { + r.TypedSpec().VIP.IP = netip.MustParseAddr("3.4.5.6") + + return nil + }) + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertRunning( + []string{"dhcp4/eth0", "vip/eth0"}, func(op *mockOperator) error { + switch op.spec.Operator { //nolint:exhaustive + case network.OperatorDHCP4: + suite.Assert().EqualValues(1024, op.spec.DHCP4.RouteMetric) + case network.OperatorVIP: + if op.spec.VIP.IP.Compare(netip.MustParseAddr("3.4.5.6")) != 0 { + return retry.ExpectedErrorf("unexpected vip: %s", op.spec.VIP.IP) + } + default: + panic("unreachable") + } + + return nil + }, + ) + }, + ), + ) + + // bring down the interface, operator should be stopped + ctest.UpdateWithConflicts(suite, linkState, func(r *network.LinkStatus) error { + r.TypedSpec().OperationalState = nethelpers.OperStateDown + + return nil + }) + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertRunning( + []string{"vip/eth0"}, func(op *mockOperator) error { + return nil + }, + ) + }, + ), + ) +} + +func (suite *OperatorSpecSuite) TestPanic() { + specPanic := network.NewOperatorSpec(network.NamespaceName, "dhcp6/eth0") + *specPanic.TypedSpec() = network.OperatorSpecSpec{ + Operator: network.OperatorDHCP6, + LinkName: "eth0", + RequireUp: true, + DHCP6: network.DHCP6OperatorSpec{ + RouteMetric: 1024, + }, + } + + suite.Require().NoError(suite.state.Create(suite.ctx, specPanic)) + + linkState := network.NewLinkStatus(network.NamespaceName, "eth0") + *linkState.TypedSpec() = network.LinkStatusSpec{ + OperationalState: nethelpers.OperStateUp, + } + + suite.Require().NoError(suite.state.Create(suite.ctx, linkState)) + + // DHCP6 operator should panic and then restart + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertRunning([]string{"dhcp6/eth0"}, func(op *mockOperator) error { return nil }) + }, + ), + ) + + // bring down the interface, operator should be stopped + ctest.UpdateWithConflicts(suite, linkState, func(r *network.LinkStatus) error { + r.TypedSpec().OperationalState = nethelpers.OperStateDown + + return nil + }) + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertRunning( + nil, func(op *mockOperator) error { + return nil + }, + ) + }, + ), + ) +} + +func (suite *OperatorSpecSuite) TestOperatorOutputs() { + specDHCP := network.NewOperatorSpec(network.NamespaceName, "dhcp4/eth0") + *specDHCP.TypedSpec() = network.OperatorSpecSpec{ + Operator: network.OperatorDHCP4, + LinkName: "eth0", + RequireUp: true, + DHCP4: network.DHCP4OperatorSpec{ + RouteMetric: 1024, + }, + } + + suite.Require().NoError(suite.state.Create(suite.ctx, specDHCP)) + + linkState := network.NewLinkStatus(network.NamespaceName, "eth0") + *linkState.TypedSpec() = network.LinkStatusSpec{ + OperationalState: nethelpers.OperStateUp, + } + + suite.Require().NoError(suite.state.Create(suite.ctx, linkState)) + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertRunning( + []string{"dhcp4/eth0"}, func(op *mockOperator) error { + return nil + }, + ) + }, + ), + ) + + // pretend dhcp has some specs ready + runningOperatorsMu.Lock() + dhcpMock := runningOperators["dhcp4/eth0"] + runningOperatorsMu.Unlock() + + dhcpMock.mu.Lock() + dhcpMock.addresses = []network.AddressSpecSpec{ + { + Address: netip.MustParsePrefix("10.5.0.2/24"), + LinkName: "eth0", + Family: nethelpers.FamilyInet4, + Scope: nethelpers.ScopeGlobal, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + ConfigLayer: network.ConfigOperator, + }, + } + dhcpMock.links = []network.LinkSpecSpec{ + { + Name: "eth0", + Up: true, + ConfigLayer: network.ConfigOperator, + }, + } + dhcpMock.hostname = []network.HostnameSpecSpec{ + { + Hostname: "foo", + ConfigLayer: network.ConfigOperator, + }, + } + dhcpMock.mu.Unlock() + + dhcpMock.notify() + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertResources(network.AddressSpecType, []string{"dhcp4/eth0/eth0/10.5.0.2/24"}) + }, + ), + ) + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertResources(network.LinkSpecType, []string{"dhcp4/eth0/eth0"}) + }, + ), + ) + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertResources(network.HostnameSpecType, []string{"dhcp4/eth0/hostname"}) + }, + ), + ) + + // update specs + dhcpMock.mu.Lock() + dhcpMock.addresses = []network.AddressSpecSpec{ + { + Address: netip.MustParsePrefix("10.5.0.3/24"), + LinkName: "eth0", + Family: nethelpers.FamilyInet4, + Scope: nethelpers.ScopeGlobal, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + ConfigLayer: network.ConfigOperator, + }, + } + dhcpMock.mu.Unlock() + + dhcpMock.notify() + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertResources(network.AddressSpecType, []string{"dhcp4/eth0/eth0/10.5.0.3/24"}) + }, + ), + ) +} + +func (suite *OperatorSpecSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestOperatorSpecSuite(t *testing.T) { + suite.Run(t, new(OperatorSpecSuite)) +} diff --git a/internal/app/machined/pkg/controllers/network/operator_vip_config.go b/internal/app/machined/pkg/controllers/network/operator_vip_config.go new file mode 100644 index 0000000..6931107 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/operator_vip_config.go @@ -0,0 +1,240 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "context" + "fmt" + "net/netip" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/hashicorp/go-multierror" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/go-procfs/procfs" + "go.uber.org/zap" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network/operator/vip" + talosconfig "github.com/siderolabs/talos/pkg/machinery/config/config" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// OperatorVIPConfigController manages network.OperatorSpec for virtual IPs based on machine configuration. +type OperatorVIPConfigController struct { + Cmdline *procfs.Cmdline +} + +// Name implements controller.Controller interface. +func (ctrl *OperatorVIPConfigController) Name() string { + return "network.OperatorVIPConfigController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *OperatorVIPConfigController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: network.NamespaceName, + Type: network.DeviceConfigSpecType, + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *OperatorVIPConfigController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.OperatorSpecType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *OperatorVIPConfigController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + touchedIDs := make(map[resource.ID]struct{}) + + items, err := r.List(ctx, resource.NewMetadata(network.NamespaceName, network.DeviceConfigSpecType, "", resource.VersionUndefined)) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting config: %w", err) + } + } + + devices := xslices.Map(items.Items, func(item resource.Resource) talosconfig.Device { + return item.(*network.DeviceConfigSpec).TypedSpec().Device + }) + + ignoredInterfaces := map[string]struct{}{} + + if ctrl.Cmdline != nil { + var settings CmdlineNetworking + + settings, err = ParseCmdlineNetwork(ctrl.Cmdline) + if err != nil { + logger.Warn("ignored cmdline parse failure", zap.Error(err)) + } + + for _, link := range settings.IgnoreInterfaces { + ignoredInterfaces[link] = struct{}{} + } + } + + var ( + specs []network.OperatorSpecSpec + specErrors *multierror.Error + ) + + // operators from the config + if len(devices) > 0 { + for _, device := range devices { + if device.Ignore() { + ignoredInterfaces[device.Interface()] = struct{}{} + } + + if _, ignore := ignoredInterfaces[device.Interface()]; ignore { + continue + } + + if device.VIPConfig() != nil { + if spec, specErr := handleVIP(ctx, device.VIPConfig(), device.Interface(), logger); specErr != nil { + specErrors = multierror.Append(specErrors, specErr) + } else { + specs = append(specs, spec) + } + } + + for _, vlan := range device.Vlans() { + if vlan.VIPConfig() != nil { + linkName := nethelpers.VLANLinkName(device.Interface(), vlan.ID()) + if spec, specErr := handleVIP(ctx, vlan.VIPConfig(), linkName, logger); specErr != nil { + specErrors = multierror.Append(specErrors, specErr) + } else { + specs = append(specs, spec) + } + } + } + } + } + + var ids []string + + ids, err = ctrl.apply(ctx, r, specs) + if err != nil { + return fmt.Errorf("error applying operator specs: %w", err) + } + + for _, id := range ids { + touchedIDs[id] = struct{}{} + } + + // list specs for cleanup + list, err := r.List(ctx, resource.NewMetadata(network.ConfigNamespaceName, network.OperatorSpecType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing resources: %w", err) + } + + for _, res := range list.Items { + if res.Metadata().Owner() != ctrl.Name() { + // skip specs created by other controllers + continue + } + + if _, ok := touchedIDs[res.Metadata().ID()]; !ok { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return fmt.Errorf("error cleaning up routes: %w", err) + } + } + } + + // last, check if some specs failed to build; fail last so that other operator specs are applied successfully + if err = specErrors.ErrorOrNil(); err != nil { + return err + } + + r.ResetRestartBackoff() + } +} + +//nolint:dupl +func (ctrl *OperatorVIPConfigController) apply(ctx context.Context, r controller.Runtime, specs []network.OperatorSpecSpec) ([]resource.ID, error) { + ids := make([]string, 0, len(specs)) + + for _, spec := range specs { + id := network.LayeredID(spec.ConfigLayer, network.OperatorID(spec.Operator, spec.LinkName)) + + if err := r.Modify( + ctx, + network.NewOperatorSpec(network.ConfigNamespaceName, id), + func(r resource.Resource) error { + *r.(*network.OperatorSpec).TypedSpec() = spec + + return nil + }, + ); err != nil { + return ids, err + } + + ids = append(ids, id) + } + + return ids, nil +} + +func handleVIP(ctx context.Context, vlanConfig talosconfig.VIPConfig, deviceName string, logger *zap.Logger) (network.OperatorSpecSpec, error) { + var sharedIP netip.Addr + + sharedIP, err := netip.ParseAddr(vlanConfig.IP()) + if err != nil { + logger.Warn("ignoring vip parse failure", zap.Error(err), zap.String("link", deviceName)) + + return network.OperatorSpecSpec{}, err + } + + spec := network.OperatorSpecSpec{ + Operator: network.OperatorVIP, + LinkName: deviceName, + RequireUp: true, + VIP: network.VIPOperatorSpec{ + IP: sharedIP, + GratuitousARP: true, + }, + ConfigLayer: network.ConfigMachineConfiguration, + } + + switch { + // Equinix Metal VIP + case vlanConfig.EquinixMetal() != nil: + spec.VIP.GratuitousARP = false + spec.VIP.EquinixMetal.APIToken = vlanConfig.EquinixMetal().APIToken() + + if err = vip.GetProjectAndDeviceIDs(ctx, &spec.VIP.EquinixMetal); err != nil { + return network.OperatorSpecSpec{}, err + } + // Hetzner Cloud VIP + case vlanConfig.HCloud() != nil: + spec.VIP.GratuitousARP = false + spec.VIP.HCloud.APIToken = vlanConfig.HCloud().APIToken() + + if err = vip.GetNetworkAndDeviceIDs(ctx, &spec.VIP.HCloud, sharedIP); err != nil { + return network.OperatorSpecSpec{}, err + } + // Regular layer 2 VIP + default: + } + + return spec, nil +} diff --git a/internal/app/machined/pkg/controllers/network/operator_vip_config_test.go b/internal/app/machined/pkg/controllers/network/operator_vip_config_test.go new file mode 100644 index 0000000..295be3c --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/operator_vip_config_test.go @@ -0,0 +1,169 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network_test + +import ( + "context" + "log" + "net/netip" + "net/url" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-pointer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type OperatorVIPConfigSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *OperatorVIPConfigSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.DeviceConfigController{})) +} + +func (suite *OperatorVIPConfigSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *OperatorVIPConfigSuite) assertOperators( + requiredIDs []string, + check func(*network.OperatorSpec, *assert.Assertions), +) { + assertResources(suite.ctx, suite.T(), suite.state, requiredIDs, check, rtestutils.WithNamespace(network.ConfigNamespaceName)) +} + +func (suite *OperatorVIPConfigSuite) TestMachineConfigurationVIP() { + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.OperatorVIPConfigController{})) + + suite.startRuntime() + + u, err := url.Parse("https://foo:6443") + suite.Require().NoError(err) + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineNetwork: &v1alpha1.NetworkConfig{ + NetworkInterfaces: []*v1alpha1.Device{ + { + DeviceInterface: "eth1", + DeviceDHCP: pointer.To(true), + DeviceVIPConfig: &v1alpha1.DeviceVIPConfig{ + SharedIP: "2.3.4.5", + }, + }, + { + DeviceInterface: "eth2", + DeviceDHCP: pointer.To(true), + DeviceVIPConfig: &v1alpha1.DeviceVIPConfig{ + SharedIP: "fd7a:115c:a1e0:ab12:4843:cd96:6277:2302", + }, + }, + { + DeviceInterface: "eth3", + DeviceDHCP: pointer.To(true), + DeviceVlans: []*v1alpha1.Vlan{ + { + VlanID: 26, + VlanVIP: &v1alpha1.DeviceVIPConfig{ + SharedIP: "5.5.4.4", + }, + }, + }, + }, + }, + }, + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: u, + }, + }, + }, + }, + ), + ) + + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + suite.assertOperators( + []string{ + "configuration/vip/eth1", + "configuration/vip/eth2", + "configuration/vip/eth3.26", + }, func(r *network.OperatorSpec, asrt *assert.Assertions) { + asrt.Equal(network.OperatorVIP, r.TypedSpec().Operator) + asrt.True(r.TypedSpec().RequireUp) + + switch r.Metadata().ID() { + case "configuration/vip/eth1": + asrt.Equal("eth1", r.TypedSpec().LinkName) + asrt.EqualValues(netip.MustParseAddr("2.3.4.5"), r.TypedSpec().VIP.IP) + case "configuration/vip/eth2": + asrt.Equal("eth2", r.TypedSpec().LinkName) + asrt.EqualValues( + netip.MustParseAddr("fd7a:115c:a1e0:ab12:4843:cd96:6277:2302"), + r.TypedSpec().VIP.IP, + ) + case "configuration/vip/eth3.26": + asrt.Equal("eth3.26", r.TypedSpec().LinkName) + asrt.EqualValues(netip.MustParseAddr("5.5.4.4"), r.TypedSpec().VIP.IP) + } + }, + ) +} + +func (suite *OperatorVIPConfigSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestOperatorVIPConfigSuite(t *testing.T) { + suite.Run(t, new(OperatorVIPConfigSuite)) +} diff --git a/internal/app/machined/pkg/controllers/network/platform_config.go b/internal/app/machined/pkg/controllers/network/platform_config.go new file mode 100644 index 0000000..6cf3cf1 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/platform_config.go @@ -0,0 +1,607 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "bytes" + "context" + "fmt" + "net/netip" + "os" + "path/filepath" + "sync" + "time" + + "github.com/cenkalti/backoff/v4" + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + "gopkg.in/yaml.v3" + + v1alpha1runtime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +// Virtual link name for external IPs. +const externalLink = "external" + +// PlatformConfigController manages updates hostnames and addressstatuses based on platform information. +type PlatformConfigController struct { + V1alpha1Platform v1alpha1runtime.Platform + PlatformState state.State + StatePath string +} + +// Name implements controller.Controller interface. +func (ctrl *PlatformConfigController) Name() string { + return "network.PlatformConfigController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *PlatformConfigController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: v1alpha1.NamespaceName, + Type: runtimeres.MountStatusType, + ID: optional.Some(constants.StatePartitionLabel), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *PlatformConfigController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.AddressSpecType, + Kind: controller.OutputShared, + }, + { + Type: network.LinkSpecType, + Kind: controller.OutputShared, + }, + { + Type: network.RouteSpecType, + Kind: controller.OutputShared, + }, + { + Type: network.HostnameSpecType, + Kind: controller.OutputShared, + }, + { + Type: network.ResolverSpecType, + Kind: controller.OutputShared, + }, + { + Type: network.TimeServerSpecType, + Kind: controller.OutputShared, + }, + { + Type: network.AddressStatusType, + Kind: controller.OutputShared, + }, + { + Type: network.OperatorSpecType, + Kind: controller.OutputShared, + }, + { + Type: network.ProbeSpecType, + Kind: controller.OutputShared, + }, + { + Type: runtimeres.PlatformMetadataType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *PlatformConfigController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + if ctrl.StatePath == "" { + ctrl.StatePath = constants.StateMountPoint + } + + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + if ctrl.V1alpha1Platform == nil { + // no platform, no work to be done + return nil + } + + platformCtx, platformCtxCancel := context.WithCancel(ctx) + defer platformCtxCancel() + + platformCh := make(chan *v1alpha1runtime.PlatformNetworkConfig, 1) + + var platformWg sync.WaitGroup + + platformWg.Add(1) + + go func() { + defer platformWg.Done() + + ctrl.runWithRestarts(platformCtx, logger, func() error { + return ctrl.V1alpha1Platform.NetworkConfiguration(platformCtx, ctrl.PlatformState, platformCh) + }) + }() + + defer platformWg.Wait() + + r.QueueReconcile() + + var cachedNetworkConfig, networkConfig *v1alpha1runtime.PlatformNetworkConfig + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + case networkConfig = <-platformCh: + } + + var stateMounted bool + + if _, err := r.Get(ctx, resource.NewMetadata(v1alpha1.NamespaceName, runtimeres.MountStatusType, constants.StatePartitionLabel, resource.VersionUndefined)); err == nil { + stateMounted = true + } else { + if state.IsNotFoundError(err) { + // in container mode STATE is always mounted + if ctrl.V1alpha1Platform.Mode() == v1alpha1runtime.ModeContainer { + stateMounted = true + } + } else { + return fmt.Errorf("error reading mount status: %w", err) + } + } + + if stateMounted && cachedNetworkConfig == nil { + var err error + + cachedNetworkConfig, err = ctrl.loadConfig(filepath.Join(ctrl.StatePath, constants.PlatformNetworkConfigFilename)) + if err != nil { + logger.Warn("ignored failure loading cached platform network config", zap.Error(err)) + } else if cachedNetworkConfig != nil { + logger.Debug("loaded cached platform network config") + } + } + + if stateMounted && networkConfig != nil { + if err := ctrl.storeConfig(filepath.Join(ctrl.StatePath, constants.PlatformNetworkConfigFilename), networkConfig); err != nil { + return fmt.Errorf("error saving platform network config: %w", err) + } + + logger.Debug("stored cached platform network config") + + cachedNetworkConfig = networkConfig + } + + switch { + // prefer live network config over cached config always + case networkConfig != nil: + if err := ctrl.apply(ctx, r, networkConfig); err != nil { + return err + } + // cached network is only used as last resort + case cachedNetworkConfig != nil: + if err := ctrl.apply(ctx, r, cachedNetworkConfig); err != nil { + return err + } + } + + r.ResetRestartBackoff() + } +} + +//nolint:dupl,gocyclo +func (ctrl *PlatformConfigController) apply(ctx context.Context, r controller.Runtime, networkConfig *v1alpha1runtime.PlatformNetworkConfig) error { + metadataLength := 0 + + if networkConfig.Metadata != nil { + metadataLength = 1 + } + + // handle all network specs in a loop as all specs can be handled in a similar way + for _, specType := range []struct { + length int + getter func(i int) interface{} + idBuilder func(spec interface{}) (resource.ID, error) + resourceBuilder func(id string) resource.Resource + resourceModifier func(newSpec interface{}) func(r resource.Resource) error + }{ + // AddressSpec + { + length: len(networkConfig.Addresses), + getter: func(i int) interface{} { + return networkConfig.Addresses[i] + }, + idBuilder: func(spec interface{}) (resource.ID, error) { + addressSpec := spec.(network.AddressSpecSpec) //nolint:errcheck,forcetypeassert + + return network.LayeredID(network.ConfigPlatform, network.AddressID(addressSpec.LinkName, addressSpec.Address)), nil + }, + resourceBuilder: func(id string) resource.Resource { + return network.NewAddressSpec(network.ConfigNamespaceName, id) + }, + resourceModifier: func(newSpec interface{}) func(r resource.Resource) error { + return func(r resource.Resource) error { + spec := r.(*network.AddressSpec).TypedSpec() + + *spec = newSpec.(network.AddressSpecSpec) //nolint:errcheck,forcetypeassert + spec.ConfigLayer = network.ConfigPlatform + + return nil + } + }, + }, + // LinkSpec + { + length: len(networkConfig.Links), + getter: func(i int) interface{} { + return networkConfig.Links[i] + }, + idBuilder: func(spec interface{}) (resource.ID, error) { + linkSpec := spec.(network.LinkSpecSpec) //nolint:errcheck,forcetypeassert + + return network.LayeredID(network.ConfigPlatform, network.LinkID(linkSpec.Name)), nil + }, + resourceBuilder: func(id string) resource.Resource { + return network.NewLinkSpec(network.ConfigNamespaceName, id) + }, + resourceModifier: func(newSpec interface{}) func(r resource.Resource) error { + return func(r resource.Resource) error { + spec := r.(*network.LinkSpec).TypedSpec() + + *spec = newSpec.(network.LinkSpecSpec) //nolint:errcheck,forcetypeassert + spec.ConfigLayer = network.ConfigPlatform + + return nil + } + }, + }, + // RouteSpec + { + length: len(networkConfig.Routes), + getter: func(i int) interface{} { + return networkConfig.Routes[i] + }, + idBuilder: func(spec interface{}) (resource.ID, error) { + routeSpec := spec.(network.RouteSpecSpec) //nolint:errcheck,forcetypeassert + + return network.LayeredID( + network.ConfigPlatform, + network.RouteID(routeSpec.Table, routeSpec.Family, routeSpec.Destination, routeSpec.Gateway, routeSpec.Priority, routeSpec.OutLinkName), + ), nil + }, + resourceBuilder: func(id string) resource.Resource { + return network.NewRouteSpec(network.ConfigNamespaceName, id) + }, + resourceModifier: func(newSpec interface{}) func(r resource.Resource) error { + return func(r resource.Resource) error { + spec := r.(*network.RouteSpec).TypedSpec() + + *spec = newSpec.(network.RouteSpecSpec) //nolint:errcheck,forcetypeassert + spec.ConfigLayer = network.ConfigPlatform + + return nil + } + }, + }, + // HostnameSpec + { + length: len(networkConfig.Hostnames), + getter: func(i int) interface{} { + return networkConfig.Hostnames[i] + }, + idBuilder: func(spec interface{}) (resource.ID, error) { + return network.LayeredID(network.ConfigPlatform, network.HostnameID), nil + }, + resourceBuilder: func(id string) resource.Resource { + return network.NewHostnameSpec(network.ConfigNamespaceName, id) + }, + resourceModifier: func(newSpec interface{}) func(r resource.Resource) error { + return func(r resource.Resource) error { + spec := r.(*network.HostnameSpec).TypedSpec() + + *spec = newSpec.(network.HostnameSpecSpec) //nolint:errcheck,forcetypeassert + spec.ConfigLayer = network.ConfigPlatform + + return nil + } + }, + }, + // ResolverSpec + { + length: len(networkConfig.Resolvers), + getter: func(i int) interface{} { + return networkConfig.Resolvers[i] + }, + idBuilder: func(spec interface{}) (resource.ID, error) { + return network.LayeredID(network.ConfigPlatform, network.ResolverID), nil + }, + resourceBuilder: func(id string) resource.Resource { + return network.NewResolverSpec(network.ConfigNamespaceName, id) + }, + resourceModifier: func(newSpec interface{}) func(r resource.Resource) error { + return func(r resource.Resource) error { + spec := r.(*network.ResolverSpec).TypedSpec() + + *spec = newSpec.(network.ResolverSpecSpec) //nolint:errcheck,forcetypeassert + spec.ConfigLayer = network.ConfigPlatform + + return nil + } + }, + }, + // TimeServerSpec + { + length: len(networkConfig.TimeServers), + getter: func(i int) interface{} { + return networkConfig.TimeServers[i] + }, + idBuilder: func(spec interface{}) (resource.ID, error) { + return network.LayeredID(network.ConfigPlatform, network.TimeServerID), nil + }, + resourceBuilder: func(id string) resource.Resource { + return network.NewTimeServerSpec(network.ConfigNamespaceName, id) + }, + resourceModifier: func(newSpec interface{}) func(r resource.Resource) error { + return func(r resource.Resource) error { + spec := r.(*network.TimeServerSpec).TypedSpec() + + *spec = newSpec.(network.TimeServerSpecSpec) //nolint:errcheck,forcetypeassert + spec.ConfigLayer = network.ConfigPlatform + + return nil + } + }, + }, + // OperatorSpec + { + length: len(networkConfig.Operators), + getter: func(i int) interface{} { + return networkConfig.Operators[i] + }, + idBuilder: func(spec interface{}) (resource.ID, error) { + operatorSpec := spec.(network.OperatorSpecSpec) //nolint:errcheck,forcetypeassert + + return network.LayeredID(network.ConfigPlatform, network.OperatorID(operatorSpec.Operator, operatorSpec.LinkName)), nil + }, + resourceBuilder: func(id string) resource.Resource { + return network.NewOperatorSpec(network.ConfigNamespaceName, id) + }, + resourceModifier: func(newSpec interface{}) func(r resource.Resource) error { + return func(r resource.Resource) error { + spec := r.(*network.OperatorSpec).TypedSpec() + + *spec = newSpec.(network.OperatorSpecSpec) //nolint:errcheck,forcetypeassert + spec.ConfigLayer = network.ConfigPlatform + + return nil + } + }, + }, + // ExternalIPs + { + length: len(networkConfig.ExternalIPs), + getter: func(i int) interface{} { + return networkConfig.ExternalIPs[i] + }, + idBuilder: func(spec interface{}) (resource.ID, error) { + ipAddr := spec.(netip.Addr) //nolint:errcheck,forcetypeassert + ipPrefix := netip.PrefixFrom(ipAddr, ipAddr.BitLen()) + + return network.AddressID(externalLink, ipPrefix), nil + }, + resourceBuilder: func(id string) resource.Resource { + return network.NewAddressStatus(network.NamespaceName, id) + }, + resourceModifier: func(newSpec interface{}) func(r resource.Resource) error { + return func(r resource.Resource) error { + ipAddr := newSpec.(netip.Addr) //nolint:errcheck,forcetypeassert + ipPrefix := netip.PrefixFrom(ipAddr, ipAddr.BitLen()) + + status := r.(*network.AddressStatus).TypedSpec() + + status.Address = ipPrefix + status.LinkName = externalLink + + if ipAddr.Is4() { + status.Family = nethelpers.FamilyInet4 + } else { + status.Family = nethelpers.FamilyInet6 + } + + status.Scope = nethelpers.ScopeGlobal + + return nil + } + }, + }, + // ProbeSpec + { + length: len(networkConfig.Probes), + getter: func(i int) interface{} { + return networkConfig.Probes[i] + }, + idBuilder: func(spec interface{}) (resource.ID, error) { + probeSpec := spec.(network.ProbeSpecSpec) //nolint:errcheck,forcetypeassert + + return probeSpec.ID() + }, + resourceBuilder: func(id string) resource.Resource { + return network.NewProbeSpec(network.NamespaceName, id) + }, + resourceModifier: func(newSpec interface{}) func(r resource.Resource) error { + return func(r resource.Resource) error { + spec := r.(*network.ProbeSpec).TypedSpec() + + *spec = newSpec.(network.ProbeSpecSpec) //nolint:errcheck,forcetypeassert + spec.ConfigLayer = network.ConfigPlatform + + return nil + } + }, + }, + // Platform metadata + { + length: metadataLength, + getter: func(i int) interface{} { + return networkConfig.Metadata + }, + idBuilder: func(spec interface{}) (resource.ID, error) { + return runtimeres.PlatformMetadataID, nil + }, + resourceBuilder: func(id string) resource.Resource { + return runtimeres.NewPlatformMetadataSpec(runtimeres.NamespaceName, id) + }, + resourceModifier: func(newSpec interface{}) func(r resource.Resource) error { + return func(r resource.Resource) error { + metadata := newSpec.(*runtimeres.PlatformMetadataSpec) //nolint:errcheck,forcetypeassert + + *r.(*runtimeres.PlatformMetadata).TypedSpec() = *metadata + + return nil + } + }, + }, + } { + touchedIDs := make(map[resource.ID]struct{}, specType.length) + + resourceEmpty := specType.resourceBuilder("") + resourceNamespace := resourceEmpty.Metadata().Namespace() + resourceType := resourceEmpty.Metadata().Type() + + for i := range specType.length { + spec := specType.getter(i) + + id, err := specType.idBuilder(spec) + if err != nil { + return fmt.Errorf("error building resource %s ID: %w", resourceType, err) + } + + if err = r.Modify(ctx, specType.resourceBuilder(id), specType.resourceModifier(spec)); err != nil { + return fmt.Errorf("error modifying resource %s: %w", resourceType, err) + } + + touchedIDs[id] = struct{}{} + } + + list, err := r.List(ctx, resource.NewMetadata(resourceNamespace, resourceType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing resources: %w", err) + } + + for _, res := range list.Items { + if res.Metadata().Owner() != ctrl.Name() { + continue + } + + if _, ok := touchedIDs[res.Metadata().ID()]; ok { + continue + } + + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return fmt.Errorf("error deleting %s: %w", res, err) + } + } + } + + return nil +} + +func (ctrl *PlatformConfigController) runWithRestarts(ctx context.Context, logger *zap.Logger, f func() error) { + backoff := backoff.NewExponentialBackOff() + + // disable number of retries limit + backoff.MaxElapsedTime = 0 + + for ctx.Err() == nil { + var err error + + if err = ctrl.runWithPanicHandler(logger, f); err == nil { + // operator finished without an error + return + } + + // skip restarting if context is already done + select { + case <-ctx.Done(): + return + default: + } + + interval := backoff.NextBackOff() + + logger.Error("restarting platform network config", zap.Duration("interval", interval), zap.Error(err)) + + select { + case <-ctx.Done(): + return + case <-time.After(interval): + } + } +} + +func (ctrl *PlatformConfigController) runWithPanicHandler(logger *zap.Logger, f func() error) (err error) { + defer func() { + if p := recover(); p != nil { + err = fmt.Errorf("panic: %v", p) + + logger.Error("platform panicked", zap.Stack("stack"), zap.Error(err)) + } + }() + + err = f() + + return +} + +func (ctrl *PlatformConfigController) loadConfig(path string) (*v1alpha1runtime.PlatformNetworkConfig, error) { + marshaled, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + + return nil, err + } + + var networkConfig v1alpha1runtime.PlatformNetworkConfig + + if err = yaml.Unmarshal(marshaled, &networkConfig); err != nil { + return nil, fmt.Errorf("error unmarshaling network config: %w", err) + } + + return &networkConfig, nil +} + +func (ctrl *PlatformConfigController) storeConfig(path string, networkConfig *v1alpha1runtime.PlatformNetworkConfig) error { + marshaled, err := yaml.Marshal(networkConfig) + if err != nil { + return fmt.Errorf("error marshaling network config: %w", err) + } + + if _, err := os.Stat(path); err == nil { + existing, err := os.ReadFile(path) + if err == nil && bytes.Equal(marshaled, existing) { + // existing contents are identical, skip writing to avoid no-op writes + return nil + } + } + + return os.WriteFile(path, marshaled, 0o400) +} diff --git a/internal/app/machined/pkg/controllers/network/platform_config_test.go b/internal/app/machined/pkg/controllers/network/platform_config_test.go new file mode 100644 index 0000000..5147b60 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/platform_config_test.go @@ -0,0 +1,846 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:dupl +package network_test + +import ( + "context" + "fmt" + "log" + "net/netip" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-procfs/procfs" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + v1alpha1runtime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +type PlatformConfigSuite struct { + suite.Suite + + state state.State + + statePath string + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *PlatformConfigSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.statePath = suite.T().TempDir() +} + +func (suite *PlatformConfigSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *PlatformConfigSuite) assertResources( + resourceNamespace resource.Namespace, + resourceType resource.Type, + requiredIDs []string, + check func(resource.Resource) error, +) error { + missingIDs := make(map[string]struct{}, len(requiredIDs)) + + for _, id := range requiredIDs { + missingIDs[id] = struct{}{} + } + + resources, err := suite.state.List( + suite.ctx, + resource.NewMetadata(resourceNamespace, resourceType, "", resource.VersionUndefined), + ) + if err != nil { + return err + } + + for _, res := range resources.Items { + if _, ok := missingIDs[res.Metadata().ID()]; ok { + if err = check(res); err != nil { + return retry.ExpectedError(err) + } + } + + delete(missingIDs, res.Metadata().ID()) + } + + if len(missingIDs) > 0 { + return retry.ExpectedErrorf("some resources are missing: %q", missingIDs) + } + + return nil +} + +func (suite *PlatformConfigSuite) assertNoResource(resourceType resource.Type, id string) error { + resources, err := suite.state.List( + suite.ctx, + resource.NewMetadata(network.ConfigNamespaceName, resourceType, "", resource.VersionUndefined), + ) + if err != nil { + return err + } + + for _, res := range resources.Items { + if res.Metadata().ID() == id { + return retry.ExpectedErrorf("spec %q is still there", id) + } + } + + return nil +} + +func (suite *PlatformConfigSuite) TestNoPlatform() { + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.PlatformConfigController{})) + + suite.startRuntime() + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertNoResource(network.HostnameSpecType, "platform/hostname") + }, + ), + ) +} + +func (suite *PlatformConfigSuite) TestPlatformMockHostname() { + suite.Require().NoError( + suite.runtime.RegisterController( + &netctrl.PlatformConfigController{ + V1alpha1Platform: &platformMock{hostname: []byte("talos-e2e-897b4e49-gcp-controlplane-jvcnl.c.talos-testbed.internal")}, + StatePath: suite.statePath, + }, + ), + ) + + suite.startRuntime() + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertResources( + network.ConfigNamespaceName, network.HostnameSpecType, []string{ + "platform/hostname", + }, func(r resource.Resource) error { + spec := r.(*network.HostnameSpec).TypedSpec() + + suite.Assert().Equal("talos-e2e-897b4e49-gcp-controlplane-jvcnl", spec.Hostname) + suite.Assert().Equal("c.talos-testbed.internal", spec.Domainname) + suite.Assert().Equal(network.ConfigPlatform, spec.ConfigLayer) + + return nil + }, + ) + }, + ), + ) +} + +func (suite *PlatformConfigSuite) TestPlatformMockHostnameNoDomain() { + suite.Require().NoError( + suite.runtime.RegisterController( + &netctrl.PlatformConfigController{ + V1alpha1Platform: &platformMock{hostname: []byte("talos-e2e-897b4e49-gcp-controlplane-jvcnl")}, + StatePath: suite.statePath, + PlatformState: suite.state, + }, + ), + ) + + suite.startRuntime() + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertResources( + network.ConfigNamespaceName, network.HostnameSpecType, []string{ + "platform/hostname", + }, func(r resource.Resource) error { + spec := r.(*network.HostnameSpec).TypedSpec() + + suite.Assert().Equal("talos-e2e-897b4e49-gcp-controlplane-jvcnl", spec.Hostname) + suite.Assert().Equal("", spec.Domainname) + suite.Assert().Equal(network.ConfigPlatform, spec.ConfigLayer) + + return nil + }, + ) + }, + ), + ) +} + +func (suite *PlatformConfigSuite) TestPlatformMockAddresses() { + suite.Require().NoError( + suite.runtime.RegisterController( + &netctrl.PlatformConfigController{ + V1alpha1Platform: &platformMock{ + addresses: []netip.Prefix{ + netip.MustParsePrefix("192.168.1.24/24"), + netip.MustParsePrefix("2001:fd::3/64"), + }, + }, + StatePath: suite.statePath, + PlatformState: suite.state, + }, + ), + ) + + suite.startRuntime() + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertResources( + network.ConfigNamespaceName, network.AddressSpecType, []string{ + "platform/eth0/192.168.1.24/24", + "platform/eth0/2001:fd::3/64", + }, func(r resource.Resource) error { + spec := r.(*network.AddressSpec).TypedSpec() + + switch r.Metadata().ID() { + case "platform/eth0/192.168.1.24/24": + suite.Assert().Equal(nethelpers.FamilyInet4, spec.Family) + suite.Assert().Equal("192.168.1.24/24", spec.Address.String()) + case "platform/eth0/2001:fd::3/64": + suite.Assert().Equal(nethelpers.FamilyInet6, spec.Family) + suite.Assert().Equal("2001:fd::3/64", spec.Address.String()) + } + + suite.Assert().Equal(network.ConfigPlatform, spec.ConfigLayer) + + return nil + }, + ) + }, + ), + ) +} + +func (suite *PlatformConfigSuite) TestPlatformMockLinks() { + suite.Require().NoError( + suite.runtime.RegisterController( + &netctrl.PlatformConfigController{ + V1alpha1Platform: &platformMock{ + linksUp: []string{"eth0", "eth1"}, + }, + StatePath: suite.statePath, + PlatformState: suite.state, + }, + ), + ) + + suite.startRuntime() + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertResources( + network.ConfigNamespaceName, network.LinkSpecType, []string{ + "platform/eth0", + "platform/eth1", + }, func(r resource.Resource) error { + spec := r.(*network.LinkSpec).TypedSpec() + + suite.Assert().True(spec.Up) + suite.Assert().Equal(network.ConfigPlatform, spec.ConfigLayer) + + return nil + }, + ) + }, + ), + ) +} + +func (suite *PlatformConfigSuite) TestPlatformMockRoutes() { + suite.Require().NoError( + suite.runtime.RegisterController( + &netctrl.PlatformConfigController{ + V1alpha1Platform: &platformMock{ + defaultRoutes: []netip.Addr{netip.MustParseAddr("10.0.0.1")}, + }, + StatePath: suite.statePath, + PlatformState: suite.state, + }, + ), + ) + + suite.startRuntime() + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertResources( + network.ConfigNamespaceName, network.RouteSpecType, []string{ + "platform/inet4/10.0.0.1//1024", + }, func(r resource.Resource) error { + spec := r.(*network.RouteSpec).TypedSpec() + + suite.Assert().Equal("10.0.0.1", spec.Gateway.String()) + suite.Assert().Equal(network.ConfigPlatform, spec.ConfigLayer) + + return nil + }, + ) + }, + ), + ) +} + +func (suite *PlatformConfigSuite) TestPlatformMockOperators() { + suite.Require().NoError( + suite.runtime.RegisterController( + &netctrl.PlatformConfigController{ + V1alpha1Platform: &platformMock{ + dhcp4Links: []string{"eth1", "eth2"}, + }, + StatePath: suite.statePath, + PlatformState: suite.state, + }, + ), + ) + + suite.startRuntime() + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertResources( + network.ConfigNamespaceName, network.OperatorSpecType, []string{ + "platform/dhcp4/eth1", + "platform/dhcp4/eth2", + }, func(r resource.Resource) error { + spec := r.(*network.OperatorSpec).TypedSpec() + + suite.Assert().Equal(network.OperatorDHCP4, spec.Operator) + suite.Assert().Equal(network.ConfigPlatform, spec.ConfigLayer) + + return nil + }, + ) + }, + ), + ) +} + +func (suite *PlatformConfigSuite) TestPlatformMockResolvers() { + suite.Require().NoError( + suite.runtime.RegisterController( + &netctrl.PlatformConfigController{ + V1alpha1Platform: &platformMock{ + resolvers: []netip.Addr{netip.MustParseAddr("1.1.1.1")}, + }, + StatePath: suite.statePath, + PlatformState: suite.state, + }, + ), + ) + + suite.startRuntime() + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertResources( + network.ConfigNamespaceName, network.ResolverSpecType, []string{ + "platform/resolvers", + }, func(r resource.Resource) error { + spec := r.(*network.ResolverSpec).TypedSpec() + + suite.Assert().Equal("[1.1.1.1]", fmt.Sprintf("%s", spec.DNSServers)) + suite.Assert().Equal(network.ConfigPlatform, spec.ConfigLayer) + + return nil + }, + ) + }, + ), + ) +} + +func (suite *PlatformConfigSuite) TestPlatformMockTimeServers() { + suite.Require().NoError( + suite.runtime.RegisterController( + &netctrl.PlatformConfigController{ + V1alpha1Platform: &platformMock{ + timeServers: []string{"pool.ntp.org"}, + }, + StatePath: suite.statePath, + PlatformState: suite.state, + }, + ), + ) + + suite.startRuntime() + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertResources( + network.ConfigNamespaceName, network.TimeServerSpecType, []string{ + "platform/timeservers", + }, func(r resource.Resource) error { + spec := r.(*network.TimeServerSpec).TypedSpec() + + suite.Assert().Equal("[pool.ntp.org]", fmt.Sprintf("%s", spec.NTPServers)) + suite.Assert().Equal(network.ConfigPlatform, spec.ConfigLayer) + + return nil + }, + ) + }, + ), + ) +} + +func (suite *PlatformConfigSuite) TestPlatformMockProbes() { + suite.Require().NoError( + suite.runtime.RegisterController( + &netctrl.PlatformConfigController{ + V1alpha1Platform: &platformMock{ + tcpProbes: []string{"example.com:80", "example.com:443"}, + }, + StatePath: suite.statePath, + PlatformState: suite.state, + }, + ), + ) + + suite.startRuntime() + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertResources( + network.NamespaceName, network.ProbeSpecType, []string{ + "tcp:example.com:80", + "tcp:example.com:443", + }, func(r resource.Resource) error { + spec := r.(*network.ProbeSpec).TypedSpec() + + suite.Assert().Equal(time.Second, spec.Interval) + suite.Assert().Equal(network.ConfigPlatform, spec.ConfigLayer) + + return nil + }, + ) + }, + ), + ) +} + +func (suite *PlatformConfigSuite) TestPlatformMockExternalIPs() { + suite.Require().NoError( + suite.runtime.RegisterController( + &netctrl.PlatformConfigController{ + V1alpha1Platform: &platformMock{ + externalIPs: []netip.Addr{ + netip.MustParseAddr("10.3.4.5"), + netip.MustParseAddr("2001:470:6d:30e:96f4:4219:5733:b860"), + }, + }, + StatePath: suite.statePath, + PlatformState: suite.state, + }, + ), + ) + + suite.startRuntime() + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertResources( + network.NamespaceName, network.AddressStatusType, []string{ + "external/10.3.4.5/32", + "external/2001:470:6d:30e:96f4:4219:5733:b860/128", + }, func(r resource.Resource) error { + spec := r.(*network.AddressStatus).TypedSpec() + + suite.Assert().Equal("external", spec.LinkName) + suite.Assert().Equal(nethelpers.ScopeGlobal, spec.Scope) + + if r.Metadata().ID() == "external/10.3.4.5/32" { + suite.Assert().Equal(nethelpers.FamilyInet4, spec.Family) + } else { + suite.Assert().Equal(nethelpers.FamilyInet6, spec.Family) + } + + return nil + }, + ) + }, + ), + ) +} + +func (suite *PlatformConfigSuite) TestPlatformMockMetadata() { + suite.Require().NoError( + suite.runtime.RegisterController( + &netctrl.PlatformConfigController{ + V1alpha1Platform: &platformMock{ + metadata: &runtimeres.PlatformMetadataSpec{ + Platform: "mock", + Zone: "mock-zone", + }, + }, + StatePath: suite.statePath, + PlatformState: suite.state, + }, + ), + ) + + suite.startRuntime() + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertResources( + runtimeres.NamespaceName, runtimeres.PlatformMetadataType, []string{ + runtimeres.PlatformMetadataID, + }, func(r resource.Resource) error { + spec := r.(*runtimeres.PlatformMetadata).TypedSpec() + + suite.Assert().Equal("mock", spec.Platform) + suite.Assert().Equal("mock-zone", spec.Zone) + + return nil + }, + ) + }, + ), + ) +} + +const sampleStoredConfig = "addresses: []\nlinks: []\nroutes: []\nhostnames:\n - hostname: talos-e2e-897b4e49-gcp-controlplane-jvcnl\n domainname: \"\"\n layer: default\nresolvers: []\ntimeServers: []\noperators: []\nexternalIPs:\n - 10.3.4.5\n - 2001:470:6d:30e:96f4:4219:5733:b860\n" //nolint:lll + +func (suite *PlatformConfigSuite) TestStoreConfig() { + suite.Require().NoError( + suite.runtime.RegisterController( + &netctrl.PlatformConfigController{ + V1alpha1Platform: &platformMock{ + hostname: []byte("talos-e2e-897b4e49-gcp-controlplane-jvcnl"), + externalIPs: []netip.Addr{ + netip.MustParseAddr("10.3.4.5"), + netip.MustParseAddr("2001:470:6d:30e:96f4:4219:5733:b860"), + }, + }, + StatePath: suite.statePath, + PlatformState: suite.state, + }, + ), + ) + + suite.startRuntime() + + stateMount := runtimeres.NewMountStatus(v1alpha1.NamespaceName, constants.StatePartitionLabel) + + suite.Assert().NoError(suite.state.Create(suite.ctx, stateMount)) + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + contents, err := os.ReadFile(filepath.Join(suite.statePath, constants.PlatformNetworkConfigFilename)) + if err != nil { + if os.IsNotExist(err) { + return retry.ExpectedError(err) + } + + return err + } + + suite.Assert().Equal(sampleStoredConfig, string(contents)) + + return nil + }, + ), + ) +} + +func (suite *PlatformConfigSuite) TestLoadConfig() { + suite.Require().NoError( + suite.runtime.RegisterController( + &netctrl.PlatformConfigController{ + V1alpha1Platform: &platformMock{ + noData: true, + }, + StatePath: suite.statePath, + PlatformState: suite.state, + }, + ), + ) + + suite.startRuntime() + + suite.Require().NoError( + os.WriteFile( + filepath.Join(suite.statePath, constants.PlatformNetworkConfigFilename), + []byte(sampleStoredConfig), + 0o400, + ), + ) + + stateMount := runtimeres.NewMountStatus(v1alpha1.NamespaceName, constants.StatePartitionLabel) + + suite.Assert().NoError(suite.state.Create(suite.ctx, stateMount)) + + // controller should pick up cached network configuration + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertResources( + network.NamespaceName, network.AddressStatusType, []string{ + "external/10.3.4.5/32", + "external/2001:470:6d:30e:96f4:4219:5733:b860/128", + }, func(r resource.Resource) error { + spec := r.(*network.AddressStatus).TypedSpec() + + suite.Assert().Equal("external", spec.LinkName) + suite.Assert().Equal(nethelpers.ScopeGlobal, spec.Scope) + + if r.Metadata().ID() == "external/10.3.4.5/32" { + suite.Assert().Equal(nethelpers.FamilyInet4, spec.Family) + } else { + suite.Assert().Equal(nethelpers.FamilyInet6, spec.Family) + } + + return nil + }, + ) + }, + ), + ) + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertResources( + network.ConfigNamespaceName, network.HostnameSpecType, []string{ + "platform/hostname", + }, func(r resource.Resource) error { + spec := r.(*network.HostnameSpec).TypedSpec() + + suite.Assert().Equal("talos-e2e-897b4e49-gcp-controlplane-jvcnl", spec.Hostname) + suite.Assert().Equal("", spec.Domainname) + suite.Assert().Equal(network.ConfigPlatform, spec.ConfigLayer) + + return nil + }, + ) + }, + ), + ) +} + +func (suite *PlatformConfigSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestPlatformConfigSuite(t *testing.T) { + suite.Run(t, new(PlatformConfigSuite)) +} + +type platformMock struct { + noData bool + + hostname []byte + externalIPs []netip.Addr + addresses []netip.Prefix + defaultRoutes []netip.Addr + linksUp []string + resolvers []netip.Addr + timeServers []string + dhcp4Links []string + tcpProbes []string + + metadata *runtimeres.PlatformMetadataSpec +} + +func (mock *platformMock) Name() string { + return "mock" +} + +func (mock *platformMock) Configuration(context.Context, state.State) ([]byte, error) { + return nil, nil +} + +func (mock *platformMock) Metadata(context.Context, state.State) (runtimeres.PlatformMetadataSpec, error) { + return runtimeres.PlatformMetadataSpec{Platform: mock.Name()}, nil +} + +func (mock *platformMock) Mode() v1alpha1runtime.Mode { + return v1alpha1runtime.ModeCloud +} + +func (mock *platformMock) KernelArgs(string) procfs.Parameters { + return nil +} + +//nolint:gocyclo +func (mock *platformMock) NetworkConfiguration( + ctx context.Context, + st state.State, + ch chan<- *v1alpha1runtime.PlatformNetworkConfig, +) error { + if mock.noData { + return nil + } + + networkConfig := &v1alpha1runtime.PlatformNetworkConfig{ + ExternalIPs: mock.externalIPs, + } + + if mock.hostname != nil { + hostnameSpec := network.HostnameSpecSpec{} + if err := hostnameSpec.ParseFQDN(string(mock.hostname)); err != nil { + return err + } + + networkConfig.Hostnames = []network.HostnameSpecSpec{hostnameSpec} + } + + for _, addr := range mock.addresses { + family := nethelpers.FamilyInet4 + if addr.Addr().Is6() { + family = nethelpers.FamilyInet6 + } + + networkConfig.Addresses = append( + networkConfig.Addresses, + network.AddressSpecSpec{ + ConfigLayer: network.ConfigPlatform, + LinkName: "eth0", + Address: addr, + Scope: nethelpers.ScopeGlobal, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + Family: family, + }, + ) + } + + for _, gw := range mock.defaultRoutes { + family := nethelpers.FamilyInet4 + if gw.Is6() { + family = nethelpers.FamilyInet6 + } + + route := network.RouteSpecSpec{ + ConfigLayer: network.ConfigPlatform, + Gateway: gw, + OutLinkName: "eth0", + Table: nethelpers.TableMain, + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + Family: family, + Priority: 1024, + } + + route.Normalize() + + networkConfig.Routes = append(networkConfig.Routes, route) + } + + for _, link := range mock.linksUp { + networkConfig.Links = append( + networkConfig.Links, network.LinkSpecSpec{ + ConfigLayer: network.ConfigPlatform, + Name: link, + Up: true, + }, + ) + } + + if len(mock.resolvers) > 0 { + networkConfig.Resolvers = append( + networkConfig.Resolvers, network.ResolverSpecSpec{ + ConfigLayer: network.ConfigPlatform, + DNSServers: mock.resolvers, + }, + ) + } + + if len(mock.timeServers) > 0 { + networkConfig.TimeServers = append( + networkConfig.TimeServers, network.TimeServerSpecSpec{ + ConfigLayer: network.ConfigPlatform, + NTPServers: mock.timeServers, + }, + ) + } + + for _, link := range mock.dhcp4Links { + networkConfig.Operators = append( + networkConfig.Operators, network.OperatorSpecSpec{ + ConfigLayer: network.ConfigPlatform, + LinkName: link, + Operator: network.OperatorDHCP4, + DHCP4: network.DHCP4OperatorSpec{}, + }, + ) + } + + for _, endpoint := range mock.tcpProbes { + networkConfig.Probes = append( + networkConfig.Probes, network.ProbeSpecSpec{ + Interval: time.Second, + TCP: network.TCPProbeSpec{ + Endpoint: endpoint, + Timeout: time.Second, + }, + ConfigLayer: network.ConfigPlatform, + }) + } + + networkConfig.Metadata = mock.metadata + + select { + case ch <- networkConfig: + case <-ctx.Done(): + return ctx.Err() + } + + return nil +} diff --git a/internal/app/machined/pkg/controllers/network/probe.go b/internal/app/machined/pkg/controllers/network/probe.go new file mode 100644 index 0000000..fe889fb --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/probe.go @@ -0,0 +1,154 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "go.uber.org/zap" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network/internal/probe" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// ProbeController runs network probes configured with ProbeSpecs and outputs ProbeStatuses. +type ProbeController struct { + runners map[string]*probe.Runner +} + +// Name implements controller.Controller interface. +func (ctrl *ProbeController) Name() string { + return "network.ProbeController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *ProbeController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: network.NamespaceName, + Type: network.ProbeSpecType, + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *ProbeController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.ProbeStatusType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +func (ctrl *ProbeController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + notifyCh := make(chan probe.Notification) + + ctrl.runners = make(map[string]*probe.Runner) + + defer func() { + for _, runner := range ctrl.runners { + runner.Stop() + } + }() + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + if err := ctrl.reconcileRunners(ctx, r, logger, notifyCh); err != nil { + return err + } + case ev := <-notifyCh: + if err := ctrl.reconcileOutputs(ctx, r, ev); err != nil { + return err + } + } + + r.ResetRestartBackoff() + } +} + +//nolint:gocyclo +func (ctrl *ProbeController) reconcileRunners(ctx context.Context, r controller.Runtime, logger *zap.Logger, notifyCh chan<- probe.Notification) error { + specList, err := safe.ReaderListAll[*network.ProbeSpec](ctx, r) + if err != nil { + return fmt.Errorf("error listing probe specs: %w", err) + } + + // figure out which operators should run + shouldRun := make(map[string]network.ProbeSpecSpec) + + for iter := specList.Iterator(); iter.Next(); { + shouldRun[iter.Value().Metadata().ID()] = *iter.Value().TypedSpec() + } + + // stop running probes which shouldn't run + for id := range ctrl.runners { + if _, exists := shouldRun[id]; !exists { + logger.Debug("stopping probe", zap.String("probe", id)) + + ctrl.runners[id].Stop() + delete(ctrl.runners, id) + } else if !shouldRun[id].Equal(ctrl.runners[id].Spec) { + logger.Debug("replacing probe", zap.String("probe", id)) + + ctrl.runners[id].Stop() + delete(ctrl.runners, id) + } + } + + // start probes which aren't running + for id := range shouldRun { + if _, exists := ctrl.runners[id]; !exists { + ctrl.runners[id] = &probe.Runner{ + ID: id, + Spec: shouldRun[id], + } + + logger.Debug("starting probe", zap.String("probe", id)) + ctrl.runners[id].Start(ctx, notifyCh, logger) + } + } + + // clean up statuses which should no longer exist + statusList, err := safe.ReaderListAll[*network.ProbeStatus](ctx, r) + if err != nil { + return fmt.Errorf("error listing probe statuses: %w", err) + } + + for iter := statusList.Iterator(); iter.Next(); { + if _, exists := shouldRun[iter.Value().Metadata().ID()]; exists { + continue + } + + if err = r.Destroy(ctx, iter.Value().Metadata()); err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error destroying probe status: %w", err) + } + } + + return nil +} + +func (ctrl *ProbeController) reconcileOutputs(ctx context.Context, r controller.Runtime, ev probe.Notification) error { + if _, exists := ctrl.runners[ev.ID]; !exists { + // probe was already removed, late notification, ignore it + return nil + } + + return safe.WriterModify(ctx, r, network.NewProbeStatus(network.NamespaceName, ev.ID), + func(status *network.ProbeStatus) error { + *status.TypedSpec() = ev.Status + + return nil + }) +} diff --git a/internal/app/machined/pkg/controllers/network/probe_test.go b/internal/app/machined/pkg/controllers/network/probe_test.go new file mode 100644 index 0000000..e8e20b1 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/probe_test.go @@ -0,0 +1,91 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network_test + +import ( + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + networkctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type ProbeSuite struct { + ctest.DefaultSuite +} + +func (suite *ProbeSuite) TestReconcile() { + googleProbeSpec := network.ProbeSpecSpec{ + Interval: 100 * time.Millisecond, + TCP: network.TCPProbeSpec{ + Endpoint: "google.com:80", + Timeout: 5 * time.Second, + }, + } + googleProbeSpecID, err := googleProbeSpec.ID() + suite.Require().NoError(err) + + probeGoogle := network.NewProbeSpec(network.NamespaceName, googleProbeSpecID) + *probeGoogle.TypedSpec() = googleProbeSpec + suite.Require().NoError(suite.State().Create(suite.Ctx(), probeGoogle)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{googleProbeSpecID}, func(r *network.ProbeStatus, assert *assert.Assertions) { + assert.Equal(network.ProbeStatusSpec{ + Success: true, + }, *r.TypedSpec()) + }) + + failingProbeSpec := network.ProbeSpecSpec{ + Interval: 100 * time.Millisecond, + FailureThreshold: 1, + TCP: network.TCPProbeSpec{ + Endpoint: "google.com:81", + Timeout: time.Second, + }, + } + failingProbeSpecID, err := failingProbeSpec.ID() + suite.Require().NoError(err) + + probeFailing := network.NewProbeSpec(network.NamespaceName, failingProbeSpecID) + *probeFailing.TypedSpec() = failingProbeSpec + suite.Require().NoError(suite.State().Create(suite.Ctx(), probeFailing)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{failingProbeSpecID}, func(r *network.ProbeStatus, assert *assert.Assertions) { + assert.False(r.TypedSpec().Success) + }) + + probeFailing.TypedSpec().TCP.Endpoint = "google.com:443" + suite.Require().NoError(suite.State().Update(suite.Ctx(), probeFailing)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{failingProbeSpecID}, func(r *network.ProbeStatus, assert *assert.Assertions) { + assert.Equal(network.ProbeStatusSpec{ + Success: true, + }, *r.TypedSpec()) + }) + + suite.Require().NoError(suite.State().Destroy(suite.Ctx(), probeFailing.Metadata())) + suite.Require().NoError(suite.State().Destroy(suite.Ctx(), probeGoogle.Metadata())) + + rtestutils.AssertNoResource[*network.ProbeStatus](suite.Ctx(), suite.T(), suite.State(), failingProbeSpecID) + rtestutils.AssertNoResource[*network.ProbeStatus](suite.Ctx(), suite.T(), suite.State(), googleProbeSpecID) +} + +// TestProbeSuite runs the ProbeSuite. +func TestProbeSuite(t *testing.T) { + suite.Run(t, &ProbeSuite{ + DefaultSuite: ctest.DefaultSuite{ + Timeout: 20 * time.Second, + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(&networkctrl.ProbeController{})) + }, + }, + }) +} diff --git a/internal/app/machined/pkg/controllers/network/resolver_config.go b/internal/app/machined/pkg/controllers/network/resolver_config.go new file mode 100644 index 0000000..5528bfa --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/resolver_config.go @@ -0,0 +1,211 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "context" + "fmt" + "net/netip" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/go-procfs/procfs" + "go.uber.org/zap" + + talosconfig "github.com/siderolabs/talos/pkg/machinery/config" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// ResolverConfigController manages network.ResolverSpec based on machine configuration, kernel cmdline. +type ResolverConfigController struct { + Cmdline *procfs.Cmdline +} + +// Name implements controller.Controller interface. +func (ctrl *ResolverConfigController) Name() string { + return "network.ResolverConfigController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *ResolverConfigController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: config.MachineConfigType, + ID: optional.Some(config.V1Alpha1ID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *ResolverConfigController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.ResolverSpecType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *ResolverConfigController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + touchedIDs := make(map[resource.ID]struct{}) + + var cfgProvider talosconfig.Config + + cfg, err := safe.ReaderGetByID[*config.MachineConfig](ctx, r, config.V1Alpha1ID) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting config: %w", err) + } + } else if cfg.Config().Machine() != nil { + cfgProvider = cfg.Config() + } + + var specs []network.ResolverSpecSpec + + // defaults + specs = append(specs, ctrl.getDefault()) + + // parse kernel cmdline for the default gateway + cmdlineServers := ctrl.parseCmdline(logger) + if cmdlineServers.DNSServers != nil { + specs = append(specs, cmdlineServers) + } + + // parse machine configuration for specs + if cfgProvider != nil { + configServers := ctrl.parseMachineConfiguration(logger, cfgProvider) + + if configServers.DNSServers != nil { + specs = append(specs, configServers) + } + } + + var ids []string + + ids, err = ctrl.apply(ctx, r, specs) + if err != nil { + return fmt.Errorf("error applying specs: %w", err) + } + + for _, id := range ids { + touchedIDs[id] = struct{}{} + } + + // list specs for cleanup + list, err := r.List(ctx, resource.NewMetadata(network.ConfigNamespaceName, network.ResolverSpecType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing resources: %w", err) + } + + for _, res := range list.Items { + if res.Metadata().Owner() != ctrl.Name() { + // skip specs created by other controllers + continue + } + + if _, ok := touchedIDs[res.Metadata().ID()]; !ok { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return fmt.Errorf("error cleaning up specs: %w", err) + } + } + } + + r.ResetRestartBackoff() + } +} + +//nolint:dupl +func (ctrl *ResolverConfigController) apply(ctx context.Context, r controller.Runtime, specs []network.ResolverSpecSpec) ([]resource.ID, error) { + ids := make([]string, 0, len(specs)) + + for _, spec := range specs { + id := network.LayeredID(spec.ConfigLayer, network.ResolverID) + + if err := r.Modify( + ctx, + network.NewResolverSpec(network.ConfigNamespaceName, id), + func(r resource.Resource) error { + *r.(*network.ResolverSpec).TypedSpec() = spec + + return nil + }, + ); err != nil { + return ids, err + } + + ids = append(ids, id) + } + + return ids, nil +} + +func (ctrl *ResolverConfigController) getDefault() (spec network.ResolverSpecSpec) { + spec.DNSServers = []netip.Addr{netip.MustParseAddr(constants.DefaultPrimaryResolver), netip.MustParseAddr(constants.DefaultSecondaryResolver)} + spec.ConfigLayer = network.ConfigDefault + + return spec +} + +func (ctrl *ResolverConfigController) parseCmdline(logger *zap.Logger) (spec network.ResolverSpecSpec) { + if ctrl.Cmdline == nil { + return + } + + settings, err := ParseCmdlineNetwork(ctrl.Cmdline) + if err != nil { + logger.Warn("ignoring error", zap.Error(err)) + + return + } + + if len(settings.DNSAddresses) == 0 { + return + } + + spec.DNSServers = settings.DNSAddresses + spec.ConfigLayer = network.ConfigCmdline + + return spec +} + +func (ctrl *ResolverConfigController) parseMachineConfiguration(logger *zap.Logger, cfgProvider talosconfig.Config) (spec network.ResolverSpecSpec) { + resolvers := cfgProvider.Machine().Network().Resolvers() + + if len(resolvers) == 0 { + return + } + + for i := range resolvers { + server, err := netip.ParseAddr(resolvers[i]) + if err != nil { + logger.Warn("failed to parse DNS server", zap.String("server", resolvers[i]), zap.Error(err)) + + continue + } + + spec.DNSServers = append(spec.DNSServers, server) + } + + spec.ConfigLayer = network.ConfigMachineConfiguration + + return spec +} diff --git a/internal/app/machined/pkg/controllers/network/resolver_config_test.go b/internal/app/machined/pkg/controllers/network/resolver_config_test.go new file mode 100644 index 0000000..3cee212 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/resolver_config_test.go @@ -0,0 +1,209 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network_test + +import ( + "context" + "log" + "net/netip" + "net/url" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-procfs/procfs" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type ResolverConfigSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *ResolverConfigSuite) State() state.State { return suite.state } + +func (suite *ResolverConfigSuite) Ctx() context.Context { return suite.ctx } + +func (suite *ResolverConfigSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) +} + +func (suite *ResolverConfigSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *ResolverConfigSuite) assertResolvers(requiredIDs []string, check func(*network.ResolverSpec, *assert.Assertions)) { + assertResources(suite.ctx, suite.T(), suite.state, requiredIDs, check, rtestutils.WithNamespace(network.ConfigNamespaceName)) +} + +func (suite *ResolverConfigSuite) assertNoResolver(id string) error { + resources, err := suite.state.List( + suite.ctx, + resource.NewMetadata(network.ConfigNamespaceName, network.ResolverSpecType, "", resource.VersionUndefined), + ) + if err != nil { + return err + } + + for _, res := range resources.Items { + if res.Metadata().ID() == id { + return retry.ExpectedErrorf("spec %q is still there", id) + } + } + + return nil +} + +func (suite *ResolverConfigSuite) TestDefaults() { + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.ResolverConfigController{})) + + suite.startRuntime() + + suite.assertResolvers( + []string{ + "default/resolvers", + }, func(r *network.ResolverSpec, asrt *assert.Assertions) { + asrt.Equal( + []netip.Addr{ + netip.MustParseAddr(constants.DefaultPrimaryResolver), + netip.MustParseAddr(constants.DefaultSecondaryResolver), + }, r.TypedSpec().DNSServers, + ) + asrt.Equal(network.ConfigDefault, r.TypedSpec().ConfigLayer) + }, + ) +} + +func (suite *ResolverConfigSuite) TestCmdline() { + suite.Require().NoError( + suite.runtime.RegisterController( + &netctrl.ResolverConfigController{ + Cmdline: procfs.NewCmdline("ip=172.20.0.2:172.21.0.1:172.20.0.1:255.255.255.0:master1:eth1::10.0.0.1:10.0.0.2:10.0.0.1"), + }, + ), + ) + + suite.startRuntime() + + suite.assertResolvers( + []string{ + "cmdline/resolvers", + }, func(r *network.ResolverSpec, asrt *assert.Assertions) { + asrt.Equal( + []netip.Addr{ + netip.MustParseAddr("10.0.0.1"), + netip.MustParseAddr("10.0.0.2"), + }, r.TypedSpec().DNSServers, + ) + }, + ) +} + +func (suite *ResolverConfigSuite) TestMachineConfiguration() { + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.ResolverConfigController{})) + + suite.startRuntime() + + u, err := url.Parse("https://foo:6443") + suite.Require().NoError(err) + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineNetwork: &v1alpha1.NetworkConfig{ + NameServers: []string{"2.2.2.2", "3.3.3.3"}, + }, + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: u, + }, + }, + }, + }, + ), + ) + + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + suite.assertResolvers( + []string{ + "configuration/resolvers", + }, func(r *network.ResolverSpec, asrt *assert.Assertions) { + asrt.Equal( + []netip.Addr{ + netip.MustParseAddr("2.2.2.2"), + netip.MustParseAddr("3.3.3.3"), + }, r.TypedSpec().DNSServers, + ) + }, + ) + + ctest.UpdateWithConflicts(suite, cfg, func(r *config.MachineConfig) error { + r.Container().RawV1Alpha1().MachineConfig.MachineNetwork.NameServers = nil + + return nil + }) + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertNoResolver("configuration/resolvers") + }, + ), + ) +} + +func (suite *ResolverConfigSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestResolverConfigSuite(t *testing.T) { + suite.Run(t, new(ResolverConfigSuite)) +} diff --git a/internal/app/machined/pkg/controllers/network/resolver_merge.go b/internal/app/machined/pkg/controllers/network/resolver_merge.go new file mode 100644 index 0000000..7ebb016 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/resolver_merge.go @@ -0,0 +1,132 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package network provides controllers which manage network resources. +// +//nolint:dupl +package network + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// ResolverMergeController merges network.ResolverSpec in network.ConfigNamespace and produces final network.ResolverSpec in network.Namespace. +type ResolverMergeController struct{} + +// Name implements controller.Controller interface. +func (ctrl *ResolverMergeController) Name() string { + return "network.ResolverMergeController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *ResolverMergeController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: network.ConfigNamespaceName, + Type: network.ResolverSpecType, + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.ResolverSpecType, + Kind: controller.InputDestroyReady, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *ResolverMergeController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.ResolverSpecType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *ResolverMergeController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + // list source network configuration resources + list, err := r.List(ctx, resource.NewMetadata(network.ConfigNamespaceName, network.ResolverSpecType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing source network addresses: %w", err) + } + + // simply merge by layers, overriding with the next configuration layer + var final network.ResolverSpecSpec + + for _, res := range list.Items { + spec := res.(*network.ResolverSpec) //nolint:errcheck,forcetypeassert + + if final.DNSServers != nil && spec.TypedSpec().ConfigLayer < final.ConfigLayer { + // skip this spec, as existing one is higher layer + continue + } + + if spec.TypedSpec().ConfigLayer == final.ConfigLayer { + // merge server lists on the same level + final.DNSServers = append(final.DNSServers, spec.TypedSpec().DNSServers...) + } else { + // otherwise, replace the lists + final = *spec.TypedSpec() + } + } + + if final.DNSServers != nil { + if err = r.Modify(ctx, network.NewResolverSpec(network.NamespaceName, network.ResolverID), func(res resource.Resource) error { + spec := res.(*network.ResolverSpec) //nolint:errcheck,forcetypeassert + + *spec.TypedSpec() = final + + return nil + }); err != nil { + if state.IsPhaseConflictError(err) { + // conflict + final.DNSServers = nil + + r.QueueReconcile() + } else { + return fmt.Errorf("error updating resource: %w", err) + } + } + } + + if final.DNSServers == nil { + // remove existing + var okToDestroy bool + + md := resource.NewMetadata(network.NamespaceName, network.ResolverSpecType, network.ResolverID, resource.VersionUndefined) + + okToDestroy, err = r.Teardown(ctx, md) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error cleaning up specs: %w", err) + } + + if okToDestroy { + if err = r.Destroy(ctx, md); err != nil { + return fmt.Errorf("error cleaning up specs: %w", err) + } + } + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/network/resolver_merge_test.go b/internal/app/machined/pkg/controllers/network/resolver_merge_test.go new file mode 100644 index 0000000..e6f9682 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/resolver_merge_test.go @@ -0,0 +1,132 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:dupl +package network_test + +import ( + "context" + "log" + "net/netip" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type ResolverMergeSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *ResolverMergeSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.ResolverMergeController{})) + + suite.startRuntime() +} + +func (suite *ResolverMergeSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *ResolverMergeSuite) assertResolvers(requiredIDs []string, check func(*network.ResolverSpec, *assert.Assertions)) { + assertResources(suite.ctx, suite.T(), suite.state, requiredIDs, check) +} + +func (suite *ResolverMergeSuite) TestMerge() { + def := network.NewResolverSpec(network.ConfigNamespaceName, "default/resolvers") + *def.TypedSpec() = network.ResolverSpecSpec{ + DNSServers: []netip.Addr{ + netip.MustParseAddr(constants.DefaultPrimaryResolver), + netip.MustParseAddr(constants.DefaultSecondaryResolver), + }, + ConfigLayer: network.ConfigDefault, + } + + dhcp1 := network.NewResolverSpec(network.ConfigNamespaceName, "dhcp/eth0") + *dhcp1.TypedSpec() = network.ResolverSpecSpec{ + DNSServers: []netip.Addr{netip.MustParseAddr("1.1.2.0")}, + ConfigLayer: network.ConfigOperator, + } + + dhcp2 := network.NewResolverSpec(network.ConfigNamespaceName, "dhcp/eth1") + *dhcp2.TypedSpec() = network.ResolverSpecSpec{ + DNSServers: []netip.Addr{netip.MustParseAddr("1.1.2.1")}, + ConfigLayer: network.ConfigOperator, + } + + static := network.NewResolverSpec(network.ConfigNamespaceName, "configuration/resolvers") + *static.TypedSpec() = network.ResolverSpecSpec{ + DNSServers: []netip.Addr{netip.MustParseAddr("2.2.2.2")}, + ConfigLayer: network.ConfigMachineConfiguration, + } + + for _, res := range []resource.Resource{def, dhcp1, dhcp2, static} { + suite.Require().NoError(suite.state.Create(suite.ctx, res), "%v", res.Spec()) + } + + suite.assertResolvers( + []string{ + "resolvers", + }, func(r *network.ResolverSpec, asrt *assert.Assertions) { + asrt.Equal(*static.TypedSpec(), *r.TypedSpec()) + }, + ) + + suite.Require().NoError(suite.state.Destroy(suite.ctx, static.Metadata())) + + suite.assertResolvers( + []string{ + "resolvers", + }, func(r *network.ResolverSpec, asrt *assert.Assertions) { + asrt.Equal(r.TypedSpec().DNSServers, []netip.Addr{netip.MustParseAddr("1.1.2.0"), netip.MustParseAddr("1.1.2.1")}) + }, + ) +} + +func (suite *ResolverMergeSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestResolverMergeSuite(t *testing.T) { + suite.Run(t, new(ResolverMergeSuite)) +} diff --git a/internal/app/machined/pkg/controllers/network/resolver_spec.go b/internal/app/machined/pkg/controllers/network/resolver_spec.go new file mode 100644 index 0000000..d6613d2 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/resolver_spec.go @@ -0,0 +1,106 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// ResolverSpecController applies network.ResolverSpec to the actual interfaces. +type ResolverSpecController struct{} + +// Name implements controller.Controller interface. +func (ctrl *ResolverSpecController) Name() string { + return "network.ResolverSpecController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *ResolverSpecController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: network.NamespaceName, + Type: network.ResolverSpecType, + Kind: controller.InputStrong, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *ResolverSpecController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.ResolverStatusType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *ResolverSpecController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + // list source network configuration resources + list, err := r.List(ctx, resource.NewMetadata(network.NamespaceName, network.ResolverSpecType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing source network addresses: %w", err) + } + + // add finalizers for all live resources + for _, res := range list.Items { + if res.Metadata().Phase() != resource.PhaseRunning { + continue + } + + if err = r.AddFinalizer(ctx, res.Metadata(), ctrl.Name()); err != nil { + return fmt.Errorf("error adding finalizer: %w", err) + } + } + + // loop over specs and sync to statuses + for _, res := range list.Items { + spec := res.(*network.ResolverSpec) //nolint:forcetypeassert,errcheck + + switch spec.Metadata().Phase() { + case resource.PhaseTearingDown: + if err = r.Destroy(ctx, resource.NewMetadata(network.NamespaceName, network.ResolverStatusType, spec.Metadata().ID(), resource.VersionUndefined)); err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error destroying status: %w", err) + } + + if err = r.RemoveFinalizer(ctx, spec.Metadata(), ctrl.Name()); err != nil { + return fmt.Errorf("error removing finalizer: %w", err) + } + case resource.PhaseRunning: + logger.Info("setting resolvers", zap.Stringers("resolvers", spec.TypedSpec().DNSServers)) + + if err = r.Modify(ctx, network.NewResolverStatus(network.NamespaceName, spec.Metadata().ID()), func(r resource.Resource) error { + status := r.(*network.ResolverStatus) //nolint:forcetypeassert,errcheck + + status.TypedSpec().DNSServers = spec.TypedSpec().DNSServers + + return nil + }); err != nil { + return fmt.Errorf("error modifying status: %w", err) + } + } + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/network/resolver_spec_test.go b/internal/app/machined/pkg/controllers/network/resolver_spec_test.go new file mode 100644 index 0000000..006f805 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/resolver_spec_test.go @@ -0,0 +1,120 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:dupl +package network_test + +import ( + "context" + "log" + "net/netip" + "reflect" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type ResolverSpecSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *ResolverSpecSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.ResolverSpecController{})) + + suite.startRuntime() +} + +func (suite *ResolverSpecSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *ResolverSpecSuite) assertStatus(id string, servers ...netip.Addr) error { + r, err := suite.state.Get( + suite.ctx, + resource.NewMetadata(network.NamespaceName, network.ResolverStatusType, id, resource.VersionUndefined), + ) + if err != nil { + if state.IsNotFoundError(err) { + return retry.ExpectedError(err) + } + + return err + } + + status := r.(*network.ResolverStatus) //nolint:errcheck,forcetypeassert + + if !reflect.DeepEqual(status.TypedSpec().DNSServers, servers) { + return retry.ExpectedErrorf("server list mismatch: %q != %q", status.TypedSpec().DNSServers, servers) + } + + return nil +} + +func (suite *ResolverSpecSuite) TestSpec() { + spec := network.NewResolverSpec(network.NamespaceName, "resolvers") + *spec.TypedSpec() = network.ResolverSpecSpec{ + DNSServers: []netip.Addr{netip.MustParseAddr(constants.DefaultPrimaryResolver)}, + ConfigLayer: network.ConfigDefault, + } + + for _, res := range []resource.Resource{spec} { + suite.Require().NoError(suite.state.Create(suite.ctx, res), "%v", res.Spec()) + } + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertStatus("resolvers", netip.MustParseAddr(constants.DefaultPrimaryResolver)) + }, + ), + ) +} + +func (suite *ResolverSpecSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestResolverSpecSuite(t *testing.T) { + suite.Run(t, new(ResolverSpecSuite)) +} diff --git a/internal/app/machined/pkg/controllers/network/route_config.go b/internal/app/machined/pkg/controllers/network/route_config.go new file mode 100644 index 0000000..b819a71 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/route_config.go @@ -0,0 +1,325 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "context" + "fmt" + "net/netip" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/value" + "github.com/siderolabs/go-procfs/procfs" + "go.uber.org/zap" + + talosconfig "github.com/siderolabs/talos/pkg/machinery/config/config" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// RouteConfigController manages network.RouteSpec based on machine configuration, kernel cmdline. +type RouteConfigController struct { + Cmdline *procfs.Cmdline +} + +// Name implements controller.Controller interface. +func (ctrl *RouteConfigController) Name() string { + return "network.RouteConfigController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *RouteConfigController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: network.NamespaceName, + Type: network.DeviceConfigSpecType, + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *RouteConfigController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.RouteSpecType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *RouteConfigController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + touchedIDs := make(map[resource.ID]struct{}) + + items, err := r.List(ctx, resource.NewMetadata(network.NamespaceName, network.DeviceConfigSpecType, "", resource.VersionUndefined)) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting config: %w", err) + } + } + + ignoredInterfaces := map[string]struct{}{} + + devices := make([]talosconfig.Device, len(items.Items)) + + for i, item := range items.Items { + device := item.(*network.DeviceConfigSpec).TypedSpec().Device + + devices[i] = device + + if device.Ignore() { + ignoredInterfaces[device.Interface()] = struct{}{} + } + } + + if len(devices) > 0 { + for _, device := range devices { + if device.Ignore() { + ignoredInterfaces[device.Interface()] = struct{}{} + } + } + } + + // parse kernel cmdline for the default gateway + cmdlineRoutes := ctrl.parseCmdline(logger) + for _, cmdlineRoute := range cmdlineRoutes { + if _, ignored := ignoredInterfaces[cmdlineRoute.OutLinkName]; !ignored { + var ids []string + + ids, err = ctrl.apply(ctx, r, []network.RouteSpecSpec{cmdlineRoute}) + if err != nil { + return fmt.Errorf("error applying cmdline route: %w", err) + } + + for _, id := range ids { + touchedIDs[id] = struct{}{} + } + } + } + + // parse machine configuration for static routes + if len(devices) > 0 { + addresses := ctrl.processDevicesConfiguration(logger, devices) + + var ids []string + + ids, err = ctrl.apply(ctx, r, addresses) + if err != nil { + return fmt.Errorf("error applying machine configuration address: %w", err) + } + + for _, id := range ids { + touchedIDs[id] = struct{}{} + } + } + + // list routes for cleanup + list, err := r.List(ctx, resource.NewMetadata(network.ConfigNamespaceName, network.RouteSpecType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing resources: %w", err) + } + + for _, res := range list.Items { + if res.Metadata().Owner() != ctrl.Name() { + // skip specs created by other controllers + continue + } + + if _, ok := touchedIDs[res.Metadata().ID()]; !ok { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return fmt.Errorf("error cleaning up routes: %w", err) + } + } + } + + r.ResetRestartBackoff() + } +} + +func (ctrl *RouteConfigController) apply(ctx context.Context, r controller.Runtime, routes []network.RouteSpecSpec) ([]resource.ID, error) { + ids := make([]string, 0, len(routes)) + + for _, route := range routes { + id := network.LayeredID(route.ConfigLayer, network.RouteID(route.Table, route.Family, route.Destination, route.Gateway, route.Priority, route.OutLinkName)) + + if err := r.Modify( + ctx, + network.NewRouteSpec(network.ConfigNamespaceName, id), + func(r resource.Resource) error { + *r.(*network.RouteSpec).TypedSpec() = route + + return nil + }, + ); err != nil { + return ids, err + } + + ids = append(ids, id) + } + + return ids, nil +} + +func (ctrl *RouteConfigController) parseCmdline(logger *zap.Logger) (routes []network.RouteSpecSpec) { + if ctrl.Cmdline == nil { + return + } + + settings, err := ParseCmdlineNetwork(ctrl.Cmdline) + if err != nil { + logger.Info("ignoring error", zap.Error(err)) + + return + } + + for idx, linkConfig := range settings.LinkConfigs { + if value.IsZero(linkConfig.Gateway) { + continue + } + + // add a default gateway route + defaultGatewayRoute := network.RouteSpecSpec{ + Gateway: linkConfig.Gateway, + Scope: nethelpers.ScopeGlobal, + Table: nethelpers.TableMain, + Priority: network.DefaultRouteMetric + uint32(idx), // set different priorities to avoid a conflict + Protocol: nethelpers.ProtocolBoot, + Type: nethelpers.TypeUnicast, + OutLinkName: linkConfig.LinkName, + ConfigLayer: network.ConfigCmdline, + } + + if defaultGatewayRoute.Gateway.Is6() { + defaultGatewayRoute.Family = nethelpers.FamilyInet6 + } else { + defaultGatewayRoute.Family = nethelpers.FamilyInet4 + } + + defaultGatewayRoute.Normalize() + + routes = append(routes, defaultGatewayRoute) + + // for IPv4, if the gateway is not directly reachable on the link, add a link-scope route for the gateway + if linkConfig.Gateway.Is4() && !linkConfig.Address.Contains(linkConfig.Gateway) { + routes = append(routes, network.RouteSpecSpec{ + Family: nethelpers.FamilyInet4, + Destination: netip.PrefixFrom(linkConfig.Gateway, linkConfig.Gateway.BitLen()), + Source: linkConfig.Address.Addr(), + OutLinkName: linkConfig.LinkName, + Table: nethelpers.TableMain, + Priority: defaultGatewayRoute.Priority, + Scope: nethelpers.ScopeLink, + Type: nethelpers.TypeUnicast, + Protocol: nethelpers.ProtocolBoot, + ConfigLayer: network.ConfigCmdline, + }) + } + } + + return routes +} + +//nolint:gocyclo,cyclop +func (ctrl *RouteConfigController) processDevicesConfiguration(logger *zap.Logger, devices []talosconfig.Device) (routes []network.RouteSpecSpec) { + convert := func(linkName string, in talosconfig.Route) (route network.RouteSpecSpec, err error) { + if in.Network() != "" { + route.Destination, err = netip.ParsePrefix(in.Network()) + if err != nil { + return route, fmt.Errorf("error parsing route network: %w", err) + } + } + + if in.Gateway() != "" { + route.Gateway, err = netip.ParseAddr(in.Gateway()) + if err != nil { + return route, fmt.Errorf("error parsing route gateway: %w", err) + } + } + + if in.Source() != "" { + route.Source, err = netip.ParseAddr(in.Source()) + if err != nil { + return route, fmt.Errorf("error parsing route source: %w", err) + } + } + + route.Normalize() + + route.Priority = in.Metric() + if route.Priority == 0 { + route.Priority = network.DefaultRouteMetric + } + + route.MTU = in.MTU() + + switch { + case !value.IsZero(route.Gateway) && route.Gateway.Is6(): + route.Family = nethelpers.FamilyInet6 + case !value.IsZero(route.Destination) && route.Destination.Addr().Is6(): + route.Family = nethelpers.FamilyInet6 + default: + route.Family = nethelpers.FamilyInet4 + } + + route.Table = nethelpers.TableMain + route.Protocol = nethelpers.ProtocolStatic + route.OutLinkName = linkName + route.ConfigLayer = network.ConfigMachineConfiguration + + route.Type = nethelpers.TypeUnicast + + if route.Destination.Addr().IsMulticast() { + route.Type = nethelpers.TypeMulticast + } + + return route, nil + } + + for _, device := range devices { + if device.Ignore() { + continue + } + + for _, route := range device.Routes() { + routeSpec, err := convert(device.Interface(), route) + if err != nil { + logger.Sugar().Infof("skipping route %q -> %q on interface %q: %s", route.Network(), route.Gateway(), device.Interface(), err) + + continue + } + + routes = append(routes, routeSpec) + } + + for _, vlan := range device.Vlans() { + vlanLinkName := nethelpers.VLANLinkName(device.Interface(), vlan.ID()) + + for _, route := range vlan.Routes() { + routeSpec, err := convert(vlanLinkName, route) + if err != nil { + logger.Sugar().Infof("skipping route %q -> %q on interface %q: %s", route.Network(), route.Gateway(), vlanLinkName, err) + + continue + } + + routes = append(routes, routeSpec) + } + } + } + + return routes +} diff --git a/internal/app/machined/pkg/controllers/network/route_config_test.go b/internal/app/machined/pkg/controllers/network/route_config_test.go new file mode 100644 index 0000000..94b08bc --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/route_config_test.go @@ -0,0 +1,280 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network_test + +import ( + "context" + "log" + "net/netip" + "net/url" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-pointer" + "github.com/siderolabs/go-procfs/procfs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type RouteConfigSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *RouteConfigSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.DeviceConfigController{})) +} + +func (suite *RouteConfigSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *RouteConfigSuite) assertRoutes(requiredIDs []string, check func(*network.RouteSpec, *assert.Assertions)) { + assertResources(suite.ctx, suite.T(), suite.state, requiredIDs, check, rtestutils.WithNamespace(network.ConfigNamespaceName)) +} + +func (suite *RouteConfigSuite) TestCmdline() { + suite.Require().NoError( + suite.runtime.RegisterController( + &netctrl.RouteConfigController{ + Cmdline: procfs.NewCmdline("ip=172.20.0.2::172.20.0.1:255.255.255.0::eth1::::: ip=eth3:dhcp ip=10.3.5.7::10.3.5.1:255.255.255.0::eth4"), + }, + ), + ) + + suite.startRuntime() + + suite.assertRoutes( + []string{ + "cmdline/inet4/172.20.0.1//1024", + "cmdline/inet4/10.3.5.1//1026", + }, func(r *network.RouteSpec, asrt *assert.Assertions) { + asrt.Equal(network.ConfigCmdline, r.TypedSpec().ConfigLayer) + asrt.Equal(nethelpers.FamilyInet4, r.TypedSpec().Family) + + switch r.Metadata().ID() { + case "cmdline/inet4/172.20.0.1//1024": + asrt.Equal("eth1", r.TypedSpec().OutLinkName) + asrt.EqualValues(network.DefaultRouteMetric, r.TypedSpec().Priority) + case "cmdline/inet4/10.3.5.1//1025": + asrt.Equal("eth4", r.TypedSpec().OutLinkName) + asrt.EqualValues(network.DefaultRouteMetric+2, r.TypedSpec().Priority) + } + }, + ) +} + +func (suite *RouteConfigSuite) TestCmdlineNotReachable() { + suite.Require().NoError( + suite.runtime.RegisterController( + &netctrl.RouteConfigController{ + Cmdline: procfs.NewCmdline("ip=172.20.0.2::172.20.0.1:255.255.255.255::eth1:::::"), + }, + ), + ) + + suite.startRuntime() + + suite.assertRoutes( + []string{ + "cmdline/inet4/172.20.0.1//1024", + "cmdline/inet4//172.20.0.1/32/1024", + }, func(r *network.RouteSpec, asrt *assert.Assertions) { + asrt.Equal(network.ConfigCmdline, r.TypedSpec().ConfigLayer) + asrt.Equal(nethelpers.FamilyInet4, r.TypedSpec().Family) + + switch r.Metadata().ID() { + case "cmdline/inet4/172.20.0.1//1024": + asrt.Equal("eth1", r.TypedSpec().OutLinkName) + asrt.EqualValues(network.DefaultRouteMetric, r.TypedSpec().Priority) + case "cmdline/inet4//172.20.0.1/32/1024": + asrt.Equal("eth1", r.TypedSpec().OutLinkName) + asrt.Equal(netip.Addr{}, r.TypedSpec().Gateway) + asrt.Equal(netip.MustParsePrefix("172.20.0.1/32"), r.TypedSpec().Destination) + asrt.EqualValues(network.DefaultRouteMetric, r.TypedSpec().Priority) + } + }, + ) +} + +func (suite *RouteConfigSuite) TestMachineConfiguration() { + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.RouteConfigController{})) + + suite.startRuntime() + + u, err := url.Parse("https://foo:6443") + suite.Require().NoError(err) + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineNetwork: &v1alpha1.NetworkConfig{ + NetworkInterfaces: []*v1alpha1.Device{ + { + DeviceInterface: "eth3", + DeviceAddresses: []string{"192.168.0.24/28"}, + DeviceRoutes: []*v1alpha1.Route{ + { + RouteNetwork: "192.168.0.0/18", + RouteGateway: "192.168.0.25", + RouteMetric: 25, + }, + { + RouteNetwork: "169.254.254.254/32", + }, + }, + }, + { + DeviceIgnore: pointer.To(true), + DeviceInterface: "eth4", + DeviceAddresses: []string{"192.168.0.24/28"}, + DeviceRoutes: []*v1alpha1.Route{ + { + RouteNetwork: "192.168.0.0/18", + RouteGateway: "192.168.0.26", + RouteMetric: 25, + }, + }, + }, + { + DeviceInterface: "eth2", + DeviceAddresses: []string{"2001:470:6d:30e:8ed2:b60c:9d2f:803a/64"}, + DeviceRoutes: []*v1alpha1.Route{ + { + RouteGateway: "2001:470:6d:30e:8ed2:b60c:9d2f:803b", + }, + }, + }, + { + DeviceInterface: "eth0", + DeviceVlans: []*v1alpha1.Vlan{ + { + VlanID: 24, + VlanAddresses: []string{ + "10.0.0.1/8", + }, + VlanRoutes: []*v1alpha1.Route{ + { + RouteNetwork: "10.0.3.0/24", + RouteGateway: "10.0.3.1", + }, + }, + }, + }, + }, + { + DeviceInterface: "eth1", + DeviceRoutes: []*v1alpha1.Route{ + { + RouteNetwork: "192.244.0.0/24", + RouteGateway: "192.244.0.1", + RouteSource: "192.244.0.10", + }, + }, + }, + }, + }, + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: u, + }, + }, + }, + }, + ), + ) + + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + suite.assertRoutes( + []string{ + "configuration/eth2/inet6/2001:470:6d:30e:8ed2:b60c:9d2f:803b//1024", + "configuration/inet4/10.0.3.1/10.0.3.0/24/1024", + "configuration/inet4/192.168.0.25/192.168.0.0/18/25", + "configuration/inet4/192.244.0.1/192.244.0.0/24/1024", + "configuration/inet4//169.254.254.254/32/1024", + }, func(r *network.RouteSpec, asrt *assert.Assertions) { + switch r.Metadata().ID() { + case "configuration/inet6/2001:470:6d:30e:8ed2:b60c:9d2f:803b//1024": + asrt.Equal("eth2", r.TypedSpec().OutLinkName) + asrt.Equal(nethelpers.FamilyInet6, r.TypedSpec().Family) + asrt.EqualValues(network.DefaultRouteMetric, r.TypedSpec().Priority) + case "configuration/inet4/10.0.3.1/10.0.3.0/24/1024": + asrt.Equal("eth0.24", r.TypedSpec().OutLinkName) + asrt.Equal(nethelpers.FamilyInet4, r.TypedSpec().Family) + asrt.EqualValues(network.DefaultRouteMetric, r.TypedSpec().Priority) + case "configuration/inet4/192.168.0.25/192.168.0.0/18/25": + asrt.Equal("eth3", r.TypedSpec().OutLinkName) + asrt.Equal(nethelpers.FamilyInet4, r.TypedSpec().Family) + asrt.EqualValues(25, r.TypedSpec().Priority) + case "configuration/inet4/192.244.0.1/192.244.0.0/24/1024": + asrt.Equal("eth1", r.TypedSpec().OutLinkName) + asrt.Equal(nethelpers.FamilyInet4, r.TypedSpec().Family) + asrt.EqualValues(network.DefaultRouteMetric, r.TypedSpec().Priority) + asrt.EqualValues(netip.MustParseAddr("192.244.0.10"), r.TypedSpec().Source) + case "configuration/inet4//169.254.254.254/32/1024": + asrt.Equal("eth3", r.TypedSpec().OutLinkName) + asrt.Equal(nethelpers.FamilyInet4, r.TypedSpec().Family) + asrt.EqualValues(network.DefaultRouteMetric, r.TypedSpec().Priority) + asrt.Equal(nethelpers.ScopeLink, r.TypedSpec().Scope) + asrt.Equal("169.254.254.254/32", r.TypedSpec().Destination.String()) + } + + asrt.Equal(network.ConfigMachineConfiguration, r.TypedSpec().ConfigLayer) + }, + ) +} + +func (suite *RouteConfigSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestRouteConfigSuite(t *testing.T) { + suite.Run(t, new(RouteConfigSuite)) +} diff --git a/internal/app/machined/pkg/controllers/network/route_merge.go b/internal/app/machined/pkg/controllers/network/route_merge.go new file mode 100644 index 0000000..8a054fb --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/route_merge.go @@ -0,0 +1,138 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package network provides controllers which manage network resources. +package network + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// RouteMergeController merges network.RouteSpec in network.ConfigNamespace and produces final network.RouteSpec in network.Namespace. +type RouteMergeController struct{} + +// Name implements controller.Controller interface. +func (ctrl *RouteMergeController) Name() string { + return "network.RouteMergeController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *RouteMergeController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: network.ConfigNamespaceName, + Type: network.RouteSpecType, + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.RouteSpecType, + Kind: controller.InputDestroyReady, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *RouteMergeController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.RouteSpecType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *RouteMergeController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + // list source network configuration resources + list, err := r.List(ctx, resource.NewMetadata(network.ConfigNamespaceName, network.RouteSpecType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing source network routes: %w", err) + } + + // route is allowed as long as it's not duplicate, for duplicate higher layer takes precedence + routes := map[string]*network.RouteSpec{} + + for _, res := range list.Items { + route := res.(*network.RouteSpec) //nolint:errcheck,forcetypeassert + id := network.RouteID(route.TypedSpec().Table, route.TypedSpec().Family, route.TypedSpec().Destination, route.TypedSpec().Gateway, route.TypedSpec().Priority, route.TypedSpec().OutLinkName) + + existing, ok := routes[id] + if ok && existing.TypedSpec().ConfigLayer > route.TypedSpec().ConfigLayer { + // skip this route, as existing one is higher layer + continue + } + + routes[id] = route + } + + conflictsDetected := 0 + + for id, route := range routes { + if err = r.Modify(ctx, network.NewRouteSpec(network.NamespaceName, id), func(res resource.Resource) error { + rt := res.(*network.RouteSpec) //nolint:errcheck,forcetypeassert + + *rt.TypedSpec() = *route.TypedSpec() + + return nil + }); err != nil { + if state.IsPhaseConflictError(err) { + // phase conflict, resource is being torn down, skip updating it and trigger reconcile + // later by failing the + conflictsDetected++ + + delete(routes, id) + } else { + return fmt.Errorf("error updating resource: %w", err) + } + } + } + + // list routes for cleanup + list, err = r.List(ctx, resource.NewMetadata(network.NamespaceName, network.RouteSpecType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing resources: %w", err) + } + + for _, res := range list.Items { + if _, ok := routes[res.Metadata().ID()]; !ok { + var okToDestroy bool + + okToDestroy, err = r.Teardown(ctx, res.Metadata()) + if err != nil { + return fmt.Errorf("error cleaning up routes: %w", err) + } + + if okToDestroy { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return fmt.Errorf("error cleaning up routes: %w", err) + } + } + } + } + + if conflictsDetected > 0 { + return fmt.Errorf("%d conflict(s) detected", conflictsDetected) + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/network/route_merge_test.go b/internal/app/machined/pkg/controllers/network/route_merge_test.go new file mode 100644 index 0000000..60290dc --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/route_merge_test.go @@ -0,0 +1,357 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:dupl +package network_test + +import ( + "context" + "log" + "net/netip" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + "golang.org/x/sync/errgroup" + + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type RouteMergeSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *RouteMergeSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.RouteMergeController{})) + + suite.startRuntime() +} + +func (suite *RouteMergeSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *RouteMergeSuite) assertRoutes(requiredIDs []string, check func(*network.RouteSpec) error) error { + missingIDs := make(map[string]struct{}, len(requiredIDs)) + + for _, id := range requiredIDs { + missingIDs[id] = struct{}{} + } + + resources, err := suite.state.List( + suite.ctx, + resource.NewMetadata(network.NamespaceName, network.RouteSpecType, "", resource.VersionUndefined), + ) + if err != nil { + return err + } + + for _, res := range resources.Items { + _, required := missingIDs[res.Metadata().ID()] + if !required { + continue + } + + delete(missingIDs, res.Metadata().ID()) + + if err = check(res.(*network.RouteSpec)); err != nil { + return retry.ExpectedError(err) + } + } + + if len(missingIDs) > 0 { + return retry.ExpectedErrorf("some resources are missing: %q", missingIDs) + } + + return nil +} + +func (suite *RouteMergeSuite) assertNoRoute(id string) error { + resources, err := suite.state.List( + suite.ctx, + resource.NewMetadata(network.NamespaceName, network.RouteSpecType, "", resource.VersionUndefined), + ) + if err != nil { + return err + } + + for _, res := range resources.Items { + if res.Metadata().ID() == id { + return retry.ExpectedErrorf("address %q is still there", id) + } + } + + return nil +} + +func (suite *RouteMergeSuite) TestMerge() { + cmdline := network.NewRouteSpec(network.ConfigNamespaceName, "cmdline/inet4//10.5.0.3/50") + *cmdline.TypedSpec() = network.RouteSpecSpec{ + Gateway: netip.MustParseAddr("10.5.0.3"), + OutLinkName: "eth0", + Family: nethelpers.FamilyInet4, + Scope: nethelpers.ScopeGlobal, + Type: nethelpers.TypeUnicast, + Table: nethelpers.TableMain, + Priority: 50, + ConfigLayer: network.ConfigCmdline, + } + + dhcp := network.NewRouteSpec(network.ConfigNamespaceName, "dhcp/inet4//10.5.0.3/50") + *dhcp.TypedSpec() = network.RouteSpecSpec{ + Gateway: netip.MustParseAddr("10.5.0.3"), + OutLinkName: "eth0", + Family: nethelpers.FamilyInet4, + Scope: nethelpers.ScopeGlobal, + Type: nethelpers.TypeUnicast, + Table: nethelpers.TableMain, + Priority: 50, + ConfigLayer: network.ConfigOperator, + } + + static := network.NewRouteSpec(network.ConfigNamespaceName, "configuration/inet4/10.0.0.35/32/10.0.0.34/1024") + *static.TypedSpec() = network.RouteSpecSpec{ + Destination: netip.MustParsePrefix("10.0.0.35/32"), + Gateway: netip.MustParseAddr("10.0.0.34"), + OutLinkName: "eth0", + Family: nethelpers.FamilyInet4, + Scope: nethelpers.ScopeGlobal, + Type: nethelpers.TypeUnicast, + Table: nethelpers.TableMain, + Priority: 1024, + ConfigLayer: network.ConfigMachineConfiguration, + } + + for _, res := range []resource.Resource{cmdline, dhcp, static} { + suite.Require().NoError(suite.state.Create(suite.ctx, res), "%v", res.Spec()) + } + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertRoutes( + []string{ + "inet4/10.5.0.3//50", + "inet4/10.0.0.34/10.0.0.35/32/1024", + }, func(r *network.RouteSpec) error { + suite.Assert().Equal(resource.PhaseRunning, r.Metadata().Phase()) + + switch r.Metadata().ID() { + case "inet4/10.5.0.3//50": + suite.Assert().Equal(*dhcp.TypedSpec(), *r.TypedSpec()) + case "inet4/10.0.0.34/10.0.0.35/32/1024": + suite.Assert().Equal(*static.TypedSpec(), *r.TypedSpec()) + } + + return nil + }, + ) + }, + ), + ) + + suite.Require().NoError(suite.state.Destroy(suite.ctx, dhcp.Metadata())) + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertRoutes( + []string{ + "inet4/10.5.0.3//50", + "inet4/10.0.0.34/10.0.0.35/32/1024", + }, func(r *network.RouteSpec) error { + suite.Assert().Equal(resource.PhaseRunning, r.Metadata().Phase()) + + switch r.Metadata().ID() { + case "inet4/10.5.0.3//50": + if *cmdline.TypedSpec() != *r.TypedSpec() { + // using retry here, as it might not be reconciled immediately + return retry.ExpectedErrorf("not equal yet") + } + case "inet4/10.0.0.34/10.0.0.35/32/1024": + suite.Assert().Equal(*static.TypedSpec(), *r.TypedSpec()) + } + + return nil + }, + ) + }, + ), + ) + + suite.Require().NoError(suite.state.Destroy(suite.ctx, static.Metadata())) + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertNoRoute("inet4/10.0.0.34/10.0.0.35/32/1024") + }, + ), + ) +} + +//nolint:gocyclo +func (suite *RouteMergeSuite) TestMergeFlapping() { + // simulate two conflicting default route definitions which are getting removed/added constantly + cmdline := network.NewRouteSpec(network.ConfigNamespaceName, "cmdline/inet4//10.5.0.3/50") + *cmdline.TypedSpec() = network.RouteSpecSpec{ + Gateway: netip.MustParseAddr("10.5.0.3"), + OutLinkName: "eth0", + Family: nethelpers.FamilyInet4, + Scope: nethelpers.ScopeGlobal, + Type: nethelpers.TypeUnicast, + Table: nethelpers.TableMain, + Priority: 50, + ConfigLayer: network.ConfigCmdline, + } + + dhcp := network.NewRouteSpec(network.ConfigNamespaceName, "dhcp/inet4//10.5.0.3/50") + *dhcp.TypedSpec() = network.RouteSpecSpec{ + Gateway: netip.MustParseAddr("10.5.0.3"), + OutLinkName: "eth1", + Family: nethelpers.FamilyInet4, + Scope: nethelpers.ScopeGlobal, + Type: nethelpers.TypeUnicast, + Table: nethelpers.TableMain, + Priority: 50, + ConfigLayer: network.ConfigOperator, + } + + resources := []resource.Resource{cmdline, dhcp} + + flipflop := func(idx int) func() error { + return func() error { + for range 500 { + if err := suite.state.Create(suite.ctx, resources[idx]); err != nil { + return err + } + + if err := suite.state.Destroy(suite.ctx, resources[idx].Metadata()); err != nil { + return err + } + + time.Sleep(time.Millisecond) + } + + return suite.state.Create(suite.ctx, resources[idx]) + } + } + + var eg errgroup.Group + + eg.Go(flipflop(0)) + eg.Go(flipflop(1)) + eg.Go( + func() error { + // add/remove finalizer to the merged resource + for range 1000 { + if err := suite.state.AddFinalizer( + suite.ctx, + resource.NewMetadata( + network.NamespaceName, + network.RouteSpecType, + "inet4/10.5.0.3//50", + resource.VersionUndefined, + ), + "foo", + ); err != nil { + if !state.IsNotFoundError(err) { + return err + } + + continue + } + + suite.T().Log("finalizer added") + + time.Sleep(10 * time.Millisecond) + + if err := suite.state.RemoveFinalizer( + suite.ctx, + resource.NewMetadata( + network.NamespaceName, + network.RouteSpecType, + "inet4/10.5.0.3//50", + resource.VersionUndefined, + ), + "foo", + ); err != nil && !state.IsNotFoundError(err) { + return err + } + } + + return nil + }, + ) + + suite.Require().NoError(eg.Wait()) + + suite.Assert().NoError( + retry.Constant(15*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertRoutes( + []string{ + "inet4/10.5.0.3//50", + }, func(r *network.RouteSpec) error { + if r.Metadata().Phase() != resource.PhaseRunning { + return retry.ExpectedErrorf("resource phase is %s", r.Metadata().Phase()) + } + + if *dhcp.TypedSpec() != *r.TypedSpec() { + // using retry here, as it might not be reconciled immediately + return retry.ExpectedErrorf("not equal yet") + } + + return nil + }, + ) + }, + ), + ) +} + +func (suite *RouteMergeSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestRouteMergeSuite(t *testing.T) { + suite.Run(t, new(RouteMergeSuite)) +} diff --git a/internal/app/machined/pkg/controllers/network/route_spec.go b/internal/app/machined/pkg/controllers/network/route_spec.go new file mode 100644 index 0000000..fc2b604 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/route_spec.go @@ -0,0 +1,309 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "context" + "fmt" + "net/netip" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/hashicorp/go-multierror" + "github.com/jsimonetti/rtnetlink" + "github.com/siderolabs/gen/value" + "go.uber.org/zap" + "golang.org/x/sys/unix" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network/watch" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// RouteSpecController applies network.RouteSpec to the actual interfaces. +type RouteSpecController struct{} + +// Name implements controller.Controller interface. +func (ctrl *RouteSpecController) Name() string { + return "network.RouteSpecController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *RouteSpecController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: network.NamespaceName, + Type: network.RouteSpecType, + Kind: controller.InputStrong, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *RouteSpecController) Outputs() []controller.Output { + return nil +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *RouteSpecController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + // watch link changes as some routes might need to be re-applied if the link appears + watcher, err := watch.NewRtNetlink(watch.NewDefaultRateLimitedTrigger(ctx, r), unix.RTMGRP_LINK|unix.RTMGRP_IPV4_ROUTE) + if err != nil { + return err + } + + defer watcher.Done() + + conn, err := rtnetlink.Dial(nil) + if err != nil { + return fmt.Errorf("error dialing rtnetlink socket: %w", err) + } + + defer conn.Close() //nolint:errcheck + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + // list source network configuration resources + list, err := r.List(ctx, resource.NewMetadata(network.NamespaceName, network.RouteSpecType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing source network addresses: %w", err) + } + + // add finalizers for all live resources + for _, res := range list.Items { + if res.Metadata().Phase() != resource.PhaseRunning { + continue + } + + if err = r.AddFinalizer(ctx, res.Metadata(), ctrl.Name()); err != nil { + return fmt.Errorf("error adding finalizer: %w", err) + } + } + + // list rtnetlink links (interfaces) + links, err := conn.Link.List() + if err != nil { + return fmt.Errorf("error listing links: %w", err) + } + + // list rtnetlink routes + routes, err := conn.Route.List() + if err != nil { + return fmt.Errorf("error listing addresses: %w", err) + } + + var multiErr *multierror.Error + + // loop over routes and make reconcile decision + for _, res := range list.Items { + route := res.(*network.RouteSpec) //nolint:forcetypeassert,errcheck + + if err = ctrl.syncRoute(ctx, r, logger, conn, links, routes, route); err != nil { + multiErr = multierror.Append(multiErr, err) + } + } + + if err = multiErr.ErrorOrNil(); err != nil { + return err + } + + r.ResetRestartBackoff() + } +} + +// netipPrefixBitsCorrected returns the number of bits in the prefix, corrected for zero value to have bits of 0. +// +// Go stdlib returns -1 for zero value, which is not what we want. +func netipPrefixBitsCorrected(p netip.Prefix) int { + if p.Addr().AsSlice() == nil { + return 0 + } + + return p.Bits() +} + +func findMatchingRoutes(existingRoutes []rtnetlink.RouteMessage, expected *network.RouteSpecSpec) []*rtnetlink.RouteMessage { + var result []*rtnetlink.RouteMessage //nolint:prealloc + + for i, route := range existingRoutes { + if route.Family != uint8(expected.Family) { + continue + } + + if int(route.DstLength) != netipPrefixBitsCorrected(expected.Destination) { + continue + } + + if !route.Attributes.Dst.Equal(expected.Destination.Addr().AsSlice()) { + continue + } + + if !route.Attributes.Gateway.Equal(expected.Gateway.AsSlice()) { + continue + } + + if nethelpers.RoutingTable(route.Table) != expected.Table { + continue + } + + if route.Attributes.Priority != expected.Priority { + continue + } + + result = append(result, &existingRoutes[i]) + } + + return result +} + +//nolint:gocyclo,cyclop +func (ctrl *RouteSpecController) syncRoute(ctx context.Context, r controller.Runtime, logger *zap.Logger, conn *rtnetlink.Conn, + links []rtnetlink.LinkMessage, routes []rtnetlink.RouteMessage, route *network.RouteSpec, +) error { + linkIndex := resolveLinkName(links, route.TypedSpec().OutLinkName) + + destinationStr := route.TypedSpec().Destination.String() + if value.IsZero(route.TypedSpec().Destination) { + destinationStr = "default" + } + + sourceStr := route.TypedSpec().Source.String() + if value.IsZero(route.TypedSpec().Source) { + sourceStr = "" + } + + gatewayStr := route.TypedSpec().Gateway.String() + if value.IsZero(route.TypedSpec().Gateway) { + gatewayStr = "" + } + + switch route.Metadata().Phase() { + case resource.PhaseTearingDown: + for _, existing := range findMatchingRoutes(routes, route.TypedSpec()) { + // delete route + if err := conn.Route.Delete(existing); err != nil { + return fmt.Errorf("error removing route: %w", err) + } + + logger.Info("deleted route", + zap.String("destination", destinationStr), + zap.String("gateway", gatewayStr), + zap.Stringer("table", route.TypedSpec().Table), + zap.String("link", route.TypedSpec().OutLinkName), + zap.Uint32("priority", route.TypedSpec().Priority), + zap.Stringer("family", route.TypedSpec().Family), + ) + } + + // now remove finalizer as address was deleted + if err := r.RemoveFinalizer(ctx, route.Metadata(), ctrl.Name()); err != nil { + return fmt.Errorf("error removing finalizer: %w", err) + } + case resource.PhaseRunning: + if linkIndex == 0 && route.TypedSpec().OutLinkName != "" { + // route can't be created as link doesn't exist (yet), skip it + return nil + } + + matchFound := false + + for _, existing := range findMatchingRoutes(routes, route.TypedSpec()) { + var existingMTU uint32 + + if existing.Attributes.Metrics != nil { + existingMTU = existing.Attributes.Metrics.MTU + } + + // check if existing route matches the spec: if it does, skip update + if existing.Scope == uint8(route.TypedSpec().Scope) && nethelpers.RouteFlags(existing.Flags).Equal(route.TypedSpec().Flags) && + existing.Protocol == uint8(route.TypedSpec().Protocol) && + existing.Attributes.OutIface == linkIndex && + (value.IsZero(route.TypedSpec().Source) || + existing.Attributes.Src.Equal(route.TypedSpec().Source.AsSlice())) && + existingMTU == route.TypedSpec().MTU { + matchFound = true + + continue + } + + // delete the route, it doesn't match the spec + if err := conn.Route.Delete(existing); err != nil { + return fmt.Errorf("error removing route: %w", err) + } + + logger.Debug("removed route due to mismatch", + zap.String("destination", destinationStr), + zap.String("gateway", gatewayStr), + zap.Stringer("table", route.TypedSpec().Table), + zap.String("link", route.TypedSpec().OutLinkName), + zap.Uint32("priority", route.TypedSpec().Priority), + zap.Stringer("family", route.TypedSpec().Family), + zap.Stringer("old_scope", nethelpers.Scope(existing.Scope)), + zap.Stringer("new_scope", route.TypedSpec().Scope), + zap.Stringer("old_flags", nethelpers.RouteFlags(existing.Flags)), + zap.Stringer("new_flags", route.TypedSpec().Flags), + zap.Stringer("old_protocol", nethelpers.RouteProtocol(existing.Protocol)), + zap.Stringer("new_protocol", route.TypedSpec().Protocol), + zap.Uint32("old_link_index", existing.Attributes.OutIface), + zap.Uint32("new_link_index", linkIndex), + zap.Stringer("old_source", existing.Attributes.Src), + zap.String("new_source", sourceStr), + ) + } + + if matchFound { + return nil + } + + routeAttributes := rtnetlink.RouteAttributes{ + Dst: route.TypedSpec().Destination.Addr().AsSlice(), + Src: route.TypedSpec().Source.AsSlice(), + Gateway: route.TypedSpec().Gateway.AsSlice(), + OutIface: linkIndex, + Priority: route.TypedSpec().Priority, + Table: uint32(route.TypedSpec().Table), + } + + if route.TypedSpec().MTU != 0 { + routeAttributes.Metrics = &rtnetlink.RouteMetrics{ + MTU: route.TypedSpec().MTU, + } + } + + // add route + msg := &rtnetlink.RouteMessage{ + Family: uint8(route.TypedSpec().Family), + DstLength: uint8(netipPrefixBitsCorrected(route.TypedSpec().Destination)), + SrcLength: 0, + Protocol: uint8(route.TypedSpec().Protocol), + Scope: uint8(route.TypedSpec().Scope), + Type: uint8(route.TypedSpec().Type), + Flags: uint32(route.TypedSpec().Flags), + Attributes: routeAttributes, + } + + if err := conn.Route.Add(msg); err != nil { + return fmt.Errorf("error adding route: %w, message %+v", err, *msg) + } + + logger.Info("created route", + zap.String("destination", destinationStr), + zap.String("gateway", gatewayStr), + zap.Stringer("table", route.TypedSpec().Table), + zap.String("link", route.TypedSpec().OutLinkName), + zap.Uint32("priority", route.TypedSpec().Priority), + zap.Stringer("family", route.TypedSpec().Family), + ) + } + + return nil +} diff --git a/internal/app/machined/pkg/controllers/network/route_spec_test.go b/internal/app/machined/pkg/controllers/network/route_spec_test.go new file mode 100644 index 0000000..73e9ded --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/route_spec_test.go @@ -0,0 +1,570 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network_test + +import ( + "context" + "fmt" + "log" + "math/rand" + "net" + "net/netip" + "os" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/jsimonetti/rtnetlink" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + "golang.org/x/sys/unix" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type RouteSpecSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *RouteSpecSuite) State() state.State { return suite.state } + +func (suite *RouteSpecSuite) Ctx() context.Context { return suite.ctx } + +func (suite *RouteSpecSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.RouteSpecController{})) + + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.DeviceConfigController{})) + + suite.startRuntime() +} + +func (suite *RouteSpecSuite) uniqueDummyInterface() string { + return fmt.Sprintf("dummy%02x%02x%02x", rand.Int31()&0xff, rand.Int31()&0xff, rand.Int31()&0xff) +} + +func (suite *RouteSpecSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *RouteSpecSuite) assertRoute( + destination netip.Prefix, + gateway netip.Addr, + check func(rtnetlink.RouteMessage) error, +) error { + conn, err := rtnetlink.Dial(nil) + suite.Require().NoError(err) + + defer conn.Close() //nolint:errcheck + + routes, err := conn.Route.List() + suite.Require().NoError(err) + + matching := 0 + + for _, route := range routes { + if !route.Attributes.Gateway.Equal(gateway.AsSlice()) { + continue + } + + if !(int(route.DstLength) == destination.Bits() || (route.DstLength == 0 && destination.Bits() == -1)) { + continue + } + + if !route.Attributes.Dst.Equal(destination.Addr().AsSlice()) { + continue + } + + matching++ + + if err = check(route); err != nil { + return retry.ExpectedError(err) + } + } + + switch { + case matching == 1: + return nil + case matching == 0: + return retry.ExpectedErrorf("route to %s via %s not found", destination, gateway) + default: + return retry.ExpectedErrorf("route to %s via %s found %d matches", destination, gateway, matching) + } +} + +func (suite *RouteSpecSuite) assertNoRoute(destination netip.Prefix, gateway netip.Addr) error { + conn, err := rtnetlink.Dial(nil) + suite.Require().NoError(err) + + defer conn.Close() //nolint:errcheck + + routes, err := conn.Route.List() + suite.Require().NoError(err) + + for _, route := range routes { + if route.Attributes.Gateway.Equal(gateway.AsSlice()) && + (destination.Bits() == int(route.DstLength) || (destination.Bits() == -1 && route.DstLength == 0)) && + route.Attributes.Dst.Equal(destination.Addr().AsSlice()) { + return retry.ExpectedErrorf("route to %s via %s is present", destination, gateway) + } + } + + return nil +} + +func (suite *RouteSpecSuite) TestLoopback() { + loopback := network.NewRouteSpec(network.NamespaceName, "loopback") + *loopback.TypedSpec() = network.RouteSpecSpec{ + Family: nethelpers.FamilyInet4, + Destination: netip.MustParsePrefix("127.0.11.0/24"), + Gateway: netip.MustParseAddr("127.0.11.1"), + OutLinkName: "lo", + Scope: nethelpers.ScopeGlobal, + Table: nethelpers.TableMain, + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + ConfigLayer: network.ConfigMachineConfiguration, + } + + for _, res := range []resource.Resource{loopback} { + suite.Require().NoError(suite.state.Create(suite.ctx, res), "%v", res.Spec()) + } + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertRoute( + netip.MustParsePrefix("127.0.11.0/24"), + netip.MustParseAddr("127.0.11.1"), + func(route rtnetlink.RouteMessage) error { + suite.Assert().EqualValues(0, route.Attributes.Priority) + + return nil + }, + ) + }, + ), + ) + + // teardown the route + for { + ready, err := suite.state.Teardown(suite.ctx, loopback.Metadata()) + suite.Require().NoError(err) + + if ready { + break + } + + time.Sleep(100 * time.Millisecond) + } + + // torn down address should be removed immediately + suite.Assert().NoError( + suite.assertNoRoute( + netip.MustParsePrefix("127.0.11.0/24"), + netip.MustParseAddr("127.0.11.1"), + ), + ) + + suite.Require().NoError(suite.state.Destroy(suite.ctx, loopback.Metadata())) +} + +func (suite *RouteSpecSuite) TestDefaultRoute() { + // adding default route with high metric to avoid messing up with the actual default route + def := network.NewRouteSpec(network.NamespaceName, "default") + *def.TypedSpec() = network.RouteSpecSpec{ + Family: nethelpers.FamilyInet4, + Destination: netip.Prefix{}, + Gateway: netip.MustParseAddr("127.0.11.2"), + Scope: nethelpers.ScopeGlobal, + Table: nethelpers.TableMain, + OutLinkName: "lo", + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + Priority: 1048576, + ConfigLayer: network.ConfigMachineConfiguration, + } + + for _, res := range []resource.Resource{def} { + suite.Require().NoError(suite.state.Create(suite.ctx, res), "%v", res.Spec()) + } + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertRoute( + netip.Prefix{}, netip.MustParseAddr("127.0.11.2"), func(route rtnetlink.RouteMessage) error { + suite.Assert().Nil(route.Attributes.Dst) + suite.Assert().EqualValues(1048576, route.Attributes.Priority) + // make sure not extra route metric attributes are set + suite.Assert().Empty(route.Attributes.Metrics) + + return nil + }, + ) + }, + ), + ) + + // update the route metric and mtu + ctest.UpdateWithConflicts(suite, def, func(defR *network.RouteSpec) error { + defR.TypedSpec().MTU = 1700 + + return nil + }) + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertRoute( + netip.Prefix{}, netip.MustParseAddr("127.0.11.2"), func(route rtnetlink.RouteMessage) error { + suite.Assert().Nil(route.Attributes.Dst) + + if route.Attributes.Metrics == nil || route.Attributes.Metrics.MTU == 0 { + return fmt.Errorf("route metric wasn't updated: %v", route.Attributes.Metrics) + } + + suite.Assert().EqualValues(1700, route.Attributes.Metrics.MTU) + + return nil + }, + ) + }, + ), + ) + + // remove mtu and make sure it's unset + ctest.UpdateWithConflicts(suite, def, func(defR *network.RouteSpec) error { + defR.TypedSpec().MTU = 0 + + return nil + }) + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertRoute( + netip.Prefix{}, netip.MustParseAddr("127.0.11.2"), func(route rtnetlink.RouteMessage) error { + suite.Assert().Nil(route.Attributes.Dst) + + if route.Attributes.Metrics != nil { + return retry.ExpectedErrorf("route mtu expected to be empty, got: %d", route.Attributes.Metrics.MTU) + } + + suite.Assert().Empty(route.Attributes.Metrics) + + return nil + }, + ) + }, + ), + ) + + // teardown the route + for { + ready, err := suite.state.Teardown(suite.ctx, def.Metadata()) + suite.Require().NoError(err) + + if ready { + break + } + + time.Sleep(100 * time.Millisecond) + } + + // torn down route should be removed immediately + suite.Assert().NoError(suite.assertNoRoute(netip.Prefix{}, netip.MustParseAddr("127.0.11.2"))) + + suite.Require().NoError(suite.state.Destroy(suite.ctx, def.Metadata())) +} + +func (suite *RouteSpecSuite) TestDefaultAndInterfaceRoutes() { + dummyInterface := suite.uniqueDummyInterface() + + conn, err := rtnetlink.Dial(nil) + suite.Require().NoError(err) + + defer conn.Close() //nolint:errcheck + + suite.Require().NoError( + conn.Link.New( + &rtnetlink.LinkMessage{ + Type: unix.ARPHRD_ETHER, + Flags: unix.IFF_UP, + Change: unix.IFF_UP, + Attributes: &rtnetlink.LinkAttributes{ + Name: dummyInterface, + MTU: 1400, + Info: &rtnetlink.LinkInfo{ + Kind: "dummy", + }, + }, + }, + ), + ) + + iface, err := net.InterfaceByName(dummyInterface) + suite.Require().NoError(err) + + defer conn.Link.Delete(uint32(iface.Index)) //nolint:errcheck + + localIP := net.ParseIP("10.28.0.27").To4() + + suite.Require().NoError( + conn.Address.New( + &rtnetlink.AddressMessage{ + Family: unix.AF_INET, + PrefixLength: 32, + Scope: unix.RT_SCOPE_UNIVERSE, + Index: uint32(iface.Index), + Attributes: &rtnetlink.AddressAttributes{ + Address: localIP, + Local: localIP, + }, + }, + ), + ) + + def := network.NewRouteSpec(network.NamespaceName, "default") + *def.TypedSpec() = network.RouteSpecSpec{ + Family: nethelpers.FamilyInet4, + Destination: netip.Prefix{}, + Gateway: netip.MustParseAddr("10.28.0.1"), + Source: netip.MustParseAddr("10.28.0.27"), + Table: nethelpers.TableMain, + OutLinkName: dummyInterface, + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + Priority: 1048576, + ConfigLayer: network.ConfigMachineConfiguration, + } + def.TypedSpec().Normalize() + + host := network.NewRouteSpec(network.NamespaceName, "aninterface") + *host.TypedSpec() = network.RouteSpecSpec{ + Family: nethelpers.FamilyInet4, + Destination: netip.MustParsePrefix("10.28.0.1/32"), + Gateway: netip.MustParseAddr("0.0.0.0"), + Source: netip.MustParseAddr("10.28.0.27"), + Table: nethelpers.TableMain, + OutLinkName: dummyInterface, + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + Priority: 1048576, + ConfigLayer: network.ConfigMachineConfiguration, + } + host.TypedSpec().Normalize() + + for _, res := range []resource.Resource{def, host} { + suite.Require().NoError(suite.state.Create(suite.ctx, res), "%v", res.Spec()) + } + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + if err := suite.assertRoute( + netip.Prefix{}, netip.MustParseAddr("10.28.0.1"), func(route rtnetlink.RouteMessage) error { + suite.Assert().Nil(route.Attributes.Dst) + suite.Assert().EqualValues(1048576, route.Attributes.Priority) + + return nil + }, + ); err != nil { + return err + } + + return suite.assertRoute( + netip.MustParsePrefix("10.28.0.1/32"), netip.Addr{}, func(route rtnetlink.RouteMessage) error { + suite.Assert().Nil(route.Attributes.Gateway) + suite.Assert().EqualValues(1048576, route.Attributes.Priority) + + return nil + }, + ) + }, + ), + ) + + // teardown the routes + for { + ready, err := suite.state.Teardown(suite.ctx, def.Metadata()) + suite.Require().NoError(err) + + if ready { + break + } + + time.Sleep(100 * time.Millisecond) + } + + for { + ready, err := suite.state.Teardown(suite.ctx, host.Metadata()) + suite.Require().NoError(err) + + if ready { + break + } + + time.Sleep(100 * time.Millisecond) + } + + // torn down route should be removed immediately + suite.Assert().NoError(suite.assertNoRoute(netip.Prefix{}, netip.MustParseAddr("10.28.0.1"))) + suite.Assert().NoError(suite.assertNoRoute(netip.MustParsePrefix("10.28.0.1/32"), netip.Addr{})) + + suite.Require().NoError(suite.state.Destroy(suite.ctx, def.Metadata())) +} + +func (suite *RouteSpecSuite) TestLinkLocalRoute() { + dummyInterface := suite.uniqueDummyInterface() + + conn, err := rtnetlink.Dial(nil) + suite.Require().NoError(err) + + defer conn.Close() //nolint:errcheck + + suite.Require().NoError( + conn.Link.New( + &rtnetlink.LinkMessage{ + Type: unix.ARPHRD_ETHER, + Flags: unix.IFF_UP, + Change: unix.IFF_UP, + Attributes: &rtnetlink.LinkAttributes{ + Name: dummyInterface, + MTU: 1500, + Info: &rtnetlink.LinkInfo{ + Kind: "dummy", + }, + }, + }, + ), + ) + + iface, err := net.InterfaceByName(dummyInterface) + suite.Require().NoError(err) + + defer conn.Link.Delete(uint32(iface.Index)) //nolint:errcheck + + localIP := net.ParseIP("10.28.0.27").To4() + + suite.Require().NoError( + conn.Address.New( + &rtnetlink.AddressMessage{ + Family: unix.AF_INET, + PrefixLength: 24, + Scope: unix.RT_SCOPE_UNIVERSE, + Index: uint32(iface.Index), + Attributes: &rtnetlink.AddressAttributes{ + Address: localIP, + Local: localIP, + }, + }, + ), + ) + + ll := network.NewRouteSpec(network.NamespaceName, "ll") + *ll.TypedSpec() = network.RouteSpecSpec{ + Family: nethelpers.FamilyInet4, + Destination: netip.MustParsePrefix("169.254.169.254/32"), + Gateway: netip.MustParseAddr("10.28.0.1"), + Source: netip.MustParseAddr("10.28.0.27"), + Table: nethelpers.TableMain, + OutLinkName: dummyInterface, + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + Priority: 1048576, + ConfigLayer: network.ConfigMachineConfiguration, + } + ll.TypedSpec().Normalize() + + for _, res := range []resource.Resource{ll} { + suite.Require().NoError(suite.state.Create(suite.ctx, res), "%v", res.Spec()) + } + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertRoute( + netip.MustParsePrefix("169.254.169.254/32"), + netip.MustParseAddr("10.28.0.1"), + func(route rtnetlink.RouteMessage) error { + suite.Assert().EqualValues(1048576, route.Attributes.Priority) + + return nil + }, + ) + }, + ), + ) + + // teardown the routes + for { + ready, err := suite.state.Teardown(suite.ctx, ll.Metadata()) + suite.Require().NoError(err) + + if ready { + break + } + + time.Sleep(100 * time.Millisecond) + } + + // torn down route should be removed immediately + suite.Assert().NoError( + suite.assertNoRoute( + netip.MustParsePrefix("169.254.169.254/32"), + netip.MustParseAddr("10.28.0.1"), + ), + ) + + suite.Require().NoError(suite.state.Destroy(suite.ctx, ll.Metadata())) +} + +func (suite *RouteSpecSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestRouteSpecSuite(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("requires root") + } + + suite.Run(t, new(RouteSpecSuite)) +} diff --git a/internal/app/machined/pkg/controllers/network/route_status.go b/internal/app/machined/pkg/controllers/network/route_status.go new file mode 100644 index 0000000..481966c --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/route_status.go @@ -0,0 +1,147 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "context" + "fmt" + "net/netip" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/jsimonetti/rtnetlink" + "go.uber.org/zap" + "golang.org/x/sys/unix" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network/watch" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// RouteStatusController manages secrets.Etcd based on configuration. +type RouteStatusController struct{} + +// Name implements controller.Controller interface. +func (ctrl *RouteStatusController) Name() string { + return "network.RouteStatusController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *RouteStatusController) Inputs() []controller.Input { + return nil +} + +// Outputs implements controller.Controller interface. +func (ctrl *RouteStatusController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.RouteStatusType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *RouteStatusController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + watcher, err := watch.NewRtNetlink(watch.NewDefaultRateLimitedTrigger(ctx, r), unix.RTMGRP_IPV4_MROUTE|unix.RTMGRP_IPV4_ROUTE|unix.RTMGRP_IPV6_MROUTE|unix.RTMGRP_IPV6_ROUTE) + if err != nil { + return err + } + + defer watcher.Done() + + conn, err := rtnetlink.Dial(nil) + if err != nil { + return fmt.Errorf("error dialing rtnetlink socket: %w", err) + } + + defer conn.Close() //nolint:errcheck + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + // build links lookup table + links, err := conn.Link.List() + if err != nil { + return fmt.Errorf("error listing links: %w", err) + } + + linkLookup := make(map[uint32]string, len(links)) + + for _, link := range links { + linkLookup[link.Index] = link.Attributes.Name + } + + // list resources for cleanup + list, err := r.List(ctx, resource.NewMetadata(network.NamespaceName, network.RouteStatusType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing resources: %w", err) + } + + itemsToDelete := map[resource.ID]struct{}{} + + for _, r := range list.Items { + itemsToDelete[r.Metadata().ID()] = struct{}{} + } + + routes, err := conn.Route.List() + if err != nil { + return fmt.Errorf("error listing routes: %w", err) + } + + for _, route := range routes { + dstAddr, _ := netip.AddrFromSlice(route.Attributes.Dst) + dstPrefix := netip.PrefixFrom(dstAddr, int(route.DstLength)) + srcAddr, _ := netip.AddrFromSlice(route.Attributes.Src) + gatewayAddr, _ := netip.AddrFromSlice(route.Attributes.Gateway) + outLinkName := linkLookup[route.Attributes.OutIface] + + id := network.RouteID(nethelpers.RoutingTable(route.Table), nethelpers.Family(route.Family), dstPrefix, gatewayAddr, route.Attributes.Priority, outLinkName) + + if err = r.Modify(ctx, network.NewRouteStatus(network.NamespaceName, id), func(r resource.Resource) error { + status := r.(*network.RouteStatus).TypedSpec() + + status.Family = nethelpers.Family(route.Family) + status.Destination = dstPrefix + status.Source = srcAddr + status.Gateway = gatewayAddr + status.OutLinkIndex = route.Attributes.OutIface + status.OutLinkName = outLinkName + status.Priority = route.Attributes.Priority + status.Table = nethelpers.RoutingTable(route.Table) + status.Scope = nethelpers.Scope(route.Scope) + status.Type = nethelpers.RouteType(route.Type) + status.Protocol = nethelpers.RouteProtocol(route.Protocol) + status.Flags = nethelpers.RouteFlags(route.Flags) + + if route.Attributes.Metrics != nil { + status.MTU = route.Attributes.Metrics.MTU + } else { + status.MTU = 0 + } + + return nil + }); err != nil { + return fmt.Errorf("error modifying resource: %w", err) + } + + delete(itemsToDelete, id) + } + + for id := range itemsToDelete { + if err = r.Destroy(ctx, resource.NewMetadata(network.NamespaceName, network.RouteStatusType, id, resource.VersionUndefined)); err != nil { + return fmt.Errorf("error deleting route status %q: %w", id, err) + } + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/network/route_status_test.go b/internal/app/machined/pkg/controllers/network/route_status_test.go new file mode 100644 index 0000000..1d5561e --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/route_status_test.go @@ -0,0 +1,93 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:dupl +package network_test + +import ( + "context" + "log" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type RouteStatusSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *RouteStatusSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.RouteStatusController{})) + + suite.startRuntime() +} + +func (suite *RouteStatusSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *RouteStatusSuite) assertRoutes(requiredIDs []string, check func(*network.RouteStatus, *assert.Assertions)) { + assertResources(suite.ctx, suite.T(), suite.state, requiredIDs, check) +} + +func (suite *RouteStatusSuite) TestRoutes() { + suite.assertRoutes( + []string{"local/inet4//127.0.0.0/8/0"}, func(r *network.RouteStatus, asrt *assert.Assertions) { + asrt.True(r.TypedSpec().Source.IsLoopback()) + asrt.Equal("lo", r.TypedSpec().OutLinkName) + asrt.Equal(nethelpers.TableLocal, r.TypedSpec().Table) + asrt.Equal(nethelpers.ScopeHost, r.TypedSpec().Scope) + asrt.Equal(nethelpers.TypeLocal, r.TypedSpec().Type) + asrt.Equal(nethelpers.ProtocolKernel, r.TypedSpec().Protocol) + asrt.EqualValues(0, r.TypedSpec().MTU) + }, + ) +} + +func (suite *RouteStatusSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestRouteStatusSuite(t *testing.T) { + suite.Run(t, new(RouteStatusSuite)) +} diff --git a/internal/app/machined/pkg/controllers/network/status.go b/internal/app/machined/pkg/controllers/network/status.go new file mode 100644 index 0000000..05d70fa --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/status.go @@ -0,0 +1,178 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/value" + "go.uber.org/zap" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/files" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// StatusController manages network.Status based on state of other resources. +type StatusController struct { + V1Alpha1Mode runtime.Mode +} + +// Name implements controller.Controller interface. +func (ctrl *StatusController) Name() string { + return "network.StatusController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *StatusController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: network.NamespaceName, + Type: network.NodeAddressType, + ID: optional.Some(network.NodeAddressCurrentID), + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.RouteStatusType, + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.HostnameStatusType, + Kind: controller.InputWeak, + }, + { + Namespace: files.NamespaceName, + Type: files.EtcFileStatusType, + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.ProbeStatusType, + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *StatusController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.StatusType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *StatusController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + result := network.StatusSpec{} + + // addresses + currentAddresses, err := safe.ReaderGet[*network.NodeAddress](ctx, r, resource.NewMetadata(network.NamespaceName, network.NodeAddressType, network.NodeAddressCurrentID, resource.VersionUndefined)) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting resource: %w", err) + } + } else { + result.AddressReady = len(currentAddresses.TypedSpec().Addresses) > 0 + } + + // connectivity + // if any probes are defined, use their status, otherwise rely on presence of the default gateway + probeStatuses, err := safe.ReaderListAll[*network.ProbeStatus](ctx, r) + if err != nil { + return fmt.Errorf("error getting probe statuses: %w", err) + } + + allProbesSuccess := true + + for iter := probeStatuses.Iterator(); iter.Next(); { + if !iter.Value().TypedSpec().Success { + allProbesSuccess = false + + break + } + } + + if probeStatuses.Len() > 0 && allProbesSuccess { + result.ConnectivityReady = true + } else if probeStatuses.Len() == 0 { + var routes safe.List[*network.RouteStatus] + + routes, err = safe.ReaderListAll[*network.RouteStatus](ctx, r) + if err != nil { + return fmt.Errorf("error getting routes: %w", err) + } + + for iter := routes.Iterator(); iter.Next(); { + if value.IsZero(iter.Value().TypedSpec().Destination) { + result.ConnectivityReady = true + + break + } + } + } + + // hostname + _, err = r.Get(ctx, resource.NewMetadata(network.NamespaceName, network.HostnameStatusType, network.HostnameID, resource.VersionUndefined)) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting resource: %w", err) + } + } else { + result.HostnameReady = true + } + + // etc files + result.EtcFilesReady = true + + for _, requiredFile := range []string{"hosts", "resolv.conf"} { + // in container mode, ignore resolv.conf, it's managed by the container runtime + if ctrl.V1Alpha1Mode.InContainer() && requiredFile == "resolv.conf" { + continue + } + + _, err = r.Get(ctx, resource.NewMetadata(files.NamespaceName, files.EtcFileStatusType, requiredFile, resource.VersionUndefined)) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting resource: %w", err) + } + + result.EtcFilesReady = false + + break + } + } + + // update output status + if err = safe.WriterModify(ctx, r, network.NewStatus(network.NamespaceName, network.StatusID), + func(r *network.Status) error { + *r.TypedSpec() = result + + return nil + }); err != nil { + return fmt.Errorf("error modifying output status: %w", err) + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/network/status_test.go b/internal/app/machined/pkg/controllers/network/status_test.go new file mode 100644 index 0000000..88a91e6 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/status_test.go @@ -0,0 +1,113 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network_test + +import ( + "net/netip" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/files" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type StatusSuite struct { + ctest.DefaultSuite +} + +func (suite *StatusSuite) TestNone() { + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{network.StatusID}, func(r *network.Status, assert *assert.Assertions) { + assert.Equal(network.StatusSpec{}, *r.TypedSpec()) + }) +} + +func (suite *StatusSuite) TestAddresses() { + nodeAddress := network.NewNodeAddress(network.NamespaceName, network.NodeAddressCurrentID) + nodeAddress.TypedSpec().Addresses = []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")} + + suite.Require().NoError(suite.State().Create(suite.Ctx(), nodeAddress)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{network.StatusID}, func(r *network.Status, assert *assert.Assertions) { + assert.Equal(network.StatusSpec{AddressReady: true}, *r.TypedSpec()) + }) +} + +func (suite *StatusSuite) TestRoutes() { + route := network.NewRouteStatus(network.NamespaceName, "foo") + route.TypedSpec().Gateway = netip.MustParseAddr("10.0.0.1") + + suite.Require().NoError(suite.State().Create(suite.Ctx(), route)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{network.StatusID}, func(r *network.Status, assert *assert.Assertions) { + assert.Equal(network.StatusSpec{ConnectivityReady: true}, *r.TypedSpec()) + }) +} + +func (suite *StatusSuite) TestProbeStatuses() { + probeStatus := network.NewProbeStatus(network.NamespaceName, "foo") + probeStatus.TypedSpec().Success = true + suite.Require().NoError(suite.State().Create(suite.Ctx(), probeStatus)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{network.StatusID}, func(r *network.Status, assert *assert.Assertions) { + assert.Equal(network.StatusSpec{ConnectivityReady: true}, *r.TypedSpec()) + }) + + // failing probe make status not ready + route := network.NewRouteStatus(network.NamespaceName, "foo") + route.TypedSpec().Gateway = netip.MustParseAddr("10.0.0.1") + + suite.Require().NoError(suite.State().Create(suite.Ctx(), route)) + + probeStatusFail := network.NewProbeStatus(network.NamespaceName, "failing") + suite.Require().NoError(suite.State().Create(suite.Ctx(), probeStatusFail)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{network.StatusID}, func(r *network.Status, assert *assert.Assertions) { + assert.Equal(network.StatusSpec{}, *r.TypedSpec()) + }) +} + +func (suite *StatusSuite) TestHostname() { + hostname := network.NewHostnameStatus(network.NamespaceName, network.HostnameID) + hostname.TypedSpec().Hostname = "foo" + + suite.Require().NoError(suite.State().Create(suite.Ctx(), hostname)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{network.StatusID}, func(r *network.Status, assert *assert.Assertions) { + assert.Equal(network.StatusSpec{HostnameReady: true}, *r.TypedSpec()) + }) +} + +func (suite *StatusSuite) TestEtcFiles() { + for _, f := range []string{"hosts", "resolv.conf"} { + suite.Require().NoError(suite.State().Create(suite.Ctx(), files.NewEtcFileStatus(files.NamespaceName, f))) + } + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{network.StatusID}, func(r *network.Status, assert *assert.Assertions) { + assert.Equal(network.StatusSpec{EtcFilesReady: true}, *r.TypedSpec()) + }) +} + +func TestStatusSuite(t *testing.T) { + suite.Run(t, &StatusSuite{ + DefaultSuite: ctest.DefaultSuite{ + Timeout: 3 * time.Second, + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController( + &netctrl.StatusController{ + V1Alpha1Mode: runtime.ModeMetal, + }, + )) + }, + }, + }) +} diff --git a/internal/app/machined/pkg/controllers/network/timeserver_config.go b/internal/app/machined/pkg/controllers/network/timeserver_config.go new file mode 100644 index 0000000..de42585 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/timeserver_config.go @@ -0,0 +1,203 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "context" + "fmt" + "slices" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/go-procfs/procfs" + "go.uber.org/zap" + + talosconfig "github.com/siderolabs/talos/pkg/machinery/config" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// TimeServerConfigController manages network.TimeServerSpec based on machine configuration, kernel cmdline. +type TimeServerConfigController struct { + Cmdline *procfs.Cmdline +} + +// Name implements controller.Controller interface. +func (ctrl *TimeServerConfigController) Name() string { + return "network.TimeServerConfigController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *TimeServerConfigController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: config.MachineConfigType, + ID: optional.Some(config.V1Alpha1ID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *TimeServerConfigController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.TimeServerSpecType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *TimeServerConfigController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + touchedIDs := make(map[resource.ID]struct{}) + + var cfgProvider talosconfig.Config + + cfg, err := safe.ReaderGetByID[*config.MachineConfig](ctx, r, config.V1Alpha1ID) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting config: %w", err) + } + } else if cfg.Config().Machine() != nil { + cfgProvider = cfg.Config() + } + + var specs []network.TimeServerSpecSpec + + // defaults + specs = append(specs, ctrl.getDefault()) + + // parse kernel cmdline for the default gateway + cmdlineServers := ctrl.parseCmdline(logger) + if cmdlineServers.NTPServers != nil { + specs = append(specs, cmdlineServers) + } + + // parse machine configuration for specs + if cfgProvider != nil { + configServers := ctrl.parseMachineConfiguration(cfgProvider) + + if configServers.NTPServers != nil { + specs = append(specs, configServers) + } + } + + var ids []string + + ids, err = ctrl.apply(ctx, r, specs) + if err != nil { + return fmt.Errorf("error applying specs: %w", err) + } + + for _, id := range ids { + touchedIDs[id] = struct{}{} + } + + // list specs for cleanup + list, err := r.List(ctx, resource.NewMetadata(network.ConfigNamespaceName, network.TimeServerSpecType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing resources: %w", err) + } + + for _, res := range list.Items { + if res.Metadata().Owner() != ctrl.Name() { + // skip specs created by other controllers + continue + } + + if _, ok := touchedIDs[res.Metadata().ID()]; !ok { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return fmt.Errorf("error cleaning up specs: %w", err) + } + } + } + + r.ResetRestartBackoff() + } +} + +//nolint:dupl +func (ctrl *TimeServerConfigController) apply(ctx context.Context, r controller.Runtime, specs []network.TimeServerSpecSpec) ([]resource.ID, error) { + ids := make([]string, 0, len(specs)) + + for _, spec := range specs { + id := network.LayeredID(spec.ConfigLayer, network.TimeServerID) + + if err := r.Modify( + ctx, + network.NewTimeServerSpec(network.ConfigNamespaceName, id), + func(r resource.Resource) error { + *r.(*network.TimeServerSpec).TypedSpec() = spec + + return nil + }, + ); err != nil { + return ids, err + } + + ids = append(ids, id) + } + + return ids, nil +} + +func (ctrl *TimeServerConfigController) getDefault() (spec network.TimeServerSpecSpec) { + spec.NTPServers = []string{constants.DefaultNTPServer} + spec.ConfigLayer = network.ConfigDefault + + return spec +} + +func (ctrl *TimeServerConfigController) parseCmdline(logger *zap.Logger) (spec network.TimeServerSpecSpec) { + if ctrl.Cmdline == nil { + return + } + + settings, err := ParseCmdlineNetwork(ctrl.Cmdline) + if err != nil { + logger.Warn("ignoring error", zap.Error(err)) + + return + } + + if len(settings.NTPAddresses) == 0 { + return + } + + spec.NTPServers = make([]string, len(settings.NTPAddresses)) + spec.ConfigLayer = network.ConfigCmdline + + for i := range settings.NTPAddresses { + spec.NTPServers[i] = settings.NTPAddresses[i].String() + } + + return spec +} + +func (ctrl *TimeServerConfigController) parseMachineConfiguration(cfgProvider talosconfig.Config) (spec network.TimeServerSpecSpec) { + if len(cfgProvider.Machine().Time().Servers()) == 0 { + return + } + + spec.NTPServers = slices.Clone(cfgProvider.Machine().Time().Servers()) + spec.ConfigLayer = network.ConfigMachineConfiguration + + return spec +} diff --git a/internal/app/machined/pkg/controllers/network/timeserver_config_test.go b/internal/app/machined/pkg/controllers/network/timeserver_config_test.go new file mode 100644 index 0000000..aaec300 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/timeserver_config_test.go @@ -0,0 +1,196 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network_test + +import ( + "context" + "log" + "net/url" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-procfs/procfs" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type TimeServerConfigSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *TimeServerConfigSuite) State() state.State { return suite.state } + +func (suite *TimeServerConfigSuite) Ctx() context.Context { return suite.ctx } + +func (suite *TimeServerConfigSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) +} + +func (suite *TimeServerConfigSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *TimeServerConfigSuite) assertTimeServers( + requiredIDs []string, + check func(*network.TimeServerSpec, *assert.Assertions), +) { + assertResources(suite.ctx, suite.T(), suite.state, requiredIDs, check, rtestutils.WithNamespace(network.ConfigNamespaceName)) +} + +func (suite *TimeServerConfigSuite) assertNoTimeServer(id string) error { + resources, err := suite.state.List( + suite.ctx, + resource.NewMetadata(network.ConfigNamespaceName, network.TimeServerSpecType, "", resource.VersionUndefined), + ) + if err != nil { + return err + } + + for _, res := range resources.Items { + if res.Metadata().ID() == id { + return retry.ExpectedErrorf("spec %q is still there", id) + } + } + + return nil +} + +func (suite *TimeServerConfigSuite) TestDefaults() { + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.TimeServerConfigController{})) + + suite.startRuntime() + + suite.assertTimeServers( + []string{ + "default/timeservers", + }, func(r *network.TimeServerSpec, asrt *assert.Assertions) { + asrt.Equal([]string{constants.DefaultNTPServer}, r.TypedSpec().NTPServers) + asrt.Equal(network.ConfigDefault, r.TypedSpec().ConfigLayer) + }, + ) +} + +func (suite *TimeServerConfigSuite) TestCmdline() { + suite.Require().NoError( + suite.runtime.RegisterController( + &netctrl.TimeServerConfigController{ + Cmdline: procfs.NewCmdline("ip=172.20.0.2:172.21.0.1:172.20.0.1:255.255.255.0:master1:eth1::10.0.0.1:10.0.0.2:10.0.0.1"), + }, + ), + ) + + suite.startRuntime() + + suite.assertTimeServers( + []string{ + "cmdline/timeservers", + }, func(r *network.TimeServerSpec, asrt *assert.Assertions) { + asrt.Equal([]string{"10.0.0.1"}, r.TypedSpec().NTPServers) + }, + ) +} + +func (suite *TimeServerConfigSuite) TestMachineConfiguration() { + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.TimeServerConfigController{})) + + suite.startRuntime() + + u, err := url.Parse("https://foo:6443") + suite.Require().NoError(err) + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineTime: &v1alpha1.TimeConfig{ + TimeServers: []string{"za.pool.ntp.org", "pool.ntp.org"}, + }, + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: u, + }, + }, + }, + }, + ), + ) + + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + suite.assertTimeServers( + []string{ + "configuration/timeservers", + }, func(r *network.TimeServerSpec, asrt *assert.Assertions) { + asrt.Equal([]string{"za.pool.ntp.org", "pool.ntp.org"}, r.TypedSpec().NTPServers) + }, + ) + + ctest.UpdateWithConflicts(suite, cfg, func(r *config.MachineConfig) error { + r.Container().RawV1Alpha1().MachineConfig.MachineTime = nil + + return nil + }) + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertNoTimeServer("configuration/timeservers") + }, + ), + ) +} + +func (suite *TimeServerConfigSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestTimeServerConfigSuite(t *testing.T) { + suite.Run(t, new(TimeServerConfigSuite)) +} diff --git a/internal/app/machined/pkg/controllers/network/timeserver_merge.go b/internal/app/machined/pkg/controllers/network/timeserver_merge.go new file mode 100644 index 0000000..a9410bd --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/timeserver_merge.go @@ -0,0 +1,132 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package network provides controllers which manage network resources. +// +//nolint:dupl +package network + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// TimeServerMergeController merges network.TimeServerSpec in network.ConfigNamespace and produces final network.TimeServerSpec in network.Namespace. +type TimeServerMergeController struct{} + +// Name implements controller.Controller interface. +func (ctrl *TimeServerMergeController) Name() string { + return "network.TimeServerMergeController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *TimeServerMergeController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: network.ConfigNamespaceName, + Type: network.TimeServerSpecType, + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.TimeServerSpecType, + Kind: controller.InputDestroyReady, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *TimeServerMergeController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.TimeServerSpecType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *TimeServerMergeController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + // list source network configuration resources + list, err := r.List(ctx, resource.NewMetadata(network.ConfigNamespaceName, network.TimeServerSpecType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing source network addresses: %w", err) + } + + // simply merge by layers, overriding with the next configuration layer + var final network.TimeServerSpecSpec + + for _, res := range list.Items { + spec := res.(*network.TimeServerSpec) //nolint:errcheck,forcetypeassert + + if final.NTPServers != nil && spec.TypedSpec().ConfigLayer < final.ConfigLayer { + // skip this spec, as existing one is higher layer + continue + } + + if spec.TypedSpec().ConfigLayer == final.ConfigLayer { + // merge server lists on the same level + final.NTPServers = append(final.NTPServers, spec.TypedSpec().NTPServers...) + } else { + // otherwise, replace the lists + final = *spec.TypedSpec() + } + } + + if final.NTPServers != nil { + if err = r.Modify(ctx, network.NewTimeServerSpec(network.NamespaceName, network.TimeServerID), func(res resource.Resource) error { + spec := res.(*network.TimeServerSpec) //nolint:errcheck,forcetypeassert + + *spec.TypedSpec() = final + + return nil + }); err != nil { + if state.IsPhaseConflictError(err) { + // conflict + final.NTPServers = nil + + r.QueueReconcile() + } else { + return fmt.Errorf("error updating resource: %w", err) + } + } + } + + if final.NTPServers == nil { + // remove existing + var okToDestroy bool + + md := resource.NewMetadata(network.NamespaceName, network.TimeServerSpecType, network.TimeServerID, resource.VersionUndefined) + + okToDestroy, err = r.Teardown(ctx, md) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error cleaning up specs: %w", err) + } + + if okToDestroy { + if err = r.Destroy(ctx, md); err != nil { + return fmt.Errorf("error cleaning up specs: %w", err) + } + } + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/network/timeserver_merge_test.go b/internal/app/machined/pkg/controllers/network/timeserver_merge_test.go new file mode 100644 index 0000000..a1560cc --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/timeserver_merge_test.go @@ -0,0 +1,131 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:dupl +package network_test + +import ( + "context" + "log" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type TimeServerMergeSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *TimeServerMergeSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.TimeServerMergeController{})) + + suite.startRuntime() +} + +func (suite *TimeServerMergeSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *TimeServerMergeSuite) assertTimeServers( + requiredIDs []string, + check func(*network.TimeServerSpec, *assert.Assertions), +) { + assertResources(suite.ctx, suite.T(), suite.state, requiredIDs, check) +} + +func (suite *TimeServerMergeSuite) TestMerge() { + def := network.NewTimeServerSpec(network.ConfigNamespaceName, "default/timeservers") + *def.TypedSpec() = network.TimeServerSpecSpec{ + NTPServers: []string{constants.DefaultNTPServer}, + ConfigLayer: network.ConfigDefault, + } + + dhcp1 := network.NewTimeServerSpec(network.ConfigNamespaceName, "dhcp/eth0") + *dhcp1.TypedSpec() = network.TimeServerSpecSpec{ + NTPServers: []string{"ntp.eth0"}, + ConfigLayer: network.ConfigOperator, + } + + dhcp2 := network.NewTimeServerSpec(network.ConfigNamespaceName, "dhcp/eth1") + *dhcp2.TypedSpec() = network.TimeServerSpecSpec{ + NTPServers: []string{"ntp.eth1"}, + ConfigLayer: network.ConfigOperator, + } + + static := network.NewTimeServerSpec(network.ConfigNamespaceName, "configuration/timeservers") + *static.TypedSpec() = network.TimeServerSpecSpec{ + NTPServers: []string{"my.ntp"}, + ConfigLayer: network.ConfigMachineConfiguration, + } + + for _, res := range []resource.Resource{def, dhcp1, dhcp2, static} { + suite.Require().NoError(suite.state.Create(suite.ctx, res), "%v", res.Spec()) + } + + suite.assertTimeServers( + []string{ + "timeservers", + }, func(r *network.TimeServerSpec, asrt *assert.Assertions) { + asrt.Equal(*static.TypedSpec(), *r.TypedSpec()) + }, + ) + + suite.Require().NoError(suite.state.Destroy(suite.ctx, static.Metadata())) + + suite.assertTimeServers( + []string{ + "timeservers", + }, func(r *network.TimeServerSpec, asrt *assert.Assertions) { + asrt.Equal([]string{"ntp.eth0", "ntp.eth1"}, r.TypedSpec().NTPServers) + }, + ) +} + +func (suite *TimeServerMergeSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestTimeServerMergeSuite(t *testing.T) { + suite.Run(t, new(TimeServerMergeSuite)) +} diff --git a/internal/app/machined/pkg/controllers/network/timeserver_spec.go b/internal/app/machined/pkg/controllers/network/timeserver_spec.go new file mode 100644 index 0000000..c50d656 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/timeserver_spec.go @@ -0,0 +1,114 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package network + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// TimeServerSpecController applies network.TimeServerSpec to the actual interfaces. +type TimeServerSpecController struct{} + +// Name implements controller.Controller interface. +func (ctrl *TimeServerSpecController) Name() string { + return "network.TimeServerSpecController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *TimeServerSpecController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: network.NamespaceName, + Type: network.TimeServerSpecType, + Kind: controller.InputStrong, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *TimeServerSpecController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.TimeServerStatusType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *TimeServerSpecController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + // as there's nothing to do actually apply time servers, simply copy spec to status + + // list source network configuration resources + list, err := r.List(ctx, resource.NewMetadata(network.NamespaceName, network.TimeServerSpecType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error listing source network addresses: %w", err) + } + + // add finalizers for all live resources + for _, res := range list.Items { + if res.Metadata().Phase() != resource.PhaseRunning { + continue + } + + if err = r.AddFinalizer(ctx, res.Metadata(), ctrl.Name()); err != nil { + return fmt.Errorf("error adding finalizer: %w", err) + } + } + + // loop over specs and sync to statuses + for _, res := range list.Items { + spec := res.(*network.TimeServerSpec) //nolint:forcetypeassert,errcheck + + switch spec.Metadata().Phase() { + case resource.PhaseTearingDown: + if err = r.Destroy(ctx, resource.NewMetadata(network.NamespaceName, network.TimeServerStatusType, spec.Metadata().ID(), resource.VersionUndefined)); err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error destroying status: %w", err) + } + + if err = r.RemoveFinalizer(ctx, spec.Metadata(), ctrl.Name()); err != nil { + return fmt.Errorf("error removing finalizer: %w", err) + } + case resource.PhaseRunning: + ntps := make([]string, len(spec.TypedSpec().NTPServers)) + + for i := range ntps { + ntps[i] = spec.TypedSpec().NTPServers[i] + } + + logger.Info("setting time servers", zap.Strings("addresses", ntps)) + + if err = r.Modify(ctx, network.NewTimeServerStatus(network.NamespaceName, spec.Metadata().ID()), func(r resource.Resource) error { + status := r.(*network.TimeServerStatus) //nolint:forcetypeassert,errcheck + + status.TypedSpec().NTPServers = spec.TypedSpec().NTPServers + + return nil + }); err != nil { + return fmt.Errorf("error modifying status: %w", err) + } + } + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/network/timeserver_spec_test.go b/internal/app/machined/pkg/controllers/network/timeserver_spec_test.go new file mode 100644 index 0000000..4a93a83 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/timeserver_spec_test.go @@ -0,0 +1,119 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:dupl +package network_test + +import ( + "context" + "log" + "reflect" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + netctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +type TimeServerSpecSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *TimeServerSpecSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer())) + suite.Require().NoError(err) + + suite.Require().NoError(suite.runtime.RegisterController(&netctrl.TimeServerSpecController{})) + + suite.startRuntime() +} + +func (suite *TimeServerSpecSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *TimeServerSpecSuite) assertStatus(id string, servers ...string) error { + r, err := suite.state.Get( + suite.ctx, + resource.NewMetadata(network.NamespaceName, network.TimeServerStatusType, id, resource.VersionUndefined), + ) + if err != nil { + if state.IsNotFoundError(err) { + return retry.ExpectedError(err) + } + + return err + } + + status := r.(*network.TimeServerStatus) //nolint:errcheck,forcetypeassert + + if !reflect.DeepEqual(status.TypedSpec().NTPServers, servers) { + return retry.ExpectedErrorf("server list mismatch: %q != %q", status.TypedSpec().NTPServers, servers) + } + + return nil +} + +func (suite *TimeServerSpecSuite) TestSpec() { + spec := network.NewTimeServerSpec(network.NamespaceName, "timeservers") + *spec.TypedSpec() = network.TimeServerSpecSpec{ + NTPServers: []string{constants.DefaultNTPServer}, + ConfigLayer: network.ConfigDefault, + } + + for _, res := range []resource.Resource{spec} { + suite.Require().NoError(suite.state.Create(suite.ctx, res), "%v", res.Spec()) + } + + suite.Assert().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertStatus("timeservers", constants.DefaultNTPServer) + }, + ), + ) +} + +func (suite *TimeServerSpecSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestTimeServerSpecSuite(t *testing.T) { + suite.Run(t, new(TimeServerSpecSuite)) +} diff --git a/internal/app/machined/pkg/controllers/network/utils/utils.go b/internal/app/machined/pkg/controllers/network/utils/utils.go new file mode 100644 index 0000000..af5bf2e --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/utils/utils.go @@ -0,0 +1,66 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package networkutils provides utilities for controllers to interact with network resources. +package networkutils + +import ( + "context" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// WaitForNetworkReady waits for devices to be ready. +// +// It is a helper function for controllers. +func WaitForNetworkReady(ctx context.Context, r controller.Runtime, condition func(*network.StatusSpec) bool, nextInputs []controller.Input) error { + // set inputs temporarily to a service only + if err := r.UpdateInputs([]controller.Input{ + { + Namespace: network.NamespaceName, + Type: network.StatusType, + ID: optional.Some(network.StatusID), + Kind: controller.InputWeak, + }, + }); err != nil { + return err + } + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-r.EventCh(): + } + + status, err := safe.ReaderGetByID[*network.Status](ctx, r, network.StatusID) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + if condition(status.TypedSpec()) { + // condition met + break + } + } + + // restore inputs + if err := r.UpdateInputs(nextInputs); err != nil { + return err + } + + // queue an update to reprocess with new inputs + r.QueueReconcile() + + return nil +} diff --git a/internal/app/machined/pkg/controllers/network/watch/ethtool.go b/internal/app/machined/pkg/controllers/network/watch/ethtool.go new file mode 100644 index 0000000..eb2d79d --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/watch/ethtool.go @@ -0,0 +1,77 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package watch + +import ( + "errors" + "fmt" + "sync" + + "github.com/mdlayher/genetlink" + "golang.org/x/sys/unix" +) + +type ethtoolWatcher struct { + wg sync.WaitGroup + conn *genetlink.Conn +} + +// NewEthtool starts ethtool watch. +func NewEthtool(trigger Trigger) (Watcher, error) { + watcher := ðtoolWatcher{} + + var err error + + watcher.conn, err = genetlink.Dial(nil) + if err != nil { + return nil, fmt.Errorf("error dialing ethtool watch socket: %w", err) + } + + ethFamily, err := watcher.conn.GetFamily(unix.ETHTOOL_GENL_NAME) + if err != nil { + return nil, fmt.Errorf("error getting family information for ethtool: %w", err) + } + + var monitorID uint32 + + for _, g := range ethFamily.Groups { + if g.Name == unix.ETHTOOL_MCGRP_MONITOR_NAME { + monitorID = g.ID + + break + } + } + + if monitorID == 0 { + return nil, errors.New("could not find monitor multicast group ID for ethtool") + } + + if err = watcher.conn.JoinGroup(monitorID); err != nil { + return nil, fmt.Errorf("error joing multicast group for ethtool: %w", err) + } + + watcher.wg.Add(1) + + go func() { + defer watcher.wg.Done() + + for { + _, _, watchErr := watcher.conn.Receive() + if watchErr != nil { + return + } + + trigger.QueueReconcile() + } + }() + + return watcher, nil +} + +func (watcher *ethtoolWatcher) Done() { + watcher.conn.Close() //nolint:errcheck + + watcher.wg.Wait() +} diff --git a/internal/app/machined/pkg/controllers/network/watch/rtnetlink.go b/internal/app/machined/pkg/controllers/network/watch/rtnetlink.go new file mode 100644 index 0000000..3d21490 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/watch/rtnetlink.go @@ -0,0 +1,55 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package watch + +import ( + "fmt" + "sync" + + "github.com/jsimonetti/rtnetlink" + "github.com/mdlayher/netlink" +) + +type rtnetlinkWatcher struct { + wg sync.WaitGroup + conn *rtnetlink.Conn +} + +// NewRtNetlink starts rtnetlink watch over specified groups. +func NewRtNetlink(trigger Trigger, groups uint32) (Watcher, error) { + watcher := &rtnetlinkWatcher{} + + var err error + + watcher.conn, err = rtnetlink.Dial(&netlink.Config{ + Groups: groups, + }) + if err != nil { + return nil, fmt.Errorf("error dialing watch socket: %w", err) + } + + watcher.wg.Add(1) + + go func() { + defer watcher.wg.Done() + + for { + _, _, watchErr := watcher.conn.Receive() + if watchErr != nil { + return + } + + trigger.QueueReconcile() + } + }() + + return watcher, nil +} + +func (watcher *rtnetlinkWatcher) Done() { + watcher.conn.Close() //nolint:errcheck + + watcher.wg.Wait() +} diff --git a/internal/app/machined/pkg/controllers/network/watch/trigger.go b/internal/app/machined/pkg/controllers/network/watch/trigger.go new file mode 100644 index 0000000..94b20fa --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/watch/trigger.go @@ -0,0 +1,74 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package watch + +import ( + "context" + + "golang.org/x/time/rate" +) + +// RateLimitedTrigger wraps a Trigger with rate limiting. +type RateLimitedTrigger struct { + trigger Trigger + limiter *rate.Limiter + ch chan struct{} +} + +// Interface check. +var _ Trigger = &RateLimitedTrigger{} + +// NewRateLimitedTrigger creates a new RateLimitedTrigger with specified params. +// +// Trigger's goroutine exists when the context is canceled. +func NewRateLimitedTrigger(ctx context.Context, trigger Trigger, rateLimit rate.Limit, burst int) *RateLimitedTrigger { + t := &RateLimitedTrigger{ + trigger: trigger, + limiter: rate.NewLimiter(rateLimit, burst), + ch: make(chan struct{}), + } + + go t.run(ctx) + + return t +} + +// NewDefaultRateLimitedTrigger creates a new RateLimitedTrigger with default params. +func NewDefaultRateLimitedTrigger(ctx context.Context, trigger Trigger) *RateLimitedTrigger { + const ( + defaultRate = 10 // 10 events per second + defaultBurst = 5 // 5 events + ) + + return NewRateLimitedTrigger(ctx, trigger, defaultRate, defaultBurst) +} + +// QueueReconcile implements Trigger interface. +// +// The event is queued if the goroutine is ready to accept it (otherwise it's already +// busy processing a previous event). +// This function returns immediately. +func (t *RateLimitedTrigger) QueueReconcile() { + select { + case t.ch <- struct{}{}: + default: + } +} + +func (t *RateLimitedTrigger) run(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case <-t.ch: + } + + if err := t.limiter.Wait(ctx); err != nil { + return + } + + t.trigger.QueueReconcile() + } +} diff --git a/internal/app/machined/pkg/controllers/network/watch/trigger_test.go b/internal/app/machined/pkg/controllers/network/watch/trigger_test.go new file mode 100644 index 0000000..452232d --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/watch/trigger_test.go @@ -0,0 +1,45 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package watch_test + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network/watch" +) + +type mockTrigger struct { + count atomic.Int64 +} + +func (t *mockTrigger) QueueReconcile() { + t.count.Add(1) +} + +func (t *mockTrigger) Get() int64 { + return t.count.Load() +} + +func TestRateLimitedTrigger(t *testing.T) { + mock := &mockTrigger{} + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + trigger := watch.NewRateLimitedTrigger(ctx, mock, 10, 5) + + start := time.Now() + + for time.Since(start) < time.Second { + trigger.QueueReconcile() + } + + assert.InDelta(t, int64(14), mock.Get(), 5) +} diff --git a/internal/app/machined/pkg/controllers/network/watch/watch.go b/internal/app/machined/pkg/controllers/network/watch/watch.go new file mode 100644 index 0000000..1251469 --- /dev/null +++ b/internal/app/machined/pkg/controllers/network/watch/watch.go @@ -0,0 +1,16 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package watch provides netlink watchers via multicast groups. +package watch + +// Watcher interface allows to stop watching. +type Watcher interface { + Done() +} + +// Trigger is used by watcher to trigger reconcile loops. +type Trigger interface { + QueueReconcile() +} diff --git a/internal/app/machined/pkg/controllers/perf/perf.go b/internal/app/machined/pkg/controllers/perf/perf.go new file mode 100644 index 0000000..9027e7d --- /dev/null +++ b/internal/app/machined/pkg/controllers/perf/perf.go @@ -0,0 +1,113 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package perf + +import ( + "context" + "time" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/prometheus/procfs" + "go.uber.org/zap" + + perfadapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/perf" + "github.com/siderolabs/talos/pkg/machinery/resources/perf" +) + +const updateInterval = time.Second * 30 + +// StatsController manages v1alpha1.Stats which is the current snaphot of the machine CPU and Memory consumption. +type StatsController struct{} + +// Name implements controller.StatsController interface. +func (ctrl *StatsController) Name() string { + return "perf.StatsController" +} + +// Inputs implements controller.StatsController interface. +func (ctrl *StatsController) Inputs() []controller.Input { + return nil +} + +// Outputs implements controller.StatsController interface. +func (ctrl *StatsController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: perf.CPUType, + Kind: controller.OutputExclusive, + }, + { + Type: perf.MemoryType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.StatsController interface. +func (ctrl *StatsController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + ticker := time.NewTicker(updateInterval) + + defer ticker.Stop() + + var ( + fs procfs.FS + err error + ) + + fs, err = procfs.NewDefaultFS() + if err != nil { + return err + } + + for { + select { + case <-r.EventCh(): + case <-ctx.Done(): + return nil + case <-ticker.C: + } + + if err := ctrl.updateMemory(ctx, r, &fs); err != nil { + return err + } + + if err := ctrl.updateCPU(ctx, r, &fs); err != nil { + return err + } + + r.ResetRestartBackoff() + } +} + +func (ctrl *StatsController) updateCPU(ctx context.Context, r controller.Runtime, fs *procfs.FS) error { + cpu := perf.NewCPU() + + stat, err := fs.Stat() + if err != nil { + return err + } + + return r.Modify(ctx, cpu, func(r resource.Resource) error { + perfadapter.CPU(r.(*perf.CPU)).Update(&stat) + + return nil + }) +} + +func (ctrl *StatsController) updateMemory(ctx context.Context, r controller.Runtime, fs *procfs.FS) error { + mem := perf.NewMemory() + + info, err := fs.Meminfo() + if err != nil { + return err + } + + return r.Modify(ctx, mem, func(r resource.Resource) error { + perfadapter.Memory(r.(*perf.Memory)).Update(&info) + + return nil + }) +} diff --git a/internal/app/machined/pkg/controllers/perf/perf_test.go b/internal/app/machined/pkg/controllers/perf/perf_test.go new file mode 100644 index 0000000..85ee54a --- /dev/null +++ b/internal/app/machined/pkg/controllers/perf/perf_test.go @@ -0,0 +1,128 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package perf_test + +import ( + "context" + "log" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/perf" + "github.com/siderolabs/talos/pkg/logging" + perfresource "github.com/siderolabs/talos/pkg/machinery/resources/perf" +) + +type PerfSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + //nolint:containedctx + ctx context.Context + ctxCancel context.CancelFunc +} + +func (suite *PerfSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + logger := logging.Wrap(log.Writer()) + + suite.runtime, err = runtime.NewRuntime(suite.state, logger) + suite.Require().NoError(err) +} + +func (suite *PerfSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *PerfSuite) TestReconcile() { + suite.Require().NoError(suite.runtime.RegisterController(&perf.StatsController{})) + + suite.startRuntime() + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + cpu, err := suite.state.Get( + suite.ctx, + resource.NewMetadata( + perfresource.NamespaceName, + perfresource.CPUType, + perfresource.CPUID, + resource.VersionUndefined, + ), + ) + if err != nil { + if state.IsNotFoundError(err) { + return retry.ExpectedError(err) + } + + return err + } + + mem, err := suite.state.Get( + suite.ctx, + resource.NewMetadata( + perfresource.NamespaceName, + perfresource.MemoryType, + perfresource.MemoryID, + resource.VersionUndefined, + ), + ) + if err != nil { + if state.IsNotFoundError(err) { + return retry.ExpectedError(err) + } + + return err + } + + cpuSpec := cpu.(*perfresource.CPU).TypedSpec() + memSpec := mem.(*perfresource.Memory).TypedSpec() + + if len(cpuSpec.CPU) == 0 || memSpec.MemTotal == 0 { + return retry.ExpectedErrorf("cpu spec does not contain any CPU or Total memory is zero") + } + + return nil + }, + ), + ) +} + +func (suite *PerfSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestPerfSuite(t *testing.T) { + suite.Run(t, new(PerfSuite)) +} diff --git a/internal/app/machined/pkg/controllers/runtime/common_test.go b/internal/app/machined/pkg/controllers/runtime/common_test.go new file mode 100644 index 0000000..faad27d --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/common_test.go @@ -0,0 +1,91 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime_test + +import ( + "context" + "errors" + "log" + "sync" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + "github.com/siderolabs/talos/pkg/logging" +) + +const ( + fsFileMax = "fs.file-max" + procSysfsFileMax = "proc.sys.fs.file-max" + sysfsFileMax = "sys.fs.file-max" +) + +type RuntimeSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *RuntimeSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + logger := logging.Wrap(log.Writer()) + + suite.runtime, err = runtime.NewRuntime(suite.state, logger) + suite.Require().NoError(err) +} + +func (suite *RuntimeSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *RuntimeSuite) assertResource(md resource.Metadata, compare func(res resource.Resource) bool) func() error { + return func() error { + r, err := suite.state.Get(suite.ctx, md) + if err != nil { + if state.IsNotFoundError(err) { + return retry.ExpectedError(err) + } + + return err + } + + if !compare(r) { + return errors.New("resource is not equal to the expected one") + } + + return nil + } +} + +func (suite *RuntimeSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} diff --git a/internal/app/machined/pkg/controllers/runtime/cri_image_gc.go b/internal/app/machined/pkg/controllers/runtime/cri_image_gc.go new file mode 100644 index 0000000..26025cb --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/cri_image_gc.go @@ -0,0 +1,314 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/benbjohnson/clock" + "github.com/containerd/containerd" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/namespaces" + "github.com/containerd/containerd/reference/docker" + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/xslices" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/etcd" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +// ImageCleanupInterval is the interval at which the image GC controller runs. +const ImageCleanupInterval = 15 * time.Minute + +// ImageGCGracePeriod is the minimum age of an image before it can be deleted. +const ImageGCGracePeriod = 4 * ImageCleanupInterval + +// CRIImageGCController renders manifests based on templates and config/secrets. +type CRIImageGCController struct { + ImageServiceProvider func() (ImageServiceProvider, error) + Clock clock.Clock + + imageFirstSeenUnreferenced map[string]time.Time +} + +// ImageServiceProvider wraps the containerd image service. +type ImageServiceProvider interface { + ImageService() images.Store + Close() error +} + +// Name implements controller.Controller interface. +func (ctrl *CRIImageGCController) Name() string { + return "runtime.CRIImageGCController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *CRIImageGCController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: v1alpha1.NamespaceName, + Type: v1alpha1.ServiceType, + ID: optional.Some("cri"), + Kind: controller.InputWeak, + }, + { + Namespace: k8s.NamespaceName, + Type: k8s.KubeletSpecType, + ID: optional.Some(k8s.KubeletID), + Kind: controller.InputWeak, + }, + { + Namespace: etcd.NamespaceName, + Type: etcd.SpecType, + ID: optional.Some(etcd.SpecID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *CRIImageGCController) Outputs() []controller.Output { + return nil +} + +func defaultImageServiceProvider() (ImageServiceProvider, error) { + criClient, err := containerd.New(constants.CRIContainerdAddress) + if err != nil { + return nil, fmt.Errorf("error creating CRI containerd client: %w", err) + } + + return &containerdImageServiceProvider{ + criClient: criClient, + }, nil +} + +type containerdImageServiceProvider struct { + criClient *containerd.Client +} + +func (s *containerdImageServiceProvider) ImageService() images.Store { + return s.criClient.ImageService() +} + +func (s *containerdImageServiceProvider) Close() error { + return s.criClient.Close() +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *CRIImageGCController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + if ctrl.ImageServiceProvider == nil { + ctrl.ImageServiceProvider = defaultImageServiceProvider + } + + if ctrl.Clock == nil { + ctrl.Clock = clock.New() + } + + if ctrl.imageFirstSeenUnreferenced == nil { + ctrl.imageFirstSeenUnreferenced = map[string]time.Time{} + } + + var ( + criIsUp bool + expectedImages []string + imageServiceProvider ImageServiceProvider + ) + + ticker := ctrl.Clock.Ticker(ImageCleanupInterval) + defer ticker.Stop() + + defer func() { + if imageServiceProvider != nil { + imageServiceProvider.Close() //nolint:errcheck + } + }() + + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + if !criIsUp || len(expectedImages) == 0 { + continue + } + + if imageServiceProvider == nil { + var err error + + imageServiceProvider, err = ctrl.ImageServiceProvider() + if err != nil { + return fmt.Errorf("error creating image service provider: %w", err) + } + } + + if err := ctrl.cleanup(ctx, logger, imageServiceProvider.ImageService(), expectedImages); err != nil { + return fmt.Errorf("error running image cleanup: %w", err) + } + case <-r.EventCh(): + criService, err := safe.ReaderGet[*v1alpha1.Service](ctx, r, resource.NewMetadata(v1alpha1.NamespaceName, v1alpha1.ServiceType, "cri", resource.VersionUndefined)) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error getting CRI service: %w", err) + } + + criIsUp = criService != nil && criService.TypedSpec().Running && criService.TypedSpec().Healthy + + expectedImages = nil + + etcdSpec, err := safe.ReaderGet[*etcd.Spec](ctx, r, resource.NewMetadata(etcd.NamespaceName, etcd.SpecType, etcd.SpecID, resource.VersionUndefined)) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error getting etcd spec: %w", err) + } + + if etcdSpec != nil { + expectedImages = append(expectedImages, etcdSpec.TypedSpec().Image) + } + + kubeletSpec, err := safe.ReaderGet[*k8s.KubeletSpec](ctx, r, resource.NewMetadata(k8s.NamespaceName, k8s.KubeletSpecType, k8s.KubeletID, resource.VersionUndefined)) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error getting etcd spec: %w", err) + } + + if kubeletSpec != nil { + expectedImages = append(expectedImages, kubeletSpec.TypedSpec().Image) + } + } + + r.ResetRestartBackoff() + } +} + +//nolint:gocyclo +func buildExpectedImageNames(logger *zap.Logger, actualImages []images.Image, expectedImages []string) (map[string]struct{}, error) { + var parseErrors []error + + expectedReferences := xslices.Map(expectedImages, func(ref string) docker.Named { + res, parseErr := docker.ParseNamed(ref) + + parseErrors = append(parseErrors, parseErr) + + return res + }) + + if err := errors.Join(parseErrors...); err != nil { + return nil, fmt.Errorf("error parsing expected images: %w", err) + } + + expectedImageNames := map[string]struct{}{} + + for _, image := range actualImages { + imageRef, err := docker.ParseAnyReference(image.Name) + if err != nil { + logger.Debug("failed to parse image reference", zap.Error(err), zap.String("image", image.Name)) + + continue + } + + digest := image.Target.Digest.String() + + switch ref := imageRef.(type) { + case docker.NamedTagged: + for _, expectedRef := range expectedReferences { + if expectedRef.Name() != ref.Name() { + continue + } + + if expectedTagged, ok := expectedRef.(docker.Tagged); ok && ref.Tag() == expectedTagged.Tag() { + // this is expected image by tag, inject other forms of the ref + expectedImageNames[digest] = struct{}{} + expectedImageNames[expectedRef.Name()+":"+expectedTagged.Tag()] = struct{}{} + expectedImageNames[expectedRef.Name()+"@"+digest] = struct{}{} + } + } + case docker.Canonical: + for _, expectedRef := range expectedReferences { + if expectedRef.Name() != ref.Name() { + continue + } + + if expectedDigested, ok := expectedRef.(docker.Digested); ok && ref.Digest() == expectedDigested.Digest() { + // this is expected image by digest, inject other forms of the ref + expectedImageNames[digest] = struct{}{} + expectedImageNames[expectedRef.Name()+"@"+digest] = struct{}{} + + // if the image is also tagged, inject the tagged version of it + if expectedTagged, ok := expectedRef.(docker.Tagged); ok { + expectedImageNames[expectedRef.Name()+":"+expectedTagged.Tag()] = struct{}{} + } + } + } + } + } + + return expectedImageNames, nil +} + +func (ctrl *CRIImageGCController) cleanup(ctx context.Context, logger *zap.Logger, imageService images.Store, expectedImages []string) error { + logger.Debug("running image cleanup") + + ctx = namespaces.WithNamespace(ctx, constants.SystemContainerdNamespace) + + actualImages, err := imageService.List(ctx) + if err != nil { + return fmt.Errorf("error listing images: %w", err) + } + + // first pass: scan actualImages and expand expectedReferences with other non-canonical refs + expectedImageNames, err := buildExpectedImageNames(logger, actualImages, expectedImages) + if err != nil { + return err + } + + // second pass, drop whatever is not expected + for _, image := range actualImages { + _, shouldKeep := expectedImageNames[image.Name] + + if shouldKeep { + logger.Debug("image is referenced, skipping garbage collection", zap.String("image", image.Name)) + + delete(ctrl.imageFirstSeenUnreferenced, image.Name) + + continue + } + + if _, ok := ctrl.imageFirstSeenUnreferenced[image.Name]; !ok { + ctrl.imageFirstSeenUnreferenced[image.Name] = ctrl.Clock.Now() + } + + // calculate image age two ways, and pick the minimum: + // * as CRI reports it, which is the time image got pulled + // * as we see it, this means the image won't be deleted until it reaches the age of ImageGCGracePeriod from the moment it became unreferenced + imageAgeCRI := ctrl.Clock.Since(image.CreatedAt) + imageAgeInternal := ctrl.Clock.Since(ctrl.imageFirstSeenUnreferenced[image.Name]) + + imageAge := min(imageAgeCRI, imageAgeInternal) + + if imageAge < ImageGCGracePeriod { + logger.Debug("skipping image cleanup, as it's below minimum age", zap.String("image", image.Name), zap.Duration("age", imageAge)) + + continue + } + + if err = imageService.Delete(ctx, image.Name); err != nil { + return fmt.Errorf("failed to delete an image %s: %w", image.Name, err) + } + + delete(ctrl.imageFirstSeenUnreferenced, image.Name) + logger.Info("deleted an image", zap.String("image", image.Name)) + } + + return nil +} diff --git a/internal/app/machined/pkg/controllers/runtime/cri_image_gc_test.go b/internal/app/machined/pkg/controllers/runtime/cri_image_gc_test.go new file mode 100644 index 0000000..618532a --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/cri_image_gc_test.go @@ -0,0 +1,298 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime_test + +import ( + "context" + "reflect" + "slices" + "sort" + "sync" + "testing" + "time" + + "github.com/benbjohnson/clock" + "github.com/containerd/containerd/images" + "github.com/opencontainers/go-digest" + v1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/siderolabs/gen/maps" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "go.uber.org/zap/zaptest" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + runtimectrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/etcd" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +func TestCRIImageGC(t *testing.T) { + mockImageService := &mockImageService{} + fakeClock := clock.NewMock() + + suite.Run(t, &CRIImageGCSuite{ + mockImageService: mockImageService, + fakeClock: fakeClock, + DefaultSuite: ctest.DefaultSuite{ + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(&runtimectrl.CRIImageGCController{ + ImageServiceProvider: func() (runtimectrl.ImageServiceProvider, error) { + return mockImageService, nil + }, + Clock: fakeClock, + })) + }, + }, + }) +} + +type mockImageService struct { + mu sync.Mutex + + images []images.Image +} + +func (m *mockImageService) ImageService() images.Store { + return m +} + +func (m *mockImageService) Close() error { + return nil +} + +func (m *mockImageService) Get(ctx context.Context, name string) (images.Image, error) { + panic("not implemented") +} + +func (m *mockImageService) List(ctx context.Context, filters ...string) ([]images.Image, error) { + m.mu.Lock() + defer m.mu.Unlock() + + return slices.Clone(m.images), nil +} + +func (m *mockImageService) Create(ctx context.Context, image images.Image) (images.Image, error) { + panic("not implemented") +} + +func (m *mockImageService) Update(ctx context.Context, image images.Image, fieldpaths ...string) (images.Image, error) { + panic("not implemented") +} + +func (m *mockImageService) Delete(ctx context.Context, name string, opts ...images.DeleteOpt) error { + m.mu.Lock() + defer m.mu.Unlock() + + m.images = xslices.FilterInPlace(m.images, func(i images.Image) bool { return i.Name != name }) + + return nil +} + +type CRIImageGCSuite struct { + ctest.DefaultSuite + + mockImageService *mockImageService + fakeClock *clock.Mock +} + +func (suite *CRIImageGCSuite) TestReconcile() { + storedImages := []images.Image{ + { + Name: "registry.io/org/image1:v1.3.5@sha256:6b094bd0b063a1172eec7da249eccbb48cc48333800569363d67c747960cfa0a", + CreatedAt: suite.fakeClock.Now().Add(-2 * runtimectrl.ImageGCGracePeriod), + Target: v1.Descriptor{ + Digest: must(digest.Parse("sha256:6b094bd0b063a1172eec7da249eccbb48cc48333800569363d67c747960cfa0a")), + }, + }, // ok to be gc'd + { + Name: "sha256:6b094bd0b063a1172eec7da249eccbb48cc48333800569363d67c747960cfa0a", + // the image age is more than the grace period, but the controller won't remove due to the check on the last seen unreferenced timestamp + CreatedAt: suite.fakeClock.Now().Add(-4 * runtimectrl.ImageGCGracePeriod), + Target: v1.Descriptor{ + Digest: must(digest.Parse("sha256:6b094bd0b063a1172eec7da249eccbb48cc48333800569363d67c747960cfa0a")), + }, + }, // ok to be gc'd, same as above, another ref + { + Name: "registry.io/org/image1:v1.3.7", + CreatedAt: suite.fakeClock.Now().Add(-2 * runtimectrl.ImageGCGracePeriod), + Target: v1.Descriptor{ + Digest: must(digest.Parse("sha256:7051a34bcd2522e58a2291d1aa065667f225fd07e4445590b091e86c6799b135")), + }, + }, // current image`` + { + Name: "registry.io/org/image1@sha256:7051a34bcd2522e58a2291d1aa065667f225fd07e4445590b091e86c6799b135", + CreatedAt: suite.fakeClock.Now().Add(-2 * runtimectrl.ImageGCGracePeriod), + Target: v1.Descriptor{ + Digest: must(digest.Parse("sha256:7051a34bcd2522e58a2291d1aa065667f225fd07e4445590b091e86c6799b135")), + }, + }, // current image, canonical ref + { + Name: "sha256:7051a34bcd2522e58a2291d1aa065667f225fd07e4445590b091e86c6799b135", + CreatedAt: suite.fakeClock.Now().Add(-2 * runtimectrl.ImageGCGracePeriod), + Target: v1.Descriptor{ + Digest: must(digest.Parse("sha256:7051a34bcd2522e58a2291d1aa065667f225fd07e4445590b091e86c6799b135")), + }, + }, // current image, digest ref + { + Name: "registry.io/org/image1:v1.3.8", + CreatedAt: suite.fakeClock.Now().Add(runtimectrl.ImageGCGracePeriod), + Target: v1.Descriptor{ + Digest: must(digest.Parse("sha256:fd03335dd2e7163e5e36e933a0c735d7fec6f42b33ddafad0bc54f333e4a23c0")), + }, + }, // not ok to clean up, too new + { + Name: "registry.io/org/image2@sha256:2f794176e9bd8a28501fa185693dc1073013a048c51585022ebce4f84b469db8", + CreatedAt: suite.fakeClock.Now().Add(-2 * runtimectrl.ImageGCGracePeriod), + Target: v1.Descriptor{ + Digest: must(digest.Parse("sha256:2f794176e9bd8a28501fa185693dc1073013a048c51585022ebce4f84b469db8")), + }, + }, // current image + } + + suite.mockImageService.images = storedImages + + criService := v1alpha1.NewService("cri") + criService.TypedSpec().Healthy = true + criService.TypedSpec().Running = true + + suite.Require().NoError(suite.State().Create(suite.Ctx(), criService)) + + kubelet := k8s.NewKubeletSpec(k8s.NamespaceName, k8s.KubeletID) + kubelet.TypedSpec().Image = "registry.io/org/image1:v1.3.7" + suite.Require().NoError(suite.State().Create(suite.Ctx(), kubelet)) + + etcd := etcd.NewSpec(etcd.NamespaceName, etcd.SpecID) + etcd.TypedSpec().Image = "registry.io/org/image2:v3.5.9@sha256:2f794176e9bd8a28501fa185693dc1073013a048c51585022ebce4f84b469db8" + suite.Require().NoError(suite.State().Create(suite.Ctx(), etcd)) + + expectedImages := xslices.Map(storedImages[2:7], func(i images.Image) string { return i.Name }) + + suite.Assert().NoError(retry.Constant(5*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(func() error { + suite.fakeClock.Add(runtimectrl.ImageCleanupInterval) + + imageList, _ := suite.mockImageService.List(suite.Ctx()) //nolint:errcheck + actualImages := xslices.Map(imageList, func(i images.Image) string { return i.Name }) + + if reflect.DeepEqual(expectedImages, actualImages) { + return nil + } + + return retry.ExpectedErrorf("images don't match: expected %v actual %v", expectedImages, actualImages) + })) +} + +func TestBuildExpectedImageNames(t *testing.T) { + actualImages := []images.Image{ + { + Name: "registry.io/org/image1:v1.3.5@sha256:6b094bd0b063a1172eec7da249eccbb48cc48333800569363d67c747960cfa0a", + Target: v1.Descriptor{ + Digest: must(digest.Parse("sha256:6b094bd0b063a1172eec7da249eccbb48cc48333800569363d67c747960cfa0a")), + }, + }, + { + Name: "sha256:6b094bd0b063a1172eec7da249eccbb48cc48333800569363d67c747960cfa0a", + Target: v1.Descriptor{ + Digest: must(digest.Parse("sha256:6b094bd0b063a1172eec7da249eccbb48cc48333800569363d67c747960cfa0a")), + }, + }, + { + Name: "registry.io/org/image1:v1.3.7", + Target: v1.Descriptor{ + Digest: must(digest.Parse("sha256:7051a34bcd2522e58a2291d1aa065667f225fd07e4445590b091e86c6799b135")), + }, + }, + { + Name: "registry.io/org/image1@sha256:7051a34bcd2522e58a2291d1aa065667f225fd07e4445590b091e86c6799b135", + Target: v1.Descriptor{ + Digest: must(digest.Parse("sha256:7051a34bcd2522e58a2291d1aa065667f225fd07e4445590b091e86c6799b135")), + }, + }, + { + Name: "sha256:7051a34bcd2522e58a2291d1aa065667f225fd07e4445590b091e86c6799b135", + Target: v1.Descriptor{ + Digest: must(digest.Parse("sha256:7051a34bcd2522e58a2291d1aa065667f225fd07e4445590b091e86c6799b135")), + }, + }, + { + Name: "registry.io/org/image1:v1.3.8", + Target: v1.Descriptor{ + Digest: must(digest.Parse("sha256:fd03335dd2e7163e5e36e933a0c735d7fec6f42b33ddafad0bc54f333e4a23c0")), + }, + }, + { + Name: "registry.io/org/image2@sha256:2f794176e9bd8a28501fa185693dc1073013a048c51585022ebce4f84b469db8", + Target: v1.Descriptor{ + Digest: must(digest.Parse("sha256:2f794176e9bd8a28501fa185693dc1073013a048c51585022ebce4f84b469db8")), + }, + }, + } + + logger := zaptest.NewLogger(t) + + for _, test := range []struct { + name string + expectedImages []string + + expectedImageNames []string + }{ + { + name: "empty", + }, + { + name: "by tag", + expectedImages: []string{ + "registry.io/org/image1:v1.3.7", + }, + expectedImageNames: []string{ + "registry.io/org/image1:v1.3.7", + "registry.io/org/image1@sha256:7051a34bcd2522e58a2291d1aa065667f225fd07e4445590b091e86c6799b135", + "sha256:7051a34bcd2522e58a2291d1aa065667f225fd07e4445590b091e86c6799b135", + }, + }, + { + name: "by digest", + expectedImages: []string{ + "registry.io/org/image1@sha256:7051a34bcd2522e58a2291d1aa065667f225fd07e4445590b091e86c6799b135", + }, + expectedImageNames: []string{ + "registry.io/org/image1@sha256:7051a34bcd2522e58a2291d1aa065667f225fd07e4445590b091e86c6799b135", + "sha256:7051a34bcd2522e58a2291d1aa065667f225fd07e4445590b091e86c6799b135", + }, + }, + { + name: "by digest and tag", + expectedImages: []string{ + "registry.io/org/image1:v1.3.7@sha256:7051a34bcd2522e58a2291d1aa065667f225fd07e4445590b091e86c6799b135", + }, + expectedImageNames: []string{ + "registry.io/org/image1:v1.3.7", + "registry.io/org/image1@sha256:7051a34bcd2522e58a2291d1aa065667f225fd07e4445590b091e86c6799b135", + "sha256:7051a34bcd2522e58a2291d1aa065667f225fd07e4445590b091e86c6799b135", + }, + }, + { + name: "not found", + expectedImages: []string{ + "registry.io/org/image1:v1.3.9", + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + expectedImages, err := runtimectrl.BuildExpectedImageNames(logger, actualImages, test.expectedImages) + require.NoError(t, err) + + expectedImageNames := maps.Keys(expectedImages) + + sort.Strings(test.expectedImageNames) + sort.Strings(expectedImageNames) + + assert.Equal(t, test.expectedImageNames, expectedImageNames) + }) + } +} diff --git a/internal/app/machined/pkg/controllers/runtime/devices_status.go b/internal/app/machined/pkg/controllers/runtime/devices_status.go new file mode 100644 index 0000000..cbe7086 --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/devices_status.go @@ -0,0 +1,68 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "go.uber.org/zap" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/v1alpha1" + machineruntime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// DevicesStatusController loads extensions.yaml and updates DevicesStatus resources. +type DevicesStatusController struct { + V1Alpha1Mode machineruntime.Mode +} + +// Name implements controller.Controller interface. +func (ctrl *DevicesStatusController) Name() string { + return "runtime.DevicesStatusController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *DevicesStatusController) Inputs() []controller.Input { + return nil +} + +// Outputs implements controller.Controller interface. +func (ctrl *DevicesStatusController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: runtime.DevicesStatusType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +func (ctrl *DevicesStatusController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + // in container mode, devices are always ready + if ctrl.V1Alpha1Mode != machineruntime.ModeContainer { + if err := v1alpha1.WaitForServiceHealthy(ctx, r, "udevd", nil); err != nil { + return err + } + } + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + if err := safe.WriterModify(ctx, r, runtime.NewDevicesStatus(runtime.NamespaceName, runtime.DevicesID), func(status *runtime.DevicesStatus) error { + status.TypedSpec().Ready = true + + return nil + }); err != nil { + return err + } + } +} diff --git a/internal/app/machined/pkg/controllers/runtime/drop_upgrade_fallback.go b/internal/app/machined/pkg/controllers/runtime/drop_upgrade_fallback.go new file mode 100644 index 0000000..c302d54 --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/drop_upgrade_fallback.go @@ -0,0 +1,94 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + machineruntime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/pkg/meta" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// MetaProvider wraps acquiring meta. +type MetaProvider interface { + Meta() machineruntime.Meta +} + +// DropUpgradeFallbackController removes upgrade fallback key once machine reaches ready & running. +type DropUpgradeFallbackController struct { + MetaProvider MetaProvider +} + +// Name implements controller.Controller interface. +func (ctrl *DropUpgradeFallbackController) Name() string { + return "runtime.DropUpgradeFallbackController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *DropUpgradeFallbackController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: runtime.NamespaceName, + Type: runtime.MachineStatusType, + ID: optional.Some(runtime.MachineStatusID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *DropUpgradeFallbackController) Outputs() []controller.Output { + return nil +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *DropUpgradeFallbackController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + machineStatus, err := safe.ReaderGetByID[*runtime.MachineStatus](ctx, r, runtime.MachineStatusID) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting machine status: %w", err) + } + + if !(machineStatus.TypedSpec().Stage == runtime.MachineStageRunning && machineStatus.TypedSpec().Status.Ready) { + continue + } + + ok, err := ctrl.MetaProvider.Meta().DeleteTag(ctx, meta.Upgrade) + if err != nil { + return err + } + + if ok { + logger.Info("removing fallback entry") + + if err = ctrl.MetaProvider.Meta().Flush(); err != nil { + return err + } + } + + // terminating the controller here, as removing fallback is required only once on boot after upgrade + return nil + } +} diff --git a/internal/app/machined/pkg/controllers/runtime/drop_upgrade_fallback_test.go b/internal/app/machined/pkg/controllers/runtime/drop_upgrade_fallback_test.go new file mode 100644 index 0000000..c489ac7 --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/drop_upgrade_fallback_test.go @@ -0,0 +1,98 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime_test + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/runtime" + machineruntime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/pkg/meta" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +type DropUpgradeFallbackControllerSuite struct { + ctest.DefaultSuite + + meta *meta.Meta +} + +type metaProvider struct { + meta *meta.Meta +} + +func (m metaProvider) Meta() machineruntime.Meta { + return m.meta +} + +func TestUpgradeFallbackControllerSuite(t *testing.T) { + tmpDir := t.TempDir() + + path := filepath.Join(tmpDir, "meta") + + f, err := os.Create(path) + require.NoError(t, err) + require.NoError(t, f.Truncate(1024*1024)) + require.NoError(t, f.Close()) + + st := state.WrapCore(namespaced.NewState(inmem.Build)) + + m, err := meta.New(context.Background(), st, meta.WithFixedPath(path)) + require.NoError(t, err) + + suite.Run(t, &DropUpgradeFallbackControllerSuite{ + meta: m, + DefaultSuite: ctest.DefaultSuite{ + AfterSetup: func(s *ctest.DefaultSuite) { + s.Require().NoError(s.Runtime().RegisterController(&runtime.DropUpgradeFallbackController{ + MetaProvider: metaProvider{meta: m}, + })) + }, + }, + }) +} + +func (suite *DropUpgradeFallbackControllerSuite) TestDropUpgradeFallback() { + _, err := suite.meta.SetTag(suite.Ctx(), meta.Upgrade, "A") + suite.Require().NoError(err) + + machineStatus := runtimeres.NewMachineStatus() + machineStatus.TypedSpec().Stage = runtimeres.MachineStageBooting + machineStatus.TypedSpec().Status.Ready = false + suite.Require().NoError(suite.State().Create(suite.Ctx(), machineStatus)) + + time.Sleep(time.Second) + + // controller should not remove the tag + val, ok := suite.meta.ReadTag(meta.Upgrade) + suite.Require().True(ok) + suite.Require().Equal("A", val) + + // update machine status to ready + machineStatus.TypedSpec().Status.Ready = true + machineStatus.TypedSpec().Stage = runtimeres.MachineStageRunning + suite.Require().NoError(suite.State().Update(suite.Ctx(), machineStatus)) + + suite.AssertWithin(time.Second, 10*time.Millisecond, func() error { + _, ok = suite.meta.ReadTag(meta.Upgrade) + if ok { + return retry.ExpectedErrorf("tag is still present") + } + + return nil + }) +} diff --git a/internal/app/machined/pkg/controllers/runtime/events_sink.go b/internal/app/machined/pkg/controllers/runtime/events_sink.go new file mode 100644 index 0000000..2391a55 --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/events_sink.go @@ -0,0 +1,208 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/rs/xid" + "github.com/siderolabs/gen/channel" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/siderolink/api/events" + "go.uber.org/zap" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/protobuf/types/known/anypb" + + networkutils "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network/utils" + machinedruntime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// EventsSinkController watches events and forwards them to the events sink server +// if it's configured. +type EventsSinkController struct { + V1Alpha1Events machinedruntime.Watcher + Drainer *machinedruntime.Drainer + + drainSub *machinedruntime.DrainSubscription + eventID xid.ID +} + +// Name implements controller.Controller interface. +func (ctrl *EventsSinkController) Name() string { + return "v1alpha1.EventsSinkController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *EventsSinkController) Inputs() []controller.Input { + return nil +} + +// Outputs implements controller.Controller interface. +func (ctrl *EventsSinkController) Outputs() []controller.Output { + return nil +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *EventsSinkController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + if ctrl.drainSub == nil { + ctrl.drainSub = ctrl.Drainer.Subscribe() + } + + defer func() { + if ctrl.drainSub != nil { + ctrl.drainSub.Cancel() + } + }() + + if err := networkutils.WaitForNetworkReady(ctx, r, + func(status *network.StatusSpec) bool { + return status.AddressReady + }, + []controller.Input{ + { + Namespace: runtime.NamespaceName, + Type: runtime.EventSinkConfigType, + ID: optional.Some(runtime.EventSinkConfigID), + Kind: controller.InputWeak, + }, + }, + ); err != nil { + return fmt.Errorf("error waiting for network: %w", err) + } + + var ( + conn *grpc.ClientConn + client events.EventSinkServiceClient + watchCh, consumeWatchCh chan machinedruntime.EventInfo + backlog int + draining bool + ) + + defer func() { + if conn != nil { + conn.Close() //nolint:errcheck + } + }() + + for { + select { + case <-ctx.Done(): + return nil + case <-ctrl.drainSub.EventCh(): + // drain started, return immediately if there's no backlog + draining = true + + if backlog == 0 { + return nil + } + case event := <-consumeWatchCh: + // if consumeWatchCh is not nil, client connection was established + backlog = event.Backlog + + data, err := anypb.New(event.Payload) + if err != nil { + return err + } + + req := &events.EventRequest{ + Id: event.ID.String(), + Data: data, + ActorId: event.ActorID, + } + + _, err = client.Publish(ctx, req) + if err != nil { + return fmt.Errorf("error publishing event: %w", err) + } + + // adjust last consumed event + ctrl.eventID = event.ID + + // if draining and backlog is 0, return immediately + if draining && backlog == 0 { + return nil + } + case <-r.EventCh(): + // configuration changed, re-establish connection + cfg, err := safe.ReaderGetByID[*runtime.EventSinkConfig](ctx, r, runtime.EventSinkConfigID) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error getting event sink config: %w", err) + } + + if conn != nil { + logger.Debug("closing connection to event sink") + + conn.Close() //nolint:errcheck + conn = nil + client = nil + consumeWatchCh = nil // stop consuming events + backlog = 0 + } + + if cfg == nil { + // no config, no event streaming + continue + } + + // establish connection + logger.Debug("establishing connection to event sink", zap.String("endpoint", cfg.TypedSpec().Endpoint)) + + conn, err = grpc.Dial( + cfg.TypedSpec().Endpoint, + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithSharedWriteBuffer(true), + ) + if err != nil { + return fmt.Errorf("error establishing connection to event sink: %w", err) + } + + client = events.NewEventSinkServiceClient(conn) + + // start watching events if we haven't already done so + // + // watch is only established with the first live connection to make sure we don't miss any events + if watchCh == nil { + watchCh = make(chan machinedruntime.EventInfo) + + opts := []machinedruntime.WatchOptionFunc{} + if ctrl.eventID.IsNil() { + opts = append(opts, machinedruntime.WithTailEvents(-1)) + } else { + opts = append(opts, machinedruntime.WithTailID(ctrl.eventID)) + } + + // Watch returns immediately, setting up a goroutine which will copy events to `watchCh` + if err = ctrl.V1Alpha1Events.Watch(func(eventCh <-chan machinedruntime.EventInfo) { + for { + select { + case <-ctx.Done(): + return + case event := <-eventCh: + if !channel.SendWithContext(ctx, watchCh, event) { + return + } + } + } + }, opts...); err != nil { + return err + } + } + + consumeWatchCh = watchCh + } + } +} diff --git a/internal/app/machined/pkg/controllers/runtime/events_sink_config.go b/internal/app/machined/pkg/controllers/runtime/events_sink_config.go new file mode 100644 index 0000000..92664d0 --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/events_sink_config.go @@ -0,0 +1,101 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/go-procfs/procfs" + "go.uber.org/zap" + + v1alpha1runtime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// EventsSinkConfigController generates configuration for kmsg log delivery. +type EventsSinkConfigController struct { + Cmdline *procfs.Cmdline + V1Alpha1Mode v1alpha1runtime.Mode +} + +// Name implements controller.Controller interface. +func (ctrl *EventsSinkConfigController) Name() string { + return "runtime.EventsSinkConfigController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *EventsSinkConfigController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: config.MachineConfigType, + ID: optional.Some(config.V1Alpha1ID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *EventsSinkConfigController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: runtime.EventSinkConfigType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *EventsSinkConfigController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) (err error) { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + var endpoint string + + if ctrl.Cmdline != nil && ctrl.V1Alpha1Mode != v1alpha1runtime.ModeContainer { + if val := ctrl.Cmdline.Get(constants.KernelParamEventsSink).First(); val != nil { + endpoint = *val + } + } + + cfg, err := safe.ReaderGetByID[*config.MachineConfig](ctx, r, config.V1Alpha1ID) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error getting machine config: %w", err) + } + + if cfg != nil && cfg.Config().Runtime().EventsEndpoint() != nil { + endpoint = *cfg.Config().Runtime().EventsEndpoint() + } + + r.StartTrackingOutputs() + + if endpoint != "" { + if err = safe.WriterModify(ctx, r, runtime.NewEventSinkConfig(), func(cfg *runtime.EventSinkConfig) error { + cfg.TypedSpec().Endpoint = endpoint + + return nil + }); err != nil { + return fmt.Errorf("error updating kmsg log config: %w", err) + } + } + + if err = safe.CleanupOutputs[*runtime.EventSinkConfig](ctx, r); err != nil { + return err + } + } +} diff --git a/internal/app/machined/pkg/controllers/runtime/events_sink_config_test.go b/internal/app/machined/pkg/controllers/runtime/events_sink_config_test.go new file mode 100644 index 0000000..061986f --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/events_sink_config_test.go @@ -0,0 +1,80 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime_test + +import ( + "testing" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/siderolabs/go-procfs/procfs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + runtimectrls "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/runtime" + "github.com/siderolabs/talos/pkg/machinery/config/container" + runtimecfg "github.com/siderolabs/talos/pkg/machinery/config/types/runtime" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +type EventsSinkConfigSuite struct { + ctest.DefaultSuite +} + +func TestEventsSinkConfigSuite(t *testing.T) { + suite.Run(t, new(EventsSinkConfigSuite)) +} + +func (suite *EventsSinkConfigSuite) TestEventSinkConfigNone() { + suite.Require().NoError(suite.Runtime().RegisterController(&runtimectrls.EventsSinkConfigController{})) + + rtestutils.AssertNoResource[*runtime.EventSinkConfig](suite.Ctx(), suite.T(), suite.State(), runtime.EventSinkConfigID) +} + +func (suite *EventsSinkConfigSuite) TestEventSinkConfigMachineConfig() { + suite.Require().NoError(suite.Runtime().RegisterController(&runtimectrls.EventsSinkConfigController{})) + + eventSinkConfig := &runtimecfg.EventSinkV1Alpha1{ + Endpoint: "10.0.0.2:4444", + } + + cfg, err := container.New(eventSinkConfig) + suite.Require().NoError(err) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), config.NewMachineConfig(cfg))) + + rtestutils.AssertResources[*runtime.EventSinkConfig](suite.Ctx(), suite.T(), suite.State(), []resource.ID{runtime.EventSinkConfigID}, + func(cfg *runtime.EventSinkConfig, asrt *assert.Assertions) { + asrt.Equal( + "10.0.0.2:4444", + cfg.TypedSpec().Endpoint, + ) + }) +} + +func (suite *EventsSinkConfigSuite) TestEventSinkConfigCmdline() { + cmdline := procfs.NewCmdline("") + cmdline.Append(constants.KernelParamEventsSink, "10.0.0.1:3333") + + cfg, err := container.New() + suite.Require().NoError(err) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), config.NewMachineConfig(cfg))) + + suite.Require().NoError(suite.Runtime().RegisterController(&runtimectrls.EventsSinkConfigController{ + Cmdline: cmdline, + })) + + rtestutils.AssertResources[*runtime.EventSinkConfig](suite.Ctx(), suite.T(), suite.State(), []resource.ID{runtime.EventSinkConfigID}, + func(cfg *runtime.EventSinkConfig, asrt *assert.Assertions) { + asrt.Equal( + "10.0.0.1:3333", + cfg.TypedSpec().Endpoint, + ) + }) +} diff --git a/internal/app/machined/pkg/controllers/runtime/events_sink_test.go b/internal/app/machined/pkg/controllers/runtime/events_sink_test.go new file mode 100644 index 0000000..56580eb --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/events_sink_test.go @@ -0,0 +1,305 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime_test + +import ( + "context" + "net" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-retry/retry" + eventsapi "github.com/siderolabs/siderolink/api/events" + "github.com/siderolabs/siderolink/pkg/events" + "github.com/stretchr/testify/suite" + "go.uber.org/zap/zaptest" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc" + + controllerruntime "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/runtime" + talosruntime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/proto" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +type handler struct { + eventsMu sync.Mutex + events []events.Event +} + +// HandleEvent implements events.Adapter. +func (s *handler) HandleEvent(ctx context.Context, e events.Event) error { + s.eventsMu.Lock() + defer s.eventsMu.Unlock() + + s.events = append(s.events, e) + + return nil +} + +type EventsSinkSuite struct { + suite.Suite + + events *v1alpha1.Events + state state.State + handler *handler + server *grpc.Server + sink *events.Sink + + runtime *runtime.Runtime + drainer *talosruntime.Drainer + wg sync.WaitGroup + eg errgroup.Group + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *EventsSinkSuite) SetupTest() { + suite.events = v1alpha1.NewEvents(1000, 10) + + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, zaptest.NewLogger(suite.T())) + suite.Require().NoError(err) + + suite.handler = &handler{} + suite.drainer = talosruntime.NewDrainer() + + suite.Require().NoError( + suite.runtime.RegisterController( + &controllerruntime.EventsSinkController{ + V1Alpha1Events: suite.events, + Drainer: suite.drainer, + }, + ), + ) + + status := network.NewStatus(network.NamespaceName, network.StatusID) + status.TypedSpec().AddressReady = true + + suite.Require().NoError(suite.state.Create(suite.ctx, status)) + + suite.startRuntime() +} + +func (suite *EventsSinkSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *EventsSinkSuite) startServer(ctx context.Context) string { + suite.sink = events.NewSink( + suite.handler, + []proto.Message{ + &machine.AddressEvent{}, + &machine.PhaseEvent{}, + }) + + lis, err := net.Listen("tcp", "localhost:0") + suite.Require().NoError(err) + + suite.server = grpc.NewServer() + eventsapi.RegisterEventSinkServiceServer(suite.server, suite.sink) + + suite.eg.Go( + func() error { + <-ctx.Done() + + suite.server.Stop() + + return nil + }, + ) + + suite.eg.Go( + func() error { + return suite.server.Serve(lis) + }, + ) + + return lis.Addr().String() +} + +func (suite *EventsSinkSuite) TestPublish() { + ctx, cancel := context.WithCancel(suite.ctx) + defer cancel() + + suite.events.Publish( + ctx, + &machine.AddressEvent{ + Hostname: "localhost", + }, + ) + + suite.events.Publish( + ctx, + &machine.PhaseEvent{ + Phase: "test", + Action: machine.PhaseEvent_START, + }, + ) + + suite.Require().Equal(0, len(suite.handler.events)) + + endpoint := suite.startServer(ctx) + config := runtimeres.NewEventSinkConfig() + config.TypedSpec().Endpoint = endpoint + suite.Require().NoError(suite.state.Create(ctx, config)) + + suite.Require().NoError(retry.Constant(time.Second*5, retry.WithUnits(time.Millisecond*100)).Retry( + func() error { + suite.handler.eventsMu.Lock() + defer suite.handler.eventsMu.Unlock() + + if len(suite.handler.events) != 2 { + return retry.ExpectedErrorf("expected 2 events, got %d", len(suite.handler.events)) + } + + return nil + }, + )) + + suite.events.Publish( + ctx, + &machine.PhaseEvent{ + Phase: "test", + Action: machine.PhaseEvent_STOP, + }, + ) + + suite.Require().NoError(retry.Constant(time.Second*5, retry.WithUnits(time.Millisecond*100)).Retry( + func() error { + suite.handler.eventsMu.Lock() + defer suite.handler.eventsMu.Unlock() + + if len(suite.handler.events) != 3 { + return retry.ExpectedErrorf("expected 3 events, got %d", len(suite.handler.events)) + } + + return nil + }, + )) +} + +func (suite *EventsSinkSuite) TestDrain() { + ctx, cancel := context.WithCancel(suite.ctx) + defer cancel() + + for range 10 { + suite.events.Publish( + ctx, + &machine.PhaseEvent{ + Phase: "test", + Action: machine.PhaseEvent_START, + }, + ) + suite.events.Publish( + ctx, + &machine.PhaseEvent{ + Phase: "test", + Action: machine.PhaseEvent_STOP, + }, + ) + } + + suite.Require().Equal(0, len(suite.handler.events)) + + // first, publish wrong endpoint + badLis, err := net.Listen("tcp", "localhost:0") + suite.Require().NoError(err) + + badEndpoint := badLis.Addr().String() + suite.Require().NoError(badLis.Close()) + + config := runtimeres.NewEventSinkConfig() + config.TypedSpec().Endpoint = badEndpoint + suite.Require().NoError(suite.state.Create(ctx, config)) + + suite.T().Logf("%s starting bad server at %s", time.Now().Format(time.RFC3339), badEndpoint) + + time.Sleep(time.Second * 1) + + drainCtx, drainCtxCancel := context.WithTimeout(ctx, time.Second*5) + defer drainCtxCancel() + + var eg errgroup.Group + + eg.Go( + func() error { + suite.T().Logf("%s starting drain", time.Now().Format(time.RFC3339)) + + return suite.drainer.Drain(drainCtx) + }, + ) + + eg.Go( + func() error { + // start real server with delay + time.Sleep(300 * time.Millisecond) + + endpoint := suite.startServer(ctx) + + suite.T().Logf("%s starting real server at %s", time.Now().Format(time.RFC3339), endpoint) + + _, updateErr := safe.StateUpdateWithConflicts( + ctx, suite.state, runtimeres.NewEventSinkConfig().Metadata(), + func(cfg *runtimeres.EventSinkConfig) error { + cfg.TypedSpec().Endpoint = endpoint + + return nil + }) + + return updateErr + }, + ) + + suite.Require().NoError(retry.Constant(time.Second*5, retry.WithUnits(time.Millisecond*100)).Retry( + func() error { + suite.handler.eventsMu.Lock() + defer suite.handler.eventsMu.Unlock() + + if len(suite.handler.events) != 20 { + return retry.ExpectedErrorf("expected 20 events, got %d", len(suite.handler.events)) + } + + return nil + }, + )) + + suite.Require().NoError(eg.Wait()) +} + +func (suite *EventsSinkSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.Require().NoError(suite.eg.Wait()) + + suite.wg.Wait() +} + +func TestEventsSinkSuite(t *testing.T) { + suite.Run(t, new(EventsSinkSuite)) +} diff --git a/internal/app/machined/pkg/controllers/runtime/export_test.go b/internal/app/machined/pkg/controllers/runtime/export_test.go new file mode 100644 index 0000000..1254f91 --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/export_test.go @@ -0,0 +1,8 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +// BuildExpectedImageNames is exported for testing. +var BuildExpectedImageNames = buildExpectedImageNames diff --git a/internal/app/machined/pkg/controllers/runtime/extension_service.go b/internal/app/machined/pkg/controllers/runtime/extension_service.go new file mode 100644 index 0000000..4e12cee --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/extension_service.go @@ -0,0 +1,225 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "go.uber.org/zap" + "gopkg.in/yaml.v3" + + "github.com/aenix-io/talm/internal/app/machined/pkg/system" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/services" + extservices "github.com/siderolabs/talos/pkg/machinery/extensions/services" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// ServiceManager is the interface to the v1alpha1 services subsystems. +type ServiceManager interface { + IsRunning(id string) (system.Service, bool, error) + Load(services ...system.Service) []string + Stop(ctx context.Context, serviceIDs ...string) (err error) + Start(serviceIDs ...string) error +} + +// ExtensionServiceController creates extension services based on the extension service configuration found in the rootfs. +type ExtensionServiceController struct { + V1Alpha1Services ServiceManager + ConfigPath string + + configStatusCache map[string]string +} + +// Name implements controller.Controller interface. +func (ctrl *ExtensionServiceController) Name() string { + return "runtime.ExtensionServiceController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *ExtensionServiceController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: runtime.NamespaceName, + Type: runtime.ExtensionServiceConfigStatusType, + Kind: controller.InputStrong, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *ExtensionServiceController) Outputs() []controller.Output { + return nil +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *ExtensionServiceController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + // wait for controller runtime to be ready + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + // extensions loading only needs to run once, as services are static + serviceFiles, err := os.ReadDir(ctrl.ConfigPath) + if err != nil { + if os.IsNotExist(err) { + // directory not present, skip completely + logger.Debug("extension service directory is not found") + + return nil + } + + return err + } + + // load initial state of configStatuses + if ctrl.configStatusCache == nil { + configStatuses, err := safe.ReaderListAll[*runtime.ExtensionServiceConfigStatus](ctx, r) + if err != nil { + return fmt.Errorf("error listing extension services config: %w", err) + } + + ctrl.configStatusCache = make(map[string]string, configStatuses.Len()) + + for iter := configStatuses.Iterator(); iter.Next(); { + ctrl.configStatusCache[iter.Value().Metadata().ID()] = iter.Value().TypedSpec().SpecVersion + } + } + + // load services from definitions into the service runner framework + extServices := map[string]struct{}{} + + for _, serviceFile := range serviceFiles { + if filepath.Ext(serviceFile.Name()) != ".yaml" { + logger.Debug("skipping config file", zap.String("filename", serviceFile.Name())) + + continue + } + + spec, err := ctrl.loadSpec(filepath.Join(ctrl.ConfigPath, serviceFile.Name())) + if err != nil { + logger.Error("error loading extension service spec", zap.String("filename", serviceFile.Name()), zap.Error(err)) + + continue + } + + if err = spec.Validate(); err != nil { + logger.Error("error validating extension service spec", zap.String("filename", serviceFile.Name()), zap.Error(err)) + + continue + } + + if _, exists := extServices[spec.Name]; exists { + logger.Error("duplicate service spec", zap.String("filename", serviceFile.Name()), zap.String("name", spec.Name)) + + continue + } + + extServices[spec.Name] = struct{}{} + + svc := &services.Extension{ + Spec: spec, + } + + ctrl.V1Alpha1Services.Load(svc) + + if err = ctrl.V1Alpha1Services.Start(svc.ID(nil)); err != nil { + return fmt.Errorf("error starting %q service: %w", spec.Name, err) + } + } + + // watch for changes in the configStatuses + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + configStatuses, err := safe.ReaderListAll[*runtime.ExtensionServiceConfigStatus](ctx, r) + if err != nil { + return fmt.Errorf("error listing extension services config: %w", err) + } + + configStatusesPresent := map[string]struct{}{} + + for iter := configStatuses.Iterator(); iter.Next(); { + configStatusesPresent[iter.Value().Metadata().ID()] = struct{}{} + + if ctrl.configStatusCache[iter.Value().Metadata().ID()] == iter.Value().TypedSpec().SpecVersion { + continue + } + + if err = ctrl.handleRestart(ctx, logger, "ext-"+iter.Value().Metadata().ID(), iter.Value().TypedSpec().SpecVersion); err != nil { + return err + } + + ctrl.configStatusCache[iter.Value().Metadata().ID()] = iter.Value().TypedSpec().SpecVersion + } + + // cleanup configStatusesCache + for id := range ctrl.configStatusCache { + if _, ok := configStatusesPresent[id]; !ok { + if err = ctrl.handleRestart(ctx, logger, "ext-"+id, "nan"); err != nil { + return err + } + + delete(ctrl.configStatusCache, id) + } + } + } +} + +func (ctrl *ExtensionServiceController) loadSpec(path string) (extservices.Spec, error) { + var spec extservices.Spec + + f, err := os.Open(path) + if err != nil { + return spec, err + } + + defer f.Close() //nolint:errcheck + + if err = yaml.NewDecoder(f).Decode(&spec); err != nil { + return spec, fmt.Errorf("error unmarshalling extension service config: %w", err) + } + + return spec, nil +} + +func (ctrl *ExtensionServiceController) handleRestart(ctx context.Context, logger *zap.Logger, svcName, specVersion string) error { + _, running, err := ctrl.V1Alpha1Services.IsRunning(svcName) + if err != nil { + return nil //nolint:nilerr // IsRunning returns an error only if the service is not found, so ignore it + } + + // this means it's a new config and the service runner is already waiting for the config to start the service + // we don't need restart it again since it will be started automatically + if running && specVersion == "1" { + return nil + } + + logger.Warn("extension service config changed, restarting", zap.String("service", svcName)) + + if running { + if err = ctrl.V1Alpha1Services.Stop(ctx, svcName); err != nil { + return fmt.Errorf("error stopping extension service %s: %w", svcName, err) + } + } + + if err = ctrl.V1Alpha1Services.Start(svcName); err != nil { + return fmt.Errorf("error starting extension service %s: %w", svcName, err) + } + + return nil +} diff --git a/internal/app/machined/pkg/controllers/runtime/extension_service_config.go b/internal/app/machined/pkg/controllers/runtime/extension_service_config.go new file mode 100644 index 0000000..430fa4d --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/extension_service_config.go @@ -0,0 +1,94 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/xslices" + "go.uber.org/zap" + + extconfig "github.com/siderolabs/talos/pkg/machinery/config/config" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// ExtensionServiceConfigController watches v1alpha1.Config, creates/updates/deletes extension services config. +type ExtensionServiceConfigController struct{} + +// Name implements controller.Controller interface. +func (ctrl *ExtensionServiceConfigController) Name() string { + return "runtime.ExtensionServiceConfigController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *ExtensionServiceConfigController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: config.MachineConfigType, + ID: optional.Some(config.V1Alpha1ID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *ExtensionServiceConfigController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: runtime.ExtensionServiceConfigType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *ExtensionServiceConfigController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + cfg, err := safe.ReaderGetByID[*config.MachineConfig](ctx, r, config.V1Alpha1ID) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error getting machine config: %w", err) + } + + r.StartTrackingOutputs() + + if cfg != nil && cfg.Config() != nil { + for _, extConfig := range cfg.Config().ExtensionServiceConfigs() { + if err = safe.WriterModify(ctx, r, runtime.NewExtensionServiceConfigSpec(runtime.NamespaceName, extConfig.Name()), func(spec *runtime.ExtensionServiceConfig) error { + spec.TypedSpec().Files = xslices.Map(extConfig.ConfigFiles(), func(c extconfig.ExtensionServiceConfigFile) runtime.ExtensionServiceConfigFile { + return runtime.ExtensionServiceConfigFile{ + Content: c.Content(), + MountPath: c.MountPath(), + } + }) + + spec.TypedSpec().Environment = extConfig.Environment() + + return nil + }); err != nil { + return err + } + } + } + + if err = safe.CleanupOutputs[*runtime.ExtensionServiceConfig](ctx, r); err != nil { + return err + } + } +} diff --git a/internal/app/machined/pkg/controllers/runtime/extension_service_config_files.go b/internal/app/machined/pkg/controllers/runtime/extension_service_config_files.go new file mode 100644 index 0000000..48b10f6 --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/extension_service_config_files.go @@ -0,0 +1,124 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "go.uber.org/zap" + + v1alpha1runtime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// ExtensionServiceConfigFilesController writes down the config files for extension services. +type ExtensionServiceConfigFilesController struct { + V1Alpha1Mode v1alpha1runtime.Mode + ExtensionsConfigBaseDir string +} + +// Name implements controller.Controller interface. +func (ctrl *ExtensionServiceConfigFilesController) Name() string { + return "runtime.ExtensionServiceConfigFilesController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *ExtensionServiceConfigFilesController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: runtime.NamespaceName, + Type: runtime.ExtensionServiceConfigType, + Kind: controller.InputStrong, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *ExtensionServiceConfigFilesController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: runtime.ExtensionServiceConfigStatusType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *ExtensionServiceConfigFilesController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + if ctrl.V1Alpha1Mode == v1alpha1runtime.ModeContainer { + return nil + } + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + list, err := safe.ReaderListAll[*runtime.ExtensionServiceConfig](ctx, r) + if err != nil { + return fmt.Errorf("error listing extension services config: %w", err) + } + + r.StartTrackingOutputs() + + touchedFiles := map[string]struct{}{} + + for iter := list.Iterator(); iter.Next(); { + extensionConfigPath := filepath.Join(ctrl.ExtensionsConfigBaseDir, iter.Value().Metadata().ID()) + + if err = os.MkdirAll(extensionConfigPath, 0o755); err != nil { + return fmt.Errorf("error creating directory %q: %w", extensionConfigPath, err) + } + + touchedFiles[extensionConfigPath] = struct{}{} + + for _, file := range iter.Value().TypedSpec().Files { + fileName := filepath.Join(extensionConfigPath, strings.ReplaceAll(strings.TrimPrefix(file.MountPath, "/"), "/", "-")) + + if err = updateFile(fileName, []byte(file.Content), 0o644); err != nil { + return fmt.Errorf("error writing file %q: %w", fileName, err) + } + + touchedFiles[fileName] = struct{}{} + } + + if err = safe.WriterModify(ctx, r, runtime.NewExtensionServiceConfigStatusSpec(runtime.NamespaceName, iter.Value().Metadata().ID()), func(spec *runtime.ExtensionServiceConfigStatus) error { + spec.TypedSpec().SpecVersion = iter.Value().Metadata().Version().String() + + return nil + }); err != nil { + return err + } + } + + // remove all files not managed by us + if err = filepath.WalkDir(ctrl.ExtensionsConfigBaseDir, func(path string, d fs.DirEntry, walkErr error) error { + if _, ok := touchedFiles[path]; path != ctrl.ExtensionsConfigBaseDir && !ok { + if err = os.RemoveAll(path); err != nil { + return err + } + } + + return nil + }); err != nil { + return err + } + + if err = safe.CleanupOutputs[*runtime.ExtensionServiceConfigStatus](ctx, r); err != nil { + return err + } + } +} diff --git a/internal/app/machined/pkg/controllers/runtime/extension_service_config_files_test.go b/internal/app/machined/pkg/controllers/runtime/extension_service_config_files_test.go new file mode 100644 index 0000000..99ff97b --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/extension_service_config_files_test.go @@ -0,0 +1,129 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. +package runtime_test + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/siderolabs/gen/xslices" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/runtime" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +type ExtensionServiceConfigFilesSuite struct { + ctest.DefaultSuite + extensionsConfigDir string +} + +func TestExtensionServiceConfigFilesSuite(t *testing.T) { + extensionsConfigDir := t.TempDir() + + suite.Run(t, &ExtensionServiceConfigFilesSuite{ + DefaultSuite: ctest.DefaultSuite{ + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(&runtime.ExtensionServiceConfigFilesController{ + ExtensionsConfigBaseDir: extensionsConfigDir, + })) + }, + }, + extensionsConfigDir: extensionsConfigDir, + }) +} + +func (suite *ExtensionServiceConfigFilesSuite) TestReconcileExtensionServiceConfigFiles() { + for _, tt := range []struct { + extensionName string + configFiles []struct { + content string + mountPath string + } + }{ + { + extensionName: "test-extension-a", + configFiles: []struct { + content string + mountPath string + }{ + { + content: "test-content-a", + mountPath: "/etc/test", + }, + }, + }, + { + extensionName: "test-extension-b", + configFiles: []struct { + content string + mountPath string + }{ + { + content: "test-content-b", + mountPath: "/etc/bar", + }, + { + content: "test-content-c", + mountPath: "/var/etc/foo", + }, + }, + }, + } { + extensionServiceConfigFiles := runtimeres.NewExtensionServiceConfigSpec(runtimeres.NamespaceName, tt.extensionName) + extensionServiceConfigFiles.TypedSpec().Files = xslices.Map(tt.configFiles, func(config struct { + content string + mountPath string + }, + ) runtimeres.ExtensionServiceConfigFile { + return runtimeres.ExtensionServiceConfigFile{ + Content: config.content, + MountPath: config.mountPath, + } + }) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), extensionServiceConfigFiles)) + + ctest.AssertResource(suite, tt.extensionName, + func(status *runtimeres.ExtensionServiceConfigStatus, asrt *assert.Assertions) { + asrt.Equal(extensionServiceConfigFiles.Metadata().Version().String(), status.TypedSpec().SpecVersion) + }, + ) + + for _, file := range tt.configFiles { + content, err := os.ReadFile(filepath.Join(suite.extensionsConfigDir, tt.extensionName, strings.ReplaceAll(strings.TrimPrefix(file.mountPath, "/"), "/", "-"))) + suite.Require().NoError(err) + + suite.Assert().Equal(file.content, string(content)) + } + } + + // create a directory and file manually in the extensions config directory + // ensure that the controller deletes the manually created directory/file + // also ensure that an update doesn't update existing files timestamp + suite.Assert().NoError(os.Mkdir(filepath.Join(suite.extensionsConfigDir, "test"), 0o755)) + suite.Assert().NoError(os.WriteFile(filepath.Join(suite.extensionsConfigDir, "test", "testdata"), []byte("{}"), 0o644)) + + extensionAConfigFileInfo, err := os.Stat(filepath.Join(suite.extensionsConfigDir, "test-extension-a", "etc-test")) + suite.Assert().NoError(err) + + // delete test-extension-b resource + suite.Assert().NoError(suite.State().Destroy(suite.Ctx(), runtimeres.NewExtensionServiceConfigSpec(runtimeres.NamespaceName, "test-extension-b").Metadata())) + ctest.AssertNoResource[*runtimeres.ExtensionServiceConfigStatus](suite, "test-extension-b") + + suite.Assert().NoFileExists(filepath.Join(suite.extensionsConfigDir, "test", "testdata")) + suite.Assert().NoDirExists(filepath.Join(suite.extensionsConfigDir, "test")) + suite.Assert().NoFileExists(filepath.Join(suite.extensionsConfigDir, "test-extension-b", "etc-bar")) + suite.Assert().NoFileExists(filepath.Join(suite.extensionsConfigDir, "test-extension-b", "var-etc-foo")) + suite.Assert().NoDirExists(filepath.Join(suite.extensionsConfigDir, "test-extension-b")) + + extensionAConfigFileInfoAfterUpdate, err := os.Stat(filepath.Join(suite.extensionsConfigDir, "test-extension-a", "etc-test")) + suite.Require().NoError(err) + + suite.Assert().Equal(extensionAConfigFileInfo.ModTime(), extensionAConfigFileInfoAfterUpdate.ModTime()) +} diff --git a/internal/app/machined/pkg/controllers/runtime/extension_service_config_test.go b/internal/app/machined/pkg/controllers/runtime/extension_service_config_test.go new file mode 100644 index 0000000..4add3d6 --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/extension_service_config_test.go @@ -0,0 +1,141 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime_test + +import ( + "testing" + + "github.com/siderolabs/gen/xslices" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/runtime" + cntrconfig "github.com/siderolabs/talos/pkg/machinery/config/config" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/runtime/extensions" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +type ExtensionServiceConfigSuite struct { + ctest.DefaultSuite +} + +func TestExtensionServiceConfigSuite(t *testing.T) { + suite.Run(t, &ExtensionServiceConfigSuite{ + DefaultSuite: ctest.DefaultSuite{ + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(&runtime.ExtensionServiceConfigController{})) + }, + }, + }) +} + +func (suite *ExtensionServiceConfigSuite) TestReconcileExtensionServiceConfig() { + extensionServiceConfigs := []struct { + extensionName string + configFiles []struct { + content string + mountPath string + } + environment []string + }{ + { + extensionName: "test-extension-a", + configFiles: []struct { + content string + mountPath string + }{ + { + content: "test-content-a", + mountPath: "/etc/test", + }, + }, + }, + { + extensionName: "test-extension-b", + configFiles: []struct { + content string + mountPath string + }{ + { + content: "test-content-b", + mountPath: "/etc/bar", + }, + { + content: "test-content-c", + mountPath: "/var/etc/foo", + }, + }, + environment: []string{ + "FOO=BAR", + }, + }, + } + + cfgs := xslices.Map(extensionServiceConfigs, func(tt struct { + extensionName string + configFiles []struct { + content string + mountPath string + } + environment []string + }, + ) cntrconfig.Document { + cfg := extensions.NewServicesConfigV1Alpha1() + cfg.ServiceName = tt.extensionName + cfg.ServiceConfigFiles = xslices.Map(tt.configFiles, func(config struct { + content string + mountPath string + }, + ) extensions.ConfigFile { + return extensions.ConfigFile{ + ConfigFileContent: config.content, + ConfigFileMountPath: config.mountPath, + } + }) + cfg.ServiceEnvironment = tt.environment + + return cfg + }) + + cntr, err := container.New(cfgs...) + suite.Require().NoError(err) + + machineConfig := config.NewMachineConfig(cntr) + suite.Require().NoError(suite.State().Create(suite.Ctx(), machineConfig)) + + for _, tt := range extensionServiceConfigs { + ctest.AssertResource(suite, tt.extensionName, func(config *runtimeres.ExtensionServiceConfig, asrt *assert.Assertions) { + spec := config.TypedSpec() + + configFileData := xslices.Map(tt.configFiles, func(config struct { + content string + mountPath string + }, + ) runtimeres.ExtensionServiceConfigFile { + return runtimeres.ExtensionServiceConfigFile{ + Content: config.content, + MountPath: config.mountPath, + } + }) + + suite.Assert().Equal(configFileData, spec.Files) + suite.Assert().Equal(tt.environment, spec.Environment) + }) + } + + // test deletion + cfg := extensions.NewServicesConfigV1Alpha1() + cfg.ServiceName = "test-extension-a" + cntr, err = container.New(cfg) + suite.Require().NoError(err) + + machineConfig = config.NewMachineConfig(cntr) + suite.Require().NoError(suite.State().Destroy(suite.Ctx(), machineConfig.Metadata())) + + ctest.AssertNoResource[*runtimeres.ExtensionServiceConfig](suite, "test-extension-a") +} diff --git a/internal/app/machined/pkg/controllers/runtime/extension_service_test.go b/internal/app/machined/pkg/controllers/runtime/extension_service_test.go new file mode 100644 index 0000000..d1637e1 --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/extension_service_test.go @@ -0,0 +1,248 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. +package runtime_test + +import ( + "context" + "fmt" + "reflect" + "sort" + "sync" + "testing" + "time" + + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + runtimecontrollers "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/system" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/services" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +type ExtensionServiceSuite struct { + RuntimeSuite +} + +type serviceMock struct { + mu sync.Mutex + services map[string]system.Service + running map[string]bool + timesStarted map[string]int + timesStopped map[string]int +} + +func (mock *serviceMock) Load(services ...system.Service) []string { + mock.mu.Lock() + defer mock.mu.Unlock() + + ids := []string{} + + for _, svc := range services { + mock.services[svc.ID(nil)] = svc + ids = append(ids, svc.ID(nil)) + } + + return ids +} + +func (mock *serviceMock) Start(serviceIDs ...string) error { + mock.mu.Lock() + defer mock.mu.Unlock() + + for _, id := range serviceIDs { + mock.running[id] = true + mock.timesStarted[id]++ + } + + return nil +} + +func (mock *serviceMock) IsRunning(id string) (system.Service, bool, error) { + mock.mu.Lock() + defer mock.mu.Unlock() + + svc, exists := mock.services[id] + if !exists { + return nil, false, fmt.Errorf("service %q not found", id) + } + + _, running := mock.running[id] + + return svc, running, nil +} + +func (mock *serviceMock) Stop(ctx context.Context, serviceIDs ...string) error { + mock.mu.Lock() + defer mock.mu.Unlock() + + for _, id := range serviceIDs { + mock.running[id] = false + mock.timesStopped[id]++ + } + + return nil +} + +func (mock *serviceMock) getIDs() []string { + mock.mu.Lock() + defer mock.mu.Unlock() + + ids := []string{} + + for id := range mock.services { + ids = append(ids, id) + } + + sort.Strings(ids) + + return ids +} + +type serviceStartStopInfo struct { + started int + stopped int +} + +func (mock *serviceMock) getTimesStartedStopped() map[string]serviceStartStopInfo { + mock.mu.Lock() + defer mock.mu.Unlock() + + result := map[string]serviceStartStopInfo{} + + for id := range mock.services { + result[id] = serviceStartStopInfo{ + started: mock.timesStarted[id], + stopped: mock.timesStopped[id], + } + } + + return result +} + +func (mock *serviceMock) get(id string) system.Service { + mock.mu.Lock() + defer mock.mu.Unlock() + + return mock.services[id] +} + +func (suite *ExtensionServiceSuite) TestReconcile() { + svcMock := &serviceMock{ + services: map[string]system.Service{}, + running: map[string]bool{}, + timesStarted: map[string]int{}, + timesStopped: map[string]int{}, + } + + suite.Require().NoError(suite.runtime.RegisterController(&runtimecontrollers.ExtensionServiceController{ + V1Alpha1Services: svcMock, + ConfigPath: "testdata/extservices/", + })) + + suite.startRuntime() + + suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + ids := svcMock.getIDs() + + if !reflect.DeepEqual(ids, []string{"ext-frr", "ext-hello-world"}) { + return retry.ExpectedErrorf("services registered: %q", ids) + } + + return nil + }, + )) + + helloSvc := svcMock.get("ext-hello-world") + suite.Require().IsType(&services.Extension{}, helloSvc) + + suite.Assert().Equal("./hello-world", helloSvc.(*services.Extension).Spec.Container.Entrypoint) + + suite.Assert().Equal( + map[string]serviceStartStopInfo{ + "ext-hello-world": { + started: 1, + }, + "ext-frr": { + started: 1, + }, + }, + svcMock.getTimesStartedStopped(), + ) + + helloConfig := runtime.NewExtensionServiceConfigStatusSpec(runtime.NamespaceName, "hello-world") + helloConfig.TypedSpec().SpecVersion = "1" + suite.Require().NoError(suite.state.Create(suite.ctx, helloConfig)) + + assertTimesStartedStopped := func(expected map[string]serviceStartStopInfo) { + suite.Assert().NoError(retry.Constant(5*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + actual := svcMock.getTimesStartedStopped() + + if !reflect.DeepEqual(actual, expected) { + return retry.ExpectedErrorf("services restart status expected %v, actual %v", expected, actual) + } + + return nil + }, + )) + } + + // specVersion is 1, and ext-hello-world is already started, so it should not be restarted + assertTimesStartedStopped(map[string]serviceStartStopInfo{ + "ext-hello-world": { + started: 1, + stopped: 0, + }, + "ext-frr": { + started: 1, + }, + }) + + unexpectedConfig := runtime.NewExtensionServiceConfigStatusSpec(runtime.NamespaceName, "unexpected") + unexpectedConfig.TypedSpec().SpecVersion = "1" + suite.Require().NoError(suite.state.Create(suite.ctx, unexpectedConfig)) + + assertTimesStartedStopped(map[string]serviceStartStopInfo{ + "ext-hello-world": { + started: 1, + stopped: 0, + }, + "ext-frr": { + started: 1, + }, + }) + + // update config for hello service + helloConfig.TypedSpec().SpecVersion = "2" + suite.Require().NoError(suite.state.Update(suite.ctx, helloConfig)) + + assertTimesStartedStopped(map[string]serviceStartStopInfo{ + "ext-hello-world": { + started: 2, + stopped: 1, + }, + "ext-frr": { + started: 1, + }, + }) + + // destroy config for hello service + suite.Require().NoError(suite.state.Destroy(suite.ctx, helloConfig.Metadata())) + + assertTimesStartedStopped(map[string]serviceStartStopInfo{ + "ext-hello-world": { + started: 3, + stopped: 2, + }, + "ext-frr": { + started: 1, + }, + }) +} + +func TestExtensionServiceSuite(t *testing.T) { + suite.Run(t, new(ExtensionServiceSuite)) +} diff --git a/internal/app/machined/pkg/controllers/runtime/extension_status.go b/internal/app/machined/pkg/controllers/runtime/extension_status.go new file mode 100644 index 0000000..4c855c8 --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/extension_status.go @@ -0,0 +1,79 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + "errors" + "fmt" + "io" + "strings" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/extensions" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// ExtensionStatusController loads extensions.yaml and updates ExtensionStatus resources. +type ExtensionStatusController struct{} + +// Name implements controller.Controller interface. +func (ctrl *ExtensionStatusController) Name() string { + return "runtime.ExtensionStatusController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *ExtensionStatusController) Inputs() []controller.Input { + return nil +} + +// Outputs implements controller.Controller interface. +func (ctrl *ExtensionStatusController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: runtime.ExtensionStatusType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +func (ctrl *ExtensionStatusController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + // controller runs once, as extensions are static + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + var cfg extensions.Config + + if err := cfg.Read(constants.ExtensionsRuntimeConfigFile); err != nil { + if errors.Is(err, io.EOF) { + // no extensions installed + return nil + } + + return fmt.Errorf("failed loading extensions config: %w", err) + } + + for _, layer := range cfg.Layers { + id := strings.TrimSuffix(layer.Image, ".sqsh") + + if err := r.Modify(ctx, runtime.NewExtensionStatus(runtime.NamespaceName, id), func(res resource.Resource) error { + *res.(*runtime.ExtensionStatus).TypedSpec() = *layer + + return nil + }); err != nil { + return err + } + } + + return nil +} diff --git a/internal/app/machined/pkg/controllers/runtime/kernel_module_config.go b/internal/app/machined/pkg/controllers/runtime/kernel_module_config.go new file mode 100644 index 0000000..49fad76 --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/kernel_module_config.go @@ -0,0 +1,89 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// KernelModuleConfigController watches v1alpha1.Config, creates/updates/deletes kernel module specs. +type KernelModuleConfigController struct{} + +// Name implements controller.Controller interface. +func (ctrl *KernelModuleConfigController) Name() string { + return "runtime.KernelModuleConfigController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *KernelModuleConfigController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: config.MachineConfigType, + ID: optional.Some(config.V1Alpha1ID), + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *KernelModuleConfigController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: runtime.KernelModuleSpecType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *KernelModuleConfigController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + cfg, err := safe.ReaderGetByID[*config.MachineConfig](ctx, r, config.V1Alpha1ID) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting config: %w", err) + } + } + + r.StartTrackingOutputs() + + if cfg != nil && cfg.Config().Machine() != nil { + for _, module := range cfg.Config().Machine().Kernel().Modules() { + item := runtime.NewKernelModuleSpec(runtime.NamespaceName, module.Name()) + + if err = safe.WriterModify(ctx, r, item, func(res *runtime.KernelModuleSpec) error { + res.TypedSpec().Name = module.Name() + res.TypedSpec().Parameters = module.Parameters() + + return nil + }); err != nil { + return err + } + } + } + + if err = safe.CleanupOutputs[*runtime.KernelModuleSpec](ctx, r); err != nil { + return err + } + } +} diff --git a/internal/app/machined/pkg/controllers/runtime/kernel_module_config_test.go b/internal/app/machined/pkg/controllers/runtime/kernel_module_config_test.go new file mode 100644 index 0000000..fe3b7d2 --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/kernel_module_config_test.go @@ -0,0 +1,104 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. +package runtime_test + +import ( + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + runtimecontrollers "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/runtime" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + runtimeresource "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +type KernelModuleConfigSuite struct { + RuntimeSuite +} + +func (suite *KernelModuleConfigSuite) TestReconcileConfig() { + suite.Require().NoError(suite.runtime.RegisterController(&runtimecontrollers.KernelModuleConfigController{})) + + suite.startRuntime() + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineKernel: &v1alpha1.KernelConfig{ + KernelModules: []*v1alpha1.KernelModuleConfig{ + { + ModuleName: "brtfs", + }, + { + ModuleName: "e1000", + }, + }, + }, + }, + ClusterConfig: &v1alpha1.ClusterConfig{}, + }, + ), + ) + + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + specMD := resource.NewMetadata(runtimeresource.NamespaceName, runtimeresource.KernelModuleSpecType, "e1000", resource.VersionUndefined) + + suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertResource( + specMD, + func(res resource.Resource) bool { + return res.(*runtimeresource.KernelModuleSpec).TypedSpec().Name == "e1000" + }, + ), + )) + + old := cfg.Metadata().Version() + cfg = config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineKernel: nil, + }, + ClusterConfig: &v1alpha1.ClusterConfig{}, + }, + ), + ) + + cfg.Metadata().SetVersion(old) + suite.Require().NoError(suite.state.Update(suite.ctx, cfg)) + + var err error + + // wait for the resource to be removed + suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + for _, md := range []resource.Metadata{specMD} { + _, err = suite.state.Get(suite.ctx, md) + if err != nil { + if state.IsNotFoundError(err) { + return nil + } + + return err + } + } + + return retry.ExpectedErrorf("resource still exists") + }, + )) +} + +func TestKernelModuleConfigSuite(t *testing.T) { + suite.Run(t, new(KernelModuleConfigSuite)) +} diff --git a/internal/app/machined/pkg/controllers/runtime/kernel_module_spec.go b/internal/app/machined/pkg/controllers/runtime/kernel_module_spec.go new file mode 100644 index 0000000..b50dbb4 --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/kernel_module_spec.go @@ -0,0 +1,90 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/pmorjan/kmod" + "go.uber.org/zap" + + v1alpha1runtime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// KernelModuleSpecController watches KernelModuleSpecs, sets/resets kernel params. +type KernelModuleSpecController struct { + V1Alpha1Mode v1alpha1runtime.Mode +} + +// Name implements controller.Controller interface. +func (ctrl *KernelModuleSpecController) Name() string { + return "runtime.KernelModuleSpecController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *KernelModuleSpecController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: runtime.NamespaceName, + Type: runtime.KernelModuleSpecType, + Kind: controller.InputStrong, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *KernelModuleSpecController) Outputs() []controller.Output { + return nil +} + +// Run implements controller.Controller interface. +func (ctrl *KernelModuleSpecController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + if ctrl.V1Alpha1Mode == v1alpha1runtime.ModeContainer { + // not supported in container mode + return nil + } + + manager, err := kmod.New() + if err != nil { + return fmt.Errorf("error initializing kmod manager: %w", err) + } + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + modules, err := safe.ReaderListAll[*runtime.KernelModuleSpec](ctx, r) + if err != nil { + return err + } + + var multiErr error + + // note: this code doesn't support module unloading in any way for now + for iter := modules.Iterator(); iter.Next(); { + module := iter.Value().TypedSpec() + parameters := strings.Join(module.Parameters, " ") + + if err = manager.Load(module.Name, parameters, 0); err != nil { + multiErr = errors.Join(multiErr, fmt.Errorf("error loading module %q: %w", module.Name, err)) + } + } + + if multiErr != nil { + return multiErr + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/runtime/kernel_param_config.go b/internal/app/machined/pkg/controllers/runtime/kernel_param_config.go new file mode 100644 index 0000000..179e690 --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/kernel_param_config.go @@ -0,0 +1,100 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/kernel" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// KernelParamConfigController watches v1alpha1.Config, creates/updates/deletes kernel param specs. +type KernelParamConfigController struct{} + +// Name implements controller.Controller interface. +func (ctrl *KernelParamConfigController) Name() string { + return "runtime.KernelParamConfigController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *KernelParamConfigController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: config.MachineConfigType, + ID: optional.Some(config.V1Alpha1ID), + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *KernelParamConfigController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: runtime.KernelParamSpecType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *KernelParamConfigController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + cfg, err := safe.ReaderGetByID[*config.MachineConfig](ctx, r, config.V1Alpha1ID) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting config: %w", err) + } + } + + r.StartTrackingOutputs() + + setKernelParam := func(kind, key, value string) error { + item := runtime.NewKernelParamSpec(runtime.NamespaceName, kind+"."+key) + + return r.Modify(ctx, item, func(res resource.Resource) error { + res.(*runtime.KernelParamSpec).TypedSpec().Value = value + + return nil + }) + } + + if cfg != nil && cfg.Config().Machine() != nil { + for key, value := range cfg.Config().Machine().Sysctls() { + if err = setKernelParam(kernel.Sysctl, key, value); err != nil { + return err + } + } + + for key, value := range cfg.Config().Machine().Sysfs() { + if err = setKernelParam(kernel.Sysfs, key, value); err != nil { + return err + } + } + } + + if err = safe.CleanupOutputs[*runtime.KernelParamSpec](ctx, r); err != nil { + return err + } + } +} diff --git a/internal/app/machined/pkg/controllers/runtime/kernel_param_config_test.go b/internal/app/machined/pkg/controllers/runtime/kernel_param_config_test.go new file mode 100644 index 0000000..06ea0a4 --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/kernel_param_config_test.go @@ -0,0 +1,121 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. +package runtime_test + +import ( + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + runtimecontrollers "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/runtime" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + runtimeresource "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +type KernelParamConfigSuite struct { + RuntimeSuite +} + +func (suite *KernelParamConfigSuite) TestReconcileConfig() { + suite.Require().NoError(suite.runtime.RegisterController(&runtimecontrollers.KernelParamConfigController{})) + + suite.startRuntime() + + value := "500000" + valueSysfs := "600000" + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineSysctls: map[string]string{ + fsFileMax: value, + }, + MachineSysfs: map[string]string{ + fsFileMax: valueSysfs, + }, + }, + ClusterConfig: &v1alpha1.ClusterConfig{}, + }, + ), + ) + + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + sysctlMD := resource.NewMetadata(runtimeresource.NamespaceName, runtimeresource.KernelParamSpecType, procSysfsFileMax, resource.VersionUndefined) + + suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertResource( + sysctlMD, + func(res resource.Resource) bool { + spec := res.(*runtimeresource.KernelParamSpec).TypedSpec() + + return suite.Assert().Equal(value, spec.Value) + }, + ), + )) + + sysfsMD := resource.NewMetadata(runtimeresource.NamespaceName, runtimeresource.KernelParamSpecType, sysfsFileMax, resource.VersionUndefined) + + suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertResource( + sysfsMD, + func(res resource.Resource) bool { + spec := res.(*runtimeresource.KernelParamSpec).TypedSpec() + + return suite.Assert().Equal(valueSysfs, spec.Value) + }, + ), + )) + + old := cfg.Metadata().Version() + cfg = config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineSysctls: map[string]string{}, + MachineSysfs: map[string]string{ + fsFileMax: valueSysfs, + }, + }, + ClusterConfig: &v1alpha1.ClusterConfig{}, + }, + ), + ) + + cfg.Metadata().SetVersion(old) + suite.Require().NoError(suite.state.Update(suite.ctx, cfg)) + + var err error + + // wait for the resource to be removed + suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + for _, md := range []resource.Metadata{sysctlMD} { + _, err = suite.state.Get(suite.ctx, md) + if err != nil { + if state.IsNotFoundError(err) { + return nil + } + + return err + } + } + + return retry.ExpectedErrorf("resource still exists") + }, + )) +} + +func TestKernelParamConfigSuite(t *testing.T) { + suite.Run(t, new(KernelParamConfigSuite)) +} diff --git a/internal/app/machined/pkg/controllers/runtime/kernel_param_defaults.go b/internal/app/machined/pkg/controllers/runtime/kernel_param_defaults.go new file mode 100644 index 0000000..911faa0 --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/kernel_param_defaults.go @@ -0,0 +1,151 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + "errors" + "os" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "go.uber.org/zap" + + v1alpha1runtime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/kernel/kspp" + "github.com/siderolabs/talos/pkg/machinery/kernel" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// KernelParamDefaultsController creates default kernel params. +type KernelParamDefaultsController struct { + V1Alpha1Mode v1alpha1runtime.Mode +} + +// Name implements controller.Controller interface. +func (ctrl *KernelParamDefaultsController) Name() string { + return "runtime.KernelParamDefaultsController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *KernelParamDefaultsController) Inputs() []controller.Input { + return nil +} + +// Outputs implements controller.Controller interface. +func (ctrl *KernelParamDefaultsController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: runtime.KernelParamDefaultSpecType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +func (ctrl *KernelParamDefaultsController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + kernelParams := ctrl.getKernelParams() + if ctrl.V1Alpha1Mode != v1alpha1runtime.ModeContainer { + kernelParams = append(kernelParams, kspp.GetKernelParams()...) + } + + for _, prop := range kernelParams { + value := prop.Value + item := runtime.NewKernelParamDefaultSpec(runtime.NamespaceName, prop.Key) + + if err := r.Modify(ctx, item, func(res resource.Resource) error { + res.(*runtime.KernelParamDefaultSpec).TypedSpec().Value = value + + return nil + }); err != nil { + return err + } + } + } + + return nil +} + +func (ctrl *KernelParamDefaultsController) getKernelParams() []*kernel.Param { + res := []*kernel.Param{ + { + Key: "proc.sys.net.ipv4.ip_forward", + Value: "1", + }, + } + + if ctrl.V1Alpha1Mode != v1alpha1runtime.ModeContainer { + res = append(res, []*kernel.Param{ + { + Key: "proc.sys.net.bridge.bridge-nf-call-iptables", + Value: "1", + }, + { + Key: "proc.sys.net.bridge.bridge-nf-call-ip6tables", + Value: "1", + }, + }...) + } + + // Apply IPv6 defaults only if IPv6 is enabled. + // NB: we only prevent the application of these rules if the IPv6 node does not exist. + // Other errors should be ignored here so that they bubble up later, where errors can be logged and handled. + _, err := os.Stat("/proc/sys/net/ipv6/conf/default/accept_ra") + if err == nil || !errors.Is(err, os.ErrNotExist) { + res = append(res, []*kernel.Param{ + { + Key: "proc.sys.net.ipv6.conf.default.forwarding", + Value: "1", + }, + { + Key: "proc.sys.net.ipv6.conf.default.accept_ra", + Value: "2", + }, + }...) + } + + res = append(res, []*kernel.Param{ + // ipvs/conntrack tcp keepalive refresh. + { + Key: "proc.sys.net.ipv4.tcp_keepalive_time", + Value: "600", + }, + { + Key: "proc.sys.net.ipv4.tcp_keepalive_intvl", + Value: "60", + }, + { + Key: "proc.sys.kernel.panic", + Value: "10", + }, + { + Key: "proc.sys.kernel.pid_max", + Value: "262144", + }, + { + Key: "proc.sys.vm.overcommit_memory", + Value: "1", + }, + }...) + + // kernel optimization for kubernetes workloads. + res = append(res, []*kernel.Param{ + // configs inotify. + { + Key: "proc.sys.fs.inotify.max_user_instances", + Value: "8192", + }, + { + Key: "proc.sys.fs.aio-max-nr", + Value: "1048576", + }, + }...) + + return res +} diff --git a/internal/app/machined/pkg/controllers/runtime/kernel_param_defaults_test.go b/internal/app/machined/pkg/controllers/runtime/kernel_param_defaults_test.go new file mode 100644 index 0000000..a70a404 --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/kernel_param_defaults_test.go @@ -0,0 +1,115 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime_test + +import ( + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + runtimecontrollers "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/kernel" + runtimeresource "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +type KernelParamDefaultsSuite struct { + RuntimeSuite +} + +func getParams(mode runtime.Mode) []*kernel.Param { + res := []*kernel.Param{ + { + Key: "proc.sys.net.ipv4.ip_forward", + Value: "1", + }, + { + Key: "proc.sys.net.ipv6.conf.default.forwarding", + Value: "1", + }, + { + Key: "proc.sys.net.ipv6.conf.default.accept_ra", + Value: "2", + }, + { + Key: "proc.sys.kernel.panic", + Value: "10", + }, + { + Key: "proc.sys.kernel.pid_max", + Value: "262144", + }, + { + Key: "proc.sys.vm.overcommit_memory", + Value: "1", + }, + } + + if mode != runtime.ModeContainer { + res = append(res, []*kernel.Param{ + { + Key: "proc.sys.net.bridge.bridge-nf-call-iptables", + Value: "1", + }, + { + Key: "proc.sys.net.bridge.bridge-nf-call-ip6tables", + Value: "1", + }, + }...) + } + + return res +} + +//nolint:dupl +func (suite *KernelParamDefaultsSuite) TestContainerMode() { + controller := &runtimecontrollers.KernelParamDefaultsController{ + runtime.ModeContainer, + } + + suite.Require().NoError(suite.runtime.RegisterController(controller)) + + suite.startRuntime() + + for _, prop := range getParams(runtime.ModeContainer) { + suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertResource( + resource.NewMetadata(runtimeresource.NamespaceName, runtimeresource.KernelParamDefaultSpecType, prop.Key, resource.VersionUndefined), + func(res resource.Resource) bool { + return res.(runtimeresource.KernelParam).TypedSpec().Value == prop.Value + }, + ), + )) + } +} + +//nolint:dupl +func (suite *KernelParamDefaultsSuite) TestMetalMode() { + controller := &runtimecontrollers.KernelParamDefaultsController{ + runtime.ModeMetal, + } + + suite.Require().NoError(suite.runtime.RegisterController(controller)) + + suite.startRuntime() + + for _, prop := range getParams(runtime.ModeMetal) { + suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertResource( + resource.NewMetadata(runtimeresource.NamespaceName, runtimeresource.KernelParamDefaultSpecType, prop.Key, resource.VersionUndefined), + func(res resource.Resource) bool { + return res.(runtimeresource.KernelParam).TypedSpec().Value == prop.Value + }, + ), + )) + } +} + +func TestKernelParamDefaultsSuite(t *testing.T) { + suite.Run(t, new(KernelParamDefaultsSuite)) +} diff --git a/internal/app/machined/pkg/controllers/runtime/kernel_param_spec.go b/internal/app/machined/pkg/controllers/runtime/kernel_param_spec.go new file mode 100644 index 0000000..713020d --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/kernel_param_spec.go @@ -0,0 +1,206 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + "errors" + "os" + "strings" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/hashicorp/go-multierror" + "go.uber.org/zap" + + krnl "github.com/siderolabs/talos/pkg/kernel" + "github.com/siderolabs/talos/pkg/kernel/kspp" + "github.com/siderolabs/talos/pkg/machinery/kernel" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// KernelParamSpecController watches KernelParamSpecs, sets/resets kernel params. +type KernelParamSpecController struct { + defaults map[string]string + state map[string]string +} + +// Name implements controller.Controller interface. +func (ctrl *KernelParamSpecController) Name() string { + return "runtime.KernelParamSpecController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *KernelParamSpecController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: runtime.NamespaceName, + Type: runtime.KernelParamDefaultSpecType, + Kind: controller.InputStrong, + }, + { + Namespace: runtime.NamespaceName, + Type: runtime.KernelParamSpecType, + Kind: controller.InputStrong, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *KernelParamSpecController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: runtime.KernelParamStatusType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *KernelParamSpecController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + if ctrl.state == nil { + ctrl.state = map[string]string{} + } + + if ctrl.defaults == nil { + ctrl.defaults = map[string]string{} + } + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + ksppParams := map[string]struct{}{} + + for _, param := range kspp.GetKernelParams() { + ksppParams[param.Key] = struct{}{} + } + + defaults, err := r.List(ctx, resource.NewMetadata(runtime.NamespaceName, runtime.KernelParamDefaultSpecType, "", resource.VersionUndefined)) + if err != nil { + return err + } + + configs, err := r.List(ctx, resource.NewMetadata(runtime.NamespaceName, runtime.KernelParamSpecType, "", resource.VersionUndefined)) + if err != nil { + return err + } + + configsCounts := len(configs.Items) + + list := configs.Items + + list = append(list, defaults.Items...) + + touchedIDs := map[string]string{} + + var errs *multierror.Error + + for i, item := range list { + spec := item.(runtime.KernelParam).TypedSpec() + id := item.Metadata().ID() + + if value, duplicate := touchedIDs[id]; i >= configsCounts && duplicate { + if _, ok := ksppParams[id]; ok { + logger.Warn("overriding KSPP enforced parameter, this is not recommended", zap.String("key", id), zap.String("value", value)) + } + + continue + } + + if err = ctrl.updateKernelParam(ctx, r, id, spec.Value); err != nil { + if errors.Is(err, os.ErrNotExist) && spec.IgnoreErrors { + status := runtime.NewKernelParamStatus(runtime.NamespaceName, id) + + if e := r.Modify(ctx, status, func(res resource.Resource) error { + res.(*runtime.KernelParamStatus).TypedSpec().Unsupported = true + + return nil + }); e != nil { + errs = multierror.Append(errs, e) + } + } else { + errs = multierror.Append(errs, err) + } + + continue + } + + touchedIDs[id] = spec.Value + } + + for key := range ctrl.state { + if _, ok := touchedIDs[key]; ok { + continue + } + + if err = ctrl.resetKernelParam(ctx, r, key); err != nil { + errs = multierror.Append(errs, err) + } + } + + if errs != nil { + return errs + } + } + + r.ResetRestartBackoff() + } +} + +func (ctrl *KernelParamSpecController) updateKernelParam(ctx context.Context, r controller.Runtime, key, value string) error { + prop := &kernel.Param{ + Key: key, + Value: value, + } + + if _, ok := ctrl.defaults[key]; !ok { + if data, err := krnl.ReadParam(prop); err == nil { + ctrl.defaults[key] = string(data) + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + } + + if err := krnl.WriteParam(prop); err != nil { + return err + } + + ctrl.state[key] = value + + status := runtime.NewKernelParamStatus(runtime.NamespaceName, key) + + return r.Modify(ctx, status, func(res resource.Resource) error { + res.(*runtime.KernelParamStatus).TypedSpec().Current = value + res.(*runtime.KernelParamStatus).TypedSpec().Default = strings.TrimSpace(ctrl.defaults[key]) + + return nil + }) +} + +func (ctrl *KernelParamSpecController) resetKernelParam(ctx context.Context, r controller.Runtime, key string) error { + var err error + + if def, ok := ctrl.defaults[key]; ok { + err = krnl.WriteParam(&kernel.Param{ + Key: key, + Value: def, + }) + } else { + err = krnl.DeleteParam(&kernel.Param{Key: key}) + } + + if err != nil { + return err + } + + delete(ctrl.defaults, key) + delete(ctrl.state, key) + + return r.Destroy(ctx, resource.NewMetadata(runtime.NamespaceName, runtime.KernelParamStatusType, key, resource.VersionUndefined)) +} diff --git a/internal/app/machined/pkg/controllers/runtime/kernel_param_spec_test.go b/internal/app/machined/pkg/controllers/runtime/kernel_param_spec_test.go new file mode 100644 index 0000000..96a521c --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/kernel_param_spec_test.go @@ -0,0 +1,116 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime_test + +import ( + "os" + "strings" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + runtimecontrollers "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/runtime" + krnl "github.com/siderolabs/talos/pkg/kernel" + "github.com/siderolabs/talos/pkg/machinery/kernel" + runtimeresource "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +type KernelParamSpecSuite struct { + RuntimeSuite +} + +func (suite *KernelParamSpecSuite) TestParamsSynced() { + suite.Require().NoError(suite.runtime.RegisterController(&runtimecontrollers.KernelParamSpecController{})) + + suite.startRuntime() + + value := "500000" + def := "" + + spec := runtimeresource.NewKernelParamSpec(runtimeresource.NamespaceName, procSysfsFileMax) + spec.TypedSpec().Value = value + + param := &kernel.Param{Key: procSysfsFileMax} + + suite.Require().NoError(suite.state.Create(suite.ctx, spec)) + + statusMD := resource.NewMetadata(runtimeresource.NamespaceName, runtimeresource.KernelParamStatusType, procSysfsFileMax, resource.VersionUndefined) + + suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertResource( + statusMD, + func(res resource.Resource) bool { + def = res.(*runtimeresource.KernelParamStatus).TypedSpec().Default + + return res.(*runtimeresource.KernelParamStatus).TypedSpec().Current == value + }, + ), + )) + + prop, err := krnl.ReadParam(param) + suite.Assert().NoError(err) + suite.Require().Equal(value, strings.TrimSpace(string(prop))) + + suite.Require().NoError(suite.state.Destroy(suite.ctx, spec.Metadata())) + + // wait for the resource to be removed + suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + for _, md := range []resource.Metadata{statusMD} { + _, err = suite.state.Get(suite.ctx, md) + if err != nil { + if state.IsNotFoundError(err) { + return nil + } + + return err + } + } + + return retry.ExpectedErrorf("resource still exists") + }, + )) + + prop, err = krnl.ReadParam(param) + suite.Assert().NoError(err) + suite.Require().Equal(def, strings.TrimSpace(string(prop))) +} + +func (suite *KernelParamSpecSuite) TestParamsUnsupported() { + suite.Require().NoError(suite.runtime.RegisterController(&runtimecontrollers.KernelParamSpecController{})) + + suite.startRuntime() + + id := "proc.sys.some.really.not.existing.sysctl" + + spec := runtimeresource.NewKernelParamSpec(runtimeresource.NamespaceName, id) + spec.TypedSpec().Value = "value" + spec.TypedSpec().IgnoreErrors = true + + suite.Require().NoError(suite.state.Create(suite.ctx, spec)) + + statusMD := resource.NewMetadata(runtimeresource.NamespaceName, runtimeresource.KernelParamStatusType, id, resource.VersionUndefined) + + suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + suite.assertResource( + statusMD, + func(res resource.Resource) bool { + return res.(*runtimeresource.KernelParamStatus).TypedSpec().Unsupported == true + }, + ), + )) +} + +func TestKernelParamSpecSuite(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("skipping test because it requires root privileges") + } + + suite.Run(t, new(KernelParamSpecSuite)) +} diff --git a/internal/app/machined/pkg/controllers/runtime/kmsg_log.go b/internal/app/machined/pkg/controllers/runtime/kmsg_log.go new file mode 100644 index 0000000..c4e063d --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/kmsg_log.go @@ -0,0 +1,274 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + "errors" + "fmt" + "net/url" + "time" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/go-kmsg" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + networkutils "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network/utils" + machinedruntime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/logging" + "github.com/siderolabs/talos/pkg/machinery/config/config" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +const ( + drainTimeout = 100 * time.Millisecond + logSendTimeout = 5 * time.Second + logRetryTimeout = 1 * time.Second + logCloseTimeout = 5 * time.Second +) + +// KmsgLogDeliveryController watches events and forwards them to the events sink server +// if it's configured. +type KmsgLogDeliveryController struct { + Drainer *machinedruntime.Drainer + + drainSub *machinedruntime.DrainSubscription +} + +// Name implements controller.Controller interface. +func (ctrl *KmsgLogDeliveryController) Name() string { + return "runtime.KmsgLogDeliveryController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *KmsgLogDeliveryController) Inputs() []controller.Input { + return nil +} + +// Outputs implements controller.Controller interface. +func (ctrl *KmsgLogDeliveryController) Outputs() []controller.Output { + return nil +} + +// Run implements controller.Controller interface. +func (ctrl *KmsgLogDeliveryController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + if err := networkutils.WaitForNetworkReady(ctx, r, + func(status *network.StatusSpec) bool { + return status.AddressReady + }, + []controller.Input{ + { + Namespace: runtime.NamespaceName, + Type: runtime.KmsgLogConfigType, + ID: optional.Some(runtime.KmsgLogConfigID), + Kind: controller.InputWeak, + }, + }, + ); err != nil { + return fmt.Errorf("error waiting for network: %w", err) + } + + // initilalize kmsg reader early, so that we don't lose position on config changes + reader, err := kmsg.NewReader(kmsg.Follow()) + if err != nil { + return fmt.Errorf("error reading kernel messages: %w", err) + } + + defer reader.Close() //nolint:errcheck + + kmsgCh := reader.Scan(ctx) + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + cfg, err := safe.ReaderGetByID[*runtime.KmsgLogConfig](ctx, r, runtime.KmsgLogConfigID) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error getting configuration: %w", err) + } + + if cfg == nil { + // no config, wait for the next event + continue + } + + if err = ctrl.deliverLogs(ctx, r, logger, kmsgCh, cfg.TypedSpec().Destinations); err != nil { + return fmt.Errorf("error delivering logs: %w", err) + } + + r.ResetRestartBackoff() + } +} + +type logConfig struct { + endpoint *url.URL +} + +func (c logConfig) Format() string { + return constants.LoggingFormatJSONLines +} + +func (c logConfig) Endpoint() *url.URL { + return c.endpoint +} + +func (c logConfig) ExtraTags() map[string]string { + return nil +} + +//nolint:gocyclo +func (ctrl *KmsgLogDeliveryController) deliverLogs(ctx context.Context, r controller.Runtime, logger *zap.Logger, kmsgCh <-chan kmsg.Packet, destURLs []*url.URL) error { + if ctrl.drainSub == nil { + ctrl.drainSub = ctrl.Drainer.Subscribe() + } + + // initialize all log senders + destLogConfigs := xslices.Map(destURLs, func(u *url.URL) config.LoggingDestination { + return logConfig{endpoint: u} + }) + senders := xslices.Map(destLogConfigs, logging.NewJSONLines) + + defer func() { + closeCtx, closeCtxCancel := context.WithTimeout(context.Background(), logCloseTimeout) + defer closeCtxCancel() + + for _, sender := range senders { + if err := sender.Close(closeCtx); err != nil { + logger.Error("error closing log sender", zap.Error(err)) + } + } + }() + + var ( + drainTimer *time.Timer + drainTimerCh <-chan time.Time + ) + + for { + var msg kmsg.Packet + + select { + case <-ctx.Done(): + ctrl.drainSub.Cancel() + + return nil + case <-r.EventCh(): + // config changed, restart the loop + return nil + case <-ctrl.drainSub.EventCh(): + // drain started, assume that ksmg is drained if there're no new messages in drainTimeout + drainTimer = time.NewTimer(drainTimeout) + drainTimerCh = drainTimer.C + + continue + case <-drainTimerCh: + ctrl.drainSub.Cancel() + + return nil + case msg = <-kmsgCh: + if drainTimer != nil { + // if draining, reset the timer as there's a new message + if !drainTimer.Stop() { + <-drainTimer.C + } + + drainTimer.Reset(drainTimeout) + } + } + + if msg.Err != nil { + return fmt.Errorf("error receiving kernel logs: %w", msg.Err) + } + + event := machinedruntime.LogEvent{ + Msg: msg.Message.Message, + Time: msg.Message.Timestamp, + Level: kmsgPriorityToLevel(msg.Message.Priority), + Fields: map[string]interface{}{ + "facility": msg.Message.Facility.String(), + "seq": msg.Message.SequenceNumber, + "clock": msg.Message.Clock, + "priority": msg.Message.Priority.String(), + }, + } + + if err := ctrl.resend(ctx, r, logger, senders, &event); err != nil { + return fmt.Errorf("error sending log event: %w", err) + } + } +} + +//nolint:gocyclo +func (ctrl *KmsgLogDeliveryController) resend(ctx context.Context, r controller.Runtime, logger *zap.Logger, senders []machinedruntime.LogSender, e *machinedruntime.LogEvent) error { + for { + sendCtx, sendCancel := context.WithTimeout(ctx, logSendTimeout) + sendErrors := make(chan error, len(senders)) + + for _, sender := range senders { + go func() { + sendErrors <- sender.Send(sendCtx, e) + }() + } + + var dontRetry bool + + for range senders { + err := <-sendErrors + + // don't retry if at least one sender succeed to avoid implementing per-sender queue, etc + if err == nil { + dontRetry = true + + continue + } + + logger.Debug("error sending log event", zap.Error(err)) + + if errors.Is(err, machinedruntime.ErrDontRetry) || errors.Is(err, context.Canceled) { + dontRetry = true + } + } + + sendCancel() + + if dontRetry { + return nil + } + + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + // config changed, restart the loop + return errors.New("config changed") + case <-time.After(logRetryTimeout): + } + } +} + +func kmsgPriorityToLevel(pri kmsg.Priority) zapcore.Level { + switch pri { + case kmsg.Alert, kmsg.Crit, kmsg.Emerg, kmsg.Err: + return zapcore.ErrorLevel + case kmsg.Debug: + return zapcore.DebugLevel + case kmsg.Info, kmsg.Notice: + return zapcore.InfoLevel + case kmsg.Warning: + return zapcore.WarnLevel + default: + return zapcore.ErrorLevel + } +} diff --git a/internal/app/machined/pkg/controllers/runtime/kmsg_log_config.go b/internal/app/machined/pkg/controllers/runtime/kmsg_log_config.go new file mode 100644 index 0000000..a8c9a10 --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/kmsg_log_config.go @@ -0,0 +1,113 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + "fmt" + "net/url" + "slices" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/go-procfs/procfs" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// KmsgLogConfigController generates configuration for kmsg log delivery. +type KmsgLogConfigController struct { + Cmdline *procfs.Cmdline +} + +// Name implements controller.Controller interface. +func (ctrl *KmsgLogConfigController) Name() string { + return "runtime.KmsgLogConfigController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *KmsgLogConfigController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: config.MachineConfigType, + ID: optional.Some(config.V1Alpha1ID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *KmsgLogConfigController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: runtime.KmsgLogConfigType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *KmsgLogConfigController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) (err error) { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + destinations := []*url.URL{} + + if ctrl.Cmdline != nil { + if val := ctrl.Cmdline.Get(constants.KernelParamLoggingKernel).First(); val != nil { + destURL, err := url.Parse(*val) + if err != nil { + return fmt.Errorf("error parsing %q: %w", constants.KernelParamLoggingKernel, err) + } + + destinations = append(destinations, destURL) + } + } + + cfg, err := safe.ReaderGetByID[*config.MachineConfig](ctx, r, config.V1Alpha1ID) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error getting machine config: %w", err) + } + + if cfg != nil { + // remove duplicate URLs in case same destination is specified in both machine config and kernel args + destinations = append(destinations, xslices.Filter(cfg.Config().Runtime().KmsgLogURLs(), + func(u *url.URL) bool { + return !slices.ContainsFunc(destinations, func(v *url.URL) bool { + return v.String() == u.String() + }) + })...) + } + + r.StartTrackingOutputs() + + if len(destinations) > 0 { + if err = safe.WriterModify(ctx, r, runtime.NewKmsgLogConfig(), func(cfg *runtime.KmsgLogConfig) error { + cfg.TypedSpec().Destinations = destinations + + return nil + }); err != nil { + return fmt.Errorf("error updating kmsg log config: %w", err) + } + } + + if err = safe.CleanupOutputs[*runtime.KmsgLogConfig](ctx, r); err != nil { + return err + } + } +} diff --git a/internal/app/machined/pkg/controllers/runtime/kmsg_log_config_test.go b/internal/app/machined/pkg/controllers/runtime/kmsg_log_config_test.go new file mode 100644 index 0000000..0ed5f94 --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/kmsg_log_config_test.go @@ -0,0 +1,104 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime_test + +import ( + "net/url" + "testing" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/go-procfs/procfs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + runtimectrls "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/runtime" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/meta" + runtimecfg "github.com/siderolabs/talos/pkg/machinery/config/types/runtime" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +type KmsgLogConfigSuite struct { + ctest.DefaultSuite +} + +func TestKmsgLogConfigSuite(t *testing.T) { + suite.Run(t, new(KmsgLogConfigSuite)) +} + +func (suite *KmsgLogConfigSuite) TestKmsgLogConfigNone() { + suite.Require().NoError(suite.Runtime().RegisterController(&runtimectrls.KmsgLogConfigController{})) + + rtestutils.AssertNoResource[*runtime.KmsgLogConfig](suite.Ctx(), suite.T(), suite.State(), runtime.KmsgLogConfigID) +} + +func (suite *KmsgLogConfigSuite) TestKmsgLogConfigMachineConfig() { + cmdline := procfs.NewCmdline("") + cmdline.Append(constants.KernelParamLoggingKernel, "https://10.0.0.1:3333/logs") + + suite.Require().NoError(suite.Runtime().RegisterController(&runtimectrls.KmsgLogConfigController{ + Cmdline: cmdline, + })) + + kmsgLogConfig1 := &runtimecfg.KmsgLogV1Alpha1{ + MetaName: "1", + KmsgLogURL: meta.URL{ + URL: must(url.Parse("https://10.0.0.2:4444/logs")), + }, + } + + kmsgLogConfig2 := &runtimecfg.KmsgLogV1Alpha1{ + MetaName: "2", + KmsgLogURL: meta.URL{ + URL: must(url.Parse("https://10.0.0.1:3333/logs")), + }, + } + + cfg, err := container.New(kmsgLogConfig1, kmsgLogConfig2) + suite.Require().NoError(err) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), config.NewMachineConfig(cfg))) + + rtestutils.AssertResources[*runtime.KmsgLogConfig](suite.Ctx(), suite.T(), suite.State(), []resource.ID{runtime.KmsgLogConfigID}, + func(cfg *runtime.KmsgLogConfig, asrt *assert.Assertions) { + asrt.Equal( + []string{ + "https://10.0.0.1:3333/logs", + "https://10.0.0.2:4444/logs", + }, + xslices.Map(cfg.TypedSpec().Destinations, func(u *url.URL) string { return u.String() }), + ) + }) +} + +func (suite *KmsgLogConfigSuite) TestKmsgLogConfigCmdline() { + cmdline := procfs.NewCmdline("") + cmdline.Append(constants.KernelParamLoggingKernel, "https://10.0.0.1:3333/logs") + + suite.Require().NoError(suite.Runtime().RegisterController(&runtimectrls.KmsgLogConfigController{ + Cmdline: cmdline, + })) + + rtestutils.AssertResources[*runtime.KmsgLogConfig](suite.Ctx(), suite.T(), suite.State(), []resource.ID{runtime.KmsgLogConfigID}, + func(cfg *runtime.KmsgLogConfig, asrt *assert.Assertions) { + asrt.Equal( + []string{"https://10.0.0.1:3333/logs"}, + xslices.Map(cfg.TypedSpec().Destinations, func(u *url.URL) string { return u.String() }), + ) + }) +} + +func must[T any](t T, err error) T { + if err != nil { + panic(err) + } + + return t +} diff --git a/internal/app/machined/pkg/controllers/runtime/kmsg_log_test.go b/internal/app/machined/pkg/controllers/runtime/kmsg_log_test.go new file mode 100644 index 0000000..fc6912f --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/kmsg_log_test.go @@ -0,0 +1,270 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime_test + +import ( + "context" + "net" + "net/netip" + "net/url" + "os" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-retry/retry" + "github.com/siderolabs/siderolink/pkg/logreceiver" + "github.com/stretchr/testify/suite" + "go.uber.org/zap/zaptest" + + runtimectrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/runtime" + talosruntime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +type logHandler struct { + mu sync.Mutex + count int +} + +// HandleLog implements logreceiver.Handler. +func (s *logHandler) HandleLog(srcAddr netip.Addr, msg map[string]interface{}) { + s.mu.Lock() + defer s.mu.Unlock() + + s.count++ +} + +func (s *logHandler) getCount() int { + s.mu.Lock() + defer s.mu.Unlock() + + return s.count +} + +type KmsgLogDeliverySuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + drainer *talosruntime.Drainer + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc + + handler1, handler2 *logHandler + + listener1, listener2 net.Listener + srv1, srv2 *logreceiver.Server +} + +func (suite *KmsgLogDeliverySuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 10*time.Second) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + logger := zaptest.NewLogger(suite.T()) + + var err error + + suite.runtime, err = runtime.NewRuntime(suite.state, logger) + suite.Require().NoError(err) + + suite.handler1 = &logHandler{} + suite.handler2 = &logHandler{} + + suite.listener1, err = net.Listen("tcp", "localhost:0") + suite.Require().NoError(err) + + suite.listener2, err = net.Listen("tcp", "localhost:0") + suite.Require().NoError(err) + + suite.srv1 = logreceiver.NewServer(logger, suite.listener1, suite.handler1.HandleLog) + + suite.srv2 = logreceiver.NewServer(logger, suite.listener2, suite.handler2.HandleLog) + + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.srv1.Serve() //nolint:errcheck + }() + + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.srv2.Serve() //nolint:errcheck + }() + + suite.drainer = talosruntime.NewDrainer() + + suite.Require().NoError( + suite.runtime.RegisterController( + &runtimectrl.KmsgLogDeliveryController{ + Drainer: suite.drainer, + }, + ), + ) + + status := network.NewStatus(network.NamespaceName, network.StatusID) + status.TypedSpec().AddressReady = true + + suite.Require().NoError(suite.state.Create(suite.ctx, status)) +} + +func (suite *KmsgLogDeliverySuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *KmsgLogDeliverySuite) TestDeliverySingleDestination() { + suite.startRuntime() + + kmsgLogConfig := runtimeres.NewKmsgLogConfig() + kmsgLogConfig.TypedSpec().Destinations = []*url.URL{ + { + Scheme: "tcp", + Host: suite.listener1.Addr().String(), + }, + } + + suite.Require().NoError(suite.state.Create(suite.ctx, kmsgLogConfig)) + + // controller should deliver some kernel logs from host's kmsg buffer + suite.assertLogsSeen(suite.handler1) +} + +func (suite *KmsgLogDeliverySuite) TestDeliveryMultipleDestinations() { + suite.startRuntime() + + kmsgLogConfig := runtimeres.NewKmsgLogConfig() + kmsgLogConfig.TypedSpec().Destinations = []*url.URL{ + { + Scheme: "tcp", + Host: suite.listener1.Addr().String(), + }, + { + Scheme: "tcp", + Host: suite.listener2.Addr().String(), + }, + } + + suite.Require().NoError(suite.state.Create(suite.ctx, kmsgLogConfig)) + + // controller should deliver logs to both destinations + suite.assertLogsSeen(suite.handler1) + suite.assertLogsSeen(suite.handler2) +} + +func (suite *KmsgLogDeliverySuite) TestDeliveryOneDeadDestination() { + suite.startRuntime() + + // stop one listener + suite.Require().NoError(suite.listener1.Close()) + + kmsgLogConfig := runtimeres.NewKmsgLogConfig() + kmsgLogConfig.TypedSpec().Destinations = []*url.URL{ + { + Scheme: "tcp", + Host: suite.listener1.Addr().String(), + }, + { + Scheme: "tcp", + Host: suite.listener2.Addr().String(), + }, + } + + suite.Require().NoError(suite.state.Create(suite.ctx, kmsgLogConfig)) + + // controller should deliver logs to live destination + suite.assertLogsSeen(suite.handler2) +} + +func (suite *KmsgLogDeliverySuite) TestDeliveryAllDeadDestinations() { + suite.startRuntime() + + // stop all listeners + suite.Require().NoError(suite.listener1.Close()) + suite.Require().NoError(suite.listener2.Close()) + + kmsgLogConfig := runtimeres.NewKmsgLogConfig() + kmsgLogConfig.TypedSpec().Destinations = []*url.URL{ + { + Scheme: "tcp", + Host: suite.listener1.Addr().String(), + }, + { + Scheme: "tcp", + Host: suite.listener2.Addr().String(), + }, + } + + suite.Require().NoError(suite.state.Create(suite.ctx, kmsgLogConfig)) +} + +func (suite *KmsgLogDeliverySuite) TestDrain() { + suite.startRuntime() + + kmsgLogConfig := runtimeres.NewKmsgLogConfig() + kmsgLogConfig.TypedSpec().Destinations = []*url.URL{ + { + Scheme: "tcp", + Host: suite.listener1.Addr().String(), + }, + } + + suite.Require().NoError(suite.state.Create(suite.ctx, kmsgLogConfig)) + + // wait for controller to start delivering some logs + suite.assertLogsSeen(suite.handler1) + + // drain should be successful, i.e. controller should stop on its own before context is canceled + suite.Assert().NoError(suite.drainer.Drain(suite.ctx)) +} + +func (suite *KmsgLogDeliverySuite) assertLogsSeen(handler *logHandler) { + err := retry.Constant(time.Second*5, retry.WithUnits(time.Millisecond*100)).Retry( + func() error { + if handler.getCount() == 0 { + return retry.ExpectedErrorf("no logs received") + } + + return nil + }, + ) + suite.Require().NoError(err) +} + +func (suite *KmsgLogDeliverySuite) TearDownTest() { + suite.srv1.Stop() + suite.srv2.Stop() + + suite.ctxCancel() + + suite.wg.Wait() +} + +func TestKmsgLogDeliverySuite(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("requires root") + } + + suite.Run(t, new(KmsgLogDeliverySuite)) +} diff --git a/internal/app/machined/pkg/controllers/runtime/machine_status.go b/internal/app/machined/pkg/controllers/runtime/machine_status.go new file mode 100644 index 0000000..d391d1b --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/machine_status.go @@ -0,0 +1,476 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + v1 "k8s.io/api/core/v1" + + k8sadapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/k8s" + v1alpha1runtime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/api/common" + machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/time" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +// MachineStatusController watches MachineStatuss, sets/resets kernel params. +type MachineStatusController struct { + V1Alpha1Events v1alpha1runtime.Watcher + + setupOnce sync.Once + notifyOnce sync.Once + + notifyCh chan struct{} + + mu sync.Mutex + currentStage runtime.MachineStage +} + +// Name implements controller.Controller interface. +func (ctrl *MachineStatusController) Name() string { + return "runtime.MachineStatusController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *MachineStatusController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: v1alpha1.NamespaceName, + Type: time.StatusType, + ID: optional.Some(time.StatusID), + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.StatusType, + ID: optional.Some(network.StatusID), + Kind: controller.InputWeak, + }, + { + Namespace: v1alpha1.NamespaceName, + Type: v1alpha1.ServiceType, + Kind: controller.InputWeak, + }, + { + Namespace: k8s.NamespaceName, + Type: k8s.StaticPodStatusType, + Kind: controller.InputWeak, + }, + { + Namespace: config.NamespaceName, + Type: config.MachineTypeType, + ID: optional.Some(config.MachineTypeID), + Kind: controller.InputWeak, + }, + { + Namespace: k8s.NamespaceName, + Type: k8s.NodenameType, + ID: optional.Some(k8s.NodenameID), + Kind: controller.InputWeak, + }, + { + Namespace: k8s.NamespaceName, + Type: k8s.NodeStatusType, + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *MachineStatusController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: runtime.MachineStatusType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *MachineStatusController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + ctrl.setupOnce.Do(func() { + // watcher is started once and runs for all controller runs, as if we reconnect to the event stream, + // we might lose some state which was in the events, but it got "scrolled away" from the buffer. + ctrl.notifyCh = make(chan struct{}, 1) + go ctrl.watchEvents() + }) + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + case <-ctrl.notifyCh: + } + + machineTypeResource, err := safe.ReaderGet[*config.MachineType](ctx, r, config.NewMachineType().Metadata()) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting machine type: %w", err) + } + } + + var machineType machine.Type + + if machineTypeResource != nil { + machineType = machineTypeResource.MachineType() + } + + ctrl.mu.Lock() + currentStage := ctrl.currentStage + ctrl.mu.Unlock() + + ready := true + + var unmetConditions []runtime.UnmetCondition + + for _, check := range ctrl.getReadinessChecks(currentStage, machineType) { + if err := check.f(ctx, r); err != nil { + ready = false + + unmetConditions = append(unmetConditions, runtime.UnmetCondition{ + Name: check.name, + Reason: err.Error(), + }) + } + } + + if err := safe.WriterModify(ctx, r, runtime.NewMachineStatus(), func(ms *runtime.MachineStatus) error { + ms.TypedSpec().Stage = currentStage + ms.TypedSpec().Status.Ready = ready + ms.TypedSpec().Status.UnmetConditions = unmetConditions + + return nil + }); err != nil { + return fmt.Errorf("error updating machine status: %w", err) + } + + if currentStage == runtime.MachineStageRunning && ready { + ctrl.notifyOnce.Do(func() { + logger.Info("machine is running and ready") + }) + } + + r.ResetRestartBackoff() + } +} + +type readinessCheck struct { + name string + f func(context.Context, controller.Runtime) error +} + +func (ctrl *MachineStatusController) getReadinessChecks(stage runtime.MachineStage, machineType machine.Type) []readinessCheck { + requiredServices := []string{ + "apid", + "machined", + "kubelet", + } + + if machineType.IsControlPlane() { + requiredServices = append(requiredServices, + "etcd", + "trustd", + ) + } + + switch stage { //nolint:exhaustive + case runtime.MachineStageBooting, runtime.MachineStageRunning: + return []readinessCheck{ + { + name: "time", + f: ctrl.timeSyncCheck, + }, + { + name: "network", + f: ctrl.networkReadyCheck, + }, + { + name: "services", + f: ctrl.servicesCheck(requiredServices), + }, + { + name: "staticPods", + f: ctrl.staticPodsCheck, + }, + { + name: "nodeReady", + f: ctrl.nodeReadyCheck, + }, + } + default: + return nil + } +} + +func (ctrl *MachineStatusController) timeSyncCheck(ctx context.Context, r controller.Runtime) error { + timeSyncStatus, err := safe.ReaderGet[*time.Status](ctx, r, time.NewStatus().Metadata()) + if err != nil { + return err + } + + if !timeSyncStatus.TypedSpec().Synced { + return errors.New("time is not synced") + } + + return nil +} + +func (ctrl *MachineStatusController) networkReadyCheck(ctx context.Context, r controller.Runtime) error { + networkStatus, err := safe.ReaderGet[*network.Status](ctx, r, network.NewStatus(network.NamespaceName, network.StatusID).Metadata()) + if err != nil { + return err + } + + var notReady []string + + if !networkStatus.TypedSpec().AddressReady { + notReady = append(notReady, "address") + } + + if !networkStatus.TypedSpec().ConnectivityReady { + notReady = append(notReady, "connectivity") + } + + if !networkStatus.TypedSpec().EtcFilesReady { + notReady = append(notReady, "etc-files") + } + + if !networkStatus.TypedSpec().HostnameReady { + notReady = append(notReady, "hostname") + } + + if len(notReady) == 0 { + return nil + } + + return fmt.Errorf("waiting on: %s", strings.Join(notReady, ", ")) +} + +func (ctrl *MachineStatusController) servicesCheck(requiredServices []string) func(ctx context.Context, r controller.Runtime) error { + return func(ctx context.Context, r controller.Runtime) error { + serviceList, err := safe.ReaderListAll[*v1alpha1.Service](ctx, r) + if err != nil { + return err + } + + var problems []string + + runningServices := map[string]struct{}{} + + for it := serviceList.Iterator(); it.Next(); { + service := it.Value() + + if !service.TypedSpec().Running { + problems = append(problems, fmt.Sprintf("%s not running", service.Metadata().ID())) + + continue + } + + runningServices[service.Metadata().ID()] = struct{}{} + + if !service.TypedSpec().Unknown && !service.TypedSpec().Healthy { + problems = append(problems, fmt.Sprintf("%s not healthy", service.Metadata().ID())) + } + } + + for _, svc := range requiredServices { + if _, running := runningServices[svc]; !running { + problems = append(problems, fmt.Sprintf("%s not running", svc)) + } + } + + if len(problems) == 0 { + return nil + } + + return fmt.Errorf("%s", strings.Join(problems, ", ")) + } +} + +//nolint:gocyclo +func (ctrl *MachineStatusController) staticPodsCheck(ctx context.Context, r controller.Runtime) error { + staticPodList, err := safe.ReaderListAll[*k8s.StaticPodStatus](ctx, r) + if err != nil { + return err + } + + var problems []string + + for it := staticPodList.Iterator(); it.Next(); { + status, err := k8sadapter.StaticPodStatus(it.Value()).Status() + if err != nil { + return err + } + + switch status.Phase { + case v1.PodPending, v1.PodFailed, v1.PodUnknown: + problems = append(problems, fmt.Sprintf("%s %s", it.Value().Metadata().ID(), strings.ToLower(string(status.Phase)))) + case v1.PodSucceeded: + // do nothing, terminal phase + case v1.PodRunning: + // check readiness + ready := false + + for _, condition := range status.Conditions { + if condition.Type == v1.PodReady { + ready = condition.Status == v1.ConditionTrue + + break + } + } + + if !ready { + problems = append(problems, fmt.Sprintf("%s not ready", it.Value().Metadata().ID())) + } + } + } + + if len(problems) == 0 { + return nil + } + + return fmt.Errorf("%s", strings.Join(problems, ", ")) +} + +func (ctrl *MachineStatusController) nodeReadyCheck(ctx context.Context, r controller.Runtime) error { + nodename, err := safe.ReaderGetByID[*k8s.Nodename](ctx, r, k8s.NodenameID) + if err != nil { + if state.IsNotFoundError(err) { + // nodename not established yet, skip + return nil + } + + return fmt.Errorf("failed to get nodename: %w", err) + } + + if nodename.TypedSpec().SkipNodeRegistration { + // node registration skipped, skip the check + return nil + } + + nodeStatus, err := safe.ReaderGetByID[*k8s.NodeStatus](ctx, r, nodename.TypedSpec().Nodename) + if err != nil { + if state.IsNotFoundError(err) { + // node not established yet, skip + return fmt.Errorf("node %q status is not available yet", nodename.TypedSpec().Nodename) + } + + return fmt.Errorf("failed to get node status: %w", err) + } + + if !nodeStatus.TypedSpec().NodeReady { + return fmt.Errorf("node %q is not ready", nodename.TypedSpec().Nodename) + } + + return nil +} + +//nolint:gocyclo,cyclop +func (ctrl *MachineStatusController) watchEvents() { + // the interface of the Watch function is weird (blaming myself @smira) + // + // at the same time as it is events based, it's impossible to reconcile the current state + // from the events, so what we're doing is watching the events forever as soon as the controller starts, + // and aggregating the state into the stage variable, notifying the controller whenever the state changes. + ctrl.V1Alpha1Events.Watch(func(eventCh <-chan v1alpha1runtime.EventInfo) { //nolint:errcheck + var ( + oldStage runtime.MachineStage + currentSequence string + ) + + for ev := range eventCh { + newStage := oldStage + + switch event := ev.Event.Payload.(type) { + case *machineapi.SequenceEvent: + currentSequence = event.Sequence + + switch event.Action { + case machineapi.SequenceEvent_START: + // mostly interested in sequence start events + switch event.Sequence { + case v1alpha1runtime.SequenceBoot.String(), v1alpha1runtime.SequenceInitialize.String(): + newStage = runtime.MachineStageBooting + case v1alpha1runtime.SequenceInstall.String(): + // install sequence is run always, even if the machine is already installed, so we'll catch it by phase name + case v1alpha1runtime.SequenceShutdown.String(): + newStage = runtime.MachineStageShuttingDown + case v1alpha1runtime.SequenceUpgrade.String(), v1alpha1runtime.SequenceStageUpgrade.String(), v1alpha1runtime.SequenceMaintenanceUpgrade.String(): + newStage = runtime.MachineStageUpgrading + case v1alpha1runtime.SequenceReset.String(): + newStage = runtime.MachineStageResetting + case v1alpha1runtime.SequenceReboot.String(): + newStage = runtime.MachineStageRebooting + } + case machineapi.SequenceEvent_NOOP: + if event.Error != nil && event.Error.Code == common.Code_FATAL { + // fatal errors lead to reboot + newStage = runtime.MachineStageRebooting + } + case machineapi.SequenceEvent_STOP: + if event.Sequence == v1alpha1runtime.SequenceBoot.String() && event.Error == nil { + newStage = runtime.MachineStageRunning + } + + // sequence finished, it doesn't matter whether if it was successful or not + currentSequence = "" + } + case *machineapi.PhaseEvent: + if event.Action == machineapi.PhaseEvent_START { + switch { + case currentSequence == v1alpha1runtime.SequenceInstall.String() && event.Phase == "install": + newStage = runtime.MachineStageInstalling + case (currentSequence == v1alpha1runtime.SequenceInstall.String() || + currentSequence == v1alpha1runtime.SequenceUpgrade.String() || + currentSequence == v1alpha1runtime.SequenceStageUpgrade.String() || + currentSequence == v1alpha1runtime.SequenceMaintenanceUpgrade.String()) && event.Phase == "kexec": + newStage = runtime.MachineStageRebooting + } + } + case *machineapi.TaskEvent: + if event.Task == "runningMaintenance" { + switch event.Action { + case machineapi.TaskEvent_START: + newStage = runtime.MachineStageMaintenance + case machineapi.TaskEvent_STOP: + newStage = runtime.MachineStageBooting + } + } + } + + if oldStage != newStage { + ctrl.mu.Lock() + ctrl.currentStage = newStage + ctrl.mu.Unlock() + + select { + case ctrl.notifyCh <- struct{}{}: + default: + } + } + + oldStage = newStage + } + }, v1alpha1runtime.WithTailEvents(-1)) +} diff --git a/internal/app/machined/pkg/controllers/runtime/machine_status_publisher.go b/internal/app/machined/pkg/controllers/runtime/machine_status_publisher.go new file mode 100644 index 0000000..4d27796 --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/machine_status_publisher.go @@ -0,0 +1,84 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/xslices" + "go.uber.org/zap" + + v1alpha1runtime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// MachineStatusPublisherController watches MachineStatusPublishers, sets/resets kernel params. +type MachineStatusPublisherController struct { + V1Alpha1Events v1alpha1runtime.Publisher +} + +// Name implements controller.Controller interface. +func (ctrl *MachineStatusPublisherController) Name() string { + return "runtime.MachineStatusPublisherController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *MachineStatusPublisherController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: runtime.NamespaceName, + Type: runtime.MachineStatusType, + ID: optional.Some(runtime.MachineStatusID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *MachineStatusPublisherController) Outputs() []controller.Output { + return nil +} + +// Run implements controller.Controller interface. +func (ctrl *MachineStatusPublisherController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + machineStatus, err := safe.ReaderGet[*runtime.MachineStatus](ctx, r, runtime.NewMachineStatus().Metadata()) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error reading machine status: %w", err) + } + + ctrl.V1Alpha1Events.Publish(ctx, &machine.MachineStatusEvent{ + Stage: machine.MachineStatusEvent_MachineStage(machineStatus.TypedSpec().Stage), + Status: &machine.MachineStatusEvent_MachineStatus{ + Ready: machineStatus.TypedSpec().Status.Ready, + UnmetConditions: xslices.Map(machineStatus.TypedSpec().Status.UnmetConditions, + func(unmetCondition runtime.UnmetCondition) *machine.MachineStatusEvent_MachineStatus_UnmetCondition { + return &machine.MachineStatusEvent_MachineStatus_UnmetCondition{ + Name: unmetCondition.Name, + Reason: unmetCondition.Reason, + } + }), + }, + }) + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/runtime/machine_status_test.go b/internal/app/machined/pkg/controllers/runtime/machine_status_test.go new file mode 100644 index 0000000..5ce93da --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/machine_status_test.go @@ -0,0 +1,159 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime_test + +import ( + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/siderolabs/gen/xslices" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + runtimectrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/runtime" + v1alpha1runtime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" + timeres "github.com/siderolabs/talos/pkg/machinery/resources/time" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +func TestMachineStatusSuite(t *testing.T) { + eventCh := make(chan v1alpha1runtime.EventInfo) + + suite.Run(t, &MachineStatusSuite{ + eventCh: eventCh, + DefaultSuite: ctest.DefaultSuite{ + Timeout: 5 * time.Second, + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(&runtimectrl.MachineStatusController{ + V1Alpha1Events: &mockWatcher{eventCh: eventCh}, + })) + }, + }, + }) +} + +type mockWatcher struct { + eventCh chan v1alpha1runtime.EventInfo +} + +func (m *mockWatcher) Watch(f v1alpha1runtime.WatchFunc, opt ...v1alpha1runtime.WatchOptionFunc) error { + f(m.eventCh) + + return nil +} + +type MachineStatusSuite struct { + ctest.DefaultSuite + + eventCh chan v1alpha1runtime.EventInfo +} + +func (suite *MachineStatusSuite) assertMachineStatus(stage runtime.MachineStage, ready bool, unmetConditions []string) { + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{runtime.MachineStatusID}, + func(machineStatus *runtime.MachineStatus, asrt *assert.Assertions) { + asrt.Equal(stage, machineStatus.TypedSpec().Stage) + asrt.Equal(ready, machineStatus.TypedSpec().Status.Ready) + + asrt.Equal(unmetConditions, + xslices.Map(machineStatus.TypedSpec().Status.UnmetConditions, func(c runtime.UnmetCondition) string { return c.Name })) + }) +} + +func (suite *MachineStatusSuite) TestReconcile() { + suite.assertMachineStatus(runtime.MachineStageUnknown, true, nil) + + suite.eventCh <- v1alpha1runtime.EventInfo{ + Event: v1alpha1runtime.Event{ + Payload: &machineapi.SequenceEvent{ + Sequence: v1alpha1runtime.SequenceInitialize.String(), + Action: machineapi.SequenceEvent_START, + }, + }, + } + + suite.assertMachineStatus(runtime.MachineStageBooting, false, []string{"time", "network", "services"}) + + machineType := config.NewMachineType() + machineType.SetMachineType(machine.TypeControlPlane) + suite.Require().NoError(suite.State().Create(suite.Ctx(), machineType)) + + timeStatus := timeres.NewStatus() + timeStatus.TypedSpec().Synced = true + suite.Require().NoError(suite.State().Create(suite.Ctx(), timeStatus)) + + suite.eventCh <- v1alpha1runtime.EventInfo{ + Event: v1alpha1runtime.Event{ + Payload: &machineapi.SequenceEvent{ + Sequence: v1alpha1runtime.SequenceBoot.String(), + Action: machineapi.SequenceEvent_START, + }, + }, + } + + suite.assertMachineStatus(runtime.MachineStageBooting, false, []string{"network", "services"}) + + suite.eventCh <- v1alpha1runtime.EventInfo{ + Event: v1alpha1runtime.Event{ + Payload: &machineapi.SequenceEvent{ + Sequence: v1alpha1runtime.SequenceBoot.String(), + Action: machineapi.SequenceEvent_STOP, + }, + }, + } + + networkStatus := network.NewStatus(network.NamespaceName, network.StatusID) + networkStatus.TypedSpec().AddressReady = true + networkStatus.TypedSpec().ConnectivityReady = true + networkStatus.TypedSpec().EtcFilesReady = true + networkStatus.TypedSpec().HostnameReady = true + suite.Require().NoError(suite.State().Create(suite.Ctx(), networkStatus)) + + suite.assertMachineStatus(runtime.MachineStageRunning, false, []string{"services"}) + + for _, service := range []string{"apid", "etcd", "kubelet", "machined", "trustd"} { + serviceStatus := v1alpha1.NewService(service) + serviceStatus.TypedSpec().Running = true + serviceStatus.TypedSpec().Healthy = true + suite.Require().NoError(suite.State().Create(suite.Ctx(), serviceStatus)) + } + + suite.assertMachineStatus(runtime.MachineStageRunning, true, nil) + + nodename := k8s.NewNodename(k8s.NamespaceName, k8s.NodenameID) + nodename.TypedSpec().Nodename = "test" + suite.Require().NoError(suite.State().Create(suite.Ctx(), nodename)) + + suite.assertMachineStatus(runtime.MachineStageRunning, false, []string{"nodeReady"}) + + nodeStatus := k8s.NewNodeStatus(k8s.NamespaceName, "test") + suite.Require().NoError(suite.State().Create(suite.Ctx(), nodeStatus)) + + suite.assertMachineStatus(runtime.MachineStageRunning, false, []string{"nodeReady"}) + + nodeStatus.TypedSpec().NodeReady = true + suite.Require().NoError(suite.State().Update(suite.Ctx(), nodeStatus)) + + suite.assertMachineStatus(runtime.MachineStageRunning, true, nil) + + suite.eventCh <- v1alpha1runtime.EventInfo{ + Event: v1alpha1runtime.Event{ + Payload: &machineapi.SequenceEvent{ + Sequence: v1alpha1runtime.SequenceReboot.String(), + Action: machineapi.SequenceEvent_START, + }, + }, + } + + suite.assertMachineStatus(runtime.MachineStageRebooting, true, nil) +} diff --git a/internal/app/machined/pkg/controllers/runtime/maintenance_config.go b/internal/app/machined/pkg/controllers/runtime/maintenance_config.go new file mode 100644 index 0000000..d19ea31 --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/maintenance_config.go @@ -0,0 +1,129 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + "fmt" + "net/netip" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/xslices" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/siderolink" +) + +// MaintenanceConfigController manages Maintenance Service config: which address it should listen on, etc. +type MaintenanceConfigController struct{} + +// Name implements controller.Controller interface. +func (ctrl *MaintenanceConfigController) Name() string { + return "runtime.MaintenanceConfigController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *MaintenanceConfigController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: siderolink.ConfigType, + ID: optional.Some(siderolink.ConfigID), + }, + { + Namespace: network.NamespaceName, + Type: network.NodeAddressType, + ID: optional.Some(network.NodeAddressCurrentID), + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *MaintenanceConfigController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: runtime.MaintenanceServiceConfigType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *MaintenanceConfigController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + nodeAddresses, err := safe.ReaderGetByID[*network.NodeAddress](ctx, r, network.NodeAddressCurrentID) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error getting node address: %w", err) + } + + var ( + listenAddress string + reachableAddresses []netip.Addr + ) + + if nodeAddresses != nil { + reachableAddresses = nodeAddresses.TypedSpec().IPs() + } + + _, err = safe.ReaderGetByID[*siderolink.Config](ctx, r, siderolink.ConfigID) + + // check if SideroLink config exists: + switch { + // * if it exists, find the SideroLink address and listen only on it + case err == nil: + if nodeAddresses != nil { + sideroLinkAddresses := xslices.Filter(nodeAddresses.TypedSpec().IPs(), func(addr netip.Addr) bool { + return network.IsULA(addr, network.ULASideroLink) + }) + + if len(sideroLinkAddresses) > 0 { + listenAddress = nethelpers.JoinHostPort(sideroLinkAddresses[0].String(), constants.ApidPort) + reachableAddresses = sideroLinkAddresses[:1] + } + } + // * if it doesn't exist, listen on '*' + case state.IsNotFoundError(err): + listenAddress = fmt.Sprintf(":%d", constants.ApidPort) + default: + return fmt.Errorf("error getting siderolink config: %w", err) + } + + if listenAddress == "" { + // drop config + if err = r.Destroy(ctx, runtime.NewMaintenanceServiceConfig().Metadata()); err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error destroying maintenance config: %w", err) + } + } else { + // create/update config + if err = safe.WriterModify[*runtime.MaintenanceServiceConfig](ctx, r, runtime.NewMaintenanceServiceConfig(), + func(config *runtime.MaintenanceServiceConfig) error { + config.TypedSpec().ListenAddress = listenAddress + config.TypedSpec().ReachableAddresses = reachableAddresses + + return nil + }); err != nil { + return fmt.Errorf("error updating maintenance config: %w", err) + } + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/runtime/maintenance_config_test.go b/internal/app/machined/pkg/controllers/runtime/maintenance_config_test.go new file mode 100644 index 0000000..f508f56 --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/maintenance_config_test.go @@ -0,0 +1,64 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime_test + +import ( + "net/netip" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + runtimectrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/siderolink" +) + +func TestMaintenanceConfigSuite(t *testing.T) { + suite.Run(t, &MaintenanceConfigSuite{ + DefaultSuite: ctest.DefaultSuite{ + Timeout: 5 * time.Second, + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(&runtimectrl.MaintenanceConfigController{})) + }, + }, + }) +} + +type MaintenanceConfigSuite struct { + ctest.DefaultSuite +} + +func (suite *MaintenanceConfigSuite) TestReconcile() { + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{runtime.MaintenanceServiceConfigID}, + func(cfg *runtime.MaintenanceServiceConfig, asrt *assert.Assertions) { + asrt.Equal(":50000", cfg.TypedSpec().ListenAddress) + asrt.Nil(cfg.TypedSpec().ReachableAddresses) + }) + + siderolinkConfig := siderolink.NewConfig(config.NamespaceName, siderolink.ConfigID) + suite.Require().NoError(suite.State().Create(suite.Ctx(), siderolinkConfig)) + + rtestutils.AssertNoResource[*runtime.MaintenanceServiceConfig](suite.Ctx(), suite.T(), suite.State(), runtime.MaintenanceServiceConfigID) + + nodeAddresses := network.NewNodeAddress(network.NamespaceName, network.NodeAddressCurrentID) + nodeAddresses.TypedSpec().Addresses = []netip.Prefix{ + netip.MustParsePrefix("172.16.0.1/24"), + netip.MustParsePrefix("fdae:41e4:649b:9303:2a07:9c7:5b08:aef7/64"), + } + suite.Require().NoError(suite.State().Create(suite.Ctx(), nodeAddresses)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{runtime.MaintenanceServiceConfigID}, + func(cfg *runtime.MaintenanceServiceConfig, asrt *assert.Assertions) { + asrt.Equal("[fdae:41e4:649b:9303:2a07:9c7:5b08:aef7]:50000", cfg.TypedSpec().ListenAddress) + asrt.Equal([]netip.Addr{netip.MustParseAddr("fdae:41e4:649b:9303:2a07:9c7:5b08:aef7")}, cfg.TypedSpec().ReachableAddresses) + }) +} diff --git a/internal/app/machined/pkg/controllers/runtime/maintenance_service.go b/internal/app/machined/pkg/controllers/runtime/maintenance_service.go new file mode 100644 index 0000000..4fde82d --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/maintenance_service.go @@ -0,0 +1,307 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + "fmt" + "net" + "net/netip" + "reflect" + "sync" + "time" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/crypto/x509" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/go-debug" + "go.uber.org/zap" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + + "github.com/aenix-io/talm/internal/app/maintenance" + "github.com/siderolabs/talos/pkg/grpc/factory" + "github.com/siderolabs/talos/pkg/grpc/middleware/authz" + machineryconfig "github.com/siderolabs/talos/pkg/machinery/config" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" +) + +// MaintenanceServiceController runs the maintenance service based on the configuration. +type MaintenanceServiceController struct { + SiderolinkPeerCheckFunc authz.SideroLinkPeerCheckFunc +} + +// Name implements controller.Controller interface. +func (ctrl *MaintenanceServiceController) Name() string { + return "runtime.MaintenanceServiceController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *MaintenanceServiceController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: runtime.NamespaceName, + Type: runtime.MaintenanceServiceRequestType, + ID: optional.Some(runtime.MaintenanceServiceRequestID), + Kind: controller.InputStrong, + }, + { + Namespace: runtime.NamespaceName, + Type: runtime.MaintenanceServiceConfigType, + ID: optional.Some(runtime.MaintenanceServiceConfigID), + Kind: controller.InputWeak, + }, + { + Namespace: secrets.NamespaceName, + Type: secrets.MaintenanceServiceCertsType, + ID: optional.Some(secrets.MaintenanceServiceCertsID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *MaintenanceServiceController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: config.MachineConfigType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *MaintenanceServiceController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + var ( + server *grpc.Server + serverWg sync.WaitGroup + listener net.Listener + lastReachableAddresses []string + lastCertificateFingerprint string + lastListenAddress string + usagePrinted bool + ) + + shutdownServer := func(ctx context.Context) { + if server != nil { + shutdownCtx, shutdownCancel := context.WithTimeout(ctx, 5*time.Second) + defer shutdownCancel() + + factory.ServerGracefulStop(server, shutdownCtx) + + serverWg.Wait() + + server = nil + } + + if listener != nil { + listener.Close() //nolint:errcheck + + listener = nil + lastReachableAddresses = nil + lastListenAddress = "" + } + + // clean up maintenance machine config, as we are done with the maintenance service + err := r.Destroy(ctx, config.NewMachineConfigWithID(nil, config.MaintenanceID).Metadata()) + if err != nil && !state.IsNotFoundError(err) { + logger.Error("failed to destroy maintenance machine config", zap.String("id", config.MaintenanceID), zap.Error(err)) + } + } + + defer shutdownServer(context.Background()) + + cfgCh := make(chan machineryconfig.Provider) + srv := maintenance.New(cfgCh) + + injector := &authz.Injector{ + Mode: authz.ReadOnlyWithAdminOnSiderolink, + SideroLinkPeerCheckFunc: ctrl.SiderolinkPeerCheckFunc, + } + + if debug.Enabled { + injector.Logger = logger.Sugar().Infof + } + + tlsProvider := maintenance.NewTLSProvider() + + for { + select { + case <-ctx.Done(): + return nil + case cfg := <-cfgCh: + configResource := config.NewMachineConfigWithID(cfg, config.MaintenanceID) + + oldConfigResource, err := safe.ReaderGetByID[*config.MachineConfig](ctx, r, config.MaintenanceID) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("failed to get machine config: %w", err) + } + + if state.IsNotFoundError(err) { + if err = r.Create(ctx, configResource); err != nil { + return fmt.Errorf("failed to create machine config: %w", err) + } + } else { + configResource.Metadata().SetVersion(oldConfigResource.Metadata().Version()) + + if err = configResource.Metadata().SetOwner(oldConfigResource.Metadata().Owner()); err != nil { + return fmt.Errorf("error setting owner: %w", err) + } + + if err = r.Update(ctx, configResource); err != nil { + return fmt.Errorf("failed to update machine config: %w", err) + } + } + + continue + case <-r.EventCh(): + } + + request, err := safe.ReaderGetByID[*runtime.MaintenanceServiceRequest](ctx, r, runtime.MaintenanceServiceRequestID) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("failed to get maintenance service request: %w", err) + } + + if request == nil { + // no request, nothing to do + shutdownServer(ctx) + + continue + } + + if request.Metadata().Phase() == resource.PhaseTearingDown { + // stop the server & remove the finalizer + shutdownServer(ctx) + + if err = r.RemoveFinalizer(ctx, request.Metadata(), ctrl.Name()); err != nil { + return fmt.Errorf("failed to remove finalizer: %w", err) + } + + continue + } + + cfg, err := safe.ReaderGetByID[*runtime.MaintenanceServiceConfig](ctx, r, runtime.MaintenanceServiceConfigID) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("failed to get maintenance service config: %w", err) + } + + cert, err := safe.ReaderGetByID[*secrets.MaintenanceServiceCerts](ctx, r, secrets.MaintenanceServiceCertsID) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("failed to get maintenance service certs: %w", err) + } + + if cert != nil { + if err = tlsProvider.Update(cert); err != nil { + return fmt.Errorf("failed to update tls provider: %w", err) + } + } + + // immediately add a finalizer + if err = r.AddFinalizer(ctx, request.Metadata(), ctrl.Name()); err != nil { + return fmt.Errorf("failed to add finalizer: %w", err) + } + + if cfg == nil { + // no config, nothing to do + shutdownServer(ctx) + + continue + } + + if listener != nil && cfg.TypedSpec().ListenAddress != lastListenAddress { + // listen address changed, restart the server + shutdownServer(ctx) + } + + if listener == nil { + listener, err = net.Listen("tcp", cfg.TypedSpec().ListenAddress) + if err != nil { + return fmt.Errorf("failed to listen: %w", err) + } + + lastListenAddress = cfg.TypedSpec().ListenAddress + } + + if server == nil { + tlsConfig, err := tlsProvider.TLSConfig() + if err != nil { + return fmt.Errorf("failed to get tls config: %w", err) + } + + server = factory.NewServer( + srv, + factory.WithDefaultLog(), + factory.ServerOptions( + grpc.Creds( + credentials.NewTLS(tlsConfig), + ), + ), + + factory.WithUnaryInterceptor(injector.UnaryInterceptor()), + factory.WithStreamInterceptor(injector.StreamInterceptor()), + ) + + serverWg.Add(1) + + go func() { + defer serverWg.Done() + + //nolint:errcheck + server.Serve(listener) + }() + } + + // print additional information for the user on important state changes + reachableAddresses := xslices.Map(cfg.TypedSpec().ReachableAddresses, netip.Addr.String) + + if !reflect.DeepEqual(lastReachableAddresses, reachableAddresses) { + logger.Info("this machine is reachable at:") + + for _, addr := range reachableAddresses { + logger.Info("\t" + addr) + } + + lastReachableAddresses = reachableAddresses + } + + if cert != nil { + certificateFingerprint, err := x509.SPKIFingerprintFromPEM(cert.TypedSpec().Server.Crt) + if err != nil { + return fmt.Errorf("failed to get certificate fingerprint: %w", err) + } + + fingerprint := certificateFingerprint.String() + + if fingerprint != lastCertificateFingerprint { + logger.Info("server certificate issued", zap.String("fingerprint", fingerprint)) + } + + lastCertificateFingerprint = fingerprint + } + + if !usagePrinted && len(reachableAddresses) > 0 && lastCertificateFingerprint != "" { + firstIP := reachableAddresses[0] + + logger.Sugar().Info("upload configuration using talosctl:") + logger.Sugar().Infof("\ttalosctl apply-config --insecure --nodes %s --file ", firstIP) + logger.Sugar().Info("or apply configuration using talosctl interactive installer:") + logger.Sugar().Infof("\ttalosctl apply-config --insecure --nodes %s --mode=interactive", firstIP) + logger.Sugar().Info("optionally with node fingerprint check:") + logger.Sugar().Infof("\ttalosctl apply-config --insecure --nodes %s --cert-fingerprint '%s' --file ", firstIP, lastCertificateFingerprint) + + usagePrinted = true + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/runtime/maintenance_service_test.go b/internal/app/machined/pkg/controllers/runtime/maintenance_service_test.go new file mode 100644 index 0000000..7c56132 --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/maintenance_service_test.go @@ -0,0 +1,299 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime_test + +import ( + "context" + "crypto/tls" + "net" + "net/netip" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/registry" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "google.golang.org/grpc/metadata" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + runtimectrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/secrets" + talosruntime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/metal" + "github.com/aenix-io/talm/internal/app/maintenance" + machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/client" + "github.com/siderolabs/talos/pkg/machinery/config" + configres "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/hardware" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +const isSiderolinkPeerHeaderKey = "is-siderolink-peer" + +func TestMaintenanceServiceSuite(t *testing.T) { + suite.Run(t, &MaintenanceServiceSuite{ + DefaultSuite: ctest.DefaultSuite{ + Timeout: 5 * time.Second, + AfterSetup: func(suite *ctest.DefaultSuite) { + maintenance.InjectController(mockController{s: suite.State()}) + + suite.Require().NoError(suite.Runtime().RegisterController(&secrets.MaintenanceRootController{})) + suite.Require().NoError(suite.Runtime().RegisterController(&secrets.MaintenanceCertSANsController{})) + suite.Require().NoError(suite.Runtime().RegisterController(&secrets.MaintenanceController{})) + suite.Require().NoError(suite.Runtime().RegisterController(&runtimectrl.MaintenanceServiceController{ + SiderolinkPeerCheckFunc: func(ctx context.Context) (netip.Addr, bool) { + isSiderolinkPeer := len(metadata.ValueFromIncomingContext(ctx, isSiderolinkPeerHeaderKey)) > 0 + if isSiderolinkPeer { + return netip.MustParseAddr("127.0.0.42"), true + } + + return netip.Addr{}, false + }, + })) + }, + }, + }) +} + +type MaintenanceServiceSuite struct { + ctest.DefaultSuite +} + +func (suite *MaintenanceServiceSuite) findListenAddr() string { + l, err := net.Listen("tcp", "127.0.0.1:0") + suite.Require().NoError(err) + + addr := l.Addr().String() + + suite.Require().NoError(l.Close()) + + return addr +} + +func (suite *MaintenanceServiceSuite) TestRunService() { + nodeAddresses := network.NewNodeAddress(network.NamespaceName, network.NodeAddressAccumulativeID) + nodeAddresses.TypedSpec().Addresses = []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")} + suite.Require().NoError(suite.State().Create(suite.Ctx(), nodeAddresses)) + + maintenanceConfig := runtime.NewMaintenanceServiceConfig() + maintenanceConfig.TypedSpec().ListenAddress = suite.findListenAddr() + maintenanceConfig.TypedSpec().ReachableAddresses = []netip.Addr{netip.MustParseAddr("10.0.0.1")} + suite.Require().NoError(suite.State().Create(suite.Ctx(), maintenanceConfig)) + + maintenanceRequest := runtime.NewMaintenanceServiceRequest() + suite.Require().NoError(suite.State().Create(suite.Ctx(), maintenanceRequest)) + + // wait for the service to be up + suite.AssertWithin(time.Second, 10*time.Millisecond, func() error { + c, err := tls.Dial("tcp", maintenanceConfig.TypedSpec().ListenAddress, + &tls.Config{ + InsecureSkipVerify: true, + }, + ) + + if c != nil { + c.Close() //nolint:errcheck + } + + return retry.ExpectedError(err) + }) + + // test API + mc, err := client.New(suite.Ctx(), + client.WithTLSConfig(&tls.Config{ + InsecureSkipVerify: true, + }), client.WithEndpoints(maintenanceConfig.TypedSpec().ListenAddress), + ) + suite.Require().NoError(err) + + _, err = mc.Version(suite.Ctx()) + suite.Require().ErrorContains(err, "API is not implemented in maintenance mode") + + // apply partial machine config + _, err = mc.ApplyConfiguration(suite.Ctx(), &machineapi.ApplyConfigurationRequest{ + Data: []byte(` +apiVersion: v1alpha1 +kind: KmsgLogConfig +name: test +url: "tcp://127.0.0.42:1234" +`), + }) + suite.Require().NoError(err) + + // assert that the config with the maintenance ID is created + rtestutils.AssertResource[*configres.MachineConfig](suite.Ctx(), suite.T(), suite.State(), configres.MaintenanceID, func(r *configres.MachineConfig, assertion *assert.Assertions) { + configBytes, configBytesErr := r.Container().Bytes() + assertion.NoError(configBytesErr) + + assertion.Contains(string(configBytes), "tcp://127.0.0.42:1234") + }) + + suite.Require().NoError(mc.Close()) + + // change the listen address + oldListenAddress := maintenanceConfig.TypedSpec().ListenAddress + maintenanceConfig.TypedSpec().ListenAddress = suite.findListenAddr() + suite.Require().NoError(suite.State().Update(suite.Ctx(), maintenanceConfig)) + + // wait for the service to be up on the new address + suite.AssertWithin(time.Second, 10*time.Millisecond, func() error { + var c *tls.Conn + + c, err = tls.Dial("tcp", maintenanceConfig.TypedSpec().ListenAddress, + &tls.Config{ + InsecureSkipVerify: true, + }, + ) + + if c != nil { + c.Close() //nolint:errcheck + } + + return retry.ExpectedError(err) + }) + + // verify that old address returns connection refused + _, err = net.Dial("tcp", oldListenAddress) + suite.Require().ErrorContains(err, "connection refused") + + // test the API again over SideroLink - the Admin role must be injected to the call + mc, err = client.New(suite.Ctx(), + client.WithTLSConfig(&tls.Config{ + InsecureSkipVerify: true, + }), client.WithEndpoints(maintenanceConfig.TypedSpec().ListenAddress), + ) + suite.Require().NoError(err) + + siderolinkCtx := metadata.AppendToOutgoingContext(suite.Ctx(), isSiderolinkPeerHeaderKey, "yep") + + _, err = mc.Version(siderolinkCtx) + suite.Require().NoError(err) + + // teardown the maintenance service + _, err = suite.State().Teardown(suite.Ctx(), maintenanceRequest.Metadata()) + suite.Require().NoError(err) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{runtime.MaintenanceServiceRequestID}, + func(r *runtime.MaintenanceServiceRequest, asrt *assert.Assertions) { + asrt.Empty(r.Metadata().Finalizers()) + }) + + suite.Require().NoError(suite.State().Destroy(suite.Ctx(), maintenanceRequest.Metadata())) + + _, err = net.Dial("tcp", maintenanceConfig.TypedSpec().ListenAddress) + suite.Require().ErrorContains(err, "connection refused") + + // assert that the maintenance service is removed from the config after the service was shut down + rtestutils.AssertNoResource[*configres.MachineConfig](suite.Ctx(), suite.T(), suite.State(), configres.MaintenanceID) +} + +type mockController struct { + s state.State +} + +type mockState struct { + s state.State +} + +func (mock mockController) Runtime() talosruntime.Runtime { + return mock +} + +func (mockController) Sequencer() talosruntime.Sequencer { + return nil +} + +func (mockController) Run(context.Context, talosruntime.Sequence, any, ...talosruntime.LockOption) error { + return nil +} + +func (mockController) V1Alpha2() talosruntime.V1Alpha2Controller { + return nil +} + +func (mock mockController) Config() config.Config { + return nil +} + +func (mock mockController) ConfigContainer() config.Container { + return nil +} + +func (mock mockController) RollbackToConfigAfter(time.Duration) error { + return nil +} + +func (mock mockController) CancelConfigRollbackTimeout() { +} + +func (mock mockController) SetConfig(config.Provider) error { + return nil +} + +func (mock mockController) CanApplyImmediate(config.Provider) error { + return nil +} + +func (mock mockController) GetSystemInformation(context.Context) (*hardware.SystemInformation, error) { + return nil, nil +} + +func (mock mockController) State() talosruntime.State { + return mockState(mock) +} + +func (mock mockController) Events() talosruntime.EventStream { + return nil +} + +func (mock mockController) Logging() talosruntime.LoggingManager { + return nil +} + +func (mock mockController) NodeName() (string, error) { + return "", nil +} + +func (mock mockController) IsBootstrapAllowed() bool { + return false +} + +func (mock mockState) Platform() talosruntime.Platform { + return &metal.Metal{} // required for ApplyConfiguration to not fail +} + +func (mock mockState) Machine() talosruntime.MachineState { + return nil +} + +func (mock mockState) Cluster() talosruntime.ClusterState { + return nil +} + +func (mock mockState) V1Alpha2() talosruntime.V1Alpha2State { + return mock +} + +func (mock mockState) Resources() state.State { + return mock.s +} + +func (mock mockState) NamespaceRegistry() *registry.NamespaceRegistry { + return nil +} + +func (mock mockState) ResourceRegistry() *registry.ResourceRegistry { + return nil +} + +func (mock mockState) SetConfig(config.Provider) error { + return nil +} diff --git a/internal/app/machined/pkg/controllers/runtime/runtime.go b/internal/app/machined/pkg/controllers/runtime/runtime.go new file mode 100644 index 0000000..25fd607 --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/runtime.go @@ -0,0 +1,6 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package runtime provides the runtime implementation. +package runtime diff --git a/internal/app/machined/pkg/controllers/runtime/security_state.go b/internal/app/machined/pkg/controllers/runtime/security_state.go new file mode 100644 index 0000000..bd95df6 --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/security_state.go @@ -0,0 +1,136 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "bytes" + "context" + "crypto/sha256" + "crypto/x509" + "encoding/hex" + "encoding/pem" + "errors" + "fmt" + "os" + "strings" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/foxboron/go-uefi/efi" + "go.uber.org/zap" + + machineruntime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/constants" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +// SecurityStateController is a controller that updates the security state of Talos. +type SecurityStateController struct { + V1Alpha1Mode machineruntime.Mode +} + +// Name implements controller.Controller interface. +func (ctrl *SecurityStateController) Name() string { + return "runtime.SecurityStateController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *SecurityStateController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: v1alpha1.NamespaceName, + Type: v1alpha1.ServiceType, + Kind: controller.OutputExclusive, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *SecurityStateController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: runtimeres.SecurityStateType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *SecurityStateController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + // wait for the `machined` service to start, as by that time initial mounts will be done + _, err := safe.ReaderGetByID[*v1alpha1.Service](ctx, r, "machined") + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("failed to get machined state: %w", err) + } + + var ( + secureBootState bool + pcrSigningKeyFingerprint string + ) + + // in container mode, never populate the fields + if ctrl.V1Alpha1Mode != machineruntime.ModeContainer { + if efi.GetSecureBoot() && !efi.GetSetupMode() { + secureBootState = true + } + + if pcrPublicKeyData, err := os.ReadFile(constants.PCRPublicKey); err == nil { + block, _ := pem.Decode(pcrPublicKeyData) + if block == nil { + return errors.New("failed to decode PEM block for PCR public key") + } + + cert := x509.Certificate{ + Raw: block.Bytes, + } + + pcrSigningKeyFingerprint = x509CertFingerprint(cert) + } + } + + if err := safe.WriterModify(ctx, r, runtimeres.NewSecurityStateSpec(runtimeres.NamespaceName), func(state *runtimeres.SecurityState) error { + state.TypedSpec().SecureBoot = secureBootState + state.TypedSpec().PCRSigningKeyFingerprint = pcrSigningKeyFingerprint + + return nil + }); err != nil { + return err + } + + // terminating the controller here, as we need to only populate securitystate once + return nil + } +} + +func x509CertFingerprint(cert x509.Certificate) string { + hash := sha256.Sum256(cert.Raw) + + var buf bytes.Buffer + + for i, b := range hex.EncodeToString(hash[:]) { + if i > 0 && i%2 == 0 { + buf.WriteByte(':') + } + + buf.WriteString(strings.ToUpper(string(b))) + } + + return buf.String() +} diff --git a/internal/app/machined/pkg/controllers/runtime/testdata/extservices/foo.bar b/internal/app/machined/pkg/controllers/runtime/testdata/extservices/foo.bar new file mode 100644 index 0000000..e69de29 diff --git a/internal/app/machined/pkg/controllers/runtime/testdata/extservices/frr.yaml b/internal/app/machined/pkg/controllers/runtime/testdata/extservices/frr.yaml new file mode 100644 index 0000000..7cfd66a --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/testdata/extservices/frr.yaml @@ -0,0 +1,10 @@ +name: frr +container: + entrypoint: ./frr + args: + - --msg + - BGP FRR +depends: + - network: + - addresses +restart: always diff --git a/internal/app/machined/pkg/controllers/runtime/testdata/extservices/hello.yaml b/internal/app/machined/pkg/controllers/runtime/testdata/extservices/hello.yaml new file mode 100644 index 0000000..4829118 --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/testdata/extservices/hello.yaml @@ -0,0 +1,10 @@ +name: hello-world +container: + entrypoint: ./hello-world + args: + - --msg + - Talos Linux Extension Service +depends: + - network: + - addresses +restart: always diff --git a/internal/app/machined/pkg/controllers/runtime/testdata/extservices/invalid.yaml b/internal/app/machined/pkg/controllers/runtime/testdata/extservices/invalid.yaml new file mode 100644 index 0000000..ada8c1d --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/testdata/extservices/invalid.yaml @@ -0,0 +1,9 @@ +name: invalid +container: + entrypoint: ./hello-world + args: + - --msg + - Talos Linux Extension Service +depends: + - nothing: true +restart: random diff --git a/internal/app/machined/pkg/controllers/runtime/testdata/extservices/zduplicate.yaml b/internal/app/machined/pkg/controllers/runtime/testdata/extservices/zduplicate.yaml new file mode 100644 index 0000000..2c8fde5 --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/testdata/extservices/zduplicate.yaml @@ -0,0 +1,9 @@ +name: hello-world +container: + entrypoint: ./duplicate + args: + - should not get registered +depends: + - network: + - addresses +restart: always diff --git a/internal/app/machined/pkg/controllers/runtime/unique_token.go b/internal/app/machined/pkg/controllers/runtime/unique_token.go new file mode 100644 index 0000000..07c733a --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/unique_token.go @@ -0,0 +1,56 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/controller/generic/transform" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/aenix-io/talm/internal/pkg/meta" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// UniqueMachineTokenController provides a unique token the machine. +type UniqueMachineTokenController = transform.Controller[*runtime.MetaLoaded, *runtime.UniqueMachineToken] + +// NewUniqueMachineTokenController instanciates the controller. +func NewUniqueMachineTokenController() *UniqueMachineTokenController { + return transform.NewController( + transform.Settings[*runtime.MetaLoaded, *runtime.UniqueMachineToken]{ + Name: "runtime.UniqueMachineTokenController", + MapMetadataFunc: func(in *runtime.MetaLoaded) *runtime.UniqueMachineToken { + return runtime.NewUniqueMachineToken() + }, + TransformFunc: func(ctx context.Context, r controller.Reader, logger *zap.Logger, _ *runtime.MetaLoaded, out *runtime.UniqueMachineToken) error { + uniqueToken, err := safe.ReaderGetByID[*runtime.MetaKey](ctx, r, runtime.MetaKeyTagToID(meta.UniqueMachineToken)) + if state.IsNotFoundError(err) { + out.TypedSpec().Token = "" + + return nil + } else if err != nil { + return err + } + + out.TypedSpec().Token = uniqueToken.TypedSpec().Value + + return nil + }, + }, + transform.WithExtraInputs( + controller.Input{ + Namespace: runtime.NamespaceName, + Type: runtime.MetaKeyType, + ID: optional.Some(runtime.MetaKeyTagToID(meta.UniqueMachineToken)), + Kind: controller.InputWeak, + }, + ), + ) +} diff --git a/internal/app/machined/pkg/controllers/runtime/utils.go b/internal/app/machined/pkg/controllers/runtime/utils.go new file mode 100644 index 0000000..579fc8a --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/utils.go @@ -0,0 +1,78 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "bytes" + "context" + "os" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// WaitForDevicesReady waits for devices to be ready. +// +// It is a helper function for controllers. +func WaitForDevicesReady(ctx context.Context, r controller.Runtime, nextInputs []controller.Input) error { + // set inputs temporarily to a service only + if err := r.UpdateInputs([]controller.Input{ + { + Namespace: runtime.NamespaceName, + Type: runtime.DevicesStatusType, + ID: optional.Some(runtime.DevicesID), + Kind: controller.InputWeak, + }, + }); err != nil { + return err + } + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-r.EventCh(): + } + + status, err := safe.ReaderGetByID[*runtime.DevicesStatus](ctx, r, runtime.DevicesID) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + if status.TypedSpec().Ready { + // condition met + break + } + } + + // restore inputs + if err := r.UpdateInputs(nextInputs); err != nil { + return err + } + + // queue an update to reprocess with new inputs + r.QueueReconcile() + + return nil +} + +// updateFile is like `os.WriteFile`, but it will only update the file if the +// contents have changed. +func updateFile(filename string, contents []byte, mode os.FileMode) error { + oldContents, err := os.ReadFile(filename) + if err == nil && bytes.Equal(oldContents, contents) { + return nil + } + + return os.WriteFile(filename, contents, mode) +} diff --git a/internal/app/machined/pkg/controllers/runtime/watchdog_timer.go b/internal/app/machined/pkg/controllers/runtime/watchdog_timer.go new file mode 100644 index 0000000..a95e057 --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/watchdog_timer.go @@ -0,0 +1,174 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + "fmt" + "os" + "syscall" + "time" + "unsafe" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + "golang.org/x/sys/unix" + + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// WatchdogTimerController watches v1alpha1.Config, creates/updates/deletes kernel module specs. +type WatchdogTimerController struct{} + +// Name implements controller.Controller interface. +func (ctrl *WatchdogTimerController) Name() string { + return "runtime.WatchdogTimerController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *WatchdogTimerController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: runtime.NamespaceName, + Type: runtime.WatchdogTimerConfigType, + ID: optional.Some(runtime.WatchdogTimerConfigID), + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *WatchdogTimerController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: runtime.WatchdogTimerStatusType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *WatchdogTimerController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + var ( + ticker *time.Ticker + tickerC <-chan time.Time + ) + + tickerStop := func() { + if ticker == nil { + return + } + + ticker.Stop() + + ticker = nil + tickerC = nil + } + + defer tickerStop() + + var wd *os.File + + wdClose := func() { + if wd == nil { + return + } + + logger.Info("closing hardware watchdog", zap.String("path", wd.Name())) + + // Magic close: make sure old watchdog won't trip after we close it + if _, err := wd.WriteString("V"); err != nil { + logger.Error("failed to send magic close to watchdog", zap.String("path", wd.Name())) + } + + if err := wd.Close(); err != nil { + logger.Error("failed to close watchdog", zap.String("path", wd.Name())) + } + + wd = nil + } + + defer wdClose() + + for { + select { + case <-ctx.Done(): + return nil + case <-tickerC: + if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, wd.Fd(), unix.WDIOC_KEEPALIVE, 0); err != 0 { + return fmt.Errorf("failed to feed watchdog: %w", err) + } + + continue + case <-r.EventCh(): + } + + cfg, err := safe.ReaderGetByID[*runtime.WatchdogTimerConfig](ctx, r, runtime.WatchdogTimerConfigID) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting watchdog config: %w", err) + } + } + + r.StartTrackingOutputs() + + if cfg == nil { + tickerStop() + wdClose() + } else { + // close the watchdog if requested to use new one + if wd != nil && wd.Name() != cfg.TypedSpec().Device { + wdClose() + } + + if wd == nil { + wd, err = os.OpenFile(cfg.TypedSpec().Device, syscall.O_RDWR, 0o600) + if err != nil { + return fmt.Errorf("failed to open watchdog device: %s", err) + } + + logger.Info("opened hardware watchdog", zap.String("path", cfg.TypedSpec().Device)) + } + + timeout := int(cfg.TypedSpec().Timeout.Seconds()) + + if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, wd.Fd(), uintptr(unix.WDIOC_SETTIMEOUT), uintptr(unsafe.Pointer(&timeout))); err != 0 { + return fmt.Errorf("failed to set watchdog timeout: %w", err) + } + + tickerStop() + + // 3 pings per timeout should suffice in any case + feedInterval := cfg.TypedSpec().Timeout / 3 + + ticker = time.NewTicker(feedInterval) + tickerC = ticker.C + + if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, wd.Fd(), uintptr(unix.WDIOC_KEEPALIVE), 0); err != 0 { + return fmt.Errorf("failed to feed watchdog: %w", err) + } + + logger.Info("set hardware watchdog timeout", zap.Duration("timeout", cfg.TypedSpec().Timeout), zap.Duration("feed_interval", feedInterval)) + + if err = safe.WriterModify(ctx, r, runtime.NewWatchdogTimerStatus(cfg.Metadata().ID()), func(status *runtime.WatchdogTimerStatus) error { + status.TypedSpec().Device = cfg.TypedSpec().Device + status.TypedSpec().Timeout = cfg.TypedSpec().Timeout + status.TypedSpec().FeedInterval = feedInterval + + return nil + }); err != nil { + return fmt.Errorf("error updating watchdog status: %w", err) + } + } + + if err = safe.CleanupOutputs[*runtime.WatchdogTimerStatus](ctx, r); err != nil { + return err + } + } +} diff --git a/internal/app/machined/pkg/controllers/runtime/watchdog_timer_config.go b/internal/app/machined/pkg/controllers/runtime/watchdog_timer_config.go new file mode 100644 index 0000000..7a2e969 --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/watchdog_timer_config.go @@ -0,0 +1,84 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// WatchdogTimerConfigController generates configuration for watchdog timers. +type WatchdogTimerConfigController struct{} + +// Name implements controller.Controller interface. +func (ctrl *WatchdogTimerConfigController) Name() string { + return "runtime.WatchdogTimerConfigController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *WatchdogTimerConfigController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: config.MachineConfigType, + ID: optional.Some(config.V1Alpha1ID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *WatchdogTimerConfigController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: runtime.WatchdogTimerConfigType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +func (ctrl *WatchdogTimerConfigController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) (err error) { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + cfg, err := safe.ReaderGetByID[*config.MachineConfig](ctx, r, config.V1Alpha1ID) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error getting machine config: %w", err) + } + + r.StartTrackingOutputs() + + if cfg != nil { + if watchdogConfig := cfg.Config().Runtime().WatchdogTimer(); watchdogConfig != nil { + if err = safe.WriterModify(ctx, r, runtime.NewWatchdogTimerConfig(), func(cfg *runtime.WatchdogTimerConfig) error { + cfg.TypedSpec().Device = watchdogConfig.Device() + cfg.TypedSpec().Timeout = watchdogConfig.Timeout() + + return nil + }); err != nil { + return fmt.Errorf("error updating kmsg log config: %w", err) + } + } + } + + if err = safe.CleanupOutputs[*runtime.WatchdogTimerConfig](ctx, r); err != nil { + return err + } + } +} diff --git a/internal/app/machined/pkg/controllers/runtime/watchdog_timer_config_test.go b/internal/app/machined/pkg/controllers/runtime/watchdog_timer_config_test.go new file mode 100644 index 0000000..58fdc4f --- /dev/null +++ b/internal/app/machined/pkg/controllers/runtime/watchdog_timer_config_test.go @@ -0,0 +1,60 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime_test + +import ( + "testing" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + runtimectrls "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/runtime" + "github.com/siderolabs/talos/pkg/machinery/config/container" + runtimecfg "github.com/siderolabs/talos/pkg/machinery/config/types/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +type WatchdogTimerConfigSuite struct { + ctest.DefaultSuite +} + +func TestWatchdogTimerConfigSuite(t *testing.T) { + suite.Run(t, new(WatchdogTimerConfigSuite)) +} + +func (suite *WatchdogTimerConfigSuite) TestWatchdogTimerConfigNone() { + suite.Require().NoError(suite.Runtime().RegisterController(&runtimectrls.WatchdogTimerConfigController{})) + + rtestutils.AssertNoResource[*runtime.WatchdogTimerConfig](suite.Ctx(), suite.T(), suite.State(), runtime.WatchdogTimerConfigID) +} + +func (suite *WatchdogTimerConfigSuite) TestWatchdogTimerConfigMachineConfig() { + suite.Require().NoError(suite.Runtime().RegisterController(&runtimectrls.WatchdogTimerConfigController{})) + + watchdogTimerConfig := &runtimecfg.WatchdogTimerV1Alpha1{ + WatchdogDevice: "/dev/watchdog0", + } + + cfg, err := container.New(watchdogTimerConfig) + suite.Require().NoError(err) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), config.NewMachineConfig(cfg))) + + rtestutils.AssertResources[*runtime.WatchdogTimerConfig](suite.Ctx(), suite.T(), suite.State(), []resource.ID{runtime.WatchdogTimerConfigID}, + func(cfg *runtime.WatchdogTimerConfig, asrt *assert.Assertions) { + asrt.Equal( + "/dev/watchdog0", + cfg.TypedSpec().Device, + ) + asrt.Equal( + runtimecfg.DefaultWatchdogTimeout, + cfg.TypedSpec().Timeout, + ) + }) +} diff --git a/internal/app/machined/pkg/controllers/secrets/api.go b/internal/app/machined/pkg/controllers/secrets/api.go new file mode 100644 index 0000000..72791fa --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/api.go @@ -0,0 +1,428 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets + +import ( + "context" + stdlibx509 "crypto/x509" + "errors" + "fmt" + "time" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/crypto/x509" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/grpc/gen" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" + timeresource "github.com/siderolabs/talos/pkg/machinery/resources/time" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/role" +) + +// APIController manages secrets.API based on configuration to provide apid certificate. +type APIController struct{} + +// Name implements controller.Controller interface. +func (ctrl *APIController) Name() string { + return "secrets.APIController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *APIController) Inputs() []controller.Input { + // initial set of inputs: wait for machine type to be known and network to be partially configured + return []controller.Input{ + { + Namespace: network.NamespaceName, + Type: network.StatusType, + ID: optional.Some(network.StatusID), + Kind: controller.InputWeak, + }, + { + Namespace: config.NamespaceName, + Type: config.MachineTypeType, + ID: optional.Some(config.MachineTypeID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *APIController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: secrets.APIType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *APIController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + // reset inputs back to what they were initially + if err := r.UpdateInputs(ctrl.Inputs()); err != nil { + return err + } + + machineTypeRes, err := r.Get(ctx, resource.NewMetadata(config.NamespaceName, config.MachineTypeType, config.MachineTypeID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting machine type: %w", err) + } + + machineType := machineTypeRes.(*config.MachineType).MachineType() + + networkResource, err := r.Get(ctx, resource.NewMetadata(network.NamespaceName, network.StatusType, network.StatusID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + networkStatus := networkResource.(*network.Status).TypedSpec() + + if !(networkStatus.AddressReady && networkStatus.HostnameReady) { + continue + } + + // machine type is known and network is ready, we can now proceed to one or another reconcile loop + switch machineType { + case machine.TypeInit, machine.TypeControlPlane: + if err = ctrl.reconcile(ctx, r, logger, true); err != nil { + return err + } + case machine.TypeWorker: + if err = ctrl.reconcile(ctx, r, logger, false); err != nil { + return err + } + case machine.TypeUnknown: + // machine configuration is not loaded yet, do nothing + default: + panic(fmt.Sprintf("unexpected machine type %v", machineType)) + } + + if err = ctrl.teardownAll(ctx, r); err != nil { + return err + } + + r.ResetRestartBackoff() + } +} + +//nolint:gocyclo,cyclop,dupl +func (ctrl *APIController) reconcile(ctx context.Context, r controller.Runtime, logger *zap.Logger, isControlplane bool) error { + inputs := []controller.Input{ + { + Namespace: secrets.NamespaceName, + Type: secrets.OSRootType, + ID: optional.Some(secrets.OSRootID), + Kind: controller.InputWeak, + }, + { + Namespace: secrets.NamespaceName, + Type: secrets.CertSANType, + ID: optional.Some(secrets.CertSANAPIID), + Kind: controller.InputWeak, + }, + { + Namespace: config.NamespaceName, + Type: config.MachineTypeType, + ID: optional.Some(config.MachineTypeID), + Kind: controller.InputWeak, + }, + // time status isn't fetched, but the fact that it is in dependencies means + // that certs will be regenerated on time sync/jump (as reconcile will be triggered) + { + Namespace: v1alpha1.NamespaceName, + Type: timeresource.StatusType, + ID: optional.Some(timeresource.StatusID), + Kind: controller.InputWeak, + }, + } + + if !isControlplane { + // worker nodes depend on endpoint list + inputs = append(inputs, controller.Input{ + Namespace: k8s.ControlPlaneNamespaceName, + Type: k8s.EndpointType, + Kind: controller.InputWeak, + }) + } + + if err := r.UpdateInputs(inputs); err != nil { + return fmt.Errorf("error updating inputs: %w", err) + } + + r.QueueReconcile() + + refreshTicker := time.NewTicker(x509.DefaultCertificateValidityDuration / 2) + defer refreshTicker.Stop() + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + case <-refreshTicker.C: + } + + machineTypeRes, err := r.Get(ctx, resource.NewMetadata(config.NamespaceName, config.MachineTypeType, config.MachineTypeID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting machine type: %w", err) + } + + machineType := machineTypeRes.(*config.MachineType).MachineType() + + switch machineType { + case machine.TypeInit, machine.TypeControlPlane: + if !isControlplane { + return errors.New("machine type changed") + } + case machine.TypeWorker: + if isControlplane { + return errors.New("machine type changed") + } + case machine.TypeUnknown: + return errors.New("machine type changed") + default: + panic(fmt.Sprintf("unexpected machine type %v", machineType)) + } + + rootResource, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.OSRootType, secrets.OSRootID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + if err = ctrl.teardownAll(ctx, r); err != nil { + return fmt.Errorf("error destroying resources: %w", err) + } + + continue + } + + return fmt.Errorf("error getting etcd root secrets: %w", err) + } + + rootSpec := rootResource.(*secrets.OSRoot).TypedSpec() + + certSANResource, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.CertSANType, secrets.CertSANAPIID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting certSANs: %w", err) + } + + certSANs := certSANResource.(*secrets.CertSAN).TypedSpec() + + var endpointsStr []string + + if !isControlplane { + endpointResources, err := r.List(ctx, resource.NewMetadata(k8s.ControlPlaneNamespaceName, k8s.EndpointType, "", resource.VersionUndefined)) + if err != nil { + return fmt.Errorf("error getting endpoints resources: %w", err) + } + + var endpointAddrs k8s.EndpointList + + // merge all endpoints into a single list + for _, res := range endpointResources.Items { + endpointAddrs = endpointAddrs.Merge(res.(*k8s.Endpoint)) + } + + if len(endpointAddrs) == 0 { + continue + } + + endpointsStr = endpointAddrs.Strings() + } + + if isControlplane { + if err := ctrl.generateControlPlane(ctx, r, logger, rootSpec, certSANs); err != nil { + return err + } + } else { + if err := ctrl.generateWorker(ctx, r, logger, rootSpec, endpointsStr, certSANs); err != nil { + return err + } + } + + r.ResetRestartBackoff() + } +} + +func (ctrl *APIController) generateControlPlane(ctx context.Context, r controller.Runtime, logger *zap.Logger, rootSpec *secrets.OSRootSpec, certSANs *secrets.CertSANSpec) error { + ca, err := x509.NewCertificateAuthorityFromCertificateAndKey(rootSpec.IssuingCA) + if err != nil { + return fmt.Errorf("failed to parse CA certificate: %w", err) + } + + serverCert, err := x509.NewKeyPair(ca, + x509.IPAddresses(certSANs.StdIPs()), + x509.DNSNames(certSANs.DNSNames), + x509.CommonName(certSANs.FQDN), + x509.NotAfter(time.Now().Add(x509.DefaultCertificateValidityDuration)), + x509.KeyUsage(stdlibx509.KeyUsageDigitalSignature), + x509.ExtKeyUsage([]stdlibx509.ExtKeyUsage{ + stdlibx509.ExtKeyUsageServerAuth, + }), + ) + if err != nil { + return fmt.Errorf("failed to generate API server cert: %w", err) + } + + clientCert, err := x509.NewKeyPair(ca, + x509.CommonName(certSANs.FQDN), + x509.Organization(string(role.Impersonator)), + x509.NotAfter(time.Now().Add(x509.DefaultCertificateValidityDuration)), + x509.KeyUsage(stdlibx509.KeyUsageDigitalSignature), + x509.ExtKeyUsage([]stdlibx509.ExtKeyUsage{ + stdlibx509.ExtKeyUsageClientAuth, + }), + ) + if err != nil { + return fmt.Errorf("failed to generate API client cert: %w", err) + } + + if err := r.Modify(ctx, secrets.NewAPI(), + func(r resource.Resource) error { + apiSecrets := r.(*secrets.API).TypedSpec() + + apiSecrets.AcceptedCAs = rootSpec.AcceptedCAs + apiSecrets.Server = x509.NewCertificateAndKeyFromKeyPair(serverCert) + apiSecrets.Client = x509.NewCertificateAndKeyFromKeyPair(clientCert) + + return nil + }); err != nil { + return fmt.Errorf("error modifying resource: %w", err) + } + + clientFingerprint, _ := x509.SPKIFingerprintFromDER(clientCert.Certificate.Certificate[0]) //nolint:errcheck + serverFingerprint, _ := x509.SPKIFingerprintFromDER(serverCert.Certificate.Certificate[0]) //nolint:errcheck + + logger.Debug("generated new certificates", + zap.Stringer("client", clientFingerprint), + zap.Stringer("server", serverFingerprint), + ) + + return nil +} + +func (ctrl *APIController) generateWorker(ctx context.Context, r controller.Runtime, logger *zap.Logger, + rootSpec *secrets.OSRootSpec, endpointsStr []string, certSANs *secrets.CertSANSpec, +) error { + remoteGen, err := gen.NewRemoteGenerator(rootSpec.Token, endpointsStr, rootSpec.IssuingCA) + if err != nil { + return fmt.Errorf("failed creating trustd client: %w", err) + } + + defer remoteGen.Close() //nolint:errcheck + + serverCSR, serverCert, err := x509.NewEd25519CSRAndIdentity( + x509.IPAddresses(certSANs.StdIPs()), + x509.DNSNames(certSANs.DNSNames), + x509.CommonName(certSANs.FQDN), + ) + if err != nil { + return fmt.Errorf("failed to generate API server CSR: %w", err) + } + + logger.Debug("sending CSR", zap.Strings("endpoints", endpointsStr)) + + var ca []byte + + // run the CSR generation in a goroutine, so we can abort the request if the inputs change + errCh := make(chan error) + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + go func() { + ca, serverCert.Crt, err = remoteGen.IdentityContext(ctx, serverCSR) + errCh <- err + }() + + select { + case <-r.EventCh(): + // there's an update to the inputs, terminate the attempt, and let the controller handle the retry + cancel() + + // re-queue the reconcile event, so that controller retries with new inputs + r.QueueReconcile() + + // wait for the goroutine to finish, ignoring the error (should be context.Canceled) + <-errCh + + return nil + case err = <-errCh: + } + + if err != nil { + return fmt.Errorf("failed to sign API server CSR: %w", err) + } + + if err := r.Modify(ctx, secrets.NewAPI(), + func(r resource.Resource) error { + apiSecrets := r.(*secrets.API).TypedSpec() + + apiSecrets.AcceptedCAs = []*x509.PEMEncodedCertificate{ + { + Crt: ca, + }, + } + apiSecrets.Server = serverCert + + return nil + }); err != nil { + return fmt.Errorf("error modifying resource: %w", err) + } + + serverFingerprint, _ := x509.SPKIFingerprintFromPEM(serverCert.Crt) //nolint:errcheck + + logger.Debug("generated new certificates", + zap.Stringer("server", serverFingerprint), + ) + + return nil +} + +func (ctrl *APIController) teardownAll(ctx context.Context, r controller.Runtime) error { + list, err := r.List(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.APIType, "", resource.VersionUndefined)) + if err != nil { + return err + } + + for _, res := range list.Items { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return err + } + } + + return nil +} diff --git a/internal/app/machined/pkg/controllers/secrets/api_cert_sans.go b/internal/app/machined/pkg/controllers/secrets/api_cert_sans.go new file mode 100644 index 0000000..40c3985 --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/api_cert_sans.go @@ -0,0 +1,154 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" +) + +// APICertSANsController manages secrets.APICertSANs based on configuration. +type APICertSANsController struct{} + +// Name implements controller.Controller interface. +func (ctrl *APICertSANsController) Name() string { + return "secrets.APICertSANsController" +} + +// Inputs implements controller.Controller interface. +// +//nolint:dupl +func (ctrl *APICertSANsController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: secrets.NamespaceName, + Type: secrets.OSRootType, + ID: optional.Some(secrets.OSRootID), + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.HostnameStatusType, + ID: optional.Some(network.HostnameID), + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.NodeAddressType, + ID: optional.Some(network.FilteredNodeAddressID(network.NodeAddressAccumulativeID, k8s.NodeAddressFilterNoK8s)), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *APICertSANsController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: secrets.CertSANType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *APICertSANsController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + apiRootRes, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.OSRootType, secrets.OSRootID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + if err = ctrl.teardownAll(ctx, r); err != nil { + return fmt.Errorf("error destroying resources: %w", err) + } + + continue + } + + return fmt.Errorf("error getting root k8s secrets: %w", err) + } + + apiRoot := apiRootRes.(*secrets.OSRoot).TypedSpec() + + hostnameResource, err := r.Get(ctx, resource.NewMetadata(network.NamespaceName, network.HostnameStatusType, network.HostnameID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + hostnameStatus := hostnameResource.(*network.HostnameStatus).TypedSpec() + + addressesResource, err := r.Get(ctx, + resource.NewMetadata(network.NamespaceName, network.NodeAddressType, network.FilteredNodeAddressID(network.NodeAddressAccumulativeID, k8s.NodeAddressFilterNoK8s), resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + nodeAddresses := addressesResource.(*network.NodeAddress).TypedSpec() + + if err = r.Modify(ctx, secrets.NewCertSAN(secrets.NamespaceName, secrets.CertSANAPIID), func(r resource.Resource) error { + spec := r.(*secrets.CertSAN).TypedSpec() + + spec.Reset() + + spec.AppendIPs(apiRoot.CertSANIPs...) + spec.AppendIPs(nodeAddresses.IPs()...) + + spec.AppendDNSNames(apiRoot.CertSANDNSNames...) + spec.AppendDNSNames(hostnameStatus.Hostname, hostnameStatus.FQDN()) + + spec.FQDN = hostnameStatus.FQDN() + + spec.Sort() + + return nil + }); err != nil { + return err + } + + r.ResetRestartBackoff() + } +} + +func (ctrl *APICertSANsController) teardownAll(ctx context.Context, r controller.Runtime) error { + list, err := r.List(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.CertSANType, "", resource.VersionUndefined)) + if err != nil { + return err + } + + for _, res := range list.Items { + if res.Metadata().Owner() == ctrl.Name() { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return err + } + } + } + + return nil +} diff --git a/internal/app/machined/pkg/controllers/secrets/api_cert_sans_test.go b/internal/app/machined/pkg/controllers/secrets/api_cert_sans_test.go new file mode 100644 index 0000000..f487d57 --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/api_cert_sans_test.go @@ -0,0 +1,119 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets_test + +import ( + "fmt" + "net/netip" + "reflect" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + secretsctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/secrets" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" +) + +func TestAPICertSANsSuite(t *testing.T) { + suite.Run(t, &APICertSANsSuite{ + DefaultSuite: ctest.DefaultSuite{ + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(&secretsctrl.APICertSANsController{})) + }, + }, + }) +} + +type APICertSANsSuite struct { + ctest.DefaultSuite +} + +func (suite *APICertSANsSuite) TestReconcileControlPlane() { + rootSecrets := secrets.NewOSRoot(secrets.OSRootID) + + rootSecrets.TypedSpec().CertSANDNSNames = []string{"some.org"} + rootSecrets.TypedSpec().CertSANIPs = []netip.Addr{netip.MustParseAddr("10.4.3.2"), netip.MustParseAddr("10.2.1.3")} + suite.Require().NoError(suite.State().Create(suite.Ctx(), rootSecrets)) + + hostnameStatus := network.NewHostnameStatus(network.NamespaceName, network.HostnameID) + hostnameStatus.TypedSpec().Hostname = "bar" + hostnameStatus.TypedSpec().Domainname = "some.org" + suite.Require().NoError(suite.State().Create(suite.Ctx(), hostnameStatus)) + + nodeAddresses := network.NewNodeAddress( + network.NamespaceName, + network.FilteredNodeAddressID(network.NodeAddressAccumulativeID, k8s.NodeAddressFilterNoK8s), + ) + nodeAddresses.TypedSpec().Addresses = []netip.Prefix{ + netip.MustParsePrefix("10.2.1.3/24"), + netip.MustParsePrefix("172.16.0.1/32"), + } + suite.Require().NoError(suite.State().Create(suite.Ctx(), nodeAddresses)) + + suite.AssertWithin(10*time.Second, 100*time.Millisecond, func() error { + certSANs, err := ctest.Get[*secrets.CertSAN]( + suite, + resource.NewMetadata( + secrets.NamespaceName, + secrets.CertSANType, + secrets.CertSANAPIID, + resource.VersionUndefined, + ), + ) + if err != nil { + if state.IsNotFoundError(err) { + return retry.ExpectedError(err) + } + + return err + } + + spec := certSANs.TypedSpec() + + suite.Assert().Equal([]string{"bar", "bar.some.org", "some.org"}, spec.DNSNames) + suite.Assert().Equal("[10.2.1.3 10.4.3.2 172.16.0.1]", fmt.Sprintf("%v", spec.IPs)) + suite.Assert().Equal("bar.some.org", spec.FQDN) + + return nil + }) + + ctest.UpdateWithConflicts(suite, rootSecrets, func(rootSecrets *secrets.OSRoot) error { + rootSecrets.TypedSpec().CertSANDNSNames = []string{"other.org"} + + return nil + }) + + suite.AssertWithin(10*time.Second, 100*time.Millisecond, func() error { + certSANs, err := ctest.Get[*secrets.CertSAN]( + suite, + resource.NewMetadata( + secrets.NamespaceName, + secrets.CertSANType, + secrets.CertSANAPIID, + resource.VersionUndefined, + ), + ) + if err != nil { + return err + } + + spec := certSANs.TypedSpec() + + expectedDNSNames := []string{"bar", "bar.some.org", "other.org"} + + if !reflect.DeepEqual(expectedDNSNames, spec.DNSNames) { + return retry.ExpectedErrorf("expected %v, got %v", expectedDNSNames, spec.DNSNames) + } + + return nil + }) +} diff --git a/internal/app/machined/pkg/controllers/secrets/api_test.go b/internal/app/machined/pkg/controllers/secrets/api_test.go new file mode 100644 index 0000000..85b0e31 --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/api_test.go @@ -0,0 +1,148 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets_test + +import ( + stdlibx509 "crypto/x509" + "fmt" + "net/netip" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/crypto/x509" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + secretsctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/secrets" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" + "github.com/siderolabs/talos/pkg/machinery/role" +) + +func TestAPISuite(t *testing.T) { + suite.Run(t, &APISuite{ + DefaultSuite: ctest.DefaultSuite{ + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(&secretsctrl.APIController{})) + }, + }, + }) +} + +type APISuite struct { + ctest.DefaultSuite +} + +func (suite *APISuite) TestReconcileControlPlane() { + rootSecrets := secrets.NewOSRoot(secrets.OSRootID) + + talosCA, err := x509.NewSelfSignedCertificateAuthority( + x509.Organization("talos"), + ) + suite.Require().NoError(err) + + rootSecrets.TypedSpec().IssuingCA = &x509.PEMEncodedCertificateAndKey{ + Crt: talosCA.CrtPEM, + Key: talosCA.KeyPEM, + } + rootSecrets.TypedSpec().AcceptedCAs = []*x509.PEMEncodedCertificate{ + { + Crt: talosCA.CrtPEM, + }, + } + rootSecrets.TypedSpec().CertSANDNSNames = []string{"example.com"} + rootSecrets.TypedSpec().CertSANIPs = []netip.Addr{netip.MustParseAddr("10.4.3.2"), netip.MustParseAddr("10.2.1.3")} + rootSecrets.TypedSpec().Token = "something" + suite.Require().NoError(suite.State().Create(suite.Ctx(), rootSecrets)) + + machineType := config.NewMachineType() + machineType.SetMachineType(machine.TypeControlPlane) + suite.Require().NoError(suite.State().Create(suite.Ctx(), machineType)) + + networkStatus := network.NewStatus(network.NamespaceName, network.StatusID) + networkStatus.TypedSpec().AddressReady = true + networkStatus.TypedSpec().HostnameReady = true + suite.Require().NoError(suite.State().Create(suite.Ctx(), networkStatus)) + + certSANs := secrets.NewCertSAN(secrets.NamespaceName, secrets.CertSANAPIID) + certSANs.TypedSpec().Append( + "example.com", + "foo", + "foo.example.com", + "10.2.1.3", + "10.4.3.2", + "172.16.0.1", + ) + + certSANs.TypedSpec().FQDN = "foo.example.com" + + suite.Require().NoError(suite.State().Create(suite.Ctx(), certSANs)) + suite.AssertWithin(10*time.Second, 100*time.Millisecond, func() error { + certs, err := ctest.Get[*secrets.API]( + suite, + resource.NewMetadata( + secrets.NamespaceName, + secrets.APIType, + secrets.APIID, + resource.VersionUndefined, + ), + ) + if err != nil { + if state.IsNotFoundError(err) { + return retry.ExpectedError(err) + } + + return err + } + + apiCerts := certs.TypedSpec() + + suite.Assert().Equal( + []*x509.PEMEncodedCertificate{ + { + Crt: talosCA.CrtPEM, + }, + }, + apiCerts.AcceptedCAs, + ) + + serverCert, err := apiCerts.Server.GetCert() + suite.Require().NoError(err) + + suite.Assert().Equal([]string{"example.com", "foo", "foo.example.com"}, serverCert.DNSNames) + suite.Assert().Equal("[10.2.1.3 10.4.3.2 172.16.0.1]", fmt.Sprintf("%v", serverCert.IPAddresses)) + + suite.Assert().Equal("foo.example.com", serverCert.Subject.CommonName) + suite.Assert().Empty(serverCert.Subject.Organization) + + suite.Assert().Equal( + stdlibx509.KeyUsageDigitalSignature, + serverCert.KeyUsage, + ) + suite.Assert().Equal([]stdlibx509.ExtKeyUsage{stdlibx509.ExtKeyUsageServerAuth}, serverCert.ExtKeyUsage) + + clientCert, err := apiCerts.Client.GetCert() + suite.Require().NoError(err) + + suite.Assert().Empty(clientCert.DNSNames) + suite.Assert().Empty(clientCert.IPAddresses) + + suite.Assert().Equal("foo.example.com", clientCert.Subject.CommonName) + suite.Assert().Equal([]string{string(role.Impersonator)}, clientCert.Subject.Organization) + + suite.Assert().Equal( + stdlibx509.KeyUsageDigitalSignature, + clientCert.KeyUsage, + ) + suite.Assert().Equal([]stdlibx509.ExtKeyUsage{stdlibx509.ExtKeyUsageClientAuth}, clientCert.ExtKeyUsage) + + return nil + }) +} diff --git a/internal/app/machined/pkg/controllers/secrets/etcd.go b/internal/app/machined/pkg/controllers/secrets/etcd.go new file mode 100644 index 0000000..6d667a8 --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/etcd.go @@ -0,0 +1,221 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/aenix-io/talm/internal/pkg/etcd" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" + "github.com/siderolabs/talos/pkg/machinery/resources/time" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +// EtcdController manages secrets.Etcd based on configuration. +type EtcdController struct{} + +// Name implements controller.Controller interface. +func (ctrl *EtcdController) Name() string { + return "secrets.EtcdController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *EtcdController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: secrets.NamespaceName, + Type: secrets.EtcdRootType, + ID: optional.Some(secrets.EtcdRootID), + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.StatusType, + ID: optional.Some(network.StatusID), + Kind: controller.InputWeak, + }, + { + Namespace: v1alpha1.NamespaceName, + Type: time.StatusType, + ID: optional.Some(time.StatusID), + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.HostnameStatusType, + ID: optional.Some(network.HostnameID), + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.NodeAddressType, + ID: optional.Some(network.FilteredNodeAddressID(network.NodeAddressAccumulativeID, k8s.NodeAddressFilterNoK8s)), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *EtcdController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: secrets.EtcdType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *EtcdController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + etcdRootRes, err := safe.ReaderGet[*secrets.EtcdRoot](ctx, r, resource.NewMetadata(secrets.NamespaceName, secrets.EtcdRootType, secrets.EtcdRootID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + if err = ctrl.teardownAll(ctx, r); err != nil { + return fmt.Errorf("error destroying resources: %w", err) + } + + continue + } + + return fmt.Errorf("error getting etcd root secrets: %w", err) + } + + etcdRoot := etcdRootRes.TypedSpec() + + // wait for network to be ready as it might change IPs/hostname + networkResource, err := safe.ReaderGet[*network.Status](ctx, r, resource.NewMetadata(network.NamespaceName, network.StatusType, network.StatusID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + networkStatus := networkResource.TypedSpec() + + if !(networkStatus.AddressReady && networkStatus.HostnameReady) { + continue + } + + // wait for time sync as certs depend on current time + timeSyncResource, err := r.Get(ctx, resource.NewMetadata(v1alpha1.NamespaceName, time.StatusType, time.StatusID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + if !timeSyncResource.(*time.Status).TypedSpec().Synced { + continue + } + + hostnameStatus, err := safe.ReaderGet[*network.HostnameStatus](ctx, r, resource.NewMetadata(network.NamespaceName, network.HostnameStatusType, network.HostnameID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting hostname status: %w", err) + } + + nodeAddrs, err := safe.ReaderGet[*network.NodeAddress]( + ctx, + r, + resource.NewMetadata( + network.NamespaceName, + network.NodeAddressType, + network.FilteredNodeAddressID(network.NodeAddressAccumulativeID, k8s.NodeAddressFilterNoK8s), + resource.VersionUndefined, + ), + ) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting addresses: %w", err) + } + + if err = safe.WriterModify(ctx, r, secrets.NewEtcd(), func(r *secrets.Etcd) error { + return ctrl.updateSecrets(etcdRoot, nodeAddrs, hostnameStatus, r.TypedSpec()) + }); err != nil { + return err + } + + r.ResetRestartBackoff() + } +} + +func (ctrl *EtcdController) updateSecrets(etcdRoot *secrets.EtcdRootSpec, nodeAddress *network.NodeAddress, hostnameStatus *network.HostnameStatus, etcdCerts *secrets.EtcdCertsSpec) error { + generator := etcd.CertificateGenerator{ + CA: etcdRoot.EtcdCA, + + NodeAddresses: nodeAddress, + HostnameStatus: hostnameStatus, + } + + var err error + + etcdCerts.Etcd, err = generator.GenerateServerCert() + if err != nil { + return fmt.Errorf("error generating etcd client certs: %w", err) + } + + etcdCerts.EtcdPeer, err = generator.GeneratePeerCert() + if err != nil { + return fmt.Errorf("error generating etcd peer certs: %w", err) + } + + etcdCerts.EtcdAdmin, err = generator.GenerateClientCert("talos") + if err != nil { + return fmt.Errorf("error generating admin client certs: %w", err) + } + + etcdCerts.EtcdAPIServer, err = generator.GenerateClientCert("kube-apiserver") + if err != nil { + return fmt.Errorf("error generating kube-apiserver etcd client certs: %w", err) + } + + return nil +} + +func (ctrl *EtcdController) teardownAll(ctx context.Context, r controller.Runtime) error { + list, err := r.List(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.EtcdType, "", resource.VersionUndefined)) + if err != nil { + return err + } + + // TODO: change this to proper teardown sequence + + for _, res := range list.Items { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return err + } + } + + return nil +} diff --git a/internal/app/machined/pkg/controllers/secrets/etcd_test.go b/internal/app/machined/pkg/controllers/secrets/etcd_test.go new file mode 100644 index 0000000..8e9daf7 --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/etcd_test.go @@ -0,0 +1,175 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets_test + +import ( + "fmt" + "net/netip" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/crypto/x509" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + secretsctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/secrets" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" + timeres "github.com/siderolabs/talos/pkg/machinery/resources/time" +) + +func TestEtcdSuite(t *testing.T) { + suite.Run(t, &EtcdSuite{ + DefaultSuite: ctest.DefaultSuite{ + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(&secretsctrl.EtcdController{})) + }, + }, + }) +} + +type EtcdSuite struct { + ctest.DefaultSuite +} + +func (suite *EtcdSuite) TestReconcile() { + rootSecrets := secrets.NewEtcdRoot(secrets.EtcdRootID) + + etcdCA, err := x509.NewSelfSignedCertificateAuthority( + x509.Organization("talos"), + x509.ECDSA(true), + ) + suite.Require().NoError(err) + + rootSecrets.TypedSpec().EtcdCA = &x509.PEMEncodedCertificateAndKey{ + Crt: etcdCA.CrtPEM, + Key: etcdCA.KeyPEM, + } + suite.Require().NoError(suite.State().Create(suite.Ctx(), rootSecrets)) + + networkStatus := network.NewStatus(network.NamespaceName, network.StatusID) + networkStatus.TypedSpec().AddressReady = true + networkStatus.TypedSpec().HostnameReady = true + suite.Require().NoError(suite.State().Create(suite.Ctx(), networkStatus)) + + hostnameStatus := network.NewHostnameStatus(network.NamespaceName, network.HostnameID) + hostnameStatus.TypedSpec().Hostname = "host" + hostnameStatus.TypedSpec().Domainname = "domain" + suite.Require().NoError(suite.State().Create(suite.Ctx(), hostnameStatus)) + + nodeAddresses := network.NewNodeAddress(network.NamespaceName, network.FilteredNodeAddressID(network.NodeAddressAccumulativeID, k8s.NodeAddressFilterNoK8s)) + nodeAddresses.TypedSpec().Addresses = []netip.Prefix{ + netip.MustParsePrefix("10.3.4.5/24"), + netip.MustParsePrefix("2001:db8::1eaf/64"), + } + suite.Require().NoError(suite.State().Create(suite.Ctx(), nodeAddresses)) + + timeSync := timeres.NewStatus() + timeSync.TypedSpec().Synced = true + suite.Require().NoError(suite.State().Create(suite.Ctx(), timeSync)) + + suite.AssertWithin(3*time.Second, 100*time.Millisecond, + ctest.WrapRetry(func(assert *assert.Assertions, require *require.Assertions) { + certs, err := ctest.Get[*secrets.Etcd]( + suite, + resource.NewMetadata( + secrets.NamespaceName, + secrets.EtcdType, + secrets.EtcdID, + resource.VersionUndefined, + ), + ) + if err != nil { + if state.IsNotFoundError(err) { + assert.NoError(err) + } else { + require.NoError(err) + } + + return + } + + etcdCerts := certs.TypedSpec() + + serverCert, err := etcdCerts.Etcd.GetCert() + require.NoError(err) + + assert.Equal([]string{"host", "host.domain", "localhost"}, serverCert.DNSNames) + assert.Equal("[10.3.4.5 2001:db8::1eaf 127.0.0.1 ::1]", fmt.Sprintf("%v", serverCert.IPAddresses)) + + assert.Equal("host", serverCert.Subject.CommonName) + + peerCert, err := etcdCerts.EtcdPeer.GetCert() + require.NoError(err) + + assert.Equal([]string{"host", "host.domain"}, peerCert.DNSNames) + assert.Equal("[10.3.4.5 2001:db8::1eaf]", fmt.Sprintf("%v", peerCert.IPAddresses)) + + assert.Equal("host", peerCert.Subject.CommonName) + + adminCert, err := etcdCerts.EtcdAdmin.GetCert() + require.NoError(err) + + assert.Empty(adminCert.DNSNames) + assert.Empty(adminCert.IPAddresses) + + assert.Equal("talos", adminCert.Subject.CommonName) + + kubeAPICert, err := etcdCerts.EtcdAPIServer.GetCert() + require.NoError(err) + + assert.Empty(kubeAPICert.DNSNames) + assert.Empty(kubeAPICert.IPAddresses) + + assert.Equal("kube-apiserver", kubeAPICert.Subject.CommonName) + })) + + // update node addresses, certs should be updated + nodeAddresses.TypedSpec().Addresses = []netip.Prefix{ + netip.MustParsePrefix("10.3.4.5/24"), + } + suite.Require().NoError(suite.State().Update(suite.Ctx(), nodeAddresses)) + + suite.AssertWithin(3*time.Second, 100*time.Millisecond, + ctest.WrapRetry(func(assert *assert.Assertions, require *require.Assertions) { + certs, err := ctest.Get[*secrets.Etcd]( + suite, + resource.NewMetadata( + secrets.NamespaceName, + secrets.EtcdType, + secrets.EtcdID, + resource.VersionUndefined, + ), + ) + if err != nil { + require.NoError(err) + + return + } + + etcdCerts := certs.TypedSpec() + + serverCert, err := etcdCerts.Etcd.GetCert() + require.NoError(err) + + assert.Equal([]string{"host", "host.domain", "localhost"}, serverCert.DNSNames) + assert.Equal("[10.3.4.5 127.0.0.1]", fmt.Sprintf("%v", serverCert.IPAddresses)) + + assert.Equal("host", serverCert.Subject.CommonName) + + peerCert, err := etcdCerts.EtcdPeer.GetCert() + require.NoError(err) + + assert.Equal([]string{"host", "host.domain"}, peerCert.DNSNames) + assert.Equal("[10.3.4.5]", fmt.Sprintf("%v", peerCert.IPAddresses)) + + assert.Equal("host", peerCert.Subject.CommonName) + })) +} diff --git a/internal/app/machined/pkg/controllers/secrets/kubelet.go b/internal/app/machined/pkg/controllers/secrets/kubelet.go new file mode 100644 index 0000000..fbe3520 --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/kubelet.go @@ -0,0 +1,87 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets + +import ( + "context" + "errors" + "fmt" + "net/url" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/controller/generic/transform" + "github.com/siderolabs/crypto/x509" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" +) + +// KubeletController manages secrets.Kubelet based on configuration. +type KubeletController = transform.Controller[*config.MachineConfig, *secrets.Kubelet] + +// NewKubeletController instanciates the controller. +func NewKubeletController() *KubeletController { + return transform.NewController( + transform.Settings[*config.MachineConfig, *secrets.Kubelet]{ + Name: "secrets.KubeletController", + MapMetadataOptionalFunc: func(cfg *config.MachineConfig) optional.Optional[*secrets.Kubelet] { + if cfg.Metadata().ID() != config.V1Alpha1ID { + return optional.None[*secrets.Kubelet]() + } + + if cfg.Config().Cluster() == nil || cfg.Config().Machine() == nil { + return optional.None[*secrets.Kubelet]() + } + + return optional.Some(secrets.NewKubelet(secrets.KubeletID)) + }, + TransformFunc: func(ctx context.Context, r controller.Reader, logger *zap.Logger, cfg *config.MachineConfig, res *secrets.Kubelet) error { + cfgProvider := cfg.Config() + kubeletSecrets := res.TypedSpec() + + switch { + case cfgProvider.Machine().Features().KubePrism().Enabled(): + // use cluster endpoint for controlplane nodes with loadbalancer support + localEndpoint, err := url.Parse(fmt.Sprintf("https://127.0.0.1:%d", cfgProvider.Machine().Features().KubePrism().Port())) + if err != nil { + return err + } + + kubeletSecrets.Endpoint = localEndpoint + case cfgProvider.Machine().Type().IsControlPlane(): + // use localhost endpoint for controlplane nodes + localEndpoint, err := url.Parse(fmt.Sprintf("https://localhost:%d", cfgProvider.Cluster().LocalAPIServerPort())) + if err != nil { + return err + } + + kubeletSecrets.Endpoint = localEndpoint + default: + // use cluster endpoint for workers + kubeletSecrets.Endpoint = cfgProvider.Cluster().Endpoint() + } + + kubeletSecrets.AcceptedCAs = nil + + if cfgProvider.Cluster().IssuingCA() != nil { + kubeletSecrets.AcceptedCAs = append(kubeletSecrets.AcceptedCAs, &x509.PEMEncodedCertificate{Crt: cfgProvider.Cluster().IssuingCA().Crt}) + } + + kubeletSecrets.AcceptedCAs = append(kubeletSecrets.AcceptedCAs, cfgProvider.Cluster().AcceptedCAs()...) + + if len(kubeletSecrets.AcceptedCAs) == 0 { + return errors.New("missing accepted Kubernetes CAs") + } + + kubeletSecrets.BootstrapTokenID = cfgProvider.Cluster().Token().ID() + kubeletSecrets.BootstrapTokenSecret = cfgProvider.Cluster().Token().Secret() + + return nil + }, + }, + ) +} diff --git a/internal/app/machined/pkg/controllers/secrets/kubelet_test.go b/internal/app/machined/pkg/controllers/secrets/kubelet_test.go new file mode 100644 index 0000000..ede6241 --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/kubelet_test.go @@ -0,0 +1,100 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets_test + +import ( + "net/url" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/crypto/x509" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + secretsctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/secrets" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" +) + +func TestKubeletSuite(t *testing.T) { + suite.Run(t, &KubeletSuite{ + DefaultSuite: ctest.DefaultSuite{ + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(secretsctrl.NewKubeletController())) + }, + }, + }) +} + +type KubeletSuite struct { + ctest.DefaultSuite +} + +func (suite *KubeletSuite) TestReconcile() { + u, err := url.Parse("https://foo:6443") + suite.Require().NoError(err) + + ca, err := x509.NewSelfSignedCertificateAuthority(x509.RSA(false)) + suite.Require().NoError(err) + + k8sCA := x509.NewCertificateAndKeyFromCertificateAuthority(ca) + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{}, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: u, + }, + }, + ClusterCA: k8sCA, + BootstrapToken: "abc.def", + }, + }, + ), + ) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + kubeletSecrets, err := ctest.Get[*secrets.Kubelet]( + suite, + resource.NewMetadata( + secrets.NamespaceName, + secrets.KubeletType, + secrets.KubeletID, + resource.VersionUndefined, + ), + ) + if err != nil { + if state.IsNotFoundError(err) { + return retry.ExpectedError(err) + } + + return err + } + + spec := kubeletSecrets.TypedSpec() + + suite.Assert().Equal("https://foo:6443", spec.Endpoint.String()) + suite.Assert().Equal([]*x509.PEMEncodedCertificate{{Crt: k8sCA.Crt}}, spec.AcceptedCAs) + suite.Assert().Equal("abc", spec.BootstrapTokenID) + suite.Assert().Equal("def", spec.BootstrapTokenSecret) + + return nil + }, + ), + ) +} diff --git a/internal/app/machined/pkg/controllers/secrets/kubernetes.go b/internal/app/machined/pkg/controllers/secrets/kubernetes.go new file mode 100644 index 0000000..1d0e5d4 --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/kubernetes.go @@ -0,0 +1,245 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets + +import ( + "bytes" + "context" + "fmt" + "net/url" + "time" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/crypto/x509" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/kubeconfig" + "github.com/siderolabs/talos/pkg/machinery/config/config" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" + timeresource "github.com/siderolabs/talos/pkg/machinery/resources/time" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +// KubernetesCertificateValidityDuration is the validity duration for the certificates created with this controller. +// +// Controller automatically refreshes certs at 50% of CertificateValidityDuration. +const KubernetesCertificateValidityDuration = constants.KubernetesDefaultCertificateValidityDuration + +// KubernetesController manages secrets.Kubernetes based on configuration. +type KubernetesController struct{} + +// Name implements controller.Controller interface. +func (ctrl *KubernetesController) Name() string { + return "secrets.KubernetesController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *KubernetesController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: secrets.NamespaceName, + Type: secrets.KubernetesRootType, + ID: optional.Some(secrets.KubernetesRootID), + Kind: controller.InputWeak, + }, + { + Namespace: v1alpha1.NamespaceName, + Type: timeresource.StatusType, + ID: optional.Some(timeresource.StatusID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *KubernetesController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: secrets.KubernetesType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *KubernetesController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + refreshTicker := time.NewTicker(KubernetesCertificateValidityDuration / 2) + defer refreshTicker.Stop() + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + case <-refreshTicker.C: + } + + k8sRoot, err := safe.ReaderGet[*secrets.KubernetesRoot](ctx, r, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesRootType, secrets.KubernetesRootID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + if err = ctrl.teardownAll(ctx, r); err != nil { + return fmt.Errorf("error destroying resources: %w", err) + } + + continue + } + + return fmt.Errorf("error getting root k8s secrets: %w", err) + } + + // wait for time sync as certs depend on current time + timeSync, err := safe.ReaderGet[*timeresource.Status](ctx, r, resource.NewMetadata(v1alpha1.NamespaceName, timeresource.StatusType, timeresource.StatusID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + if !timeSync.TypedSpec().Synced { + continue + } + + if err = safe.WriterModify(ctx, r, secrets.NewKubernetes(), func(r *secrets.Kubernetes) error { + return ctrl.updateSecrets(k8sRoot.TypedSpec(), r.TypedSpec()) + }); err != nil { + return err + } + + r.ResetRestartBackoff() + } +} + +func (ctrl *KubernetesController) updateSecrets(k8sRoot *secrets.KubernetesRootSpec, k8sSecrets *secrets.KubernetesCertsSpec) error { + var buf bytes.Buffer + + if err := kubeconfig.Generate(&kubeconfig.GenerateInput{ + ClusterName: k8sRoot.Name, + + IssuingCA: k8sRoot.IssuingCA, + AcceptedCAs: k8sRoot.AcceptedCAs, + CertificateLifetime: KubernetesCertificateValidityDuration, + + CommonName: constants.KubernetesControllerManagerOrganization, + Organization: constants.KubernetesControllerManagerOrganization, + + Endpoint: k8sRoot.LocalEndpoint.String(), + Username: constants.KubernetesControllerManagerOrganization, + ContextName: "default", + }, &buf); err != nil { + return fmt.Errorf("failed to generate controller manager kubeconfig: %w", err) + } + + k8sSecrets.ControllerManagerKubeconfig = buf.String() + + buf.Reset() + + if err := kubeconfig.Generate(&kubeconfig.GenerateInput{ + ClusterName: k8sRoot.Name, + + IssuingCA: k8sRoot.IssuingCA, + AcceptedCAs: k8sRoot.AcceptedCAs, + CertificateLifetime: KubernetesCertificateValidityDuration, + + CommonName: constants.KubernetesSchedulerOrganization, + Organization: constants.KubernetesSchedulerOrganization, + + Endpoint: k8sRoot.LocalEndpoint.String(), + Username: constants.KubernetesSchedulerOrganization, + ContextName: "default", + }, &buf); err != nil { + return fmt.Errorf("failed to generate scheduler kubeconfig: %w", err) + } + + k8sSecrets.SchedulerKubeconfig = buf.String() + + buf.Reset() + + if err := kubeconfig.GenerateAdmin(&generateAdminAdapter{ + k8sRoot: k8sRoot, + endpoint: k8sRoot.Endpoint, + }, &buf); err != nil { + return fmt.Errorf("failed to generate admin kubeconfig: %w", err) + } + + k8sSecrets.AdminKubeconfig = buf.String() + + buf.Reset() + + if err := kubeconfig.GenerateAdmin(&generateAdminAdapter{ + k8sRoot: k8sRoot, + endpoint: k8sRoot.LocalEndpoint, + }, &buf); err != nil { + return fmt.Errorf("failed to generate admin kubeconfig: %w", err) + } + + k8sSecrets.LocalhostAdminKubeconfig = buf.String() + + return nil +} + +func (ctrl *KubernetesController) teardownAll(ctx context.Context, r controller.Runtime) error { + list, err := r.List(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesType, "", resource.VersionUndefined)) + if err != nil { + return err + } + + // TODO: change this to proper teardown sequence + + for _, res := range list.Items { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return err + } + } + + return nil +} + +// generateAdminAdapter allows to translate input config into GenerateAdmin input. +type generateAdminAdapter struct { + k8sRoot *secrets.KubernetesRootSpec + endpoint *url.URL +} + +func (adapter *generateAdminAdapter) Name() string { + return adapter.k8sRoot.Name +} + +func (adapter *generateAdminAdapter) Endpoint() *url.URL { + return adapter.endpoint +} + +func (adapter *generateAdminAdapter) IssuingCA() *x509.PEMEncodedCertificateAndKey { + return adapter.k8sRoot.IssuingCA +} + +func (adapter *generateAdminAdapter) AcceptedCAs() []*x509.PEMEncodedCertificate { + return adapter.k8sRoot.AcceptedCAs +} + +func (adapter *generateAdminAdapter) AdminKubeconfig() config.AdminKubeconfig { + return adapter +} + +func (adapter *generateAdminAdapter) CertLifetime() time.Duration { + // this certificate is not delivered to the user, it's used only internally by control plane components + return KubernetesCertificateValidityDuration +} + +func (adapter *generateAdminAdapter) CommonName() string { + return constants.KubernetesTalosAdminCertCommonName +} + +func (adapter *generateAdminAdapter) CertOrganization() string { + return constants.KubernetesAdminCertOrganization +} diff --git a/internal/app/machined/pkg/controllers/secrets/kubernetes_cert_sans.go b/internal/app/machined/pkg/controllers/secrets/kubernetes_cert_sans.go new file mode 100644 index 0000000..b00f9fc --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/kubernetes_cert_sans.go @@ -0,0 +1,167 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets + +import ( + "context" + "fmt" + "net/netip" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" +) + +// KubernetesCertSANsController manages secrets.KubernetesCertSANs based on configuration. +type KubernetesCertSANsController struct{} + +// Name implements controller.Controller interface. +func (ctrl *KubernetesCertSANsController) Name() string { + return "secrets.KubernetesCertSANsController" +} + +// Inputs implements controller.Controller interface. +// +//nolint:dupl +func (ctrl *KubernetesCertSANsController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: secrets.NamespaceName, + Type: secrets.KubernetesRootType, + ID: optional.Some(secrets.KubernetesRootID), + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.HostnameStatusType, + ID: optional.Some(network.HostnameID), + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.NodeAddressType, + ID: optional.Some(network.FilteredNodeAddressID(network.NodeAddressAccumulativeID, k8s.NodeAddressFilterNoK8s)), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *KubernetesCertSANsController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: secrets.CertSANType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *KubernetesCertSANsController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + k8sRootRes, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesRootType, secrets.KubernetesRootID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + if err = ctrl.teardownAll(ctx, r); err != nil { + return fmt.Errorf("error destroying resources: %w", err) + } + + continue + } + + return fmt.Errorf("error getting root k8s secrets: %w", err) + } + + k8sRoot := k8sRootRes.(*secrets.KubernetesRoot).TypedSpec() + + hostnameResource, err := r.Get(ctx, resource.NewMetadata(network.NamespaceName, network.HostnameStatusType, network.HostnameID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + hostnameStatus := hostnameResource.(*network.HostnameStatus).TypedSpec() + + addressesResource, err := r.Get(ctx, + resource.NewMetadata(network.NamespaceName, network.NodeAddressType, network.FilteredNodeAddressID(network.NodeAddressAccumulativeID, k8s.NodeAddressFilterNoK8s), resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + nodeAddresses := addressesResource.(*network.NodeAddress).TypedSpec() + + if err = r.Modify(ctx, secrets.NewCertSAN(secrets.NamespaceName, secrets.CertSANKubernetesID), func(r resource.Resource) error { + spec := r.(*secrets.CertSAN).TypedSpec() + + spec.Reset() + + spec.Append(k8sRoot.Endpoint.Hostname()) + spec.Append(k8sRoot.CertSANs...) + + spec.AppendDNSNames( + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + "kubernetes.default.svc."+k8sRoot.DNSDomain, + "localhost", + ) + + spec.Append( + hostnameStatus.Hostname, + hostnameStatus.FQDN(), + ) + + spec.AppendIPs(k8sRoot.APIServerIPs...) + spec.AppendIPs(nodeAddresses.IPs()...) + spec.AppendIPs(netip.MustParseAddr("127.0.0.1")) + + spec.Sort() + + return nil + }); err != nil { + return err + } + + r.ResetRestartBackoff() + } +} + +func (ctrl *KubernetesCertSANsController) teardownAll(ctx context.Context, r controller.Runtime) error { + list, err := r.List(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.CertSANType, "", resource.VersionUndefined)) + if err != nil { + return err + } + + for _, res := range list.Items { + if res.Metadata().Owner() == ctrl.Name() { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return err + } + } + } + + return nil +} diff --git a/internal/app/machined/pkg/controllers/secrets/kubernetes_cert_sans_test.go b/internal/app/machined/pkg/controllers/secrets/kubernetes_cert_sans_test.go new file mode 100644 index 0000000..5d67932 --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/kubernetes_cert_sans_test.go @@ -0,0 +1,152 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets_test + +import ( + "fmt" + "net/netip" + "net/url" + "reflect" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + secretsctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/secrets" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" +) + +type KubernetesCertSANsSuite struct { + ctest.DefaultSuite +} + +func TestKubernetesCertSANsSuite(t *testing.T) { + suite.Run(t, &KubernetesCertSANsSuite{ + DefaultSuite: ctest.DefaultSuite{ + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(&secretsctrl.KubernetesCertSANsController{})) + }, + }, + }) +} + +func (suite *KubernetesCertSANsSuite) TestReconcile() { + rootSecrets := secrets.NewKubernetesRoot(secrets.KubernetesRootID) + + var err error + + rootSecrets.TypedSpec().CertSANs = []string{"example.com"} + rootSecrets.TypedSpec().APIServerIPs = []netip.Addr{netip.MustParseAddr("10.4.3.2"), netip.MustParseAddr("10.2.1.3")} + rootSecrets.TypedSpec().DNSDomain = "cluster.remote" + rootSecrets.TypedSpec().Endpoint, err = url.Parse("https://some.url:6443/") + suite.Require().NoError(err) + rootSecrets.TypedSpec().LocalEndpoint, err = url.Parse("https://localhost:6443/") + suite.Require().NoError(err) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), rootSecrets)) + + hostnameStatus := network.NewHostnameStatus(network.NamespaceName, network.HostnameID) + hostnameStatus.TypedSpec().Hostname = "foo" + hostnameStatus.TypedSpec().Domainname = "example.com" + suite.Require().NoError(suite.State().Create(suite.Ctx(), hostnameStatus)) + + nodeAddresses := network.NewNodeAddress( + network.NamespaceName, + network.FilteredNodeAddressID(network.NodeAddressAccumulativeID, k8s.NodeAddressFilterNoK8s), + ) + nodeAddresses.TypedSpec().Addresses = []netip.Prefix{ + netip.MustParsePrefix("10.2.1.3/24"), + netip.MustParsePrefix("172.16.0.1/32"), + } + suite.Require().NoError(suite.State().Create(suite.Ctx(), nodeAddresses)) + + suite.AssertWithin(10*time.Second, 100*time.Millisecond, func() error { + certSANs, err := ctest.Get[*secrets.CertSAN]( + suite, + resource.NewMetadata( + secrets.NamespaceName, + secrets.CertSANType, + secrets.CertSANKubernetesID, + resource.VersionUndefined, + ), + ) + if err != nil { + if state.IsNotFoundError(err) { + return retry.ExpectedError(err) + } + + return err + } + + spec := certSANs.TypedSpec() + + suite.Assert().Equal( + []string{ + "example.com", + "foo", + "foo.example.com", + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + "kubernetes.default.svc.cluster.remote", + "localhost", + "some.url", + }, spec.DNSNames, + ) + suite.Assert().Equal("[10.2.1.3 10.4.3.2 127.0.0.1 172.16.0.1]", fmt.Sprintf("%v", spec.IPs)) + + return nil + }) + + ctest.UpdateWithConflicts(suite, rootSecrets, func(rootSecrets *secrets.KubernetesRoot) error { + var err error + rootSecrets.TypedSpec().Endpoint, err = url.Parse("https://some.other.url:6443/") + + return err + }) + + suite.AssertWithin(10*time.Second, 100*time.Millisecond, func() error { + var certSANs resource.Resource + + certSANs, err := ctest.Get[*secrets.CertSAN]( + suite, + resource.NewMetadata( + secrets.NamespaceName, + secrets.CertSANType, + secrets.CertSANKubernetesID, + resource.VersionUndefined, + ), + ) + if err != nil { + return err + } + + spec := certSANs.(*secrets.CertSAN).TypedSpec() + + expectedDNSNames := []string{ + "example.com", + "foo", + "foo.example.com", + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + "kubernetes.default.svc.cluster.remote", + "localhost", + "some.other.url", + } + + if !reflect.DeepEqual(spec.DNSNames, expectedDNSNames) { + return retry.ExpectedErrorf("expected %v, got %v", expectedDNSNames, spec.DNSNames) + } + + return nil + }) +} diff --git a/internal/app/machined/pkg/controllers/secrets/kubernetes_dynamic_certs.go b/internal/app/machined/pkg/controllers/secrets/kubernetes_dynamic_certs.go new file mode 100644 index 0000000..21149bb --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/kubernetes_dynamic_certs.go @@ -0,0 +1,248 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets + +import ( + "context" + stdlibx509 "crypto/x509" + "fmt" + "time" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/crypto/x509" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" + timeresource "github.com/siderolabs/talos/pkg/machinery/resources/time" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +// KubernetesDynamicCertsController manages secrets.KubernetesDynamicCerts based on configuration. +type KubernetesDynamicCertsController struct{} + +// Name implements controller.Controller interface. +func (ctrl *KubernetesDynamicCertsController) Name() string { + return "secrets.KubernetesDynamicCertsController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *KubernetesDynamicCertsController) Inputs() []controller.Input { + return nil +} + +// Outputs implements controller.Controller interface. +func (ctrl *KubernetesDynamicCertsController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: secrets.KubernetesDynamicCertsType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *KubernetesDynamicCertsController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + // wait for the network to be ready first, then switch to regular inputs + if err := r.UpdateInputs([]controller.Input{ + { + Namespace: network.NamespaceName, + Type: network.StatusType, + ID: optional.Some(network.StatusID), + Kind: controller.InputWeak, + }, + }); err != nil { + return fmt.Errorf("error updating inputs: %w", err) + } + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + // wait for network to be ready as it might change IPs/hostname + networkStatus, err := safe.ReaderGet[*network.Status](ctx, r, resource.NewMetadata(network.NamespaceName, network.StatusType, network.StatusID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + if networkStatus.TypedSpec().AddressReady && networkStatus.TypedSpec().HostnameReady { + break + } + } + + // switch to regular inputs once the network is ready + if err := r.UpdateInputs([]controller.Input{ + { + Namespace: secrets.NamespaceName, + Type: secrets.KubernetesRootType, + ID: optional.Some(secrets.KubernetesRootID), + Kind: controller.InputWeak, + }, + { + Namespace: v1alpha1.NamespaceName, + Type: timeresource.StatusType, + ID: optional.Some(timeresource.StatusID), + Kind: controller.InputWeak, + }, + { + Namespace: secrets.NamespaceName, + Type: secrets.CertSANType, + ID: optional.Some(secrets.CertSANKubernetesID), + Kind: controller.InputWeak, + }, + }); err != nil { + return fmt.Errorf("error updating inputs: %w", err) + } + + r.QueueReconcile() + + refreshTicker := time.NewTicker(KubernetesCertificateValidityDuration / 2) + defer refreshTicker.Stop() + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + case <-refreshTicker.C: + } + + k8sRoot, err := safe.ReaderGet[*secrets.KubernetesRoot](ctx, r, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesRootType, secrets.KubernetesRootID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + if err = ctrl.teardownAll(ctx, r); err != nil { + return fmt.Errorf("error destroying resources: %w", err) + } + + continue + } + + return fmt.Errorf("error getting root k8s secrets: %w", err) + } + + // wait for time sync as certs depend on current time + timeSync, err := safe.ReaderGet[*timeresource.Status](ctx, r, resource.NewMetadata(v1alpha1.NamespaceName, timeresource.StatusType, timeresource.StatusID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + if !timeSync.TypedSpec().Synced { + continue + } + + certSANs, err := safe.ReaderGet[*secrets.CertSAN](ctx, r, resource.NewMetadata(secrets.NamespaceName, secrets.CertSANType, secrets.CertSANKubernetesID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + if err = safe.WriterModify(ctx, r, secrets.NewKubernetesDynamicCerts(), func(r *secrets.KubernetesDynamicCerts) error { + return ctrl.updateSecrets(k8sRoot.TypedSpec(), r.TypedSpec(), certSANs.TypedSpec()) + }); err != nil { + return err + } + + r.ResetRestartBackoff() + } +} + +func (ctrl *KubernetesDynamicCertsController) updateSecrets(k8sRoot *secrets.KubernetesRootSpec, k8sCerts *secrets.KubernetesDynamicCertsSpec, + certSANs *secrets.CertSANSpec, +) error { + ca, err := x509.NewCertificateAuthorityFromCertificateAndKey(k8sRoot.IssuingCA) + if err != nil { + return fmt.Errorf("failed to parse CA certificate: %w", err) + } + + apiServer, err := x509.NewKeyPair(ca, + x509.IPAddresses(certSANs.StdIPs()), + x509.DNSNames(certSANs.DNSNames), + x509.CommonName("kube-apiserver"), + x509.Organization("kube-master"), + x509.NotAfter(time.Now().Add(KubernetesCertificateValidityDuration)), + x509.KeyUsage(stdlibx509.KeyUsageDigitalSignature|stdlibx509.KeyUsageKeyEncipherment), + x509.ExtKeyUsage([]stdlibx509.ExtKeyUsage{ + stdlibx509.ExtKeyUsageServerAuth, + }), + ) + if err != nil { + return fmt.Errorf("failed to generate api-server cert: %w", err) + } + + k8sCerts.APIServer = x509.NewCertificateAndKeyFromKeyPair(apiServer) + + apiServerKubeletClient, err := x509.NewKeyPair(ca, + x509.CommonName(constants.KubernetesAPIServerKubeletClientCommonName), + x509.Organization(constants.KubernetesAdminCertOrganization), + x509.NotAfter(time.Now().Add(KubernetesCertificateValidityDuration)), + x509.KeyUsage(stdlibx509.KeyUsageDigitalSignature|stdlibx509.KeyUsageKeyEncipherment), + x509.ExtKeyUsage([]stdlibx509.ExtKeyUsage{ + stdlibx509.ExtKeyUsageClientAuth, + }), + ) + if err != nil { + return fmt.Errorf("failed to generate api-server cert: %w", err) + } + + k8sCerts.APIServerKubeletClient = x509.NewCertificateAndKeyFromKeyPair(apiServerKubeletClient) + + aggregatorCA, err := x509.NewCertificateAuthorityFromCertificateAndKey(k8sRoot.AggregatorCA) + if err != nil { + return fmt.Errorf("failed to parse aggregator CA: %w", err) + } + + frontProxy, err := x509.NewKeyPair(aggregatorCA, + x509.CommonName("front-proxy-client"), + x509.NotAfter(time.Now().Add(KubernetesCertificateValidityDuration)), + x509.KeyUsage(stdlibx509.KeyUsageDigitalSignature|stdlibx509.KeyUsageKeyEncipherment), + x509.ExtKeyUsage([]stdlibx509.ExtKeyUsage{ + stdlibx509.ExtKeyUsageClientAuth, + }), + ) + if err != nil { + return fmt.Errorf("failed to generate aggregator cert: %w", err) + } + + k8sCerts.FrontProxy = x509.NewCertificateAndKeyFromKeyPair(frontProxy) + + return nil +} + +func (ctrl *KubernetesDynamicCertsController) teardownAll(ctx context.Context, r controller.Runtime) error { + list, err := r.List(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesDynamicCertsType, "", resource.VersionUndefined)) + if err != nil { + return err + } + + // TODO: change this to proper teardown sequence + + for _, res := range list.Items { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return err + } + } + + return nil +} diff --git a/internal/app/machined/pkg/controllers/secrets/kubernetes_dynamic_certs_test.go b/internal/app/machined/pkg/controllers/secrets/kubernetes_dynamic_certs_test.go new file mode 100644 index 0000000..fd7e28a --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/kubernetes_dynamic_certs_test.go @@ -0,0 +1,200 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets_test + +import ( + stdlibx509 "crypto/x509" + "fmt" + "net/netip" + "net/url" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/siderolabs/crypto/x509" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + secretsctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/secrets" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" + timeresource "github.com/siderolabs/talos/pkg/machinery/resources/time" +) + +func TestKubernetesDynamicCertsSuite(t *testing.T) { + suite.Run(t, &KubernetesDynamicCertsSuite{ + DefaultSuite: ctest.DefaultSuite{ + Timeout: 5 * time.Second, + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(&secretsctrl.KubernetesDynamicCertsController{})) + }, + }, + }) +} + +type KubernetesDynamicCertsSuite struct { + ctest.DefaultSuite +} + +func (suite *KubernetesDynamicCertsSuite) TestReconcile() { + rootSecrets := secrets.NewKubernetesRoot(secrets.KubernetesRootID) + + k8sCA, err := x509.NewSelfSignedCertificateAuthority( + x509.Organization("kubernetes"), + x509.ECDSA(true), + ) + suite.Require().NoError(err) + + aggregatorCA, err := x509.NewSelfSignedCertificateAuthority( + x509.Organization("kubernetes"), + x509.ECDSA(true), + ) + suite.Require().NoError(err) + + serviceAccount, err := x509.NewECDSAKey() + suite.Require().NoError(err) + + rootSecrets.TypedSpec().Name = "cluster1" + rootSecrets.TypedSpec().Endpoint, err = url.Parse("https://some.url:6443/") + suite.Require().NoError(err) + rootSecrets.TypedSpec().LocalEndpoint, err = url.Parse("https://localhost:6443/") + suite.Require().NoError(err) + + rootSecrets.TypedSpec().IssuingCA = &x509.PEMEncodedCertificateAndKey{ + Crt: k8sCA.CrtPEM, + Key: k8sCA.KeyPEM, + } + rootSecrets.TypedSpec().AggregatorCA = &x509.PEMEncodedCertificateAndKey{ + Crt: aggregatorCA.CrtPEM, + Key: aggregatorCA.KeyPEM, + } + rootSecrets.TypedSpec().ServiceAccount = &x509.PEMEncodedKey{ + Key: serviceAccount.KeyPEM, + } + rootSecrets.TypedSpec().CertSANs = []string{"example.com"} + rootSecrets.TypedSpec().APIServerIPs = []netip.Addr{netip.MustParseAddr("10.4.3.2"), netip.MustParseAddr("10.2.1.3")} + rootSecrets.TypedSpec().DNSDomain = "cluster.remote" + suite.Require().NoError(suite.State().Create(suite.Ctx(), rootSecrets)) + + machineType := config.NewMachineType() + machineType.SetMachineType(machine.TypeControlPlane) + suite.Require().NoError(suite.State().Create(suite.Ctx(), machineType)) + + networkStatus := network.NewStatus(network.NamespaceName, network.StatusID) + networkStatus.TypedSpec().AddressReady = true + networkStatus.TypedSpec().HostnameReady = true + suite.Require().NoError(suite.State().Create(suite.Ctx(), networkStatus)) + + certSANs := secrets.NewCertSAN(secrets.NamespaceName, secrets.CertSANKubernetesID) + certSANs.TypedSpec().Append( + "example.com", + "foo", + "foo.example.com", + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + "kubernetes.default.svc.cluster.remote", + "localhost", + "some.url", + "10.2.1.3", + "10.4.3.2", + "172.16.0.1", + ) + suite.Require().NoError(suite.State().Create(suite.Ctx(), certSANs)) + + timeSync := timeresource.NewStatus() + *timeSync.TypedSpec() = timeresource.StatusSpec{ + Synced: true, + } + suite.Require().NoError(suite.State().Create(suite.Ctx(), timeSync)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{secrets.KubernetesDynamicCertsID}, + func(certs *secrets.KubernetesDynamicCerts, assertion *assert.Assertions) { + kubernetesCerts := certs.TypedSpec() + + apiCert, err := kubernetesCerts.APIServer.GetCert() + assertion.NoError(err) + + if err != nil { + return + } + + assertion.Equal( + []string{ + "example.com", + "foo", + "foo.example.com", + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + "kubernetes.default.svc.cluster.remote", + "localhost", + "some.url", + }, apiCert.DNSNames, + ) + assertion.Equal("[10.2.1.3 10.4.3.2 172.16.0.1]", fmt.Sprintf("%v", apiCert.IPAddresses)) + + assertion.Equal("kube-apiserver", apiCert.Subject.CommonName) + assertion.Equal([]string{"kube-master"}, apiCert.Subject.Organization) + + assertion.Equal( + stdlibx509.KeyUsageDigitalSignature|stdlibx509.KeyUsageKeyEncipherment, + apiCert.KeyUsage, + ) + assertion.Equal([]stdlibx509.ExtKeyUsage{stdlibx509.ExtKeyUsageServerAuth}, apiCert.ExtKeyUsage) + + clientCert, err := kubernetesCerts.APIServerKubeletClient.GetCert() + assertion.NoError(err) + + if err != nil { + return + } + + assertion.Empty(clientCert.DNSNames) + assertion.Empty(clientCert.IPAddresses) + + assertion.Equal( + constants.KubernetesAPIServerKubeletClientCommonName, + clientCert.Subject.CommonName, + ) + assertion.Equal( + []string{constants.KubernetesAdminCertOrganization}, + clientCert.Subject.Organization, + ) + + assertion.Equal( + stdlibx509.KeyUsageDigitalSignature|stdlibx509.KeyUsageKeyEncipherment, + clientCert.KeyUsage, + ) + assertion.Equal([]stdlibx509.ExtKeyUsage{stdlibx509.ExtKeyUsageClientAuth}, clientCert.ExtKeyUsage) + + frontProxyCert, err := kubernetesCerts.FrontProxy.GetCert() + assertion.NoError(err) + + if err != nil { + return + } + + assertion.Empty(frontProxyCert.DNSNames) + assertion.Empty(frontProxyCert.IPAddresses) + + assertion.Equal("front-proxy-client", frontProxyCert.Subject.CommonName) + assertion.Empty(frontProxyCert.Subject.Organization) + + assertion.Equal( + stdlibx509.KeyUsageDigitalSignature|stdlibx509.KeyUsageKeyEncipherment, + frontProxyCert.KeyUsage, + ) + assertion.Equal( + []stdlibx509.ExtKeyUsage{stdlibx509.ExtKeyUsageClientAuth}, + frontProxyCert.ExtKeyUsage, + ) + }) +} diff --git a/internal/app/machined/pkg/controllers/secrets/kubernetes_test.go b/internal/app/machined/pkg/controllers/secrets/kubernetes_test.go new file mode 100644 index 0000000..2eaa671 --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/kubernetes_test.go @@ -0,0 +1,113 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets_test + +import ( + "net/netip" + "net/url" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/siderolabs/crypto/x509" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "k8s.io/client-go/tools/clientcmd" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + secretsctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/secrets" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" + timeresource "github.com/siderolabs/talos/pkg/machinery/resources/time" +) + +func TestKubernetesSuite(t *testing.T) { + suite.Run(t, &KubernetesSuite{ + DefaultSuite: ctest.DefaultSuite{ + Timeout: 5 * time.Second, + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(&secretsctrl.KubernetesController{})) + }, + }, + }) +} + +type KubernetesSuite struct { + ctest.DefaultSuite +} + +func (suite *KubernetesSuite) TestReconcile() { + rootSecrets := secrets.NewKubernetesRoot(secrets.KubernetesRootID) + + k8sCA, err := x509.NewSelfSignedCertificateAuthority( + x509.Organization("kubernetes"), + x509.ECDSA(true), + ) + suite.Require().NoError(err) + + aggregatorCA, err := x509.NewSelfSignedCertificateAuthority( + x509.Organization("kubernetes"), + x509.ECDSA(true), + ) + suite.Require().NoError(err) + + serviceAccount, err := x509.NewECDSAKey() + suite.Require().NoError(err) + + rootSecrets.TypedSpec().Name = "cluster1" + rootSecrets.TypedSpec().Endpoint, err = url.Parse("https://some.url:6443/") + suite.Require().NoError(err) + rootSecrets.TypedSpec().LocalEndpoint, err = url.Parse("https://localhost:6443/") + suite.Require().NoError(err) + + rootSecrets.TypedSpec().IssuingCA = &x509.PEMEncodedCertificateAndKey{ + Crt: k8sCA.CrtPEM, + Key: k8sCA.KeyPEM, + } + rootSecrets.TypedSpec().AggregatorCA = &x509.PEMEncodedCertificateAndKey{ + Crt: aggregatorCA.CrtPEM, + Key: aggregatorCA.KeyPEM, + } + rootSecrets.TypedSpec().ServiceAccount = &x509.PEMEncodedKey{ + Key: serviceAccount.KeyPEM, + } + rootSecrets.TypedSpec().CertSANs = []string{"example.com"} + rootSecrets.TypedSpec().APIServerIPs = []netip.Addr{netip.MustParseAddr("10.4.3.2"), netip.MustParseAddr("10.2.1.3")} + rootSecrets.TypedSpec().DNSDomain = "cluster.svc" + suite.Require().NoError(suite.State().Create(suite.Ctx(), rootSecrets)) + + machineType := config.NewMachineType() + machineType.SetMachineType(machine.TypeControlPlane) + suite.Require().NoError(suite.State().Create(suite.Ctx(), machineType)) + + timeSync := timeresource.NewStatus() + *timeSync.TypedSpec() = timeresource.StatusSpec{ + Synced: true, + } + suite.Require().NoError(suite.State().Create(suite.Ctx(), timeSync)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{secrets.KubernetesID}, + func(certs *secrets.Kubernetes, assertion *assert.Assertions) { + kubernetesCerts := certs.TypedSpec() + + for _, kubeconfig := range []string{ + kubernetesCerts.ControllerManagerKubeconfig, + kubernetesCerts.SchedulerKubeconfig, + kubernetesCerts.LocalhostAdminKubeconfig, + kubernetesCerts.AdminKubeconfig, + } { + config, err := clientcmd.Load([]byte(kubeconfig)) + assertion.NoError(err) + + if err != nil { + return + } + + assertion.NoError(clientcmd.ConfirmUsable(*config, config.CurrentContext)) + } + }) +} diff --git a/internal/app/machined/pkg/controllers/secrets/maintenance.go b/internal/app/machined/pkg/controllers/secrets/maintenance.go new file mode 100644 index 0000000..e3e13bd --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/maintenance.go @@ -0,0 +1,143 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets + +import ( + "context" + stdlibx509 "crypto/x509" + "fmt" + "time" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/crypto/x509" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" + timeresource "github.com/siderolabs/talos/pkg/machinery/resources/time" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +// MaintenanceController manages secrets.MaintenanceServiceCerts. +type MaintenanceController struct{} + +// Name implements controller.Controller interface. +func (ctrl *MaintenanceController) Name() string { + return "secrets.MaintenanceController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *MaintenanceController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: secrets.NamespaceName, + Type: secrets.MaintenanceRootType, + ID: optional.Some(secrets.MaintenanceRootID), + Kind: controller.InputWeak, + }, + { + Namespace: secrets.NamespaceName, + Type: secrets.CertSANType, + ID: optional.Some(secrets.CertSANMaintenanceID), + Kind: controller.InputWeak, + }, + // time status isn't fetched, but the fact that it is in dependencies means + // that certs will be regenerated on time sync/jump (as reconcile will be triggered) + { + Namespace: v1alpha1.NamespaceName, + Type: timeresource.StatusType, + ID: optional.Some(timeresource.StatusID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *MaintenanceController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: secrets.MaintenanceServiceCertsType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *MaintenanceController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + refreshTicker := time.NewTicker(x509.DefaultCertificateValidityDuration / 2) + defer refreshTicker.Stop() + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + case <-refreshTicker.C: + } + + rootSecrets, err := safe.ReaderGetByID[*secrets.MaintenanceRoot](ctx, r, secrets.MaintenanceRootID) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting maintenance root secrets: %w", err) + } + + certSANs, err := safe.ReaderGetByID[*secrets.CertSAN](ctx, r, secrets.CertSANMaintenanceID) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting certSANs: %w", err) + } + + ca, err := x509.NewCertificateAuthorityFromCertificateAndKey(rootSecrets.TypedSpec().CA) + if err != nil { + return fmt.Errorf("failed to parse CA certificate: %w", err) + } + + serverCert, err := x509.NewKeyPair(ca, + x509.IPAddresses(certSANs.TypedSpec().StdIPs()), + x509.DNSNames(certSANs.TypedSpec().DNSNames), + x509.CommonName(certSANs.TypedSpec().FQDN), + x509.NotAfter(time.Now().Add(x509.DefaultCertificateValidityDuration)), + x509.KeyUsage(stdlibx509.KeyUsageDigitalSignature), + x509.ExtKeyUsage([]stdlibx509.ExtKeyUsage{ + stdlibx509.ExtKeyUsageServerAuth, + }), + ) + if err != nil { + return fmt.Errorf("failed to generate maintenance server cert: %w", err) + } + + if err = safe.WriterModify(ctx, r, secrets.NewMaintenanceServiceCerts(), + func(maintenanceSecrets *secrets.MaintenanceServiceCerts) error { + spec := maintenanceSecrets.TypedSpec() + + spec.CA = &x509.PEMEncodedCertificateAndKey{ + Crt: rootSecrets.TypedSpec().CA.Crt, + } + spec.Server = x509.NewCertificateAndKeyFromKeyPair(serverCert) + + return nil + }); err != nil { + return fmt.Errorf("error modifying resource: %w", err) + } + + serverFingerprint, _ := x509.SPKIFingerprintFromDER(serverCert.Certificate.Certificate[0]) //nolint:errcheck + + logger.Debug("generated new certificates", + zap.Stringer("server", serverFingerprint), + ) + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/secrets/maintenance_cert_sans.go b/internal/app/machined/pkg/controllers/secrets/maintenance_cert_sans.go new file mode 100644 index 0000000..60efebe --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/maintenance_cert_sans.go @@ -0,0 +1,107 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets + +import ( + "context" + "fmt" + "net/netip" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" +) + +// MaintenanceCertSANsController manages secrets.APICertSANs based on configuration. +type MaintenanceCertSANsController struct{} + +// Name implements controller.Controller interface. +func (ctrl *MaintenanceCertSANsController) Name() string { + return "secrets.MaintenanceCertSANsController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *MaintenanceCertSANsController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: network.NamespaceName, + Type: network.HostnameStatusType, + ID: optional.Some(network.HostnameID), + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.NodeAddressType, + ID: optional.Some(network.NodeAddressAccumulativeID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *MaintenanceCertSANsController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: secrets.CertSANType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +func (ctrl *MaintenanceCertSANsController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + hostnameStatus, err := safe.ReaderGetByID[*network.HostnameStatus](ctx, r, network.HostnameID) + if err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("failed to get hostname status: %w", err) + } + + nodeAddresses, err := safe.ReaderGetByID[*network.NodeAddress](ctx, r, network.NodeAddressAccumulativeID) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + if err = r.Modify(ctx, secrets.NewCertSAN(secrets.NamespaceName, secrets.CertSANMaintenanceID), func(r resource.Resource) error { + spec := r.(*secrets.CertSAN).TypedSpec() + + spec.Reset() + + spec.AppendIPs(nodeAddresses.TypedSpec().IPs()...) + spec.AppendIPs(netip.MustParseAddr("127.0.0.1")) + spec.AppendIPs(netip.MustParseAddr("::1")) + + if hostnameStatus != nil { + spec.AppendDNSNames(hostnameStatus.TypedSpec().DNSNames()...) + } + + spec.FQDN = constants.MaintenanceServiceCommonName + + spec.Sort() + + return nil + }); err != nil { + return err + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/secrets/maintenance_cert_sans_test.go b/internal/app/machined/pkg/controllers/secrets/maintenance_cert_sans_test.go new file mode 100644 index 0000000..d19925c --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/maintenance_cert_sans_test.go @@ -0,0 +1,67 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets_test + +import ( + "fmt" + "net/netip" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + secretsctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/secrets" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" +) + +func TestMaintenanceCertSANsSuite(t *testing.T) { + suite.Run(t, &MaintenanceCertSANsSuite{ + DefaultSuite: ctest.DefaultSuite{ + Timeout: 2 * time.Second, + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(&secretsctrl.MaintenanceCertSANsController{})) + }, + }, + }) +} + +type MaintenanceCertSANsSuite struct { + ctest.DefaultSuite +} + +func (suite *MaintenanceCertSANsSuite) TestReconcile() { + nodeAddresses := network.NewNodeAddress( + network.NamespaceName, + network.NodeAddressAccumulativeID, + ) + nodeAddresses.TypedSpec().Addresses = []netip.Prefix{ + netip.MustParsePrefix("10.2.1.3/24"), + netip.MustParsePrefix("172.16.0.1/32"), + } + suite.Require().NoError(suite.State().Create(suite.Ctx(), nodeAddresses)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{secrets.CertSANMaintenanceID}, + func(certSANs *secrets.CertSAN, asrt *assert.Assertions) { + asrt.Empty(certSANs.TypedSpec().DNSNames) + asrt.Equal("[10.2.1.3 127.0.0.1 172.16.0.1 ::1]", fmt.Sprintf("%v", certSANs.TypedSpec().IPs)) + asrt.Equal(constants.MaintenanceServiceCommonName, certSANs.TypedSpec().FQDN) + }) + + hostnameStatus := network.NewHostnameStatus(network.NamespaceName, network.HostnameID) + hostnameStatus.TypedSpec().Hostname = "bar" + hostnameStatus.TypedSpec().Domainname = "some.org" + suite.Require().NoError(suite.State().Create(suite.Ctx(), hostnameStatus)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{secrets.CertSANMaintenanceID}, + func(certSANs *secrets.CertSAN, asrt *assert.Assertions) { + asrt.Equal([]string{"bar", "bar.some.org"}, certSANs.TypedSpec().DNSNames) + }) +} diff --git a/internal/app/machined/pkg/controllers/secrets/maintenance_root.go b/internal/app/machined/pkg/controllers/secrets/maintenance_root.go new file mode 100644 index 0000000..407c1e8 --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/maintenance_root.go @@ -0,0 +1,61 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/siderolabs/crypto/x509" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" +) + +// MaintenanceRootController manages secrets.Root based on configuration. +type MaintenanceRootController struct{} + +// Name implements controller.Controller interface. +func (ctrl *MaintenanceRootController) Name() string { + return "secrets.MaintenanceRootController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *MaintenanceRootController) Inputs() []controller.Input { + return nil +} + +// Outputs implements controller.Controller interface. +func (ctrl *MaintenanceRootController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: secrets.MaintenanceRootType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +func (ctrl *MaintenanceRootController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + // run this controller only once, as the CA never changes + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + return safe.WriterModify(ctx, r, secrets.NewMaintenanceRoot(secrets.MaintenanceRootID), func(root *secrets.MaintenanceRoot) error { + ca, err := x509.NewSelfSignedCertificateAuthority() + if err != nil { + return fmt.Errorf("failed to generate self-signed CA: %w", err) + } + + root.TypedSpec().CA = x509.NewCertificateAndKeyFromCertificateAuthority(ca) + + return nil + }) +} diff --git a/internal/app/machined/pkg/controllers/secrets/maintenance_root_test.go b/internal/app/machined/pkg/controllers/secrets/maintenance_root_test.go new file mode 100644 index 0000000..394fda5 --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/maintenance_root_test.go @@ -0,0 +1,39 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets_test + +import ( + "testing" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + secretsctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/secrets" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" +) + +func TestMaintenanceRootSuite(t *testing.T) { + suite.Run(t, &MaintenanceRootSuite{ + DefaultSuite: ctest.DefaultSuite{ + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(&secretsctrl.MaintenanceRootController{})) + }, + }, + }) +} + +type MaintenanceRootSuite struct { + ctest.DefaultSuite +} + +func (suite *MaintenanceRootSuite) TestReconcile() { + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{secrets.MaintenanceRootID}, + func(root *secrets.MaintenanceRoot, asrt *assert.Assertions) { + asrt.NotEmpty(root.TypedSpec().CA) + }) +} diff --git a/internal/app/machined/pkg/controllers/secrets/maintenance_test.go b/internal/app/machined/pkg/controllers/secrets/maintenance_test.go new file mode 100644 index 0000000..8014f34 --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/maintenance_test.go @@ -0,0 +1,89 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets_test + +import ( + stdlibx509 "crypto/x509" + "fmt" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/siderolabs/crypto/x509" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + secretsctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/secrets" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" +) + +func TestMaintenanceSuite(t *testing.T) { + suite.Run(t, &MaintenanceSuite{ + DefaultSuite: ctest.DefaultSuite{ + Timeout: 2 * time.Second, + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(&secretsctrl.MaintenanceController{})) + }, + }, + }) +} + +type MaintenanceSuite struct { + ctest.DefaultSuite +} + +func (suite *MaintenanceSuite) TestReconcile() { + rootSecrets := secrets.NewMaintenanceRoot(secrets.MaintenanceRootID) + + rootCA, err := x509.NewSelfSignedCertificateAuthority( + x509.Organization("talos"), + ) + suite.Require().NoError(err) + + rootSecrets.TypedSpec().CA = &x509.PEMEncodedCertificateAndKey{ + Crt: rootCA.CrtPEM, + Key: rootCA.KeyPEM, + } + suite.Require().NoError(suite.State().Create(suite.Ctx(), rootSecrets)) + + certSANs := secrets.NewCertSAN(secrets.NamespaceName, secrets.CertSANMaintenanceID) + certSANs.TypedSpec().Append( + "example.com", + "foo", + "10.2.1.3", + ) + + certSANs.TypedSpec().FQDN = "maintenance-service" + suite.Require().NoError(suite.State().Create(suite.Ctx(), certSANs)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{secrets.MaintenanceServiceCertsID}, + func(certs *secrets.MaintenanceServiceCerts, asrt *assert.Assertions) { + spec := certs.TypedSpec() + + asrt.Equal(rootCA.CrtPEM, spec.CA.Crt) + asrt.Nil(spec.CA.Key) + + serverCert, err := spec.Server.GetCert() + asrt.NoError(err) + + if err != nil { + return + } + + asrt.Equal([]string{"example.com", "foo"}, serverCert.DNSNames) + asrt.Equal("[10.2.1.3]", fmt.Sprintf("%v", serverCert.IPAddresses)) + + asrt.Equal("maintenance-service", serverCert.Subject.CommonName) + asrt.Empty(serverCert.Subject.Organization) + + asrt.Equal( + stdlibx509.KeyUsageDigitalSignature, + serverCert.KeyUsage, + ) + asrt.Equal([]stdlibx509.ExtKeyUsage{stdlibx509.ExtKeyUsageServerAuth}, serverCert.ExtKeyUsage) + }) +} diff --git a/internal/app/machined/pkg/controllers/secrets/root.go b/internal/app/machined/pkg/controllers/secrets/root.go new file mode 100644 index 0000000..ac7d4c5 --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/root.go @@ -0,0 +1,200 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets + +import ( + "context" + "errors" + "fmt" + "net/netip" + "net/url" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/controller/generic" + "github.com/cosi-project/runtime/pkg/controller/generic/transform" + "github.com/siderolabs/crypto/x509" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" +) + +func rootMapFunc[Output generic.ResourceWithRD](output Output, requireControlPlane bool) func(cfg *config.MachineConfig) optional.Optional[Output] { + return func(cfg *config.MachineConfig) optional.Optional[Output] { + if cfg.Metadata().ID() != config.V1Alpha1ID { + return optional.None[Output]() + } + + if cfg.Config().Cluster() == nil || cfg.Config().Machine() == nil { + return optional.None[Output]() + } + + if requireControlPlane && !cfg.Config().Machine().Type().IsControlPlane() { + return optional.None[Output]() + } + + return optional.Some(output) + } +} + +// RootEtcdController manages secrets.EtcdRoot based on configuration. +type RootEtcdController = transform.Controller[*config.MachineConfig, *secrets.EtcdRoot] + +// NewRootEtcdController instanciates the controller. +func NewRootEtcdController() *RootEtcdController { + return transform.NewController( + transform.Settings[*config.MachineConfig, *secrets.EtcdRoot]{ + Name: "secrets.RootEtcdController", + MapMetadataOptionalFunc: rootMapFunc(secrets.NewEtcdRoot(secrets.EtcdRootID), true), + TransformFunc: func(ctx context.Context, r controller.Reader, logger *zap.Logger, cfg *config.MachineConfig, res *secrets.EtcdRoot) error { + cfgProvider := cfg.Config() + etcdSecrets := res.TypedSpec() + + etcdSecrets.EtcdCA = cfgProvider.Cluster().Etcd().CA() + + if etcdSecrets.EtcdCA == nil { + return errors.New("missing cluster.etcdCA secret") + } + + return nil + }, + }, + ) +} + +// RootKubernetesController manages secrets.KubernetesRoot based on configuration. +type RootKubernetesController = transform.Controller[*config.MachineConfig, *secrets.KubernetesRoot] + +// NewRootKubernetesController instanciates the controller. +func NewRootKubernetesController() *RootKubernetesController { + return transform.NewController( + transform.Settings[*config.MachineConfig, *secrets.KubernetesRoot]{ + Name: "secrets.RootKubernetesController", + MapMetadataOptionalFunc: rootMapFunc(secrets.NewKubernetesRoot(secrets.KubernetesRootID), true), + TransformFunc: func(ctx context.Context, r controller.Reader, logger *zap.Logger, cfg *config.MachineConfig, res *secrets.KubernetesRoot) error { + cfgProvider := cfg.Config() + k8sSecrets := res.TypedSpec() + + var ( + err error + localEndpoint *url.URL + ) + + if cfgProvider.Machine().Features().KubePrism().Enabled() { + localEndpoint, err = url.Parse(fmt.Sprintf("https://127.0.0.1:%d", cfgProvider.Machine().Features().KubePrism().Port())) + if err != nil { + return err + } + } else { + localEndpoint, err = url.Parse(fmt.Sprintf("https://localhost:%d", cfgProvider.Cluster().LocalAPIServerPort())) + if err != nil { + return err + } + } + + k8sSecrets.Name = cfgProvider.Cluster().Name() + k8sSecrets.Endpoint = cfgProvider.Cluster().Endpoint() + k8sSecrets.LocalEndpoint = localEndpoint + k8sSecrets.CertSANs = cfgProvider.Cluster().CertSANs() + k8sSecrets.DNSDomain = cfgProvider.Cluster().Network().DNSDomain() + + k8sSecrets.APIServerIPs, err = cfgProvider.Cluster().Network().APIServerIPs() + if err != nil { + return fmt.Errorf("error building API service IPs: %w", err) + } + + k8sSecrets.AggregatorCA = cfgProvider.Cluster().AggregatorCA() + + if k8sSecrets.AggregatorCA == nil { + return errors.New("missing cluster.aggregatorCA secret") + } + + k8sSecrets.IssuingCA = cfgProvider.Cluster().IssuingCA() + k8sSecrets.AcceptedCAs = cfgProvider.Cluster().AcceptedCAs() + + if k8sSecrets.IssuingCA != nil { + k8sSecrets.AcceptedCAs = append(k8sSecrets.AcceptedCAs, &x509.PEMEncodedCertificate{ + Crt: k8sSecrets.IssuingCA.Crt, + }) + } + + if len(k8sSecrets.IssuingCA.Key) == 0 { + // drop incomplete issuing CA, as the machine config for workers contains just the cert + k8sSecrets.IssuingCA = nil + } + + if len(k8sSecrets.AcceptedCAs) == 0 { + return errors.New("missing cluster.CA secret") + } + + k8sSecrets.ServiceAccount = cfgProvider.Cluster().ServiceAccount() + + k8sSecrets.AESCBCEncryptionSecret = cfgProvider.Cluster().AESCBCEncryptionSecret() + k8sSecrets.SecretboxEncryptionSecret = cfgProvider.Cluster().SecretboxEncryptionSecret() + + k8sSecrets.BootstrapTokenID = cfgProvider.Cluster().Token().ID() + k8sSecrets.BootstrapTokenSecret = cfgProvider.Cluster().Token().Secret() + + return nil + }, + }, + ) +} + +// RootOSController manages secrets.OSRoot based on configuration. +type RootOSController = transform.Controller[*config.MachineConfig, *secrets.OSRoot] + +// NewRootOSController instanciates the controller. +func NewRootOSController() *RootOSController { + return transform.NewController( + transform.Settings[*config.MachineConfig, *secrets.OSRoot]{ + Name: "secrets.RootOSController", + MapMetadataOptionalFunc: rootMapFunc(secrets.NewOSRoot(secrets.OSRootID), false), + TransformFunc: func(ctx context.Context, r controller.Reader, logger *zap.Logger, cfg *config.MachineConfig, res *secrets.OSRoot) error { + cfgProvider := cfg.Config() + osSecrets := res.TypedSpec() + + osSecrets.IssuingCA = cfgProvider.Machine().Security().IssuingCA() + osSecrets.AcceptedCAs = cfgProvider.Machine().Security().AcceptedCAs() + + if osSecrets.IssuingCA != nil { + osSecrets.AcceptedCAs = append(osSecrets.AcceptedCAs, &x509.PEMEncodedCertificate{ + Crt: osSecrets.IssuingCA.Crt, + }) + } + + if len(osSecrets.IssuingCA.Key) == 0 { + // drop incomplete issuing CA, as the machine config for workers contains just the cert + osSecrets.IssuingCA = nil + } + + osSecrets.CertSANIPs = nil + osSecrets.CertSANDNSNames = nil + + for _, san := range cfgProvider.Machine().Security().CertSANs() { + if ip, err := netip.ParseAddr(san); err == nil { + osSecrets.CertSANIPs = append(osSecrets.CertSANIPs, ip) + } else { + osSecrets.CertSANDNSNames = append(osSecrets.CertSANDNSNames, san) + } + } + + if cfgProvider.Machine().Features().KubernetesTalosAPIAccess().Enabled() { + // add Kubernetes Talos service name to the list of SANs + osSecrets.CertSANDNSNames = append(osSecrets.CertSANDNSNames, + constants.KubernetesTalosAPIServiceName, + constants.KubernetesTalosAPIServiceName+"."+constants.KubernetesTalosAPIServiceNamespace, + ) + } + + osSecrets.Token = cfgProvider.Machine().Security().Token() + + return nil + }, + }, + ) +} diff --git a/internal/app/machined/pkg/controllers/secrets/root_test.go b/internal/app/machined/pkg/controllers/secrets/root_test.go new file mode 100644 index 0000000..eebe0a9 --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/root_test.go @@ -0,0 +1,121 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets_test + +import ( + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/siderolabs/crypto/x509" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + secretsctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/secrets" + talosconfig "github.com/siderolabs/talos/pkg/machinery/config" + "github.com/siderolabs/talos/pkg/machinery/config/generate" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" +) + +func TestRootSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, &RootSuite{ + DefaultSuite: ctest.DefaultSuite{ + Timeout: 10 * time.Second, + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(secretsctrl.NewRootEtcdController())) + suite.Require().NoError(suite.Runtime().RegisterController(secretsctrl.NewRootKubernetesController())) + suite.Require().NoError(suite.Runtime().RegisterController(secretsctrl.NewRootOSController())) + }, + }, + }) +} + +type RootSuite struct { + ctest.DefaultSuite +} + +func (suite *RootSuite) genConfig(controlplane bool) talosconfig.Config { + input, err := generate.NewInput("test-cluster", "http://localhost:6443", "") + suite.Require().NoError(err) + + var cfg talosconfig.Provider + + if controlplane { + cfg, err = input.Config(machine.TypeControlPlane) + } else { + cfg, err = input.Config(machine.TypeWorker) + } + + suite.Require().NoError(err) + + machineCfg := config.NewMachineConfig(cfg) + suite.Require().NoError(suite.State().Create(suite.Ctx(), machineCfg)) + + return cfg +} + +func (suite *RootSuite) TestReconcileControlPlane() { + cfg := suite.genConfig(true) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{secrets.EtcdRootID}, + func(res *secrets.EtcdRoot, asrt *assert.Assertions) { + asrt.Equal(res.TypedSpec().EtcdCA, cfg.Cluster().Etcd().CA()) + }, + ) + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{secrets.KubernetesRootID}, + func(res *secrets.KubernetesRoot, asrt *assert.Assertions) { + asrt.Equal(res.TypedSpec().IssuingCA, cfg.Cluster().IssuingCA()) + asrt.Equal( + []*x509.PEMEncodedCertificate{ + { + Crt: cfg.Cluster().IssuingCA().Crt, + }, + }, + res.TypedSpec().AcceptedCAs, + ) + }, + ) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{secrets.OSRootID}, + func(res *secrets.OSRoot, asrt *assert.Assertions) { + asrt.Equal(res.TypedSpec().IssuingCA, cfg.Machine().Security().IssuingCA()) + asrt.Equal( + []*x509.PEMEncodedCertificate{ + { + Crt: cfg.Machine().Security().IssuingCA().Crt, + }, + }, + res.TypedSpec().AcceptedCAs, + ) + }, + ) +} + +func (suite *RootSuite) TestReconcileWorker() { + cfg := suite.genConfig(false) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{secrets.OSRootID}, + func(res *secrets.OSRoot, asrt *assert.Assertions) { + asrt.Nil(res.TypedSpec().IssuingCA) + asrt.Equal( + []*x509.PEMEncodedCertificate{ + { + Crt: cfg.Machine().Security().IssuingCA().Crt, + }, + }, + res.TypedSpec().AcceptedCAs, + ) + }, + ) + + rtestutils.AssertNoResource[*secrets.Etcd](suite.Ctx(), suite.T(), suite.State(), secrets.EtcdRootID) + rtestutils.AssertNoResource[*secrets.Kubernetes](suite.Ctx(), suite.T(), suite.State(), secrets.KubernetesRootID) +} diff --git a/internal/app/machined/pkg/controllers/secrets/secrets.go b/internal/app/machined/pkg/controllers/secrets/secrets.go new file mode 100644 index 0000000..3149da1 --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/secrets.go @@ -0,0 +1,6 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package secrets provides controllers which manage secret resources. +package secrets diff --git a/internal/app/machined/pkg/controllers/secrets/trustd.go b/internal/app/machined/pkg/controllers/secrets/trustd.go new file mode 100644 index 0000000..e460495 --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/trustd.go @@ -0,0 +1,272 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets + +import ( + "context" + stdlibx509 "crypto/x509" + "errors" + "fmt" + "time" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/crypto/x509" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" + timeresource "github.com/siderolabs/talos/pkg/machinery/resources/time" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +// TrustdController manages secrets.API based on configuration to provide apid certificate. +type TrustdController struct{} + +// Name implements controller.Controller interface. +func (ctrl *TrustdController) Name() string { + return "secrets.TrustdController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *TrustdController) Inputs() []controller.Input { + // initial set of inputs: wait for machine type to be known and network to be partially configured + return []controller.Input{ + { + Namespace: network.NamespaceName, + Type: network.StatusType, + ID: optional.Some(network.StatusID), + Kind: controller.InputWeak, + }, + { + Namespace: config.NamespaceName, + Type: config.MachineTypeType, + ID: optional.Some(config.MachineTypeID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *TrustdController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: secrets.TrustdType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *TrustdController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + // reset inputs back to what they were initially + if err := r.UpdateInputs(ctrl.Inputs()); err != nil { + return err + } + + machineTypeRes, err := safe.ReaderGet[*config.MachineType](ctx, r, resource.NewMetadata(config.NamespaceName, config.MachineTypeType, config.MachineTypeID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting machine type: %w", err) + } + + machineType := machineTypeRes.MachineType() + + networkResource, err := safe.ReaderGet[*network.Status](ctx, r, resource.NewMetadata(network.NamespaceName, network.StatusType, network.StatusID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + networkStatus := networkResource.TypedSpec() + + if !(networkStatus.AddressReady && networkStatus.HostnameReady) { + continue + } + + // machine type is known and network is ready, we can now proceed to one or another reconcile loop + if machineType.IsControlPlane() { + if err = ctrl.reconcile(ctx, r, logger); err != nil { + return err + } + } else { + if err = ctrl.teardownAll(ctx, r); err != nil { + return err + } + } + + r.ResetRestartBackoff() + } +} + +//nolint:gocyclo,dupl +func (ctrl *TrustdController) reconcile(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + inputs := []controller.Input{ + { + Namespace: secrets.NamespaceName, + Type: secrets.OSRootType, + ID: optional.Some(secrets.OSRootID), + Kind: controller.InputWeak, + }, + { + Namespace: secrets.NamespaceName, + Type: secrets.CertSANType, + ID: optional.Some(secrets.CertSANAPIID), + Kind: controller.InputWeak, + }, + { + Namespace: config.NamespaceName, + Type: config.MachineTypeType, + ID: optional.Some(config.MachineTypeID), + Kind: controller.InputWeak, + }, + // time status isn't fetched, but the fact that it is in dependencies means + // that certs will be regenerated on time sync/jump (as reconcile will be triggered) + { + Namespace: v1alpha1.NamespaceName, + Type: timeresource.StatusType, + ID: optional.Some(timeresource.StatusID), + Kind: controller.InputWeak, + }, + } + + if err := r.UpdateInputs(inputs); err != nil { + return fmt.Errorf("error updating inputs: %w", err) + } + + r.QueueReconcile() + + refreshTicker := time.NewTicker(x509.DefaultCertificateValidityDuration / 2) + defer refreshTicker.Stop() + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + case <-refreshTicker.C: + } + + machineTypeRes, err := safe.ReaderGet[*config.MachineType](ctx, r, resource.NewMetadata(config.NamespaceName, config.MachineTypeType, config.MachineTypeID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting machine type: %w", err) + } + + machineType := machineTypeRes.MachineType() + + if !machineType.IsControlPlane() { + return errors.New("machine type changed") + } + + rootResource, err := safe.ReaderGet[*secrets.OSRoot](ctx, r, resource.NewMetadata(secrets.NamespaceName, secrets.OSRootType, secrets.OSRootID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + if err = ctrl.teardownAll(ctx, r); err != nil { + return fmt.Errorf("error destroying resources: %w", err) + } + + continue + } + + return fmt.Errorf("error getting etcd root secrets: %w", err) + } + + rootSpec := rootResource.TypedSpec() + + certSANResource, err := safe.ReaderGet[*secrets.CertSAN](ctx, r, resource.NewMetadata(secrets.NamespaceName, secrets.CertSANType, secrets.CertSANAPIID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting certSANs: %w", err) + } + + certSANs := certSANResource.TypedSpec() + + if err := ctrl.generateControlPlane(ctx, r, logger, rootSpec, certSANs); err != nil { + return err + } + } +} + +func (ctrl *TrustdController) generateControlPlane(ctx context.Context, r controller.Runtime, logger *zap.Logger, rootSpec *secrets.OSRootSpec, certSANs *secrets.CertSANSpec) error { + ca, err := x509.NewCertificateAuthorityFromCertificateAndKey(rootSpec.IssuingCA) + if err != nil { + return fmt.Errorf("failed to parse CA certificate: %w", err) + } + + serverCert, err := x509.NewKeyPair(ca, + x509.IPAddresses(certSANs.StdIPs()), + x509.DNSNames(certSANs.DNSNames), + x509.CommonName(certSANs.FQDN), + x509.NotAfter(time.Now().Add(x509.DefaultCertificateValidityDuration)), + x509.KeyUsage(stdlibx509.KeyUsageDigitalSignature|stdlibx509.KeyUsageKeyEncipherment), + x509.ExtKeyUsage([]stdlibx509.ExtKeyUsage{ + stdlibx509.ExtKeyUsageServerAuth, + }), + ) + if err != nil { + return fmt.Errorf("failed to generate API server cert: %w", err) + } + + if err := safe.WriterModify(ctx, r, secrets.NewTrustd(), + func(r *secrets.Trustd) error { + trustdSecrets := r.TypedSpec() + + trustdSecrets.AcceptedCAs = rootSpec.AcceptedCAs + trustdSecrets.Server = x509.NewCertificateAndKeyFromKeyPair(serverCert) + + return nil + }); err != nil { + return fmt.Errorf("error modifying resource: %w", err) + } + + serverFingerprint, _ := x509.SPKIFingerprintFromDER(serverCert.Certificate.Certificate[0]) //nolint:errcheck + + logger.Debug("generated new certificates", + zap.Stringer("server", serverFingerprint), + ) + + return nil +} + +func (ctrl *TrustdController) teardownAll(ctx context.Context, r controller.Runtime) error { + list, err := r.List(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.TrustdType, "", resource.VersionUndefined)) + if err != nil { + return err + } + + for _, res := range list.Items { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return err + } + } + + return nil +} diff --git a/internal/app/machined/pkg/controllers/secrets/trustd_test.go b/internal/app/machined/pkg/controllers/secrets/trustd_test.go new file mode 100644 index 0000000..5a98ed6 --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/trustd_test.go @@ -0,0 +1,134 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets_test + +import ( + stdlibx509 "crypto/x509" + "fmt" + "net/netip" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/crypto/x509" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + secretsctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/secrets" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" +) + +func TestTrustdSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, &TrustdSuite{ + DefaultSuite: ctest.DefaultSuite{ + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(&secretsctrl.TrustdController{})) + }, + }, + }) +} + +type TrustdSuite struct { + ctest.DefaultSuite +} + +func (suite *TrustdSuite) TestReconcileControlPlane() { + rootSecrets := secrets.NewOSRoot(secrets.OSRootID) + + talosCA, err := x509.NewSelfSignedCertificateAuthority( + x509.Organization("talos"), + ) + suite.Require().NoError(err) + + rootSecrets.TypedSpec().IssuingCA = &x509.PEMEncodedCertificateAndKey{ + Crt: talosCA.CrtPEM, + Key: talosCA.KeyPEM, + } + rootSecrets.TypedSpec().AcceptedCAs = []*x509.PEMEncodedCertificate{ + { + Crt: talosCA.CrtPEM, + }, + } + rootSecrets.TypedSpec().CertSANDNSNames = []string{"example.com"} + rootSecrets.TypedSpec().CertSANIPs = []netip.Addr{netip.MustParseAddr("10.4.3.2"), netip.MustParseAddr("10.2.1.3")} + rootSecrets.TypedSpec().Token = "something" + suite.Require().NoError(suite.State().Create(suite.Ctx(), rootSecrets)) + + machineType := config.NewMachineType() + machineType.SetMachineType(machine.TypeControlPlane) + suite.Require().NoError(suite.State().Create(suite.Ctx(), machineType)) + + networkStatus := network.NewStatus(network.NamespaceName, network.StatusID) + networkStatus.TypedSpec().AddressReady = true + networkStatus.TypedSpec().HostnameReady = true + suite.Require().NoError(suite.State().Create(suite.Ctx(), networkStatus)) + + certSANs := secrets.NewCertSAN(secrets.NamespaceName, secrets.CertSANAPIID) + certSANs.TypedSpec().Append( + "example.com", + "foo", + "foo.example.com", + "10.2.1.3", + "10.4.3.2", + "172.16.0.1", + ) + + certSANs.TypedSpec().FQDN = "foo.example.com" + + suite.Require().NoError(suite.State().Create(suite.Ctx(), certSANs)) + suite.AssertWithin(10*time.Second, 100*time.Millisecond, func() error { + certs, err := ctest.Get[*secrets.Trustd]( + suite, + resource.NewMetadata( + secrets.NamespaceName, + secrets.TrustdType, + secrets.TrustdID, + resource.VersionUndefined, + ), + ) + if err != nil { + if state.IsNotFoundError(err) { + return retry.ExpectedError(err) + } + + return err + } + + trustdCerts := certs.TypedSpec() + + suite.Assert().Equal( + []*x509.PEMEncodedCertificate{ + { + Crt: talosCA.CrtPEM, + }, + }, + trustdCerts.AcceptedCAs, + ) + + serverCert, err := trustdCerts.Server.GetCert() + suite.Require().NoError(err) + + suite.Assert().Equal([]string{"example.com", "foo", "foo.example.com"}, serverCert.DNSNames) + suite.Assert().Equal("[10.2.1.3 10.4.3.2 172.16.0.1]", fmt.Sprintf("%v", serverCert.IPAddresses)) + + suite.Assert().Equal("foo.example.com", serverCert.Subject.CommonName) + suite.Assert().Empty(serverCert.Subject.Organization) + + suite.Assert().Equal( + stdlibx509.KeyUsageDigitalSignature|stdlibx509.KeyUsageKeyEncipherment, + serverCert.KeyUsage, + ) + suite.Assert().Equal([]stdlibx509.ExtKeyUsage{stdlibx509.ExtKeyUsageServerAuth}, serverCert.ExtKeyUsage) + + return nil + }) +} diff --git a/internal/app/machined/pkg/controllers/siderolink/config.go b/internal/app/machined/pkg/controllers/siderolink/config.go new file mode 100644 index 0000000..0e24f45 --- /dev/null +++ b/internal/app/machined/pkg/controllers/siderolink/config.go @@ -0,0 +1,117 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package siderolink + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/go-procfs/procfs" + "go.uber.org/zap" + + v1alpha1runtime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/pkg/endpoint" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/siderolink" +) + +// ConfigController interacts with SideroLink API and brings up the SideroLink Wireguard interface. +type ConfigController struct { + Cmdline *procfs.Cmdline + V1Alpha1Mode v1alpha1runtime.Mode +} + +// Name implements controller.Controller interface. +func (ctrl *ConfigController) Name() string { + return "siderolink.ConfigController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *ConfigController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: config.MachineConfigType, + ID: optional.Some(config.V1Alpha1ID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *ConfigController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: siderolink.ConfigType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *ConfigController) Run(ctx context.Context, r controller.Runtime, _ *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + cfg, err := safe.ReaderGetByID[*config.MachineConfig](ctx, r, config.V1Alpha1ID) + if err != nil && !state.IsNotFoundError(err) { + return err + } + + r.StartTrackingOutputs() + + if ep := ctrl.apiEndpoint(cfg); ep != "" { + var parsed endpoint.Endpoint + + parsed, err = endpoint.Parse(ep) + if err != nil { + return fmt.Errorf("failed to parse siderolink API endpoint: %w", err) + } + + if err = safe.WriterModify(ctx, r, siderolink.NewConfig(config.NamespaceName, siderolink.ConfigID), func(c *siderolink.Config) error { + c.TypedSpec().APIEndpoint = ep + c.TypedSpec().Host = parsed.Host + c.TypedSpec().JoinToken = parsed.GetParam("jointoken") + c.TypedSpec().Insecure = parsed.Insecure + c.TypedSpec().Tunnel = parsed.GetParam("grpc_tunnel") == "true" || parsed.GetParam("grpc_tunnel") == "y" + + return nil + }); err != nil { + return fmt.Errorf("failed to update config: %w", err) + } + } + + if err = safe.CleanupOutputs[*siderolink.Config](ctx, r); err != nil { + return err + } + } +} + +func (ctrl *ConfigController) apiEndpoint(machineConfig *config.MachineConfig) string { + if machineConfig != nil && machineConfig.Config().SideroLink() != nil && machineConfig.Config().SideroLink().APIUrl() != nil { + return machineConfig.Config().SideroLink().APIUrl().String() + } + + if ctrl.V1Alpha1Mode == v1alpha1runtime.ModeContainer { + return "" + } + + if ctrl.Cmdline == nil || ctrl.Cmdline.Get(constants.KernelParamSideroLink).First() == nil { + return "" + } + + return *ctrl.Cmdline.Get(constants.KernelParamSideroLink).First() +} diff --git a/internal/app/machined/pkg/controllers/siderolink/config_test.go b/internal/app/machined/pkg/controllers/siderolink/config_test.go new file mode 100644 index 0000000..835c86e --- /dev/null +++ b/internal/app/machined/pkg/controllers/siderolink/config_test.go @@ -0,0 +1,81 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package siderolink_test + +import ( + "net/url" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/siderolabs/gen/xtesting/must" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + siderolinkctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/siderolink" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/meta" + siderolinkcfg "github.com/siderolabs/talos/pkg/machinery/config/types/siderolink" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/siderolink" +) + +type ConfigSuite struct { + ctest.DefaultSuite +} + +func TestConfigSuite(t *testing.T) { + suite.Run(t, &ConfigSuite{ + DefaultSuite: ctest.DefaultSuite{ + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(&siderolinkctrl.ConfigController{})) + }, + Timeout: time.Second, + }, + }) +} + +func (suite *ConfigSuite) TestConfig() { + rtestutils.AssertNoResource[*siderolink.Config](suite.Ctx(), suite.T(), suite.State(), siderolink.ConfigID) + + siderolinkConfig := &siderolinkcfg.ConfigV1Alpha1{ + APIUrlConfig: meta.URL{ + URL: must.Value(url.Parse("https://api.sidero.dev"))(suite.T()), + }, + } + + cfg, err := container.New(siderolinkConfig) + suite.Require().NoError(err) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), config.NewMachineConfig(cfg))) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{siderolink.ConfigID}, + func(c *siderolink.Config, assert *assert.Assertions) { + assert.Equal("https://api.sidero.dev", c.TypedSpec().APIEndpoint) + }) +} + +func (suite *ConfigSuite) TestConfigTunnel() { + rtestutils.AssertNoResource[*siderolink.Config](suite.Ctx(), suite.T(), suite.State(), siderolink.ConfigID) + + siderolinkConfig := &siderolinkcfg.ConfigV1Alpha1{ + APIUrlConfig: meta.URL{ + URL: must.Value(url.Parse("https://api.sidero.dev?grpc_tunnel=true"))(suite.T()), + }, + } + + cfg, err := container.New(siderolinkConfig) + suite.Require().NoError(err) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), config.NewMachineConfig(cfg))) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{siderolink.ConfigID}, + func(c *siderolink.Config, assert *assert.Assertions) { + assert.Equal("https://api.sidero.dev?grpc_tunnel=true", c.TypedSpec().APIEndpoint) + assert.True(c.TypedSpec().Tunnel) + }) +} diff --git a/internal/app/machined/pkg/controllers/siderolink/manager.go b/internal/app/machined/pkg/controllers/siderolink/manager.go new file mode 100644 index 0000000..b532172 --- /dev/null +++ b/internal/app/machined/pkg/controllers/siderolink/manager.go @@ -0,0 +1,510 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package siderolink + +import ( + "bytes" + "context" + "crypto/tls" + "errors" + "fmt" + "net/netip" + "os" + "time" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/go-pointer" + pb "github.com/siderolabs/siderolink/api/siderolink" + "github.com/siderolabs/siderolink/pkg/wireguard" + "go.uber.org/zap" + "golang.zx2c4.com/wireguard/wgctrl" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + + networkutils "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network/utils" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/hardware" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/siderolink" + "github.com/siderolabs/talos/pkg/machinery/version" +) + +// ManagerController interacts with SideroLink API and brings up the SideroLink Wireguard interface. +type ManagerController struct { + nodeKey wgtypes.Key + pd provisionData +} + +// Name implements controller.Controller interface. +func (ctrl *ManagerController) Name() string { + return "siderolink.ManagerController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *ManagerController) Inputs() []controller.Input { + return nil +} + +// Outputs implements controller.Controller interface. +func (ctrl *ManagerController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: network.AddressSpecType, + Kind: controller.OutputShared, + }, + { + Type: network.LinkSpecType, + Kind: controller.OutputShared, + }, + { + Type: siderolink.TunnelType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *ManagerController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + // initially, wait for the network address status to be ready + if err := networkutils.WaitForNetworkReady(ctx, r, + func(status *network.StatusSpec) bool { + return status.AddressReady + }, + []controller.Input{ + { + Namespace: config.NamespaceName, + Type: siderolink.ConfigType, + ID: optional.Some(siderolink.ConfigID), + Kind: controller.InputWeak, + }, + { + Namespace: hardware.NamespaceName, + Type: hardware.SystemInformationType, + ID: optional.Some(hardware.SystemInformationID), + Kind: controller.InputWeak, + }, + { + Namespace: runtime.NamespaceName, + Type: runtime.UniqueMachineTokenType, + ID: optional.Some(runtime.UniqueMachineTokenID), + Kind: controller.InputWeak, + }, + }, + ); err != nil { + return fmt.Errorf("error waiting for network: %w", err) + } + + // normal reconcile loop + wgClient, wgClientErr := wgctrl.New() + if wgClientErr != nil { + return wgClientErr + } + + defer func() { + if closeErr := wgClient.Close(); closeErr != nil { + logger.Error("failed to close wg client", zap.Error(closeErr)) + } + }() + + var zeroKey wgtypes.Key + + if bytes.Equal(ctrl.nodeKey[:], zeroKey[:]) { + var err error + + ctrl.nodeKey, err = wgtypes.GeneratePrivateKey() + if err != nil { + return fmt.Errorf("error generating Wireguard key: %w", err) + } + } + + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + reconnect, err := ctrl.shouldReconnect(wgClient) + if err != nil { + return err + } + + if !reconnect { + // nothing to do + continue + } + case <-r.EventCh(): + } + + if ctrl.pd.IsEmpty() { + provision, err := ctrl.provision(ctx, r, logger) + if err != nil { + return fmt.Errorf("error provisioning: %w", err) + } + + if !provision.IsPresent() { + continue + } + + ctrl.pd = provision.ValueOrZero() + } + + serverAddress, err := netip.ParseAddr(ctrl.pd.ServerAddress) + if err != nil { + return fmt.Errorf("error parsing server address: %w", err) + } + + nodeAddress, err := netip.ParsePrefix(ctrl.pd.NodeAddressPrefix) + if err != nil { + return fmt.Errorf("error parsing node address: %w", err) + } + + linkSpec := network.NewLinkSpec(network.ConfigNamespaceName, network.LayeredID(network.ConfigOperator, network.LinkID(constants.SideroLinkName))) + addressSpec := network.NewAddressSpec(network.ConfigNamespaceName, network.LayeredID(network.ConfigOperator, network.AddressID(constants.SideroLinkName, nodeAddress))) + + // Rotate through the endpoints. + ep, ok := ctrl.pd.TakeEndpoint() + if !ok { + return errors.New("host returned no endpoints") + } + + logger.Info( + "configuring siderolink connection", + zap.String("peer_endpoint", ep), + zap.String("next_peer_endpoint", ctrl.pd.PeekNextEndpoint()), + ) + + if err = safe.WriterModify(ctx, r, linkSpec, + func(res *network.LinkSpec) error { + spec := res.TypedSpec() + + spec.ConfigLayer = network.ConfigOperator + spec.Name = constants.SideroLinkName + spec.Type = nethelpers.LinkNone + spec.Kind = "wireguard" + spec.Up = true + spec.Logical = ctrl.pd.grpcPeerAddrPort == "" + spec.MTU = wireguard.LinkMTU + + spec.Wireguard = network.WireguardSpec{ + PrivateKey: ctrl.nodeKey.String(), + Peers: []network.WireguardPeer{ + { + PublicKey: ctrl.pd.ServerPublicKey, + Endpoint: ep, + AllowedIPs: []netip.Prefix{ + netip.PrefixFrom(serverAddress, serverAddress.BitLen()), + }, + // make sure Talos pings SideroLink endpoint, so that tunnel is established: + // SideroLink doesn't know Talos endpoint. + PersistentKeepaliveInterval: constants.SideroLinkDefaultPeerKeepalive, + }, + }, + } + spec.Wireguard.Sort() + + return nil + }); err != nil { + return fmt.Errorf("error creating siderolink spec: %w", err) + } + + if err = safe.WriterModify(ctx, r, addressSpec, + func(res *network.AddressSpec) error { + spec := res.TypedSpec() + + spec.ConfigLayer = network.ConfigOperator + spec.Address = nodeAddress + spec.Family = nethelpers.FamilyInet6 + spec.Flags = nethelpers.AddressFlags(nethelpers.AddressPermanent) + spec.LinkName = constants.SideroLinkName + spec.Scope = nethelpers.ScopeGlobal + + return nil + }); err != nil { + return fmt.Errorf("error creating address spec: %w", err) + } + + if ctrl.pd.grpcPeerAddrPort != "" { + var ourAddr netip.AddrPort + + ourAddr, err = netip.ParseAddrPort(ctrl.pd.grpcPeerAddrPort) + if err != nil { + return err + } + + if err = safe.WriterModify(ctx, r, siderolink.NewTunnel(), + func(tunnel *siderolink.Tunnel) error { + tunnel.TypedSpec().APIEndpoint = ctrl.pd.apiEndpont + tunnel.TypedSpec().LinkName = constants.SideroLinkName + tunnel.TypedSpec().MTU = wireguard.LinkMTU + tunnel.TypedSpec().NodeAddress = ourAddr + + return nil + }, + ); err != nil { + return fmt.Errorf("error creating tunnel spec: %w", err) + } + } else { + if err = r.Destroy(ctx, siderolink.NewTunnel().Metadata()); err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("error destroying tunnel spec: %w", err) + } + } + + keepLinkSpecSet := map[resource.ID]struct{}{ + linkSpec.Metadata().ID(): {}, + } + + keepAddressSpecSet := map[resource.ID]struct{}{ + addressSpec.Metadata().ID(): {}, + } + + if err := ctrl.cleanup(ctx, r, keepLinkSpecSet, keepAddressSpecSet, logger); err != nil { + return err + } + + logger.Info( + "siderolink connection configured", + zap.String("endpoint", ctrl.pd.apiEndpont), + zap.String("node_uuid", ctrl.pd.nodeUUID), + zap.String("node_address", nodeAddress.String()), + ) + } +} + +//nolint:gocyclo +func (ctrl *ManagerController) provision(ctx context.Context, r controller.Runtime, logger *zap.Logger) (optional.Optional[provisionData], error) { + cfg, err := safe.ReaderGetByID[*siderolink.Config](ctx, r, siderolink.ConfigID) + if err != nil { + if state.IsNotFoundError(err) { + if cleanupErr := ctrl.cleanup(ctx, r, nil, nil, logger); cleanupErr != nil { + return optional.None[provisionData](), fmt.Errorf("failed to do cleanup: %w", cleanupErr) + } + + // no config + return optional.None[provisionData](), nil + } + + return optional.None[provisionData](), fmt.Errorf("failed to get siderolink config: %w", err) + } + + sysInfo, err := safe.ReaderGetByID[*hardware.SystemInformation](ctx, r, hardware.SystemInformationID) + if err != nil { + if state.IsNotFoundError(err) { + // no system information + return optional.None[provisionData](), nil + } + + return optional.None[provisionData](), fmt.Errorf("failed to get system information: %w", err) + } + + nodeUUID := sysInfo.TypedSpec().UUID + + provision := func() (*pb.ProvisionResponse, error) { + conn, connErr := grpc.Dial( + cfg.TypedSpec().Host, + withTransportCredentials(cfg.TypedSpec().Insecure), + grpc.WithSharedWriteBuffer(true), + ) + if connErr != nil { + return nil, fmt.Errorf("error dialing SideroLink endpoint %q: %w", cfg.TypedSpec().Host, connErr) + } + + defer func() { + if closeErr := conn.Close(); closeErr != nil { + logger.Error("failed to close SideroLink provisioning GRPC connection", zap.Error(closeErr)) + } + }() + + uniqTokenRes, rdrErr := safe.ReaderGetByID[*runtime.UniqueMachineToken](ctx, r, runtime.UniqueMachineTokenID) + if rdrErr != nil { + return nil, fmt.Errorf("failed to get unique token: %w", rdrErr) + } + + var wgOverGRPC *bool + + if cfg.TypedSpec().Tunnel { + wgOverGRPC = pointer.To(true) + } + + sideroLinkClient := pb.NewProvisionServiceClient(conn) + request := &pb.ProvisionRequest{ + NodeUuid: nodeUUID, + NodePublicKey: ctrl.nodeKey.PublicKey().String(), + NodeUniqueToken: pointer.To(uniqTokenRes.TypedSpec().Token), + TalosVersion: pointer.To(version.Tag), + WireguardOverGrpc: wgOverGRPC, + } + + token := cfg.TypedSpec().JoinToken + + if token != "" { + request.JoinToken = pointer.To(token) + } + + return sideroLinkClient.Provision(ctx, request) + } + + resp, err := provision() + if err != nil { + return optional.None[provisionData](), err + } + + return optional.Some(provisionData{ + nodeUUID: nodeUUID, + apiEndpont: cfg.TypedSpec().APIEndpoint, + ServerAddress: resp.ServerAddress, + ServerPublicKey: resp.ServerPublicKey, + NodeAddressPrefix: resp.NodeAddressPrefix, + endpoints: resp.GetEndpoints(), + grpcPeerAddrPort: resp.GrpcPeerAddrPort, + }), nil +} + +type provisionData struct { + nodeUUID string + apiEndpont string + ServerAddress string + ServerPublicKey string + NodeAddressPrefix string + endpoints []string + grpcPeerAddrPort string +} + +func (d *provisionData) IsEmpty() bool { + return d == nil || len(d.endpoints) == 0 +} + +func (d *provisionData) TakeEndpoint() (string, bool) { + if d.IsEmpty() { + return "", false + } + + ep := d.endpoints[0] + d.endpoints = d.endpoints[1:] + + return ep, true +} + +func (d *provisionData) PeekNextEndpoint() string { + if d.IsEmpty() { + return "" + } + + return d.endpoints[0] +} + +func (ctrl *ManagerController) cleanup( + ctx context.Context, + r controller.Runtime, + keepLinkSpecIDSet, keepAddressSpecIDSet map[resource.ID]struct{}, + logger *zap.Logger, +) error { + if err := ctrl.cleanupLinkSpecs(ctx, r, keepLinkSpecIDSet, logger); err != nil { + return err + } + + return ctrl.cleanupAddressSpecs(ctx, r, keepAddressSpecIDSet, logger) +} + +//nolint:dupl +func (ctrl *ManagerController) cleanupLinkSpecs(ctx context.Context, r controller.Runtime, keepSet map[resource.ID]struct{}, logger *zap.Logger) error { + list, err := safe.ReaderList[*network.LinkSpec](ctx, r, network.NewLinkSpec(network.ConfigNamespaceName, "").Metadata()) + if err != nil { + return err + } + + for iter := list.Iterator(); iter.Next(); { + link := iter.Value() + + if link.Metadata().Owner() != ctrl.Name() { + continue + } + + if _, ok := keepSet[link.Metadata().ID()]; ok { + continue + } + + if destroyErr := r.Destroy(ctx, link.Metadata()); destroyErr != nil && !state.IsNotFoundError(destroyErr) { + return destroyErr + } + + logger.Info("destroyed link spec", zap.String("link_id", link.Metadata().ID())) + } + + return nil +} + +//nolint:dupl +func (ctrl *ManagerController) cleanupAddressSpecs(ctx context.Context, r controller.Runtime, keepSet map[resource.ID]struct{}, logger *zap.Logger) error { + list, err := safe.ReaderList[*network.AddressSpec](ctx, r, network.NewAddressSpec(network.ConfigNamespaceName, "").Metadata()) + if err != nil { + return err + } + + for iter := list.Iterator(); iter.Next(); { + address := iter.Value() + + if address.Metadata().Owner() != ctrl.Name() { + continue + } + + if _, ok := keepSet[address.Metadata().ID()]; ok { + continue + } + + if destroyErr := r.Destroy(ctx, address.Metadata()); destroyErr != nil && !state.IsNotFoundError(destroyErr) { + return destroyErr + } + + logger.Info("destroyed address spec", zap.String("address_id", address.Metadata().ID())) + } + + return nil +} + +func (ctrl *ManagerController) shouldReconnect(wgClient *wgctrl.Client) (bool, error) { + wgDevice, err := wgClient.Device(constants.SideroLinkName) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + // no Wireguard device, so no need to reconnect + return false, nil + } + + return false, fmt.Errorf("error reading Wireguard device: %w", err) + } + + if len(wgDevice.Peers) != 1 { + return false, fmt.Errorf("unexpected number of Wireguard peers: %d", len(wgDevice.Peers)) + } + + peer := wgDevice.Peers[0] + since := time.Since(peer.LastHandshakeTime) + + return since >= wireguard.PeerDownInterval, nil +} + +func withTransportCredentials(insec bool) grpc.DialOption { + var transportCredentials credentials.TransportCredentials + + if insec { + transportCredentials = insecure.NewCredentials() + } else { + transportCredentials = credentials.NewTLS(&tls.Config{}) + } + + return grpc.WithTransportCredentials(transportCredentials) +} diff --git a/internal/app/machined/pkg/controllers/siderolink/manager_test.go b/internal/app/machined/pkg/controllers/siderolink/manager_test.go new file mode 100644 index 0000000..5b7043c --- /dev/null +++ b/internal/app/machined/pkg/controllers/siderolink/manager_test.go @@ -0,0 +1,176 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package siderolink_test + +import ( + "context" + "fmt" + "net" + "net/netip" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-pointer" + "github.com/siderolabs/go-procfs/procfs" + "github.com/siderolabs/go-retry/retry" + pb "github.com/siderolabs/siderolink/api/siderolink" + "github.com/stretchr/testify/suite" + "google.golang.org/grpc" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + siderolinkctrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/siderolink" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/hardware" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/siderolink" +) + +func TestManagerSuite(t *testing.T) { + var m ManagerSuite + m.AfterSetup = func(suite *ctest.DefaultSuite) { + lis, err := net.Listen("tcp", "localhost:0") + suite.Require().NoError(err) + + m.s = grpc.NewServer() + pb.RegisterProvisionServiceServer(m.s, mockServer{}) + + go func() { + suite.Require().NoError(m.s.Serve(lis)) + }() + + cmdline := procfs.NewCmdline(fmt.Sprintf("%s=%s", constants.KernelParamSideroLink, lis.Addr().String())) + configController := siderolinkctrl.ConfigController{Cmdline: cmdline} + + suite.Require().NoError(suite.Runtime().RegisterController(&siderolinkctrl.ManagerController{})) + suite.Require().NoError(suite.Runtime().RegisterController(&configController)) + } + + suite.Run(t, &m) +} + +type ManagerSuite struct { + ctest.DefaultSuite + s *grpc.Server +} + +type mockServer struct { + pb.UnimplementedProvisionServiceServer +} + +const ( + mockServerEndpoint = "127.0.0.11:51820" + mockServerAddress = "fdae:41e4:649b:9303:b6db:d99c:215e:dfc4" + mockServerPublicKey = "2aq/V91QyrHAoH24RK0bldukgo2rWk+wqE5Eg6TArCM=" + mockNodeAddressPrefix = "fdae:41e4:649b:9303:2a07:9c7:5b08:aef7/64" +) + +func (srv mockServer) Provision(_ context.Context, _ *pb.ProvisionRequest) (*pb.ProvisionResponse, error) { + return &pb.ProvisionResponse{ + ServerEndpoint: pb.MakeEndpoints(mockServerEndpoint), + ServerAddress: mockServerAddress, + ServerPublicKey: mockServerPublicKey, + NodeAddressPrefix: mockNodeAddressPrefix, + }, nil +} + +func (suite *ManagerSuite) TestReconcile() { + networkStatus := network.NewStatus(network.NamespaceName, network.StatusID) + networkStatus.TypedSpec().AddressReady = true + + suite.Require().NoError(suite.State().Create(suite.Ctx(), networkStatus)) + + systemInformation := hardware.NewSystemInformation(hardware.SystemInformationID) + systemInformation.TypedSpec().UUID = "71233efd-7a07-43f8-b6ba-da90fae0e88b" + + suite.Require().NoError(suite.State().Create(suite.Ctx(), systemInformation)) + + uniqToken := runtime.NewUniqueMachineToken() + uniqToken.TypedSpec().Token = "random-token" + + suite.Require().NoError(suite.State().Create(suite.Ctx(), uniqToken)) + + nodeAddress := netip.MustParsePrefix(mockNodeAddressPrefix) + + addressSpec := network.NewAddressSpec(network.ConfigNamespaceName, network.LayeredID(network.ConfigOperator, network.AddressID(constants.SideroLinkName, nodeAddress))) + linkSpec := network.NewLinkSpec(network.ConfigNamespaceName, network.LayeredID(network.ConfigOperator, network.LinkID(constants.SideroLinkName))) + + suite.AssertWithin(10*time.Second, 100*time.Millisecond, func() error { + addressResource, err := ctest.Get[*network.AddressSpec](suite, addressSpec.Metadata()) + if err != nil { + if state.IsNotFoundError(err) { + return retry.ExpectedError(err) + } + + return err + } + + address := addressResource.TypedSpec() + + suite.Assert().Equal(nodeAddress, address.Address) + suite.Assert().Equal(network.ConfigOperator, address.ConfigLayer) + suite.Assert().Equal(nethelpers.FamilyInet6, address.Family) + suite.Assert().Equal(constants.SideroLinkName, address.LinkName) + + linkResource, err := ctest.Get[*network.LinkSpec](suite, linkSpec.Metadata()) + if err != nil { + if state.IsNotFoundError(err) { + return retry.ExpectedError(err) + } + + return err + } + + link := linkResource.TypedSpec() + + suite.Assert().Equal("wireguard", link.Kind) + suite.Assert().Equal(network.ConfigOperator, link.ConfigLayer) + suite.Assert().NotEmpty(link.Wireguard.PrivateKey) + suite.Assert().Len(link.Wireguard.Peers, 1) + suite.Assert().Equal(mockServerEndpoint, link.Wireguard.Peers[0].Endpoint) + suite.Assert().Equal(mockServerPublicKey, link.Wireguard.Peers[0].PublicKey) + suite.Assert().Equal( + []netip.Prefix{ + netip.PrefixFrom( + netip.MustParseAddr(mockServerAddress), + 128, + ), + }, link.Wireguard.Peers[0].AllowedIPs, + ) + suite.Assert().Equal( + constants.SideroLinkDefaultPeerKeepalive, + link.Wireguard.Peers[0].PersistentKeepaliveInterval, + ) + + return nil + }) + + // remove config + configPtr := siderolink.NewConfig(config.NamespaceName, siderolink.ConfigID).Metadata() + destroyErr := suite.State().Destroy(suite.Ctx(), configPtr, + state.WithDestroyOwner(pointer.To(siderolinkctrl.ConfigController{}).Name())) + suite.Require().NoError(destroyErr) + + suite.AssertWithin(10*time.Second, 100*time.Millisecond, func() error { + _, err := ctest.Get[*network.LinkSpec](suite, linkSpec.Metadata()) + if err == nil { + return retry.ExpectedErrorf("link resource still exists") + } + + suite.Assert().Truef(state.IsNotFoundError(err), "unexpected error: %v", err) + + _, err = ctest.Get[*network.AddressSpec](suite, addressSpec.Metadata()) + if err == nil { + return retry.ExpectedErrorf("address resource still exists") + } + + suite.Assert().Truef(state.IsNotFoundError(err), "unexpected error: %v", err) + + return nil + }) +} diff --git a/internal/app/machined/pkg/controllers/siderolink/siderolink.go b/internal/app/machined/pkg/controllers/siderolink/siderolink.go new file mode 100644 index 0000000..d34db5f --- /dev/null +++ b/internal/app/machined/pkg/controllers/siderolink/siderolink.go @@ -0,0 +1,6 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package siderolink provides controllers which manage file resources. +package siderolink diff --git a/internal/app/machined/pkg/controllers/siderolink/userspace.go b/internal/app/machined/pkg/controllers/siderolink/userspace.go new file mode 100644 index 0000000..313bbb5 --- /dev/null +++ b/internal/app/machined/pkg/controllers/siderolink/userspace.go @@ -0,0 +1,265 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package siderolink + +import ( + "context" + "fmt" + "net/netip" + "sync" + "time" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "github.com/siderolabs/siderolink/pkg/wgtunnel" + "github.com/siderolabs/siderolink/pkg/wgtunnel/wgbind" + "github.com/siderolabs/siderolink/pkg/wgtunnel/wggrpc" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" + + "github.com/aenix-io/talm/internal/pkg/ctxutil" + "github.com/aenix-io/talm/internal/pkg/endpoint" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/siderolink" +) + +// UserspaceWireguardController imlements a controller that manages a Wireguard over GRPC tunnel in userspace. +type UserspaceWireguardController struct { + RelayRetryTimeout time.Duration + DebugDataStream bool +} + +// Name implements controller.Controller interface. +func (ctrl *UserspaceWireguardController) Name() string { + return "siderolink.UserspaceWireguardController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *UserspaceWireguardController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: siderolink.TunnelType, + ID: optional.Some(siderolink.TunnelID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *UserspaceWireguardController) Outputs() []controller.Output { + return []controller.Output{} +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *UserspaceWireguardController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + eg, ctx := errgroup.WithContext(ctx) + + var ( + relayRetryTimer resettableTimer + tunnelDevice tunnelDeviceProps + tunnelRelay tunnelProps + ) + + defer func() { + tunnelRelay.relay.Close() + tunnelDevice.device.Close() + }() + + const ( + // maxPendingServerMessages is the maximum number of messages that can be pending in the queue before blocking. + maxPendingServerMessages = 100 + // maxPendingClientMessages is the maximum number of messages that can be pending in the ring before being overwritten. + maxPendingClientMessages = 100 + ) + + qp := wgbind.NewQueuePair(maxPendingServerMessages, maxPendingClientMessages) + + for { + select { + case <-ctx.Done(): + return ctxutil.Cause(ctx) + case <-r.EventCh(): + case <-relayRetryTimer.C(): + relayRetryTimer.Clear() + } + + res, err := safe.ReaderGetByID[*siderolink.Tunnel](ctx, r, siderolink.TunnelID) + if err != nil { + if state.IsNotFoundError(err) { + tunnelRelay.relay.Close() + tunnelDevice.device.Close() + + continue + } + + return fmt.Errorf("failed to read link spec: %w", err) + } + + if tunnelDevice.device.IsClosed() { + tunnelDevice.device.Close() + + dev, err := wgtunnel.NewTunnelDevice(res.TypedSpec().LinkName, res.TypedSpec().MTU, qp, ctrl.makeLogger(logger)) + if err != nil { + return fmt.Errorf("failed to create tunnel device: %w", err) + } + + // Store in outer scope because modifying the same variable will lead to the data race below + tunnelDevice = tunnelDeviceProps{device: dev, linkName: res.TypedSpec().LinkName, mtu: res.TypedSpec().MTU} + + logger.Info("wg over grpc tunnel device created", zap.String("link_name", res.TypedSpec().LinkName)) + + eg.Go(func() error { + logger.Debug("tunnel device running") + defer logger.Debug("tunnel device exited") + + return dev.Run() + }) + } + + ep, err := endpoint.Parse(res.TypedSpec().APIEndpoint) + if err != nil { + return fmt.Errorf("failed to parse siderolink API endpoint: %w", err) + } + + dstHost := ep.Host + ourAddrPort := res.TypedSpec().NodeAddress + + if tunnelRelay.relay.IsClosed() || tunnelRelay.dstHost != dstHost || tunnelRelay.ourAddrPort != ourAddrPort { + // Reset timer because we are going to start tunnel anyway + relayRetryTimer.Reset(0) + + tunnelRelay.relay.Close() + + logger.Info( + "updating tunnel relay", + zap.String("old_endpoint", tunnelRelay.dstHost), + zap.Stringer("old_node_address", tunnelRelay.ourAddrPort), + zap.String("new_endpoint", dstHost), + zap.Stringer("new_node_address", ourAddrPort), + ) + + relay, err := wggrpc.NewRelayToHost(dstHost, ctrl.RelayRetryTimeout, qp, ourAddrPort, withTransportCredentials(ep.Insecure)) + if err != nil { + return fmt.Errorf("failed to create tunnel relay: %w", err) + } + + // Store in outer scope because modifying the same variable will lead to the data race below + tunnelRelay = tunnelProps{relay: relay, dstHost: dstHost, ourAddrPort: ourAddrPort} + + eg.Go(func() error { + logger.Debug("running tunnel relay") + + err := relay.Run(ctx, ctrl.makeLogger(logger)) + if err == nil { + logger.Debug("tunnel relay exited gracefully", + zap.String("endpoint", dstHost), + zap.Stringer("node_address", ourAddrPort), + ) + + return nil + } + + // Relay returned an error, close the relay and print the error, device should be kept running. + relay.Close() + + const retryIn = 5 * time.Second + + logger.Error("tunnel relay failed, retrying", + zap.Duration("timeout", retryIn), + zap.String("endpoint", dstHost), + zap.Stringer("node_address", ourAddrPort), + zap.Error(err), + ) + + relayRetryTimer.Reset(retryIn) + + return nil + }) + } + } +} + +// makeLogger ensures that we do not spam like crazy into our ring buffer loggers unless we explicitly want to. +func (ctrl *UserspaceWireguardController) makeLogger(logger *zap.Logger) *zap.Logger { + if ctrl.DebugDataStream { + return logger + } + + return logger.WithOptions(zap.IncreaseLevel(zap.InfoLevel)) +} + +type tunnelProps struct { + relay *wggrpc.Relay + dstHost string + ourAddrPort netip.AddrPort +} + +type tunnelDeviceProps struct { + device *wgtunnel.TunnelDevice + linkName string + mtu int +} + +// resettableTimer wraps time.Timer to allow resetting the timer to any duration. +type resettableTimer struct { + mx sync.Mutex + timer *time.Timer +} + +// Reset resets the timer to the given duration. +// +// If the duration is zero, the timer is removed (and stopped as needed). +// If the duration is non-zero, the timer is created if it doesn't exist, or reset if it does. +func (rt *resettableTimer) Reset(delay time.Duration) { + rt.mx.Lock() + defer rt.mx.Unlock() + + if delay == 0 { + if rt.timer != nil { + if !rt.timer.Stop() { + <-rt.timer.C + } + + rt.timer = nil + } + } else { + if rt.timer == nil { + rt.timer = time.NewTimer(delay) + } else { + if !rt.timer.Stop() { + <-rt.timer.C + } + + rt.timer.Reset(delay) + } + } +} + +// Clear should be called after receiving from the timer channel. +func (rt *resettableTimer) Clear() { + rt.mx.Lock() + defer rt.mx.Unlock() + + rt.timer = nil +} + +// C returns the timer channel. +// +// If the timer was not reset to a non-zero duration, nil is returned. +func (rt *resettableTimer) C() <-chan time.Time { + rt.mx.Lock() + defer rt.mx.Unlock() + + if rt.timer == nil { + return nil + } + + return rt.timer.C +} diff --git a/internal/app/machined/pkg/controllers/time/adjtime_status.go b/internal/app/machined/pkg/controllers/time/adjtime_status.go new file mode 100644 index 0000000..5a1d9fb --- /dev/null +++ b/internal/app/machined/pkg/controllers/time/adjtime_status.go @@ -0,0 +1,97 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package time + +import ( + "context" + "fmt" + stdtime "time" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "go.uber.org/zap" + "golang.org/x/sys/unix" + + v1alpha1runtime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/pkg/timex" + "github.com/siderolabs/talos/pkg/machinery/resources/time" +) + +// AdjtimeStatusController manages time.AdjtimeStatus based on Linux kernel info. +type AdjtimeStatusController struct { + V1Alpha1Mode v1alpha1runtime.Mode +} + +// Name implements controller.Controller interface. +func (ctrl *AdjtimeStatusController) Name() string { + return "time.AdjtimeStatusController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *AdjtimeStatusController) Inputs() []controller.Input { + return nil +} + +// Outputs implements controller.Controller interface. +func (ctrl *AdjtimeStatusController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: time.AdjtimeStatusType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +func (ctrl *AdjtimeStatusController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + if ctrl.V1Alpha1Mode == v1alpha1runtime.ModeContainer { + // in container mode, clock is managed by the host + return nil + } + + const pollInterval = 30 * stdtime.Second + + pollTicker := stdtime.NewTicker(pollInterval) + defer pollTicker.Stop() + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + case <-pollTicker.C: + } + + var timexBuf unix.Timex + + state, err := timex.Adjtimex(&timexBuf) + if err != nil { + return fmt.Errorf("failed to get adjtimex state: %w", err) + } + + scale := stdtime.Nanosecond + + if timexBuf.Status&unix.STA_NANO == 0 { + scale = stdtime.Microsecond + } + + if err := safe.WriterModify(ctx, r, time.NewAdjtimeStatus(), func(status *time.AdjtimeStatus) error { + status.TypedSpec().Offset = stdtime.Duration(timexBuf.Offset) * scale //nolint:durationcheck + status.TypedSpec().FrequencyAdjustmentRatio = 1 + float64(timexBuf.Freq)/65536.0/1000000.0 + status.TypedSpec().MaxError = stdtime.Duration(timexBuf.Maxerror) * stdtime.Microsecond //nolint:durationcheck + status.TypedSpec().EstError = stdtime.Duration(timexBuf.Esterror) * stdtime.Microsecond //nolint:durationcheck + status.TypedSpec().Status = timex.Status(timexBuf.Status).String() + status.TypedSpec().State = state.String() + status.TypedSpec().Constant = int(timexBuf.Constant) + status.TypedSpec().SyncStatus = timexBuf.Status&unix.STA_UNSYNC == 0 + + return nil + }); err != nil { + return fmt.Errorf("failed to update adjtime status: %w", err) + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/time/sync.go b/internal/app/machined/pkg/controllers/time/sync.go new file mode 100644 index 0000000..0afa721 --- /dev/null +++ b/internal/app/machined/pkg/controllers/time/sync.go @@ -0,0 +1,244 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package time + +import ( + "context" + "fmt" + "sync" + stdtime "time" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + v1alpha1runtime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/pkg/ntp" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/time" +) + +// SyncController manages v1alpha1.TimeSync based on configuration and NTP sync process. +type SyncController struct { + V1Alpha1Mode v1alpha1runtime.Mode + NewNTPSyncer NewNTPSyncerFunc + + bootTime stdtime.Time +} + +// Name implements controller.Controller interface. +func (ctrl *SyncController) Name() string { + return "time.SyncController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *SyncController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: network.NamespaceName, + Type: network.TimeServerStatusType, + ID: optional.Some(network.TimeServerID), + Kind: controller.InputWeak, + }, + { + Namespace: config.NamespaceName, + Type: config.MachineConfigType, + ID: optional.Some(config.V1Alpha1ID), + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *SyncController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: time.StatusType, + Kind: controller.OutputExclusive, + }, + } +} + +// NTPSyncer interface is implemented by ntp.Syncer, interface for mocking. +type NTPSyncer interface { + Run(ctx context.Context) + Synced() <-chan struct{} + EpochChange() <-chan struct{} + SetTimeServers([]string) +} + +// NewNTPSyncerFunc function allows to replace ntp.Syncer with the mock. +type NewNTPSyncerFunc func(*zap.Logger, []string) NTPSyncer + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *SyncController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + if ctrl.bootTime.IsZero() { + ctrl.bootTime = stdtime.Now() + } + + if ctrl.NewNTPSyncer == nil { + ctrl.NewNTPSyncer = func(logger *zap.Logger, timeServers []string) NTPSyncer { + return ntp.NewSyncer(logger, timeServers) + } + } + + var ( + syncCtx context.Context + syncCtxCancel context.CancelFunc + syncWg sync.WaitGroup + + syncCh <-chan struct{} + epochCh <-chan struct{} + syncer NTPSyncer + + timeSynced bool + epoch int + + timeSyncTimeoutTimer *stdtime.Timer + timeSyncTimeoutCh <-chan stdtime.Time + ) + + defer func() { + if syncer != nil { + syncCtxCancel() + + syncWg.Wait() + } + + if timeSyncTimeoutTimer != nil { + timeSyncTimeoutTimer.Stop() + } + }() + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + case <-syncCh: + syncCh = nil + timeSynced = true + case <-epochCh: + epoch++ + case <-timeSyncTimeoutCh: + timeSynced = true + timeSyncTimeoutTimer = nil + } + + timeServersStatus, err := r.Get(ctx, resource.NewMetadata(network.NamespaceName, network.TimeServerStatusType, network.TimeServerID, resource.VersionUndefined)) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting time server status: %w", err) + } + + // time server list is not ready yet, wait for the next reconcile + continue + } + + timeServers := timeServersStatus.(*network.TimeServerStatus).TypedSpec().NTPServers + + cfg, err := safe.ReaderGetByID[*config.MachineConfig](ctx, r, config.V1Alpha1ID) + if err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error getting config: %w", err) + } + } + + var syncTimeout stdtime.Duration + + syncDisabled := false + + if ctrl.V1Alpha1Mode == v1alpha1runtime.ModeContainer { + syncDisabled = true + } + + if cfg != nil && cfg.Config().Machine() != nil { + if cfg.Config().Machine().Time().Disabled() { + syncDisabled = true + } + + syncTimeout = cfg.Config().Machine().Time().BootTimeout() + } + + if !timeSynced { + sinceBoot := stdtime.Since(ctrl.bootTime) + + switch { + case syncTimeout == 0: + // disable sync timeout + if timeSyncTimeoutTimer != nil { + timeSyncTimeoutTimer.Stop() + } + + timeSyncTimeoutCh = nil + case sinceBoot > syncTimeout: + // over sync timeout already, so in sync + timeSynced = true + default: + // make sure timer fires in whatever time is left till the timeout + if timeSyncTimeoutTimer == nil || !timeSyncTimeoutTimer.Reset(syncTimeout-sinceBoot) { + timeSyncTimeoutTimer = stdtime.NewTimer(syncTimeout - sinceBoot) + timeSyncTimeoutCh = timeSyncTimeoutTimer.C + } + } + } + + switch { + case syncDisabled && syncer != nil: + // stop syncing + syncCtxCancel() + + syncWg.Wait() + + syncer = nil + syncCh = nil + epochCh = nil + case !syncDisabled && syncer == nil: + // start syncing + syncer = ctrl.NewNTPSyncer(logger, timeServers) + syncCh = syncer.Synced() + epochCh = syncer.EpochChange() + + timeSynced = false + + syncCtx, syncCtxCancel = context.WithCancel(ctx) //nolint:govet + + syncWg.Add(1) + + go func() { + defer syncWg.Done() + + syncer.Run(syncCtx) + }() + } + + if syncer != nil { + syncer.SetTimeServers(timeServers) + } + + if syncDisabled { + timeSynced = true + } + + if err = r.Modify(ctx, time.NewStatus(), func(r resource.Resource) error { + *r.(*time.Status).TypedSpec() = time.StatusSpec{ + Epoch: epoch, + Synced: timeSynced, + SyncDisabled: syncDisabled, + } + + return nil + }); err != nil { + return fmt.Errorf("error updating objects: %w", err) //nolint:govet + } + + r.ResetRestartBackoff() + } +} diff --git a/internal/app/machined/pkg/controllers/time/sync_test.go b/internal/app/machined/pkg/controllers/time/sync_test.go new file mode 100644 index 0000000..7b31cec --- /dev/null +++ b/internal/app/machined/pkg/controllers/time/sync_test.go @@ -0,0 +1,519 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package time_test + +import ( + "context" + "log" + "reflect" + "slices" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-pointer" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + "go.uber.org/zap" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/ctest" + timectrl "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/time" + v1alpha1runtime "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + timeresource "github.com/siderolabs/talos/pkg/machinery/resources/time" + v1alpha1resource "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +type SyncSuite struct { + suite.Suite + + state state.State + + runtime *runtime.Runtime + wg sync.WaitGroup + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc + + syncerMu sync.Mutex + syncer *mockSyncer +} + +func (suite *SyncSuite) State() state.State { return suite.state } + +func (suite *SyncSuite) Ctx() context.Context { return suite.ctx } + +func (suite *SyncSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute) + + suite.state = state.WrapCore(namespaced.NewState(inmem.Build)) + + var err error + + logger := logging.Wrap(log.Writer()) + + suite.runtime, err = runtime.NewRuntime(suite.state, logger) + suite.Require().NoError(err) +} + +func (suite *SyncSuite) startRuntime() { + suite.wg.Add(1) + + go func() { + defer suite.wg.Done() + + suite.Assert().NoError(suite.runtime.Run(suite.ctx)) + }() +} + +func (suite *SyncSuite) assertTimeStatus(spec timeresource.StatusSpec) error { + r, err := suite.state.Get( + suite.ctx, + resource.NewMetadata( + v1alpha1resource.NamespaceName, + timeresource.StatusType, + timeresource.StatusID, + resource.VersionUndefined, + ), + ) + if err != nil { + if state.IsNotFoundError(err) { + return retry.ExpectedError(err) + } + + return err + } + + status := r.(*timeresource.Status) //nolint:errcheck,forcetypeassert + + if *status.TypedSpec() != spec { + return retry.ExpectedErrorf("time status doesn't match: %v != %v", *status.TypedSpec(), spec) + } + + return nil +} + +func (suite *SyncSuite) TestReconcileContainerMode() { + suite.Require().NoError( + suite.runtime.RegisterController( + &timectrl.SyncController{ + V1Alpha1Mode: v1alpha1runtime.ModeContainer, + NewNTPSyncer: suite.newMockSyncer, + }, + ), + ) + + timeServers := network.NewTimeServerStatus(network.NamespaceName, network.TimeServerID) + timeServers.TypedSpec().NTPServers = []string{constants.DefaultNTPServer} + suite.Require().NoError(suite.state.Create(suite.ctx, timeServers)) + + suite.startRuntime() + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertTimeStatus( + timeresource.StatusSpec{ + Synced: true, + Epoch: 0, + SyncDisabled: true, + }, + ) + }, + ), + ) +} + +func (suite *SyncSuite) TestReconcileSyncDisabled() { + suite.Require().NoError( + suite.runtime.RegisterController( + &timectrl.SyncController{ + V1Alpha1Mode: v1alpha1runtime.ModeMetal, + NewNTPSyncer: suite.newMockSyncer, + }, + ), + ) + + suite.startRuntime() + + timeServers := network.NewTimeServerStatus(network.NamespaceName, network.TimeServerID) + timeServers.TypedSpec().NTPServers = []string{constants.DefaultNTPServer} + suite.Require().NoError(suite.state.Create(suite.ctx, timeServers)) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertTimeStatus( + timeresource.StatusSpec{ + Synced: false, + Epoch: 0, + SyncDisabled: false, + }, + ) + }, + ), + ) + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineTime: &v1alpha1.TimeConfig{ + TimeDisabled: pointer.To(true), + }, + }, + ClusterConfig: &v1alpha1.ClusterConfig{}, + }, + ), + ) + + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertTimeStatus( + timeresource.StatusSpec{ + Synced: true, + Epoch: 0, + SyncDisabled: true, + }, + ) + }, + ), + ) +} + +func (suite *SyncSuite) TestReconcileSyncDefaultConfig() { + suite.Require().NoError( + suite.runtime.RegisterController( + &timectrl.SyncController{ + V1Alpha1Mode: v1alpha1runtime.ModeMetal, + NewNTPSyncer: suite.newMockSyncer, + }, + ), + ) + + suite.startRuntime() + + timeServers := network.NewTimeServerStatus(network.NamespaceName, network.TimeServerID) + timeServers.TypedSpec().NTPServers = []string{constants.DefaultNTPServer} + suite.Require().NoError(suite.state.Create(suite.ctx, timeServers)) + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{}, + ClusterConfig: &v1alpha1.ClusterConfig{}, + }, + ), + ) + + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertTimeStatus( + timeresource.StatusSpec{ + Synced: false, + Epoch: 0, + SyncDisabled: false, + }, + ) + }, + ), + ) +} + +func (suite *SyncSuite) TestReconcileSyncChangeConfig() { + suite.Require().NoError( + suite.runtime.RegisterController( + &timectrl.SyncController{ + V1Alpha1Mode: v1alpha1runtime.ModeMetal, + NewNTPSyncer: suite.newMockSyncer, + }, + ), + ) + + suite.startRuntime() + + timeServers := network.NewTimeServerStatus(network.NamespaceName, network.TimeServerID) + timeServers.TypedSpec().NTPServers = []string{constants.DefaultNTPServer} + suite.Require().NoError(suite.state.Create(suite.ctx, timeServers)) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertTimeStatus( + timeresource.StatusSpec{ + Synced: false, + Epoch: 0, + SyncDisabled: false, + }, + ) + }, + ), + ) + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{}, + ClusterConfig: &v1alpha1.ClusterConfig{}, + }, + ), + ) + + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + var mockSyncer *mockSyncer + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + mockSyncer = suite.getMockSyncer() + + if mockSyncer == nil { + return retry.ExpectedErrorf("syncer not created yet") + } + + return nil + }, + ), + ) + + suite.Assert().Equal([]string{constants.DefaultNTPServer}, mockSyncer.getTimeServers()) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertTimeStatus( + timeresource.StatusSpec{ + Synced: false, + Epoch: 0, + SyncDisabled: false, + }, + ) + }, + ), + ) + + close(mockSyncer.syncedCh) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertTimeStatus( + timeresource.StatusSpec{ + Synced: true, + Epoch: 0, + SyncDisabled: false, + }, + ) + }, + ), + ) + + ctest.UpdateWithConflicts(suite, timeServers, func(r *network.TimeServerStatus) error { + r.TypedSpec().NTPServers = []string{"127.0.0.1"} + + return nil + }) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + if !reflect.DeepEqual(mockSyncer.getTimeServers(), []string{"127.0.0.1"}) { + return retry.ExpectedErrorf("time servers not updated yet") + } + + return nil + }, + ), + ) + + mockSyncer.epochCh <- struct{}{} + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertTimeStatus( + timeresource.StatusSpec{ + Synced: true, + Epoch: 1, + SyncDisabled: false, + }, + ) + }, + ), + ) + + ctest.UpdateWithConflicts(suite, cfg, func(r *config.MachineConfig) error { + r.Container().RawV1Alpha1().MachineConfig.MachineTime = &v1alpha1.TimeConfig{ + TimeDisabled: pointer.To(true), + } + + return nil + }) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertTimeStatus( + timeresource.StatusSpec{ + Synced: true, + Epoch: 1, + SyncDisabled: true, + }, + ) + }, + ), + ) +} + +func (suite *SyncSuite) TestReconcileSyncBootTimeout() { + suite.Require().NoError( + suite.runtime.RegisterController( + &timectrl.SyncController{ + V1Alpha1Mode: v1alpha1runtime.ModeMetal, + NewNTPSyncer: suite.newMockSyncer, + }, + ), + ) + + suite.startRuntime() + + timeServers := network.NewTimeServerStatus(network.NamespaceName, network.TimeServerID) + timeServers.TypedSpec().NTPServers = []string{constants.DefaultNTPServer} + suite.Require().NoError(suite.state.Create(suite.ctx, timeServers)) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertTimeStatus( + timeresource.StatusSpec{ + Synced: false, + Epoch: 0, + SyncDisabled: false, + }, + ) + }, + ), + ) + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineTime: &v1alpha1.TimeConfig{ + TimeBootTimeout: 5 * time.Second, + }, + }, + ClusterConfig: &v1alpha1.ClusterConfig{}, + }, + ), + ) + + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertTimeStatus( + timeresource.StatusSpec{ + Synced: true, + Epoch: 0, + SyncDisabled: false, + }, + ) + }, + ), + ) +} + +func (suite *SyncSuite) TearDownTest() { + suite.T().Log("tear down") + + suite.ctxCancel() + + suite.wg.Wait() +} + +func (suite *SyncSuite) newMockSyncer(logger *zap.Logger, servers []string) timectrl.NTPSyncer { + suite.syncerMu.Lock() + defer suite.syncerMu.Unlock() + + suite.syncer = newMockSyncer(logger, servers) + + return suite.syncer +} + +func (suite *SyncSuite) getMockSyncer() *mockSyncer { + suite.syncerMu.Lock() + defer suite.syncerMu.Unlock() + + return suite.syncer +} + +func TestSyncSuite(t *testing.T) { + suite.Run(t, new(SyncSuite)) +} + +type mockSyncer struct { + mu sync.Mutex + + timeServers []string + syncedCh chan struct{} + epochCh chan struct{} +} + +func (mock *mockSyncer) Run(ctx context.Context) { + <-ctx.Done() +} + +func (mock *mockSyncer) Synced() <-chan struct{} { + return mock.syncedCh +} + +func (mock *mockSyncer) EpochChange() <-chan struct{} { + return mock.epochCh +} + +func (mock *mockSyncer) getTimeServers() (servers []string) { + mock.mu.Lock() + defer mock.mu.Unlock() + + return slices.Clone(mock.timeServers) +} + +func (mock *mockSyncer) SetTimeServers(servers []string) { + mock.mu.Lock() + defer mock.mu.Unlock() + + mock.timeServers = slices.Clone(servers) +} + +func newMockSyncer(_ *zap.Logger, servers []string) *mockSyncer { + return &mockSyncer{ + timeServers: slices.Clone(servers), + syncedCh: make(chan struct{}, 1), + epochCh: make(chan struct{}, 1), + } +} diff --git a/internal/app/machined/pkg/controllers/time/time.go b/internal/app/machined/pkg/controllers/time/time.go new file mode 100644 index 0000000..a117764 --- /dev/null +++ b/internal/app/machined/pkg/controllers/time/time.go @@ -0,0 +1,6 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package time contains controllers managing time, synchronization, etc. +package time diff --git a/internal/app/machined/pkg/controllers/utils.go b/internal/app/machined/pkg/controllers/utils.go new file mode 100644 index 0000000..a511e4e --- /dev/null +++ b/internal/app/machined/pkg/controllers/utils.go @@ -0,0 +1,61 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package controllers provides common methods for controller operations. +package controllers + +import ( + "errors" + "fmt" + "os" + "reflect" + + yaml "gopkg.in/yaml.v3" +) + +// LoadOrNewFromFile either loads value from file.yaml or generates new values and saves as file.yaml. +func LoadOrNewFromFile(path string, empty interface{}, generate func(interface{}) error) error { + f, err := os.OpenFile(path, os.O_RDONLY, 0) + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("error reading state file: %w", err) + } + + // file doesn't exist yet, generate new value and save it + if f == nil { + if err = generate(empty); err != nil { + return err + } + + f, err = os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0o600) + if err != nil { + return fmt.Errorf("error creating state file: %w", err) + } + + defer f.Close() //nolint:errcheck + + encoder := yaml.NewEncoder(f) + if err = encoder.Encode(empty); err != nil { + return fmt.Errorf("error marshaling: %w", err) + } + + if err = encoder.Close(); err != nil { + return err + } + + return f.Close() + } + + // read existing cached value + defer f.Close() //nolint:errcheck + + if err = yaml.NewDecoder(f).Decode(empty); err != nil { + return fmt.Errorf("error unmarshaling: %w", err) + } + + if reflect.ValueOf(empty).Elem().IsZero() { + return errors.New("value is still zero after unmarshaling") + } + + return f.Close() +} diff --git a/internal/app/machined/pkg/controllers/v1alpha1/service.go b/internal/app/machined/pkg/controllers/v1alpha1/service.go new file mode 100644 index 0000000..a0062fa --- /dev/null +++ b/internal/app/machined/pkg/controllers/v1alpha1/service.go @@ -0,0 +1,104 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package v1alpha1 + +import ( + "context" + "sync" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "go.uber.org/zap" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +// ServiceController manages v1alpha1.Service based on services subsystem state. +type ServiceController struct { + V1Alpha1Events runtime.Watcher +} + +// Name implements controller.Controller interface. +func (ctrl *ServiceController) Name() string { + return "v1alpha1.ServiceController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *ServiceController) Inputs() []controller.Input { + return nil +} + +// Outputs implements controller.Controller interface. +func (ctrl *ServiceController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: v1alpha1.ServiceType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *ServiceController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + var wg sync.WaitGroup + + wg.Add(1) + + if err := ctrl.V1Alpha1Events.Watch(func(eventCh <-chan runtime.EventInfo) { + defer wg.Done() + + for { + var ( + event runtime.EventInfo + ok bool + ) + + select { + case <-ctx.Done(): + return + case event, ok = <-eventCh: + if !ok { + return + } + } + + if msg, ok := event.Payload.(*machine.ServiceStateEvent); ok { + service := v1alpha1.NewService(msg.Service) + + switch msg.Action { //nolint:exhaustive + case machine.ServiceStateEvent_RUNNING: + if err := r.Modify(ctx, service, func(r resource.Resource) error { + svc := r.(*v1alpha1.Service) //nolint:errcheck,forcetypeassert + + *svc.TypedSpec() = v1alpha1.ServiceSpec{ + Running: true, + Healthy: msg.GetHealth().GetHealthy(), + Unknown: msg.GetHealth().GetUnknown(), + } + + return nil + }); err != nil { + logger.Info("failed creating service resource", zap.String("id", service.Metadata().ID()), zap.Error(err)) + } + default: + if err := r.Destroy(ctx, service.Metadata()); err != nil && !state.IsNotFoundError(err) { + logger.Info("failed destroying service resource", zap.String("id", service.Metadata().ID()), zap.Error(err)) + } + } + } + } + }, runtime.WithTailEvents(-1)); err != nil { + return err + } + + wg.Wait() + + return nil +} diff --git a/internal/app/machined/pkg/controllers/v1alpha1/utils.go b/internal/app/machined/pkg/controllers/v1alpha1/utils.go new file mode 100644 index 0000000..ed75fc6 --- /dev/null +++ b/internal/app/machined/pkg/controllers/v1alpha1/utils.go @@ -0,0 +1,65 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package v1alpha1 + +import ( + "context" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +// WaitForServiceHealthy waits for a service to be healthy. +// +// It is a helper function for controllers. +func WaitForServiceHealthy(ctx context.Context, r controller.Runtime, serviceID string, nextInputs []controller.Input) error { + // set inputs temporarily to a service only + if err := r.UpdateInputs([]controller.Input{ + { + Namespace: v1alpha1.NamespaceName, + Type: v1alpha1.ServiceType, + ID: optional.Some(serviceID), + Kind: controller.InputWeak, + }, + }); err != nil { + return err + } + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-r.EventCh(): + } + + service, err := safe.ReaderGetByID[*v1alpha1.Service](ctx, r, serviceID) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + if service.TypedSpec().Running && service.TypedSpec().Healthy { + // condition met + break + } + } + + // restore inputs + if err := r.UpdateInputs(nextInputs); err != nil { + return err + } + + // queue an update to reprocess with new inputs + r.QueueReconcile() + + return nil +} diff --git a/internal/app/machined/pkg/controllers/v1alpha1/v1alpha1.go b/internal/app/machined/pkg/controllers/v1alpha1/v1alpha1.go new file mode 100644 index 0000000..117a373 --- /dev/null +++ b/internal/app/machined/pkg/controllers/v1alpha1/v1alpha1.go @@ -0,0 +1,6 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package v1alpha1 provides controllers managing v1alpha1 resources. +package v1alpha1 diff --git a/internal/app/machined/pkg/runtime/board.go b/internal/app/machined/pkg/runtime/board.go new file mode 100644 index 0000000..0c957af --- /dev/null +++ b/internal/app/machined/pkg/runtime/board.go @@ -0,0 +1,31 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import "github.com/siderolabs/go-procfs/procfs" + +// PartitionOptions are the board specific options for customizing the +// partition table. +type PartitionOptions struct { + PartitionsOffset uint64 +} + +// BoardInstallOptions are the board specific options for installation of various boot assets. +type BoardInstallOptions struct { + InstallDisk string + MountPrefix string + DTBPath string + UBootPath string + RPiFirmwarePath string + Printf func(string, ...any) +} + +// Board defines the requirements for a SBC. +type Board interface { + Name() string + Install(options BoardInstallOptions) error + KernelArgs() procfs.Parameters + PartitionOptions() *PartitionOptions +} diff --git a/internal/app/machined/pkg/runtime/controller.go b/internal/app/machined/pkg/runtime/controller.go new file mode 100644 index 0000000..6170fba --- /dev/null +++ b/internal/app/machined/pkg/runtime/controller.go @@ -0,0 +1,64 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + "log" + + "github.com/cosi-project/runtime/pkg/controller" +) + +// TaskSetupFunc defines the function that a task will execute for a specific runtime +// mode. +type TaskSetupFunc func(seq Sequence, data any) (TaskExecutionFunc, string) + +// TaskExecutionFunc defines the function that a task will execute for a specific runtime +// mode. +type TaskExecutionFunc func(context.Context, *log.Logger, Runtime) error + +// Phase represents a collection of tasks to be performed concurrently. +type Phase struct { + Name string + Tasks []TaskSetupFunc + CheckFunc func() bool +} + +// LockOptions represents the options for a controller. +type LockOptions struct { + Takeover bool +} + +// LockOption represents an option setter. +type LockOption func(o *LockOptions) error + +// WithTakeover sets the take option to true. +func WithTakeover() LockOption { + return func(o *LockOptions) error { + o.Takeover = true + + return nil + } +} + +// DefaultControllerOptions returns the default controller options. +func DefaultControllerOptions() LockOptions { + return LockOptions{} +} + +// Controller represents the controller responsible for managing the execution +// of sequences. +type Controller interface { + Runtime() Runtime + Sequencer() Sequencer + Run(context.Context, Sequence, interface{}, ...LockOption) error + V1Alpha2() V1Alpha2Controller +} + +// V1Alpha2Controller provides glue into v2alpha1 controller runtime. +type V1Alpha2Controller interface { + Run(context.Context, *Drainer) error + DependencyGraph() (*controller.DependencyGraph, error) +} diff --git a/internal/app/machined/pkg/runtime/disk/disk.go b/internal/app/machined/pkg/runtime/disk/disk.go new file mode 100644 index 0000000..a4b70b3 --- /dev/null +++ b/internal/app/machined/pkg/runtime/disk/disk.go @@ -0,0 +1,6 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package disk contains abstract utility function to filter disks in MachineState.Disk call. +package disk diff --git a/internal/app/machined/pkg/runtime/disk/options.go b/internal/app/machined/pkg/runtime/disk/options.go new file mode 100644 index 0000000..20f5e36 --- /dev/null +++ b/internal/app/machined/pkg/runtime/disk/options.go @@ -0,0 +1,20 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package disk + +// Option defines a function that can alter MachineState.Disk() method output. +type Option func(options *Options) + +// Options contains disk selection options. +type Options struct { + Label string +} + +// WithPartitionLabel select a disk which has the partition labeled. +func WithPartitionLabel(label string) Option { + return func(opts *Options) { + opts.Label = label + } +} diff --git a/internal/app/machined/pkg/runtime/doc.go b/internal/app/machined/pkg/runtime/doc.go new file mode 100644 index 0000000..f55345f --- /dev/null +++ b/internal/app/machined/pkg/runtime/doc.go @@ -0,0 +1,7 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package runtime defines interfaces for accessing runtime specific settings, +// and state. +package runtime diff --git a/internal/app/machined/pkg/runtime/drainer.go b/internal/app/machined/pkg/runtime/drainer.go new file mode 100644 index 0000000..c9d79f4 --- /dev/null +++ b/internal/app/machined/pkg/runtime/drainer.go @@ -0,0 +1,116 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + "errors" + "sync" +) + +// NewDrainer creates new drainer. +func NewDrainer() *Drainer { + return &Drainer{ + shutdown: make(chan struct{}, 1), + } +} + +// Drainer is used in controllers to ensure graceful shutdown. +type Drainer struct { + subscriptionsMu sync.Mutex + draining bool + subscriptions []*DrainSubscription + + shutdown chan struct{} +} + +// Drain initializes drain sequence waits for it to succeed until the context is canceled. +func (d *Drainer) Drain(ctx context.Context) error { + d.subscriptionsMu.Lock() + if d.draining { + d.subscriptionsMu.Unlock() + + return errors.New("already draining") + } + + d.draining = true + + for _, s := range d.subscriptions { + select { + case s.events <- DrainEvent{}: + default: + } + } + d.subscriptionsMu.Unlock() + + for { + d.subscriptionsMu.Lock() + l := len(d.subscriptions) + d.subscriptionsMu.Unlock() + + if l == 0 { + return nil + } + + select { + case <-d.shutdown: + case <-ctx.Done(): + return ctx.Err() + } + } +} + +// Subscribe should be called from a controller that needs graceful shutdown. +func (d *Drainer) Subscribe() *DrainSubscription { + d.subscriptionsMu.Lock() + defer d.subscriptionsMu.Unlock() + + subscription := &DrainSubscription{ + events: make(chan DrainEvent, 1), + drainer: d, + } + + if d.draining { + subscription.events <- DrainEvent{} + } + + d.subscriptions = append(d.subscriptions, subscription) + + return subscription +} + +// DrainSubscription keeps ingoing and outgoing events channels. +type DrainSubscription struct { + drainer *Drainer + events chan DrainEvent +} + +// EventCh returns drain events channel. +func (s *DrainSubscription) EventCh() <-chan DrainEvent { + return s.events +} + +// Cancel the subscription which triggers drain to shutdown. +func (s *DrainSubscription) Cancel() { + s.drainer.subscriptionsMu.Lock() + + for i, sub := range s.drainer.subscriptions { + if sub == s { + s.drainer.subscriptions = append(s.drainer.subscriptions[:i], s.drainer.subscriptions[i+1:]...) + + break + } + } + + s.drainer.subscriptionsMu.Unlock() + + select { + case s.drainer.shutdown <- struct{}{}: + default: + } +} + +// DrainEvent is sent to the events channel when drainer starts the shutdown sequence. +type DrainEvent struct{} diff --git a/internal/app/machined/pkg/runtime/drainer_test.go b/internal/app/machined/pkg/runtime/drainer_test.go new file mode 100644 index 0000000..de00f03 --- /dev/null +++ b/internal/app/machined/pkg/runtime/drainer_test.go @@ -0,0 +1,99 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" +) + +//nolint:gocyclo +func TestDrainer(t *testing.T) { + drainer := runtime.NewDrainer() + + sub1 := drainer.Subscribe() + sub2 := drainer.Subscribe() + + errCh := make(chan error) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + go func() { + errCh <- drainer.Drain(ctx) + }() + + select { + case <-sub1.EventCh(): + case <-time.After(time.Second): + require.Fail(t, "should be notified") + } + + select { + case <-sub2.EventCh(): + case <-time.After(time.Second): + require.Fail(t, "should be notified") + } + + select { + case <-errCh: + require.Fail(t, "shouldn't be drained now") + default: + } + + sub1.Cancel() + + select { + case <-errCh: + require.Fail(t, "shouldn't be drained now") + default: + } + + sub3 := drainer.Subscribe() + + select { + case <-sub3.EventCh(): + case <-time.After(time.Second): + require.Fail(t, "should be notified") + } + + sub3.Cancel() + sub2.Cancel() + + select { + case err := <-errCh: + assert.NoError(t, err) + case <-time.After(time.Second): + require.Fail(t, "should be drained now") + } +} + +func TestDrainTimeout(t *testing.T) { + drainer := runtime.NewDrainer() + + drainer.Subscribe() + + errCh := make(chan error) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + go func() { + errCh <- drainer.Drain(ctx) + }() + + select { + case err := <-errCh: + assert.ErrorIs(t, err, context.DeadlineExceeded) + case <-time.After(5 * time.Second): + require.Fail(t, "should be drained now") + } +} diff --git a/internal/app/machined/pkg/runtime/emergency/emergency.go b/internal/app/machined/pkg/runtime/emergency/emergency.go new file mode 100644 index 0000000..1085c84 --- /dev/null +++ b/internal/app/machined/pkg/runtime/emergency/emergency.go @@ -0,0 +1,19 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package emergency provides values to handle emergency (panic/unrecoverable error) handling for machined. +package emergency + +import ( + "sync/atomic" + + "golang.org/x/sys/unix" +) + +// RebootCmd is a command to reboot the system after an unrecoverable error. +var RebootCmd atomic.Int64 + +func init() { + RebootCmd.Store(unix.LINUX_REBOOT_CMD_RESTART) +} diff --git a/internal/app/machined/pkg/runtime/errors.go b/internal/app/machined/pkg/runtime/errors.go new file mode 100644 index 0000000..033535c --- /dev/null +++ b/internal/app/machined/pkg/runtime/errors.go @@ -0,0 +1,39 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "errors" + "fmt" +) + +var ( + // ErrLocked indicates that the sequencer is currently locked, and processing + // another sequence. + ErrLocked = errors.New("locked") + + // ErrInvalidSequenceData indicates that the sequencer got data the wrong + // data type for a sequence. + ErrInvalidSequenceData = errors.New("invalid sequence data") + + // ErrUndefinedRuntime indicates that the sequencer's runtime is not defined. + ErrUndefinedRuntime = errors.New("undefined runtime") +) + +// RebootError encapsulates unix.Reboot() cmd argument. +type RebootError struct { + Cmd int +} + +func (e RebootError) Error() string { + return fmt.Sprintf("unix.Reboot(%x)", e.Cmd) +} + +// IsRebootError checks whether given error is RebootError. +func IsRebootError(err error) bool { + var rebootErr RebootError + + return errors.As(err, &rebootErr) +} diff --git a/internal/app/machined/pkg/runtime/events.go b/internal/app/machined/pkg/runtime/events.go new file mode 100644 index 0000000..969fcd7 --- /dev/null +++ b/internal/app/machined/pkg/runtime/events.go @@ -0,0 +1,158 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/rs/xid" + "google.golang.org/protobuf/types/known/anypb" + + "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/proto" +) + +// ActorIDCtxKey is the context key used for event actor id. +type ActorIDCtxKey struct{} + +// Event is what is sent on the wire. +type Event struct { + TypeURL string + ID xid.ID + Payload proto.Message + ActorID string +} + +// EventInfo unifies event and queue information for the WatchFunc. +type EventInfo struct { + Event + Backlog int +} + +// WatchFunc defines the watcher callback function. +type WatchFunc func(<-chan EventInfo) + +// WatchOptions defines options for the watch call. +// +// Only one of TailEvents, TailID or TailDuration should be non-zero. +type WatchOptions struct { + // Return that many past events. + // + // If TailEvents is negative, return all the events available. + TailEvents int + // Start at ID > specified. + TailID xid.ID + // Start at timestamp Now() - TailDuration. + TailDuration time.Duration + // ActorID to ID of the actor to filter events by. + ActorID string +} + +// WatchOptionFunc defines the options for the watcher. +type WatchOptionFunc func(opts *WatchOptions) error + +// WithTailEvents sets up Watcher to return specified number of past events. +// +// If number is negative, all the available past events are returned. +func WithTailEvents(number int) WatchOptionFunc { + return func(opts *WatchOptions) error { + if !opts.TailID.IsNil() || opts.TailDuration != 0 { + return errors.New("WithTailEvents can't be specified at the same time with WithTailID or WithTailDuration") + } + + opts.TailEvents = number + + return nil + } +} + +// WithTailID sets up Watcher to return events with ID > TailID. +func WithTailID(id xid.ID) WatchOptionFunc { + return func(opts *WatchOptions) error { + if opts.TailEvents != 0 || opts.TailDuration != 0 { + return errors.New("WithTailID can't be specified at the same time with WithTailEvents or WithTailDuration") + } + + opts.TailID = id + + return nil + } +} + +// WithTailDuration sets up Watcher to return events with timestamp >= (now - tailDuration). +func WithTailDuration(dur time.Duration) WatchOptionFunc { + return func(opts *WatchOptions) error { + if opts.TailEvents != 0 || !opts.TailID.IsNil() { + return errors.New("WithTailDuration can't be specified at the same time with WithTailEvents or WithTailID") + } + + opts.TailDuration = dur + + return nil + } +} + +// WithActorID sets up Watcher to return events filtered by given actor id. +func WithActorID(actorID string) WatchOptionFunc { + return func(opts *WatchOptions) error { + opts.ActorID = actorID + + return nil + } +} + +// Watcher defines a runtime event watcher. +type Watcher interface { + Watch(WatchFunc, ...WatchOptionFunc) error +} + +// Publisher defines a runtime event publisher. +type Publisher interface { + Publish(context.Context, proto.Message) +} + +// EventStream defines the runtime event stream. +type EventStream interface { + Watcher + Publisher +} + +// NewEvent creates a new event with the provided payload and actor ID. +func NewEvent(payload proto.Message, actorID string) Event { + typeURL := "" + if payload != nil { + typeURL = fmt.Sprintf("talos/runtime/%s", payload.ProtoReflect().Descriptor().FullName()) + } + + return Event{ + // In the future, we can publish `talos/runtime`, and + // `talos/plugin/` (or something along those lines) events. + // TypeURL: fmt.Sprintf("talos/runtime/%s", protoreflect.MessageDescriptor.FullName(msg)), + TypeURL: typeURL, + Payload: payload, + ID: xid.New(), + ActorID: actorID, + } +} + +// ToMachineEvent serializes Event as proto message machine.Event. +func (event *Event) ToMachineEvent() (*machine.Event, error) { + value, err := proto.Marshal(event.Payload) + if err != nil { + return nil, err + } + + return &machine.Event{ + Data: &anypb.Any{ + TypeUrl: event.TypeURL, + Value: value, + }, + Id: event.ID.String(), + ActorId: event.ActorID, + }, nil +} diff --git a/internal/app/machined/pkg/runtime/logging.go b/internal/app/machined/pkg/runtime/logging.go new file mode 100644 index 0000000..4836e4a --- /dev/null +++ b/internal/app/machined/pkg/runtime/logging.go @@ -0,0 +1,90 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + "errors" + "io" + "time" + + "go.uber.org/zap/zapcore" +) + +// LoggingManager provides unified interface to publish and consume logs. +type LoggingManager interface { + // ServiceLog privides a log handler for a given service (that may not exist). + ServiceLog(service string) LogHandler + + // SetSenders sets log senders for all derived log handlers + // and returns the previous ones for closing. + // + // SetSenders should be thread-safe. + SetSenders(senders []LogSender) []LogSender + + // RegisteredLogs returns a list of registered logs containers. + RegisteredLogs() []string +} + +// LogOptions for LogHandler.Reader. +type LogOptions struct { + Follow bool + TailLines *int +} + +// LogOption provides functional options for LogHandler.Reader. +type LogOption func(*LogOptions) error + +// WithFollow enables follow mode for the logs. +func WithFollow() LogOption { + return func(o *LogOptions) error { + o.Follow = true + + return nil + } +} + +// WithTailLines starts log reading from lines from the tail of the log. +func WithTailLines(lines int) LogOption { + return func(o *LogOptions) error { + o.TailLines = &lines + + return nil + } +} + +// LogHandler provides interface to access particular log source. +type LogHandler interface { + Writer() (io.WriteCloser, error) + Reader(opt ...LogOption) (io.ReadCloser, error) +} + +// LogEvent represents a log message to be send. +type LogEvent struct { + Msg string + Time time.Time + Level zapcore.Level + Fields map[string]interface{} +} + +// ErrDontRetry indicates that log event should not be resent. +var ErrDontRetry = errors.New("don't retry") + +// LogSender provides common interface for log senders. +type LogSender interface { + // Send tries to send the log event once, exiting on success, error, or context cancelation. + // + // Returned error is nil on success, non-nil otherwise. + // As a special case, Send can return (possibly wrapped) ErrDontRetry if the log event should not be resent + // (if it is invalid, if it was sent partially, etc). + // + // Send should be thread-safe. + Send(ctx context.Context, e *LogEvent) error + + // Close stops the sender gracefully if possible, or forcefully on context cancelation. + // + // Close should be thread-safe. + Close(ctx context.Context) error +} diff --git a/internal/app/machined/pkg/runtime/logging/circular.go b/internal/app/machined/pkg/runtime/logging/circular.go new file mode 100644 index 0000000..ed88cfd --- /dev/null +++ b/internal/app/machined/pkg/runtime/logging/circular.go @@ -0,0 +1,320 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package logging + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "io" + "log" + "sync" + "time" + + "github.com/siderolabs/go-circular" + "github.com/siderolabs/go-debug" + "github.com/siderolabs/go-tail" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" +) + +// These constants should some day move to config. +const ( + // Some logs are tiny, no need to reserve too much memory. + InitialCapacity = 16384 + // Cap each log at 1M. + MaxCapacity = 1048576 + // Safety gap to avoid buffer overruns. + SafetyGap = 2048 +) + +// CircularBufferLoggingManager implements logging to circular fixed size buffer. +type CircularBufferLoggingManager struct { + fallbackLogger *log.Logger + + buffers sync.Map + + sendersRW sync.RWMutex + senders []runtime.LogSender + sendersChanged chan struct{} +} + +// NewCircularBufferLoggingManager initializes new CircularBufferLoggingManager. +func NewCircularBufferLoggingManager(fallbackLogger *log.Logger) *CircularBufferLoggingManager { + return &CircularBufferLoggingManager{ + fallbackLogger: fallbackLogger, + sendersChanged: make(chan struct{}), + } +} + +// ServiceLog implements runtime.LoggingManager interface. +func (manager *CircularBufferLoggingManager) ServiceLog(id string) runtime.LogHandler { + return &circularHandler{ + manager: manager, + id: id, + fields: map[string]interface{}{ + // use field name that is not used by anything else + "talos-service": id, + }, + } +} + +// SetSenders implements runtime.LoggingManager interface. +func (manager *CircularBufferLoggingManager) SetSenders(senders []runtime.LogSender) []runtime.LogSender { + manager.sendersRW.Lock() + + prevChanged := manager.sendersChanged + manager.sendersChanged = make(chan struct{}) + + prevSenders := manager.senders + manager.senders = senders + + manager.sendersRW.Unlock() + + close(prevChanged) + + return prevSenders +} + +// getSenders waits for senders to be set and returns them. +func (manager *CircularBufferLoggingManager) getSenders() []runtime.LogSender { + for { + manager.sendersRW.RLock() + + senders, changed := manager.senders, manager.sendersChanged + + manager.sendersRW.RUnlock() + + if len(senders) > 0 { + return senders + } + + <-changed + } +} + +func (manager *CircularBufferLoggingManager) getBuffer(id string, create bool) (*circular.Buffer, error) { + buf, ok := manager.buffers.Load(id) + if !ok { + if !create { + return nil, nil + } + + b, err := circular.NewBuffer( + circular.WithInitialCapacity(InitialCapacity), + circular.WithMaxCapacity(MaxCapacity), + circular.WithSafetyGap(SafetyGap)) + if err != nil { + return nil, err // only configuration issue might raise error + } + + buf, _ = manager.buffers.LoadOrStore(id, b) + } + + return buf.(*circular.Buffer), nil +} + +// RegisteredLogs implements runtime.LoggingManager interface. +func (manager *CircularBufferLoggingManager) RegisteredLogs() []string { + var result []string + + manager.buffers.Range(func(key, val any) bool { + result = append(result, key.(string)) + + return true + }) + + return result +} + +type circularHandler struct { + manager *CircularBufferLoggingManager + id string + fields map[string]interface{} + + buf *circular.Buffer +} + +type nopCloser struct { + io.Writer +} + +func (nopCloser) Close() error { + return nil +} + +// Writer implements runtime.LogHandler interface. +func (handler *circularHandler) Writer() (io.WriteCloser, error) { + if handler.buf == nil { + var err error + + handler.buf, err = handler.manager.getBuffer(handler.id, true) + if err != nil { + return nil, err + } + + go func() { + if err := handler.runSenders(); err != nil { + handler.manager.fallbackLogger.Printf("log senders stopped: %s", err) + } + }() + } + + switch handler.id { + case "machined": + return &timeStampWriter{w: handler.buf}, nil + default: + return nopCloser{handler.buf}, nil + } +} + +// Reader implements runtime.LogHandler interface. +func (handler *circularHandler) Reader(opts ...runtime.LogOption) (io.ReadCloser, error) { + if handler.buf == nil { + var err error + + handler.buf, err = handler.manager.getBuffer(handler.id, false) + if err != nil { + return nil, err + } + + if handler.buf == nil { + // only Writer() operation creates new buffers + return nil, fmt.Errorf("log %q was not registered", handler.id) + } + } + + var opt runtime.LogOptions + + for _, o := range opts { + if err := o(&opt); err != nil { + return nil, err + } + } + + var r interface { + io.ReadCloser + io.Seeker + } + + if opt.Follow { + r = handler.buf.GetStreamingReader() + } else { + r = handler.buf.GetReader() + } + + if opt.TailLines != nil { + err := tail.SeekLines(r, *opt.TailLines) + if err != nil { + r.Close() //nolint:errcheck + + return nil, fmt.Errorf("error tailing log: %w", err) + } + } + + return r, nil +} + +func (handler *circularHandler) runSenders() error { + r, err := handler.Reader(runtime.WithFollow()) + if err != nil { + return err + } + defer r.Close() //nolint:errcheck + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + l := bytes.TrimSpace(scanner.Bytes()) + if len(l) == 0 { + continue + } + + e := parseLogLine(l, time.Now()) + if e.Fields == nil { + e.Fields = handler.fields + } else { + for k, v := range handler.fields { + e.Fields[k] = v + } + } + + handler.resend(e) + } + + return fmt.Errorf("scanner: %w", scanner.Err()) +} + +// resend sends and resends given event until success or ErrDontRetry error. +func (handler *circularHandler) resend(e *runtime.LogEvent) { + for { + senders := handler.manager.getSenders() + + sendCtx, sendCancel := context.WithTimeout(context.TODO(), 5*time.Second) + sendErrors := make(chan error, len(senders)) + + for _, sender := range senders { + go func() { + sendErrors <- sender.Send(sendCtx, e) + }() + } + + var dontRetry bool + + for range senders { + err := <-sendErrors + + // don't retry if at least one sender succeed to avoid implementing per-sender queue, etc + if err == nil { + dontRetry = true + + continue + } + + if debug.Enabled { + handler.manager.fallbackLogger.Print(err) + } + + if errors.Is(err, runtime.ErrDontRetry) { + dontRetry = true + } + } + + sendCancel() + + if dontRetry { + return + } + + time.Sleep(time.Second) + } +} + +// timeStampWriter is a writer that adds a timestamp to each line. +type timeStampWriter struct { + w io.Writer +} + +// Write implements the io.Writer interface. +func (t *timeStampWriter) Write(p []byte) (int, error) { + // Current log.Logger implementation always adds a newline to the message, so we don't need to wait for it. + var buf bytes.Buffer + + buf.WriteString(time.Now().Format("2006/01/02 15:04:05.000000")) + buf.WriteByte(' ') + buf.Write(p) + + return t.w.Write(buf.Bytes()) +} + +// Close implements the io.Closer interface. +func (t *timeStampWriter) Close() error { + if c, ok := t.w.(io.Closer); ok { + return c.Close() + } + + return nil +} diff --git a/internal/app/machined/pkg/runtime/logging/extract.go b/internal/app/machined/pkg/runtime/logging/extract.go new file mode 100644 index 0000000..4efc869 --- /dev/null +++ b/internal/app/machined/pkg/runtime/logging/extract.go @@ -0,0 +1,124 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package logging + +import ( + "bytes" + "encoding/json" + "math" + "strings" + "time" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" +) + +var maxEpochTS = float64(time.Date(3000, 1, 1, 0, 0, 0, 0, time.UTC).Unix()) + +//nolint:gocyclo +func parseLogLine(l []byte, now time.Time) *runtime.LogEvent { + msg, m := parseJSONLogLine(l) + e := &runtime.LogEvent{ + Msg: msg, + Time: now, + Level: zapcore.InfoLevel, + } + + if m == nil { + return e + } + + for _, k := range []string{"time", "ts"} { + var t time.Time + switch ts := m[k].(type) { + case string: + t, _ = time.Parse(time.RFC3339Nano, ts) //nolint:errcheck + case float64: + // seconds or milliseconds since epoch + sec, fsec := math.Modf(ts) + if sec > maxEpochTS { + sec, fsec = math.Modf(ts / 1000) + } + + t = time.Unix(int64(sec), int64(fsec*float64(time.Second))) + } + + if !t.IsZero() { + e.Time = t.UTC() + + delete(m, k) + + break + } + } + + if levelS, ok := m["level"].(string); ok { + levelS = strings.ToLower(levelS) + + // convert containerd's logrus' level to zap's level + if levelS == "warning" { + levelS = "warn" + } + + var level zapcore.Level + if err := level.UnmarshalText([]byte(levelS)); err == nil { + e.Level = level + + delete(m, "level") + } + } + + if msgS, ok := m["msg"].(string); ok { + // in case we have both message before JSON and "msg" JSON field + if e.Msg != "" { + e.Msg += " " + } + + e.Msg += strings.TrimSpace(msgS) + + delete(m, "msg") + } + + if errS, ok := m["err"].(string); ok { + if e.Level < zap.WarnLevel { + e.Level = zap.WarnLevel + } + + if e.Msg != "" { + e.Msg += ": " + } + + e.Msg += strings.TrimSpace(errS) + + delete(m, "err") + } + + e.Fields = m + + return e +} + +func parseJSONLogLine(l []byte) (msg string, m map[string]interface{}) { + // the whole line is valid JSON + if err := json.Unmarshal(l, &m); err == nil { + return + } + + // the line is a message followed by JSON + if i := bytes.Index(l, []byte("{")); i != -1 { + if err := json.Unmarshal(l[i:], &m); err == nil { + msg = string(bytes.TrimSpace(l[:i])) + + return + } + } + + // no JSON found + msg = string(l) + + return +} diff --git a/internal/app/machined/pkg/runtime/logging/extract_test.go b/internal/app/machined/pkg/runtime/logging/extract_test.go new file mode 100644 index 0000000..41e018a --- /dev/null +++ b/internal/app/machined/pkg/runtime/logging/extract_test.go @@ -0,0 +1,104 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package logging //nolint:testpackage + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap/zapcore" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" +) + +func TestParseLogLine(t *testing.T) { + t.Parallel() + + now := time.Date(2021, 10, 19, 12, 42, 37, 123456789, time.UTC) + + for name, tc := range map[string]struct { + l string + expected *runtime.LogEvent + }{ + "machined": { + l: `[talos] task updateBootloader (1/1): done, 219.885384ms`, + expected: &runtime.LogEvent{ + Msg: `[talos] task updateBootloader (1/1): done, 219.885384ms`, + Time: now, + Level: zapcore.InfoLevel, + }, + }, + "controller-runtime": { + l: `reconfigured wireguard link {"component": "controller-runtime", "controller": "network.LinkSpecController", "link": "kubespan", "peers": 4}`, + expected: &runtime.LogEvent{ + Msg: `reconfigured wireguard link`, + Time: now, + Level: zapcore.InfoLevel, + Fields: map[string]interface{}{ + "component": "controller-runtime", + "controller": "network.LinkSpecController", + "link": "kubespan", + "peers": float64(4), + }, + }, + }, + "etcd-zap": { + l: `{"level":"info","ts":"2021-10-19T14:53:05.815Z","caller":"mvcc/kvstore_compaction.go:57","msg":"finished scheduled compaction","compact-revision":34567,"took":"21.041639ms"}`, + expected: &runtime.LogEvent{ + Msg: `finished scheduled compaction`, + Time: time.Date(2021, 10, 19, 14, 53, 5, 815000000, time.UTC), + Level: zapcore.InfoLevel, + Fields: map[string]interface{}{ + "caller": "mvcc/kvstore_compaction.go:57", + "compact-revision": float64(34567), + "took": "21.041639ms", + }, + }, + }, + "cri-logrus": { + l: `{"level":"warning","msg":"cleanup warnings time=\"2021-10-19T14:52:20Z\" level=info msg=\"starting signal loop\" namespace=k8s.io pid=2629\n","time":"2021-10-19T14:52:20.578858689Z"}`, + expected: &runtime.LogEvent{ + Msg: `cleanup warnings time="2021-10-19T14:52:20Z" level=info msg="starting signal loop" namespace=k8s.io pid=2629`, + Time: time.Date(2021, 10, 19, 14, 52, 20, 578858689, time.UTC), + Level: zapcore.WarnLevel, + Fields: map[string]interface{}{}, + }, + }, + "kubelet": { + l: `{"ts":1635266764792.703,"caller":"topologymanager/scope.go:110","msg":"RemoveContainer","v":0,"containerID":"0194fac91ac1d3949497f6912f3c7e73a062c3bf29b6d3da05557d4db2f8482b"}`, + expected: &runtime.LogEvent{ + Msg: `RemoveContainer`, + Time: time.Date(2021, 10, 26, 16, 46, 4, 792702913, time.UTC), + Level: zapcore.InfoLevel, + Fields: map[string]interface{}{ + "caller": "topologymanager/scope.go:110", + "containerID": "0194fac91ac1d3949497f6912f3c7e73a062c3bf29b6d3da05557d4db2f8482b", + "v": float64(0), + }, + }, + }, + "kubelet-err": { + l: `{"ts":1635266751595.943,"caller":"kubelet/kubelet.go:1703","msg":"Failed creating a mirror pod for",` + + `"pod":"kube-system/kube-controller-manager-talos-dev-qemu-master-1","err":"pods \"kube-controller-manager-talos-dev-qemu-master-1\" already exists"}`, + expected: &runtime.LogEvent{ + Msg: `Failed creating a mirror pod for: pods "kube-controller-manager-talos-dev-qemu-master-1" already exists`, + Time: time.Date(2021, 10, 26, 16, 45, 51, 595943212, time.UTC), + Level: zapcore.WarnLevel, + Fields: map[string]interface{}{ + "caller": "kubelet/kubelet.go:1703", + "pod": "kube-system/kube-controller-manager-talos-dev-qemu-master-1", + }, + }, + }, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + actual := parseLogLine([]byte(tc.l), now) + assert.Equal(t, tc.expected, actual) + }) + } +} diff --git a/internal/app/machined/pkg/runtime/logging/file.go b/internal/app/machined/pkg/runtime/logging/file.go new file mode 100644 index 0000000..f742ff3 --- /dev/null +++ b/internal/app/machined/pkg/runtime/logging/file.go @@ -0,0 +1,129 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package logging + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/siderolabs/gen/containers" + "github.com/siderolabs/go-tail" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/follow" +) + +// FileLoggingManager implements simple logging to files. +type FileLoggingManager struct { + logDirectory string + + registeredLogs containers.ConcurrentMap[string, struct{}] +} + +// NewFileLoggingManager initializes new FileLoggingManager. +func NewFileLoggingManager(logDirectory string) *FileLoggingManager { + return &FileLoggingManager{ + logDirectory: logDirectory, + } +} + +// ServiceLog implements runtime.LoggingManager interface. +func (manager *FileLoggingManager) ServiceLog(id string) runtime.LogHandler { + return &fileLogHandler{ + logDirectory: manager.logDirectory, + id: id, + manager: manager, + } +} + +// SetSenders implements runtime.LoggingManager interface (by doing nothing). +func (manager *FileLoggingManager) SetSenders([]runtime.LogSender) []runtime.LogSender { + return nil +} + +// RegisteredLogs implements runtime.LoggingManager interface. +func (manager *FileLoggingManager) RegisteredLogs() []string { + var result []string + + manager.registeredLogs.ForEach(func(key string, _ struct{}) { + result = append(result, key) + }) + + return result +} + +type fileLogHandler struct { + path string + + logDirectory string + id string + manager *FileLoggingManager +} + +func (handler *fileLogHandler) buildPath() error { + if strings.ContainsAny(handler.id, string(os.PathSeparator)+".") { + return errors.New("service ID is invalid") + } + + handler.path = filepath.Join(handler.logDirectory, handler.id+".log") + + return nil +} + +// Writer implements runtime.LogHandler interface. +func (handler *fileLogHandler) Writer() (io.WriteCloser, error) { + if err := handler.buildPath(); err != nil { + return nil, err + } + + result, err := os.OpenFile(handler.path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o666) + if err != nil { + return nil, err + } + + handler.manager.registeredLogs.GetOrCreate(handler.id, struct{}{}) + + return result, nil +} + +// Reader implements runtime.LogHandler interface. +func (handler *fileLogHandler) Reader(opts ...runtime.LogOption) (io.ReadCloser, error) { + var opt runtime.LogOptions + + for _, o := range opts { + if err := o(&opt); err != nil { + return nil, err + } + } + + if err := handler.buildPath(); err != nil { + return nil, err + } + + f, err := os.OpenFile(handler.path, os.O_RDONLY, 0) + if err != nil { + return nil, err + } + + if opt.TailLines != nil { + err = tail.SeekLines(f, *opt.TailLines) + if err != nil { + f.Close() //nolint:errcheck + + return nil, fmt.Errorf("error tailing log: %w", err) + } + } + + if opt.Follow { + return follow.NewReader(context.Background(), f), nil + } + + return f, nil +} diff --git a/internal/app/machined/pkg/runtime/logging/logging.go b/internal/app/machined/pkg/runtime/logging/logging.go new file mode 100644 index 0000000..cb10633 --- /dev/null +++ b/internal/app/machined/pkg/runtime/logging/logging.go @@ -0,0 +1,6 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package logging provides implementations of runtime.LoggingManager. +package logging diff --git a/internal/app/machined/pkg/runtime/logging/null.go b/internal/app/machined/pkg/runtime/logging/null.go new file mode 100644 index 0000000..97b1cb8 --- /dev/null +++ b/internal/app/machined/pkg/runtime/logging/null.go @@ -0,0 +1,45 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package logging + +import ( + "io" + "os" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" +) + +// NullLoggingManager sends all the logs to /dev/null. +type NullLoggingManager struct{} + +// NewNullLoggingManager initializes NullLoggingManager. +func NewNullLoggingManager() *NullLoggingManager { + return &NullLoggingManager{} +} + +// ServiceLog implements LoggingManager. +func (*NullLoggingManager) ServiceLog(id string) runtime.LogHandler { + return &nullLogHandler{} +} + +// SetSenders implements runtime.LoggingManager interface (by doing nothing). +func (*NullLoggingManager) SetSenders([]runtime.LogSender) []runtime.LogSender { + return nil +} + +// RegisteredLogs implements runtime.LoggingManager interface (by doing nothing). +func (*NullLoggingManager) RegisteredLogs() []string { + return nil +} + +type nullLogHandler struct{} + +func (*nullLogHandler) Writer() (io.WriteCloser, error) { + return os.OpenFile(os.DevNull, os.O_WRONLY, 0) +} + +func (*nullLogHandler) Reader(...runtime.LogOption) (io.ReadCloser, error) { + return os.OpenFile(os.DevNull, os.O_RDONLY, 0) +} diff --git a/internal/app/machined/pkg/runtime/logging/sender_jsonlines.go b/internal/app/machined/pkg/runtime/logging/sender_jsonlines.go new file mode 100644 index 0000000..7c386a0 --- /dev/null +++ b/internal/app/machined/pkg/runtime/logging/sender_jsonlines.go @@ -0,0 +1,144 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package logging + +import ( + "context" + "encoding/json" + "fmt" + "net" + "net/url" + "time" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/config/config" +) + +type jsonLinesSender struct { + endpoint *url.URL + extraTags map[string]string + + sema chan struct{} + conn net.Conn +} + +// NewJSONLines returns log sender that sends logs in JSON over TCP (newline-delimited) +// or UDP (one message per packet). +func NewJSONLines(cfg config.LoggingDestination) runtime.LogSender { + sema := make(chan struct{}, 1) + sema <- struct{}{} + + return &jsonLinesSender{ + endpoint: cfg.Endpoint(), + extraTags: cfg.ExtraTags(), + + sema: sema, + } +} + +func (j *jsonLinesSender) tryLock(ctx context.Context) (unlock func()) { + select { + case <-j.sema: + unlock = func() { j.sema <- struct{}{} } + case <-ctx.Done(): + unlock = nil + } + + return +} + +func (j *jsonLinesSender) marshalJSON(e *runtime.LogEvent) ([]byte, error) { + m := make(map[string]interface{}, len(e.Fields)+3) + for k, v := range e.Fields { + m[k] = v + } + + m["msg"] = e.Msg + m["talos-time"] = e.Time.Format(time.RFC3339Nano) + m["talos-level"] = e.Level.String() + + for k, v := range j.extraTags { + m[k] = v + } + + return json.Marshal(m) +} + +// Send implements LogSender interface. +func (j *jsonLinesSender) Send(ctx context.Context, e *runtime.LogEvent) error { + b, err := j.marshalJSON(e) + if err != nil { + return fmt.Errorf("%w: %s", runtime.ErrDontRetry, err) + } + + if j.endpoint.Scheme == "tcp" { + b = append(b, '\n') + } + + unlock := j.tryLock(ctx) + if unlock == nil { + return ctx.Err() + } + + defer unlock() + + // Connect (or "connect" for UDP) if no connection is established already. + if j.conn == nil { + conn, err := new(net.Dialer).DialContext(ctx, j.endpoint.Scheme, j.endpoint.Host) + if err != nil { + return err + } + + j.conn = conn + } + + d, _ := ctx.Deadline() + j.conn.SetWriteDeadline(d) //nolint:errcheck + + // Close connection on send error. + if n, err := j.conn.Write(b); err != nil { + j.conn.Close() //nolint:errcheck + j.conn = nil + + // skip partially sent events to avoid partial duplicates in the receiver + if n > 0 { + err = fmt.Errorf("%w: %s", runtime.ErrDontRetry, err) + } + + return err + } + + return nil +} + +// Close implements LogSender interface. +func (j *jsonLinesSender) Close(ctx context.Context) error { + unlock := j.tryLock(ctx) + if unlock == nil { + return ctx.Err() + } + + defer unlock() + + if j.conn == nil { + return nil + } + + conn := j.conn + j.conn = nil + + closed := make(chan error, 1) + + go func() { + closed <- conn.Close() + }() + + select { + case <-ctx.Done(): + return ctx.Err() + case err := <-closed: + return err + } +} diff --git a/internal/app/machined/pkg/runtime/logging/sender_jsonlines_test.go b/internal/app/machined/pkg/runtime/logging/sender_jsonlines_test.go new file mode 100644 index 0000000..30bd0df --- /dev/null +++ b/internal/app/machined/pkg/runtime/logging/sender_jsonlines_test.go @@ -0,0 +1,336 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package logging_test + +import ( + "bufio" + "context" + "encoding/json" + "net" + "net/url" + "sync" + "testing" + "time" + + "github.com/siderolabs/gen/channel" + "github.com/siderolabs/gen/ensure" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/logging" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +func udpHandler(ctx context.Context, t *testing.T, conn net.PacketConn, sendCh chan<- []byte) { + t.Helper() + + for { + select { + case <-ctx.Done(): + return + default: + } + + if err := conn.SetReadDeadline(time.Now().Add(10 * time.Millisecond)); err != nil { + t.Logf("failed to set read deadline: %v", err) + + return + } + + buf := make([]byte, 1024) + + n, _, err := conn.ReadFrom(buf) + if err != nil { + if netErr, ok := err.(net.Error); ok && netErr.Timeout() { + continue + } + + t.Logf("failed to read from UDP connection: %v", err) + + return + } + + if !channel.SendWithContext(ctx, sendCh, buf[:n]) { + return + } + } +} + +func tcpHandler(ctx context.Context, t *testing.T, conn net.Listener, sendCh chan<- []byte) { + t.Helper() + + for { + select { + case <-ctx.Done(): + return + default: + } + + if err := conn.(*net.TCPListener).SetDeadline(time.Now().Add(10 * time.Millisecond)); err != nil { + t.Logf("failed to set accept deadline: %v", err) + + return + } + + c, err := conn.Accept() + if err != nil { + if netErr, ok := err.(net.Error); ok && netErr.Timeout() { + continue + } + + t.Logf("failed to accept UDP connection: %v", err) + + return + } + + go func() { + defer c.Close() //nolint:errcheck + + scanner := bufio.NewScanner(c) + + for scanner.Scan() { + if !channel.SendWithContext(ctx, sendCh, scanner.Bytes()) { + return + } + } + }() + } +} + +type loggingDestination struct { + endpoint *url.URL + extraTags map[string]string +} + +func (l *loggingDestination) Endpoint() *url.URL { + return l.endpoint +} + +func (l *loggingDestination) ExtraTags() map[string]string { + return l.extraTags +} + +func (l *loggingDestination) Format() string { + return constants.LoggingFormatJSONLines +} + +func TestSenderJSONLines(t *testing.T) { //nolint:tparallel + t.Parallel() + + lisUDP, err := net.ListenPacket("udp", "127.0.0.1:0") + require.NoError(t, err) + + t.Cleanup(func() { + require.NoError(t, lisUDP.Close()) + }) + + lisTCP, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + + t.Cleanup(func() { + require.NoError(t, lisTCP.Close()) + }) + + udpEndpoint := lisUDP.LocalAddr().String() + tcpEndpoint := lisTCP.Addr().String() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + t.Cleanup(cancel) + + sendCh := make(chan []byte, 32) + + var wg sync.WaitGroup + + wg.Add(1) + + go func() { + defer wg.Done() + + udpHandler(ctx, t, lisUDP, sendCh) + }() + + wg.Add(1) + + go func() { + defer wg.Done() + + tcpHandler(ctx, t, lisTCP, sendCh) + }() + + t.Cleanup(wg.Wait) + + for _, test := range []struct { + name string + + endpoint *url.URL + extraTags map[string]string + + messages []*runtime.LogEvent + + expected []map[string]any + }{ + { + name: "UDP", + + endpoint: ensure.Value(url.Parse("udp://" + udpEndpoint)), + + messages: []*runtime.LogEvent{ + { + Msg: "msg1", + Time: ensure.Value(time.Parse(time.RFC3339Nano, "2021-01-01T00:00:00Z")), + Level: zapcore.InfoLevel, + Fields: map[string]any{ + "field1": "value1", + }, + }, + { + Msg: "msg2", + Time: ensure.Value(time.Parse(time.RFC3339Nano, "2021-01-01T00:00:01Z")), + Level: zapcore.DebugLevel, + }, + }, + + expected: []map[string]any{ + { + "field1": "value1", + "msg": "msg1", + "talos-level": "info", + "talos-time": "2021-01-01T00:00:00Z", + }, + { + "msg": "msg2", + "talos-level": "debug", + "talos-time": "2021-01-01T00:00:01Z", + }, + }, + }, + { + name: "UDP with extra tags", + + endpoint: ensure.Value(url.Parse("udp://" + udpEndpoint)), + extraTags: map[string]string{ + "extra1": "value1", + }, + + messages: []*runtime.LogEvent{ + { + Msg: "msg1", + Time: ensure.Value(time.Parse(time.RFC3339Nano, "2021-01-01T00:00:00Z")), + Level: zapcore.InfoLevel, + Fields: map[string]any{ + "field1": "value1", + }, + }, + { + Msg: "msg2", + Time: ensure.Value(time.Parse(time.RFC3339Nano, "2021-01-01T00:00:01Z")), + Level: zapcore.DebugLevel, + }, + }, + + expected: []map[string]any{ + { + "field1": "value1", + "extra1": "value1", + "msg": "msg1", + "talos-level": "info", + "talos-time": "2021-01-01T00:00:00Z", + }, + { + "msg": "msg2", + "extra1": "value1", + "talos-level": "debug", + "talos-time": "2021-01-01T00:00:01Z", + }, + }, + }, + { + name: "TCP", + + endpoint: ensure.Value(url.Parse("tcp://" + tcpEndpoint)), + + messages: []*runtime.LogEvent{ + { + Msg: "hello", + Time: ensure.Value(time.Parse(time.RFC3339Nano, "2021-01-01T00:00:00Z")), + Level: zapcore.InfoLevel, + Fields: map[string]any{ + "field1": "value1", + }, + }, + }, + + expected: []map[string]any{ + { + "field1": "value1", + "msg": "hello", + "talos-level": "info", + "talos-time": "2021-01-01T00:00:00Z", + }, + }, + }, + { + name: "TCP with extra tags", + + endpoint: ensure.Value(url.Parse("tcp://" + tcpEndpoint)), + extraTags: map[string]string{ + "extra1": "value1", + }, + + messages: []*runtime.LogEvent{ + { + Msg: "hello", + Time: ensure.Value(time.Parse(time.RFC3339Nano, "2021-01-01T00:00:00Z")), + Level: zapcore.InfoLevel, + Fields: map[string]any{ + "field1": "value1", + }, + }, + }, + + expected: []map[string]any{ + { + "field1": "value1", + "extra1": "value1", + "msg": "hello", + "talos-level": "info", + "talos-time": "2021-01-01T00:00:00Z", + }, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + // not parallel - need sequential execution + loggingCfg := &loggingDestination{ + endpoint: test.endpoint, + extraTags: test.extraTags, + } + + sender := logging.NewJSONLines(loggingCfg) + + for _, msg := range test.messages { + require.NoError(t, sender.Send(ctx, msg)) + } + + for _, expected := range test.expected { + select { + case <-time.After(time.Second): + t.Fatalf("timed out waiting for message") + case msg := <-sendCh: + var m map[string]any + + require.NoError(t, json.Unmarshal(msg, &m)) + + require.Equal(t, expected, m) + } + } + + require.NoError(t, sender.Close(ctx)) + }) + } + + cancel() +} diff --git a/internal/app/machined/pkg/runtime/mode.go b/internal/app/machined/pkg/runtime/mode.go new file mode 100644 index 0000000..ec6ca69 --- /dev/null +++ b/internal/app/machined/pkg/runtime/mode.go @@ -0,0 +1,92 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "fmt" +) + +// Mode is a runtime mode. +type Mode int + +// ModeCapability describes mode capability flags. +type ModeCapability uint64 + +const ( + // ModeCloud is the cloud runtime mode. + ModeCloud Mode = iota + // ModeContainer is the container runtime mode. + ModeContainer + // ModeMetal is the metal runtime mode. + ModeMetal +) + +const ( + // Reboot node reboot. + Reboot ModeCapability = 1 << iota + // Rollback node rollback. + Rollback + // Shutdown node shutdown. + Shutdown + // Upgrade node upgrade. + Upgrade + // MetaKV is META partition. + MetaKV +) + +const ( + cloud = "cloud" + container = "container" + metal = "metal" +) + +// String returns the string representation of a Mode. +func (m Mode) String() string { + return [...]string{cloud, container, metal}[m] +} + +// RequiresInstall implements config.RuntimeMode. +func (m Mode) RequiresInstall() bool { + return m == ModeMetal +} + +// InContainer implements config.RuntimeMode. +func (m Mode) InContainer() bool { + return m == ModeContainer +} + +// Supports returns mode capability. +func (m Mode) Supports(feature ModeCapability) bool { + return (m.capabilities() & uint64(feature)) != 0 +} + +// ParseMode returns a `Mode` that matches the specified string. +func ParseMode(s string) (mod Mode, err error) { + switch s { + case cloud: + mod = ModeCloud + case container: + mod = ModeContainer + case metal: + mod = ModeMetal + default: + return mod, fmt.Errorf("unknown runtime mode: %q", s) + } + + return mod, nil +} + +func (m Mode) capabilities() uint64 { + all := ^uint64(0) + + return [...]uint64{ + // metal + all, + // container + all ^ uint64(Reboot|Shutdown|Upgrade|Rollback|MetaKV), + // cloud + all, + }[m] +} diff --git a/internal/app/machined/pkg/runtime/mode_test.go b/internal/app/machined/pkg/runtime/mode_test.go new file mode 100644 index 0000000..1a27565 --- /dev/null +++ b/internal/app/machined/pkg/runtime/mode_test.go @@ -0,0 +1,98 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:scopelint +package runtime_test + +import ( + "reflect" + "testing" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" +) + +func TestMode_String(t *testing.T) { + tests := []struct { + name string + m runtime.Mode + want string + }{ + { + name: "cloud", + m: runtime.ModeCloud, + want: "cloud", + }, + { + name: "container", + m: runtime.ModeContainer, + want: "container", + }, + { + name: "metal", + m: runtime.ModeMetal, + want: "metal", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.m.String(); got != tt.want { + t.Errorf("Mode.String() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestParseMode(t *testing.T) { + type args struct { + s string + } + + tests := []struct { + name string + args args + wantM runtime.Mode + wantErr bool + }{ + { + name: "cloud", + args: args{"cloud"}, + wantM: runtime.ModeCloud, + wantErr: false, + }, + { + name: "container", + args: args{"container"}, + wantM: runtime.ModeContainer, + wantErr: false, + }, + { + name: "metal", + args: args{"metal"}, + wantM: runtime.ModeMetal, + wantErr: false, + }, + { + name: "invalid", + args: args{"invalid"}, + wantM: 0, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotM, err := runtime.ParseMode(tt.args.s) + if (err != nil) != tt.wantErr { + t.Errorf("ParseMode() error = %v, wantErr %v", err, tt.wantErr) + + return + } + + if !reflect.DeepEqual(gotM, tt.wantM) { + t.Errorf("ParseMode() = %v, want %v", gotM, tt.wantM) + } + }) + } +} diff --git a/internal/app/machined/pkg/runtime/platform.go b/internal/app/machined/pkg/runtime/platform.go new file mode 100644 index 0000000..70853c5 --- /dev/null +++ b/internal/app/machined/pkg/runtime/platform.go @@ -0,0 +1,63 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + "net/netip" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-procfs/procfs" + + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// Platform defines the requirements for a platform. +type Platform interface { + // Name returns platform name. + Name() string + + // Mode returns platform mode (metal, cloud or container). + Mode() Mode + + // Configuration fetches the machine configuration from platform-specific location. + // + // On cloud-like platform it is user-data in metadata service. + // For metal platform that is either `talos.config=` URL or mounted ISO image. + Configuration(context.Context, state.State) ([]byte, error) + + // KernelArgs returns additional kernel arguments which should be injected for the kernel boot. + KernelArgs(arch string) procfs.Parameters + + // NetworkConfiguration fetches network configuration from the platform metadata. + // + // Controller will run this in function a separate goroutine, restarting it + // on error. Platform is expected to deliver network configuration over the channel, + // including updates to the configuration over time. + NetworkConfiguration(context.Context, state.State, chan<- *PlatformNetworkConfig) error +} + +// PlatformNetworkConfig describes the network configuration produced by the platform. +// +// This structure is marshaled to STATE partition to persist cached network configuration across +// reboots. +type PlatformNetworkConfig struct { + Addresses []network.AddressSpecSpec `yaml:"addresses"` + Links []network.LinkSpecSpec `yaml:"links"` + Routes []network.RouteSpecSpec `yaml:"routes"` + + Hostnames []network.HostnameSpecSpec `yaml:"hostnames"` + Resolvers []network.ResolverSpecSpec `yaml:"resolvers"` + TimeServers []network.TimeServerSpecSpec `yaml:"timeServers"` + + Operators []network.OperatorSpecSpec `yaml:"operators"` + + ExternalIPs []netip.Addr `yaml:"externalIPs"` + + Probes []network.ProbeSpecSpec `yaml:"probes,omitempty"` + + Metadata *runtime.PlatformMetadataSpec `yaml:"metadata,omitempty"` +} diff --git a/internal/app/machined/pkg/runtime/runtime.go b/internal/app/machined/pkg/runtime/runtime.go new file mode 100644 index 0000000..5db0066 --- /dev/null +++ b/internal/app/machined/pkg/runtime/runtime.go @@ -0,0 +1,29 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + "time" + + "github.com/siderolabs/talos/pkg/machinery/config" + "github.com/siderolabs/talos/pkg/machinery/resources/hardware" +) + +// Runtime defines the runtime parameters. +type Runtime interface { //nolint:interfacebloat + Config() config.Config + ConfigContainer() config.Container + RollbackToConfigAfter(time.Duration) error + CancelConfigRollbackTimeout() + SetConfig(config.Provider) error + CanApplyImmediate(config.Provider) error + State() State + Events() EventStream + Logging() LoggingManager + NodeName() (string, error) + IsBootstrapAllowed() bool + GetSystemInformation(ctx context.Context) (*hardware.SystemInformation, error) +} diff --git a/internal/app/machined/pkg/runtime/sequencer.go b/internal/app/machined/pkg/runtime/sequencer.go new file mode 100644 index 0000000..9288fb0 --- /dev/null +++ b/internal/app/machined/pkg/runtime/sequencer.go @@ -0,0 +1,166 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "fmt" + + "github.com/siderolabs/talos/pkg/machinery/api/machine" +) + +// Sequence represents a sequence type. +type Sequence int + +const ( + // SequenceNoop is the noop sequence. + SequenceNoop Sequence = iota + // SequenceBoot is the boot sequence. + SequenceBoot + // SequenceInitialize is the initialize sequence. + SequenceInitialize + // SequenceInstall is the install sequence. + SequenceInstall + // SequenceShutdown is the shutdown sequence. + SequenceShutdown + // SequenceUpgrade is the upgrade sequence. + SequenceUpgrade + // SequenceStageUpgrade is the stage upgrade sequence. + SequenceStageUpgrade + // SequenceMaintenanceUpgrade is the upgrade sequence in maintenance mode. + SequenceMaintenanceUpgrade + // SequenceReset is the reset sequence. + SequenceReset + // SequenceReboot is the reboot sequence. + SequenceReboot +) + +const ( + boot = "boot" + initialize = "initialize" + install = "install" + shutdown = "shutdown" + upgrade = "upgrade" + stageUpgrade = "stageUpgrade" + maintenanceUpgrade = "maintenanceUpgrade" + reset = "reset" + reboot = "reboot" + noop = "noop" +) + +var sequenceTakeOver = map[Sequence]map[Sequence]struct{}{ + SequenceInitialize: { + SequenceMaintenanceUpgrade: {}, + }, + SequenceBoot: { + SequenceReboot: {}, + SequenceReset: {}, + SequenceUpgrade: {}, + }, + SequenceReboot: { + SequenceReboot: {}, + }, + SequenceReset: { + SequenceReboot: {}, + }, +} + +// String returns the string representation of a `Sequence`. +func (s Sequence) String() string { + return [...]string{noop, boot, initialize, install, shutdown, upgrade, stageUpgrade, maintenanceUpgrade, reset, reboot}[s] +} + +// CanTakeOver defines sequences priority. +// +// | what is running (columns) what is requested (rows) | boot | reboot | reset | upgrade | +// |----------------------------------------------------|------|--------|-------|---------| +// | reboot | Y | Y | Y | N | +// | reset | Y | N | N | N | +// | upgrade | Y | N | N | N |. +func (s Sequence) CanTakeOver(running Sequence) bool { + if running == SequenceNoop { + return true + } + + if sequences, ok := sequenceTakeOver[running]; ok { + if _, ok = sequences[s]; ok { + return true + } + } + + return false +} + +// ParseSequence returns a `Sequence` that matches the specified string. +// +//nolint:gocyclo +func ParseSequence(s string) (seq Sequence, err error) { + switch s { + case boot: + seq = SequenceBoot + case initialize: + seq = SequenceInitialize + case install: + seq = SequenceInstall + case shutdown: + seq = SequenceShutdown + case upgrade: + seq = SequenceUpgrade + case stageUpgrade: + seq = SequenceStageUpgrade + case maintenanceUpgrade: + seq = SequenceMaintenanceUpgrade + case reset: + seq = SequenceReset + case reboot: + seq = SequenceReboot + case noop: + seq = SequenceNoop + default: + return seq, fmt.Errorf("unknown runtime sequence: %q", s) + } + + return seq, nil +} + +// ResetOptions are parameters to Reset sequence. +type ResetOptions interface { + GetGraceful() bool + GetReboot() bool + GetMode() machine.ResetRequest_WipeMode + GetUserDisksToWipe() []string + GetSystemDiskTargets() []PartitionTarget +} + +// PartitionTarget provides interface to the disk partition. +type PartitionTarget interface { + fmt.Stringer + Format(func(string, ...any)) error + GetLabel() string +} + +// Sequencer describes the set of sequences required for the lifecycle +// management of the operating system. +type Sequencer interface { + Boot(Runtime) []Phase + Initialize(Runtime) []Phase + Install(Runtime) []Phase + Reboot(Runtime) []Phase + Reset(Runtime, ResetOptions) []Phase + Shutdown(Runtime, *machine.ShutdownRequest) []Phase + StageUpgrade(Runtime, *machine.UpgradeRequest) []Phase + Upgrade(Runtime, *machine.UpgradeRequest) []Phase + MaintenanceUpgrade(Runtime, *machine.UpgradeRequest) []Phase +} + +// EventSequenceStart represents the sequence start event. +type EventSequenceStart struct { + Sequence Sequence +} + +// EventFatalSequencerError represents a fatal sequencer error. +type EventFatalSequencerError struct { + Error error + Sequence Sequence +} diff --git a/internal/app/machined/pkg/runtime/sequencer_test.go b/internal/app/machined/pkg/runtime/sequencer_test.go new file mode 100644 index 0000000..66de55c --- /dev/null +++ b/internal/app/machined/pkg/runtime/sequencer_test.go @@ -0,0 +1,141 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:scopelint +package runtime_test + +import ( + "testing" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" +) + +func TestSequence_String(t *testing.T) { + tests := []struct { + name string + s runtime.Sequence + want string + }{ + { + name: "boot", + s: runtime.SequenceBoot, + want: "boot", + }, + { + name: "initialize", + s: runtime.SequenceInitialize, + want: "initialize", + }, + { + name: "shutdown", + s: runtime.SequenceShutdown, + want: "shutdown", + }, + { + name: "upgrade", + s: runtime.SequenceUpgrade, + want: "upgrade", + }, + { + name: "stageUpgrade", + s: runtime.SequenceStageUpgrade, + want: "stageUpgrade", + }, + { + name: "reboot", + s: runtime.SequenceReboot, + want: "reboot", + }, + { + name: "reset", + s: runtime.SequenceReset, + want: "reset", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.s.String(); got != tt.want { + t.Errorf("Sequence.String() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestParseSequence(t *testing.T) { + type args struct { + s string + } + + tests := []struct { + name string + args args + wantSeq runtime.Sequence + wantErr bool + }{ + { + name: "boot", + args: args{"boot"}, + wantSeq: runtime.SequenceBoot, + wantErr: false, + }, + { + name: "initialize", + args: args{"initialize"}, + wantSeq: runtime.SequenceInitialize, + wantErr: false, + }, + { + name: "shutdown", + args: args{"shutdown"}, + wantSeq: runtime.SequenceShutdown, + wantErr: false, + }, + { + name: "upgrade", + args: args{"upgrade"}, + wantSeq: runtime.SequenceUpgrade, + wantErr: false, + }, + { + name: "stageUpgrade", + args: args{"stageUpgrade"}, + wantSeq: runtime.SequenceStageUpgrade, + wantErr: false, + }, + { + name: "reboot", + args: args{"reboot"}, + wantSeq: runtime.SequenceReboot, + wantErr: false, + }, + { + name: "reset", + args: args{"reset"}, + wantSeq: runtime.SequenceReset, + wantErr: false, + }, + { + name: "invalid", + args: args{"invalid"}, + wantSeq: 0, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotSeq, err := runtime.ParseSequence(tt.args.s) + if (err != nil) != tt.wantErr { + t.Errorf("ParseSequence() error = %v, wantErr %v", err, tt.wantErr) + + return + } + + if gotSeq != tt.wantSeq { + t.Errorf("ParseSequence() = %v, want %v", gotSeq, tt.wantSeq) + } + }) + } +} diff --git a/internal/app/machined/pkg/runtime/state.go b/internal/app/machined/pkg/runtime/state.go new file mode 100644 index 0000000..113e7a5 --- /dev/null +++ b/internal/app/machined/pkg/runtime/state.go @@ -0,0 +1,76 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime + +import ( + "context" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/registry" + "github.com/siderolabs/go-blockdevice/blockdevice/probe" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/disk" + configcore "github.com/siderolabs/talos/pkg/machinery/config" + "github.com/siderolabs/talos/pkg/machinery/config/config" +) + +// State defines the state. +type State interface { + Platform() Platform + Machine() MachineState + Cluster() ClusterState + V1Alpha2() V1Alpha2State +} + +// Machine defines the runtime parameters. +type Machine interface { + State() MachineState + Config() config.MachineConfig +} + +// MachineState defines the machined state. +type MachineState interface { + Disk(options ...disk.Option) *probe.ProbedBlockDevice + Close() error + Installed() bool + IsInstallStaged() bool + StagedInstallImageRef() string + StagedInstallOptions() []byte + KexecPrepared(bool) + IsKexecPrepared() bool + DBus() DBusState + Meta() Meta +} + +// Meta defines the access to META partition. +type Meta interface { + ReadTag(t uint8) (val string, ok bool) + ReadTagBytes(t uint8) (val []byte, ok bool) + SetTag(ctx context.Context, t uint8, val string) (bool, error) + SetTagBytes(ctx context.Context, t uint8, val []byte) (bool, error) + DeleteTag(ctx context.Context, t uint8) (bool, error) + Reload(ctx context.Context) error + Flush() error +} + +// ClusterState defines the cluster state. +type ClusterState interface{} + +// V1Alpha2State defines the next generation (v2) interface binding into v1 runtime. +type V1Alpha2State interface { + Resources() state.State + + NamespaceRegistry() *registry.NamespaceRegistry + ResourceRegistry() *registry.ResourceRegistry + + SetConfig(configcore.Provider) error +} + +// DBusState defines the D-Bus logind mock. +type DBusState interface { + Start() error + Stop() error + WaitShutdown(ctx context.Context) error +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/acpi/acpi.go b/internal/app/machined/pkg/runtime/v1alpha1/acpi/acpi.go new file mode 100644 index 0000000..6e84316 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/acpi/acpi.go @@ -0,0 +1,108 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package acpi + +import ( + "errors" + "fmt" + "log" + "os" + "strings" + + "github.com/hashicorp/go-multierror" + "github.com/mdlayher/genetlink" + "github.com/mdlayher/netlink" +) + +const ( + // PowerButtonEvent is the ACPI event name associated with the power off + // button. + PowerButtonEvent = "button/power" + // See https://github.com/torvalds/linux/blob/master/drivers/acpi/event.c + acpiGenlFamilyName = "acpi_event" + acpiGenlMcastGroupName = "acpi_mc_group" +) + +// StartACPIListener starts listening for ACPI netlink events. +// +//nolint:gocyclo +func StartACPIListener() (err error) { + // Get the acpi_event family. + conn, err := genetlink.Dial(nil) + if err != nil { + return err + } + + f, err := conn.GetFamily(acpiGenlFamilyName) + if errors.Is(err, os.ErrNotExist) { + //nolint:errcheck + conn.Close() + + return fmt.Errorf(acpiGenlFamilyName+" not available: %w", err) + } + + var id uint32 + + for _, group := range f.Groups { + if group.Name == acpiGenlMcastGroupName { + id = group.ID + } + } + + if err = conn.JoinGroup(id); err != nil { + //nolint:errcheck + conn.Close() + + return err + } + + //nolint:errcheck + defer conn.Close() + + for { + msgs, _, err := conn.Receive() + if err != nil { + return fmt.Errorf("error reading from ACPI channel: %w", err) + } + + if len(msgs) > 0 { + ok, err := parse(msgs, PowerButtonEvent) + if err != nil { + log.Printf("failed to parse netlink message: %v", err) + + continue + } + + if !ok { + continue + } + + return nil + } + } +} + +func parse(msgs []genetlink.Message, event string) (bool, error) { + var result *multierror.Error + + for _, msg := range msgs { + ad, err := netlink.NewAttributeDecoder(msg.Data) + if err != nil { + result = multierror.Append(result, fmt.Errorf("failed to create attribute decoder: %w", err)) + + continue + } + + for ad.Next() { + if strings.HasPrefix(ad.String(), event) { + return true, nil + } + + log.Printf("ignoring ACPI event: %q", ad.String()) + } + } + + return false, result.ErrorOrNil() +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/acpi/acpi_test.go b/internal/app/machined/pkg/runtime/v1alpha1/acpi/acpi_test.go new file mode 100644 index 0000000..2dd9c2e --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/acpi/acpi_test.go @@ -0,0 +1,76 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:scopelint,testpackage +package acpi + +import ( + "testing" + + "github.com/mdlayher/genetlink" +) + +func Test_parse(t *testing.T) { + type args struct { + msgs []genetlink.Message + event string + } + + tests := []struct { + name string + args args + want bool + wantErr bool + }{ + { + name: PowerButtonEvent, + args: args{ + msgs: []genetlink.Message{ + { + Header: genetlink.Header{ + Command: 1, + Version: 1, + }, + Data: []byte{48, 0, 1, 0, 98, 117, 116, 116, 111, 110, 47, 112, 111, 119, 101, 114, 0, 0, 0, 0, 0, 0, 0, 0, 76, 78, 88, 80, 87, 82, 66, 78, 58, 48, 48, 0, 0, 0, 0, 0, 128, 0, 0, 0, 1, 0, 0, 0}, + }, + }, + event: PowerButtonEvent, + }, + want: true, + wantErr: false, + }, + { + name: "battery", + args: args{ + msgs: []genetlink.Message{ + { + Header: genetlink.Header{ + Command: 1, + Version: 1, + }, + Data: []byte{48, 0, 1, 0, 98, 117, 116, 116, 111, 110, 47, 112, 111, 119, 101, 114, 0, 0, 0, 0, 0, 0, 0, 0, 76, 78, 88, 80, 87, 82, 66, 78, 58, 48, 48, 0, 0, 0, 0, 0, 128, 0, 0, 0, 1, 0, 0, 0}, + }, + }, + event: "battery", + }, + want: false, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parse(tt.args.msgs, tt.args.event) + if (err != nil) != tt.wantErr { + t.Errorf("parse() error = %v, wantErr %v", err, tt.wantErr) + + return + } + + if got != tt.want { + t.Errorf("parse() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/board/bananapi_m64/bananapi_m64.go b/internal/app/machined/pkg/runtime/v1alpha1/board/bananapi_m64/bananapi_m64.go new file mode 100644 index 0000000..0610aaf --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/board/bananapi_m64/bananapi_m64.go @@ -0,0 +1,102 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package bananapim64 provides the Banana Pi M64 board implementation. +package bananapim64 + +import ( + "os" + "path/filepath" + + "github.com/siderolabs/go-copy/copy" + "github.com/siderolabs/go-procfs/procfs" + "golang.org/x/sys/unix" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +var ( + bin = constants.BoardBananaPiM64 + "/u-boot-sunxi-with-spl.bin" + off int64 = 1024 * 8 + dtb = "allwinner/sun50i-a64-bananapi-m64.dtb" +) + +// BananaPiM64 represents the Banana Pi M64. +// +// References: +// - http://www.banana-pi.org/m64.html +// - http://wiki.banana-pi.org/Banana_Pi_BPI-M64 +// - https://linux-sunxi.org/Banana_Pi_M64 +type BananaPiM64 struct{} + +// Name implements the runtime.Board. +func (b *BananaPiM64) Name() string { + return constants.BoardBananaPiM64 +} + +// Install implements the runtime.Board. +func (b *BananaPiM64) Install(options runtime.BoardInstallOptions) (err error) { + var f *os.File + + if f, err = os.OpenFile(options.InstallDisk, os.O_RDWR|unix.O_CLOEXEC, 0o666); err != nil { + return err + } + //nolint:errcheck + defer f.Close() + + var uboot []byte + + uboot, err = os.ReadFile(filepath.Join(options.UBootPath, bin)) + if err != nil { + return err + } + + options.Printf("writing %s at offset %d", bin, off) + + var n int + + n, err = f.WriteAt(uboot, off) + if err != nil { + return err + } + + options.Printf("wrote %d bytes", n) + + // NB: In the case that the block device is a loopback device, we sync here + // to esure that the file is written before the loopback device is + // unmounted. + err = f.Sync() + if err != nil { + return err + } + + src := filepath.Join(options.DTBPath, dtb) + dst := filepath.Join(options.MountPrefix, "/boot/EFI/dtb", dtb) + + err = os.MkdirAll(filepath.Dir(dst), 0o600) + if err != nil { + return err + } + + err = copy.File(src, dst) + if err != nil { + return err + } + + return nil +} + +// KernelArgs implements the runtime.Board. +func (b *BananaPiM64) KernelArgs() procfs.Parameters { + return []*procfs.Parameter{ + procfs.NewParameter("console").Append("tty0").Append("ttyS0,115200"), + procfs.NewParameter(constants.KernelParamDashboardDisabled).Append("1"), + } +} + +// PartitionOptions implements the runtime.Board. +func (b *BananaPiM64) PartitionOptions() *runtime.PartitionOptions { + return &runtime.PartitionOptions{PartitionsOffset: 2048} +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/board/board.go b/internal/app/machined/pkg/runtime/v1alpha1/board/board.go new file mode 100644 index 0000000..c83a9a0 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/board/board.go @@ -0,0 +1,56 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package board provides the function to discover the current board. +package board + +import ( + "fmt" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + bananapim64 "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/board/bananapi_m64" + jetsonnano "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/board/jetson_nano" + libretechallh3cch5 "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/board/libretech_all_h3_cc_h5" + nanopir4s "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/board/nanopi_r4s" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/board/pine64" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/board/rock64" + rockpi4 "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/board/rockpi4" + rockpi4c "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/board/rockpi4c" + rpigeneric "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/board/rpi_generic" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// NewBoard initializes and returns a runtime.Board. +// Deprecated: Not supported anymore, use overlays instead. +func NewBoard(board string) (b runtime.Board, err error) { + return newBoard(board) +} + +//gocyclo:ignore +func newBoard(board string) (b runtime.Board, err error) { + switch board { + case constants.BoardLibretechAllH3CCH5: + b = &libretechallh3cch5.LibretechAllH3CCH5{} + case constants.BoardRPiGeneric: + b = &rpigeneric.RPiGeneric{} + case constants.BoardBananaPiM64: + b = &bananapim64.BananaPiM64{} + case constants.BoardPine64: + b = &pine64.Pine64{} + case constants.BoardRock64: + b = &rock64.Rock64{} + case constants.BoardRockpi4: + b = &rockpi4.Rockpi4{} + case constants.BoardRockpi4c: + b = &rockpi4c.Rockpi4c{} + case constants.BoardJetsonNano: + b = &jetsonnano.JetsonNano{} + case constants.BoardNanoPiR4S: + b = &nanopir4s.NanoPiR4S{} + default: + return nil, fmt.Errorf("unsupported board: %q", board) + } + + return b, nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/board/jetson_nano/jetson_nano.go b/internal/app/machined/pkg/runtime/v1alpha1/board/jetson_nano/jetson_nano.go new file mode 100644 index 0000000..69d63f3 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/board/jetson_nano/jetson_nano.go @@ -0,0 +1,90 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package jetsonnano provides the Jetson Nano board implementation. +package jetsonnano + +import ( + "os" + "path/filepath" + + "github.com/siderolabs/go-copy/copy" + "github.com/siderolabs/go-procfs/procfs" + "golang.org/x/sys/unix" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// References +// - https://github.com/u-boot/u-boot/blob/v2021.10/configs/p3450-0000_defconfig#L8 +// - https://github.com/u-boot/u-boot/blob/v2021.10/include/configs/tegra-common.h#L53 +// - https://github.com/u-boot/u-boot/blob/v2021.10/include/configs/tegra210-common.h#L49 +var dtb = "nvidia/tegra210-p3450-0000.dtb" + +// JetsonNano represents the JetsonNano board +// +// References: +// - https://developer.nvidia.com/embedded/jetson-nano-developer-kit +type JetsonNano struct{} + +// Name implements the runtime.Board. +func (b *JetsonNano) Name() string { + return constants.BoardJetsonNano +} + +// Install implements the runtime.Board. +func (b JetsonNano) Install(options runtime.BoardInstallOptions) (err error) { + var f *os.File + + if f, err = os.OpenFile(options.InstallDisk, os.O_RDWR|unix.O_CLOEXEC, 0o666); err != nil { + return err + } + //nolint:errcheck + defer f.Close() + + // NB: In the case that the block device is a loopback device, we sync here + // to ensure that the file is written before the loopback device is + // unmounted. + err = f.Sync() + if err != nil { + return err + } + + src := filepath.Join(options.DTBPath, dtb) + dst := filepath.Join(options.MountPrefix, "/boot/EFI/dtb", dtb) + + err = os.MkdirAll(filepath.Dir(dst), 0o600) + if err != nil { + return err + } + + err = copy.File(src, dst) + if err != nil { + return err + } + + return nil +} + +// KernelArgs implements the runtime.Board. +// +// References: +// - https://elinux.org/Jetson/Nano/Upstream to enable early console +// - http://en.techinfodepot.shoutwiki.com/wiki/NVIDIA_Jetson_Nano_Developer_Kit for other chips on the SoC +func (b JetsonNano) KernelArgs() procfs.Parameters { + return []*procfs.Parameter{ + procfs.NewParameter("console").Append("tty0").Append("ttyS0,115200"), + // even though PSCI works perfectly on the Jetson Nano, the kernel is stuck + // trying to kexec. Seems the drivers state is not reset properly. + // disabling kexec until we have further knowledge on this + procfs.NewParameter("sysctl.kernel.kexec_load_disabled").Append("1"), + procfs.NewParameter(constants.KernelParamDashboardDisabled).Append("1"), + } +} + +// PartitionOptions implements the runtime.Board. +func (b JetsonNano) PartitionOptions() *runtime.PartitionOptions { + return nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/board/libretech_all_h3_cc_h5/libretech_all_h3_cc_h5.go b/internal/app/machined/pkg/runtime/v1alpha1/board/libretech_all_h3_cc_h5/libretech_all_h3_cc_h5.go new file mode 100644 index 0000000..8ef9fc4 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/board/libretech_all_h3_cc_h5/libretech_all_h3_cc_h5.go @@ -0,0 +1,99 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package libretechallh3cch5 provides the LibretechAllH3CCH5 board implementation. +package libretechallh3cch5 + +import ( + "os" + "path/filepath" + + "github.com/siderolabs/go-copy/copy" + "github.com/siderolabs/go-procfs/procfs" + "golang.org/x/sys/unix" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +var ( + bin = constants.BoardLibretechAllH3CCH5 + "/u-boot-sunxi-with-spl.bin" + off int64 = 1024 * 8 + dtb = "allwinner/sun50i-h5-libretech-all-h3-cc.dtb" +) + +// LibretechAllH3CCH5 represents the Libre Computer ALL-H3-CC (Tritium). +// +// Reference: https://libre.computer/products/boards/all-h3-cc/ +type LibretechAllH3CCH5 struct{} + +// Name implements the runtime.Board. +func (l *LibretechAllH3CCH5) Name() string { + return constants.BoardLibretechAllH3CCH5 +} + +// Install implements the runtime.Board. +func (l *LibretechAllH3CCH5) Install(options runtime.BoardInstallOptions) (err error) { + var f *os.File + + if f, err = os.OpenFile(options.InstallDisk, os.O_RDWR|unix.O_CLOEXEC, 0o666); err != nil { + return err + } + //nolint:errcheck + defer f.Close() + + var uboot []byte + + uboot, err = os.ReadFile(filepath.Join(options.UBootPath, bin)) + if err != nil { + return err + } + + options.Printf("writing %s at offset %d", bin, off) + + var n int + + n, err = f.WriteAt(uboot, off) + if err != nil { + return err + } + + options.Printf("wrote %d bytes", n) + + // NB: In the case that the block device is a loopback device, we sync here + // to esure that the file is written before the loopback device is + // unmounted. + err = f.Sync() + if err != nil { + return err + } + + src := filepath.Join(options.DTBPath, dtb) + dst := filepath.Join(options.MountPrefix, "/boot/EFI/dtb", dtb) + + err = os.MkdirAll(filepath.Dir(dst), 0o600) + if err != nil { + return err + } + + err = copy.File(src, dst) + if err != nil { + return err + } + + return nil +} + +// KernelArgs implements the runtime.Board. +func (l *LibretechAllH3CCH5) KernelArgs() procfs.Parameters { + return []*procfs.Parameter{ + procfs.NewParameter("console").Append("tty0").Append("ttyS0,115200"), + procfs.NewParameter(constants.KernelParamDashboardDisabled).Append("1"), + } +} + +// PartitionOptions implements the runtime.Board. +func (l *LibretechAllH3CCH5) PartitionOptions() *runtime.PartitionOptions { + return &runtime.PartitionOptions{PartitionsOffset: 2048} +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/board/nanopi_r4s/nanopi_r4s.go b/internal/app/machined/pkg/runtime/v1alpha1/board/nanopi_r4s/nanopi_r4s.go new file mode 100644 index 0000000..bc20e41 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/board/nanopi_r4s/nanopi_r4s.go @@ -0,0 +1,88 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package nanopir4s provides the Nano Pi R4S board implementation. +package nanopir4s + +import ( + "os" + "path/filepath" + + "github.com/siderolabs/go-copy/copy" + "github.com/siderolabs/go-procfs/procfs" + "golang.org/x/sys/unix" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +var ( + bin = constants.BoardNanoPiR4S + "/u-boot-rockchip.bin" + off int64 = 512 * 64 + dtb = "rockchip/rk3399-nanopi-r4s.dtb" +) + +// NanoPiR4S represents the Friendlyelec Nano Pi R4S board. +// +// Reference: https://wiki.friendlyelec.com/wiki/index.php/NanoPi_R4S +type NanoPiR4S struct{} + +// Name implements the runtime.Board. +func (n *NanoPiR4S) Name() string { + return constants.BoardNanoPiR4S +} + +// Install implements the runtime.Board. +func (n *NanoPiR4S) Install(options runtime.BoardInstallOptions) (err error) { + file, err := os.OpenFile(options.InstallDisk, os.O_RDWR|unix.O_CLOEXEC, 0o666) + if err != nil { + return err + } + + defer file.Close() //nolint:errcheck + + uboot, err := os.ReadFile(filepath.Join(options.UBootPath, bin)) + if err != nil { + return err + } + + options.Printf("writing %s at offset %d", bin, off) + + amount, err := file.WriteAt(uboot, off) + if err != nil { + return err + } + + options.Printf("wrote %d bytes", amount) + + // NB: In the case that the block device is a loopback device, we sync here + // to esure that the file is written before the loopback device is + // unmounted. + if err := file.Sync(); err != nil { + return err + } + + src := filepath.Join(options.DTBPath, dtb) + dst := filepath.Join(options.MountPrefix, "/boot/EFI/dtb", dtb) + + if err := os.MkdirAll(filepath.Dir(dst), 0o600); err != nil { + return err + } + + return copy.File(src, dst) +} + +// KernelArgs implements the runtime.Board. +func (n *NanoPiR4S) KernelArgs() procfs.Parameters { + return []*procfs.Parameter{ + procfs.NewParameter("console").Append("tty0").Append("ttyS2,1500000n8"), + procfs.NewParameter("sysctl.kernel.kexec_load_disabled").Append("1"), + procfs.NewParameter(constants.KernelParamDashboardDisabled).Append("1"), + } +} + +// PartitionOptions implements the runtime.Board. +func (n *NanoPiR4S) PartitionOptions() *runtime.PartitionOptions { + return &runtime.PartitionOptions{PartitionsOffset: 2048 * 10} +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/board/pine64/pine64.go b/internal/app/machined/pkg/runtime/v1alpha1/board/pine64/pine64.go new file mode 100644 index 0000000..c2204be --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/board/pine64/pine64.go @@ -0,0 +1,100 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package pine64 provides the Pine64 board implementation. +package pine64 + +import ( + "os" + "path/filepath" + + "github.com/siderolabs/go-copy/copy" + "github.com/siderolabs/go-procfs/procfs" + "golang.org/x/sys/unix" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +var ( + bin = constants.BoardPine64 + "/u-boot-sunxi-with-spl.bin" + off int64 = 1024 * 8 + dtb = "allwinner/sun50i-a64-pine64-plus.dtb" +) + +// Pine64 represents the Pine64 board +// +// References: +// - http://linux-sunxi.org/Pine64 +type Pine64 struct{} + +// Name implements the runtime.Board. +func (b *Pine64) Name() string { + return constants.BoardPine64 +} + +// Install implements the runtime.Board. +func (b Pine64) Install(options runtime.BoardInstallOptions) (err error) { + var f *os.File + + if f, err = os.OpenFile(options.InstallDisk, os.O_RDWR|unix.O_CLOEXEC, 0o666); err != nil { + return err + } + //nolint:errcheck + defer f.Close() + + var uboot []byte + + uboot, err = os.ReadFile(filepath.Join(options.UBootPath, bin)) + if err != nil { + return err + } + + options.Printf("writing %s at offset %d", bin, off) + + var n int + + n, err = f.WriteAt(uboot, off) + if err != nil { + return err + } + + options.Printf("wrote %d bytes", n) + + // NB: In the case that the block device is a loopback device, we sync here + // to esure that the file is written before the loopback device is + // unmounted. + err = f.Sync() + if err != nil { + return err + } + + src := filepath.Join(options.DTBPath, dtb) + dst := filepath.Join(options.MountPrefix, "/boot/EFI/dtb", dtb) + + err = os.MkdirAll(filepath.Dir(dst), 0o600) + if err != nil { + return err + } + + err = copy.File(src, dst) + if err != nil { + return err + } + + return nil +} + +// KernelArgs implements the runtime.Board. +func (b Pine64) KernelArgs() procfs.Parameters { + return []*procfs.Parameter{ + procfs.NewParameter("console").Append("tty0").Append("ttyS0,115200"), + procfs.NewParameter(constants.KernelParamDashboardDisabled).Append("1"), + } +} + +// PartitionOptions implements the runtime.Board. +func (b Pine64) PartitionOptions() *runtime.PartitionOptions { + return &runtime.PartitionOptions{PartitionsOffset: 2048} +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/board/rock64/rock64.go b/internal/app/machined/pkg/runtime/v1alpha1/board/rock64/rock64.go new file mode 100644 index 0000000..ace9c52 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/board/rock64/rock64.go @@ -0,0 +1,99 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package rock64 provides the Pine64 Rock64 board implementation. +package rock64 + +import ( + "os" + "path/filepath" + + "github.com/siderolabs/go-copy/copy" + "github.com/siderolabs/go-procfs/procfs" + "golang.org/x/sys/unix" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +var ( + bin = constants.BoardRock64 + "/u-boot-rockchip.bin" + off int64 = 512 * 64 + dtb = "rockchip/rk3328-rock64.dtb" +) + +// Rock64 represents the Pine64 Rock64 board. +// +// Reference: https://www.pine64.org/devices/single-board-computers/rock64/ +type Rock64 struct{} + +// Name implements the runtime.Board. +func (r *Rock64) Name() string { + return constants.BoardRock64 +} + +// Install implements the runtime.Board. +func (r *Rock64) Install(options runtime.BoardInstallOptions) (err error) { + var f *os.File + + if f, err = os.OpenFile(options.InstallDisk, os.O_RDWR|unix.O_CLOEXEC, 0o666); err != nil { + return err + } + //nolint:errcheck + defer f.Close() + + var uboot []byte + + uboot, err = os.ReadFile(filepath.Join(options.UBootPath, bin)) + if err != nil { + return err + } + + options.Printf("writing %s at offset %d", bin, off) + + var n int + + n, err = f.WriteAt(uboot, off) + if err != nil { + return err + } + + options.Printf("wrote %d bytes", n) + + // NB: In the case that the block device is a loopback device, we sync here + // to esure that the file is written before the loopback device is + // unmounted. + err = f.Sync() + if err != nil { + return err + } + + src := filepath.Join(options.DTBPath, dtb) + dst := filepath.Join(options.MountPrefix, "/boot/EFI/dtb", dtb) + + err = os.MkdirAll(filepath.Dir(dst), 0o600) + if err != nil { + return err + } + + err = copy.File(src, dst) + if err != nil { + return err + } + + return nil +} + +// KernelArgs implements the runtime.Board. +func (r *Rock64) KernelArgs() procfs.Parameters { + return []*procfs.Parameter{ + procfs.NewParameter("console").Append("tty0").Append("ttyS2,115200n8"), + procfs.NewParameter(constants.KernelParamDashboardDisabled).Append("1"), + } +} + +// PartitionOptions implements the runtime.Board. +func (r *Rock64) PartitionOptions() *runtime.PartitionOptions { + return &runtime.PartitionOptions{PartitionsOffset: 2048 * 10} +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/board/rockpi4/rockpi4.go b/internal/app/machined/pkg/runtime/v1alpha1/board/rockpi4/rockpi4.go new file mode 100644 index 0000000..f9b5cf4 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/board/rockpi4/rockpi4.go @@ -0,0 +1,95 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package rockpi4 provides the Radxa rock pi implementation. +package rockpi4 + +import ( + "os" + "path/filepath" + + "github.com/siderolabs/go-copy/copy" + "github.com/siderolabs/go-procfs/procfs" + "golang.org/x/sys/unix" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +var ( + bin = constants.BoardRockpi4 + "/u-boot-rockchip.bin" + off int64 = 512 * 64 + // https://github.com/u-boot/u-boot/blob/4de720e98d552dfda9278516bf788c4a73b3e56f/configs/rock-pi-4-rk3399_defconfig#L7= + // 4a and 4b uses the same overlay. + dtb = "rockchip/rk3399-rock-pi-4b.dtb" +) + +// Rockpi4 represents the Radxa rock pi board. +// +// Reference: https://rockpi.org/ +type Rockpi4 struct{} + +// Name implements the runtime.Board. +func (r *Rockpi4) Name() string { + return constants.BoardRockpi4 +} + +// Install implements the runtime.Board. +func (r *Rockpi4) Install(options runtime.BoardInstallOptions) (err error) { + var f *os.File + + if f, err = os.OpenFile(options.InstallDisk, os.O_RDWR|unix.O_CLOEXEC, 0o666); err != nil { + return err + } + + defer f.Close() //nolint:errcheck + + uboot, err := os.ReadFile(filepath.Join(options.UBootPath, bin)) + if err != nil { + return err + } + + options.Printf("writing %s at offset %d", bin, off) + + var n int + + n, err = f.WriteAt(uboot, off) + if err != nil { + return err + } + + options.Printf("wrote %d bytes", n) + + // NB: In the case that the block device is a loopback device, we sync here + // to esure that the file is written before the loopback device is + // unmounted. + err = f.Sync() + if err != nil { + return err + } + + src := filepath.Join(options.DTBPath, dtb) + dst := filepath.Join(options.MountPrefix, "/boot/EFI/dtb", dtb) + + err = os.MkdirAll(filepath.Dir(dst), 0o600) + if err != nil { + return err + } + + return copy.File(src, dst) +} + +// KernelArgs implements the runtime.Board. +func (r *Rockpi4) KernelArgs() procfs.Parameters { + return []*procfs.Parameter{ + procfs.NewParameter("console").Append("tty0").Append("ttyS2,1500000n8"), + procfs.NewParameter("sysctl.kernel.kexec_load_disabled").Append("1"), + procfs.NewParameter(constants.KernelParamDashboardDisabled).Append("1"), + } +} + +// PartitionOptions implements the runtime.Board. +func (r *Rockpi4) PartitionOptions() *runtime.PartitionOptions { + return &runtime.PartitionOptions{PartitionsOffset: 2048 * 10} +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/board/rockpi4c/rockpi4c.go b/internal/app/machined/pkg/runtime/v1alpha1/board/rockpi4c/rockpi4c.go new file mode 100644 index 0000000..db08864 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/board/rockpi4c/rockpi4c.go @@ -0,0 +1,94 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package rockpi4c provides the Radxa rock pi implementation. +package rockpi4c + +import ( + "os" + "path/filepath" + + "github.com/siderolabs/go-copy/copy" + "github.com/siderolabs/go-procfs/procfs" + "golang.org/x/sys/unix" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +var ( + bin = constants.BoardRockpi4 + "/u-boot-rockchip.bin" + off int64 = 512 * 64 + // https://github.com/u-boot/u-boot/blob/4de720e98d552dfda9278516bf788c4a73b3e56f/configs/rock-pi-4c-rk3399_defconfig#L7= + dtb = "rockchip/rk3399-rock-pi-4c.dtb" +) + +// Rockpi4c represents the Radxa rock pi board. +// +// Reference: https://rockpi.org/ +type Rockpi4c struct{} + +// Name implements the runtime.Board. +func (r *Rockpi4c) Name() string { + return constants.BoardRockpi4 +} + +// Install implements the runtime.Board. +func (r *Rockpi4c) Install(options runtime.BoardInstallOptions) (err error) { + var f *os.File + + if f, err = os.OpenFile(options.InstallDisk, os.O_RDWR|unix.O_CLOEXEC, 0o666); err != nil { + return err + } + + defer f.Close() //nolint:errcheck + + uboot, err := os.ReadFile(filepath.Join(options.UBootPath, bin)) + if err != nil { + return err + } + + options.Printf("writing %s at offset %d", bin, off) + + var n int + + n, err = f.WriteAt(uboot, off) + if err != nil { + return err + } + + options.Printf("wrote %d bytes", n) + + // NB: In the case that the block device is a loopback device, we sync here + // to esure that the file is written before the loopback device is + // unmounted. + err = f.Sync() + if err != nil { + return err + } + + src := filepath.Join(options.DTBPath, dtb) + dst := filepath.Join(options.MountPrefix, "/boot/EFI/dtb", dtb) + + err = os.MkdirAll(filepath.Dir(dst), 0o600) + if err != nil { + return err + } + + return copy.File(src, dst) +} + +// KernelArgs implements the runtime.Board. +func (r *Rockpi4c) KernelArgs() procfs.Parameters { + return []*procfs.Parameter{ + procfs.NewParameter("console").Append("tty0").Append("ttyS2,1500000n8"), + procfs.NewParameter("sysctl.kernel.kexec_load_disabled").Append("1"), + procfs.NewParameter(constants.KernelParamDashboardDisabled).Append("1"), + } +} + +// PartitionOptions implements the runtime.Board. +func (r *Rockpi4c) PartitionOptions() *runtime.PartitionOptions { + return &runtime.PartitionOptions{PartitionsOffset: 2048 * 10} +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/board/rpi_generic/config.txt b/internal/app/machined/pkg/runtime/v1alpha1/board/rpi_generic/config.txt new file mode 100644 index 0000000..727cb39 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/board/rpi_generic/config.txt @@ -0,0 +1,19 @@ +# See https://www.raspberrypi.com/documentation/computers/configuration.html +# Reduce GPU memory to give more to CPU. +gpu_mem=32 +# Enable maximum compatibility on both HDMI ports; +# only the one closest to the power/USB-C port will work in practice. +hdmi_safe:0=1 +hdmi_safe:1=1 +# Load U-Boot. +kernel=u-boot.bin +# Forces the kernel loading system to assume a 64-bit kernel. +arm_64bit=1 +# Run as fast as firmware / board allows. +arm_boost=1 +# Enable the primary/console UART. +enable_uart=1 +# Disable Bluetooth. +dtoverlay=disable-bt +# Disable Wireless Lan. +dtoverlay=disable-wifi \ No newline at end of file diff --git a/internal/app/machined/pkg/runtime/v1alpha1/board/rpi_generic/rpi_generic.go b/internal/app/machined/pkg/runtime/v1alpha1/board/rpi_generic/rpi_generic.go new file mode 100644 index 0000000..a103224 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/board/rpi_generic/rpi_generic.go @@ -0,0 +1,60 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package rpigeneric provides the Raspberry Pi Compute Module 4 implementation. +package rpigeneric + +import ( + _ "embed" + "os" + "path/filepath" + + "github.com/siderolabs/go-copy/copy" + "github.com/siderolabs/go-procfs/procfs" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +//go:embed config.txt +var configTxt []byte + +// RPiGeneric represents the Raspberry Pi Compute Module 4. +// +// Reference: https://www.raspberrypi.com/products/compute-module-4/ +type RPiGeneric struct{} + +// Name implements the runtime.Board. +func (r *RPiGeneric) Name() string { + return constants.BoardRPiGeneric +} + +// Install implements the runtime.Board. +func (r *RPiGeneric) Install(options runtime.BoardInstallOptions) (err error) { + err = copy.Dir(filepath.Join(options.RPiFirmwarePath, "boot"), filepath.Join(options.MountPrefix, "/boot/EFI")) + if err != nil { + return err + } + + err = copy.File(filepath.Join(options.UBootPath, "rpi_generic/u-boot.bin"), filepath.Join(options.MountPrefix, "/boot/EFI/u-boot.bin")) + if err != nil { + return err + } + + return os.WriteFile(filepath.Join(options.MountPrefix, "/boot/EFI/config.txt"), configTxt, 0o600) +} + +// KernelArgs implements the runtime.Board. +func (r *RPiGeneric) KernelArgs() procfs.Parameters { + return []*procfs.Parameter{ + procfs.NewParameter("console").Append("tty0").Append("ttyAMA0,115200"), + procfs.NewParameter("sysctl.kernel.kexec_load_disabled").Append("1"), + procfs.NewParameter(constants.KernelParamDashboardDisabled).Append("1"), + } +} + +// PartitionOptions implements the runtime.Board. +func (r *RPiGeneric) PartitionOptions() *runtime.PartitionOptions { + return nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/bootloader/bootloader.go b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/bootloader.go new file mode 100644 index 0000000..d0f92c5 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/bootloader.go @@ -0,0 +1,75 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package bootloader provides bootloader implementation. +package bootloader + +import ( + "context" + "os" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/bootloader/options" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/bootloader/sdboot" + "github.com/siderolabs/talos/pkg/imager/quirks" +) + +// Bootloader describes a bootloader. +type Bootloader interface { + // Install installs the bootloader + Install(options options.InstallOptions) error + // Revert reverts the bootloader entry to the previous state. + Revert(ctx context.Context) error + // PreviousLabel returns the previous bootloader label. + PreviousLabel() string + // UEFIBoot returns true if the bootloader is UEFI-only. + UEFIBoot() bool +} + +// Probe checks if any supported bootloaders are installed. +// +// If 'disk' is empty, it will probe all disks. +// Returns nil if it cannot detect any supported bootloader. +func Probe(ctx context.Context, disk string) (Bootloader, error) { + grubBootloader, err := grub.Probe(ctx, disk) + if err != nil { + return nil, err + } + + if grubBootloader != nil { + return grubBootloader, nil + } + + sdbootBootloader, err := sdboot.Probe(ctx, disk) + if err != nil { + return nil, err + } + + if sdbootBootloader != nil { + return sdbootBootloader, nil + } + + return nil, os.ErrNotExist +} + +// NewAuto returns a new bootloader based on auto-detection. +func NewAuto() Bootloader { + if sdboot.IsBootedUsingSDBoot() { + return sdboot.New() + } + + return grub.NewConfig() +} + +// New returns a new bootloader based on the secureboot flag. +func New(secureboot bool, talosVersion string) Bootloader { + if secureboot { + return sdboot.New() + } + + g := grub.NewConfig() + g.AddResetOption = quirks.New(talosVersion).SupportsResetGRUBOption() + + return g +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/boot_label.go b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/boot_label.go new file mode 100644 index 0000000..9a44bdd --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/boot_label.go @@ -0,0 +1,62 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package grub + +import ( + "fmt" + "strings" +) + +// flipBootLabel flips the boot label. +func flipBootLabel(e BootLabel) (BootLabel, error) { + switch e { + case BootA: + return BootB, nil + case BootB: + return BootA, nil + case BootReset: + fallthrough + default: + return "", fmt.Errorf("invalid entry: %s", e) + } +} + +// Flip flips the default boot label. +func (c *Config) flip() error { + if _, exists := c.Entries[c.Default]; !exists { + return nil + } + + current := c.Default + + next, err := flipBootLabel(c.Default) + if err != nil { + return err + } + + c.Default = next + c.Fallback = current + + return nil +} + +// PreviousLabel returns the previous bootloader label. +func (c *Config) PreviousLabel() string { + return string(c.Fallback) +} + +// ParseBootLabel parses the given human-readable boot label to a BootLabel. +func ParseBootLabel(name string) (BootLabel, error) { + switch { + case strings.HasPrefix(name, string(BootA)): + return BootA, nil + case strings.HasPrefix(name, string(BootB)): + return BootB, nil + case strings.HasPrefix(name, "Reset"): + return BootReset, nil + default: + return "", fmt.Errorf("could not parse boot entry from name: %s", name) + } +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/constants.go b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/constants.go new file mode 100644 index 0000000..1f50806 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/constants.go @@ -0,0 +1,29 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package grub + +import ( + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// BootLabel represents a boot label, e.g. A or B. +type BootLabel string + +const ( + // ConfigPath is the path to the grub config. + ConfigPath = constants.BootMountPoint + "/grub/grub.cfg" + // BootA is a bootloader label. + BootA BootLabel = "A" + // BootB is a bootloader label. + BootB BootLabel = "B" + // BootReset is a bootloader label. + BootReset BootLabel = "Reset" +) + +const ( + bootloaderNotInstalled = "bootloader not installed" +) + +type bootloaderNotInstalledError struct{} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/decode.go b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/decode.go new file mode 100644 index 0000000..8166a6d --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/decode.go @@ -0,0 +1,159 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package grub + +import ( + "errors" + "fmt" + "os" + "regexp" +) + +var ( + defaultEntryRegex = regexp.MustCompile(`(?m)^\s*set default="(.*)"\s*$`) + fallbackEntryRegex = regexp.MustCompile(`(?m)^\s*set fallback="(.*)"\s*$`) + menuEntryRegex = regexp.MustCompile(`(?ms)^menuentry\s+"(.+?)" {(.+?)[^\\]}`) + linuxRegex = regexp.MustCompile(`(?m)^\s*linux\s+(.+?)\s+(.*)$`) + initrdRegex = regexp.MustCompile(`(?m)^\s*initrd\s+(.+)$`) +) + +// Read reads the grub configuration from the disk. +func Read(path string) (*Config, error) { + c, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + + if err != nil { + return nil, err + } + + return Decode(c) +} + +// Decode parses the grub configuration from the given bytes. +func Decode(c []byte) (*Config, error) { + defaultEntryMatches := defaultEntryRegex.FindAllSubmatch(c, -1) + if len(defaultEntryMatches) != 1 { + return nil, errors.New("failed to find default") + } + + fallbackEntryMatches := fallbackEntryRegex.FindAllSubmatch(c, -1) + if len(fallbackEntryMatches) > 1 { + return nil, errors.New("found multiple fallback entries") + } + + var fallbackEntry BootLabel + + if len(fallbackEntryMatches) == 1 { + if len(fallbackEntryMatches[0]) != 2 { + return nil, errors.New("failed to parse fallback entry") + } + + entry, err := ParseBootLabel(string(fallbackEntryMatches[0][1])) + if err != nil { + return nil, err + } + + fallbackEntry = entry + } + + if len(defaultEntryMatches[0]) != 2 { + return nil, fmt.Errorf("default entry: expected 2 matches, got %d", len(defaultEntryMatches[0])) + } + + defaultEntry, err := ParseBootLabel(string(defaultEntryMatches[0][1])) + if err != nil { + return nil, err + } + + entries, hasResetOption, err := parseEntries(c) + if err != nil { + return nil, err + } + + conf := Config{ + Default: defaultEntry, + Fallback: fallbackEntry, + Entries: entries, + AddResetOption: hasResetOption, + } + + return &conf, nil +} + +func parseEntries(conf []byte) (map[BootLabel]MenuEntry, bool, error) { + entries := make(map[BootLabel]MenuEntry) + hasResetOption := false + + matches := menuEntryRegex.FindAllSubmatch(conf, -1) + for _, m := range matches { + if len(m) != 3 { + return nil, false, fmt.Errorf("conf block: expected 3 matches, got %d", len(m)) + } + + confBlock := m[2] + + linux, cmdline, initrd, err := parseConfBlock(confBlock) + if err != nil { + return nil, false, err + } + + name := string(m[1]) + + bootEntry, err := ParseBootLabel(name) + if err != nil { + return nil, false, err + } + + if bootEntry == BootReset { + hasResetOption = true + + continue + } + + entries[bootEntry] = MenuEntry{ + Name: name, + Linux: linux, + Cmdline: cmdline, + Initrd: initrd, + } + } + + return entries, hasResetOption, nil +} + +func parseConfBlock(block []byte) (linux, cmdline, initrd string, err error) { + block = []byte(Unquote(string(block))) + + linuxMatches := linuxRegex.FindAllSubmatch(block, -1) + if len(linuxMatches) != 1 { + return "", "", "", + fmt.Errorf("linux: expected 1 match, got %d", len(linuxMatches)) + } + + if len(linuxMatches[0]) != 3 { + return "", "", "", + fmt.Errorf("linux: expected 3 matches, got %d", len(linuxMatches[0])) + } + + linux = string(linuxMatches[0][1]) + cmdline = string(linuxMatches[0][2]) + + initrdMatches := initrdRegex.FindAllSubmatch(block, -1) + if len(initrdMatches) != 1 { + return "", "", "", + fmt.Errorf("initrd: expected 1 match, got %d: %s", len(initrdMatches), string(block)) + } + + if len(initrdMatches[0]) != 2 { + return "", "", "", + fmt.Errorf("initrd: expected 2 matches, got %d", len(initrdMatches[0])) + } + + initrd = string(initrdMatches[0][1]) + + return linux, cmdline, initrd, nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/encode.go b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/encode.go new file mode 100644 index 0000000..cbf5b63 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/encode.go @@ -0,0 +1,76 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package grub + +import ( + "bytes" + "io" + "os" + "path/filepath" + "text/template" +) + +const confTemplate = `set default="{{ (index .Entries .Default).Name }}" +{{ with (index .Entries .Fallback).Name -}} +set fallback="{{ . }}" +{{- end }} +set timeout=3 + +insmod all_video + +terminal_input console +terminal_output console + +{{ range $key, $entry := .Entries -}} +menuentry "{{ $entry.Name }}" { + set gfxmode=auto + set gfxpayload=text + linux {{ $entry.Linux }} {{ quote $entry.Cmdline }} + initrd {{ $entry.Initrd }} +} +{{ end -}} + +{{ if .AddResetOption -}} +{{ $defaultEntry := index .Entries .Default -}} +menuentry "Reset Talos installation and return to maintenance mode" { + set gfxmode=auto + set gfxpayload=text + linux {{ $defaultEntry.Linux }} {{ quote $defaultEntry.Cmdline }} talos.experimental.wipe=system:EPHEMERAL,STATE + initrd {{ $defaultEntry.Initrd }} +} +{{ end -}} +` + +// Write the grub configuration to the given file. +func (c *Config) Write(path string, printf func(string, ...any)) error { + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, os.ModeDir); err != nil { + return err + } + + wr := new(bytes.Buffer) + + err := c.Encode(wr) + if err != nil { + return err + } + + printf("writing %s to disk", path) + + return os.WriteFile(path, wr.Bytes(), 0o600) +} + +// Encode writes the grub configuration to the given writer. +func (c *Config) Encode(wr io.Writer) error { + if err := c.validate(); err != nil { + return err + } + + t := template.Must(template.New("grub").Funcs(template.FuncMap{ + "quote": Quote, + }).Parse(confTemplate)) + + return t.Execute(wr, c) +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/grub.go b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/grub.go new file mode 100644 index 0000000..e0f877b --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/grub.go @@ -0,0 +1,84 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package grub provides the interface to the GRUB bootloader: config management, installation, etc. +package grub + +import ( + "errors" + "fmt" + "path/filepath" + + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/version" +) + +// Config represents a grub configuration file (grub.cfg). +type Config struct { + Default BootLabel + Fallback BootLabel + Entries map[BootLabel]MenuEntry + AddResetOption bool +} + +// MenuEntry represents a grub menu entry in the grub config file. +type MenuEntry struct { + Name string + Linux string + Cmdline string + Initrd string +} + +func (e bootloaderNotInstalledError) Error() string { + return bootloaderNotInstalled +} + +// NewConfig creates a new grub configuration (nothing is written to disk). +func NewConfig() *Config { + return &Config{ + Default: BootA, + Entries: map[BootLabel]MenuEntry{}, + AddResetOption: true, + } +} + +// UEFIBoot returns true if bootloader is UEFI-only. +func (c *Config) UEFIBoot() bool { + // grub supports BIOS boot, so false here. + return false +} + +// Put puts a new menu entry to the grub config (nothing is written to disk). +func (c *Config) Put(entry BootLabel, cmdline, version string) error { + c.Entries[entry] = buildMenuEntry(entry, cmdline, version) + + return nil +} + +func (c *Config) validate() error { + if _, ok := c.Entries[c.Default]; !ok { + return fmt.Errorf("invalid default entry: %s", c.Default) + } + + if c.Fallback != "" { + if _, ok := c.Entries[c.Fallback]; !ok { + return fmt.Errorf("invalid fallback entry: %s", c.Fallback) + } + } + + if c.Default == c.Fallback { + return errors.New("default and fallback entries must not be the same") + } + + return nil +} + +func buildMenuEntry(entry BootLabel, cmdline, versionTag string) MenuEntry { + return MenuEntry{ + Name: fmt.Sprintf("%s - %s %s", entry, version.Name, versionTag), + Linux: filepath.Join("/", string(entry), constants.KernelAsset), + Cmdline: cmdline, + Initrd: filepath.Join("/", string(entry), constants.InitramfsAsset), + } +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/grub_test.go b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/grub_test.go new file mode 100644 index 0000000..0547ed3 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/grub_test.go @@ -0,0 +1,278 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package grub_test + +import ( + "bufio" + "bytes" + _ "embed" + "fmt" + "io" + "os" + "regexp" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub" + "github.com/siderolabs/talos/pkg/machinery/version" +) + +var ( + //go:embed testdata/grub_parse_test.cfg + grubCfg []byte + + //go:embed testdata/grub_write_test.cfg + newConfig string + + //go:embed testdata/grub_write_no_reset_test.cfg + newNoResetConfig string +) + +func TestDecode(t *testing.T) { + conf, err := grub.Decode(grubCfg) + assert.NoError(t, err) + + assert.Equal(t, grub.BootA, conf.Default) + assert.Equal(t, grub.BootB, conf.Fallback) + + assert.Len(t, conf.Entries, 2) + + a := conf.Entries[grub.BootA] + assert.Equal(t, "A - v1", a.Name) + assert.True(t, strings.HasPrefix(a.Linux, "/A/")) + assert.True(t, strings.HasPrefix(a.Initrd, "/A/")) + assert.Equal(t, "cmdline A", a.Cmdline) + + b := conf.Entries[grub.BootB] + assert.Equal(t, "B - v2", b.Name) + assert.Equal(t, "cmdline B", b.Cmdline) + assert.True(t, strings.HasPrefix(b.Linux, "/B/")) + assert.True(t, strings.HasPrefix(b.Initrd, "/B/")) + + assert.True(t, conf.AddResetOption) +} + +func TestEncodeDecode(t *testing.T) { + config := grub.NewConfig() + require.NoError(t, config.Put(grub.BootA, "talos.platform=metal talos.config=https://my-metadata.server/talos/config?hostname=${hostname}&mac=${mac}", "v1.2.3")) + require.NoError(t, config.Put(grub.BootB, "talos.platform=metal talos.config=https://my-metadata.server/talos/config?uuid=${uuid}", "v1.3.4")) + + var b bytes.Buffer + + require.NoError(t, config.Encode(&b)) + + t.Logf("config encoded to:\n%s", b.String()) + + config2, err := grub.Decode(b.Bytes()) + require.NoError(t, err) + + assert.Equal(t, config, config2) +} + +func TestParseBootLabel(t *testing.T) { + label, err := grub.ParseBootLabel("A - v1") + assert.NoError(t, err) + assert.Equal(t, grub.BootA, label) + + label, err = grub.ParseBootLabel("B - v2") + assert.NoError(t, err) + assert.Equal(t, grub.BootB, label) + + label, err = grub.ParseBootLabel("Reset Talos installation and return to maintenance mode\n") + assert.NoError(t, err) + assert.Equal(t, grub.BootReset, label) + + _, err = grub.ParseBootLabel("C - v3") + assert.Error(t, err) +} + +//nolint:errcheck +func TestWrite(t *testing.T) { + oldName := version.Name + + t.Cleanup(func() { + version.Name = oldName + }) + + version.Name = "Test" + + tempFile, _ := os.CreateTemp("", "talos-test-grub-*.cfg") + + t.Cleanup(func() { require.NoError(t, os.Remove(tempFile.Name())) }) + + config := grub.NewConfig() + require.NoError(t, config.Put(grub.BootA, "cmdline A", "v0.0.1")) + + err := config.Write(tempFile.Name(), t.Logf) + assert.NoError(t, err) + + written, _ := os.ReadFile(tempFile.Name()) + assert.Equal(t, newConfig, string(written)) +} + +//nolint:errcheck +func TestWriteNoReset(t *testing.T) { + oldName := version.Name + + t.Cleanup(func() { + version.Name = oldName + }) + + version.Name = "TestOld" + + tempFile, _ := os.CreateTemp("", "talos-test-grub-*.cfg") + + t.Cleanup(func() { require.NoError(t, os.Remove(tempFile.Name())) }) + + config := grub.NewConfig() + config.AddResetOption = false + require.NoError(t, config.Put(grub.BootA, "cmdline A", "v0.0.1")) + + err := config.Write(tempFile.Name(), t.Logf) + assert.NoError(t, err) + + written, _ := os.ReadFile(tempFile.Name()) + assert.Equal(t, newNoResetConfig, string(written)) +} + +func TestPut(t *testing.T) { + config := grub.NewConfig() + require.NoError(t, config.Put(grub.BootA, "cmdline A", "v1.2.3")) + + err := config.Put(grub.BootB, "cmdline B", "v1.0.0") + + assert.NoError(t, err) + + assert.Len(t, config.Entries, 2) + assert.Equal(t, "cmdline B", config.Entries[grub.BootB].Cmdline) + + err = config.Put(grub.BootA, "cmdline A 2", "v1.3.4") + assert.NoError(t, err) + + assert.Equal(t, "cmdline A 2", config.Entries[grub.BootA].Cmdline) +} + +//nolint:errcheck +func TestFallback(t *testing.T) { + config := grub.NewConfig() + require.NoError(t, config.Put(grub.BootA, "cmdline A", "v1.0.0")) + + _ = config.Put(grub.BootB, "cmdline B", "1.2.0") + + config.Fallback = grub.BootB + + var buf bytes.Buffer + _ = config.Encode(&buf) + + result := buf.String() + + assert.Contains(t, result, `set fallback="B - `) + + buf.Reset() + + config.Fallback = "" + _ = config.Encode(&buf) + + result = buf.String() + assert.NotContains(t, result, "set fallback") +} + +type bootEntry struct { + Linux string + Initrd string + Cmdline string +} + +// oldParser is the kexec parser used before the GRUB parser was rewritten. +// +// This makes sure Talos 0.14 can kexec into newly written GRUB config. +// +//nolint:gocyclo +func oldParser(r io.Reader) (*bootEntry, error) { + scanner := bufio.NewScanner(r) + + entry := &bootEntry{} + + var ( + defaultEntry string + currentEntry string + ) + + for scanner.Scan() { + line := scanner.Text() + + switch { + case strings.HasPrefix(line, "set default"): + matches := regexp.MustCompile(`set default="(.*)"`).FindStringSubmatch(line) + if len(matches) != 2 { + return nil, fmt.Errorf("malformed default entry: %q", line) + } + + defaultEntry = matches[1] + case strings.HasPrefix(line, "menuentry"): + matches := regexp.MustCompile(`menuentry "(.*)"`).FindStringSubmatch(line) + if len(matches) != 2 { + return nil, fmt.Errorf("malformed menuentry: %q", line) + } + + currentEntry = matches[1] + case strings.HasPrefix(line, " linux "): + if currentEntry != defaultEntry { + continue + } + + parts := strings.SplitN(line[8:], " ", 2) + + entry.Linux = parts[0] + if len(parts) == 2 { + entry.Cmdline = parts[1] + } + case strings.HasPrefix(line, " initrd "): + if currentEntry != defaultEntry { + continue + } + + entry.Initrd = line[9:] + } + } + + if entry.Linux == "" || entry.Initrd == "" { + return nil, scanner.Err() + } + + return entry, scanner.Err() +} + +func TestBackwardsCompat(t *testing.T) { + oldName := version.Name + + t.Cleanup(func() { + version.Name = oldName + }) + + version.Name = "Test" + + var buf bytes.Buffer + + config := grub.NewConfig() + require.NoError(t, config.Put(grub.BootA, "cmdline A", "v0.0.1")) + require.NoError(t, config.Put(grub.BootB, "cmdline B", "v0.0.1")) + config.Default = grub.BootB + + err := config.Encode(&buf) + assert.NoError(t, err) + + entry, err := oldParser(&buf) + require.NoError(t, err) + + assert.Equal(t, &bootEntry{ + Linux: "/B/vmlinuz", + Initrd: "/B/initramfs.xz", + Cmdline: "cmdline B", + }, entry) +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/install.go b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/install.go new file mode 100644 index 0000000..e1868bb --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/install.go @@ -0,0 +1,120 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package grub + +import ( + "fmt" + "path/filepath" + "runtime" + "strings" + + "github.com/siderolabs/go-blockdevice/blockdevice" + "github.com/siderolabs/go-cmd/pkg/cmd" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/bootloader/options" + "github.com/siderolabs/talos/pkg/imager/utils" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +const ( + amd64 = "amd64" + arm64 = "arm64" +) + +// Install validates the grub configuration and writes it to the disk. +// +//nolint:gocyclo +func (c *Config) Install(options options.InstallOptions) error { + if err := c.flip(); err != nil { + return err + } + + if err := utils.CopyFiles( + options.Printf, + utils.SourceDestination( + options.BootAssets.KernelPath, + filepath.Join(options.MountPrefix, constants.BootMountPoint, string(c.Default), constants.KernelAsset), + ), + utils.SourceDestination( + options.BootAssets.InitramfsPath, + filepath.Join(options.MountPrefix, constants.BootMountPoint, string(c.Default), constants.InitramfsAsset), + ), + ); err != nil { + return err + } + + if err := c.Put(c.Default, options.Cmdline, options.Version); err != nil { + return err + } + + if err := c.Write(filepath.Join(options.MountPrefix, ConfigPath), options.Printf); err != nil { + return err + } + + blk, err := getBlockDeviceName(options.BootDisk) + if err != nil { + return err + } + + var platforms []string + + switch options.Arch { + case amd64: + platforms = []string{"x86_64-efi", "i386-pc"} + case arm64: + platforms = []string{"arm64-efi"} + } + + if runtime.GOARCH == amd64 && options.Arch == amd64 && !options.ImageMode { + // let grub choose the platform automatically if not building an image + platforms = []string{""} + } + + for _, platform := range platforms { + args := []string{ + "--boot-directory=" + filepath.Join(options.MountPrefix, constants.BootMountPoint), + "--efi-directory=" + filepath.Join(options.MountPrefix, constants.EFIMountPoint), + "--removable", + } + + if options.ImageMode { + args = append(args, "--no-nvram") + } + + if platform != "" { + args = append(args, "--target="+platform) + } + + args = append(args, blk) + + options.Printf("executing: grub-install %s", strings.Join(args, " ")) + + if _, err := cmd.Run("grub-install", args...); err != nil { + return fmt.Errorf("failed to install grub: %w", err) + } + } + + return nil +} + +func getBlockDeviceName(bootDisk string) (string, error) { + dev, err := blockdevice.Open(bootDisk, blockdevice.WithMode(blockdevice.ReadonlyMode)) + if err != nil { + return "", err + } + + //nolint:errcheck + defer dev.Close() + + // verify that BootDisk has boot partition + _, err = dev.GetPartition(constants.BootPartitionLabel) + if err != nil { + return "", err + } + + blk := dev.Device().Name() + + return blk, nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/probe.go b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/probe.go new file mode 100644 index 0000000..efaaee4 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/probe.go @@ -0,0 +1,35 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package grub provides the interface to the GRUB bootloader: config management, installation, etc. +package grub + +import ( + "context" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/bootloader/mount" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// Probe probes a block device for GRUB bootloader. +// +// If the 'disk' is passed, search happens on that disk only, otherwise searches all partitions. +func Probe(ctx context.Context, disk string) (*Config, error) { + var grubConf *Config + + if err := mount.PartitionOp(ctx, disk, constants.BootPartitionLabel, func() error { + var err error + + grubConf, err = Read(ConfigPath) + if err != nil { + return err + } + + return nil + }); err != nil { + return nil, err + } + + return grubConf, nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/quote.go b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/quote.go new file mode 100644 index 0000000..e3adbeb --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/quote.go @@ -0,0 +1,29 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package grub + +import ( + "strings" +) + +// Quote according to (incomplete) GRUB quoting rules. +// +// See https://www.gnu.org/software/grub/manual/grub/html_node/Shell_002dlike-scripting.html +func Quote(s string) string { + for _, c := range `\{}&$|;<>"` { + s = strings.ReplaceAll(s, string(c), `\`+string(c)) + } + + return s +} + +// Unquote according to (incomplete) GRUB quoting rules. +func Unquote(s string) string { + for _, c := range `{}&$|;<>\"` { + s = strings.ReplaceAll(s, `\`+string(c), string(c)) + } + + return s +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/quote_test.go b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/quote_test.go new file mode 100644 index 0000000..660dcef --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/quote_test.go @@ -0,0 +1,105 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package grub_test + +import ( + "testing" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub" +) + +//nolint:dupl +func TestQuote(t *testing.T) { + t.Parallel() + + for _, test := range []struct { + name string + input string + expected string + }{ + { + name: "empty", + input: "", + expected: "", + }, + { + name: "no special characters", + input: "foo", + expected: "foo", + }, + { + name: "backslash", + input: `foo\`, + expected: `foo\\`, + }, + { + name: "escaped backslash", + input: `foo\$`, + expected: `foo\\\$`, + }, + { + name: "url", + input: "http://my-host/config.yaml?uuid=${uuid}&serial=${serial}&mac=${mac}&hostname=${hostname}", + expected: "http://my-host/config.yaml?uuid=\\$\\{uuid\\}\\&serial=\\$\\{serial\\}\\&mac=\\$\\{mac\\}\\&hostname=\\$\\{hostname\\}", + }, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + actual := grub.Quote(test.input) + + if actual != test.expected { + t.Fatalf("expected %q, got %q", test.expected, actual) + } + }) + } +} + +//nolint:dupl +func TestUnquote(t *testing.T) { + t.Parallel() + + for _, test := range []struct { + name string + input string + expected string + }{ + { + name: "empty", + input: "", + expected: "", + }, + { + name: "no special characters", + input: "foo", + expected: "foo", + }, + { + name: "backslash", + input: `foo\\`, + expected: `foo\`, + }, + { + name: "escaped backslash", + input: `foo\\\$`, + expected: `foo\$`, + }, + { + name: "url", + input: "http://my-host/config.yaml?uuid=\\$\\{uuid\\}\\&serial=\\$\\{serial\\}\\&mac=\\$\\{mac\\}\\&hostname=\\$\\{hostname\\}", + expected: "http://my-host/config.yaml?uuid=${uuid}&serial=${serial}&mac=${mac}&hostname=${hostname}", + }, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + actual := grub.Unquote(test.input) + + if actual != test.expected { + t.Fatalf("expected %q, got %q", test.expected, actual) + } + }) + } +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/revert.go b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/revert.go new file mode 100644 index 0000000..e0e1b41 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/revert.go @@ -0,0 +1,79 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package grub provides the interface to the GRUB bootloader: config management, installation, etc. +package grub + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "path/filepath" + + "github.com/siderolabs/go-blockdevice/blockdevice/probe" + + "github.com/aenix-io/talm/internal/pkg/mount" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// Revert reverts the bootloader to the previous version. +// +//nolint:gocyclo +func (c *Config) Revert(ctx context.Context) error { + if c == nil { + return fmt.Errorf("cannot revert bootloader: %w", bootloaderNotInstalledError{}) + } + + if err := c.flip(); err != nil { + return err + } + + // attempt to probe BOOT partition directly + dev, err := probe.GetDevWithPartitionName(constants.BootPartitionLabel) + if os.IsNotExist(err) { + // no BOOT partition, nothing to revert + return nil + } + + if err != nil { + return err + } + + defer dev.Close() //nolint:errcheck + + mp, err := mount.SystemMountPointForLabel(ctx, dev.BlockDevice, constants.BootPartitionLabel) + if err != nil { + return err + } + + // if no BOOT partition nothing to revert + if mp == nil { + return nil + } + + alreadyMounted, err := mp.IsMounted() + if err != nil { + return err + } + + if !alreadyMounted { + if err = mp.Mount(); err != nil { + return err + } + + defer mp.Unmount() //nolint:errcheck + } + + if _, err = os.Stat(filepath.Join(constants.BootMountPoint, string(c.Default))); errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("cannot rollback to %q, label does not exist", "") + } + + if err := c.Write(ConfigPath, log.Printf); err != nil { + return fmt.Errorf("failed to revert bootloader: %v", err) + } + + return nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/testdata/grub_parse_test.cfg b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/testdata/grub_parse_test.cfg new file mode 100644 index 0000000..90b2a25 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/testdata/grub_parse_test.cfg @@ -0,0 +1,29 @@ +set default="A - v1" +set timeout=3 +set fallback="B - v2" + +insmod all_video + +terminal_input console +terminal_output console + +menuentry "A - v1" { + set gfxmode=auto + set gfxpayload=text + linux /A/vmlinuz cmdline A + initrd /A/initramfs.xz +} + +menuentry "B - v2" { + set gfxmode=auto + set gfxpayload=text + linux /B/vmlinuz cmdline B + initrd /B/initramfs.xz +} + +menuentry "Reset Talos installation and return to maintenance mode" { + set gfxmode=auto + set gfxpayload=text + linux /A/vmlinuz cmdline A talos.experimental.wipe=system:EPHEMERAL,STATE + initrd /A/initramfs.xz +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/testdata/grub_write_no_reset_test.cfg b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/testdata/grub_write_no_reset_test.cfg new file mode 100644 index 0000000..44632d3 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/testdata/grub_write_no_reset_test.cfg @@ -0,0 +1,15 @@ +set default="A - TestOld v0.0.1" + +set timeout=3 + +insmod all_video + +terminal_input console +terminal_output console + +menuentry "A - TestOld v0.0.1" { + set gfxmode=auto + set gfxpayload=text + linux /A/vmlinuz cmdline A + initrd /A/initramfs.xz +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/testdata/grub_write_test.cfg b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/testdata/grub_write_test.cfg new file mode 100644 index 0000000..e76ec5c --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/testdata/grub_write_test.cfg @@ -0,0 +1,21 @@ +set default="A - Test v0.0.1" + +set timeout=3 + +insmod all_video + +terminal_input console +terminal_output console + +menuentry "A - Test v0.0.1" { + set gfxmode=auto + set gfxpayload=text + linux /A/vmlinuz cmdline A + initrd /A/initramfs.xz +} +menuentry "Reset Talos installation and return to maintenance mode" { + set gfxmode=auto + set gfxpayload=text + linux /A/vmlinuz cmdline A talos.experimental.wipe=system:EPHEMERAL,STATE + initrd /A/initramfs.xz +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/bootloader/mount/mount.go b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/mount/mount.go new file mode 100644 index 0000000..403b111 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/mount/mount.go @@ -0,0 +1,107 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package mount provides bootloader mount operations. +package mount + +import ( + "context" + "errors" + "fmt" + "os" + + "github.com/siderolabs/go-blockdevice/blockdevice" + "github.com/siderolabs/go-blockdevice/blockdevice/probe" + + "github.com/aenix-io/talm/internal/pkg/mount" +) + +// PartitionOp mounts a partition with the specified label, executes the operation func, and unmounts the partition. +// +//nolint:gocyclo +func PartitionOp(ctx context.Context, disk string, partitionLabel string, opFunc func() error) error { + var probedBlockDevice *blockdevice.BlockDevice + + switch { + case disk != "": + dev, err := blockdevice.Open(disk, blockdevice.WithMode(blockdevice.ReadonlyMode)) + if err != nil { + return err + } + + defer dev.Close() //nolint:errcheck + + _, err = dev.GetPartition(partitionLabel) + if err != nil { + if errors.Is(err, blockdevice.ErrMissingPartitionTable) || os.IsNotExist(err) { + return nil + } + + return err + } + + probedBlockDevice = dev + case disk == "": + // attempt to probe partition with partitionLabel on any disk + dev, err := probe.GetDevWithPartitionName(partitionLabel) + if os.IsNotExist(err) { + // no EFI partition, nothing to do + return nil + } + + if err != nil { + return err + } + + defer dev.Close() //nolint:errcheck + + probedBlockDevice = dev.BlockDevice + } + + mp, err := mount.SystemMountPointForLabel(ctx, probedBlockDevice, partitionLabel, mount.WithFlags(mount.ReadOnly)) + if err != nil { + return err + } + + // no mountpoint defined for this partition, should not happen + if mp == nil { + return fmt.Errorf("no mountpoint defined for %s", partitionLabel) + } + + alreadyMounted, err := mp.IsMounted() + if err != nil { + return err + } + + if !alreadyMounted { + if err = mp.Mount(); err != nil { + return err + } + + defer mp.Unmount() //nolint:errcheck + } + + return opFunc() +} + +// GetBlockDeviceName returns the block device name for the specified boot disk and partition label. +func GetBlockDeviceName(bootDisk, partitionLabel string) (string, error) { + dev, err := blockdevice.Open(bootDisk, blockdevice.WithMode(blockdevice.ReadonlyMode)) + if err != nil { + return "", err + } + + //nolint:errcheck + defer dev.Close() + + // verify that BootDisk has partition with the specified label + _, err = dev.GetPartition(partitionLabel) + if err != nil { + return "", err + } + + blk := dev.Device().Name() + + return blk, nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/bootloader/options/options.go b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/options/options.go new file mode 100644 index 0000000..4ff9719 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/options/options.go @@ -0,0 +1,82 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package options provides bootloader options. +package options + +import ( + "fmt" + + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// InstallOptions configures bootloader installation. +type InstallOptions struct { + // The disk to install to. + BootDisk string + // Target architecture. + Arch string + // Kernel command line (grub only). + Cmdline string + // Talos version. + Version string + + // Are we running in image mode? + ImageMode bool + + // Mount prefix for /boot-like partitions. + MountPrefix string + + // Boot assets to install. + BootAssets BootAssets + + // Printf-like function to use. + Printf func(format string, v ...any) +} + +// BootAssets describes the assets to be installed by the bootloader. +type BootAssets struct { + KernelPath string + InitramfsPath string + + UKIPath string + SDBootPath string + + DTBPath string + UBootPath string + RPiFirmwarePath string +} + +// FillDefaults fills in default paths to be used when in the context of the installer. +func (assets *BootAssets) FillDefaults(arch string) { + if assets.KernelPath == "" { + assets.KernelPath = fmt.Sprintf(constants.KernelAssetPath, arch) + } + + if assets.InitramfsPath == "" { + assets.InitramfsPath = fmt.Sprintf(constants.InitramfsAssetPath, arch) + } + + if assets.UKIPath == "" { + assets.UKIPath = fmt.Sprintf(constants.UKIAssetPath, arch) + } + + if assets.SDBootPath == "" { + assets.SDBootPath = fmt.Sprintf(constants.SDBootAssetPath, arch) + } + + if arch == "arm64" { + if assets.DTBPath == "" { + assets.DTBPath = fmt.Sprintf(constants.DTBAssetPath, arch) + } + + if assets.UBootPath == "" { + assets.UBootPath = fmt.Sprintf(constants.UBootAssetPath, arch) + } + + if assets.RPiFirmwarePath == "" { + assets.RPiFirmwarePath = fmt.Sprintf(constants.RPiFirmwareAssetPath, arch) + } + } +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/bootloader/sdboot/efivars.go b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/sdboot/efivars.go new file mode 100644 index 0000000..0b27ab7 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/sdboot/efivars.go @@ -0,0 +1,83 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package sdboot + +import ( + "errors" + + "github.com/ecks/uefi/efi/efiguid" + "github.com/ecks/uefi/efi/efivario" + "golang.org/x/sys/unix" + "golang.org/x/text/encoding/unicode" + + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +const ( + // SystemdBootGUIDString is the GUID of the SystemdBoot EFI variables. + SystemdBootGUIDString = "4a67b082-0a4c-41cf-b6c7-440b29bb8c4f" + // SystemdBootStubInfoPath is the path to the SystemdBoot StubInfo EFI variable. + SystemdBootStubInfoPath = constants.EFIVarsMountPoint + "/" + "StubInfo-" + SystemdBootGUIDString +) + +// SystemdBootGUID is the GUID of the SystemdBoot EFI variables. +var SystemdBootGUID = efiguid.MustFromString(SystemdBootGUIDString) + +// Variable names. +const ( + LoaderEntryDefaultName = "LoaderEntryDefault" + LoaderEntrySelectedName = "LoaderEntrySelected" + LoaderConfigTimeoutName = "LoaderConfigTimeout" +) + +// ReadVariable reads a SystemdBoot EFI variable. +func ReadVariable(c efivario.Context, name string) (string, error) { + _, data, err := efivario.ReadAll(c, name, SystemdBootGUID) + if err != nil { + if errors.Is(err, efivario.ErrNotFound) { + return "", nil + } + + return "", err + } + + out := make([]byte, len(data)) + + decoder := unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM).NewDecoder() + + n, _, err := decoder.Transform(out, data, true) + if err != nil { + return "", err + } + + if n > 0 && out[n-1] == 0 { + n-- + } + + return string(out[:n]), nil +} + +// WriteVariable reads a SystemdBoot EFI variable. +func WriteVariable(c efivario.Context, name, value string) error { + // mount EFI vars as rw + if err := unix.Mount("efivarfs", constants.EFIVarsMountPoint, "efivarfs", unix.MS_REMOUNT, ""); err != nil { + return err + } + + defer unix.Mount("efivarfs", constants.EFIVarsMountPoint, "efivarfs", unix.MS_REMOUNT|unix.MS_RDONLY, "") //nolint:errcheck + + out := make([]byte, (len(value)+1)*2) + + encoder := unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM).NewEncoder() + + n, _, err := encoder.Transform(out, []byte(value), true) + if err != nil { + return err + } + + out = append(out[:n], 0, 0) + + return c.Set(name, SystemdBootGUID, efivario.BootServiceAccess|efivario.RuntimeAccess|efivario.NonVolatile, out) +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/bootloader/sdboot/sdboot.go b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/sdboot/sdboot.go new file mode 100644 index 0000000..f2cc10d --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/sdboot/sdboot.go @@ -0,0 +1,233 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package sdboot provides the interface to the Systemd-Boot bootloader: config management, installation, etc. +package sdboot + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "path/filepath" + "strings" + + "github.com/ecks/uefi/efi/efivario" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/bootloader/mount" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/bootloader/options" + "github.com/siderolabs/talos/pkg/imager/utils" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// Config describe sd-boot state. +type Config struct { + Default string + Fallback string +} + +func isUEFIBoot() bool { + // https://renenyffenegger.ch/notes/Linux/fhs/sys/firmware/efi/index + _, err := os.Stat("/sys/firmware/efi") + + return err == nil +} + +// IsBootedUsingSDBoot returns true if the system is booted using sd-boot. +func IsBootedUsingSDBoot() bool { + // https://www.freedesktop.org/software/systemd/man/systemd-stub.html#EFI%20Variables + // https://www.freedesktop.org/software/systemd/man/systemd-stub.html#StubInfo + _, err := os.Stat(SystemdBootStubInfoPath) + + return err == nil +} + +// New creates a new sdboot bootloader config. +func New() *Config { + return &Config{} +} + +// Probe for existing sd-boot bootloader. +// +//nolint:gocyclo +func Probe(ctx context.Context, disk string) (*Config, error) { + // if not UEFI boot, nothing to do + if !isUEFIBoot() { + return nil, nil + } + + if !IsBootedUsingSDBoot() { + return nil, nil + } + + // read /boot/EFI and find if sd-boot is already being used + // this is to make sure sd-boot from Talos is being used and not sd-boot from another distro + if err := mount.PartitionOp(ctx, disk, constants.EFIPartitionLabel, func() error { + // list existing boot*.efi files in boot folder + files, err := filepath.Glob(filepath.Join(constants.EFIMountPoint, "EFI", "boot", "BOOT*.efi")) + if err != nil { + return err + } + + if len(files) == 0 { + return fmt.Errorf("no boot*.efi files found in %q", filepath.Join(constants.EFIMountPoint, "EFI", "boot")) + } + + return nil + }); err != nil { + return nil, err + } + + // here we need to read the EFI vars to see if we have any defaults + // and populate config accordingly + // https://www.freedesktop.org/software/systemd/man/systemd-boot.html#LoaderEntryDefault + // this should be set on install/upgrades + + efiCtx := efivario.NewDefaultContext() + + bootedEntry, err := ReadVariable(efiCtx, LoaderEntrySelectedName) + if err != nil { + return nil, err + } + + log.Printf("booted entry: %q", bootedEntry) + + if opErr := mount.PartitionOp(ctx, disk, constants.EFIPartitionLabel, func() error { + // list existing UKIs, and check if the current one is present + files, err := filepath.Glob(filepath.Join(constants.EFIMountPoint, "EFI", "Linux", "Talos-*.efi")) + if err != nil { + return err + } + + for _, file := range files { + if strings.EqualFold(filepath.Base(file), bootedEntry) { + return nil + } + } + + return fmt.Errorf("booted entry %q not found", bootedEntry) + }); opErr != nil { + return nil, opErr + } + + return &Config{ + Default: bootedEntry, + }, nil +} + +// UEFIBoot returns true if bootloader is UEFI-only. +func (c *Config) UEFIBoot() bool { + return true +} + +// Install the bootloader. +// +// Assumes that EFI partition is already mounted. +// Writes down the UKI and updates the EFI variables. +// +//nolint:gocyclo +func (c *Config) Install(options options.InstallOptions) error { + var sdbootFilename string + + switch options.Arch { + case "amd64": + sdbootFilename = "BOOTX64.efi" + case "arm64": + sdbootFilename = "BOOTAA64.efi" + default: + return fmt.Errorf("unsupported architecture: %s", options.Arch) + } + + // list existing UKIs, and clean up all but the current one (used to boot) + files, err := filepath.Glob(filepath.Join(options.MountPrefix, constants.EFIMountPoint, "EFI", "Linux", "Talos-*.efi")) + if err != nil { + return err + } + + // writing UKI by version-based filename here + ukiPath := fmt.Sprintf("%s-%s.efi", "Talos", options.Version) + + for _, file := range files { + if strings.EqualFold(filepath.Base(file), c.Default) { + if !strings.EqualFold(c.Default, ukiPath) { + // set fallback to the current default unless it matches the new install + c.Fallback = c.Default + } + + continue + } + + options.Printf("removing old UKI: %s", file) + + if err = os.Remove(file); err != nil { + return err + } + } + + if err := utils.CopyFiles( + options.Printf, + utils.SourceDestination( + options.BootAssets.UKIPath, + filepath.Join(options.MountPrefix, constants.EFIMountPoint, "EFI", "Linux", ukiPath), + ), + utils.SourceDestination( + options.BootAssets.SDBootPath, + filepath.Join(options.MountPrefix, constants.EFIMountPoint, "EFI", "boot", sdbootFilename), + ), + ); err != nil { + return err + } + + // don't update EFI variables if we're installing to a loop device + if !options.ImageMode { + options.Printf("updating EFI variables") + + efiCtx := efivario.NewDefaultContext() + + // set the new entry as a default one + if err := WriteVariable(efiCtx, LoaderEntryDefaultName, ukiPath); err != nil { + return err + } + + // set default 5 second boot timeout + if err := WriteVariable(efiCtx, LoaderConfigTimeoutName, "5"); err != nil { + return err + } + } + + return nil +} + +// PreviousLabel returns the label of the previous bootloader version. +func (c *Config) PreviousLabel() string { + return c.Fallback +} + +// Revert the bootloader to the previous version. +func (c *Config) Revert(ctx context.Context) error { + if err := mount.PartitionOp(ctx, "", constants.EFIPartitionLabel, func() error { + // use c.Default as the current entry, list other UKIs, find the one which is not c.Default, and update EFI var + files, err := filepath.Glob(filepath.Join(constants.EFIMountPoint, "EFI", "Linux", "Talos-*.efi")) + if err != nil { + return err + } + + for _, file := range files { + if strings.EqualFold(filepath.Base(file), c.Default) { + continue + } + + log.Printf("reverting to previous UKI: %s", file) + + return WriteVariable(efivario.NewDefaultContext(), LoaderEntryDefaultName, filepath.Base(file)) + } + + return errors.New("previous UKI not found") + }); err != nil { + return err + } + + return nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/doc.go b/internal/app/machined/pkg/runtime/v1alpha1/doc.go new file mode 100644 index 0000000..e642017 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/doc.go @@ -0,0 +1,6 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package v1alpha1 implements a `Runtime`. +package v1alpha1 diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/akamai/akamai.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/akamai/akamai.go new file mode 100644 index 0000000..432f010 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/akamai/akamai.go @@ -0,0 +1,206 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package akamai contains the Akamai implementation of the [platform.Platform]. +package akamai + +import ( + "context" + "fmt" + "net/netip" + "strconv" + "strings" + + "github.com/cosi-project/runtime/pkg/state" + akametadata "github.com/linode/go-metadata" + "github.com/siderolabs/go-procfs/procfs" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/internal/netutils" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// Akamai is the concrete type that implements the platform.Platform interface. +type Akamai struct{} + +// Name implements the platform.Platform interface. +func (a *Akamai) Name() string { + return "akamai" +} + +// ParseMetadata converts Akamai platform metadata into platform network config. +func (a *Akamai) ParseMetadata(metadata *akametadata.InstanceData, interfaceAddresses *akametadata.NetworkData) (*runtime.PlatformNetworkConfig, error) { + networkConfig := &runtime.PlatformNetworkConfig{} + + if metadata.Label != "" { + hostnameSpec := network.HostnameSpecSpec{ + ConfigLayer: network.ConfigPlatform, + } + + if err := hostnameSpec.ParseFQDN(metadata.Label); err != nil { + return nil, err + } + + networkConfig.Hostnames = append(networkConfig.Hostnames, hostnameSpec) + } + + publicIPs := make([]string, 0, len(interfaceAddresses.IPv4.Public)+len(interfaceAddresses.IPv6.Ranges)) + + // external IP + for _, iface := range interfaceAddresses.IPv4.Public { + publicIPs = append(publicIPs, iface.Addr().String()) + networkConfig.Addresses = append(networkConfig.Addresses, + network.AddressSpecSpec{ + ConfigLayer: network.ConfigPlatform, + LinkName: "eth0", + Address: iface, + Scope: nethelpers.ScopeGlobal, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + Family: nethelpers.FamilyInet4, + }, + ) + } + + for _, iface := range interfaceAddresses.IPv4.Private { + networkConfig.Addresses = append(networkConfig.Addresses, + network.AddressSpecSpec{ + ConfigLayer: network.ConfigPlatform, + LinkName: "eth0", + Address: iface, + Scope: nethelpers.ScopeGlobal, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + Family: nethelpers.FamilyInet4, + }, + ) + } + + for _, iface := range interfaceAddresses.IPv6.Ranges { + publicIPs = append(publicIPs, iface.Addr().String()) + + networkConfig.Addresses = append(networkConfig.Addresses, + network.AddressSpecSpec{ + ConfigLayer: network.ConfigPlatform, + LinkName: "eth0", + Address: iface, + Scope: nethelpers.ScopeGlobal, + Flags: nethelpers.AddressFlags(nethelpers.AddressManagementTemp), + Family: nethelpers.FamilyInet6, + }, + ) + } + + networkConfig.Addresses = append(networkConfig.Addresses, + network.AddressSpecSpec{ + ConfigLayer: network.ConfigPlatform, + LinkName: "eth0", + Address: interfaceAddresses.IPv6.LinkLocal, + Scope: nethelpers.ScopeLink, + Family: nethelpers.FamilyInet6, + }, + ) + + ipv6gw, err := netip.ParseAddr(strings.Split(interfaceAddresses.IPv6.LinkLocal.String(), ":")[0] + "::1") + if err != nil { + return nil, err + } + + route := network.RouteSpecSpec{ + ConfigLayer: network.ConfigPlatform, + Gateway: ipv6gw, + OutLinkName: "eth0", + Destination: interfaceAddresses.IPv6.LinkLocal, + Table: nethelpers.TableMain, + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + Family: nethelpers.FamilyInet6, + Priority: 1024, + } + + route.Normalize() + + networkConfig.Routes = append(networkConfig.Routes, route) + + for _, ipStr := range publicIPs { + if ip, err := netip.ParseAddr(ipStr); err == nil { + networkConfig.ExternalIPs = append(networkConfig.ExternalIPs, ip) + } + } + + networkConfig.Metadata = &runtimeres.PlatformMetadataSpec{ + Platform: a.Name(), + Hostname: metadata.Label, + Region: metadata.Region, + InstanceType: metadata.Type, + InstanceID: strconv.Itoa(metadata.ID), + ProviderID: fmt.Sprintf("linode://%d", metadata.ID), + } + + return networkConfig, nil +} + +// Configuration implements the platform.Platform interface. +func (a *Akamai) Configuration(ctx context.Context, r state.State) ([]byte, error) { + if err := netutils.Wait(ctx, r); err != nil { + return nil, err + } + + metadataClient, err := akametadata.NewClient(ctx) + if err != nil { + return nil, fmt.Errorf("new metadata client: %w", err) + } + + userData, err := metadataClient.GetUserData(ctx) + if err != nil { + return nil, fmt.Errorf("get user data: %w", err) + } + + return []byte(userData), err +} + +// Mode implements the platform.Platform interface. +func (a *Akamai) Mode() runtime.Mode { + return runtime.ModeCloud +} + +// KernelArgs implements the runtime.Platform interface. +func (a *Akamai) KernelArgs(string) procfs.Parameters { + return []*procfs.Parameter{ + procfs.NewParameter("console").Append("ttyS0").Append("tty0").Append("tty1"), + procfs.NewParameter(constants.KernelParamNetIfnames).Append("0"), + } +} + +// NetworkConfiguration implements the runtime.Platform interface. +func (a *Akamai) NetworkConfiguration(ctx context.Context, _ state.State, ch chan<- *runtime.PlatformNetworkConfig) error { + metadataClient, err := akametadata.NewClient(ctx) + if err != nil { + return fmt.Errorf("new metadata client: %w", err) + } + + metadata, err := metadataClient.GetInstance(ctx) + if err != nil { + return fmt.Errorf("get instance data: %w", err) + } + + metadataNetworkConfig, err := metadataClient.GetNetwork(ctx) + if err != nil { + return fmt.Errorf("get network data: %w", err) + } + + networkConfig, err := a.ParseMetadata(metadata, metadataNetworkConfig) + if err != nil { + return fmt.Errorf("parse metadata: %w", err) + } + + select { + case ch <- networkConfig: + case <-ctx.Done(): + return ctx.Err() + } + + return nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/akamai/akamai_test.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/akamai/akamai_test.go new file mode 100644 index 0000000..6611858 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/akamai/akamai_test.go @@ -0,0 +1,48 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package akamai_test + +import ( + _ "embed" + "encoding/json" + "testing" + + akametadata "github.com/linode/go-metadata" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/akamai" +) + +//go:embed testdata/instance.json +var rawMetadata []byte + +//go:embed testdata/network.json + +var rawNetwork []byte + +//go:embed testdata/expected.yaml +var expectedNetworkConfig string + +func TestParseMetadata(t *testing.T) { + p := &akamai.Akamai{} + + var metadata akametadata.InstanceData + + var interfaceConfig akametadata.NetworkData + + require.NoError(t, json.Unmarshal(rawMetadata, &metadata)) + + require.NoError(t, json.Unmarshal(rawNetwork, &interfaceConfig)) + + networkConfig, err := p.ParseMetadata(&metadata, &interfaceConfig) + require.NoError(t, err) + + marshaled, err := yaml.Marshal(networkConfig) + require.NoError(t, err) + + assert.Equal(t, expectedNetworkConfig, string(marshaled)) +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/akamai/testdata/expected.yaml b/internal/app/machined/pkg/runtime/v1alpha1/platform/akamai/testdata/expected.yaml new file mode 100644 index 0000000..4fa14ba --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/akamai/testdata/expected.yaml @@ -0,0 +1,56 @@ +addresses: + - address: 172.1.2.3/32 + linkName: eth0 + family: inet4 + scope: global + flags: permanent + layer: platform + - address: 192.1.2.3/32 + linkName: eth0 + family: inet4 + scope: global + flags: permanent + layer: platform + - address: 2600:3c05:d011:797::/64 + linkName: eth0 + family: inet6 + scope: global + flags: mngmtmpaddr + layer: platform + - address: fe80::f03c:93ff:fe6e:5cd9/128 + linkName: eth0 + family: inet6 + scope: link + flags: "" + layer: platform +links: [] +routes: + - family: inet6 + dst: fe80::f03c:93ff:fe6e:5cd9/128 + src: "" + gateway: fe80::1 + outLinkName: eth0 + table: main + priority: 1024 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform +hostnames: + - hostname: talos + domainname: "" + layer: platform +resolvers: [] +timeServers: [] +operators: [] +externalIPs: + - 172.1.2.3 + - '2600:3c05:d011:797::' +metadata: + platform: akamai + hostname: talos + region: us-east + instanceType: g6-standard-1 + instanceId: "123456" + providerId: linode://123456 diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/akamai/testdata/instance.json b/internal/app/machined/pkg/runtime/v1alpha1/platform/akamai/testdata/instance.json new file mode 100644 index 0000000..2af1a22 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/akamai/testdata/instance.json @@ -0,0 +1,19 @@ +{ + "id": 123456, + "label": "talos", + "region": "us-east", + "type": "g6-standard-1", + "specs": { + "vcpus": 1, + "memory": 2048, + "gpus": 0, + "transfer": 2000, + "disk": 51200 + }, + "backups": { + "enabled": false, + "status": null + }, + "host_uuid": "0c2897331ea446f483f754852b18a67c", + "tags": [] +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/akamai/testdata/network.json b/internal/app/machined/pkg/runtime/v1alpha1/platform/akamai/testdata/network.json new file mode 100644 index 0000000..1a11c13 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/akamai/testdata/network.json @@ -0,0 +1,20 @@ +{ + "interfaces": [], + "ipv4": { + "public": [ + "172.1.2.3/32" + ], + "private": [ + "192.1.2.3/32" + ], + "shared": [] + }, + "ipv6": { + "slaac": "2600:3c06::f03c:93ff:fe6e:5cd9/128", + "ranges": [ + "2600:3c05:d011:797::/64" + ], + "link_local": "fe80::f03c:93ff:fe6e:5cd9/128", + "shared_ranges": [] + } +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/aws/aws.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/aws/aws.go new file mode 100644 index 0000000..ac3e892 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/aws/aws.go @@ -0,0 +1,171 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package aws contains the AWS implementation of the [platform.Platform]. +package aws + +import ( + "context" + "fmt" + "io" + "log" + "net/netip" + "strings" + + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/feature/ec2/imds" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-procfs/procfs" + "github.com/siderolabs/go-retry/retry" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/errors" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/internal/netutils" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// AWS is the concrete type that implements the runtime.Platform interface. +type AWS struct { + metadataClient *imds.Client +} + +// NewAWS initializes AWS platform building the IMDS client. +func NewAWS() (*AWS, error) { + a := &AWS{} + + cfg, err := config.LoadDefaultConfig(context.TODO()) + if err != nil { + return nil, fmt.Errorf("error initializing AWS default config: %w", err) + } + + a.metadataClient = imds.NewFromConfig(cfg) + + return a, nil +} + +// ParseMetadata converts AWS platform metadata into platform network config. +func (a *AWS) ParseMetadata(metadata *MetadataConfig) (*runtime.PlatformNetworkConfig, error) { + networkConfig := &runtime.PlatformNetworkConfig{} + + if metadata.Hostname != "" { + hostnameSpec := network.HostnameSpecSpec{ + ConfigLayer: network.ConfigPlatform, + } + + if err := hostnameSpec.ParseFQDN(metadata.Hostname); err != nil { + return nil, err + } + + networkConfig.Hostnames = append(networkConfig.Hostnames, hostnameSpec) + } + + publicIPs := []string{} + + if metadata.PublicIPv4 != "" { + publicIPs = append(publicIPs, metadata.PublicIPv4) + } + + if metadata.PublicIPv6 != "" { + publicIPs = append(publicIPs, metadata.PublicIPv6) + } + + for _, ipStr := range publicIPs { + if ip, err := netip.ParseAddr(ipStr); err == nil { + networkConfig.ExternalIPs = append(networkConfig.ExternalIPs, ip) + } + } + + networkConfig.Metadata = &runtimeres.PlatformMetadataSpec{ + Platform: a.Name(), + Hostname: metadata.Hostname, + Region: metadata.Region, + Zone: metadata.Zone, + InstanceType: metadata.InstanceType, + InstanceID: metadata.InstanceID, + ProviderID: fmt.Sprintf("aws:///%s/%s", metadata.Zone, metadata.InstanceID), + Spot: metadata.InstanceLifeCycle == "spot", + } + + return networkConfig, nil +} + +// Name implements the runtime.Platform interface. +func (a *AWS) Name() string { + return "aws" +} + +// Configuration implements the runtime.Platform interface. +func (a *AWS) Configuration(ctx context.Context, r state.State) ([]byte, error) { + if err := netutils.Wait(ctx, r); err != nil { + return nil, err + } + + log.Printf("fetching machine config from AWS") + + userdata, err := netutils.RetryFetch(ctx, a.fetchConfiguration) + if err != nil { + return nil, err + } + + if strings.TrimSpace(userdata) == "" { + return nil, errors.ErrNoConfigSource + } + + return []byte(userdata), nil +} + +func (a *AWS) fetchConfiguration(ctx context.Context) (string, error) { + resp, err := a.metadataClient.GetUserData(ctx, &imds.GetUserDataInput{}) + if err != nil { + if isNotFoundError(err) { + return "", errors.ErrNoConfigSource + } + + return "", retry.ExpectedErrorf("failed to fetch EC2 userdata: %w", err) + } + + defer resp.Content.Close() //nolint:errcheck + + userdata, err := io.ReadAll(resp.Content) + + return string(userdata), err +} + +// Mode implements the runtime.Platform interface. +func (a *AWS) Mode() runtime.Mode { + return runtime.ModeCloud +} + +// KernelArgs implements the runtime.Platform interface. +func (a *AWS) KernelArgs(string) procfs.Parameters { + return []*procfs.Parameter{ + procfs.NewParameter("console").Append("tty1").Append("ttyS0"), + procfs.NewParameter(constants.KernelParamNetIfnames).Append("0"), + } +} + +// NetworkConfiguration implements the runtime.Platform interface. +func (a *AWS) NetworkConfiguration(ctx context.Context, _ state.State, ch chan<- *runtime.PlatformNetworkConfig) error { + log.Printf("fetching aws instance config") + + metadata, err := a.getMetadata(ctx) + if err != nil { + return err + } + + networkConfig, err := a.ParseMetadata(metadata) + if err != nil { + return err + } + + select { + case ch <- networkConfig: + case <-ctx.Done(): + return ctx.Err() + } + + return nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/aws/aws_test.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/aws/aws_test.go new file mode 100644 index 0000000..756ad82 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/aws/aws_test.go @@ -0,0 +1,39 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package aws_test + +import ( + _ "embed" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/aws" +) + +//go:embed testdata/metadata.json +var rawMetadata []byte + +//go:embed testdata/expected.yaml +var expectedNetworkConfig string + +func TestEmpty(t *testing.T) { + p := &aws.AWS{} + + var metadata aws.MetadataConfig + + require.NoError(t, json.Unmarshal(rawMetadata, &metadata)) + + networkConfig, err := p.ParseMetadata(&metadata) + require.NoError(t, err) + + marshaled, err := yaml.Marshal(networkConfig) + require.NoError(t, err) + + assert.Equal(t, expectedNetworkConfig, string(marshaled)) +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/aws/metadata.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/aws/metadata.go new file mode 100644 index 0000000..9b62966 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/aws/metadata.go @@ -0,0 +1,99 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package aws + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + + "github.com/aws/aws-sdk-go-v2/feature/ec2/imds" + smithyhttp "github.com/aws/smithy-go/transport/http" +) + +// MetadataConfig represents a metadata AWS instance. +type MetadataConfig struct { + Hostname string `json:"hostname,omitempty"` + InstanceID string `json:"instance-id,omitempty"` + InstanceType string `json:"instance-type,omitempty"` + InstanceLifeCycle string `json:"instance-life-cycle,omitempty"` + PublicIPv4 string `json:"public-ipv4,omitempty"` + PublicIPv6 string `json:"ipv6,omitempty"` + Region string `json:"region,omitempty"` + Zone string `json:"zone,omitempty"` +} + +//nolint:gocyclo +func (a *AWS) getMetadata(ctx context.Context) (*MetadataConfig, error) { + getMetadataKey := func(key string) (string, error) { + resp, err := a.metadataClient.GetMetadata(ctx, &imds.GetMetadataInput{ + Path: key, + }) + if err != nil { + if isNotFoundError(err) { + return "", nil + } + + return "", fmt.Errorf("failed to fetch %q from IMDS: %w", key, err) + } + + defer resp.Content.Close() //nolint:errcheck + + v, err := io.ReadAll(resp.Content) + + return string(v), err + } + + var ( + metadata MetadataConfig + err error + ) + + if metadata.Hostname, err = getMetadataKey("hostname"); err != nil { + return nil, err + } + + if metadata.InstanceType, err = getMetadataKey("instance-type"); err != nil { + return nil, err + } + + if metadata.InstanceLifeCycle, err = getMetadataKey("instance-life-cycle"); err != nil { + return nil, err + } + + if metadata.InstanceID, err = getMetadataKey("instance-id"); err != nil { + return nil, err + } + + if metadata.PublicIPv4, err = getMetadataKey("public-ipv4"); err != nil { + return nil, err + } + + if metadata.PublicIPv6, err = getMetadataKey("ipv6"); err != nil { + return nil, err + } + + if metadata.Region, err = getMetadataKey("placement/region"); err != nil { + return nil, err + } + + if metadata.Zone, err = getMetadataKey("placement/availability-zone"); err != nil { + return nil, err + } + + return &metadata, nil +} + +func isNotFoundError(err error) bool { + var awsErr *smithyhttp.ResponseError + + if errors.As(err, &awsErr) { + return awsErr.HTTPStatusCode() == http.StatusNotFound + } + + return false +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/aws/testdata/expected.yaml b/internal/app/machined/pkg/runtime/v1alpha1/platform/aws/testdata/expected.yaml new file mode 100644 index 0000000..15bec75 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/aws/testdata/expected.yaml @@ -0,0 +1,19 @@ +addresses: [] +links: [] +routes: [] +hostnames: + - hostname: talos + domainname: "" + layer: platform +resolvers: [] +timeServers: [] +operators: [] +externalIPs: + - 1.2.3.4 +metadata: + platform: aws + hostname: talos + region: us-east-1 + zone: us-east-1a + instanceId: i-0a0a0a0a0a0a0a0a0 + providerId: aws:///us-east-1a/i-0a0a0a0a0a0a0a0a0 diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/aws/testdata/metadata.json b/internal/app/machined/pkg/runtime/v1alpha1/platform/aws/testdata/metadata.json new file mode 100644 index 0000000..a0358b1 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/aws/testdata/metadata.json @@ -0,0 +1,7 @@ +{ + "hostname": "talos", + "instance-id": "i-0a0a0a0a0a0a0a0a0", + "public-ipv4": "1.2.3.4", + "region": "us-east-1", + "zone": "us-east-1a" +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/azure/azure.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/azure/azure.go new file mode 100644 index 0000000..487deb1 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/azure/azure.go @@ -0,0 +1,346 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package azure contains the Azure implementation of the [platform.Platform]. +package azure + +import ( + "context" + "encoding/base64" + "encoding/json" + "encoding/xml" + stderrors "errors" + "fmt" + "log" + "net/netip" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-procfs/procfs" + "golang.org/x/sys/unix" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/errors" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/internal/netutils" + "github.com/siderolabs/talos/pkg/download" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// NetworkConfig holds network interface meta config. +type NetworkConfig struct { + IPv4 struct { + IPAddresses []IPAddresses `json:"ipAddress"` + } `json:"ipv4"` + IPv6 struct { + IPAddresses []IPAddresses `json:"ipAddress"` + } `json:"ipv6"` +} + +// IPAddresses holds public/private IPs. +type IPAddresses struct { + PrivateIPAddress string `json:"privateIpAddress"` + PublicIPAddress string `json:"publicIpAddress"` +} + +// LoadBalancerMetadata represents load balancer metadata in IMDS. +type LoadBalancerMetadata struct { + LoadBalancer struct { + PublicIPAddresses []struct { + FrontendIPAddress string `json:"frontendIpAddress,omitempty"` + PrivateIPAddress string `json:"privateIpAddress,omitempty"` + } `json:"publicIpAddresses,omitempty"` + } `json:"loadbalancer,omitempty"` +} + +// Azure is the concrete type that implements the platform.Platform interface. +type Azure struct{} + +// ovfXML is a simple struct to help us fish custom data out from the ovf-env.xml file. +type ovfXML struct { + XMLName xml.Name `xml:"Environment"` + CustomData string `xml:"ProvisioningSection>LinuxProvisioningConfigurationSet>CustomData"` +} + +// Name implements the platform.Platform interface. +func (a *Azure) Name() string { + return "azure" +} + +// ParseMetadata parses Azure network metadata into the platform network config. +// +//nolint:gocyclo +func (a *Azure) ParseMetadata(metadata *ComputeMetadata, interfaceAddresses []NetworkConfig, host []byte) (*runtime.PlatformNetworkConfig, error) { + var networkConfig runtime.PlatformNetworkConfig + + // hostname + if len(host) > 0 { + hostnameSpec := network.HostnameSpecSpec{ + ConfigLayer: network.ConfigPlatform, + } + + if err := hostnameSpec.ParseFQDN(string(host)); err != nil { + return nil, err + } + + networkConfig.Hostnames = append(networkConfig.Hostnames, hostnameSpec) + } + + publicIPs := []string{} + + // external IP + for _, iface := range interfaceAddresses { + for _, ipv4addr := range iface.IPv4.IPAddresses { + publicIPs = append(publicIPs, ipv4addr.PublicIPAddress) + } + + for _, ipv6addr := range iface.IPv6.IPAddresses { + publicIPs = append(publicIPs, ipv6addr.PublicIPAddress) + } + } + + // DHCP6 for enabled interfaces + for idx, iface := range interfaceAddresses { + ipv6 := false + + for _, ipv6addr := range iface.IPv6.IPAddresses { + ipv6 = ipv6addr.PublicIPAddress != "" || ipv6addr.PrivateIPAddress != "" + } + + if ipv6 { + ifname := fmt.Sprintf("eth%d", idx) + + networkConfig.Operators = append(networkConfig.Operators, network.OperatorSpecSpec{ + Operator: network.OperatorDHCP6, + LinkName: ifname, + RequireUp: true, + DHCP6: network.DHCP6OperatorSpec{ + RouteMetric: 2 * network.DefaultRouteMetric, + }, + }) + + // If accept_ra is not set, use the default gateway. + route := network.RouteSpecSpec{ + ConfigLayer: network.ConfigPlatform, + Gateway: netip.MustParseAddr("fe80::1234:5678:9abc"), + OutLinkName: ifname, + Table: nethelpers.TableMain, + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + Family: nethelpers.FamilyInet6, + Priority: 4 * network.DefaultRouteMetric, + } + + route.Normalize() + + networkConfig.Routes = append(networkConfig.Routes, route) + } + } + + for _, ipStr := range publicIPs { + if ip, err := netip.ParseAddr(ipStr); err == nil { + networkConfig.ExternalIPs = append(networkConfig.ExternalIPs, ip) + } + } + + zone := metadata.FaultDomain + if metadata.Zone != "" { + zone = fmt.Sprintf("%s-%s", metadata.Location, metadata.Zone) + } + + networkConfig.Metadata = &runtimeres.PlatformMetadataSpec{ + Platform: a.Name(), + Hostname: metadata.OSProfile.ComputerName, + Region: strings.ToLower(metadata.Location), + Zone: strings.ToLower(zone), + InstanceType: metadata.VMSize, + InstanceID: metadata.ResourceID, + ProviderID: fmt.Sprintf("azure://%s", metadata.ResourceID), + Spot: metadata.EvictionPolicy != "", + } + + return &networkConfig, nil +} + +// ParseLoadBalancerIP parses Azure LoadBalancer metadata into the platform external ip list. +func (a *Azure) ParseLoadBalancerIP(lbConfig LoadBalancerMetadata, exIP []netip.Addr) ([]netip.Addr, error) { + lbAddresses := exIP + + for _, addr := range lbConfig.LoadBalancer.PublicIPAddresses { + ipaddr := addr.FrontendIPAddress + + if i := strings.IndexByte(ipaddr, ']'); i != -1 { + ipaddr = strings.TrimPrefix(ipaddr[:i], "[") + } + + if ip, err := netip.ParseAddr(ipaddr); err == nil { + lbAddresses = append(lbAddresses, ip) + } + } + + return lbAddresses, nil +} + +// Configuration implements the platform.Platform interface. +func (a *Azure) Configuration(ctx context.Context, r state.State) ([]byte, error) { + defer func() { + if err := netutils.Wait(ctx, r); err != nil { + log.Printf("failed to wait for network, err: %s", err) + } + + if err := linuxAgent(ctx); err != nil { + log.Printf("failed to update instance status, err: %s", err) + } + }() + + log.Printf("fetching machine config from ovf-env.xml") + + // Custom data is not available in IMDS, so trying to find it on CDROM. + return a.configFromCD() +} + +// Mode implements the platform.Platform interface. +func (a *Azure) Mode() runtime.Mode { + return runtime.ModeCloud +} + +// KernelArgs implements the runtime.Platform interface. +func (a *Azure) KernelArgs(string) procfs.Parameters { + return []*procfs.Parameter{ + procfs.NewParameter("console").Append("ttyS0,115200n8"), + procfs.NewParameter("earlyprintk").Append("ttyS0,115200"), + procfs.NewParameter("rootdelay").Append("300"), + procfs.NewParameter(constants.KernelParamNetIfnames).Append("0"), + procfs.NewParameter(constants.KernelParamDashboardDisabled).Append("1"), + } +} + +// configFromCD handles looking for devices and trying to mount/fetch xml to get the custom data. +// +//nolint:gocyclo +func (a *Azure) configFromCD() ([]byte, error) { + devList, err := os.ReadDir("/dev") + if err != nil { + return nil, err + } + + diskRegex := regexp.MustCompile("(sr[0-9]|hd[c-z]|cdrom[0-9]|cd[0-9])") + + for _, dev := range devList { + if diskRegex.MatchString(dev.Name()) { + fmt.Printf("found matching device. checking for ovf-env.xml: %s\n", dev.Name()) + + // Mount and slurp xml from disk + if err = unix.Mount(filepath.Join("/dev", dev.Name()), mnt, "udf", unix.MS_RDONLY, ""); err != nil { + fmt.Printf("unable to mount %s, possibly not udf: %s", dev.Name(), err.Error()) + + continue + } + + ovfEnvFile, err := os.ReadFile(filepath.Join(mnt, "ovf-env.xml")) + if err != nil { + // Device mount worked, but it wasn't the "CD" that contains the xml file + if os.IsNotExist(err) { + continue + } + + return nil, fmt.Errorf("failed to read config: %w", err) + } + + if err = unix.Unmount(mnt, 0); err != nil { + return nil, fmt.Errorf("failed to unmount: %w", err) + } + + // Unmarshall xml we slurped + ovfEnvData := ovfXML{} + + err = xml.Unmarshal(ovfEnvFile, &ovfEnvData) + if err != nil { + return nil, err + } + + if len(ovfEnvData.CustomData) > 0 { + b64CustomData, err := base64.StdEncoding.DecodeString(ovfEnvData.CustomData) + if err != nil { + return nil, err + } + + return b64CustomData, nil + } + + return nil, errors.ErrNoConfigSource + } + } + + return nil, errors.ErrNoConfigSource +} + +// NetworkConfiguration implements the runtime.Platform interface. +// +//nolint:gocyclo +func (a *Azure) NetworkConfiguration(ctx context.Context, _ state.State, ch chan<- *runtime.PlatformNetworkConfig) error { + log.Printf("fetching azure instance config from: %q", AzureMetadataEndpoint) + + metadata, err := a.getMetadata(ctx) + if err != nil { + return err + } + + log.Printf("fetching network config from %q", AzureInterfacesEndpoint) + + metadataNetworkConfig, err := download.Download(ctx, AzureInterfacesEndpoint, + download.WithHeaders(map[string]string{"Metadata": "true"})) + if err != nil { + return fmt.Errorf("failed to fetch network config from metadata service: %w", err) + } + + var interfaceAddresses []NetworkConfig + + if err = json.Unmarshal(metadataNetworkConfig, &interfaceAddresses); err != nil { + return err + } + + networkConfig, err := a.ParseMetadata(metadata, interfaceAddresses, []byte(metadata.OSProfile.ComputerName)) + if err != nil { + return fmt.Errorf("failed to parse network metadata: %w", err) + } + + log.Printf("fetching load balancer metadata from: %q", AzureLoadbalancerEndpoint) + + var loadBalancerAddresses LoadBalancerMetadata + + lbConfig, err := download.Download(ctx, AzureLoadbalancerEndpoint, + download.WithHeaders(map[string]string{"Metadata": "true"}), + download.WithErrorOnNotFound(errors.ErrNoConfigSource), + download.WithErrorOnEmptyResponse(errors.ErrNoConfigSource)) + if err != nil && !stderrors.Is(err, errors.ErrNoConfigSource) { + log.Printf("failed to fetch load balancer config from metadata service: %s", err) + + lbConfig = nil + } + + if len(lbConfig) > 0 { + if err = json.Unmarshal(lbConfig, &loadBalancerAddresses); err != nil { + return fmt.Errorf("failed to parse loadbalancer metadata: %w", err) + } + + networkConfig.ExternalIPs, err = a.ParseLoadBalancerIP(loadBalancerAddresses, networkConfig.ExternalIPs) + if err != nil { + return fmt.Errorf("failed to define externalIPs: %w", err) + } + } + + select { + case ch <- networkConfig: + case <-ctx.Done(): + return ctx.Err() + } + + return nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/azure/azure_test.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/azure/azure_test.go new file mode 100644 index 0000000..1b2849a --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/azure/azure_test.go @@ -0,0 +1,56 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package azure_test + +import ( + _ "embed" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/azure" +) + +//go:embed testdata/interfaces.json +var rawInterfaces []byte + +//go:embed testdata/compute.json +var rawCompute []byte + +//go:embed testdata/loadbalancer.json +var rawLoadBalancerMetadata []byte + +//go:embed testdata/expected.yaml +var expectedNetworkConfig string + +func TestParseMetadata(t *testing.T) { + a := &azure.Azure{} + + var interfacesMetadata []azure.NetworkConfig + + require.NoError(t, json.Unmarshal(rawInterfaces, &interfacesMetadata)) + + var computeMetadata azure.ComputeMetadata + + require.NoError(t, json.Unmarshal(rawCompute, &computeMetadata)) + + networkConfig, err := a.ParseMetadata(&computeMetadata, interfacesMetadata, []byte("some.fqdn")) + require.NoError(t, err) + + var lb azure.LoadBalancerMetadata + + require.NoError(t, json.Unmarshal(rawLoadBalancerMetadata, &lb)) + + networkConfig.ExternalIPs, err = a.ParseLoadBalancerIP(lb, networkConfig.ExternalIPs) + require.NoError(t, err) + + marshaled, err := yaml.Marshal(networkConfig) + require.NoError(t, err) + + assert.Equal(t, expectedNetworkConfig, string(marshaled)) +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/azure/metadata.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/azure/metadata.go new file mode 100644 index 0000000..04c25d6 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/azure/metadata.go @@ -0,0 +1,67 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package azure + +import ( + "context" + "encoding/json" + stderrors "errors" + "fmt" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/errors" + "github.com/siderolabs/talos/pkg/download" +) + +const ( + // AzureInternalEndpoint is the Azure Internal Channel IP + // https://blogs.msdn.microsoft.com/mast/2015/05/18/what-is-the-ip-address-168-63-129-16/ + AzureInternalEndpoint = "http://168.63.129.16" + // AzureMetadataEndpoint is the local endpoint for the metadata. + AzureMetadataEndpoint = "http://169.254.169.254/metadata/instance/compute?api-version=2021-12-13&format=json" + // AzureInterfacesEndpoint is the local endpoint to get external IPs. + AzureInterfacesEndpoint = "http://169.254.169.254/metadata/instance/network/interface?api-version=2021-12-13&format=json" + // AzureLoadbalancerEndpoint is the local endpoint for load balancer config. + AzureLoadbalancerEndpoint = "http://169.254.169.254/metadata/loadbalancer?api-version=2021-05-01&format=json" + + mnt = "/mnt" +) + +// ComputeMetadata represents metadata compute information. +type ComputeMetadata struct { + Environment string `json:"azEnvironment,omitempty"` + SKU string `json:"sku,omitempty"` + Name string `json:"name,omitempty"` + Zone string `json:"zone,omitempty"` + VMSize string `json:"vmSize,omitempty"` + OSType string `json:"osType,omitempty"` + OSProfile struct { + ComputerName string `json:"computerName,omitempty"` + } `json:"osProfile,omitempty"` + Location string `json:"location,omitempty"` + FaultDomain string `json:"platformFaultDomain,omitempty"` + PlatformSubFaultDomain string `json:"platformSubFaultDomain,omitempty"` + UpdateDomain string `json:"platformUpdateDomain,omitempty"` + ResourceGroup string `json:"resourceGroupName,omitempty"` + ResourceID string `json:"resourceId,omitempty"` + VMScaleSetName string `json:"vmScaleSetName,omitempty"` + SubscriptionID string `json:"subscriptionId,omitempty"` + EvictionPolicy string `json:"evictionPolicy,omitempty"` +} + +func (a *Azure) getMetadata(ctx context.Context) (*ComputeMetadata, error) { + metadataDl, err := download.Download(ctx, AzureMetadataEndpoint, + download.WithHeaders(map[string]string{"Metadata": "true"})) + if err != nil && !stderrors.Is(err, errors.ErrNoHostname) { + return nil, fmt.Errorf("error fetching metadata: %w", err) + } + + var metadata ComputeMetadata + + if err = json.Unmarshal(metadataDl, &metadata); err != nil { + return nil, fmt.Errorf("failed to parse compute metadata: %w", err) + } + + return &metadata, nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/azure/register.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/azure/register.go new file mode 100644 index 0000000..83602cb --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/azure/register.go @@ -0,0 +1,214 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package azure + +import ( + "bytes" + "context" + "encoding/xml" + "fmt" + "io" + "net/http" + "net/url" + + "github.com/siderolabs/talos/pkg/download" +) + +// This should provide the bare minimum to trigger a node in ready condition to allow +// azure to be happy with the node and let it on it's lawn. +func linuxAgent(ctx context.Context) (err error) { + var gs *GoalState + + gs, err = goalState(ctx) + if err != nil { + return fmt.Errorf("failed to register with Azure and fetch GoalState XML: %w", err) + } + + return reportHealth(ctx, gs.Incarnation, gs.Container.ContainerID, gs.Container.RoleInstanceList.RoleInstance.InstanceID) +} + +func goalState(ctx context.Context) (gs *GoalState, err error) { + body, err := download.Download(ctx, AzureInternalEndpoint+"/machine/?comp=goalstate", + download.WithHeaders(map[string]string{ + "x-ms-agent-name": "WALinuxAgent", + "x-ms-version": "2015-04-05", + "Content-Type": "text/xml;charset=utf-8", + })) + if err != nil { + return nil, err + } + + gs = &GoalState{} + err = xml.Unmarshal(body, gs) + + return gs, err +} + +func reportHealth(ctx context.Context, gsIncarnation, gsContainerID, gsInstanceID string) (err error) { + // Construct health response + h := &Health{ + Xsi: "http://www.w3.org/2001/XMLSchema-instance", + Xsd: "http://www.w3.org/2001/XMLSchema", + WAAgent: WAAgent{ + GoalStateIncarnation: gsIncarnation, + Container: &Container{ + ContainerID: gsContainerID, + RoleInstanceList: &RoleInstanceList{ + Role: &RoleInstance{ + InstanceID: gsInstanceID, + Health: &HealthStatus{ + State: "Ready", + }, + }, + }, + }, + }, + } + + // Encode health response as xml + b := new(bytes.Buffer) + b.WriteString(xml.Header) + + err = xml.NewEncoder(b).Encode(h) + if err != nil { + return err + } + + var u *url.URL + + u, err = url.Parse(AzureInternalEndpoint + "/machine/?comp=health") + if err != nil { + return nil + } + + var ( + req *http.Request + resp *http.Response + ) + + req, err = http.NewRequestWithContext(ctx, http.MethodPost, u.String(), b) + if err != nil { + return err + } + + addHeaders(req) + + client := &http.Client{} + + resp, err = client.Do(req) + if err != nil { + return err + } + + // TODO probably should do some better check here ( verify status code ) + //nolint:errcheck + defer resp.Body.Close() + + _, err = io.ReadAll(resp.Body) + if err != nil { + return err + } + + return err +} + +func addHeaders(req *http.Request) { + req.Header.Add("x-ms-agent-name", "WALinuxAgent") + req.Header.Add("x-ms-version", "2015-04-05") + req.Header.Add("Content-Type", "text/xml;charset=utf-8") +} + +// GoalState is the response from the Azure platform when a machine +// starts up. Ref: +// https://github.com/Azure/WALinuxAgent/blob/b26feb7822f7d4a19507b6762fe1bd280c2ba2de/bin/waagent2.0#L4331 +// https://github.com/Azure/WALinuxAgent/blob/3be3e1fbf2330303f76961b87d891672e847ce4e/azurelinuxagent/common/protocol/wire.py#L216 +type GoalState struct { + XMLName xml.Name `xml:"GoalState"` + Xsi string `xml:"xsi,attr"` + Xsd string `xml:"xsd,attr"` + WAAgent +} + +// Health is the response from the local machine to Azure to denote current +// machine state. +type Health struct { + XMLName xml.Name `xml:"Health"` + Xsi string `xml:"xmlns:xsi,attr"` + Xsd string `xml:"xmlns:xsd,attr"` + WAAgent +} + +// WAAgent contains the meat of the data format that is passed between the +// Azure platform and the machine. +// Mostly, we just care about the Incarnation and Container fields here. +type WAAgent struct { + Text string `xml:",chardata"` + Version string `xml:"Version,omitempty"` + Incarnation string `xml:"Incarnation,omitempty"` + GoalStateIncarnation string `xml:"GoalStateIncarnation,omitempty"` + Machine *Machine `xml:"Machine,omitempty"` + Container *Container `xml:"Container,omitempty"` +} + +// Container holds the interesting details about a provisioned machine. +type Container struct { + Text string `xml:",chardata"` + ContainerID string `xml:"ContainerId"` + RoleInstanceList *RoleInstanceList `xml:"RoleInstanceList"` +} + +// RoleInstanceList is a list but only has a single item which is cool I guess. +type RoleInstanceList struct { + Text string `xml:",chardata"` + RoleInstance *RoleInstance `xml:"RoleInstance,omitempty"` + Role *RoleInstance `xml:"Role,omitempty"` +} + +// RoleInstance contains the specifics for the provisioned VM. +type RoleInstance struct { + Text string `xml:",chardata"` + InstanceID string `xml:"InstanceId"` + State string `xml:"State,omitempty"` + Configuration *Configuration `xml:"Configuration,omitempty"` + Health *HealthStatus `xml:"Health,omitempty"` +} + +// Configuration seems important but isnt really used right now. We could +// very well not include it because we have no use for it right now, but +// since we want completeness, we're going to include it. +type Configuration struct { + Text string `xml:",chardata"` + HostingEnvironmentConfig string `xml:"HostingEnvironmentConfig"` + SharedConfig string `xml:"SharedConfig"` + ExtensionsConfig string `xml:"ExtensionsConfig"` + FullConfig string `xml:"FullConfig"` + Certificates string `xml:"Certificates"` + ConfigName string `xml:"ConfigName"` +} + +// Machine holds no useful information for us. +type Machine struct { + Text string `xml:",chardata"` + ExpectedState string `xml:"ExpectedState"` + StopRolesDeadlineHint string `xml:"StopRolesDeadlineHint"` + LBProbePorts *struct { + Text string `xml:",chardata"` + Port string `xml:"Port"` + } `xml:"LBProbePorts,omitempty"` + ExpectHealthReport string `xml:"ExpectHealthReport"` +} + +// HealthStatus provides mechanism to trigger Azure to understand that our +// machine has transitioned to a 'Ready' state and is good to go. +// We can fill out details if we want to be more verbose... +type HealthStatus struct { + Text string `xml:",chardata"` + State string `xml:"State"` + Details *struct { + Text string `xml:",chardata"` + SubStatus string `xml:"SubStatus"` + Description string `xml:"Description"` + } `xml:"Details,omitempty"` +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/azure/testdata/compute.json b/internal/app/machined/pkg/runtime/v1alpha1/platform/azure/testdata/compute.json new file mode 100644 index 0000000..b57a05c --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/azure/testdata/compute.json @@ -0,0 +1,14 @@ +{ + "location": "CentralUS", + "name": "IMDSCanary", + "offer": "RHEL", + "osType": "Linux", + "platformFaultDomain": "0", + "platformUpdateDomain": "0", + "publisher": "RedHat", + "resourceId": "000-000-000-000-000", + "sku": "7.2", + "version": "7.2.20161026", + "vmId": "5c08b38e-4d57-4c23-ac45-aca61037f084", + "vmSize": "Standard_DS2" +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/azure/testdata/expected.yaml b/internal/app/machined/pkg/runtime/v1alpha1/platform/azure/testdata/expected.yaml new file mode 100644 index 0000000..a899b09 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/azure/testdata/expected.yaml @@ -0,0 +1,39 @@ +addresses: [] +links: [] +routes: + - family: inet6 + dst: "" + src: "" + gateway: fe80::1234:5678:9abc + outLinkName: eth0 + table: main + priority: 4096 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform +hostnames: + - hostname: some + domainname: fqdn + layer: platform +resolvers: [] +timeServers: [] +operators: + - operator: dhcp6 + linkName: eth0 + requireUp: true + dhcp6: + routeMetric: 2048 + layer: default +externalIPs: + - 1.2.3.4 + - 2603:1020:10:5::34 + - 20.10.5.34 +metadata: + platform: azure + region: centralus + zone: "0" + instanceType: Standard_DS2 + instanceId: 000-000-000-000-000 + providerId: azure://000-000-000-000-000 diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/azure/testdata/interfaces.json b/internal/app/machined/pkg/runtime/v1alpha1/platform/azure/testdata/interfaces.json new file mode 100644 index 0000000..163ed0d --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/azure/testdata/interfaces.json @@ -0,0 +1,27 @@ +[ + { + "ipv4": { + "ipAddress": [ + { + "privateIpAddress": "172.18.1.10", + "publicIpAddress": "1.2.3.4" + } + ], + "subnet": [ + { + "address": "172.18.1.0", + "prefix": "24" + } + ] + }, + "ipv6": { + "ipAddress": [ + { + "privateIpAddress": "fd00::10", + "publicIpAddress": "" + } + ] + }, + "macAddress": "000D3AD631EE" + } +] diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/azure/testdata/loadbalancer.json b/internal/app/machined/pkg/runtime/v1alpha1/platform/azure/testdata/loadbalancer.json new file mode 100644 index 0000000..fd49514 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/azure/testdata/loadbalancer.json @@ -0,0 +1,31 @@ +{ + "loadbalancer": { + "publicIpAddresses": [ + { + "frontendIpAddress": "[2603:1020:10:5::34]", + "privateIpAddress": "[fd00::10]" + }, + { + "frontendIpAddress": "20.10.5.34", + "privateIpAddress": "172.18.1.10" + } + ], + "inboundRules": [ + { + "frontendIpAddress": "[fd60:172:16:88::5]", + "protocol": "Tcp", + "frontendPort": 6443, + "backendPort": 6443, + "privateIpAddress": "[fd00::10]" + }, + { + "frontendIpAddress": "172.16.136.5", + "protocol": "Tcp", + "frontendPort": 6443, + "backendPort": 6443, + "privateIpAddress": "172.18.1.10" + } + ], + "outboundRules": [] + } +} \ No newline at end of file diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/container/container.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/container/container.go new file mode 100644 index 0000000..1e6844a --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/container/container.go @@ -0,0 +1,91 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package container contains the Container implementation of the [platform.Platform]. +package container + +import ( + "context" + "encoding/base64" + "log" + "os" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-procfs/procfs" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/container/internal/files" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/errors" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// Container is a platform for installing Talos via an Container image. +type Container struct{} + +// Name implements the platform.Platform interface. +func (c *Container) Name() string { + return "container" +} + +// Configuration implements the platform.Platform interface. +func (c *Container) Configuration(context.Context, state.State) ([]byte, error) { + log.Printf("fetching machine config from: USERDATA environment variable") + + s := os.Getenv("USERDATA") + if s == "" { + return nil, errors.ErrNoConfigSource + } + + decoded, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return nil, err + } + + return decoded, nil +} + +// Mode implements the platform.Platform interface. +func (c *Container) Mode() runtime.Mode { + return runtime.ModeContainer +} + +// KernelArgs implements the runtime.Platform interface. +func (c *Container) KernelArgs(string) procfs.Parameters { + return nil +} + +// NetworkConfiguration implements the runtime.Platform interface. +func (c *Container) NetworkConfiguration(ctx context.Context, _ state.State, ch chan<- *runtime.PlatformNetworkConfig) error { + networkConfig := &runtime.PlatformNetworkConfig{} + + hostnameSpec, err := files.ReadHostname("/etc/hostname") + if err != nil { + return err + } + + networkConfig.Hostnames = append(networkConfig.Hostnames, hostnameSpec) + + resolverSpec, err := files.ReadResolvConf("/etc/resolv.conf") + if err != nil { + return err + } + + if len(resolverSpec.DNSServers) > 0 { + networkConfig.Resolvers = append(networkConfig.Resolvers, resolverSpec) + } + + networkConfig.Metadata = &runtimeres.PlatformMetadataSpec{ + Platform: c.Name(), + Hostname: hostnameSpec.FQDN(), + InstanceType: os.Getenv("TALOSSKU"), + } + + select { + case ch <- networkConfig: + case <-ctx.Done(): + return ctx.Err() + } + + return nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/container/internal/files/hostname.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/container/internal/files/hostname.go new file mode 100644 index 0000000..955c2fb --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/container/internal/files/hostname.go @@ -0,0 +1,33 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package files provides internal methods to container platform to read files. +package files + +import ( + "bytes" + "os" + + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// ReadHostname reads and parses /etc/hostname file. +func ReadHostname(path string) (network.HostnameSpecSpec, error) { + hostname, err := os.ReadFile(path) + if err != nil { + return network.HostnameSpecSpec{}, err + } + + hostname = bytes.TrimSpace(hostname) + + hostnameSpec := network.HostnameSpecSpec{ + ConfigLayer: network.ConfigPlatform, + } + + if err = hostnameSpec.ParseFQDN(string(hostname)); err != nil { + return network.HostnameSpecSpec{}, err + } + + return hostnameSpec, nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/container/internal/files/hostname_test.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/container/internal/files/hostname_test.go new file mode 100644 index 0000000..c890c4c --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/container/internal/files/hostname_test.go @@ -0,0 +1,23 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package files_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/container/internal/files" +) + +func TestReadHostname(t *testing.T) { + t.Parallel() + + spec, err := files.ReadHostname("testdata/hostname") + require.NoError(t, err) + + require.Equal(t, "foo", spec.Hostname) + require.Equal(t, "example.com", spec.Domainname) +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/container/internal/files/resolv.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/container/internal/files/resolv.go new file mode 100644 index 0000000..9995b37 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/container/internal/files/resolv.go @@ -0,0 +1,42 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package files + +import ( + "bytes" + "net/netip" + "os" + + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// ReadResolvConf reads and parses /etc/resolv.conf file. +func ReadResolvConf(path string) (network.ResolverSpecSpec, error) { + resolverSpec := network.ResolverSpecSpec{ + ConfigLayer: network.ConfigPlatform, + } + + resolvers, err := os.ReadFile(path) + if err != nil { + return resolverSpec, err + } + + for _, line := range bytes.Split(resolvers, []byte("\n")) { + line = bytes.TrimSpace(line) + line, _, _ = bytes.Cut(line, []byte("#")) + + if !bytes.HasPrefix(line, []byte("nameserver")) { + continue + } + + line = bytes.TrimSpace(bytes.TrimPrefix(line, []byte("nameserver"))) + + if addr, err := netip.ParseAddr(string(line)); err == nil { + resolverSpec.DNSServers = append(resolverSpec.DNSServers, addr) + } + } + + return resolverSpec, nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/container/internal/files/resolv_test.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/container/internal/files/resolv_test.go new file mode 100644 index 0000000..71c8d74 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/container/internal/files/resolv_test.go @@ -0,0 +1,26 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package files_test + +import ( + "net/netip" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/container/internal/files" +) + +func TestReadResolvConf(t *testing.T) { + t.Parallel() + + spec, err := files.ReadResolvConf("testdata/resolv.conf") + require.NoError(t, err) + + require.Equal(t, []netip.Addr{ + netip.MustParseAddr("127.0.0.53"), + netip.MustParseAddr("::1"), + }, spec.DNSServers) +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/container/internal/files/testdata/hostname b/internal/app/machined/pkg/runtime/v1alpha1/platform/container/internal/files/testdata/hostname new file mode 100644 index 0000000..7488fab --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/container/internal/files/testdata/hostname @@ -0,0 +1 @@ +foo.example.com diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/container/internal/files/testdata/resolv.conf b/internal/app/machined/pkg/runtime/v1alpha1/platform/container/internal/files/testdata/resolv.conf new file mode 100644 index 0000000..cf5777b --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/container/internal/files/testdata/resolv.conf @@ -0,0 +1,8 @@ +# This is /run/systemd/resolve/stub-resolv.conf managed by man:systemd-resolved(8). +# Do not edit. + +nameserver 127.0.0.53 # v4 one +options edns0 trust-ad +search . + + nameserver ::1 # this is V6 diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean/digitalocean.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean/digitalocean.go new file mode 100644 index 0000000..b7a3cd0 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean/digitalocean.go @@ -0,0 +1,287 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package digitalocean contains the Digital Ocean implementation of the [platform.Platform]. +package digitalocean + +import ( + "context" + "fmt" + "log" + "net/netip" + "strconv" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-procfs/procfs" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/errors" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/internal/address" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/internal/netutils" + "github.com/siderolabs/talos/pkg/download" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// DigitalOcean is the concrete type that implements the platform.Platform interface. +type DigitalOcean struct{} + +// Name implements the platform.Platform interface. +func (d *DigitalOcean) Name() string { + return "digital-ocean" +} + +// ParseMetadata converts DigitalOcean platform metadata into platform network config. +// +//nolint:gocyclo,cyclop +func (d *DigitalOcean) ParseMetadata(metadata *MetadataConfig) (*runtime.PlatformNetworkConfig, error) { + networkConfig := &runtime.PlatformNetworkConfig{} + + if metadata.Hostname != "" { + hostnameSpec := network.HostnameSpecSpec{ + ConfigLayer: network.ConfigPlatform, + } + + if err := hostnameSpec.ParseFQDN(metadata.Hostname); err != nil { + return nil, err + } + + networkConfig.Hostnames = append(networkConfig.Hostnames, hostnameSpec) + } + + if len(metadata.DNS.Nameservers) > 0 { + var dnsIPs []netip.Addr + + for _, dnsIP := range metadata.DNS.Nameservers { + if ip, err := netip.ParseAddr(dnsIP); err == nil { + dnsIPs = append(dnsIPs, ip) + } + } + + networkConfig.Resolvers = append(networkConfig.Resolvers, network.ResolverSpecSpec{ + DNSServers: dnsIPs, + ConfigLayer: network.ConfigPlatform, + }) + } + + networkConfig.Links = append(networkConfig.Links, network.LinkSpecSpec{ + Name: "eth0", + Up: true, + ConfigLayer: network.ConfigPlatform, + }) + + publicIPs := []string{} + + for _, iface := range metadata.Interfaces["public"] { + if iface.IPv4 != nil { + ifAddr, err := address.IPPrefixFrom(iface.IPv4.IPAddress, iface.IPv4.Netmask) + if err != nil { + return nil, fmt.Errorf("failed to parse ip address: %w", err) + } + + publicIPs = append(publicIPs, iface.IPv4.IPAddress) + + networkConfig.Addresses = append(networkConfig.Addresses, + network.AddressSpecSpec{ + ConfigLayer: network.ConfigPlatform, + LinkName: "eth0", + Address: ifAddr, + Scope: nethelpers.ScopeGlobal, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + Family: nethelpers.FamilyInet4, + }, + ) + + if iface.IPv4.Gateway != "" { + gw, err := netip.ParseAddr(iface.IPv4.Gateway) + if err != nil { + return nil, fmt.Errorf("failed to parse gateway ip: %w", err) + } + + route := network.RouteSpecSpec{ + ConfigLayer: network.ConfigPlatform, + Gateway: gw, + OutLinkName: "eth0", + Table: nethelpers.TableMain, + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + Family: nethelpers.FamilyInet4, + Priority: network.DefaultRouteMetric, + } + + route.Normalize() + + networkConfig.Routes = append(networkConfig.Routes, route) + + metaServer, _ := netip.ParsePrefix("169.254.169.254/32") //nolint:errcheck + + networkConfig.Routes = append(networkConfig.Routes, network.RouteSpecSpec{ + ConfigLayer: network.ConfigPlatform, + OutLinkName: "eth0", + Destination: metaServer, + Gateway: gw, + Table: nethelpers.TableMain, + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + Family: nethelpers.FamilyInet4, + Priority: 512, + }) + } + } + + if iface.IPv6 != nil { + ifAddr, err := address.IPPrefixFrom(iface.IPv6.IPAddress, strconv.Itoa(iface.IPv6.CIDR)) + if err != nil { + return nil, fmt.Errorf("failed to parse ip address: %w", err) + } + + publicIPs = append(publicIPs, iface.IPv6.IPAddress) + networkConfig.Addresses = append(networkConfig.Addresses, + network.AddressSpecSpec{ + ConfigLayer: network.ConfigPlatform, + LinkName: "eth0", + Address: ifAddr, + Scope: nethelpers.ScopeGlobal, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + Family: nethelpers.FamilyInet6, + }, + ) + + if iface.IPv6.Gateway != "" { + gw, err := netip.ParseAddr(iface.IPv6.Gateway) + if err != nil { + return nil, fmt.Errorf("failed to parse gateway ip: %w", err) + } + + route := network.RouteSpecSpec{ + ConfigLayer: network.ConfigPlatform, + Gateway: gw, + OutLinkName: "eth0", + Table: nethelpers.TableMain, + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + Family: nethelpers.FamilyInet6, + Priority: 2 * network.DefaultRouteMetric, + } + + route.Normalize() + + networkConfig.Routes = append(networkConfig.Routes, route) + } + } + + if iface.AnchorIPv4 != nil { + ifAddr, err := address.IPPrefixFrom(iface.AnchorIPv4.IPAddress, iface.AnchorIPv4.Netmask) + if err != nil { + return nil, fmt.Errorf("failed to parse ip address: %w", err) + } + + networkConfig.Addresses = append(networkConfig.Addresses, + network.AddressSpecSpec{ + ConfigLayer: network.ConfigPlatform, + LinkName: "eth0", + Address: ifAddr, + Scope: nethelpers.ScopeLink, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + Family: nethelpers.FamilyInet4, + }, + ) + } + } + + for idx, iface := range metadata.Interfaces["private"] { + ifName := fmt.Sprintf("eth%d", idx+1) + + networkConfig.Links = append(networkConfig.Links, network.LinkSpecSpec{ + Name: ifName, + Up: true, + ConfigLayer: network.ConfigPlatform, + }) + + if iface.IPv4 != nil { + ifAddr, err := address.IPPrefixFrom(iface.IPv4.IPAddress, iface.IPv4.Netmask) + if err != nil { + return nil, fmt.Errorf("failed to parse ip address: %w", err) + } + + networkConfig.Addresses = append(networkConfig.Addresses, + network.AddressSpecSpec{ + ConfigLayer: network.ConfigPlatform, + LinkName: ifName, + Address: ifAddr, + Scope: nethelpers.ScopeGlobal, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + Family: nethelpers.FamilyInet4, + }, + ) + } + } + + for _, ipStr := range publicIPs { + if ip, err := netip.ParseAddr(ipStr); err == nil { + networkConfig.ExternalIPs = append(networkConfig.ExternalIPs, ip) + } + } + + networkConfig.Metadata = &runtimeres.PlatformMetadataSpec{ + Platform: d.Name(), + Hostname: metadata.Hostname, + Region: metadata.Region, + InstanceID: strconv.Itoa(metadata.DropletID), + ProviderID: fmt.Sprintf("digitalocean://%d", metadata.DropletID), + } + + return networkConfig, nil +} + +// Configuration implements the platform.Platform interface. +func (d *DigitalOcean) Configuration(ctx context.Context, r state.State) ([]byte, error) { + if err := netutils.Wait(ctx, r); err != nil { + return nil, err + } + + log.Printf("fetching machine config from: %q", DigitalOceanUserDataEndpoint) + + return download.Download(ctx, DigitalOceanUserDataEndpoint, + download.WithErrorOnNotFound(errors.ErrNoConfigSource), + download.WithErrorOnEmptyResponse(errors.ErrNoConfigSource)) +} + +// Mode implements the platform.Platform interface. +func (d *DigitalOcean) Mode() runtime.Mode { + return runtime.ModeCloud +} + +// KernelArgs implements the runtime.Platform interface. +func (d *DigitalOcean) KernelArgs(string) procfs.Parameters { + return []*procfs.Parameter{ + procfs.NewParameter("console").Append("ttyS0").Append("tty0").Append("tty1"), + procfs.NewParameter(constants.KernelParamNetIfnames).Append("0"), + } +} + +// NetworkConfiguration implements the runtime.Platform interface. +func (d *DigitalOcean) NetworkConfiguration(ctx context.Context, _ state.State, ch chan<- *runtime.PlatformNetworkConfig) error { + log.Printf("fetching DigitalOcean instance config from: %q ", DigitalOceanMetadataEndpoint) + + metadata, err := d.getMetadata(ctx) + if err != nil { + return err + } + + networkConfig, err := d.ParseMetadata(metadata) + if err != nil { + return err + } + + select { + case ch <- networkConfig: + case <-ctx.Done(): + return ctx.Err() + } + + return nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean/digitalocean_test.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean/digitalocean_test.go new file mode 100644 index 0000000..2c667f3 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean/digitalocean_test.go @@ -0,0 +1,39 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package digitalocean_test + +import ( + _ "embed" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean" +) + +//go:embed testdata/metadata.json +var rawMetadata []byte + +//go:embed testdata/expected.yaml +var expectedNetworkConfig string + +func TestParseMetadata(t *testing.T) { + p := &digitalocean.DigitalOcean{} + + var metadata digitalocean.MetadataConfig + + require.NoError(t, json.Unmarshal(rawMetadata, &metadata)) + + networkConfig, err := p.ParseMetadata(&metadata) + require.NoError(t, err) + + marshaled, err := yaml.Marshal(networkConfig) + require.NoError(t, err) + + assert.Equal(t, expectedNetworkConfig, string(marshaled)) +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean/metadata.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean/metadata.go new file mode 100644 index 0000000..d4c9727 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean/metadata.go @@ -0,0 +1,81 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package digitalocean + +import ( + "context" + "encoding/json" + stderrors "errors" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/errors" + "github.com/siderolabs/talos/pkg/download" +) + +const ( + // DigitalOceanExternalIPEndpoint displays all external addresses associated with the instance. + DigitalOceanExternalIPEndpoint = "http://169.254.169.254/metadata/v1/interfaces/public/0/ipv4/address" + // DigitalOceanMetadataEndpoint is the local endpoint for the platform metadata. + DigitalOceanMetadataEndpoint = "http://169.254.169.254/metadata/v1.json" + // DigitalOceanUserDataEndpoint is the local endpoint for the config. + DigitalOceanUserDataEndpoint = "http://169.254.169.254/metadata/v1/user-data" +) + +// MetadataConfig represents a metadata Digital Ocean instance. +type MetadataConfig struct { + Hostname string `json:"hostname,omitempty"` + DropletID int `json:"droplet_id,omitempty"` + Region string `json:"region,omitempty"` + PublicIPv4 string `json:"public-ipv4,omitempty"` + Tags []string `json:"tags,omitempty"` + + DNS struct { + Nameservers []string `json:"nameservers,omitempty"` + } `json:"dns,omitempty"` + Interfaces map[string][]struct { + MACAddress string `json:"mac,omitempty"` + Type string `json:"type,omitempty"` + + IPv4 *struct { + IPAddress string `json:"ip_address,omitempty"` + Netmask string `json:"netmask,omitempty"` + Gateway string `json:"gateway,omitempty"` + } `json:"ipv4,omitempty"` + IPv6 *struct { + IPAddress string `json:"ip_address,omitempty"` + CIDR int `json:"cidr,omitempty"` + Gateway string `json:"gateway,omitempty"` + } `json:"ipv6,omitempty"` + AnchorIPv4 *struct { + IPAddress string `json:"ip_address,omitempty"` + Netmask string `json:"netmask,omitempty"` + Gateway string `json:"gateway,omitempty"` + } `json:"anchor_ipv4,omitempty"` + } `json:"interfaces,omitempty"` +} + +func (d *DigitalOcean) getMetadata(ctx context.Context) (*MetadataConfig, error) { + metaConfigDl, err := download.Download(ctx, DigitalOceanMetadataEndpoint, + download.WithErrorOnNotFound(errors.ErrNoHostname), + download.WithErrorOnEmptyResponse(errors.ErrNoHostname)) + if err != nil && !stderrors.Is(err, errors.ErrNoHostname) { + return nil, err + } + + var metadata MetadataConfig + if err = json.Unmarshal(metaConfigDl, &metadata); err != nil { + return nil, err + } + + extIP, err := download.Download(ctx, DigitalOceanExternalIPEndpoint, + download.WithErrorOnNotFound(errors.ErrNoExternalIPs), + download.WithErrorOnEmptyResponse(errors.ErrNoExternalIPs)) + if err != nil && !stderrors.Is(err, errors.ErrNoExternalIPs) { + return nil, err + } + + metadata.PublicIPv4 = string(extIP) + + return &metadata, nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean/testdata/expected.yaml b/internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean/testdata/expected.yaml new file mode 100644 index 0000000..d40e6cb --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean/testdata/expected.yaml @@ -0,0 +1,97 @@ +addresses: + - address: 128.199.52.32/19 + linkName: eth0 + family: inet4 + scope: global + flags: permanent + layer: platform + - address: 2a03:b0c0:2:d0::1478:3001/64 + linkName: eth0 + family: inet6 + scope: global + flags: permanent + layer: platform + - address: 10.18.0.5/16 + linkName: eth0 + family: inet4 + scope: link + flags: permanent + layer: platform + - address: 10.133.0.2/16 + linkName: eth1 + family: inet4 + scope: global + flags: permanent + layer: platform +links: + - name: eth0 + logical: false + up: true + mtu: 0 + kind: "" + type: netrom + layer: platform + - name: eth1 + logical: false + up: true + mtu: 0 + kind: "" + type: netrom + layer: platform +routes: + - family: inet4 + dst: "" + src: "" + gateway: 128.199.32.1 + outLinkName: eth0 + table: main + priority: 1024 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform + - family: inet4 + dst: 169.254.169.254/32 + src: "" + gateway: 128.199.32.1 + outLinkName: eth0 + table: main + priority: 512 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform + - family: inet6 + dst: "" + src: "" + gateway: 2a03:b0c0:2:d0::1 + outLinkName: eth0 + table: main + priority: 2048 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform +hostnames: + - hostname: debian-s-1vcpu-512mb-10gb-ams3-01 + domainname: "" + layer: platform +resolvers: + - dnsServers: + - 67.207.67.2 + - 67.207.67.3 + layer: platform +timeServers: [] +operators: [] +externalIPs: + - 128.199.52.32 + - 2a03:b0c0:2:d0::1478:3001 +metadata: + platform: digital-ocean + hostname: debian-s-1vcpu-512mb-10gb-ams3-01 + region: ams3 + instanceId: "320206672" + providerId: digitalocean://320206672 diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean/testdata/metadata.json b/internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean/testdata/metadata.json new file mode 100644 index 0000000..3667ab9 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean/testdata/metadata.json @@ -0,0 +1,70 @@ +{ + "droplet_id": 320206672, + "hostname": "debian-s-1vcpu-512mb-10gb-ams3-01", + "user_data": "", + "vendor_data": "", + "public_keys": [], + "auth_key": "490eac0a2fc04503267ef85064407f2f", + "region": "ams3", + "interfaces": { + "private": [ + { + "ipv4": { + "ip_address": "10.133.0.2", + "netmask": "255.255.0.0", + "gateway": "10.133.0.1" + }, + "mac": "2a:3c:79:3d:f3:b7", + "type": "private" + } + ], + "public": [ + { + "ipv4": { + "ip_address": "128.199.52.32", + "netmask": "255.255.224.0", + "gateway": "128.199.32.1" + }, + "ipv6": { + "ip_address": "2A03:B0C0:0002:00D0:0000:0000:1478:3001", + "cidr": 64, + "gateway": "2a03:b0c0:2:d0::1" + }, + "anchor_ipv4": { + "ip_address": "10.18.0.5", + "netmask": "255.255.0.0", + "gateway": "10.18.0.1" + }, + "mac": "12:2f:49:0c:eb:c0", + "type": "public" + } + ] + }, + "floating_ip": { + "ipv4": { + "active": false + } + }, + "reserved_ip": { + "ipv4": { + "active": false + } + }, + "dns": { + "nameservers": [ + "67.207.67.2", + "67.207.67.3" + ] + }, + "tags": [ + "label123" + ], + "features": { + "dhcp_enabled": false + }, + "modify_index": 113261986, + "dotty_status": "running", + "ssh_info": { + "port": 22 + } +} \ No newline at end of file diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/equinixmetal/equinix.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/equinixmetal/equinix.go new file mode 100644 index 0000000..719d241 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/equinixmetal/equinix.go @@ -0,0 +1,467 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package equinixmetal contains the Equinix Metal implementation of the [platform.Platform]. +package equinixmetal + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/netip" + "time" + + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-procfs/procfs" + "github.com/siderolabs/go-retry/retry" + + networkadapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/network" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/errors" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/internal/netutils" + "github.com/siderolabs/talos/pkg/download" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// Event holds data to pass to the Equinix Metal event URL. +type Event struct { + Type string `json:"type"` + Message string `json:"msg"` +} + +// Network holds network info from the equinixmetal metadata. +type Network struct { + Bonding Bonding `json:"bonding"` + Interfaces []Interface `json:"interfaces"` + Addresses []Address `json:"addresses"` +} + +// Bonding holds bonding info from the equinixmetal metadata. +type Bonding struct { + Mode int `json:"mode"` +} + +// Interface holds interface info from the equinixmetal metadata. +type Interface struct { + Name string `json:"name"` + MAC string `json:"mac"` + Bond string `json:"bond"` +} + +// Address holds address info from the equinixmetal metadata. +type Address struct { + Public bool `json:"public"` + Management bool `json:"management"` + Enabled bool `json:"enabled"` + CIDR int `json:"cidr"` + Family int `json:"address_family"` + Netmask string `json:"netmask"` + Network string `json:"network"` + Address string `json:"address"` + Gateway string `json:"gateway"` +} + +// BGPNeighbor holds BGP neighbor info from the equinixmetal metadata. +type BGPNeighbor struct { + AddressFamily int `json:"address_family"` + PeerIPs []string `json:"peer_ips"` +} + +const ( + // EquinixMetalUserDataEndpoint is the local metadata endpoint for Equinix. + EquinixMetalUserDataEndpoint = "https://metadata.platformequinix.com/userdata" + // EquinixMetalMetaDataEndpoint is the local endpoint for machine info like networking. + EquinixMetalMetaDataEndpoint = "https://metadata.platformequinix.com/metadata" +) + +// EquinixMetal is a platform for EquinixMetal Metal cloud. +type EquinixMetal struct{} + +// Name implements the platform.Platform interface. +func (p *EquinixMetal) Name() string { + return "equinixMetal" +} + +// Configuration implements the platform.Platform interface. +func (p *EquinixMetal) Configuration(ctx context.Context, r state.State) ([]byte, error) { + if err := netutils.Wait(ctx, r); err != nil { + return nil, err + } + + log.Printf("fetching machine config from: %q", EquinixMetalUserDataEndpoint) + + return download.Download(ctx, EquinixMetalUserDataEndpoint, + download.WithErrorOnNotFound(errors.ErrNoConfigSource), + download.WithErrorOnEmptyResponse(errors.ErrNoConfigSource)) +} + +// Mode implements the platform.Platform interface. +func (p *EquinixMetal) Mode() runtime.Mode { + return runtime.ModeMetal +} + +// KernelArgs implements the runtime.Platform interface. +func (p *EquinixMetal) KernelArgs(arch string) procfs.Parameters { + switch arch { + case "amd64": + return []*procfs.Parameter{ + procfs.NewParameter("console").Append("ttyS1,115200n8"), + } + case "arm64": + return []*procfs.Parameter{ + procfs.NewParameter("console").Append("ttyAMA0,115200"), + } + default: + return nil + } +} + +// ParseMetadata converts Equinix Metal metadata into Talos network configuration. +// +//nolint:gocyclo,cyclop +func (p *EquinixMetal) ParseMetadata(ctx context.Context, equinixMetadata *MetadataConfig, st state.State) (*runtime.PlatformNetworkConfig, error) { + networkConfig := &runtime.PlatformNetworkConfig{} + + // 1. Links + + // translate the int returned from bond mode metadata to the type needed by network resources + bondMode := nethelpers.BondMode(uint8(equinixMetadata.Network.Bonding.Mode)) + + hostInterfaces, err := safe.StateListAll[*network.LinkStatus](ctx, st) + if err != nil { + return nil, fmt.Errorf("error listing host interfaces: %w", err) + } + + bondSlaveIndexes := map[string]int{} + firstBond := "" + + for _, iface := range equinixMetadata.Network.Interfaces { + if iface.Bond == "" { + continue + } + + if firstBond == "" { + firstBond = iface.Bond + } + + found := false + + hostInterfaceIter := hostInterfaces.Iterator() + + for hostInterfaceIter.Next() { + // match using permanent MAC address: + // - bond interfaces don't have permanent addresses set, so we skip them this way + // - if the bond is already configured, regular hardware address is overwritten with bond address + if hostInterfaceIter.Value().TypedSpec().PermanentAddr.String() == iface.MAC { + found = true + + slaveIndex := bondSlaveIndexes[iface.Bond] + + networkConfig.Links = append(networkConfig.Links, + network.LinkSpecSpec{ + Name: hostInterfaceIter.Value().Metadata().ID(), + Up: true, + BondSlave: network.BondSlave{ + MasterName: iface.Bond, + SlaveIndex: slaveIndex, + }, + ConfigLayer: network.ConfigPlatform, + }) + + bondSlaveIndexes[iface.Bond]++ + + break + } + } + + if !found { + log.Printf("interface with MAC %q wasn't found on the host, adding with the name from metadata", iface.MAC) + + slaveIndex := bondSlaveIndexes[iface.Bond] + + networkConfig.Links = append(networkConfig.Links, + network.LinkSpecSpec{ + ConfigLayer: network.ConfigPlatform, + Name: iface.Name, + Up: true, + BondSlave: network.BondSlave{ + MasterName: iface.Bond, + SlaveIndex: slaveIndex, + }, + }) + + bondSlaveIndexes[iface.Bond]++ + } + } + + for bondName := range bondSlaveIndexes { + bondLink := network.LinkSpecSpec{ + ConfigLayer: network.ConfigPlatform, + Name: bondName, + Logical: true, + Up: true, + Kind: network.LinkKindBond, + Type: nethelpers.LinkEther, + BondMaster: network.BondMasterSpec{ + Mode: bondMode, + DownDelay: 200, + MIIMon: 100, + UpDelay: 200, + HashPolicy: nethelpers.BondXmitPolicyLayer34, + }, + } + + networkadapter.BondMasterSpec(&bondLink.BondMaster).FillDefaults() + + networkConfig.Links = append(networkConfig.Links, bondLink) + } + + // 2. addresses + + publicIPs := []string{} + + for _, addr := range equinixMetadata.Network.Addresses { + if !(addr.Enabled && addr.Management) { + continue + } + + if addr.Public { + publicIPs = append(publicIPs, addr.Address) + } + + ipAddr, err := netip.ParsePrefix(fmt.Sprintf("%s/%d", addr.Address, addr.CIDR)) + if err != nil { + return nil, err + } + + family := nethelpers.FamilyInet4 + if ipAddr.Addr().Is6() { + family = nethelpers.FamilyInet6 + } + + networkConfig.Addresses = append(networkConfig.Addresses, + network.AddressSpecSpec{ + ConfigLayer: network.ConfigPlatform, + LinkName: firstBond, + Address: ipAddr, + Scope: nethelpers.ScopeGlobal, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + Family: family, + }, + ) + } + + for _, ipStr := range publicIPs { + if ip, err := netip.ParseAddr(ipStr); err == nil { + networkConfig.ExternalIPs = append(networkConfig.ExternalIPs, ip) + } + } + + // 3. routes + var privateGateway netip.Addr + + for _, addr := range equinixMetadata.Network.Addresses { + if !(addr.Enabled && addr.Management) { + continue + } + + ipAddr, err := netip.ParsePrefix(fmt.Sprintf("%s/%d", addr.Address, addr.CIDR)) + if err != nil { + return nil, err + } + + family := nethelpers.FamilyInet4 + if ipAddr.Addr().Is6() { + family = nethelpers.FamilyInet6 + } + + if addr.Public { + // for "Public" address add the default route + gw, err := netip.ParseAddr(addr.Gateway) + if err != nil { + return nil, err + } + + route := network.RouteSpecSpec{ + ConfigLayer: network.ConfigPlatform, + Gateway: gw, + OutLinkName: firstBond, + Table: nethelpers.TableMain, + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + Family: family, + Priority: network.DefaultRouteMetric, + } + + if addr.Family == 6 { + route.Priority = 2 * network.DefaultRouteMetric + } + + route.Normalize() + + networkConfig.Routes = append(networkConfig.Routes, route) + } else { + // for "Private" addresses, we add a route that goes out the gateway for the private subnets. + for _, privSubnet := range equinixMetadata.PrivateSubnets { + gw, err := netip.ParseAddr(addr.Gateway) + if err != nil { + return nil, err + } + + privateGateway = gw + + dest, err := netip.ParsePrefix(privSubnet) + if err != nil { + return nil, err + } + + route := network.RouteSpecSpec{ + ConfigLayer: network.ConfigPlatform, + Gateway: gw, + Destination: dest, + OutLinkName: firstBond, + Table: nethelpers.TableMain, + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + Family: family, + } + + route.Normalize() + + networkConfig.Routes = append(networkConfig.Routes, route) + } + } + } + + // 4. hostname + + if equinixMetadata.Hostname != "" { + hostnameSpec := network.HostnameSpecSpec{ + ConfigLayer: network.ConfigPlatform, + } + + if err := hostnameSpec.ParseFQDN(equinixMetadata.Hostname); err != nil { + return nil, err + } + + networkConfig.Hostnames = append(networkConfig.Hostnames, hostnameSpec) + } + + // 5. platform metadata + + networkConfig.Metadata = &runtimeres.PlatformMetadataSpec{ + Platform: p.Name(), + Hostname: equinixMetadata.Hostname, + Region: equinixMetadata.Metro, + Zone: equinixMetadata.Facility, + InstanceType: equinixMetadata.Plan, + InstanceID: equinixMetadata.ID, + ProviderID: fmt.Sprintf("equinixmetal://%s", equinixMetadata.ID), + } + + // 6. BGP neighbors + + for _, bgpNeighbor := range equinixMetadata.BGPNeighbors { + if bgpNeighbor.AddressFamily != 4 { + continue + } + + for _, peerIP := range bgpNeighbor.PeerIPs { + peer, err := netip.ParseAddr(peerIP) + if err != nil { + return nil, err + } + + route := network.RouteSpecSpec{ + ConfigLayer: network.ConfigPlatform, + Gateway: privateGateway, + Destination: netip.PrefixFrom(peer, 32), + OutLinkName: firstBond, + Table: nethelpers.TableMain, + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + Family: nethelpers.FamilyInet4, + } + + route.Normalize() + + networkConfig.Routes = append(networkConfig.Routes, route) + } + } + + return networkConfig, nil +} + +// NetworkConfiguration implements the runtime.Platform interface. +func (p *EquinixMetal) NetworkConfiguration(ctx context.Context, st state.State, ch chan<- *runtime.PlatformNetworkConfig) error { + log.Printf("fetching equinix network config from: %q", EquinixMetalMetaDataEndpoint) + + metadataConfig, err := download.Download(ctx, EquinixMetalMetaDataEndpoint) + if err != nil { + return err + } + + var meta MetadataConfig + if err = json.Unmarshal(metadataConfig, &meta); err != nil { + return err + } + + networkConfig, err := p.ParseMetadata(ctx, &meta, st) + if err != nil { + return err + } + + select { + case <-ctx.Done(): + return ctx.Err() + case ch <- networkConfig: + } + + return nil +} + +// FireEvent will take an event and pass it to an events server. +// nb: This is currently only used with Equinix Metal but we may find interesting ways +// to extend it for other event servers (Azure may have something similar?) +func (p *EquinixMetal) FireEvent(ctx context.Context, event Event) error { + var eventURL *string + if eventURL = procfs.ProcCmdline().Get(constants.KernelParamEquinixMetalEvents).First(); eventURL == nil { + return errors.ErrNoEventURL + } + + eventData, err := json.Marshal(event) + if err != nil { + return err + } + + err = retry.Constant(5*time.Minute, + retry.WithUnits(time.Second), + retry.WithErrorLogging(true)).RetryWithContext( + ctx, + func(ctx context.Context) error { + req, reqErr := http.NewRequestWithContext(ctx, http.MethodPost, *eventURL, bytes.NewReader(eventData)) + if reqErr != nil { + return reqErr + } + + resp, reqErr := http.DefaultClient.Do(req) + if resp != nil { + io.Copy(io.Discard, io.LimitReader(resp.Body, 4*1024*1024)) //nolint:errcheck + resp.Body.Close() //nolint:errcheck + } + + return retry.ExpectedError(reqErr) + }, + ) + + return err +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/equinixmetal/equinix_test.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/equinixmetal/equinix_test.go new file mode 100644 index 0000000..b6b1cd7 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/equinixmetal/equinix_test.go @@ -0,0 +1,57 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package equinixmetal_test + +import ( + "context" + _ "embed" + "encoding/json" + "testing" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/equinixmetal" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +//go:embed testdata/metadata.json +var rawMetadata []byte + +//go:embed testdata/expected.yaml +var expectedNetworkConfig string + +func TestParseMetadata(t *testing.T) { + p := &equinixmetal.EquinixMetal{} + + var m equinixmetal.MetadataConfig + + require.NoError(t, json.Unmarshal(rawMetadata, &m)) + + ctx := context.Background() + + st := state.WrapCore(namespaced.NewState(inmem.Build)) + + eth1 := network.NewLinkStatus(network.NamespaceName, "eth1") + eth1.TypedSpec().PermanentAddr = nethelpers.HardwareAddr{0x68, 0x05, 0xca, 0xb8, 0xf1, 0xf8} + require.NoError(t, st.Create(ctx, eth1)) + + eth2 := network.NewLinkStatus(network.NamespaceName, "eth2") + eth2.TypedSpec().PermanentAddr = nethelpers.HardwareAddr{0x68, 0x05, 0xca, 0xb8, 0xf1, 0xf9} + require.NoError(t, st.Create(ctx, eth2)) + + networkConfig, err := p.ParseMetadata(ctx, &m, st) + require.NoError(t, err) + + marshaled, err := yaml.Marshal(networkConfig) + require.NoError(t, err) + + assert.Equal(t, expectedNetworkConfig, string(marshaled)) +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/equinixmetal/metadata.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/equinixmetal/metadata.go new file mode 100644 index 0000000..53c4827 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/equinixmetal/metadata.go @@ -0,0 +1,17 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package equinixmetal + +// MetadataConfig holds equinixmetal metadata info. +type MetadataConfig struct { + ID string `json:"id"` + Hostname string `json:"hostname"` + Plan string `json:"plan"` + Metro string `json:"metro"` + Facility string `json:"facility"` + Network Network `json:"network"` + BGPNeighbors []BGPNeighbor `json:"bgp_neighbors"` + PrivateSubnets []string `json:"private_subnets"` +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/equinixmetal/testdata/expected.yaml b/internal/app/machined/pkg/runtime/v1alpha1/platform/equinixmetal/testdata/expected.yaml new file mode 100644 index 0000000..c6870e9 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/equinixmetal/testdata/expected.yaml @@ -0,0 +1,137 @@ +addresses: + - address: 147.75.78.41/31 + linkName: bond0 + family: inet4 + scope: global + flags: permanent + layer: platform + - address: 2604:1380:45d1:fd00::11/127 + linkName: bond0 + family: inet6 + scope: global + flags: permanent + layer: platform + - address: 10.66.142.17/31 + linkName: bond0 + family: inet4 + scope: global + flags: permanent + layer: platform +links: + - name: eth1 + logical: false + up: true + mtu: 0 + kind: "" + type: netrom + masterName: bond0 + layer: platform + - name: eth2 + logical: false + up: true + mtu: 0 + kind: "" + type: netrom + masterName: bond0 + slaveIndex: 1 + layer: platform + - name: bond0 + logical: true + up: true + mtu: 0 + kind: bond + type: ether + bondMaster: + mode: 802.3ad + xmitHashPolicy: layer3+4 + lacpRate: slow + arpValidate: none + arpAllTargets: any + primaryReselect: always + failOverMac: 0 + miimon: 100 + updelay: 200 + downdelay: 200 + resendIgmp: 1 + lpInterval: 1 + packetsPerSlave: 1 + numPeerNotif: 1 + tlbLogicalLb: 1 + adActorSysPrio: 65535 + layer: platform +routes: + - family: inet4 + dst: "" + src: "" + gateway: 147.75.78.40 + outLinkName: bond0 + table: main + priority: 1024 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform + - family: inet6 + dst: "" + src: "" + gateway: 2604:1380:45d1:fd00::10 + outLinkName: bond0 + table: main + priority: 2048 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform + - family: inet4 + dst: 10.0.0.0/8 + src: "" + gateway: 10.66.142.16 + outLinkName: bond0 + table: main + scope: global + type: unicast + flags: "" + protocol: static + layer: platform + - family: inet4 + dst: 169.254.255.1/32 + src: "" + gateway: 10.66.142.16 + outLinkName: bond0 + table: main + scope: global + type: unicast + flags: "" + protocol: static + layer: platform + - family: inet4 + dst: 169.254.255.2/32 + src: "" + gateway: 10.66.142.16 + outLinkName: bond0 + table: main + scope: global + type: unicast + flags: "" + protocol: static + layer: platform +hostnames: + - hostname: infra-green-ci + domainname: "" + layer: platform +resolvers: [] +timeServers: [] +operators: [] +externalIPs: + - 147.75.78.41 + - 2604:1380:45d1:fd00::11 +metadata: + platform: equinixMetal + hostname: infra-green-ci + region: ny + zone: ny5 + instanceType: c3.medium.x86 + instanceId: X + providerId: equinixmetal://X diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/equinixmetal/testdata/metadata.json b/internal/app/machined/pkg/runtime/v1alpha1/platform/equinixmetal/testdata/metadata.json new file mode 100644 index 0000000..1df1e1e --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/equinixmetal/testdata/metadata.json @@ -0,0 +1,149 @@ +{ + "id": "X", + "hostname": "infra-green-ci", + "plan": "c3.medium.x86", + "reserved": false, + "class": "c3.medium.x86", + "facility": "ny5", + "metro": "ny", + "private_subnets": [ + "10.0.0.0/8" + ], + "tags": [], + "ssh_keys": [ + ], + "customdata": {}, + "network": { + "bonding": { + "mode": 4, + "link_aggregation": "mlag_ha", + "mac": "68:05:ca:b8:f1:f8" + }, + "interfaces": [ + { + "name": "eth0", + "mac": "68:05:ca:b8:f1:f8", + "bond": "bond0" + }, + { + "name": "eth1", + "mac": "68:05:ca:b8:f1:f9", + "bond": "bond0" + } + ], + "addresses": [ + { + "id": "d6be5d63-50f8-452c-b5cd-6cba42fbd5b3", + "address_family": 4, + "netmask": "255.255.255.254", + "created_at": "2021-11-24T20:24:54Z", + "public": true, + "cidr": 31, + "management": true, + "enabled": true, + "network": "147.75.78.40", + "address": "147.75.78.41", + "gateway": "147.75.78.40", + "parent_block": { + "network": "147.75.78.40", + "netmask": "255.255.255.254", + "cidr": 31, + "href": "/ips/e5cc5a4e-1d80-42c8-b5ea-39644effb407" + } + }, + { + "id": "09c743e9-52e4-4125-9e88-da56c8e62ae4", + "address_family": 6, + "netmask": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:fffe", + "created_at": "2021-11-24T20:24:54Z", + "public": true, + "cidr": 127, + "management": true, + "enabled": true, + "network": "2604:1380:45d1:fd00::10", + "address": "2604:1380:45d1:fd00::11", + "gateway": "2604:1380:45d1:fd00::10", + "parent_block": { + "network": "2604:1380:45d1:fd00:0000:0000:0000:0000", + "netmask": "ffff:ffff:ffff:ff00:0000:0000:0000:0000", + "cidr": 56, + "href": "/ips/a76e6dd1-a22a-4f8a-a04d-7b68b4f358e5" + } + }, + { + "id": "c7d3cd31-beae-460a-b008-29776c95562b", + "address_family": 4, + "netmask": "255.255.255.255", + "created_at": "2021-12-10T13:41:14Z", + "public": true, + "cidr": 32, + "management": false, + "enabled": true, + "network": "147.75.195.143", + "address": "147.75.195.143", + "gateway": "147.75.195.143", + "parent_block": { + "network": "147.75.195.143", + "netmask": "255.255.255.255", + "cidr": 32, + "href": "/ips/77e054ac-cd40-473a-9c59-8f6c322c5a20" + } + }, + { + "id": "5e0bc796-9e7f-46a5-9472-ced35b8acb6d", + "address_family": 4, + "netmask": "255.255.255.254", + "created_at": "2021-11-24T20:24:54Z", + "public": false, + "cidr": 31, + "management": true, + "enabled": true, + "network": "10.66.142.16", + "address": "10.66.142.17", + "gateway": "10.66.142.16", + "parent_block": { + "network": "10.66.142.0", + "netmask": "255.255.255.128", + "cidr": 25, + "href": "/ips/045b5dd5-6a32-48e6-870d-8ea9a39169d6" + } + } + ], + "metal_gateways": [] + }, + "bgp_neighbors": [ + { + "address_family": 4, + "customer_as": 65000, + "customer_ip": "10.67.50.1", + "md5_enabled": false, + "md5_password": null, + "multihop": true, + "peer_as": 65530, + "peer_ips": [ + "169.254.255.1", + "169.254.255.2" + ], + "routes_in": [], + "routes_out": [] + }, + { + "address_family": 6, + "customer_as": 65000, + "customer_ip": "2604:1380:45e1:5000::1", + "md5_enabled": false, + "md5_password": null, + "multihop": true, + "peer_as": 65530, + "peer_ips": [ + "fc00:0000:0000:0000:0000:0000:0000:000e", + "fc00:0000:0000:0000:0000:0000:0000:000f" + ], + "routes_in": [], + "routes_out": [] + } + ], + "api_url": "https://metadata.packet.net", + "phone_home_url": "http://tinkerbell.ny5.packet.net/phone-home", + "user_state_url": "http://tinkerbell.ny5.packet.net/events" +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/errors/errors.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/errors/errors.go new file mode 100644 index 0000000..2c726a0 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/errors/errors.go @@ -0,0 +1,23 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package errors contains errors used by the platform package. +package errors + +import "errors" + +// ErrNoConfigSource indicates that the platform does not have a configured source for the configuration. +var ErrNoConfigSource = errors.New("no configuration source") + +// ErrNoHostname indicates that the meta server does not have a instance hostname. +var ErrNoHostname = errors.New("failed to fetch hostname from metadata service") + +// ErrNoExternalIPs indicates that the meta server does not have a external addresses. +var ErrNoExternalIPs = errors.New("failed to fetch external addresses from metadata service") + +// ErrNoEventURL indicates that the platform does not have an expected events URL in the kernel params. +var ErrNoEventURL = errors.New("no event URL") + +// ErrMetadataNotReady indicates that the platform does not have metadata yet. +var ErrMetadataNotReady = errors.New("platform metadata is not ready") diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/exoscale/exoscale.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/exoscale/exoscale.go new file mode 100644 index 0000000..3960cba --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/exoscale/exoscale.go @@ -0,0 +1,117 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package exoscale contains the Exoscale platform implementation. +package exoscale + +import ( + "context" + "fmt" + "log" + "net/netip" + "strings" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-procfs/procfs" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/errors" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/internal/netutils" + "github.com/siderolabs/talos/pkg/download" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// Exoscale is the concrete type that implements the runtime.Platform interface. +type Exoscale struct{} + +// ParseMetadata converts Exoscale platform metadata into platform network config. +func (e *Exoscale) ParseMetadata(metadata *MetadataConfig) (*runtime.PlatformNetworkConfig, error) { + networkConfig := &runtime.PlatformNetworkConfig{} + + if metadata.Hostname != "" { + hostnameSpec := network.HostnameSpecSpec{ + ConfigLayer: network.ConfigPlatform, + } + + if err := hostnameSpec.ParseFQDN(metadata.Hostname); err != nil { + return nil, err + } + + networkConfig.Hostnames = append(networkConfig.Hostnames, hostnameSpec) + } + + if metadata.PublicIPv4 != "" { + if ip, err := netip.ParseAddr(metadata.PublicIPv4); err == nil { + networkConfig.ExternalIPs = append(networkConfig.ExternalIPs, ip) + } + } + + networkConfig.Metadata = &runtimeres.PlatformMetadataSpec{ + Platform: e.Name(), + Hostname: metadata.Hostname, + Region: metadata.Zone, + Zone: metadata.Zone, + InstanceType: strings.ToLower(strings.SplitN(metadata.InstanceType, " ", 2)[0]), + InstanceID: metadata.InstanceID, + ProviderID: fmt.Sprintf("exoscale://%s", metadata.InstanceID), + } + + return networkConfig, nil +} + +// Name implements the runtime.Platform interface. +func (e *Exoscale) Name() string { + return "exoscale" +} + +// Configuration implements the runtime.Platform interface. +func (e *Exoscale) Configuration(ctx context.Context, r state.State) ([]byte, error) { + if err := netutils.Wait(ctx, r); err != nil { + return nil, err + } + + log.Printf("fetching machine config from %q", ExoscaleUserDataEndpoint) + + return download.Download(ctx, ExoscaleUserDataEndpoint, + download.WithErrorOnNotFound(errors.ErrNoConfigSource), + download.WithErrorOnEmptyResponse(errors.ErrNoConfigSource)) +} + +// Mode implements the runtime.Platform interface. +func (e *Exoscale) Mode() runtime.Mode { + return runtime.ModeCloud +} + +// KernelArgs implements the runtime.Platform interface. +func (e *Exoscale) KernelArgs(string) procfs.Parameters { + return []*procfs.Parameter{ + procfs.NewParameter("console").Append("tty1").Append("ttyS0"), + procfs.NewParameter(constants.KernelParamNetIfnames).Append("0"), + } +} + +// NetworkConfiguration implements the runtime.Platform interface. +func (e *Exoscale) NetworkConfiguration(ctx context.Context, _ state.State, ch chan<- *runtime.PlatformNetworkConfig) error { + log.Printf("fetching exoscale instance config from: %q", ExoscaleMetadataEndpoint) + + metadata, err := e.getMetadata(ctx) + if err != nil { + return err + } + + networkConfig, err := e.ParseMetadata(metadata) + if err != nil { + return err + } + + select { + case ch <- networkConfig: + case <-ctx.Done(): + return ctx.Err() + } + + return nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/exoscale/exoscale_test.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/exoscale/exoscale_test.go new file mode 100644 index 0000000..68e7e3e --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/exoscale/exoscale_test.go @@ -0,0 +1,39 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package exoscale_test + +import ( + _ "embed" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/exoscale" +) + +//go:embed testdata/metadata.json +var rawMetadata []byte + +//go:embed testdata/expected.yaml +var expectedNetworkConfig string + +func TestEmpty(t *testing.T) { + p := &exoscale.Exoscale{} + + var m exoscale.MetadataConfig + + require.NoError(t, json.Unmarshal(rawMetadata, &m)) + + networkConfig, err := p.ParseMetadata(&m) + require.NoError(t, err) + + marshaled, err := yaml.Marshal(networkConfig) + require.NoError(t, err) + + assert.Equal(t, expectedNetworkConfig, string(marshaled)) +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/exoscale/metadata.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/exoscale/metadata.go new file mode 100644 index 0000000..063b958 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/exoscale/metadata.go @@ -0,0 +1,67 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package exoscale + +import ( + "context" + stderrors "errors" + "fmt" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/errors" + "github.com/siderolabs/talos/pkg/download" +) + +const ( + // ExoscaleMetadataEndpoint is the local Exoscale endpoint. + ExoscaleMetadataEndpoint = "http://169.254.169.254/1.0/meta-data" + // ExoscaleUserDataEndpoint is the local Exoscale endpoint for the config. + ExoscaleUserDataEndpoint = "http://169.254.169.254/1.0/user-data" +) + +// MetadataConfig represents a metadata Exoscale instance. +type MetadataConfig struct { + Hostname string `json:"local-hostname,omitempty"` + InstanceID string `json:"instance-id,omitempty"` + InstanceType string `json:"service-offering,omitempty"` + PublicIPv4 string `json:"public-ipv4,omitempty"` + Zone string `json:"availability-zone,omitempty"` +} + +func (e *Exoscale) getMetadata(ctx context.Context) (metadata *MetadataConfig, err error) { + getMetadataKey := func(key string) (string, error) { + res, metaerr := download.Download(ctx, fmt.Sprintf("%s/%s", ExoscaleMetadataEndpoint, key), + download.WithErrorOnNotFound(errors.ErrNoConfigSource), + download.WithErrorOnEmptyResponse(errors.ErrNoConfigSource)) + if metaerr != nil && !stderrors.Is(metaerr, errors.ErrNoConfigSource) { + return "", fmt.Errorf("failed to fetch %q from IMDS: %w", key, metaerr) + } + + return string(res), nil + } + + metadata = &MetadataConfig{} + + if metadata.Hostname, err = getMetadataKey("local-hostname"); err != nil { + return nil, err + } + + if metadata.InstanceType, err = getMetadataKey("service-offering"); err != nil { + return nil, err + } + + if metadata.InstanceID, err = getMetadataKey("instance-id"); err != nil { + return nil, err + } + + if metadata.PublicIPv4, err = getMetadataKey("public-ipv4"); err != nil { + return nil, err + } + + if metadata.Zone, err = getMetadataKey("availability-zone"); err != nil { + return nil, err + } + + return metadata, nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/exoscale/testdata/expected.yaml b/internal/app/machined/pkg/runtime/v1alpha1/platform/exoscale/testdata/expected.yaml new file mode 100644 index 0000000..cdc180d --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/exoscale/testdata/expected.yaml @@ -0,0 +1,18 @@ +addresses: [] +links: [] +routes: [] +hostnames: + - hostname: talos + domainname: fqdn + layer: platform +resolvers: [] +timeServers: [] +operators: [] +externalIPs: + - 1.2.3.4 +metadata: + platform: exoscale + hostname: talos.fqdn + instanceType: standard.tiny + instanceId: 3085c764-b270-45b0-b974-68c55a9c2d53 + providerId: exoscale://3085c764-b270-45b0-b974-68c55a9c2d53 diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/exoscale/testdata/metadata.json b/internal/app/machined/pkg/runtime/v1alpha1/platform/exoscale/testdata/metadata.json new file mode 100644 index 0000000..4f60835 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/exoscale/testdata/metadata.json @@ -0,0 +1,6 @@ +{ + "local-hostname": "talos.fqdn", + "instance-id": "3085c764-b270-45b0-b974-68c55a9c2d53", + "public-ipv4": "1.2.3.4", + "service-offering": "standard.tiny" +} \ No newline at end of file diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/gcp/gcp.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/gcp/gcp.go new file mode 100644 index 0000000..df89b90 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/gcp/gcp.go @@ -0,0 +1,236 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package gcp contains the GCP implementation of the [platform.Platform]. +package gcp + +import ( + "context" + "fmt" + "log" + "net/netip" + "strconv" + "strings" + + "cloud.google.com/go/compute/metadata" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-procfs/procfs" + "github.com/siderolabs/go-retry/retry" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/errors" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/internal/netutils" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// GCP is the concrete type that implements the platform.Platform interface. +type GCP struct{} + +// Name implements the platform.Platform interface. +func (g *GCP) Name() string { + return "gcp" +} + +// ParseMetadata converts GCP platform metadata into platform network config. +// +//nolint:gocyclo +func (g *GCP) ParseMetadata(metadata *MetadataConfig, interfaces []NetworkInterfaceConfig) (*runtime.PlatformNetworkConfig, error) { + networkConfig := &runtime.PlatformNetworkConfig{} + + if metadata.Hostname != "" { + hostnameSpec := network.HostnameSpecSpec{ + ConfigLayer: network.ConfigPlatform, + } + + if err := hostnameSpec.ParseFQDN(metadata.Hostname); err != nil { + return nil, err + } + + networkConfig.Hostnames = append(networkConfig.Hostnames, hostnameSpec) + } + + dns, _ := netip.ParseAddr(gcpResolverServer) //nolint:errcheck + + networkConfig.Resolvers = append(networkConfig.Resolvers, network.ResolverSpecSpec{ + DNSServers: []netip.Addr{dns}, + ConfigLayer: network.ConfigPlatform, + }) + + networkConfig.TimeServers = append(networkConfig.TimeServers, network.TimeServerSpecSpec{ + NTPServers: []string{gcpTimeServer}, + ConfigLayer: network.ConfigPlatform, + }) + + region := metadata.Zone + + if idx := strings.LastIndex(region, "-"); idx != -1 { + region = region[:idx] + } + + for idx, iface := range interfaces { + ifname := fmt.Sprintf("eth%d", idx) + + networkConfig.Links = append(networkConfig.Links, network.LinkSpecSpec{ + Name: ifname, + Up: true, + MTU: uint32(iface.MTU), + ConfigLayer: network.ConfigPlatform, + }) + + networkConfig.Operators = append(networkConfig.Operators, network.OperatorSpecSpec{ + Operator: network.OperatorDHCP4, + LinkName: ifname, + DHCP4: network.DHCP4OperatorSpec{ + RouteMetric: network.DefaultRouteMetric, + }, + RequireUp: true, + ConfigLayer: network.ConfigPlatform, + }) + + for _, ipv6addr := range iface.IPv6 { + if ipv6addr == "" || iface.GatewayIPv6 == "" { + continue + } + + ipPrefix, err := netip.ParsePrefix(ipv6addr) + if err != nil { + return nil, fmt.Errorf("failed to parse ip address: %w", err) + } + + networkConfig.Addresses = append(networkConfig.Addresses, + network.AddressSpecSpec{ + ConfigLayer: network.ConfigPlatform, + LinkName: ifname, + Address: ipPrefix, + Scope: nethelpers.ScopeGlobal, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + Family: nethelpers.FamilyInet6, + }, + ) + + gw, err := netip.ParseAddr(iface.GatewayIPv6) + if err != nil { + return nil, err + } + + route := network.RouteSpecSpec{ + ConfigLayer: network.ConfigPlatform, + Gateway: gw, + OutLinkName: ifname, + Table: nethelpers.TableMain, + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + Family: nethelpers.FamilyInet6, + Priority: 2 * network.DefaultRouteMetric, + } + + route.Normalize() + + networkConfig.Routes = append(networkConfig.Routes, route) + } + } + + for _, iface := range interfaces { + for _, ipStr := range iface.AccessConfigs { + if ipStr.Type == "ONE_TO_ONE_NAT" { + if ip, err := netip.ParseAddr(ipStr.ExternalIP); err == nil { + networkConfig.ExternalIPs = append(networkConfig.ExternalIPs, ip) + } + } + } + } + + preempted, _ := strconv.ParseBool(metadata.Preempted) //nolint:errcheck + + networkConfig.Metadata = &runtimeres.PlatformMetadataSpec{ + Platform: g.Name(), + Hostname: metadata.Hostname, + Region: region, + Zone: metadata.Zone, + InstanceType: metadata.InstanceType, + InstanceID: metadata.InstanceID, + ProviderID: fmt.Sprintf("gce://%s/%s/%s", metadata.ProjectID, metadata.Zone, metadata.Name), + Spot: preempted, + } + + return networkConfig, nil +} + +// Configuration implements the platform.Platform interface. +func (g *GCP) Configuration(ctx context.Context, r state.State) ([]byte, error) { + if err := netutils.Wait(ctx, r); err != nil { + return nil, err + } + + log.Printf("fetching machine config from GCP metadata service") + + userdata, err := netutils.RetryFetch(ctx, g.fetchConfiguration) + if err != nil { + return nil, err + } + + if strings.TrimSpace(userdata) == "" { + return nil, errors.ErrNoConfigSource + } + + return []byte(userdata), nil +} + +func (g *GCP) fetchConfiguration(_ context.Context) (string, error) { + userdata, err := metadata.InstanceAttributeValue("user-data") + if err != nil { + if _, ok := err.(metadata.NotDefinedError); ok { + return "", errors.ErrNoConfigSource + } + + return "", retry.ExpectedError(err) + } + + return userdata, nil +} + +// Mode implements the platform.Platform interface. +func (g *GCP) Mode() runtime.Mode { + return runtime.ModeCloud +} + +// KernelArgs implements the runtime.Platform interface. +func (g *GCP) KernelArgs(string) procfs.Parameters { + return []*procfs.Parameter{ + procfs.NewParameter("console").Append("ttyS0"), + procfs.NewParameter(constants.KernelParamNetIfnames).Append("0"), + procfs.NewParameter(constants.KernelParamDashboardDisabled).Append("1"), + } +} + +// NetworkConfiguration implements the runtime.Platform interface. +func (g *GCP) NetworkConfiguration(ctx context.Context, st state.State, ch chan<- *runtime.PlatformNetworkConfig) error { + log.Printf("fetching gcp instance config") + + metadata, err := g.getMetadata(ctx) + if err != nil { + return fmt.Errorf("failed to receive GCP metadata: %w", err) + } + + network, err := g.getNetworkMetadata(ctx) + if err != nil { + return fmt.Errorf("failed to receive GCP network metadata: %w", err) + } + + networkConfig, err := g.ParseMetadata(metadata, network) + if err != nil { + return err + } + + select { + case ch <- networkConfig: + case <-ctx.Done(): + return ctx.Err() + } + + return nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/gcp/gcp_test.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/gcp/gcp_test.go new file mode 100644 index 0000000..5e86aac --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/gcp/gcp_test.go @@ -0,0 +1,46 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package gcp_test + +import ( + _ "embed" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/gcp" +) + +//go:embed testdata/metadata.json +var rawMetadata []byte + +//go:embed testdata/interfaces.json +var rawInterfaces []byte + +//go:embed testdata/expected.yaml +var expectedNetworkConfig string + +func TestParseMetadata(t *testing.T) { + p := &gcp.GCP{} + + var ( + metadata gcp.MetadataConfig + interfaces []gcp.NetworkInterfaceConfig + ) + + require.NoError(t, json.Unmarshal(rawMetadata, &metadata)) + require.NoError(t, json.Unmarshal(rawInterfaces, &interfaces)) + + networkConfig, err := p.ParseMetadata(&metadata, interfaces) + require.NoError(t, err) + + marshaled, err := yaml.Marshal(networkConfig) + require.NoError(t, err) + + assert.Equal(t, expectedNetworkConfig, string(marshaled)) +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/gcp/metadata.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/gcp/metadata.go new file mode 100644 index 0000000..b4126ec --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/gcp/metadata.go @@ -0,0 +1,98 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package gcp + +import ( + "context" + "encoding/json" + "strings" + + "cloud.google.com/go/compute/metadata" +) + +const ( + gcpResolverServer = "169.254.169.254" + gcpTimeServer = "metadata.google.internal" +) + +// MetadataConfig holds meta info. +type MetadataConfig struct { + ProjectID string `json:"project-id"` + Name string `json:"name,omitempty"` + Hostname string `json:"hostname,omitempty"` + Zone string `json:"zone,omitempty"` + InstanceType string `json:"machine-type"` + InstanceID string `json:"id"` + Preempted string `json:"preempted"` +} + +// NetworkInterfaceConfig holds network meta info. +type NetworkInterfaceConfig struct { + AccessConfigs []struct { + ExternalIP string `json:"externalIp,omitempty"` + Type string `json:"type,omitempty"` + } `json:"accessConfigs,omitempty"` + GatewayIPv4 string `json:"gateway,omitempty"` + GatewayIPv6 string `json:"gatewayIpv6,omitempty"` + IPv4 string `json:"ip,omitempty"` + IPv6 []string `json:"ipv6,omitempty"` + MTU int `json:"mtu,omitempty"` +} + +func (g *GCP) getMetadata(context.Context) (*MetadataConfig, error) { + var ( + meta MetadataConfig + err error + ) + + if meta.ProjectID, err = metadata.ProjectID(); err != nil { + return nil, err + } + + if meta.Name, err = metadata.InstanceName(); err != nil { + return nil, err + } + + instanceType, err := metadata.Get("instance/machine-type") + if err != nil { + return nil, err + } + + meta.InstanceType = strings.TrimSpace(instanceType[strings.LastIndex(instanceType, "/")+1:]) + + if meta.InstanceID, err = metadata.InstanceID(); err != nil { + return nil, err + } + + if meta.Hostname, err = metadata.Hostname(); err != nil { + return nil, err + } + + if meta.Zone, err = metadata.Zone(); err != nil { + return nil, err + } + + meta.Preempted, err = metadata.Get("instance/scheduling/preemptible") + if err != nil { + return nil, err + } + + return &meta, nil +} + +func (g *GCP) getNetworkMetadata(context.Context) ([]NetworkInterfaceConfig, error) { + metadataNetworkConfigDl, err := metadata.Get("instance/network-interfaces/?recursive=true&alt=json") + if err != nil { + return nil, err + } + + var unmarshalledNetworkConfig []NetworkInterfaceConfig + + if err = json.Unmarshal([]byte(metadataNetworkConfigDl), &unmarshalledNetworkConfig); err != nil { + return nil, err + } + + return unmarshalledNetworkConfig, nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/gcp/testdata/expected.yaml b/internal/app/machined/pkg/runtime/v1alpha1/platform/gcp/testdata/expected.yaml new file mode 100644 index 0000000..c5b2f91 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/gcp/testdata/expected.yaml @@ -0,0 +1,57 @@ +addresses: + - address: fd20:172:1610:7003:0:1::/96 + linkName: eth0 + family: inet6 + scope: global + flags: permanent + layer: platform +links: + - name: eth0 + logical: false + up: true + mtu: 1500 + kind: "" + type: netrom + layer: platform +routes: + - family: inet6 + dst: "" + src: "" + gateway: fe80::4001:acff:fe10:1 + outLinkName: eth0 + table: main + priority: 2048 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform +hostnames: + - hostname: talos + domainname: "" + layer: platform +resolvers: + - dnsServers: + - 169.254.169.254 + layer: platform +timeServers: + - timeServers: + - metadata.google.internal + layer: platform +operators: + - operator: dhcp4 + linkName: eth0 + requireUp: true + dhcp4: + routeMetric: 1024 + layer: platform +externalIPs: + - 35.1.2.3 +metadata: + platform: gcp + hostname: talos + region: us-central1 + zone: us-central1-a + instanceType: n1-standard-1 + instanceId: "0" + providerId: gce://123/us-central1-a/my-server diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/gcp/testdata/interfaces.json b/internal/app/machined/pkg/runtime/v1alpha1/platform/gcp/testdata/interfaces.json new file mode 100644 index 0000000..411303c --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/gcp/testdata/interfaces.json @@ -0,0 +1,32 @@ +[ + { + "accessConfigs": [ + { + "externalIp": "35.1.2.3", + "type": "ONE_TO_ONE_NAT" + } + ], + "dhcpv6Refresh": "2219726792944608985", + "dnsServers": [ + "169.254.169.254" + ], + "forwardedIps": [ + "172.16.0.230" + ], + "gateway": "172.16.0.1", + "gatewayIpv6": "fe80::4001:acff:fe10:1", + "ip": "172.16.0.4", + "ipAliases": [], + "ipv6": [ + "fd20:172:1610:7003:0:1::/96" + ], + "ipv6s": [ + "fd20:172:1610:7003:0:1:0:0" + ], + "mac": "42:01:ac:10:00:04", + "mtu": 1500, + "network": "projects/123/networks/main", + "subnetmask": "255.255.255.0", + "targetInstanceIps": [] + } +] \ No newline at end of file diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/gcp/testdata/metadata.json b/internal/app/machined/pkg/runtime/v1alpha1/platform/gcp/testdata/metadata.json new file mode 100644 index 0000000..d864fab --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/gcp/testdata/metadata.json @@ -0,0 +1,9 @@ +{ + "project-id": "123", + "hostname": "talos", + "id": "0", + "zone": "us-central1-a", + "name": "my-server", + "machine-type": "n1-standard-1", + "preempted": "FALSE" +} \ No newline at end of file diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/hcloud/hcloud.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/hcloud/hcloud.go new file mode 100644 index 0000000..8004c92 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/hcloud/hcloud.go @@ -0,0 +1,213 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package hcloud contains the Hcloud implementation of the [platform.Platform]. +package hcloud + +import ( + "context" + "fmt" + "log" + "net/netip" + "strings" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-procfs/procfs" + "gopkg.in/yaml.v3" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/errors" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/internal/netutils" + "github.com/siderolabs/talos/pkg/download" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// Hcloud is the concrete type that implements the runtime.Platform interface. +type Hcloud struct{} + +// Name implements the runtime.Platform interface. +func (h *Hcloud) Name() string { + return "hcloud" +} + +// ParseMetadata converts HCloud metadata to platform network configuration. +// +//nolint:gocyclo +func (h *Hcloud) ParseMetadata(unmarshalledNetworkConfig *NetworkConfig, metadata *MetadataConfig) (*runtime.PlatformNetworkConfig, error) { + networkConfig := &runtime.PlatformNetworkConfig{} + + if metadata.Hostname != "" { + hostnameSpec := network.HostnameSpecSpec{ + ConfigLayer: network.ConfigPlatform, + } + + if err := hostnameSpec.ParseFQDN(metadata.Hostname); err != nil { + return nil, err + } + + networkConfig.Hostnames = append(networkConfig.Hostnames, hostnameSpec) + } + + publicIPs := []string{} + + if metadata.PublicIPv4 != "" { + publicIPs = append(publicIPs, metadata.PublicIPv4) + } + + for _, ntwrk := range unmarshalledNetworkConfig.Config { + if ntwrk.Type != "physical" { + continue + } + + networkConfig.Links = append(networkConfig.Links, network.LinkSpecSpec{ + Name: ntwrk.Interfaces, + Up: true, + ConfigLayer: network.ConfigPlatform, + }) + + for _, subnet := range ntwrk.Subnets { + if subnet.Type == "dhcp" && subnet.Ipv4 { + networkConfig.Operators = append(networkConfig.Operators, network.OperatorSpecSpec{ + Operator: network.OperatorDHCP4, + LinkName: ntwrk.Interfaces, + DHCP4: network.DHCP4OperatorSpec{ + RouteMetric: network.DefaultRouteMetric, + }, + ConfigLayer: network.ConfigPlatform, + }) + } + + if subnet.Type == "static" { + ipAddr, err := netip.ParsePrefix(subnet.Address) + if err != nil { + return nil, err + } + + family := nethelpers.FamilyInet4 + + if ipAddr.Addr().Is6() { + publicIPs = append(publicIPs, strings.SplitN(subnet.Address, "/", 2)[0]) + family = nethelpers.FamilyInet6 + } + + networkConfig.Addresses = append(networkConfig.Addresses, + network.AddressSpecSpec{ + ConfigLayer: network.ConfigPlatform, + LinkName: ntwrk.Interfaces, + Address: ipAddr, + Scope: nethelpers.ScopeGlobal, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + Family: family, + }, + ) + } + + if subnet.Gateway != "" && subnet.Ipv6 { + gw, err := netip.ParseAddr(subnet.Gateway) + if err != nil { + return nil, err + } + + route := network.RouteSpecSpec{ + ConfigLayer: network.ConfigPlatform, + Gateway: gw, + OutLinkName: ntwrk.Interfaces, + Table: nethelpers.TableMain, + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + Family: nethelpers.FamilyInet6, + Priority: network.DefaultRouteMetric, + } + + route.Normalize() + + networkConfig.Routes = append(networkConfig.Routes, route) + } + } + } + + for _, ipStr := range publicIPs { + if ip, err := netip.ParseAddr(ipStr); err == nil { + networkConfig.ExternalIPs = append(networkConfig.ExternalIPs, ip) + } + } + + networkConfig.Metadata = &runtimeres.PlatformMetadataSpec{ + Platform: h.Name(), + Hostname: metadata.Hostname, + Region: metadata.Region, + Zone: metadata.AvailabilityZone, + InstanceID: metadata.InstanceID, + ProviderID: fmt.Sprintf("hcloud://%s", metadata.InstanceID), + } + + return networkConfig, nil +} + +// Configuration implements the runtime.Platform interface. +func (h *Hcloud) Configuration(ctx context.Context, r state.State) ([]byte, error) { + if err := netutils.Wait(ctx, r); err != nil { + return nil, err + } + + log.Printf("fetching machine config from: %q", HCloudUserDataEndpoint) + + return download.Download(ctx, HCloudUserDataEndpoint, + download.WithErrorOnNotFound(errors.ErrNoConfigSource), + download.WithErrorOnEmptyResponse(errors.ErrNoConfigSource)) +} + +// Mode implements the runtime.Platform interface. +func (h *Hcloud) Mode() runtime.Mode { + return runtime.ModeCloud +} + +// KernelArgs implements the runtime.Platform interface. +func (h *Hcloud) KernelArgs(string) procfs.Parameters { + return []*procfs.Parameter{ + procfs.NewParameter("console").Append("tty1").Append("ttyS0"), + procfs.NewParameter(constants.KernelParamNetIfnames).Append("0"), + } +} + +// NetworkConfiguration implements the runtime.Platform interface. +func (h *Hcloud) NetworkConfiguration(ctx context.Context, _ state.State, ch chan<- *runtime.PlatformNetworkConfig) error { + metadata, err := h.getMetadata(ctx) + if err != nil { + return err + } + + log.Printf("fetching hcloud network config from: %q", HCloudNetworkEndpoint) + + metadataNetworkConfig, err := download.Download(ctx, HCloudNetworkEndpoint) + if err != nil { + return fmt.Errorf("failed to fetch network config from metadata service: %w", err) + } + + var unmarshalledNetworkConfig NetworkConfig + + if err = yaml.Unmarshal(metadataNetworkConfig, &unmarshalledNetworkConfig); err != nil { + return err + } + + if unmarshalledNetworkConfig.Version != 1 { + return fmt.Errorf("network-config metadata version=%d is not supported", unmarshalledNetworkConfig.Version) + } + + networkConfig, err := h.ParseMetadata(&unmarshalledNetworkConfig, metadata) + if err != nil { + return err + } + + select { + case ch <- networkConfig: + case <-ctx.Done(): + return ctx.Err() + } + + return nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/hcloud/hcloud_test.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/hcloud/hcloud_test.go new file mode 100644 index 0000000..d13ed51 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/hcloud/hcloud_test.go @@ -0,0 +1,46 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package hcloud_test + +import ( + _ "embed" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/hcloud" +) + +//go:embed testdata/metadata.yaml +var rawMetadata []byte + +//go:embed testdata/expected.yaml +var expectedNetworkConfig string + +func TestParseMetadata(t *testing.T) { + h := &hcloud.Hcloud{} + + metadata := &hcloud.MetadataConfig{ + Hostname: "talos.fqdn", + PublicIPv4: "1.2.3.4", + InstanceID: "0", + Region: "hel1", + AvailabilityZone: "hel1-dc2", + } + + var m hcloud.NetworkConfig + + require.NoError(t, yaml.Unmarshal(rawMetadata, &m)) + + networkConfig, err := h.ParseMetadata(&m, metadata) + require.NoError(t, err) + + marshaled, err := yaml.Marshal(networkConfig) + require.NoError(t, err) + + assert.Equal(t, expectedNetworkConfig, string(marshaled)) +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/hcloud/metadata.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/hcloud/metadata.go new file mode 100644 index 0000000..8dfc177 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/hcloud/metadata.go @@ -0,0 +1,91 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package hcloud + +import ( + "context" + stderrors "errors" + "fmt" + "strings" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/errors" + "github.com/siderolabs/talos/pkg/download" +) + +const ( + // HCloudMetadataEndpoint is the local HCloud metadata endpoint. + HCloudMetadataEndpoint = "http://169.254.169.254/hetzner/v1/metadata" + + // HCloudNetworkEndpoint is the local HCloud metadata endpoint for the network-config. + HCloudNetworkEndpoint = "http://169.254.169.254/hetzner/v1/metadata/network-config" + + // HCloudUserDataEndpoint is the local HCloud metadata endpoint for the config. + HCloudUserDataEndpoint = "http://169.254.169.254/hetzner/v1/userdata" +) + +// MetadataConfig holds meta info. +type MetadataConfig struct { + Hostname string `yaml:"hostname,omitempty"` + Region string `yaml:"region,omitempty"` + AvailabilityZone string `yaml:"availability-zone,omitempty"` + InstanceID string `yaml:"instance-id,omitempty"` + PublicIPv4 string `yaml:"public-ipv4,omitempty"` +} + +// NetworkConfig holds hcloud network-config info. +type NetworkConfig struct { + Version int `yaml:"version"` + Config []struct { + Mac string `yaml:"mac_address"` + Interfaces string `yaml:"name"` + Subnets []struct { + NameServers []string `yaml:"dns_nameservers,omitempty"` + Address string `yaml:"address,omitempty"` + Gateway string `yaml:"gateway,omitempty"` + Ipv4 bool `yaml:"ipv4,omitempty"` + Ipv6 bool `yaml:"ipv6,omitempty"` + Type string `yaml:"type"` + } `yaml:"subnets"` + Type string `yaml:"type"` + } `yaml:"config"` +} + +func (h *Hcloud) getMetadata(ctx context.Context) (metadata *MetadataConfig, err error) { + getMetadataKey := func(key string) (string, error) { + res, metaerr := download.Download(ctx, fmt.Sprintf("%s/%s", HCloudMetadataEndpoint, key), + download.WithErrorOnNotFound(errors.ErrNoConfigSource), + download.WithErrorOnEmptyResponse(errors.ErrNoConfigSource)) + if metaerr != nil && !stderrors.Is(metaerr, errors.ErrNoConfigSource) { + return "", fmt.Errorf("failed to fetch %q from IMDS: %w", key, metaerr) + } + + return string(res), nil + } + + metadata = &MetadataConfig{} + + if metadata.Hostname, err = getMetadataKey("hostname"); err != nil { + return nil, err + } + + if metadata.InstanceID, err = getMetadataKey("instance-id"); err != nil { + return nil, err + } + + if metadata.AvailabilityZone, err = getMetadataKey("availability-zone"); err != nil { + return nil, err + } + + // Original CCM/CSI uses first part of availability-zone to define region name. + // But metadata has different value. + // We will follow official behavior. + metadata.Region = strings.SplitN(metadata.AvailabilityZone, "-", 2)[0] + + if metadata.PublicIPv4, err = getMetadataKey("public-ipv4"); err != nil { + return nil, err + } + + return metadata, nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/hcloud/testdata/expected.yaml b/internal/app/machined/pkg/runtime/v1alpha1/platform/hcloud/testdata/expected.yaml new file mode 100644 index 0000000..08de327 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/hcloud/testdata/expected.yaml @@ -0,0 +1,51 @@ +addresses: + - address: 2a01:4f8:1:2::1/64 + linkName: eth0 + family: inet6 + scope: global + flags: permanent + layer: platform +links: + - name: eth0 + logical: false + up: true + mtu: 0 + kind: "" + type: netrom + layer: platform +routes: + - family: inet6 + dst: "" + src: "" + gateway: fe80::1 + outLinkName: eth0 + table: main + priority: 1024 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform +hostnames: + - hostname: talos + domainname: fqdn + layer: platform +resolvers: [] +timeServers: [] +operators: + - operator: dhcp4 + linkName: eth0 + requireUp: false + dhcp4: + routeMetric: 1024 + layer: platform +externalIPs: + - 1.2.3.4 + - 2a01:4f8:1:2::1 +metadata: + platform: hcloud + hostname: talos.fqdn + region: hel1 + zone: hel1-dc2 + instanceId: "0" + providerId: hcloud://0 diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/hcloud/testdata/metadata.yaml b/internal/app/machined/pkg/runtime/v1alpha1/platform/hcloud/testdata/metadata.yaml new file mode 100644 index 0000000..127b7d0 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/hcloud/testdata/metadata.yaml @@ -0,0 +1,17 @@ +config: +- mac_address: 96:00:00:1:2:3 + name: eth0 + subnets: + - ipv4: true + type: dhcp + - address: 2a01:4f8:1:2::1/64 + gateway: fe80::1 + ipv6: true + type: static + type: physical +- address: + - 185.12.64.2 + - 185.12.64.1 + interface: eth0 + type: nameserver +version: 1 diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/internal/address/address.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/internal/address/address.go new file mode 100644 index 0000000..adcfb10 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/internal/address/address.go @@ -0,0 +1,58 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package address provides utility functions for address parsing. +package address + +import ( + "errors" + "fmt" + "net" + "net/netip" + "strconv" + "strings" +) + +// IPPrefixFrom make netip.Prefix from cidr-address and netmask strings. +// address can be IP or CIDR (1.1.1.1 or 1.1.1.1/8 or 1.1.1.1/255.0.0.0) +// netmask can be IP or number (255.255.255.0 or 24 or empty). +func IPPrefixFrom(address, netmask string) (netip.Prefix, error) { + cidr := strings.SplitN(address, "/", 2) + if len(cidr) == 1 { + address = cidr[0] + } else { + address = cidr[0] + netmask = cidr[1] + } + + ip, err := netip.ParseAddr(address) + if err != nil { + return netip.Prefix{}, fmt.Errorf("failed to parse ip address: %w", err) + } + + if netmask == "" { + if ip.Is4() { + netmask = "32" + } else { + netmask = "128" + } + } + + bits, err := strconv.Atoi(netmask) + if err != nil { + netmask, err := netip.ParseAddr(netmask) + if err != nil { + return netip.Prefix{}, fmt.Errorf("failed to parse netmask: %w", err) + } + + mask, _ := netmask.MarshalBinary() //nolint:errcheck // never fails + bits, _ = net.IPMask(mask).Size() + } + + if ip.Is4() && bits > 32 { + return netip.Prefix{}, errors.New("failed netmask should be the same address family") + } + + return netip.PrefixFrom(ip, bits), nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/internal/netutils/netutils.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/internal/netutils/netutils.go new file mode 100644 index 0000000..c358e99 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/internal/netutils/netutils.go @@ -0,0 +1,101 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package netutils provides network-related helpers for platform implementation. +package netutils + +import ( + "context" + "fmt" + "log" + "time" + + "github.com/cenkalti/backoff/v4" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-retry/retry" + + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// Wait for the network to be ready to interact with platform metadata services. +func Wait(ctx context.Context, r state.State) error { + log.Printf("waiting for network to be ready") + + return network.NewReadyCondition(r, network.AddressReady).Wait(ctx) +} + +// WaitInterfaces for the interfaces to be up to interact with platform metadata services. +func WaitInterfaces(ctx context.Context, r state.State) error { + backoff := backoff.NewExponentialBackOff() + backoff.MaxInterval = 2 * time.Second + backoff.MaxElapsedTime = 30 * time.Second + + for ctx.Err() == nil { + hostInterfaces, err := safe.StateListAll[*network.LinkStatus](ctx, r) + if err != nil { + return fmt.Errorf("error listing host interfaces: %w", err) + } + + numPhysical := 0 + + for iter := hostInterfaces.Iterator(); iter.Next(); { + iface := iter.Value() + + if iface.TypedSpec().Physical() { + numPhysical++ + } + } + + if numPhysical > 0 { + return nil + } + + log.Printf("waiting for physical network interfaces to appear...") + + interval := backoff.NextBackOff() + + select { + case <-ctx.Done(): + return nil + case <-time.After(interval): + } + } + + return nil +} + +// WaitForDevicesReady waits for devices to be ready. +func WaitForDevicesReady(ctx context.Context, r state.State) error { + log.Printf("waiting for devices to be ready...") + + return runtime.NewDevicesStatusCondition(r).Wait(ctx) +} + +// RetryFetch retries fetching from metadata service. +func RetryFetch(ctx context.Context, f func(ctx context.Context) (string, error)) (string, error) { + var ( + userdata string + err error + ) + + err = retry.Exponential( + constants.ConfigLoadTimeout, + retry.WithUnits(time.Second), + retry.WithJitter(time.Second), + retry.WithErrorLogging(true), + ).RetryWithContext( + ctx, func(ctx context.Context) error { + userdata, err = f(ctx) + + return err + }) + if err != nil { + return "", err + } + + return userdata, err +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/metal.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/metal.go new file mode 100644 index 0000000..2620d79 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/metal.go @@ -0,0 +1,245 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package metal contains the metal implementation of the [platform.Platform]. +package metal + +import ( + "context" + stderrors "errors" + "fmt" + "log" + "os" + "path/filepath" + "time" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/channel" + "github.com/siderolabs/go-blockdevice/blockdevice/filesystem" + "github.com/siderolabs/go-blockdevice/blockdevice/probe" + "github.com/siderolabs/go-pointer" + "github.com/siderolabs/go-procfs/procfs" + "github.com/siderolabs/go-retry/retry" + "golang.org/x/sys/unix" + "gopkg.in/yaml.v3" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/errors" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/internal/netutils" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/oauth2" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url" + "github.com/aenix-io/talm/internal/pkg/meta" + "github.com/siderolabs/talos/pkg/download" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/hardware" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +const ( + mnt = "/mnt" +) + +// Metal is a discoverer for non-cloud environments. +type Metal struct{} + +// Name implements the platform.Platform interface. +func (m *Metal) Name() string { + return constants.PlatformMetal +} + +// Configuration implements the platform.Platform interface. +func (m *Metal) Configuration(ctx context.Context, r state.State) ([]byte, error) { + var option *string + if option = procfs.ProcCmdline().Get(constants.KernelParamConfig).First(); option == nil { + return nil, errors.ErrNoConfigSource + } + + if *option == constants.ConfigNone { + return nil, errors.ErrNoConfigSource + } + + getURL := func(ctx context.Context) (string, error) { + // give a shorter timeout to populate the URL, leave the rest of the time to the actual download + ctx, cancel := context.WithTimeout(ctx, constants.ConfigLoadAttemptTimeout/2) + defer cancel() + + downloadEndpoint, err := url.Populate(ctx, *option, r) + if err != nil { + log.Printf("failed to populate talos.config fetch URL %q: %s", *option, err.Error()) + } + + log.Printf("fetching machine config from: %q", downloadEndpoint) + + return downloadEndpoint, nil + } + + switch *option { + case constants.MetalConfigISOLabel: + return readConfigFromISO(ctx, r) + default: + if err := netutils.Wait(ctx, r); err != nil { + return nil, err + } + + oauth2Cfg, err := oauth2.NewConfig(procfs.ProcCmdline(), *option) + if err != nil && !os.IsNotExist(err) { + return nil, fmt.Errorf("failed to parse OAuth2 config: %w", err) + } + + var extraHeaders map[string]string + + // perform OAuth2 device auth flow first to acquire extra headers + if oauth2Cfg != nil { + if err = retry.Constant(constants.ConfigLoadTimeout, retry.WithUnits(30*time.Second)).RetryWithContext(ctx, func(ctx context.Context) error { + return oauth2Cfg.DeviceAuthFlow(ctx, r) + }); err != nil { + return nil, fmt.Errorf("OAuth2 device auth flow failed: %w", err) + } + + extraHeaders = oauth2Cfg.ExtraHeaders() + } + + return download.Download( + ctx, + *option, + download.WithEndpointFunc(getURL), + download.WithTimeout(constants.ConfigLoadTimeout), + download.WithRetryOptions( + // give a timeout per attempt, max 50% of that is dedicated for URL interpolation, the rest is for the actual download + retry.WithAttemptTimeout(constants.ConfigLoadAttemptTimeout), + ), + download.WithHeaders(extraHeaders), + ) + } +} + +// Mode implements the platform.Platform interface. +func (m *Metal) Mode() runtime.Mode { + return runtime.ModeMetal +} + +func readConfigFromISO(ctx context.Context, r state.State) ([]byte, error) { + if err := netutils.WaitForDevicesReady(ctx, r); err != nil { + return nil, fmt.Errorf("failed to wait for devices: %w", err) + } + + dev, err := probe.GetDevWithFileSystemLabel(constants.MetalConfigISOLabel) + if err != nil { + return nil, fmt.Errorf("failed to find %s iso: %w", constants.MetalConfigISOLabel, err) + } + + //nolint:errcheck + defer dev.Close() + + sb, err := filesystem.Probe(dev.Device().Name()) + if err != nil { + return nil, err + } + + if sb == nil { + return nil, stderrors.New("error while substituting filesystem type") + } + + if err = unix.Mount(dev.Device().Name(), mnt, sb.Type(), unix.MS_RDONLY, ""); err != nil { + return nil, fmt.Errorf("failed to mount iso: %w", err) + } + + b, err := os.ReadFile(filepath.Join(mnt, filepath.Base(constants.ConfigPath))) + if err != nil { + return nil, fmt.Errorf("read config: %s", err.Error()) + } + + if err = unix.Unmount(mnt, 0); err != nil { + return nil, fmt.Errorf("failed to unmount: %w", err) + } + + return b, nil +} + +// KernelArgs implements the runtime.Platform interface. +func (m *Metal) KernelArgs(arch string) procfs.Parameters { + switch arch { + case "amd64": + return []*procfs.Parameter{ + procfs.NewParameter("console").Append("ttyS0").Append("tty0"), + } + case "arm64": + return []*procfs.Parameter{ + procfs.NewParameter("console").Append("ttyAMA0").Append("tty0"), + } + default: + return nil + } +} + +// NetworkConfiguration implements the runtime.Platform interface. +// +//nolint:gocyclo +func (m *Metal) NetworkConfiguration(ctx context.Context, st state.State, ch chan<- *runtime.PlatformNetworkConfig) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + watchCh := make(chan state.Event) + + if err := st.Watch(ctx, hardware.NewSystemInformation(hardware.SystemInformationID).Metadata(), watchCh); err != nil { + return err + } + + if err := st.Watch(ctx, runtimeres.NewMetaKey(runtimeres.NamespaceName, runtimeres.MetaKeyTagToID(meta.MetalNetworkPlatformConfig)).Metadata(), watchCh); err != nil { + return err + } + + // network config from META partition + var metaCfg runtime.PlatformNetworkConfig + + // fixed metadata filled by this function + metadata := &runtimeres.PlatformMetadataSpec{} + metadata.Platform = m.Name() + + if option := procfs.ProcCmdline().Get(constants.KernelParamHostname).First(); option != nil { + metadata.Hostname = *option + } + + for { + var event state.Event + + select { + case <-ctx.Done(): + return ctx.Err() + case event = <-watchCh: + } + + switch event.Type { + case state.Errored: + return fmt.Errorf("watch failed: %w", event.Error) + case state.Bootstrapped: + // ignored, should not happen + case state.Created, state.Updated: + switch r := event.Resource.(type) { + case *hardware.SystemInformation: + metadata.InstanceID = r.TypedSpec().UUID + case *runtimeres.MetaKey: + metaCfg = runtime.PlatformNetworkConfig{} + + if err := yaml.Unmarshal([]byte(r.TypedSpec().Value), &metaCfg); err != nil { + return fmt.Errorf("failed to unmarshal metal network config from META: %w", err) + } + } + case state.Destroyed: + switch event.Resource.(type) { + case *hardware.SystemInformation: + metadata.InstanceID = "" + case *runtimeres.MetaKey: + metaCfg = runtime.PlatformNetworkConfig{} + } + } + + cfg := metaCfg + cfg.Metadata = pointer.To(metadata.DeepCopy()) + + if !channel.SendWithContext(ctx, ch, &cfg) { + return ctx.Err() + } + } +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/metal_test.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/metal_test.go new file mode 100644 index 0000000..bcd088b --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/metal_test.go @@ -0,0 +1,120 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package metal_test + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/metal" + "github.com/aenix-io/talm/internal/pkg/meta" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/hardware" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +//nolint:gocyclo +func TestNetworkConfig(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + t.Cleanup(cancel) + + p := &metal.Metal{} + + ch := make(chan *runtime.PlatformNetworkConfig, 1) + + st := state.WrapCore(namespaced.NewState(inmem.Build)) + + uuid := hardware.NewSystemInformation(hardware.SystemInformationID) + uuid.TypedSpec().UUID = "0123-4567-89ab-cdef" + require.NoError(t, st.Create(ctx, uuid)) + + errCh := make(chan error) + + go func() { + errCh <- p.NetworkConfiguration(ctx, st, ch) + }() + + // platform might see updates coming in different order, so we need to wait a bit for the final state +outerLoop: + for { + select { + case <-ctx.Done(): + require.FailNow(t, "timed out waiting for network config") + case cfg := <-ch: + assert.Equal(t, constants.PlatformMetal, cfg.Metadata.Platform) + + if cfg.Metadata.InstanceID == "" { + continue + } + + assert.Equal(t, uuid.TypedSpec().UUID, cfg.Metadata.InstanceID) + + break outerLoop + } + } + + metaKey := runtimeres.NewMetaKey(runtimeres.NamespaceName, runtimeres.MetaKeyTagToID(meta.MetalNetworkPlatformConfig)) + metaKey.TypedSpec().Value = `{"externalIPs": ["1.2.3.4"]}` + require.NoError(t, st.Create(ctx, metaKey)) + + // platform might see updates coming in different order, so we need to wait a bit for the final state +outerLoop2: + for { + select { + case <-ctx.Done(): + require.FailNow(t, "timed out waiting for network config") + case cfg := <-ch: + assert.Equal(t, constants.PlatformMetal, cfg.Metadata.Platform) + assert.Equal(t, uuid.TypedSpec().UUID, cfg.Metadata.InstanceID) + + if len(cfg.ExternalIPs) == 0 { + continue + } + + assert.Equal(t, "[1.2.3.4]", fmt.Sprintf("%v", cfg.ExternalIPs)) + + break outerLoop2 + } + } + + metaKey.TypedSpec().Value = `{"hostnames": [{"hostname": "talos", "domainname": "fqdn", "layer": "platform"}]}` + require.NoError(t, st.Update(ctx, metaKey)) + + select { + case <-ctx.Done(): + require.FailNow(t, "timed out waiting for network config") + case cfg := <-ch: + assert.Equal(t, constants.PlatformMetal, cfg.Metadata.Platform) + assert.Equal(t, uuid.TypedSpec().UUID, cfg.Metadata.InstanceID) + + assert.Equal(t, "[]", fmt.Sprintf("%v", cfg.ExternalIPs)) + assert.Equal(t, "[{talos fqdn platform}]", fmt.Sprintf("%v", cfg.Hostnames)) + } + + require.NoError(t, st.Destroy(ctx, metaKey.Metadata())) + + select { + case <-ctx.Done(): + require.FailNow(t, "timed out waiting for network config") + case cfg := <-ch: + assert.Equal(t, constants.PlatformMetal, cfg.Metadata.Platform) + assert.Equal(t, uuid.TypedSpec().UUID, cfg.Metadata.InstanceID) + + assert.Equal(t, "[]", fmt.Sprintf("%v", cfg.ExternalIPs)) + assert.Equal(t, "[]", fmt.Sprintf("%v", cfg.Hostnames)) + } + + cancel() + require.ErrorIs(t, <-errCh, context.Canceled) +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/oauth2/oauth2.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/oauth2/oauth2.go new file mode 100644 index 0000000..968c0c3 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/oauth2/oauth2.go @@ -0,0 +1,204 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package oauth2 implements OAuth2 Device Flow to authenticate machine config download. +package oauth2 + +import ( + "bytes" + "context" + "fmt" + "log" + "net/http" + "net/url" + "os" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/hashicorp/go-cleanhttp" + "github.com/mdp/qrterminal/v3" + "github.com/siderolabs/go-procfs/procfs" + "golang.org/x/oauth2" + + metalurl "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url" + "github.com/siderolabs/talos/pkg/httpdefaults" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// Config represents the OAuth2 configuration. +type Config struct { + ClientID string + ClientSecret string + Audience string + Scopes []string + + ExtraVariables []string + + DeviceAuthURL string + TokenURL string + + extraHeaders map[string]string +} + +// NewConfig returns a new Config from cmdline. +// +// If OAuth2 is not configured, it returns os.ErrNotExist. +// +//nolint:gocyclo +func NewConfig(cmdline *procfs.Cmdline, downloadURL string) (*Config, error) { + var cfg Config + + clientID := cmdline.Get(constants.KernelParamConfigOAuthClientID).First() + + if clientID == nil { + return nil, os.ErrNotExist + } + + cfg.ClientID = *clientID + + if clientSecret := cmdline.Get(constants.KernelParamConfigOAuthClientSecret).First(); clientSecret != nil { + cfg.ClientSecret = *clientSecret + } + + if audience := cmdline.Get(constants.KernelParamConfigOAuthAudience).First(); audience != nil { + cfg.Audience = *audience + } + + for i := 0; ; i++ { + scope := cmdline.Get(constants.KernelParamConfigOAuthScope).Get(i) + + if scope == nil { + break + } + + cfg.Scopes = append(cfg.Scopes, *scope) + } + + for i := 0; ; i++ { + extra := cmdline.Get(constants.KernelParamConfigOAuthExtraVariable).Get(i) + + if extra == nil { + break + } + + cfg.ExtraVariables = append(cfg.ExtraVariables, *extra) + } + + if deviceAuthURL := cmdline.Get(constants.KernelParamConfigOAuthDeviceAuthURL).First(); deviceAuthURL != nil { + cfg.DeviceAuthURL = *deviceAuthURL + } else { + u, err := url.Parse(downloadURL) + if err != nil { + return nil, err + } + + u.Path = "/device/code" + + cfg.DeviceAuthURL = u.String() + } + + if tokenURL := cmdline.Get(constants.KernelParamConfigOAuthTokenURL).First(); tokenURL != nil { + cfg.TokenURL = *tokenURL + } else { + u, err := url.Parse(downloadURL) + if err != nil { + return nil, err + } + + u.Path = "/token" + + cfg.TokenURL = u.String() + } + + return &cfg, nil +} + +// DeviceAuthFlow represents the device auth flow response. +func (c *Config) DeviceAuthFlow(ctx context.Context, st state.State) error { + transport := httpdefaults.PatchTransport(cleanhttp.DefaultTransport()) + + client := &http.Client{ + Transport: transport, + } + + // register the HTTP client with OAuth2 flow + ctx = context.WithValue(ctx, oauth2.HTTPClient, client) + + cfg := oauth2.Config{ + ClientID: c.ClientID, + Scopes: c.Scopes, + Endpoint: oauth2.Endpoint{ + DeviceAuthURL: c.DeviceAuthURL, + TokenURL: c.TokenURL, + }, + } + + log.Printf("[OAuth] starting the authentication device flow with the following settings:") + log.Printf("[OAuth] - client ID: %q", c.ClientID) + log.Printf("[OAuth] - device auth URL: %q", c.DeviceAuthURL) + log.Printf("[OAuth] - token URL: %q", c.TokenURL) + log.Printf("[OAuth] - extra variables: %q", c.ExtraVariables) + + // acquire device variables + variables, err := c.getVariableValues(ctx, st) + if err != nil { + return fmt.Errorf("failed to get variable values: %w", err) + } + + var deviceAuthOptions []oauth2.AuthCodeOption //nolint:prealloc + + if c.Audience != "" { + deviceAuthOptions = append(deviceAuthOptions, oauth2.SetAuthURLParam("audience", c.Audience)) + } + + for k, v := range variables { + deviceAuthOptions = append(deviceAuthOptions, oauth2.SetAuthURLParam(k, v)) + } + + deviceAuthResponse, err := cfg.DeviceAuth(ctx, deviceAuthOptions...) + if err != nil { + return fmt.Errorf("failed to get device auth response: %w", err) + } + + log.Printf("[OAuth] please visit the URL %s and enter the code %s", deviceAuthResponse.VerificationURI, deviceAuthResponse.UserCode) + + if deviceAuthResponse.VerificationURIComplete != "" { + var qrBuf bytes.Buffer + + qrterminal.GenerateHalfBlock(deviceAuthResponse.VerificationURIComplete, qrterminal.L, &qrBuf) + + log.Printf("[OAuth] or scan the following QR code:\n%s", qrBuf.String()) + } + + log.Printf("[OAuth] waiting for the device to be authorized (expires at %s)...", deviceAuthResponse.Expiry.Format("15:04:05")) + + if c.ClientSecret != "" { + deviceAuthOptions = append(deviceAuthOptions, oauth2.SetAuthURLParam("client_secret", c.ClientSecret)) + } + + token, err := cfg.DeviceAccessToken(ctx, deviceAuthResponse, deviceAuthOptions...) + if err != nil { + return fmt.Errorf("failed to get device access token: %w", err) + } + + log.Printf("[OAuth] device authorized successfully") + + c.extraHeaders = map[string]string{ + "Authorization": token.Type() + " " + token.AccessToken, + } + + return nil +} + +// getVariableValues returns the variable values to include in the device auth request. +func (c *Config) getVariableValues(ctx context.Context, st state.State) (map[string]string, error) { + ctx, cancel := context.WithTimeout(ctx, constants.ConfigLoadAttemptTimeout/2) + defer cancel() + + return metalurl.MapValues(ctx, st, c.ExtraVariables) +} + +// ExtraHeaders returns the extra headers to include in the download request. +func (c *Config) ExtraHeaders() map[string]string { + return c.extraHeaders +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/oauth2/oauth2_test.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/oauth2/oauth2_test.go new file mode 100644 index 0000000..65c0285 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/oauth2/oauth2_test.go @@ -0,0 +1,117 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package oauth2_test + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" + + "github.com/siderolabs/go-procfs/procfs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/oauth2" +) + +func TestNewConfig(t *testing.T) { //nolint:tparallel + t.Parallel() + + for _, test := range []struct { + name string + + cmdline string + expected *oauth2.Config + }{ + { + name: "no config", + }, + { + name: "only client ID", + cmdline: `talos.config.oauth.client_id=device_client_id`, + expected: &oauth2.Config{ + ClientID: "device_client_id", + TokenURL: "https://example.com/token", + DeviceAuthURL: "https://example.com/device/code", + }, + }, + { + name: "client ID and custom URLs", + cmdline: `talos.config.oauth.client_id=device_client_id talos.config.oauth.token_url=https://google.com/token talos.config.oauth.device_auth_url=https://google.com/device/code`, + expected: &oauth2.Config{ + ClientID: "device_client_id", + TokenURL: "https://google.com/token", + DeviceAuthURL: "https://google.com/device/code", + }, + }, + { + name: "complete config", + cmdline: `talos.config.oauth.client_id=device_client_id talos.config.oauth.client_secret=device_secret ` + + `talos.config.oauth.token_url=https://google.com/token talos.config.oauth.device_auth_url=https://google.com/device/code ` + + `talos.config.oauth.scope=foo talos.config.oauth.scope=bar talos.config.oauth.audience=world ` + + `talos.config.oauth.extra_variable=uuid talos.config.oauth.extra_variable=mac`, + expected: &oauth2.Config{ + ClientID: "device_client_id", + ClientSecret: "device_secret", + Audience: "world", + Scopes: []string{"foo", "bar"}, + ExtraVariables: []string{"uuid", "mac"}, + TokenURL: "https://google.com/token", + DeviceAuthURL: "https://google.com/device/code", + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + cfg, err := oauth2.NewConfig(procfs.NewCmdline(test.cmdline), "https://example.com/my/config") + if test.expected == nil { + require.Error(t, err) + assert.True(t, os.IsNotExist(err)) + + return + } + + require.NoError(t, err) + assert.Equal(t, test.expected, cfg) + }) + } +} + +func TestDeviceAuthFlow(t *testing.T) { + t.Parallel() + + cfg := &oauth2.Config{ + ClientID: "device_client_id", + } + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() //nolint:errcheck + + t.Logf("received request: %s %s", r.Method, r.RequestURI) + + switch r.Method + r.RequestURI { + case "POST/device/code": + w.Header().Add("Content-Type", "application/json") + w.Write([]byte(`{"device_code":"abcd", "user_code":"1234", "verification_uri":"https://example.com/verify","verification_uri_complete":"https://example.com/verify/1234","interval":1,"expires_in":36000}`)) //nolint:errcheck,lll + case "POST/token": + w.Header().Add("Content-Type", "application/json") + w.Write([]byte(`{"access_token":"abcd","token_type":"bearer","expires_in":3600,"refresh_token":"efgh","id_token":"ijkl"}`)) //nolint:errcheck + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(ts.Close) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + t.Cleanup(cancel) + + cfg.DeviceAuthURL = ts.URL + "/device/code" + cfg.TokenURL = ts.URL + "/token" + + require.NoError(t, cfg.DeviceAuthFlow(ctx, nil)) + assert.Equal(t, map[string]string{"Authorization": "Bearer abcd"}, cfg.ExtraHeaders()) +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url/map.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url/map.go new file mode 100644 index 0000000..a1ea092 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url/map.go @@ -0,0 +1,85 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package url + +import ( + "context" + "fmt" + "log" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/maps" + "github.com/siderolabs/gen/xslices" +) + +// MapValues maps variable names to values. +// +//nolint:gocyclo +func MapValues(ctx context.Context, st state.State, variableNames []string) (map[string]string, error) { + // happy case + if len(variableNames) == 0 { + return nil, nil + } + + availableVariables := AllVariables() + activeVariables := make(map[string]*Variable, len(variableNames)) + + for _, variableName := range variableNames { + if v, ok := availableVariables[variableName]; ok { + activeVariables[variableName] = v + } else { + return nil, fmt.Errorf("unsupported variable name: %q", variableName) + } + } + + // setup watches + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + watchCh := make(chan state.Event) + + for _, variable := range activeVariables { + if err := variable.Value.RegisterWatch(ctx, st, watchCh); err != nil { + return nil, fmt.Errorf("error watching variable %q: %w", variable.Key, err) + } + } + + pendingVariables := xslices.ToSet(maps.Values(activeVariables)) + + // wait for all variables to be populated +waitLoop: + for len(pendingVariables) > 0 { + log.Printf("waiting for variables: %v", xslices.Map(maps.Keys(pendingVariables), func(v *Variable) string { return v.Key })) + + var ev state.Event + + select { + case <-ctx.Done(): + // context was canceled, return what we have + break waitLoop + case ev = <-watchCh: + } + + switch ev.Type { + case state.Errored: + return nil, fmt.Errorf("error watching variables: %w", ev.Error) + case state.Bootstrapped: + // ignored + case state.Created, state.Updated, state.Destroyed: + for _, variable := range activeVariables { + handled, err := variable.Value.EventHandler(ev) + if err != nil { + return nil, fmt.Errorf("error handling variable %q: %w", variable.Key, err) + } + + if handled { + delete(pendingVariables, variable) + } + } + } + } + + return maps.Map(activeVariables, func(k string, v *Variable) (string, string) { return k, v.Value.Get() }), nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url/map_test.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url/map_test.go new file mode 100644 index 0000000..903e3f4 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url/map_test.go @@ -0,0 +1,106 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package url_test + +import ( + "context" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url" +) + +func TestMapValues(t *testing.T) { + t.Parallel() + + for _, test := range []struct { + name string + variableNames []string + + preSetup []setupFunc + parallelSetup []setupFunc + + expected map[string]string + }{ + { + name: "no variables", + }, + { + name: "multiple variables", + variableNames: []string{"uuid", "mac", "hostname", "code"}, + expected: map[string]string{ + "code": "top-secret", + "hostname": "some-node", + "mac": "12:34:56:78:90:ce", + "uuid": "0000-0000", + }, + preSetup: []setupFunc{ + createSysInfo("0000-0000", "12345"), + createMac("12:34:56:78:90:ce"), + createHostname("some-node"), + createCode("top-secret"), + }, + }, + { + name: "mixed wait variables", + variableNames: []string{"uuid", "mac", "hostname", "code"}, + expected: map[string]string{ + "code": "", + "hostname": "another-node", + "mac": "12:34:56:78:90:ab", + "uuid": "0000-1234", + }, + preSetup: []setupFunc{ + createSysInfo("0000-1234", "12345"), + createMac("12:34:56:78:90:ab"), + createHostname("example-node"), + }, + parallelSetup: []setupFunc{ + sleep(time.Second), + updateHostname("another-node"), + sleep(time.Second / 2), + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + st := state.WrapCore(namespaced.NewState(inmem.Build)) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + for _, f := range test.preSetup { + f(ctx, t, st) + } + + errCh := make(chan error) + + var result map[string]string + + go func() { + var e error + + result, e = url.MapValues(ctx, st, test.variableNames) + errCh <- e + }() + + for _, f := range test.parallelSetup { + f(ctx, t, st) + } + + err := <-errCh + require.NoError(t, err) + + assert.Equal(t, test.expected, result) + }) + } +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url/url.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url/url.go new file mode 100644 index 0000000..05d199c --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url/url.go @@ -0,0 +1,118 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package url handles expansion of the download URL for the config. +package url + +import ( + "context" + "fmt" + "log" + "net/url" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/maps" + "github.com/siderolabs/gen/xslices" +) + +// Populate populates the config download URL with values replacing variables. +func Populate(ctx context.Context, downloadURL string, st state.State) (string, error) { + return PopulateVariables(ctx, downloadURL, st, maps.Values(AllVariables())) +} + +// PopulateVariables populates the config download URL with values replacing variables. +// +//nolint:gocyclo +func PopulateVariables(ctx context.Context, downloadURL string, st state.State, variables []*Variable) (string, error) { + u, err := url.Parse(downloadURL) + if err != nil { + return "", fmt.Errorf("failed to parse URL: %w", err) + } + + query := u.Query() + + var activeVariables []*Variable + + for _, variable := range variables { + if variable.Matches(query) { + activeVariables = append(activeVariables, variable) + } + } + + // happy path: no variables + if len(activeVariables) == 0 { + return downloadURL, nil + } + + // setup watches + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + watchCh := make(chan state.Event) + + for _, variable := range activeVariables { + if err = variable.Value.RegisterWatch(ctx, st, watchCh); err != nil { + return "", fmt.Errorf("error watching variable %q: %w", variable.Key, err) + } + } + + pendingVariables := xslices.ToSet(activeVariables) + + // wait for all variables to be populated + for len(pendingVariables) > 0 { + log.Printf("waiting for URL variables: %v", xslices.Map(maps.Keys(pendingVariables), func(v *Variable) string { return v.Key })) + + var ev state.Event + + select { + case <-ctx.Done(): + // context was canceled, return the URL as is + u.RawQuery = query.Encode() + + return u.String(), ctx.Err() + case ev = <-watchCh: + } + + switch ev.Type { + case state.Errored: + return "", fmt.Errorf("error watching variables: %w", ev.Error) + case state.Bootstrapped: + // ignored + case state.Created, state.Updated, state.Destroyed: + anyHandled := false + + for _, variable := range activeVariables { + handled, err := variable.Value.EventHandler(ev) + if err != nil { + return "", fmt.Errorf("error handling variable %q: %w", variable.Key, err) + } + + if handled { + delete(pendingVariables, variable) + + anyHandled = true + } + } + + if !anyHandled { + continue + } + + // perform another round of replacing + query = u.Query() + + for _, variable := range activeVariables { + if _, pending := pendingVariables[variable]; pending { + continue + } + + variable.Replace(query) + } + } + } + + u.RawQuery = query.Encode() + + return u.String(), nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url/url_test.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url/url_test.go new file mode 100644 index 0000000..01e46dc --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url/url_test.go @@ -0,0 +1,176 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package url_test + +import ( + "context" + "net" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url" + "github.com/aenix-io/talm/internal/pkg/meta" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/hardware" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +type setupFunc func(context.Context, *testing.T, state.State) + +func TestPopulate(t *testing.T) { + t.Parallel() + + for _, test := range []struct { + name string + url string + + preSetup []setupFunc + parallelSetup []setupFunc + + expected string + }{ + { + name: "no variables", + url: "https://example.com?foo=bar", + expected: "https://example.com?foo=bar", + }, + { + name: "legacy UUID", + url: "https://example.com?uuid=", + expected: "https://example.com?uuid=0000-0000", + preSetup: []setupFunc{ + createSysInfo("0000-0000", ""), + }, + }, + { + name: "sys info", + url: "https://example.com?uuid=${uuid}&no=${serial}", + expected: "https://example.com?no=12345&uuid=0000-0000", + preSetup: []setupFunc{ + createSysInfo("0000-0000", "12345"), + }, + }, + { + name: "multiple variables", + url: "https://example.com?uuid=${uuid}&mac=${mac}&hostname=${hostname}&code=${code}", + expected: "https://example.com?code=top-secret&hostname=example-node&mac=12%3A34%3A56%3A78%3A90%3Aab&uuid=0000-0000", + preSetup: []setupFunc{ + createSysInfo("0000-0000", "12345"), + createMac("12:34:56:78:90:ab"), + createHostname("example-node"), + createCode("top-secret"), + }, + }, + { + name: "mixed wait variables", + url: "https://example.com?uuid=${uuid}&mac=${mac}&hostname=${hostname}&code=${code}", + expected: "https://example.com?code=top-secret&hostname=another-node&mac=12%3A34%3A56%3A78%3A90%3Aab&uuid=0000-1234", + preSetup: []setupFunc{ + createSysInfo("0000-1234", "12345"), + createMac("12:34:56:78:90:ab"), + createHostname("example-node"), + }, + parallelSetup: []setupFunc{ + sleep(time.Second), + updateHostname("another-node"), + sleep(time.Second), + createCode("top-secret"), + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + st := state.WrapCore(namespaced.NewState(inmem.Build)) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + for _, f := range test.preSetup { + f(ctx, t, st) + } + + errCh := make(chan error) + + var result string + + go func() { + var e error + + result, e = url.Populate(ctx, test.url, st) + errCh <- e + }() + + for _, f := range test.parallelSetup { + f(ctx, t, st) + } + + err := <-errCh + require.NoError(t, err) + + assert.Equal(t, test.expected, result) + }) + } +} + +func createSysInfo(uuid, serial string) setupFunc { + return func(ctx context.Context, t *testing.T, st state.State) { + sysInfo := hardware.NewSystemInformation(hardware.SystemInformationID) + sysInfo.TypedSpec().UUID = uuid + sysInfo.TypedSpec().SerialNumber = serial + require.NoError(t, st.Create(ctx, sysInfo)) + } +} + +func createMac(mac string) setupFunc { + return func(ctx context.Context, t *testing.T, st state.State) { + addr, err := net.ParseMAC(mac) + require.NoError(t, err) + + hwAddr := network.NewHardwareAddr(network.NamespaceName, network.FirstHardwareAddr) + hwAddr.TypedSpec().HardwareAddr = nethelpers.HardwareAddr(addr) + require.NoError(t, st.Create(ctx, hwAddr)) + } +} + +func createHostname(hostname string) setupFunc { + return func(ctx context.Context, t *testing.T, st state.State) { + hn := network.NewHostnameStatus(network.NamespaceName, network.HostnameID) + hn.TypedSpec().Hostname = hostname + require.NoError(t, st.Create(ctx, hn)) + } +} + +func updateHostname(hostname string) setupFunc { + return func(ctx context.Context, t *testing.T, st state.State) { + hn, err := safe.StateGet[*network.HostnameStatus](ctx, st, network.NewHostnameStatus(network.NamespaceName, network.HostnameID).Metadata()) + require.NoError(t, err) + + hn.TypedSpec().Hostname = hostname + require.NoError(t, st.Update(ctx, hn)) + } +} + +func createCode(code string) setupFunc { + return func(ctx context.Context, t *testing.T, st state.State) { + mk := runtime.NewMetaKey(runtime.NamespaceName, runtime.MetaKeyTagToID(meta.DownloadURLCode)) + mk.TypedSpec().Value = code + require.NoError(t, st.Create(ctx, mk)) + } +} + +func sleep(d time.Duration) setupFunc { + return func(ctx context.Context, t *testing.T, st state.State) { + time.Sleep(d) + } +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url/value.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url/value.go new file mode 100644 index 0000000..fef9e35 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url/value.go @@ -0,0 +1,148 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package url + +import ( + "context" + "sync" + + "github.com/cosi-project/runtime/pkg/state" + + "github.com/aenix-io/talm/internal/pkg/meta" + "github.com/siderolabs/talos/pkg/machinery/resources/hardware" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// Value of a variable. +type Value interface { + // Get the value. + Get() string + // RegisterWatch handles registering a watch for the variable. + RegisterWatch(ctx context.Context, st state.State, ch chan<- state.Event) error + // EventHandler is called for each watch event, returns when the variable value is ready. + EventHandler(event state.Event) (bool, error) +} + +type value struct { + mu sync.Mutex + val string + + registerWatch func(ctx context.Context, st state.State, ch chan<- state.Event) error + eventHandler func(event state.Event) (string, error) +} + +func (v *value) Get() string { + v.mu.Lock() + defer v.mu.Unlock() + + return v.val +} + +func (v *value) RegisterWatch(ctx context.Context, st state.State, ch chan<- state.Event) error { + return v.registerWatch(ctx, st, ch) +} + +func (v *value) EventHandler(event state.Event) (bool, error) { + val, err := v.eventHandler(event) + if err != nil { + return false, err + } + + if val == "" { + return false, nil + } + + v.mu.Lock() + v.val = val + v.mu.Unlock() + + return true, nil +} + +// UUIDValue is a value for UUID variable. +func UUIDValue() Value { + return &value{ + registerWatch: func(ctx context.Context, st state.State, ch chan<- state.Event) error { + return st.Watch(ctx, hardware.NewSystemInformation(hardware.SystemInformationID).Metadata(), ch) + }, + eventHandler: func(event state.Event) (string, error) { + sysInfo, ok := event.Resource.(*hardware.SystemInformation) + if !ok { + return "", nil + } + + return sysInfo.TypedSpec().UUID, nil + }, + } +} + +// SerialNumberValue is a value for SerialNumber variable. +func SerialNumberValue() Value { + return &value{ + registerWatch: func(ctx context.Context, st state.State, ch chan<- state.Event) error { + return st.Watch(ctx, hardware.NewSystemInformation(hardware.SystemInformationID).Metadata(), ch) + }, + eventHandler: func(event state.Event) (string, error) { + sysInfo, ok := event.Resource.(*hardware.SystemInformation) + if !ok { + return "", nil + } + + return sysInfo.TypedSpec().SerialNumber, nil + }, + } +} + +// MACValue is a value for MAC variable. +func MACValue() Value { + return &value{ + registerWatch: func(ctx context.Context, st state.State, ch chan<- state.Event) error { + return st.Watch(ctx, network.NewHardwareAddr(network.NamespaceName, network.FirstHardwareAddr).Metadata(), ch) + }, + eventHandler: func(event state.Event) (string, error) { + hwAddr, ok := event.Resource.(*network.HardwareAddr) + if !ok { + return "", nil + } + + return hwAddr.TypedSpec().HardwareAddr.String(), nil + }, + } +} + +// HostnameValue is a value for Hostname variable. +func HostnameValue() Value { + return &value{ + registerWatch: func(ctx context.Context, st state.State, ch chan<- state.Event) error { + return st.Watch(ctx, network.NewHostnameStatus(network.NamespaceName, network.HostnameID).Metadata(), ch) + }, + eventHandler: func(event state.Event) (string, error) { + hostname, ok := event.Resource.(*network.HostnameStatus) + if !ok { + return "", nil + } + + return hostname.TypedSpec().Hostname, nil + }, + } +} + +// CodeValue is a value for Code variable. +func CodeValue() Value { + return &value{ + registerWatch: func(ctx context.Context, st state.State, ch chan<- state.Event) error { + return st.Watch(ctx, runtime.NewMetaKey(runtime.NamespaceName, runtime.MetaKeyTagToID(meta.DownloadURLCode)).Metadata(), ch) + }, + eventHandler: func(event state.Event) (string, error) { + code, ok := event.Resource.(*runtime.MetaKey) + if !ok { + return "", nil + } + + return code.TypedSpec().Value, nil + }, + } +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url/variable.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url/variable.go new file mode 100644 index 0000000..9e29309 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url/variable.go @@ -0,0 +1,108 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package url + +import ( + "net/url" + "regexp" + "strings" + "sync" + + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// Variable represents a variable substitution in the download URL. +type Variable struct { + // Key is the variable name. + Key string + // MatchOnArg is set for variables which are match on the arg name with empty value. + // + // Required to support legacy `?uuid=` style of the download URL. + MatchOnArg bool + // Value is the variable value. + Value Value + + rOnce sync.Once + r *regexp.Regexp +} + +// AllVariables is a map of all supported variables. +func AllVariables() map[string]*Variable { + return map[string]*Variable{ + constants.UUIDKey: { + Key: constants.UUIDKey, + MatchOnArg: true, + Value: UUIDValue(), + }, + constants.SerialNumberKey: { + Key: constants.SerialNumberKey, + Value: SerialNumberValue(), + }, + constants.MacKey: { + Key: constants.MacKey, + Value: MACValue(), + }, + constants.HostnameKey: { + Key: constants.HostnameKey, + Value: HostnameValue(), + }, + constants.CodeKey: { + Key: constants.CodeKey, + Value: CodeValue(), + }, + } +} + +func keyToVar(key string) string { + return `${` + key + `}` +} + +func (v *Variable) init() { + v.rOnce.Do(func() { + v.r = regexp.MustCompile(`(?i)` + regexp.QuoteMeta(keyToVar(v.Key))) + }) +} + +// Matches checks if the variable is present in the URL. +func (v *Variable) Matches(query url.Values) bool { + v.init() + + for arg, values := range query { + if v.MatchOnArg { + if arg == v.Key && !(len(values) == 1 && strings.TrimSpace(values[0]) != "") { + return true + } + } + + for _, value := range values { + if v.r.MatchString(value) { + return true + } + } + } + + return false +} + +// Replace modifies the URL query replacing the variable with the value. +func (v *Variable) Replace(query url.Values) { + v.init() + + for arg, values := range query { + if v.MatchOnArg { + if arg == v.Key && !(len(values) == 1 && strings.TrimSpace(values[0]) != "") { + query.Set(arg, v.Value.Get()) + + continue + } + } + + for idx, value := range values { + values[idx] = v.r.ReplaceAllString(value, v.Value.Get()) + } + + query[arg] = values + } +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url/variable_test.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url/variable_test.go new file mode 100644 index 0000000..44dba6e --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url/variable_test.go @@ -0,0 +1,156 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package url_test + +import ( + "context" + neturl "net/url" + "testing" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +func TestVariableMatches(t *testing.T) { + t.Parallel() + + for _, test := range []struct { + name string + url string + shouldMatch map[string]struct{} + }{ + { + name: "no matches", + url: "https://example.com?foo=bar", + }, + { + name: "legacy UUID", + url: "https://example.com?uuid=&foo=bar", + shouldMatch: map[string]struct{}{ + constants.UUIDKey: {}, + }, + }, + { + name: "UUID static", + url: "https://example.com?uuid=0000-0000&foo=bar", + }, + { + name: "more variables", + url: "https://example.com?uuid=${uuid}&foo=bar&serial=${serial}&mac=${mac}&hostname=fixed&hostname=${hostname}", + shouldMatch: map[string]struct{}{ + constants.UUIDKey: {}, + constants.SerialNumberKey: {}, + constants.MacKey: {}, + constants.HostnameKey: {}, + }, + }, + { + name: "case insensitive", + url: "https://example.com?uuid=${UUId}&foo=bar&serial=${SeRiaL}", + shouldMatch: map[string]struct{}{ + constants.UUIDKey: {}, + constants.SerialNumberKey: {}, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + u, err := neturl.Parse(test.url) + require.NoError(t, err) + + for _, variable := range url.AllVariables() { + if _, ok := test.shouldMatch[variable.Key]; ok { + assert.True(t, variable.Matches(u.Query())) + } else { + assert.False(t, variable.Matches(u.Query())) + } + } + }) + } +} + +type mockValue struct { + value string +} + +func (v mockValue) Get() string { + return v.value +} + +func (v mockValue) RegisterWatch(context.Context, state.State, chan<- state.Event) error { + return nil +} + +func (v mockValue) EventHandler(state.Event) (bool, error) { + return true, nil +} + +func TestVariableReplace(t *testing.T) { + t.Parallel() + + var1 := &url.Variable{ + Key: "var1", + MatchOnArg: true, + Value: mockValue{ + value: "value1", + }, + } + + var2 := &url.Variable{ + Key: "var2", + Value: mockValue{ + value: "value2", + }, + } + + for _, test := range []struct { + name string + url string + expected string + }{ + { + name: "no matches", + url: "https://example.com?foo=bar", + expected: "https://example.com?foo=bar", + }, + { + name: "legacy match", + url: "https://example.com?var1=&foo=bar", + expected: "https://example.com?foo=bar&var1=value1", + }, + { + name: "variable match", + url: "https://example.com?a=${var1}-suffix&foo=bar&b=${var2}&b=xyz&b=${var2}", + expected: "https://example.com?a=value1-suffix&b=value2&b=xyz&b=value2&foo=bar", + }, + { + name: "case insensitive", + url: "https://example.com?a=${VAR1}", + expected: "https://example.com?a=value1", + }, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + u, err := neturl.Parse(test.url) + require.NoError(t, err) + + query := u.Query() + + for _, variable := range []*url.Variable{var1, var2} { + variable.Replace(query) + } + + u.RawQuery = query.Encode() + + assert.Equal(t, test.expected, u.String()) + }) + } +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url_test.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url_test.go new file mode 100644 index 0000000..7414c83 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/metal/url_test.go @@ -0,0 +1,129 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package metal_test + +import ( + "context" + "net" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/siderolabs/go-procfs/procfs" + "github.com/stretchr/testify/assert" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/metal" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/hardware" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +func createOrUpdate(ctx context.Context, st state.State, r resource.Resource) error { + oldRes, err := st.Get(ctx, r.Metadata()) + if err != nil && !state.IsNotFoundError(err) { + return err + } + + if oldRes == nil { + err = st.Create(ctx, r) + if err != nil { + return err + } + } else { + r.Metadata().SetVersion(oldRes.Metadata().Version()) + + err = st.Update(ctx, r) + if err != nil { + return err + } + } + + return nil +} + +func setup(ctx context.Context, t *testing.T, st state.State, mockUUID, mockSerialNumber, mockHostname, mockMAC string) { + sysInfo := hardware.NewSystemInformation(hardware.SystemInformationID) + sysInfo.TypedSpec().UUID = mockUUID + sysInfo.TypedSpec().SerialNumber = mockSerialNumber + assert.NoError(t, createOrUpdate(ctx, st, sysInfo)) + + hostnameSpec := network.NewHostnameStatus(network.NamespaceName, network.HostnameID) + hostnameSpec.TypedSpec().Hostname = mockHostname + assert.NoError(t, createOrUpdate(ctx, st, hostnameSpec)) + + linkStatusSpec := network.NewHardwareAddr(network.NamespaceName, network.FirstHardwareAddr) + parsedMockMAC, err := net.ParseMAC(mockMAC) + assert.NoError(t, err) + + linkStatusSpec.TypedSpec().HardwareAddr = nethelpers.HardwareAddr(parsedMockMAC) + assert.NoError(t, createOrUpdate(ctx, st, linkStatusSpec)) + + netStatus := network.NewStatus(network.NamespaceName, network.StatusID) + netStatus.TypedSpec().AddressReady = true + assert.NoError(t, createOrUpdate(ctx, st, netStatus)) +} + +func TestRepopulateOnRetry(t *testing.T) { + st := state.WrapCore(namespaced.NewState(inmem.Build)) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + nCalls := 0 + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch nCalls { + case 0: + assert.Equal(t, "h=myTestHostname&m=52%3A2f%3Afd%3Adf%3Afc%3Ac0&s=0OCZJ19N65&u=40dcbd19-3b10-444e-bfff-aaee44a51fda", r.URL.RawQuery) + w.WriteHeader(http.StatusNotFound) + + // After the first call we change the resources that should be substituted in the next call. + uuid2 := "9fba530f-767d-40f9-9410-bb1fed5d2134" + mac2 := "aa:aa:bb:bb:cc:cc" + serialNumber2 := "111AAA9N65" + hostname2 := "anotherHostname" + + setup(ctx, t, st, uuid2, serialNumber2, hostname2, mac2) + case 1: + // Before the second call Configuration() should have resubstituted all the new parameters in the URL. + assert.Equal(t, "h=anotherHostname&m=aa%3Aaa%3Abb%3Abb%3Acc%3Acc&s=111AAA9N65&u=9fba530f-767d-40f9-9410-bb1fed5d2134", r.URL.RawQuery) + w.WriteHeader(http.StatusOK) + } + + nCalls++ + })) + defer server.Close() + + uuid1 := "40dcbd19-3b10-444e-bfff-aaee44a51fda" + mac1 := "52:2f:fd:df:fc:c0" + serialNumber1 := "0OCZJ19N65" + hostname1 := "myTestHostname" + + setup(ctx, t, st, uuid1, serialNumber1, hostname1, mac1) + + downloadURL := server.URL + "/metadata?h=${hostname}&m=${mac}&s=${serial}&u=${uuid}" + + param := procfs.NewParameter(constants.KernelParamConfig) + param.Append(downloadURL) + + procfs.ProcCmdline().Set(constants.KernelParamConfig, param) + defer procfs.ProcCmdline().Set(constants.KernelParamConfig, nil) + + go func() { + testObj := metal.Metal{} + _, err := testObj.Configuration(ctx, st) + assert.NoError(t, err) + + cancel() + }() + + <-ctx.Done() +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/metadata.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/metadata.go new file mode 100644 index 0000000..b45793f --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/metadata.go @@ -0,0 +1,677 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package nocloud provides the NoCloud platform implementation. +package nocloud + +import ( + "context" + "fmt" + "log" + "net" + "net/netip" + "net/url" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/maps" + "github.com/siderolabs/go-blockdevice/blockdevice/filesystem" + "github.com/siderolabs/go-blockdevice/blockdevice/probe" + "golang.org/x/sys/unix" + yaml "gopkg.in/yaml.v3" + + networkadapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/network" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/errors" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/internal/netutils" + "github.com/aenix-io/talm/internal/pkg/smbios" + "github.com/siderolabs/talos/pkg/download" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +const ( + configISOLabel = "cidata" + configNetworkConfigPath = "network-config" + configMetaDataPath = "meta-data" + configUserDataPath = "user-data" + mnt = "/mnt" +) + +// NetworkConfig holds network-config info. +type NetworkConfig struct { + Version int `yaml:"version"` + Config []struct { + Mac string `yaml:"mac_address,omitempty"` + Interfaces string `yaml:"name,omitempty"` + MTU uint32 `yaml:"mtu,omitempty"` + Subnets []struct { + Address string `yaml:"address,omitempty"` + Netmask string `yaml:"netmask,omitempty"` + Gateway string `yaml:"gateway,omitempty"` + Type string `yaml:"type"` + } `yaml:"subnets,omitempty"` + Address []string `yaml:"address,omitempty"` + Type string `yaml:"type"` + } `yaml:"config,omitempty"` + Ethernets map[string]Ethernet `yaml:"ethernets,omitempty"` + Bonds map[string]Bonds `yaml:"bonds,omitempty"` +} + +// Ethernet holds network interface info. +type Ethernet struct { + Match struct { + Name string `yaml:"name,omitempty"` + HWAddr string `yaml:"macaddress,omitempty"` + } `yaml:"match,omitempty"` + DHCPv4 bool `yaml:"dhcp4,omitempty"` + DHCPv6 bool `yaml:"dhcp6,omitempty"` + Address []string `yaml:"addresses,omitempty"` + Gateway4 string `yaml:"gateway4,omitempty"` + Gateway6 string `yaml:"gateway6,omitempty"` + MTU uint32 `yaml:"mtu,omitempty"` + NameServers struct { + Search []string `yaml:"search,omitempty"` + Address []string `yaml:"addresses,omitempty"` + } `yaml:"nameservers,omitempty"` + Routes []struct { + To string `yaml:"to,omitempty"` + Via string `yaml:"via,omitempty"` + Metric string `yaml:"metric,omitempty"` + Table uint32 `yaml:"table,omitempty"` + } `yaml:"routes,omitempty"` + RoutingPolicy []struct { // TODO + From string `yaml:"froom,omitempty"` + Table uint32 `yaml:"table,omitempty"` + } `yaml:"routing-policy,omitempty"` +} + +// Bonds holds bonding interface info. +type Bonds struct { + Ethernet `yaml:",inline"` + Interfaces []string `yaml:"interfaces,omitempty"` + Params struct { + Mode string `yaml:"mode,omitempty"` + LACPRate string `yaml:"lacp-rate,omitempty"` + HashPolicy string `yaml:"transmit-hash-policy,omitempty"` + MIIMon uint32 `yaml:"mii-monitor-interval,omitempty"` + UpDelay uint32 `yaml:"up-delay,omitempty"` + DownDelay uint32 `yaml:"down-delay,omitempty"` + } `yaml:"parameters,omitempty"` +} + +// MetadataConfig holds meta info. +type MetadataConfig struct { + Hostname string `yaml:"hostname,omitempty"` + LocalHostname string `yaml:"local-hostname,omitempty"` + InstanceID string `yaml:"instance-id,omitempty"` + InstanceType string `yaml:"instance-type,omitempty"` + ProviderID string `yaml:"provider-id,omitempty"` + Region string `yaml:"region,omitempty"` + Zone string `yaml:"zone,omitempty"` +} + +func (n *Nocloud) configFromNetwork(ctx context.Context, metaBaseURL string, r state.State) (metaConfig []byte, networkConfig []byte, machineConfig []byte, err error) { + log.Printf("fetching meta config from: %q", metaBaseURL+configMetaDataPath) + + if err = netutils.Wait(ctx, r); err != nil { + return nil, nil, nil, err + } + + metaConfig, err = download.Download(ctx, metaBaseURL+configMetaDataPath) + if err != nil { + metaConfig = nil + } + + log.Printf("fetching network config from: %q", metaBaseURL+configNetworkConfigPath) + + networkConfig, err = download.Download(ctx, metaBaseURL+configNetworkConfigPath) + if err != nil { + networkConfig = nil + } + + log.Printf("fetching machine config from: %q", metaBaseURL+configUserDataPath) + + machineConfig, err = download.Download(ctx, metaBaseURL+configUserDataPath, + download.WithErrorOnNotFound(errors.ErrNoConfigSource), + download.WithErrorOnEmptyResponse(errors.ErrNoConfigSource)) + + return metaConfig, networkConfig, machineConfig, err +} + +//nolint:gocyclo +func (n *Nocloud) configFromCD(ctx context.Context, r state.State) (metaConfig []byte, networkConfig []byte, machineConfig []byte, err error) { + if err := netutils.WaitForDevicesReady(ctx, r); err != nil { + return nil, nil, nil, fmt.Errorf("failed to wait for devices: %w", err) + } + + var dev *probe.ProbedBlockDevice + + dev, err = probe.GetDevWithFileSystemLabel(strings.ToLower(configISOLabel)) + if err != nil { + dev, err = probe.GetDevWithFileSystemLabel(strings.ToUpper(configISOLabel)) + if err != nil { + return nil, nil, nil, errors.ErrNoConfigSource + } + } + + //nolint:errcheck + defer dev.Close() + + sb, err := filesystem.Probe(dev.Path) + if err != nil || sb == nil { + return nil, nil, nil, errors.ErrNoConfigSource + } + + log.Printf("found config disk (cidata) at %s", dev.Path) + + if err = unix.Mount(dev.Path, mnt, sb.Type(), unix.MS_RDONLY, ""); err != nil { + return nil, nil, nil, errors.ErrNoConfigSource + } + + log.Printf("fetching meta config from: cidata/%s", configMetaDataPath) + + metaConfig, err = os.ReadFile(filepath.Join(mnt, configMetaDataPath)) + if err != nil { + log.Printf("failed to read %s", configMetaDataPath) + + metaConfig = nil + } + + log.Printf("fetching network config from: cidata/%s", configNetworkConfigPath) + + networkConfig, err = os.ReadFile(filepath.Join(mnt, configNetworkConfigPath)) + if err != nil { + log.Printf("failed to read %s", configNetworkConfigPath) + + networkConfig = nil + } + + log.Printf("fetching machine config from: cidata/%s", configUserDataPath) + + machineConfig, err = os.ReadFile(filepath.Join(mnt, configUserDataPath)) + if err != nil { + log.Printf("failed to read %s", configUserDataPath) + + machineConfig = nil + } + + if err = unix.Unmount(mnt, 0); err != nil { + return nil, nil, nil, fmt.Errorf("failed to unmount: %w", err) + } + + return metaConfig, networkConfig, machineConfig, nil +} + +//nolint:gocyclo +func (n *Nocloud) acquireConfig(ctx context.Context, r state.State) (metadataConfigDl, metadataNetworkConfigDl, machineConfigDl []byte, metadata *MetadataConfig, err error) { + s, err := smbios.GetSMBIOSInfo() + if err != nil { + return nil, nil, nil, nil, err + } + + var ( + metaBaseURL, hostname string + networkSource bool + ) + + options := strings.Split(s.SystemInformation.SerialNumber, ";") + for _, option := range options { + parts := strings.SplitN(option, "=", 2) + if len(parts) == 2 { + switch parts[0] { + case "ds": + if parts[1] == "nocloud-net" { + networkSource = true + } + case "s": + var u *url.URL + + u, err = url.Parse(parts[1]) + if err == nil && strings.HasPrefix(u.Scheme, "http") { + if strings.HasSuffix(u.Path, "/") { + metaBaseURL = parts[1] + } else { + metaBaseURL = parts[1] + "/" + } + } + case "h": + hostname = parts[1] + } + } + } + + if networkSource && metaBaseURL != "" { + metadataConfigDl, metadataNetworkConfigDl, machineConfigDl, err = n.configFromNetwork(ctx, metaBaseURL, r) + } else { + metadataConfigDl, metadataNetworkConfigDl, machineConfigDl, err = n.configFromCD(ctx, r) + } + + metadata = &MetadataConfig{} + + if metadataConfigDl != nil { + _ = yaml.Unmarshal(metadataConfigDl, metadata) //nolint:errcheck + } + + if hostname != "" { + metadata.Hostname = hostname + } + + // Some providers may provide the hostname via user-data instead of meta-data (e.g. Proxmox VE) + // As long as the user doesn't use it for machine config, it can still be used to obtain the hostname + if metadata.Hostname == "" && metadata.LocalHostname == "" && machineConfigDl != nil { + fallbackMetadata := &MetadataConfig{} + _ = yaml.Unmarshal(machineConfigDl, fallbackMetadata) //nolint:errcheck + metadata.Hostname = fallbackMetadata.Hostname + metadata.LocalHostname = fallbackMetadata.LocalHostname + } + + return metadataConfigDl, metadataNetworkConfigDl, machineConfigDl, metadata, err +} + +//nolint:gocyclo,cyclop +func (n *Nocloud) applyNetworkConfigV1(config *NetworkConfig, st state.State, networkConfig *runtime.PlatformNetworkConfig) error { + ctx := context.TODO() + + if err := netutils.WaitInterfaces(ctx, st); err != nil { + return err + } + + hostInterfaces, err := safe.StateListAll[*network.LinkStatus](ctx, st) + if err != nil { + return fmt.Errorf("error listing host interfaces: %w", err) + } + + for _, ntwrk := range config.Config { + switch ntwrk.Type { + case "nameserver": + dnsIPs := make([]netip.Addr, 0, len(ntwrk.Address)) + + for i := range ntwrk.Address { + if ip, err := netip.ParseAddr(ntwrk.Address[i]); err == nil { + dnsIPs = append(dnsIPs, ip) + } else { + return err + } + } + + networkConfig.Resolvers = append(networkConfig.Resolvers, network.ResolverSpecSpec{ + DNSServers: dnsIPs, + ConfigLayer: network.ConfigPlatform, + }) + case "physical": + name := ntwrk.Interfaces + + if ntwrk.Mac != "" { + macAddressMatched := false + hostInterfaceIter := hostInterfaces.Iterator() + + for hostInterfaceIter.Next() { + macAddress := hostInterfaceIter.Value().TypedSpec().PermanentAddr.String() + if macAddress == ntwrk.Mac { + name = hostInterfaceIter.Value().Metadata().ID() + macAddressMatched = true + + break + } + } + + if !macAddressMatched { + log.Printf("nocloud: no link with matching MAC address %q, defaulted to use name %s instead", ntwrk.Mac, name) + } + } + + networkConfig.Links = append(networkConfig.Links, network.LinkSpecSpec{ + Name: name, + Up: true, + ConfigLayer: network.ConfigPlatform, + }) + + for _, subnet := range ntwrk.Subnets { + switch subnet.Type { + case "dhcp", "dhcp4": + networkConfig.Operators = append(networkConfig.Operators, network.OperatorSpecSpec{ + Operator: network.OperatorDHCP4, + LinkName: name, + RequireUp: true, + DHCP4: network.DHCP4OperatorSpec{ + RouteMetric: network.DefaultRouteMetric, + }, + ConfigLayer: network.ConfigPlatform, + }) + case "static", "static6": + family := nethelpers.FamilyInet4 + + if subnet.Type == "static6" { + family = nethelpers.FamilyInet6 + } + + ipPrefix, err := netip.ParsePrefix(subnet.Address) + if err != nil { + ip, err := netip.ParseAddr(subnet.Address) + if err != nil { + return err + } + + netmask, err := netip.ParseAddr(subnet.Netmask) + if err != nil { + return err + } + + mask, _ := netmask.MarshalBinary() //nolint:errcheck // never fails + ones, _ := net.IPMask(mask).Size() + ipPrefix = netip.PrefixFrom(ip, ones) + } + + networkConfig.Addresses = append(networkConfig.Addresses, + network.AddressSpecSpec{ + ConfigLayer: network.ConfigPlatform, + LinkName: name, + Address: ipPrefix, + Scope: nethelpers.ScopeGlobal, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + Family: family, + }, + ) + + if subnet.Gateway != "" { + gw, err := netip.ParseAddr(subnet.Gateway) + if err != nil { + return err + } + + route := network.RouteSpecSpec{ + ConfigLayer: network.ConfigPlatform, + Gateway: gw, + OutLinkName: name, + Table: nethelpers.TableMain, + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + Family: family, + Priority: network.DefaultRouteMetric, + } + + if family == nethelpers.FamilyInet6 { + route.Priority = 2 * network.DefaultRouteMetric + } + + route.Normalize() + + networkConfig.Routes = append(networkConfig.Routes, route) + } + case "ipv6_dhcpv6-stateful": + networkConfig.Operators = append(networkConfig.Operators, network.OperatorSpecSpec{ + Operator: network.OperatorDHCP6, + LinkName: name, + RequireUp: true, + DHCP6: network.DHCP6OperatorSpec{ + RouteMetric: 2 * network.DefaultRouteMetric, + }, + ConfigLayer: network.ConfigPlatform, + }) + } + } + } + } + + return nil +} + +//nolint:gocyclo +func applyNetworkConfigV2Ethernet(name string, eth Ethernet, networkConfig *runtime.PlatformNetworkConfig, dnsIPs *[]netip.Addr) error { + if eth.DHCPv4 { + networkConfig.Operators = append(networkConfig.Operators, network.OperatorSpecSpec{ + Operator: network.OperatorDHCP4, + LinkName: name, + RequireUp: true, + DHCP4: network.DHCP4OperatorSpec{ + RouteMetric: network.DefaultRouteMetric, + }, + ConfigLayer: network.ConfigPlatform, + }) + } + + if eth.DHCPv6 { + networkConfig.Operators = append(networkConfig.Operators, network.OperatorSpecSpec{ + Operator: network.OperatorDHCP6, + LinkName: name, + RequireUp: true, + DHCP6: network.DHCP6OperatorSpec{ + RouteMetric: network.DefaultRouteMetric, + }, + ConfigLayer: network.ConfigPlatform, + }) + } + + for _, addr := range eth.Address { + ipPrefix, err := netip.ParsePrefix(addr) + if err != nil { + return err + } + + family := nethelpers.FamilyInet4 + + if ipPrefix.Addr().Is6() { + family = nethelpers.FamilyInet6 + } + + networkConfig.Addresses = append(networkConfig.Addresses, + network.AddressSpecSpec{ + ConfigLayer: network.ConfigPlatform, + LinkName: name, + Address: ipPrefix, + Scope: nethelpers.ScopeGlobal, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + Family: family, + }, + ) + } + + if eth.Gateway4 != "" { + gw, err := netip.ParseAddr(eth.Gateway4) + if err != nil { + return err + } + + route := network.RouteSpecSpec{ + ConfigLayer: network.ConfigPlatform, + Gateway: gw, + OutLinkName: name, + Table: nethelpers.TableMain, + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + Family: nethelpers.FamilyInet4, + Priority: network.DefaultRouteMetric, + } + + route.Normalize() + + networkConfig.Routes = append(networkConfig.Routes, route) + } + + if eth.Gateway6 != "" { + gw, err := netip.ParseAddr(eth.Gateway6) + if err != nil { + return err + } + + route := network.RouteSpecSpec{ + ConfigLayer: network.ConfigPlatform, + Gateway: gw, + OutLinkName: name, + Table: nethelpers.TableMain, + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + Family: nethelpers.FamilyInet6, + Priority: 2 * network.DefaultRouteMetric, + } + + route.Normalize() + + networkConfig.Routes = append(networkConfig.Routes, route) + } + + for _, addr := range eth.NameServers.Address { + if ip, err := netip.ParseAddr(addr); err == nil { + *dnsIPs = append(*dnsIPs, ip) + } else { + return err + } + } + + for _, route := range eth.Routes { + gw, err := netip.ParseAddr(route.Via) + if err != nil { + return fmt.Errorf("failed to parse route gateway: %w", err) + } + + dest, err := netip.ParsePrefix(route.To) + if err != nil { + return fmt.Errorf("failed to parse route destination: %w", err) + } + + route := network.RouteSpecSpec{ + ConfigLayer: network.ConfigPlatform, + Destination: dest, + Gateway: gw, + OutLinkName: name, + Table: nethelpers.RoutingTable(route.Table), + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + Family: nethelpers.FamilyInet4, + Priority: network.DefaultRouteMetric, + } + + if gw.Is6() { + route.Family = nethelpers.FamilyInet6 + route.Priority = 2 * network.DefaultRouteMetric + } + + route.Normalize() + + networkConfig.Routes = append(networkConfig.Routes, route) + } + + return nil +} + +//nolint:gocyclo +func (n *Nocloud) applyNetworkConfigV2(config *NetworkConfig, st state.State, networkConfig *runtime.PlatformNetworkConfig) error { + var dnsIPs []netip.Addr + + hostInterfaces, err := safe.StateListAll[*network.LinkStatus](context.TODO(), st) + if err != nil { + return fmt.Errorf("error listing host interfaces: %w", err) + } + + ethernetNames := maps.Keys(config.Ethernets) + sort.Strings(ethernetNames) + + for _, name := range ethernetNames { + eth := config.Ethernets[name] + + var bondSlave network.BondSlave + + for bondName, bond := range config.Bonds { + for _, iface := range bond.Interfaces { + if iface == name { + bondSlave.MasterName = bondName + bondSlave.SlaveIndex = 1 + } + } + } + + if eth.Match.HWAddr != "" { + var availableMACAddresses []string + + macAddressMatched := false + hostInterfaceIter := hostInterfaces.Iterator() + + for hostInterfaceIter.Next() { + macAddress := hostInterfaceIter.Value().TypedSpec().PermanentAddr.String() + if macAddress == eth.Match.HWAddr { + name = hostInterfaceIter.Value().Metadata().ID() + macAddressMatched = true + + break + } + + availableMACAddresses = append(availableMACAddresses, macAddress) + } + + if !macAddressMatched { + log.Printf("nocloud: no link with matching MAC address %q (available %v), defaulted to use name %s instead", eth.Match.HWAddr, availableMACAddresses, name) + } + } + + networkConfig.Links = append(networkConfig.Links, network.LinkSpecSpec{ + Name: name, + Up: true, + MTU: eth.MTU, + ConfigLayer: network.ConfigPlatform, + BondSlave: bondSlave, + }) + + err := applyNetworkConfigV2Ethernet(name, eth, networkConfig, &dnsIPs) + if err != nil { + return err + } + } + + for name, bond := range config.Bonds { + mode, err := nethelpers.BondModeByName(bond.Params.Mode) + if err != nil { + return fmt.Errorf("invalid mode: %w", err) + } + + hashPolicy, err := nethelpers.BondXmitHashPolicyByName(bond.Params.HashPolicy) + if err != nil { + return fmt.Errorf("invalid transmit-hash-policy: %w", err) + } + + lacpRate, err := nethelpers.LACPRateByName(bond.Params.LACPRate) + if err != nil { + return fmt.Errorf("invalid lacp-rate: %w", err) + } + + bondLink := network.LinkSpecSpec{ + ConfigLayer: network.ConfigPlatform, + Name: name, + Logical: true, + Up: true, + MTU: bond.Ethernet.MTU, + Kind: network.LinkKindBond, + Type: nethelpers.LinkEther, + BondMaster: network.BondMasterSpec{ + Mode: mode, + HashPolicy: hashPolicy, + MIIMon: bond.Params.MIIMon, + UpDelay: bond.Params.UpDelay, + DownDelay: bond.Params.DownDelay, + LACPRate: lacpRate, + }, + } + + networkadapter.BondMasterSpec(&bondLink.BondMaster).FillDefaults() + networkConfig.Links = append(networkConfig.Links, bondLink) + + err = applyNetworkConfigV2Ethernet(name, bond.Ethernet, networkConfig, &dnsIPs) + if err != nil { + return err + } + } + + if len(dnsIPs) > 0 { + networkConfig.Resolvers = append(networkConfig.Resolvers, network.ResolverSpecSpec{ + DNSServers: dnsIPs, + ConfigLayer: network.ConfigPlatform, + }) + } + + return nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/nocloud.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/nocloud.go new file mode 100644 index 0000000..3e855b2 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/nocloud.go @@ -0,0 +1,141 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package nocloud + +import ( + "bytes" + "context" + stderrors "errors" + "fmt" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-procfs/procfs" + yaml "gopkg.in/yaml.v3" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/errors" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// Nocloud is the concrete type that implements the runtime.Platform interface. +type Nocloud struct{} + +// Name implements the runtime.Platform interface. +func (n *Nocloud) Name() string { + return "nocloud" +} + +// ParseMetadata converts nocloud metadata to platform network config. +func (n *Nocloud) ParseMetadata(unmarshalledNetworkConfig *NetworkConfig, st state.State, metadata *MetadataConfig) (*runtime.PlatformNetworkConfig, error) { + networkConfig := &runtime.PlatformNetworkConfig{} + + hostname := metadata.Hostname + if hostname == "" { + hostname = metadata.LocalHostname + } + + if hostname != "" { + hostnameSpec := network.HostnameSpecSpec{ + ConfigLayer: network.ConfigPlatform, + } + + if err := hostnameSpec.ParseFQDN(hostname); err != nil { + return nil, err + } + + networkConfig.Hostnames = append(networkConfig.Hostnames, hostnameSpec) + } + + switch unmarshalledNetworkConfig.Version { + case 1: + if err := n.applyNetworkConfigV1(unmarshalledNetworkConfig, st, networkConfig); err != nil { + return nil, err + } + case 2: + if err := n.applyNetworkConfigV2(unmarshalledNetworkConfig, st, networkConfig); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("network-config metadata version=%d is not supported", unmarshalledNetworkConfig.Version) + } + + networkConfig.Metadata = &runtimeres.PlatformMetadataSpec{ + Platform: n.Name(), + Hostname: hostname, + InstanceID: metadata.InstanceID, + InstanceType: metadata.InstanceType, + ProviderID: metadata.ProviderID, + Region: metadata.Region, + Zone: metadata.Zone, + } + + return networkConfig, nil +} + +// Configuration implements the runtime.Platform interface. +func (n *Nocloud) Configuration(ctx context.Context, r state.State) ([]byte, error) { + _, _, machineConfigDl, _, err := n.acquireConfig(ctx, r) //nolint:dogsled + if err != nil { + return nil, err + } + + if bytes.HasPrefix(machineConfigDl, []byte("#cloud-config")) { + return nil, errors.ErrNoConfigSource + } + + return machineConfigDl, nil +} + +// Mode implements the runtime.Platform interface. +func (n *Nocloud) Mode() runtime.Mode { + return runtime.ModeCloud +} + +// KernelArgs implements the runtime.Platform interface. +func (n *Nocloud) KernelArgs(string) procfs.Parameters { + return []*procfs.Parameter{ + procfs.NewParameter("console").Append("tty1").Append("ttyS0"), + procfs.NewParameter(constants.KernelParamNetIfnames).Append("0"), + } +} + +// NetworkConfiguration implements the runtime.Platform interface. +func (n *Nocloud) NetworkConfiguration(ctx context.Context, st state.State, ch chan<- *runtime.PlatformNetworkConfig) error { + metadataConfigDl, metadataNetworkConfigDl, _, metadata, err := n.acquireConfig(ctx, st) + if stderrors.Is(err, errors.ErrNoConfigSource) { + err = nil + } + + if err != nil { + return err + } + + if metadataConfigDl == nil && metadataNetworkConfigDl == nil { + // no data, use cached network configuration if available + return nil + } + + var unmarshalledNetworkConfig NetworkConfig + if metadataNetworkConfigDl != nil { + if err = yaml.Unmarshal(metadataNetworkConfigDl, &unmarshalledNetworkConfig); err != nil { + return err + } + } + + networkConfig, err := n.ParseMetadata(&unmarshalledNetworkConfig, st, metadata) + if err != nil { + return err + } + + select { + case ch <- networkConfig: + case <-ctx.Done(): + return ctx.Err() + } + + return nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/nocloud_test.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/nocloud_test.go new file mode 100644 index 0000000..3a03e7e --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/nocloud_test.go @@ -0,0 +1,106 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package nocloud_test + +import ( + "context" + _ "embed" + "fmt" + "testing" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +//go:embed testdata/metadata-v1.yaml +var rawMetadataV1 []byte + +//go:embed testdata/metadata-v2.yaml +var rawMetadataV2 []byte + +//go:embed testdata/expected-v1.yaml +var expectedNetworkConfigV1 string + +//go:embed testdata/expected-v2.yaml +var expectedNetworkConfigV2 string + +func TestParseMetadata(t *testing.T) { + for _, tt := range []struct { + name string + raw []byte + expected string + }{ + { + name: "V1", + raw: rawMetadataV1, + expected: expectedNetworkConfigV1, + }, + { + name: "V2", + raw: rawMetadataV2, + expected: expectedNetworkConfigV2, + }, + } { + t.Run(tt.name, func(t *testing.T) { + n := &nocloud.Nocloud{} + + st := state.WrapCore(namespaced.NewState(inmem.Build)) + + eth0 := network.NewLinkStatus(network.NamespaceName, "eth0") + eth0.TypedSpec().PermanentAddr = nethelpers.HardwareAddr{0x68, 0x05, 0xca, 0xb8, 0xf1, 0xf7} + eth0.TypedSpec().Type = nethelpers.LinkEther + eth0.TypedSpec().Kind = "" + require.NoError(t, st.Create(context.TODO(), eth0)) + + eth1 := network.NewLinkStatus(network.NamespaceName, "eth1") + eth1.TypedSpec().PermanentAddr = nethelpers.HardwareAddr{0x68, 0x05, 0xca, 0xb8, 0xf1, 0xf8} + eth1.TypedSpec().Type = nethelpers.LinkEther + eth1.TypedSpec().Kind = "" + require.NoError(t, st.Create(context.TODO(), eth1)) + + eth2 := network.NewLinkStatus(network.NamespaceName, "eth2") + eth2.TypedSpec().PermanentAddr = nethelpers.HardwareAddr{0x68, 0x05, 0xca, 0xb8, 0xf1, 0xf9} + eth2.TypedSpec().Type = nethelpers.LinkEther + eth2.TypedSpec().Kind = "" + require.NoError(t, st.Create(context.TODO(), eth2)) + + var m nocloud.NetworkConfig + + require.NoError(t, yaml.Unmarshal(tt.raw, &m)) + + mc := nocloud.MetadataConfig{ + Hostname: "talos.fqdn", + InstanceID: "0", + } + mc2 := nocloud.MetadataConfig{ + LocalHostname: "talos.fqdn", + InstanceID: "0", + } + + networkConfig, err := n.ParseMetadata(&m, st, &mc) + require.NoError(t, err) + networkConfig2, err := n.ParseMetadata(&m, st, &mc2) + require.NoError(t, err) + + marshaled, err := yaml.Marshal(networkConfig) + require.NoError(t, err) + marshaled2, err := yaml.Marshal(networkConfig2) + require.NoError(t, err) + + fmt.Print(string(marshaled)) + + assert.Equal(t, tt.expected, string(marshaled)) + assert.Equal(t, tt.expected, string(marshaled2)) + }) + } +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/testdata/expected-v1.yaml b/internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/testdata/expected-v1.yaml new file mode 100644 index 0000000..bc4c610 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/testdata/expected-v1.yaml @@ -0,0 +1,104 @@ +addresses: + - address: 192.168.1.11/24 + linkName: eth0 + family: inet4 + scope: global + flags: permanent + layer: platform + - address: 2001:2:3:4:5:6:7:f7/64 + linkName: eth0 + family: inet6 + scope: global + flags: permanent + layer: platform + - address: 192.168.2.11/24 + linkName: eth2 + family: inet4 + scope: global + flags: permanent + layer: platform + - address: 2001:2:3:4:5:6:7:f9/64 + linkName: eth2 + family: inet6 + scope: global + flags: permanent + layer: platform +links: + - name: eth0 + logical: false + up: true + mtu: 0 + kind: "" + type: netrom + layer: platform + - name: eth2 + logical: false + up: true + mtu: 0 + kind: "" + type: netrom + layer: platform +routes: + - family: inet4 + dst: "" + src: "" + gateway: 192.168.1.1 + outLinkName: eth0 + table: main + priority: 1024 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform + - family: inet6 + dst: "" + src: "" + gateway: fe80::1 + outLinkName: eth0 + table: main + priority: 2048 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform + - family: inet4 + dst: "" + src: "" + gateway: 192.168.2.1 + outLinkName: eth2 + table: main + priority: 1024 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform + - family: inet6 + dst: "" + src: "" + gateway: fe80::2 + outLinkName: eth2 + table: main + priority: 2048 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform +hostnames: + - hostname: talos + domainname: fqdn + layer: platform +resolvers: + - dnsServers: + - 192.168.1.1 + layer: platform +timeServers: [] +operators: [] +externalIPs: [] +metadata: + platform: nocloud + hostname: talos.fqdn + instanceId: "0" diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/testdata/expected-v2.yaml b/internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/testdata/expected-v2.yaml new file mode 100644 index 0000000..ec6d8cc --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/testdata/expected-v2.yaml @@ -0,0 +1,153 @@ +addresses: + - address: 192.168.14.2/24 + linkName: eth0 + family: inet4 + scope: global + flags: permanent + layer: platform + - address: 2001:1::1/64 + linkName: eth0 + family: inet6 + scope: global + flags: permanent + layer: platform + - address: 10.10.4.140/29 + linkName: bond0 + family: inet4 + scope: global + flags: permanent + layer: platform +links: + - name: eth0 + logical: false + up: true + mtu: 0 + kind: "" + type: netrom + layer: platform + - name: eth1 + logical: false + up: true + mtu: 0 + kind: "" + type: netrom + masterName: bond0 + slaveIndex: 1 + layer: platform + - name: eth2 + logical: false + up: true + mtu: 0 + kind: "" + type: netrom + masterName: bond0 + slaveIndex: 1 + layer: platform + - name: bond0 + logical: true + up: true + mtu: 1500 + kind: bond + type: ether + bondMaster: + mode: 802.3ad + xmitHashPolicy: layer3+4 + lacpRate: fast + arpValidate: none + arpAllTargets: any + primaryReselect: always + failOverMac: 0 + miimon: 100 + updelay: 200 + downdelay: 200 + resendIgmp: 1 + lpInterval: 1 + packetsPerSlave: 1 + numPeerNotif: 1 + tlbLogicalLb: 1 + adActorSysPrio: 65535 + layer: platform +routes: + - family: inet4 + dst: "" + src: "" + gateway: 192.168.14.1 + outLinkName: eth0 + table: main + priority: 1024 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform + - family: inet6 + dst: "" + src: "" + gateway: 2001:1::2 + outLinkName: eth0 + table: main + priority: 2048 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform + - family: inet4 + dst: 10.0.0.0/8 + src: "" + gateway: 10.10.4.147 + outLinkName: bond0 + table: unspec + priority: 1024 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform + - family: inet4 + dst: 192.168.0.0/16 + src: "" + gateway: 10.10.4.147 + outLinkName: bond0 + table: unspec + priority: 1024 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform + - family: inet4 + dst: 188.42.208.0/21 + src: "" + gateway: 10.10.4.147 + outLinkName: bond0 + table: unspec + priority: 1024 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform +hostnames: + - hostname: talos + domainname: fqdn + layer: platform +resolvers: + - dnsServers: + - 8.8.8.8 + - 1.1.1.1 + - 2.2.2.2 + layer: platform +timeServers: [] +operators: + - operator: dhcp4 + linkName: eth0 + requireUp: true + dhcp4: + routeMetric: 1024 + layer: platform +externalIPs: [] +metadata: + platform: nocloud + hostname: talos.fqdn + instanceId: "0" diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/testdata/metadata-v1.yaml b/internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/testdata/metadata-v1.yaml new file mode 100644 index 0000000..bfe9ae0 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/testdata/metadata-v1.yaml @@ -0,0 +1,29 @@ +version: 1 +config: + - type: physical + name: eth0 + mac_address: '68:05:ca:b8:f1:f7' + subnets: + - type: static + address: '192.168.1.11' + netmask: '255.255.255.0' + gateway: '192.168.1.1' + - type: static6 + address: '2001:2:3:4:5:6:7:f7/64' + gateway: 'fe80::1' + - type: physical + name: eth1 + mac_address: '68:05:ca:b8:f1:f9' + subnets: + - type: static + address: '192.168.2.11' + netmask: '255.255.255.0' + gateway: '192.168.2.1' + - type: static6 + address: '2001:2:3:4:5:6:7:f9/64' + gateway: 'fe80::2' + - type: nameserver + address: + - '192.168.1.1' + search: + - 'lan' diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/testdata/metadata-v2.yaml b/internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/testdata/metadata-v2.yaml new file mode 100644 index 0000000..5d7b0d0 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/testdata/metadata-v2.yaml @@ -0,0 +1,49 @@ +version: 2 +ethernets: + eth0: + match: + macaddress: '00:20:6e:1f:f9:a8' + dhcp4: true + addresses: + - 192.168.14.2/24 + - 2001:1::1/64 + gateway4: 192.168.14.1 + gateway6: 2001:1::2 + nameservers: + search: [foo.local, bar.local] + addresses: [8.8.8.8] + + ext1: + match: + macaddress: 68:05:ca:b8:f1:f8 + ext2: + match: + macaddress: 68:05:ca:b8:f1:f9 + +bonds: + bond0: + interfaces: + - ext1 + - ext2 + macaddress: e4:3d:1a:4d:6a:28 + mtu: 1500 + parameters: + mode: 802.3ad + mii-monitor-interval: 100 + down-delay: 200 + up-delay: 200 + lacp-rate: fast + transmit-hash-policy: layer3+4 + addresses: + - 10.10.4.140/29 + nameservers: + addresses: + - 1.1.1.1 + - 2.2.2.2 + routes: + - to: 10.0.0.0/8 + via: 10.10.4.147 + - to: 192.168.0.0/16 + via: 10.10.4.147 + - to: 188.42.208.0/21 + via: 10.10.4.147 diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/testdata/metadata-v3.yaml b/internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/testdata/metadata-v3.yaml new file mode 100644 index 0000000..b6fe782 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud/testdata/metadata-v3.yaml @@ -0,0 +1,49 @@ +version: 2 +ethernets: + eth0: + match: + macaddress: '00:20:6e:1f:f9:a8' + dhcp4: true + addresses: + - 192.168.14.2/24 + - 2001:1::1/64 + gateway4: 192.168.14.1 + gateway6: 2001:1::2 + nameservers: + search: [foo.local, bar.local] + addresses: [8.8.8.8] + + ext1: + match: + macaddress: f4:33:01:16:3d:90 + ext2: + match: + macaddress: 50:a5:1b:e8:7b:db + +bonds: + bond0: + interfaces: + - ext1 + - ext2 + macaddress: e4:3d:1a:4d:6a:28 + mtu: 1500 + parameters: + mode: 802.3ad + mii-monitor-interval: 100 + down-delay: 200 + up-delay: 200 + lacp-rate: fast + transmit-hash-policy: layer3+4 + addresses: + - 10.10.4.140/29 + nameservers: + addresses: + - 1.1.1.1 + - 2.2.2.2 + routes: + - to: 10.0.0.0/8 + via: 10.10.4.147 + - to: 192.168.0.0/16 + via: 10.10.4.147 + - to: 188.42.208.0/21 + via: 10.10.4.147 diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/opennebula/metadata.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/opennebula/metadata.go new file mode 100644 index 0000000..aa3723f --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/opennebula/metadata.go @@ -0,0 +1,76 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package opennebula provides the OpenNebula platform implementation. +package opennebula + +import ( + "context" + "fmt" + "log" + "os" + "path/filepath" + "strings" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-blockdevice/blockdevice/filesystem" + "github.com/siderolabs/go-blockdevice/blockdevice/probe" + "golang.org/x/sys/unix" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/errors" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/internal/netutils" +) + +const ( + configISOLabel = "context" + oneContextPath = "context.sh" + mnt = "/mnt" +) + +func (o *OpenNebula) contextFromCD(ctx context.Context, r state.State) (oneContext []byte, err error) { + if err := netutils.WaitForDevicesReady(ctx, r); err != nil { + return nil, fmt.Errorf("failed to wait for devices: %w", err) + } + + var dev *probe.ProbedBlockDevice + + dev, err = probe.GetDevWithFileSystemLabel(strings.ToLower(configISOLabel)) + if err != nil { + dev, err = probe.GetDevWithFileSystemLabel(strings.ToUpper(configISOLabel)) + if err != nil { + return nil, fmt.Errorf("failed to find %s iso: %w", configISOLabel, err) + } + } + + //nolint:errcheck + defer dev.Close() + + sb, err := filesystem.Probe(dev.Path) + if err != nil || sb == nil { + return nil, errors.ErrNoConfigSource + } + + log.Printf("found config disk (context) at %s", dev.Path) + + if err = unix.Mount(dev.Path, mnt, sb.Type(), unix.MS_RDONLY, ""); err != nil { + return nil, fmt.Errorf("failed to mount iso: %w", err) + } + + log.Printf("fetching context from: %s/", oneContextPath) + + oneContext, err = os.ReadFile(filepath.Join(mnt, oneContextPath)) + if err != nil { + return nil, fmt.Errorf("read config: %s", err.Error()) + } + + if err = unix.Unmount(mnt, 0); err != nil { + return nil, fmt.Errorf("failed to unmount: %w", err) + } + + if oneContext == nil { + return nil, errors.ErrNoConfigSource + } + + return oneContext, nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/opennebula/opennebula.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/opennebula/opennebula.go new file mode 100644 index 0000000..72837d9 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/opennebula/opennebula.go @@ -0,0 +1,258 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package opennebula + +import ( + "bytes" + "context" + "encoding/base64" + stderrors "errors" + "fmt" + "net/netip" + "strconv" + "strings" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/hashicorp/go-envparse" + "github.com/siderolabs/go-procfs/procfs" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/errors" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/internal/address" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// OpenNebula is the concrete type that implements the runtime.Platform interface. +type OpenNebula struct{} + +// Name implements the runtime.Platform interface. +func (o *OpenNebula) Name() string { + return "opennebula" +} + +// ParseMetadata converts opennebula metadata to platform network config. +// +//nolint:gocyclo +func (o *OpenNebula) ParseMetadata(st state.State, oneContextPlain []byte) (*runtime.PlatformNetworkConfig, error) { + // Initialize the PlatformNetworkConfig + networkConfig := &runtime.PlatformNetworkConfig{} + + oneContext, err := envparse.Parse(bytes.NewReader(oneContextPlain)) + if err != nil { + return nil, fmt.Errorf("failed to parse context file %q: %w", oneContextPlain, err) + } + + // Create HostnameSpecSpec entry + hostnameValue := oneContext["HOSTNAME"] + if hostnameValue == "" { + hostnameValue = oneContext["SET_HOSTNAME"] + if hostnameValue == "" { + hostnameValue = oneContext["NAME"] + } + } + + if oneContext["NETWORK"] == "YES" { + // Iterate through parsed environment variables + for key := range oneContext { + // Dereference the pointer here + if strings.HasPrefix(key, "ETH") && strings.HasSuffix(key, "_MAC") { + ifaceName := strings.TrimSuffix(key, "_MAC") + ifaceNameLower := strings.ToLower(ifaceName) + + if oneContext[ifaceName+"_METHOD"] == "dhcp" { + // Create DHCP4 OperatorSpec entry + networkConfig.Operators = append(networkConfig.Operators, + network.OperatorSpecSpec{ + Operator: network.OperatorDHCP4, + LinkName: ifaceNameLower, + RequireUp: true, + DHCP4: network.DHCP4OperatorSpec{ + RouteMetric: 1024, + SkipHostnameRequest: true, + }, + ConfigLayer: network.ConfigPlatform, + }, + ) + } else { + // Parse IP address and create AddressSpecSpec entry + ipPrefix, err := address.IPPrefixFrom(oneContext[ifaceName+"_IP"], oneContext[ifaceName+"_MASK"]) + if err != nil { + return nil, fmt.Errorf("failed to parse IP address: %w", err) + } + + networkConfig.Addresses = append(networkConfig.Addresses, + network.AddressSpecSpec{ + Address: ipPrefix, + LinkName: ifaceNameLower, + Family: nethelpers.FamilyInet4, + Scope: nethelpers.ScopeGlobal, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + AnnounceWithARP: false, + ConfigLayer: network.ConfigPlatform, + }, + ) + + var mtu uint32 + + if oneContext[ifaceName+"_MTU"] == "" { + mtu = 0 + } else { + var mtu64 uint64 + + mtu64, err = strconv.ParseUint(oneContext[ifaceName+"_MTU"], 10, 32) + // check if any error happened + if err != nil { + return nil, fmt.Errorf("failed to parse MTU: %w", err) + } + + mtu = uint32(mtu64) + } + + // Create LinkSpecSpec entry + networkConfig.Links = append(networkConfig.Links, + network.LinkSpecSpec{ + Name: ifaceNameLower, + Logical: false, + Up: true, + MTU: mtu, + Kind: "", + Type: nethelpers.LinkEther, + ParentName: "", + ConfigLayer: network.ConfigPlatform, + }, + ) + + if oneContext[ifaceName+"_GATEWAY"] != "" { + // Parse gateway address and create RouteSpecSpec entry + gateway, err := netip.ParseAddr(oneContext[ifaceName+"_GATEWAY"]) + if err != nil { + return nil, fmt.Errorf("failed to parse gateway ip: %w", err) + } + + route := network.RouteSpecSpec{ + ConfigLayer: network.ConfigPlatform, + Gateway: gateway, + OutLinkName: ifaceNameLower, + Table: nethelpers.TableMain, + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + Family: nethelpers.FamilyInet4, + Priority: network.DefaultRouteMetric, + } + + route.Normalize() + + networkConfig.Routes = append(networkConfig.Routes, route) + } + + // Parse DNS servers + dnsServers := strings.Fields(oneContext[ifaceName+"_DNS"]) + + var dnsIPs []netip.Addr + + for _, dnsServer := range dnsServers { + ip, err := netip.ParseAddr(dnsServer) + if err != nil { + return nil, fmt.Errorf("failed to parse DNS server IP: %w", err) + } + + dnsIPs = append(dnsIPs, ip) + } + + // Create ResolverSpecSpec entry with multiple DNS servers + networkConfig.Resolvers = append(networkConfig.Resolvers, + network.ResolverSpecSpec{ + DNSServers: dnsIPs, + ConfigLayer: network.ConfigPlatform, + }, + ) + } + } + } + } + // Create HostnameSpecSpec entry + networkConfig.Hostnames = append(networkConfig.Hostnames, + network.HostnameSpecSpec{ + Hostname: hostnameValue, + Domainname: oneContext["DNS_HOSTNAME"], + ConfigLayer: network.ConfigPlatform, + }, + ) + + // Create Metadata entry + networkConfig.Metadata = &runtimeres.PlatformMetadataSpec{ + Platform: o.Name(), + Hostname: hostnameValue, + InstanceID: oneContext["VMID"], + } + + return networkConfig, nil +} + +// Configuration implements the runtime.Platform interface. +func (o *OpenNebula) Configuration(ctx context.Context, r state.State) (machineConfig []byte, err error) { + oneContextPlain, err := o.contextFromCD(ctx, r) + if err != nil { + return nil, err + } + + oneContext, err := envparse.Parse(bytes.NewReader(oneContextPlain)) + if err != nil { + return nil, fmt.Errorf("failed to parse environment file %q: %w", oneContextPlain, err) + } + + userData, ok := oneContext["USER_DATA"] + if !ok { + return nil, errors.ErrNoConfigSource + } + + machineConfig, err = base64.StdEncoding.DecodeString(userData) + if err != nil { + return nil, fmt.Errorf("failed to decode USER_DATA: %v", err) + } + + return machineConfig, nil +} + +// Mode implements the runtime.Platform interface. +func (o *OpenNebula) Mode() runtime.Mode { + return runtime.ModeCloud +} + +// KernelArgs implements the runtime.Platform interface. +func (o *OpenNebula) KernelArgs(string) procfs.Parameters { + return []*procfs.Parameter{ + procfs.NewParameter("console").Append("tty1").Append("ttyS0"), + procfs.NewParameter(constants.KernelParamNetIfnames).Append("0"), + } +} + +// NetworkConfiguration implements the runtime.Platform interface. +func (o *OpenNebula) NetworkConfiguration(ctx context.Context, st state.State, ch chan<- *runtime.PlatformNetworkConfig) error { + oneContext, err := o.contextFromCD(ctx, st) + if stderrors.Is(err, errors.ErrNoConfigSource) { + err = nil + } + + if err != nil { + return err + } + + networkConfig, err := o.ParseMetadata(st, oneContext) + if err != nil { + return err + } + + select { + case ch <- networkConfig: + case <-ctx.Done(): + return ctx.Err() + } + + return nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/opennebula/opennebula_test.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/opennebula/opennebula_test.go new file mode 100644 index 0000000..705d328 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/opennebula/opennebula_test.go @@ -0,0 +1,40 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package opennebula_test + +import ( + _ "embed" + "fmt" + "testing" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/opennebula" +) + +//go:embed testdata/metadata.yaml +var oneContextPlain []byte + +//go:embed testdata/expected.yaml +var expectedNetworkConfig string + +func TestParseMetadata(t *testing.T) { + o := &opennebula.OpenNebula{} + st := state.WrapCore(namespaced.NewState(inmem.Build)) + + networkConfig, err := o.ParseMetadata(st, oneContextPlain) + require.NoError(t, err) + + marshaled, err := yaml.Marshal(networkConfig) + require.NoError(t, err) + + fmt.Print(marshaled) + assert.Equal(t, expectedNetworkConfig, string(marshaled)) +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/opennebula/testdata/expected.yaml b/internal/app/machined/pkg/runtime/v1alpha1/platform/opennebula/testdata/expected.yaml new file mode 100644 index 0000000..0151c63 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/opennebula/testdata/expected.yaml @@ -0,0 +1,45 @@ +addresses: + - address: 192.168.1.92/24 + linkName: eth0 + family: inet4 + scope: global + flags: permanent + layer: platform +links: + - name: eth0 + logical: false + up: true + mtu: 0 + kind: "" + type: ether + layer: platform +routes: + - family: inet4 + dst: "" + src: "" + gateway: 192.168.1.1 + outLinkName: eth0 + table: main + priority: 1024 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform +hostnames: + - hostname: code-server + domainname: "" + layer: platform +resolvers: + - dnsServers: + - 192.168.1.1 + - 8.8.8.8 + - 1.1.1.1 + layer: platform +timeServers: [] +operators: [] +externalIPs: [] +metadata: + platform: opennebula + hostname: code-server + instanceId: "14" diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/opennebula/testdata/metadata.yaml b/internal/app/machined/pkg/runtime/v1alpha1/platform/opennebula/testdata/metadata.yaml new file mode 100644 index 0000000..cae9d34 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/opennebula/testdata/metadata.yaml @@ -0,0 +1,28 @@ +# Context variables generated by OpenNebula +DISK_ID = "1" +ETH0_DNS = "192.168.1.1 8.8.8.8 1.1.1.1" +ETH0_EXTERNAL = "" +ETH0_GATEWAY = "192.168.1.1" +ETH0_IP = "192.168.1.92" +ETH0_IP6 = "" +ETH0_IP6_GATEWAY = "" +ETH0_IP6_METHOD = "" +ETH0_IP6_METRIC = "" +ETH0_IP6_PREFIX_LENGTH = "" +ETH0_IP6_ULA = "" +ETH0_MAC = "02:00:c0:a8:01:5c" +ETH0_MASK = "255.255.255.0" +ETH0_METHOD = "" +ETH0_METRIC = "" +ETH0_MTU = "" +ETH0_NETWORK = "192.168.1.0" +ETH0_SEARCH_DOMAIN = "" +ETH0_VLAN_ID = "3" +ETH0_VROUTER_IP = "" +ETH0_VROUTER_IP6 = "" +ETH0_VROUTER_MANAGEMENT = "" +NETWORK = "YES" +SSH_PUBLIC_KEY = "" +TARGET = "hda" +VMID = "14" +NAME = "code-server" \ No newline at end of file diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/openstack/metadata.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/openstack/metadata.go new file mode 100644 index 0000000..74b5bb8 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/openstack/metadata.go @@ -0,0 +1,206 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package openstack + +import ( + "context" + "fmt" + "log" + "net/netip" + "os" + "path/filepath" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-blockdevice/blockdevice/filesystem" + "github.com/siderolabs/go-blockdevice/blockdevice/probe" + "golang.org/x/sys/unix" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/errors" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/internal/netutils" + "github.com/siderolabs/talos/pkg/download" +) + +const ( + // mnt is folder to mount config drive. + mnt = "/mnt" + + // config-drive configs path. + configISOLabel = "config-2" + configMetadataPath = "openstack/latest/meta_data.json" + configNetworkDataPath = "openstack/latest/network_data.json" + configUserDataPath = "openstack/latest/user_data" + + endpoint = "http://169.254.169.254/" + + // OpenstackExternalIPEndpoint is the local Openstack endpoint for the external IP. + OpenstackExternalIPEndpoint = endpoint + "latest/meta-data/public-ipv4" + // OpenstackInstanceTypeEndpoint is the local Openstack endpoint for the instance-type. + OpenstackInstanceTypeEndpoint = endpoint + "latest/meta-data/instance-type" + // OpenstackMetaDataEndpoint is the local Openstack endpoint for the meta config. + OpenstackMetaDataEndpoint = endpoint + configMetadataPath + // OpenstackNetworkDataEndpoint is the local Openstack endpoint for the network config. + OpenstackNetworkDataEndpoint = endpoint + configNetworkDataPath + // OpenstackUserDataEndpoint is the local Openstack endpoint for the config. + OpenstackUserDataEndpoint = endpoint + configUserDataPath +) + +// NetworkConfig holds NetworkData config. +type NetworkConfig struct { + Links []struct { + ID string `json:"id,omitempty"` + Type string `json:"type"` + Mac string `json:"ethernet_mac_address,omitempty"` + MTU int `json:"mtu,omitempty"` + BondMode string `json:"bond_mode,omitempty"` + BondLinks []string `json:"bond_links,omitempty"` + BondMIIMon uint32 `json:"bond_miimon,string,omitempty"` + BondHashPolicy string `json:"bond_xmit_hash_policy,omitempty"` + } `json:"links"` + Networks []struct { + ID string `json:"id,omitempty"` + Link string `json:"link"` + Type string `json:"type"` + Address string `json:"ip_address,omitempty"` + Netmask string `json:"netmask,omitempty"` + Gateway string `json:"gateway,omitempty"` + Routes []struct { + Network string `json:"network,omitempty"` + Netmask string `json:"netmask,omitempty"` + Gateway string `json:"gateway,omitempty"` + } `json:"routes,omitempty"` + } `json:"networks"` + Services []struct { + Type string `json:"type"` + Address string `json:"address"` + } `json:"services,omitempty"` +} + +// MetadataConfig holds meta info. +type MetadataConfig struct { + UUID string `json:"uuid,omitempty"` + Hostname string `json:"hostname,omitempty"` + AvailabilityZone string `json:"availability_zone,omitempty"` + ProjectID string `json:"project_id"` + InstanceType string `json:"instance_type"` +} + +func (o *Openstack) configFromNetwork(ctx context.Context) (metaConfig []byte, networkConfig []byte, machineConfig []byte, err error) { + log.Printf("fetching meta config from: %q", OpenstackMetaDataEndpoint) + + metaConfig, err = download.Download(ctx, OpenstackMetaDataEndpoint) + if err != nil { + metaConfig = nil + } + + log.Printf("fetching network config from: %q", OpenstackNetworkDataEndpoint) + + networkConfig, err = download.Download(ctx, OpenstackNetworkDataEndpoint) + if err != nil { + networkConfig = nil + } + + log.Printf("fetching machine config from: %q", OpenstackUserDataEndpoint) + + machineConfig, err = download.Download(ctx, OpenstackUserDataEndpoint, + download.WithErrorOnNotFound(errors.ErrNoConfigSource), + download.WithErrorOnEmptyResponse(errors.ErrNoConfigSource)) + + return metaConfig, networkConfig, machineConfig, err +} + +//nolint:gocyclo +func (o *Openstack) configFromCD(ctx context.Context, r state.State) (metaConfig []byte, networkConfig []byte, machineConfig []byte, err error) { + if err := netutils.WaitForDevicesReady(ctx, r); err != nil { + return nil, nil, nil, fmt.Errorf("failed to wait for devices: %w", err) + } + + var dev *probe.ProbedBlockDevice + + dev, err = probe.GetDevWithFileSystemLabel(configISOLabel) + if err != nil { + return nil, nil, nil, errors.ErrNoConfigSource + } + + //nolint:errcheck + defer dev.Close() + + sb, err := filesystem.Probe(dev.Path) + if err != nil || sb == nil { + return nil, nil, nil, errors.ErrNoConfigSource + } + + log.Printf("found config disk (config-drive) at %s", dev.Path) + + if err = unix.Mount(dev.Path, mnt, sb.Type(), unix.MS_RDONLY, ""); err != nil { + return nil, nil, nil, errors.ErrNoConfigSource + } + + log.Printf("fetching meta config from: config-drive/%s", configMetadataPath) + + metaConfig, err = os.ReadFile(filepath.Join(mnt, configMetadataPath)) + if err != nil { + log.Printf("failed to read %s", configMetadataPath) + + metaConfig = nil + } + + log.Printf("fetching network config from: config-drive/%s", configNetworkDataPath) + + networkConfig, err = os.ReadFile(filepath.Join(mnt, configNetworkDataPath)) + if err != nil { + log.Printf("failed to read %s", configNetworkDataPath) + + networkConfig = nil + } + + log.Printf("fetching machine config from: config-drive/%s", configUserDataPath) + + machineConfig, err = os.ReadFile(filepath.Join(mnt, configUserDataPath)) + if err != nil { + log.Printf("failed to read %s", configUserDataPath) + + machineConfig = nil + } + + if err = unix.Unmount(mnt, 0); err != nil { + return nil, nil, nil, fmt.Errorf("failed to unmount: %w", err) + } + + if machineConfig == nil { + return metaConfig, networkConfig, machineConfig, errors.ErrNoConfigSource + } + + return metaConfig, networkConfig, machineConfig, nil +} + +func (o *Openstack) instanceType(ctx context.Context) string { + log.Printf("fetching instance-type from: %q", OpenstackInstanceTypeEndpoint) + + sku, err := download.Download(ctx, OpenstackInstanceTypeEndpoint) + if err != nil { + return "" + } + + return string(sku) +} + +func (o *Openstack) externalIPs(ctx context.Context) (addrs []netip.Addr) { + log.Printf("fetching externalIP from: %q", OpenstackExternalIPEndpoint) + + exIP, err := download.Download(ctx, OpenstackExternalIPEndpoint, + download.WithErrorOnNotFound(errors.ErrNoExternalIPs), + download.WithErrorOnEmptyResponse(errors.ErrNoExternalIPs)) + if err != nil { + log.Printf("failed to fetch external IPs, ignored: %s", err) + + return nil + } + + if addr, err := netip.ParseAddr(string(exIP)); err == nil { + addrs = append(addrs, addr) + } + + return addrs +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/openstack/openstack.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/openstack/openstack.go new file mode 100644 index 0000000..f7ce197 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/openstack/openstack.go @@ -0,0 +1,437 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package openstack provides the Openstack platform implementation. +package openstack + +import ( + "bytes" + "context" + "encoding/json" + stderrors "errors" + "fmt" + "log" + "net/netip" + "strings" + + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-procfs/procfs" + + networkadapter "github.com/aenix-io/talm/internal/app/machined/pkg/adapters/network" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/errors" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/internal/address" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/internal/netutils" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// Openstack is the concrete type that implements the runtime.Platform interface. +type Openstack struct{} + +// Name implements the runtime.Platform interface. +func (o *Openstack) Name() string { + return "openstack" +} + +// ParseMetadata converts OpenStack metadata to platform network configuration. +// +//nolint:gocyclo,cyclop +func (o *Openstack) ParseMetadata( + ctx context.Context, + unmarshalledNetworkConfig *NetworkConfig, + extIPs []netip.Addr, + metadata *MetadataConfig, + st state.State, +) (*runtime.PlatformNetworkConfig, error) { + networkConfig := &runtime.PlatformNetworkConfig{} + + if metadata.Hostname != "" { + hostnameSpec := network.HostnameSpecSpec{ + ConfigLayer: network.ConfigPlatform, + } + + if err := hostnameSpec.ParseFQDN(metadata.Hostname); err != nil { + return nil, err + } + + networkConfig.Hostnames = append(networkConfig.Hostnames, hostnameSpec) + } + + networkConfig.ExternalIPs = extIPs + + var dnsIPs []netip.Addr + + for _, netsvc := range unmarshalledNetworkConfig.Services { + if netsvc.Type == "dns" && netsvc.Address != "" { + if ip, err := netip.ParseAddr(netsvc.Address); err == nil { + dnsIPs = append(dnsIPs, ip) + } else { + return nil, fmt.Errorf("failed to parse dns service ip: %w", err) + } + } + } + + if len(dnsIPs) > 0 { + networkConfig.Resolvers = append(networkConfig.Resolvers, network.ResolverSpecSpec{ + DNSServers: dnsIPs, + ConfigLayer: network.ConfigPlatform, + }) + } + + hostInterfaces, err := safe.StateListAll[*network.LinkStatus](ctx, st) + if err != nil { + return nil, fmt.Errorf("error listing host interfaces: %w", err) + } + + ifaces := make(map[string]string) + bondLinks := make(map[string]string) + + // Bonds + + bondIndex := 0 + + for _, netLink := range unmarshalledNetworkConfig.Links { + if netLink.Type != "bond" { + continue + } + + mode, err := nethelpers.BondModeByName(netLink.BondMode) + if err != nil { + return nil, fmt.Errorf("invalid bond_mode: %w", err) + } + + hashPolicy, err := nethelpers.BondXmitHashPolicyByName(netLink.BondHashPolicy) + if err != nil { + return nil, fmt.Errorf("invalid bond_xmit_hash_policy: %w", err) + } + + bondName := fmt.Sprintf("bond%d", bondIndex) + ifaces[netLink.ID] = bondName + + bondLink := network.LinkSpecSpec{ + ConfigLayer: network.ConfigPlatform, + Name: bondName, + Logical: true, + Up: true, + MTU: uint32(netLink.MTU), + Kind: network.LinkKindBond, + Type: nethelpers.LinkEther, + BondMaster: network.BondMasterSpec{ + Mode: mode, + MIIMon: netLink.BondMIIMon, + HashPolicy: hashPolicy, + UpDelay: 200, + DownDelay: 200, + LACPRate: nethelpers.LACPRateFast, + }, + } + + networkadapter.BondMasterSpec(&bondLink.BondMaster).FillDefaults() + networkConfig.Links = append(networkConfig.Links, bondLink) + + for _, link := range netLink.BondLinks { + bondLinks[link] = bondName + } + + bondIndex++ + } + + bondSlaveIndexes := make(map[string]int) + + // Interfaces + + for idx, netLink := range unmarshalledNetworkConfig.Links { + // OpenStack network metadata schema: + // "type": { + // "$id": "#/definitions/l2_link/properties/type", + // "type": "string", + // "enum": [ + // "bridge", + // "dvs", + // "hw_veb", + // "hyperv", + // "ovs", + // "tap", + // "vhostuser", + // "vif", + // "phy" + // ], + // "title": "Interface type", + // "examples": [ + // "bridge" + // ] + // }, + // "vif_id": { + // "$ref": "#/definitions/l2_vif_id" + // } + switch netLink.Type { + case "phy", "vif", "ovs", "bridge", "tap", "vhostuser", "hw_veb": + linkName := "" + + for hostInterfaceIter := hostInterfaces.Iterator(); hostInterfaceIter.Next(); { + if strings.EqualFold(hostInterfaceIter.Value().TypedSpec().PermanentAddr.String(), netLink.Mac) { + linkName = hostInterfaceIter.Value().Metadata().ID() + + break + } + } + + if linkName == "" { + linkName = fmt.Sprintf("eth%d", idx) + + log.Printf("failed to find interface with MAC %q, using %q", netLink.Mac, linkName) + } + + ifaces[netLink.ID] = linkName + + link := network.LinkSpecSpec{ + Name: ifaces[netLink.ID], + Up: true, + MTU: uint32(netLink.MTU), + ConfigLayer: network.ConfigPlatform, + } + + if bondName, ok := bondLinks[netLink.ID]; ok { + link.BondSlave = network.BondSlave{ + MasterName: bondName, + SlaveIndex: bondSlaveIndexes[bondName], + } + + bondSlaveIndexes[bondName]++ + } + + networkConfig.Links = append(networkConfig.Links, link) + } + } + + for _, ntwrk := range unmarshalledNetworkConfig.Networks { + if ntwrk.ID == "" || ifaces[ntwrk.Link] == "" { + continue + } + + iface := ifaces[ntwrk.Link] + + switch ntwrk.Type { + case "ipv4_dhcp": + networkConfig.Operators = append(networkConfig.Operators, network.OperatorSpecSpec{ + Operator: network.OperatorDHCP4, + LinkName: iface, + RequireUp: true, + DHCP4: network.DHCP4OperatorSpec{ + RouteMetric: network.DefaultRouteMetric, + SkipHostnameRequest: true, + }, + ConfigLayer: network.ConfigPlatform, + }) + case "ipv6_dhcp", "ipv6_dhcpv6-stateless", "ipv6_dhcpv6-stateful": + networkConfig.Operators = append(networkConfig.Operators, network.OperatorSpecSpec{ + Operator: network.OperatorDHCP6, + LinkName: iface, + RequireUp: true, + DHCP6: network.DHCP6OperatorSpec{ + RouteMetric: 2 * network.DefaultRouteMetric, + SkipHostnameRequest: true, + }, + ConfigLayer: network.ConfigPlatform, + }) + case "ipv4", "ipv6", "ipv6_slaac": + // FIXME: we need to switch on/off slaac here + default: + log.Printf("network type %s is not supported", ntwrk.Type) + + continue + } + + if ntwrk.Address != "" { + ipPrefix, err := address.IPPrefixFrom(ntwrk.Address, ntwrk.Netmask) + if err != nil { + return nil, fmt.Errorf("failed to parse ip address: %w", err) + } + + family := nethelpers.FamilyInet4 + if ipPrefix.Addr().Is6() { + family = nethelpers.FamilyInet6 + } + + networkConfig.Addresses = append(networkConfig.Addresses, + network.AddressSpecSpec{ + ConfigLayer: network.ConfigPlatform, + LinkName: iface, + Address: ipPrefix, + Scope: nethelpers.ScopeGlobal, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + Family: family, + }, + ) + + if ntwrk.Gateway != "" { + gw, err := netip.ParseAddr(ntwrk.Gateway) + if err != nil { + return nil, fmt.Errorf("failed to parse gateway ip: %w", err) + } + + priority := uint32(network.DefaultRouteMetric) + + if family == nethelpers.FamilyInet6 { + priority *= 2 + } + + route := network.RouteSpecSpec{ + ConfigLayer: network.ConfigPlatform, + Gateway: gw, + OutLinkName: iface, + Table: nethelpers.TableMain, + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + Family: family, + Priority: priority, + } + + route.Normalize() + + networkConfig.Routes = append(networkConfig.Routes, route) + } + } + + for _, route := range ntwrk.Routes { + gw, err := netip.ParseAddr(route.Gateway) + if err != nil { + return nil, fmt.Errorf("failed to parse route gateway: %w", err) + } + + dest, err := address.IPPrefixFrom(route.Network, route.Netmask) + if err != nil { + return nil, fmt.Errorf("failed to parse route network: %w", err) + } + + family := nethelpers.FamilyInet4 + if dest.Addr().Is6() { + family = nethelpers.FamilyInet6 + } + + route := network.RouteSpecSpec{ + ConfigLayer: network.ConfigPlatform, + Destination: dest, + Gateway: gw, + OutLinkName: iface, + Table: nethelpers.TableMain, + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + Family: family, + Priority: network.DefaultRouteMetric, + } + + route.Normalize() + + // double the priority of the route if it is actually the default gateway and IPv6 + if route.Destination == (netip.Prefix{}) && family == nethelpers.FamilyInet6 { + route.Priority *= 2 + } + + networkConfig.Routes = append(networkConfig.Routes, route) + } + } + + networkConfig.Metadata = &runtimeres.PlatformMetadataSpec{ + Platform: o.Name(), + Hostname: metadata.Hostname, + Zone: metadata.AvailabilityZone, + InstanceID: metadata.UUID, + InstanceType: metadata.InstanceType, + ProviderID: fmt.Sprintf("openstack:///%s", metadata.UUID), + } + + return networkConfig, nil +} + +// Configuration implements the runtime.Platform interface. +func (o *Openstack) Configuration(ctx context.Context, r state.State) (machineConfig []byte, err error) { + _, _, machineConfig, err = o.configFromCD(ctx, r) + if err != nil { + if err = netutils.Wait(ctx, r); err != nil { + return nil, err + } + + _, _, machineConfig, err = o.configFromNetwork(ctx) + if err != nil { + return nil, err + } + } + + // Some openstack setups does not allow you to change user-data, + // so skip this case. + if bytes.HasPrefix(machineConfig, []byte("#cloud-config")) { + return nil, errors.ErrNoConfigSource + } + + return machineConfig, nil +} + +// Mode implements the runtime.Platform interface. +func (o *Openstack) Mode() runtime.Mode { + return runtime.ModeCloud +} + +// KernelArgs implements the runtime.Platform interface. +func (o *Openstack) KernelArgs(string) procfs.Parameters { + return []*procfs.Parameter{ + procfs.NewParameter("console").Append("tty1").Append("ttyS0"), + procfs.NewParameter(constants.KernelParamNetIfnames).Append("0"), + } +} + +// NetworkConfiguration implements the runtime.Platform interface. +func (o *Openstack) NetworkConfiguration(ctx context.Context, st state.State, ch chan<- *runtime.PlatformNetworkConfig) error { + networkSource := false + + metadataConfigDl, metadataNetworkConfigDl, _, err := o.configFromCD(ctx, st) + if err != nil { + metadataConfigDl, metadataNetworkConfigDl, _, err = o.configFromNetwork(ctx) + if stderrors.Is(err, errors.ErrNoConfigSource) { + err = nil + } + + if err != nil { + return err + } + + networkSource = true + } + + var ( + meta MetadataConfig + unmarshalledNetworkConfig NetworkConfig + ) + + // ignore errors unmarshaling, empty configs work just fine as empty default + _ = json.Unmarshal(metadataConfigDl, &meta) //nolint:errcheck + _ = json.Unmarshal(metadataNetworkConfigDl, &unmarshalledNetworkConfig) //nolint:errcheck + + var extIPs []netip.Addr + + if networkSource { + extIPs = o.externalIPs(ctx) + + if meta.InstanceType == "" { + meta.InstanceType = o.instanceType(ctx) + } + } + + networkConfig, err := o.ParseMetadata(ctx, &unmarshalledNetworkConfig, extIPs, &meta, st) + if err != nil { + return err + } + + select { + case ch <- networkConfig: + case <-ctx.Done(): + return ctx.Err() + } + + return nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/openstack/openstack_test.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/openstack/openstack_test.go new file mode 100644 index 0000000..c6f5613 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/openstack/openstack_test.go @@ -0,0 +1,79 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package openstack_test + +import ( + "context" + _ "embed" + "encoding/json" + "net/netip" + "testing" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/openstack" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +//go:embed testdata/metadata.json +var rawMetadata []byte + +//go:embed testdata/network.json +var rawNetwork []byte + +//go:embed testdata/expected.yaml +var expectedNetworkConfig string + +func TestParseMetadata(t *testing.T) { + o := &openstack.Openstack{} + + var metadata openstack.MetadataConfig + + require.NoError(t, json.Unmarshal(rawMetadata, &metadata)) + + var n openstack.NetworkConfig + + require.NoError(t, json.Unmarshal(rawNetwork, &n)) + + ctx := context.Background() + + st := state.WrapCore(namespaced.NewState(inmem.Build)) + + eth0 := network.NewLinkStatus(network.NamespaceName, "eth0") + eth0.TypedSpec().PermanentAddr = nethelpers.HardwareAddr{0xa4, 0xbf, 0x00, 0x10, 0x20, 0x30} + require.NoError(t, st.Create(ctx, eth0)) + + eth1 := network.NewLinkStatus(network.NamespaceName, "eth1") + eth1.TypedSpec().PermanentAddr = nethelpers.HardwareAddr{0xa4, 0xbf, 0x00, 0x10, 0x20, 0x31} + require.NoError(t, st.Create(ctx, eth1)) + + eth2 := network.NewLinkStatus(network.NamespaceName, "eth2") + eth2.TypedSpec().PermanentAddr = nethelpers.HardwareAddr{0xa4, 0xbf, 0x00, 0x10, 0x20, 0x33} + require.NoError(t, st.Create(ctx, eth2)) + + // Bond slaves + + eth3 := network.NewLinkStatus(network.NamespaceName, "eth3") + eth3.TypedSpec().PermanentAddr = nethelpers.HardwareAddr{0x4c, 0xd9, 0x8f, 0xb3, 0x34, 0xf8} + require.NoError(t, st.Create(ctx, eth3)) + + eth4 := network.NewLinkStatus(network.NamespaceName, "eth4") + eth4.TypedSpec().PermanentAddr = nethelpers.HardwareAddr{0x4c, 0xd9, 0x8f, 0xb3, 0x34, 0xf7} + require.NoError(t, st.Create(ctx, eth4)) + + networkConfig, err := o.ParseMetadata(ctx, &n, []netip.Addr{netip.MustParseAddr("1.2.3.4")}, &metadata, st) + require.NoError(t, err) + + marshaled, err := yaml.Marshal(networkConfig) + require.NoError(t, err) + + assert.Equal(t, expectedNetworkConfig, string(marshaled)) +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/openstack/testdata/expected.yaml b/internal/app/machined/pkg/runtime/v1alpha1/platform/openstack/testdata/expected.yaml new file mode 100644 index 0000000..1898589 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/openstack/testdata/expected.yaml @@ -0,0 +1,200 @@ +addresses: + - address: 2000:0:100::/56 + linkName: eth0 + family: inet6 + scope: global + flags: permanent + layer: platform + - address: 10.184.0.244/20 + linkName: eth1 + family: inet4 + scope: global + flags: permanent + layer: platform + - address: 2001:db8::3257:9652/64 + linkName: eth1 + family: inet6 + scope: global + flags: permanent + layer: platform + - address: fd60:172:16:84:f816:3eff:fe73:5901/64 + linkName: eth2 + family: inet6 + scope: global + flags: permanent + layer: platform + - address: 94.156.45.48/24 + linkName: bond0 + family: inet4 + scope: global + flags: permanent + layer: platform +links: + - name: bond0 + logical: true + up: true + mtu: 1500 + kind: bond + type: ether + bondMaster: + mode: 802.3ad + xmitHashPolicy: layer2+3 + lacpRate: fast + arpValidate: none + arpAllTargets: any + primaryReselect: always + failOverMac: 0 + miimon: 100 + updelay: 200 + downdelay: 200 + resendIgmp: 1 + lpInterval: 1 + packetsPerSlave: 1 + numPeerNotif: 1 + tlbLogicalLb: 1 + adActorSysPrio: 65535 + layer: platform + - name: eth0 + logical: false + up: true + mtu: 1450 + kind: "" + type: netrom + layer: platform + - name: eth1 + logical: false + up: true + mtu: 9000 + kind: "" + type: netrom + layer: platform + - name: eth2 + logical: false + up: true + mtu: 0 + kind: "" + type: netrom + layer: platform + - name: eth3 + logical: false + up: true + mtu: 0 + kind: "" + type: netrom + masterName: bond0 + layer: platform + - name: eth4 + logical: false + up: true + mtu: 0 + kind: "" + type: netrom + masterName: bond0 + slaveIndex: 1 + layer: platform +routes: + - family: inet6 + dst: "" + src: "" + gateway: 2000:0:100:2fff:ff:ff:ff:ff + outLinkName: eth0 + table: main + priority: 2048 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform + - family: inet6 + dst: 2000:0:100:2f00::/58 + src: "" + gateway: 2000:0:100:2fff:ff:ff:ff:f0 + outLinkName: eth0 + table: main + priority: 1024 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform + - family: inet4 + dst: 192.168.0.0/16 + src: "" + gateway: 10.184.0.1 + outLinkName: eth1 + table: main + priority: 1024 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform + - family: inet4 + dst: "" + src: "" + gateway: 10.184.0.1 + outLinkName: eth1 + table: main + priority: 1024 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform + - family: inet6 + dst: "" + src: "" + gateway: fd00::1 + outLinkName: eth1 + table: main + priority: 2048 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform + - family: inet4 + dst: "" + src: "" + gateway: 94.156.45.1 + outLinkName: bond0 + table: main + priority: 1024 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform +hostnames: + - hostname: talos + domainname: "" + layer: platform +resolvers: + - dnsServers: + - 8.8.8.8 + - 1.1.1.1 + layer: platform +timeServers: [] +operators: + - operator: dhcp4 + linkName: eth0 + requireUp: true + dhcp4: + routeMetric: 1024 + skipHostnameRequest: true + layer: platform + - operator: dhcp6 + linkName: eth2 + requireUp: true + dhcp6: + routeMetric: 2048 + skipHostnameRequest: true + layer: platform +externalIPs: + - 1.2.3.4 +metadata: + platform: openstack + hostname: talos + zone: nova + instanceId: 39073b0a-1234-1234-1234-5e76a4bd64b2 + providerId: openstack:///39073b0a-1234-1234-1234-5e76a4bd64b2 diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/openstack/testdata/metadata.json b/internal/app/machined/pkg/runtime/v1alpha1/platform/openstack/testdata/metadata.json new file mode 100644 index 0000000..3d3cd78 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/openstack/testdata/metadata.json @@ -0,0 +1,11 @@ +{ + "availability_zone": "nova", + "devices": [], + "hostname": "talos", + "keys": [], + "launch_index": 0, + "name": "talos", + "project_id": "39073b0a-1234-1234-1234-5e76a4bd64b2", + "public_keys": {}, + "uuid": "39073b0a-1234-1234-1234-5e76a4bd64b2" +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/openstack/testdata/network.json b/internal/app/machined/pkg/runtime/v1alpha1/platform/openstack/testdata/network.json new file mode 100644 index 0000000..cb850b8 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/openstack/testdata/network.json @@ -0,0 +1,153 @@ +{ + "links": [ + { + "ethernet_mac_address": "A4:BF:00:10:20:30", + "id": "aae16046-6c74-4f33-acf2-a16e9ab093eb", + "type": "phy", + "mtu": 1450, + "vif_id": "7607af2d-c24d-4bfb-909e-c447b119f4e2" + }, + { + "ethernet_mac_address": "A4:BF:00:10:20:31", + "id": "aae16046-6c74-4f33-acf2-a16e9ab093ec", + "type": "ovs", + "mtu": 9000, + "vif_id": "c816df7e-7bcc-45ca-9eb2-3d3d3dca0639" + }, + { + "ethernet_mac_address": "A4:BF:00:10:20:33", + "id": "aae16046-6c74-4f33-acf2-a16e9ab093ed", + "type": "vif", + "vif_id": "c816df7e-7bcc-45ca-9eb2-3d3d3dca063a" + }, + { + "id": "tap7819ff08-20", + "vif_id": "7819ff08-204b-4193-8c4d-1fff242932e5", + "type": "bond", + "mtu": 1500, + "ethernet_mac_address": "4c:d9:8f:b3:34:f7", + "bond_mode": "802.3ad", + "bond_links": [ + "411f3980-d8f9-4bf0-a09a-9c01c81e1022", + "83f59825-bf2d-4ea7-98be-edc772fe82de" + ], + "bond_miimon": "100", + "bond_xmit_hash_policy": "layer2+3" + }, + { + "id": "411f3980-d8f9-4bf0-a09a-9c01c81e1022", + "type": "bridge", + "ethernet_mac_address": "4c:d9:8f:b3:34:f8" + }, + { + "id": "83f59825-bf2d-4ea7-98be-edc772fe82de", + "type": "phy", + "ethernet_mac_address": "4c:d9:8f:b3:34:f7" + } + ], + "networks": [ + { + "id": "publicnet-ipv4", + "link": "aae16046-6c74-4f33-acf2-a16e9ab093eb", + "network_id": "66374c4d-5123-4f11-8fa9-8a6dea2b4fe7", + "type": "ipv4_dhcp" + }, + { + "routes": [ + { + "network": "2000:0:100:2f00::", + "gateway": "2000:0:100:2fff:ff:ff:ff:f0", + "netmask": "ffff:ffff:ffff:ffc0::" + } + ], + "dns_nameservers": [ + "2000:0:100::1" + ], + "gateway": "2000:0:100:2fff:ff:ff:ff:ff", + "link": "aae16046-6c74-4f33-acf2-a16e9ab093eb", + "ip_address": "2000:0:100::/56", + "network_id": "39b48637-d98a-4dfc-a05b-d61e8d88fafe", + "id": "publicnet-ipv6", + "type": "ipv6" + }, + { + "id": "privatnet-ipv4", + "link": "aae16046-6c74-4f33-acf2-a16e9ab093ec", + "network_id": "66374c4d-5123-4f11-8fa9-8a6dea2b4fe7", + "type": "ipv4", + "ip_address": "10.184.0.244", + "netmask": "255.255.240.0", + "routes": [ + { + "network": "192.168.0.0", + "netmask": "255.255.0.0", + "gateway": "10.184.0.1" + }, + { + "network": "0.0.0.0", + "netmask": "0.0.0.0", + "gateway": "10.184.0.1" + } + ] + }, + { + "id": "privatnet-ipv6", + "link": "aae16046-6c74-4f33-acf2-a16e9ab093ec", + "network_id": "66374c4d-5123-4f11-8fa9-8a6dea2b4fe7", + "type": "ipv6", + "ip_address": "2001:db8::3257:9652", + "netmask": "ffff:ffff:ffff:ffff::", + "routes": [ + { + "network": "::", + "netmask": "::", + "gateway": "fd00::1" + } + ] + }, + { + "id": "privatnet-ipv6-2", + "link": "aae16046-6c74-4f33-acf2-a16e9ab093ed", + "network_id": "66374c4d-5123-4f11-8fa9-8a6dea2b4fe7", + "type": "ipv6_dhcpv6-stateless", + "ip_address": "fd60:172:16:84:f816:3eff:fe73:5901", + "netmask": "ffff:ffff:ffff:ffff::", + "routes": [] + }, + { + "id": "network0", + "type": "ipv4", + "link": "tap7819ff08-20", + "ip_address": "94.156.45.48", + "netmask": "255.255.255.0", + "routes": [ + { + "network": "0.0.0.0", + "netmask": "0.0.0.0", + "gateway": "94.156.45.1" + } + ], + "network_id": "dc4b2e65-869b-46b1-b53b-538774dbdc16", + "services": [ + { + "type": "dns", + "address": "8.8.8.8" + }, + { + "type": "dns", + "address": "8.8.4.4" + } + ] + } + ], + "services": [ + { + "address": "8.8.8.8", + "type": "dns" + }, + { + "address": "1.1.1.1", + "type": "dns" + } + ] +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/oracle/metadata.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/oracle/metadata.go new file mode 100644 index 0000000..6e2150c --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/oracle/metadata.go @@ -0,0 +1,54 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package oracle + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/errors" + "github.com/siderolabs/talos/pkg/download" +) + +// Ref: https://docs.oracle.com/en-us/iaas/Content/Compute/Tasks/gettingmetadata.htm +const ( + // OracleMetadataEndpoint is the local metadata endpoint for the hostname. + OracleMetadataEndpoint = "http://169.254.169.254/opc/v2/instance/" + // OracleUserDataEndpoint is the local metadata endpoint inside of Oracle Cloud. + OracleUserDataEndpoint = "http://169.254.169.254/opc/v2/instance/metadata/user_data" + // OracleNetworkEndpoint is the local network metadata endpoint inside of Oracle Cloud. + OracleNetworkEndpoint = "http://169.254.169.254/opc/v2/vnics/" + + oracleResolverServer = "169.254.169.254" + oracleTimeServer = "169.254.169.254" +) + +// MetadataConfig represents a metadata Oracle instance. +type MetadataConfig struct { + Hostname string `json:"hostname,omitempty"` + ID string `json:"id,omitempty"` + Region string `json:"region,omitempty"` + AvailabilityDomain string `json:"availabilityDomain,omitempty"` + FaultDomain string `json:"faultDomain,omitempty"` + Shape string `json:"shape,omitempty"` +} + +func (o *Oracle) getMetadata(ctx context.Context) (*MetadataConfig, error) { + metaConfigDl, err := download.Download(ctx, OracleMetadataEndpoint, + download.WithHeaders(map[string]string{"Authorization": "Bearer Oracle"}), + download.WithErrorOnNotFound(errors.ErrNoHostname), + download.WithErrorOnEmptyResponse(errors.ErrNoHostname)) + if err != nil { + return nil, fmt.Errorf("error fetching metadata: %w", err) + } + + var meta MetadataConfig + if err = json.Unmarshal(metaConfigDl, &meta); err != nil { + return nil, err + } + + return &meta, nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/oracle/oracle.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/oracle/oracle.go new file mode 100644 index 0000000..9333e9e --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/oracle/oracle.go @@ -0,0 +1,205 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package oracle provides the Oracle platform implementation. +package oracle + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "log" + "net/netip" + "strings" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-procfs/procfs" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/errors" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/internal/netutils" + "github.com/siderolabs/talos/pkg/download" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// NetworkConfig holds network interface meta config. +type NetworkConfig struct { + HWAddr string `json:"macAddr"` + PrivateIP string `json:"privateIp"` + VirtualRouterIP string `json:"virtualRouterIp"` + SubnetCidrBlock string `json:"subnetCidrBlock"` + Ipv6SubnetCidrBlock string `json:"ipv6SubnetCidrBlock,omitempty"` + Ipv6VirtualRouterIP string `json:"ipv6VirtualRouterIp,omitempty"` + Ipv6Addresses []string `json:"ipv6Addresses,omitempty"` +} + +// Oracle is the concrete type that implements the platform.Platform interface. +type Oracle struct{} + +// Name implements the platform.Platform interface. +func (o *Oracle) Name() string { + return "oracle" +} + +// ParseMetadata converts Oracle Cloud metadata into platform network configuration. +func (o *Oracle) ParseMetadata(interfaceAddresses []NetworkConfig, metadata *MetadataConfig) (*runtime.PlatformNetworkConfig, error) { + networkConfig := &runtime.PlatformNetworkConfig{} + + if metadata.Hostname != "" { + hostnameSpec := network.HostnameSpecSpec{ + ConfigLayer: network.ConfigPlatform, + } + + if err := hostnameSpec.ParseFQDN(metadata.Hostname); err != nil { + return nil, err + } + + networkConfig.Hostnames = append(networkConfig.Hostnames, hostnameSpec) + } + + for idx, iface := range interfaceAddresses { + ifname := fmt.Sprintf("eth%d", idx) + + if iface.Ipv6SubnetCidrBlock != "" && iface.Ipv6VirtualRouterIP != "" { + networkConfig.Operators = append(networkConfig.Operators, network.OperatorSpecSpec{ + Operator: network.OperatorDHCP6, + LinkName: ifname, + RequireUp: true, + DHCP6: network.DHCP6OperatorSpec{ + RouteMetric: network.DefaultRouteMetric, + }, + ConfigLayer: network.ConfigPlatform, + }) + + gw, err := netip.ParseAddr(iface.Ipv6VirtualRouterIP) + if err != nil { + return nil, err + } + + route := network.RouteSpecSpec{ + ConfigLayer: network.ConfigPlatform, + Gateway: gw, + OutLinkName: ifname, + Table: nethelpers.TableMain, + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + Family: nethelpers.FamilyInet6, + Priority: 2 * network.DefaultRouteMetric, + } + + route.Normalize() + + networkConfig.Routes = append(networkConfig.Routes, route) + } + } + + dns, _ := netip.ParseAddr(oracleResolverServer) //nolint:errcheck + + networkConfig.Resolvers = append(networkConfig.Resolvers, network.ResolverSpecSpec{ + DNSServers: []netip.Addr{dns}, + ConfigLayer: network.ConfigPlatform, + }) + + networkConfig.TimeServers = append(networkConfig.TimeServers, network.TimeServerSpecSpec{ + NTPServers: []string{oracleTimeServer}, + ConfigLayer: network.ConfigPlatform, + }) + + zone := metadata.AvailabilityDomain + + if idx := strings.LastIndex(zone, ":"); idx != -1 { + zone = zone[idx+1:] + } + + networkConfig.Metadata = &runtimeres.PlatformMetadataSpec{ + Platform: o.Name(), + Hostname: metadata.Hostname, + Region: metadata.Region, + Zone: zone, + InstanceType: metadata.Shape, + InstanceID: metadata.ID, + ProviderID: fmt.Sprintf("oci://%s", metadata.ID), + } + + return networkConfig, nil +} + +// Configuration implements the platform.Platform interface. +func (o *Oracle) Configuration(ctx context.Context, r state.State) ([]byte, error) { + if err := netutils.Wait(ctx, r); err != nil { + return nil, err + } + + log.Printf("fetching machine config from: %q", OracleUserDataEndpoint) + + machineConfigDl, err := download.Download(ctx, OracleUserDataEndpoint, + download.WithHeaders(map[string]string{"Authorization": "Bearer Oracle"}), + download.WithErrorOnNotFound(errors.ErrNoConfigSource), + download.WithErrorOnEmptyResponse(errors.ErrNoConfigSource)) + if err != nil { + return nil, err + } + + machineConfig, err := base64.StdEncoding.DecodeString(string(machineConfigDl)) + if err != nil { + return nil, errors.ErrNoConfigSource + } + + return machineConfig, nil +} + +// Mode implements the platform.Platform interface. +func (o *Oracle) Mode() runtime.Mode { + return runtime.ModeCloud +} + +// KernelArgs implements the runtime.Platform interface. +func (o *Oracle) KernelArgs(string) procfs.Parameters { + return []*procfs.Parameter{ + procfs.NewParameter("console").Append("tty1").Append("ttyS0"), + procfs.NewParameter(constants.KernelParamNetIfnames).Append("0"), + procfs.NewParameter(constants.KernelParamDashboardDisabled).Append("1"), + } +} + +// NetworkConfiguration implements the runtime.Platform interface. +func (o *Oracle) NetworkConfiguration(ctx context.Context, _ state.State, ch chan<- *runtime.PlatformNetworkConfig) error { + log.Printf("fetching oracle metadata from: %q", OracleMetadataEndpoint) + + metadata, err := o.getMetadata(ctx) + if err != nil { + return err + } + + log.Printf("fetching network config from %q", OracleNetworkEndpoint) + + metadataNetworkConfigDl, err := download.Download(ctx, OracleNetworkEndpoint, + download.WithHeaders(map[string]string{"Authorization": "Bearer Oracle"})) + if err != nil { + return fmt.Errorf("failed to fetch network config from: %w", err) + } + + var interfaceAddresses []NetworkConfig + + if err = json.Unmarshal(metadataNetworkConfigDl, &interfaceAddresses); err != nil { + return err + } + + networkConfig, err := o.ParseMetadata(interfaceAddresses, metadata) + if err != nil { + return err + } + + select { + case ch <- networkConfig: + case <-ctx.Done(): + return ctx.Err() + } + + return nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/oracle/oracle_test.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/oracle/oracle_test.go new file mode 100644 index 0000000..0a8e38e --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/oracle/oracle_test.go @@ -0,0 +1,46 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package oracle_test + +import ( + _ "embed" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/oracle" +) + +//go:embed testdata/metadata.json +var rawMetadata []byte + +//go:embed testdata/metadatanetwork.json +var rawMetadataNetwork []byte + +//go:embed testdata/expected.yaml +var expectedNetworkConfig string + +func TestParseMetadata(t *testing.T) { + p := &oracle.Oracle{} + + var metadata oracle.MetadataConfig + + require.NoError(t, json.Unmarshal(rawMetadata, &metadata)) + + var m []oracle.NetworkConfig + + require.NoError(t, json.Unmarshal(rawMetadataNetwork, &m)) + + networkConfig, err := p.ParseMetadata(m, &metadata) + require.NoError(t, err) + + marshaled, err := yaml.Marshal(networkConfig) + require.NoError(t, err) + + assert.Equal(t, expectedNetworkConfig, string(marshaled)) +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/oracle/testdata/expected.yaml b/internal/app/machined/pkg/runtime/v1alpha1/platform/oracle/testdata/expected.yaml new file mode 100644 index 0000000..f68ebae --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/oracle/testdata/expected.yaml @@ -0,0 +1,43 @@ +addresses: [] +links: [] +routes: + - family: inet6 + dst: "" + src: "" + gateway: fe80::a:b:c:d + outLinkName: eth0 + table: main + priority: 2048 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform +hostnames: + - hostname: talos + domainname: "" + layer: platform +resolvers: + - dnsServers: + - 169.254.169.254 + layer: platform +timeServers: + - timeServers: + - 169.254.169.254 + layer: platform +operators: + - operator: dhcp6 + linkName: eth0 + requireUp: true + dhcp6: + routeMetric: 1024 + layer: platform +externalIPs: [] +metadata: + platform: oracle + hostname: talos + region: phx + zone: PHX-AD-1 + instanceType: VM.Standard.E3.Flex + instanceId: ocid1.instance.oc1.phx.exampleuniqueID + providerId: oci://ocid1.instance.oc1.phx.exampleuniqueID diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/oracle/testdata/metadata.json b/internal/app/machined/pkg/runtime/v1alpha1/platform/oracle/testdata/metadata.json new file mode 100644 index 0000000..37ebee9 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/oracle/testdata/metadata.json @@ -0,0 +1,53 @@ +{ + "availabilityDomain": "EMIr:PHX-AD-1", + "faultDomain": "FAULT-DOMAIN-3", + "compartmentId": "ocid1.tenancy.oc1..exampleuniqueID", + "displayName": "talos-instance", + "hostname": "talos", + "id": "ocid1.instance.oc1.phx.exampleuniqueID", + "image": "ocid1.image.oc1.phx.exampleuniqueID", + "metadata": {}, + "region": "phx", + "canonicalRegionName": "us-phoenix-1", + "ociAdName": "phx-ad-1", + "regionInfo": { + "realmKey": "oc1", + "realmDomainComponent": "oraclecloud.com", + "regionKey": "PHX", + "regionIdentifier": "us-phoenix-1" + }, + "shape": "VM.Standard.E3.Flex", + "state": "Running", + "timeCreated": 1600381928581, + "agentConfig": { + "monitoringDisabled": false, + "managementDisabled": false, + "allPluginsDisabled": false, + "pluginsConfig": [ + { + "name": "OS Management Service Agent", + "desiredState": "ENABLED" + }, + { + "name": "Custom Logs Monitoring", + "desiredState": "ENABLED" + }, + { + "name": "Compute Instance Run Command", + "desiredState": "ENABLED" + }, + { + "name": "Compute Instance Monitoring", + "desiredState": "ENABLED" + } + ] + }, + "freeformTags": { + "Department": "Finance" + }, + "definedTags": { + "Operations": { + "CostCenter": "42" + } + } +} \ No newline at end of file diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/oracle/testdata/metadatanetwork.json b/internal/app/machined/pkg/runtime/v1alpha1/platform/oracle/testdata/metadatanetwork.json new file mode 100644 index 0000000..be56c48 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/oracle/testdata/metadatanetwork.json @@ -0,0 +1,15 @@ +[ + { + "vnicId": "ocid1.vnic.oc1.eu-amsterdam-1.asdasd", + "privateIp": "172.16.1.11", + "vlanTag": 1, + "macAddr": "02:00:17:00:00:00", + "virtualRouterIp": "172.16.1.1", + "subnetCidrBlock": "172.16.1.0/24", + "ipv6SubnetCidrBlock": "2603:a:b:c::/64", + "ipv6VirtualRouterIp": "fe80::a:b:c:d", + "ipv6Addresses": [ + "2603:a:b:c::1234" + ] + } +] \ No newline at end of file diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/platform.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/platform.go new file mode 100644 index 0000000..dafcedd --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/platform.go @@ -0,0 +1,162 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package platform provides functions to get the [runtime.Platform]. +package platform + +import ( + "context" + "errors" + "fmt" + "log" + "os" + + "github.com/siderolabs/go-procfs/procfs" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/akamai" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/aws" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/azure" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/container" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/equinixmetal" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/exoscale" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/gcp" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/hcloud" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/metal" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/opennebula" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/openstack" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/oracle" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/scaleway" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/upcloud" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/vmware" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/vultr" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// Event is a struct used below in FireEvent +// in hopes that we can reuse some of this eventing in other platforms if possible. +type Event struct { + Type string + Message string + Error error +} + +// nb: these events currently map to those expected by +// equinix metal. if/when we do other platforms, we should +// maybe generalize this and map the events inside each platform. +const ( + // EventTypeActivate is the activate event string. + EventTypeActivate = "activate" + // EventTypeFailure is the failure event string. + EventTypeFailure = "failure" + // EventTypeInfo is the info event string. + EventTypeInfo = "info" + // EventTypeConfigLoaded is the config loaded event string. + EventTypeConfigLoaded = "talos.prov.config.loaded" + // EventTypeRebooted is the reboot event string. + EventTypeRebooted = "talos.prov.host.rebooted" + // EventTypeInstalled is the installation event string. + EventTypeInstalled = "talos.prov.os.installed" + // EventTypeUpgraded is the upgrade event string. + EventTypeUpgraded = "talos.prov.os.upgraded" +) + +// CurrentPlatform is a helper func for discovering the current platform. +func CurrentPlatform() (p runtime.Platform, err error) { + var platform string + + if p := procfs.ProcCmdline().Get(constants.KernelParamPlatform).First(); p != nil { + platform = *p + } + + if p, ok := os.LookupEnv("PLATFORM"); ok { + platform = p + } + + if platform == "" { + return nil, errors.New("failed to determine platform") + } + + return newPlatform(platform) +} + +// NewPlatform initializes and returns a runtime.Platform. +func NewPlatform(platform string) (p runtime.Platform, err error) { + return newPlatform(platform) +} + +//nolint:gocyclo +func newPlatform(platform string) (p runtime.Platform, err error) { + switch platform { + case "akamai": + p = &akamai.Akamai{} + case "aws": + return aws.NewAWS() + case "azure": + p = &azure.Azure{} + case "container": + p = &container.Container{} + case "digital-ocean": + p = &digitalocean.DigitalOcean{} + case "gcp": + p = &gcp.GCP{} + case "hcloud": + p = &hcloud.Hcloud{} + case constants.PlatformMetal: + p = &metal.Metal{} + case "opennebula": + p = &opennebula.OpenNebula{} + case "openstack": + p = &openstack.Openstack{} + case "oracle": + p = &oracle.Oracle{} + case "nocloud": + p = &nocloud.Nocloud{} + // "packet" kept for backwards compatibility + case "equinixMetal", "packet": + p = &equinixmetal.EquinixMetal{} + case "exoscale": + p = &exoscale.Exoscale{} + case "scaleway": + p = &scaleway.Scaleway{} + case "upcloud": + p = &upcloud.UpCloud{} + case "vmware": + p = &vmware.VMware{} + case "vultr": + p = &vultr.Vultr{} + default: + return nil, fmt.Errorf("unknown platform: %q", platform) + } + + return p, nil +} + +// FireEvent will call the implemented platform's event function if we know it has one. +// Error logging is handled in this function and we don't return any error values to the sequencer. +func FireEvent(ctx context.Context, p runtime.Platform, e Event) { + switch platType := p.(type) { + case *equinixmetal.EquinixMetal: + emEvent := equinixmetal.Event{ + Type: e.Type, + Message: e.Message, + } + + if e.Error != nil { + emEvent.Message = fmt.Sprintf("%s: %s", e.Message, e.Error) + } + + eventErr := platType.FireEvent(ctx, emEvent) + + if eventErr != nil { + log.Printf("failed sending event: %s", eventErr) + } + + default: + // Treat anything else as a no-op b/c we don't support event firing + return + } +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/scaleway/metadata.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/scaleway/metadata.go new file mode 100644 index 0000000..a135f7a --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/scaleway/metadata.go @@ -0,0 +1,36 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package scaleway + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/scaleway/scaleway-sdk-go/api/instance/v1" + + "github.com/siderolabs/talos/pkg/download" +) + +const ( + // ScalewayMetadataEndpoint is the local Scaleway endpoint. + ScalewayMetadataEndpoint = "http://169.254.42.42/conf?format=json" + // ScalewayUserDataEndpoint is the local Scaleway endpoint for the config. + ScalewayUserDataEndpoint = "http://169.254.42.42/user_data/cloud-init" +) + +func (u *Scaleway) getMetadata(ctx context.Context) (*instance.Metadata, error) { + metaConfigDl, err := download.Download(ctx, ScalewayMetadataEndpoint) + if err != nil { + return nil, fmt.Errorf("error fetching metadata: %w", err) + } + + var meta instance.Metadata + if err = json.Unmarshal(metaConfigDl, &meta); err != nil { + return nil, err + } + + return &meta, nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/scaleway/scaleway.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/scaleway/scaleway.go new file mode 100644 index 0000000..a2219dd --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/scaleway/scaleway.go @@ -0,0 +1,214 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package scaleway provides the Scaleway platform implementation. +package scaleway + +import ( + "context" + "fmt" + "log" + "net/netip" + "strconv" + "strings" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/scaleway/scaleway-sdk-go/api/instance/v1" + "github.com/siderolabs/go-procfs/procfs" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/errors" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/internal/netutils" + "github.com/siderolabs/talos/pkg/download" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// Scaleway is the concrete type that implements the runtime.Platform interface. +type Scaleway struct{} + +// Name implements the runtime.Platform interface. +func (s *Scaleway) Name() string { + return "scaleway" +} + +// ParseMetadata converts Scaleway platform metadata into platform network config. +// +//nolint:gocyclo +func (s *Scaleway) ParseMetadata(metadata *instance.Metadata) (*runtime.PlatformNetworkConfig, error) { + networkConfig := &runtime.PlatformNetworkConfig{} + + if metadata.Hostname != "" { + hostnameSpec := network.HostnameSpecSpec{ + ConfigLayer: network.ConfigPlatform, + } + + if err := hostnameSpec.ParseFQDN(metadata.Hostname); err != nil { + return nil, err + } + + networkConfig.Hostnames = append(networkConfig.Hostnames, hostnameSpec) + } + + publicIPs := []string{} + + if metadata.PublicIP.Address != "" { + publicIPs = append(publicIPs, metadata.PublicIP.Address) + } + + networkConfig.Links = append(networkConfig.Links, network.LinkSpecSpec{ + Name: "eth0", + Up: true, + ConfigLayer: network.ConfigPlatform, + }) + + gw, _ := netip.ParsePrefix("169.254.42.42/32") //nolint:errcheck + route := network.RouteSpecSpec{ + ConfigLayer: network.ConfigPlatform, + OutLinkName: "eth0", + Destination: gw, + Table: nethelpers.TableMain, + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + Family: nethelpers.FamilyInet4, + Priority: network.DefaultRouteMetric, + } + + route.Normalize() + networkConfig.Routes = []network.RouteSpecSpec{route} + + networkConfig.Operators = append(networkConfig.Operators, network.OperatorSpecSpec{ + Operator: network.OperatorDHCP4, + LinkName: "eth0", + RequireUp: true, + DHCP4: network.DHCP4OperatorSpec{ + RouteMetric: network.DefaultRouteMetric, + }, + ConfigLayer: network.ConfigPlatform, + }) + + if metadata.IPv6.Address != "" { + bits, err := strconv.Atoi(metadata.IPv6.Netmask) + if err != nil { + return nil, err + } + + ip, err := netip.ParseAddr(metadata.IPv6.Address) + if err != nil { + return nil, err + } + + addr := netip.PrefixFrom(ip, bits) + + publicIPs = append(publicIPs, metadata.IPv6.Address) + networkConfig.Addresses = append(networkConfig.Addresses, + network.AddressSpecSpec{ + ConfigLayer: network.ConfigPlatform, + LinkName: "eth0", + Address: addr, + Scope: nethelpers.ScopeGlobal, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + Family: nethelpers.FamilyInet6, + }, + ) + + gw, err := netip.ParseAddr(metadata.IPv6.Gateway) + if err != nil { + return nil, err + } + + route := network.RouteSpecSpec{ + ConfigLayer: network.ConfigPlatform, + Gateway: gw, + OutLinkName: "eth0", + Table: nethelpers.TableMain, + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + Family: nethelpers.FamilyInet6, + Priority: 2 * network.DefaultRouteMetric, + } + + route.Normalize() + + networkConfig.Routes = append(networkConfig.Routes, route) + } + + zoneParts := strings.Split(metadata.Location.ZoneID, "-") + if len(zoneParts) > 2 { + zoneParts = zoneParts[:2] + } + + for _, ipStr := range publicIPs { + if ip, err := netip.ParseAddr(ipStr); err == nil { + networkConfig.ExternalIPs = append(networkConfig.ExternalIPs, ip) + } + } + + networkConfig.Metadata = &runtimeres.PlatformMetadataSpec{ + Platform: s.Name(), + Hostname: metadata.Hostname, + Region: strings.Join(zoneParts, "-"), + Zone: metadata.Location.ZoneID, + InstanceType: metadata.CommercialType, + InstanceID: metadata.ID, + ProviderID: fmt.Sprintf("scaleway://instance/%s/%s", metadata.Location.ZoneID, metadata.ID), + } + + return networkConfig, nil +} + +// Configuration implements the runtime.Platform interface. +// +//nolint:stylecheck +func (s *Scaleway) Configuration(ctx context.Context, r state.State) ([]byte, error) { + if err := netutils.Wait(ctx, r); err != nil { + return nil, err + } + + log.Printf("fetching machine config from %q", ScalewayUserDataEndpoint) + + return download.Download(ctx, ScalewayUserDataEndpoint, + download.WithLowSrcPort(), + download.WithErrorOnNotFound(errors.ErrNoConfigSource), + download.WithErrorOnEmptyResponse(errors.ErrNoConfigSource)) +} + +// Mode implements the runtime.Platform interface. +func (s *Scaleway) Mode() runtime.Mode { + return runtime.ModeCloud +} + +// KernelArgs implements the runtime.Platform interface. +func (s *Scaleway) KernelArgs(string) procfs.Parameters { + return []*procfs.Parameter{ + procfs.NewParameter("console").Append("tty1").Append("ttyS0"), + procfs.NewParameter(constants.KernelParamNetIfnames).Append("0"), + procfs.NewParameter(constants.KernelParamDashboardDisabled).Append("1"), + } +} + +// NetworkConfiguration implements the runtime.Platform interface. +func (s *Scaleway) NetworkConfiguration(ctx context.Context, _ state.State, ch chan<- *runtime.PlatformNetworkConfig) error { + log.Printf("fetching scaleway instance config from: %q", ScalewayMetadataEndpoint) + + metadata, err := s.getMetadata(ctx) + if err != nil { + return err + } + + networkConfig, err := s.ParseMetadata(metadata) + if err != nil { + return err + } + + select { + case ch <- networkConfig: + case <-ctx.Done(): + return ctx.Err() + } + + return nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/scaleway/scaleway_test.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/scaleway/scaleway_test.go new file mode 100644 index 0000000..2a05f91 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/scaleway/scaleway_test.go @@ -0,0 +1,40 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package scaleway_test + +import ( + _ "embed" + "encoding/json" + "testing" + + "github.com/scaleway/scaleway-sdk-go/api/instance/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/scaleway" +) + +//go:embed testdata/metadata.json +var rawMetadata []byte + +//go:embed testdata/expected.yaml +var expectedNetworkConfig string + +func TestParseMetadata(t *testing.T) { + p := &scaleway.Scaleway{} + + var metadata instance.Metadata + + require.NoError(t, json.Unmarshal(rawMetadata, &metadata)) + + networkConfig, err := p.ParseMetadata(&metadata) + require.NoError(t, err) + + marshaled, err := yaml.Marshal(networkConfig) + require.NoError(t, err) + + assert.Equal(t, expectedNetworkConfig, string(marshaled)) +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/scaleway/testdata/expected.yaml b/internal/app/machined/pkg/runtime/v1alpha1/platform/scaleway/testdata/expected.yaml new file mode 100644 index 0000000..61def78 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/scaleway/testdata/expected.yaml @@ -0,0 +1,64 @@ +addresses: + - address: 2001:111:222:3333::1/64 + linkName: eth0 + family: inet6 + scope: global + flags: permanent + layer: platform +links: + - name: eth0 + logical: false + up: true + mtu: 0 + kind: "" + type: netrom + layer: platform +routes: + - family: inet4 + dst: 169.254.42.42/32 + src: "" + gateway: "" + outLinkName: eth0 + table: main + priority: 1024 + scope: link + type: unicast + flags: "" + protocol: static + layer: platform + - family: inet6 + dst: "" + src: "" + gateway: '2001:111:222:3333::' + outLinkName: eth0 + table: main + priority: 2048 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform +hostnames: + - hostname: scw-talos + domainname: "" + layer: platform +resolvers: [] +timeServers: [] +operators: + - operator: dhcp4 + linkName: eth0 + requireUp: true + dhcp4: + routeMetric: 1024 + layer: platform +externalIPs: + - 11.22.222.222 + - 2001:111:222:3333::1 +metadata: + platform: scaleway + hostname: scw-talos + region: zone-name + zone: zone-name-1 + instanceType: DEV1-S + instanceId: 11111111-1111-1111-1111-111111111111 + providerId: scaleway://instance/zone-name-1/11111111-1111-1111-1111-111111111111 diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/scaleway/testdata/metadata.json b/internal/app/machined/pkg/runtime/v1alpha1/platform/scaleway/testdata/metadata.json new file mode 100644 index 0000000..b7bc97a --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/scaleway/testdata/metadata.json @@ -0,0 +1,22 @@ +{ + "id": "11111111-1111-1111-1111-111111111111", + "name": "scw-talos", + "commercial_type": "DEV1-S", + "hostname": "scw-talos", + "tags": [], + "state_detail": "booted", + "public_ip": { + "id": "11111111-1111-1111-1111-111111111111", + "address": "11.22.222.222", + "dynamic": false + }, + "private_ip": "10.00.222.222", + "ipv6": { + "address": "2001:111:222:3333::1", + "gateway": "2001:111:222:3333::", + "netmask": "64" + }, + "location": { + "zone_id": "zone-name-1" + } +} \ No newline at end of file diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/upcloud/metadata.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/upcloud/metadata.go new file mode 100644 index 0000000..c9c1358 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/upcloud/metadata.go @@ -0,0 +1,63 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package upcloud + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/siderolabs/talos/pkg/download" +) + +const ( + // UpCloudMetadataEndpoint is the local UpCloud endpoint. + UpCloudMetadataEndpoint = "http://169.254.169.254/metadata/v1.json" + + // UpCloudUserDataEndpoint is the local UpCloud endpoint for the config. + UpCloudUserDataEndpoint = "http://169.254.169.254/metadata/v1/user_data" +) + +// MetadataConfig represents a metadata Upcloud instance. +type MetadataConfig struct { + Hostname string `json:"hostname,omitempty"` + InstanceID string `json:"instance_id,omitempty"` + PublicKeys []string `json:"public_keys,omitempty"` + Zone string `json:"region,omitempty"` + Tags []string `json:"tags,omitempty"` + + Network struct { + Interfaces []struct { + Index int `json:"index,omitempty"` + IPAddresses []struct { + Address string `json:"address,omitempty"` + DHCP bool `json:"dhcp,omitempty"` + DNS []string `json:"dns,omitempty"` + Family string `json:"family,omitempty"` + Floating bool `json:"floating,omitempty"` + Gateway string `json:"gateway,omitempty"` + Network string `json:"network,omitempty"` + } `json:"ip_addresses,omitempty"` + MAC string `json:"mac,omitempty"` + NetworkType string `json:"type,omitempty"` + NetworkID string `json:"network_id,omitempty"` + } `json:"interfaces,omitempty"` + DNS []string `json:"dns,omitempty"` + } `json:"network,omitempty"` +} + +func (u *UpCloud) getMetadata(ctx context.Context) (*MetadataConfig, error) { + metaConfigDl, err := download.Download(ctx, UpCloudMetadataEndpoint) + if err != nil { + return nil, fmt.Errorf("error fetching metadata: %w", err) + } + + var meta MetadataConfig + if err = json.Unmarshal(metaConfigDl, &meta); err != nil { + return nil, err + } + + return &meta, nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/upcloud/testdata/expected.yaml b/internal/app/machined/pkg/runtime/v1alpha1/platform/upcloud/testdata/expected.yaml new file mode 100644 index 0000000..6a962b9 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/upcloud/testdata/expected.yaml @@ -0,0 +1,81 @@ +addresses: + - address: 185.70.197.3/32 + linkName: eth0 + family: inet4 + scope: global + flags: permanent + layer: platform + - address: 2a04:3544:8000:1000:0:1111:2222:3333/64 + linkName: eth2 + family: inet6 + scope: global + flags: permanent + layer: platform +links: + - name: eth0 + logical: false + up: true + mtu: 0 + kind: "" + type: netrom + layer: platform + - name: eth1 + logical: false + up: true + mtu: 0 + kind: "" + type: netrom + layer: platform + - name: eth2 + logical: false + up: true + mtu: 0 + kind: "" + type: netrom + layer: platform +routes: + - family: inet6 + dst: 2a04:3544:8000:1000::/64 + src: "" + gateway: 2a04:3544:8000:1000::1 + outLinkName: eth2 + table: main + priority: 1024 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform +hostnames: + - hostname: talos + domainname: "" + layer: platform +resolvers: + - dnsServers: + - 94.237.127.9 + - 94.237.40.9 + - 2a04:3540:53::1 + - 2a04:3544:53::1 + layer: platform +timeServers: [] +operators: + - operator: dhcp4 + linkName: eth0 + requireUp: true + dhcp4: + routeMetric: 1024 + layer: platform + - operator: dhcp4 + linkName: eth1 + requireUp: true + dhcp4: + routeMetric: 1024 + layer: platform +externalIPs: + - 185.70.197.2 + - 2a04:3544:8000:1000:0:1111:2222:3333 +metadata: + platform: upcloud + hostname: talos + instanceId: 00123456-1111-2222-3333-123456789012 + providerId: upcloud://00123456-1111-2222-3333-123456789012 diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/upcloud/testdata/metadata.json b/internal/app/machined/pkg/runtime/v1alpha1/platform/upcloud/testdata/metadata.json new file mode 100644 index 0000000..688be61 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/upcloud/testdata/metadata.json @@ -0,0 +1,83 @@ +{ + "cloud_name": "upcloud", + "instance_id": "00123456-1111-2222-3333-123456789012", + "hostname": "talos", + "network": { + "interfaces": [ + { + "index": 1, + "ip_addresses": [ + { + "address": "185.70.197.2", + "dhcp": true, + "dns": [ + "94.237.127.9", + "94.237.40.9" + ], + "family": "IPv4", + "floating": false, + "gateway": "185.70.196.1", + "network": "185.70.196.0/22" + }, + { + "address": "185.70.197.3", + "dhcp": false, + "dns": null, + "family": "IPv4", + "floating": true, + "gateway": "", + "network": "185.70.197.3/32" + } + ], + "mac": "5e:bf:5e:02:28:07", + "network_id": "035ef879-1111-2222-3333-123456789012", + "type": "public" + }, + { + "index": 2, + "ip_addresses": [ + { + "address": "10.11.0.2", + "dhcp": true, + "dns": null, + "family": "IPv4", + "floating": false, + "gateway": "10.11.0.1", + "network": "10.11.0.0/22" + } + ], + "mac": "5e:bf:5e:02:cd:e0", + "network_id": "031c9f9c-1111-2222-3333-123456789012", + "type": "utility" + }, + { + "index": 3, + "ip_addresses": [ + { + "address": "2a04:3544:8000:1000:0000:1111:2222:3333", + "dhcp": false, + "dns": [ + "2a04:3540:53::1", + "2a04:3544:53::1" + ], + "family": "IPv6", + "floating": false, + "gateway": "2a04:3544:8000:1000::1", + "network": "2a04:3544:8000:1000::/64" + } + ], + "mac": "5e:bf:5e:02:78:a4", + "network_id": "03b326a2-1111-2222-3333-123456789012", + "type": "public" + } + ], + "dns": [ + "94.237.127.9", + "94.237.40.9" + ] + }, + "storage": {}, + "tags": [], + "user_data": "", + "vendor_data": "" +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/upcloud/upcloud.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/upcloud/upcloud.go new file mode 100644 index 0000000..2b448b9 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/upcloud/upcloud.go @@ -0,0 +1,225 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package upcloud provides the UpCloud platform implementation. +package upcloud + +import ( + "context" + "fmt" + "log" + "net/netip" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-procfs/procfs" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/errors" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/internal/netutils" + "github.com/siderolabs/talos/pkg/download" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// UpCloud is the concrete type that implements the runtime.Platform interface. +type UpCloud struct{} + +// Name implements the runtime.Platform interface. +func (u *UpCloud) Name() string { + return "upcloud" +} + +// ParseMetadata converts Upcloud metadata into platform network configuration. +// +//nolint:gocyclo +func (u *UpCloud) ParseMetadata(metadata *MetadataConfig) (*runtime.PlatformNetworkConfig, error) { + networkConfig := &runtime.PlatformNetworkConfig{} + + if metadata.Hostname != "" { + hostnameSpec := network.HostnameSpecSpec{ + ConfigLayer: network.ConfigPlatform, + } + + if err := hostnameSpec.ParseFQDN(metadata.Hostname); err != nil { + return nil, err + } + + networkConfig.Hostnames = append(networkConfig.Hostnames, hostnameSpec) + } + + var ( + publicIPs []string + dnsIPs []netip.Addr + ) + + firstIP := true + + for _, addr := range metadata.Network.Interfaces { + if addr.Index <= 0 { // protect from negative interface name + continue + } + + iface := fmt.Sprintf("eth%d", addr.Index-1) + + networkConfig.Links = append(networkConfig.Links, network.LinkSpecSpec{ + Name: iface, + Up: true, + ConfigLayer: network.ConfigPlatform, + }) + + for _, ip := range addr.IPAddresses { + if firstIP { + publicIPs = append(publicIPs, ip.Address) + + firstIP = false + } + + for _, addr := range ip.DNS { + if ipAddr, err := netip.ParseAddr(addr); err == nil { + dnsIPs = append(dnsIPs, ipAddr) + } + } + + if ip.DHCP && ip.Family == "IPv4" { + networkConfig.Operators = append(networkConfig.Operators, network.OperatorSpecSpec{ + Operator: network.OperatorDHCP4, + LinkName: iface, + RequireUp: true, + DHCP4: network.DHCP4OperatorSpec{ + RouteMetric: network.DefaultRouteMetric, + }, + ConfigLayer: network.ConfigPlatform, + }) + } + + if !ip.DHCP { + ntwrk, err := netip.ParsePrefix(ip.Network) + if err != nil { + return nil, err + } + + addr, err := netip.ParseAddr(ip.Address) + if err != nil { + return nil, err + } + + ipPrefix := netip.PrefixFrom(addr, ntwrk.Bits()) + + family := nethelpers.FamilyInet4 + + if addr.Is6() { + publicIPs = append(publicIPs, ip.Address) + family = nethelpers.FamilyInet6 + } + + networkConfig.Addresses = append(networkConfig.Addresses, + network.AddressSpecSpec{ + ConfigLayer: network.ConfigPlatform, + LinkName: iface, + Address: ipPrefix, + Scope: nethelpers.ScopeGlobal, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + Family: family, + }, + ) + + if ip.Gateway != "" { + gw, err := netip.ParseAddr(ip.Gateway) + if err != nil { + return nil, err + } + + route := network.RouteSpecSpec{ + ConfigLayer: network.ConfigPlatform, + Gateway: gw, + Destination: ntwrk, + OutLinkName: iface, + Table: nethelpers.TableMain, + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + Family: family, + Priority: network.DefaultRouteMetric, + } + + route.Normalize() + + networkConfig.Routes = append(networkConfig.Routes, route) + } + } + } + } + + if len(dnsIPs) > 0 { + networkConfig.Resolvers = append(networkConfig.Resolvers, network.ResolverSpecSpec{ + DNSServers: dnsIPs, + ConfigLayer: network.ConfigPlatform, + }) + } + + for _, ipStr := range publicIPs { + if ip, err := netip.ParseAddr(ipStr); err == nil { + networkConfig.ExternalIPs = append(networkConfig.ExternalIPs, ip) + } + } + + networkConfig.Metadata = &runtimeres.PlatformMetadataSpec{ + Platform: u.Name(), + Hostname: metadata.Hostname, + Zone: metadata.Zone, + InstanceID: metadata.InstanceID, + ProviderID: fmt.Sprintf("upcloud://%s", metadata.InstanceID), + } + + return networkConfig, nil +} + +// Configuration implements the runtime.Platform interface. +func (u *UpCloud) Configuration(ctx context.Context, r state.State) ([]byte, error) { + if err := netutils.Wait(ctx, r); err != nil { + return nil, err + } + + log.Printf("fetching machine config from: %q", UpCloudUserDataEndpoint) + + return download.Download(ctx, UpCloudUserDataEndpoint, + download.WithErrorOnNotFound(errors.ErrNoConfigSource), + download.WithErrorOnEmptyResponse(errors.ErrNoConfigSource)) +} + +// Mode implements the runtime.Platform interface. +func (u *UpCloud) Mode() runtime.Mode { + return runtime.ModeCloud +} + +// KernelArgs implements the runtime.Platform interface. +func (u *UpCloud) KernelArgs(string) procfs.Parameters { + return []*procfs.Parameter{ + procfs.NewParameter(constants.KernelParamNetIfnames).Append("0"), + } +} + +// NetworkConfiguration implements the runtime.Platform interface. +func (u *UpCloud) NetworkConfiguration(ctx context.Context, _ state.State, ch chan<- *runtime.PlatformNetworkConfig) error { + log.Printf("fetching UpCloud instance config from: %q", UpCloudMetadataEndpoint) + + metadata, err := u.getMetadata(ctx) + if err != nil { + return err + } + + networkConfig, err := u.ParseMetadata(metadata) + if err != nil { + return err + } + + select { + case ch <- networkConfig: + case <-ctx.Done(): + return ctx.Err() + } + + return nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/upcloud/upcloud_test.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/upcloud/upcloud_test.go new file mode 100644 index 0000000..3ce63f0 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/upcloud/upcloud_test.go @@ -0,0 +1,39 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package upcloud_test + +import ( + _ "embed" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/upcloud" +) + +//go:embed testdata/metadata.json +var rawMetadata []byte + +//go:embed testdata/expected.yaml +var expectedNetworkConfig string + +func TestParseMetadata(t *testing.T) { + p := &upcloud.UpCloud{} + + var metadata upcloud.MetadataConfig + + require.NoError(t, json.Unmarshal(rawMetadata, &metadata)) + + networkConfig, err := p.ParseMetadata(&metadata) + require.NoError(t, err) + + marshaled, err := yaml.Marshal(networkConfig) + require.NoError(t, err) + + assert.Equal(t, expectedNetworkConfig, string(marshaled)) +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/metadata.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/metadata.go new file mode 100644 index 0000000..af23612 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/metadata.go @@ -0,0 +1,266 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package vmware provides the VMware platform implementation. +package vmware + +import ( + "context" + "fmt" + "log" + "net/netip" + "strings" + + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// NetworkConfig maps to VMware GuestInfo metadata. +// See also definition of GuestInfo in CAPV https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/blob/main/pkg/util/constants.go +type NetworkConfig struct { + InstanceID string `yaml:"instance-id"` + LocalHostname string `yaml:"local-hostname"` + // Talos doesn't block on network, it will reconfigure itself as network information becomes available. WaitOnNetwork is not used. + WaitOnNetwork struct { + Ipv4 bool `yaml:"ipv4"` + Ipv6 bool `yaml:"ipv6"` + } `yaml:"wait-on-network,omitempty"` + Network struct { + Version int `yaml:"version"` + Ethernets map[string]Ethernet `yaml:"ethernets"` + } + Routes []Route `yaml:"routes,omitempty"` +} + +// Ethernet holds network interface info. +type Ethernet struct { + Match struct { + Name string `yaml:"name,omitempty"` + HWAddr string `yaml:"macaddress,omitempty"` + } `yaml:"match,omitempty"` + SetName string `yaml:"set-name,omitempty"` + Wakeonlan bool `yaml:"wakeonlan,omitempty"` + DHCPv4 bool `yaml:"dhcp4,omitempty"` + DHCP4Overrides DHCPOverrides `yaml:"dhcp4-overrides,omitempty"` + DHCPv6 bool `yaml:"dhcp6,omitempty"` + DHCP6Overrides DHCPOverrides `yaml:"dhcp6-overrides,omitempty"` + Address []string `yaml:"addresses,omitempty"` + Gateway4 string `yaml:"gateway4,omitempty"` + Gateway6 string `yaml:"gateway6,omitempty"` + MTU int `yaml:"mtu,omitempty"` + NameServers struct { + Search []string `yaml:"search,omitempty"` + Address []string `yaml:"addresses,omitempty"` + } `yaml:"nameservers,omitempty"` + Routes []Route `yaml:"routes,omitempty"` +} + +// Route configuration. Not used. +type Route struct { + To string `yaml:"to,omitempty"` + Via string `yaml:"via,omitempty"` + Metric string `yaml:"metric,omitempty"` +} + +// DHCPOverrides is partial implemented. Only RouteMetric is use, the other elements are not processed. +type DHCPOverrides struct { + Hostname string `yaml:"hostname,omitempty"` + RouteMetric uint32 `yaml:"route-metric,omitempty"` + SendHostname string `yaml:"send-hostname,omitempty"` + UseDNS string `yaml:"use-dns,omitempty"` + UseDomains string `yaml:"use-domains,omitempty"` + UseHostname string `yaml:"use-hostname,omitempty"` + UseMTU string `yaml:"use-mtu,omitempty"` + UseNTP string `yaml:"use-ntp,omitempty"` + UseRoutes string `yaml:"use-routes,omitempty"` +} + +// ApplyNetworkConfigV2 gets GuestInfo and applies to the Talos runtime platform network configuration. +// +//nolint:gocyclo,cyclop +func (v *VMware) ApplyNetworkConfigV2(ctx context.Context, st state.State, config *NetworkConfig, networkConfig *runtime.PlatformNetworkConfig) error { + var dnsIPs []netip.Addr + + hostInterfaces, err := safe.StateListAll[*network.LinkStatus](ctx, st) + if err != nil { + return fmt.Errorf("error listing host interfaces: %w", err) + } + + for name, eth := range config.Network.Ethernets { + if eth.SetName != "" { + name = eth.SetName + } + + if !strings.HasPrefix(name, "eth") { + continue + } + + if eth.Match.HWAddr != "" { + var availableMACAddresses []string + + macAddressMatched := false + hostInterfaceIter := hostInterfaces.Iterator() + + for hostInterfaceIter.Next() { + macAddress := hostInterfaceIter.Value().TypedSpec().PermanentAddr.String() + if macAddress == eth.Match.HWAddr { + name = hostInterfaceIter.Value().Metadata().ID() + macAddressMatched = true + + break + } + + availableMACAddresses = append(availableMACAddresses, macAddress) + } + + if !macAddressMatched { + log.Printf("vmware: no link with matching MAC address %q (available %v), defaulted to use name %s instead", eth.Match.HWAddr, availableMACAddresses, name) + } + } + + networkConfig.Links = append(networkConfig.Links, network.LinkSpecSpec{ + Name: name, + Up: true, + MTU: uint32(eth.MTU), + ConfigLayer: network.ConfigPlatform, + }) + + if eth.DHCPv4 { + routeMetric := uint32(network.DefaultRouteMetric) + + if eth.DHCP4Overrides.RouteMetric != 0 { + routeMetric = eth.DHCP4Overrides.RouteMetric + } + + networkConfig.Operators = append(networkConfig.Operators, network.OperatorSpecSpec{ + Operator: network.OperatorDHCP4, + LinkName: name, + RequireUp: true, + DHCP4: network.DHCP4OperatorSpec{ + RouteMetric: routeMetric, + }, + ConfigLayer: network.ConfigPlatform, + }) + } + + if eth.DHCPv6 { + routeMetric := uint32(2 * network.DefaultRouteMetric) + + if eth.DHCP4Overrides.RouteMetric != 0 { + routeMetric = eth.DHCP6Overrides.RouteMetric + } + + networkConfig.Operators = append(networkConfig.Operators, network.OperatorSpecSpec{ + Operator: network.OperatorDHCP6, + LinkName: name, + RequireUp: true, + DHCP6: network.DHCP6OperatorSpec{ + RouteMetric: routeMetric, + }, + ConfigLayer: network.ConfigPlatform, + }) + } + + for _, addr := range eth.Address { + ipPrefix, err := netip.ParsePrefix(addr) + if err != nil { + return err + } + + family := nethelpers.FamilyInet4 + + if ipPrefix.Addr().Is6() { + family = nethelpers.FamilyInet6 + } + + networkConfig.Addresses = append(networkConfig.Addresses, + network.AddressSpecSpec{ + ConfigLayer: network.ConfigPlatform, + LinkName: name, + Address: ipPrefix, + Scope: nethelpers.ScopeGlobal, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + Family: family, + }, + ) + } + + if eth.Gateway4 != "" { + gw, err := netip.ParseAddr(eth.Gateway4) + if err != nil { + return err + } + + route := network.RouteSpecSpec{ + ConfigLayer: network.ConfigPlatform, + Gateway: gw, + OutLinkName: name, + Table: nethelpers.TableMain, + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + Family: nethelpers.FamilyInet4, + Priority: network.DefaultRouteMetric, + } + + route.Normalize() + + networkConfig.Routes = append(networkConfig.Routes, route) + } + + if eth.Gateway6 != "" { + gw, err := netip.ParseAddr(eth.Gateway6) + if err != nil { + return err + } + + route := network.RouteSpecSpec{ + ConfigLayer: network.ConfigPlatform, + Gateway: gw, + OutLinkName: name, + Table: nethelpers.TableMain, + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + Family: nethelpers.FamilyInet6, + Priority: 2 * network.DefaultRouteMetric, + } + + route.Normalize() + + networkConfig.Routes = append(networkConfig.Routes, route) + } + + for _, addr := range eth.NameServers.Address { + if ip, err := netip.ParseAddr(addr); err == nil { + dnsIPs = append(dnsIPs, ip) + } else { + return err + } + } + } + + if config.LocalHostname != "" { + hostnameSpec := network.HostnameSpecSpec{ + ConfigLayer: network.ConfigPlatform, + } + + if err := hostnameSpec.ParseFQDN(config.LocalHostname); err != nil { + return err + } + + networkConfig.Hostnames = append(networkConfig.Hostnames, hostnameSpec) + } + + if len(dnsIPs) > 0 { + networkConfig.Resolvers = append(networkConfig.Resolvers, network.ResolverSpecSpec{ + DNSServers: dnsIPs, + ConfigLayer: network.ConfigPlatform, + }) + } + + return nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/testdata/expected-match-by-mac.yaml b/internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/testdata/expected-match-by-mac.yaml new file mode 100644 index 0000000..0679bb0 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/testdata/expected-match-by-mac.yaml @@ -0,0 +1,36 @@ +addresses: + - address: 192.168.0.230/24 + linkName: eth2 + family: inet4 + scope: global + flags: permanent + layer: platform +links: + - name: eth2 + logical: false + up: true + mtu: 0 + kind: "" + type: netrom + layer: platform +routes: + - family: inet4 + dst: "" + src: "" + gateway: 192.168.0.1 + outLinkName: eth2 + table: main + priority: 1024 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform +hostnames: + - hostname: vmware-test-controlplane-zhnhr + domainname: "" + layer: platform +resolvers: [] +timeServers: [] +operators: [] +externalIPs: [] diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/testdata/expected-match-by-name.yaml b/internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/testdata/expected-match-by-name.yaml new file mode 100644 index 0000000..249d71b --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/testdata/expected-match-by-name.yaml @@ -0,0 +1,36 @@ +addresses: + - address: 192.168.0.230/24 + linkName: eth1 + family: inet4 + scope: global + flags: permanent + layer: platform +links: + - name: eth1 + logical: false + up: true + mtu: 0 + kind: "" + type: netrom + layer: platform +routes: + - family: inet4 + dst: "" + src: "" + gateway: 192.168.0.1 + outLinkName: eth1 + table: main + priority: 1024 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform +hostnames: + - hostname: vmware-test-controlplane-zhnhr + domainname: "" + layer: platform +resolvers: [] +timeServers: [] +operators: [] +externalIPs: [] diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/testdata/metadata-match-by-mac.yaml b/internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/testdata/metadata-match-by-mac.yaml new file mode 100644 index 0000000..fae7138 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/testdata/metadata-match-by-mac.yaml @@ -0,0 +1,16 @@ +instance-id: "1" +local-hostname: "vmware-test-controlplane-zhnhr" +wait-on-network: + ipv4: false + ipv6: false +network: + version: 2 + ethernets: + id0: + match: + macaddress: "68:05:ca:b8:f1:f9" + set-name: "eth0" + wakeonlan: true + addresses: + - "192.168.0.230/24" + gateway4: "192.168.0.1" \ No newline at end of file diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/testdata/metadata-match-by-name.yaml b/internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/testdata/metadata-match-by-name.yaml new file mode 100644 index 0000000..4e217d5 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/testdata/metadata-match-by-name.yaml @@ -0,0 +1,16 @@ +instance-id: "1" +local-hostname: "vmware-test-controlplane-zhnhr" +wait-on-network: + ipv4: false + ipv6: false +network: + version: 2 + ethernets: + id0: + match: + name: "eth1" + set-name: "eth1" + wakeonlan: true + addresses: + - "192.168.0.230/24" + gateway4: "192.168.0.1" \ No newline at end of file diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/vmware_amd64.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/vmware_amd64.go new file mode 100644 index 0000000..4d7bc93 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/vmware_amd64.go @@ -0,0 +1,293 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//go:build amd64 + +// Package vmware provides the VMware platform implementation. +package vmware + +import ( + "context" + "encoding/base64" + "encoding/xml" + "errors" + "fmt" + "log" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-procfs/procfs" + "github.com/vmware/vmw-guestinfo/rpcvmx" + "github.com/vmware/vmw-guestinfo/vmcheck" + yaml "gopkg.in/yaml.v3" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + platformerrors "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/errors" + "github.com/siderolabs/talos/pkg/machinery/constants" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// VMware is the concrete type that implements the platform.Platform interface. +type VMware struct{} + +// Name implements the platform.Platform interface. +func (v *VMware) Name() string { + return "vmware" +} + +// Read and de-base64 a property from `extraConfig`. This is commonly referred to as `guestinfo`. +func readConfigFromExtraConfig(extraConfig *rpcvmx.Config, key string) ([]byte, error) { + val, err := extraConfig.String(key, "") + if err != nil { + return nil, fmt.Errorf("failed to get extraConfig %s: %w", key, err) + } + + if val == "" { // not present + log.Printf("empty (thus absent) %s", key) + + return nil, nil + } + + decoded, err := base64.StdEncoding.DecodeString(val) + if err != nil { + return nil, fmt.Errorf("failed to decode extraConfig %s: %w", key, err) + } + + if len(decoded) == 0 { + log.Printf("skipping zero-length config in extraConfig") + + return nil, nil + } + + return decoded, nil +} + +// ofvEnv and related types are extracted from github.com/vmware/govmomi/ovf/env.go. +type ovfEnvFile struct { + XMLName xml.Name `xml:"http://schemas.dmtf.org/ovf/environment/1 Environment"` + ID string `xml:"id,attr"` + EsxID string `xml:"http://www.vmware.com/schema/ovfenv esxId,attr"` + + Platform *ovfPlatformSection `xml:"PlatformSection"` + Property *ovfPropertySection `xml:"PropertySection"` +} + +type ovfPlatformSection struct { + Kind string `xml:"Kind"` + Version string `xml:"Version"` + Vendor string `xml:"Vendor"` + Locale string `xml:"Locale"` +} + +type ovfPropertySection struct { + Properties []ovfEnvProperty `xml:"Property"` +} + +type ovfEnvProperty struct { + Key string `xml:"key,attr"` + Value string `xml:"value,attr"` +} + +// Read and de-base64 a property from the OVF env. This is different way to pass data to your VM. +// This is how data gets passed when using vCloud Director. +func readConfigFromOvf(extraConfig *rpcvmx.Config, key string) ([]byte, error) { + ovfXML, err := extraConfig.String(constants.VMwareGuestInfoOvfEnvKey, "") + if err != nil { + return nil, fmt.Errorf("failed to read extraConfig var '%s': %w", key, err) + } + + if ovfXML == "" { // value empty (probably because not present) + return nil, nil + } + + var ovfEnv ovfEnvFile + + err = xml.Unmarshal([]byte(ovfXML), &ovfEnv) + if err != nil { + return nil, fmt.Errorf("failed to unmarshall XML from OVF env: %w", err) + } + + if ovfEnv.Property == nil || ovfEnv.Property.Properties == nil { // no data in OVF env + log.Printf("empty OVF env") + + return nil, nil + } + + log.Printf("searching for property '%s' in OVF", key) + + for _, property := range ovfEnv.Property.Properties { // iterate to check if our key is present + if property.Key == key { + log.Printf("it is there, decoding") + + decoded, err := base64.StdEncoding.DecodeString(property.Value) + if err != nil { + return nil, fmt.Errorf("failed to decode OVF property %s: %w", property.Key, err) + } + + if len(decoded) == 0 { + log.Printf("skipping zero-length config in OVF") + + return nil, nil + } + + return decoded, nil + } + } + + return nil, nil +} + +// Configuration implements the platform.Platform interface. +// +//nolint:gocyclo +func (v *VMware) Configuration(context.Context, state.State) ([]byte, error) { + var option *string + if option = procfs.ProcCmdline().Get(constants.KernelParamConfig).First(); option == nil { + return nil, fmt.Errorf("%s not found", constants.KernelParamConfig) + } + + if *option == constants.ConfigGuestInfo { + log.Printf("fetching machine config from VMware extraConfig or OVF env") + + ok, err := vmcheck.IsVirtualWorld(true) + if err != nil { + return nil, fmt.Errorf("error checking if we are virtual: %w", err) + } + + if !ok { + return nil, errors.New("not a virtual world") + } + + extraConfig := rpcvmx.NewConfig() + + // try to fetch `talos.config` from plain extraConfig (ie, the old behavior) + log.Printf("trying to find '%s' in extraConfig", constants.VMwareGuestInfoConfigKey) + + config, err := readConfigFromExtraConfig(extraConfig, constants.VMwareGuestInfoConfigKey) + if err != nil { + return nil, err + } + + if config != nil { + return config, nil + } + + // try to fetch `userdata` from plain extraConfig (ie, the old behavior) + log.Printf("trying to find '%s' in extraConfig", constants.VMwareGuestInfoFallbackKey) + + config, err = readConfigFromExtraConfig(extraConfig, constants.VMwareGuestInfoFallbackKey) + if err != nil { + return nil, err + } + + if config != nil { + return config, nil + } + + // try to fetch `talos.config` from OVF + log.Printf("trying to find '%s' in OVF env", constants.VMwareGuestInfoConfigKey) + + config, err = readConfigFromOvf(extraConfig, constants.VMwareGuestInfoConfigKey) + if err != nil { + return nil, err + } + + if config != nil { + return config, nil + } + + // try to fetch `userdata` from OVF + log.Printf("trying to find '%s' in OVF env", constants.VMwareGuestInfoFallbackKey) + + config, err = readConfigFromOvf(extraConfig, constants.VMwareGuestInfoFallbackKey) + if err != nil { + return nil, err + } + + if config != nil { + return config, nil + } + + return nil, platformerrors.ErrNoConfigSource + } + + return nil, nil +} + +// Mode implements the platform.Platform interface. +func (v *VMware) Mode() runtime.Mode { + return runtime.ModeCloud +} + +// KernelArgs implements the runtime.Platform interface. +func (v *VMware) KernelArgs(string) procfs.Parameters { + return []*procfs.Parameter{ + procfs.NewParameter(constants.KernelParamConfig).Append(constants.ConfigGuestInfo), + procfs.NewParameter("console").Append("tty0").Append("ttyS0"), + procfs.NewParameter("earlyprintk").Append("ttyS0,115200"), + procfs.NewParameter(constants.KernelParamNetIfnames).Append("0"), + } +} + +// Read VMware GuestInfo metadata if available. +func (v *VMware) readMetadata(extraConfig *rpcvmx.Config) ([]byte, error) { + guestInfoMetadata, err := readConfigFromExtraConfig(extraConfig, constants.VMwareGuestInfoMetadataKey) + if err != nil { + return nil, err + } + + if guestInfoMetadata == nil { + guestInfoMetadata, err = readConfigFromOvf(extraConfig, constants.VMwareGuestInfoMetadataKey) + } + + if err != nil { + return nil, err + } + + return guestInfoMetadata, nil +} + +// NetworkConfiguration implements the runtime.Platform interface. +func (v *VMware) NetworkConfiguration(ctx context.Context, st state.State, ch chan<- *runtime.PlatformNetworkConfig) error { + extraConfig := rpcvmx.NewConfig() + + guestInfoMetadata, err := v.readMetadata(extraConfig) + if err != nil { + return fmt.Errorf("failed to read GuestInfo: %w", err) + } + + networkConfig := &runtime.PlatformNetworkConfig{ + Metadata: &runtimeres.PlatformMetadataSpec{Platform: v.Name()}, + } + + if guestInfoMetadata != nil { + var unmarshalledNetworkConfig NetworkConfig + if err = yaml.Unmarshal(guestInfoMetadata, &unmarshalledNetworkConfig); err != nil { + return fmt.Errorf("failed to unmarshall metadata '%s'. Error '%w'", guestInfoMetadata, err) + } + + switch unmarshalledNetworkConfig.Network.Version { + case 2: + err := v.ApplyNetworkConfigV2(ctx, st, &unmarshalledNetworkConfig, networkConfig) + if err != nil { + return fmt.Errorf("failed to apply metadata '%s'. Error '%w'", guestInfoMetadata, err) + } + + networkConfig.Metadata = &runtimeres.PlatformMetadataSpec{ + Platform: v.Name(), + Hostname: unmarshalledNetworkConfig.LocalHostname, + InstanceID: unmarshalledNetworkConfig.InstanceID, + } + default: + return fmt.Errorf("GuestInfo version=%d is not supported. GuestInfo: %s", unmarshalledNetworkConfig.Network.Version, guestInfoMetadata) + } + } + + select { + case <-ctx.Done(): + return ctx.Err() + case ch <- networkConfig: + } + + return nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/vmware_other.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/vmware_other.go new file mode 100644 index 0000000..4472188 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/vmware_other.go @@ -0,0 +1,45 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//go:build !amd64 + +package vmware + +import ( + "context" + "errors" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-procfs/procfs" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" +) + +// VMware is the concrete type that implements the platform.Platform interface. +type VMware struct{} + +// Name implements the platform.Platform interface. +func (v *VMware) Name() string { + return "vmware" +} + +// Configuration implements the platform.Platform interface. +func (v *VMware) Configuration(context.Context, state.State) ([]byte, error) { + return nil, errors.New("arch not supported") +} + +// Mode implements the platform.Platform interface. +func (v *VMware) Mode() runtime.Mode { + return runtime.ModeCloud +} + +// KernelArgs implements the runtime.Platform interface. +func (v *VMware) KernelArgs(string) procfs.Parameters { + return []*procfs.Parameter{} +} + +// NetworkConfiguration implements the runtime.Platform interface. +func (v *VMware) NetworkConfiguration(ctx context.Context, _ state.State, ch chan<- *runtime.PlatformNetworkConfig) error { + return nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/vmware_test.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/vmware_test.go new file mode 100644 index 0000000..5faad31 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/vmware/vmware_test.go @@ -0,0 +1,85 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package vmware_test + +import ( + "context" + _ "embed" + "fmt" + "testing" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/vmware" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +//go:embed testdata/metadata-match-by-mac.yaml +var rawMetadataMatchByMAC []byte + +//go:embed testdata/expected-match-by-mac.yaml +var expectedNetworkConfigMatchByMAC string + +//go:embed testdata/metadata-match-by-name.yaml +var rawMetadataMatchByName []byte + +//go:embed testdata/expected-match-by-name.yaml +var expectedNetworkConfigMatchByName string + +func TestApplyNetworkConfigV2a(t *testing.T) { + for _, tt := range []struct { + name string + raw []byte + expected string + }{ + { + name: "byMAC", + raw: rawMetadataMatchByMAC, + expected: expectedNetworkConfigMatchByMAC, + }, + { + name: "byName", + raw: rawMetadataMatchByName, + expected: expectedNetworkConfigMatchByName, + }, + } { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + st := state.WrapCore(namespaced.NewState(inmem.Build)) + + eth1 := network.NewLinkStatus(network.NamespaceName, "eth1") + eth1.TypedSpec().PermanentAddr = nethelpers.HardwareAddr{0x68, 0x05, 0xca, 0xb8, 0xf1, 0xf8} + require.NoError(t, st.Create(ctx, eth1)) + + eth2 := network.NewLinkStatus(network.NamespaceName, "eth2") + eth2.TypedSpec().PermanentAddr = nethelpers.HardwareAddr{0x68, 0x05, 0xca, 0xb8, 0xf1, 0xf9} + require.NoError(t, st.Create(ctx, eth2)) + + var metadata vmware.NetworkConfig + + require.NoError(t, yaml.Unmarshal(tt.raw, &metadata)) + + v := &vmware.VMware{} + networkConfig := &runtime.PlatformNetworkConfig{} + + err := v.ApplyNetworkConfigV2(ctx, st, &metadata, networkConfig) + require.NoError(t, err) + + marshaled, err := yaml.Marshal(networkConfig) + require.NoError(t, err) + + fmt.Print(string(marshaled)) + + assert.Equal(t, tt.expected, string(marshaled)) + }) + } +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/vultr/metadata.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/vultr/metadata.go new file mode 100644 index 0000000..f685ea6 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/vultr/metadata.go @@ -0,0 +1,38 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package vultr + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/vultr/metadata" + + "github.com/siderolabs/talos/pkg/download" +) + +const ( + // VultrMetadataEndpoint is the local Vultr endpoint fot the instance metadata. + VultrMetadataEndpoint = "http://169.254.169.254/v1.json" + // VultrExternalIPEndpoint is the local Vultr endpoint for the external IP. + VultrExternalIPEndpoint = "http://169.254.169.254/latest/meta-data/public-ipv4" + // VultrUserDataEndpoint is the local Vultr endpoint for the config. + VultrUserDataEndpoint = "http://169.254.169.254/latest/user-data" +) + +func (g *Vultr) getMetadata(ctx context.Context) (*metadata.MetaData, error) { + metaConfigDl, err := download.Download(ctx, VultrMetadataEndpoint) + if err != nil { + return nil, fmt.Errorf("error fetching metadata: %w", err) + } + + var meta metadata.MetaData + if err = json.Unmarshal(metaConfigDl, &meta); err != nil { + return nil, err + } + + return &meta, nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/vultr/testdata/expected.yaml b/internal/app/machined/pkg/runtime/v1alpha1/platform/vultr/testdata/expected.yaml new file mode 100644 index 0000000..f7d68a6 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/vultr/testdata/expected.yaml @@ -0,0 +1,56 @@ +addresses: + - address: 95.111.222.111/23 + linkName: eth0 + family: inet4 + scope: global + flags: permanent + layer: platform + - address: 10.7.96.3/20 + linkName: eth1 + family: inet4 + scope: global + flags: permanent + layer: platform +links: + - name: eth0 + logical: false + up: true + mtu: 0 + kind: "" + type: netrom + layer: platform + - name: eth1 + logical: false + up: true + mtu: 1450 + kind: "" + type: netrom + layer: platform +routes: + - family: inet4 + dst: "" + src: "" + gateway: 95.111.222.1 + outLinkName: eth0 + table: main + scope: global + type: unicast + flags: "" + protocol: static + layer: platform +hostnames: + - hostname: talos + domainname: "" + layer: platform +resolvers: [] +timeServers: [] +operators: [] +externalIPs: + - 95.111.222.111 + - 2001:19f0:5001:2095:1111:2222:3333:4444 +metadata: + platform: vultr + hostname: talos + region: AMS + instanceId: 91b07056-af72-4551-b15b-d57d34071be9 + providerId: vultr://91b07056-af72-4551-b15b-d57d34071be9 diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/vultr/testdata/metadata.json b/internal/app/machined/pkg/runtime/v1alpha1/platform/vultr/testdata/metadata.json new file mode 100644 index 0000000..9280380 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/vultr/testdata/metadata.json @@ -0,0 +1,61 @@ +{ + "bgp": { + "ipv4": { + "my-address": "", + "my-asn": "", + "peer-address": "", + "peer-asn": "" + }, + "ipv6": { + "my-address": "", + "my-asn": "", + "peer-address": "", + "peer-asn": "" + } + }, + "hostname": "talos", + "instance-v2-id": "91b07056-af72-4551-b15b-d57d34071be9", + "instanceid": "50190000", + "interfaces": [ + { + "ipv4": { + "additional": [], + "address": "95.111.222.111", + "gateway": "95.111.222.1", + "netmask": "255.255.254.0" + }, + "ipv6": { + "additional": [], + "address": "2001:19f0:5001:2095:1111:2222:3333:4444", + "network": "2001:19f0:5001:2095::", + "prefix": "64" + }, + "mac": "56:00:03:89:53:e0", + "network-type": "public" + }, + { + "ipv4": { + "additional": [], + "address": "10.7.96.3", + "gateway": "", + "netmask": "255.255.240.0" + }, + "ipv6": { + "additional": [], + "network": "", + "prefix": "" + }, + "mac": "5a:00:03:89:53:e0", + "network-type": "private", + "network-v2-id": "dadc2b30-0b55-4fa1-8c29-f67215bd5ac4", + "networkid": "net6126811851cd7" + } + ], + "public-keys": [ + "ssh-ed25519" + ], + "region": { + "regioncode": "AMS" + }, + "user-defined": [] +} \ No newline at end of file diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/vultr/vultr.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/vultr/vultr.go new file mode 100644 index 0000000..eee13a2 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/vultr/vultr.go @@ -0,0 +1,206 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package vultr provides the Vultr platform implementation. +package vultr + +import ( + "context" + "fmt" + "log" + "net" + "net/netip" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-procfs/procfs" + "github.com/vultr/metadata" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/errors" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/internal/netutils" + "github.com/siderolabs/talos/pkg/download" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// Vultr is the concrete type that implements the runtime.Platform interface. +type Vultr struct{} + +// Name implements the runtime.Platform interface. +func (v *Vultr) Name() string { + return "vultr" +} + +// ParseMetadata converts Vultr platform metadata into platform network config. +// +//nolint:gocyclo +func (v *Vultr) ParseMetadata(metadata *metadata.MetaData) (*runtime.PlatformNetworkConfig, error) { + networkConfig := &runtime.PlatformNetworkConfig{} + + if metadata.Hostname != "" { + hostnameSpec := network.HostnameSpecSpec{ + ConfigLayer: network.ConfigPlatform, + } + + if err := hostnameSpec.ParseFQDN(metadata.Hostname); err != nil { + return nil, err + } + + networkConfig.Hostnames = append(networkConfig.Hostnames, hostnameSpec) + } + + publicIPs := []string{} + + for i, addr := range metadata.Interfaces { + iface := fmt.Sprintf("eth%d", i) + + link := network.LinkSpecSpec{ + Name: iface, + Up: true, + ConfigLayer: network.ConfigPlatform, + } + + if addr.NetworkType == "private" { + link.MTU = 1450 + } + + networkConfig.Links = append(networkConfig.Links, link) + + if addr.IPv4.Address != "" { + if addr.NetworkType == "public" { + publicIPs = append(publicIPs, addr.IPv4.Address) + } + + ip, err := netip.ParseAddr(addr.IPv4.Address) + if err != nil { + return nil, err + } + + netmask, err := netip.ParseAddr(addr.IPv4.Netmask) + if err != nil { + return nil, err + } + + mask, _ := netmask.MarshalBinary() //nolint:errcheck // never fails + ones, _ := net.IPMask(mask).Size() + ipAddr := netip.PrefixFrom(ip, ones) + + networkConfig.Addresses = append(networkConfig.Addresses, + network.AddressSpecSpec{ + ConfigLayer: network.ConfigPlatform, + LinkName: iface, + Address: ipAddr, + Scope: nethelpers.ScopeGlobal, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + Family: nethelpers.FamilyInet4, + }, + ) + + if addr.IPv4.Gateway != "" { + gw, err := netip.ParseAddr(addr.IPv4.Gateway) + if err != nil { + return nil, err + } + + route := network.RouteSpecSpec{ + ConfigLayer: network.ConfigPlatform, + Gateway: gw, + OutLinkName: iface, + Table: nethelpers.TableMain, + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + Family: nethelpers.FamilyInet4, + } + + route.Normalize() + + networkConfig.Routes = append(networkConfig.Routes, route) + } + } else { + networkConfig.Operators = append(networkConfig.Operators, network.OperatorSpecSpec{ + Operator: network.OperatorDHCP4, + LinkName: iface, + RequireUp: true, + DHCP4: network.DHCP4OperatorSpec{ + RouteMetric: network.DefaultRouteMetric, + }, + ConfigLayer: network.ConfigPlatform, + }) + } + + if addr.IPv6.Address != "" { + if addr.NetworkType == "public" { + publicIPs = append(publicIPs, addr.IPv6.Address) + } + } + } + + for _, ipStr := range publicIPs { + if ip, err := netip.ParseAddr(ipStr); err == nil { + networkConfig.ExternalIPs = append(networkConfig.ExternalIPs, ip) + } + } + + networkConfig.Metadata = &runtimeres.PlatformMetadataSpec{ + Platform: v.Name(), + Hostname: metadata.Hostname, + Region: metadata.Region.RegionCode, + InstanceID: metadata.InstanceV2ID, + ProviderID: fmt.Sprintf("vultr://%s", metadata.InstanceV2ID), + } + + return networkConfig, nil +} + +// Configuration implements the runtime.Platform interface. +// +//nolint:stylecheck +func (v *Vultr) Configuration(ctx context.Context, r state.State) ([]byte, error) { + if err := netutils.Wait(ctx, r); err != nil { + return nil, err + } + + log.Printf("fetching machine config from: %q", VultrUserDataEndpoint) + + return download.Download(ctx, VultrUserDataEndpoint, + download.WithErrorOnNotFound(errors.ErrNoConfigSource), + download.WithErrorOnEmptyResponse(errors.ErrNoConfigSource)) +} + +// Mode implements the runtime.Platform interface. +func (v *Vultr) Mode() runtime.Mode { + return runtime.ModeCloud +} + +// KernelArgs implements the runtime.Platform interface. +func (v *Vultr) KernelArgs(string) procfs.Parameters { + return []*procfs.Parameter{ + procfs.NewParameter(constants.KernelParamNetIfnames).Append("0"), + } +} + +// NetworkConfiguration implements the runtime.Platform interface. +func (v *Vultr) NetworkConfiguration(ctx context.Context, _ state.State, ch chan<- *runtime.PlatformNetworkConfig) error { + log.Printf("fetching Vultr instance metadata from: %q", VultrMetadataEndpoint) + + metadata, err := v.getMetadata(ctx) + if err != nil { + return err + } + + networkConfig, err := v.ParseMetadata(metadata) + if err != nil { + return err + } + + select { + case ch <- networkConfig: + case <-ctx.Done(): + return ctx.Err() + } + + return nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/vultr/vultr_test.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/vultr/vultr_test.go new file mode 100644 index 0000000..5e83043 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/vultr/vultr_test.go @@ -0,0 +1,40 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package vultr_test + +import ( + _ "embed" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vultr/metadata" + "gopkg.in/yaml.v3" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform/vultr" +) + +//go:embed testdata/metadata.json +var rawMetadata []byte + +//go:embed testdata/expected.yaml +var expectedNetworkConfig string + +func TestParseMetadata(t *testing.T) { + p := &vultr.Vultr{} + + var metadata metadata.MetaData + + require.NoError(t, json.Unmarshal(rawMetadata, &metadata)) + + networkConfig, err := p.ParseMetadata(&metadata) + require.NoError(t, err) + + marshaled, err := yaml.Marshal(networkConfig) + require.NoError(t, err) + + assert.Equal(t, expectedNetworkConfig, string(marshaled)) +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_controller.go b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_controller.go new file mode 100644 index 0000000..c065179 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_controller.go @@ -0,0 +1,415 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package v1alpha1 + +import ( + "context" + "errors" + "fmt" + "io" + "log" + "os" + "os/signal" + "syscall" + "time" + + "github.com/siderolabs/go-kmsg" + "golang.org/x/sync/errgroup" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/logging" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/acpi" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha2" + krnl "github.com/siderolabs/talos/pkg/kernel" + "github.com/siderolabs/talos/pkg/machinery/api/common" + "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/kernel" +) + +// Controller represents the controller responsible for managing the execution +// of sequences. +type Controller struct { + s runtime.Sequencer + r *Runtime + v2 *v1alpha2.Controller + + priorityLock *PriorityLock[runtime.Sequence] +} + +// NewController intializes and returns a controller. +func NewController() (*Controller, error) { + s, err := NewState() + if err != nil { + return nil, err + } + + // TODO: this should be streaming capacity and probably some constant + e := NewEvents(1000, 10) + + l := logging.NewCircularBufferLoggingManager(log.New(os.Stdout, "machined fallback logger: ", log.Flags())) + + ctlr := &Controller{ + r: NewRuntime(s, e, l), + s: NewSequencer(), + priorityLock: NewPriorityLock[runtime.Sequence](), + } + + ctlr.v2, err = v1alpha2.NewController(ctlr.r) + if err != nil { + return nil, err + } + + if err := ctlr.setupLogging(); err != nil { + return nil, err + } + + return ctlr, nil +} + +func (c *Controller) setupLogging() error { + machinedLog, err := c.r.Logging().ServiceLog("machined").Writer() + if err != nil { + return err + } + + if c.r.State().Platform().Mode() == runtime.ModeContainer { + // send all the logs to machinedLog as well, but skip /dev/kmsg logging + log.SetOutput(io.MultiWriter(log.Writer(), machinedLog)) + log.SetPrefix("[talos] ") + + return nil + } + + // disable ratelimiting for kmsg, otherwise logs might be not visible. + // this should be set via kernel arg, but in case it's not set, try to force it. + if err = krnl.WriteParam(&kernel.Param{ + Key: "proc.sys.kernel.printk_devkmsg", + Value: "on\n", + }); err != nil { + var serr syscall.Errno + + if !(errors.As(err, &serr) && serr == syscall.EINVAL) { // ignore EINVAL which is returned when kernel arg is set + log.Printf("failed setting kernel.printk_devkmsg: %s, error ignored", err) + } + } + + if err = kmsg.SetupLogger(nil, "[talos]", machinedLog); err != nil { + return fmt.Errorf("failed to setup logging: %w", err) + } + + return nil +} + +// Run executes all phases known to the controller in serial. `Controller` +// aborts immediately if any phase fails. +func (c *Controller) Run(ctx context.Context, seq runtime.Sequence, data interface{}, setters ...runtime.LockOption) error { + // We must ensure that the runtime is configured since all sequences depend + // on the runtime. + if c.r == nil { + return runtime.ErrUndefinedRuntime + } + + ctx, err := c.priorityLock.Lock(ctx, time.Minute, seq, setters...) + if err != nil { + if errors.Is(err, runtime.ErrLocked) { + c.Runtime().Events().Publish(context.Background(), &machine.SequenceEvent{ + Sequence: seq.String(), + Action: machine.SequenceEvent_NOOP, + Error: &common.Error{ + Code: common.Code_LOCKED, + Message: fmt.Sprintf("sequence not started: %s", runtime.ErrLocked.Error()), + }, + }) + } + + return err + } + + defer c.priorityLock.Unlock() + + phases, err := c.phases(seq, data) + if err != nil { + return err + } + + err = c.run(ctx, seq, phases, data) + if err != nil { + code := common.Code_FATAL + + if errors.Is(err, context.Canceled) { + code = common.Code_CANCELED + } + + c.Runtime().Events().Publish(ctx, &machine.SequenceEvent{ + Sequence: seq.String(), + Action: machine.SequenceEvent_NOOP, + Error: &common.Error{ + Code: code, + Message: fmt.Sprintf("sequence failed: %s", err.Error()), + }, + }) + + return err + } + + return nil +} + +// V1Alpha2 implements the controller interface. +func (c *Controller) V1Alpha2() runtime.V1Alpha2Controller { + return c.v2 +} + +// Runtime implements the controller interface. +func (c *Controller) Runtime() runtime.Runtime { + return c.r +} + +// Sequencer implements the controller interface. +func (c *Controller) Sequencer() runtime.Sequencer { + return c.s +} + +// ListenForEvents starts the event listener. The listener will trigger a +// shutdown in response to a SIGTERM signal and ACPI button/power event. +func (c *Controller) ListenForEvents(ctx context.Context) error { + sigs := make(chan os.Signal, 1) + + signal.Notify(sigs, syscall.SIGTERM) + + errCh := make(chan error, 2) + + go func() { + <-sigs + signal.Stop(sigs) + + log.Printf("shutdown via SIGTERM received") + + if err := c.Run(ctx, runtime.SequenceShutdown, &machine.ShutdownRequest{Force: true}, runtime.WithTakeover()); err != nil { + if !runtime.IsRebootError(err) { + log.Printf("shutdown failed: %v", err) + } + } + + errCh <- nil + }() + + if c.r.State().Platform().Mode() == runtime.ModeContainer { + return nil + } + + go func() { + if err := acpi.StartACPIListener(); err != nil { + errCh <- err + + return + } + + log.Printf("shutdown via ACPI received") + + if err := c.Run(ctx, runtime.SequenceShutdown, &machine.ShutdownRequest{Force: true}, runtime.WithTakeover()); err != nil { + if !runtime.IsRebootError(err) { + log.Printf("failed to run shutdown sequence: %s", err) + } + } + + errCh <- nil + }() + + err := <-errCh + + return err +} + +func (c *Controller) run(ctx context.Context, seq runtime.Sequence, phases []runtime.Phase, data interface{}) error { + c.Runtime().Events().Publish(ctx, &machine.SequenceEvent{ + Sequence: seq.String(), + Action: machine.SequenceEvent_START, + }) + + defer c.Runtime().Events().Publish(ctx, &machine.SequenceEvent{ + Sequence: seq.String(), + Action: machine.SequenceEvent_STOP, + }) + + start := time.Now() + + var ( + number int + phase runtime.Phase + err error + ) + + log.Printf("%s sequence: %d phase(s)", seq.String(), len(phases)) + + defer func() { + if err != nil { + if !runtime.IsRebootError(err) { + log.Printf("%s sequence: failed", seq.String()) + } + } else { + log.Printf("%s sequence: done: %s", seq.String(), time.Since(start)) + } + }() + + for number, phase = range phases { + if phase.CheckFunc != nil && !phase.CheckFunc() { + continue + } + + // Make the phase number human friendly. + number++ + + start := time.Now() + + progress := fmt.Sprintf("%d/%d", number, len(phases)) + + log.Printf("phase %s (%s): %d tasks(s)", phase.Name, progress, len(phase.Tasks)) + + if err = c.runPhase(ctx, phase, seq, data); err != nil { + if !runtime.IsRebootError(err) { + log.Printf("phase %s (%s): failed", phase.Name, progress) + } + + return fmt.Errorf("error running phase %d in %s sequence: %w", number, seq.String(), err) + } + + log.Printf("phase %s (%s): done, %s", phase.Name, progress, time.Since(start)) + + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + } + + return nil +} + +func (c *Controller) runPhase(ctx context.Context, phase runtime.Phase, seq runtime.Sequence, data interface{}) error { + c.Runtime().Events().Publish(ctx, &machine.PhaseEvent{ + Phase: phase.Name, + Action: machine.PhaseEvent_START, + }) + + defer c.Runtime().Events().Publish(ctx, &machine.PhaseEvent{ + Phase: phase.Name, + Action: machine.PhaseEvent_STOP, + }) + + eg, ctx := errgroup.WithContext(ctx) + + for number, task := range phase.Tasks { + // Make the task number human friendly. + number++ + + eg.Go(func() error { + progress := fmt.Sprintf("%d/%d", number, len(phase.Tasks)) + + if err := c.runTask(ctx, progress, task, seq, data); err != nil { + return fmt.Errorf("task %s: failed, %w", progress, err) + } + + return nil + }) + } + + return eg.Wait() +} + +func (c *Controller) runTask(ctx context.Context, progress string, f runtime.TaskSetupFunc, seq runtime.Sequence, data interface{}) error { + task, taskName := f(seq, data) + if task == nil { + return nil + } + + start := time.Now() + + c.Runtime().Events().Publish(ctx, &machine.TaskEvent{ + Task: taskName, + Action: machine.TaskEvent_START, + }) + + var err error + + log.Printf("task %s (%s): starting", taskName, progress) + + defer func() { + if err != nil { + if !runtime.IsRebootError(err) { + log.Printf("task %s (%s): failed: %s", taskName, progress, err) + } + } else { + log.Printf("task %s (%s): done, %s", taskName, progress, time.Since(start)) + } + }() + + defer c.Runtime().Events().Publish(ctx, &machine.TaskEvent{ + Task: taskName, + Action: machine.TaskEvent_STOP, + }) + + logger := log.New(log.Writer(), fmt.Sprintf("[talos] task %s (%s): ", taskName, progress), log.Flags()) + + err = task(ctx, logger, c.r) + + return err +} + +//nolint:gocyclo +func (c *Controller) phases(seq runtime.Sequence, data interface{}) ([]runtime.Phase, error) { + var phases []runtime.Phase + + switch seq { + case runtime.SequenceBoot: + phases = c.s.Boot(c.r) + case runtime.SequenceInitialize: + phases = c.s.Initialize(c.r) + case runtime.SequenceInstall: + phases = c.s.Install(c.r) + case runtime.SequenceShutdown: + in, ok := data.(*machine.ShutdownRequest) + if !ok { + return nil, runtime.ErrInvalidSequenceData + } + + phases = c.s.Shutdown(c.r, in) + case runtime.SequenceReboot: + phases = c.s.Reboot(c.r) + case runtime.SequenceUpgrade: + in, ok := data.(*machine.UpgradeRequest) + if !ok { + return nil, runtime.ErrInvalidSequenceData + } + + phases = c.s.Upgrade(c.r, in) + case runtime.SequenceStageUpgrade: + in, ok := data.(*machine.UpgradeRequest) + if !ok { + return nil, runtime.ErrInvalidSequenceData + } + + phases = c.s.StageUpgrade(c.r, in) + case runtime.SequenceMaintenanceUpgrade: + in, ok := data.(*machine.UpgradeRequest) + if !ok { + return nil, runtime.ErrInvalidSequenceData + } + + phases = c.s.MaintenanceUpgrade(c.r, in) + case runtime.SequenceReset: + in, ok := data.(runtime.ResetOptions) + if !ok { + return nil, runtime.ErrInvalidSequenceData + } + + phases = c.s.Reset(c.r, in) + case runtime.SequenceNoop: + default: + return nil, fmt.Errorf("sequence not implemented: %q", seq) + } + + return phases, nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_controller_test.go b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_controller_test.go new file mode 100644 index 0000000..66d6135 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_controller_test.go @@ -0,0 +1,215 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:scopelint,testpackage +package v1alpha1 + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" + + v1alpha1server "github.com/aenix-io/talm/internal/app/machined/internal/server/v1alpha1" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/logging" + "github.com/siderolabs/talos/pkg/machinery/api/machine" +) + +type mockSequencer struct { + callsMu sync.Mutex + calls map[runtime.Sequence]int + + phases map[runtime.Sequence]PhaseList +} + +func (m *mockSequencer) Boot(r runtime.Runtime) []runtime.Phase { + return m.phases[runtime.SequenceBoot] +} + +func (m *mockSequencer) Initialize(r runtime.Runtime) []runtime.Phase { + return m.phases[runtime.SequenceInitialize] +} + +func (m *mockSequencer) Install(r runtime.Runtime) []runtime.Phase { + return m.phases[runtime.SequenceInstall] +} + +func (m *mockSequencer) Reboot(r runtime.Runtime) []runtime.Phase { + return m.phases[runtime.SequenceReboot] +} + +func (m *mockSequencer) Reset(r runtime.Runtime, opts runtime.ResetOptions) []runtime.Phase { + return m.phases[runtime.SequenceReset] +} + +func (m *mockSequencer) Shutdown(r runtime.Runtime, req *machine.ShutdownRequest) []runtime.Phase { + return m.phases[runtime.SequenceShutdown] +} + +func (m *mockSequencer) StageUpgrade(r runtime.Runtime, req *machine.UpgradeRequest) []runtime.Phase { + return m.phases[runtime.SequenceStageUpgrade] +} + +func (m *mockSequencer) MaintenanceUpgrade(r runtime.Runtime, req *machine.UpgradeRequest) []runtime.Phase { + return m.phases[runtime.SequenceMaintenanceUpgrade] +} + +func (m *mockSequencer) Upgrade(r runtime.Runtime, req *machine.UpgradeRequest) []runtime.Phase { + return m.phases[runtime.SequenceUpgrade] +} + +func (m *mockSequencer) trackCall(name string, doneCh chan struct{}) func(runtime.Sequence, interface{}) (runtime.TaskExecutionFunc, string) { + return func(seq runtime.Sequence, data interface{}) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) error { + if doneCh != nil { + defer func() { + select { + case doneCh <- struct{}{}: + case <-time.After(time.Second): + } + }() + } + + m.callsMu.Lock() + defer m.callsMu.Unlock() + + m.calls[seq]++ + + return nil + }, name + } +} + +func TestRun(t *testing.T) { + require := require.New(t) + + tests := []struct { + name string + from runtime.Sequence + to runtime.Sequence + expectError error + dataFrom interface{} + dataTo interface{} + }{ + { + name: "reboot should take over boot", + from: runtime.SequenceBoot, + to: runtime.SequenceReboot, + expectError: context.Canceled, + }, + { + name: "reset should take over boot", + from: runtime.SequenceBoot, + to: runtime.SequenceReset, + expectError: context.Canceled, + dataTo: &v1alpha1server.ResetOptions{}, + }, + { + name: "upgrade should take over boot", + from: runtime.SequenceBoot, + to: runtime.SequenceUpgrade, + expectError: context.Canceled, + dataTo: &machine.UpgradeRequest{}, + }, + { + name: "boot should not take over reboot", + from: runtime.SequenceReboot, + to: runtime.SequenceBoot, + expectError: runtime.ErrLocked, + }, + { + name: "reset should not take over upgrade", + from: runtime.SequenceUpgrade, + to: runtime.SequenceReset, + expectError: runtime.ErrLocked, + dataFrom: &machine.UpgradeRequest{}, + dataTo: &v1alpha1server.ResetOptions{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := NewEvents(1000, 10) + + t.Setenv("PLATFORM", "container") + + s, err := NewState() + require.NoError(err) + + sequencer := &mockSequencer{ + calls: map[runtime.Sequence]int{}, + phases: map[runtime.Sequence]PhaseList{}, + } + + var ( + eg errgroup.Group + doneCh = make(chan struct{}) + ) + + sequencer.phases[tt.from] = sequencer.phases[tt.from]. + Append(tt.from.String(), sequencer.trackCall(tt.from.String(), doneCh)). + Append("wait", wait) + + sequencer.phases[tt.to] = sequencer.phases[tt.to].Append(tt.to.String(), sequencer.trackCall(tt.to.String(), nil)) + + l := logging.NewCircularBufferLoggingManager(log.New(os.Stdout, "machined fallback logger: ", log.Flags())) + + r := NewRuntime(s, e, l) + + controller := Controller{ + r: r, + s: sequencer, + priorityLock: NewPriorityLock[runtime.Sequence](), + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*200) + defer cancel() + + eg.Go(func() error { + return controller.Run(ctx, tt.from, tt.dataFrom) + }) + + eg.Go(func() error { + select { + case <-doneCh: + case <-time.After(time.Second): + return fmt.Errorf("timed out waiting for %s sequence to start", tt.from.String()) + } + + return controller.Run(ctx, tt.to, tt.dataTo) + }) + + require.ErrorIs(eg.Wait(), tt.expectError) + + if errors.Is(tt.expectError, runtime.ErrLocked) { + return + } + + sequencer.callsMu.Lock() + defer sequencer.callsMu.Unlock() + + require.Equal(1, sequencer.calls[tt.to]) + }) + } +} + +func wait(seq runtime.Sequence, data interface{}) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Second * 1): + } + + return nil + }, "wait" +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_dbus.go b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_dbus.go new file mode 100644 index 0000000..462cd21 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_dbus.go @@ -0,0 +1,95 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package v1alpha1 + +import ( + "context" + "errors" + "os" + "path/filepath" + "time" + + "github.com/aenix-io/talm/internal/pkg/logind" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// DBusState implements the logind mock. +type DBusState struct { + broker *logind.DBusBroker + logindMock *logind.ServiceMock + errCh chan error + cancel context.CancelFunc +} + +// Start the D-Bus broker and logind mock. +func (dbus *DBusState) Start() error { + for _, path := range []string{constants.DBusServiceSocketPath, constants.DBusClientSocketPath} { + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return err + } + } + + var err error + + dbus.broker, err = logind.NewBroker(constants.DBusServiceSocketPath, constants.DBusClientSocketPath) + if err != nil { + return err + } + + var ctx context.Context + + ctx, dbus.cancel = context.WithCancel(context.Background()) + + dbus.errCh = make(chan error) + + go func() { + dbus.errCh <- dbus.broker.Run(ctx) + }() + + dbus.logindMock, err = logind.NewServiceMock(constants.DBusServiceSocketPath) + + return err +} + +// Stop the D-Bus broker and logind mock. +func (dbus *DBusState) Stop() error { + if dbus.cancel == nil { + return nil + } + + dbus.cancel() + + if dbus.logindMock == nil || dbus.broker == nil { + return nil + } + + if err := dbus.logindMock.Close(); err != nil { + return err + } + + if err := dbus.broker.Close(); err != nil { + return err + } + + select { + case <-time.After(time.Second): + return errors.New("timed out stopping D-Bus broker") + case err := <-dbus.errCh: + return err + } +} + +// WaitShutdown signals the shutdown over the D-Bus and waits for the inhibit lock to be released. +func (dbus *DBusState) WaitShutdown(ctx context.Context) error { + if dbus.logindMock == nil { + return nil + } + + if err := dbus.logindMock.EmitShutdown(); err != nil { + return err + } + + return dbus.logindMock.WaitLockRelease(ctx) +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_events.go b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_events.go new file mode 100644 index 0000000..d5e1599 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_events.go @@ -0,0 +1,234 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package v1alpha1 + +import ( + "context" + "sort" + "sync" + "time" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/proto" +) + +// Events represents the runtime event stream. +// +// Events internally is implemented as circular buffer of `runtime.Event`. +// `e.stream` slice is allocated to the initial capacity and slice size doesn't change +// throughout the lifetime of Events. +// +// To explain the internals, let's call `Publish()` method 'Publisher' (there might be +// multiple callers for it), and each `Watch()` handler as 'Consumer'. +// +// For Publisher, `Events` keeps `e.writePos`, `e.writePos` is write offset into `e.stream`. +// Offset `e.writePos` is always incremeneted, real write index is `e.writePos % e.cap` +// +// Each Consumer captures initial position it starts consumption from as `pos` which is +// local to each Consumer, as Consumers are free to work on their own pace. Following diagram shows +// Publisher and three Consumers: +// +// Consumer 3 Consumer 2 +// pos = 27 pos = 34 +// e.stream []Event | | +// | | +// +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+ +// | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 |17 | +// +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+ +// | | +// | | +// Consumer 1 Publisher +// pos = 43 e.writePos = 50 +// +// Capacity of Events in this diagram is 18, Publisher published already 50 events, so it +// already overwrote `e.stream` twice fully. +// +// Consumer1 is trying to keep up with the publisher, it has 14-7 = 7 events to catch up. +// +// Consumer2 is reading events published by Publisher before last wraparound, it has +// 50-34 = 16 events to catch up. Consumer 2 has a lot of events to catch up, but as it stays +// on track, it can still do that. +// +// Consumer3 is doing bad: 50-27 = 23 > 18 (capacity), so its read position has already been +// overwritten, it can't read consistent data, soit should error out. +// +// Synchronization: at the moment single mutex protects `e.stream` and `e.writePos`, consumers keep their +// position as local variable, so it doesn't require synchronization. If Consumer catches up with Publisher, +// it sleeps on condition variable to be woken up by Publisher on next publish. +type Events struct { + // stream is used as ring buffer of events + stream []runtime.Event + + // writePos is the index in streams for the next write (publish) + // + // writePos gets always incremented, real position in slice is (writePos % cap) + writePos int64 + + // cap is a capacity of the stream + cap int + // gap is a safety gap between consumers and publishers + gap int + + // mutext protects access to writePos and stream + mu sync.Mutex + c *sync.Cond +} + +// NewEvents initializes and returns the v1alpha1 runtime event stream. +// +// Argument cap is a maximum event stream capacity (available event history). +// Argument gap is a safety gap to separate consumer from the publisher. +// Maximum available event history is (cap-gap). +func NewEvents(capacity, gap int) *Events { + e := &Events{ + stream: make([]runtime.Event, capacity), + cap: capacity, + gap: gap, + } + + if gap >= capacity { + // we should never reach this, but if we do, panic so that we know. + panic("NewEvents: gap >= capacity") + } + + e.c = sync.NewCond(&e.mu) + + return e +} + +// Watch implements the Events interface. +// +//nolint:gocyclo +func (e *Events) Watch(f runtime.WatchFunc, opt ...runtime.WatchOptionFunc) error { + var opts runtime.WatchOptions + + for _, o := range opt { + if err := o(&opts); err != nil { + return err + } + } + + // context is used to abort the loop when WatchFunc exits + ctx, ctxCancel := context.WithCancel(context.Background()) + + ch := make(chan runtime.EventInfo) + + go func() { + defer ctxCancel() + + f(ch) + }() + + e.mu.Lock() + + // capture initial consumer position: by default, consumer starts consuming from the next + // event to be published + pos := e.writePos + minPos := e.writePos - int64(e.cap-e.gap) + + if minPos < 0 { + minPos = 0 + } + + // calculate initial position based on options + switch { + case opts.TailEvents != 0: + if opts.TailEvents < 0 { + pos = minPos + } else { + pos -= int64(opts.TailEvents) + + if pos < minPos { + pos = minPos + } + } + case !opts.TailID.IsNil(): + pos = minPos + int64(sort.Search(int(pos-minPos), func(i int) bool { + event := e.stream[(minPos+int64(i))%int64(e.cap)] + + return event.ID.Compare(opts.TailID) > 0 + })) + case opts.TailDuration != 0: + timestamp := time.Now().Add(-opts.TailDuration) + + pos = minPos + int64(sort.Search(int(pos-minPos), func(i int) bool { + event := e.stream[(minPos+int64(i))%int64(e.cap)] + + return event.ID.Time().After(timestamp) + })) + } + + e.mu.Unlock() + + go func() { + defer close(ch) + + for { + e.mu.Lock() + // while there's no data to consume (pos == e.writePos), wait for Condition variable signal, + // then recheck the condition to be true. + for pos == e.writePos { + e.c.Wait() + + select { + case <-ctx.Done(): + e.mu.Unlock() + + return + default: + } + } + + if e.writePos-pos >= int64(e.cap) { + // buffer overrun, there's no way to signal error in this case, + // so for now just return + e.mu.Unlock() + + return + } + + event := e.stream[pos%int64(e.cap)] + pos++ + backlog := int(e.writePos - pos) + + e.mu.Unlock() + + // if actor id filter is specified and does not match the event, skip it + if opts.ActorID != "" && event.ActorID != opts.ActorID { + continue + } + + // send event to WatchFunc, wait for it to process the event + select { + case ch <- runtime.EventInfo{ + Event: event, + Backlog: backlog, + }: + case <-ctx.Done(): + return + } + } + }() + + return nil +} + +// Publish implements the Events interface. +func (e *Events) Publish(ctx context.Context, msg proto.Message) { + actorID, ok := ctx.Value(runtime.ActorIDCtxKey{}).(string) + if !ok { + actorID = "" + } + + event := runtime.NewEvent(msg, actorID) + + e.mu.Lock() + defer e.mu.Unlock() + + e.stream[e.writePos%int64(e.cap)] = event + e.writePos++ + + e.c.Broadcast() +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_events_test.go b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_events_test.go new file mode 100644 index 0000000..33f1866 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_events_test.go @@ -0,0 +1,306 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:scopelint,testpackage +package v1alpha1 + +import ( + "context" + "fmt" + "strconv" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "golang.org/x/time/rate" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/api/machine" +) + +func TestEvents_Publish(t *testing.T) { + tests := []struct { + name string + cap int + watchers int + messages int + }{ + { + name: "nowatchers", + cap: 100, + watchers: 0, + messages: 100, + }, + { + name: "onemessage", + cap: 100, + watchers: 10, + messages: 1, + }, + { + name: "manymessages_singlewatcher", + cap: 100, + watchers: 1, + messages: 50, + }, + { + name: "manymessages_manywatchers", + cap: 100, + watchers: 20, + messages: 50, + }, + { + name: "manymessages_overcap", + cap: 10, + watchers: 5, + messages: 200, + }, + { + name: "megamessages_overcap", + cap: 1000, + watchers: 1, + messages: 2000, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := NewEvents(tt.cap, tt.cap/10) + + var wg sync.WaitGroup + + wg.Add(tt.watchers) + + got := uint32(0) + + for range tt.watchers { + if err := e.Watch(func(events <-chan runtime.EventInfo) { + defer wg.Done() + + l := rate.NewLimiter(500, tt.cap*8/10) + + for j := range tt.messages { + event, ok := <-events + + if !ok { + // on buffer overrun Watch() closes the channel + t.Fatalf("buffer overrun") + } + + seq, err := strconv.Atoi(event.Payload.(*machine.SequenceEvent).Sequence) + if err != nil { + t.Fatalf("failed to convert sequence to number: %s", err) + } + + if seq != j { + t.Fatalf("unexpected sequence: %d != %d", seq, j) + } + + atomic.AddUint32(&got, 1) + + _ = l.Wait(context.Background()) //nolint:errcheck + } + }); err != nil { + t.Errorf("Watch error %s", err) + } + } + + l := rate.NewLimiter(500, tt.cap/2) + + for i := range tt.messages { + _ = l.Wait(context.Background()) //nolint:errcheck + + e.Publish(context.Background(), &machine.SequenceEvent{ + Sequence: strconv.Itoa(i), + }) + } + + wg.Wait() + + if got != uint32(tt.messages*tt.watchers) { + t.Errorf("Watch() = got %v, want %v", got, tt.messages*tt.watchers) + } + }) + } +} + +func receive(t *testing.T, e runtime.Watcher, n int, opts ...runtime.WatchOptionFunc) (result []runtime.EventInfo) { + var wg sync.WaitGroup + + wg.Add(1) + + if err := e.Watch(func(events <-chan runtime.EventInfo) { + defer wg.Done() + + for range n { + event, ok := <-events + if !ok { + t.Fatalf("Watch: chanel closed") + } + + result = append(result, event) + } + + select { + case _, ok := <-events: + if ok { + t.Fatal("received extra events") + } else { + t.Fatalf("Watch: chanel closed") + } + case <-time.After(50 * time.Millisecond): + } + }, opts...); err != nil { + t.Fatalf("Watch() error %s", err) + } + + wg.Wait() + + return result +} + +func extractSeq(t *testing.T, events []runtime.EventInfo) (result []int) { + for _, event := range events { + seq, err := strconv.Atoi(event.Payload.(*machine.SequenceEvent).Sequence) + if err != nil { + t.Fatalf("failed to convert sequence to number: %s", err) + } + + result = append(result, seq) + } + + return result +} + +func gen(k, l int) (result []int) { + for j := k; j < l; j++ { + result = append(result, j) + } + + return +} + +func TestEvents_WatchOptionsTailEvents(t *testing.T) { + e := NewEvents(100, 10) + + for i := range 200 { + e.Publish(context.Background(), &machine.SequenceEvent{ + Sequence: strconv.Itoa(i), + }) + } + + assert.Equal(t, []int(nil), extractSeq(t, receive(t, e, 0))) + assert.Equal(t, gen(199, 200), extractSeq(t, receive(t, e, 1, runtime.WithTailEvents(1)))) + assert.Equal(t, gen(195, 200), extractSeq(t, receive(t, e, 5, runtime.WithTailEvents(5)))) + assert.Equal(t, gen(111, 200), extractSeq(t, receive(t, e, 89, runtime.WithTailEvents(89)))) + assert.Equal(t, gen(110, 200), extractSeq(t, receive(t, e, 90, runtime.WithTailEvents(90)))) + assert.Equal(t, gen(110, 200), extractSeq(t, receive(t, e, 90, runtime.WithTailEvents(91)))) // can't tail more than cap-gap + assert.Equal(t, gen(110, 200), extractSeq(t, receive(t, e, 90, runtime.WithTailEvents(1000)))) // can't tail more than cap-gap + assert.Equal(t, gen(110, 200), extractSeq(t, receive(t, e, 90, runtime.WithTailEvents(-1)))) // tail all events + + e = NewEvents(100, 10) + + for i := range 30 { + e.Publish(context.Background(), &machine.SequenceEvent{ + Sequence: strconv.Itoa(i), + }) + } + + assert.Equal(t, []int(nil), extractSeq(t, receive(t, e, 0))) + assert.Equal(t, gen(29, 30), extractSeq(t, receive(t, e, 1, runtime.WithTailEvents(1)))) + assert.Equal(t, gen(28, 30), extractSeq(t, receive(t, e, 2, runtime.WithTailEvents(2)))) + assert.Equal(t, gen(25, 30), extractSeq(t, receive(t, e, 5, runtime.WithTailEvents(5)))) + assert.Equal(t, gen(0, 30), extractSeq(t, receive(t, e, 30, runtime.WithTailEvents(40)))) +} + +func TestEvents_WatchOptionsTailSeconds(t *testing.T) { + e := NewEvents(100, 10) + + for i := range 20 { + e.Publish(context.Background(), &machine.SequenceEvent{ + Sequence: strconv.Itoa(i), + }) + } + + // sleep to get time gap between two series of events + time.Sleep(3 * time.Second) + + for i := 20; i < 30; i++ { + e.Publish(context.Background(), &machine.SequenceEvent{ + Sequence: strconv.Itoa(i), + }) + } + + assert.Equal(t, []int(nil), extractSeq(t, receive(t, e, 0, runtime.WithTailDuration(0)))) + assert.Equal(t, gen(20, 30), extractSeq(t, receive(t, e, 10, runtime.WithTailDuration(2*time.Second)))) + assert.Equal(t, gen(0, 30), extractSeq(t, receive(t, e, 30, runtime.WithTailDuration(10*time.Second)))) +} + +func TestEvents_WatchOptionsTailID(t *testing.T) { + e := NewEvents(100, 10) + + for i := range 20 { + e.Publish(context.Background(), &machine.SequenceEvent{ + Sequence: strconv.Itoa(i), + }) + } + + events := receive(t, e, 20, runtime.WithTailEvents(-1)) + + for i, event := range events { + assert.Equal(t, gen(i+1, 20), extractSeq(t, receive(t, e, 20-i-1, runtime.WithTailID(event.ID)))) + } +} + +func BenchmarkWatch(b *testing.B) { + e := NewEvents(100, 10) + + var wg sync.WaitGroup + + wg.Add(b.N) + + for range b.N { + _ = e.Watch(func(events <-chan runtime.EventInfo) { wg.Done() }) //nolint:errcheck + } + + wg.Wait() +} + +func BenchmarkPublish(bb *testing.B) { + for _, watchers := range []int{0, 1, 10} { + bb.Run(fmt.Sprintf("Watchers-%d", watchers), func(b *testing.B) { + e := NewEvents(10000, 10) + + var wg sync.WaitGroup + + watchers := 10 + + wg.Add(watchers) + + for range watchers { + _ = e.Watch(func(events <-chan runtime.EventInfo) { //nolint:errcheck + defer wg.Done() + + for range b.N { + if _, ok := <-events; !ok { + return + } + } + }) + } + + ev := machine.SequenceEvent{} + + b.ResetTimer() + + for range b.N { + e.Publish(context.Background(), &ev) + } + + wg.Wait() + }) + } +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_priority_lock.go b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_priority_lock.go new file mode 100644 index 0000000..9fe5c13 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_priority_lock.go @@ -0,0 +1,122 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package v1alpha1 + +import ( + "context" + "errors" + "sync" + "time" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" +) + +// Priority describes the running priority of a process. +// +// If CanTakeOver returns true, current process with "lower" priority +// will be canceled and "higher" priority process will be run. +type Priority[T any] interface { + comparable + CanTakeOver(another T) bool +} + +// PriorityLock is a lock that makes sure that only a single process can run at a time. +// +// If a process with "higher" priority tries to acquire the lock, previous process is stopped +// and new process with "higher" priority is run. +type PriorityLock[T Priority[T]] struct { + runningCh chan struct{} + takeoverCh chan struct{} + + mu sync.Mutex + runningPriority T + cancelCtx context.CancelFunc +} + +// NewPriorityLock returns a new PriorityLock. +func NewPriorityLock[T Priority[T]]() *PriorityLock[T] { + runningCh := make(chan struct{}, 1) + runningCh <- struct{}{} + + return &PriorityLock[T]{ + runningCh: runningCh, + takeoverCh: make(chan struct{}, 1), + } +} + +func (lock *PriorityLock[T]) getRunningPriority() (T, context.CancelFunc) { + lock.mu.Lock() + defer lock.mu.Unlock() + + return lock.runningPriority, lock.cancelCtx +} + +func (lock *PriorityLock[T]) setRunningPriority(seq T, cancelCtx context.CancelFunc) { + lock.mu.Lock() + defer lock.mu.Unlock() + + var zeroSeq T + + if seq == zeroSeq && lock.cancelCtx != nil { + lock.cancelCtx() + } + + lock.runningPriority, lock.cancelCtx = seq, cancelCtx +} + +// Lock acquires the lock according the priority rules and returns a context that should be used within the process. +// +// Process should terminate as soon as the context is canceled. +// Argument seq defines the priority of the process. +// Argument takeOverTimeout defines the maximum time to wait for the low-priority process to terminate. +func (lock *PriorityLock[T]) Lock(ctx context.Context, takeOverTimeout time.Duration, seq T, options ...runtime.LockOption) (context.Context, error) { + opts := runtime.DefaultControllerOptions() + for _, o := range options { + if err := o(&opts); err != nil { + return nil, err + } + } + + takeOverTimer := time.NewTimer(takeOverTimeout) + defer takeOverTimer.Stop() + + select { + case lock.takeoverCh <- struct{}{}: + case <-takeOverTimer.C: + return nil, errors.New("failed to acquire lock: timeout") + } + + defer func() { + <-lock.takeoverCh + }() + + sequence, cancelCtx := lock.getRunningPriority() + + if !seq.CanTakeOver(sequence) && !opts.Takeover { + return nil, runtime.ErrLocked + } + + if cancelCtx != nil { + cancelCtx() + } + + select { + case <-lock.runningCh: + seqCtx, seqCancel := context.WithCancel(ctx) + lock.setRunningPriority(seq, seqCancel) + + return seqCtx, nil + case <-takeOverTimer.C: + return nil, errors.New("failed to acquire lock: timeout") + } +} + +// Unlock releases the lock. +func (lock *PriorityLock[T]) Unlock() { + var zeroSeq T + + lock.setRunningPriority(zeroSeq, nil) + lock.runningCh <- struct{}{} +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_priority_lock_test.go b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_priority_lock_test.go new file mode 100644 index 0000000..712972a --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_priority_lock_test.go @@ -0,0 +1,154 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package v1alpha1_test + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1" +) + +type testSequenceNumber int + +func (candidate testSequenceNumber) CanTakeOver(running testSequenceNumber) bool { + return candidate > running +} + +func TestPriorityLock(t *testing.T) { + require := require.New(t) + + lock := v1alpha1.NewPriorityLock[testSequenceNumber]() + ctx := context.Background() + + ctx1, err := lock.Lock(ctx, time.Second, 2) + require.NoError(err) + + select { + case <-ctx1.Done(): + require.FailNow("should not be canceled") + default: + } + + _, err = lock.Lock(ctx, time.Millisecond, 1) + require.Error(err) + require.EqualError(err, runtime.ErrLocked.Error()) + + errCh := make(chan error) + + go func() { + _, err2 := lock.Lock(ctx, time.Second, 3) + errCh <- err2 + }() + + select { + case <-ctx1.Done(): + case <-time.After(time.Second): + require.FailNow("should be canceled") + } + + select { + case <-errCh: + require.FailNow("should not be reached") + default: + } + + lock.Unlock() + + select { + case err = <-errCh: + require.NoError(err) + case <-time.After(time.Second): + require.FailNow("should be canceled") + } +} + +func TestPriorityLockSequential(t *testing.T) { + require := require.New(t) + + lock := v1alpha1.NewPriorityLock[testSequenceNumber]() + ctx := context.Background() + + _, err := lock.Lock(ctx, time.Second, 2) + require.NoError(err) + + lock.Unlock() + + _, err = lock.Lock(ctx, time.Second, 1) + require.NoError(err) + + lock.Unlock() +} + +//nolint:gocyclo +func TestPriorityLockConcurrent(t *testing.T) { + require := require.New(t) + + lock := v1alpha1.NewPriorityLock[testSequenceNumber]() + + globalCtx, globalCtxCancel := context.WithCancel(context.Background()) + defer globalCtxCancel() + + var eg errgroup.Group + + sequenceCh := make(chan testSequenceNumber) + + for seq := testSequenceNumber(1); seq <= 20; seq++ { + eg.Go(func() error { + ctx, err := lock.Lock(globalCtx, time.Second, seq) + if errors.Is(err, runtime.ErrLocked) { + return nil + } + + if err != nil { + return err + } + + select { + case sequenceCh <- seq: + <-ctx.Done() + case <-ctx.Done(): + } + + lock.Unlock() + + return nil + }) + } + + timer := time.NewTimer(5 * time.Second) + defer timer.Stop() + + var prevSeq testSequenceNumber + + for { + select { + case <-timer.C: + require.FailNow("timeout") + case seq := <-sequenceCh: + t.Logf("sequence running: %d", seq) + + if prevSeq >= seq { + require.FailNowf("can take over inversion", "sequence %d should be greater than %d", seq, prevSeq) + } + + prevSeq = seq + } + + if prevSeq == 20 { + globalCtxCancel() + + break + } + } + + require.NoError(eg.Wait()) +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_runtime.go b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_runtime.go new file mode 100644 index 0000000..854b6cd --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_runtime.go @@ -0,0 +1,234 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package v1alpha1 + +import ( + "context" + "errors" + "fmt" + "log" + "reflect" + "sync" + "sync/atomic" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/google/go-cmp/cmp" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/system" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/services" + "github.com/siderolabs/talos/pkg/machinery/config" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/resources/hardware" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +// Runtime implements the Runtime interface. +type Runtime struct { + c atomicInterface[config.Provider] + s runtime.State + e runtime.EventStream + l runtime.LoggingManager + + rollbackTimerMu sync.Mutex + rollbackTimer *time.Timer +} + +// NewRuntime initializes and returns the v1alpha1 runtime. +func NewRuntime(s runtime.State, e runtime.EventStream, l runtime.LoggingManager) *Runtime { + return &Runtime{ + s: s, + e: e, + l: l, + } +} + +// Config implements the Runtime interface. +func (r *Runtime) Config() config.Config { + return r.c.Load() +} + +// ConfigContainer implements the Runtime interface. +func (r *Runtime) ConfigContainer() config.Container { + return r.c.Load() +} + +// RollbackToConfigAfter implements the Runtime interface. +func (r *Runtime) RollbackToConfigAfter(timeout time.Duration) error { + cfgProvider := r.c.Load() + + r.CancelConfigRollbackTimeout() + + r.rollbackTimer = time.AfterFunc(timeout, func() { + log.Println("rolling back the configuration") + + if err := r.SetConfig(cfgProvider); err != nil { + log.Printf("config rollback failed %s", err) + } + }) + + return nil +} + +// CancelConfigRollbackTimeout implements the Runtime interface. +func (r *Runtime) CancelConfigRollbackTimeout() { + r.rollbackTimerMu.Lock() + defer r.rollbackTimerMu.Unlock() + + if r.rollbackTimer != nil { + r.rollbackTimer.Stop() + r.rollbackTimer = nil + } +} + +// SetConfig implements the Runtime interface. +func (r *Runtime) SetConfig(cfg config.Provider) error { + r.c.Store(cfg) + + return r.s.V1Alpha2().SetConfig(cfg) +} + +// CanApplyImmediate implements the Runtime interface. +func (r *Runtime) CanApplyImmediate(cfg config.Provider) error { + cfgProv := r.c.Load() + if cfgProv == nil { + return errors.New("no current config") + } + + currentConfig := cfgProv.RawV1Alpha1() + if currentConfig == nil { + return errors.New("current config is not v1alpha1") + } + + newConfig := cfg.RawV1Alpha1() + if newConfig == nil { + return errors.New("new config is not v1alpha1") + } + + // copy the config as we're going to modify it + newConfig = newConfig.DeepCopy() + + // the config changes allowed to be applied immediately are: + // * .debug + // * .cluster + // * .machine.ca + // * .machine.acceptedCAs + // * .machine.time + // * .machine.certCANs + // * .machine.install + // * .machine.network + // * .machine.sysfs + // * .machine.sysctls + // * .machine.logging + // * .machine.controlplane + // * .machine.kubelet + // * .machine.kernel + // * .machine.registries (note that auth is not applied immediately, containerd limitation) + // * .machine.pods + // * .machine.seccompProfiles + // * .machine.nodeLabels + // * .machine.nodeTaints + // * .machine.features.kubernetesTalosAPIAccess + // * .machine.features.kubePrism + // * .machine.features.localDNS + newConfig.ConfigDebug = currentConfig.ConfigDebug + newConfig.ClusterConfig = currentConfig.ClusterConfig + + if newConfig.MachineConfig != nil && currentConfig.MachineConfig != nil { + newConfig.MachineConfig.MachineCA = currentConfig.MachineConfig.MachineCA + newConfig.MachineConfig.MachineAcceptedCAs = currentConfig.MachineConfig.MachineAcceptedCAs + newConfig.MachineConfig.MachineTime = currentConfig.MachineConfig.MachineTime + newConfig.MachineConfig.MachineCertSANs = currentConfig.MachineConfig.MachineCertSANs + newConfig.MachineConfig.MachineInstall = currentConfig.MachineConfig.MachineInstall + newConfig.MachineConfig.MachineNetwork = currentConfig.MachineConfig.MachineNetwork + newConfig.MachineConfig.MachineSysfs = currentConfig.MachineConfig.MachineSysfs + newConfig.MachineConfig.MachineSysctls = currentConfig.MachineConfig.MachineSysctls + newConfig.MachineConfig.MachineLogging = currentConfig.MachineConfig.MachineLogging + newConfig.MachineConfig.MachineControlPlane = currentConfig.MachineConfig.MachineControlPlane + newConfig.MachineConfig.MachineKubelet = currentConfig.MachineConfig.MachineKubelet + newConfig.MachineConfig.MachineKernel = currentConfig.MachineConfig.MachineKernel + newConfig.MachineConfig.MachineRegistries = currentConfig.MachineConfig.MachineRegistries + newConfig.MachineConfig.MachinePods = currentConfig.MachineConfig.MachinePods + newConfig.MachineConfig.MachineSeccompProfiles = currentConfig.MachineConfig.MachineSeccompProfiles + newConfig.MachineConfig.MachineNodeLabels = currentConfig.MachineConfig.MachineNodeLabels + newConfig.MachineConfig.MachineNodeTaints = currentConfig.MachineConfig.MachineNodeTaints + + if newConfig.MachineConfig.MachineFeatures != nil && currentConfig.MachineConfig.MachineFeatures != nil { + newConfig.MachineConfig.MachineFeatures.KubernetesTalosAPIAccessConfig = currentConfig.MachineConfig.MachineFeatures.KubernetesTalosAPIAccessConfig + newConfig.MachineConfig.MachineFeatures.KubePrismSupport = currentConfig.MachineConfig.MachineFeatures.KubePrismSupport + newConfig.MachineConfig.MachineFeatures.HostDNSSupport = currentConfig.MachineConfig.MachineFeatures.HostDNSSupport + } + } + + if !reflect.DeepEqual(currentConfig, newConfig) { + diff := cmp.Diff(currentConfig, newConfig, cmp.AllowUnexported(v1alpha1.InstallDiskSizeMatcher{})) + + return fmt.Errorf("this config change can't be applied in immediate mode\ndiff: %s", diff) + } + + return nil +} + +// State implements the Runtime interface. +func (r *Runtime) State() runtime.State { + return r.s +} + +// Events implements the Runtime interface. +func (r *Runtime) Events() runtime.EventStream { + return r.e +} + +// Logging implements the Runtime interface. +func (r *Runtime) Logging() runtime.LoggingManager { + return r.l +} + +// NodeName implements the Runtime interface. +func (r *Runtime) NodeName() (string, error) { + nodenameResource, err := r.s.V1Alpha2().Resources().Get(context.Background(), resource.NewMetadata(k8s.NamespaceName, k8s.NodenameType, k8s.NodenameID, resource.VersionUndefined)) + if err != nil { + return "", fmt.Errorf("error getting nodename resource: %w", err) + } + + return nodenameResource.(*k8s.Nodename).TypedSpec().Nodename, nil +} + +// IsBootstrapAllowed checks for CRI to be up, checked in the bootstrap method. +func (r *Runtime) IsBootstrapAllowed() bool { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + svc := &services.CRI{} + if err := system.WaitForService(system.StateEventUp, svc.ID(r)).Wait(ctx); err != nil { + return false + } + + return true +} + +// GetSystemInformation returns system information resource if it exists. +func (r *Runtime) GetSystemInformation(ctx context.Context) (*hardware.SystemInformation, error) { + return safe.StateGet[*hardware.SystemInformation](ctx, r.State().V1Alpha2().Resources(), hardware.NewSystemInformation(hardware.SystemInformationID).Metadata()) +} + +// atomicInterface is a typed wrapper around atomic.Value. It's only useful for storing the interfaces, because +// you don't need another layer of indirection (unlike atomic.Pointer[T]) to load the value. For concrete types +// please use atomic.Pointer. +type atomicInterface[T any] struct{ v atomic.Value } + +func (a *atomicInterface[T]) Load() T { + if val := a.v.Load(); val != nil { + return val.(T) //nolint:forcetypeassert + } + + var zero T + + return zero +} + +func (a *atomicInterface[T]) Store(v T) { a.v.Store(v) } diff --git a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer.go b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer.go new file mode 100644 index 0000000..d8cbeca --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer.go @@ -0,0 +1,578 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package v1alpha1 + +import ( + "strconv" + + "github.com/siderolabs/go-pointer" + "github.com/siderolabs/go-procfs/procfs" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// Sequencer implements the sequencer interface. +type Sequencer struct{} + +// NewSequencer intializes and returns a sequencer. +func NewSequencer() *Sequencer { + return &Sequencer{} +} + +// PhaseList represents a list of phases. +type PhaseList []runtime.Phase + +// Append appends a task to the phase list. +func (p PhaseList) Append(name string, tasks ...runtime.TaskSetupFunc) PhaseList { + p = append(p, runtime.Phase{ + Name: name, + Tasks: tasks, + }) + + return p +} + +// AppendWhen appends a task to the phase list when `when` is `true`. +func (p PhaseList) AppendWhen(when bool, name string, tasks ...runtime.TaskSetupFunc) PhaseList { + if when { + p = p.Append(name, tasks...) + } + + return p +} + +// AppendWithDeferredCheck appends a task to the phase list but skips the sequence if `check func()` returns `false` during execution. +func (p PhaseList) AppendWithDeferredCheck(check func() bool, name string, tasks ...runtime.TaskSetupFunc) PhaseList { + p = append(p, runtime.Phase{ + Name: name, + Tasks: tasks, + CheckFunc: check, + }) + + return p +} + +// AppendList appends an additional PhaseList to the existing one. +func (p PhaseList) AppendList(list PhaseList) PhaseList { + return append(p, list...) +} + +// Initialize is the initialize sequence. The primary goals of this sequence is +// to load the config and enforce kernel security requirements. +func (*Sequencer) Initialize(r runtime.Runtime) []runtime.Phase { + phases := PhaseList{} + + switch r.State().Platform().Mode() { //nolint:exhaustive + case runtime.ModeContainer: + phases = phases.Append( + "systemRequirements", + SetupSystemDirectory, + ).Append( + "etc", + CreateSystemCgroups, + CreateOSReleaseFile, + SetUserEnvVars, + ).Append( + "machined", + StartMachined, + StartContainerd, + ).Append( + "config", + LoadConfig, + ) + default: + phases = phases.Append( + "systemRequirements", + EnforceKSPPRequirements, + SetupSystemDirectory, + MountBPFFS, + MountCgroups, + MountPseudoFilesystems, + SetRLimit, + ).Append( + "integrity", + WriteIMAPolicy, + ).Append( + "etc", + CreateSystemCgroups, + CreateOSReleaseFile, + SetUserEnvVars, + ).Append( + "earlyServices", + StartUdevd, + StartMachined, + StartSyslogd, + StartContainerd, + ).Append( + "usb", + WaitForUSB, + ).Append( + "meta", + ReloadMeta, + ).AppendWithDeferredCheck( + func() bool { + disabledStr := procfs.ProcCmdline().Get(constants.KernelParamDashboardDisabled).First() + disabled, _ := strconv.ParseBool(pointer.SafeDeref(disabledStr)) //nolint:errcheck + + return !disabled + }, + "dashboard", + StartDashboard, + ).AppendWithDeferredCheck( + func() bool { + wipe := procfs.ProcCmdline().Get(constants.KernelParamWipe).First() + + return pointer.SafeDeref(wipe) != "" + }, + "wipeDisks", + ResetSystemDiskPartitions, + ).AppendWithDeferredCheck( + func() bool { + return r.State().Machine().Installed() + }, + "mountSystem", + MountStatePartition, + ).Append( + "config", + LoadConfig, + ).AppendWithDeferredCheck( + func() bool { + return r.State().Machine().Installed() + }, + "unmountSystem", + UnmountStatePartition, + ) + } + + return phases +} + +// Install is the install sequence. +func (*Sequencer) Install(r runtime.Runtime) []runtime.Phase { + phases := PhaseList{} + + switch r.State().Platform().Mode() { //nolint:exhaustive + case runtime.ModeContainer: + return nil + default: + if !r.State().Machine().Installed() || r.State().Machine().IsInstallStaged() { + phases = phases.Append( + "env", + SetUserEnvVars, + ).Append( + "install", + Install, + ).Append( + "meta", + ReloadMeta, + ).Append( + "saveMeta", // saving META here to merge in-memory changes with the on-disk ones from the installer + FlushMeta, + ).Append( + "saveStateEncryptionConfig", + SaveStateEncryptionConfig, + ).Append( + "mountState", + MountStatePartition, + ).Append( + "saveConfig", + SaveConfig, + ).Append( + "unmountState", + UnmountStatePartition, + ).Append( + "stopEverything", + StopAllServices, + ).Append( + "kexec", + KexecPrepare, + ).Append( + "reboot", + Reboot, + ) + } + } + + return phases +} + +// Boot is the boot sequence. This primary goal if this sequence is to apply +// user supplied settings and start the services for the specific machine type. +// This sequence should never be reached if an installation is not found. +func (*Sequencer) Boot(r runtime.Runtime) []runtime.Phase { + phases := PhaseList{} + + phases = phases.AppendWhen( + r.State().Platform().Mode() != runtime.ModeContainer, + "saveStateEncryptionConfig", + SaveStateEncryptionConfig, + ).AppendWhen( + r.State().Platform().Mode() != runtime.ModeContainer, + "mountState", + MountStatePartition, + ).Append( + "saveConfig", + SaveConfig, + ).Append( + "memorySizeCheck", + MemorySizeCheck, + ).Append( + "diskSizeCheck", + DiskSizeCheck, + ).Append( + "env", + SetUserEnvVars, + ).Append( + "dbus", + StartDBus, + ).AppendWhen( + r.State().Platform().Mode() == runtime.ModeContainer, + "sharedFilesystems", + SetupSharedFilesystems, + ).AppendWhen( + r.State().Platform().Mode() != runtime.ModeContainer, + "ephemeral", + MountEphemeralPartition, + ).Append( + "var", + SetupVarDirectory, + ).AppendWhen( + r.State().Platform().Mode() != runtime.ModeContainer, + "overlay", + MountOverlayFilesystems, + ).AppendWhen( + r.State().Platform().Mode() != runtime.ModeContainer, + "udevSetup", + WriteUdevRules, + ).AppendWhen( + r.State().Platform().Mode() != runtime.ModeContainer, + "userDisks", + pauseOnFailure(MountUserDisks, constants.FailurePauseTimeout), + ).Append( + "userSetup", + pauseOnFailure(WriteUserFiles, constants.FailurePauseTimeout), + ).AppendWhen( + r.State().Platform().Mode() != runtime.ModeContainer, + "lvm", + ActivateLogicalVolumes, + ).Append( + "extendPCRStartAll", + ExtendPCRStartAll, + ).Append( + "startEverything", + StartAllServices, + ) + + return phases +} + +// Reboot is the reboot sequence. +func (*Sequencer) Reboot(r runtime.Runtime) []runtime.Phase { + phases := PhaseList{}.Append( + "cleanup", + StopAllPods, + ).Append( + "dbus", + StopDBus, + ). + AppendList(stopAllPhaselist(r, true)). + Append("reboot", Reboot) + + return phases +} + +// Reset is the reset sequence. +// +//nolint:gocyclo +func (*Sequencer) Reset(r runtime.Runtime, in runtime.ResetOptions) []runtime.Phase { + phases := PhaseList{} + + // Use kexec if we don't wipe the boot partition. + withKexec := false + if len(in.GetSystemDiskTargets()) > 0 { + withKexec = !bootPartitionInTargets(in.GetSystemDiskTargets()) + } + + var ( + resetUserDisks bool + resetSystemDisk bool + ) + + switch in.GetMode() { + case machineapi.ResetRequest_ALL: + resetUserDisks = true + resetSystemDisk = true + case machineapi.ResetRequest_USER_DISKS: + resetUserDisks = true + case machineapi.ResetRequest_SYSTEM_DISK: + resetSystemDisk = true + } + + switch r.State().Platform().Mode() { //nolint:exhaustive + case runtime.ModeContainer: + phases = phases.AppendList(stopAllPhaselist(r, false)). + Append( + "shutdown", + Shutdown, + ) + default: + phases = phases.AppendWhen( + in.GetGraceful() && !r.Config().Machine().Kubelet().SkipNodeRegistration(), + "drain", + taskErrorHandler(logError, CordonAndDrainNode), + ).AppendWhen( + in.GetGraceful(), + "cleanup", + taskErrorHandler(logError, RemoveAllPods), + ).AppendWhen( + !in.GetGraceful(), + "cleanup", + taskErrorHandler(logError, StopAllPods), + ).Append( + "dbus", + StopDBus, + ).AppendWhen( + in.GetGraceful() && (r.Config().Machine().Type() != machine.TypeWorker), + "leave", + LeaveEtcd, + ).Append( + "preReset", + SendResetSignal, + ).AppendList( + phaseListErrorHandler(logError, stopAllPhaselist(r, withKexec)...), + ).Append( + "forceCleanup", + ForceCleanup, + ).AppendWhen( + len(in.GetSystemDiskTargets()) == 0 && resetSystemDisk, + "reset", + ResetSystemDisk, + ).AppendWhen( + len(in.GetSystemDiskTargets()) > 0 && resetSystemDisk, + "resetSpec", + ResetSystemDiskSpec, + ).AppendWhen( + len(in.GetUserDisksToWipe()) > 0 && resetUserDisks, + "resetUserDisks", + ResetUserDisks, + ).AppendWhen( + in.GetReboot(), + "reboot", + Reboot, + ).AppendWhen( + !in.GetReboot(), + "shutdown", + Shutdown, + ) + } + + return phases +} + +// Shutdown is the shutdown sequence. +func (*Sequencer) Shutdown(r runtime.Runtime, in *machineapi.ShutdownRequest) []runtime.Phase { + phases := PhaseList{}.Append( + "storeShudown", + StoreShutdownEmergency, + ).AppendWhen( + !in.GetForce() && !r.Config().Machine().Kubelet().SkipNodeRegistration(), + "drain", + CordonAndDrainNode, + ).Append( + "cleanup", + StopAllPods, + ).Append( + "dbus", + StopDBus, + ). + AppendList(stopAllPhaselist(r, false)). + Append("shutdown", Shutdown) + + return phases +} + +// StageUpgrade is the stage upgrade sequence. +func (*Sequencer) StageUpgrade(r runtime.Runtime, in *machineapi.UpgradeRequest) []runtime.Phase { + phases := PhaseList{} + + switch r.State().Platform().Mode() { //nolint:exhaustive + case runtime.ModeContainer: + return nil + default: + phases = phases.Append( + "cleanup", + StopAllPods, + ).Append( + "dbus", + StopDBus, + ).AppendWhen( + !in.GetPreserve() && (r.Config().Machine().Type() != machine.TypeWorker), + "leave", + LeaveEtcd, + ).AppendList( + stopAllPhaselist(r, in.GetRebootMode() == machineapi.UpgradeRequest_DEFAULT), + ).Append( + "reboot", + Reboot, + ) + } + + return phases +} + +// MaintenanceUpgrade is the upgrade sequence in maintenance mode. +func (*Sequencer) MaintenanceUpgrade(r runtime.Runtime, in *machineapi.UpgradeRequest) []runtime.Phase { + phases := PhaseList{} + + switch r.State().Platform().Mode() { //nolint:exhaustive + case runtime.ModeContainer: + return nil + default: + phases = phases.Append( + "verifyDisk", + VerifyDiskAvailability, + ).Append( + "upgrade", + Upgrade, + ).Append( + "meta", + ReloadMeta, + ).AppendWhen( + in.GetRebootMode() == machineapi.UpgradeRequest_DEFAULT, + "kexec", + KexecPrepare, + ).Append( + "stopEverything", + StopAllServices, + ).Append( + "reboot", + Reboot, + ) + } + + return phases +} + +// Upgrade is the upgrade sequence. +func (*Sequencer) Upgrade(r runtime.Runtime, in *machineapi.UpgradeRequest) []runtime.Phase { + phases := PhaseList{} + + switch r.State().Platform().Mode() { //nolint:exhaustive + case runtime.ModeContainer: + return nil + default: + phases = phases.AppendWhen( + !r.Config().Machine().Kubelet().SkipNodeRegistration(), + "drain", + CordonAndDrainNode, + ).AppendWhen( + !in.GetPreserve(), + "cleanup", + RemoveAllPods, + ).AppendWhen( + in.GetPreserve(), + "cleanup", + StopAllPods, + ).Append( + "dbus", + StopDBus, + ).AppendWhen( + !in.GetPreserve() && (r.Config().Machine().Type() != machine.TypeWorker), + "leave", + LeaveEtcd, + ).Append( + "stopServices", + StopServicesEphemeral, + ).Append( + "unmountUser", + UnmountUserDisks, + ).Append( + "unmount", + UnmountOverlayFilesystems, + UnmountPodMounts, + ).Append( + "unmountBind", + UnmountSystemDiskBindMounts, + ).Append( + "unmountSystem", + UnmountEphemeralPartition, + UnmountStatePartition, + ).Append( + "verifyDisk", + VerifyDiskAvailability, + ).Append( + "upgrade", + Upgrade, + ).Append( + "meta", + ReloadMeta, + ).AppendWhen( + in.GetRebootMode() == machineapi.UpgradeRequest_DEFAULT, + "kexec", + KexecPrepare, + ).Append( + "stopEverything", + StopAllServices, + ).Append( + "reboot", + Reboot, + ) + } + + return phases +} + +func stopAllPhaselist(r runtime.Runtime, enableKexec bool) PhaseList { + phases := PhaseList{} + + switch r.State().Platform().Mode() { //nolint:exhaustive + case runtime.ModeContainer: + phases = phases.Append( + "stopEverything", + StopAllServices, + ) + default: + phases = phases.Append( + "stopServices", + StopServicesEphemeral, + ).Append( + "unmountUser", + UnmountUserDisks, + ).Append( + "umount", + UnmountOverlayFilesystems, + UnmountPodMounts, + ).Append( + "unmountBind", + UnmountSystemDiskBindMounts, + ).Append( + "unmountSystem", + UnmountEphemeralPartition, + UnmountStatePartition, + ).AppendWhen( + enableKexec, + "kexec", + KexecPrepare, + ).Append( + "stopEverything", + StopAllServices, + ) + } + + return phases +} + +func bootPartitionInTargets(targets []runtime.PartitionTarget) bool { + for _, target := range targets { + if target.GetLabel() == constants.BootPartitionLabel { + return true + } + } + + return false +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go new file mode 100644 index 0000000..fa664c5 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go @@ -0,0 +1,2348 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package v1alpha1 + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "syscall" + "time" + + "github.com/containerd/cgroups/v3" + "github.com/containerd/cgroups/v3/cgroup1" + "github.com/containerd/cgroups/v3/cgroup2" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/dustin/go-humanize" + "github.com/hashicorp/go-multierror" + "github.com/opencontainers/runtime-spec/specs-go" + pprocfs "github.com/prometheus/procfs" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/go-blockdevice/blockdevice" + "github.com/siderolabs/go-blockdevice/blockdevice/partition/gpt" + "github.com/siderolabs/go-blockdevice/blockdevice/util" + "github.com/siderolabs/go-cmd/pkg/cmd" + "github.com/siderolabs/go-cmd/pkg/cmd/proc" + "github.com/siderolabs/go-pointer" + "github.com/siderolabs/go-procfs/procfs" + "github.com/siderolabs/go-retry/retry" + clientv3 "go.etcd.io/etcd/client/v3" + "golang.org/x/sys/unix" + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + + installer "github.com/siderolabs/talos/cmd/installer/pkg/install" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/disk" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/emergency" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform" + "github.com/aenix-io/talm/internal/app/machined/pkg/system" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/events" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/services" + "github.com/aenix-io/talm/internal/pkg/cgroup" + "github.com/aenix-io/talm/internal/pkg/cri" + "github.com/aenix-io/talm/internal/pkg/environment" + "github.com/aenix-io/talm/internal/pkg/etcd" + "github.com/aenix-io/talm/internal/pkg/install" + "github.com/aenix-io/talm/internal/pkg/logind" + "github.com/aenix-io/talm/internal/pkg/meta" + "github.com/aenix-io/talm/internal/pkg/mount" + "github.com/aenix-io/talm/internal/pkg/partition" + "github.com/aenix-io/talm/internal/pkg/secureboot" + "github.com/aenix-io/talm/internal/pkg/secureboot/tpm2" + "github.com/siderolabs/talos/pkg/conditions" + "github.com/siderolabs/talos/pkg/images" + "github.com/siderolabs/talos/pkg/kernel/kspp" + "github.com/siderolabs/talos/pkg/kubernetes" + machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/config/config" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/constants" + metamachinery "github.com/siderolabs/talos/pkg/machinery/meta" + resourcefiles "github.com/siderolabs/talos/pkg/machinery/resources/files" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + resourceruntime "github.com/siderolabs/talos/pkg/machinery/resources/runtime" + resourcev1alpha1 "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/version" + "github.com/siderolabs/talos/pkg/minimal" +) + +// WaitForUSB represents the WaitForUSB task. +func WaitForUSB(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) error { + // Wait for USB storage in the case that the install disk is supplied over + // USB. If we don't wait, there is the chance that we will fail to detect the + // install disk. + file := "/sys/module/usb_storage/parameters/delay_use" + + _, err := os.Stat(file) + if err != nil { + if os.IsNotExist(err) { + return nil + } + + return err + } + + b, err := os.ReadFile(file) + if err != nil { + return err + } + + val := strings.TrimSuffix(string(b), "\n") + + var i int + + i, err = strconv.Atoi(val) + if err != nil { + return err + } + + logger.Printf("waiting %d second(s) for USB storage", i) + + time.Sleep(time.Duration(i) * time.Second) + + return nil + }, "waitForUSB" +} + +// EnforceKSPPRequirements represents the EnforceKSPPRequirements task. +func EnforceKSPPRequirements(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + if err = resourceruntime.NewKernelParamsSetCondition(r.State().V1Alpha2().Resources(), kspp.GetKernelParams()...).Wait(ctx); err != nil { + return err + } + + return kspp.EnforceKSPPKernelParameters() + }, "enforceKSPPRequirements" +} + +// SetupSystemDirectory represents the SetupSystemDirectory task. +func SetupSystemDirectory(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + for _, p := range []string{constants.SystemEtcPath, constants.SystemVarPath, constants.StateMountPoint} { + if err = os.MkdirAll(p, 0o700); err != nil { + return err + } + } + + for _, p := range []string{constants.SystemRunPath} { + if err = os.MkdirAll(p, 0o751); err != nil { + return err + } + } + + return nil + }, "setupSystemDirectory" +} + +// CreateSystemCgroups represents the CreateSystemCgroups task. +// +//nolint:gocyclo +func CreateSystemCgroups(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + // in container mode cgroups mode depends on cgroups provided by the container runtime + if r.State().Platform().Mode() != runtime.ModeContainer { + // assert that cgroupsv2 is being used when running not in container mode, + // as Talos sets up cgroupsv2 on its own + if cgroups.Mode() != cgroups.Unified && !mount.ForceGGroupsV1() { + return errors.New("cgroupsv2 should be used") + } + } + + // Initialize cgroups root path. + if err = cgroup.InitRoot(); err != nil { + return fmt.Errorf("error initializing cgroups root path: %w", err) + } + + groups := []struct { + name string + resources *cgroup2.Resources + }{ + { + name: constants.CgroupInit, + resources: &cgroup2.Resources{ + Memory: &cgroup2.Memory{ + Min: pointer.To[int64](constants.CgroupInitReservedMemory), + Low: pointer.To[int64](constants.CgroupInitReservedMemory * 2), + }, + }, + }, + { + name: constants.CgroupSystem, + resources: &cgroup2.Resources{ + Memory: &cgroup2.Memory{ + Min: pointer.To[int64](constants.CgroupSystemReservedMemory), + Low: pointer.To[int64](constants.CgroupSystemReservedMemory * 2), + }, + }, + }, + { + name: constants.CgroupSystemRuntime, + resources: &cgroup2.Resources{}, + }, + { + name: constants.CgroupUdevd, + resources: &cgroup2.Resources{}, + }, + { + name: constants.CgroupPodRuntime, + resources: &cgroup2.Resources{ + Memory: &cgroup2.Memory{ + Min: pointer.To[int64](constants.CgroupPodRuntimeReservedMemory), + Low: pointer.To[int64](constants.CgroupPodRuntimeReservedMemory * 2), + }, + }, + }, + { + name: constants.CgroupKubelet, + resources: &cgroup2.Resources{ + Memory: &cgroup2.Memory{ + Min: pointer.To[int64](constants.CgroupKubeletReservedMemory), + Low: pointer.To[int64](constants.CgroupKubeletReservedMemory * 2), + }, + }, + }, + { + name: constants.CgroupDashboard, + resources: &cgroup2.Resources{ + Memory: &cgroup2.Memory{ + Min: pointer.To[int64](constants.CgroupDashboardReservedMemory), + Low: pointer.To[int64](constants.CgroupDashboardLowMemory), + }, + }, + }, + } + + for _, c := range groups { + if cgroups.Mode() == cgroups.Unified { + resources := c.resources + + if r.State().Platform().Mode() == runtime.ModeContainer { + // don't attempt to set resources in container mode, as they might conflict with the parent cgroup tree + resources = &cgroup2.Resources{} + } + + cg, err := cgroup2.NewManager(constants.CgroupMountPath, cgroup.Path(c.name), resources) + if err != nil { + return fmt.Errorf("failed to create cgroup: %w", err) + } + + if c.name == constants.CgroupInit { + if err := cg.AddProc(uint64(os.Getpid())); err != nil { + return fmt.Errorf("failed to move init process to cgroup: %w", err) + } + } + } else { + cg, err := cgroup1.New(cgroup1.StaticPath(c.name), &specs.LinuxResources{}) + if err != nil { + return fmt.Errorf("failed to create cgroup: %w", err) + } + + if c.name == constants.CgroupInit { + if err := cg.Add(cgroup1.Process{ + Pid: os.Getpid(), + }); err != nil { + return fmt.Errorf("failed to move init process to cgroup: %w", err) + } + } + } + } + + return nil + }, "CreateSystemCgroups" +} + +// MountBPFFS represents the MountBPFFS task. +func MountBPFFS(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + var mountpoints *mount.Points + + mountpoints, err = mount.BPFMountPoints() + if err != nil { + return err + } + + return mount.Mount(mountpoints) + }, "mountBPFFS" +} + +// MountCgroups represents the MountCgroups task. +func MountCgroups(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + var mountpoints *mount.Points + + mountpoints, err = mount.CGroupMountPoints() + if err != nil { + return err + } + + return mount.Mount(mountpoints) + }, "mountCgroups" +} + +// MountPseudoFilesystems represents the MountPseudoFilesystems task. +func MountPseudoFilesystems(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + var mountpoints *mount.Points + + mountpoints, err = mount.PseudoSubMountPoints() + if err != nil { + return err + } + + return mount.Mount(mountpoints) + }, "mountPseudoFilesystems" +} + +// SetRLimit represents the SetRLimit task. +func SetRLimit(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + // TODO(andrewrynhard): Should we read limit from /proc/sys/fs/nr_open? + return unix.Setrlimit(unix.RLIMIT_NOFILE, &unix.Rlimit{Cur: 1048576, Max: 1048576}) + }, "setRLimit" +} + +// See https://www.kernel.org/doc/Documentation/ABI/testing/ima_policy +var rules = []string{ + "dont_measure fsmagic=0x9fa0", // PROC_SUPER_MAGIC + "dont_measure fsmagic=0x62656572", // SYSFS_MAGIC + "dont_measure fsmagic=0x64626720", // DEBUGFS_MAGIC + "dont_measure fsmagic=0x1021994", // TMPFS_MAGIC + "dont_measure fsmagic=0x1cd1", // DEVPTS_SUPER_MAGIC + "dont_measure fsmagic=0x42494e4d", // BINFMTFS_MAGIC + "dont_measure fsmagic=0x73636673", // SECURITYFS_MAGIC + "dont_measure fsmagic=0xf97cff8c", // SELINUX_MAGIC + "dont_measure fsmagic=0x43415d53", // SMACK_MAGIC + "dont_measure fsmagic=0x27e0eb", // CGROUP_SUPER_MAGIC + "dont_measure fsmagic=0x63677270", // CGROUP2_SUPER_MAGIC + "dont_measure fsmagic=0x6e736673", // NSFS_MAGIC + "dont_measure fsmagic=0xde5e81e4", // EFIVARFS_MAGIC + "dont_measure fsmagic=0x58465342", // XFS_MAGIC + "dont_measure fsmagic=0x794c7630", // OVERLAYFS_SUPER_MAGIC + "dont_measure fsmagic=0x9123683e", // BTRFS_SUPER_MAGIC + "dont_measure fsmagic=0x72b6", // JFFS2_SUPER_MAGIC + "dont_measure fsmagic=0x4d44", // MSDOS_SUPER_MAGIC + "dont_measure fsmagic=0x2011bab0", // EXFAT_SUPER_MAGIC + "dont_measure fsmagic=0x6969", // NFS_SUPER_MAGIC + "dont_measure fsmagic=0x5346544e", // NTFS_SB_MAGIC + "dont_measure fsmagic=0x9660", // ISOFS_SUPER_MAGIC + "dont_measure fsmagic=0x15013346", // UDF_SUPER_MAGIC + "dont_measure fsmagic=0x52654973", // REISERFS_SUPER_MAGIC + "dont_measure fsmagic=0x137d", // EXT_SUPER_MAGIC + "dont_measure fsmagic=0xef51", // EXT2_OLD_SUPER_MAGIC + "dont_measure fsmagic=0xef53", // EXT2_SUPER_MAGIC / EXT3_SUPER_MAGIC / EXT4_SUPER_MAGIC + "dont_measure fsmagic=0x00c36400", // CEPH_SUPER_MAGIC + "dont_measure fsmagic=0x65735543", // FUSE_CTL_SUPER_MAGIC + "measure func=MMAP_CHECK mask=MAY_EXEC", + "measure func=BPRM_CHECK mask=MAY_EXEC", + "measure func=FILE_CHECK mask=^MAY_READ euid=0", + "measure func=FILE_CHECK mask=^MAY_READ uid=0", + "measure func=MODULE_CHECK", + "measure func=FIRMWARE_CHECK", + "measure func=POLICY_CHECK", +} + +// WriteIMAPolicy represents the WriteIMAPolicy task. +func WriteIMAPolicy(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + if _, err = os.Stat("/sys/kernel/security/ima/policy"); os.IsNotExist(err) { + return fmt.Errorf("policy file does not exist: %w", err) + } + + f, err := os.OpenFile("/sys/kernel/security/ima/policy", os.O_APPEND|os.O_WRONLY, 0o644) + if err != nil { + return err + } + + defer f.Close() //nolint:errcheck + + for _, line := range rules { + if _, err = f.WriteString(line + "\n"); err != nil { + return fmt.Errorf("rule %q is invalid", err) + } + } + + return nil + }, "writeIMAPolicy" +} + +// OSRelease renders a valid /etc/os-release file and writes it to disk. The +// node's OS Image field is reported by the node from /etc/os-release. +func OSRelease() (err error) { + if err = createBindMount(filepath.Join(constants.SystemEtcPath, "os-release"), "/etc/os-release"); err != nil { + return err + } + + contents, err := version.OSRelease() + if err != nil { + return err + } + + return os.WriteFile(filepath.Join(constants.SystemEtcPath, "os-release"), contents, 0o644) +} + +// createBindMount creates a common way to create a writable source file with a +// bind mounted destination. This is most commonly used for well known files +// under /etc that need to be adjusted during startup. +func createBindMount(src, dst string) (err error) { + var f *os.File + + if f, err = os.OpenFile(src, os.O_WRONLY|os.O_CREATE, 0o644); err != nil { + return err + } + + if err = f.Close(); err != nil { + return err + } + + if err = unix.Mount(src, dst, "", unix.MS_BIND, ""); err != nil { + return fmt.Errorf("failed to create bind mount for %s: %w", dst, err) + } + + return nil +} + +// CreateOSReleaseFile represents the CreateOSReleaseFile task. +func CreateOSReleaseFile(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + // Create /etc/os-release. + return OSRelease() + }, "createOSReleaseFile" +} + +// LoadConfig represents the LoadConfig task. +func LoadConfig(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) error { + // create a request to initialize the process acquisition process + request := resourcev1alpha1.NewAcquireConfigSpec() + if err := r.State().V1Alpha2().Resources().Create(ctx, request); err != nil { + return fmt.Errorf("failed to create config request: %w", err) + } + + // wait for the config to be acquired + status := resourcev1alpha1.NewAcquireConfigStatus() + if _, err := r.State().V1Alpha2().Resources().WatchFor(ctx, status.Metadata(), state.WithEventTypes(state.Created)); err != nil { + return err + } + + // clean up request to make sure controller doesn't work after this point + return r.State().V1Alpha2().Resources().Destroy(ctx, request.Metadata()) + }, "loadConfig" +} + +// SaveConfig represents the SaveConfig task. +func SaveConfig(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + var b []byte + + b, err = r.ConfigContainer().Bytes() + if err != nil { + return err + } + + return os.WriteFile(constants.ConfigPath, b, 0o600) + }, "saveConfig" +} + +// MemorySizeCheck represents the MemorySizeCheck task. +func MemorySizeCheck(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) error { + if r.State().Platform().Mode() == runtime.ModeContainer { + log.Println("skipping memory size check in the container") + + return nil + } + + pc, err := pprocfs.NewDefaultFS() + if err != nil { + return fmt.Errorf("failed to open procfs: %w", err) + } + + info, err := pc.Meminfo() + if err != nil { + return fmt.Errorf("failed to read meminfo: %w", err) + } + + minimum, recommended, err := minimal.Memory(r.Config().Machine().Type()) + if err != nil { + return err + } + + switch memTotal := pointer.SafeDeref(info.MemTotal) * humanize.KiByte; { + case memTotal < minimum: + log.Println("WARNING: memory size is less than recommended") + log.Println("WARNING: Talos may not work properly") + log.Println("WARNING: minimum memory size is", minimum/humanize.MiByte, "MiB") + log.Println("WARNING: recommended memory size is", recommended/humanize.MiByte, "MiB") + log.Println("WARNING: current total memory size is", memTotal/humanize.MiByte, "MiB") + case memTotal < recommended: + log.Println("NOTE: recommended memory size is", recommended/humanize.MiByte, "MiB") + log.Println("NOTE: current total memory size is", memTotal/humanize.MiByte, "MiB") + default: + log.Println("memory size is OK") + log.Println("memory size is", memTotal/humanize.MiByte, "MiB") + } + + return nil + }, "memorySizeCheck" +} + +// DiskSizeCheck represents the DiskSizeCheck task. +func DiskSizeCheck(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) error { + if r.State().Platform().Mode() == runtime.ModeContainer { + log.Println("skipping disk size check in the container") + + return nil + } + + disk := r.State().Machine().Disk() // get ephemeral disk state + if disk == nil { + return errors.New("failed to get ephemeral disk state") + } + + diskSize, err := disk.Size() + if err != nil { + return fmt.Errorf("failed to get ephemeral disk size: %w", err) + } + + if minimum := minimal.DiskSize(); diskSize < minimum { + log.Println("WARNING: disk size is less than recommended") + log.Println("WARNING: Talos may not work properly") + log.Println("WARNING: minimum recommended disk size is", minimum/humanize.MiByte, "MiB") + log.Println("WARNING: current total disk size is", diskSize/humanize.MiByte, "MiB") + } else { + log.Println("disk size is OK") + log.Println("disk size is", diskSize/humanize.MiByte, "MiB") + } + + return nil + }, "diskSizeCheck" +} + +// SetUserEnvVars represents the SetUserEnvVars task. +func SetUserEnvVars(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + for _, env := range environment.Get(r.Config()) { + key, val, _ := strings.Cut(env, "=") + + if err = os.Setenv(key, val); err != nil { + return fmt.Errorf("failed to set enivronment variable: %w", err) + } + } + + return nil + }, "setUserEnvVars" +} + +// StartContainerd represents the task to start containerd. +func StartContainerd(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + svc := &services.Containerd{} + + system.Services(r).LoadAndStart(svc) + + ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + + return system.WaitForService(system.StateEventUp, svc.ID(r)).Wait(ctx) + }, "startContainerd" +} + +// WriteUdevRules is the task that writes udev rules to a udev rules file. +// TODO: frezbo: move this to controller based since writing udev rules doesn't need a restart. +func WriteUdevRules(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + rules := r.Config().Machine().Udev().Rules() + + var content strings.Builder + + for _, rule := range rules { + content.WriteString(strings.ReplaceAll(rule, "\n", "\\\n")) + content.WriteByte('\n') + } + + if err = os.WriteFile(constants.UdevRulesPath, []byte(content.String()), 0o644); err != nil { + return fmt.Errorf("failed writing custom udev rules: %w", err) + } + + if len(rules) > 0 { + if _, err := cmd.RunContext(ctx, "/sbin/udevadm", "control", "--reload"); err != nil { + return err + } + + if _, err := cmd.RunContext(ctx, "/sbin/udevadm", "trigger", "--type=devices", "--action=add"); err != nil { + return err + } + + if _, err := cmd.RunContext(ctx, "/sbin/udevadm", "trigger", "--type=subsystems", "--action=add"); err != nil { + return err + } + + // This ensures that `udevd` finishes processing kernel events, triggered by + // `udevd trigger`, to prevent a race condition when a user specifies a path + // under `/dev/disk/*` in any disk definitions. + _, err := cmd.RunContext(ctx, "/sbin/udevadm", "settle", "--timeout=50") + + return err + } + + return nil + }, "writeUdevRules" +} + +// StartMachined represents the task to start machined. +func StartMachined(_ runtime.Sequence, _ any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) error { + if err := tpm2.PCRExtent(secureboot.UKIPCR, []byte(secureboot.EnterMachined)); err != nil { + return err + } + + svc := &services.Machined{} + + id := svc.ID(r) + + err := system.Services(r).Start(id) + if err != nil { + return fmt.Errorf("failed to start machined service: %w", err) + } + + ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + + return system.WaitForService(system.StateEventUp, id).Wait(ctx) + }, "startMachined" +} + +// StartSyslogd represents the task to start syslogd. +func StartSyslogd(r runtime.Sequence, _ any) (runtime.TaskExecutionFunc, string) { + return func(_ context.Context, _ *log.Logger, r runtime.Runtime) error { + system.Services(r).LoadAndStart(&services.Syslogd{}) + + return nil + }, "startSyslogd" +} + +// StartDashboard represents the task to start dashboard. +func StartDashboard(_ runtime.Sequence, _ interface{}) (runtime.TaskExecutionFunc, string) { + return func(_ context.Context, _ *log.Logger, r runtime.Runtime) error { + system.Services(r).LoadAndStart(&services.Dashboard{}) + + return nil + }, "startDashboard" +} + +// StartUdevd represents the task to start udevd. +func StartUdevd(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + mp := mount.NewMountPoints() + mp.Set("udev-data", mount.NewMountPoint("", constants.UdevDir, "", unix.MS_I_VERSION, "", mount.WithFlags(mount.Overlay|mount.SystemOverlay|mount.Shared))) + + if err = mount.Mount(mp); err != nil { + return err + } + + svc := &services.Udevd{} + + system.Services(r).LoadAndStart(svc) + + ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + + return system.WaitForService(system.StateEventUp, svc.ID(r)).Wait(ctx) + }, "startUdevd" +} + +// ExtendPCRStartAll represents the task to extend the PCR with the StartTheWorld PCR phase. +func ExtendPCRStartAll(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + return tpm2.PCRExtent(secureboot.UKIPCR, []byte(secureboot.StartTheWorld)) + }, "extendPCRStartAll" +} + +// StartAllServices represents the task to start the system services. +func StartAllServices(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + // nb: Treating the beginning of "service starts" as the activate event for a normal + // non-maintenance mode boot. At this point, we'd expect the user to + // start interacting with the system for troubleshooting at least. + platform.FireEvent( + ctx, + r.State().Platform(), + platform.Event{ + Type: platform.EventTypeActivate, + Message: "Talos is ready for user interaction.", + }, + ) + + svcs := system.Services(r) + + // load the kubelet service, but don't start it; + // KubeletServiceController will start it once it's ready. + svcs.Load( + &services.Kubelet{}, + ) + + serviceList := []system.Service{ + &services.CRI{}, + } + + switch t := r.Config().Machine().Type(); t { + case machine.TypeInit: + serviceList = append(serviceList, + &services.Trustd{}, + &services.Etcd{Bootstrap: true}, + ) + case machine.TypeControlPlane: + serviceList = append(serviceList, + &services.Trustd{}, + &services.Etcd{}, + ) + case machine.TypeWorker: + // nothing + case machine.TypeUnknown: + fallthrough + default: + panic(fmt.Sprintf("unexpected machine type %v", t)) + } + + svcs.LoadAndStart(serviceList...) + + all := []conditions.Condition{} + + logger.Printf("waiting for %d services", len(svcs.List())) + + for _, svc := range svcs.List() { + cond := system.WaitForService(system.StateEventUp, svc.AsProto().GetId()) + all = append(all, cond) + } + + ctx, cancel := context.WithTimeout(ctx, constants.BootTimeout) + defer cancel() + + aggregateCondition := conditions.WaitForAll(all...) + + errChan := make(chan error) + + go func() { + errChan <- aggregateCondition.Wait(ctx) + }() + + ticker := time.NewTicker(15 * time.Second) + defer ticker.Stop() + + for { + logger.Printf("%s", aggregateCondition.String()) + + select { + case err := <-errChan: + return err + case <-ticker.C: + } + } + }, "startAllServices" +} + +// StopServicesEphemeral represents the StopServicesEphemeral task. +func StopServicesEphemeral(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + // stopping 'cri' service stops everything which depends on it (kubelet, etcd, ...) + return system.Services(nil).StopWithRevDepenencies(ctx, "cri", "udevd", "trustd") + }, "stopServicesForUpgrade" +} + +// StopAllServices represents the StopAllServices task. +func StopAllServices(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + system.Services(nil).Shutdown(ctx) + + return nil + }, "stopAllServices" +} + +// MountOverlayFilesystems represents the MountOverlayFilesystems task. +func MountOverlayFilesystems(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + var mountpoints *mount.Points + + mountpoints, err = mount.OverlayMountPoints() + if err != nil { + return err + } + + return mount.Mount(mountpoints) + }, "mountOverlayFilesystems" +} + +// SetupSharedFilesystems represents the SetupSharedFilesystems task. +func SetupSharedFilesystems(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + targets := []string{"/", "/var", "/etc/cni", "/run"} + for _, t := range targets { + if err = unix.Mount("", t, "", unix.MS_SHARED|unix.MS_REC, ""); err != nil { + return err + } + } + + return nil + }, "setupSharedFilesystems" +} + +// SetupVarDirectory represents the SetupVarDirectory task. +func SetupVarDirectory(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + for _, p := range []string{"/var/log/audit", "/var/log/containers", "/var/log/pods", "/var/lib/kubelet", "/var/run/lock", constants.SeccompProfilesDirectory} { + if err = os.MkdirAll(p, 0o700); err != nil { + return err + } + } + + // Handle Kubernetes directories which need different ownership + for _, p := range []string{constants.KubernetesAuditLogDir} { + if err = os.MkdirAll(p, 0o700); err != nil { + return err + } + + if err = os.Chown(p, constants.KubernetesAPIServerRunUser, constants.KubernetesAPIServerRunGroup); err != nil { + return fmt.Errorf("failed to chown %s: %w", p, err) + } + } + + return nil + }, "setupVarDirectory" +} + +// MountUserDisks represents the MountUserDisks task. +func MountUserDisks(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + if err = partitionAndFormatDisks(logger, r); err != nil { + return err + } + + return mountDisks(logger, r) + }, "mountUserDisks" +} + +// TODO(andrewrynhard): We shouldn't pull in the installer command package +// here. +func partitionAndFormatDisks(logger *log.Logger, r runtime.Runtime) error { + m := &installer.Manifest{ + Devices: map[string]installer.Device{}, + Targets: map[string][]*installer.Target{}, + Printf: logger.Printf, + } + + for _, disk := range r.Config().Machine().Disks() { + if err := func() error { + bd, err := blockdevice.Open(disk.Device(), blockdevice.WithMode(blockdevice.ReadonlyMode)) + if err != nil { + return err + } + + deviceName := bd.Device().Name() + + if disk.Device() != deviceName { + logger.Printf("using device name %q instead of %q", deviceName, disk.Device()) + } + + //nolint:errcheck + defer bd.Close() + + var pt *gpt.GPT + + pt, err = bd.PartitionTable() + if err != nil { + if !errors.Is(err, blockdevice.ErrMissingPartitionTable) { + return err + } + } + + // Partitions will be created/recreated if either of the following + // conditions are true: + // - a partition table exists AND there are no partitions + // - a partition table does not exist + + if pt != nil { + if len(pt.Partitions().Items()) > 0 { + logger.Printf(("skipping setup of %q, found existing partitions"), deviceName) + + return nil + } + } + + m.Devices[deviceName] = installer.Device{ + Device: deviceName, + ResetPartitionTable: true, + SkipOverlayMountsCheck: true, + } + + for _, part := range disk.Partitions() { + extraTarget := &installer.Target{ + Device: deviceName, + FormatOptions: &partition.FormatOptions{ + Force: true, + FileSystemType: partition.FilesystemTypeXFS, + }, + Options: &partition.Options{ + Size: part.Size(), + PartitionType: partition.LinuxFilesystemData, + }, + } + + m.Targets[deviceName] = append(m.Targets[deviceName], extraTarget) + } + + return nil + }(); err != nil { + return err + } + } + + return m.Execute() +} + +func mountDisks(logger *log.Logger, r runtime.Runtime) (err error) { + mountpoints := mount.NewMountPoints() + + for _, disk := range r.Config().Machine().Disks() { + bd, err := blockdevice.Open(disk.Device(), blockdevice.WithMode(blockdevice.ReadonlyMode)) + if err != nil { + return err + } + + deviceName := bd.Device().Name() + + if disk.Device() != deviceName { + logger.Printf("using device name %q instead of %q", deviceName, disk.Device()) + } + + if err = bd.Close(); err != nil { + return err + } + + for i, part := range disk.Partitions() { + var partname string + + partname, err = util.PartPath(deviceName, i+1) + if err != nil { + return err + } + + if _, err = os.Stat(part.MountPoint()); errors.Is(err, os.ErrNotExist) { + if err = os.MkdirAll(part.MountPoint(), 0o700); err != nil { + return err + } + } + + mountpoints.Set(partname, + mount.NewMountPoint(partname, part.MountPoint(), "xfs", unix.MS_NOATIME, "", + mount.WithProjectQuota(r.Config().Machine().Features().DiskQuotaSupportEnabled()), + ), + ) + } + } + + return mount.Mount(mountpoints) +} + +// WriteUserFiles represents the WriteUserFiles task. +// +//nolint:gocyclo,cyclop +func WriteUserFiles(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + var result *multierror.Error + + files, err := r.Config().Machine().Files() + if err != nil { + return fmt.Errorf("error generating extra files: %w", err) + } + + for _, f := range files { + content := f.Content() + + switch f.Op() { + case "create": + // Allow create at all times. + case "overwrite": + if err = existsAndIsFile(f.Path()); err != nil { + result = multierror.Append(result, err) + + continue + } + case "append": + if err = existsAndIsFile(f.Path()); err != nil { + result = multierror.Append(result, err) + + continue + } + + var existingFileContents []byte + + existingFileContents, err = os.ReadFile(f.Path()) + if err != nil { + result = multierror.Append(result, err) + + continue + } + + content = string(existingFileContents) + "\n" + f.Content() + default: + result = multierror.Append(result, fmt.Errorf("unknown operation for file %q: %q", f.Path(), f.Op())) + + continue + } + + if filepath.Dir(f.Path()) == constants.ManifestsDirectory { + if err = os.WriteFile(f.Path(), []byte(content), f.Permissions()); err != nil { + result = multierror.Append(result, err) + + continue + } + + if err = os.Chmod(f.Path(), f.Permissions()); err != nil { + result = multierror.Append(result, err) + + continue + } + + continue + } + + // CRI configuration customization + if f.Path() == filepath.Join("/etc", constants.CRICustomizationConfigPart) { + if err = injectCRIConfigPatch(ctx, r.State().V1Alpha2().Resources(), []byte(f.Content())); err != nil { + result = multierror.Append(result, err) + } + + continue + } + + // Determine if supplied path is in /var or not. + // If not, we'll write it to /var anyways and bind mount below + p := f.Path() + inVar := true + parts := strings.Split( + strings.TrimLeft(f.Path(), "/"), + string(os.PathSeparator), + ) + + if parts[0] != "var" { + p = filepath.Join("/var", f.Path()) + inVar = false + } + + // We do not want to support creating new files anywhere outside of + // /var. If a valid use case comes up, we can reconsider then. + if !inVar && f.Op() == "create" { + return fmt.Errorf("create operation not allowed outside of /var: %q", f.Path()) + } + + if err = os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + result = multierror.Append(result, err) + + continue + } + + if err = os.WriteFile(p, []byte(content), f.Permissions()); err != nil { + result = multierror.Append(result, err) + + continue + } + + if err = os.Chmod(p, f.Permissions()); err != nil { + result = multierror.Append(result, err) + + continue + } + + if !inVar { + if err = unix.Mount(p, f.Path(), "", unix.MS_BIND|unix.MS_RDONLY, ""); err != nil { + result = multierror.Append(result, fmt.Errorf("failed to create bind mount for %s: %w", p, err)) + } + } + } + + return result.ErrorOrNil() + }, "writeUserFiles" +} + +func injectCRIConfigPatch(ctx context.Context, st state.State, content []byte) error { + // limit overall waiting time + ctx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + + ch := make(chan state.Event) + + // wait for the CRI config to be created + if err := st.Watch(ctx, resourcefiles.NewEtcFileSpec(resourcefiles.NamespaceName, constants.CRIConfig).Metadata(), ch); err != nil { + return err + } + + // first update should be received about the existing resource + select { + case <-ch: + case <-ctx.Done(): + return ctx.Err() + } + + etcFileSpec := resourcefiles.NewEtcFileSpec(resourcefiles.NamespaceName, constants.CRICustomizationConfigPart) + etcFileSpec.TypedSpec().Mode = 0o600 + etcFileSpec.TypedSpec().Contents = content + + if err := st.Create(ctx, etcFileSpec); err != nil { + return err + } + + // wait for the CRI config parts controller to generate the merged file + var version resource.Version + + select { + case ev := <-ch: + version = ev.Resource.Metadata().Version() + case <-ctx.Done(): + return ctx.Err() + } + + // wait for the file to be rendered + _, err := st.WatchFor(ctx, resourcefiles.NewEtcFileStatus(resourcefiles.NamespaceName, constants.CRIConfig).Metadata(), state.WithCondition(func(r resource.Resource) (bool, error) { + fileStatus, ok := r.(*resourcefiles.EtcFileStatus) + if !ok { + return false, nil + } + + return fileStatus.TypedSpec().SpecVersion == version.String(), nil + })) + + return err +} + +func existsAndIsFile(p string) (err error) { + var info os.FileInfo + + info, err = os.Stat(p) + if err != nil { + if !os.IsNotExist(err) { + return err + } + + return fmt.Errorf("file must exist: %q", p) + } + + if !info.Mode().IsRegular() { + return fmt.Errorf("invalid mode: %q", info.Mode().String()) + } + + return nil +} + +// UnmountOverlayFilesystems represents the UnmountOverlayFilesystems task. +func UnmountOverlayFilesystems(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + var mountpoints *mount.Points + + mountpoints, err = mount.OverlayMountPoints() + if err != nil { + return err + } + + return mount.Unmount(mountpoints) + }, "unmountOverlayFilesystems" +} + +// UnmountUserDisks represents the UnmountUserDisks task. +func UnmountUserDisks(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + if r.Config() == nil { + return nil + } + + mountpoints := mount.NewMountPoints() + + for _, disk := range r.Config().Machine().Disks() { + bd, err := blockdevice.Open(disk.Device(), blockdevice.WithMode(blockdevice.ReadonlyMode)) + if err != nil { + return err + } + + deviceName := bd.Device().Name() + + if deviceName != disk.Device() { + logger.Printf("using device name %q instead of %q", deviceName, disk.Device()) + } + + if err = bd.Close(); err != nil { + return err + } + + for i, part := range disk.Partitions() { + var partname string + + partname, err = util.PartPath(deviceName, i+1) + if err != nil { + return err + } + + mountpoints.Set(partname, mount.NewMountPoint(partname, part.MountPoint(), "xfs", unix.MS_NOATIME, "")) + } + } + + return mount.Unmount(mountpoints) + }, "unmountUserDisks" +} + +// UnmountPodMounts represents the UnmountPodMounts task. +func UnmountPodMounts(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + var b []byte + + if b, err = os.ReadFile("/proc/self/mounts"); err != nil { + return err + } + + rdr := bytes.NewReader(b) + + scanner := bufio.NewScanner(rdr) + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + + if len(fields) < 2 { + continue + } + + mountpoint := fields[1] + if strings.HasPrefix(mountpoint, constants.EphemeralMountPoint+"/") { + logger.Printf("unmounting %s\n", mountpoint) + + if err = mount.SafeUnmount(ctx, logger, mountpoint); err != nil { + if errors.Is(err, syscall.EINVAL) { + log.Printf("ignoring unmount error %s: %v", mountpoint, err) + } else { + return fmt.Errorf("error unmounting %s: %w", mountpoint, err) + } + } + } + } + + return scanner.Err() + }, "unmountPodMounts" +} + +// UnmountSystemDiskBindMounts represents the UnmountSystemDiskBindMounts task. +func UnmountSystemDiskBindMounts(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + systemDisk := r.State().Machine().Disk() + if systemDisk == nil { + return nil + } + + devname := systemDisk.BlockDevice.Device().Name() + + f, err := os.Open("/proc/mounts") + if err != nil { + return err + } + + defer f.Close() //nolint:errcheck + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + + if len(fields) < 2 { + continue + } + + device := strings.ReplaceAll(fields[0], "/dev/mapper", "/dev") + mountpoint := fields[1] + + if strings.HasPrefix(device, devname) && device != devname { + logger.Printf("unmounting %s\n", mountpoint) + + if err = mount.SafeUnmount(ctx, logger, mountpoint); err != nil { + if errors.Is(err, syscall.EINVAL) { + log.Printf("ignoring unmount error %s: %v", mountpoint, err) + } else { + return fmt.Errorf("error unmounting %s: %w", mountpoint, err) + } + } + } + } + + return scanner.Err() + }, "unmountSystemDiskBindMounts" +} + +// CordonAndDrainNode represents the task for stop all containerd tasks in the +// k8s.io namespace. +func CordonAndDrainNode(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + // skip not exist error as it means that the node hasn't fully joined yet + if _, err = os.Stat("/var/lib/kubelet/pki/kubelet-client-current.pem"); err != nil { + if os.IsNotExist(err) { + return nil + } + + return err + } + + var nodename string + + if nodename, err = r.NodeName(); err != nil { + return err + } + + // controllers will automatically cordon the node when the node enters appropriate phase, + // so here we just wait for the node to be cordoned + if err = waitForNodeCordoned(ctx, logger, r, nodename); err != nil { + return err + } + + var kubeHelper *kubernetes.Client + + if kubeHelper, err = kubernetes.NewClientFromKubeletKubeconfig(); err != nil { + return err + } + + defer kubeHelper.Close() //nolint:errcheck + + return kubeHelper.Drain(ctx, nodename) + }, "cordonAndDrainNode" +} + +func waitForNodeCordoned(ctx context.Context, logger *log.Logger, r runtime.Runtime, nodename string) error { + ctx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + + logger.Print("waiting for node to be cordoned") + + _, err := r.State().V1Alpha2().Resources().WatchFor( + ctx, + k8s.NewNodeStatus(k8s.NamespaceName, nodename).Metadata(), + state.WithCondition(func(r resource.Resource) (bool, error) { + if resource.IsTombstone(r) { + return false, nil + } + + nodeStatus, ok := r.(*k8s.NodeStatus) + if !ok { + return false, nil + } + + return nodeStatus.TypedSpec().Unschedulable, nil + }), + ) + + return err +} + +// LeaveEtcd represents the task for removing a control plane node from etcd. +// +//nolint:gocyclo +func LeaveEtcd(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + _, err = os.Stat(filepath.Join(constants.EtcdDataPath, "/member")) + if err != nil { + if os.IsNotExist(err) { + return nil + } + + return err + } + + etcdID := (&services.Etcd{}).ID(r) + + services := system.Services(r).List() + + shouldLeaveEtcd := false + + for _, service := range services { + if service.AsProto().Id != etcdID { + continue + } + + switch service.GetState() { //nolint:exhaustive + case events.StateRunning: + fallthrough + case events.StateStopping: + fallthrough + case events.StateFailed: + shouldLeaveEtcd = true + } + + break + } + + if !shouldLeaveEtcd { + return nil + } + + client, err := etcd.NewClientFromControlPlaneIPs(ctx, r.State().V1Alpha2().Resources()) + if err != nil { + return fmt.Errorf("failed to create etcd client: %w", err) + } + + //nolint:errcheck + defer client.Close() + + ctx = clientv3.WithRequireLeader(ctx) + + if err = client.LeaveCluster(ctx, r.State().V1Alpha2().Resources()); err != nil { + return fmt.Errorf("failed to leave cluster: %w", err) + } + + return nil + }, "leaveEtcd" +} + +// RemoveAllPods represents the task for stopping and removing all pods. +func RemoveAllPods(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return stopAndRemoveAllPods(cri.StopAndRemove), "removeAllPods" +} + +// StopAllPods represents the task for stopping all pods. +func StopAllPods(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return stopAndRemoveAllPods(cri.StopOnly), "stopAllPods" +} + +func waitForKubeletLifecycleFinalizers(ctx context.Context, logger *log.Logger, r runtime.Runtime) error { + logger.Printf("waiting for kubelet lifecycle finalizers") + + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + lifecycle := resource.NewMetadata(k8s.NamespaceName, k8s.KubeletLifecycleType, k8s.KubeletLifecycleID, resource.VersionUndefined) + + for { + ok, err := r.State().V1Alpha2().Resources().Teardown(ctx, lifecycle) + if err != nil { + return err + } + + if ok { + break + } + + _, err = r.State().V1Alpha2().Resources().WatchFor(ctx, lifecycle, state.WithFinalizerEmpty()) + if err != nil { + return err + } + } + + return r.State().V1Alpha2().Resources().Destroy(ctx, lifecycle) +} + +func stopAndRemoveAllPods(stopAction cri.StopAction) runtime.TaskExecutionFunc { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + if err = waitForKubeletLifecycleFinalizers(ctx, logger, r); err != nil { + logger.Printf("failed waiting for kubelet lifecycle finalizers: %s", err) + } + + logger.Printf("shutting down kubelet gracefully") + + shutdownCtx, shutdownCtxCancel := context.WithTimeout(ctx, logind.InhibitMaxDelay) + defer shutdownCtxCancel() + + if err = r.State().Machine().DBus().WaitShutdown(shutdownCtx); err != nil { + logger.Printf("failed waiting for inhibit shutdown lock: %s", err) + } + + if err = system.Services(nil).Stop(ctx, "kubelet"); err != nil { + return err + } + + // check that the CRI is running and the socket is available, if not, skip the rest + if _, err = os.Stat(constants.CRIContainerdAddress); os.IsNotExist(err) { + return nil + } + + client, err := cri.NewClient("unix://"+constants.CRIContainerdAddress, 10*time.Second) + if err != nil { + return err + } + + //nolint:errcheck + defer client.Close() + + ctx, cancel := context.WithTimeout(ctx, time.Minute*3) + defer cancel() + + // We remove pods with POD network mode first so that the CNI can perform + // any cleanup tasks. If we don't do this, we run the risk of killing the + // CNI, preventing the CRI from cleaning up the pod's networking. + + if err = client.StopAndRemovePodSandboxes(ctx, stopAction, runtimeapi.NamespaceMode_POD, runtimeapi.NamespaceMode_CONTAINER); err != nil { + return err + } + + // With the POD network mode pods out of the way, we kill the remaining + // pods. + + return client.StopAndRemovePodSandboxes(ctx, stopAction) + } +} + +// ResetSystemDiskPartitions represents the task for wiping the system disk partitions. +func ResetSystemDiskPartitions(seq runtime.Sequence, _ any) (runtime.TaskExecutionFunc, string) { + wipeStr := procfs.ProcCmdline().Get(constants.KernelParamWipe).First() + reboot, _ := Reboot(seq, nil) + + if pointer.SafeDeref(wipeStr) == "" { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) error { + return errors.New("no wipe target specified") + }, "wipeSystemDisk" + } + + if *wipeStr == "system" { + resetSystemDisk, _ := ResetSystemDisk(seq, nil) + + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) error { + logger.Printf("resetting system disks") + + err := resetSystemDisk(ctx, logger, r) + if err != nil { + logger.Printf("resetting system disks failed") + + return err + } + + logger.Printf("finished resetting system disks") + + return reboot(ctx, logger, r) // only reboot when we wiped boot partition + }, "wipeSystemDisk" + } + + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) error { + targets, err := parseTargets(r, *wipeStr) + if err != nil { + return err + } + + fn, _ := ResetSystemDiskSpec(seq, targets) + diskTargets := targets.GetSystemDiskTargets() + + logger.Printf("resetting system disks %s", diskTargets) + + err = fn(ctx, logger, r) + if err != nil { + logger.Printf("resetting system disks %s failed", diskTargets) + + return err + } + + logger.Printf("finished resetting system disks %s", diskTargets) + + bootWiped := slices.ContainsFunc(diskTargets, func(t runtime.PartitionTarget) bool { + return t.GetLabel() == constants.BootPartitionLabel + }) + + if bootWiped { + return reboot(ctx, logger, r) // only reboot when we wiped boot partition + } + + return nil + }, "wipeSystemDiskPartitions" +} + +// ResetSystemDisk represents the task to reset the system disk. +func ResetSystemDisk(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + var dev *blockdevice.BlockDevice + + disk := r.State().Machine().Disk() + + if disk == nil { + return nil + } + + dev, err = blockdevice.Open(disk.Device().Name()) + if err != nil { + return err + } + + defer dev.Close() //nolint:errcheck + + return dev.FastWipe() + }, "resetSystemDisk" +} + +// ResetUserDisks represents the task to reset the user disks. +func ResetUserDisks(_ runtime.Sequence, data any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + in, ok := data.(runtime.ResetOptions) + if !ok { + return errors.New("unexpected runtime data") + } + + wipeDevice := func(deviceName string) error { + dev, err := blockdevice.Open(deviceName) + if err != nil { + return err + } + + defer func() { + if closeErr := dev.Close(); closeErr != nil { + logger.Printf("failed to close device %s: %s", deviceName, closeErr) + } + }() + + logger.Printf("wiping user disk %s", deviceName) + + return dev.FastWipe() + } + + for _, deviceName := range in.GetUserDisksToWipe() { + if err := wipeDevice(deviceName); err != nil { + return err + } + } + + return nil + }, "resetUserDisks" +} + +type targets struct { + systemDiskTargets []*installer.Target +} + +func (opt targets) GetSystemDiskTargets() []runtime.PartitionTarget { + return xslices.Map(opt.systemDiskTargets, func(t *installer.Target) runtime.PartitionTarget { return t }) +} + +func parseTargets(r runtime.Runtime, wipeStr string) (targets, error) { + after, found := strings.CutPrefix(wipeStr, "system:") + if !found { + return targets{}, fmt.Errorf("invalid wipe labels string: %q", wipeStr) + } + + var result []*installer.Target //nolint:prealloc + + for _, part := range strings.Split(after, ",") { + bd := r.State().Machine().Disk().BlockDevice + + target, err := installer.ParseTarget(part, bd.Device().Name()) + if err != nil { + return targets{}, fmt.Errorf("error parsing target label %q: %w", part, err) + } + + pt, err := bd.PartitionTable() + if err != nil { + return targets{}, fmt.Errorf("error reading partition table: %w", err) + } + + _, err = target.Locate(pt) + if err != nil { + return targets{}, fmt.Errorf("error locating partition %q: %w", part, err) + } + + result = append(result, target) + } + + if len(result) == 0 { + return targets{}, errors.New("no wipe labels specified") + } + + return targets{systemDiskTargets: result}, nil +} + +// SystemDiskTargets represents the interface for getting the system disk targets. +// It's a subset of [runtime.ResetOptions]. +type SystemDiskTargets interface { + GetSystemDiskTargets() []runtime.PartitionTarget +} + +// ResetSystemDiskSpec represents the task to reset the system disk by spec. +func ResetSystemDiskSpec(_ runtime.Sequence, data any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + in, ok := data.(SystemDiskTargets) + if !ok { + return errors.New("unexpected runtime data") + } + + for _, target := range in.GetSystemDiskTargets() { + if err = target.Format(logger.Printf); err != nil { + return fmt.Errorf("failed wiping partition %s: %w", target, err) + } + } + + stateWiped := slices.ContainsFunc(in.GetSystemDiskTargets(), func(t runtime.PartitionTarget) bool { + return t.GetLabel() == constants.StatePartitionLabel + }) + + metaWiped := slices.ContainsFunc(in.GetSystemDiskTargets(), func(t runtime.PartitionTarget) bool { + return t.GetLabel() == constants.MetaPartitionLabel + }) + + if stateWiped && !metaWiped { + var removed bool + + removed, err = r.State().Machine().Meta().DeleteTag(ctx, meta.StateEncryptionConfig) + if err != nil { + return fmt.Errorf("failed to remove state encryption META config tag: %w", err) + } + + if removed { + if err = r.State().Machine().Meta().Flush(); err != nil { + return fmt.Errorf("failed to flush META: %w", err) + } + + logger.Printf("reset the state encryption META config tag") + } + } + + logger.Printf("successfully reset system disk by the spec") + + return nil + }, "resetSystemDiskSpec" +} + +// VerifyDiskAvailability represents the task for verifying that the system +// disk is not in use. +func VerifyDiskAvailability(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + devname := r.State().Machine().Disk().BlockDevice.Device().Name() + + // We MUST close this in order to avoid EBUSY. + if err = r.State().Machine().Close(); err != nil { + return err + } + + // TODO(andrewrynhard): This should be more dynamic. If we ever change the + // partition scheme there is the chance that 2 is not the correct parition to + // check. + partname, err := util.PartPath(devname, 2) + if err != nil { + return err + } + + if _, err = os.Stat(partname); errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("ephemeral partition not found: %w", err) + } + + mountsReported := false + + return retry.Constant(3*time.Minute, retry.WithUnits(500*time.Millisecond)).Retry(func() error { + if err = tryLock(partname); err != nil { + if err == unix.EBUSY { + if !mountsReported { + // if disk is busy, report mounts for debugging purposes but just once + // otherwise console might be flooded with messages + dumpMounts(logger) + + mountsReported = true + } + + return retry.ExpectedErrorf("ephemeral partition in use: %q", partname) + } + + return fmt.Errorf("failed to verify ephemeral partition not in use: %w", err) + } + + return nil + }) + }, "verifyDiskAvailability" +} + +func tryLock(path string) error { + fd, errno := unix.Open(path, unix.O_RDONLY|unix.O_EXCL|unix.O_CLOEXEC, 0) + + //nolint:errcheck + defer unix.Close(fd) + + return errno +} + +func dumpMounts(logger *log.Logger) { + mounts, err := os.Open("/proc/mounts") + if err != nil { + logger.Printf("failed to read mounts: %s", err) + + return + } + + defer mounts.Close() //nolint:errcheck + + logger.Printf("contents of /proc/mounts:") + + _, _ = io.Copy(log.Writer(), mounts) //nolint:errcheck +} + +// Upgrade represents the task for performing an upgrade. +func Upgrade(_ runtime.Sequence, data any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + // This should be checked by the gRPC server, but we double check here just + // to be safe. + in, ok := data.(*machineapi.UpgradeRequest) + if !ok { + return runtime.ErrInvalidSequenceData + } + + devname := r.State().Machine().Disk().BlockDevice.Device().Name() + + logger.Printf("performing upgrade via %q", in.GetImage()) + + // We pull the installer image when we receive an upgrade request. No need + // to pull it again. + err = install.RunInstallerContainer( + devname, r.State().Platform().Name(), + in.GetImage(), + r.Config(), + r.ConfigContainer(), + install.OptionsFromUpgradeRequest(r, in)..., + ) + if err != nil { + return err + } + + logger.Println("upgrade successful") + + return nil + }, "upgrade" +} + +// Reboot represents the Reboot task. +func Reboot(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + rebootCmd := unix.LINUX_REBOOT_CMD_RESTART + + if r.State().Machine().IsKexecPrepared() { + rebootCmd = unix.LINUX_REBOOT_CMD_KEXEC + } + + r.Events().Publish(ctx, &machineapi.RestartEvent{ + Cmd: int64(rebootCmd), + }) + + platform.FireEvent( + ctx, + r.State().Platform(), + platform.Event{ + Type: platform.EventTypeRebooted, + Message: "Talos rebooted.", + }, + ) + + return runtime.RebootError{Cmd: rebootCmd} + }, "reboot" +} + +// Shutdown represents the Shutdown task. +func Shutdown(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + cmd := unix.LINUX_REBOOT_CMD_POWER_OFF + + if p := procfs.ProcCmdline().Get(constants.KernelParamShutdown).First(); p != nil { + if *p == "halt" { + cmd = unix.LINUX_REBOOT_CMD_HALT + } + } + + r.Events().Publish(ctx, &machineapi.RestartEvent{ + Cmd: int64(cmd), + }) + + return runtime.RebootError{Cmd: cmd} + }, "shutdown" +} + +// SaveStateEncryptionConfig saves state partition encryption info in the meta partition. +func SaveStateEncryptionConfig(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + config := r.Config() + if config == nil { + return nil + } + + encryption := config.Machine().SystemDiskEncryption().Get(constants.StatePartitionLabel) + if encryption == nil { + return nil + } + + var data []byte + + if data, err = json.Marshal(encryption); err != nil { + return err + } + + ok, err := r.State().Machine().Meta().SetTagBytes(ctx, meta.StateEncryptionConfig, data) + if err != nil { + return err + } + + if !ok { + return errors.New("failed to save state encryption config in the META partition") + } + + return r.State().Machine().Meta().Flush() + }, "SaveStateEncryptionConfig" +} + +// MountEFIPartition mounts the EFI partition. +func MountEFIPartition(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + return mount.SystemPartitionMount(ctx, r, logger, constants.EFIPartitionLabel) + }, "mountEFIPartition" +} + +// UnmountEFIPartition unmounts the EFI partition. +func UnmountEFIPartition(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) error { + return mount.SystemPartitionUnmount(r, logger, constants.EFIPartitionLabel) + }, "unmountEFIPartition" +} + +// MountStatePartition mounts the system partition. +func MountStatePartition(seq runtime.Sequence, _ any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + flags := mount.SkipIfMounted + + if seq == runtime.SequenceInitialize { + flags |= mount.SkipIfNoFilesystem + } + + opts := []mount.Option{mount.WithFlags(flags)} + + var encryption config.Encryption + // first try reading encryption from the config + // which always has the priority here + if r.Config() != nil && r.Config().Machine() != nil { + encryption = r.Config().Machine().SystemDiskEncryption().Get(constants.StatePartitionLabel) + } + + // then try reading it from the META partition + if encryption == nil { + var encryptionFromMeta *v1alpha1.EncryptionConfig + + data, ok := r.State().Machine().Meta().ReadTagBytes(meta.StateEncryptionConfig) + if ok { + if err = json.Unmarshal(data, &encryptionFromMeta); err != nil { + return err + } + + encryption = encryptionFromMeta + } + } + + if encryption != nil { + opts = append(opts, mount.WithEncryptionConfig(encryption), mount.WithSystemInformationGetter(r.GetSystemInformation)) + } + + return mount.SystemPartitionMount(ctx, r, logger, constants.StatePartitionLabel, opts...) + }, "mountStatePartition" +} + +// UnmountStatePartition unmounts the system partition. +func UnmountStatePartition(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) error { + return mount.SystemPartitionUnmount(r, logger, constants.StatePartitionLabel) + }, "unmountStatePartition" +} + +// MountEphemeralPartition mounts the ephemeral partition. +func MountEphemeralPartition(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) error { + return mount.SystemPartitionMount(ctx, r, logger, constants.EphemeralPartitionLabel, + mount.WithFlags(mount.Resize), + mount.WithProjectQuota(r.Config().Machine().Features().DiskQuotaSupportEnabled())) + }, "mountEphemeralPartition" +} + +// UnmountEphemeralPartition unmounts the ephemeral partition. +func UnmountEphemeralPartition(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + return mount.SystemPartitionUnmount(r, logger, constants.EphemeralPartitionLabel) + }, "unmountEphemeralPartition" +} + +// Install mounts or installs the system partitions. +func Install(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + switch { + case !r.State().Machine().Installed(): + installerImage := r.Config().Machine().Install().Image() + if installerImage == "" { + installerImage = images.DefaultInstallerImage + } + + var disk string + + disk, err = r.Config().Machine().Install().Disk() + if err != nil { + return err + } + + err = install.RunInstallerContainer( + disk, + r.State().Platform().Name(), + installerImage, + r.Config(), + r.ConfigContainer(), + install.WithForce(true), + install.WithZero(r.Config().Machine().Install().Zero()), + install.WithExtraKernelArgs(r.Config().Machine().Install().ExtraKernelArgs()), + ) + if err != nil { + platform.FireEvent( + ctx, + r.State().Platform(), + platform.Event{ + Type: platform.EventTypeFailure, + Message: "Talos install failed.", + Error: err, + }, + ) + + return err + } + + platform.FireEvent( + ctx, + r.State().Platform(), + platform.Event{ + Type: platform.EventTypeInstalled, + Message: "Talos installed successfully.", + }, + ) + + logger.Println("install successful") + + case r.State().Machine().IsInstallStaged(): + devname := r.State().Machine().Disk().BlockDevice.Device().Name() + + var options install.Options + + if err = json.Unmarshal(r.State().Machine().StagedInstallOptions(), &options); err != nil { + return fmt.Errorf("error unserializing install options: %w", err) + } + + logger.Printf("performing staged upgrade via %q", r.State().Machine().StagedInstallImageRef()) + + err = install.RunInstallerContainer( + devname, r.State().Platform().Name(), + r.State().Machine().StagedInstallImageRef(), + r.Config(), + r.ConfigContainer(), + install.WithOptions(options), + ) + if err != nil { + platform.FireEvent( + ctx, + r.State().Platform(), + platform.Event{ + Type: platform.EventTypeFailure, + Message: "Talos staged upgrade failed.", + Error: err, + }, + ) + + return err + } + + // nb: we don't fire an "activate" event after this one + // b/c we'd only ever get here if Talos was already + // installed I believe. + platform.FireEvent( + ctx, + r.State().Platform(), + platform.Event{ + Type: platform.EventTypeUpgraded, + Message: "Talos staged upgrade successful.", + }, + ) + + logger.Println("staged upgrade successful") + + default: + return errors.New("unsupported configuration for install task") + } + + return nil + }, "install" +} + +// ActivateLogicalVolumes represents the task for activating logical volumes. +func ActivateLogicalVolumes(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + if _, err = cmd.Run("/sbin/lvm", "vgchange", "-ay"); err != nil { + return fmt.Errorf("failed to activate logical volumes: %w", err) + } + + return nil + }, "activateLogicalVolumes" +} + +// KexecPrepare loads next boot kernel via kexec_file_load. +// +//nolint:gocyclo +func KexecPrepare(_ runtime.Sequence, data any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) error { + if req, ok := data.(*machineapi.RebootRequest); ok { + if req.Mode == machineapi.RebootRequest_POWERCYCLE { + log.Print("kexec skipped as reboot with power cycle was requested") + + return nil + } + } + + if r.Config() == nil { + return nil + } + + // check if partition with label BOOT exists + if device := r.State().Machine().Disk(disk.WithPartitionLabel(constants.BootPartitionLabel)); device == nil { + return nil + } + + // BOOT partition exists and we can mount it + if err := mount.SystemPartitionMount(ctx, r, logger, constants.BootPartitionLabel); err != nil { + return err + } + + defer mount.SystemPartitionUnmount(r, logger, constants.BootPartitionLabel) //nolint:errcheck + + conf, err := grub.Read(grub.ConfigPath) + if err != nil { + return err + } + + if conf == nil { + return nil + } + + defaultEntry, ok := conf.Entries[conf.Default] + if !ok { + return nil + } + + kernelPath := filepath.Join(constants.BootMountPoint, defaultEntry.Linux) + initrdPath := filepath.Join(constants.BootMountPoint, defaultEntry.Initrd) + + kernel, err := os.Open(kernelPath) + if err != nil { + return err + } + + defer kernel.Close() //nolint:errcheck + + initrd, err := os.Open(initrdPath) + if err != nil { + return err + } + + defer initrd.Close() //nolint:errcheck + + cmdline := strings.TrimSpace(defaultEntry.Cmdline) + + if err = unix.KexecFileLoad(int(kernel.Fd()), int(initrd.Fd()), cmdline, 0); err != nil { + switch { + case errors.Is(err, unix.ENOSYS): + log.Printf("kexec support is disabled in the kernel") + + return nil + case errors.Is(err, unix.EPERM): + log.Printf("kexec support is disabled via sysctl") + + return nil + case errors.Is(err, unix.EBUSY): + log.Printf("kexec is busy") + + return nil + default: + return fmt.Errorf("error loading kernel for kexec: %w", err) + } + } + + log.Printf("prepared kexec environment kernel=%q initrd=%q cmdline=%q", kernelPath, initrdPath, cmdline) + + r.State().Machine().KexecPrepared(true) + + return nil + }, "kexecPrepare" +} + +// StartDBus starts the D-Bus mock. +func StartDBus(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) error { + return r.State().Machine().DBus().Start() + }, "startDBus" +} + +// StopDBus stops the D-Bus mock. +func StopDBus(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) error { + if err := r.State().Machine().DBus().Stop(); err != nil { + logger.Printf("error stopping D-Bus: %s, ignored", err) + } + + return nil + }, "stopDBus" +} + +// ForceCleanup kills remaining procs and forces partitions unmount. +func ForceCleanup(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) error { + if err := proc.KillAll(); err != nil { + logger.Printf("error killing all procs: %s", err) + } + + if err := mount.UnmountAll(); err != nil { + logger.Printf("error unmounting: %s", err) + } + + return nil + }, "forceCleanup" +} + +// ReloadMeta reloads META partition after disk mount, installer run, etc. +// +//nolint:gocyclo +func ReloadMeta(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) error { + err := r.State().Machine().Meta().Reload(ctx) + if err != nil && !os.IsNotExist(err) { + return err + } + + // attempt to populate meta from the environment if Talos is not installed (yet) + if os.IsNotExist(err) { + env := environment.Get(r.Config()) + + prefix := constants.MetaValuesEnvVar + "=" + + for _, e := range env { + if !strings.HasPrefix(e, prefix) { + continue + } + + values, err := metamachinery.DecodeValues(e[len(prefix):]) + if err != nil { + return fmt.Errorf("error decoding meta values: %w", err) + } + + for _, value := range values { + _, err = r.State().Machine().Meta().SetTag(ctx, value.Key, value.Value) + if err != nil { + return fmt.Errorf("error setting meta tag %x: %w", value.Key, err) + } + } + } + } + + if _, err := safe.ReaderGetByID[*resourceruntime.MetaLoaded]( + ctx, + r.State().V1Alpha2().Resources(), + resourceruntime.MetaLoadedID, + ); err != nil { + if !state.IsNotFoundError(err) { + return fmt.Errorf("error reading MetaLoaded resource: %w", err) + } + + // create MetaLoaded resource signaling that META is now loaded + loaded := resourceruntime.NewMetaLoaded() + loaded.TypedSpec().Done = true + + err = r.State().V1Alpha2().Resources().Create(ctx, loaded) + if err != nil { + return fmt.Errorf("error creating MetaLoaded resource: %w", err) + } + } + + return nil + }, "reloadMeta" +} + +// FlushMeta flushes META partition after install run. +func FlushMeta(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) error { + return r.State().Machine().Meta().Flush() + }, "flushMeta" +} + +// StoreShutdownEmergency stores shutdown emergency state. +func StoreShutdownEmergency(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) error { + // for shutdown sequence, store power_off as the intent, it will be picked up + // by emergency handled in machined/main.go if the Shutdown sequence fails + emergency.RebootCmd.Store(unix.LINUX_REBOOT_CMD_POWER_OFF) + + return nil + }, "storeShutdownEmergency" +} + +// SendResetSignal func represents the task to send the final reset signal. +func SendResetSignal(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { + return r.State().V1Alpha2().Resources().Create(ctx, resourceruntime.NewMachineResetSignal()) + }, "sendResetSignal" +} + +func pauseOnFailure(callback func(runtime.Sequence, any) (runtime.TaskExecutionFunc, string), + timeout time.Duration, +) func(seq runtime.Sequence, data any) (runtime.TaskExecutionFunc, string) { + return func(seq runtime.Sequence, data any) (runtime.TaskExecutionFunc, string) { + f, name := callback(seq, data) + + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) error { + err := f(ctx, logger, r) + if err != nil { + logger.Printf("%s failed, rebooting in %.0f minutes. You can use talosctl apply-config or talosctl edit mc to fix the issues, error:\n%s", name, timeout.Minutes(), err) + + timer := time.NewTimer(time.Minute * 5) + defer timer.Stop() + + select { + case <-timer.C: + case <-ctx.Done(): + } + } + + return err + }, name + } +} + +func taskErrorHandler(handler func(error, *log.Logger) error, task runtime.TaskSetupFunc) runtime.TaskSetupFunc { + return func(seq runtime.Sequence, data any) (runtime.TaskExecutionFunc, string) { + f, name := task(seq, data) + + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) error { + err := f(ctx, logger, r) + if err != nil { + return handler(err, logger) + } + + return nil + }, name + } +} + +func phaseListErrorHandler(handler func(error, *log.Logger) error, phases ...runtime.Phase) PhaseList { + for _, phase := range phases { + for i, task := range phase.Tasks { + phase.Tasks[i] = taskErrorHandler(handler, task) + } + } + + return phases +} + +func logError(err error, logger *log.Logger) error { + logger.Printf("WARNING: task failed: %s", err) + + return nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_test.go b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_test.go new file mode 100644 index 0000000..53c96bd --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_test.go @@ -0,0 +1,112 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:scopelint,testpackage +package v1alpha1 + +import ( + "reflect" + "testing" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" +) + +func TestNewSequencer(t *testing.T) { + tests := []struct { + name string + want *Sequencer + }{ + { + name: "test", + want: &Sequencer{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := NewSequencer(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("NewSequencer() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPhaseList_Append(t *testing.T) { + t.Skip("temporarily disabling until reflect.DeepEqual responds as expected") + + type args struct { + name string + tasks []runtime.TaskSetupFunc + } + + tests := []struct { + name string + p PhaseList + args args + want PhaseList + }{ + { + name: "test", + p: PhaseList{}, + args: args{ + name: "mount", + tasks: []runtime.TaskSetupFunc{KexecPrepare}, + }, + want: PhaseList{runtime.Phase{Name: "mount", Tasks: []runtime.TaskSetupFunc{KexecPrepare}}}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.p = tt.p.Append(tt.args.name, tt.args.tasks...); !reflect.DeepEqual(tt.p, tt.want) { + t.Errorf("PhaseList.Append() = %v, want %v", tt.p, tt.want) + } + }) + } +} + +func TestPhaseList_AppendWhen(t *testing.T) { + t.Skip("temporarily disabling until reflect.DeepEqual responds as expected") + + type args struct { + when bool + name string + tasks []runtime.TaskSetupFunc + } + + tests := []struct { + name string + p PhaseList + args args + want PhaseList + }{ + { + name: "true", + p: PhaseList{}, + args: args{ + when: true, + name: "mount", + tasks: []runtime.TaskSetupFunc{KexecPrepare}, + }, + want: PhaseList{runtime.Phase{Name: "mount", Tasks: []runtime.TaskSetupFunc{KexecPrepare}}}, + }, + { + name: "false", + p: PhaseList{}, + args: args{ + when: false, + tasks: []runtime.TaskSetupFunc{KexecPrepare}, + }, + want: PhaseList{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.p = tt.p.AppendWhen(tt.args.when, tt.args.name, tt.args.tasks...); !reflect.DeepEqual(tt.p, tt.want) { + t.Errorf("PhaseList.AppendWhen() = %v, want %v", tt.p, tt.want) + } + }) + } +} diff --git a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_state.go b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_state.go new file mode 100644 index 0000000..4da5ade --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_state.go @@ -0,0 +1,346 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package v1alpha1 + +import ( + "context" + "errors" + "log" + "os" + "sync" + + "github.com/cosi-project/runtime/pkg/state" + multierror "github.com/hashicorp/go-multierror" + "github.com/siderolabs/go-blockdevice/blockdevice/probe" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/disk" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha2" + "github.com/aenix-io/talm/internal/pkg/meta" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// State implements the state interface. +type State struct { + platform runtime.Platform + machine *MachineState + cluster *ClusterState + v2 runtime.V1Alpha2State +} + +// MachineState represents the machine's state. +type MachineState struct { + platform runtime.Platform + resources state.State + + disks map[string]*probe.ProbedBlockDevice + + meta *meta.Meta + metaOnce sync.Once + + stagedInstall bool + stagedInstallImageRef string + stagedInstallOptions []byte + + kexecPrepared bool + + dbus DBusState +} + +// ClusterState represents the cluster's state. +type ClusterState struct{} + +// NewState initializes and returns the v1alpha1 state. +func NewState() (s *State, err error) { + p, err := platform.CurrentPlatform() + if err != nil { + return nil, err + } + + v2State, err := v1alpha2.NewState() + if err != nil { + return nil, err + } + + machine := &MachineState{ + platform: p, + resources: v2State.Resources(), + } + + err = machine.probeDisks() + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return nil, err + } + } + + cluster := &ClusterState{} + + s = &State{ + platform: p, + cluster: cluster, + machine: machine, + v2: v2State, + } + + return s, nil +} + +// Platform implements the state interface. +func (s *State) Platform() runtime.Platform { + return s.platform +} + +// Machine implements the state interface. +func (s *State) Machine() runtime.MachineState { + return s.machine +} + +// Cluster implements the state interface. +func (s *State) Cluster() runtime.ClusterState { + return s.cluster +} + +// V1Alpha2 implements the state interface. +func (s *State) V1Alpha2() runtime.V1Alpha2State { + return s.v2 +} + +func (s *MachineState) probeDisks(labels ...string) error { + if s.platform.Mode() == runtime.ModeContainer { + return os.ErrNotExist + } + + if len(labels) == 0 { + labels = []string{constants.EphemeralPartitionLabel, constants.BootPartitionLabel, constants.EFIPartitionLabel, constants.StatePartitionLabel} + } + + if s.disks == nil { + s.disks = map[string]*probe.ProbedBlockDevice{} + } + + for _, label := range labels { + if _, ok := s.disks[label]; ok { + continue + } + + var dev *probe.ProbedBlockDevice + + dev, err := probe.GetDevWithPartitionName(label) + if err != nil { + return err + } + + s.disks[label] = dev + } + + return nil +} + +// Meta implements the runtime.MachineState interface. +func (s *MachineState) Meta() runtime.Meta { + // no META in container mode + if s.platform.Mode() == runtime.ModeContainer { + return s + } + + var ( + justLoaded bool + loadErr error + ) + + s.metaOnce.Do(func() { + s.meta, loadErr = meta.New(context.Background(), s.resources) + if loadErr != nil { + if !os.IsNotExist(loadErr) { + log.Printf("META: failed to load: %s", loadErr) + } + } else { + s.probeMeta() + } + + justLoaded = true + }) + + return metaWrapper{ + MachineState: s, + justLoaded: justLoaded, + loadErr: loadErr, + } +} + +// ReadTag implements the runtime.Meta interface. +func (s *MachineState) ReadTag(t uint8) (val string, ok bool) { + if s.platform.Mode() == runtime.ModeContainer { + return "", false + } + + return s.meta.ReadTag(t) +} + +// ReadTagBytes implements the runtime.Meta interface. +func (s *MachineState) ReadTagBytes(t uint8) (val []byte, ok bool) { + if s.platform.Mode() == runtime.ModeContainer { + return nil, false + } + + return s.meta.ReadTagBytes(t) +} + +// SetTag implements the runtime.Meta interface. +func (s *MachineState) SetTag(ctx context.Context, t uint8, val string) (bool, error) { + if s.platform.Mode() == runtime.ModeContainer { + return false, nil + } + + return s.meta.SetTag(ctx, t, val) +} + +// SetTagBytes implements the runtime.Meta interface. +func (s *MachineState) SetTagBytes(ctx context.Context, t uint8, val []byte) (bool, error) { + if s.platform.Mode() == runtime.ModeContainer { + return false, nil + } + + return s.meta.SetTagBytes(ctx, t, val) +} + +// DeleteTag implements the runtime.Meta interface. +func (s *MachineState) DeleteTag(ctx context.Context, t uint8) (bool, error) { + if s.platform.Mode() == runtime.ModeContainer { + return false, nil + } + + return s.meta.DeleteTag(ctx, t) +} + +// Reload implements the runtime.Meta interface. +func (s *MachineState) Reload(ctx context.Context) error { + if s.platform.Mode() == runtime.ModeContainer { + return nil + } + + err := s.meta.Reload(ctx) + if err == nil { + s.probeMeta() + } + + return err +} + +// Flush implements the runtime.Meta interface. +func (s *MachineState) Flush() error { + if s.platform.Mode() == runtime.ModeContainer { + return nil + } + + return s.meta.Flush() +} + +func (s *MachineState) probeMeta() { + stagedInstallImageRef, ok1 := s.meta.ReadTag(meta.StagedUpgradeImageRef) + stagedInstallOptions, ok2 := s.meta.ReadTag(meta.StagedUpgradeInstallOptions) + + s.stagedInstall = ok1 && ok2 + + if s.stagedInstall { + // clear the staged install flags + _, err1 := s.meta.DeleteTag(context.Background(), meta.StagedUpgradeImageRef) + _, err2 := s.meta.DeleteTag(context.Background(), meta.StagedUpgradeInstallOptions) + + if err := s.meta.Flush(); err != nil || err1 != nil || err2 != nil { + // failed to delete staged install tags, clear the stagedInstall to prevent boot looping + s.stagedInstall = false + } + + s.stagedInstallImageRef = stagedInstallImageRef + s.stagedInstallOptions = []byte(stagedInstallOptions) + } +} + +// Disk implements the machine state interface. +func (s *MachineState) Disk(options ...disk.Option) *probe.ProbedBlockDevice { + opts := &disk.Options{ + Label: constants.EphemeralPartitionLabel, + } + + for _, opt := range options { + opt(opts) + } + + s.probeDisks(opts.Label) //nolint:errcheck + + return s.disks[opts.Label] +} + +// Close implements the machine state interface. +func (s *MachineState) Close() error { + var result *multierror.Error + + for label, disk := range s.disks { + if err := disk.Close(); err != nil { + e := multierror.Append(result, err) + if e != nil { + return e + } + } + + delete(s.disks, label) + } + + return result.ErrorOrNil() +} + +// Installed implements the machine state interface. +func (s *MachineState) Installed() bool { + return s.Disk( + disk.WithPartitionLabel(constants.EphemeralPartitionLabel), + ) != nil +} + +// IsInstallStaged implements the machine state interface. +func (s *MachineState) IsInstallStaged() bool { + return s.stagedInstall +} + +// StagedInstallImageRef implements the machine state interface. +func (s *MachineState) StagedInstallImageRef() string { + return s.stagedInstallImageRef +} + +// StagedInstallOptions implements the machine state interface. +func (s *MachineState) StagedInstallOptions() []byte { + return s.stagedInstallOptions +} + +// KexecPrepared implements the machine state interface. +func (s *MachineState) KexecPrepared(prepared bool) { + s.kexecPrepared = prepared +} + +// IsKexecPrepared implements the machine state interface. +func (s *MachineState) IsKexecPrepared() bool { + return s.kexecPrepared +} + +// DBus implements the machine state interface. +func (s *MachineState) DBus() runtime.DBusState { + return &s.dbus +} + +type metaWrapper struct { + *MachineState + justLoaded bool + loadErr error +} + +func (m metaWrapper) Reload(ctx context.Context) error { + if m.justLoaded { + return m.loadErr + } + + return m.MachineState.Reload(ctx) +} diff --git a/internal/app/machined/pkg/runtime/v1alpha2/adapters.go b/internal/app/machined/pkg/runtime/v1alpha2/adapters.go new file mode 100644 index 0000000..9037cb1 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha2/adapters.go @@ -0,0 +1,48 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package v1alpha2 + +import ( + "context" + + "github.com/cosi-project/runtime/pkg/state" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/config" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/platform" +) + +// platformConfigurator adapts a runtime.Platform to the config.PlatformConfigurator interface. +type platformConfigurator struct { + platform runtime.Platform + state state.State +} + +// Check interfaces. +var ( + _ config.PlatformConfigurator = &platformConfigurator{} +) + +func (p *platformConfigurator) Name() string { + return p.platform.Name() +} + +func (p *platformConfigurator) Configuration(ctx context.Context) ([]byte, error) { + return p.platform.Configuration(ctx, p.state) +} + +// platformEventer adapts a runtime.Platform to the config.PlatformEventer interface. +type platformEventer struct { + platform runtime.Platform +} + +// Check interfaces. +var ( + _ config.PlatformEventer = &platformEventer{} +) + +func (p *platformEventer) FireEvent(ctx context.Context, event platform.Event) { + platform.FireEvent(ctx, p.platform, event) +} diff --git a/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2.go b/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2.go new file mode 100644 index 0000000..f7776df --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2.go @@ -0,0 +1,6 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package v1alpha2 provides runtime implementation based on os-runtime. +package v1alpha2 diff --git a/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go b/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go new file mode 100644 index 0000000..cc1bb3b --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go @@ -0,0 +1,512 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package v1alpha2 + +import ( + "context" + "fmt" + "net/url" + "sync" + "time" + + "github.com/cosi-project/runtime/pkg/controller" + osruntime "github.com/cosi-project/runtime/pkg/controller/runtime" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/go-procfs/procfs" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/block" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/cluster" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/config" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/cri" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/etcd" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/files" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/hardware" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/k8s" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/kubeaccess" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/kubespan" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/network" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/perf" + runtimecontrollers "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/secrets" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/siderolink" + timecontrollers "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/time" + "github.com/aenix-io/talm/internal/app/machined/pkg/controllers/v1alpha1" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + runtimelogging "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/logging" + "github.com/aenix-io/talm/internal/app/machined/pkg/system" + "github.com/siderolabs/talos/pkg/logging" + talosconfig "github.com/siderolabs/talos/pkg/machinery/config/config" + "github.com/siderolabs/talos/pkg/machinery/constants" + configresource "github.com/siderolabs/talos/pkg/machinery/resources/config" +) + +// Controller implements runtime.V1alpha2Controller. +type Controller struct { + controllerRuntime *osruntime.Runtime + + loggingManager runtime.LoggingManager + consoleLogLevel zap.AtomicLevel + logger *zap.Logger + + v1alpha1Runtime runtime.Runtime +} + +// NewController creates Controller. +func NewController(v1alpha1Runtime runtime.Runtime) (*Controller, error) { + ctrl := &Controller{ + consoleLogLevel: zap.NewAtomicLevel(), + loggingManager: v1alpha1Runtime.Logging(), + v1alpha1Runtime: v1alpha1Runtime, + } + + var err error + + ctrl.logger, err = ctrl.makeLogger("controller-runtime") + if err != nil { + return nil, err + } + + ctrl.controllerRuntime, err = osruntime.NewRuntime(v1alpha1Runtime.State().V1Alpha2().Resources(), ctrl.logger) + + return ctrl, err +} + +// Run the controller runtime. +func (ctrl *Controller) Run(ctx context.Context, drainer *runtime.Drainer) error { + // adjust the log level based on machine configuration + go ctrl.watchMachineConfig(ctx) + + dnsCacheLogger, err := ctrl.makeLogger("dns-resolve-cache") + if err != nil { + return err + } + + for _, c := range []controller.Controller{ + &block.DevicesController{ + V1Alpha1Mode: ctrl.v1alpha1Runtime.State().Platform().Mode(), + }, + &block.DiscoveryController{}, + &cluster.AffiliateMergeController{}, + cluster.NewConfigController(), + &cluster.DiscoveryServiceController{}, + &cluster.EndpointController{}, + cluster.NewInfoController(), + &cluster.KubernetesPullController{}, + &cluster.KubernetesPushController{}, + &cluster.LocalAffiliateController{}, + &cluster.MemberController{}, + &cluster.NodeIdentityController{ + V1Alpha1Mode: ctrl.v1alpha1Runtime.State().Platform().Mode(), + }, + &config.AcquireController{ + PlatformConfiguration: &platformConfigurator{ + platform: ctrl.v1alpha1Runtime.State().Platform(), + state: ctrl.v1alpha1Runtime.State().V1Alpha2().Resources(), + }, + PlatformEvent: &platformEventer{ + platform: ctrl.v1alpha1Runtime.State().Platform(), + }, + ConfigSetter: ctrl.v1alpha1Runtime, + EventPublisher: ctrl.v1alpha1Runtime.Events(), + ValidationMode: ctrl.v1alpha1Runtime.State().Platform().Mode(), + }, + &config.MachineTypeController{}, + &cri.SeccompProfileController{}, + &cri.SeccompProfileFileController{ + V1Alpha1Mode: ctrl.v1alpha1Runtime.State().Platform().Mode(), + SeccompProfilesDirectory: constants.SeccompProfilesDirectory, + }, + &etcd.AdvertisedPeerController{}, + etcd.NewConfigController(), + &etcd.PKIController{}, + &etcd.SpecController{}, + &etcd.MemberController{}, + &files.CRIConfigPartsController{}, + &files.CRIRegistryConfigController{}, + &files.EtcFileController{ + EtcPath: "/etc", + ShadowPath: constants.SystemEtcPath, + }, + &hardware.SystemInfoController{ + V1Alpha1Mode: ctrl.v1alpha1Runtime.State().Platform().Mode(), + }, + &k8s.AddressFilterController{}, + k8s.NewControlPlaneAPIServerController(), + k8s.NewControlPlaneAdmissionControlController(), + k8s.NewControlPlaneAuditPolicyController(), + k8s.NewControlPlaneBootstrapManifestsController(), + k8s.NewControlPlaneControllerManagerController(), + k8s.NewControlPlaneExtraManifestsController(), + k8s.NewControlPlaneSchedulerController(), + &k8s.ControlPlaneStaticPodController{}, + &k8s.EndpointController{}, + &k8s.ExtraManifestController{}, + k8s.NewKubeletConfigController(), + &k8s.KubeletServiceController{ + V1Alpha1Services: system.Services(ctrl.v1alpha1Runtime), + V1Alpha1Mode: ctrl.v1alpha1Runtime.State().Platform().Mode(), + }, + &k8s.KubeletSpecController{ + V1Alpha1Mode: ctrl.v1alpha1Runtime.State().Platform().Mode(), + }, + &k8s.KubeletStaticPodController{}, + k8s.NewKubePrismEndpointsController(), + k8s.NewKubePrismConfigController(), + &k8s.KubePrismController{}, + &k8s.ManifestApplyController{}, + &k8s.ManifestController{}, + k8s.NewNodeIPConfigController(), + &k8s.NodeIPController{}, + &k8s.NodeApplyController{}, + &k8s.NodeCordonedSpecController{}, + &k8s.NodeLabelSpecController{}, + &k8s.NodeStatusController{}, + &k8s.NodeTaintSpecController{}, + &k8s.NodenameController{}, + &k8s.RenderConfigsStaticPodController{}, + &k8s.RenderSecretsStaticPodController{}, + &k8s.StaticEndpointController{}, + &k8s.StaticPodConfigController{}, + &k8s.StaticPodServerController{}, + kubeaccess.NewConfigController(), + &kubeaccess.CRDController{}, + &kubeaccess.EndpointController{}, + kubespan.NewConfigController(), + &kubespan.EndpointController{}, + &kubespan.IdentityController{}, + &kubespan.ManagerController{}, + &kubespan.PeerSpecController{}, + &network.AddressConfigController{ + Cmdline: procfs.ProcCmdline(), + V1Alpha1Mode: ctrl.v1alpha1Runtime.State().Platform().Mode(), + }, + &network.AddressEventController{ + V1Alpha1Events: ctrl.v1alpha1Runtime.Events(), + }, + &network.AddressMergeController{}, + &network.AddressSpecController{}, + &network.AddressStatusController{}, + &network.DeviceConfigController{}, + &network.DNSResolveCacheController{ + State: ctrl.v1alpha1Runtime.State().V1Alpha2().Resources(), + Logger: dnsCacheLogger, + }, + &network.DNSUpstreamController{}, + &network.EtcFileController{ + PodResolvConfPath: constants.PodResolvConfPath, + V1Alpha1Mode: ctrl.v1alpha1Runtime.State().Platform().Mode(), + }, + &network.HardwareAddrController{}, + &network.HostDNSConfigController{}, + &network.HostnameConfigController{ + Cmdline: procfs.ProcCmdline(), + }, + &network.HostnameMergeController{}, + &network.HostnameSpecController{ + V1Alpha1Mode: ctrl.v1alpha1Runtime.State().Platform().Mode(), + }, + &network.LinkConfigController{ + Cmdline: procfs.ProcCmdline(), + }, + &network.LinkMergeController{}, + &network.LinkSpecController{}, + &network.LinkStatusController{}, + &network.NfTablesChainConfigController{}, + &network.NfTablesChainController{}, + &network.NodeAddressController{}, + &network.OperatorConfigController{ + Cmdline: procfs.ProcCmdline(), + }, + &network.OperatorMergeController{}, + &network.OperatorSpecController{ + V1alpha1Platform: ctrl.v1alpha1Runtime.State().Platform(), + State: ctrl.v1alpha1Runtime.State().V1Alpha2().Resources(), + }, + &network.OperatorVIPConfigController{ + Cmdline: procfs.ProcCmdline(), + }, + &network.PlatformConfigController{ + V1alpha1Platform: ctrl.v1alpha1Runtime.State().Platform(), + PlatformState: ctrl.v1alpha1Runtime.State().V1Alpha2().Resources(), + }, + &network.ProbeController{}, + &network.ResolverConfigController{ + Cmdline: procfs.ProcCmdline(), + }, + &network.ResolverMergeController{}, + &network.ResolverSpecController{}, + &network.RouteConfigController{ + Cmdline: procfs.ProcCmdline(), + }, + &network.RouteMergeController{}, + &network.RouteSpecController{}, + &network.RouteStatusController{}, + &network.StatusController{ + V1Alpha1Mode: ctrl.v1alpha1Runtime.State().Platform().Mode(), + }, + &network.TimeServerConfigController{ + Cmdline: procfs.ProcCmdline(), + }, + &network.TimeServerMergeController{}, + &network.TimeServerSpecController{}, + &perf.StatsController{}, + &runtimecontrollers.CRIImageGCController{}, + &runtimecontrollers.DevicesStatusController{ + V1Alpha1Mode: ctrl.v1alpha1Runtime.State().Platform().Mode(), + }, + &runtimecontrollers.DropUpgradeFallbackController{ + MetaProvider: ctrl.v1alpha1Runtime.State().Machine(), + }, + &runtimecontrollers.ExtensionServiceConfigController{}, + &runtimecontrollers.ExtensionServiceConfigFilesController{ + V1Alpha1Mode: ctrl.v1alpha1Runtime.State().Platform().Mode(), + ExtensionsConfigBaseDir: constants.ExtensionServiceUserConfigPath, + }, + &runtimecontrollers.EventsSinkConfigController{ + Cmdline: procfs.ProcCmdline(), + V1Alpha1Mode: ctrl.v1alpha1Runtime.State().Platform().Mode(), + }, + &runtimecontrollers.EventsSinkController{ + V1Alpha1Events: ctrl.v1alpha1Runtime.Events(), + Drainer: drainer, + }, + &runtimecontrollers.ExtensionServiceController{ + V1Alpha1Services: system.Services(ctrl.v1alpha1Runtime), + ConfigPath: constants.ExtensionServiceConfigPath, + }, + &runtimecontrollers.ExtensionStatusController{}, + &runtimecontrollers.KernelModuleConfigController{}, + &runtimecontrollers.KernelModuleSpecController{ + V1Alpha1Mode: ctrl.v1alpha1Runtime.State().Platform().Mode(), + }, + &runtimecontrollers.KernelParamConfigController{}, + &runtimecontrollers.KernelParamDefaultsController{ + V1Alpha1Mode: ctrl.v1alpha1Runtime.State().Platform().Mode(), + }, + &runtimecontrollers.KernelParamSpecController{}, + &runtimecontrollers.KmsgLogConfigController{ + Cmdline: procfs.ProcCmdline(), + }, + &runtimecontrollers.KmsgLogDeliveryController{ + Drainer: drainer, + }, + &runtimecontrollers.MaintenanceConfigController{}, + &runtimecontrollers.MaintenanceServiceController{}, + &runtimecontrollers.MachineStatusController{ + V1Alpha1Events: ctrl.v1alpha1Runtime.Events(), + }, + &runtimecontrollers.MachineStatusPublisherController{ + V1Alpha1Events: ctrl.v1alpha1Runtime.Events(), + }, + &runtimecontrollers.SecurityStateController{ + V1Alpha1Mode: ctrl.v1alpha1Runtime.State().Platform().Mode(), + }, + runtimecontrollers.NewUniqueMachineTokenController(), + &runtimecontrollers.WatchdogTimerConfigController{}, + &runtimecontrollers.WatchdogTimerController{}, + &secrets.APICertSANsController{}, + &secrets.APIController{}, + &secrets.EtcdController{}, + secrets.NewKubeletController(), + &secrets.KubernetesCertSANsController{}, + &secrets.KubernetesDynamicCertsController{}, + &secrets.KubernetesController{}, + &secrets.MaintenanceController{}, + &secrets.MaintenanceCertSANsController{}, + &secrets.MaintenanceRootController{}, + secrets.NewRootEtcdController(), + secrets.NewRootKubernetesController(), + secrets.NewRootOSController(), + &secrets.TrustdController{}, + &siderolink.ConfigController{ + Cmdline: procfs.ProcCmdline(), + V1Alpha1Mode: ctrl.v1alpha1Runtime.State().Platform().Mode(), + }, + &siderolink.ManagerController{}, + &siderolink.UserspaceWireguardController{ + RelayRetryTimeout: 10 * time.Second, + }, + &timecontrollers.AdjtimeStatusController{ + V1Alpha1Mode: ctrl.v1alpha1Runtime.State().Platform().Mode(), + }, + &timecontrollers.SyncController{ + V1Alpha1Mode: ctrl.v1alpha1Runtime.State().Platform().Mode(), + }, + &v1alpha1.ServiceController{ + V1Alpha1Events: ctrl.v1alpha1Runtime.Events(), + }, + } { + if err := ctrl.controllerRuntime.RegisterController(c); err != nil { + return err + } + } + + return ctrl.controllerRuntime.Run(ctx) +} + +// DependencyGraph returns controller-resources dependencies. +func (ctrl *Controller) DependencyGraph() (*controller.DependencyGraph, error) { + return ctrl.controllerRuntime.GetDependencyGraph() +} + +type loggingDestination struct { + Format string + Endpoint *url.URL + ExtraTags map[string]string +} + +func (a *loggingDestination) Equal(b *loggingDestination) bool { + if a.Format != b.Format { + return false + } + + if a.Endpoint.String() != b.Endpoint.String() { + return false + } + + if len(a.ExtraTags) != len(b.ExtraTags) { + return false + } + + for k, v := range a.ExtraTags { + if vv, ok := b.ExtraTags[k]; !ok || vv != v { + return false + } + } + + return true +} + +func (ctrl *Controller) watchMachineConfig(ctx context.Context) { + watchCh := make(chan state.Event) + + if err := ctrl.v1alpha1Runtime.State().V1Alpha2().Resources().Watch( + ctx, + resource.NewMetadata(configresource.NamespaceName, configresource.MachineConfigType, configresource.V1Alpha1ID, resource.VersionUndefined), + watchCh, + ); err != nil { + ctrl.logger.Warn("error watching machine configuration", zap.Error(err)) + + return + } + + var loggingDestinations []loggingDestination + + for { + var cfg talosconfig.Config + select { + case event := <-watchCh: + if event.Type != state.Created && event.Type != state.Updated { + continue + } + + cfg = event.Resource.(*configresource.MachineConfig).Config() + + case <-ctx.Done(): + return + } + + ctrl.updateConsoleLoggingConfig(cfg.Debug()) + + if cfg.Machine() == nil { + ctrl.updateLoggingConfig(ctx, nil, &loggingDestinations) + } else { + ctrl.updateLoggingConfig(ctx, cfg.Machine().Logging().Destinations(), &loggingDestinations) + } + } +} + +func (ctrl *Controller) updateConsoleLoggingConfig(debug bool) { + newLogLevel := zapcore.InfoLevel + if debug { + newLogLevel = zapcore.DebugLevel + } + + if newLogLevel != ctrl.consoleLogLevel.Level() { + ctrl.logger.Info("setting console log level", zap.Stringer("level", newLogLevel)) + ctrl.consoleLogLevel.SetLevel(newLogLevel) + } +} + +func (ctrl *Controller) updateLoggingConfig(ctx context.Context, dests []talosconfig.LoggingDestination, prevLoggingDestinations *[]loggingDestination) { + loggingDestinations := make([]loggingDestination, len(dests)) + + for i, dest := range dests { + switch f := dest.Format(); f { + case constants.LoggingFormatJSONLines: + loggingDestinations[i] = loggingDestination{ + Format: f, + Endpoint: dest.Endpoint(), + ExtraTags: dest.ExtraTags(), + } + default: + // should not be possible due to validation + panic(fmt.Sprintf("unhandled log destination format %q", f)) + } + } + + loggingChanged := len(*prevLoggingDestinations) != len(loggingDestinations) + if !loggingChanged { + for i, u := range *prevLoggingDestinations { + if !u.Equal(&loggingDestinations[i]) { + loggingChanged = true + + break + } + } + } + + if !loggingChanged { + return + } + + *prevLoggingDestinations = loggingDestinations + + var prevSenders []runtime.LogSender + + if len(loggingDestinations) > 0 { + senders := xslices.Map(dests, runtimelogging.NewJSONLines) + + ctrl.logger.Info("enabling JSON logging") + prevSenders = ctrl.loggingManager.SetSenders(senders) + } else { + ctrl.logger.Info("disabling JSON logging") + prevSenders = ctrl.loggingManager.SetSenders(nil) + } + + closeCtx, closeCancel := context.WithTimeout(ctx, 3*time.Second) + defer closeCancel() + + var wg sync.WaitGroup + + for _, sender := range prevSenders { + wg.Add(1) + + go func() { + defer wg.Done() + + err := sender.Close(closeCtx) + ctrl.logger.Info("log sender closed", zap.Error(err)) + }() + } + + wg.Wait() +} + +func (ctrl *Controller) makeLogger(s string) (*zap.Logger, error) { + logWriter, err := ctrl.loggingManager.ServiceLog(s).Writer() + if err != nil { + return nil, err + } + + return logging.ZapLogger( + logging.NewLogDestination(logWriter, zapcore.DebugLevel, logging.WithColoredLevels()), + logging.NewLogDestination(logging.StdWriter, ctrl.consoleLogLevel, logging.WithoutTimestamp(), logging.WithoutLogLevels()), + ).With(logging.Component(s)), nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_state.go b/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_state.go new file mode 100644 index 0000000..b254358 --- /dev/null +++ b/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_state.go @@ -0,0 +1,260 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package v1alpha2 + +import ( + "context" + + "github.com/cosi-project/runtime/pkg/resource/meta" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/cosi-project/runtime/pkg/state/registry" + + talosconfig "github.com/siderolabs/talos/pkg/machinery/config" + "github.com/siderolabs/talos/pkg/machinery/resources/block" + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/cri" + "github.com/siderolabs/talos/pkg/machinery/resources/etcd" + "github.com/siderolabs/talos/pkg/machinery/resources/files" + "github.com/siderolabs/talos/pkg/machinery/resources/hardware" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/kubeaccess" + "github.com/siderolabs/talos/pkg/machinery/resources/kubespan" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/perf" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" + "github.com/siderolabs/talos/pkg/machinery/resources/siderolink" + "github.com/siderolabs/talos/pkg/machinery/resources/time" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +// State implements runtime.V1alpha2State interface. +type State struct { + resources state.State + + namespaceRegistry *registry.NamespaceRegistry + resourceRegistry *registry.ResourceRegistry +} + +// NewState creates State. +func NewState() (*State, error) { + s := &State{} + + ctx := context.TODO() + + s.resources = state.WrapCore(namespaced.NewState( + func(ns string) state.CoreState { + return inmem.NewStateWithOptions( + inmem.WithHistoryInitialCapacity(8), + inmem.WithHistoryMaxCapacity(1024), + inmem.WithHistoryGap(4), + )(ns) + }, + )) + s.namespaceRegistry = registry.NewNamespaceRegistry(s.resources) + s.resourceRegistry = registry.NewResourceRegistry(s.resources) + + if err := s.namespaceRegistry.RegisterDefault(ctx); err != nil { + return nil, err + } + + if err := s.resourceRegistry.RegisterDefault(ctx); err != nil { + return nil, err + } + + // register Talos namespaces + for _, ns := range []struct { + name string + description string + }{ + {v1alpha1.NamespaceName, "Talos v1alpha1 subsystems glue resources."}, + {cluster.NamespaceName, "Cluster configuration and discovery resources."}, + {cluster.RawNamespaceName, "Cluster unmerged raw resources."}, + {config.NamespaceName, "Talos node configuration."}, + {etcd.NamespaceName, "etcd resources."}, + {files.NamespaceName, "Files and file-like resources."}, + {hardware.NamespaceName, "Hardware resources."}, + {k8s.NamespaceName, "Kubernetes all node types resources."}, + {k8s.ControlPlaneNamespaceName, "Kubernetes control plane resources."}, + {kubespan.NamespaceName, "KubeSpan resources."}, + {network.NamespaceName, "Networking resources."}, + {network.ConfigNamespaceName, "Networking configuration resources."}, + {cri.NamespaceName, "CRI Seccomp resources."}, + {secrets.NamespaceName, "Resources with secret material."}, + {perf.NamespaceName, "Stats resources."}, + } { + if err := s.namespaceRegistry.Register(ctx, ns.name, ns.description); err != nil { + return nil, err + } + } + + // register Talos resources + for _, r := range []meta.ResourceWithRD{ + &block.Device{}, + &block.DiscoveredVolume{}, + &cluster.Affiliate{}, + &cluster.Config{}, + &cluster.Identity{}, + &cluster.Info{}, + &cluster.Member{}, + &config.MachineConfig{}, + &config.MachineType{}, + &cri.SeccompProfile{}, + &etcd.Config{}, + &etcd.PKIStatus{}, + &etcd.Spec{}, + &etcd.Member{}, + &files.EtcFileSpec{}, + &files.EtcFileStatus{}, + &hardware.Processor{}, + &hardware.MemoryModule{}, + &hardware.SystemInformation{}, + &k8s.AdmissionControlConfig{}, + &k8s.AuditPolicyConfig{}, + &k8s.APIServerConfig{}, + &k8s.KubePrismEndpoints{}, + &k8s.ConfigStatus{}, + &k8s.ControllerManagerConfig{}, + &k8s.Endpoint{}, + &k8s.ExtraManifestsConfig{}, + &k8s.KubeletConfig{}, + &k8s.KubeletLifecycle{}, + &k8s.KubeletSpec{}, + &k8s.KubePrismConfig{}, + &k8s.KubePrismStatuses{}, + &k8s.Manifest{}, + &k8s.ManifestStatus{}, + &k8s.BootstrapManifestsConfig{}, + &k8s.NodeCordonedSpec{}, + &k8s.NodeIP{}, + &k8s.NodeIPConfig{}, + &k8s.NodeLabelSpec{}, + &k8s.Nodename{}, + &k8s.NodeStatus{}, + &k8s.NodeTaintSpec{}, + &k8s.SchedulerConfig{}, + &k8s.StaticPod{}, + &k8s.StaticPodServerStatus{}, + &k8s.StaticPodStatus{}, + &k8s.SecretsStatus{}, + &kubeaccess.Config{}, + &kubespan.Config{}, + &kubespan.Endpoint{}, + &kubespan.Identity{}, + &kubespan.PeerSpec{}, + &kubespan.PeerStatus{}, + &network.AddressStatus{}, + &network.AddressSpec{}, + &network.DeviceConfigSpec{}, + &network.DNSResolveCache{}, + &network.DNSUpstream{}, + &network.HardwareAddr{}, + &network.HostDNSConfig{}, + &network.HostnameStatus{}, + &network.HostnameSpec{}, + &network.LinkRefresh{}, + &network.LinkStatus{}, + &network.LinkSpec{}, + &network.NfTablesChain{}, + &network.NodeAddress{}, + &network.NodeAddressFilter{}, + &network.OperatorSpec{}, + &network.ProbeSpec{}, + &network.ProbeStatus{}, + &network.ResolverStatus{}, + &network.ResolverSpec{}, + &network.RouteStatus{}, + &network.RouteSpec{}, + &network.Status{}, + &network.TimeServerStatus{}, + &network.TimeServerSpec{}, + &perf.CPU{}, + &perf.Memory{}, + &runtime.DevicesStatus{}, + &runtime.EventSinkConfig{}, + &runtime.ExtensionServiceConfig{}, + &runtime.ExtensionServiceConfigStatus{}, + &runtime.ExtensionStatus{}, + &runtime.KernelModuleSpec{}, + &runtime.KernelParamSpec{}, + &runtime.KernelParamDefaultSpec{}, + &runtime.KernelParamStatus{}, + &runtime.KmsgLogConfig{}, + &runtime.MaintenanceServiceConfig{}, + &runtime.MaintenanceServiceRequest{}, + &runtime.MachineResetSignal{}, + &runtime.MachineStatus{}, + &runtime.MetaKey{}, + &runtime.MetaLoaded{}, + &runtime.MountStatus{}, + &runtime.PlatformMetadata{}, + &runtime.SecurityState{}, + &runtime.UniqueMachineToken{}, + &runtime.WatchdogTimerConfig{}, + &runtime.WatchdogTimerStatus{}, + &secrets.API{}, + &secrets.CertSAN{}, + &secrets.Etcd{}, + &secrets.EtcdRoot{}, + &secrets.Kubelet{}, + &secrets.Kubernetes{}, + &secrets.KubernetesDynamicCerts{}, + &secrets.KubernetesRoot{}, + &secrets.MaintenanceServiceCerts{}, + &secrets.MaintenanceRoot{}, + &secrets.OSRoot{}, + &secrets.Trustd{}, + &siderolink.Config{}, + &siderolink.Tunnel{}, + &time.AdjtimeStatus{}, + &time.Status{}, + &v1alpha1.AcquireConfigSpec{}, + &v1alpha1.AcquireConfigStatus{}, + &v1alpha1.Service{}, + } { + if err := s.resourceRegistry.Register(ctx, r); err != nil { + return nil, err + } + } + + return s, nil +} + +// Resources implements runtime.V1alpha2State interface. +func (s *State) Resources() state.State { + return s.resources +} + +// NamespaceRegistry implements runtime.V1alpha2State interface. +func (s *State) NamespaceRegistry() *registry.NamespaceRegistry { + return s.namespaceRegistry +} + +// ResourceRegistry implements runtime.V1alpha2State interface. +func (s *State) ResourceRegistry() *registry.ResourceRegistry { + return s.resourceRegistry +} + +// SetConfig implements runtime.V1alpha2State interface. +func (s *State) SetConfig(cfg talosconfig.Provider) error { + cfgResource := config.NewMachineConfig(cfg) + ctx := context.TODO() + + oldCfg, err := s.resources.Get(ctx, cfgResource.Metadata()) + if err != nil { + if state.IsNotFoundError(err) { + return s.resources.Create(ctx, cfgResource) + } + + return err + } + + cfgResource.Metadata().SetVersion(oldCfg.Metadata().Version()) + + return s.resources.Update(ctx, cfgResource) +} diff --git a/internal/app/machined/pkg/system/events/events.go b/internal/app/machined/pkg/system/events/events.go new file mode 100644 index 0000000..656982e --- /dev/null +++ b/internal/app/machined/pkg/system/events/events.go @@ -0,0 +1,148 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package events + +import ( + "time" + + "github.com/siderolabs/gen/xslices" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/aenix-io/talm/internal/app/machined/pkg/system/health" + machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" +) + +// MaxEventsToKeep is maximum number of events to keep per service before dropping old entries. +const MaxEventsToKeep = 64 + +// ServiceState is enum of service run states. +type ServiceState int + +// ServiceState constants. +const ( + StateInitialized ServiceState = iota + StatePreparing + StateWaiting + StateRunning + StateStopping + StateFinished + StateFailed + StateSkipped + StateStarting +) + +func (state ServiceState) String() string { + switch state { + case StateInitialized: + return "Initialized" + case StateStarting: + return "Starting" + case StatePreparing: + return "Preparing" + case StateWaiting: + return "Waiting" + case StateRunning: + return "Running" + case StateStopping: + return "Stopping" + case StateFinished: + return "Finished" + case StateFailed: + return "Failed" + case StateSkipped: + return "Skipped" + default: + return "Unknown" + } +} + +// ServiceEvent describes state change of the running service. +type ServiceEvent struct { + Message string + State ServiceState + Health health.Status + Timestamp time.Time +} + +// AsProto returns protobuf representation of respective machined event. +func (event *ServiceEvent) AsProto(service string) *machineapi.ServiceStateEvent { + return &machineapi.ServiceStateEvent{ + Service: service, + Action: machineapi.ServiceStateEvent_Action(event.State), + Message: event.Message, + Health: event.Health.AsProto(), + } +} + +// ServiceEvents is a fixed length history of events. +type ServiceEvents struct { + events []ServiceEvent + pos int + discarded uint +} + +// Push appends new event to the history popping out oldest event on overflow. +func (events *ServiceEvents) Push(event ServiceEvent) { + if events.events == nil { + events.events = make([]ServiceEvent, MaxEventsToKeep) + } + + if events.events[events.pos].Message != "" { + // overwriting some entry + events.discarded++ + } + + events.events[events.pos] = event + events.pos = (events.pos + 1) % len(events.events) +} + +// Get return a copy of event history, with most recent event being the last one. +func (events *ServiceEvents) Get(count int) (result []ServiceEvent) { + if events.events == nil { + return + } + + if count > MaxEventsToKeep { + count = MaxEventsToKeep + } + + n := len(events.events) + + for i := (events.pos - count + n) % n; count > 0; i = (i + 1) % n { + if events.events[i].Message != "" { + result = append(result, events.events[i]) + } + + count-- + } + + return +} + +// AsProto returns protobuf-ready serialized snapshot. +func (events *ServiceEvents) AsProto(count int) *machineapi.ServiceEvents { + eventList := events.Get(count) + + fn := func(event ServiceEvent) *machineapi.ServiceEvent { + tspb := timestamppb.New(event.Timestamp) + + return &machineapi.ServiceEvent{ + Msg: event.Message, + State: event.State.String(), + Ts: tspb, + } + } + + return &machineapi.ServiceEvents{ + Events: xslices.Map(eventList, fn), + } +} + +// Recorder adds new event to the history of events, formatting message with args using Sprintf. +type Recorder func(newstate ServiceState, message string, args ...interface{}) + +// NullRecorder discards events. +func NullRecorder(newstate ServiceState, message string, args ...interface{}) { +} diff --git a/internal/app/machined/pkg/system/events/events_test.go b/internal/app/machined/pkg/system/events/events_test.go new file mode 100644 index 0000000..0af519f --- /dev/null +++ b/internal/app/machined/pkg/system/events/events_test.go @@ -0,0 +1,78 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package events_test + +import ( + "strconv" + "testing" + + "github.com/siderolabs/gen/xslices" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/system/events" +) + +type EventsSuite struct { + suite.Suite +} + +func (suite *EventsSuite) assertEvents(expectedMessages []string, evs []events.ServiceEvent) { + messages := xslices.Map(evs, func(ev events.ServiceEvent) string { return ev.Message }) + suite.Assert().Equal(expectedMessages, messages) +} + +func (suite *EventsSuite) TestEmpty() { + var e events.ServiceEvents + + suite.Assert().Equal([]events.ServiceEvent(nil), e.Get(100)) +} + +func (suite *EventsSuite) TestSome() { + var e events.ServiceEvents + + for i := range 5 { + e.Push(events.ServiceEvent{ + Message: strconv.Itoa(i), + }) + } + + suite.Assert().Equal([]events.ServiceEvent(nil), e.Get(0)) + suite.assertEvents([]string{"4"}, e.Get(1)) + suite.assertEvents([]string{"1", "2", "3", "4"}, e.Get(4)) + suite.assertEvents([]string{"0", "1", "2", "3", "4"}, e.Get(5)) + suite.assertEvents([]string{"0", "1", "2", "3", "4"}, e.Get(6)) + suite.assertEvents([]string{"0", "1", "2", "3", "4"}, e.Get(100)) + + protoEvents := e.AsProto(1) + suite.Assert().Len(protoEvents.Events, 1) + suite.Assert().Equal("4", protoEvents.Events[0].Msg) + suite.Assert().Equal("Initialized", protoEvents.Events[0].State) +} + +func (suite *EventsSuite) TestOverflow() { + var e events.ServiceEvents + + numEvents := events.MaxEventsToKeep*2 + 3 + + for i := range numEvents { + e.Push(events.ServiceEvent{ + Message: strconv.Itoa(i), + }) + } + + suite.Assert().Equal([]events.ServiceEvent(nil), e.Get(0)) + suite.assertEvents([]string{strconv.Itoa(numEvents - 1)}, e.Get(1)) + + expected := []string{} + for i := numEvents - events.MaxEventsToKeep; i < numEvents; i++ { + expected = append(expected, strconv.Itoa(i)) + } + suite.assertEvents(expected, e.Get(events.MaxEventsToKeep*10)) + suite.assertEvents(expected[len(expected)-3:], e.Get(3)) +} + +func TestEventsSuite(t *testing.T) { + suite.Run(t, new(EventsSuite)) +} diff --git a/internal/app/machined/pkg/system/export_test.go b/internal/app/machined/pkg/system/export_test.go new file mode 100644 index 0000000..e6079ec --- /dev/null +++ b/internal/app/machined/pkg/system/export_test.go @@ -0,0 +1,18 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package system + +import ( + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/conditions" +) + +func NewServices(runtime runtime.Runtime) *singleton { //nolint:revive + return newServices(runtime) +} + +func WaitForServiceWithInstance(instance *singleton, event StateEvent, service string) conditions.Condition { + return waitForService(instance, event, service) +} diff --git a/internal/app/machined/pkg/system/health/check.go b/internal/app/machined/pkg/system/health/check.go new file mode 100644 index 0000000..9b00c1d --- /dev/null +++ b/internal/app/machined/pkg/system/health/check.go @@ -0,0 +1,64 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package health + +import ( + "context" + "time" +) + +// Check runs the health check under given context. +// +// Healthcheck is considered successful when func returns no error. +// Func should terminate when context is canceled. +type Check func(ctx context.Context) error + +// Run the health check publishing the results to state. +// +// Run aborts when context is canceled. +func Run(ctx context.Context, settings *Settings, state *State, check Check) error { + state.Init() + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(settings.InitialDelay): + } + + ticker := time.NewTicker(settings.Period) + defer ticker.Stop() + + var ( + err error + healthy bool + message string + checkCtx context.Context + checkCtxCancel context.CancelFunc + ) + + for { + err = func() error { + checkCtx, checkCtxCancel = context.WithTimeout(ctx, settings.Timeout) + defer checkCtxCancel() + + return check(checkCtx) + }() + + healthy = err == nil + message = "" + + if !healthy { + message = err.Error() + } + + state.Update(healthy, message) + + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + } +} diff --git a/internal/app/machined/pkg/system/health/health_test.go b/internal/app/machined/pkg/system/health/health_test.go new file mode 100644 index 0000000..de60e48 --- /dev/null +++ b/internal/app/machined/pkg/system/health/health_test.go @@ -0,0 +1,208 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package health_test + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/system/health" +) + +type CheckSuite struct { + suite.Suite +} + +func (suite *CheckSuite) TestHealthy() { + settings := health.Settings{ + InitialDelay: time.Millisecond, + Period: 10 * time.Millisecond, + Timeout: time.Millisecond, + } + + var called uint32 + + //nolint:unparam + check := func(context.Context) error { + atomic.AddUint32(&called, 1) + + return nil + } + + var state health.State + + errCh := make(chan error) + ctx, ctxCancel := context.WithCancel(context.Background()) + + go func() { + errCh <- health.Run(ctx, &settings, &state, check) + }() + + for range 20 { + time.Sleep(10 * time.Millisecond) + + if atomic.LoadUint32(&called) > 2 { + break + } + } + + ctxCancel() + + suite.Assert().EqualError(<-errCh, context.Canceled.Error()) + suite.Assert().True(called > 2) + + protoHealth := state.AsProto() + suite.Assert().False(protoHealth.Unknown) + suite.Assert().True(protoHealth.Healthy) + suite.Assert().Equal("", protoHealth.LastMessage) +} + +func (suite *CheckSuite) TestHealthChange() { + settings := health.Settings{ + InitialDelay: time.Millisecond, + Period: time.Millisecond, + Timeout: time.Millisecond, + } + + var healthy uint32 + + check := func(context.Context) error { + if atomic.LoadUint32(&healthy) == 1 { + return nil + } + + return errors.New("health failed") + } + + var state health.State + + notifyCh := make(chan health.StateChange, 2) + state.Subscribe(notifyCh) + + errCh := make(chan error) + ctx, ctxCancel := context.WithCancel(context.Background()) + + go func() { + errCh <- health.Run(ctx, &settings, &state, check) + }() + + // wait for the first health change + for range 20 { + if state.Get().Healthy != nil { + break + } + + time.Sleep(50 * time.Millisecond) + } + + suite.Require().False(*state.Get().Healthy) + suite.Require().Equal("health failed", state.Get().LastMessage) + + atomic.StoreUint32(&healthy, 1) + + for range 10 { + time.Sleep(20 * time.Millisecond) + + if *state.Get().Healthy { + break + } + } + + suite.Require().True(*state.Get().Healthy) + suite.Require().Equal("", state.Get().LastMessage) + + ctxCancel() + + suite.Assert().EqualError(<-errCh, context.Canceled.Error()) + + state.Unsubscribe(notifyCh) + + close(notifyCh) + + change := <-notifyCh + suite.Assert().Nil(change.Old.Healthy) + suite.Assert().False(*change.New.Healthy) + + change = <-notifyCh + suite.Assert().False(*change.Old.Healthy) + suite.Assert().True(*change.New.Healthy) +} + +func (suite *CheckSuite) TestCheckAbort() { + settings := health.Settings{ + InitialDelay: time.Millisecond, + Period: time.Millisecond, + Timeout: time.Millisecond, + } + + check := func(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(50 * time.Second): + // should never be triggered, as Timeout is 1ms + return nil + } + } + + var state health.State + + errCh := make(chan error) + ctx, ctxCancel := context.WithCancel(context.Background()) + + go func() { + errCh <- health.Run(ctx, &settings, &state, check) + }() + + // wait for the first health change + for range 20 { + if state.Get().Healthy != nil { + break + } + + time.Sleep(50 * time.Millisecond) + } + + suite.Require().False(*state.Get().Healthy) + suite.Require().Equal("context deadline exceeded", state.Get().LastMessage) + + ctxCancel() + + suite.Assert().EqualError(<-errCh, context.Canceled.Error()) +} + +func (suite *CheckSuite) TestInitialState() { + settings := health.Settings{ + InitialDelay: 5 * time.Minute, + } + + var state health.State + + ctx, ctxCancel := context.WithCancel(context.Background()) + + errCh := make(chan error) + + go func() { + errCh <- health.Run(ctx, &settings, &state, nil) + }() + + time.Sleep(100 * time.Millisecond) + + suite.Require().Nil(state.Get().Healthy) + suite.Require().Equal("Unknown", state.Get().LastMessage) + + ctxCancel() + + suite.Assert().EqualError(<-errCh, context.Canceled.Error()) +} + +func TestCheckSuite(t *testing.T) { + suite.Run(t, new(CheckSuite)) +} diff --git a/internal/app/machined/pkg/system/health/settings.go b/internal/app/machined/pkg/system/health/settings.go new file mode 100644 index 0000000..906dd33 --- /dev/null +++ b/internal/app/machined/pkg/system/health/settings.go @@ -0,0 +1,23 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package health + +import "time" + +// Settings configures health check +// +// Fields are similar to k8s pod probe definitions. +type Settings struct { + InitialDelay time.Duration + Period time.Duration + Timeout time.Duration +} + +// DefaultSettings provides some default health check settings. +var DefaultSettings = Settings{ + InitialDelay: time.Second, + Period: 5 * time.Second, + Timeout: 500 * time.Millisecond, +} diff --git a/internal/app/machined/pkg/system/health/status.go b/internal/app/machined/pkg/system/health/status.go new file mode 100644 index 0000000..5956140 --- /dev/null +++ b/internal/app/machined/pkg/system/health/status.go @@ -0,0 +1,131 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package health + +import ( + "slices" + "sync" + "time" + + "google.golang.org/protobuf/types/known/timestamppb" + + machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" +) + +// Status of the healthcheck. +type Status struct { + Healthy *bool + LastChange time.Time + LastMessage string +} + +// AsProto returns protobuf-ready health state. +func (status *Status) AsProto() *machineapi.ServiceHealth { + tspb := timestamppb.New(status.LastChange) + + return &machineapi.ServiceHealth{ + Unknown: status.Healthy == nil, + Healthy: status.Healthy != nil && *status.Healthy, + LastMessage: status.LastMessage, + LastChange: tspb, + } +} + +// StateChange is used to notify about status changes. +type StateChange struct { + Old Status + New Status +} + +// State provides proper locking around health state. +type State struct { + sync.Mutex + + status Status + subscribers []chan<- StateChange +} + +// Update health status (locked). +func (state *State) Update(healthy bool, message string) { + state.Lock() + + oldStatus := state.status + notify := false + + if state.status.Healthy == nil || *state.status.Healthy != healthy { + notify = true + state.status.Healthy = &healthy + state.status.LastChange = time.Now() + } + + state.status.LastMessage = message + + newStatus := state.status + + var subscribers []chan<- StateChange + if notify { + subscribers = slices.Clone(state.subscribers) + } + + state.Unlock() + + if notify { + for _, ch := range subscribers { + select { + case ch <- StateChange{oldStatus, newStatus}: + default: // drop messages to clients which don't consume them + } + } + } +} + +// Subscribe for the notifications on state changes. +func (state *State) Subscribe(ch chan<- StateChange) { + state.Lock() + defer state.Unlock() + + state.subscribers = append(state.subscribers, ch) +} + +// Unsubscribe from state changes. +func (state *State) Unsubscribe(ch chan<- StateChange) { + state.Lock() + defer state.Unlock() + + for i := 0; i < len(state.subscribers); { + if state.subscribers[i] == ch { + state.subscribers[i] = state.subscribers[len(state.subscribers)-1] + state.subscribers[len(state.subscribers)-1] = nil + state.subscribers = state.subscribers[:len(state.subscribers)-1] + } else { + i++ + } + } +} + +// Init health status (locked). +func (state *State) Init() { + state.Lock() + defer state.Unlock() + + state.status.LastMessage = "Unknown" + state.status.LastChange = time.Now() + state.status.Healthy = nil +} + +// Get returns health status (locked). +func (state *State) Get() Status { + state.Lock() + defer state.Unlock() + + return state.status +} + +// AsProto returns protobuf-ready health state. +func (state *State) AsProto() *machineapi.ServiceHealth { + status := state.Get() + + return status.AsProto() +} diff --git a/internal/app/machined/pkg/system/integration_test.go b/internal/app/machined/pkg/system/integration_test.go new file mode 100644 index 0000000..dd64aa9 --- /dev/null +++ b/internal/app/machined/pkg/system/integration_test.go @@ -0,0 +1,93 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package system_test + +import ( + "context" + "io" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/system" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/events" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/goroutine" + "github.com/siderolabs/talos/pkg/conditions" +) + +type TestCondition struct{} + +func (TestCondition) String() string { + return "test-condition" +} + +func (TestCondition) Wait(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(10 * time.Millisecond): + return nil + } +} + +type TestService struct{} + +func (TestService) ID(runtime.Runtime) string { + return "test-service" +} + +func (TestService) PreFunc(ctx context.Context, r runtime.Runtime) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + return nil + } +} + +func (TestService) Runner(r runtime.Runtime) (runner.Runner, error) { + return goroutine.NewRunner(r, "test-service", func(ctx context.Context, r runtime.Runtime, logOutput io.Writer) error { + <-ctx.Done() + + return nil + }), nil +} + +func (TestService) PostFunc(runtime.Runtime, events.ServiceState) error { + return nil +} + +func (TestService) Condition(runtime.Runtime) conditions.Condition { + return TestCondition{} +} + +func (TestService) DependsOn(runtime.Runtime) []string { + return nil +} + +func TestRestartService(t *testing.T) { + deadline, ok := t.Deadline() + if !ok { + deadline = time.Now().Add(15 * time.Second) + } + + ctx, cancel := context.WithDeadline(context.Background(), deadline) + defer cancel() + + services := system.NewServices(nil) + + services.Load(TestService{}) + + for range 100 { + require.NoError(t, services.Start("test-service")) + + require.NoError(t, system.WaitForServiceWithInstance(services, system.StateEventUp, "test-service").Wait(ctx)) + + require.NoError(t, services.Stop(ctx, "test-service")) + } +} diff --git a/internal/app/machined/pkg/system/mocks_test.go b/internal/app/machined/pkg/system/mocks_test.go new file mode 100644 index 0000000..7d0f964 --- /dev/null +++ b/internal/app/machined/pkg/system/mocks_test.go @@ -0,0 +1,150 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package system_test + +import ( + "context" + "errors" + "sync/atomic" + "time" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/events" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/health" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner" + "github.com/siderolabs/talos/pkg/conditions" +) + +type MockService struct { + name string + preError error + runnerError error + nilRunner bool + runner runner.Runner + condition conditions.Condition + postError error + dependencies []string +} + +func (m *MockService) ID(runtime.Runtime) string { + if m.name != "" { + return m.name + } + + return "MockRunner" +} + +func (m *MockService) PreFunc(context.Context, runtime.Runtime) error { + return m.preError +} + +func (m *MockService) Runner(runtime.Runtime) (runner.Runner, error) { + if m.runner != nil { + return m.runner, m.runnerError + } + + if m.nilRunner { + return nil, nil + } + + return &MockRunner{exitCh: make(chan error)}, m.runnerError +} + +func (m *MockService) PostFunc(runtime.Runtime, events.ServiceState) error { + return m.postError +} + +func (m *MockService) Condition(runtime.Runtime) conditions.Condition { + return m.condition +} + +func (m *MockService) DependsOn(runtime.Runtime) []string { + return m.dependencies +} + +type MockHealthcheckedService struct { + MockService + + notHealthy uint32 +} + +func (m *MockHealthcheckedService) SetHealthy(healthy bool) { + if healthy { + atomic.StoreUint32(&m.notHealthy, 0) + } else { + atomic.StoreUint32(&m.notHealthy, 1) + } +} + +func (m *MockHealthcheckedService) HealthFunc(runtime.Runtime) health.Check { + return func(context.Context) error { + if atomic.LoadUint32(&m.notHealthy) == 0 { + return nil + } + + return errors.New("not healthy") + } +} + +func (m *MockHealthcheckedService) HealthSettings(runtime.Runtime) *health.Settings { + return &health.Settings{ + InitialDelay: time.Millisecond, + Timeout: time.Second, + Period: time.Millisecond, + } +} + +type MockRunner struct { + exitCh chan error +} + +func (m *MockRunner) Open() error { + return nil +} + +func (m *MockRunner) Close() error { + return nil +} + +func (m *MockRunner) Run(eventSink events.Recorder) error { + eventSink(events.StateRunning, "Running") + + return <-m.exitCh +} + +func (m *MockRunner) Stop() error { + close(m.exitCh) + + return nil +} + +func (m *MockRunner) String() string { + return "MockRunner()" +} + +type MockCondition struct { + done chan struct{} + desc string +} + +func NewMockCondition(desc string) *MockCondition { + return &MockCondition{ + done: make(chan struct{}), + desc: desc, + } +} + +func (m *MockCondition) String() string { + return m.desc +} + +func (m *MockCondition) Wait(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-m.done: + return nil + } +} diff --git a/internal/app/machined/pkg/system/runner/containerd/containerd.go b/internal/app/machined/pkg/system/runner/containerd/containerd.go new file mode 100644 index 0000000..d04d200 --- /dev/null +++ b/internal/app/machined/pkg/system/runner/containerd/containerd.go @@ -0,0 +1,373 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package containerd + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "syscall" + "time" + + "github.com/containerd/containerd" + "github.com/containerd/containerd/cio" + "github.com/containerd/containerd/contrib/seccomp" + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/namespaces" + "github.com/containerd/containerd/oci" + + "github.com/aenix-io/talm/internal/app/machined/pkg/system/events" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner" + "github.com/aenix-io/talm/internal/pkg/cgroup" +) + +// containerdRunner is a runner.Runner that runs container in containerd. +type containerdRunner struct { + args *runner.Args + opts *runner.Options + debug bool + + stop chan struct{} + stopped chan struct{} + + client *containerd.Client + ctx context.Context //nolint:containedctx + container containerd.Container + stdinCloser *StdinCloser +} + +// NewRunner creates runner.Runner that runs a container in containerd. +func NewRunner(debug bool, args *runner.Args, setters ...runner.Option) runner.Runner { + r := &containerdRunner{ + args: args, + opts: runner.DefaultOptions(), + debug: debug, + stop: make(chan struct{}), + stopped: make(chan struct{}), + } + + for _, setter := range setters { + setter(r.opts) + } + + return r +} + +// Open implements the Runner interface. +func (c *containerdRunner) Open() error { + // Create the containerd client. + var err error + + c.ctx = namespaces.WithNamespace(context.Background(), c.opts.Namespace) + + c.client, err = containerd.New(c.opts.ContainerdAddress) + if err != nil { + return err + } + + var image containerd.Image + + if c.opts.ContainerImage != "" { + image, err = c.client.GetImage(c.ctx, c.opts.ContainerImage) + if err != nil { + return err + } + } + + // See if there's previous container/snapshot to clean up + var oldcontainer containerd.Container + + if oldcontainer, err = c.client.LoadContainer(c.ctx, c.args.ID); err == nil { + if err = oldcontainer.Delete(c.ctx, containerd.WithSnapshotCleanup); err != nil { + return fmt.Errorf("error deleting old container instance: %w", err) + } + } + + if err = c.client.SnapshotService("").Remove(c.ctx, c.args.ID); err != nil && !errdefs.IsNotFound(err) { + return fmt.Errorf("error cleaning up stale snapshot: %w", err) + } + + // Create the container. + specOpts := c.newOCISpecOpts(image) + containerOpts := c.newContainerOpts(image, specOpts) + + c.container, err = c.client.NewContainer( + c.ctx, + c.args.ID, + containerOpts..., + ) + if err != nil { + return fmt.Errorf("failed to create container %q: %w", c.args.ID, err) + } + + return nil +} + +// Close implements runner.Runner interface. +func (c *containerdRunner) Close() error { + if c.container != nil { + err := c.container.Delete(c.ctx, containerd.WithSnapshotCleanup) + if err != nil { + return err + } + } + + if c.client == nil { + return nil + } + + return c.client.Close() +} + +// Run implements runner.Runner interface +// +//nolint:gocyclo,cyclop +func (c *containerdRunner) Run(eventSink events.Recorder) error { + defer close(c.stopped) + + var ( + task containerd.Task + logW io.WriteCloser + err error + ) + + // attempt to clean up a task if it already exists + task, err = c.container.Task(c.ctx, nil) + if err == nil { + var s <-chan containerd.ExitStatus + + s, err = task.Wait(c.ctx) + if err != nil { + return fmt.Errorf("failed to wait for the task %q: %w", c.args.ID, err) + } + + err = task.Kill(c.ctx, syscall.SIGKILL, containerd.WithKillAll) + if err != nil && !errdefs.IsNotFound(err) { + return fmt.Errorf("failed to kill the task %q: %w", c.args.ID, err) + } + + select { + case <-s: + case <-c.stop: + return nil + } + + if _, err = task.Delete(c.ctx); err != nil { + return fmt.Errorf("failed to clean up task %q: %w", c.args.ID, err) + } + } + + logW, err = c.opts.LoggingManager.ServiceLog(c.args.ID).Writer() + if err != nil { + return fmt.Errorf("error creating log: %w", err) + } + + defer logW.Close() //nolint:errcheck + + var w io.Writer = logW + + if c.debug { + w = io.MultiWriter(w, os.Stdout) + } + + r, err := c.StdinReader() + if err != nil { + return fmt.Errorf("failed to create stdin reader: %w", err) + } + + creator := cio.NewCreator(cio.WithStreams(r, w, w)) + + // Create the task and start it. + task, err = c.container.NewTask(c.ctx, creator) + if err != nil { + return fmt.Errorf("failed to create task: %q: %w", c.args.ID, err) + } + + if r != nil { + // See https://github.com/containerd/containerd/issues/4489. + go c.stdinCloser.WaitAndClose(c.ctx, task) + } + + defer task.Delete(c.ctx) //nolint:errcheck + + if err = task.Start(c.ctx); err != nil { + return fmt.Errorf("failed to start task: %q: %w", c.args.ID, err) + } + + eventSink(events.StateRunning, "Started task %s (PID %d) for container %s", task.ID(), task.Pid(), c.container.ID()) + + statusC, err := task.Wait(c.ctx) + if err != nil { + return fmt.Errorf("failed waiting for task: %q: %w", c.args.ID, err) + } + + select { + case status := <-statusC: + code := status.ExitCode() + if code != 0 { + return fmt.Errorf("task %q failed: exit code %d", c.args.ID, code) + } + + return nil + case <-c.stop: + // graceful stop the task + eventSink( + events.StateStopping, + "Sending SIGTERM to task %s (PID %d, container %s)", + task.ID(), + task.Pid(), + c.container.ID(), + ) + + if err = task.Kill(c.ctx, syscall.SIGTERM, containerd.WithKillAll); err != nil { + return fmt.Errorf("error sending SIGTERM: %w", err) + } + } + + select { + case <-statusC: + // stopped process exited + return nil + case <-time.After(c.opts.GracefulShutdownTimeout): + // kill the process + eventSink( + events.StateStopping, + "Sending SIGKILL to task %s (PID %d, container %s)", + task.ID(), + task.Pid(), + c.container.ID(), + ) + + if err = task.Kill(c.ctx, syscall.SIGKILL, containerd.WithKillAll); err != nil { + return fmt.Errorf("error sending SIGKILL: %w", err) + } + } + + <-statusC + + return logW.Close() +} + +// Stop implements runner.Runner interface. +func (c *containerdRunner) Stop() error { + close(c.stop) + + <-c.stopped + + c.stop = make(chan struct{}) + c.stopped = make(chan struct{}) + + return nil +} + +func (c *containerdRunner) newContainerOpts( + image containerd.Image, + specOpts []oci.SpecOpts, +) []containerd.NewContainerOpts { + containerOpts := []containerd.NewContainerOpts{} + + if image != nil { + containerOpts = append( + containerOpts, + containerd.WithImage(image), + containerd.WithNewSnapshot(c.args.ID, image), + ) + } + + containerOpts = append( + containerOpts, + containerd.WithNewSpec(specOpts...), + ) + + containerOpts = append( + containerOpts, + c.opts.ContainerOpts..., + ) + + return containerOpts +} + +func (c *containerdRunner) newOCISpecOpts(image oci.Image) []oci.SpecOpts { + specOpts := []oci.SpecOpts{} + + if image != nil { + specOpts = append( + specOpts, + oci.WithImageConfig(image), + ) + } + + specOpts = append( + specOpts, + oci.WithProcessArgs(c.args.ProcessArgs...), + oci.WithEnv(c.opts.Env), + oci.WithHostHostsFile, + oci.WithHostResolvconf, + oci.WithNoNewPrivileges, + ) + + if c.opts.OOMScoreAdj != 0 { + specOpts = append( + specOpts, + WithOOMScoreAdj(c.opts.OOMScoreAdj), + ) + } + + if c.opts.CgroupPath != "" { + specOpts = append( + specOpts, + oci.WithCgroup(cgroup.Path(c.opts.CgroupPath)), + ) + } + + specOpts = append( + specOpts, + c.opts.OCISpecOpts..., + ) + + if c.opts.OverrideSeccompProfile != nil { + specOpts = append( + specOpts, + WithCustomSeccompProfile(c.opts.OverrideSeccompProfile), + ) + } else { + specOpts = append( + specOpts, + seccomp.WithDefaultProfile(), // add seccomp profile last, as it depends on process capabilities + ) + } + + return specOpts +} + +func (c *containerdRunner) String() string { + return fmt.Sprintf("Containerd(%v)", c.args.ID) +} + +func (c *containerdRunner) StdinReader() (io.Reader, error) { + if c.opts.Stdin == nil { + return nil, nil + } + + if _, err := c.opts.Stdin.Seek(0, 0); err != nil { + return nil, err + } + + // copy the input buffer as containerd API seems to be buggy: + // * if the task fails to start, IO loop is not stopped properly, so after a restart there are two goroutines concurrently reading from stdin + contents, err := io.ReadAll(c.opts.Stdin) + if err != nil { + return nil, err + } + + c.stdinCloser = &StdinCloser{ + Stdin: bytes.NewReader(contents), + Closer: make(chan struct{}), + } + + return c.stdinCloser, nil +} diff --git a/internal/app/machined/pkg/system/runner/containerd/containerd_test.go b/internal/app/machined/pkg/system/runner/containerd/containerd_test.go new file mode 100644 index 0000000..3d261d4 --- /dev/null +++ b/internal/app/machined/pkg/system/runner/containerd/containerd_test.go @@ -0,0 +1,439 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package containerd_test + +import ( + "bytes" + "context" + "encoding/hex" + "fmt" + "io" + "log" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/containerd/cgroups/v3" + "github.com/containerd/cgroups/v3/cgroup1" + "github.com/containerd/cgroups/v3/cgroup2" + "github.com/containerd/containerd" + "github.com/containerd/containerd/namespaces" + "github.com/containerd/containerd/oci" + "github.com/google/uuid" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/logging" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/events" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner" + containerdrunner "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/containerd" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/process" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/restart" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +const ( + busyboxImage = "docker.io/library/busybox:latest" +) + +func MockEventSink(state events.ServiceState, message string, args ...interface{}) { + log.Printf("state %s: %s", state, fmt.Sprintf(message, args...)) +} + +type ContainerdSuite struct { + suite.Suite + + tmpDir string + + loggingManager runtime.LoggingManager + + containerdNamespace string + containerdRunner runner.Runner + containerdWg sync.WaitGroup + containerdAddress string + + containerID string + + client *containerd.Client + image containerd.Image +} + +func (suite *ContainerdSuite) SetupSuite() { + var err error + + suite.tmpDir = suite.T().TempDir() + + suite.loggingManager = logging.NewFileLoggingManager(suite.tmpDir) + + stateDir, rootDir := filepath.Join(suite.tmpDir, "state"), filepath.Join(suite.tmpDir, "root") + suite.Require().NoError(os.Mkdir(stateDir, 0o777)) + suite.Require().NoError(os.Mkdir(rootDir, 0o777)) + + if cgroups.Mode() == cgroups.Unified { + var manager *cgroup2.Manager + + manager, err = cgroup2.NewManager(constants.CgroupMountPath, "/"+suite.T().Name(), &cgroup2.Resources{}) + suite.Require().NoError(err) + + // when using buildkit runner, parent `cgroup.type` is set to `domain threaded`, so child cgroups have to explicitly specify + // `cgroup.type` to "threaded" https://www.kernel.org/doc/html/v5.0/admin-guide/cgroup-v2.html#threads + suite.Require().NoError(os.WriteFile(filepath.Join(constants.CgroupMountPath, suite.T().Name(), "cgroup.type"), []byte("threaded"), 0o644)) + + defer manager.Delete() //nolint:errcheck + } else { + var manager cgroup1.Cgroup + + manager, err = cgroup1.New(cgroup1.NestedPath(suite.tmpDir), &specs.LinuxResources{}) + suite.Require().NoError(err) + + defer manager.Delete() //nolint:errcheck + } + + suite.containerdAddress = filepath.Join(suite.tmpDir, "run.sock") + + args := &runner.Args{ + ID: "containerd", + ProcessArgs: []string{ + "/bin/containerd", + "--address", suite.containerdAddress, + "--state", stateDir, + "--root", rootDir, + "--config", constants.CRIContainerdConfig, + }, + } + + suite.containerdRunner = process.NewRunner( + false, + args, + runner.WithLoggingManager(suite.loggingManager), + runner.WithEnv([]string{"PATH=/bin:" + constants.PATH}), + runner.WithCgroupPath("/"+suite.T().Name()), + ) + suite.Require().NoError(suite.containerdRunner.Open()) + suite.containerdWg.Add(1) + + go func() { + defer suite.containerdWg.Done() + defer suite.containerdRunner.Close() //nolint:errcheck + suite.containerdRunner.Run(MockEventSink) //nolint:errcheck + }() + + suite.client, err = containerd.New(suite.containerdAddress) + suite.Require().NoError(err) + + namespace := ([16]byte)(uuid.New()) + suite.containerdNamespace = "talos" + hex.EncodeToString(namespace[:]) + + ctx := namespaces.WithNamespace(context.Background(), suite.containerdNamespace) + + suite.image, err = suite.client.Pull(ctx, busyboxImage, containerd.WithPullUnpack) + suite.Require().NoError(err) +} + +func (suite *ContainerdSuite) SetupTest() { + suite.containerID = uuid.New().String() +} + +func (suite *ContainerdSuite) TearDownSuite() { + suite.Require().NoError(suite.client.Close()) + + suite.Require().NoError(suite.containerdRunner.Stop()) + suite.containerdWg.Wait() +} + +func (suite *ContainerdSuite) getLogContents(filename string) []byte { + logFile, err := os.Open(filepath.Join(suite.tmpDir, filename)) + suite.Assert().NoError(err) + + //nolint:errcheck + defer logFile.Close() + + logContents, err := io.ReadAll(logFile) + suite.Assert().NoError(err) + + return logContents +} + +func (suite *ContainerdSuite) TestRunSuccess() { + r := containerdrunner.NewRunner(false, &runner.Args{ + ID: suite.containerID, + ProcessArgs: []string{"/bin/sh", "-c", "exit 0"}, + }, + runner.WithLoggingManager(suite.loggingManager), + runner.WithNamespace(suite.containerdNamespace), + runner.WithContainerImage(busyboxImage), + runner.WithContainerdAddress(suite.containerdAddress), + ) + + suite.Require().NoError(r.Open()) + + defer func() { suite.Assert().NoError(r.Close()) }() + + suite.Assert().NoError(r.Run(MockEventSink)) + // calling stop when Run has finished is no-op + suite.Assert().NoError(r.Stop()) +} + +func (suite *ContainerdSuite) TestRunTwice() { + r := containerdrunner.NewRunner(false, &runner.Args{ + ID: suite.containerID, + ProcessArgs: []string{"/bin/sh", "-c", "exit 0"}, + }, + runner.WithLoggingManager(suite.loggingManager), + runner.WithNamespace(suite.containerdNamespace), + runner.WithContainerImage(busyboxImage), + runner.WithContainerdAddress(suite.containerdAddress), + ) + + suite.Require().NoError(r.Open()) + + defer func() { suite.Assert().NoError(r.Close()) }() + + // running same container twice should be fine + // (checks that containerd state is cleaned up properly) + for i := range 2 { + suite.Assert().NoError(r.Run(MockEventSink)) + // calling stop when Run has finished is no-op + suite.Assert().NoError(r.Stop()) + + if i == 0 { + // wait a bit to let containerd clean up the state + time.Sleep(100 * time.Millisecond) + } + } +} + +func (suite *ContainerdSuite) TestContainerCleanup() { + // create two runners with the same container ID + // + // open first runner, but don't close it; second runner should be + // able to start the container by cleaning up container created by the first + // runner + r1 := containerdrunner.NewRunner(false, &runner.Args{ + ID: suite.containerID, + ProcessArgs: []string{"/bin/sh", "-c", "exit 1"}, + }, + runner.WithLoggingManager(suite.loggingManager), + runner.WithNamespace(suite.containerdNamespace), + runner.WithContainerImage(busyboxImage), + runner.WithContainerdAddress(suite.containerdAddress), + ) + + suite.Require().NoError(r1.Open()) + + r2 := containerdrunner.NewRunner(false, &runner.Args{ + ID: suite.containerID, + ProcessArgs: []string{"/bin/sh", "-c", "exit 0"}, + }, + runner.WithLoggingManager(suite.loggingManager), + runner.WithNamespace(suite.containerdNamespace), + runner.WithContainerImage(busyboxImage), + runner.WithContainerdAddress(suite.containerdAddress), + ) + suite.Require().NoError(r2.Open()) + + defer func() { suite.Assert().NoError(r2.Close()) }() + + suite.Assert().NoError(r2.Run(MockEventSink)) + // calling stop when Run has finished is no-op + suite.Assert().NoError(r2.Stop()) +} + +func (suite *ContainerdSuite) TestRunLogs() { + r := containerdrunner.NewRunner(false, &runner.Args{ + ID: suite.containerID, + ProcessArgs: []string{"/bin/sh", "-c", "echo -n \"Test 1\nTest 2\n\""}, + }, + runner.WithLoggingManager(suite.loggingManager), + runner.WithNamespace(suite.containerdNamespace), + runner.WithContainerImage(busyboxImage), + runner.WithContainerdAddress(suite.containerdAddress), + ) + + suite.Require().NoError(r.Open()) + + defer func() { suite.Assert().NoError(r.Close()) }() + + suite.Assert().NoError(r.Run(MockEventSink)) + + logFile, err := os.Open(filepath.Join(suite.tmpDir, suite.containerID+".log")) + suite.Assert().NoError(err) + + //nolint:errcheck + defer logFile.Close() + + logContents, err := io.ReadAll(logFile) + suite.Assert().NoError(err) + + suite.Assert().Equal([]byte("Test 1\nTest 2\n"), logContents) +} + +func (suite *ContainerdSuite) TestStopFailingAndRestarting() { + testDir := filepath.Join(suite.tmpDir, "test") + suite.Assert().NoError(os.Mkdir(testDir, 0o770)) + + testFile := filepath.Join(testDir, "talos-test") + //nolint:errcheck + _ = os.Remove(testFile) + + r := restart.New(containerdrunner.NewRunner(false, &runner.Args{ + ID: suite.containerID, + ProcessArgs: []string{"/bin/sh", "-c", "test -f " + testFile + " && echo ok || (echo fail; false)"}, + }, + runner.WithLoggingManager(suite.loggingManager), + runner.WithNamespace(suite.containerdNamespace), + runner.WithContainerImage(busyboxImage), + runner.WithOCISpecOpts( + oci.WithMounts([]specs.Mount{ + {Type: "bind", Destination: testDir, Source: testDir, Options: []string{"bind", "ro"}}, + }), + ), + runner.WithContainerdAddress(suite.containerdAddress), + ), + restart.WithType(restart.Forever), + restart.WithRestartInterval(5*time.Millisecond), + ) + + suite.Require().NoError(r.Open()) + + defer func() { suite.Assert().NoError(r.Close()) }() + + done := make(chan error, 1) + + go func() { + done <- r.Run(MockEventSink) + }() + + for range 10 { + time.Sleep(500 * time.Millisecond) + + if bytes.Contains(suite.getLogContents(suite.containerID+".log"), []byte("fail\n")) { + break + } + } + + select { + case err := <-done: + suite.Assert().Failf("task should be running", "error: %s", err) + + return + default: + } + + f, err := os.Create(testFile) + suite.Assert().NoError(err) + suite.Assert().NoError(f.Close()) + + for range 10 { + time.Sleep(500 * time.Millisecond) + + if bytes.Contains(suite.getLogContents(suite.containerID+".log"), []byte("ok\n")) { + break + } + } + + select { + case err = <-done: + suite.Assert().Failf("task should be running", "error: %s", err) + + return + default: + } + + suite.Assert().NoError(r.Stop()) + <-done + + logContents := suite.getLogContents(suite.containerID + ".log") + + suite.Assert().Truef(bytes.Contains(logContents, []byte("ok\n")), "logContents doesn't contain success entry: %v", logContents) + suite.Assert().Truef(bytes.Contains(logContents, []byte("fail\n")), "logContents doesn't contain fail entry: %v", logContents) +} + +func (suite *ContainerdSuite) TestStopSigKill() { + if cgroups.Mode() == cgroups.Unified { + suite.T().Skip("test doesn't pass under cgroupsv2") + } + + r := containerdrunner.NewRunner(false, &runner.Args{ + ID: suite.containerID, + ProcessArgs: []string{"/bin/sh", "-c", "trap -- '' SIGTERM; while :; do :; done"}, + }, + runner.WithLoggingManager(suite.loggingManager), + runner.WithNamespace(suite.containerdNamespace), + runner.WithContainerImage(busyboxImage), + runner.WithGracefulShutdownTimeout(10*time.Millisecond), + runner.WithContainerdAddress(suite.containerdAddress), + ) + + suite.Require().NoError(r.Open()) + + defer func() { suite.Assert().NoError(r.Close()) }() + + done := make(chan error, 1) + + go func() { + done <- r.Run(MockEventSink) + }() + + time.Sleep(50 * time.Millisecond) + select { + case <-done: + suite.Assert().Fail("container should be still running") + default: + } + + time.Sleep(100 * time.Millisecond) + + suite.Assert().NoError(r.Stop()) + suite.Assert().NoError(<-done) +} + +func (suite *ContainerdSuite) TestContainerStdin() { + stdin := bytes.Repeat([]byte{0xde, 0xad, 0xbe, 0xef}, 2000) + + r := containerdrunner.NewRunner(false, &runner.Args{ + ID: suite.containerID, + ProcessArgs: []string{"/bin/cat"}, + }, + runner.WithStdin(bytes.NewReader(stdin)), + runner.WithLoggingManager(suite.loggingManager), + runner.WithNamespace(suite.containerdNamespace), + runner.WithContainerImage(busyboxImage), + runner.WithContainerdAddress(suite.containerdAddress), + ) + + suite.Require().NoError(r.Open()) + + defer func() { suite.Assert().NoError(r.Close()) }() + + suite.Assert().NoError(r.Run(MockEventSink)) + + logFile, err := os.Open(filepath.Join(suite.tmpDir, suite.containerID+".log")) + suite.Assert().NoError(err) + + //nolint:errcheck + defer logFile.Close() + + logContents, err := io.ReadAll(logFile) + suite.Assert().NoError(err) + + suite.Assert().Equal(stdin, logContents) +} + +func TestContainerdSuite(t *testing.T) { + if os.Getuid() != 0 { + t.Skip("can't run the test as non-root") + } + + _, err := os.Stat("/bin/containerd") + if err != nil { + t.Skip("containerd binary is not available, skipping the test") + } + + suite.Run(t, new(ContainerdSuite)) +} diff --git a/internal/app/machined/pkg/system/runner/containerd/import.go b/internal/app/machined/pkg/system/runner/containerd/import.go new file mode 100644 index 0000000..f438d7d --- /dev/null +++ b/internal/app/machined/pkg/system/runner/containerd/import.go @@ -0,0 +1,125 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package containerd + +import ( + "context" + "fmt" + "log" + "os" + + "github.com/containerd/containerd" + "github.com/containerd/containerd/namespaces" + multierror "github.com/hashicorp/go-multierror" + + "github.com/siderolabs/talos/pkg/conditions" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// ImportRequest represents an image import request. +type ImportRequest struct { + Path string + Options []containerd.ImportOpt +} + +// Importer implements image import. +type Importer struct { + namespace string + options importerOptions +} + +type importerOptions struct { + containerdAddress string +} + +// ImporterOption configures containerd Inspector. +type ImporterOption func(*importerOptions) + +// WithContainerdAddress configures containerd address to use. +func WithContainerdAddress(address string) ImporterOption { + return func(o *importerOptions) { + o.containerdAddress = address + } +} + +// NewImporter builds new Importer. +func NewImporter(namespace string, options ...ImporterOption) *Importer { + importer := &Importer{ + namespace: namespace, + options: importerOptions{ + containerdAddress: constants.CRIContainerdAddress, + }, + } + + for _, opt := range options { + opt(&importer.options) + } + + return importer +} + +// Import imports the images specified by the import requests. +func (i *Importer) Import(ctx context.Context, reqs ...*ImportRequest) (err error) { + err = conditions.WaitForFileToExist(i.options.containerdAddress).Wait(ctx) + if err != nil { + return err + } + + ctx = namespaces.WithNamespace(ctx, i.namespace) + + client, err := containerd.New(i.options.containerdAddress) + if err != nil { + return err + } + //nolint:errcheck + defer client.Close() + + errCh := make(chan error) + + var result *multierror.Error + + for _, req := range reqs { + go func(errCh chan<- error, r *ImportRequest) { + errCh <- func() error { + tarball, err := os.Open(r.Path) + if err != nil { + return fmt.Errorf("error opening %s: %w", r.Path, err) + } + + defer tarball.Close() //nolint:errcheck + + imgs, err := client.Import(ctx, tarball, r.Options...) + if err != nil { + return fmt.Errorf("error importing %s: %w", r.Path, err) + } + if err = tarball.Close(); err != nil { + return fmt.Errorf("error closing %s: %w", r.Path, err) + } + + for _, img := range imgs { + image := containerd.NewImage(client, img) + log.Printf("unpacking %s (%s)\n", img.Name, img.Target.Digest) + err = image.Unpack(ctx, containerd.DefaultSnapshotter) + if err != nil { + return fmt.Errorf("error unpacking %s: %w", img.Name, err) + } + } + + return nil + }() + }(errCh, req) + } + + for range reqs { + result = multierror.Append(result, <-errCh) + } + + return result.ErrorOrNil() +} + +// Import imports the images specified by the import requests. +func Import(ctx context.Context, namespace string, reqs ...*ImportRequest) error { + return NewImporter(namespace).Import(ctx, reqs...) +} diff --git a/internal/app/machined/pkg/system/runner/containerd/opts.go b/internal/app/machined/pkg/system/runner/containerd/opts.go new file mode 100644 index 0000000..3d2b7a8 --- /dev/null +++ b/internal/app/machined/pkg/system/runner/containerd/opts.go @@ -0,0 +1,55 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package containerd + +import ( + "context" + + "github.com/containerd/containerd/containers" + "github.com/containerd/containerd/contrib/seccomp" + "github.com/containerd/containerd/oci" + specs "github.com/opencontainers/runtime-spec/specs-go" +) + +// WithMemoryLimit sets the linux resource memory limit field. +func WithMemoryLimit(limit int64) oci.SpecOpts { + return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error { + s.Linux.Resources.Memory = &specs.LinuxMemory{ + Limit: &limit, + // DisableOOMKiller: &disable, + } + + return nil + } +} + +// WithRootfsPropagation sets the root filesystem propagation. +func WithRootfsPropagation(rp string) oci.SpecOpts { + return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error { + s.Linux.RootfsPropagation = rp + + return nil + } +} + +// WithOOMScoreAdj sets the oom score. +func WithOOMScoreAdj(score int) oci.SpecOpts { + return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error { + s.Process.OOMScoreAdj = &score + + return nil + } +} + +// WithCustomSeccompProfile allows to override default seccomp profile. +func WithCustomSeccompProfile(override func(*specs.LinuxSeccomp)) oci.SpecOpts { + return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error { + s.Linux.Seccomp = seccomp.DefaultProfile(s) + + override(s.Linux.Seccomp) + + return nil + } +} diff --git a/internal/app/machined/pkg/system/runner/containerd/stdin.go b/internal/app/machined/pkg/system/runner/containerd/stdin.go new file mode 100644 index 0000000..d24bcf8 --- /dev/null +++ b/internal/app/machined/pkg/system/runner/containerd/stdin.go @@ -0,0 +1,38 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package containerd + +import ( + "context" + "io" + + "github.com/containerd/containerd" +) + +// StdinCloser wraps io.Reader providing a signal when reader is read till EOF. +type StdinCloser struct { + Stdin io.Reader + Closer chan struct{} +} + +func (s *StdinCloser) Read(p []byte) (int, error) { + n, err := s.Stdin.Read(p) + if err == io.EOF { + close(s.Closer) + } + + return n, err +} + +// WaitAndClose closes containerd task stdin when StdinCloser is exhausted. +func (s *StdinCloser) WaitAndClose(ctx context.Context, task containerd.Task) { + select { + case <-ctx.Done(): + return + case <-s.Closer: + //nolint:errcheck + task.CloseIO(ctx, containerd.WithStdinCloser) + } +} diff --git a/internal/app/machined/pkg/system/runner/goroutine/goroutine.go b/internal/app/machined/pkg/system/runner/goroutine/goroutine.go new file mode 100644 index 0000000..c948dfa --- /dev/null +++ b/internal/app/machined/pkg/system/runner/goroutine/goroutine.go @@ -0,0 +1,118 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package goroutine + +import ( + "context" + "fmt" + "io" + stdlibruntime "runtime" + "sync" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/events" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner" +) + +// goroutineRunner is a runner.Runner that runs a service in a goroutine. +type goroutineRunner struct { + main FuncMain + id string + runtime runtime.Runtime + + opts *runner.Options + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc + + wg sync.WaitGroup +} + +// FuncMain is a entrypoint into the service. +// +// Service should abort and return when ctx is canceled. +type FuncMain func(ctx context.Context, r runtime.Runtime, logOutput io.Writer) error + +// NewRunner creates runner.Runner that runs a service as goroutine. +func NewRunner(r runtime.Runtime, id string, main FuncMain, setters ...runner.Option) runner.Runner { + run := &goroutineRunner{ + id: id, + runtime: r, + main: main, + opts: runner.DefaultOptions(), + } + + run.ctx, run.ctxCancel = context.WithCancel(context.Background()) + + for _, setter := range setters { + setter(run.opts) + } + + return run +} + +// Open implements the Runner interface. +func (r *goroutineRunner) Open() error { + return nil +} + +// Run implements the Runner interface. +func (r *goroutineRunner) Run(eventSink events.Recorder) error { + r.wg.Add(1) + defer r.wg.Done() + + eventSink(events.StateRunning, "Service started as goroutine") + + return r.wrappedMain() +} + +func (r *goroutineRunner) wrappedMain() (err error) { + defer func() { + if r := recover(); r != nil { + buf := make([]byte, 8192) + n := stdlibruntime.Stack(buf, false) + err = fmt.Errorf("panic in service: %v\n%s", r, string(buf[:n])) + } + }() + + var w io.WriteCloser + + w, err = r.opts.LoggingManager.ServiceLog(r.id).Writer() + if err != nil { + err = fmt.Errorf("service log handler: %w", err) + + return + } + //nolint:errcheck + defer w.Close() + + err = r.main(r.ctx, r.runtime, w) + if err == context.Canceled { + // clear error if service was aborted + err = nil + } + + return err +} + +// Stop implements the Runner interface. +func (r *goroutineRunner) Stop() error { + r.ctxCancel() + + r.wg.Wait() + + r.ctx, r.ctxCancel = context.WithCancel(context.Background()) + + return nil +} + +// Close implements the Runner interface. +func (r *goroutineRunner) Close() error { + return nil +} + +func (r *goroutineRunner) String() string { + return fmt.Sprintf("Goroutine(%q)", r.id) +} diff --git a/internal/app/machined/pkg/system/runner/goroutine/goroutine_test.go b/internal/app/machined/pkg/system/runner/goroutine/goroutine_test.go new file mode 100644 index 0000000..5631b88 --- /dev/null +++ b/internal/app/machined/pkg/system/runner/goroutine/goroutine_test.go @@ -0,0 +1,173 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package goroutine_test + +import ( + "context" + "errors" + "fmt" + "io" + "log" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/logging" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/events" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/goroutine" + "github.com/siderolabs/talos/pkg/machinery/config/container" + v1alpha1cfg "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" +) + +func MockEventSink(state events.ServiceState, message string, args ...interface{}) { + log.Printf("state %s: %s", state, fmt.Sprintf(message, args...)) +} + +type GoroutineSuite struct { + suite.Suite + r runtime.Runtime + + tmpDir string + + loggingManager runtime.LoggingManager +} + +func (suite *GoroutineSuite) SetupSuite() { + var err error + + suite.tmpDir = suite.T().TempDir() + + suite.loggingManager = logging.NewFileLoggingManager(suite.tmpDir) + + s, err := v1alpha1.NewState() + suite.Require().NoError(err) + + cfg, err := container.New(&v1alpha1cfg.Config{}) + suite.Require().NoError(err) + + e := v1alpha1.NewEvents(100, 10) + + r := v1alpha1.NewRuntime(s, e, suite.loggingManager) + + suite.Require().NoError(r.SetConfig(cfg)) + + suite.r = r +} + +func (suite *GoroutineSuite) TestRunSuccess() { + r := goroutine.NewRunner(suite.r, "testsuccess", + func(context.Context, runtime.Runtime, io.Writer) error { + return nil + }, runner.WithLoggingManager(suite.loggingManager)) + + suite.Assert().NoError(r.Open()) + + defer func() { suite.Assert().NoError(r.Close()) }() + + suite.Assert().NoError(r.Run(MockEventSink)) + // calling stop when Run has finished is no-op + suite.Assert().NoError(r.Stop()) +} + +func (suite *GoroutineSuite) TestRunFail() { + r := goroutine.NewRunner(suite.r, "testfail", + func(context.Context, runtime.Runtime, io.Writer) error { + return errors.New("service failed") + }, runner.WithLoggingManager(suite.loggingManager)) + + suite.Assert().NoError(r.Open()) + + defer func() { suite.Assert().NoError(r.Close()) }() + + suite.Assert().EqualError(r.Run(MockEventSink), "service failed") + // calling stop when Run has finished is no-op + suite.Assert().NoError(r.Stop()) +} + +func (suite *GoroutineSuite) TestRunPanic() { + r := goroutine.NewRunner(suite.r, "testpanic", + func(context.Context, runtime.Runtime, io.Writer) error { + panic("service panic") + }, runner.WithLoggingManager(suite.loggingManager)) + + suite.Assert().NoError(r.Open()) + + defer func() { suite.Assert().NoError(r.Close()) }() + + err := r.Run(MockEventSink) + suite.Assert().Error(err) + suite.Assert().Regexp("^panic in service: service panic.*", err.Error()) + // calling stop when Run has finished is no-op + suite.Assert().NoError(r.Stop()) +} + +func (suite *GoroutineSuite) TestStop() { + r := goroutine.NewRunner(suite.r, "teststop", + func(ctx context.Context, data runtime.Runtime, logger io.Writer) error { + <-ctx.Done() + + return ctx.Err() + }, runner.WithLoggingManager(suite.loggingManager)) + + suite.Assert().NoError(r.Open()) + + defer func() { suite.Assert().NoError(r.Close()) }() + + errCh := make(chan error) + + go func() { + errCh <- r.Run(MockEventSink) + }() + + time.Sleep(20 * time.Millisecond) + + select { + case <-errCh: + suite.Require().Fail("should not return yet") + default: + } + + suite.Assert().NoError(r.Stop()) + suite.Assert().NoError(<-errCh) +} + +func (suite *GoroutineSuite) TestRunLogs() { + r := goroutine.NewRunner(suite.r, "logtest", + func(ctx context.Context, data runtime.Runtime, logger io.Writer) error { + //nolint:errcheck + _, _ = logger.Write([]byte("Test 1\nTest 2\n")) + + return nil + }, runner.WithLoggingManager(suite.loggingManager)) + + suite.Assert().NoError(r.Open()) + + defer func() { suite.Assert().NoError(r.Close()) }() + + suite.Assert().NoError(r.Run(MockEventSink)) + + logFile, err := os.Open(filepath.Join(suite.tmpDir, "logtest.log")) + suite.Assert().NoError(err) + + //nolint:errcheck + defer logFile.Close() + + logContents, err := io.ReadAll(logFile) + suite.Assert().NoError(err) + + suite.Assert().Equal([]byte("Test 1\nTest 2\n"), logContents) +} + +func TestGoroutineSuite(t *testing.T) { + t.Setenv("PLATFORM", "metal") + + suite.Run(t, new(GoroutineSuite)) +} diff --git a/internal/app/machined/pkg/system/runner/process/process.go b/internal/app/machined/pkg/system/runner/process/process.go new file mode 100644 index 0000000..cf7ef21 --- /dev/null +++ b/internal/app/machined/pkg/system/runner/process/process.go @@ -0,0 +1,257 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package process + +import ( + "fmt" + "io" + "os" + "os/exec" + "strings" + "syscall" + "time" + + "github.com/siderolabs/go-cmd/pkg/cmd/proc/reaper" + + "github.com/aenix-io/talm/internal/app/machined/pkg/system/events" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner" + "github.com/aenix-io/talm/internal/pkg/cgroup" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// processRunner is a runner.Runner that runs a process on the host. +type processRunner struct { + args *runner.Args + opts *runner.Options + debug bool + + stop chan struct{} + stopped chan struct{} +} + +// NewRunner creates runner.Runner that runs a process on the host. +func NewRunner(debug bool, args *runner.Args, setters ...runner.Option) runner.Runner { + r := &processRunner{ + args: args, + opts: runner.DefaultOptions(), + debug: debug, + stop: make(chan struct{}), + stopped: make(chan struct{}), + } + + for _, setter := range setters { + setter(r.opts) + } + + return r +} + +// Open implements the Runner interface. +func (p *processRunner) Open() error { + return nil +} + +// Run implements the Runner interface. +func (p *processRunner) Run(eventSink events.Recorder) error { + defer close(p.stopped) + + return p.run(eventSink) +} + +// Stop implements the Runner interface. +func (p *processRunner) Stop() error { + close(p.stop) + + <-p.stopped + + p.stop = make(chan struct{}) + p.stopped = make(chan struct{}) + + return nil +} + +// Close implements the Runner interface. +func (p *processRunner) Close() error { + return nil +} + +type commandWrapper struct { + cmd *exec.Cmd + afterStart func() + afterTermination func() error +} + +//nolint:gocyclo +func (p *processRunner) build() (commandWrapper, error) { + args := []string{ + fmt.Sprintf("-name=%s", p.args.ID), + fmt.Sprintf("-dropped-caps=%s", strings.Join(p.opts.DroppedCapabilities, ",")), + fmt.Sprintf("-cgroup-path=%s", cgroup.Path(p.opts.CgroupPath)), + fmt.Sprintf("-oom-score=%d", p.opts.OOMScoreAdj), + fmt.Sprintf("-uid=%d", p.opts.UID), + } + + args = append(args, p.args.ProcessArgs...) + + cmd := exec.Command("/sbin/wrapperd", args...) + + // Set the environment for the service. + cmd.Env = append([]string{fmt.Sprintf("PATH=%s", constants.PATH)}, p.opts.Env...) + + // Setup logging. + w, err := p.opts.LoggingManager.ServiceLog(p.args.ID).Writer() + if err != nil { + return commandWrapper{}, fmt.Errorf("service log handler: %w", err) + } + + var writer io.Writer + if p.debug { // TODO: wrap it into LoggingManager + writer = io.MultiWriter(w, os.Stdout) + } else { + writer = w + } + + // close the writer if we exit early due to an error + closeWriter := true + + defer func() { + if closeWriter { + w.Close() //nolint:errcheck + } + }() + + var afterStartFuncs []func() + + if p.opts.StdinFile != "" { + stdin, err := os.Open(p.opts.StdinFile) + if err != nil { + return commandWrapper{}, err + } + + cmd.Stdin = stdin + + afterStartFuncs = append(afterStartFuncs, func() { + stdin.Close() //nolint:errcheck + }) + } + + if p.opts.StdoutFile != "" { + stdout, err := os.OpenFile(p.opts.StdoutFile, os.O_WRONLY, 0) + if err != nil { + return commandWrapper{}, err + } + + cmd.Stdout = stdout + + afterStartFuncs = append(afterStartFuncs, func() { + stdout.Close() //nolint:errcheck + }) + } else { + cmd.Stdout = writer + } + + if p.opts.StderrFile != "" { + stderr, err := os.OpenFile(p.opts.StderrFile, os.O_WRONLY, 0) + if err != nil { + return commandWrapper{}, err + } + + cmd.Stderr = stderr + + afterStartFuncs = append(afterStartFuncs, func() { + stderr.Close() //nolint:errcheck + }) + } else { + cmd.Stderr = writer + } + + ctty, cttySet := p.opts.Ctty.Get() + if cttySet { + cmd.SysProcAttr = &syscall.SysProcAttr{ + Setsid: true, + Setctty: true, + Ctty: ctty, + } + } + + closeWriter = false + + return commandWrapper{ + cmd: cmd, + afterStart: func() { + for _, f := range afterStartFuncs { + f() + } + }, + afterTermination: func() error { + return w.Close() + }, + }, nil +} + +func (p *processRunner) run(eventSink events.Recorder) error { + cmdWrapper, err := p.build() + if err != nil { + return fmt.Errorf("error building command: %w", err) + } + + defer cmdWrapper.afterTermination() //nolint:errcheck + + notifyCh := make(chan reaper.ProcessInfo, 8) + + usingReaper := reaper.Notify(notifyCh) + if usingReaper { + defer reaper.Stop(notifyCh) + } + + err = cmdWrapper.cmd.Start() + + cmdWrapper.afterStart() + + if err != nil { + return fmt.Errorf("error starting process: %w", err) + } + + eventSink(events.StateRunning, "Process %s started with PID %d", p, cmdWrapper.cmd.Process.Pid) + + waitCh := make(chan error) + + go func() { + waitCh <- reaper.WaitWrapper(usingReaper, notifyCh, cmdWrapper.cmd) + }() + + select { + case err = <-waitCh: + // process exited + return err + case <-p.stop: + // graceful stop the service + eventSink(events.StateStopping, "Sending SIGTERM to %s", p) + + //nolint:errcheck + _ = cmdWrapper.cmd.Process.Signal(syscall.SIGTERM) + } + + select { + case <-waitCh: + // stopped process exited + return nil + case <-time.After(p.opts.GracefulShutdownTimeout): + // kill the process + eventSink(events.StateStopping, "Sending SIGKILL to %s", p) + + //nolint:errcheck + _ = cmdWrapper.cmd.Process.Signal(syscall.SIGKILL) + } + + // wait for process to terminate + <-waitCh + + return cmdWrapper.afterTermination() +} + +func (p *processRunner) String() string { + return fmt.Sprintf("Process(%q)", p.args.ProcessArgs) +} diff --git a/internal/app/machined/pkg/system/runner/process/process_test.go b/internal/app/machined/pkg/system/runner/process/process_test.go new file mode 100644 index 0000000..3d286ec --- /dev/null +++ b/internal/app/machined/pkg/system/runner/process/process_test.go @@ -0,0 +1,232 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package process_test + +import ( + "fmt" + "io" + "log" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/siderolabs/go-cmd/pkg/cmd/proc/reaper" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/logging" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/events" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/process" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/restart" +) + +func MockEventSink(state events.ServiceState, message string, args ...interface{}) { + log.Printf("state %s: %s", state, fmt.Sprintf(message, args...)) +} + +type ProcessSuite struct { + suite.Suite + + tmpDir string + runReaper bool + + loggingManager runtime.LoggingManager +} + +func (suite *ProcessSuite) SetupSuite() { + suite.tmpDir = suite.T().TempDir() + + suite.loggingManager = logging.NewFileLoggingManager(suite.tmpDir) + + if suite.runReaper { + reaper.Run() + } +} + +func (suite *ProcessSuite) TearDownSuite() { + if suite.runReaper { + reaper.Shutdown() + } +} + +func (suite *ProcessSuite) TestRunSuccess() { + r := process.NewRunner(false, &runner.Args{ + ID: "test", + ProcessArgs: []string{"/bin/sh", "-c", "exit 0"}, + }, runner.WithLoggingManager(suite.loggingManager)) + + suite.Assert().NoError(r.Open()) + + defer func() { suite.Assert().NoError(r.Close()) }() + + suite.Assert().NoError(r.Run(MockEventSink)) + // calling stop when Run has finished is no-op + suite.Assert().NoError(r.Stop()) +} + +func (suite *ProcessSuite) TestRunLogs() { + r := process.NewRunner(false, &runner.Args{ + ID: "logtest", + ProcessArgs: []string{"/bin/sh", "-c", "echo -n \"Test 1\nTest 2\n\""}, + }, runner.WithLoggingManager(suite.loggingManager)) + + suite.Assert().NoError(r.Open()) + + defer func() { suite.Assert().NoError(r.Close()) }() + + suite.Assert().NoError(r.Run(MockEventSink)) + + logFile, err := os.Open(filepath.Join(suite.tmpDir, "logtest.log")) + suite.Assert().NoError(err) + + //nolint:errcheck + defer logFile.Close() + + logContents, err := io.ReadAll(logFile) + suite.Assert().NoError(err) + + suite.Assert().Equal([]byte("Test 1\nTest 2\n"), logContents) +} + +func (suite *ProcessSuite) TestRunRestartFailed() { + testFile := filepath.Join(suite.tmpDir, "talos-test") + //nolint:errcheck + _ = os.Remove(testFile) + + r := restart.New(process.NewRunner(false, &runner.Args{ + ID: "restarter", + ProcessArgs: []string{"/bin/sh", "-c", "echo \"ran\"; test -f " + testFile}, + }, runner.WithLoggingManager(suite.loggingManager)), restart.WithType(restart.UntilSuccess), restart.WithRestartInterval(time.Millisecond)) + + suite.Assert().NoError(r.Open()) + + defer func() { suite.Assert().NoError(r.Close()) }() + + var wg sync.WaitGroup + + wg.Add(1) + + go func() { + defer wg.Done() + suite.Assert().NoError(r.Run(MockEventSink)) + }() + + fetchLog := func() []byte { + logFile, err := os.Open(filepath.Join(suite.tmpDir, "restarter.log")) + suite.Assert().NoError(err) + + //nolint:errcheck + defer logFile.Close() + + logContents, err := io.ReadAll(logFile) + suite.Assert().NoError(err) + + return logContents + } + + for range 20 { + time.Sleep(100 * time.Millisecond) + + if len(fetchLog()) > 20 { + break + } + } + + f, err := os.Create(testFile) + suite.Assert().NoError(err) + suite.Assert().NoError(f.Close()) + + wg.Wait() + + suite.Assert().True(len(fetchLog()) > 20) +} + +func (suite *ProcessSuite) TestStopFailingAndRestarting() { + testFile := filepath.Join(suite.tmpDir, "talos-test") + //nolint:errcheck + _ = os.Remove(testFile) + + r := restart.New(process.NewRunner(false, &runner.Args{ + ID: "endless", + ProcessArgs: []string{"/bin/sh", "-c", "test -f " + testFile}, + }, runner.WithLoggingManager(suite.loggingManager)), restart.WithType(restart.Forever), restart.WithRestartInterval(5*time.Millisecond)) + + suite.Assert().NoError(r.Open()) + + defer func() { suite.Assert().NoError(r.Close()) }() + + done := make(chan error, 1) + + go func() { + done <- r.Run(MockEventSink) + }() + + time.Sleep(40 * time.Millisecond) + + select { + case <-done: + suite.Assert().Fail("task should be running") + + return + default: + } + + f, err := os.Create(testFile) + suite.Assert().NoError(err) + suite.Assert().NoError(f.Close()) + + time.Sleep(40 * time.Millisecond) + + select { + case <-done: + suite.Assert().Fail("task should be running") + + return + default: + } + + suite.Assert().NoError(r.Stop()) + <-done +} + +func (suite *ProcessSuite) TestStopSigKill() { + r := process.NewRunner(false, &runner.Args{ + ID: "nokill", + ProcessArgs: []string{"/bin/sh", "-c", "trap -- '' SIGTERM; while :; do :; done"}, + }, + runner.WithLoggingManager(suite.loggingManager), + runner.WithGracefulShutdownTimeout(10*time.Millisecond), + ) + + suite.Assert().NoError(r.Open()) + + defer func() { suite.Assert().NoError(r.Close()) }() + + done := make(chan error, 1) + + go func() { + done <- r.Run(MockEventSink) + }() + + time.Sleep(100 * time.Millisecond) + + suite.Assert().NoError(r.Stop()) + <-done +} + +func TestProcessSuite(t *testing.T) { + if _, err := os.Stat("/sbin/wrapperd"); err != nil { + t.Skip("wrapperd not found") + } + + for _, runReaper := range []bool{true, false} { + func(runReaper bool) { + t.Run(fmt.Sprintf("runReaper=%v", runReaper), func(t *testing.T) { suite.Run(t, &ProcessSuite{runReaper: runReaper}) }) + }(runReaper) + } +} diff --git a/internal/app/machined/pkg/system/runner/restart/restart.go b/internal/app/machined/pkg/system/runner/restart/restart.go new file mode 100644 index 0000000..db034ea --- /dev/null +++ b/internal/app/machined/pkg/system/runner/restart/restart.go @@ -0,0 +1,175 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package restart + +import ( + "fmt" + "time" + + "github.com/aenix-io/talm/internal/app/machined/pkg/system/events" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner" +) + +type restarter struct { + wrappedRunner runner.Runner + opts *Options + + stop chan struct{} + stopped chan struct{} +} + +// New wraps runner.Runner with restart policy. +func New(wrapRunner runner.Runner, opts ...Option) runner.Runner { + r := &restarter{ + wrappedRunner: wrapRunner, + opts: DefaultOptions(), + stop: make(chan struct{}), + stopped: make(chan struct{}), + } + + for _, opt := range opts { + opt(r.opts) + } + + return r +} + +// Options is the functional options struct. +type Options struct { + // Type describes the service's restart policy. + Type Type + // RestartInterval is the interval between restarts for failed runs. + RestartInterval time.Duration +} + +// Option is the functional option func. +type Option func(*Options) + +// Type represents the service's restart policy. +type Type int + +const ( + // Forever will always restart a process. + Forever Type = iota + // Once will run process exactly once. + Once + // UntilSuccess will restart process until run succeeds. + UntilSuccess +) + +func (t Type) String() string { + switch t { + case Forever: + return "Forever" + case Once: + return "Once" + case UntilSuccess: + return "UntilSuccess" + default: + return "Unknown" + } +} + +// DefaultOptions describes the default options to a runner. +func DefaultOptions() *Options { + return &Options{ + Type: Forever, + RestartInterval: 5 * time.Second, + } +} + +// WithType sets the type of a service. +func WithType(o Type) Option { + return func(args *Options) { + args.Type = o + } +} + +// WithRestartInterval sets the interval between restarts of the failed task. +func WithRestartInterval(interval time.Duration) Option { + return func(args *Options) { + args.RestartInterval = interval + } +} + +// Open implements the Runner interface. +func (r *restarter) Open() error { + return r.wrappedRunner.Open() +} + +// Run implements the Runner interface +// +//nolint:gocyclo +func (r *restarter) Run(eventSink events.Recorder) error { + defer close(r.stopped) + + for { + errCh := make(chan error) + + go func() { + errCh <- r.wrappedRunner.Run(eventSink) + }() + + var err error + + select { + case <-r.stop: + //nolint:errcheck + _ = r.wrappedRunner.Stop() + + return <-errCh + case err = <-errCh: + } + + errStop := r.wrappedRunner.Stop() + if errStop != nil { + return errStop + } + + switch r.opts.Type { + case Once: + return err + case UntilSuccess: + if err == nil { + return nil + } + + eventSink(events.StateWaiting, "Error running %s, going to restart until it succeeds: %v", r.wrappedRunner, err) + case Forever: + if err == nil { + eventSink(events.StateWaiting, "Runner %s exited without error, going to restart it", r.wrappedRunner) + } else { + eventSink(events.StateWaiting, "Error running %v, going to restart forever: %v", r.wrappedRunner, err) + } + } + + select { + case <-r.stop: + eventSink(events.StateStopping, "Aborting restart sequence") + + return nil + case <-time.After(r.opts.RestartInterval): + } + } +} + +// Stop implements the Runner interface. +func (r *restarter) Stop() error { + close(r.stop) + + <-r.stopped + + return nil +} + +// Close implements the Runner interface. +func (r *restarter) Close() error { + return r.wrappedRunner.Close() +} + +// String implements the Runner interface. +func (r *restarter) String() string { + return fmt.Sprintf("Restart(%s, %s)", r.opts.Type, r.wrappedRunner) +} diff --git a/internal/app/machined/pkg/system/runner/restart/restart_test.go b/internal/app/machined/pkg/system/runner/restart/restart_test.go new file mode 100644 index 0000000..0fb9853 --- /dev/null +++ b/internal/app/machined/pkg/system/runner/restart/restart_test.go @@ -0,0 +1,182 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package restart_test + +import ( + "errors" + "fmt" + "log" + "testing" + "time" + + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/system/events" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/restart" +) + +type RestartSuite struct { + suite.Suite +} + +type MockRunner struct { + exitCh chan error + times int + stop chan struct{} + stopped chan struct{} +} + +func (m *MockRunner) Open() error { + m.stop = make(chan struct{}) + m.stopped = make(chan struct{}) + + return nil +} + +func (m *MockRunner) Close() error { + close(m.exitCh) + + return nil +} + +func (m *MockRunner) Run(eventSink events.Recorder) error { + defer close(m.stopped) + + select { + case err := <-m.exitCh: + m.times++ + + return err + case <-m.stop: + return nil + } +} + +func (m *MockRunner) Stop() error { + close(m.stop) + + <-m.stopped + + m.stop = make(chan struct{}) + m.stopped = make(chan struct{}) + + return nil +} + +func (m *MockRunner) String() string { + return "MockRunner()" +} + +func MockEventSink(state events.ServiceState, message string, args ...interface{}) { + log.Printf("state %s: %s", state, fmt.Sprintf(message, args...)) +} + +func (suite *RestartSuite) TestString() { + suite.Assert().Equal("Restart(UntilSuccess, MockRunner())", restart.New(&MockRunner{}, restart.WithType(restart.UntilSuccess)).String()) +} + +func (suite *RestartSuite) TestRunOnce() { + mock := MockRunner{ + exitCh: make(chan error), + } + + r := restart.New(&mock, restart.WithType(restart.Once)) + suite.Assert().NoError(r.Open()) + + defer func() { suite.Assert().NoError(r.Close()) }() + + failed := errors.New("failed") + + go func() { + mock.exitCh <- failed + }() + + suite.Assert().EqualError(r.Run(MockEventSink), failed.Error()) + suite.Assert().NoError(r.Stop()) +} + +func (suite *RestartSuite) TestRunOnceStop() { + mock := MockRunner{ + exitCh: make(chan error), + } + + r := restart.New(&mock, restart.WithType(restart.Once)) + suite.Assert().NoError(r.Open()) + + defer func() { suite.Assert().NoError(r.Close()) }() + + errCh := make(chan error) + + go func() { + errCh <- r.Run(MockEventSink) + }() + + suite.Assert().NoError(r.Stop()) + suite.Assert().NoError(<-errCh) +} + +func (suite *RestartSuite) TestRunUntilSuccess() { + mock := MockRunner{ + exitCh: make(chan error), + } + + r := restart.New(&mock, restart.WithType(restart.UntilSuccess), restart.WithRestartInterval(time.Millisecond)) + suite.Assert().NoError(r.Open()) + + defer func() { suite.Assert().NoError(r.Close()) }() + + failed := errors.New("failed") + errCh := make(chan error) + + go func() { + errCh <- r.Run(MockEventSink) + }() + + mock.exitCh <- failed + mock.exitCh <- failed + mock.exitCh <- failed + mock.exitCh <- nil + + suite.Assert().NoError(<-errCh) + suite.Assert().NoError(r.Stop()) + suite.Assert().Equal(4, mock.times) +} + +func (suite *RestartSuite) TestRunForever() { + mock := MockRunner{ + exitCh: make(chan error), + } + + r := restart.New(&mock, restart.WithType(restart.Forever), restart.WithRestartInterval(time.Millisecond)) + suite.Assert().NoError(r.Open()) + + defer func() { suite.Assert().NoError(r.Close()) }() + + failed := errors.New("failed") + errCh := make(chan error) + + go func() { + errCh <- r.Run(MockEventSink) + }() + + mock.exitCh <- failed + mock.exitCh <- nil + mock.exitCh <- failed + mock.exitCh <- nil + + select { + case <-errCh: + suite.Assert().Fail("runner should be still running") + default: + } + + suite.Assert().NoError(r.Stop()) + suite.Assert().NoError(<-errCh) + suite.Assert().Equal(4, mock.times) +} + +func TestRestartSuite(t *testing.T) { + suite.Run(t, new(RestartSuite)) +} diff --git a/internal/app/machined/pkg/system/runner/runner.go b/internal/app/machined/pkg/system/runner/runner.go new file mode 100644 index 0000000..5c4b9ea --- /dev/null +++ b/internal/app/machined/pkg/system/runner/runner.go @@ -0,0 +1,222 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package runner provides a runner for running services. +package runner + +import ( + "fmt" + "io" + "time" + + "github.com/containerd/containerd" + "github.com/containerd/containerd/oci" + "github.com/opencontainers/runtime-spec/specs-go" + "github.com/siderolabs/gen/maps" + "github.com/siderolabs/gen/optional" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/logging" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/events" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// Runner describes the requirements for running a process. +type Runner interface { + fmt.Stringer + Open() error + Run(events.Recorder) error + Stop() error + Close() error +} + +// Args represents the required options for services. +type Args struct { + ID string + ProcessArgs []string +} + +// Options is the functional options struct. +type Options struct { + // LoggingManager provides service log handling. + LoggingManager runtime.LoggingManager + // Env describes the service's environment variables. Elements should be in + // the format + Env []string + // ContainerdAddress is containerd socket address. + ContainerdAddress string + // ContainerOpts describes the container options. + ContainerOpts []containerd.NewContainerOpts + // OCISpecOpts describes the OCI spec options. + OCISpecOpts []oci.SpecOpts + // ContainerImage is the container's image. + ContainerImage string + // Namespace is the containerd namespace. + Namespace string + // GracefulShutdownTimeout is the time to wait for process to exit after SIGTERM + // before sending SIGKILL + GracefulShutdownTimeout time.Duration + // Stdin is the process standard input. + Stdin io.ReadSeeker + // Specify an oom_score_adj for the process. + OOMScoreAdj int + // CgroupPath (optional) sets the cgroup path to use + CgroupPath string + // OverrideSeccompProfile default Linux seccomp profile. + OverrideSeccompProfile func(*specs.LinuxSeccomp) + // DroppedCapabilities is the list of capabilities to drop. + DroppedCapabilities []string + // StdinFile is the path to the file to use as stdin. + StdinFile string + // StdoutFile is the path to the file to use as stdout. + StdoutFile string + // StderrFile is the path to the file to use as stderr. + StderrFile string + // Ctty is the controlling tty. + Ctty optional.Optional[int] + // UID is the user id of the process. + UID uint32 +} + +// Option is the functional option func. +type Option func(*Options) + +// DefaultOptions describes the default options to a runner. +func DefaultOptions() *Options { + return &Options{ + LoggingManager: logging.NewNullLoggingManager(), + Env: []string{}, + Namespace: constants.SystemContainerdNamespace, + GracefulShutdownTimeout: 10 * time.Second, + ContainerdAddress: constants.CRIContainerdAddress, + Stdin: nil, + OOMScoreAdj: 0, + } +} + +// WithEnv sets the environment variables of a service. +func WithEnv(o []string) Option { + return func(args *Options) { + args.Env = o + } +} + +// WithNamespace sets the tar file to load. +func WithNamespace(o string) Option { + return func(args *Options) { + args.Namespace = o + } +} + +// WithContainerdAddress sets the containerd socket path. +func WithContainerdAddress(a string) Option { + return func(args *Options) { + args.ContainerdAddress = a + } +} + +// WithContainerImage sets the image ref. +func WithContainerImage(o string) Option { + return func(args *Options) { + args.ContainerImage = o + } +} + +// WithContainerOpts sets the containerd container options. +func WithContainerOpts(o ...containerd.NewContainerOpts) Option { + return func(args *Options) { + args.ContainerOpts = o + } +} + +// WithOCISpecOpts sets the OCI spec options. +func WithOCISpecOpts(o ...oci.SpecOpts) Option { + return func(args *Options) { + args.OCISpecOpts = o + } +} + +// WithLoggingManager sets the LoggingManager option. +func WithLoggingManager(manager runtime.LoggingManager) Option { + return func(args *Options) { + args.LoggingManager = manager + } +} + +// WithGracefulShutdownTimeout sets the timeout for the task to terminate before sending SIGKILL. +func WithGracefulShutdownTimeout(timeout time.Duration) Option { + return func(args *Options) { + args.GracefulShutdownTimeout = timeout + } +} + +// WithStdin sets the standard input. +func WithStdin(stdin io.ReadSeeker) Option { + return func(args *Options) { + args.Stdin = stdin + } +} + +// WithOOMScoreAdj sets the oom_score_adj. +func WithOOMScoreAdj(score int) Option { + return func(args *Options) { + args.OOMScoreAdj = score + } +} + +// WithCgroupPath sets the cgroup path. +func WithCgroupPath(path string) Option { + return func(args *Options) { + args.CgroupPath = path + } +} + +// WithCustomSeccompProfile sets the function to override seccomp profile. +func WithCustomSeccompProfile(override func(*specs.LinuxSeccomp)) Option { + return func(args *Options) { + args.OverrideSeccompProfile = override + } +} + +// WithDroppedCapabilities sets the list of capabilities to drop. +func WithDroppedCapabilities(caps map[string]struct{}) Option { + return func(args *Options) { + args.DroppedCapabilities = maps.Keys(caps) + } +} + +// WithStdinFile sets the path to the file to use as stdin. +func WithStdinFile(path string) Option { + return func(args *Options) { + args.StdinFile = path + } +} + +// WithStdoutFile sets the path to the file to use as stdout. +func WithStdoutFile(path string) Option { + return func(args *Options) { + args.StdoutFile = path + } +} + +// WithStderrFile sets the path to the file to use as stderr. +func WithStderrFile(path string) Option { + return func(args *Options) { + args.StdoutFile = path + } +} + +// WithCtty sets the controlling tty. +func WithCtty(ctty int) Option { + return func(args *Options) { + args.Ctty = optional.Some(ctty) + } +} + +// WithUID sets the user id of the process. +func WithUID(uid uint32) Option { + return func(args *Options) { + args.UID = uid + } +} diff --git a/internal/app/machined/pkg/system/runner/runner_test.go b/internal/app/machined/pkg/system/runner/runner_test.go new file mode 100644 index 0000000..feaf527 --- /dev/null +++ b/internal/app/machined/pkg/system/runner/runner_test.go @@ -0,0 +1,14 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runner_test + +import "testing" + +func TestEmpty(t *testing.T) { + // added for accurate coverage estimation + // + // please remove it once any unit-test is added + // for this package +} diff --git a/internal/app/machined/pkg/system/service.go b/internal/app/machined/pkg/system/service.go new file mode 100644 index 0000000..4fcb87f --- /dev/null +++ b/internal/app/machined/pkg/system/service.go @@ -0,0 +1,55 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package system + +import ( + "context" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/events" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/health" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner" + "github.com/siderolabs/talos/pkg/conditions" +) + +// Service is an interface describing a system service. +type Service interface { + // ID is the service id. + ID(runtime.Runtime) string + // PreFunc is invoked before a runner is created + PreFunc(context.Context, runtime.Runtime) error + // Runner creates runner for the service + Runner(runtime.Runtime) (runner.Runner, error) + // PostFunc is invoked after a runner is closed. + PostFunc(runtime.Runtime, events.ServiceState) error + // Condition describes the conditions under which a service should + // start. + Condition(runtime.Runtime) conditions.Condition + // DependsOn returns list of service IDs this service depends on. + DependsOn(runtime.Runtime) []string +} + +// HealthcheckedService is a service which provides health check. +type HealthcheckedService interface { + // HealtFunc provides function that checks health of the service + HealthFunc(runtime.Runtime) health.Check + // HealthSettings returns settings for the health check + HealthSettings(runtime.Runtime) *health.Settings +} + +// APIStartableService is a service which allows to be started via API. +type APIStartableService interface { + APIStartAllowed(runtime.Runtime) bool +} + +// APIStoppableService is a service which allows to be stopped via API. +type APIStoppableService interface { + APIStopAllowed(runtime.Runtime) bool +} + +// APIRestartableService is a service which allows to be restarted via API. +type APIRestartableService interface { + APIRestartAllowed(runtime.Runtime) bool +} diff --git a/internal/app/machined/pkg/system/service_events.go b/internal/app/machined/pkg/system/service_events.go new file mode 100644 index 0000000..b2a4e3b --- /dev/null +++ b/internal/app/machined/pkg/system/service_events.go @@ -0,0 +1,116 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package system + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/siderolabs/talos/pkg/conditions" +) + +// StateEvent is a service event (e.g. 'up', 'down'). +type StateEvent string + +// Service event list. +const ( + StateEventUp = StateEvent("up") + StateEventDown = StateEvent("down") + StateEventFinished = StateEvent("finished") +) + +type serviceCondition struct { + mu sync.Mutex + waitingRegister bool + instance *singleton + + event StateEvent + service string +} + +func (sc *serviceCondition) Wait(ctx context.Context) error { + sc.instance.mu.Lock() + svcrunner := sc.instance.state[sc.service] + sc.instance.mu.Unlock() + + if svcrunner == nil { + return sc.waitRegister(ctx) + } + + return sc.waitEvent(ctx, svcrunner) +} + +func (sc *serviceCondition) waitEvent(ctx context.Context, svcrunner *ServiceRunner) error { + notifyCh := make(chan struct{}, 1) + + svcrunner.Subscribe(sc.event, notifyCh) + defer svcrunner.Unsubscribe(sc.event, notifyCh) + + select { + case <-ctx.Done(): + return ctx.Err() + case <-notifyCh: + return nil + } +} + +func (sc *serviceCondition) waitRegister(ctx context.Context) error { + sc.mu.Lock() + sc.waitingRegister = true + sc.mu.Unlock() + + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + var svcrunner *ServiceRunner + + for { + sc.instance.mu.Lock() + svcrunner = sc.instance.state[sc.service] + sc.instance.mu.Unlock() + + if svcrunner != nil { + break + } + + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + } + sc.mu.Lock() + sc.waitingRegister = false + sc.mu.Unlock() + + return sc.waitEvent(ctx, svcrunner) +} + +func (sc *serviceCondition) String() string { + sc.mu.Lock() + waitingRegister := sc.waitingRegister + sc.mu.Unlock() + + if waitingRegister { + return fmt.Sprintf("service %q to be registered", sc.service) + } + + return fmt.Sprintf("service %q to be %q", sc.service, string(sc.event)) +} + +// WaitForService waits for service to reach some state event. +func WaitForService(event StateEvent, service string) conditions.Condition { + return waitForService(instance, event, service) +} + +func waitForService(instance *singleton, event StateEvent, service string) conditions.Condition { + return &serviceCondition{ + instance: instance, + event: event, + service: service, + } +} diff --git a/internal/app/machined/pkg/system/service_runner.go b/internal/app/machined/pkg/system/service_runner.go new file mode 100644 index 0000000..8c2fed6 --- /dev/null +++ b/internal/app/machined/pkg/system/service_runner.go @@ -0,0 +1,458 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package system + +import ( + "context" + "errors" + "fmt" + "log" + "slices" + "sync" + "time" + + "github.com/siderolabs/gen/xslices" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/events" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/health" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner" + "github.com/siderolabs/talos/pkg/conditions" + machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" +) + +// WaitConditionCheckInterval is time between checking for wait condition +// description changes. +// +// Exposed here for unit-tests to override. +var WaitConditionCheckInterval = time.Second + +// ServiceRunner wraps the state of the service (running, stopped, ...). +type ServiceRunner struct { + mu sync.Mutex + + runtime runtime.Runtime + service Service + id string + instance *singleton + + state events.ServiceState + events events.ServiceEvents + + healthState health.State + + stateSubscribers map[StateEvent][]chan<- struct{} + + stopCh chan struct{} +} + +// NewServiceRunner creates new ServiceRunner around Service instance. +func NewServiceRunner(instance *singleton, service Service, runtime runtime.Runtime) *ServiceRunner { + return &ServiceRunner{ + service: service, + instance: instance, + runtime: runtime, + id: service.ID(runtime), + state: events.StateInitialized, + stateSubscribers: make(map[StateEvent][]chan<- struct{}), + stopCh: make(chan struct{}, 1), + } +} + +// GetState implements events.Recorder. +func (svcrunner *ServiceRunner) GetState() events.ServiceState { + svcrunner.mu.Lock() + defer svcrunner.mu.Unlock() + + return svcrunner.state +} + +// UpdateState implements events.Recorder. +func (svcrunner *ServiceRunner) UpdateState(ctx context.Context, newstate events.ServiceState, message string, args ...interface{}) { + svcrunner.mu.Lock() + + event := events.ServiceEvent{ + Message: fmt.Sprintf(message, args...), + State: newstate, + Timestamp: time.Now(), + } + + svcrunner.state = newstate + svcrunner.events.Push(event) + + log.Printf("service[%s](%s): %s", svcrunner.id, svcrunner.state, event.Message) + + isUp := svcrunner.inStateLocked(StateEventUp) + isDown := svcrunner.inStateLocked(StateEventDown) + isFinished := svcrunner.inStateLocked(StateEventFinished) + svcrunner.mu.Unlock() + + if svcrunner.runtime != nil { + svcrunner.runtime.Events().Publish(ctx, event.AsProto(svcrunner.id)) + } + + if isUp { + svcrunner.notifyEvent(StateEventUp) + } + + if isDown { + svcrunner.notifyEvent(StateEventDown) + } + + if isFinished { + svcrunner.notifyEvent(StateEventFinished) + } +} + +func (svcrunner *ServiceRunner) healthUpdate(ctx context.Context, change health.StateChange) { + svcrunner.mu.Lock() + + // service not running, suppress event + if svcrunner.state != events.StateRunning { + svcrunner.mu.Unlock() + + return + } + + var message string + if *change.New.Healthy { + message = "Health check successful" + } else { + message = fmt.Sprintf("Health check failed: %s", change.New.LastMessage) + } + + event := events.ServiceEvent{ + Message: message, + State: svcrunner.state, + Health: change.New, + Timestamp: time.Now(), + } + svcrunner.events.Push(event) + + log.Printf("service[%s](%s): %s", svcrunner.id, svcrunner.state, event.Message) + + isUp := svcrunner.inStateLocked(StateEventUp) + svcrunner.mu.Unlock() + + if isUp { + svcrunner.notifyEvent(StateEventUp) + } + + if svcrunner.runtime != nil { + svcrunner.runtime.Events().Publish(ctx, event.AsProto(svcrunner.id)) + } +} + +// GetEventHistory returns history of events for this service. +func (svcrunner *ServiceRunner) GetEventHistory(count int) []events.ServiceEvent { + svcrunner.mu.Lock() + defer svcrunner.mu.Unlock() + + return svcrunner.events.Get(count) +} + +func (svcrunner *ServiceRunner) waitFor(ctx context.Context, condition conditions.Condition) error { + description := condition.String() + svcrunner.UpdateState(ctx, events.StateWaiting, "Waiting for %s", description) + + errCh := make(chan error) + + go func() { + errCh <- condition.Wait(ctx) + }() + + ticker := time.NewTicker(WaitConditionCheckInterval) + defer ticker.Stop() + + // update state if condition description changes (some conditions are satisfied) + for { + select { + case err := <-errCh: + return err + case <-ticker.C: + newDescription := condition.String() + if newDescription != description && newDescription != "" { + description = newDescription + svcrunner.UpdateState(ctx, events.StateWaiting, "Waiting for %s", description) + } + } + } +} + +// ErrSkip is returned by Run when service is skipped. +var ErrSkip = errors.New("service skipped") + +// Run initializes the service and runs it. +// +// Run returns an error when a service stops. +// +// Run should be run in a goroutine. +// +//nolint:gocyclo +func (svcrunner *ServiceRunner) Run(notifyChannels ...chan<- struct{}) error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go func() { + select { + case <-ctx.Done(): + return + case <-svcrunner.stopCh: + cancel() + } + }() + + svcrunner.UpdateState(ctx, events.StateStarting, "Starting service") + + for _, notifyCh := range notifyChannels { + close(notifyCh) + } + + condition := svcrunner.service.Condition(svcrunner.runtime) + + dependencies := svcrunner.service.DependsOn(svcrunner.runtime) + if len(dependencies) > 0 { + serviceConditions := xslices.Map(dependencies, func(dep string) conditions.Condition { return waitForService(instance, StateEventUp, dep) }) + serviceDependencies := conditions.WaitForAll(serviceConditions...) + + if condition != nil { + condition = conditions.WaitForAll(serviceDependencies, condition) + } else { + condition = serviceDependencies + } + } + + if condition != nil { + if err := svcrunner.waitFor(ctx, condition); err != nil { + return fmt.Errorf("condition failed: %w", err) + } + } + + svcrunner.UpdateState(ctx, events.StatePreparing, "Running pre state") + + if err := svcrunner.service.PreFunc(ctx, svcrunner.runtime); err != nil { + return fmt.Errorf("failed to run pre stage: %w", err) + } + + svcrunner.UpdateState(ctx, events.StatePreparing, "Creating service runner") + + runnr, err := svcrunner.service.Runner(svcrunner.runtime) + if err != nil { + return fmt.Errorf("failed to create runner: %w", err) + } + + defer func() { + // PostFunc passes in the state so that we can take actions that depend on the outcome of the run + state := svcrunner.GetState() + + if err := svcrunner.service.PostFunc(svcrunner.runtime, state); err != nil { + svcrunner.UpdateState(ctx, events.StateFailed, "Failed to run post stage: %v", err) + } + }() + + if runnr == nil { + return ErrSkip + } + + if err := svcrunner.run(ctx, runnr); err != nil { + return fmt.Errorf("failed running service: %w", err) + } + + return nil +} + +//nolint:gocyclo +func (svcrunner *ServiceRunner) run(ctx context.Context, runnr runner.Runner) error { + if runnr == nil { + // special case - run nothing (TODO: we should handle it better, e.g. in PreFunc) + return nil + } + + if err := runnr.Open(); err != nil { + return fmt.Errorf("error opening runner: %w", err) + } + + //nolint:errcheck + defer runnr.Close() + + errCh := make(chan error) + + go func() { + errCh <- runnr.Run(func(s events.ServiceState, msg string, args ...interface{}) { + svcrunner.UpdateState(ctx, s, msg, args...) + }) + }() + + if healthSvc, ok := svcrunner.service.(HealthcheckedService); ok { + var healthWg sync.WaitGroup + defer healthWg.Wait() + + healthWg.Add(1) + + go func() { + defer healthWg.Done() + + //nolint:errcheck + health.Run( + ctx, + healthSvc.HealthSettings(svcrunner.runtime), + &svcrunner.healthState, + healthSvc.HealthFunc(svcrunner.runtime), + ) + }() + + notifyCh := make(chan health.StateChange, 2) + + svcrunner.healthState.Subscribe(notifyCh) + defer svcrunner.healthState.Unsubscribe(notifyCh) + + healthWg.Add(1) + + go func() { + defer healthWg.Done() + + for { + select { + case <-ctx.Done(): + return + case change := <-notifyCh: + svcrunner.healthUpdate(ctx, change) + } + } + }() + } + + select { + case <-ctx.Done(): + err := runnr.Stop() + + <-errCh + + if err != nil { + return fmt.Errorf("error stopping service: %w", err) + } + case err := <-errCh: + if err != nil { + return fmt.Errorf("error running service: %w", err) + } + } + + return nil +} + +// Shutdown initiates shutdown of the service runner +// +// Shutdown completes when Start() returns. +func (svcrunner *ServiceRunner) Shutdown() { + select { + case svcrunner.stopCh <- struct{}{}: + default: + } +} + +// AsProto returns protobuf struct with the state of the service runner. +func (svcrunner *ServiceRunner) AsProto() *machineapi.ServiceInfo { + svcrunner.mu.Lock() + defer svcrunner.mu.Unlock() + + return &machineapi.ServiceInfo{ + Id: svcrunner.id, + State: svcrunner.state.String(), + Events: svcrunner.events.AsProto(events.MaxEventsToKeep), + Health: svcrunner.healthState.AsProto(), + } +} + +// Subscribe to a specific event for this service. +// +// Channel `ch` should be buffered or it should have listener attached to it, +// as event might be delivered before Subscribe() returns. +func (svcrunner *ServiceRunner) Subscribe(event StateEvent, ch chan<- struct{}) { + svcrunner.mu.Lock() + + if svcrunner.inStateLocked(event) { + svcrunner.mu.Unlock() + + // svcrunner is already in expected state, notify immediately + select { + case ch <- struct{}{}: + default: + } + + return + } + + svcrunner.stateSubscribers[event] = append(svcrunner.stateSubscribers[event], ch) + svcrunner.mu.Unlock() +} + +// Unsubscribe cancels subscription established with Subscribe. +func (svcrunner *ServiceRunner) Unsubscribe(event StateEvent, ch chan<- struct{}) { + svcrunner.mu.Lock() + defer svcrunner.mu.Unlock() + + channels := svcrunner.stateSubscribers[event] + + for i := 0; i < len(channels); { + if channels[i] == ch { + channels[i], channels[len(channels)-1] = channels[len(channels)-1], nil + channels = channels[:len(channels)-1] + } else { + i++ + } + } + + svcrunner.stateSubscribers[event] = channels +} + +func (svcrunner *ServiceRunner) notifyEvent(event StateEvent) { + svcrunner.mu.Lock() + channels := slices.Clone(svcrunner.stateSubscribers[event]) + svcrunner.mu.Unlock() + + for _, ch := range channels { + select { + case ch <- struct{}{}: + default: + } + } +} + +func (svcrunner *ServiceRunner) inStateLocked(event StateEvent) bool { + switch event { + case StateEventUp: + // up when: + // a) either skipped or already finished + // b) or running and healthy (if supports health checks) + switch svcrunner.state { //nolint:exhaustive + case events.StateSkipped, events.StateFinished: + return true + case events.StateRunning: + // check if service supports health checks + _, supportsHealth := svcrunner.service.(HealthcheckedService) + health := svcrunner.healthState.Get() + + return !supportsHealth || (health.Healthy != nil && *health.Healthy) + default: + return false + } + case StateEventDown: + // down when in any of the terminal states + switch svcrunner.state { //nolint:exhaustive + case events.StateFailed, events.StateFinished, events.StateSkipped: + return true + default: + return false + } + case StateEventFinished: + if svcrunner.state == events.StateFinished { + return true + } + + return false + default: + panic("unsupported event") + } +} diff --git a/internal/app/machined/pkg/system/service_runner_test.go b/internal/app/machined/pkg/system/service_runner_test.go new file mode 100644 index 0000000..2707783 --- /dev/null +++ b/internal/app/machined/pkg/system/service_runner_test.go @@ -0,0 +1,467 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package system_test + +import ( + "errors" + "testing" + "time" + + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/system" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/events" + "github.com/siderolabs/talos/pkg/conditions" +) + +type ServiceRunnerSuite struct { + suite.Suite +} + +func (suite *ServiceRunnerSuite) assertStateSequence(expectedStates []events.ServiceState, sr *system.ServiceRunner) { + states := []events.ServiceState{} + + for _, event := range sr.GetEventHistory(1000) { + states = append(states, event.State) + } + + suite.Assert().Equal(expectedStates, states) +} + +func (suite *ServiceRunnerSuite) TestFullFlow() { + sr := system.NewServiceRunner(system.Services(nil), &MockService{ + condition: conditions.None(), + }, nil) + + errCh := make(chan error) + + go func() { + errCh <- sr.Run() + }() + + suite.Require().NoError(retry.Constant(time.Minute, retry.WithUnits(10*time.Millisecond)).Retry(func() error { + state := sr.AsProto().State + if state != events.StateRunning.String() { + return retry.ExpectedErrorf("service should be running") + } + + return nil + })) + + select { + case <-errCh: + suite.Require().Fail("service running should be still running") + default: + } + + sr.Shutdown() + + suite.Assert().NoError(<-errCh) + + suite.assertStateSequence([]events.ServiceState{ + events.StateStarting, + events.StateWaiting, + events.StatePreparing, + events.StatePreparing, + events.StateRunning, + }, sr) + + protoService := sr.AsProto() + suite.Assert().Equal("MockRunner", protoService.Id) + suite.Assert().Equal("Running", protoService.State) + suite.Assert().True(protoService.Health.Unknown) + suite.Assert().Len(protoService.Events.Events, 5) +} + +func (suite *ServiceRunnerSuite) TestFullFlowHealthy() { + sr := system.NewServiceRunner(system.Services(nil), &MockHealthcheckedService{}, nil) + + errCh := make(chan error) + + go func() { + errCh <- sr.Run() + }() + + suite.Require().NoError(retry.Constant(time.Minute, retry.WithUnits(10*time.Millisecond)).Retry(func() error { + health := sr.AsProto().Health + if health.Unknown || !health.Healthy { + return retry.ExpectedErrorf("service should be healthy") + } + + return nil + })) + + select { + case <-errCh: + suite.Require().Fail("service running should be still running") + default: + } + + sr.Shutdown() + + suite.Assert().NoError(<-errCh) + + suite.assertStateSequence([]events.ServiceState{ + events.StateStarting, + events.StatePreparing, + events.StatePreparing, + events.StateRunning, + events.StateRunning, // one more notification when service is healthy + }, sr) +} + +func (suite *ServiceRunnerSuite) TestFullFlowHealthChanges() { + m := MockHealthcheckedService{ + MockService: MockService{ + condition: conditions.None(), + }, + } + sr := system.NewServiceRunner(system.Services(nil), &m, nil) + + errCh := make(chan error) + + go func() { + errCh <- sr.Run() + }() + + suite.Require().NoError(retry.Constant(time.Minute, retry.WithUnits(10*time.Millisecond)).Retry(func() error { + health := sr.AsProto().Health + if health.Unknown || !health.Healthy { + return retry.ExpectedErrorf("service should be healthy") + } + + return nil + })) + + m.SetHealthy(false) + + suite.Require().NoError(retry.Constant(time.Minute, retry.WithUnits(10*time.Millisecond)).Retry(func() error { + health := sr.AsProto().Health + if health.Unknown || health.Healthy { + return retry.ExpectedErrorf("service should be not healthy") + } + + return nil + })) + + m.SetHealthy(true) + + suite.Require().NoError(retry.Constant(time.Minute, retry.WithUnits(10*time.Millisecond)).Retry(func() error { + health := sr.AsProto().Health + if health.Unknown || !health.Healthy { + return retry.ExpectedErrorf("service should be healthy") + } + + return nil + })) + + sr.Shutdown() + + suite.Assert().NoError(<-errCh) + + suite.assertStateSequence([]events.ServiceState{ + events.StateStarting, + events.StateWaiting, + events.StatePreparing, + events.StatePreparing, + events.StateRunning, + events.StateRunning, // initial: healthy + events.StateRunning, // not healthy + events.StateRunning, // once again healthy + }, sr) +} + +func (suite *ServiceRunnerSuite) TestWaitingDescriptionChange() { + oldWaitConditionCheckInterval := system.WaitConditionCheckInterval + system.WaitConditionCheckInterval = 10 * time.Millisecond + + defer func() { + system.WaitConditionCheckInterval = oldWaitConditionCheckInterval + }() + + cond1 := NewMockCondition("cond1") + cond2 := NewMockCondition("cond2") + sr := system.NewServiceRunner(system.Services(nil), &MockService{ + condition: conditions.WaitForAll(cond1, cond2), + }, nil) + + errCh := make(chan error) + + go func() { + errCh <- sr.Run() + }() + + suite.Require().NoError(retry.Constant(time.Minute, retry.WithUnits(10*time.Millisecond)).Retry(func() error { + state := sr.AsProto().State + if state != events.StateWaiting.String() { + return retry.ExpectedErrorf("service should be waiting") + } + + return nil + })) + + select { + case <-errCh: + suite.Require().Fail("service running should be still running") + default: + } + + close(cond1.done) + + suite.Require().NoError(retry.Constant(time.Minute, retry.WithUnits(10*time.Millisecond)).Retry(func() error { + events := sr.AsProto().Events.Events + + lastMsg := events[len(events)-1].Msg + if lastMsg != "Waiting for cond2" { + return retry.ExpectedErrorf("service should be waiting on 2nd condition") + } + + return nil + })) + + select { + case <-errCh: + suite.Require().Fail("service running should be still running") + default: + } + + close(cond2.done) + + suite.Require().NoError(retry.Constant(time.Minute, retry.WithUnits(10*time.Millisecond)).Retry(func() error { + state := sr.AsProto().State + if state != events.StateRunning.String() { + return retry.ExpectedErrorf("service should be running") + } + + return nil + })) + + sr.Shutdown() + + suite.Assert().NoError(<-errCh) + + suite.assertStateSequence([]events.ServiceState{ + events.StateStarting, + events.StateWaiting, + events.StateWaiting, + events.StatePreparing, + events.StatePreparing, + events.StateRunning, + }, sr) + + events := sr.GetEventHistory(10000) + suite.Assert().Equal("Waiting for cond1, cond2", events[1].Message) + suite.Assert().Equal("Waiting for cond2", events[2].Message) +} + +func (suite *ServiceRunnerSuite) TestPreStageFail() { + svc := &MockService{ + preError: errors.New("pre failed"), + } + sr := system.NewServiceRunner(system.Services(nil), svc, nil) + err := sr.Run() + + suite.assertStateSequence([]events.ServiceState{ + events.StateStarting, + events.StatePreparing, + }, sr) + suite.Assert().EqualError(err, "failed to run pre stage: pre failed") +} + +func (suite *ServiceRunnerSuite) TestRunnerStageFail() { + svc := &MockService{ + runnerError: errors.New("runner failed"), + } + sr := system.NewServiceRunner(system.Services(nil), svc, nil) + err := sr.Run() + + suite.assertStateSequence([]events.ServiceState{ + events.StateStarting, + events.StatePreparing, + events.StatePreparing, + }, sr) + suite.Assert().EqualError(err, "failed to create runner: runner failed") +} + +func (suite *ServiceRunnerSuite) TestRunnerStageSkipped() { + svc := &MockService{ + nilRunner: true, + } + sr := system.NewServiceRunner(system.Services(nil), svc, nil) + err := sr.Run() + + suite.assertStateSequence([]events.ServiceState{ + events.StateStarting, + events.StatePreparing, + events.StatePreparing, + }, sr) + suite.Assert().ErrorIs(err, system.ErrSkip) +} + +func (suite *ServiceRunnerSuite) TestAbortOnCondition() { + svc := &MockService{ + condition: conditions.WaitForFileToExist("/doesntexistever"), + } + sr := system.NewServiceRunner(system.Services(nil), svc, nil) + + errCh := make(chan error) + + go func() { + errCh <- sr.Run() + }() + + suite.Require().NoError(retry.Constant(time.Minute, retry.WithUnits(10*time.Millisecond)).Retry(func() error { + state := sr.AsProto().State + if state != events.StateWaiting.String() { + return retry.ExpectedErrorf("service should be waiting") + } + + return nil + })) + + select { + case <-errCh: + suite.Require().Fail("service running should be still running") + default: + } + + sr.Shutdown() + + suite.Assert().EqualError(<-errCh, "condition failed: context canceled") + + suite.assertStateSequence([]events.ServiceState{ + events.StateStarting, + events.StateWaiting, + }, sr) +} + +func (suite *ServiceRunnerSuite) TestPostStateFail() { + svc := &MockService{ + condition: conditions.None(), + postError: errors.New("post failed"), + } + sr := system.NewServiceRunner(system.Services(nil), svc, nil) + + errCh := make(chan error) + runNotify := make(chan struct{}) + + go func() { + errCh <- sr.Run(runNotify) + }() + + <-runNotify + + sr.Shutdown() + + suite.Assert().NoError(<-errCh) + + suite.assertStateSequence([]events.ServiceState{ + events.StateStarting, + events.StateWaiting, + events.StatePreparing, + events.StatePreparing, + events.StateRunning, + events.StateFailed, + }, sr) +} + +func (suite *ServiceRunnerSuite) TestRunFail() { + runner := &MockRunner{exitCh: make(chan error)} + svc := &MockService{runner: runner} + sr := system.NewServiceRunner(system.Services(nil), svc, nil) + + errCh := make(chan error) + + go func() { + errCh <- sr.Run() + }() + + runner.exitCh <- errors.New("run failed") + + suite.Assert().EqualError(<-errCh, "failed running service: error running service: run failed") + + suite.assertStateSequence([]events.ServiceState{ + events.StateStarting, + events.StatePreparing, + events.StatePreparing, + events.StateRunning, + }, sr) +} + +func (suite *ServiceRunnerSuite) TestFullFlowRestart() { + sr := system.NewServiceRunner(system.Services(nil), &MockService{ + condition: conditions.None(), + }, nil) + + errCh := make(chan error) + + go func() { + errCh <- sr.Run() + }() + + suite.Require().NoError(retry.Constant(time.Minute, retry.WithUnits(10*time.Millisecond)).Retry(func() error { + state := sr.AsProto().State + if state != events.StateRunning.String() { + return retry.ExpectedErrorf("service should be running") + } + + return nil + })) + + select { + case <-errCh: + suite.Require().Fail("service running should be still running") + default: + } + + sr.Shutdown() + + suite.Assert().NoError(<-errCh) + + notifyCh := make(chan struct{}) + + go func() { + errCh <- sr.Run(notifyCh) + }() + + <-notifyCh + + suite.Require().NoError(retry.Constant(time.Minute, retry.WithUnits(10*time.Millisecond)).Retry(func() error { + state := sr.AsProto().State + if state != events.StateRunning.String() { + return retry.ExpectedErrorf("service should be running") + } + + return nil + })) + + select { + case <-errCh: + suite.Require().Fail("service running should be still running") + default: + } + + sr.Shutdown() + + suite.Assert().NoError(<-errCh) + + suite.assertStateSequence([]events.ServiceState{ + events.StateStarting, + events.StateWaiting, + events.StatePreparing, + events.StatePreparing, + events.StateRunning, + events.StateStarting, + events.StateWaiting, + events.StatePreparing, + events.StatePreparing, + events.StateRunning, + }, sr) +} + +func TestServiceRunnerSuite(t *testing.T) { + suite.Run(t, new(ServiceRunnerSuite)) +} diff --git a/internal/app/machined/pkg/system/services/apid.go b/internal/app/machined/pkg/system/services/apid.go new file mode 100644 index 0000000..0aaa749 --- /dev/null +++ b/internal/app/machined/pkg/system/services/apid.go @@ -0,0 +1,226 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:golint +package services + +import ( + "context" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "strings" + + "github.com/containerd/containerd/oci" + "github.com/containerd/containerd/pkg/cap" + "github.com/cosi-project/runtime/api/v1alpha1" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/protobuf/server" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/siderolabs/go-debug" + "google.golang.org/grpc" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/system" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/events" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/health" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/containerd" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/restart" + "github.com/aenix-io/talm/internal/pkg/environment" + "github.com/siderolabs/talos/pkg/conditions" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" +) + +var _ system.HealthcheckedService = (*APID)(nil) + +// APID implements the Service interface. It serves as the concrete type with +// the required methods. +type APID struct { + runtimeServer *grpc.Server +} + +// ID implements the Service interface. +func (o *APID) ID(r runtime.Runtime) string { + return "apid" +} + +// apidResourceFilter filters access to COSI state for apid. +func apidResourceFilter(ctx context.Context, access state.Access) error { + if !access.Verb.Readonly() { + return errors.New("write access denied") + } + + switch { + case access.ResourceNamespace == secrets.NamespaceName && access.ResourceType == secrets.APIType && access.ResourceID == secrets.APIID: + // allowed, contains apid certificates + case access.ResourceNamespace == network.NamespaceName && access.ResourceType == network.NodeAddressType: + // allowed, contains local node addresses + case access.ResourceNamespace == network.NamespaceName && access.ResourceType == network.HostnameStatusType: + // allowed, contains local node hostname + default: + return errors.New("access denied") + } + + return nil +} + +// PreFunc implements the Service interface. +func (o *APID) PreFunc(ctx context.Context, r runtime.Runtime) error { + // filter apid access to make sure apid can only access its certificates + resources := state.Filter(r.State().V1Alpha2().Resources(), apidResourceFilter) + + // ensure socket dir exists + if err := os.MkdirAll(filepath.Dir(constants.APIRuntimeSocketPath), 0o750); err != nil { + return err + } + + // set the final leaf to be world-executable to make apid connect to the socket + if err := os.Chmod(filepath.Dir(constants.APIRuntimeSocketPath), 0o751); err != nil { + return err + } + + // clean up the socket if it already exists (important for Talos in a container) + if err := os.RemoveAll(constants.APIRuntimeSocketPath); err != nil { + return err + } + + listener, err := net.Listen("unix", constants.APIRuntimeSocketPath) + if err != nil { + return err + } + + // chown the socket path to make it accessible to the apid + if err := os.Chown(constants.APIRuntimeSocketPath, constants.ApidUserID, constants.ApidUserID); err != nil { + return err + } + + o.runtimeServer = grpc.NewServer( + grpc.SharedWriteBuffer(true), + ) + v1alpha1.RegisterStateServer(o.runtimeServer, server.NewState(resources)) + + go o.runtimeServer.Serve(listener) //nolint:errcheck + + return prepareRootfs(o.ID(r)) +} + +// PostFunc implements the Service interface. +func (o *APID) PostFunc(r runtime.Runtime, state events.ServiceState) (err error) { + o.runtimeServer.Stop() + + return os.RemoveAll(constants.APIRuntimeSocketPath) +} + +// Condition implements the Service interface. +func (o *APID) Condition(r runtime.Runtime) conditions.Condition { + return secrets.NewAPIReadyCondition(r.State().V1Alpha2().Resources()) +} + +// DependsOn implements the Service interface. +func (o *APID) DependsOn(r runtime.Runtime) []string { + return []string{"containerd"} +} + +// Runner implements the Service interface. +func (o *APID) Runner(r runtime.Runtime) (runner.Runner, error) { + // Ensure socket dir exists + if err := os.MkdirAll(filepath.Dir(constants.APISocketPath), 0o750); err != nil { + return nil, err + } + + // Make sure apid user owns socket directory. + if err := os.Chown(filepath.Dir(constants.APISocketPath), constants.ApidUserID, constants.ApidUserID); err != nil { + return nil, err + } + + // Set the process arguments. + args := runner.Args{ + ID: o.ID(r), + ProcessArgs: []string{ + "/apid", + }, + } + + if r.Config().Machine().Features().RBACEnabled() { + args.ProcessArgs = append(args.ProcessArgs, "--enable-rbac") + } + + if r.Config().Machine().Features().ApidCheckExtKeyUsageEnabled() { + args.ProcessArgs = append(args.ProcessArgs, "--enable-ext-key-usage-check") + } + + // Set the mounts. + mounts := []specs.Mount{ + {Type: "bind", Destination: "/etc/ssl", Source: "/etc/ssl", Options: []string{"bind", "ro"}}, + {Type: "bind", Destination: filepath.Dir(constants.MachineSocketPath), Source: filepath.Dir(constants.MachineSocketPath), Options: []string{"rbind", "ro"}}, + {Type: "bind", Destination: filepath.Dir(constants.APISocketPath), Source: filepath.Dir(constants.APISocketPath), Options: []string{"rbind", "rw"}}, + } + + env := []string{ + constants.TcellMinimizeEnvironment, + } + + for _, value := range environment.Get(r.Config()) { + key, _, _ := strings.Cut(value, "=") + + switch strings.ToLower(key) { + // explicitly exclude proxy variables from apid since this will + // negatively impact grpc connections. + // ref: https://github.com/grpc/grpc-go/blob/0f32486dd3c9bc29705535bd7e2e43801824cbc4/clientconn.go#L199-L206 + // ref: https://github.com/grpc/grpc-go/blob/63ae68c9686cc0dd26c4f7476d66bb2f5c31789f/proxy.go#L118-L144 + case "no_proxy": + case "http_proxy": + case "https_proxy": + default: + env = append(env, value) + } + } + + if debug.RaceEnabled { + env = append(env, "GORACE=halt_on_error=1") + } + + return restart.New(containerd.NewRunner( + r.Config().Debug(), + &args, + runner.WithLoggingManager(r.Logging()), + runner.WithContainerdAddress(constants.SystemContainerdAddress), + runner.WithEnv(env), + runner.WithOCISpecOpts( + oci.WithDroppedCapabilities(cap.Known()), + oci.WithHostNamespace(specs.NetworkNamespace), + oci.WithMounts(mounts), + oci.WithRootFSPath(filepath.Join(constants.SystemLibexecPath, o.ID(r))), + oci.WithRootFSReadonly(), + oci.WithUser(fmt.Sprintf("%d:%d", constants.ApidUserID, constants.ApidUserID)), + ), + runner.WithOOMScoreAdj(-998), + ), + restart.WithType(restart.Forever), + ), nil +} + +// HealthFunc implements the HealthcheckedService interface. +func (o *APID) HealthFunc(runtime.Runtime) health.Check { + return func(ctx context.Context) error { + var d net.Dialer + + conn, err := d.DialContext(ctx, "tcp", fmt.Sprintf("%s:%d", "127.0.0.1", constants.ApidPort)) + if err != nil { + return err + } + + return conn.Close() + } +} + +// HealthSettings implements the HealthcheckedService interface. +func (o *APID) HealthSettings(runtime.Runtime) *health.Settings { + return &health.DefaultSettings +} diff --git a/internal/app/machined/pkg/system/services/containerd.go b/internal/app/machined/pkg/system/services/containerd.go new file mode 100644 index 0000000..9c4934b --- /dev/null +++ b/internal/app/machined/pkg/system/services/containerd.go @@ -0,0 +1,140 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package services + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/containerd/containerd" + "google.golang.org/grpc/health/grpc_health_v1" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/system" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/events" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/health" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/process" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/restart" + "github.com/aenix-io/talm/internal/pkg/environment" + "github.com/siderolabs/talos/pkg/conditions" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +var _ system.HealthcheckedService = (*Containerd)(nil) + +// Containerd implements the Service interface. It serves as the concrete type with +// the required methods. +type Containerd struct { + // client is a lazy-initialized containerd client. It should be accessed using the Client() method. + client *containerd.Client +} + +// Client lazy-initializes the containerd client if needed and returns it. +func (c *Containerd) Client() (*containerd.Client, error) { + if c.client != nil { + return c.client, nil + } + + client, err := containerd.New(constants.SystemContainerdAddress) + if err != nil { + return nil, err + } + + c.client = client + + return c.client, err +} + +// ID implements the Service interface. +func (c *Containerd) ID(r runtime.Runtime) string { + return "containerd" +} + +// PreFunc implements the Service interface. +func (c *Containerd) PreFunc(ctx context.Context, r runtime.Runtime) error { + return nil +} + +// PostFunc implements the Service interface. +func (c *Containerd) PostFunc(r runtime.Runtime, state events.ServiceState) (err error) { + if c.client != nil { + return c.client.Close() + } + + return nil +} + +// Condition implements the Service interface. +func (c *Containerd) Condition(r runtime.Runtime) conditions.Condition { + return nil +} + +// DependsOn implements the Service interface. +func (c *Containerd) DependsOn(r runtime.Runtime) []string { + return nil +} + +// Runner implements the Service interface. +func (c *Containerd) Runner(r runtime.Runtime) (runner.Runner, error) { + // Set the process arguments. + args := &runner.Args{ + ID: c.ID(r), + ProcessArgs: []string{ + "/bin/containerd", + "--address", + constants.SystemContainerdAddress, + "--state", + filepath.Join(constants.SystemRunPath, "containerd"), + "--root", + filepath.Join(constants.SystemVarPath, "lib", "containerd"), + }, + } + + debug := false + + if r.Config() != nil { + debug = r.Config().Debug() + } + + return restart.New(process.NewRunner( + debug, + args, + runner.WithLoggingManager(r.Logging()), + runner.WithEnv(environment.Get(r.Config())), + runner.WithOOMScoreAdj(-999), + runner.WithCgroupPath(constants.CgroupSystemRuntime), + runner.WithDroppedCapabilities(constants.DefaultDroppedCapabilities), + ), + restart.WithType(restart.Forever), + ), nil +} + +// HealthFunc implements the HealthcheckedService interface. +func (c *Containerd) HealthFunc(runtime.Runtime) health.Check { + return func(ctx context.Context) error { + client, err := c.Client() + if err != nil { + return err + } + + resp, err := client.HealthService().Check(ctx, &grpc_health_v1.HealthCheckRequest{}) + if err != nil { + return err + } + + if resp.Status != grpc_health_v1.HealthCheckResponse_SERVING { + return fmt.Errorf("unexpected serving status: %d", resp.Status) + } + + return nil + } +} + +// HealthSettings implements the HealthcheckedService interface. +func (c *Containerd) HealthSettings(runtime.Runtime) *health.Settings { + return &health.DefaultSettings +} diff --git a/internal/app/machined/pkg/system/services/cri.go b/internal/app/machined/pkg/system/services/cri.go new file mode 100644 index 0000000..4bccae9 --- /dev/null +++ b/internal/app/machined/pkg/system/services/cri.go @@ -0,0 +1,134 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package services + +import ( + "context" + "fmt" + "os" + + "github.com/containerd/containerd" + "github.com/containerd/containerd/defaults" + "google.golang.org/grpc/health/grpc_health_v1" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/system" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/events" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/health" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/process" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/restart" + "github.com/aenix-io/talm/internal/pkg/environment" + "github.com/siderolabs/talos/pkg/conditions" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +var _ system.HealthcheckedService = (*CRI)(nil) + +// CRI implements the Service interface. It serves as the concrete type with +// the required methods. +type CRI struct { + // client is a lazy-initialized containerd client. It should be accessed using the Client() method. + client *containerd.Client +} + +// Client lazy-initializes the containerd client if needed and returns it. +func (c *CRI) Client() (*containerd.Client, error) { + if c.client != nil { + return c.client, nil + } + + client, err := containerd.New(constants.CRIContainerdAddress) + if err != nil { + return nil, err + } + + c.client = client + + return c.client, err +} + +// ID implements the Service interface. +func (c *CRI) ID(r runtime.Runtime) string { + return "cri" +} + +// PreFunc implements the Service interface. +func (c *CRI) PreFunc(ctx context.Context, r runtime.Runtime) error { + return os.MkdirAll(defaults.DefaultRootDir, os.ModeDir) +} + +// PostFunc implements the Service interface. +func (c *CRI) PostFunc(r runtime.Runtime, state events.ServiceState) (err error) { + if c.client != nil { + return c.client.Close() + } + + return nil +} + +// Condition implements the Service interface. +func (c *CRI) Condition(r runtime.Runtime) conditions.Condition { + return network.NewReadyCondition(r.State().V1Alpha2().Resources(), network.AddressReady, network.HostnameReady, network.EtcFilesReady) +} + +// DependsOn implements the Service interface. +func (c *CRI) DependsOn(r runtime.Runtime) []string { + return nil +} + +// Runner implements the Service interface. +func (c *CRI) Runner(r runtime.Runtime) (runner.Runner, error) { + // Set the process arguments. + args := &runner.Args{ + ID: c.ID(r), + ProcessArgs: []string{ + "/bin/containerd", + "--address", + constants.CRIContainerdAddress, + "--config", + constants.CRIContainerdConfig, + }, + } + + return restart.New(process.NewRunner( + r.Config().Debug(), + args, + runner.WithLoggingManager(r.Logging()), + runner.WithEnv(environment.Get(r.Config())), + runner.WithOOMScoreAdj(-500), + runner.WithCgroupPath(constants.CgroupPodRuntime), + runner.WithDroppedCapabilities(constants.DefaultDroppedCapabilities), + ), + restart.WithType(restart.Forever), + ), nil +} + +// HealthFunc implements the HealthcheckedService interface. +func (c *CRI) HealthFunc(runtime.Runtime) health.Check { + return func(ctx context.Context) error { + client, err := c.Client() + if err != nil { + return err + } + + resp, err := client.HealthService().Check(ctx, &grpc_health_v1.HealthCheckRequest{}) + if err != nil { + return err + } + + if resp.Status != grpc_health_v1.HealthCheckResponse_SERVING { + return fmt.Errorf("unexpected serving status: %d", resp.Status) + } + + return nil + } +} + +// HealthSettings implements the HealthcheckedService interface. +func (c *CRI) HealthSettings(runtime.Runtime) *health.Settings { + return &health.DefaultSettings +} diff --git a/internal/app/machined/pkg/system/services/dashboard.go b/internal/app/machined/pkg/system/services/dashboard.go new file mode 100644 index 0000000..96ade72 --- /dev/null +++ b/internal/app/machined/pkg/system/services/dashboard.go @@ -0,0 +1,75 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:golint +package services + +import ( + "context" + "fmt" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/events" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/process" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/restart" + "github.com/aenix-io/talm/internal/pkg/capability" + "github.com/aenix-io/talm/internal/pkg/console" + "github.com/siderolabs/talos/pkg/conditions" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// Dashboard implements the Service interface. It serves as the concrete type with +// the required methods. +type Dashboard struct{} + +// ID implements the Service interface. +func (d *Dashboard) ID(_ runtime.Runtime) string { + return "dashboard" +} + +// PreFunc implements the Service interface. +func (d *Dashboard) PreFunc(_ context.Context, _ runtime.Runtime) error { + return console.Switch(constants.DashboardTTY) +} + +// PostFunc implements the Service interface. +func (d *Dashboard) PostFunc(_ runtime.Runtime, _ events.ServiceState) error { + return console.Switch(constants.KernelLogsTTY) +} + +// Condition implements the Service interface. +func (d *Dashboard) Condition(_ runtime.Runtime) conditions.Condition { + return conditions.WaitForFileToExist(constants.MachineSocketPath) +} + +// DependsOn implements the Service interface. +func (d *Dashboard) DependsOn(_ runtime.Runtime) []string { + return []string{machinedServiceID} +} + +// Runner implements the Service interface. +func (d *Dashboard) Runner(r runtime.Runtime) (runner.Runner, error) { + tty := fmt.Sprintf("/dev/tty%d", constants.DashboardTTY) + + return restart.New(process.NewRunner(false, &runner.Args{ + ID: d.ID(r), + ProcessArgs: []string{"/sbin/dashboard"}, + }, + runner.WithLoggingManager(r.Logging()), + runner.WithEnv([]string{ + "TERM=linux", + constants.TcellMinimizeEnvironment, + }), + runner.WithStdinFile(tty), + runner.WithStdoutFile(tty), + runner.WithCtty(1), + runner.WithOOMScoreAdj(-400), + runner.WithDroppedCapabilities(capability.AllCapabilitiesSetLowercase()), + runner.WithCgroupPath(constants.CgroupDashboard), + runner.WithUID(constants.DashboardUserID), + ), + restart.WithType(restart.Forever), + ), nil +} diff --git a/internal/app/machined/pkg/system/services/etcd.go b/internal/app/machined/pkg/system/services/etcd.go new file mode 100644 index 0000000..a8ee7f3 --- /dev/null +++ b/internal/app/machined/pkg/system/services/etcd.go @@ -0,0 +1,710 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package services + +import ( + "context" + "errors" + "fmt" + "io" + "log" + "net/netip" + "os" + goruntime "runtime" + "strings" + "time" + + containerdapi "github.com/containerd/containerd" + "github.com/containerd/containerd/namespaces" + "github.com/containerd/containerd/oci" + "github.com/containerd/containerd/pkg/cap" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/go-retry/retry" + clientv3 "go.etcd.io/etcd/client/v3" + snapshot "go.etcd.io/etcd/etcdutl/v3/snapshot" + "google.golang.org/grpc" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/system" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/events" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/health" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/containerd" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/restart" + "github.com/aenix-io/talm/internal/pkg/containers/image" + "github.com/aenix-io/talm/internal/pkg/environment" + "github.com/aenix-io/talm/internal/pkg/etcd" + "github.com/aenix-io/talm/internal/pkg/meta" + "github.com/siderolabs/talos/pkg/argsbuilder" + "github.com/siderolabs/talos/pkg/conditions" + "github.com/siderolabs/talos/pkg/filetree" + "github.com/siderolabs/talos/pkg/logging" + machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + etcdresource "github.com/siderolabs/talos/pkg/machinery/resources/etcd" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + timeresource "github.com/siderolabs/talos/pkg/machinery/resources/time" +) + +var _ system.HealthcheckedService = (*Etcd)(nil) + +// Etcd implements the Service interface. It serves as the concrete type with +// the required methods. +type Etcd struct { + Bootstrap bool + RecoverFromSnapshot bool + RecoverSkipHashCheck bool + + args []string + client *etcd.Client + + imgRef string + + // if the new member was added as a learner during the service start, its ID is kept here + learnerMemberID uint64 + + promoteCtxCancel context.CancelFunc +} + +// ID implements the Service interface. +func (e *Etcd) ID(r runtime.Runtime) string { + return "etcd" +} + +// PreFunc implements the Service interface. +// +//nolint:gocyclo +func (e *Etcd) PreFunc(ctx context.Context, r runtime.Runtime) error { + if err := os.MkdirAll(constants.EtcdDataPath, 0o700); err != nil { + return err + } + + // Data path might exist after upgrade from previous version of Talos. + if err := os.Chmod(constants.EtcdDataPath, 0o700); err != nil { + return err + } + + // Make sure etcd user can access files in the data directory. + if err := filetree.ChownRecursive(constants.EtcdDataPath, constants.EtcdUserID, constants.EtcdUserID); err != nil { + return err + } + + client, err := containerdapi.New(constants.CRIContainerdAddress) + if err != nil { + return err + } + //nolint:errcheck + defer client.Close() + + // Pull the image and unpack it. + containerdctx := namespaces.WithNamespace(ctx, constants.SystemContainerdNamespace) + + spec, err := safe.ReaderGet[*etcdresource.Spec](ctx, r.State().V1Alpha2().Resources(), etcdresource.NewSpec(etcdresource.NamespaceName, etcdresource.SpecID).Metadata()) + if err != nil { + // spec should be ready + return fmt.Errorf("failed to get etcd spec: %w", err) + } + + img, err := image.Pull(containerdctx, r.Config().Machine().Registries(), client, spec.TypedSpec().Image, image.WithSkipIfAlreadyPulled()) + if err != nil { + return fmt.Errorf("failed to pull image %q: %w", spec.TypedSpec().Image, err) + } + + e.imgRef = img.Target().Digest.String() + + // Clear any previously set learner member ID + e.learnerMemberID = 0 + + switch t := r.Config().Machine().Type(); t { + case machine.TypeInit: + if err = e.argsForInit(ctx, r, spec.TypedSpec()); err != nil { + return err + } + case machine.TypeControlPlane: + if err = e.argsForControlPlane(ctx, r, spec.TypedSpec()); err != nil { + return err + } + case machine.TypeWorker: + return fmt.Errorf("unexpected machine type: %v", t) + case machine.TypeUnknown: + fallthrough + default: + panic(fmt.Sprintf("unexpected machine type %v", t)) + } + + if err = waitPKI(ctx, r); err != nil { + return fmt.Errorf("failed to generate etcd PKI: %w", err) + } + + return nil +} + +// PostFunc implements the Service interface. +func (e *Etcd) PostFunc(r runtime.Runtime, state events.ServiceState) (err error) { + if e.promoteCtxCancel != nil { + e.promoteCtxCancel() + } + + if e.client != nil { + e.client.Close() //nolint:errcheck + } + + e.client = nil + + return nil +} + +// Condition implements the Service interface. +func (e *Etcd) Condition(r runtime.Runtime) conditions.Condition { + return conditions.WaitForAll( + timeresource.NewSyncCondition(r.State().V1Alpha2().Resources()), + network.NewReadyCondition(r.State().V1Alpha2().Resources(), network.AddressReady, network.HostnameReady, network.EtcFilesReady), + etcdresource.NewSpecReadyCondition(r.State().V1Alpha2().Resources()), + ) +} + +// DependsOn implements the Service interface. +func (e *Etcd) DependsOn(r runtime.Runtime) []string { + return []string{"cri"} +} + +// Runner implements the Service interface. +func (e *Etcd) Runner(r runtime.Runtime) (runner.Runner, error) { + // Set the process arguments. + args := runner.Args{ + ID: e.ID(r), + ProcessArgs: append([]string{"/usr/local/bin/etcd"}, e.args...), + } + + mounts := []specs.Mount{ + {Type: "bind", Destination: constants.EtcdPKIPath, Source: constants.EtcdPKIPath, Options: []string{"rbind", "ro"}}, + {Type: "bind", Destination: constants.EtcdDataPath, Source: constants.EtcdDataPath, Options: []string{"rbind", "rw"}}, + } + + env := environment.Get(r.Config()) + + if goruntime.GOARCH == "arm64" { + env = append(env, "ETCD_UNSUPPORTED_ARCH=arm64") + } + + env = append(env, "ETCD_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305") //nolint:lll + + if e.learnerMemberID != 0 { + var promoteCtx context.Context + + promoteCtx, e.promoteCtxCancel = context.WithCancel(context.Background()) + + go func() { + if err := promoteMember(promoteCtx, r, e.learnerMemberID); err != nil && !errors.Is(err, context.Canceled) { + log.Printf("failed promoting member: %s", err) + } else if err == nil { + log.Printf("successfully promoted etcd member") + } + }() + } + + return restart.New(containerd.NewRunner( + r.Config().Debug(), + &args, + runner.WithLoggingManager(r.Logging()), + runner.WithNamespace(constants.SystemContainerdNamespace), + runner.WithContainerImage(e.imgRef), + runner.WithEnv(env), + runner.WithOCISpecOpts( + oci.WithDroppedCapabilities(cap.Known()), + oci.WithHostNamespace(specs.NetworkNamespace), + oci.WithMounts(mounts), + oci.WithUser(fmt.Sprintf("%d:%d", constants.EtcdUserID, constants.EtcdUserID)), + ), + runner.WithOOMScoreAdj(-998), + ), + restart.WithType(restart.Forever), + ), nil +} + +// HealthFunc implements the HealthcheckedService interface. +func (e *Etcd) HealthFunc(runtime.Runtime) health.Check { + return func(ctx context.Context) error { + if e.client == nil { + var err error + + e.client, err = etcd.NewLocalClient(ctx) + if err != nil { + return err + } + } + + return e.client.ValidateQuorum(ctx) + } +} + +// HealthSettings implements the HealthcheckedService interface. +func (e *Etcd) HealthSettings(runtime.Runtime) *health.Settings { + return &health.Settings{ + InitialDelay: 5 * time.Second, + Period: 20 * time.Second, + Timeout: 15 * time.Second, + } +} + +func waitPKI(ctx context.Context, r runtime.Runtime) error { + _, err := r.State().V1Alpha2().Resources().WatchFor(ctx, + resource.NewMetadata(etcdresource.NamespaceName, etcdresource.PKIStatusType, etcdresource.PKIID, resource.VersionUndefined), + state.WithEventTypes(state.Created, state.Updated), + ) + + return err +} + +func addMember(ctx context.Context, r runtime.Runtime, addrs []string, name string) (*clientv3.MemberListResponse, uint64, error) { + client, err := etcd.NewClientFromControlPlaneIPs(ctx, r.State().V1Alpha2().Resources()) + if err != nil { + return nil, 0, err + } + + //nolint:errcheck + defer client.Close() + + ctx = clientv3.WithRequireLeader(ctx) + + list, err := client.MemberList(ctx) + if err != nil { + return nil, 0, fmt.Errorf("error getting etcd member list: %w", err) + } + + for _, member := range list.Members { + // addMember only gets called when the etcd data directory is empty, so the node is about to join the etcd cluster + // if there's already a member with same hostname, it should be removed, as there will be a conflict between the existing + // member and a new joining member. + // here we assume that control plane nodes have unique hostnames (if that's not the case, it will be a problem anyways) + if member.Name == name { + if _, err = client.MemberRemove(ctx, member.ID); err != nil { + return nil, 0, fmt.Errorf("error removing self from the member list: %w", err) + } + } + } + + add, err := client.MemberAddAsLearner(ctx, addrs) + if err != nil { + return nil, 0, fmt.Errorf("error adding member: %w", err) + } + + list, err = client.MemberList(ctx) + if err != nil { + return nil, 0, fmt.Errorf("error getting second etcd member list: %w", err) + } + + return list, add.Member.ID, nil +} + +func buildInitialCluster(ctx context.Context, r runtime.Runtime, name string, peerAddrs []string) (initial string, learnerMemberID uint64, err error) { + var ( + id uint64 + lastNag time.Time + ) + + err = retry.Constant(constants.EtcdJoinTimeout, + retry.WithUnits(3*time.Second), + retry.WithJitter(time.Second), + retry.WithErrorLogging(true), + ).RetryWithContext(ctx, func(ctx context.Context) error { + var resp *clientv3.MemberListResponse + + if time.Since(lastNag) > 30*time.Second { + lastNag = time.Now() + + log.Printf("etcd is waiting to join the cluster, if this node is the first node in the cluster, please run `talosctl bootstrap` against one of the following IPs:") + + // we "allow" a failure here since we want to fallthrough and attempt to add the etcd member regardless of + // whether we can print our IPs + currentAddresses, addrErr := r.State().V1Alpha2().Resources().Get(ctx, + resource.NewMetadata(network.NamespaceName, network.NodeAddressType, network.FilteredNodeAddressID(network.NodeAddressCurrentID, k8s.NodeAddressFilterNoK8s), resource.VersionUndefined)) + if addrErr != nil { + log.Printf("error getting node addresses: %s", addrErr.Error()) + } else { + ips := currentAddresses.(*network.NodeAddress).TypedSpec().IPs() + log.Printf("%s", ips) + } + } + + attemptCtx, attemptCtxCancel := context.WithTimeout(ctx, 30*time.Second) + defer attemptCtxCancel() + + resp, id, err = addMember(attemptCtx, r, peerAddrs, name) + if err != nil { + if errors.Is(err, context.Canceled) { + return err + } + + // TODO(andrewrynhard): We should check the error type here and + // handle the specific error accordingly. + return retry.ExpectedError(err) + } + + conf := []string{} + + for _, memb := range resp.Members { + for _, u := range memb.PeerURLs { + n := memb.Name + if memb.ID == id { + n = name + } + + conf = append(conf, fmt.Sprintf("%s=%s", n, u)) + } + } + + initial = strings.Join(conf, ",") + + return nil + }) + if err != nil { + return "", 0, fmt.Errorf("failed to build cluster arguments: %w", err) + } + + return initial, id, nil +} + +func (e *Etcd) argsForInit(ctx context.Context, r runtime.Runtime, spec *etcdresource.SpecSpec) error { + var upgraded bool + + _, upgraded = r.State().Machine().Meta().ReadTag(meta.Upgrade) + + denyListArgs := argsbuilder.Args{ + "name": spec.Name, + "auto-tls": "false", + "peer-auto-tls": "false", + "data-dir": constants.EtcdDataPath, + "listen-peer-urls": formatEtcdURLs(spec.ListenPeerAddresses, constants.EtcdPeerPort), + "listen-client-urls": formatEtcdURLs(spec.ListenClientAddresses, constants.EtcdClientPort), + "client-cert-auth": "true", + "cert-file": constants.EtcdCert, + "key-file": constants.EtcdKey, + "trusted-ca-file": constants.EtcdCACert, + "peer-client-cert-auth": "true", + "peer-cert-file": constants.EtcdPeerCert, + "peer-key-file": constants.EtcdPeerKey, + "peer-trusted-ca-file": constants.EtcdCACert, + "experimental-initial-corrupt-check": "true", + "experimental-watch-progress-notify-interval": "5s", + "experimental-compact-hash-check-enabled": "true", + } + + extraArgs := argsbuilder.Args(spec.ExtraArgs) + + denyList := argsbuilder.WithDenyList(denyListArgs) + + if !extraArgs.Contains("initial-cluster-state") { + denyListArgs.Set("initial-cluster-state", "new") + } + + // If the initial cluster isn't explicitly defined, we need to discover any + // existing members. + if !extraArgs.Contains("initial-cluster") { + ok, err := IsDirEmpty(constants.EtcdDataPath) + if err != nil { + return err + } + + if ok { + initialCluster := formatClusterURLs(spec.Name, getEtcdURLs(spec.AdvertisedAddresses, constants.EtcdPeerPort)) + + if upgraded { + denyListArgs.Set("initial-cluster-state", "existing") + + initialCluster, e.learnerMemberID, err = buildInitialCluster(ctx, r, spec.Name, getEtcdURLs(spec.AdvertisedAddresses, constants.EtcdPeerPort)) + if err != nil { + return err + } + } + + denyListArgs.Set("initial-cluster", initialCluster) + } else { + denyListArgs.Set("initial-cluster-state", "existing") + } + } + + if !extraArgs.Contains("initial-advertise-peer-urls") { + denyListArgs.Set("initial-advertise-peer-urls", + formatEtcdURLs(spec.AdvertisedAddresses, constants.EtcdPeerPort), + ) + } + + if !extraArgs.Contains("advertise-client-urls") { + denyListArgs.Set("advertise-client-urls", + formatEtcdURLs(spec.AdvertisedAddresses, constants.EtcdClientPort), + ) + } + + if err := denyListArgs.Merge(extraArgs, denyList); err != nil { + return err + } + + e.args = denyListArgs.Args() + + return nil +} + +//nolint:gocyclo +func (e *Etcd) argsForControlPlane(ctx context.Context, r runtime.Runtime, spec *etcdresource.SpecSpec) error { + denyListArgs := argsbuilder.Args{ + "name": spec.Name, + "auto-tls": "false", + "peer-auto-tls": "false", + "data-dir": constants.EtcdDataPath, + "listen-peer-urls": formatEtcdURLs(spec.ListenPeerAddresses, constants.EtcdPeerPort), + "listen-client-urls": formatEtcdURLs(spec.ListenClientAddresses, constants.EtcdClientPort), + "client-cert-auth": "true", + "cert-file": constants.EtcdCert, + "key-file": constants.EtcdKey, + "trusted-ca-file": constants.EtcdCACert, + "peer-client-cert-auth": "true", + "peer-cert-file": constants.EtcdPeerCert, + "peer-key-file": constants.EtcdPeerKey, + "peer-trusted-ca-file": constants.EtcdCACert, + "experimental-initial-corrupt-check": "true", + "experimental-watch-progress-notify-interval": "5s", + "experimental-compact-hash-check-enabled": "true", + } + + extraArgs := argsbuilder.Args(spec.ExtraArgs) + + denyList := argsbuilder.WithDenyList(denyListArgs) + + if e.RecoverFromSnapshot { + if err := e.recoverFromSnapshot(spec); err != nil { + return err + } + } + + ok, err := IsDirEmpty(constants.EtcdDataPath) + if err != nil { + return err + } + + // The only time that we need to build the initial cluster args, is when we + // don't have any data. + if ok { + if !extraArgs.Contains("initial-cluster-state") { + if e.Bootstrap { + denyListArgs.Set("initial-cluster-state", "new") + } else { + denyListArgs.Set("initial-cluster-state", "existing") + } + } + + if !extraArgs.Contains("initial-cluster") { + var initialCluster string + + if e.Bootstrap { + initialCluster = formatClusterURLs(spec.Name, getEtcdURLs(spec.AdvertisedAddresses, constants.EtcdPeerPort)) + } else { + initialCluster, e.learnerMemberID, err = buildInitialCluster(ctx, r, spec.Name, getEtcdURLs(spec.AdvertisedAddresses, constants.EtcdPeerPort)) + if err != nil { + return fmt.Errorf("failed to build initial etcd cluster: %w", err) + } + } + + denyListArgs.Set("initial-cluster", initialCluster) + } + + if !extraArgs.Contains("initial-advertise-peer-urls") { + denyListArgs.Set("initial-advertise-peer-urls", + formatEtcdURLs(spec.AdvertisedAddresses, constants.EtcdPeerPort), + ) + } + } + + if !extraArgs.Contains("advertise-client-urls") { + denyListArgs.Set("advertise-client-urls", + formatEtcdURLs(spec.AdvertisedAddresses, constants.EtcdClientPort), + ) + } + + if err = denyListArgs.Merge(extraArgs, denyList); err != nil { + return err + } + + e.args = denyListArgs.Args() + + return nil +} + +// recoverFromSnapshot recovers etcd data directory from the snapshot uploaded previously. +func (e *Etcd) recoverFromSnapshot(spec *etcdresource.SpecSpec) error { + manager := snapshot.NewV3(logging.Wrap(log.Writer())) + + status, err := manager.Status(constants.EtcdRecoverySnapshotPath) + if err != nil { + return fmt.Errorf("error verifying snapshot: %w", err) + } + + log.Printf("recovering etcd from snapshot: hash %08x, revision %d, total keys %d, total size %d\n", + status.Hash, status.Revision, status.TotalKey, status.TotalSize) + + if err = manager.Restore(snapshot.RestoreConfig{ + SnapshotPath: constants.EtcdRecoverySnapshotPath, + + Name: spec.Name, + OutputDataDir: constants.EtcdDataPath, + + PeerURLs: getEtcdURLs(spec.AdvertisedAddresses, constants.EtcdPeerPort), + + InitialCluster: fmt.Sprintf("%s=%s", spec.Name, formatEtcdURLs(spec.AdvertisedAddresses, constants.EtcdPeerPort)), + + SkipHashCheck: e.RecoverSkipHashCheck, + }); err != nil { + return fmt.Errorf("error recovering from the snapshot: %w", err) + } + + if err = os.Remove(constants.EtcdRecoverySnapshotPath); err != nil { + return fmt.Errorf("error deleting snapshot: %w", err) + } + + return filetree.ChownRecursive(constants.EtcdDataPath, constants.EtcdUserID, constants.EtcdUserID) +} + +func promoteMember(ctx context.Context, r runtime.Runtime, memberID uint64) error { + // try to promote a member until it succeeds (call might fail until the member catches up with the leader) + // promote member call will fail until member catches up with the master + // + // iterate over all endpoints until we find the one which works + // if we stick with the default behavior, we might hit the member being promoted, and that will never + // promote itself. + idx := 0 + + return retry.Constant(10*time.Minute, + retry.WithUnits(15*time.Second), + retry.WithAttemptTimeout(30*time.Second), + retry.WithJitter(time.Second), + retry.WithErrorLogging(true), + ).RetryWithContext(ctx, func(ctx context.Context) error { + endpoints, err := etcd.GetEndpoints(ctx, r.State().V1Alpha2().Resources()) + if err != nil { + return retry.ExpectedError(err) + } + + if len(endpoints) == 0 { + return retry.ExpectedErrorf("no endpoints") + } + + // try to iterate all available endpoints in the time available for an attempt + for range len(endpoints) { + select { + case <-ctx.Done(): + return retry.ExpectedError(ctx.Err()) + default: + } + + endpoint := endpoints[idx%len(endpoints)] + idx++ + + err = attemptPromote(ctx, endpoint, memberID) + if err == nil { + return nil + } + } + + return retry.ExpectedError(err) + }) +} + +func attemptPromote(ctx context.Context, endpoint string, memberID uint64) error { + client, err := etcd.NewClient(ctx, []string{endpoint}, grpc.WithBlock()) + if err != nil { + return err + } + + defer client.Close() //nolint:errcheck + + _, err = client.MemberPromote(ctx, memberID) + + return err +} + +// IsDirEmpty checks if a directory is empty or not. +func IsDirEmpty(name string) (bool, error) { + f, err := os.Open(name) + if err != nil { + return false, err + } + //nolint:errcheck + defer f.Close() + + _, err = f.Readdirnames(1) + if err == io.EOF { + return true, nil + } + + return false, err +} + +// BootstrapEtcd bootstraps the etcd cluster. +// +// Current instance of etcd (not joined yet) is stopped, and new instance is started in bootstrap mode. +func BootstrapEtcd(ctx context.Context, r runtime.Runtime, req *machineapi.BootstrapRequest) error { + if err := system.Services(r).Stop(ctx, "etcd"); err != nil { + return fmt.Errorf("failed to stop etcd: %w", err) + } + + // This is hack. We need to fake a finished state so that we can get the + // wait in the boot sequence to unblock. + for _, svc := range system.Services(r).List() { + if svc.AsProto().GetId() == "etcd" { + svc.UpdateState(ctx, events.StateFinished, "Bootstrap requested") + + break + } + } + + if entries, _ := os.ReadDir(constants.EtcdDataPath); len(entries) > 0 { //nolint:errcheck + return errors.New("etcd data directory is not empty") + } + + svc := &Etcd{ + Bootstrap: true, + RecoverFromSnapshot: req.RecoverEtcd, + RecoverSkipHashCheck: req.RecoverSkipHashCheck, + } + + if err := system.Services(r).Unload(ctx, svc.ID(r)); err != nil { + return err + } + + system.Services(r).Load(svc) + + if err := system.Services(r).Start(svc.ID(r)); err != nil { + return fmt.Errorf("error starting etcd in bootstrap mode: %w", err) + } + + return nil +} + +func formatEtcdURL(addr netip.Addr, port int) string { + return fmt.Sprintf("https://%s", nethelpers.JoinHostPort(addr.String(), port)) +} + +func getEtcdURLs(addrs []netip.Addr, port int) []string { + return xslices.Map(addrs, func(addr netip.Addr) string { + return formatEtcdURL(addr, port) + }) +} + +func formatEtcdURLs(addrs []netip.Addr, port int) string { + return strings.Join(getEtcdURLs(addrs, port), ",") +} + +func formatClusterURLs(name string, urls []string) string { + return strings.Join(xslices.Map(urls, func(url string) string { + return fmt.Sprintf("%s=%s", name, url) + }), ",") +} diff --git a/internal/app/machined/pkg/system/services/export_test.go b/internal/app/machined/pkg/system/services/export_test.go new file mode 100644 index 0000000..bea149c --- /dev/null +++ b/internal/app/machined/pkg/system/services/export_test.go @@ -0,0 +1,17 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package services + +import "github.com/containerd/containerd/oci" + +// GetOCIOptions gets all OCI options from an Extension. +func (svc *Extension) GetOCIOptions() ([]oci.SpecOpts, error) { + envVars, err := svc.parseEnvironment() + if err != nil { + return nil, err + } + + return svc.getOCIOptions(envVars, svc.Spec.Container.Mounts), nil +} diff --git a/internal/app/machined/pkg/system/services/extension.go b/internal/app/machined/pkg/system/services/extension.go new file mode 100644 index 0000000..c42fcb1 --- /dev/null +++ b/internal/app/machined/pkg/system/services/extension.go @@ -0,0 +1,272 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package services + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/containerd/containerd/oci" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/hashicorp/go-envparse" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/siderolabs/gen/maps" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/events" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/containerd" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/restart" + "github.com/aenix-io/talm/internal/pkg/capability" + "github.com/aenix-io/talm/internal/pkg/environment" + "github.com/aenix-io/talm/internal/pkg/mount" + "github.com/siderolabs/talos/pkg/conditions" + "github.com/siderolabs/talos/pkg/machinery/constants" + extservices "github.com/siderolabs/talos/pkg/machinery/extensions/services" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/time" +) + +// Extension service is a generic wrapper around extension services spec. +type Extension struct { + Spec extservices.Spec + + overlay *mount.Point +} + +// ID implements the Service interface. +func (svc *Extension) ID(r runtime.Runtime) string { + return "ext-" + svc.Spec.Name +} + +// PreFunc implements the Service interface. +func (svc *Extension) PreFunc(ctx context.Context, r runtime.Runtime) error { + // re-mount service rootfs as overlay rw mount to allow containerd to mount there /dev, /proc, etc. + svc.overlay = mount.NewMountPoint( + "", + filepath.Join(constants.ExtensionServiceRootfsPath, svc.Spec.Name), + "", + 0, + "", + mount.WithFlags(mount.Overlay|mount.SystemOverlay), + ) + + return svc.overlay.Mount() +} + +// PostFunc implements the Service interface. +func (svc *Extension) PostFunc(r runtime.Runtime, state events.ServiceState) (err error) { + return svc.overlay.Unmount() +} + +// Condition implements the Service interface. +func (svc *Extension) Condition(r runtime.Runtime) conditions.Condition { + conds := []conditions.Condition{} + + if svc.Spec.Container.EnvironmentFile != "" { + // add a dependency on the environment file + conds = append(conds, conditions.WaitForFileToExist(svc.Spec.Container.EnvironmentFile)) + } + + for _, dep := range svc.Spec.Depends { + switch { + case dep.Path != "": + conds = append(conds, conditions.WaitForFileToExist(dep.Path)) + case len(dep.Network) > 0: + conds = append(conds, network.NewReadyCondition(r.State().V1Alpha2().Resources(), network.StatusChecksFromStatuses(dep.Network...)...)) + case dep.Time: + conds = append(conds, time.NewSyncCondition(r.State().V1Alpha2().Resources())) + case dep.Configuration: + conds = append(conds, runtimeres.NewExtensionServiceConfigStatusCondition(r.State().V1Alpha2().Resources(), svc.Spec.Name)) + } + } + + if len(conds) == 0 { + return nil + } + + return conditions.WaitForAll(conds...) +} + +// DependsOn implements the Service interface. +func (svc *Extension) DependsOn(r runtime.Runtime) []string { + deps := []string{"containerd"} + + for _, dep := range svc.Spec.Depends { + if dep.Service != "" { + deps = append(deps, dep.Service) + } + } + + return deps +} + +func (svc *Extension) getOCIOptions(envVars []string, mounts []specs.Mount) []oci.SpecOpts { + ociOpts := []oci.SpecOpts{ + oci.WithRootFSPath(filepath.Join(constants.ExtensionServiceRootfsPath, svc.Spec.Name)), + containerd.WithRootfsPropagation(svc.Spec.Container.Security.RootfsPropagation), + oci.WithCgroup(filepath.Join(constants.CgroupExtensions, svc.Spec.Name)), + oci.WithMounts(mounts), + oci.WithHostNamespace(specs.NetworkNamespace), + oci.WithSelinuxLabel(""), + oci.WithApparmorProfile(""), + oci.WithCapabilities(capability.AllGrantableCapabilities()), + oci.WithAllDevicesAllowed, + oci.WithEnv(envVars), + } + + if !svc.Spec.Container.Security.WriteableRootfs { + ociOpts = append(ociOpts, oci.WithRootFSReadonly()) + } + + if svc.Spec.Container.Security.WriteableSysfs { + ociOpts = append(ociOpts, oci.WithWriteableSysfs) + } + + if svc.Spec.Container.Security.MaskedPaths != nil { + ociOpts = append(ociOpts, oci.WithMaskedPaths(svc.Spec.Container.Security.MaskedPaths)) + } + + if svc.Spec.Container.Security.ReadonlyPaths != nil { + ociOpts = append(ociOpts, oci.WithReadonlyPaths(svc.Spec.Container.Security.ReadonlyPaths)) + } + + return ociOpts +} + +// Runner implements the Service interface. +// +//nolint:gocyclo +func (svc *Extension) Runner(r runtime.Runtime) (runner.Runner, error) { + args := runner.Args{ + ID: svc.ID(r), + ProcessArgs: append([]string{svc.Spec.Container.Entrypoint}, svc.Spec.Container.Args...), + } + + for _, mount := range svc.Spec.Container.Mounts { + if _, err := os.Stat(mount.Source); err == nil { + // already exists, skip + continue + } else if !errors.Is(err, os.ErrNotExist) { + return nil, err + } + + if err := os.MkdirAll(mount.Source, 0o700); err != nil { + return nil, err + } + } + + mounts := append([]specs.Mount{}, svc.Spec.Container.Mounts...) + + envVars, err := svc.parseEnvironment() + if err != nil { + return nil, err + } + + configSpec, err := safe.StateGetByID[*runtimeres.ExtensionServiceConfig](context.Background(), r.State().V1Alpha2().Resources(), svc.Spec.Name) + if err == nil { + spec := configSpec.TypedSpec() + + for _, ext := range spec.Files { + mounts = append(mounts, specs.Mount{ + Source: filepath.Join(constants.ExtensionServiceUserConfigPath, svc.Spec.Name, strings.ReplaceAll(strings.TrimPrefix(ext.MountPath, "/"), "/", "-")), + Destination: ext.MountPath, + Type: "bind", + Options: []string{"ro", "bind"}, + }) + } + + envVars = append(envVars, spec.Environment...) + } else if !state.IsNotFoundError(err) { + return nil, err + } + + var restartType restart.Type + + switch svc.Spec.Restart { + case extservices.RestartAlways: + restartType = restart.Forever + case extservices.RestartNever: + restartType = restart.Once + case extservices.RestartUntilSuccess: + restartType = restart.UntilSuccess + } + + ociSpecOpts := svc.getOCIOptions(envVars, mounts) + + debug := false + + if r.Config() != nil { + debug = r.Config().Debug() + } + + return restart.New(containerd.NewRunner( + debug, + &args, + runner.WithLoggingManager(r.Logging()), + runner.WithNamespace(constants.SystemContainerdNamespace), + runner.WithContainerdAddress(constants.SystemContainerdAddress), + runner.WithEnv(environment.Get(r.Config())), + runner.WithOCISpecOpts(ociSpecOpts...), + runner.WithOOMScoreAdj(-600), + ), + restart.WithType(restartType), + ), nil +} + +// APIRestartAllowed implements APIRestartableService. +func (svc *Extension) APIRestartAllowed(runtime.Runtime) bool { + return true +} + +// APIStartAllowed implements APIStartableService. +func (svc *Extension) APIStartAllowed(runtime.Runtime) bool { + return true +} + +// APIStopAllowed implements APIStoppableService. +func (svc *Extension) APIStopAllowed(runtime.Runtime) bool { + return true +} + +func (svc *Extension) parseEnvironment() ([]string, error) { + var envVars []string + + if svc.Spec.Container.EnvironmentFile != "" { + envFile, err := os.OpenFile(svc.Spec.Container.EnvironmentFile, os.O_RDONLY, 0) + if err != nil { + return nil, err + } + + defer func() { + if closeErr := envFile.Close(); err != nil { + err = closeErr + } + }() + + parsedEnvVars, err := envparse.Parse(envFile) + if err != nil { + return nil, fmt.Errorf("failed to parse environment file %q: %w", svc.Spec.Container.EnvironmentFile, err) + } + + envVarsSlice := maps.ToSlice(parsedEnvVars, func(k, v string) string { + return fmt.Sprintf("%s=%s", k, v) + }) + + envVars = append(envVars, envVarsSlice...) + } + + if svc.Spec.Container.Environment != nil { + envVars = append(envVars, svc.Spec.Container.Environment...) + } + + return envVars, nil +} diff --git a/internal/app/machined/pkg/system/services/extension_test.go b/internal/app/machined/pkg/system/services/extension_test.go new file mode 100644 index 0000000..e8ef319 --- /dev/null +++ b/internal/app/machined/pkg/system/services/extension_test.go @@ -0,0 +1,189 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package services_test + +import ( + "context" + "os" + "testing" + + "github.com/containerd/containerd/containers" + "github.com/containerd/containerd/namespaces" + "github.com/containerd/containerd/oci" + "github.com/containerd/containerd/snapshots" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + + "github.com/aenix-io/talm/internal/app/machined/pkg/system/services" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/services/mocks" + extservices "github.com/siderolabs/talos/pkg/machinery/extensions/services" +) + +type MockClient struct { + controller *gomock.Controller +} + +func (c *MockClient) SnapshotService(snapshotterName string) snapshots.Snapshotter { + return mocks.NewMockSnapshotter(c.controller) +} + +func TestGetOCIOptions(t *testing.T) { + mockClient := MockClient{ + controller: gomock.NewController(t), + } + defer mockClient.controller.Finish() + + generateOCISpec := func(svc *services.Extension) (*oci.Spec, error) { + ociOpts, err := svc.GetOCIOptions() + if err != nil { + return nil, err + } + + return oci.GenerateSpec(namespaces.WithNamespace(context.Background(), "testNamespace"), &mockClient, &containers.Container{}, ociOpts...) + } + + t.Run("default configurations are cleared away if user passes empty arrays for MaskedPaths and ReadonlyPaths", func(t *testing.T) { + // given + svc := &services.Extension{ + Spec: extservices.Spec{ + Container: extservices.Container{ + Security: extservices.Security{ + MaskedPaths: []string{}, + ReadonlyPaths: []string{}, + }, + }, + }, + } + + // when + spec, err := generateOCISpec(svc) + + // then + assert.NoError(t, err) + assert.Equal(t, []string{}, spec.Linux.MaskedPaths) + assert.Equal(t, []string{}, spec.Linux.ReadonlyPaths) + }) + + t.Run("default configuration applies if user passes nil for MaskedPaths and ReadonlyPaths", func(t *testing.T) { + // given + svc := &services.Extension{ + Spec: extservices.Spec{ + Container: extservices.Container{ + Security: extservices.Security{ + MaskedPaths: nil, + ReadonlyPaths: nil, + }, + }, + }, + } + + // when + spec, err := generateOCISpec(svc) + + // then + assert.NoError(t, err) + assert.Equal(t, []string{ + "/proc/acpi", + "/proc/asound", + "/proc/kcore", + "/proc/keys", + "/proc/latency_stats", + "/proc/timer_list", + "/proc/timer_stats", + "/proc/sched_debug", + "/sys/firmware", + "/sys/devices/virtual/powercap", + "/proc/scsi", + }, spec.Linux.MaskedPaths) + assert.Equal(t, []string{ + "/proc/bus", + "/proc/fs", + "/proc/irq", + "/proc/sys", + "/proc/sysrq-trigger", + }, spec.Linux.ReadonlyPaths) + }) + + t.Run("root fs is readonly unless explicitly enabled", func(t *testing.T) { + // given + svc := &services.Extension{ + Spec: extservices.Spec{ + Container: extservices.Container{ + Security: extservices.Security{ + WriteableRootfs: true, + }, + }, + }, + } + + // when + spec, err := generateOCISpec(svc) + + // then + assert.NoError(t, err) + assert.Equal(t, false, spec.Root.Readonly) + }) + + t.Run("root fs is readonly by default", func(t *testing.T) { + // given + svc := &services.Extension{ + Spec: extservices.Spec{ + Container: extservices.Container{ + Security: extservices.Security{}, + }, + }, + } + + // when + spec, err := generateOCISpec(svc) + + // then + assert.NoError(t, err) + assert.Equal(t, true, spec.Root.Readonly) + }) + + t.Run("allows setting extra env vars", func(t *testing.T) { + // given + svc := &services.Extension{ + Spec: extservices.Spec{ + Container: extservices.Container{ + Environment: []string{ + "FOO=BAR", + }, + }, + }, + } + + // when + spec, err := generateOCISpec(svc) + + // then + assert.NoError(t, err) + assert.Equal(t, []string{"FOO=BAR"}, spec.Process.Env) + }) + + t.Run("allows setting extra envFile", func(t *testing.T) { + tempDir := t.TempDir() + envFile := tempDir + "/envfile" + + assert.NoError(t, os.WriteFile(envFile, []byte("FOO=BARFROMENVFILE"), 0o644)) + + // given + svc := &services.Extension{ + Spec: extservices.Spec{ + Container: extservices.Container{ + EnvironmentFile: envFile, + }, + }, + } + + // when + spec, err := generateOCISpec(svc) + + // then + assert.NoError(t, err) + assert.Equal(t, []string{"FOO=BARFROMENVFILE"}, spec.Process.Env) + }) +} diff --git a/internal/app/machined/pkg/system/services/kubelet.go b/internal/app/machined/pkg/system/services/kubelet.go new file mode 100644 index 0000000..3667c83 --- /dev/null +++ b/internal/app/machined/pkg/system/services/kubelet.go @@ -0,0 +1,237 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package services + +import ( + "context" + "fmt" + "net/http" + "os" + "time" + + containerdapi "github.com/containerd/containerd" + "github.com/containerd/containerd/namespaces" + "github.com/containerd/containerd/oci" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + specs "github.com/opencontainers/runtime-spec/specs-go" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/system" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/events" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/health" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/containerd" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/restart" + "github.com/aenix-io/talm/internal/pkg/capability" + "github.com/aenix-io/talm/internal/pkg/containers/image" + "github.com/aenix-io/talm/internal/pkg/environment" + "github.com/siderolabs/talos/pkg/conditions" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + timeresource "github.com/siderolabs/talos/pkg/machinery/resources/time" +) + +var _ system.HealthcheckedService = (*Kubelet)(nil) + +// Kubelet implements the Service interface. It serves as the concrete type with +// the required methods. +type Kubelet struct { + imgRef string +} + +// ID implements the Service interface. +func (k *Kubelet) ID(r runtime.Runtime) string { + return "kubelet" +} + +// PreFunc implements the Service interface. +func (k *Kubelet) PreFunc(ctx context.Context, r runtime.Runtime) error { + specResource, err := r.State().V1Alpha2().Resources().Get(ctx, resource.NewMetadata(k8s.NamespaceName, k8s.KubeletSpecType, k8s.KubeletID, resource.VersionUndefined)) + if err != nil { + return err + } + + spec := specResource.(*k8s.KubeletSpec).TypedSpec() + + client, err := containerdapi.New(constants.CRIContainerdAddress) + if err != nil { + return err + } + //nolint:errcheck + defer client.Close() + + // Pull the image and unpack it. + containerdctx := namespaces.WithNamespace(ctx, constants.SystemContainerdNamespace) + + img, err := image.Pull(containerdctx, r.Config().Machine().Registries(), client, spec.Image, image.WithSkipIfAlreadyPulled()) + if err != nil { + return err + } + + k.imgRef = img.Target().Digest.String() + + // Create lifecycle resource to signal that the kubelet is about to start. + err = r.State().V1Alpha2().Resources().Create(ctx, k8s.NewKubeletLifecycle(k8s.NamespaceName, k8s.KubeletLifecycleID)) + if err != nil && !state.IsConflictError(err) { // ignore if the lifecycle resource already exists + return err + } + + return nil +} + +// PostFunc implements the Service interface. +func (k *Kubelet) PostFunc(r runtime.Runtime, state events.ServiceState) (err error) { + return nil +} + +// Condition implements the Service interface. +func (k *Kubelet) Condition(r runtime.Runtime) conditions.Condition { + return conditions.WaitForAll( + timeresource.NewSyncCondition(r.State().V1Alpha2().Resources()), + network.NewReadyCondition(r.State().V1Alpha2().Resources(), network.AddressReady, network.HostnameReady, network.EtcFilesReady), + ) +} + +// DependsOn implements the Service interface. +func (k *Kubelet) DependsOn(r runtime.Runtime) []string { + return []string{"cri"} +} + +// Runner implements the Service interface. +func (k *Kubelet) Runner(r runtime.Runtime) (runner.Runner, error) { + specResource, err := r.State().V1Alpha2().Resources().Get(context.Background(), resource.NewMetadata(k8s.NamespaceName, k8s.KubeletSpecType, k8s.KubeletID, resource.VersionUndefined)) + if err != nil { + return nil, err + } + + spec := specResource.(*k8s.KubeletSpec).TypedSpec() + + // Set the process arguments. + args := runner.Args{ + ID: k.ID(r), + ProcessArgs: append([]string{"/usr/local/bin/kubelet"}, spec.Args...), + } + // Set the required kubelet mounts. + mounts := []specs.Mount{ + {Type: "bind", Destination: "/dev", Source: "/dev", Options: []string{"rbind", "rshared", "rw"}}, + {Type: "sysfs", Destination: "/sys", Source: "/sys", Options: []string{"bind", "ro"}}, + {Type: "bind", Destination: constants.CgroupMountPath, Source: constants.CgroupMountPath, Options: []string{"rbind", "rshared", "rw"}}, + {Type: "bind", Destination: "/lib/modules", Source: "/lib/modules", Options: []string{"bind", "ro"}}, + {Type: "bind", Destination: "/etc/kubernetes", Source: "/etc/kubernetes", Options: []string{"bind", "rshared", "rw"}}, + {Type: "bind", Destination: constants.KubeletCredentialProviderBinDir, Source: constants.KubeletCredentialProviderBinDir, Options: []string{"bind", "ro"}}, + {Type: "bind", Destination: "/etc/nfsmount.conf", Source: "/etc/nfsmount.conf", Options: []string{"bind", "ro"}}, + {Type: "bind", Destination: "/etc/machine-id", Source: "/etc/machine-id", Options: []string{"bind", "ro"}}, + {Type: "bind", Destination: "/etc/os-release", Source: "/etc/os-release", Options: []string{"bind", "ro"}}, + {Type: "bind", Destination: constants.PodResolvConfPath, Source: constants.PodResolvConfPath, Options: []string{"bind", "ro"}}, + {Type: "bind", Destination: "/etc/cni", Source: "/etc/cni", Options: []string{"rbind", "rshared", "rw"}}, + {Type: "bind", Destination: "/usr/libexec/kubernetes", Source: "/usr/libexec/kubernetes", Options: []string{"rbind", "rshared", "rw"}}, + {Type: "bind", Destination: "/var/run", Source: "/run", Options: []string{"rbind", "rshared", "rw"}}, + {Type: "bind", Destination: "/var/lib/containerd", Source: "/var/lib/containerd", Options: []string{"rbind", "rshared", "rw"}}, + {Type: "bind", Destination: "/var/lib/kubelet", Source: "/var/lib/kubelet", Options: []string{"rbind", "rshared", "rw"}}, + {Type: "bind", Destination: "/var/log/containers", Source: "/var/log/containers", Options: []string{"rbind", "rshared", "rw"}}, + {Type: "bind", Destination: "/var/log/pods", Source: "/var/log/pods", Options: []string{"rbind", "rshared", "rw"}}, + } + + // Add extra mounts. + // TODO(andrewrynhard): We should verify that the mount source is + // allowlisted. There is the potential that a user can expose + // sensitive information. + for _, mount := range spec.ExtraMounts { + if err = os.MkdirAll(mount.Source, 0o700); err != nil { + return nil, err + } + + mounts = append(mounts, mount) + } + + return restart.New(containerd.NewRunner( + r.Config().Debug() && r.Config().Machine().Type() == machine.TypeWorker, // enable debug logs only for the worker nodes + &args, + runner.WithLoggingManager(r.Logging()), + runner.WithNamespace(constants.SystemContainerdNamespace), + runner.WithContainerImage(k.imgRef), + runner.WithEnv(environment.Get(r.Config())), + runner.WithOCISpecOpts( + containerd.WithRootfsPropagation("shared"), + oci.WithCgroup(constants.CgroupKubelet), + oci.WithMounts(mounts), + oci.WithHostNamespace(specs.NetworkNamespace), + oci.WithHostNamespace(specs.PIDNamespace), + oci.WithParentCgroupDevices, + oci.WithMaskedPaths(nil), + oci.WithReadonlyPaths(nil), + oci.WithWriteableSysfs, + oci.WithWriteableCgroupfs, + oci.WithSelinuxLabel(""), + oci.WithApparmorProfile(""), + oci.WithAllDevicesAllowed, + oci.WithCapabilities(capability.AllGrantableCapabilities()), // TODO: kubelet doesn't need all of these, we should consider limiting capabilities + ), + runner.WithOOMScoreAdj(constants.KubeletOOMScoreAdj), + runner.WithCustomSeccompProfile(kubeletSeccomp), + ), + restart.WithType(restart.Forever), + ), nil +} + +// HealthFunc implements the HealthcheckedService interface. +func (k *Kubelet) HealthFunc(runtime.Runtime) health.Check { + return func(ctx context.Context) error { + req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1:10248/healthz", nil) + if err != nil { + return err + } + + req = req.WithContext(ctx) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + //nolint:errcheck + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("expected HTTP status OK, got %s", resp.Status) + } + + return nil + } +} + +// HealthSettings implements the HealthcheckedService interface. +func (k *Kubelet) HealthSettings(runtime.Runtime) *health.Settings { + settings := health.DefaultSettings + settings.InitialDelay = 2 * time.Second // increase initial delay as kubelet is slow on startup + + return &settings +} + +// APIRestartAllowed implements APIRestartableService. +func (k *Kubelet) APIRestartAllowed(runtime.Runtime) bool { + return true +} + +// APIStartAllowed implements APIStartableService. +func (k *Kubelet) APIStartAllowed(runtime.Runtime) bool { + return true +} + +func kubeletSeccomp(seccomp *specs.LinuxSeccomp) { + // for cephfs mounts + seccomp.Syscalls = append(seccomp.Syscalls, + specs.LinuxSyscall{ + Names: []string{ + "add_key", + "request_key", + }, + Action: specs.ActAllow, + Args: []specs.LinuxSeccompArg{}, + }, + ) +} diff --git a/internal/app/machined/pkg/system/services/machined.go b/internal/app/machined/pkg/system/services/machined.go new file mode 100644 index 0000000..45a4cf9 --- /dev/null +++ b/internal/app/machined/pkg/system/services/machined.go @@ -0,0 +1,240 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package services + +import ( + "context" + "io" + "log" + "net" + "os" + "path/filepath" + "time" + + "github.com/siderolabs/go-debug" + "google.golang.org/grpc" + + v1alpha1server "github.com/aenix-io/talm/internal/app/machined/internal/server/v1alpha1" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/system" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/events" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/health" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/goroutine" + "github.com/siderolabs/talos/pkg/conditions" + "github.com/siderolabs/talos/pkg/grpc/factory" + "github.com/siderolabs/talos/pkg/grpc/middleware/authz" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/role" +) + +const machinedServiceID = "machined" + +var rules = map[string]role.Set{ + "/cluster.ClusterService/HealthCheck": role.MakeSet(role.Admin, role.Operator, role.Reader), + + "/inspect.InspectService/ControllerRuntimeDependencies": role.MakeSet(role.Admin, role.Operator, role.Reader), + + "/machine.MachineService/ApplyConfiguration": role.MakeSet(role.Admin), + "/machine.MachineService/Bootstrap": role.MakeSet(role.Admin), + "/machine.MachineService/CPUInfo": role.MakeSet(role.Admin, role.Operator, role.Reader), + "/machine.MachineService/Containers": role.MakeSet(role.Admin, role.Operator, role.Reader), + "/machine.MachineService/Copy": role.MakeSet(role.Admin), + "/machine.MachineService/DiskStats": role.MakeSet(role.Admin, role.Operator, role.Reader), + "/machine.MachineService/DiskUsage": role.MakeSet(role.Admin, role.Operator, role.Reader), + "/machine.MachineService/Dmesg": role.MakeSet(role.Admin, role.Operator, role.Reader), + "/machine.MachineService/EtcdAlarmList": role.MakeSet(role.Admin, role.Operator), + "/machine.MachineService/EtcdAlarmDisarm": role.MakeSet(role.Admin, role.Operator), + "/machine.MachineService/EtcdDefragment": role.MakeSet(role.Admin, role.Operator), + "/machine.MachineService/EtcdForfeitLeadership": role.MakeSet(role.Admin), + "/machine.MachineService/EtcdLeaveCluster": role.MakeSet(role.Admin), + "/machine.MachineService/EtcdMemberList": role.MakeSet(role.Admin, role.Operator, role.Reader), + "/machine.MachineService/EtcdRecover": role.MakeSet(role.Admin), + "/machine.MachineService/EtcdRemoveMemberByID": role.MakeSet(role.Admin), + "/machine.MachineService/EtcdSnapshot": role.MakeSet(role.Admin, role.Operator, role.EtcdBackup), + "/machine.MachineService/EtcdStatus": role.MakeSet(role.Admin, role.Operator), + "/machine.MachineService/Events": role.MakeSet(role.Admin, role.Operator, role.Reader), + "/machine.MachineService/GenerateClientConfiguration": role.MakeSet(role.Admin), + "/machine.MachineService/GenerateConfiguration": role.MakeSet(role.Admin), + "/machine.MachineService/Hostname": role.MakeSet(role.Admin, role.Operator, role.Reader), + "/machine.MachineService/ImageList": role.MakeSet(role.Admin, role.Operator, role.Reader), + "/machine.MachineService/ImagePull": role.MakeSet(role.Admin, role.Operator), + "/machine.MachineService/Kubeconfig": role.MakeSet(role.Admin), + "/machine.MachineService/List": role.MakeSet(role.Admin, role.Operator, role.Reader), + "/machine.MachineService/LoadAvg": role.MakeSet(role.Admin, role.Operator, role.Reader), + "/machine.MachineService/Logs": role.MakeSet(role.Admin, role.Operator, role.Reader), + "/machine.MachineService/LogsContainers": role.MakeSet(role.Admin, role.Operator, role.Reader), + "/machine.MachineService/Memory": role.MakeSet(role.Admin, role.Operator, role.Reader), + "/machine.MachineService/MetaWrite": role.MakeSet(role.Admin), + "/machine.MachineService/MetaDelete": role.MakeSet(role.Admin), + "/machine.MachineService/Mounts": role.MakeSet(role.Admin, role.Operator, role.Reader), + "/machine.MachineService/NetworkDeviceStats": role.MakeSet(role.Admin, role.Operator, role.Reader), + "/machine.MachineService/Netstat": role.MakeSet(role.Admin, role.Operator, role.Reader), + "/machine.MachineService/PacketCapture": role.MakeSet(role.Admin, role.Operator), + "/machine.MachineService/Processes": role.MakeSet(role.Admin, role.Operator, role.Reader), + "/machine.MachineService/Read": role.MakeSet(role.Admin), + "/machine.MachineService/Reboot": role.MakeSet(role.Admin, role.Operator), + "/machine.MachineService/Reset": role.MakeSet(role.Admin), + "/machine.MachineService/Restart": role.MakeSet(role.Admin, role.Operator), + "/machine.MachineService/Rollback": role.MakeSet(role.Admin), + "/machine.MachineService/ServiceList": role.MakeSet(role.Admin, role.Operator, role.Reader), + "/machine.MachineService/ServiceRestart": role.MakeSet(role.Admin, role.Operator), + "/machine.MachineService/ServiceStart": role.MakeSet(role.Admin, role.Operator), + "/machine.MachineService/ServiceStop": role.MakeSet(role.Admin, role.Operator), + "/machine.MachineService/Shutdown": role.MakeSet(role.Admin, role.Operator), + "/machine.MachineService/Stats": role.MakeSet(role.Admin, role.Operator, role.Reader), + "/machine.MachineService/SystemStat": role.MakeSet(role.Admin, role.Operator, role.Reader), + "/machine.MachineService/Upgrade": role.MakeSet(role.Admin), + "/machine.MachineService/Version": role.MakeSet(role.Admin, role.Operator, role.Reader), + + // per-type authorization is handled by the service itself + "/cosi.resource.State/Create": role.MakeSet(role.Admin), + "/cosi.resource.State/Destroy": role.MakeSet(role.Admin), + "/cosi.resource.State/Get": role.MakeSet(role.Admin, role.Operator, role.Reader), + "/cosi.resource.State/List": role.MakeSet(role.Admin, role.Operator, role.Reader), + "/cosi.resource.State/Update": role.MakeSet(role.Admin), + "/cosi.resource.State/Watch": role.MakeSet(role.Admin, role.Operator, role.Reader), + + "/storage.StorageService/Disks": role.MakeSet(role.Admin, role.Operator, role.Reader), + + "/time.TimeService/Time": role.MakeSet(role.Admin, role.Operator, role.Reader), + "/time.TimeService/TimeCheck": role.MakeSet(role.Admin, role.Operator, role.Reader), +} + +type machinedService struct { + c runtime.Controller +} + +// Main is an entrypoint to the API service. +func (s *machinedService) Main(ctx context.Context, r runtime.Runtime, logWriter io.Writer) error { + injector := &authz.Injector{ + Mode: authz.MetadataOnly, + } + + if debug.Enabled { + injector.Logger = log.New(logWriter, "machined/authz/injector ", log.Flags()).Printf + } + + authorizer := &authz.Authorizer{ + Rules: rules, + FallbackRoles: role.MakeSet(role.Admin), + Logger: log.New(logWriter, "machined/authz/authorizer ", log.Flags()).Printf, + } + + // Start the API server. + server := factory.NewServer( //nolint:contextcheck + &v1alpha1server.Server{ + Controller: s.c, + // breaking the import loop cycle between services/ package and v1alpha1_server.go + EtcdBootstrapper: BootstrapEtcd, + + ShutdownCtx: ctx, + }, + factory.WithLog("machined ", logWriter), + + factory.ServerOptions( + grpc.MaxRecvMsgSize(constants.GRPCMaxMessageSize), + ), + + factory.WithUnaryInterceptor(injector.UnaryInterceptor()), + factory.WithStreamInterceptor(injector.StreamInterceptor()), //nolint:contextcheck + + factory.WithUnaryInterceptor(authorizer.UnaryInterceptor()), + factory.WithStreamInterceptor(authorizer.StreamInterceptor()), //nolint:contextcheck + ) + + // ensure socket dir exists + if err := os.MkdirAll(filepath.Dir(constants.MachineSocketPath), 0o770); err != nil { + return err + } + + // set the final leaf to be world-executable to make apid connect to the socket + if err := os.Chmod(filepath.Dir(constants.MachineSocketPath), 0o771); err != nil { + return err + } + + listener, err := factory.NewListener(factory.Network("unix"), factory.SocketPath(constants.MachineSocketPath)) //nolint:contextcheck + if err != nil { + return err + } + + // chown the socket path to make it accessible to the apid + if err := os.Chown(constants.MachineSocketPath, constants.ApidUserID, constants.ApidUserID); err != nil { + return err + } + + go func() { + //nolint:errcheck + server.Serve(listener) + }() + + <-ctx.Done() + + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second) + defer shutdownCancel() + + factory.ServerGracefulStop(server, shutdownCtx) //nolint:contextcheck + + return nil +} + +var _ system.HealthcheckedService = (*Machined)(nil) + +// Machined implements the Service interface. It serves as the concrete type with +// the required methods. +type Machined struct { + Controller runtime.Controller +} + +// ID implements the Service interface. +func (m *Machined) ID(r runtime.Runtime) string { + return machinedServiceID +} + +// PreFunc implements the Service interface. +func (m *Machined) PreFunc(ctx context.Context, r runtime.Runtime) error { + return nil +} + +// PostFunc implements the Service interface. +func (m *Machined) PostFunc(r runtime.Runtime, state events.ServiceState) (err error) { + return nil +} + +// Condition implements the Service interface. +func (m *Machined) Condition(r runtime.Runtime) conditions.Condition { + return nil +} + +// DependsOn implements the Service interface. +func (m *Machined) DependsOn(r runtime.Runtime) []string { + return nil +} + +// Runner implements the Service interface. +func (m *Machined) Runner(r runtime.Runtime) (runner.Runner, error) { + svc := &machinedService{m.Controller} + + return goroutine.NewRunner(r, machinedServiceID, svc.Main, runner.WithLoggingManager(r.Logging())), nil +} + +// HealthFunc implements the HealthcheckedService interface. +func (m *Machined) HealthFunc(runtime.Runtime) health.Check { + return func(ctx context.Context) error { + var d net.Dialer + + conn, err := d.DialContext(ctx, "unix", constants.MachineSocketPath) + if err != nil { + return err + } + + return conn.Close() + } +} + +// HealthSettings implements the HealthcheckedService interface. +func (m *Machined) HealthSettings(runtime.Runtime) *health.Settings { + return &health.DefaultSettings +} diff --git a/internal/app/machined/pkg/system/services/machined_test.go b/internal/app/machined/pkg/system/services/machined_test.go new file mode 100644 index 0000000..6b60fc4 --- /dev/null +++ b/internal/app/machined/pkg/system/services/machined_test.go @@ -0,0 +1,75 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package services //nolint:testpackage // to test unexported variable + +import ( + "fmt" + "testing" + + cosi "github.com/cosi-project/runtime/api/v1alpha1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + + "github.com/siderolabs/talos/pkg/machinery/api/cluster" + "github.com/siderolabs/talos/pkg/machinery/api/inspect" + "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/api/storage" + "github.com/siderolabs/talos/pkg/machinery/api/time" +) + +func collectMethods(t *testing.T) map[string]struct{} { + methods := make(map[string]struct{}) + + for _, service := range []grpc.ServiceDesc{ + cosi.State_ServiceDesc, + cluster.ClusterService_ServiceDesc, + inspect.InspectService_ServiceDesc, + machine.MachineService_ServiceDesc, + // security.SecurityService_ServiceDesc, - not in machined + storage.StorageService_ServiceDesc, + time.TimeService_ServiceDesc, + } { + for _, method := range service.Methods { + s := fmt.Sprintf("/%s/%s", service.ServiceName, method.MethodName) + require.NotContains(t, methods, s) + methods[s] = struct{}{} + } + + for _, stream := range service.Streams { + s := fmt.Sprintf("/%s/%s", service.ServiceName, stream.StreamName) + require.NotContains(t, methods, s) + methods[s] = struct{}{} + } + } + + return methods +} + +func TestRules(t *testing.T) { + t.Parallel() + + methods := collectMethods(t) + + // check that there are no rules without matching methods + t.Run("NoMethodForRule", func(t *testing.T) { + t.Parallel() + + for rule := range rules { + _, ok := methods[rule] + assert.True(t, ok, "no method for rule %q", rule) + } + }) + + // check that there are no methods without matching rules + t.Run("NoRuleForMethod", func(t *testing.T) { + t.Parallel() + + for method := range methods { + _, ok := rules[method] + assert.True(t, ok, "no rule for method %q", method) + } + }) +} diff --git a/internal/app/machined/pkg/system/services/mocks/snapshotter.go b/internal/app/machined/pkg/system/services/mocks/snapshotter.go new file mode 100644 index 0000000..093ba34 --- /dev/null +++ b/internal/app/machined/pkg/system/services/mocks/snapshotter.go @@ -0,0 +1,249 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Code generated by MockGen. DO NOT EDIT. +// Source: ~/go/pkg/mod/github.com/containerd/containerd@v1.6.4/snapshots/snapshotter.go + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + mount "github.com/containerd/containerd/mount" + snapshots "github.com/containerd/containerd/snapshots" + gomock "github.com/golang/mock/gomock" +) + +// MockSnapshotter is a mock of Snapshotter interface. +type MockSnapshotter struct { + ctrl *gomock.Controller + recorder *MockSnapshotterMockRecorder +} + +// MockSnapshotterMockRecorder is the mock recorder for MockSnapshotter. +type MockSnapshotterMockRecorder struct { + mock *MockSnapshotter +} + +// NewMockSnapshotter creates a new mock instance. +func NewMockSnapshotter(ctrl *gomock.Controller) *MockSnapshotter { + mock := &MockSnapshotter{ctrl: ctrl} + mock.recorder = &MockSnapshotterMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSnapshotter) EXPECT() *MockSnapshotterMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockSnapshotter) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockSnapshotterMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockSnapshotter)(nil).Close)) +} + +// Commit mocks base method. +func (m *MockSnapshotter) Commit(ctx context.Context, name, key string, opts ...snapshots.Opt) error { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, name, key} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Commit", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// Commit indicates an expected call of Commit. +func (mr *MockSnapshotterMockRecorder) Commit(ctx, name, key interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, name, key}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Commit", reflect.TypeOf((*MockSnapshotter)(nil).Commit), varargs...) +} + +// Mounts mocks base method. +func (m *MockSnapshotter) Mounts(ctx context.Context, key string) ([]mount.Mount, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Mounts", ctx, key) + ret0, _ := ret[0].([]mount.Mount) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Mounts indicates an expected call of Mounts. +func (mr *MockSnapshotterMockRecorder) Mounts(ctx, key interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Mounts", reflect.TypeOf((*MockSnapshotter)(nil).Mounts), ctx, key) +} + +// Prepare mocks base method. +func (m *MockSnapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, key, parent} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Prepare", varargs...) + ret0, _ := ret[0].([]mount.Mount) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Prepare indicates an expected call of Prepare. +func (mr *MockSnapshotterMockRecorder) Prepare(ctx, key, parent interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, key, parent}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Prepare", reflect.TypeOf((*MockSnapshotter)(nil).Prepare), varargs...) +} + +// Remove mocks base method. +func (m *MockSnapshotter) Remove(ctx context.Context, key string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Remove", ctx, key) + ret0, _ := ret[0].(error) + return ret0 +} + +// Remove indicates an expected call of Remove. +func (mr *MockSnapshotterMockRecorder) Remove(ctx, key interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Remove", reflect.TypeOf((*MockSnapshotter)(nil).Remove), ctx, key) +} + +// Stat mocks base method. +func (m *MockSnapshotter) Stat(ctx context.Context, key string) (snapshots.Info, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Stat", ctx, key) + ret0, _ := ret[0].(snapshots.Info) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Stat indicates an expected call of Stat. +func (mr *MockSnapshotterMockRecorder) Stat(ctx, key interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stat", reflect.TypeOf((*MockSnapshotter)(nil).Stat), ctx, key) +} + +// Update mocks base method. +func (m *MockSnapshotter) Update(ctx context.Context, info snapshots.Info, fieldpaths ...string) (snapshots.Info, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, info} + for _, a := range fieldpaths { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Update", varargs...) + ret0, _ := ret[0].(snapshots.Info) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Update indicates an expected call of Update. +func (mr *MockSnapshotterMockRecorder) Update(ctx, info interface{}, fieldpaths ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, info}, fieldpaths...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockSnapshotter)(nil).Update), varargs...) +} + +// Usage mocks base method. +func (m *MockSnapshotter) Usage(ctx context.Context, key string) (snapshots.Usage, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Usage", ctx, key) + ret0, _ := ret[0].(snapshots.Usage) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Usage indicates an expected call of Usage. +func (mr *MockSnapshotterMockRecorder) Usage(ctx, key interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Usage", reflect.TypeOf((*MockSnapshotter)(nil).Usage), ctx, key) +} + +// View mocks base method. +func (m *MockSnapshotter) View(ctx context.Context, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, key, parent} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "View", varargs...) + ret0, _ := ret[0].([]mount.Mount) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// View indicates an expected call of View. +func (mr *MockSnapshotterMockRecorder) View(ctx, key, parent interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, key, parent}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "View", reflect.TypeOf((*MockSnapshotter)(nil).View), varargs...) +} + +// Walk mocks base method. +func (m *MockSnapshotter) Walk(ctx context.Context, fn snapshots.WalkFunc, filters ...string) error { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, fn} + for _, a := range filters { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Walk", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// Walk indicates an expected call of Walk. +func (mr *MockSnapshotterMockRecorder) Walk(ctx, fn interface{}, filters ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, fn}, filters...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Walk", reflect.TypeOf((*MockSnapshotter)(nil).Walk), varargs...) +} + +// MockCleaner is a mock of Cleaner interface. +type MockCleaner struct { + ctrl *gomock.Controller + recorder *MockCleanerMockRecorder +} + +// MockCleanerMockRecorder is the mock recorder for MockCleaner. +type MockCleanerMockRecorder struct { + mock *MockCleaner +} + +// NewMockCleaner creates a new mock instance. +func NewMockCleaner(ctrl *gomock.Controller) *MockCleaner { + mock := &MockCleaner{ctrl: ctrl} + mock.recorder = &MockCleanerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockCleaner) EXPECT() *MockCleanerMockRecorder { + return m.recorder +} + +// Cleanup mocks base method. +func (m *MockCleaner) Cleanup(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Cleanup", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// Cleanup indicates an expected call of Cleanup. +func (mr *MockCleanerMockRecorder) Cleanup(ctx interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Cleanup", reflect.TypeOf((*MockCleaner)(nil).Cleanup), ctx) +} diff --git a/internal/app/machined/pkg/system/services/syslogd.go b/internal/app/machined/pkg/system/services/syslogd.go new file mode 100644 index 0000000..3a04bee --- /dev/null +++ b/internal/app/machined/pkg/system/services/syslogd.go @@ -0,0 +1,68 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package services + +import ( + "context" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/system" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/events" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/health" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/goroutine" + "github.com/aenix-io/talm/internal/app/syslogd" + "github.com/siderolabs/talos/pkg/conditions" +) + +const syslogServiceID = "syslogd" + +var _ system.HealthcheckedService = (*Syslogd)(nil) + +// Syslogd implements the Service interface. It serves as the concrete type with +// the required methods. +type Syslogd struct{} + +// ID implements the Service interface. +func (s *Syslogd) ID(r runtime.Runtime) string { + return syslogServiceID +} + +// PreFunc implements the Service interface. +func (s *Syslogd) PreFunc(ctx context.Context, r runtime.Runtime) error { + return nil +} + +// PostFunc implements the Service interface. +func (s *Syslogd) PostFunc(r runtime.Runtime, state events.ServiceState) (err error) { + return nil +} + +// Condition implements the Service interface. +func (s *Syslogd) Condition(r runtime.Runtime) conditions.Condition { + return nil +} + +// DependsOn implements the Service interface. +func (s *Syslogd) DependsOn(r runtime.Runtime) []string { + return []string{machinedServiceID} +} + +// Runner implements the Service interface. +func (s *Syslogd) Runner(r runtime.Runtime) (runner.Runner, error) { + return goroutine.NewRunner(r, syslogServiceID, syslogd.Main, runner.WithLoggingManager(r.Logging())), nil +} + +// HealthFunc implements the HealthcheckedService interface. +func (s *Syslogd) HealthFunc(runtime.Runtime) health.Check { + return func(ctx context.Context) error { + return nil + } +} + +// HealthSettings implements the HealthcheckedService interface. +func (s *Syslogd) HealthSettings(runtime.Runtime) *health.Settings { + return &health.DefaultSettings +} diff --git a/internal/app/machined/pkg/system/services/trustd.go b/internal/app/machined/pkg/system/services/trustd.go new file mode 100644 index 0000000..69cbb11 --- /dev/null +++ b/internal/app/machined/pkg/system/services/trustd.go @@ -0,0 +1,190 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:golint +package services + +import ( + "context" + "errors" + "fmt" + "net" + "os" + "path/filepath" + + "github.com/containerd/containerd/oci" + "github.com/containerd/containerd/pkg/cap" + "github.com/cosi-project/runtime/api/v1alpha1" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/protobuf/server" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/siderolabs/go-debug" + "google.golang.org/grpc" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/system" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/events" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/health" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/containerd" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/restart" + "github.com/aenix-io/talm/internal/pkg/environment" + "github.com/siderolabs/talos/pkg/conditions" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" + timeresource "github.com/siderolabs/talos/pkg/machinery/resources/time" +) + +var _ system.HealthcheckedService = (*Trustd)(nil) + +// Trustd implements the Service interface. It serves as the concrete type with +// the required methods. +type Trustd struct { + runtimeServer *grpc.Server +} + +// ID implements the Service interface. +func (t *Trustd) ID(r runtime.Runtime) string { + return "trustd" +} + +// PreFunc implements the Service interface. +// +//nolint:gocyclo +func (t *Trustd) PreFunc(ctx context.Context, r runtime.Runtime) error { + // filter apid access to make sure apid can only access its certificates + resources := state.Filter( + r.State().V1Alpha2().Resources(), + func(ctx context.Context, access state.Access) error { + if !access.Verb.Readonly() { + return errors.New("write access denied") + } + + switch { + case access.ResourceNamespace == secrets.NamespaceName && access.ResourceType == secrets.TrustdType && access.ResourceID == secrets.TrustdID: + case access.ResourceNamespace == secrets.NamespaceName && access.ResourceType == secrets.OSRootType && access.ResourceID == secrets.OSRootID: + default: + return errors.New("access denied") + } + + return nil + }, + ) + + // ensure socket dir exists + if err := os.MkdirAll(filepath.Dir(constants.TrustdRuntimeSocketPath), 0o750); err != nil { + return err + } + + // set the final leaf to be world-executable to make trustd connect to the socket + if err := os.Chmod(filepath.Dir(constants.TrustdRuntimeSocketPath), 0o751); err != nil { + return err + } + + // clean up the socket if it already exists (important for Talos in a container) + if err := os.RemoveAll(constants.TrustdRuntimeSocketPath); err != nil { + return err + } + + listener, err := net.Listen("unix", constants.TrustdRuntimeSocketPath) + if err != nil { + return err + } + + // chown the socket path to make it accessible to the apid + if err := os.Chown(constants.TrustdRuntimeSocketPath, constants.TrustdUserID, constants.TrustdUserID); err != nil { + return err + } + + t.runtimeServer = grpc.NewServer( + grpc.SharedWriteBuffer(true), + ) + v1alpha1.RegisterStateServer(t.runtimeServer, server.NewState(resources)) + + go t.runtimeServer.Serve(listener) //nolint:errcheck + + return prepareRootfs(t.ID(r)) +} + +// PostFunc implements the Service interface. +func (t *Trustd) PostFunc(r runtime.Runtime, state events.ServiceState) (err error) { + t.runtimeServer.Stop() + + return os.RemoveAll(constants.TrustdRuntimeSocketPath) +} + +// Condition implements the Service interface. +func (t *Trustd) Condition(r runtime.Runtime) conditions.Condition { + return conditions.WaitForAll( + timeresource.NewSyncCondition(r.State().V1Alpha2().Resources()), + network.NewReadyCondition(r.State().V1Alpha2().Resources(), network.AddressReady, network.HostnameReady), + ) +} + +// DependsOn implements the Service interface. +func (t *Trustd) DependsOn(r runtime.Runtime) []string { + return []string{"containerd"} +} + +// Runner implements the Service interface. +func (t *Trustd) Runner(r runtime.Runtime) (runner.Runner, error) { + // Set the process arguments. + args := runner.Args{ + ID: t.ID(r), + ProcessArgs: []string{"/trustd"}, + } + + // Set the mounts. + mounts := []specs.Mount{ + {Type: "bind", Destination: "/tmp", Source: "/tmp", Options: []string{"rbind", "rshared", "rw"}}, + {Type: "bind", Destination: filepath.Dir(constants.TrustdRuntimeSocketPath), Source: filepath.Dir(constants.TrustdRuntimeSocketPath), Options: []string{"rbind", "ro"}}, + } + + env := environment.Get(r.Config()) + env = append(env, constants.TcellMinimizeEnvironment) + + if debug.RaceEnabled { + env = append(env, "GORACE=halt_on_error=1") + } + + return restart.New(containerd.NewRunner( + r.Config().Debug(), + &args, + runner.WithLoggingManager(r.Logging()), + runner.WithContainerdAddress(constants.SystemContainerdAddress), + runner.WithEnv(env), + runner.WithOCISpecOpts( + containerd.WithMemoryLimit(int64(1000000*512)), + oci.WithDroppedCapabilities(cap.Known()), + oci.WithHostNamespace(specs.NetworkNamespace), + oci.WithMounts(mounts), + oci.WithRootFSPath(filepath.Join(constants.SystemLibexecPath, t.ID(r))), + oci.WithRootFSReadonly(), + oci.WithUser(fmt.Sprintf("%d:%d", constants.TrustdUserID, constants.TrustdUserID)), + ), + runner.WithOOMScoreAdj(-998), + ), + restart.WithType(restart.Forever), + ), nil +} + +// HealthFunc implements the HealthcheckedService interface. +func (t *Trustd) HealthFunc(runtime.Runtime) health.Check { + return func(ctx context.Context) error { + var d net.Dialer + + conn, err := d.DialContext(ctx, "tcp", fmt.Sprintf("%s:%d", "127.0.0.1", constants.TrustdPort)) + if err != nil { + return err + } + + return conn.Close() + } +} + +// HealthSettings implements the HealthcheckedService interface. +func (t *Trustd) HealthSettings(runtime.Runtime) *health.Settings { + return &health.DefaultSettings +} diff --git a/internal/app/machined/pkg/system/services/udevd.go b/internal/app/machined/pkg/system/services/udevd.go new file mode 100644 index 0000000..2c97bc7 --- /dev/null +++ b/internal/app/machined/pkg/system/services/udevd.go @@ -0,0 +1,138 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package services + +import ( + "context" + "time" + + "github.com/siderolabs/go-cmd/pkg/cmd" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/system" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/events" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/health" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/process" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/restart" + "github.com/siderolabs/talos/pkg/conditions" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +var _ system.HealthcheckedService = (*Udevd)(nil) + +// Udevd implements the Service interface. It serves as the concrete type with +// the required methods. +type Udevd struct { + triggered bool +} + +// ID implements the Service interface. +func (c *Udevd) ID(r runtime.Runtime) string { + return "udevd" +} + +// PreFunc implements the Service interface. +func (c *Udevd) PreFunc(ctx context.Context, r runtime.Runtime) error { + _, err := cmd.RunContext( + ctx, + "/sbin/udevadm", + "hwdb", + "--update", + "--root=/usr", + ) + + return err +} + +// PostFunc implements the Service interface. +func (c *Udevd) PostFunc(r runtime.Runtime, state events.ServiceState) (err error) { + return nil +} + +// Condition implements the Service interface. +func (c *Udevd) Condition(r runtime.Runtime) conditions.Condition { + return nil +} + +// DependsOn implements the Service interface. +func (c *Udevd) DependsOn(r runtime.Runtime) []string { + return nil +} + +// Runner implements the Service interface. +func (c *Udevd) Runner(r runtime.Runtime) (runner.Runner, error) { + // Set the process arguments. + args := &runner.Args{ + ID: c.ID(r), + ProcessArgs: []string{ + "/sbin/udevd", + "--resolve-names=never", + }, + } + + debug := false + + if r.Config() != nil { + debug = r.Config().Debug() + } + + return restart.New(process.NewRunner( + debug, + args, + runner.WithLoggingManager(r.Logging()), + runner.WithCgroupPath(constants.CgroupUdevd), + runner.WithDroppedCapabilities(constants.UdevdDroppedCapabilities), + ), + restart.WithType(restart.Forever), + ), nil +} + +// HealthFunc implements the HealthcheckedService interface. +func (c *Udevd) HealthFunc(runtime.Runtime) health.Check { + return func(ctx context.Context) error { + // checking for the existence of the udev control socket is a faster way to check + // that udevd is running, but not a complete check since the socket can persist if the process + // was not gracefully stopped + if err := conditions.WaitForFileToExist("/run/udev/control").Wait(ctx); err != nil { + return err + } + + // udevadm trigger returns with an exit code of 0 even if udevd is not fully running, + // so running `udevadm control --reload` to ensure that udevd is fully initialized + // which returns an exit code of 2 if udevd is not running. This complementes the previous check + if _, err := cmd.RunContext(ctx, "/sbin/udevadm", "control", "--reload"); err != nil { + return err + } + + if !c.triggered { + if _, err := cmd.RunContext(ctx, "/sbin/udevadm", "trigger", "--type=devices", "--action=add"); err != nil { + return err + } + + if _, err := cmd.RunContext(ctx, "/sbin/udevadm", "trigger", "--type=subsystems", "--action=add"); err != nil { + return err + } + + c.triggered = true + } + + // This ensures that `udevd` finishes processing kernel events, triggered by + // `udevd trigger`, to prevent a race condition when a user specifies a path + // under `/dev/disk/*` in any disk definitions. + _, err := cmd.RunContext(ctx, "/sbin/udevadm", "settle", "--timeout=50") // timeout here should be less than health.Settings.Timeout + + return err + } +} + +// HealthSettings implements the HealthcheckedService interface. +func (c *Udevd) HealthSettings(runtime.Runtime) *health.Settings { + return &health.Settings{ + InitialDelay: 100 * time.Millisecond, + Period: time.Minute, + Timeout: 55 * time.Second, + } +} diff --git a/internal/app/machined/pkg/system/services/utils.go b/internal/app/machined/pkg/system/services/utils.go new file mode 100644 index 0000000..307b765 --- /dev/null +++ b/internal/app/machined/pkg/system/services/utils.go @@ -0,0 +1,36 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package services + +import ( + "fmt" + "os" + "path/filepath" + + "golang.org/x/sys/unix" + + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// prepareRootfs creates /system/libexec/ rootfs and bind-mounts /sbin/init there. +func prepareRootfs(id string) error { + rootfsPath := filepath.Join(constants.SystemLibexecPath, id) + + if err := os.MkdirAll(rootfsPath, 0o711); err != nil { // rwx--x--x, non-root programs should be able to follow path + return fmt.Errorf("failed to create rootfs %q: %w", rootfsPath, err) + } + + executablePath := filepath.Join(rootfsPath, id) + + if err := os.WriteFile(executablePath, nil, 0o555); err != nil { // r-xr-xr-x, non-root programs should be able to execute & read + return fmt.Errorf("failed to create empty executable %q: %w", executablePath, err) + } + + if err := unix.Mount("/sbin/init", executablePath, "", unix.MS_BIND, ""); err != nil { + return fmt.Errorf("failed to create bind mount for %q: %w", executablePath, err) + } + + return nil +} diff --git a/internal/app/machined/pkg/system/system.go b/internal/app/machined/pkg/system/system.go new file mode 100644 index 0000000..f9a0ebe --- /dev/null +++ b/internal/app/machined/pkg/system/system.go @@ -0,0 +1,473 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package system + +import ( + "context" + "errors" + "fmt" + "log" + "sort" + "strings" + "sync" + "time" + + "github.com/hashicorp/go-multierror" + "github.com/siderolabs/gen/maps" + "github.com/siderolabs/gen/xslices" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/events" + "github.com/siderolabs/talos/pkg/conditions" +) + +// singleton the system services API interface. +type singleton struct { + runtime runtime.Runtime + + // State of running services by ID + state map[string]*ServiceRunner + + // List of running services at the moment. + // + // Service might be in any state, but service ID in the map + // implies ServiceRunner.Start() method is running at the momemnt + runningMu sync.Mutex + running map[string]struct{} + + mu sync.Mutex + wg sync.WaitGroup + terminating bool +} + +var ( + instance *singleton + once sync.Once +) + +func newServices(runtime runtime.Runtime) *singleton { + return &singleton{ + runtime: runtime, + state: map[string]*ServiceRunner{}, + running: map[string]struct{}{}, + } +} + +// Services returns the instance of the system services API. +// +//nolint:revive,golint +func Services(runtime runtime.Runtime) *singleton { + once.Do(func() { + instance = newServices(runtime) + }) + + return instance +} + +// Load adds service to the list of services managed by the runner. +// +// Load returns service IDs for each of the services. +func (s *singleton) Load(services ...Service) []string { + s.mu.Lock() + defer s.mu.Unlock() + + if s.terminating { + return nil + } + + ids := make([]string, 0, len(services)) + + for _, service := range services { + id := service.ID(s.runtime) + ids = append(ids, id) + + if _, exists := s.state[id]; exists { + // service already loaded, ignore + continue + } + + svcrunner := NewServiceRunner(s, service, s.runtime) + s.state[id] = svcrunner + } + + return ids +} + +// Unload stops the service and removes it from the list of running services. +// +// It is not an error to unload a service which was already removed or stopped. +func (s *singleton) Unload(ctx context.Context, serviceIDs ...string) error { + s.mu.Lock() + if s.terminating { + s.mu.Unlock() + + return nil + } + + servicesToRemove := make([]string, 0, len(serviceIDs)) + + for _, id := range serviceIDs { + if _, exists := s.state[id]; exists { + servicesToRemove = append(servicesToRemove, id) + } + } + s.mu.Unlock() + + if err := s.Stop(ctx, servicesToRemove...); err != nil { + return fmt.Errorf("error stopping services %v: %w", servicesToRemove, err) + } + + s.mu.Lock() + defer s.mu.Unlock() + + s.runningMu.Lock() + defer s.runningMu.Unlock() + + for _, id := range servicesToRemove { + delete(s.state, id) + delete(s.running, id) // this fixes an edge case when defer() in Start() doesn't have time to remove stopped service from running + } + + return nil +} + +// Start will invoke the service's Pre, Condition, and Type funcs. If any +// error occurs in the Pre or Condition invocations, it is up to the caller to +// restart the service. +func (s *singleton) Start(serviceIDs ...string) error { + s.mu.Lock() + defer s.mu.Unlock() + + if s.terminating { + return nil + } + + var multiErr *multierror.Error + + for _, id := range serviceIDs { + svcrunner := s.state[id] + if svcrunner == nil { + multiErr = multierror.Append(multiErr, fmt.Errorf("service %q not defined", id)) + } + + s.runningMu.Lock() + + _, running := s.running[id] + if !running { + s.running[id] = struct{}{} + } + + s.runningMu.Unlock() + + if running { + // service already running, skip + continue + } + + runNotify := make(chan struct{}) + + s.wg.Add(1) + + go func(id string, svcrunner *ServiceRunner) { + err := func() error { + defer func() { + s.runningMu.Lock() + delete(s.running, id) + s.runningMu.Unlock() + }() + defer s.wg.Done() + + return svcrunner.Run(runNotify) + }() + + switch { + case err == nil: + svcrunner.UpdateState(context.Background(), events.StateFinished, "Service finished successfully") + case errors.Is(err, ErrSkip): + svcrunner.UpdateState(context.Background(), events.StateSkipped, "Service skipped") + default: + msg := err.Error() + if len(msg) > 0 { + msg = strings.ToUpper(msg[:1]) + msg[1:] + } + + svcrunner.UpdateState(context.Background(), events.StateFailed, msg) + } + }(id, svcrunner) + + // wait for svcrunner.Run to enter the running phase, and then return + <-runNotify + } + + return multiErr.ErrorOrNil() +} + +// StartAll starts all the services. +func (s *singleton) StartAll() { + s.mu.Lock() + serviceIDs := maps.Keys(s.state) + s.mu.Unlock() + + //nolint:errcheck + s.Start(serviceIDs...) +} + +// LoadAndStart combines Load and Start into single call. +func (s *singleton) LoadAndStart(services ...Service) { + err := s.Start(s.Load(services...)...) + if err != nil { + // should never happen + panic(err) + } +} + +// Shutdown all the services. +func (s *singleton) Shutdown(ctx context.Context) { + s.mu.Lock() + if s.terminating { + s.mu.Unlock() + + return + } + + s.terminating = true + + _ = s.stopServices(ctx, nil, true) //nolint:errcheck +} + +// Stop will initiate a shutdown of the specified service. +func (s *singleton) Stop(ctx context.Context, serviceIDs ...string) (err error) { + if len(serviceIDs) == 0 { + return + } + + s.mu.Lock() + if s.terminating { + s.mu.Unlock() + + return nil + } + + return s.stopServices(ctx, serviceIDs, false) +} + +// StopWithRevDepenencies will initiate a shutdown of the specified services waiting for reverse dependencies to finish first. +// +// If reverse dependency is not stopped, this method might block waiting on it being stopped for up to 30 seconds. +func (s *singleton) StopWithRevDepenencies(ctx context.Context, serviceIDs ...string) (err error) { + if len(serviceIDs) == 0 { + return + } + + s.mu.Lock() + if s.terminating { + s.mu.Unlock() + + return nil + } + + return s.stopServices(ctx, serviceIDs, true) +} + +//nolint:gocyclo +func (s *singleton) stopServices(ctx context.Context, services []string, waitForRevDependencies bool) error { + servicesToStop := map[string]*ServiceRunner{} + + if services == nil { + for name, svcrunner := range s.state { + servicesToStop[name] = svcrunner + } + } else { + for _, name := range services { + if _, ok := s.state[name]; !ok { + continue + } + + servicesToStop[name] = s.state[name] + } + } + + // build reverse dependencies, and expand the list of services to stop + // with services which depend on the one being stopped + reverseDependencies := map[string][]string{} + + if waitForRevDependencies { + // expand the list of services to stop with the list of services which depend + // on the ones being stopped + // the loop is run as long as more dependencies are added to the list + for { + expanded := false + + for name, svcrunner := range s.state { + if _, scheduledToStop := servicesToStop[name]; scheduledToStop { + continue + } + + dependencies := svcrunner.service.DependsOn(s.runtime) + + shouldStopService := false + + for _, dependency := range dependencies { + for scheduledService := range servicesToStop { + if scheduledService == dependency { + shouldStopService = true + + break + } + } + + if shouldStopService { + break + } + } + + if shouldStopService { + servicesToStop[name] = svcrunner + expanded = true + } + } + + if !expanded { + break + } + } + + // build a list of dependencies to wait for before stopping each of the services + for name, svcrunner := range servicesToStop { + for _, dependency := range svcrunner.service.DependsOn(s.runtime) { + reverseDependencies[dependency] = append(reverseDependencies[dependency], name) + } + } + } + + s.mu.Unlock() + + // shutdown all the services waiting for rev deps + var shutdownWg sync.WaitGroup + + // wait max 30 seconds for reverse deps to shut down + shutdownCtx, shutdownCtxCancel := context.WithTimeout(ctx, 30*time.Second) + defer shutdownCtxCancel() + + stoppedConds := make([]conditions.Condition, 0, len(servicesToStop)) + + for name, svcrunner := range servicesToStop { + shutdownWg.Add(1) + + stoppedConds = append(stoppedConds, waitForService(s, StateEventDown, name)) + + go func(svcrunner *ServiceRunner, reverseDeps []string) { + defer shutdownWg.Done() + + conds := xslices.Map(reverseDeps, func(dep string) conditions.Condition { return waitForService(s, StateEventDown, dep) }) + allDeps := conditions.WaitForAll(conds...) + + if err := allDeps.Wait(shutdownCtx); err != nil { + log.Printf("gave up on %s while stopping %q", allDeps, svcrunner.id) + } + + svcrunner.Shutdown() + }(svcrunner, reverseDependencies[name]) + } + + shutdownWg.Wait() + + return conditions.WaitForAll(stoppedConds...).Wait(ctx) +} + +// List returns snapshot of ServiceRunner instances. +func (s *singleton) List() (result []*ServiceRunner) { + s.mu.Lock() + defer s.mu.Unlock() + + result = maps.Values(s.state) + + // TODO: results should be sorted properly with topological sort on dependencies + // but, we don't have dependencies yet, so sort by service id for now to get stable order + sort.Slice(result, func(i, j int) bool { return result[i].id < result[j].id }) + + return +} + +// IsRunning checks service status (started/stopped). +// +// It doesn't check if service runner was started or not, just pure +// check for service status in terms of start/stop. +func (s *singleton) IsRunning(id string) (Service, bool, error) { + s.mu.Lock() + runner, exists := s.state[id] + s.mu.Unlock() + + if !exists { + return nil, false, fmt.Errorf("service %q not defined", id) + } + + s.runningMu.Lock() + _, running := s.running[id] + s.runningMu.Unlock() + + return runner.service, running, nil +} + +// APIStart processes service start request from the API. +func (s *singleton) APIStart(ctx context.Context, id string) error { + service, running, err := s.IsRunning(id) + if err != nil { + return err + } + + if running { + // already started, skip + return nil + } + + if svc, ok := service.(APIStartableService); ok && svc.APIStartAllowed(s.runtime) { + return s.Start(id) + } + + return fmt.Errorf("service %q doesn't support start operation via API", id) +} + +// APIStop processes services stop request from the API. +func (s *singleton) APIStop(ctx context.Context, id string) error { + service, running, err := s.IsRunning(id) + if err != nil { + return err + } + + if !running { + // already stopped, skip + return nil + } + + if svc, ok := service.(APIStoppableService); ok && svc.APIStopAllowed(s.runtime) { + return s.Stop(ctx, id) + } + + return fmt.Errorf("service %q doesn't support stop operation via API", id) +} + +// APIRestart processes services restart request from the API. +func (s *singleton) APIRestart(ctx context.Context, id string) error { + service, running, err := s.IsRunning(id) + if err != nil { + return err + } + + if !running { + // restart for not running service is equivalent to Start() + return s.APIStart(ctx, id) + } + + if svc, ok := service.(APIRestartableService); ok && svc.APIRestartAllowed(s.runtime) { + if err := s.Stop(ctx, id); err != nil { + return err + } + + return s.Start(id) + } + + return fmt.Errorf("service %q doesn't support restart operation via API", id) +} diff --git a/internal/app/machined/pkg/system/system_test.go b/internal/app/machined/pkg/system/system_test.go new file mode 100644 index 0000000..1544927 --- /dev/null +++ b/internal/app/machined/pkg/system/system_test.go @@ -0,0 +1,69 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package system_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/app/machined/pkg/system" +) + +type SystemServicesSuite struct { + suite.Suite +} + +func (suite *SystemServicesSuite) TestStartShutdown() { + system.Services(nil).LoadAndStart( + &MockService{name: "containerd"}, + &MockService{name: "trustd", dependencies: []string{"containerd"}}, + &MockService{name: "machined", dependencies: []string{"containerd", "trustd"}}, + ) + time.Sleep(10 * time.Millisecond) + + suite.Require().NoError(system.Services(nil).Unload(context.Background(), "trustd", "notrunning")) +} + +func (suite *SystemServicesSuite) TestStartStop() { + system.Services(nil).LoadAndStart( + &MockService{name: "yolo"}, + ) + + time.Sleep(10 * time.Millisecond) + + err := system.Services(nil).Stop( + context.TODO(), "yolo", + ) + suite.Assert().NoError(err) +} + +func (suite *SystemServicesSuite) TestStopWithRevDeps() { + system.Services(nil).LoadAndStart( + &MockService{name: "cri"}, + &MockService{name: "networkd", dependencies: []string{"cri"}}, + &MockService{name: "vland", dependencies: []string{"networkd"}}, + ) + time.Sleep(10 * time.Millisecond) + + // stopping cri should stop all services + suite.Require().NoError(system.Services(nil).StopWithRevDepenencies(context.Background(), "cri")) + + // no services should be running + for _, name := range []string{"cri", "networkd", "vland"} { + svc, running, err := system.Services(nil).IsRunning(name) + suite.Require().NoError(err) + suite.Assert().NotNil(svc) + suite.Assert().False(running) + } + + system.Services(nil).Shutdown(context.TODO()) +} + +func TestSystemServicesSuite(t *testing.T) { + suite.Run(t, new(SystemServicesSuite)) +} diff --git a/internal/app/machined/revert.go b/internal/app/machined/revert.go new file mode 100644 index 0000000..2b30cb0 --- /dev/null +++ b/internal/app/machined/revert.go @@ -0,0 +1,69 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package main + +import ( + "context" + "log" + "os" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/v1alpha1/bootloader" + "github.com/aenix-io/talm/internal/pkg/meta" +) + +func revertBootloader(ctx context.Context) { + if err := revertBootloadInternal(ctx); err != nil { + log.Printf("failed to revert bootloader: %s", err) + } +} + +func revertBootloadInternal(ctx context.Context) error { + metaState, err := meta.New(ctx, nil) + if err != nil { + if os.IsNotExist(err) { + // no META, no way to revert + return nil + } + + return err + } + + label, ok := metaState.ReadTag(meta.Upgrade) + if !ok { + return nil + } + + if label == "" { + if _, err = metaState.DeleteTag(ctx, meta.Upgrade); err != nil { + return err + } + + return metaState.Flush() + } + + log.Printf("reverting failed upgrade, switching to %q", label) + + if err = func() error { + config, probeErr := bootloader.Probe(ctx, "") + if probeErr != nil { + if os.IsNotExist(probeErr) { + // no bootloader found, nothing to do + return nil + } + + return probeErr + } + + return config.Revert(ctx) + }(); err != nil { + return err + } + + if _, err = metaState.DeleteTag(ctx, meta.Upgrade); err != nil { + return err + } + + return metaState.Flush() +} diff --git a/internal/pkg/cgroup/cgroup.go b/internal/pkg/cgroup/cgroup.go new file mode 100644 index 0000000..a10f165 --- /dev/null +++ b/internal/pkg/cgroup/cgroup.go @@ -0,0 +1,58 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package cgroup provides cgroup utilities to handle nested cgroups. +// +// When Talos runs in a container, it might either share or not share the host cgroup namespace. +// If the cgroup namespace is not shared, PID 1 will appear in cgroup '/', otherwise it will be +// part of some pre-existing cgroup hierarchy. +// +// When Talos is running in a non-container mode, it is always at the root of the cgroup hierarchy. +// +// This package provides a transparent way to handle nested cgroups by providing a Path() function +// which returns the correct cgroup path based on the cgroup hierarchy available. +package cgroup + +import ( + "path/filepath" + + "github.com/containerd/cgroups/v3" + "github.com/containerd/cgroups/v3/cgroup2" +) + +var root = "/" + +// InitRoot initializes the root cgroup path. +// +// This function should be called once at the beginning of the program, after the cgroup +// filesystem is mounted. +// +// This function only supports cgroupv2 nesting. +func InitRoot() error { + if cgroups.Mode() != cgroups.Unified { + return nil + } + + var err error + + root, err = cgroup2.NestedGroupPath("/") + + return err +} + +// Root returns the root cgroup path. +func Root() string { + return root +} + +// Path returns the path to the cgroup. +// +// This function handles the case when the cgroups are nested. +func Path(cgroupPath string) string { + if cgroups.Mode() != cgroups.Unified { + return cgroupPath + } + + return filepath.Join(root, cgroupPath) +} diff --git a/internal/pkg/cri/client.go b/internal/pkg/cri/client.go new file mode 100644 index 0000000..a473720 --- /dev/null +++ b/internal/pkg/cri/client.go @@ -0,0 +1,58 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cri + +import ( + "context" + "fmt" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + + "github.com/siderolabs/talos/pkg/grpc/dialer" +) + +// Client is a lightweight implementation of CRI client. +type Client struct { + conn *grpc.ClientConn + runtimeClient runtimeapi.RuntimeServiceClient + imagesClient runtimeapi.ImageServiceClient +} + +// maxMsgSize use 16MB as the default message size limit. +// grpc library default is 4MB. +const maxMsgSize = 1024 * 1024 * 16 + +// NewClient builds CRI client. +func NewClient(endpoint string, connectionTimeout time.Duration) (*Client, error) { + ctx, cancel := context.WithTimeout(context.Background(), connectionTimeout) + defer cancel() + + conn, err := grpc.DialContext(ctx, endpoint, + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithBlock(), + grpc.FailOnNonTempDialError(false), + grpc.WithBackoffMaxDelay(3*time.Second), //nolint:staticcheck + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize)), + grpc.WithContextDialer(dialer.DialUnix()), + grpc.WithSharedWriteBuffer(true), + ) + if err != nil { + return nil, fmt.Errorf("error connecting to CRI: %w", err) + } + + return &Client{ + conn: conn, + runtimeClient: runtimeapi.NewRuntimeServiceClient(conn), + imagesClient: runtimeapi.NewImageServiceClient(conn), + }, nil +} + +// Close connection. +func (c *Client) Close() error { + return c.conn.Close() +} diff --git a/internal/pkg/cri/containers.go b/internal/pkg/cri/containers.go new file mode 100644 index 0000000..b688730 --- /dev/null +++ b/internal/pkg/cri/containers.go @@ -0,0 +1,117 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cri + +import ( + "context" + "fmt" + + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" +) + +// CreateContainer creates a new container in the specified PodSandbox. +func (c *Client) CreateContainer(ctx context.Context, podSandBoxID string, config *runtimeapi.ContainerConfig, sandboxConfig *runtimeapi.PodSandboxConfig) (string, error) { + resp, err := c.runtimeClient.CreateContainer(ctx, &runtimeapi.CreateContainerRequest{ + PodSandboxId: podSandBoxID, + Config: config, + SandboxConfig: sandboxConfig, + }) + if err != nil { + return "", fmt.Errorf("CreateContainer in sandbox %q from runtime service failed: %w", podSandBoxID, err) + } + + if resp.ContainerId == "" { + return "", fmt.Errorf("ContainerId is not set for container %q", config.GetMetadata()) + } + + return resp.ContainerId, nil +} + +// StartContainer starts the container. +func (c *Client) StartContainer(ctx context.Context, containerID string) error { + _, err := c.runtimeClient.StartContainer(ctx, &runtimeapi.StartContainerRequest{ + ContainerId: containerID, + }) + if err != nil { + return fmt.Errorf("StartContainer %q from runtime service failed: %w", containerID, err) + } + + return nil +} + +// StopContainer stops a running container with a grace period (i.e., timeout). +func (c *Client) StopContainer(ctx context.Context, containerID string, timeout int64) error { + _, err := c.runtimeClient.StopContainer(ctx, &runtimeapi.StopContainerRequest{ + ContainerId: containerID, + Timeout: timeout, + }) + if err != nil { + return fmt.Errorf("StopContainer %q from runtime service failed: %w", containerID, err) + } + + return nil +} + +// RemoveContainer removes the container. If the container is running, the container +// should be forced to removal. +func (c *Client) RemoveContainer(ctx context.Context, containerID string) error { + _, err := c.runtimeClient.RemoveContainer(ctx, &runtimeapi.RemoveContainerRequest{ + ContainerId: containerID, + }) + if err != nil { + return fmt.Errorf("RemoveContainer %q from runtime service failed: %w", containerID, err) + } + + return nil +} + +// ListContainers lists containers by filters. +func (c *Client) ListContainers(ctx context.Context, filter *runtimeapi.ContainerFilter) ([]*runtimeapi.Container, error) { + resp, err := c.runtimeClient.ListContainers(ctx, &runtimeapi.ListContainersRequest{ + Filter: filter, + }) + if err != nil { + return nil, fmt.Errorf("ListContainers with filter %+v from runtime service failed: %w", filter, err) + } + + return resp.Containers, nil +} + +// ContainerStatus returns the container status. +func (c *Client) ContainerStatus(ctx context.Context, containerID string, verbose bool) (*runtimeapi.ContainerStatus, map[string]string, error) { + resp, err := c.runtimeClient.ContainerStatus(ctx, &runtimeapi.ContainerStatusRequest{ + ContainerId: containerID, + Verbose: verbose, + }) + if err != nil { + return nil, nil, fmt.Errorf("ContainerStatus %q from runtime service failed: %w", containerID, err) + } + + return resp.Status, resp.Info, nil +} + +// ContainerStats returns the stats of the container. +func (c *Client) ContainerStats(ctx context.Context, containerID string) (*runtimeapi.ContainerStats, error) { + resp, err := c.runtimeClient.ContainerStats(ctx, &runtimeapi.ContainerStatsRequest{ + ContainerId: containerID, + }) + if err != nil { + return nil, fmt.Errorf("ContainerStatus %q from runtime service failed: %w", containerID, err) + } + + return resp.GetStats(), nil +} + +// ListContainerStats returns stats for all the containers matching the filter. +func (c *Client) ListContainerStats(ctx context.Context, filter *runtimeapi.ContainerStatsFilter) ([]*runtimeapi.ContainerStats, error) { + resp, err := c.runtimeClient.ListContainerStats(ctx, &runtimeapi.ListContainerStatsRequest{ + Filter: filter, + }) + if err != nil { + return nil, fmt.Errorf("ListContainerStats with filter %+v from runtime service failed: %w", filter, err) + } + + return resp.GetStats(), nil +} diff --git a/internal/pkg/cri/cri.go b/internal/pkg/cri/cri.go new file mode 100644 index 0000000..03989de --- /dev/null +++ b/internal/pkg/cri/cri.go @@ -0,0 +1,9 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package cri provides minimal CRI client. +package cri + +// This client is based on k8s version in https://github.com/kubernetes/kubernetes/tree/master/pkg/kubelet/remote, +// but it doesn't depend on k8s libs. diff --git a/internal/pkg/cri/cri_test.go b/internal/pkg/cri/cri_test.go new file mode 100644 index 0000000..93bd614 --- /dev/null +++ b/internal/pkg/cri/cri_test.go @@ -0,0 +1,252 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cri_test + +import ( + "context" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/containerd/cgroups/v3" + "github.com/containerd/cgroups/v3/cgroup1" + "github.com/containerd/cgroups/v3/cgroup2" + "github.com/opencontainers/runtime-spec/specs-go" + "github.com/stretchr/testify/suite" + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/logging" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/events" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner" + "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/process" + "github.com/aenix-io/talm/internal/pkg/cri" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +const ( + busyboxImage = "docker.io/library/busybox:1.30.1" +) + +func MockEventSink(t *testing.T) func(state events.ServiceState, message string, args ...interface{}) { + return func(state events.ServiceState, message string, args ...interface{}) { + t.Logf(message, args...) + } +} + +type CRISuite struct { + suite.Suite + + tmpDir string + + containerdRunner runner.Runner + containerdWg sync.WaitGroup + containerdAddress string + + client *cri.Client + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +func (suite *CRISuite) SetupSuite() { + if cgroups.Mode() == cgroups.Unified { + suite.T().Skip("test doesn't pass under cgroupsv2") + } + + var err error + + suite.tmpDir = suite.T().TempDir() + + stateDir, rootDir := filepath.Join(suite.tmpDir, "state"), filepath.Join(suite.tmpDir, "root") + suite.Require().NoError(os.Mkdir(stateDir, 0o777)) + suite.Require().NoError(os.Mkdir(rootDir, 0o777)) + + if cgroups.Mode() == cgroups.Unified { + var ( + groupPath string + manager *cgroup2.Manager + ) + + groupPath, err = cgroup2.NestedGroupPath(suite.tmpDir) + suite.Require().NoError(err) + + manager, err = cgroup2.NewManager(constants.CgroupMountPath, groupPath, &cgroup2.Resources{}) + suite.Require().NoError(err) + + defer manager.Delete() //nolint:errcheck + } else { + var manager cgroup1.Cgroup + + manager, err = cgroup1.New(cgroup1.NestedPath(suite.tmpDir), &specs.LinuxResources{}) + suite.Require().NoError(err) + + defer manager.Delete() //nolint:errcheck + } + + suite.containerdAddress = filepath.Join(suite.tmpDir, "run.sock") + + args := &runner.Args{ + ID: "containerd", + ProcessArgs: []string{ + "/bin/containerd", + "--address", suite.containerdAddress, + "--state", stateDir, + "--root", rootDir, + "--config", constants.CRIContainerdConfig, + }, + } + + suite.containerdRunner = process.NewRunner( + false, + args, + runner.WithLoggingManager(logging.NewFileLoggingManager(suite.tmpDir)), + runner.WithEnv([]string{"PATH=/bin:" + constants.PATH}), + runner.WithCgroupPath(suite.tmpDir), + ) + suite.Require().NoError(suite.containerdRunner.Open()) + suite.containerdWg.Add(1) + + go func() { + defer suite.containerdWg.Done() + defer suite.containerdRunner.Close() //nolint:errcheck + suite.containerdRunner.Run(MockEventSink(suite.T())) //nolint:errcheck + }() + + suite.client, err = cri.NewClient("unix:"+suite.containerdAddress, 30*time.Second) + suite.Require().NoError(err) +} + +func (suite *CRISuite) TearDownSuite() { + suite.ctxCancel() + + suite.Require().NoError(suite.client.Close()) + + suite.Require().NoError(suite.containerdRunner.Stop()) + suite.containerdWg.Wait() +} + +func (suite *CRISuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 30*time.Second) +} + +func (suite *CRISuite) TearDownTest() { + suite.ctxCancel() +} + +func (suite *CRISuite) TestRunSandboxContainer() { + podSandboxConfig := &runtimeapi.PodSandboxConfig{ + Metadata: &runtimeapi.PodSandboxMetadata{ + Name: "etcd-master-1", + Uid: "ed1a599a53090941c9b4025c7e3e883d", + Namespace: "kube-system", + Attempt: 0, + }, + Labels: map[string]string{ + "io.kubernetes.pod.name": "etcd-master-1", + "io.kubernetes.pod.namespace": "kube-system", + }, + LogDirectory: suite.tmpDir, + Linux: &runtimeapi.LinuxPodSandboxConfig{ + SecurityContext: &runtimeapi.LinuxSandboxSecurityContext{ + NamespaceOptions: &runtimeapi.NamespaceOption{ + Network: runtimeapi.NamespaceMode_NODE, + }, + }, + }, + } + + podSandboxID, err := suite.client.RunPodSandbox(suite.ctx, podSandboxConfig, "") + suite.Require().NoError(err) + suite.Require().Len(podSandboxID, 64) + + imageRef, err := suite.client.PullImage( + suite.ctx, &runtimeapi.ImageSpec{ + Image: busyboxImage, + }, podSandboxConfig, + ) + suite.Require().NoError(err) + + _, err = suite.client.ImageStatus( + suite.ctx, &runtimeapi.ImageSpec{ + Image: imageRef, + }, + ) + suite.Require().NoError(err) + + ctrID, err := suite.client.CreateContainer( + suite.ctx, podSandboxID, + &runtimeapi.ContainerConfig{ + Metadata: &runtimeapi.ContainerMetadata{ + Name: "etcd", + }, + Labels: map[string]string{ + "io.kubernetes.container.name": "etcd", + "io.kubernetes.pod.name": "etcd-master-1", + "io.kubernetes.pod.namespace": "kube-system", + }, + Annotations: map[string]string{ + "io.kubernetes.container.restartCount": "1", + }, + Image: &runtimeapi.ImageSpec{ + Image: imageRef, + }, + Command: []string{"/bin/sh", "-c", "sleep 3600"}, + }, podSandboxConfig, + ) + suite.Require().NoError(err) + suite.Require().Len(ctrID, 64) + + err = suite.client.StartContainer(suite.ctx, ctrID) + suite.Require().NoError(err) + + _, err = suite.client.ContainerStats(suite.ctx, ctrID) + suite.Require().NoError(err) + + _, _, err = suite.client.ContainerStatus(suite.ctx, ctrID, true) + suite.Require().NoError(err) + + err = suite.client.StopContainer(suite.ctx, ctrID, 10) + suite.Require().NoError(err) + + err = suite.client.RemoveContainer(suite.ctx, ctrID) + suite.Require().NoError(err) + + err = suite.client.StopPodSandbox(suite.ctx, podSandboxID) + suite.Require().NoError(err) + + err = suite.client.RemovePodSandbox(suite.ctx, podSandboxID) + suite.Require().NoError(err) +} + +func (suite *CRISuite) TestList() { + pods, err := suite.client.ListPodSandbox(suite.ctx, &runtimeapi.PodSandboxFilter{}) + suite.Require().NoError(err) + suite.Require().Len(pods, 0) + + containers, err := suite.client.ListContainers(suite.ctx, &runtimeapi.ContainerFilter{}) + suite.Require().NoError(err) + suite.Require().Len(containers, 0) + + containerStats, err := suite.client.ListContainerStats(suite.ctx, &runtimeapi.ContainerStatsFilter{}) + suite.Require().NoError(err) + suite.Require().Len(containerStats, 0) + + _, err = suite.client.ListImages(suite.ctx, &runtimeapi.ImageFilter{}) + suite.Require().NoError(err) +} + +func TestCRISuite(t *testing.T) { + if os.Getuid() != 0 { + t.Skip("can't run the test as non-root") + } + + _, err := os.Stat("/bin/containerd") + if err != nil { + t.Skip("containerd binary is not available, skipping the test") + } + + suite.Run(t, new(CRISuite)) +} diff --git a/internal/pkg/cri/images.go b/internal/pkg/cri/images.go new file mode 100644 index 0000000..43c7376 --- /dev/null +++ b/internal/pkg/cri/images.go @@ -0,0 +1,55 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cri + +import ( + "context" + "fmt" + + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" +) + +// PullImage pulls container image. +func (c *Client) PullImage(ctx context.Context, image *runtimeapi.ImageSpec, sandboxConfig *runtimeapi.PodSandboxConfig) (string, error) { + resp, err := c.imagesClient.PullImage(ctx, &runtimeapi.PullImageRequest{ + Image: image, + SandboxConfig: sandboxConfig, + }) + if err != nil { + return "", fmt.Errorf("error pulling image %s: %w", image, err) + } + + return resp.ImageRef, nil +} + +// ListImages lists available images. +func (c *Client) ListImages(ctx context.Context, filter *runtimeapi.ImageFilter) ([]*runtimeapi.Image, error) { + resp, err := c.imagesClient.ListImages(ctx, &runtimeapi.ListImagesRequest{ + Filter: filter, + }) + if err != nil { + return nil, fmt.Errorf("error listing images: %w", err) + } + + return resp.Images, nil +} + +// ImageStatus returns the status of the image. +func (c *Client) ImageStatus(ctx context.Context, image *runtimeapi.ImageSpec) (*runtimeapi.Image, error) { + resp, err := c.imagesClient.ImageStatus(ctx, &runtimeapi.ImageStatusRequest{ + Image: image, + }) + if err != nil { + return nil, fmt.Errorf("ImageStatus %q from image service failed: %w", image.Image, err) + } + + if resp.Image != nil { + if resp.Image.Id == "" || resp.Image.Size_ == 0 { + return nil, fmt.Errorf("id or size of image %q is not set", image.Image) + } + } + + return resp.Image, nil +} diff --git a/internal/pkg/cri/pods.go b/internal/pkg/cri/pods.go new file mode 100644 index 0000000..4aa8161 --- /dev/null +++ b/internal/pkg/cri/pods.go @@ -0,0 +1,248 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cri + +import ( + "context" + "fmt" + "log" + "time" + + "github.com/siderolabs/go-retry/retry" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc/codes" + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + + talosclient "github.com/siderolabs/talos/pkg/machinery/client" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// RunPodSandbox creates and starts a pod-level sandbox. Runtimes should ensure +// the sandbox is in ready state. +func (c *Client) RunPodSandbox(ctx context.Context, config *runtimeapi.PodSandboxConfig, runtimeHandler string) (string, error) { + resp, err := c.runtimeClient.RunPodSandbox(ctx, &runtimeapi.RunPodSandboxRequest{ + Config: config, + RuntimeHandler: runtimeHandler, + }) + if err != nil { + return "", err + } + + if resp.PodSandboxId == "" { + return "", fmt.Errorf("PodSandboxId is not set for sandbox %q", config.GetMetadata()) + } + + return resp.PodSandboxId, nil +} + +// StopPodSandbox stops the sandbox. If there are any running containers in the +// sandbox, they should be forced to termination. +func (c *Client) StopPodSandbox(ctx context.Context, podSandBoxID string) error { + _, err := c.runtimeClient.StopPodSandbox(ctx, &runtimeapi.StopPodSandboxRequest{ + PodSandboxId: podSandBoxID, + }) + if err != nil { + return fmt.Errorf("StopPodSandbox %q from runtime service failed: %w", podSandBoxID, err) + } + + return nil +} + +// RemovePodSandbox removes the sandbox. If there are any containers in the +// sandbox, they should be forcibly removed. +func (c *Client) RemovePodSandbox(ctx context.Context, podSandBoxID string) error { + _, err := c.runtimeClient.RemovePodSandbox(ctx, &runtimeapi.RemovePodSandboxRequest{ + PodSandboxId: podSandBoxID, + }) + if err != nil { + return fmt.Errorf("RemovePodSandbox %q from runtime service failed: %w", podSandBoxID, err) + } + + return nil +} + +// ListPodSandbox returns a list of PodSandboxes. +func (c *Client) ListPodSandbox(ctx context.Context, filter *runtimeapi.PodSandboxFilter) ([]*runtimeapi.PodSandbox, error) { + resp, err := c.runtimeClient.ListPodSandbox(ctx, &runtimeapi.ListPodSandboxRequest{ + Filter: filter, + }) + if err != nil { + return nil, fmt.Errorf("ListPodSandbox with filter %+v from runtime service failed: %w", filter, err) + } + + return resp.Items, nil +} + +// PodSandboxStatus returns the status of the PodSandbox. +func (c *Client) PodSandboxStatus(ctx context.Context, podSandBoxID string) (*runtimeapi.PodSandboxStatus, map[string]string, error) { + resp, err := c.runtimeClient.PodSandboxStatus(ctx, &runtimeapi.PodSandboxStatusRequest{ + PodSandboxId: podSandBoxID, + Verbose: true, + }) + if err != nil { + return nil, nil, err + } + + return resp.Status, resp.Info, nil +} + +// StopAction for StopAndRemovePodSandboxes. +type StopAction int + +// Stop actions. +const ( + StopOnly StopAction = iota + StopAndRemove +) + +// StopAndRemovePodSandboxes stops and removes all pods with the specified network mode. If no +// network mode is specified, all pods will be removed. +func (c *Client) StopAndRemovePodSandboxes(ctx context.Context, stopAction StopAction, modes ...runtimeapi.NamespaceMode) (err error) { + pods, err := c.ListPodSandbox(ctx, nil) + if err != nil { + return err + } + + var g errgroup.Group + + for _, pod := range pods { + g.Go(func() error { + status, _, e := c.PodSandboxStatus(ctx, pod.GetId()) + if e != nil { + if talosclient.StatusCode(e) == codes.NotFound { + return nil + } + + return e + } + + networkMode := status.GetLinux().GetNamespaces().GetOptions().GetNetwork() + + // If any modes are specified, we verify that the current pod is + // running any one of the modes. If it doesn't, we skip it. + if len(modes) > 0 && !contains(networkMode, modes) { + return nil + } + + if e = stopAndRemove(ctx, stopAction, c, pod, networkMode.String()); e != nil { + return fmt.Errorf("failed stopping pod %s/%s: %w", pod.Metadata.Namespace, pod.Metadata.Name, e) + } + + return nil + }) + } + + return g.Wait() +} + +func contains(mode runtimeapi.NamespaceMode, modes []runtimeapi.NamespaceMode) bool { + for _, m := range modes { + if mode == m { + return true + } + } + + return false +} + +//nolint:gocyclo +func stopAndRemove(ctx context.Context, stopAction StopAction, client *Client, pod *runtimeapi.PodSandbox, mode string) (err error) { + action := "stopping" + status := "stopped" + + if stopAction == StopAndRemove { + action = "removing" + status = "removed" + } + + log.Printf("%s pod %s/%s with network mode %q", action, pod.Metadata.Namespace, pod.Metadata.Name, mode) + + filter := &runtimeapi.ContainerFilter{ + PodSandboxId: pod.Id, + } + + containers, err := client.ListContainers(ctx, filter) + if err != nil { + if talosclient.StatusCode(err) == codes.NotFound { + return nil + } + + return err + } + + var g errgroup.Group + + for _, container := range containers { + g.Go(func() error { + // TODO(andrewrynhard): Can we set the timeout dynamically? + if container.State == runtimeapi.ContainerState_CONTAINER_RUNNING || container.State == runtimeapi.ContainerState_CONTAINER_UNKNOWN { + log.Printf("stopping container %s/%s:%s", pod.Metadata.Namespace, pod.Metadata.Name, container.Metadata.Name) + + if criErr := client.StopContainer(ctx, container.Id, int64(constants.KubeletShutdownGracePeriod.Seconds())); criErr != nil { + if talosclient.StatusCode(criErr) == codes.NotFound { + return nil + } + + return criErr + } + } + + if stopAction == StopAndRemove { + log.Printf("removing container %s/%s:%s", pod.Metadata.Namespace, pod.Metadata.Name, container.Metadata.Name) + + if removeErr := retry.Constant(constants.KubeletShutdownGracePeriod, retry.WithUnits(time.Second), retry.WithErrorLogging(true)).RetryWithContext(ctx, + func(ctx context.Context) error { + if criErr := client.RemoveContainer(ctx, container.Id); criErr != nil { + if talosclient.StatusCode(criErr) == codes.NotFound { + return nil + } + + return retry.ExpectedError(criErr) + } + + return nil + }); removeErr != nil { + return removeErr + } + } + + log.Printf("%s container %s/%s:%s", status, pod.Metadata.Namespace, pod.Metadata.Name, container.Metadata.Name) + + return nil + }) + } + + if err = g.Wait(); err != nil { + return err + } + + if pod.State == runtimeapi.PodSandboxState_SANDBOX_READY { + if err = client.StopPodSandbox(ctx, pod.Id); err != nil { + if talosclient.StatusCode(err) == codes.NotFound { + return nil + } + + log.Printf("error stopping pod %s/%s, ignored: %s", pod.Metadata.Namespace, pod.Metadata.Name, err) + + return nil + } + } + + if stopAction == StopAndRemove { + if err = client.RemovePodSandbox(ctx, pod.Id); err != nil { + if talosclient.StatusCode(err) == codes.NotFound { + return nil + } + + log.Printf("error removing pod %s/%s, ignored: %s", pod.Metadata.Namespace, pod.Metadata.Name, err) + + return nil + } + } + + log.Printf("%s pod %s/%s", status, pod.Metadata.Namespace, pod.Metadata.Name) + + return nil +} diff --git a/internal/pkg/dashboard/apidata/apidata.go b/internal/pkg/dashboard/apidata/apidata.go new file mode 100644 index 0000000..c39bb37 --- /dev/null +++ b/internal/pkg/dashboard/apidata/apidata.go @@ -0,0 +1,6 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package apidata implements the types and the data sources for the data sourced from various Talos APIs. +package apidata diff --git a/internal/pkg/dashboard/apidata/data.go b/internal/pkg/dashboard/apidata/data.go new file mode 100644 index 0000000..80ac1ba --- /dev/null +++ b/internal/pkg/dashboard/apidata/data.go @@ -0,0 +1,38 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package apidata implements types to handle monitoring data, calculate values from it, etc. +package apidata + +import ( + "time" +) + +const maxPoints = 1000 + +// Data represents the monitoring data retrieved via Talos API. +// +// Data structure is sent over the channel each interval. +type Data struct { + // Data per each node. + Nodes map[string]*Node + + Timestamp time.Time + Interval time.Duration +} + +// CalculateDiff with data from previous iteration. +func (data *Data) CalculateDiff(oldData *Data) { + data.Interval = data.Timestamp.Sub(oldData.Timestamp) + + for node, nodeData := range data.Nodes { + oldNodeData := oldData.Nodes[node] + if oldNodeData == nil { + continue + } + + nodeData.UpdateDiff(oldNodeData) + nodeData.UpdateSeries(oldNodeData) + } +} diff --git a/internal/pkg/dashboard/apidata/diff.go b/internal/pkg/dashboard/apidata/diff.go new file mode 100644 index 0000000..ef3efce --- /dev/null +++ b/internal/pkg/dashboard/apidata/diff.go @@ -0,0 +1,85 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package apidata + +import "github.com/siderolabs/talos/pkg/machinery/api/machine" + +func cpuInfoDiff(old, next *machine.CPUStat) *machine.CPUStat { + if old == nil || next == nil { + return &machine.CPUStat{} + } + + // TODO: support wraparound + return &machine.CPUStat{ + User: next.User - old.User, + Nice: next.Nice - old.Nice, + System: next.System - old.System, + Idle: next.Idle - old.Idle, + Iowait: next.Iowait - old.Iowait, + Irq: next.Irq - old.Irq, + SoftIrq: next.SoftIrq - old.SoftIrq, + Steal: next.Steal - old.Steal, + Guest: next.Guest - old.Guest, + GuestNice: next.GuestNice - old.GuestNice, + } +} + +func netDevDiff(old, next *machine.NetDev) *machine.NetDev { + if old == nil || next == nil { + return &machine.NetDev{} + } + + // TODO: support wraparound + return &machine.NetDev{ + Name: next.Name, + RxBytes: next.RxBytes - old.RxBytes, + RxPackets: next.RxPackets - old.RxPackets, + RxErrors: next.RxErrors - old.RxErrors, + RxDropped: next.RxDropped - old.RxDropped, + RxFifo: next.RxFifo - old.RxFifo, + RxFrame: next.RxFrame - old.RxFrame, + RxCompressed: next.RxCompressed - old.RxCompressed, + RxMulticast: next.RxMulticast - old.RxMulticast, + TxBytes: next.TxBytes - old.TxBytes, + TxPackets: next.TxPackets - old.TxPackets, + TxErrors: next.TxErrors - old.TxErrors, + TxDropped: next.TxDropped - old.TxDropped, + TxFifo: next.TxFifo - old.TxFifo, + TxCollisions: next.TxCollisions - old.TxCollisions, + TxCarrier: next.TxCarrier - old.TxCarrier, + TxCompressed: next.TxCompressed - old.TxCompressed, + } +} + +func diskStatDiff(old, next *machine.DiskStat) *machine.DiskStat { + if old == nil || next == nil { + return &machine.DiskStat{} + } + + // TODO: support wraparound + return &machine.DiskStat{ + Name: next.Name, + ReadCompleted: next.ReadCompleted - old.ReadCompleted, + ReadMerged: next.ReadMerged - old.ReadMerged, + ReadSectors: next.ReadSectors - old.ReadSectors, + WriteCompleted: next.WriteCompleted - old.WriteCompleted, + WriteMerged: next.WriteMerged - old.WriteMerged, + WriteSectors: next.WriteSectors - old.WriteSectors, + DiscardCompleted: next.DiscardCompleted - old.DiscardCompleted, + DiscardMerged: next.DiscardMerged - old.DiscardMerged, + DiscardSectors: next.DiscardSectors - old.DiscardSectors, + } +} + +func procDiff(old, next *machine.ProcessInfo) *machine.ProcessInfo { + if old == nil || next == nil { + return &machine.ProcessInfo{} + } + + // TODO: support wraparound + return &machine.ProcessInfo{ + CpuTime: next.CpuTime - old.CpuTime, + } +} diff --git a/internal/pkg/dashboard/apidata/node.go b/internal/pkg/dashboard/apidata/node.go new file mode 100644 index 0000000..373cfa8 --- /dev/null +++ b/internal/pkg/dashboard/apidata/node.go @@ -0,0 +1,209 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package apidata + +import ( + "github.com/siderolabs/gen/xslices" + + "github.com/siderolabs/talos/pkg/machinery/api/machine" +) + +// Node represents data gathered from a single node. +type Node struct { + // These fields are directly API responses. + Hostname *machine.Hostname + LoadAvg *machine.LoadAvg + Version *machine.Version + Memory *machine.Memory + SystemStat *machine.SystemStat + CPUsInfo *machine.CPUsInfo + NetDevStats *machine.NetworkDeviceStats + DiskStats *machine.DiskStats + Processes *machine.Process + ServiceList *machine.ServiceList + + // These fields are calculated as diff with Node data from previous pol. + SystemStatDiff *machine.SystemStat + NetDevStatsDiff *machine.NetworkDeviceStats + DiskStatsDiff *machine.DiskStats + ProcsDiff map[int32]*machine.ProcessInfo + + // Time-series data. + Series map[string][]float64 +} + +// MemUsage as used/total. +func (node *Node) MemUsage() float64 { + memTotal := node.Memory.GetMeminfo().GetMemtotal() + memUsed := node.Memory.GetMeminfo().GetMemtotal() - node.Memory.GetMeminfo().GetMemfree() - node.Memory.GetMeminfo().GetCached() - node.Memory.GetMeminfo().GetBuffers() + + if memTotal == 0 { + return 0 + } + + return float64(memUsed) / float64(memTotal) +} + +// CPUUsageByName returns CPU usage by name. +// +//nolint:gocyclo +func (node *Node) CPUUsageByName(name string) float64 { + if node.SystemStatDiff == nil || node.SystemStatDiff.CpuTotal == nil { + return 0 + } + + stat := node.SystemStatDiff.CpuTotal + + idle := stat.Idle + stat.Iowait + nonIdle := stat.User + stat.Nice + stat.System + stat.Irq + stat.Steal + stat.SoftIrq + total := idle + nonIdle + + if total == 0 { + return 0 + } + + switch name { + case "user": + return stat.User / total + case "system": + return stat.System / total + case "idle": + return stat.Idle / total + case "iowait": + return stat.Iowait / total + case "nice": + return stat.Nice / total + case "irq": + return stat.Irq / total + case "steal": + return stat.Steal / total + case "softirq": + return stat.SoftIrq / total + case "usage": + return (total - idle) / total + case "total": + return total + case "total_weighted": + cpuCount := len(node.CPUsInfo.GetCpuInfo()) + if cpuCount == 0 { + return total + } + + return total / float64(cpuCount) + } + + panic("unknown cpu usage name") +} + +// CtxSwitches returns diff of context switches. +func (node *Node) CtxSwitches() uint64 { + if node.SystemStatDiff == nil { + return 0 + } + + return node.SystemStatDiff.GetContextSwitches() +} + +// ProcsCreated returns diff of processes created. +func (node *Node) ProcsCreated() uint64 { + if node.SystemStatDiff == nil { + return 0 + } + + return node.SystemStatDiff.GetProcessCreated() +} + +// UpdateSeries builds time-series data based on previous iteration data. +func (node *Node) UpdateSeries(old *Node) { + node.Series = make(map[string][]float64) + + for _, graphInfo := range []struct { + name string + f func() float64 + }{ + { + "mem", + node.MemUsage, + }, + { + "user", + func() float64 { return node.CPUUsageByName("user") }, + }, + { + "system", + func() float64 { return node.CPUUsageByName("system") }, + }, + { + "loadavg", + func() float64 { return node.LoadAvg.GetLoad1() }, + }, + { + "netrxbytes", + func() float64 { return float64(node.NetDevStatsDiff.GetTotal().GetRxBytes()) }, + }, + { + "nettxbytes", + func() float64 { return float64(node.NetDevStatsDiff.GetTotal().GetTxBytes()) }, + }, + { + "diskrdsectors", + func() float64 { return float64(node.DiskStatsDiff.GetTotal().GetReadSectors()) }, + }, + { + "diskwrsectors", + func() float64 { return float64(node.DiskStatsDiff.GetTotal().GetWriteSectors()) }, + }, + } { + oldSeries := old.Series[graphInfo.name] + + off := 0 + if len(oldSeries) > maxPoints { + off = len(oldSeries) - maxPoints + } + + node.Series[graphInfo.name] = append(oldSeries[off:], graphInfo.f()) + + // TODO: bug with plot widget + for len(node.Series[graphInfo.name]) < 2 { + node.Series[graphInfo.name] = append([]float64{0.0}, node.Series[graphInfo.name]...) + } + } +} + +// UpdateDiff calculates diff with node data from previous iteration. +func (node *Node) UpdateDiff(old *Node) { + if old.SystemStat != nil { + node.SystemStatDiff = &machine.SystemStat{ + // TODO: support other fields + CpuTotal: cpuInfoDiff(old.SystemStat.GetCpuTotal(), node.SystemStat.GetCpuTotal()), + ContextSwitches: node.SystemStat.ContextSwitches - old.SystemStat.ContextSwitches, + ProcessCreated: node.SystemStat.ProcessCreated - old.SystemStat.ProcessCreated, + } + } + + if old.NetDevStats != nil { + node.NetDevStatsDiff = &machine.NetworkDeviceStats{ + // TODO: support other fields + Total: netDevDiff(old.NetDevStats.GetTotal(), node.NetDevStats.GetTotal()), + } + } + + if old.DiskStats != nil { + node.DiskStatsDiff = &machine.DiskStats{ + // TODO: support other fields + Total: diskStatDiff(old.DiskStats.GetTotal(), node.DiskStats.GetTotal()), + } + } + + if old.Processes != nil { + index := xslices.ToMap(old.Processes.GetProcesses(), func(proc *machine.ProcessInfo) (int32, *machine.ProcessInfo) { + return proc.Pid, proc + }) + + node.ProcsDiff = xslices.ToMap(node.Processes.GetProcesses(), func(proc *machine.ProcessInfo) (int32, *machine.ProcessInfo) { + return proc.Pid, procDiff(index[proc.Pid], proc) + }) + } +} diff --git a/internal/pkg/dashboard/apidata/source.go b/internal/pkg/dashboard/apidata/source.go new file mode 100644 index 0000000..e9a3da4 --- /dev/null +++ b/internal/pkg/dashboard/apidata/source.go @@ -0,0 +1,297 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package apidata + +import ( + "context" + "sync" + "time" + + "golang.org/x/sync/errgroup" + "google.golang.org/protobuf/types/known/emptypb" + + "github.com/siderolabs/talos/pkg/machinery/client" +) + +// Source is a data source that gathers information about a Talos node using Talos API. +type Source struct { + *client.Client + + Interval time.Duration + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc + + wg sync.WaitGroup +} + +// Run the data poll on interval. +func (source *Source) Run(ctx context.Context) <-chan *Data { + dataCh := make(chan *Data) + + source.ctx, source.ctxCancel = context.WithCancel(ctx) + + source.wg.Add(1) + + go source.run(dataCh) + + return dataCh +} + +// Stop the data collection process. +func (source *Source) Stop() { + source.ctxCancel() + + source.wg.Wait() +} + +func (source *Source) run(dataCh chan<- *Data) { + defer source.wg.Done() + defer close(dataCh) + + ticker := time.NewTicker(source.Interval) + defer ticker.Stop() + + var oldData, currentData *Data + + for { + currentData = source.gather() + + if oldData == nil { + currentData.CalculateDiff(currentData) + } else { + currentData.CalculateDiff(oldData) + } + + select { + case dataCh <- currentData: + case <-source.ctx.Done(): + return + } + + select { + case <-source.ctx.Done(): + return + case <-ticker.C: + } + + oldData = currentData + } +} + +//nolint:gocyclo,cyclop +func (source *Source) gather() *Data { + result := &Data{ + Timestamp: time.Now(), + Nodes: map[string]*Node{}, + } + + var resultLock sync.Mutex + + gatherFuncs := []func() error{ + func() error { + resp, err := source.MachineClient.LoadAvg(source.ctx, &emptypb.Empty{}) + if err != nil { + return err + } + + resultLock.Lock() + defer resultLock.Unlock() + + for _, msg := range resp.GetMessages() { + node := msg.GetMetadata().GetHostname() + + if _, ok := result.Nodes[node]; !ok { + result.Nodes[node] = &Node{} + } + + result.Nodes[node].LoadAvg = msg + } + + return nil + }, + func() error { + resp, err := source.MachineClient.Version(source.ctx, &emptypb.Empty{}) + if err != nil { + return err + } + + resultLock.Lock() + defer resultLock.Unlock() + + for _, msg := range resp.GetMessages() { + node := msg.GetMetadata().GetHostname() + + if _, ok := result.Nodes[node]; !ok { + result.Nodes[node] = &Node{} + } + + result.Nodes[node].Version = msg + } + + return nil + }, + func() error { + resp, err := source.MachineClient.Memory(source.ctx, &emptypb.Empty{}) + if err != nil { + return err + } + + resultLock.Lock() + defer resultLock.Unlock() + + for _, msg := range resp.GetMessages() { + node := msg.GetMetadata().GetHostname() + + if _, ok := result.Nodes[node]; !ok { + result.Nodes[node] = &Node{} + } + + result.Nodes[node].Memory = msg + } + + return nil + }, + func() error { + resp, err := source.MachineClient.SystemStat(source.ctx, &emptypb.Empty{}) + if err != nil { + return err + } + + resultLock.Lock() + defer resultLock.Unlock() + + for _, msg := range resp.GetMessages() { + node := msg.GetMetadata().GetHostname() + + if _, ok := result.Nodes[node]; !ok { + result.Nodes[node] = &Node{} + } + + result.Nodes[node].SystemStat = msg + } + + return nil + }, + func() error { + resp, err := source.MachineClient.CPUInfo(source.ctx, &emptypb.Empty{}) + if err != nil { + return err + } + + resultLock.Lock() + defer resultLock.Unlock() + + for _, msg := range resp.GetMessages() { + node := msg.GetMetadata().GetHostname() + + if _, ok := result.Nodes[node]; !ok { + result.Nodes[node] = &Node{} + } + + result.Nodes[node].CPUsInfo = msg + } + + return nil + }, + func() error { + resp, err := source.MachineClient.NetworkDeviceStats(source.ctx, &emptypb.Empty{}) + if err != nil { + return err + } + + resultLock.Lock() + defer resultLock.Unlock() + + for _, msg := range resp.GetMessages() { + node := msg.GetMetadata().GetHostname() + + if _, ok := result.Nodes[node]; !ok { + result.Nodes[node] = &Node{} + } + + result.Nodes[node].NetDevStats = msg + } + + return nil + }, + func() error { + resp, err := source.MachineClient.DiskStats(source.ctx, &emptypb.Empty{}) + if err != nil { + return err + } + + resultLock.Lock() + defer resultLock.Unlock() + + for _, msg := range resp.GetMessages() { + node := msg.GetMetadata().GetHostname() + + if _, ok := result.Nodes[node]; !ok { + result.Nodes[node] = &Node{} + } + + result.Nodes[node].DiskStats = msg + } + + return nil + }, + func() error { + resp, err := source.MachineClient.Processes(source.ctx, &emptypb.Empty{}) + if err != nil { + return err + } + + resultLock.Lock() + defer resultLock.Unlock() + + for _, msg := range resp.GetMessages() { + node := msg.GetMetadata().GetHostname() + + if _, ok := result.Nodes[node]; !ok { + result.Nodes[node] = &Node{} + } + + result.Nodes[node].Processes = msg + } + + return nil + }, + func() error { + resp, err := source.MachineClient.ServiceList(source.ctx, &emptypb.Empty{}) + if err != nil { + return err + } + + resultLock.Lock() + defer resultLock.Unlock() + + for _, msg := range resp.GetMessages() { + node := msg.GetMetadata().GetHostname() + + if _, ok := result.Nodes[node]; !ok { + result.Nodes[node] = &Node{} + } + + result.Nodes[node].ServiceList = msg + } + + return nil + }, + } + + var eg errgroup.Group + + for _, f := range gatherFuncs { + eg.Go(f) + } + + if err := eg.Wait(); err != nil { + // TODO: handle error + _ = err + } + + return result +} diff --git a/internal/pkg/dashboard/components/components.go b/internal/pkg/dashboard/components/components.go new file mode 100644 index 0000000..b75dff1 --- /dev/null +++ b/internal/pkg/dashboard/components/components.go @@ -0,0 +1,93 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package components implements specific widgets for the dashboard. +package components + +import ( + "fmt" + "strings" + + "github.com/siderolabs/gen/xslices" +) + +const ( + noData = "..." + notAvailable = "n/a" + none = "" + maxLogLines = 1000 +) + +// field represents a field in a widget consist of a name and a value, rendered next to each other. +type field struct { + Name string + Value string +} + +func (f *field) render(nameWidth int) string { + return fmt.Sprintf("[::b]%s[::-] %s", padRight(f.Name, nameWidth), f.Value) +} + +type fieldGroup struct { + fields []field +} + +// String implements the Stringer interface. +func (fg *fieldGroup) String() string { + width := fg.maxFieldNameLength() + + return strings.Join( + xslices.Map(fg.fields, func(t field) string { + return t.render(width) + }), + "\n", + ) +} + +func (fg *fieldGroup) maxFieldNameLength() int { + max := 0 + + for _, f := range fg.fields { + if len(f.Name) > max { + max = len(f.Name) + } + } + + return max +} + +// padRight pads a string to the specified width by appending spaces to the end. +func padRight(s string, width int) string { + return fmt.Sprintf("%-*s", width, s) +} + +func toHealthStatus(healthy bool) string { + if healthy { + return formatStatus("Healthy") + } + + return formatStatus("Unhealthy") +} + +func formatStatus(status any) string { + statusStr := capitalizeFirst(fmt.Sprintf("%v", status)) + + switch strings.ToLower(statusStr) { + case "running", "healthy", "true": + return fmt.Sprintf("[green]%s[-]", statusStr) + case "stopped", "unhealthy", "false": + return fmt.Sprintf("[red]%s[-]", statusStr) + default: + return statusStr + } +} + +// capitalizeFirst capitalizes the first character of string. +func capitalizeFirst(s string) string { + if s == "" { + return s + } + + return strings.ToUpper(string(s[0])) + strings.ToLower(s[1:]) +} diff --git a/internal/pkg/dashboard/components/footer.go b/internal/pkg/dashboard/components/footer.go new file mode 100644 index 0000000..2dafefc --- /dev/null +++ b/internal/pkg/dashboard/components/footer.go @@ -0,0 +1,127 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package components + +import ( + "fmt" + "sort" + "strings" + + "github.com/gdamore/tcell/v2" + "github.com/rivo/tview" + "github.com/siderolabs/gen/maps" +) + +// Footer represents the top bar with host info. +type Footer struct { + tview.TextView + + selectedNode string + nodes []string + + screenKeyToName map[string]string + + selectedScreen string +} + +// NewFooter initializes Footer. +func NewFooter(screenKeyToName map[string]string, nodes []string) *Footer { + var initialScreen string + for _, name := range screenKeyToName { + initialScreen = name + + break + } + + widget := &Footer{ + TextView: *tview.NewTextView(), + screenKeyToName: screenKeyToName, + selectedScreen: initialScreen, + nodes: nodes, + } + + widget.SetDynamicColors(true) + + // set the background to be a horizontal line + widget.SetDrawFunc(func(screen tcell.Screen, x, y, width, height int) (int, int, int, int) { + for i := x; i < x+width; i++ { + for j := y; j < y+height; j++ { + screen.SetContent( + i, + j, + tview.BoxDrawingsLightHorizontal, + nil, + tcell.StyleDefault.Foreground(tcell.ColorWhite), + ) + } + } + + return x, y, width, height + }) + + widget.refresh() + + return widget +} + +// OnNodeSelect implements the NodeSelectListener interface. +func (widget *Footer) OnNodeSelect(node string) { + widget.selectedNode = node + + widget.refresh() +} + +// SelectScreen refreshes the footer with the tabs and screens data. +func (widget *Footer) SelectScreen(screen string) { + widget.selectedScreen = screen + + widget.refresh() +} + +func (widget *Footer) refresh() { + widget.SetText(fmt.Sprintf( + "[%s] --- %s", + widget.nodesText(), + widget.screensText(), + )) +} + +func (widget *Footer) nodesText() string { + nodesCopy := make([]string, 0, len(widget.nodes)) + + for _, node := range widget.nodes { + if node == widget.selectedNode { + name := node + if name == "" { + name = "(local)" + } + + nodesCopy = append(nodesCopy, fmt.Sprintf("[red]%s[-]", name)) + } else { + nodesCopy = append(nodesCopy, node) + } + } + + return strings.Join(nodesCopy, " | ") +} + +func (widget *Footer) screensText() string { + screenKeys := maps.Keys(widget.screenKeyToName) + sort.Strings(screenKeys) + + screenTexts := make([]string, 0, len(widget.screenKeyToName)) + + for _, screenKey := range screenKeys { + screen := widget.screenKeyToName[screenKey] + + if screen == widget.selectedScreen { + screenTexts = append(screenTexts, fmt.Sprintf("[[red]%s[-]]", screen)) + } else { + screenTexts = append(screenTexts, fmt.Sprintf("[%s: %s]", screenKey, screen)) + } + } + + return strings.Join(screenTexts, " --- ") +} diff --git a/internal/pkg/dashboard/components/gauges.go b/internal/pkg/dashboard/components/gauges.go new file mode 100644 index 0000000..be3e67e --- /dev/null +++ b/internal/pkg/dashboard/components/gauges.go @@ -0,0 +1,97 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package components + +import ( + "fmt" + "math" + + ui "github.com/gizak/termui/v3" + "github.com/gizak/termui/v3/widgets" + + "github.com/aenix-io/talm/internal/pkg/dashboard/apidata" +) + +// SystemGauges quickly show CPU/mem load. +type SystemGauges struct { + *TermUIWrapper + + inner *systemGaugesInner +} + +// NewSystemGauges creates SystemGauges. +func NewSystemGauges() *SystemGauges { + inner := systemGaugesInner{ + Block: *ui.NewBlock(), + } + + inner.cpuGauge = widgets.NewGauge() + inner.cpuGauge.Border = false + inner.cpuGauge.Title = "CPU" + + inner.memGauge = widgets.NewGauge() + inner.memGauge.Title = "MEM" + inner.memGauge.Border = false + + wrapper := NewTermUIWrapper(&inner) + + widget := &SystemGauges{ + TermUIWrapper: wrapper, + inner: &inner, + } + + widget.SetBorderPadding(1, 0, 0, 0) + + return widget +} + +// OnAPIDataChange implements the APIDataListener interface. +func (widget *SystemGauges) OnAPIDataChange(node string, data *apidata.Data) { + nodeData := data.Nodes[node] + + if nodeData == nil { + widget.inner.cpuGauge.Label = noData + widget.inner.cpuGauge.Percent = 0 + widget.inner.memGauge.Label = noData + widget.inner.memGauge.Percent = 0 + } else { + memUsed := nodeData.MemUsage() + + widget.inner.memGauge.Percent = int(math.Round(memUsed * 100.0)) + widget.inner.memGauge.Label = fmt.Sprintf("%.1f%%", memUsed*100.0) + + cpuUsed := nodeData.CPUUsageByName("usage") + + widget.inner.cpuGauge.Percent = int(math.Round(cpuUsed * 100.0)) + widget.inner.cpuGauge.Label = fmt.Sprintf("%.1f%%", cpuUsed*100.0) + } +} + +type systemGaugesInner struct { + ui.Block + + cpuGauge *widgets.Gauge + memGauge *widgets.Gauge +} + +// Draw implements io.Drawable. +func (widget *systemGaugesInner) Draw(buf *ui.Buffer) { + width := widget.Dx() + height := widget.Dy() + + y := 0 + itemHeight := 2 + + for _, item := range []ui.Drawable{widget.cpuGauge, widget.memGauge} { + item.SetRect(widget.Min.X, widget.Min.Y+y, widget.Min.X+width, widget.Min.Y+y+itemHeight+1) + item.Draw(buf) + + y += itemHeight + + if y > height { + break + } + } +} diff --git a/internal/pkg/dashboard/components/graphs.go b/internal/pkg/dashboard/components/graphs.go new file mode 100644 index 0000000..123e615 --- /dev/null +++ b/internal/pkg/dashboard/components/graphs.go @@ -0,0 +1,84 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package components + +import ( + "github.com/gizak/termui/v3/widgets" + "github.com/siderolabs/gen/xslices" + + "github.com/aenix-io/talm/internal/pkg/dashboard/apidata" +) + +// BaseGraph represents the widget with some usage graph. +type BaseGraph struct { + widgets.Plot +} + +// NewBaseGraph initializes BaseGraph. +func NewBaseGraph(title string, labels []string) *BaseGraph { + widget := &BaseGraph{ + Plot: *widgets.NewPlot(), + } + + widget.Border = false + widget.Title = title + widget.DataLabels = labels + widget.ShowAxes = false + // TODO: looks to be a bug as it requires at least 2 points + widget.Data = xslices.Map(labels, func(label string) []float64 { return []float64{0, 0} }) + + return widget +} + +// OnAPIDataChange implements the APIDataListener interface. +func (widget *BaseGraph) OnAPIDataChange(node string, data *apidata.Data) { + nodeData := data.Nodes[node] + + if nodeData == nil { + for i := range widget.Data { + widget.Data[i] = []float64{0, 0} + } + + return + } + + width := widget.Inner.Dx() + + for i, name := range widget.DataLabels { + series := nodeData.Series[name] + + if len(series) < width { + width = len(series) + } + + widget.Data[i] = widget.leftPadSeries(series[len(series)-width:], 2) + } +} + +func (widget *BaseGraph) leftPadSeries(series []float64, size int) []float64 { + if len(series) >= size { + return series + } + + padded := make([]float64, size) + copy(padded[size-len(series):], series) + + return padded +} + +// NewCPUGraph creates CPU usage graph. +func NewCPUGraph() *BaseGraph { + return NewBaseGraph("CPU USER/SYSTEM", []string{"user", "system"}) +} + +// NewMemGraph creates mem usage graph. +func NewMemGraph() *BaseGraph { + return NewBaseGraph("MEM USED", []string{"mem"}) +} + +// NewLoadAvgGraph creates loadavg graph. +func NewLoadAvgGraph() *BaseGraph { + return NewBaseGraph("LOAD AVG 60sec", []string{"loadavg"}) +} diff --git a/internal/pkg/dashboard/components/header.go b/internal/pkg/dashboard/components/header.go new file mode 100644 index 0000000..29bab90 --- /dev/null +++ b/internal/pkg/dashboard/components/header.go @@ -0,0 +1,185 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package components + +import ( + "fmt" + "math" + "strconv" + "time" + + "github.com/dustin/go-humanize" + "github.com/rivo/tview" + + "github.com/aenix-io/talm/internal/pkg/dashboard/apidata" + "github.com/aenix-io/talm/internal/pkg/dashboard/resourcedata" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +const noHostname = "(no hostname)" + +type headerData struct { + hostname string + version string + uptime string + numCPUs string + cpuFreq string + totalMem string + numProcesses string + cpuUsagePercent string + memUsagePercent string +} + +// Header represents the top bar with host info. +type Header struct { + tview.TextView + + selectedNode string + nodeMap map[string]*headerData +} + +// NewHeader initializes Header. +func NewHeader() *Header { + header := &Header{ + TextView: *tview.NewTextView(), + nodeMap: make(map[string]*headerData), + } + + header.SetDynamicColors(true).SetText(noData) + + return header +} + +// OnNodeSelect implements the NodeSelectListener interface. +func (widget *Header) OnNodeSelect(node string) { + if node != widget.selectedNode { + widget.selectedNode = node + + widget.redraw() + } +} + +// OnResourceDataChange implements the ResourceDataListener interface. +func (widget *Header) OnResourceDataChange(data resourcedata.Data) { + nodeData := widget.getOrCreateNodeData(data.Node) + + switch res := data.Resource.(type) { //nolint:gocritic + case *network.HostnameStatus: + if data.Deleted { + nodeData.hostname = noHostname + } else { + nodeData.hostname = res.TypedSpec().Hostname + } + } + + if data.Node == widget.selectedNode { + widget.redraw() + } +} + +// OnAPIDataChange implements the APIDataListener interface. +func (widget *Header) OnAPIDataChange(node string, data *apidata.Data) { + nodeAPIData := data.Nodes[node] + + widget.updateNodeAPIData(node, nodeAPIData) + + if node == widget.selectedNode { + widget.redraw() + } +} + +func (widget *Header) humanizeCPUFrequency(mhz float64) string { + value := math.Round(mhz) + unit := "MHz" + + if mhz >= 1000 { + ghz := value / 1000 + value = math.Round(ghz*100) / 100 + unit = "GHz" + } + + return fmt.Sprintf("%s%s", humanize.Ftoa(value), unit) +} + +func (widget *Header) redraw() { + data := widget.getOrCreateNodeData(widget.selectedNode) + + text := fmt.Sprintf( + "[yellow::b]%s[-:-:-] (%s): uptime %s, %sx%s, %s RAM, PROCS %s, CPU %s, RAM %s", + data.hostname, + data.version, + data.uptime, + data.numCPUs, + data.cpuFreq, + data.totalMem, + data.numProcesses, + data.cpuUsagePercent, + data.memUsagePercent, + ) + + widget.SetText(text) +} + +func (widget *Header) updateNodeAPIData(node string, data *apidata.Node) { + nodeData := widget.getOrCreateNodeData(node) + + if data == nil { + return + } + + nodeData.cpuUsagePercent = fmt.Sprintf("%.1f%%", data.CPUUsageByName("usage")*100.0) + nodeData.memUsagePercent = fmt.Sprintf("%.1f%%", data.MemUsage()*100.0) + + if data.Hostname != nil { + nodeData.hostname = data.Hostname.GetHostname() + } + + if data.Version != nil { + nodeData.version = data.Version.GetVersion().GetTag() + } + + if data.SystemStat != nil { + nodeData.uptime = time.Since(time.Unix(int64(data.SystemStat.GetBootTime()), 0)).Round(time.Second).String() + } + + if data.CPUsInfo != nil { + numCPUs := len(data.CPUsInfo.GetCpuInfo()) + + nodeData.numCPUs = strconv.Itoa(numCPUs) + + if numCPUs > 0 { + nodeData.cpuFreq = widget.humanizeCPUFrequency(data.CPUsInfo.GetCpuInfo()[0].GetCpuMhz()) + } + } + + if data.Processes != nil { + nodeData.numProcesses = strconv.Itoa(len(data.Processes.GetProcesses())) + } + + if data.Memory != nil { + nodeData.totalMem = humanize.IBytes(data.Memory.GetMeminfo().GetMemtotal() << 10) + } +} + +func (widget *Header) getOrCreateNodeData(node string) *headerData { + data, ok := widget.nodeMap[node] + if !ok { + data = &headerData{ + hostname: notAvailable, + version: notAvailable, + uptime: notAvailable, + numCPUs: notAvailable, + cpuFreq: notAvailable, + totalMem: notAvailable, + numProcesses: notAvailable, + cpuUsagePercent: notAvailable, + memUsagePercent: notAvailable, + } + + widget.nodeMap[node] = data + } + + return data +} diff --git a/internal/pkg/dashboard/components/horizontalline.go b/internal/pkg/dashboard/components/horizontalline.go new file mode 100644 index 0000000..1a89882 --- /dev/null +++ b/internal/pkg/dashboard/components/horizontalline.go @@ -0,0 +1,35 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package components + +import ( + "github.com/gdamore/tcell/v2" + "github.com/rivo/tview" +) + +// HorizontalLine is a widget that draws a horizontal line. +type HorizontalLine struct { + tview.TextView +} + +// NewHorizontalLine initializes HorizontalLine. +func NewHorizontalLine() *HorizontalLine { + widget := &HorizontalLine{ + TextView: *tview.NewTextView(), + } + + // set the background to be a horizontal line + widget.SetDrawFunc(func(screen tcell.Screen, x, y, width, height int) (int, int, int, int) { + for i := x; i < x+width; i++ { + for j := y; j < y+height; j++ { + screen.SetContent(i, j, tview.BoxDrawingsLightHorizontal, nil, tcell.StyleDefault.Foreground(tcell.ColorWhite)) + } + } + + return x, y, width, height + }) + + return widget +} diff --git a/internal/pkg/dashboard/components/info.go b/internal/pkg/dashboard/components/info.go new file mode 100644 index 0000000..022b386 --- /dev/null +++ b/internal/pkg/dashboard/components/info.go @@ -0,0 +1,173 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package components + +import ( + "fmt" + + "github.com/dustin/go-humanize" + "github.com/rivo/tview" + + "github.com/aenix-io/talm/internal/pkg/dashboard/apidata" +) + +// LoadAvgInfo represents the widget with load average info. +type LoadAvgInfo struct { + tview.TextView +} + +// NewLoadAvgInfo initializes LoadAvgInfo. +func NewLoadAvgInfo() *LoadAvgInfo { + widget := &LoadAvgInfo{ + TextView: *tview.NewTextView(), + } + + widget.SetBorder(false) + widget.SetBorderPadding(1, 0, 1, 0) + widget.SetDynamicColors(true) + + return widget +} + +// OnAPIDataChange implements the APIDataListener interface. +func (widget *LoadAvgInfo) OnAPIDataChange(node string, data *apidata.Data) { + nodeData := data.Nodes[node] + + if nodeData == nil { + widget.SetText(noData) + } else { + widget.SetText(fmt.Sprintf( + "[::b]LOAD[::-]\n"+ + "1 min [::b]%6.2f[::-]\n"+ + "5 min [::b]%6.2f[::-]\n"+ + "15 min [::b]%6.2f[::-]", + nodeData.LoadAvg.GetLoad1(), + nodeData.LoadAvg.GetLoad5(), + nodeData.LoadAvg.GetLoad15(), + )) + } +} + +// ProcsInfo represents the widget with processes info. +type ProcsInfo struct { + tview.TextView +} + +// NewProcsInfo initializes ProcsInfo. +func NewProcsInfo() *ProcsInfo { + widget := &ProcsInfo{ + TextView: *tview.NewTextView(), + } + + widget.SetBorder(false) + widget.SetBorderPadding(1, 0, 1, 0) + widget.SetDynamicColors(true) + + return widget +} + +// OnAPIDataChange implements the APIDataListener interface. +func (widget *ProcsInfo) OnAPIDataChange(node string, data *apidata.Data) { + nodeData := data.Nodes[node] + + if nodeData == nil { + widget.SetText(noData) + } else { + procsCreated, suffix := humanize.ComputeSI(float64(nodeData.ProcsCreated())) + + widget.SetText(fmt.Sprintf( + "[::b]PROCS[::-]\n"+ + "Created [::b]%5.1f%s[::-]\n"+ + "Running [::b]%5d[::-]\n"+ + "Blocked [::b]%5d[::-]", + procsCreated, suffix, + nodeData.SystemStat.GetProcessRunning(), + nodeData.SystemStat.GetProcessBlocked(), + )) + } +} + +// MemInfo represents the widget with memory info. +type MemInfo struct { + tview.TextView +} + +// NewMemInfo initializes LoadAvgInfo. +func NewMemInfo() *MemInfo { + widget := &MemInfo{ + TextView: *tview.NewTextView(), + } + + widget.SetBorder(false) + widget.SetBorderPadding(1, 0, 1, 0) + widget.SetDynamicColors(true) + + return widget +} + +// OnAPIDataChange implements the APIDataListener interface. +func (widget *MemInfo) OnAPIDataChange(node string, data *apidata.Data) { + nodeData := data.Nodes[node] + + if nodeData == nil { + widget.SetText(noData) + } else { + widget.SetText(fmt.Sprintf( + "[::b]MEMORY[::-]\n"+ + "Total [::b]%8s[::-] Buffers [::b]%8s[::-]\n"+ + "Used [::b]%8s[::-] Cache [::b]%8s[::-]\n"+ + "Free [::b]%8s[::-] Avail [::b]%8s[::-]\n"+ + "Shared [::b]%8s[::-]\n", + humanize.Bytes(nodeData.Memory.GetMeminfo().GetMemtotal()<<10), + humanize.Bytes(nodeData.Memory.GetMeminfo().GetBuffers()<<10), + humanize.Bytes((nodeData.Memory.GetMeminfo().GetMemtotal()-nodeData.Memory.GetMeminfo().GetMemfree()-nodeData.Memory.GetMeminfo().GetCached()-nodeData.Memory.GetMeminfo().GetBuffers())<<10), + humanize.Bytes(nodeData.Memory.GetMeminfo().GetCached()<<10), + humanize.Bytes(nodeData.Memory.GetMeminfo().GetMemfree()<<10), + humanize.Bytes(nodeData.Memory.GetMeminfo().GetMemavailable()<<10), + humanize.Bytes(nodeData.Memory.GetMeminfo().GetShmem()<<10), + )) + } +} + +// CPUInfo represents the widget with CPU info. +type CPUInfo struct { + tview.TextView +} + +// NewCPUInfo initializes CPUInfo. +func NewCPUInfo() *CPUInfo { + widget := &CPUInfo{ + TextView: *tview.NewTextView(), + } + + widget.SetBorder(false) + widget.SetBorderPadding(1, 0, 1, 0) + widget.SetDynamicColors(true) + + return widget +} + +// OnAPIDataChange implements the APIDataListener interface. +func (widget *CPUInfo) OnAPIDataChange(node string, data *apidata.Data) { + nodeData := data.Nodes[node] + + if nodeData == nil { + widget.SetText(noData) + } else { + ctxSw, suffix := humanize.ComputeSI(float64(nodeData.CtxSwitches())) + + widget.SetText(fmt.Sprintf( + "[::b]CPU[::-]\n"+ + "User [::b]%5.1f%%[::-] Nice [::b]%5.1f%%[::-]\n"+ + "System [::b]%5.1f%%[::-] IRQ [::b]%5.1f%%[::-]\n"+ + "Idle [::b]%5.1f%%[::-] Iowait [::b]%5.1f%%[::-]\n"+ + "Steal [::b]%5.1f%%[::-] CtxSw [::b]%5.1f%s[::-]\n", + nodeData.CPUUsageByName("user")*100.0, nodeData.CPUUsageByName("nice")*100.0, + nodeData.CPUUsageByName("system")*100.0, nodeData.CPUUsageByName("irq")*100.0, + nodeData.CPUUsageByName("idle")*100.0, nodeData.CPUUsageByName("iowait")*100.0, + nodeData.CPUUsageByName("steal")*100.0, ctxSw, suffix, + )) + } +} diff --git a/internal/pkg/dashboard/components/kubernetesinfo.go b/internal/pkg/dashboard/components/kubernetesinfo.go new file mode 100644 index 0000000..5fd4666 --- /dev/null +++ b/internal/pkg/dashboard/components/kubernetesinfo.go @@ -0,0 +1,231 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package components + +import ( + "strings" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/rivo/tview" + "github.com/siderolabs/gen/maps" + + "github.com/aenix-io/talm/internal/pkg/dashboard/apidata" + "github.com/aenix-io/talm/internal/pkg/dashboard/resourcedata" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +type staticPodStatuses struct { + apiServer string + controllerManager string + scheduler string +} + +type kubernetesInfoData struct { + isControlPlane bool + kubernetesVersion string + kubeletStatus string + + podStatuses staticPodStatuses + staticPodStatusMap map[resource.ID]*k8s.StaticPodStatus +} + +// KubernetesInfo represents the kubernetes info widget. +type KubernetesInfo struct { + tview.TextView + + selectedNode string + nodeMap map[string]*kubernetesInfoData +} + +// NewKubernetesInfo initializes KubernetesInfo. +func NewKubernetesInfo() *KubernetesInfo { + kubernetes := &KubernetesInfo{ + TextView: *tview.NewTextView(), + nodeMap: make(map[string]*kubernetesInfoData), + } + + kubernetes.SetDynamicColors(true). + SetText(noData). + SetBorderPadding(1, 0, 1, 0) + + return kubernetes +} + +// OnNodeSelect implements the NodeSelectListener interface. +func (widget *KubernetesInfo) OnNodeSelect(node string) { + if node != widget.selectedNode { + widget.selectedNode = node + + widget.redraw() + } +} + +// OnResourceDataChange implements the ResourceDataListener interface. +func (widget *KubernetesInfo) OnResourceDataChange(data resourcedata.Data) { + widget.updateNodeData(data) + + if data.Node == widget.selectedNode { + widget.redraw() + } +} + +// OnAPIDataChange implements the APIDataListener interface. +func (widget *KubernetesInfo) OnAPIDataChange(node string, data *apidata.Data) { + nodeAPIData := data.Nodes[node] + + widget.updateNodeAPIData(node, nodeAPIData) + + if node == widget.selectedNode { + widget.redraw() + } +} + +func (widget *KubernetesInfo) updateNodeData(data resourcedata.Data) { + nodeData := widget.getOrCreateNodeData(data.Node) + + switch res := data.Resource.(type) { + case *k8s.KubeletSpec: + if data.Deleted { + nodeData.kubernetesVersion = notAvailable + } else { + imageParts := strings.Split(res.TypedSpec().Image, ":") + if len(imageParts) > 0 { + nodeData.kubernetesVersion = imageParts[len(imageParts)-1] + } + } + case *k8s.StaticPodStatus: + if data.Deleted { + delete(nodeData.staticPodStatusMap, res.Metadata().ID()) + } else { + nodeData.staticPodStatusMap[res.Metadata().ID()] = res + } + + nodeData.podStatuses = widget.staticPodStatuses(maps.Values(nodeData.staticPodStatusMap)) + case *config.MachineType: + nodeData.isControlPlane = !data.Deleted && res.MachineType() == machine.TypeControlPlane + } +} + +func (widget *KubernetesInfo) updateNodeAPIData(node string, data *apidata.Node) { + nodeData := widget.getOrCreateNodeData(node) + + if data != nil && data.ServiceList != nil { + for _, info := range data.ServiceList.GetServices() { + if info.Id == "kubelet" { + nodeData.kubeletStatus = toHealthStatus(info.GetHealth().Healthy) + + break + } + } + } +} + +func (widget *KubernetesInfo) getOrCreateNodeData(node string) *kubernetesInfoData { + nodeData, ok := widget.nodeMap[node] + if !ok { + nodeData = &kubernetesInfoData{ + kubernetesVersion: notAvailable, + kubeletStatus: notAvailable, + podStatuses: staticPodStatuses{ + apiServer: notAvailable, + controllerManager: notAvailable, + scheduler: notAvailable, + }, + staticPodStatusMap: make(map[resource.ID]*k8s.StaticPodStatus), + } + + widget.nodeMap[node] = nodeData + } + + return nodeData +} + +func (widget *KubernetesInfo) redraw() { + data := widget.getOrCreateNodeData(widget.selectedNode) + + fieldList := make([]field, 0, 5) + + fieldList = append(fieldList, + field{ + Name: "KUBERNETES", + Value: data.kubernetesVersion, + }, + field{ + Name: "KUBELET", + Value: data.kubeletStatus, + }) + + if data.isControlPlane { + fieldList = append(fieldList, + field{ + Name: "APISERVER", + Value: data.podStatuses.apiServer, + }, + field{ + Name: "CONTROLLER-MANAGER", + Value: data.podStatuses.controllerManager, + }, + field{ + Name: "SCHEDULER", + Value: data.podStatuses.scheduler, + }) + } + + fields := fieldGroup{ + fields: fieldList, + } + + widget.SetText(fields.String()) +} + +func (widget *KubernetesInfo) staticPodStatuses(statuses []*k8s.StaticPodStatus) staticPodStatuses { + result := staticPodStatuses{ + apiServer: notAvailable, + controllerManager: notAvailable, + scheduler: notAvailable, + } + + isReady := func(podStatus map[string]any) string { + conditions, conditionsOk := podStatus["conditions"] + if !conditionsOk { + return notAvailable + } + + conditionsSlc, conditionsSlcOk := conditions.([]any) + if !conditionsSlcOk { + return notAvailable + } + + for _, condition := range conditionsSlc { + conditionObj, conditionObjOk := condition.(map[string]any) + if !conditionObjOk { + return notAvailable + } + + if conditionObj["type"] == "Ready" { + return toHealthStatus(conditionObj["status"] == "True") + } + } + + return notAvailable + } + + for _, status := range statuses { + podStatus := status.TypedSpec().PodStatus + + switch { + case strings.Contains(status.Metadata().ID(), "kube-apiserver"): + result.apiServer = isReady(podStatus) + case strings.Contains(status.Metadata().ID(), "kube-controller-manager"): + result.controllerManager = isReady(podStatus) + case strings.Contains(status.Metadata().ID(), "kube-scheduler"): + result.scheduler = isReady(podStatus) + } + } + + return result +} diff --git a/internal/pkg/dashboard/components/logviewer.go b/internal/pkg/dashboard/components/logviewer.go new file mode 100644 index 0000000..44a918c --- /dev/null +++ b/internal/pkg/dashboard/components/logviewer.go @@ -0,0 +1,66 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package components + +import ( + "github.com/gdamore/tcell/v2" + "github.com/rivo/tview" +) + +// LogViewer represents the logs widget. +type LogViewer struct { + tview.Grid + logs tview.TextView +} + +// NewLogViewer initializes LogViewer. +func NewLogViewer() *LogViewer { + widget := &LogViewer{ + Grid: *tview.NewGrid(), + logs: *tview.NewTextView(), + } + + widget.logs.ScrollToEnd(). + SetDynamicColors(true). + SetMaxLines(maxLogLines). + SetText(noData). + SetBorderPadding(0, 0, 1, 1). + SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey { + _, _, _, pageSize := widget.logs.GetInnerRect() + lineOffset, columnOffset := widget.logs.GetScrollOffset() + + //nolint:exhaustive + switch event.Key() { + case tcell.KeyCtrlD: + widget.logs.ScrollTo(lineOffset+(pageSize/2), columnOffset) + + return nil + case tcell.KeyCtrlU: + widget.logs.ScrollTo(lineOffset-(pageSize/2), columnOffset) + + return nil + } + + return event + }) + + widget.SetRows(1, 0).SetColumns(0) + + widget.AddItem(NewHorizontalLine(), 0, 0, 1, 1, 0, 0, false) + widget.AddItem(&widget.logs, 1, 0, 1, 1, 0, 0, true) + + return widget +} + +// WriteLog writes the log line to the widget. +func (widget *LogViewer) WriteLog(logLine, logError string) { + if logError != "" { + logLine = "[red]" + tview.Escape(logError) + "[-]\n" + } else { + logLine = tview.Escape(logLine) + "\n" + } + + widget.logs.Write([]byte(logLine)) //nolint:errcheck +} diff --git a/internal/pkg/dashboard/components/networkinfo.go b/internal/pkg/dashboard/components/networkinfo.go new file mode 100644 index 0000000..a8650b6 --- /dev/null +++ b/internal/pkg/dashboard/components/networkinfo.go @@ -0,0 +1,273 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package components + +import ( + "net/netip" + "sort" + "strings" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/rivo/tview" + "github.com/siderolabs/gen/maps" + "github.com/siderolabs/gen/xslices" + + "github.com/aenix-io/talm/internal/pkg/dashboard/resourcedata" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +var ( + zeroPrefix = netip.Prefix{} + routedNoK8sID = network.FilteredNodeAddressID(network.NodeAddressRoutedID, k8s.NodeAddressFilterNoK8s) +) + +type networkInfoData struct { + hostname string + gateway string + connectivity string + resolvers string + timeservers string + + addresses string + nodeAddressRouted *network.NodeAddress + nodeAddressRoutedNoK8s *network.NodeAddress + + routeStatusMap map[resource.ID]*network.RouteStatus +} + +// NetworkInfo represents the network info widget. +type NetworkInfo struct { + tview.TextView + + selectedNode string + nodeMap map[string]*networkInfoData +} + +// NewNetworkInfo initializes NetworkInfo. +func NewNetworkInfo() *NetworkInfo { + component := &NetworkInfo{ + TextView: *tview.NewTextView(), + nodeMap: make(map[string]*networkInfoData), + } + + component.SetDynamicColors(true). + SetText(noData). + SetBorderPadding(1, 0, 1, 0) + + return component +} + +// OnNodeSelect implements the NodeSelectListener interface. +func (widget *NetworkInfo) OnNodeSelect(node string) { + if node != widget.selectedNode { + widget.selectedNode = node + + widget.redraw() + } +} + +// OnResourceDataChange implements the ResourceDataListener interface. +func (widget *NetworkInfo) OnResourceDataChange(data resourcedata.Data) { + widget.updateNodeData(data) + + if data.Node == widget.selectedNode { + widget.redraw() + } +} + +//nolint:gocyclo +func (widget *NetworkInfo) updateNodeData(data resourcedata.Data) { + nodeData := widget.getOrCreateNodeData(data.Node) + + switch res := data.Resource.(type) { + case *network.ResolverStatus: + if data.Deleted { + nodeData.resolvers = notAvailable + } else { + nodeData.resolvers = widget.resolvers(res) + } + case *network.TimeServerStatus: + if data.Deleted { + nodeData.timeservers = notAvailable + } else { + nodeData.timeservers = widget.timeservers(res) + } + case *network.Status: + if data.Deleted { + nodeData.connectivity = notAvailable + } else { + nodeData.connectivity = widget.connectivity(res) + } + case *network.HostnameStatus: + if data.Deleted { + nodeData.hostname = notAvailable + } else { + nodeData.hostname = res.TypedSpec().Hostname + } + case *network.RouteStatus: + if data.Deleted { + delete(nodeData.routeStatusMap, res.Metadata().ID()) + } else { + nodeData.routeStatusMap[res.Metadata().ID()] = res + } + + nodeData.gateway = widget.gateway(maps.Values(nodeData.routeStatusMap)) + case *network.NodeAddress: + widget.setAddresses(data, res) + } +} + +func (widget *NetworkInfo) getOrCreateNodeData(node string) *networkInfoData { + data, ok := widget.nodeMap[node] + if !ok { + data = &networkInfoData{ + hostname: notAvailable, + addresses: notAvailable, + gateway: notAvailable, + connectivity: notAvailable, + resolvers: notAvailable, + timeservers: notAvailable, + routeStatusMap: make(map[resource.ID]*network.RouteStatus), + } + + widget.nodeMap[node] = data + } + + return data +} + +func (widget *NetworkInfo) redraw() { + data := widget.getOrCreateNodeData(widget.selectedNode) + + fields := fieldGroup{ + fields: []field{ + { + Name: "HOST", + Value: data.hostname, + }, + { + Name: "IP", + Value: data.addresses, + }, + { + Name: "GW", + Value: data.gateway, + }, + { + Name: "CONNECTIVITY", + Value: data.connectivity, + }, + { + Name: "DNS", + Value: data.resolvers, + }, + { + Name: "NTP", + Value: data.timeservers, + }, + }, + } + + widget.SetText(fields.String()) +} + +func (widget *NetworkInfo) setAddresses(data resourcedata.Data, nodeAddress *network.NodeAddress) { + nodeData := widget.getOrCreateNodeData(data.Node) + + switch nodeAddress.Metadata().ID() { + case network.NodeAddressRoutedID: + if data.Deleted { + nodeData.nodeAddressRouted = nil + } else { + nodeData.nodeAddressRouted = nodeAddress + } + case routedNoK8sID: + if data.Deleted { + nodeData.nodeAddressRoutedNoK8s = nil + } else { + nodeData.nodeAddressRoutedNoK8s = nodeAddress + } + } + + formatIPs := func(res *network.NodeAddress) string { + if res == nil { + return notAvailable + } + + strs := xslices.Map(res.TypedSpec().Addresses, func(prefix netip.Prefix) string { + return prefix.String() + }) + + sort.Strings(strs) + + return strings.Join(strs, ", ") + } + + // if "routed-no-k8s" is available, use it + if nodeData.nodeAddressRoutedNoK8s != nil { + nodeData.addresses = formatIPs(nodeData.nodeAddressRoutedNoK8s) + + return + } + + // fallback to "routed" + nodeData.addresses = formatIPs(nodeData.nodeAddressRouted) +} + +func (widget *NetworkInfo) gateway(statuses []*network.RouteStatus) string { + var gatewaysV4, gatewaysV6 []string + + for _, status := range statuses { + gateway := status.TypedSpec().Gateway + if !gateway.IsValid() || + status.TypedSpec().Destination != zeroPrefix { + continue + } + + if gateway.Is4() { + gatewaysV4 = append(gatewaysV4, gateway.String()) + } else { + gatewaysV6 = append(gatewaysV6, gateway.String()) + } + } + + if len(gatewaysV4) == 0 && len(gatewaysV6) == 0 { + return notAvailable + } + + sort.Strings(gatewaysV4) + sort.Strings(gatewaysV6) + + return strings.Join(append(gatewaysV4, gatewaysV6...), ", ") +} + +func (widget *NetworkInfo) resolvers(status *network.ResolverStatus) string { + strs := xslices.Map(status.TypedSpec().DNSServers, func(t netip.Addr) string { + return t.String() + }) + + if len(strs) == 0 { + return none + } + + return strings.Join(strs, ", ") +} + +func (widget *NetworkInfo) timeservers(status *network.TimeServerStatus) string { + if len(status.TypedSpec().NTPServers) == 0 { + return none + } + + return strings.Join(status.TypedSpec().NTPServers, ", ") +} + +func (widget *NetworkInfo) connectivity(status *network.Status) string { + if status.TypedSpec().ConnectivityReady { + return "[green]OK[-]" + } + + return "[red]FAILED[-]" +} diff --git a/internal/pkg/dashboard/components/sparklines.go b/internal/pkg/dashboard/components/sparklines.go new file mode 100644 index 0000000..49b0368 --- /dev/null +++ b/internal/pkg/dashboard/components/sparklines.go @@ -0,0 +1,76 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package components + +import ( + ui "github.com/gizak/termui/v3" + "github.com/gizak/termui/v3/widgets" + + "github.com/aenix-io/talm/internal/pkg/dashboard/apidata" +) + +// BaseSparklineGroup represents the widget with some sparklines. +type BaseSparklineGroup struct { + widgets.SparklineGroup + + dataLabels []string +} + +// NewBaseSparklineGroup initializes BaseSparklineGroup. +func NewBaseSparklineGroup(title string, labels, dataLabels []string) *BaseSparklineGroup { + sparklines := make([]*widgets.Sparkline, len(labels)) + + for i := range sparklines { + sparklines[i] = widgets.NewSparkline() + sparklines[i].Title = labels[i] + sparklines[i].Data = []float64{0, 0} + sparklines[i].LineColor = ui.Theme.Plot.Lines[i] + } + + widget := &BaseSparklineGroup{ + SparklineGroup: *widgets.NewSparklineGroup(sparklines...), + dataLabels: dataLabels, + } + + widget.Border = false + widget.Title = title + + return widget +} + +// OnAPIDataChange implements the APIDataListener interface. +func (widget *BaseSparklineGroup) OnAPIDataChange(node string, data *apidata.Data) { + nodeData := data.Nodes[node] + + if nodeData == nil { + for i := range widget.Sparklines { + widget.Sparklines[i].Data = []float64{0, 0} + } + + return + } + + width := widget.Inner.Dx() + + for i, name := range widget.dataLabels { + series := nodeData.Series[name] + + if len(series) < width { + width = len(series) + } + + widget.Sparklines[i].Data = series[len(series)-width:] + } +} + +// NewNetSparkline creates network sparkline. +func NewNetSparkline() *BaseSparklineGroup { + return NewBaseSparklineGroup("NET", []string{"RX", "TX"}, []string{"netrxbytes", "nettxbytes"}) +} + +// NewDiskSparkline creates disk sparkline. +func NewDiskSparkline() *BaseSparklineGroup { + return NewBaseSparklineGroup("DISK", []string{"READ", "WRITE"}, []string{"diskrdsectors", "diskwrsectors"}) +} diff --git a/internal/pkg/dashboard/components/tables.go b/internal/pkg/dashboard/components/tables.go new file mode 100644 index 0000000..783856d --- /dev/null +++ b/internal/pkg/dashboard/components/tables.go @@ -0,0 +1,114 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package components + +import ( + "fmt" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/dustin/go-humanize" + ui "github.com/gizak/termui/v3" + "github.com/gizak/termui/v3/widgets" + + "github.com/aenix-io/talm/internal/pkg/dashboard/apidata" +) + +// ProcessTable represents the widget with process info. +type ProcessTable struct { + widgets.List +} + +// NewProcessTable initializes ProcessTable. +func NewProcessTable() *ProcessTable { + widget := &ProcessTable{ + List: *widgets.NewList(), + } + + widget.Border = false + widget.Title = fmt.Sprintf("%6s %1s %6s %6s %8s %8s %10s %4s %s", + "PID", + "S", + "CPU%", + "MEM%", + "VIRT", + "RES", + "TIME+", + "THR", + "COMMAND", + ) + widget.Rows = []string{ + noData, + } + widget.SelectedRowStyle = ui.NewStyle(ui.Theme.List.Text.Fg, ui.Theme.List.Text.Bg, ui.ModifierReverse) + + return widget +} + +// OnAPIDataChange implements the APIDataListener interface. +func (widget *ProcessTable) OnAPIDataChange(node string, data *apidata.Data) { + nodeData := data.Nodes[node] + + if nodeData == nil { + widget.Rows = []string{ + noData, + } + } else { + widget.Rows = widget.Rows[:0] + + totalMem := nodeData.Memory.GetMeminfo().GetMemtotal() * 1024 + if totalMem == 0 { + totalMem = 1 + } + + totalWeightedCPU := nodeData.CPUUsageByName("total_weighted") + if totalWeightedCPU == 0 { + totalWeightedCPU = 1 + } + + // All downstream logic relies on nodeData.Processes to be not nil + // Putting a check here to reduce cyclomatic complexity + if nodeData.Processes == nil { + return + } + + if nodeData.ProcsDiff != nil { + sort.Slice(nodeData.Processes.Processes, func(i, j int) bool { + proc1 := nodeData.Processes.Processes[i] + proc2 := nodeData.Processes.Processes[j] + + return nodeData.ProcsDiff[proc1.Pid].CpuTime > nodeData.ProcsDiff[proc2.Pid].CpuTime + }) + } + + for _, proc := range nodeData.Processes.Processes { + var args string + + switch { + case proc.Executable == "": + args = proc.Command + case proc.Args != "" && strings.Fields(proc.Args)[0] == filepath.Base(strings.Fields(proc.Executable)[0]): + args = strings.Replace(proc.Args, strings.Fields(proc.Args)[0], proc.Executable, 1) + default: + args = proc.Args + } + + line := fmt.Sprintf("%7d %s %6.1f %6.1f %8s %8s %10s %4d %s", + proc.GetPid(), + proc.State, + nodeData.ProcsDiff[proc.Pid].GetCpuTime()/totalWeightedCPU*100.0, + float64(proc.ResidentMemory)/float64(totalMem)*100.0, + humanize.Bytes(proc.VirtualMemory), + humanize.Bytes(proc.ResidentMemory), + time.Duration(proc.CpuTime)*time.Second, + proc.Threads, + args, + ) + widget.Rows = append(widget.Rows, line) + } + } +} diff --git a/internal/pkg/dashboard/components/tables_test.go b/internal/pkg/dashboard/components/tables_test.go new file mode 100644 index 0000000..468213b --- /dev/null +++ b/internal/pkg/dashboard/components/tables_test.go @@ -0,0 +1,39 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package components_test + +import ( + "testing" + + "github.com/aenix-io/talm/internal/pkg/dashboard/apidata" + "github.com/aenix-io/talm/internal/pkg/dashboard/components" + "github.com/siderolabs/talos/pkg/machinery/api/machine" +) + +func TestUpdate(t *testing.T) { + testProcessTable := components.NewProcessTable() + + testData := &apidata.Data{ + Nodes: map[string]*apidata.Node{ + "node1": { + Processes: &machine.Process{ + Processes: []*machine.ProcessInfo{}, + }, + ProcsDiff: map[int32]*machine.ProcessInfo{ + 1: {}, + }, + Series: map[string][]float64{}, + }, + "node2": { + ProcsDiff: map[int32]*machine.ProcessInfo{ + 1: {}, + }, + }, + }, + } + testProcessTable.OnAPIDataChange("node1", testData) + // Node2 does not have processes, without the check it panics + testProcessTable.OnAPIDataChange("node2", testData) +} diff --git a/internal/pkg/dashboard/components/talosinfo.go b/internal/pkg/dashboard/components/talosinfo.go new file mode 100644 index 0000000..a6c99bf --- /dev/null +++ b/internal/pkg/dashboard/components/talosinfo.go @@ -0,0 +1,216 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package components + +import ( + "fmt" + "strconv" + "strings" + + "github.com/rivo/tview" + + "github.com/aenix-io/talm/internal/pkg/dashboard/resourcedata" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/hardware" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +type talosInfoData struct { + uuid string + clusterName string + stage string + ready string + typ string + numMachinesText string + secureBootState string + statePartitionMountStatus string + ephemeralPartitionMountStatus string + + machineIDSet map[string]struct{} +} + +// TalosInfo represents the Talos info widget. +type TalosInfo struct { + tview.TextView + + selectedNode string + nodeMap map[string]*talosInfoData +} + +// NewTalosInfo initializes TalosInfo. +func NewTalosInfo() *TalosInfo { + widget := &TalosInfo{ + TextView: *tview.NewTextView(), + nodeMap: make(map[string]*talosInfoData), + } + + widget.SetDynamicColors(true). + SetText(noData). + SetBorderPadding(1, 0, 1, 0) + + return widget +} + +// OnNodeSelect implements the NodeSelectListener interface. +func (widget *TalosInfo) OnNodeSelect(node string) { + if node != widget.selectedNode { + widget.selectedNode = node + + widget.redraw() + } +} + +// OnResourceDataChange implements the ResourceDataListener interface. +func (widget *TalosInfo) OnResourceDataChange(data resourcedata.Data) { + widget.updateNodeData(data) + + if data.Node == widget.selectedNode { + widget.redraw() + } +} + +//nolint:gocyclo +func (widget *TalosInfo) updateNodeData(data resourcedata.Data) { + nodeData := widget.getOrCreateNodeData(data.Node) + + switch res := data.Resource.(type) { + case *hardware.SystemInformation: + if data.Deleted { + nodeData.uuid = notAvailable + } else { + nodeData.uuid = res.TypedSpec().UUID + } + case *cluster.Info: + clusterName := res.TypedSpec().ClusterName + if data.Deleted || clusterName == "" { + nodeData.clusterName = notAvailable + } else { + nodeData.clusterName = clusterName + } + case *runtime.MachineStatus: + if data.Deleted { + nodeData.stage = notAvailable + nodeData.ready = notAvailable + } else { + nodeData.stage = formatStatus(res.TypedSpec().Stage.String()) + nodeData.ready = formatStatus(res.TypedSpec().Status.Ready) + } + case *runtime.SecurityState: + if data.Deleted { + nodeData.secureBootState = notAvailable + } else { + nodeData.secureBootState = formatStatus(res.TypedSpec().SecureBoot) + } + case *runtime.MountStatus: + switch res.Metadata().ID() { + case constants.StatePartitionLabel: + if data.Deleted { + nodeData.statePartitionMountStatus = notAvailable + } else { + nodeData.statePartitionMountStatus = mountStatus(res.TypedSpec().Encrypted, res.TypedSpec().EncryptionProviders) + } + case constants.EphemeralPartitionLabel: + if data.Deleted { + nodeData.ephemeralPartitionMountStatus = notAvailable + } else { + nodeData.ephemeralPartitionMountStatus = mountStatus(res.TypedSpec().Encrypted, res.TypedSpec().EncryptionProviders) + } + } + + case *config.MachineType: + if data.Deleted { + nodeData.typ = notAvailable + } else { + nodeData.typ = res.MachineType().String() + } + case *cluster.Member: + if data.Deleted { + delete(nodeData.machineIDSet, res.Metadata().ID()) + } else { + nodeData.machineIDSet[res.Metadata().ID()] = struct{}{} + } + + nodeData.numMachinesText = strconv.Itoa(len(nodeData.machineIDSet)) + } +} + +func (widget *TalosInfo) getOrCreateNodeData(node string) *talosInfoData { + nodeData, ok := widget.nodeMap[node] + if !ok { + nodeData = &talosInfoData{ + uuid: notAvailable, + clusterName: notAvailable, + stage: notAvailable, + ready: notAvailable, + typ: notAvailable, + numMachinesText: notAvailable, + secureBootState: notAvailable, + statePartitionMountStatus: notAvailable, + ephemeralPartitionMountStatus: notAvailable, + machineIDSet: make(map[string]struct{}), + } + + widget.nodeMap[node] = nodeData + } + + return nodeData +} + +func (widget *TalosInfo) redraw() { + data := widget.getOrCreateNodeData(widget.selectedNode) + + fields := fieldGroup{ + fields: []field{ + { + Name: "UUID", + Value: data.uuid, + }, + { + Name: "CLUSTER", + Value: data.clusterName, + }, + { + Name: "STAGE", + Value: data.stage, + }, + { + Name: "READY", + Value: data.ready, + }, + { + Name: "TYPE", + Value: data.typ, + }, + { + Name: "MACHINES", + Value: data.numMachinesText, + }, + { + Name: "SECUREBOOT", + Value: data.secureBootState, + }, + { + Name: "STATE", + Value: data.statePartitionMountStatus, + }, + { + Name: "EPHEMERAL", + Value: data.ephemeralPartitionMountStatus, + }, + }, + } + + widget.SetText(fields.String()) +} + +func mountStatus(encrypted bool, providers []string) string { + if !encrypted { + return "[green]OK[-]" + } + + return fmt.Sprintf("[green]OK - encrypted[-] (%s)", strings.Join(providers, ",")) +} diff --git a/internal/pkg/dashboard/components/termui.go b/internal/pkg/dashboard/components/termui.go new file mode 100644 index 0000000..9ce545c --- /dev/null +++ b/internal/pkg/dashboard/components/termui.go @@ -0,0 +1,100 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package components + +import ( + "image" + + "github.com/gdamore/tcell/v2" + "github.com/gizak/termui/v3" + "github.com/rivo/tview" +) + +// TermUIWrapper is a custom tview component that wraps a legacy termui component and draws it. +type TermUIWrapper struct { + *tview.Box + termUIDrawable termui.Drawable +} + +// NewTermUIWrapper initializes a new TermUIWrapper. +func NewTermUIWrapper(drawable termui.Drawable) *TermUIWrapper { + return &TermUIWrapper{ + Box: tview.NewBox(), + termUIDrawable: drawable, + } +} + +// Draw implements the tview.Primitive interface. +func (w *TermUIWrapper) Draw(screen tcell.Screen) { + w.Box.DrawForSubclass(screen, w) + x, y, width, height := w.GetInnerRect() + + if width == 0 || height == 0 { + return + } + + w.termUIDrawable.SetRect(0, 0, width, height) + buf := termui.NewBuffer(w.termUIDrawable.GetRect()) + w.termUIDrawable.Draw(buf) + + for i := range width { + for j := range height { + cell := buf.GetCell(image.Point{X: i, Y: j}) + + style := w.convertStyle(cell.Style) + + screen.SetContent(i+x, j+y, cell.Rune, nil, style) + } + } +} + +// convertStyle converts termui style to tcell (tview) style. +func (w *TermUIWrapper) convertStyle(style termui.Style) tcell.Style { + fgColor := w.convertColor(style.Fg) + bgColor := w.convertColor(style.Bg) + + bold := false + if style.Modifier&termui.ModifierBold != 0 { + bold = true + } + + underline := false + if style.Modifier&termui.ModifierUnderline != 0 { + underline = true + } + + reverse := false + if style.Modifier&termui.ModifierReverse != 0 { + reverse = true + } + + return tcell.StyleDefault.Foreground(fgColor).Background(bgColor).Bold(bold).Underline(underline).Reverse(reverse) +} + +// convertColor converts termui color to tcell (tview) color. +func (w *TermUIWrapper) convertColor(color termui.Color) tcell.Color { + switch color { + case termui.ColorClear: + return tcell.ColorDefault + case termui.ColorBlack: + return tcell.ColorBlack + case termui.ColorRed: + return tcell.ColorRed + case termui.ColorGreen: + return tcell.ColorGreen + case termui.ColorYellow: + return tcell.ColorYellow + case termui.ColorBlue: + return tcell.ColorBlue + case termui.ColorMagenta: + return tcell.ColorPurple + case termui.ColorCyan: + return tcell.ColorTeal + case termui.ColorWhite: + return tcell.ColorWhite + default: + return tcell.ColorDefault + } +} diff --git a/internal/pkg/dashboard/configurl.go b/internal/pkg/dashboard/configurl.go new file mode 100644 index 0000000..83affd9 --- /dev/null +++ b/internal/pkg/dashboard/configurl.go @@ -0,0 +1,211 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package dashboard + +import ( + "context" + "fmt" + "strings" + + "github.com/rivo/tview" + "github.com/siderolabs/go-procfs/procfs" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/aenix-io/talm/internal/pkg/dashboard/resourcedata" + "github.com/aenix-io/talm/internal/pkg/meta" + "github.com/siderolabs/talos/pkg/machinery/constants" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +const unset = "[gray](unset)[-]" + +type configURLData struct { + existingCode string +} + +// ConfigURLGrid represents the config URL grid. +type ConfigURLGrid struct { + tview.Grid + + dashboard *Dashboard + + form *tview.Form + + template *tview.TextView + existingCode *tview.TextView + newCodeField *tview.InputField + infoView *tview.TextView + + selectedNode string + nodeMap map[string]*configURLData +} + +// NewConfigURLGrid returns a new config URL grid. +func NewConfigURLGrid(ctx context.Context, dashboard *Dashboard) *ConfigURLGrid { + grid := &ConfigURLGrid{ + Grid: *tview.NewGrid(), + dashboard: dashboard, + form: tview.NewForm(), + template: tview.NewTextView().SetDynamicColors(true).SetLabel("Template").SetScrollable(false), + existingCode: tview.NewTextView().SetDynamicColors(true).SetLabel("Existing Code").SetText(unset).SetSize(1, 0).SetScrollable(false), + newCodeField: tview.NewInputField().SetLabel("New Code"), + infoView: tview.NewTextView().SetDynamicColors(true).SetSize(2, 0).SetScrollable(false), + + nodeMap: make(map[string]*configURLData), + } + + grid.template.SetText(grid.readTemplateFromKernelArgs()) + + grid.SetRows(-1, 18, -1).SetColumns(-1, 72, -1) + + grid.form.SetBorder(true) + + grid.form.AddFormItem(grid.template) + grid.form.AddFormItem(grid.existingCode) + grid.form.AddFormItem(grid.newCodeField) + grid.form.AddButton("Save", func() { + ctx = nodeContext(ctx, grid.selectedNode) + + value := grid.newCodeField.GetText() + + if value == "" { + grid.infoView.SetText("[red]Error: No code entered[-]") + + return + } + + err := dashboard.cli.MetaWrite(ctx, meta.DownloadURLCode, []byte(value)) + if err != nil { + grid.infoView.SetText(fmt.Sprintf("[red]Error: %v[-]", err)) + + return + } + + grid.clearForm() + grid.dashboard.selectScreen(ScreenSummary) + }) + grid.form.AddButton("Delete", func() { + ctx = nodeContext(ctx, grid.selectedNode) + + err := dashboard.cli.MetaDelete(ctx, meta.DownloadURLCode) + if err != nil { + if status.Code(err) == codes.NotFound { + grid.clearForm() + grid.infoView.SetText("[green]Already deleted[-]") + + return + } + + grid.infoView.SetText(fmt.Sprintf("[red]Error: %v[-]", err)) + + return + } + + grid.clearForm() + grid.infoView.SetText("[green]Deleted successfully[-]") + }) + + grid.form.AddFormItem(grid.infoView) + + grid.AddItem(tview.NewBox(), 0, 0, 1, 3, 0, 0, false) + grid.AddItem(tview.NewBox(), 1, 0, 1, 1, 0, 0, false) + grid.AddItem(grid.form, 1, 1, 1, 1, 0, 0, false) + grid.AddItem(tview.NewBox(), 1, 2, 1, 1, 0, 0, false) + grid.AddItem(tview.NewBox(), 2, 0, 1, 3, 0, 0, false) + + return grid +} + +func (widget *ConfigURLGrid) readTemplateFromKernelArgs() (val string) { + defer func() { // catch potential panic from procfs.ProcCmdline() + if r := recover(); r != nil { + val = "error reading kernel args" + } + }() + + option := procfs.ProcCmdline().Get(constants.KernelParamConfig).First() + if option == nil { + return unset + } + + codeVar := fmt.Sprintf("${%s}", constants.CodeKey) + + return strings.ReplaceAll(tview.Escape(*option), codeVar, fmt.Sprintf("[green]%s[-]", codeVar)) +} + +// OnScreenSelect implements the screenSelectListener interface. +func (widget *ConfigURLGrid) onScreenSelect(active bool) { + if active { + widget.dashboard.app.SetFocus(widget.form) + } else { + widget.clearForm() + } +} + +// OnNodeSelect implements the NodeSelectListener interface. +func (widget *ConfigURLGrid) OnNodeSelect(node string) { + if node != widget.selectedNode { + widget.selectedNode = node + + widget.clearForm() + widget.redraw() + } +} + +// OnResourceDataChange implements the ResourceDataListener interface. +func (widget *ConfigURLGrid) OnResourceDataChange(data resourcedata.Data) { + widget.updateNodeData(data) + + if data.Node == widget.selectedNode { + widget.redraw() + } +} + +func (widget *ConfigURLGrid) updateNodeData(data resourcedata.Data) { + nodeData := widget.getOrCreateNodeData(data.Node) + + //nolint:gocritic + switch res := data.Resource.(type) { + case *runtimeres.MetaKey: + if res.Metadata().ID() == runtimeres.MetaKeyTagToID(meta.DownloadURLCode) { + if data.Deleted { + nodeData.existingCode = unset + } else { + val := res.TypedSpec().Value + if val == "" { + val = "(empty)" + } + + nodeData.existingCode = fmt.Sprintf("[blue]%s[-]", val) + } + } + } +} + +func (widget *ConfigURLGrid) redraw() { + data := widget.getOrCreateNodeData(widget.selectedNode) + + widget.existingCode.SetText(data.existingCode) +} + +func (widget *ConfigURLGrid) getOrCreateNodeData(node string) *configURLData { + nodeData, ok := widget.nodeMap[node] + if !ok { + nodeData = &configURLData{ + existingCode: unset, + } + + widget.nodeMap[node] = nodeData + } + + return nodeData +} + +func (widget *ConfigURLGrid) clearForm() { + widget.form.SetFocus(0) + widget.infoView.SetText("") + widget.newCodeField.SetText("") +} diff --git a/internal/pkg/dashboard/context.go b/internal/pkg/dashboard/context.go new file mode 100644 index 0000000..4d69828 --- /dev/null +++ b/internal/pkg/dashboard/context.go @@ -0,0 +1,28 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package dashboard + +import ( + "context" + + "google.golang.org/grpc/metadata" + + "github.com/siderolabs/talos/pkg/machinery/client" +) + +func nodeContext(ctx context.Context, selectedNode string) context.Context { + md, mdOk := metadata.FromOutgoingContext(ctx) + if mdOk { + md.Delete("nodes") + + ctx = metadata.NewOutgoingContext(ctx, md) + } + + if selectedNode != "" { + ctx = client.WithNode(ctx, selectedNode) + } + + return ctx +} diff --git a/internal/pkg/dashboard/dashboard.go b/internal/pkg/dashboard/dashboard.go new file mode 100644 index 0000000..f202bf3 --- /dev/null +++ b/internal/pkg/dashboard/dashboard.go @@ -0,0 +1,584 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package dashboard implements a text-based UI dashboard. +package dashboard + +import ( + "context" + "errors" + "fmt" + "net/netip" + "slices" + "strings" + "time" + + "github.com/gdamore/tcell/v2" + _ "github.com/gdamore/tcell/v2/terminfo/l/linux" // linux terminal is used when running on the machine, but not included with tcell_minimal + "github.com/gizak/termui/v3" + "github.com/rivo/tview" + "github.com/siderolabs/gen/maps" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/go-api-signature/pkg/message" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc/metadata" + + "github.com/aenix-io/talm/internal/pkg/dashboard/apidata" + "github.com/aenix-io/talm/internal/pkg/dashboard/components" + "github.com/aenix-io/talm/internal/pkg/dashboard/logdata" + "github.com/aenix-io/talm/internal/pkg/dashboard/resourcedata" + "github.com/siderolabs/talos/pkg/machinery/client" +) + +func init() { + // set background to be left as the default color of the terminal + tview.Styles.PrimitiveBackgroundColor = tcell.ColorDefault + + // set the titles of the termui (legacy) to be bold + termui.Theme.Block.Title.Modifier = termui.ModifierBold +} + +// Screen is a dashboard screen. +type Screen string + +const ( + pageMain = "main" + + // ScreenSummary is the summary screen. + ScreenSummary Screen = "Summary" + + // ScreenMonitor is the monitor (metrics) screen. + ScreenMonitor Screen = "Monitor" + + // ScreenNetworkConfig is the network configuration screen. + ScreenNetworkConfig Screen = "Network Config" + + // ScreenConfigURL is the config URL screen. + ScreenConfigURL Screen = "Config URL" +) + +// APIDataListener is a listener which is notified when API-sourced data is updated. +type APIDataListener interface { + OnAPIDataChange(node string, data *apidata.Data) +} + +// ResourceDataListener is a listener which is notified when a resource is updated. +type ResourceDataListener interface { + OnResourceDataChange(data resourcedata.Data) +} + +// LogDataListener is a listener which is notified when a log line is received. +type LogDataListener interface { + OnLogDataChange(node, logLine, logError string) +} + +// NodeSelectListener is a listener which is notified when a node is selected. +type NodeSelectListener interface { + OnNodeSelect(node string) +} + +type screenConfig struct { + screenKey string + screen Screen + keyCode tcell.Key + primitive screenSelectListener + allowNodeNavigation bool +} + +// screenSelectListener is a listener which is notified when a screen is selected. +type screenSelectListener interface { + tview.Primitive + + onScreenSelect(active bool) +} + +// Dashboard implements the summary dashboard. +type Dashboard struct { + cli *client.Client + interval time.Duration + + apiDataSource *apidata.Source + resourceDataSource *resourcedata.Source + logDataSource *logdata.Source + + apiDataListeners []APIDataListener + resourceDataListeners []ResourceDataListener + logDataListeners []LogDataListener + nodeSelectListeners []NodeSelectListener + + app *tview.Application + + mainGrid *tview.Grid + + pages *tview.Pages + + selectedScreenConfig *screenConfig + screenConfigs []screenConfig + footer *components.Footer + + data *apidata.Data + + selectedNodeIndex int + selectedNode string + nodeSet map[string]struct{} + ipsToNodeAliases map[string]string + nodes []string +} + +// buildDashboard initializes the summary dashboard. +// +//nolint:gocyclo +func buildDashboard(ctx context.Context, cli *client.Client, opts ...Option) (*Dashboard, error) { + defOptions := defaultOptions() + + for _, opt := range opts { + opt(defOptions) + } + + // map node IPs to their aliases (names/IPs - as specified "nodes" in context). + // this will also trigger the interactive API authentication if needed - e.g., when the API is used through Omni. + ipsToNodeAliases, err := collectNodeIPsToNodeAliases(ctx, cli) + if err != nil { + return nil, err + } + + nodes := getSortedNodeAliases(ipsToNodeAliases) + + dashboard := &Dashboard{ + cli: cli, + interval: defOptions.interval, + app: tview.NewApplication(), + nodeSet: make(map[string]struct{}), + nodes: nodes, + ipsToNodeAliases: ipsToNodeAliases, + } + + dashboard.mainGrid = tview.NewGrid(). + SetRows(1, 0, 1). + SetColumns(0) + + dashboard.pages = tview.NewPages().AddPage(pageMain, dashboard.mainGrid, true, true) + + dashboard.app.SetRoot(dashboard.pages, true).SetFocus(dashboard.pages) + + header := components.NewHeader() + dashboard.mainGrid.AddItem(header, 0, 0, 1, 1, 0, 0, false) + + if err = dashboard.initScreenConfigs(ctx, defOptions.screens); err != nil { + return nil, err + } + + screenKeyToName := xslices.ToMap(dashboard.screenConfigs, func(t screenConfig) (string, string) { + return t.screenKey, string(t.screen) + }) + + screenConfigByKeyCode := xslices.ToMap(dashboard.screenConfigs, func(config screenConfig) (tcell.Key, screenConfig) { + return config.keyCode, config + }) + + dashboard.footer = components.NewFooter(screenKeyToName, nodes) + + dashboard.app.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey { + config, screenOk := screenConfigByKeyCode[event.Key()] + + allowNodeNavigation := dashboard.selectedScreenConfig != nil && dashboard.selectedScreenConfig.allowNodeNavigation + + switch { + case screenOk: + dashboard.selectScreen(config.screen) + + return nil + case allowNodeNavigation && (event.Key() == tcell.KeyLeft || event.Rune() == 'h'): + dashboard.selectNodeByIndex(dashboard.selectedNodeIndex - 1) + + return nil + case allowNodeNavigation && (event.Key() == tcell.KeyRight || event.Rune() == 'l'): + dashboard.selectNodeByIndex(dashboard.selectedNodeIndex + 1) + + return nil + case defOptions.allowExitKeys && (event.Key() == tcell.KeyCtrlC || event.Rune() == 'q'): + dashboard.app.Stop() + + return nil + } + + return event + }) + + dashboard.mainGrid.AddItem(dashboard.footer, 2, 0, 1, 1, 0, 0, false) + + dashboard.apiDataListeners = []APIDataListener{ + header, + } + + dashboard.resourceDataListeners = []ResourceDataListener{ + header, + } + + dashboard.logDataListeners = []LogDataListener{} + + dashboard.nodeSelectListeners = []NodeSelectListener{ + header, + dashboard.footer, + } + + for _, config := range dashboard.screenConfigs { + screenPrimitive := config.primitive + + apiDataListener, ok := screenPrimitive.(APIDataListener) + if ok { + dashboard.apiDataListeners = append(dashboard.apiDataListeners, apiDataListener) + } + + resourceDataListener, ok := screenPrimitive.(ResourceDataListener) + if ok { + dashboard.resourceDataListeners = append(dashboard.resourceDataListeners, resourceDataListener) + } + + logDataListener, ok := screenPrimitive.(LogDataListener) + if ok { + dashboard.logDataListeners = append(dashboard.logDataListeners, logDataListener) + } + + nodeSelectListener, ok := screenPrimitive.(NodeSelectListener) + if ok { + dashboard.nodeSelectListeners = append(dashboard.nodeSelectListeners, nodeSelectListener) + } + } + + dashboard.apiDataSource = &apidata.Source{ + Client: cli, + Interval: defOptions.interval, + } + + dashboard.resourceDataSource = &resourcedata.Source{ + COSI: cli.COSI, + } + + dashboard.logDataSource = logdata.NewSource(cli) + + return dashboard, nil +} + +func (d *Dashboard) initScreenConfigs(ctx context.Context, screens []Screen) error { + primitiveForScreen := func(screen Screen) screenSelectListener { + switch screen { + case ScreenSummary: + return NewSummaryGrid(d.app) + case ScreenMonitor: + return NewMonitorGrid(d.app) + case ScreenNetworkConfig: + return NewNetworkConfigGrid(ctx, d) + case ScreenConfigURL: + return NewConfigURLGrid(ctx, d) + default: + return nil + } + } + + d.screenConfigs = make([]screenConfig, 0, len(screens)) + + for i, screen := range screens { + primitive := primitiveForScreen(screen) + if primitive == nil { + return fmt.Errorf("unknown screen %s", screen) + } + + config := screenConfig{ + screenKey: fmt.Sprintf("F%d", i+1), + screen: screen, + keyCode: tcell.KeyF1 + tcell.Key(i), + primitive: primitive, + allowNodeNavigation: true, + } + + if screen == ScreenNetworkConfig || screen == ScreenConfigURL { + config.allowNodeNavigation = false + } + + d.screenConfigs = append(d.screenConfigs, config) + } + + return nil +} + +// Run starts the dashboard. +func Run(ctx context.Context, cli *client.Client, opts ...Option) (runErr error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + dashboard, err := buildDashboard(ctx, cli, opts...) + if err != nil { + return err + } + + dashboard.selectNodeByIndex(0) + + // handle panic & stop dashboard gracefully on exit + defer func() { + if r := recover(); r != nil { + runErr = fmt.Errorf("dashboard panic: %v", r) + } + + dashboard.app.Stop() + }() + + dashboard.selectScreen(ScreenSummary) + + eg, ctx := errgroup.WithContext(ctx) + + stopFunc := dashboard.startDataHandler(ctx) + defer stopFunc() //nolint:errcheck + + eg.Go(func() error { + defer cancel() + + return dashboard.app.Run() + }) + + // stop dashboard when the context is canceled + eg.Go(func() error { + <-ctx.Done() + + dashboard.app.Stop() + + return nil + }) + + return eg.Wait() +} + +// startDataHandler starts the data and log update handler and returns a function to stop it. +func (d *Dashboard) startDataHandler(ctx context.Context) func() error { + var eg errgroup.Group + + ctx, cancel := context.WithCancel(ctx) + + stopFunc := func() error { + cancel() + + err := eg.Wait() + if errors.Is(err, context.Canceled) { + return nil + } + + return err + } + + eg.Go(func() error { + // start API data source + dataCh := d.apiDataSource.Run(ctx) + defer d.apiDataSource.Stop() + + // start resources data source + d.resourceDataSource.Run(ctx) + defer d.resourceDataSource.Stop() //nolint:errcheck + + // start logs data source + d.logDataSource.Start(ctx) + defer d.logDataSource.Stop() //nolint:errcheck + + lastLogTime := time.Now() + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case nodeLog := <-d.logDataSource.LogCh: + nodeAlias := d.attemptResolveIPToAlias(nodeLog.Node) + + if time.Since(lastLogTime) < 50*time.Millisecond { + d.app.QueueUpdate(func() { + d.processLog(nodeAlias, nodeLog.Log, nodeLog.Error) + }) + } else { + d.app.QueueUpdateDraw(func() { + d.processLog(nodeAlias, nodeLog.Log, nodeLog.Error) + }) + } + + lastLogTime = time.Now() + case d.data = <-dataCh: + d.data.Nodes = maps.Map(d.data.Nodes, func(key string, v *apidata.Node) (string, *apidata.Node) { + return d.attemptResolveIPToAlias(key), v + }) + + d.app.QueueUpdateDraw(func() { + d.processAPIData() + }) + case nodeResource := <-d.resourceDataSource.NodeResourceCh: + d.app.QueueUpdateDraw(func() { + d.processNodeResource(nodeResource) + }) + } + } + }) + + return stopFunc +} + +func (d *Dashboard) selectNodeByIndex(index int) { + if len(d.nodes) == 0 { + return + } + + if index < 0 { + index = 0 + } else if index >= len(d.nodes) { + index = len(d.nodes) - 1 + } + + d.selectedNode = d.nodes[index] + d.selectedNodeIndex = index + + d.processAPIData() + + for _, listener := range d.nodeSelectListeners { + listener.OnNodeSelect(d.selectedNode) + } +} + +// processAPIData re-renders the components with new API-sourced data. +func (d *Dashboard) processAPIData() { + if d.data == nil { + return + } + + for _, component := range d.apiDataListeners { + component.OnAPIDataChange(d.selectedNode, d.data) + } +} + +// processNodeResource re-renders the components with new resource data. +func (d *Dashboard) processNodeResource(nodeResource resourcedata.Data) { + for _, component := range d.resourceDataListeners { + component.OnResourceDataChange(nodeResource) + } +} + +// processLog re-renders the log components with new log data. +func (d *Dashboard) processLog(node, logLine, logError string) { + for _, component := range d.logDataListeners { + component.OnLogDataChange(node, logLine, logError) + } +} + +func (d *Dashboard) selectScreen(screen Screen) { + for _, info := range d.screenConfigs { + if info.screen == screen { + d.selectedScreenConfig = &info //nolint:exportloopref + + d.mainGrid.AddItem(info.primitive, 1, 0, 1, 1, 0, 0, false) + + info.primitive.onScreenSelect(true) + + continue + } + + d.mainGrid.RemoveItem(info.primitive) + info.primitive.onScreenSelect(false) + } + + d.footer.SelectScreen(string(screen)) +} + +// attemptResolveIPToAlias attempts to resolve the given node IP to its alias as it appears in "nodes" in the context. +// If the IP is not found in the context, the IP is returned as-is. +func (d *Dashboard) attemptResolveIPToAlias(node string) string { + if resolved, ok := d.ipsToNodeAliases[node]; ok { + return resolved + } + + return node +} + +// collectNodeIPsToNodeAliases probes all nodes in the context for their IP addresses by calling their .Version endpoint and maps them to the node aliases in the context. +// +// Sample output: +// +// 172.20.0.6 -> node-1 +// +// 10.42.0.1 -> node-1 +// +// 172.20.0.7 -> node-2 +// +// 10.42.0.2 -> node-2. +func collectNodeIPsToNodeAliases(ctx context.Context, c *client.Client) (map[string]string, error) { + ipsToNodeAliases := make(map[string]string) + + nodes := nodeAliasesInContext(ctx) + for _, node := range nodes { + ctx = client.WithNodes(ctx, node) // do not replace this with "WithNode" - it would not return the IP in the response metadata + + resp, err := c.Version(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get node %q version: %w", node, err) + } + + if len(resp.GetMessages()) == 0 { + return nil, fmt.Errorf("node %q returned no messages in version response", node) + } + + nodeIP := resp.GetMessages()[0].GetMetadata().GetHostname() + if nodeIP == "" { + return nil, fmt.Errorf("node %q returned no IP in version response", node) + } + + ipsToNodeAliases[nodeIP] = node + } + + return ipsToNodeAliases, nil +} + +// nodeAliasesInContext extracts the node aliases (IP, name etc.) from the given context which are stored in the "node" or "nodes" GRPC metadata. +func nodeAliasesInContext(ctx context.Context) []string { + md, mdOk := metadata.FromOutgoingContext(ctx) + if !mdOk { + return nil + } + + nodeVal := md.Get("node") + if len(nodeVal) > 0 { + return []string{nodeVal[0]} + } + + nodesVal := md.Get(message.NodesHeaderKey) + + return xslices.FlatMap(nodesVal, func(node string) []string { + return strings.Split(node, ",") + }) +} + +// getSortedNodeAliases returns the unique node aliases sorted by their IP address. +func getSortedNodeAliases(ipToNodeAliases map[string]string) []string { + if len(ipToNodeAliases) == 0 { // assume that it is the local node (running on TTY) + return []string{""} + } + + nodeAliases := maps.Keys(xslices.ToSet(maps.Values(ipToNodeAliases))) // eliminate duplicates + + // if the aliases are IP addresses, compare them as IPs + // otherwise, compare them as strings + // all IPs come before non-IPs + slices.SortFunc(nodeAliases, func(a, b string) int { + addrA, aErr := netip.ParseAddr(a) + addrB, bErr := netip.ParseAddr(b) + + if aErr != nil && bErr != nil { + return strings.Compare(a, b) + } + + if aErr != nil { + return 1 + } + + if bErr != nil { + return -1 + } + + return addrA.Compare(addrB) + }) + + return nodeAliases +} diff --git a/internal/pkg/dashboard/formdata.go b/internal/pkg/dashboard/formdata.go new file mode 100644 index 0000000..1718455 --- /dev/null +++ b/internal/pkg/dashboard/formdata.go @@ -0,0 +1,246 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package dashboard + +import ( + "errors" + "fmt" + "net/netip" + "strings" + "unicode" + + "github.com/hashicorp/go-multierror" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +const ( + interfaceNone = "(none)" + + // ModeDHCP is the DHCP mode for the link. + ModeDHCP = "DHCP" + + // ModeStatic is the static IP mode for the link. + ModeStatic = "Static" +) + +// NetworkConfigFormData is the form data for the network config. +type NetworkConfigFormData struct { + Base runtime.PlatformNetworkConfig + Hostname string + DNSServers string + TimeServers string + Iface string + Mode string + Addresses string + Gateway string +} + +// ToPlatformNetworkConfig converts the form data to a PlatformNetworkConfig. +// +//nolint:gocyclo +func (formData *NetworkConfigFormData) ToPlatformNetworkConfig() (*runtime.PlatformNetworkConfig, error) { + var errs error + + config := &formData.Base + + // zero-out the fields managed by the form + config.Hostnames = nil + config.Resolvers = nil + config.TimeServers = nil + config.Links = nil + config.Operators = nil + config.Addresses = nil + config.Routes = nil + + if formData.Hostname != "" { + config.Hostnames = []network.HostnameSpecSpec{ + { + Hostname: formData.Hostname, + ConfigLayer: network.ConfigPlatform, + }, + } + } + + dnsServers, err := formData.parseAddresses(formData.DNSServers) + if err != nil { + errs = multierror.Append(errs, fmt.Errorf("failed to parse DNS servers: %w", err)) + } + + if len(dnsServers) > 0 { + config.Resolvers = []network.ResolverSpecSpec{ + { + DNSServers: dnsServers, + ConfigLayer: network.ConfigPlatform, + }, + } + } + + timeServers := formData.splitInputList(formData.TimeServers) + + if len(timeServers) > 0 { + config.TimeServers = []network.TimeServerSpecSpec{ + { + NTPServers: timeServers, + ConfigLayer: network.ConfigPlatform, + }, + } + } + + ifaceSelected := formData.Iface != "" && formData.Iface != interfaceNone + if ifaceSelected { + config.Links = []network.LinkSpecSpec{ + { + Name: formData.Iface, + Logical: false, + Up: true, + Type: nethelpers.LinkEther, + ConfigLayer: network.ConfigPlatform, + }, + } + + switch formData.Mode { + case ModeDHCP: + config.Operators = []network.OperatorSpecSpec{ + { + Operator: network.OperatorDHCP4, + LinkName: formData.Iface, + RequireUp: true, + DHCP4: network.DHCP4OperatorSpec{ + RouteMetric: 1024, + }, + ConfigLayer: network.ConfigPlatform, + }, + } + case ModeStatic: + config.Addresses, err = formData.buildAddresses(formData.Iface) + if err != nil { + errs = multierror.Append(errs, err) + } + + if len(config.Addresses) == 0 { + errs = multierror.Append(errs, errors.New("no addresses specified")) + } + + config.Routes, err = formData.buildRoutes(formData.Iface) + if err != nil { + errs = multierror.Append(errs, err) + } + } + } + + if errs != nil { + return nil, errs + } + + return config, nil +} + +func (formData *NetworkConfigFormData) parseAddresses(text string) ([]netip.Addr, error) { + var errs error + + split := formData.splitInputList(text) + addresses := make([]netip.Addr, 0, len(split)) + + for _, address := range split { + addr, err := netip.ParseAddr(address) + if err != nil { + errs = multierror.Append(errs, fmt.Errorf("address: %w", err)) + + continue + } + + addresses = append(addresses, addr) + } + + if errs != nil { + return nil, errs + } + + return addresses, nil +} + +func (formData *NetworkConfigFormData) buildAddresses(linkName string) ([]network.AddressSpecSpec, error) { + var errs error + + addressesSplit := formData.splitInputList(formData.Addresses) + addresses := make([]network.AddressSpecSpec, 0, len(addressesSplit)) + + for _, address := range addressesSplit { + prefix, err := netip.ParsePrefix(address) + if err != nil { + errs = multierror.Append(errs, err) + + continue + } + + ipFamily := nethelpers.FamilyInet4 + if prefix.Addr().Is6() { + ipFamily = nethelpers.FamilyInet6 + } + + addresses = append(addresses, network.AddressSpecSpec{ + Address: prefix, + LinkName: linkName, + Family: ipFamily, + Scope: nethelpers.ScopeGlobal, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + ConfigLayer: network.ConfigPlatform, + }) + } + + if errs != nil { + return nil, errs + } + + return addresses, nil +} + +func (formData *NetworkConfigFormData) buildRoutes(linkName string) ([]network.RouteSpecSpec, error) { + gateway := strings.TrimSpace(formData.Gateway) + + gatewayAddr, err := netip.ParseAddr(gateway) + if err != nil { + return nil, fmt.Errorf("gateway: %w", err) + } + + family := nethelpers.FamilyInet4 + if gatewayAddr.Is6() { + family = nethelpers.FamilyInet6 + } + + return []network.RouteSpecSpec{ + { + Family: family, + Gateway: gatewayAddr, + OutLinkName: linkName, + Table: nethelpers.TableMain, + Scope: nethelpers.ScopeGlobal, + Type: nethelpers.TypeUnicast, + Protocol: nethelpers.ProtocolStatic, + ConfigLayer: network.ConfigPlatform, + }, + }, nil +} + +func (formData *NetworkConfigFormData) splitInputList(text string) []string { + parts := strings.FieldsFunc(text, func(r rune) bool { + return r == ',' || unicode.IsSpace(r) + }) + + result := make([]string, 0, len(parts)) + + for _, part := range parts { + trimmed := strings.TrimSpace(part) + + if trimmed != "" { + result = append(result, part) + } + } + + return result +} diff --git a/internal/pkg/dashboard/formdata_test.go b/internal/pkg/dashboard/formdata_test.go new file mode 100644 index 0000000..1188c3e --- /dev/null +++ b/internal/pkg/dashboard/formdata_test.go @@ -0,0 +1,245 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package dashboard_test + +import ( + "net/netip" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/pkg/dashboard" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +func TestEmptyFormData(t *testing.T) { + formData := dashboard.NetworkConfigFormData{} + + config, err := formData.ToPlatformNetworkConfig() + assert.NoError(t, err) + + assert.Equal(t, runtime.PlatformNetworkConfig{}, *config) +} + +func TestBaseDataZeroOut(t *testing.T) { + base := runtime.PlatformNetworkConfig{ + Addresses: []network.AddressSpecSpec{ + { + LinkName: "foobar", + }, + }, + Links: []network.LinkSpecSpec{ + { + Name: "foobar", + }, + }, + Routes: []network.RouteSpecSpec{ + { + OutLinkName: "foobar", + }, + }, + Hostnames: []network.HostnameSpecSpec{ + { + Hostname: "foobar", + }, + }, + Resolvers: []network.ResolverSpecSpec{ + { + DNSServers: []netip.Addr{ + netip.MustParseAddr("1.2.3.4"), + }, + }, + }, + TimeServers: []network.TimeServerSpecSpec{ + { + NTPServers: []string{"foobar"}, + }, + }, + Operators: []network.OperatorSpecSpec{ + { + LinkName: "foobar", + }, + }, + ExternalIPs: []netip.Addr{ + netip.MustParseAddr("2.3.4.5"), + }, + Metadata: &runtimeres.PlatformMetadataSpec{ + Platform: "foobar", + Spot: true, + }, + } + + formData := dashboard.NetworkConfigFormData{ + Base: base, + } + + config, err := formData.ToPlatformNetworkConfig() + assert.NoError(t, err) + + // assert that the fields managed by the form are zeroed out + assert.Empty(t, config.Addresses) + assert.Empty(t, config.Links) + assert.Empty(t, config.Routes) + assert.Empty(t, config.Hostnames) + assert.Empty(t, config.Resolvers) + assert.Empty(t, config.TimeServers) + assert.Empty(t, config.Operators) + + // assert that the fields not managed by the form are untouched + assert.Equal(t, base.ExternalIPs, config.ExternalIPs) + assert.Equal(t, base.Metadata, config.Metadata) +} + +func TestFilledFormNoIface(t *testing.T) { + formData := dashboard.NetworkConfigFormData{ + Base: runtime.PlatformNetworkConfig{ + Metadata: &runtimeres.PlatformMetadataSpec{ + Platform: "foobar", + }, + }, + Hostname: "foobar", + DNSServers: "1.2.3.4 5.6.7.8", + TimeServers: "a.example.com , b.example.com", + } + + config, err := formData.ToPlatformNetworkConfig() + assert.NoError(t, err) + + assert.Equal( + t, + runtime.PlatformNetworkConfig{ + Hostnames: []network.HostnameSpecSpec{{ + Hostname: "foobar", + ConfigLayer: network.ConfigPlatform, + }}, + Resolvers: []network.ResolverSpecSpec{{ + DNSServers: []netip.Addr{ + netip.MustParseAddr("1.2.3.4"), + netip.MustParseAddr("5.6.7.8"), + }, + ConfigLayer: network.ConfigPlatform, + }}, + TimeServers: []network.TimeServerSpecSpec{ + { + NTPServers: []string{"a.example.com", "b.example.com"}, + ConfigLayer: network.ConfigPlatform, + }, + }, + Metadata: &runtimeres.PlatformMetadataSpec{ + Platform: "foobar", + }, + }, + *config, + ) +} + +func TestFilledFormModeDHCP(t *testing.T) { + formData := dashboard.NetworkConfigFormData{ + Iface: "eth0", + Mode: dashboard.ModeDHCP, + } + + config, err := formData.ToPlatformNetworkConfig() + assert.NoError(t, err) + + assert.Equal(t, runtime.PlatformNetworkConfig{ + Links: []network.LinkSpecSpec{ + { + Name: formData.Iface, + Logical: false, + Up: true, + Type: nethelpers.LinkEther, + ConfigLayer: network.ConfigPlatform, + }, + }, + Operators: []network.OperatorSpecSpec{ + { + Operator: network.OperatorDHCP4, + LinkName: formData.Iface, + RequireUp: true, + DHCP4: network.DHCP4OperatorSpec{ + RouteMetric: 1024, + }, + ConfigLayer: network.ConfigPlatform, + }, + }, + }, *config) +} + +func TestFilledFormModeStaticNoAddresses(t *testing.T) { + formData := dashboard.NetworkConfigFormData{ + Iface: "eth0", + Mode: dashboard.ModeStatic, + } + + _, err := formData.ToPlatformNetworkConfig() + assert.ErrorContains(t, err, "no addresses specified") +} + +func TestFilledFormModeStaticNoGateway(t *testing.T) { + formData := dashboard.NetworkConfigFormData{ + Iface: "eth0", + Mode: dashboard.ModeStatic, + Addresses: "1.2.3.4/24", + } + + _, err := formData.ToPlatformNetworkConfig() + assert.ErrorContains(t, err, "unable to parse") +} + +func TestFilledFormModeStatic(t *testing.T) { + formData := dashboard.NetworkConfigFormData{ + Iface: "eth42", + Mode: dashboard.ModeStatic, + Addresses: "1.2.3.4/24 2.3.4.5/32", + Gateway: "3.4.5.6", + } + + config, err := formData.ToPlatformNetworkConfig() + assert.NoError(t, err) + + assert.Equal(t, runtime.PlatformNetworkConfig{ + Links: []network.LinkSpecSpec{ + { + Name: formData.Iface, + Logical: false, + Up: true, + Type: nethelpers.LinkEther, + ConfigLayer: network.ConfigPlatform, + }, + }, + Addresses: []network.AddressSpecSpec{ + { + Address: netip.MustParsePrefix("1.2.3.4/24"), + LinkName: formData.Iface, + Family: nethelpers.FamilyInet4, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + ConfigLayer: network.ConfigPlatform, + }, + { + Address: netip.MustParsePrefix("2.3.4.5/32"), + LinkName: formData.Iface, + Family: nethelpers.FamilyInet4, + Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent), + ConfigLayer: network.ConfigPlatform, + }, + }, + Routes: []network.RouteSpecSpec{ + { + Family: nethelpers.FamilyInet4, + Gateway: netip.MustParseAddr("3.4.5.6"), + OutLinkName: "eth42", + Table: nethelpers.TableMain, + Scope: nethelpers.ScopeGlobal, + Type: nethelpers.TypeUnicast, + Protocol: nethelpers.ProtocolStatic, + ConfigLayer: network.ConfigPlatform, + }, + }, + }, *config) +} diff --git a/internal/pkg/dashboard/logdata/logdata.go b/internal/pkg/dashboard/logdata/logdata.go new file mode 100644 index 0000000..e81efd2 --- /dev/null +++ b/internal/pkg/dashboard/logdata/logdata.go @@ -0,0 +1,133 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package logdata implements the types and the data sources for the data sourced from the Talos dmesg API. +package logdata + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "time" + + "golang.org/x/sync/errgroup" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" + "github.com/aenix-io/talm/internal/pkg/dashboard/util" + "github.com/siderolabs/talos/pkg/machinery/api/common" + "github.com/siderolabs/talos/pkg/machinery/client" +) + +// Data is a log line from a node. +type Data struct { + Node string + Log string + Error string +} + +// Source is a data source for Kernel (dmesg) logs. +type Source struct { + client *client.Client + + logCtxCancel context.CancelFunc + + eg errgroup.Group + once sync.Once + + LogCh chan Data +} + +// NewSource initializes and returns Source data source. +func NewSource(client *client.Client) *Source { + return &Source{ + client: client, + LogCh: make(chan Data), + } +} + +// Start starts the data source. +func (source *Source) Start(ctx context.Context) { + source.once.Do(func() { + source.start(ctx) + }) +} + +// Stop stops the data source. +func (source *Source) Stop() error { + source.logCtxCancel() + + return source.eg.Wait() +} + +func (source *Source) start(ctx context.Context) { + ctx, source.logCtxCancel = context.WithCancel(ctx) + + for _, nodeContext := range util.NodeContexts(ctx) { + source.eg.Go(func() error { + return source.tailNodeWithRetries(nodeContext.Ctx, nodeContext.Node) + }) + } +} + +func (source *Source) tailNodeWithRetries(ctx context.Context, node string) error { + for { + readErr := source.readDmesg(ctx, node) + if errors.Is(readErr, context.Canceled) || status.Code(readErr) == codes.Canceled { + return nil + } + + if readErr != nil { + source.LogCh <- Data{Node: node, Error: readErr.Error()} + } + + // back off a bit before retrying + sleepWithContext(ctx, 30*time.Second) + } +} + +func (source *Source) readDmesg(ctx context.Context, node string) error { + dmesgStream, err := source.client.Dmesg(ctx, true, false) + if err != nil { + return fmt.Errorf("dashboard: error opening dmesg stream: %w", err) + } + + readErr := helpers.ReadGRPCStream(dmesgStream, func(data *common.Data, _ string, _ bool) error { + if len(data.Bytes) == 0 { + return nil + } + + line := strings.TrimSpace(string(data.Bytes)) + if line == "" { + return nil + } + + select { + case <-ctx.Done(): + return ctx.Err() + case source.LogCh <- Data{Node: node, Log: line}: + } + + return nil + }) + if readErr != nil { + return fmt.Errorf("error reading dmesg stream: %w", readErr) + } + + return nil +} + +func sleepWithContext(ctx context.Context, d time.Duration) { + timer := time.NewTimer(d) + select { + case <-ctx.Done(): + if !timer.Stop() { + <-timer.C + } + case <-timer.C: + } +} diff --git a/internal/pkg/dashboard/monitor.go b/internal/pkg/dashboard/monitor.go new file mode 100644 index 0000000..30c3230 --- /dev/null +++ b/internal/pkg/dashboard/monitor.go @@ -0,0 +1,129 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package dashboard + +import ( + "github.com/gdamore/tcell/v2" + "github.com/rivo/tview" + + "github.com/aenix-io/talm/internal/pkg/dashboard/apidata" + "github.com/aenix-io/talm/internal/pkg/dashboard/components" +) + +// MonitorGrid represents the monitoring grid with a process table and various metrics. +type MonitorGrid struct { + tview.Grid + + app *tview.Application + + apiDataListeners []APIDataListener + + processTableInner *components.ProcessTable + processTable *components.TermUIWrapper +} + +// NewMonitorGrid initializes MonitorGrid. +func NewMonitorGrid(app *tview.Application) *MonitorGrid { + widget := &MonitorGrid{ + app: app, + Grid: *tview.NewGrid(), + } + + widget.SetRows(7, -1, -2).SetColumns(0) + + infoGrid := tview.NewGrid().SetRows(0).SetColumns(-1, -2, -1, -1, -2) + + sysGauges := components.NewSystemGauges() + cpuInfo := components.NewCPUInfo() + loadAvgInfo := components.NewLoadAvgInfo() + procsInfo := components.NewProcsInfo() + memInfo := components.NewMemInfo() + + infoGrid.AddItem(sysGauges, 0, 0, 1, 1, 0, 0, false) + infoGrid.AddItem(cpuInfo, 0, 1, 1, 1, 0, 0, false) + infoGrid.AddItem(loadAvgInfo, 0, 2, 1, 1, 0, 0, false) + infoGrid.AddItem(procsInfo, 0, 3, 1, 1, 0, 0, false) + infoGrid.AddItem(memInfo, 0, 4, 1, 1, 0, 0, false) + + graphGrid := tview.NewGrid().SetRows(0).SetColumns(0, 0, 0) + + cpuGraph := components.NewCPUGraph() + memGraph := components.NewMemGraph() + loadAvgGraph := components.NewLoadAvgGraph() + + graphGrid.AddItem(components.NewTermUIWrapper(cpuGraph), 0, 0, 1, 1, 0, 0, false) + graphGrid.AddItem(components.NewTermUIWrapper(memGraph), 0, 1, 1, 1, 0, 0, false) + graphGrid.AddItem(components.NewTermUIWrapper(loadAvgGraph), 0, 2, 1, 1, 0, 0, false) + + bottomGrid := tview.NewGrid().SetRows(0, 0).SetColumns(-1, -3) + + netSparkline := components.NewNetSparkline() + diskSparkline := components.NewDiskSparkline() + + widget.initProcessTable() + + bottomGrid.AddItem(components.NewTermUIWrapper(netSparkline), 0, 0, 1, 1, 0, 0, false) + bottomGrid.AddItem(components.NewTermUIWrapper(diskSparkline), 1, 0, 1, 1, 0, 0, false) + bottomGrid.AddItem(widget.processTable, 0, 1, 2, 1, 0, 0, false) + + widget.AddItem(infoGrid, 0, 0, 1, 1, 0, 0, false) + widget.AddItem(graphGrid, 1, 0, 1, 1, 0, 0, false) + widget.AddItem(bottomGrid, 2, 0, 1, 1, 0, 0, false) + + widget.apiDataListeners = []APIDataListener{ + sysGauges, + cpuInfo, + loadAvgInfo, + procsInfo, + memInfo, + cpuGraph, + memGraph, + loadAvgGraph, + netSparkline, + diskSparkline, + widget.processTableInner, + } + + return widget +} + +// OnAPIDataChange implements the APIDataListener interface. +func (widget *MonitorGrid) OnAPIDataChange(node string, data *apidata.Data) { + for _, dataWidget := range widget.apiDataListeners { + dataWidget.OnAPIDataChange(node, data) + } +} + +// OnScreenSelect implements the screenSelectListener interface. +func (widget *MonitorGrid) onScreenSelect(active bool) { + if active { + widget.processTableInner.ScrollTop() + widget.app.SetFocus(widget.processTable) + } +} + +func (widget *MonitorGrid) initProcessTable() { + widget.processTableInner = components.NewProcessTable() + + widget.processTable = components.NewTermUIWrapper(widget.processTableInner) + widget.processTable.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey { + switch { + case event.Key() == tcell.KeyUp, event.Rune() == 'k': + widget.processTableInner.ScrollUp() + case event.Key() == tcell.KeyDown, event.Rune() == 'j': + widget.processTableInner.ScrollDown() + case event.Key() == tcell.KeyCtrlU: + widget.processTableInner.ScrollHalfPageUp() + case event.Key() == tcell.KeyCtrlD: + widget.processTableInner.ScrollHalfPageDown() + case event.Key() == tcell.KeyCtrlB, event.Key() == tcell.KeyPgUp: + widget.processTableInner.ScrollPageUp() + case event.Key() == tcell.KeyCtrlF, event.Key() == tcell.KeyPgDn: + widget.processTableInner.ScrollPageDown() + } + + return event + }) +} diff --git a/internal/pkg/dashboard/networkconfig.go b/internal/pkg/dashboard/networkconfig.go new file mode 100644 index 0000000..3f50a6e --- /dev/null +++ b/internal/pkg/dashboard/networkconfig.go @@ -0,0 +1,451 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package dashboard + +import ( + "context" + "fmt" + "sort" + "strings" + + "github.com/gdamore/tcell/v2" + "github.com/rivo/tview" + "github.com/siderolabs/gen/maps" + "github.com/siderolabs/go-pointer" + "gopkg.in/yaml.v3" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/pkg/dashboard/resourcedata" + "github.com/aenix-io/talm/internal/pkg/meta" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +const ( + formItemHostname = "Hostname" + formItemDNSServers = "DNS Servers" + formItemTimeServers = "Time Servers" + formItemInterface = "Interface" + formItemMode = "Mode" + formItemAddresses = "Addresses" + formItemGateway = "Gateway" +) + +type networkConfigData struct { + existingConfig *runtime.PlatformNetworkConfig + newConfig *runtime.PlatformNetworkConfig + newConfigError error + linkSet map[string]struct{} +} + +// NetworkConfigGrid represents the network configuration widget. +type NetworkConfigGrid struct { + tview.Grid + + dashboard *Dashboard + + configForm *tview.Form + hostnameField *tview.InputField + dnsServersField *tview.InputField + timeServersField *tview.InputField + interfaceDropdown *tview.DropDown + modeDropdown *tview.DropDown + addressesField *tview.InputField + gatewayField *tview.InputField + + infoView *tview.TextView + existingConfigView *tview.TextView + newConfigView *tview.TextView + + selectedNode string + nodeMap map[string]*networkConfigData +} + +// NewNetworkConfigGrid initializes NetworkConfigGrid. +func NewNetworkConfigGrid(ctx context.Context, dashboard *Dashboard) *NetworkConfigGrid { + widget := &NetworkConfigGrid{ + Grid: *tview.NewGrid(), + configForm: tview.NewForm(), + infoView: tview.NewTextView(), + existingConfigView: tview.NewTextView(), + newConfigView: tview.NewTextView(), + nodeMap: make(map[string]*networkConfigData), + dashboard: dashboard, + } + + widget.configForm.SetBorder(true).SetTitle("Configure (Ctrl+Q)") + widget.SetRows(0, 3).SetColumns(0, 0, 0) + + widget.infoView. + SetDynamicColors(true). + SetScrollable(true). + SetWrap(true) + widget.existingConfigView. + SetDynamicColors(true). + SetScrollable(true). + SetBorderPadding(0, 0, 1, 0). + SetBorder(true). + SetTitle("Existing Config (Ctrl+W)") + widget.newConfigView. + SetDynamicColors(true). + SetScrollable(true). + SetBorderPadding(0, 0, 1, 0). + SetBorder(true). + SetTitle("New Config (Ctrl+E)") + + widget.AddItem(widget.configForm, 0, 0, 1, 1, 0, 0, false) + widget.AddItem(widget.infoView, 1, 0, 1, 1, 0, 0, false) + widget.AddItem(widget.existingConfigView, 0, 1, 2, 1, 0, 0, false) + widget.AddItem(widget.newConfigView, 0, 2, 2, 1, 0, 0, false) + + widget.hostnameField = tview.NewInputField().SetLabel(formItemHostname) + widget.hostnameField.SetBlurFunc(widget.formEdited) + + widget.dnsServersField = tview.NewInputField().SetLabel(formItemDNSServers) + widget.dnsServersField.SetBlurFunc(widget.formEdited) + + widget.timeServersField = tview.NewInputField().SetLabel(formItemTimeServers) + widget.timeServersField.SetBlurFunc(widget.formEdited) + + widget.interfaceDropdown = tview.NewDropDown().SetLabel(formItemInterface) + widget.interfaceDropdown.SetBlurFunc(widget.formEdited) + widget.interfaceDropdown.SetOptions([]string{interfaceNone}, func(_ string, _ int) { + widget.formEdited() + }) + widget.interfaceDropdown.SetListStyles( + tcell.StyleDefault.Foreground(tview.Styles.PrimitiveBackgroundColor).Background(tview.Styles.MoreContrastBackgroundColor), + tcell.StyleDefault.Foreground(tcell.ColorBlack).Background(tview.Styles.PrimaryTextColor), + ) + + widget.modeDropdown = tview.NewDropDown().SetLabel(formItemMode) + widget.modeDropdown.SetBlurFunc(widget.formEdited) + widget.modeDropdown.SetOptions([]string{ModeDHCP, ModeStatic}, func(_ string, _ int) { + widget.formEdited() + }) + widget.modeDropdown.SetListStyles( + tcell.StyleDefault.Foreground(tview.Styles.PrimitiveBackgroundColor).Background(tview.Styles.MoreContrastBackgroundColor), + tcell.StyleDefault.Foreground(tcell.ColorBlack).Background(tview.Styles.PrimaryTextColor), + ) + + widget.addressesField = tview.NewInputField().SetLabel(formItemAddresses) + widget.addressesField.SetBlurFunc(widget.formEdited) + + widget.gatewayField = tview.NewInputField().SetLabel(formItemGateway) + widget.gatewayField.SetBlurFunc(widget.formEdited) + + widget.configForm.AddFormItem(widget.hostnameField) + widget.configForm.AddFormItem(widget.dnsServersField) + widget.configForm.AddFormItem(widget.timeServersField) + widget.configForm.AddFormItem(widget.interfaceDropdown) + widget.configForm.AddFormItem(widget.modeDropdown) + widget.configForm.AddFormItem(widget.addressesField) + widget.configForm.AddFormItem(widget.gatewayField) + + widget.configForm.AddButton("Save", func() { + widget.save(ctx) + }) + + saveButton := widget.configForm.GetButton(0) + saveButton.SetBlurFunc(widget.formEdited) + + inputCapture := func(event *tcell.EventKey) *tcell.EventKey { + if widget.handleFocusSwitch(event) { + return nil + } + + return event + } + + widget.configForm.SetInputCapture(inputCapture) + widget.existingConfigView.SetInputCapture(inputCapture) + widget.newConfigView.SetInputCapture(inputCapture) + + widget.interfaceDropdown.SetCurrentOption(0) + widget.modeDropdown.SetCurrentOption(0) + + return widget +} + +// OnNodeSelect implements the NodeSelectListener interface. +func (widget *NetworkConfigGrid) OnNodeSelect(node string) { + if node != widget.selectedNode { + widget.selectedNode = node + + widget.clearForm() + widget.formEdited() + + widget.redraw() + } +} + +// OnResourceDataChange implements the ResourceDataListener interface. +func (widget *NetworkConfigGrid) OnResourceDataChange(data resourcedata.Data) { + widget.updateNodeData(data) + + if data.Node == widget.selectedNode { + widget.redraw() + } +} + +//nolint:gocyclo +func (widget *NetworkConfigGrid) formEdited() { + widget.infoView.SetText("") + + resetInputField := func(field *tview.InputField) { + // avoid triggering another form edit if there is nothing to change + if field.GetText() != "" { + field.SetText("") + } + } + + resetDropdown := func(dropdown *tview.DropDown) { + // avoid triggering another form edit if there is nothing to change + if currentIndex, _ := dropdown.GetCurrentOption(); currentIndex != 0 { + dropdown.SetCurrentOption(0) + } + } + + _, currentInterface := widget.interfaceDropdown.GetCurrentOption() + _, currentMode := widget.modeDropdown.GetCurrentOption() + + ifaceSelected := currentInterface != "" && currentInterface != interfaceNone + if ifaceSelected { + if itemIndex := widget.configForm.GetFormItemIndex(formItemMode); itemIndex == -1 { + widget.configForm.AddFormItem(widget.modeDropdown) + } + + switch currentMode { + case ModeDHCP: + resetInputField(widget.addressesField) + resetInputField(widget.gatewayField) + + if itemIndex := widget.configForm.GetFormItemIndex(formItemAddresses); itemIndex != -1 { + widget.configForm.RemoveFormItem(itemIndex) + } + + if itemIndex := widget.configForm.GetFormItemIndex(formItemGateway); itemIndex != -1 { + widget.configForm.RemoveFormItem(itemIndex) + } + case ModeStatic: + if itemIndex := widget.configForm.GetFormItemIndex(formItemAddresses); itemIndex == -1 { + widget.configForm.AddFormItem(widget.addressesField) + } + + if itemIndex := widget.configForm.GetFormItemIndex(formItemGateway); itemIndex == -1 { + widget.configForm.AddFormItem(widget.gatewayField) + } + } + } else { + resetDropdown(widget.modeDropdown) + resetInputField(widget.addressesField) + resetInputField(widget.gatewayField) + + if itemIndex := widget.configForm.GetFormItemIndex(formItemMode); itemIndex != -1 { + widget.configForm.RemoveFormItem(itemIndex) + } + + if itemIndex := widget.configForm.GetFormItemIndex(formItemAddresses); itemIndex != -1 { + widget.configForm.RemoveFormItem(itemIndex) + } + + if itemIndex := widget.configForm.GetFormItemIndex(formItemGateway); itemIndex != -1 { + widget.configForm.RemoveFormItem(itemIndex) + } + } + + data := widget.getOrCreateNodeData(widget.selectedNode) + + formData := NetworkConfigFormData{ + Base: pointer.SafeDeref(data.existingConfig), + Hostname: widget.hostnameField.GetText(), + DNSServers: widget.dnsServersField.GetText(), + TimeServers: widget.timeServersField.GetText(), + Iface: currentInterface, + Mode: currentMode, + Addresses: widget.addressesField.GetText(), + Gateway: widget.gatewayField.GetText(), + } + + config, err := formData.ToPlatformNetworkConfig() + if err != nil { + data.newConfig = nil + data.newConfigError = err + } else { + data.newConfig = config + data.newConfigError = nil + } + + widget.redraw() +} + +func (widget *NetworkConfigGrid) redraw() { + data := widget.getOrCreateNodeData(widget.selectedNode) + + if data.existingConfig != nil { + var buf strings.Builder + + encoder := yaml.NewEncoder(&buf) + encoder.SetIndent(2) + + err := encoder.Encode(data.existingConfig) + if err != nil { + widget.existingConfigView.SetText(fmt.Sprintf("[red]error: %v[-]", err)) + } + + widget.existingConfigView.SetText(fmt.Sprintf("[lightblue]%s[-]", tview.Escape(buf.String()))) + } else { + widget.existingConfigView.SetText("[gray]No Config[-]") + } + + if data.newConfigError != nil { + widget.newConfigView.SetText(fmt.Sprintf("[red]error: %v[-]", data.newConfigError)) + } else if data.newConfig != nil { + var buf strings.Builder + + encoder := yaml.NewEncoder(&buf) + encoder.SetIndent(2) + + err := encoder.Encode(data.newConfig) + if err != nil { + widget.newConfigView.SetText(fmt.Sprintf("[red]error: %v[-]", err)) + } + + widget.newConfigView.SetText(fmt.Sprintf("[green]%s[-]", tview.Escape(buf.String()))) + } +} + +func (widget *NetworkConfigGrid) clearForm() { + widget.hostnameField.SetText("") + widget.dnsServersField.SetText("") + widget.timeServersField.SetText("") + widget.interfaceDropdown.SetCurrentOption(0) + widget.modeDropdown.SetCurrentOption(0) + widget.addressesField.SetText("") + widget.gatewayField.SetText("") + widget.infoView.SetText("") + + widget.configForm.SetFocus(0) + + widget.formEdited() +} + +func (widget *NetworkConfigGrid) updateNodeData(data resourcedata.Data) { + nodeData := widget.getOrCreateNodeData(data.Node) + + switch res := data.Resource.(type) { + case *network.LinkStatus: + if data.Deleted { + delete(nodeData.linkSet, res.Metadata().ID()) + } else { + if !res.TypedSpec().Physical() { + return + } + + nodeData.linkSet[res.Metadata().ID()] = struct{}{} + } + + links := maps.Keys(nodeData.linkSet) + + sort.Strings(links) + + allLinks := append([]string{interfaceNone}, links...) + + widget.interfaceDropdown.SetOptions(allLinks, func(_ string, _ int) { + widget.formEdited() + }) + case *runtimeres.MetaKey: + if res.Metadata().ID() == runtimeres.MetaKeyTagToID(meta.MetalNetworkPlatformConfig) { + if data.Deleted { + nodeData.existingConfig = nil + } else { + cfg := runtime.PlatformNetworkConfig{} + + if err := yaml.Unmarshal([]byte(res.TypedSpec().Value), &cfg); err != nil { + widget.existingConfigView.SetText(fmt.Sprintf("[red]error: %v[-]", err)) + + return + } + + nodeData.existingConfig = &cfg + + widget.formEdited() + } + } + } +} + +func (widget *NetworkConfigGrid) getOrCreateNodeData(node string) *networkConfigData { + nodeData, ok := widget.nodeMap[node] + if !ok { + nodeData = &networkConfigData{ + linkSet: make(map[string]struct{}), + } + + widget.nodeMap[node] = nodeData + } + + return nodeData +} + +// OnScreenSelect implements the screenSelectListener interface. +func (widget *NetworkConfigGrid) onScreenSelect(active bool) { + if active { + widget.dashboard.app.SetFocus(widget.configForm) + } +} + +func (widget *NetworkConfigGrid) save(ctx context.Context) { + nodeData := widget.getOrCreateNodeData(widget.selectedNode) + + if nodeData.newConfig == nil { + widget.infoView.SetText("[red]Error: nothing to save[-]") + + return + } + + if nodeData.newConfigError != nil { + widget.infoView.SetText("[red]Error: cannot save, fix the errors and try again[-]") + + return + } + + configBytes, err := yaml.Marshal(nodeData.newConfig) + if err != nil { + widget.infoView.SetText(fmt.Sprintf("[red]Error: %v[-]", err)) + + return + } + + ctx = nodeContext(ctx, widget.selectedNode) + + if err = widget.dashboard.cli.MetaWrite(ctx, meta.MetalNetworkPlatformConfig, configBytes); err != nil { + widget.infoView.SetText(fmt.Sprintf("[red]Error: %v[-]", err)) + + return + } + + widget.infoView.SetText("[green]Network config saved successfully[-]") + widget.clearForm() + widget.dashboard.selectScreen(ScreenSummary) +} + +func (widget *NetworkConfigGrid) handleFocusSwitch(event *tcell.EventKey) bool { + switch event.Key() { //nolint:exhaustive + case tcell.KeyCtrlQ: + widget.dashboard.app.SetFocus(widget.configForm) + + return true + case tcell.KeyCtrlW: + widget.dashboard.app.SetFocus(widget.existingConfigView) + + return true + case tcell.KeyCtrlE: + widget.dashboard.app.SetFocus(widget.newConfigView) + + return true + default: + return false + } +} diff --git a/internal/pkg/dashboard/options.go b/internal/pkg/dashboard/options.go new file mode 100644 index 0000000..8a4a834 --- /dev/null +++ b/internal/pkg/dashboard/options.go @@ -0,0 +1,52 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package dashboard + +import ( + "time" +) + +type options struct { + interval time.Duration + allowExitKeys bool + screens []Screen +} + +func defaultOptions() *options { + return &options{ + interval: 5 * time.Second, + allowExitKeys: true, + screens: []Screen{ + ScreenSummary, + ScreenMonitor, + ScreenNetworkConfig, + }, + } +} + +// Option is a functional option for Dashboard. +type Option func(*options) + +// WithInterval sets the interval for the dashboard. +func WithInterval(interval time.Duration) Option { + return func(o *options) { + o.interval = interval + } +} + +// WithAllowExitKeys sets whether the dashboard should allow exit keys (Ctrl + C). +func WithAllowExitKeys(allowExitKeys bool) Option { + return func(o *options) { + o.allowExitKeys = allowExitKeys + } +} + +// WithScreens sets the screens to display. +// The order is preserved. +func WithScreens(screens ...Screen) Option { + return func(o *options) { + o.screens = screens + } +} diff --git a/internal/pkg/dashboard/resourcedata/resourcedata.go b/internal/pkg/dashboard/resourcedata/resourcedata.go new file mode 100644 index 0000000..2383ef7 --- /dev/null +++ b/internal/pkg/dashboard/resourcedata/resourcedata.go @@ -0,0 +1,207 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package resourcedata implements the types and the data sources for the data sourced from the Talos resource API (COSI). +package resourcedata + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/channel" + "golang.org/x/sync/errgroup" + + "github.com/aenix-io/talm/internal/pkg/dashboard/util" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/hardware" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +// Data contains a resource, whether it is deleted and the node it came from. +type Data struct { + Node string + Resource resource.Resource + Deleted bool +} + +// Source is the data source for the Talos resources. +type Source struct { + ctxCancel context.CancelFunc + + eg errgroup.Group + once sync.Once + + COSI state.State + + ch chan Data + NodeResourceCh <-chan Data +} + +// Run starts the data source. +func (source *Source) Run(ctx context.Context) { + source.once.Do(func() { + source.run(ctx) + }) +} + +// Stop stops the data source. +func (source *Source) Stop() error { + source.ctxCancel() + + return source.eg.Wait() +} + +func (source *Source) run(ctx context.Context) { + ctx, source.ctxCancel = context.WithCancel(ctx) + + source.ch = make(chan Data) + + source.NodeResourceCh = source.ch + + for _, nodeContext := range util.NodeContexts(ctx) { + source.eg.Go(func() error { + source.runResourceWatchWithRetries(nodeContext.Ctx, nodeContext.Node) + + return nil + }) + } +} + +func (source *Source) runResourceWatchWithRetries(ctx context.Context, node string) { + for { + if err := source.runResourceWatch(ctx, node); errors.Is(err, context.Canceled) { + return + } + + // wait for a second before the next retry + timer := time.NewTimer(1 * time.Second) + + select { + case <-ctx.Done(): + timer.Stop() + + return + case <-timer.C: + } + } +} + +//nolint:gocyclo,cyclop +func (source *Source) runResourceWatch(ctx context.Context, node string) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + eventCh := make(chan state.Event) + + if err := source.COSI.Watch(ctx, runtime.NewMachineStatus().Metadata(), eventCh); err != nil { + return err + } + + if err := source.COSI.Watch(ctx, runtime.NewSecurityStateSpec(v1alpha1.NamespaceName).Metadata(), eventCh); err != nil { + return err + } + + if err := source.COSI.Watch(ctx, runtime.NewMountStatus(v1alpha1.NamespaceName, constants.StatePartitionLabel).Metadata(), eventCh); err != nil { + return err + } + + if err := source.COSI.Watch(ctx, runtime.NewMountStatus(v1alpha1.NamespaceName, constants.EphemeralPartitionLabel).Metadata(), eventCh); err != nil { + return err + } + + if err := source.COSI.Watch(ctx, config.NewMachineType().Metadata(), eventCh); err != nil { + return err + } + + if err := source.COSI.Watch(ctx, k8s.NewKubeletSpec(k8s.NamespaceName, k8s.KubeletID).Metadata(), eventCh); err != nil { + return err + } + + if err := source.COSI.Watch(ctx, network.NewResolverStatus(network.NamespaceName, network.ResolverID).Metadata(), eventCh); err != nil { + return err + } + + if err := source.COSI.Watch(ctx, network.NewTimeServerStatus(network.NamespaceName, network.TimeServerID).Metadata(), eventCh); err != nil { + return err + } + + if err := source.COSI.Watch(ctx, hardware.NewSystemInformation(hardware.SystemInformationID).Metadata(), eventCh); err != nil { + return err + } + + if err := source.COSI.Watch(ctx, cluster.NewInfo().Metadata(), eventCh); err != nil { + return err + } + + if err := source.COSI.Watch(ctx, network.NewStatus(network.NamespaceName, network.StatusID).Metadata(), eventCh); err != nil { + return err + } + + if err := source.COSI.Watch(ctx, network.NewHostnameStatus(network.NamespaceName, network.HostnameID).Metadata(), eventCh); err != nil { + return err + } + + if err := source.COSI.WatchKind(ctx, runtime.NewMetaKey(runtime.NamespaceName, "").Metadata(), eventCh, state.WithBootstrapContents(true)); err != nil { + return err + } + + if err := source.COSI.WatchKind(ctx, k8s.NewStaticPodStatus(k8s.NamespaceName, "").Metadata(), eventCh, state.WithBootstrapContents(true)); err != nil { + return err + } + + if err := source.COSI.WatchKind(ctx, network.NewRouteStatus(network.NamespaceName, "").Metadata(), eventCh, state.WithBootstrapContents(true)); err != nil { + return err + } + + if err := source.COSI.WatchKind(ctx, network.NewLinkStatus(network.NamespaceName, "").Metadata(), eventCh, state.WithBootstrapContents(true)); err != nil { + return err + } + + if err := source.COSI.WatchKind(ctx, cluster.NewMember(cluster.NamespaceName, "").Metadata(), eventCh, state.WithBootstrapContents(true)); err != nil { + return err + } + + if err := source.COSI.WatchKind(ctx, network.NewNodeAddress(network.NamespaceName, "").Metadata(), eventCh, state.WithBootstrapContents(true)); err != nil { + return err + } + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case event := <-eventCh: + switch event.Type { + case state.Errored: + return fmt.Errorf("watch failed: %w", event.Error) + case state.Bootstrapped: + // ignored + case state.Created, state.Updated: + if !channel.SendWithContext(ctx, source.ch, Data{ + Node: node, + Resource: event.Resource, + }) { + return ctx.Err() + } + case state.Destroyed: + if !channel.SendWithContext(ctx, source.ch, Data{ + Node: node, + Resource: event.Resource, + Deleted: true, + }) { + return ctx.Err() + } + } + } + } +} diff --git a/internal/pkg/dashboard/summary.go b/internal/pkg/dashboard/summary.go new file mode 100644 index 0000000..2f2f794 --- /dev/null +++ b/internal/pkg/dashboard/summary.go @@ -0,0 +1,134 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package dashboard + +import ( + "github.com/rivo/tview" + + "github.com/aenix-io/talm/internal/pkg/dashboard/apidata" + "github.com/aenix-io/talm/internal/pkg/dashboard/components" + "github.com/aenix-io/talm/internal/pkg/dashboard/resourcedata" +) + +// SummaryGrid represents the summary grid with the basic node information and the logs. +type SummaryGrid struct { + tview.Grid + + app *tview.Application + + apiDataListeners []APIDataListener + resourceListeners []ResourceDataListener + nodeSelectListeners []NodeSelectListener + + active bool + node string + logViewers map[string]*components.LogViewer +} + +// NewSummaryGrid initializes SummaryGrid. +func NewSummaryGrid(app *tview.Application) *SummaryGrid { + widget := &SummaryGrid{ + app: app, + Grid: *tview.NewGrid(), + logViewers: make(map[string]*components.LogViewer), + } + + widget.SetRows(8, 0).SetColumns(-3, -2, -3) + + talosInfo := components.NewTalosInfo() + widget.AddItem(talosInfo, 0, 0, 1, 1, 0, 0, false) + + kubernetesInfo := components.NewKubernetesInfo() + widget.AddItem(kubernetesInfo, 0, 1, 1, 1, 0, 0, false) + + networkInfo := components.NewNetworkInfo() + widget.AddItem(networkInfo, 0, 2, 1, 1, 0, 0, false) + + widget.apiDataListeners = []APIDataListener{ + kubernetesInfo, + } + + widget.resourceListeners = []ResourceDataListener{ + talosInfo, + kubernetesInfo, + networkInfo, + } + + widget.nodeSelectListeners = []NodeSelectListener{ + talosInfo, + kubernetesInfo, + networkInfo, + } + + return widget +} + +// OnNodeSelect implements the NodeSelectListener interface. +func (widget *SummaryGrid) OnNodeSelect(node string) { + widget.node = node + + widget.updateLogViewer() + + for _, nodeSelectListener := range widget.nodeSelectListeners { + nodeSelectListener.OnNodeSelect(node) + } +} + +// OnAPIDataChange implements the APIDataListener interface. +func (widget *SummaryGrid) OnAPIDataChange(node string, data *apidata.Data) { + for _, dataWidget := range widget.apiDataListeners { + dataWidget.OnAPIDataChange(node, data) + } +} + +// OnResourceDataChange implements the ResourceDataListener interface. +func (widget *SummaryGrid) OnResourceDataChange(nodeResource resourcedata.Data) { + for _, resourceListener := range widget.resourceListeners { + resourceListener.OnResourceDataChange(nodeResource) + } +} + +// OnLogDataChange implements the LogDataListener interface. +func (widget *SummaryGrid) OnLogDataChange(node, logLine, logError string) { + widget.logViewer(node).WriteLog(logLine, logError) +} + +func (widget *SummaryGrid) updateLogViewer() { + if !widget.active { + return + } + + widget.logViewer(widget.node) + + for currNode, logViewer := range widget.logViewers { + if currNode == widget.node { + widget.AddItem(logViewer, 1, 0, 1, 3, 0, 0, false) + + widget.app.SetFocus(logViewer) + + return + } + + widget.RemoveItem(logViewer) + } +} + +func (widget *SummaryGrid) logViewer(node string) *components.LogViewer { + logViewer, ok := widget.logViewers[node] + if !ok { + logViewer = components.NewLogViewer() + + widget.logViewers[node] = logViewer + } + + return logViewer +} + +// OnScreenSelect implements the screenSelectListener interface. +func (widget *SummaryGrid) onScreenSelect(active bool) { + widget.active = active + + widget.updateLogViewer() +} diff --git a/internal/pkg/dashboard/util/util.go b/internal/pkg/dashboard/util/util.go new file mode 100644 index 0000000..844b222 --- /dev/null +++ b/internal/pkg/dashboard/util/util.go @@ -0,0 +1,49 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package util provides utility functions for the dashboard. +package util + +import ( + "context" + + "google.golang.org/grpc/metadata" + + "github.com/siderolabs/talos/pkg/machinery/client" +) + +// NodeContext contains the context.Context for a single node and the node name. +type NodeContext struct { + Ctx context.Context //nolint:containedctx + Node string +} + +// NodeContexts returns a list of NodeContexts from the given context. +// +// It extracts the node names from the outgoing GRPC context metadata. +// If the node name is not present in the metadata, context will be returned as-is with an empty node name. +func NodeContexts(ctx context.Context) []NodeContext { + md, mdOk := metadata.FromOutgoingContext(ctx) + if !mdOk { + return []NodeContext{{Ctx: ctx}} + } + + nodeVal := md.Get("node") + if len(nodeVal) > 0 { + return []NodeContext{{Ctx: ctx, Node: nodeVal[0]}} + } + + nodesVal := md.Get("nodes") + if len(nodesVal) == 0 { + return []NodeContext{{Ctx: ctx}} + } + + nodeContexts := make([]NodeContext, 0, len(nodesVal)) + + for _, node := range nodesVal { + nodeContexts = append(nodeContexts, NodeContext{Ctx: client.WithNode(ctx, node), Node: node}) + } + + return nodeContexts +} diff --git a/internal/pkg/environment/environment.go b/internal/pkg/environment/environment.go new file mode 100644 index 0000000..ab06178 --- /dev/null +++ b/internal/pkg/environment/environment.go @@ -0,0 +1,44 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package environment provides a set of functions to get environment variables. +package environment + +import ( + "github.com/siderolabs/go-procfs/procfs" + + "github.com/siderolabs/talos/pkg/machinery/config" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// Get the desired set of the environment variables based on kernel cmdline and machine config. +// +// The returned value is a list of strings in the form of "key=value". +func Get(cfg config.Config) []string { + return GetCmdline(procfs.ProcCmdline(), cfg) +} + +// GetCmdline the desired set of the environment variables based on kernel cmdline. +func GetCmdline(cmdline *procfs.Cmdline, cfg config.Config) []string { + var result []string + + param := cmdline.Get(constants.KernelParamEnvironment) + + for idx := 0; ; idx++ { + val := param.Get(idx) + if val == nil { + break + } + + result = append(result, *val) + } + + if cfg != nil && cfg.Machine() != nil { + for k, v := range cfg.Machine().Env() { + result = append(result, k+"="+v) + } + } + + return result +} diff --git a/internal/pkg/environment/environment_test.go b/internal/pkg/environment/environment_test.go new file mode 100644 index 0000000..9e2de09 --- /dev/null +++ b/internal/pkg/environment/environment_test.go @@ -0,0 +1,85 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package environment_test + +import ( + "testing" + + "github.com/siderolabs/go-procfs/procfs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/aenix-io/talm/internal/pkg/environment" + "github.com/siderolabs/talos/pkg/machinery/config" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" +) + +func TestGet(t *testing.T) { + t.Parallel() + + for _, test := range []struct { + name string + cmdline string + cfg map[string]string + + expected []string + }{ + { + name: "empty", + }, + { + name: "machine config only", + cfg: map[string]string{ + "http_proxy": "http://proxy.example.com:8080", + }, + expected: []string{ + "http_proxy=http://proxy.example.com:8080", + }, + }, + { + name: "cmdline only", + cmdline: "talos.environment=foo=bar talos.environment=bar=baz", + expected: []string{ + "foo=bar", + "bar=baz", + }, + }, + { + name: "cmdline and machine config", + cmdline: "talos.environment=foo=bar", + cfg: map[string]string{ + "http_proxy": "http://proxy.example.com:8080", + }, + expected: []string{ + "foo=bar", + "http_proxy=http://proxy.example.com:8080", + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + cmdline := procfs.NewCmdline(test.cmdline) + + var cfg config.Config + + if test.cfg != nil { + var err error + + cfg, err = container.New(&v1alpha1.Config{ + MachineConfig: &v1alpha1.MachineConfig{ + MachineEnv: test.cfg, + }, + }) + require.NoError(t, err) + } + + result := environment.GetCmdline(cmdline, cfg) + + assert.Equal(t, test.expected, result) + }) + } +} diff --git a/internal/pkg/etcd/certs.go b/internal/pkg/etcd/certs.go new file mode 100644 index 0000000..1db9f57 --- /dev/null +++ b/internal/pkg/etcd/certs.go @@ -0,0 +1,139 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package etcd + +import ( + stdlibx509 "crypto/x509" + "fmt" + "net" + "net/netip" + "time" + + "github.com/siderolabs/crypto/x509" + "github.com/siderolabs/gen/xslices" + + "github.com/siderolabs/talos/pkg/machinery/resources/network" +) + +// CertificateGenerator contains etcd certificate options. +type CertificateGenerator struct { + CA *x509.PEMEncodedCertificateAndKey + + NodeAddresses *network.NodeAddress + HostnameStatus *network.HostnameStatus +} + +// buildOptions set common certificate options. +func (gen *CertificateGenerator) buildOptions(autoSANs, includeLocalhost bool) []x509.Option { + addresses := gen.NodeAddresses.TypedSpec().IPs() + + if includeLocalhost { + addresses = append(addresses, netip.MustParseAddr("127.0.0.1")) + + for _, addr := range addresses { + if addr.Is6() { + addresses = append(addresses, netip.MustParseAddr("::1")) + + break + } + } + } + + hostname := gen.HostnameStatus.TypedSpec().Hostname + dnsNames := gen.HostnameStatus.TypedSpec().DNSNames() + + if includeLocalhost { + dnsNames = append(dnsNames, "localhost") + } + + result := []x509.Option{ + x509.NotAfter(time.Now().Add(87600 * time.Hour)), + x509.KeyUsage(stdlibx509.KeyUsageDigitalSignature | stdlibx509.KeyUsageKeyEncipherment), + } + + if autoSANs { + result = append(result, + x509.CommonName(hostname), + x509.DNSNames(dnsNames), + x509.IPAddresses(xslices.Map(addresses, func(addr netip.Addr) net.IP { + return addr.AsSlice() + })), + ) + } + + return result +} + +// GeneratePeerCert generates etcd peer certificate and key from etcd CA. +func (gen *CertificateGenerator) GeneratePeerCert() (*x509.PEMEncodedCertificateAndKey, error) { + opts := gen.buildOptions(true, false) + + opts = append(opts, + x509.ExtKeyUsage([]stdlibx509.ExtKeyUsage{ + stdlibx509.ExtKeyUsageServerAuth, + stdlibx509.ExtKeyUsageClientAuth, + }), + ) + + ca, err := x509.NewCertificateAuthorityFromCertificateAndKey(gen.CA) + if err != nil { + return nil, fmt.Errorf("failed loading CA from config: %w", err) + } + + keyPair, err := x509.NewKeyPair(ca, opts...) + if err != nil { + return nil, fmt.Errorf("failed generating peer key pair: %w", err) + } + + return x509.NewCertificateAndKeyFromKeyPair(keyPair), nil +} + +// GenerateServerCert generates server etcd certificate and key from etcd CA. +func (gen *CertificateGenerator) GenerateServerCert() (*x509.PEMEncodedCertificateAndKey, error) { + opts := gen.buildOptions(true, true) + + opts = append(opts, + x509.ExtKeyUsage([]stdlibx509.ExtKeyUsage{ + stdlibx509.ExtKeyUsageServerAuth, + stdlibx509.ExtKeyUsageClientAuth, + }), + ) + + ca, err := x509.NewCertificateAuthorityFromCertificateAndKey(gen.CA) + if err != nil { + return nil, fmt.Errorf("failed loading CA from config: %w", err) + } + + keyPair, err := x509.NewKeyPair(ca, opts...) + if err != nil { + return nil, fmt.Errorf("failed generating client key pair: %w", err) + } + + return x509.NewCertificateAndKeyFromKeyPair(keyPair), nil +} + +// GenerateClientCert generates client certificate and key from etcd CA. +func (gen *CertificateGenerator) GenerateClientCert(commonName string) (*x509.PEMEncodedCertificateAndKey, error) { + opts := gen.buildOptions(false, false) + + opts = append(opts, x509.CommonName(commonName)) + opts = append(opts, + x509.ExtKeyUsage([]stdlibx509.ExtKeyUsage{ + stdlibx509.ExtKeyUsageClientAuth, + }), + ) + + ca, err := x509.NewCertificateAuthorityFromCertificateAndKey(gen.CA) + if err != nil { + return nil, fmt.Errorf("failed loading CA from config: %w", err) + } + + keyPair, err := x509.NewKeyPair(ca, opts...) + if err != nil { + return nil, fmt.Errorf("failed generating client key pair: %w", err) + } + + return x509.NewCertificateAndKeyFromKeyPair(keyPair), nil +} diff --git a/internal/pkg/etcd/endpoints.go b/internal/pkg/etcd/endpoints.go new file mode 100644 index 0000000..da2f564 --- /dev/null +++ b/internal/pkg/etcd/endpoints.go @@ -0,0 +1,50 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package etcd + +import ( + "context" + "errors" + "fmt" + + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +// GetEndpoints returns expected endpoints of etcd cluster members. +// +// It is not guaranteed that etcd is running on each listed endpoint. +func GetEndpoints(ctx context.Context, resources state.State) ([]string, error) { + endpointResources, err := safe.StateListAll[*k8s.Endpoint](ctx, resources) + if err != nil { + return nil, fmt.Errorf("error getting endpoints resources: %w", err) + } + + iter := endpointResources.Iterator() + + var endpointAddrs k8s.EndpointList + + // merge all endpoints into a single list + for iter.Next() { + endpointAddrs = endpointAddrs.Merge(iter.Value()) + } + + if len(endpointAddrs) == 0 { + return nil, errors.New("no controlplane endpoints discovered yet") + } + + endpoints := endpointAddrs.Strings() + + // Etcd expects host:port format. + for i := range len(endpoints) { + endpoints[i] = nethelpers.JoinHostPort(endpoints[i], constants.EtcdClientPort) + } + + return endpoints, nil +} diff --git a/internal/pkg/etcd/etcd.go b/internal/pkg/etcd/etcd.go new file mode 100644 index 0000000..9a0c917 --- /dev/null +++ b/internal/pkg/etcd/etcd.go @@ -0,0 +1,289 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package etcd + +import ( + "context" + "errors" + "fmt" + "log" + "math/rand" + "os" + "time" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-retry/retry" + "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" + "go.etcd.io/etcd/client/pkg/v3/transport" + clientv3 "go.etcd.io/etcd/client/v3" + "go.uber.org/zap" + "google.golang.org/grpc" + + "github.com/aenix-io/talm/internal/app/machined/pkg/system" + "github.com/siderolabs/talos/pkg/machinery/config" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + etcdresource "github.com/siderolabs/talos/pkg/machinery/resources/etcd" +) + +// QuorumCheckTimeout is the amount of time to allow for KV operations before quorum is declared invalid. +const QuorumCheckTimeout = 15 * time.Second + +// Client is a wrapper around the official etcd client. +type Client struct { + *clientv3.Client +} + +// NewClient initializes and returns an etcd client configured to talk to +// a list of endpoints. +func NewClient(ctx context.Context, endpoints []string, dialOpts ...grpc.DialOption) (client *Client, err error) { + tlsInfo := transport.TLSInfo{ + CertFile: constants.EtcdAdminCert, + KeyFile: constants.EtcdAdminKey, + TrustedCAFile: constants.EtcdCACert, + } + + tlsConfig, err := tlsInfo.ClientConfig() + if err != nil { + return nil, fmt.Errorf("error building etcd client TLS config: %w", err) + } + + c, err := clientv3.New(clientv3.Config{ + Endpoints: endpoints, + DialTimeout: 5 * time.Second, + Context: ctx, + DialOptions: append(dialOpts, grpc.WithSharedWriteBuffer(true)), + TLS: tlsConfig, + Logger: zap.NewNop(), + }) + if err != nil { + return nil, fmt.Errorf("error building etcd client: %w", err) + } + + return &Client{Client: c}, nil +} + +// NewLocalClient initializes and returns etcd client configured to talk to localhost endpoint. +func NewLocalClient(ctx context.Context, dialOpts ...grpc.DialOption) (client *Client, err error) { + return NewClient( + ctx, + []string{nethelpers.JoinHostPort("localhost", constants.EtcdClientPort)}, + append([]grpc.DialOption{grpc.WithBlock()}, dialOpts...)..., + ) +} + +// NewClientFromControlPlaneIPs initializes and returns an etcd client +// configured to talk to all members. +func NewClientFromControlPlaneIPs(ctx context.Context, resources state.State, dialOpts ...grpc.DialOption) (client *Client, err error) { + endpoints, err := GetEndpoints(ctx, resources) + if err != nil { + return nil, err + } + + // Shuffle endpoints to establish random order on each call to avoid patterns based on sorted IP list. + rand.Shuffle(len(endpoints), func(i, j int) { endpoints[i], endpoints[j] = endpoints[j], endpoints[i] }) + + return NewClient(ctx, endpoints, dialOpts...) +} + +// ValidateForUpgrade validates the etcd cluster state to ensure that performing +// an upgrade is safe. +func (c *Client) ValidateForUpgrade(ctx context.Context, config config.Config, preserve bool) error { + if config.Machine().Type() == machine.TypeWorker { + return nil + } + + resp, err := c.MemberList(ctx) + if err != nil { + return err + } + + if !preserve { + if len(resp.Members) == 1 { + return errors.New("only 1 etcd member found; assuming this is not an HA setup and refusing to upgrade; if this is a single-node cluster, use --preserve to upgrade") + } + } + + if len(resp.Members) == 2 { + return fmt.Errorf("etcd member count(%d) is insufficient to maintain quorum if upgrade commences", len(resp.Members)) + } + + for _, member := range resp.Members { + // If the member is not started, the name will be an empty string. + if len(member.Name) == 0 { + return fmt.Errorf("etcd member %016x is not started, all members must be running to perform an upgrade", member.ID) + } + + if err = validateMemberHealth(ctx, member.GetClientURLs()); err != nil { + return fmt.Errorf("etcd member %016x is not healthy; all members must be healthy to perform an upgrade: %w", member.ID, err) + } + } + + return nil +} + +// ValidateQuorum performs a KV operation to make certain that quorum is good. +func (c *Client) ValidateQuorum(ctx context.Context) (err error) { + // Get a random key. As long as we can get the response without an error, quorum is good. + checkCtx, cancel := context.WithTimeout(ctx, QuorumCheckTimeout) + defer cancel() + + _, err = c.Get(checkCtx, "health") + if err == rpctypes.ErrPermissionDenied { + // Permission denied is OK since proposal goes through consensus to get this error. + err = nil + } + + if err != nil { + return err + } + + return nil +} + +func validateMemberHealth(ctx context.Context, memberURIs []string) (err error) { + c, err := NewClient(ctx, memberURIs) + if err != nil { + return fmt.Errorf("failed to create client to member: %w", err) + } + + return c.ValidateQuorum(ctx) +} + +// LeaveCluster removes the current member from the etcd cluster and nukes etcd data directory. +func (c *Client) LeaveCluster(ctx context.Context, st state.State) error { + memberID, err := GetLocalMemberID(ctx, st) + if err != nil { + return err + } + + if err := retry.Constant(5*time.Minute, retry.WithUnits(10*time.Second)).RetryWithContext(ctx, func(ctx context.Context) error { + err := c.RemoveMemberByMemberID(ctx, memberID) + if err == nil { + return nil + } + + if errors.Is(err, rpctypes.ErrUnhealthy) { + // unhealthy is returned when the member hasn't established connections with quorum other members + return retry.ExpectedError(err) + } + + if errors.Is(err, rpctypes.ErrStopped) { + // retry the stopped errors as the member might be in the process of shutting down + return retry.ExpectedError(err) + } + + return err + }); err != nil { + return err + } + + if err := system.Services(nil).Stop(ctx, "etcd"); err != nil { + return fmt.Errorf("failed to stop etcd: %w", err) + } + + // Once the member is removed, the data is no longer valid. + if err := os.RemoveAll(constants.EtcdDataPath); err != nil { + return fmt.Errorf("failed to remove %s: %w", constants.EtcdDataPath, err) + } + + return nil +} + +// GetMemberID returns the member ID of the node client is connected to. +func (c *Client) GetMemberID(ctx context.Context) (uint64, error) { + resp, err := c.Client.Maintenance.AlarmList(ctx) + if err != nil { + return 0, err + } + + return resp.Header.MemberId, nil +} + +// RemoveMemberByMemberID removes the member from the etcd cluster. +func (c *Client) RemoveMemberByMemberID(ctx context.Context, memberID uint64) error { + _, err := c.MemberRemove(ctx, memberID) + if err != nil { + return fmt.Errorf("failed to remove member %d: %w", memberID, err) + } + + return nil +} + +// ForfeitLeadership transfers leadership from the current member to another +// member. +// +//nolint:gocyclo +func (c *Client) ForfeitLeadership(ctx context.Context, memberID string) (string, error) { + resp, err := c.MemberList(ctx) + if err != nil { + return "", fmt.Errorf("failed to list etcd members: %w", err) + } + + if len(resp.Members) == 1 { + return "", errors.New("cannot forfeit leadership, only one member") + } + + var member *etcdserverpb.Member + + memberIDUint64, err := etcdresource.ParseMemberID(memberID) + if err != nil { + return "", err + } + + for _, m := range resp.Members { + if m.ID == memberIDUint64 { + member = m + + break + } + } + + if member == nil { + return "", fmt.Errorf("failed to find %q in list of etcd members", memberID) + } + + for _, ep := range member.GetClientURLs() { + var status *clientv3.StatusResponse + + status, err = c.Status(ctx, ep) + if err != nil { + return "", err + } + + if status.Leader != member.GetID() { + return "", nil + } + + for _, m := range resp.Members { + if m.GetID() != member.GetID() { + log.Printf("moving leadership from %q to %q", member.GetName(), m.GetName()) + + conn, err := c.Dial(ep) + if err != nil { + return "", err + } + + maintenance := clientv3.NewMaintenanceFromMaintenanceClient(clientv3.RetryMaintenanceClient(c.Client, conn), c.Client) + + _, err = maintenance.MoveLeader(ctx, m.GetID()) + if err != nil { + if errors.Is(err, rpctypes.ErrNotLeader) { + // this member is not a leader anymore, so nothing to be done for the forfeit leadership + return "", nil + } + + return "", err + } + + return m.GetName(), nil + } + } + } + + return "", nil +} diff --git a/internal/pkg/etcd/local.go b/internal/pkg/etcd/local.go new file mode 100644 index 0000000..ee61629 --- /dev/null +++ b/internal/pkg/etcd/local.go @@ -0,0 +1,34 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package etcd + +import ( + "context" + "fmt" + "time" + + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + + "github.com/siderolabs/talos/pkg/machinery/resources/etcd" +) + +// GetLocalMemberID gets the etcd member id of the local node via resources. +func GetLocalMemberID(ctx context.Context, s state.State) (uint64, error) { + ctx, cancel := context.WithTimeout(ctx, 3*time.Minute) + defer cancel() + + member, err := safe.StateWatchFor[*etcd.Member]( + ctx, + s, + etcd.NewMember(etcd.NamespaceName, etcd.LocalMemberID).Metadata(), + state.WithEventTypes(state.Created), + ) + if err != nil { + return 0, fmt.Errorf("failed to get local etcd member ID: %w", err) + } + + return etcd.ParseMemberID(member.TypedSpec().MemberID) +} diff --git a/internal/pkg/etcd/lock.go b/internal/pkg/etcd/lock.go new file mode 100644 index 0000000..bb9a6a7 --- /dev/null +++ b/internal/pkg/etcd/lock.go @@ -0,0 +1,44 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package etcd + +import ( + "context" + "fmt" + + "go.etcd.io/etcd/client/v3/concurrency" + "go.uber.org/zap" +) + +// WithLock executes the given function exclusively by acquiring an Etcd lock with the given key. +func WithLock(ctx context.Context, key string, logger *zap.Logger, f func() error) error { + etcdClient, err := NewLocalClient(ctx) + if err != nil { + return fmt.Errorf("error creating etcd client: %w", err) + } + + defer etcdClient.Close() //nolint:errcheck + + session, err := concurrency.NewSession(etcdClient.Client) + if err != nil { + return fmt.Errorf("error creating etcd session: %w", err) + } + + defer session.Close() //nolint:errcheck + + mutex := concurrency.NewMutex(session, key) + + logger.Debug("waiting for mutex", zap.String("key", key)) + + if err = mutex.Lock(ctx); err != nil { + return fmt.Errorf("error acquiring mutex for key %s: %w", key, err) + } + + logger.Debug("mutex acquired", zap.String("key", key)) + + defer mutex.Unlock(ctx) //nolint:errcheck + + return f() +} diff --git a/internal/pkg/install/install.go b/internal/pkg/install/install.go new file mode 100644 index 0000000..8c2d1bb --- /dev/null +++ b/internal/pkg/install/install.go @@ -0,0 +1,300 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package install + +import ( + "bytes" + "context" + "fmt" + "io" + "log" + "os" + "strconv" + + "github.com/containerd/containerd" + "github.com/containerd/containerd/cio" + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/namespaces" + "github.com/containerd/containerd/oci" + "github.com/opencontainers/runtime-spec/specs-go" + "github.com/siderolabs/go-kmsg" + "github.com/siderolabs/go-procfs/procfs" + "golang.org/x/sys/unix" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + containerdrunner "github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/containerd" + "github.com/aenix-io/talm/internal/pkg/capability" + "github.com/aenix-io/talm/internal/pkg/containers/image" + "github.com/aenix-io/talm/internal/pkg/environment" + "github.com/aenix-io/talm/internal/pkg/extensions" + machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" + configcore "github.com/siderolabs/talos/pkg/machinery/config" + "github.com/siderolabs/talos/pkg/machinery/config/config" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// RunInstallerContainer performs an installation via the installer container. +// +//nolint:gocyclo,cyclop +func RunInstallerContainer(disk, platform, ref string, cfg configcore.Config, cfgContainer configcore.Container, opts ...Option) error { + const containerID = "upgrade" + + options := DefaultInstallOptions() + + for _, opt := range opts { + if err := opt(&options); err != nil { + return err + } + } + + var ( + registriesConfig config.Registries + extensionsConfig []config.Extension + ) + + if cfg != nil && cfg.Machine() != nil { + registriesConfig = cfg.Machine().Registries() + extensionsConfig = cfg.Machine().Install().Extensions() + } else { + registriesConfig = &v1alpha1.RegistriesConfig{} + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ctx = namespaces.WithNamespace(ctx, constants.SystemContainerdNamespace) + + client, err := containerd.New(constants.SystemContainerdAddress) + if err != nil { + return err + } + + defer client.Close() //nolint:errcheck + + var done func(context.Context) error + + ctx, done, err = client.WithLease(ctx) + defer done(ctx) //nolint:errcheck + + var img containerd.Image + + if !options.Pull { + img, err = client.GetImage(ctx, ref) + } + + if img == nil || err != nil && errdefs.IsNotFound(err) { + log.Printf("pulling %q", ref) + + img, err = image.Pull(ctx, registriesConfig, client, ref) + } + + if err != nil { + return err + } + + puller, err := extensions.NewPuller(client) + if err != nil { + return err + } + + if extensionsConfig != nil { + if err = puller.PullAndMount(ctx, registriesConfig, extensionsConfig); err != nil { + return err + } + } + + defer func() { + if err = puller.Cleanup(ctx); err != nil { + log.Printf("error cleaning up pulled system extensions: %s", err) + } + }() + + // See if there's previous container/snapshot to clean up + var oldcontainer containerd.Container + + if oldcontainer, err = client.LoadContainer(ctx, containerID); err == nil { + if err = oldcontainer.Delete(ctx, containerd.WithSnapshotCleanup); err != nil { + return fmt.Errorf("error deleting old container instance: %w", err) + } + } + + if err = client.SnapshotService("").Remove(ctx, containerID); err != nil && !errdefs.IsNotFound(err) { + return fmt.Errorf("error cleaning up stale snapshot: %w", err) + } + + mounts := []specs.Mount{ + {Type: "bind", Destination: "/dev", Source: "/dev", Options: []string{"rbind", "rshared", "rw"}}, + {Type: "bind", Destination: constants.SystemExtensionsPath, Source: constants.SystemExtensionsPath, Options: []string{"rbind", "rshared", "ro"}}, + } + + // mount the machined socket into the container for upgrade pre-checks if the socket exists + if _, err = os.Stat(constants.MachineSocketPath); err == nil { + mounts = append(mounts, + specs.Mount{Type: "bind", Destination: constants.MachineSocketPath, Source: constants.MachineSocketPath, Options: []string{"rbind", "rshared", "ro"}}, + ) + } + + // mount the efivars into the container if the efivars directory exists + if _, err = os.Stat(constants.EFIVarsMountPoint); err == nil { + mounts = append(mounts, + specs.Mount{Type: "efivarfs", Source: "efivarfs", Destination: constants.EFIVarsMountPoint, Options: []string{"rw", "nosuid", "nodev", "noexec", "relatime"}}, + ) + } + + // mount the /.extra directory into the container if the directory exists + if _, err = os.Stat(constants.SDStubDynamicInitrdPath); err == nil { + mounts = append(mounts, + specs.Mount{Type: "bind", Destination: constants.SDStubDynamicInitrdPath, Source: constants.SDStubDynamicInitrdPath, Options: []string{"rbind", "rshared", "ro"}}, + ) + } + + // TODO(andrewrynhard): To handle cases when the newer version changes the + // platform name, this should be determined in the installer container. + config := constants.ConfigNone + if c := procfs.ProcCmdline().Get(constants.KernelParamConfig).First(); c != nil { + config = *c + } + + upgrade := strconv.FormatBool(options.Upgrade) + force := strconv.FormatBool(options.Force) + zero := strconv.FormatBool(options.Zero) + + args := []string{ + "/bin/installer", + "install", + "--disk=" + disk, + "--platform=" + platform, + "--config=" + config, + "--upgrade=" + upgrade, + "--force=" + force, + "--zero=" + zero, + } + + for _, arg := range options.ExtraKernelArgs { + args = append(args, "--extra-kernel-arg", arg) + } + + for _, preservedArg := range []string{ + constants.KernelParamSideroLink, + constants.KernelParamEventsSink, + constants.KernelParamLoggingKernel, + constants.KernelParamEquinixMetalEvents, + constants.KernelParamDashboardDisabled, + constants.KernelParamNetIfnames, + } { + if c := procfs.ProcCmdline().Get(preservedArg).First(); c != nil { + args = append(args, "--extra-kernel-arg", fmt.Sprintf("%s=%s", preservedArg, *c)) + } + } + + specOpts := []oci.SpecOpts{ + oci.WithImageConfig(img), + oci.WithProcessArgs(args...), + oci.WithHostNamespace(specs.NetworkNamespace), + oci.WithHostNamespace(specs.PIDNamespace), + oci.WithMounts(mounts), + oci.WithHostHostsFile, + oci.WithHostResolvconf, + oci.WithParentCgroupDevices, + oci.WithCapabilities(capability.AllGrantableCapabilities()), + oci.WithMaskedPaths(nil), + oci.WithReadonlyPaths(nil), + oci.WithWriteableSysfs, + oci.WithWriteableCgroupfs, + oci.WithSelinuxLabel(""), + oci.WithApparmorProfile(""), + oci.WithSeccompUnconfined, + oci.WithAllDevicesAllowed, + oci.WithEnv(environment.Get(cfg)), + } + + containerOpts := []containerd.NewContainerOpts{ + containerd.WithImage(img), + containerd.WithNewSnapshot(containerID, img), + containerd.WithNewSpec(specOpts...), + } + + container, err := client.NewContainer(ctx, containerID, containerOpts...) + if err != nil { + return err + } + + defer container.Delete(ctx, containerd.WithSnapshotCleanup) //nolint:errcheck + + f, err := os.OpenFile("/dev/kmsg", os.O_RDWR|unix.O_CLOEXEC|unix.O_NONBLOCK|unix.O_NOCTTY, 0o666) + if err != nil { + return fmt.Errorf("failed to open /dev/kmsg: %w", err) + } + //nolint:errcheck + defer f.Close() + + w := &kmsg.Writer{KmsgWriter: f} + + var r interface { + io.Reader + WaitAndClose(context.Context, containerd.Task) + } + + if cfgContainer != nil { + var configBytes []byte + + configBytes, err = cfgContainer.Bytes() + if err != nil { + return err + } + + r = &containerdrunner.StdinCloser{ + Stdin: bytes.NewReader(configBytes), + Closer: make(chan struct{}), + } + } + + creator := cio.NewCreator(cio.WithStreams(r, w, w)) + + t, err := container.NewTask(ctx, creator) + if err != nil { + return err + } + + if r != nil { + go r.WaitAndClose(ctx, t) + } + + defer t.Delete(ctx) //nolint:errcheck + + if err = t.Start(ctx); err != nil { + return fmt.Errorf("failed to start %q task: %w", "upgrade", err) + } + + statusC, err := t.Wait(ctx) + if err != nil { + return fmt.Errorf("failed waiting for %q task: %w", "upgrade", err) + } + + status := <-statusC + + code := status.ExitCode() + if code != 0 { + return fmt.Errorf("task %q failed: exit code %d", "upgrade", code) + } + + return nil +} + +// OptionsFromUpgradeRequest builds installer options from upgrade request. +func OptionsFromUpgradeRequest(r runtime.Runtime, in *machineapi.UpgradeRequest) []Option { + opts := []Option{ + WithPull(false), + WithUpgrade(true), + WithForce(!in.GetPreserve()), + } + + if r.Config() != nil && r.Config().Machine() != nil { + opts = append(opts, WithExtraKernelArgs(r.Config().Machine().Install().ExtraKernelArgs())) + } + + return opts +} diff --git a/internal/pkg/install/options.go b/internal/pkg/install/options.go new file mode 100644 index 0000000..4500f04 --- /dev/null +++ b/internal/pkg/install/options.go @@ -0,0 +1,89 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package install + +// Option is a functional option. +type Option func(o *Options) error + +// Options describes the install options. +type Options struct { + Pull bool + Force bool + Upgrade bool + Zero bool + ExtraKernelArgs []string +} + +// DefaultInstallOptions returns default options. +func DefaultInstallOptions() Options { + return Options{ + Pull: true, + } +} + +// Apply list of Option. +func (o *Options) Apply(opts ...Option) error { + for _, opt := range opts { + if err := opt(o); err != nil { + return err + } + } + + return nil +} + +// WithOptions sets Options as a whole. +func WithOptions(opts Options) Option { + return func(o *Options) error { + *o = opts + + return nil + } +} + +// WithPull sets the pull option. +func WithPull(b bool) Option { + return func(o *Options) error { + o.Pull = b + + return nil + } +} + +// WithForce sets the force option. +func WithForce(b bool) Option { + return func(o *Options) error { + o.Force = b + + return nil + } +} + +// WithUpgrade sets the upgrade option. +func WithUpgrade(b bool) Option { + return func(o *Options) error { + o.Upgrade = b + + return nil + } +} + +// WithZero sets the zero option. +func WithZero(b bool) Option { + return func(o *Options) error { + o.Zero = b + + return nil + } +} + +// WithExtraKernelArgs sets the extra args. +func WithExtraKernelArgs(s []string) Option { + return func(o *Options) error { + o.ExtraKernelArgs = s + + return nil + } +} diff --git a/internal/pkg/install/pull.go b/internal/pkg/install/pull.go new file mode 100644 index 0000000..6eb5e99 --- /dev/null +++ b/internal/pkg/install/pull.go @@ -0,0 +1,111 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package install + +import ( + "context" + "errors" + "fmt" + + "github.com/containerd/containerd" + "github.com/containerd/containerd/cio" + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/namespaces" + "github.com/containerd/containerd/oci" + + "github.com/aenix-io/talm/internal/pkg/containers/image" + "github.com/siderolabs/talos/pkg/machinery/config/config" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// PullAndValidateInstallerImage pulls down the installer and validates that it can run. +// +//nolint:gocyclo +func PullAndValidateInstallerImage(ctx context.Context, reg config.Registries, ref string) error { + // Pull down specified installer image early so we can bail if it doesn't exist in the upstream registry + containerdctx := namespaces.WithNamespace(ctx, constants.SystemContainerdNamespace) + + const containerID = "validate" + + client, err := containerd.New(constants.SystemContainerdAddress) + if err != nil { + return err + } + + defer client.Close() //nolint:errcheck + + img, err := image.Pull(containerdctx, reg, client, ref, image.WithSkipIfAlreadyPulled()) + if err != nil { + return err + } + + // See if there's previous container/snapshot to clean up + var oldcontainer containerd.Container + + if oldcontainer, err = client.LoadContainer(containerdctx, containerID); err == nil { + if err = oldcontainer.Delete(containerdctx, containerd.WithSnapshotCleanup); err != nil { + return fmt.Errorf("error deleting old container instance: %w", err) + } + } + + if err = client.SnapshotService("").Remove(containerdctx, containerID); err != nil && !errdefs.IsNotFound(err) { + return fmt.Errorf("error cleaning up stale snapshot: %w", err) + } + + // Launch the container with a known help command for a simple check to make sure the image is valid + args := []string{ + "/bin/installer", + "--help", + } + + specOpts := []oci.SpecOpts{ + oci.WithImageConfig(img), + oci.WithProcessArgs(args...), + } + + containerOpts := []containerd.NewContainerOpts{ + containerd.WithImage(img), + containerd.WithNewSnapshot(containerID, img), + containerd.WithNewSpec(specOpts...), + } + + container, err := client.NewContainer(containerdctx, containerID, containerOpts...) + if err != nil { + return err + } + + //nolint:errcheck + defer container.Delete(containerdctx, containerd.WithSnapshotCleanup) + + task, err := container.NewTask(containerdctx, cio.NullIO) + if err != nil { + return err + } + + //nolint:errcheck + defer task.Delete(containerdctx) + + exitStatusC, err := task.Wait(containerdctx) + if err != nil { + return err + } + + if err = task.Start(containerdctx); err != nil { + return err + } + + status := <-exitStatusC + + code, _, err := status.Result() + if err != nil { + return err + } + + if code != 0 { + return errors.New("installer help returned non-zero exit. assuming invalid installer") + } + + return nil +} diff --git a/internal/pkg/logind/broker.go b/internal/pkg/logind/broker.go new file mode 100644 index 0000000..cdda604 --- /dev/null +++ b/internal/pkg/logind/broker.go @@ -0,0 +1,272 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package logind + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "log" + "net" + "os" + "strings" + "sync" + "syscall" + "time" + + "golang.org/x/sync/errgroup" +) + +// DBusBroker implements simplified D-Bus broker which allows to connect +// kubelet D-Bus connection with Talos logind mock. +// +// Broker doesn't actually implement auth, and it is supposed that service +// connects on one socket, while a client connects on another socket. +type DBusBroker struct { + listenService, listenClient net.Listener +} + +// NewBroker initializes new broker. +func NewBroker(serviceSocketPath, clientSocketPath string) (*DBusBroker, error) { + // remove socket paths as with Docker mode paths might persist across container restarts + for _, socketPath := range []string{serviceSocketPath, clientSocketPath} { + if err := os.RemoveAll(socketPath); err != nil { + return nil, fmt.Errorf("error cleaning up D-Bus socket paths: %w", err) + } + } + + broker := &DBusBroker{} + + var err error + + broker.listenService, err = net.Listen("unix", serviceSocketPath) + if err != nil { + return nil, err + } + + broker.listenClient, err = net.Listen("unix", clientSocketPath) + if err != nil { + return nil, err + } + + return broker, nil +} + +// Close the listen sockets. +func (broker *DBusBroker) Close() error { + if err := broker.listenClient.Close(); err != nil { + return err + } + + return broker.listenService.Close() +} + +// Run the broker. +func (broker *DBusBroker) Run(ctx context.Context) error { + eg, ctx := errgroup.WithContext(ctx) + + var ( + connClient, connService net.Conn + mu sync.Mutex + ) + + eg.Go(func() error { return broker.run(ctx, broker.listenService, &mu, &connService, &connClient) }) + eg.Go(func() error { return broker.run(ctx, broker.listenClient, &mu, &connClient, &connService) }) + + return eg.Wait() +} + +func (broker *DBusBroker) run(ctx context.Context, l net.Listener, mu *sync.Mutex, ours, theirs *net.Conn) error { + for ctx.Err() == nil { + conn, err := l.Accept() + if err != nil { + if errors.Is(err, net.ErrClosed) { + return nil + } + + return err + } + + handleConn(ctx, conn.(*net.UnixConn), mu, ours, theirs) + } + + return nil +} + +func extractFiles(oob []byte) []int { + if len(oob) == 0 { + return nil + } + + var fds []int + + scms, err := syscall.ParseSocketControlMessage(oob) + if err == nil { + for _, scm := range scms { + var files []int + + files, err = syscall.ParseUnixRights(&scm) + if err == nil { + fds = append(fds, files...) + } + } + } + + return fds +} + +//nolint:gocyclo +func handleConn(ctx context.Context, conn *net.UnixConn, mu *sync.Mutex, ours, theirs *net.Conn) { + defer conn.Close() //nolint: errcheck + + r := bufio.NewReader(conn) + + if err := handleAuth(r, conn); err != nil { + log.Printf("auth failed: %s", err) + + return + } + + mu.Lock() + *ours = conn + mu.Unlock() + + defer func() { + mu.Lock() + *ours = nil + mu.Unlock() + }() + + buf := make([]byte, 4096) + oob := make([]byte, 4096) + + for ctx.Err() == nil { + var ( + n, oobn int + err error + ) + + if r.Buffered() > 0 { + // read remaining buffered data + n, err = r.Read(buf[:r.Buffered()]) + } else { + // read the message and OOB data from the UNIX socket + n, oobn, _, _, err = conn.ReadMsgUnix(buf, oob) + } + + if err != nil { + return + } + + // capture all file descriptors in the OOB message + // broker needs to close the file descriptors as they get passed to the other peer + fds := extractFiles(oob[:oobn]) + + // find the other side of the connection + var w net.Conn + + for range 10 { + mu.Lock() + w = *theirs + mu.Unlock() + + if w != nil { + break + } + + select { + case <-time.After(time.Second): + case <-ctx.Done(): + return + } + } + + if w == nil { + // drop data, as there's no other connection + continue + } + + // send the message and OOB date + // this will pass the file descriptors if they are in the OOB date + if _, _, err = w.(*net.UnixConn).WriteMsgUnix(buf[:n], oob[:oobn], nil); err != nil { + return + } + + // close fds to make sure broker doesn't hold the fds on its side + for _, fd := range fds { + syscall.Close(fd) //nolint:errcheck + } + } +} + +//nolint:gocyclo +func handleAuth(r *bufio.Reader, w io.Writer) error { + readLine := func() (string, error) { + l, err := r.ReadString('\n') + if err != nil { + return l, err + } + + l = strings.TrimRight(l, "\r\n") + + return l, nil + } + + // first, should receive AUTH command preceded by zero byte + line, err := readLine() + if err != nil { + return err + } + + if line != "\x00AUTH" { + return fmt.Errorf("unexpected line, expected AUTH: %q", line) + } + + if _, err = w.Write([]byte("REJECTED EXTERNAL\r\n")); err != nil { + return err + } + + // now real auth command + line, err = readLine() + if err != nil { + return err + } + + if !strings.HasPrefix(line, "AUTH EXTERNAL") { + return fmt.Errorf("unexpected line, expected AUTH EXTERNAL: %q", line) + } + + if _, err = w.Write([]byte("OK 1234deadbeef\r\n")); err != nil { + return err + } + + // negotiate unix FDs + line, err = readLine() + if err != nil { + return err + } + + if line != "NEGOTIATE_UNIX_FD" { + return fmt.Errorf("unexpected line, expected NEGOTIATE_UNIX_FD: %q", line) + } + + if _, err = w.Write([]byte("AGREE_UNIX_FD\r\n")); err != nil { + return err + } + + // BEGIN + line, err = readLine() + if err != nil { + return err + } + + if line != "BEGIN" { + return fmt.Errorf("unexpected line, expected BEGIN: %q", line) + } + + return nil +} diff --git a/internal/pkg/logind/dbus.go b/internal/pkg/logind/dbus.go new file mode 100644 index 0000000..831e202 --- /dev/null +++ b/internal/pkg/logind/dbus.go @@ -0,0 +1,22 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package logind + +import "github.com/godbus/dbus/v5" + +const ( + dbusPath = dbus.ObjectPath("/org/freedesktop/DBus") + dbusInterface = "org.freedesktop.DBus" +) + +type dbusMock struct{} + +func (dbusMock) Hello() (string, *dbus.Error) { + return "id", nil +} + +func (dbusMock) AddMatch(_ string) *dbus.Error { + return nil +} diff --git a/internal/pkg/logind/kubelet_mock_test.go b/internal/pkg/logind/kubelet_mock_test.go new file mode 100644 index 0000000..5a28c45 --- /dev/null +++ b/internal/pkg/logind/kubelet_mock_test.go @@ -0,0 +1,148 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package logind_test + +import ( + "errors" + "fmt" + "log" + "syscall" + "time" + + "github.com/godbus/dbus/v5" +) + +const ( + logindService = "org.freedesktop.login1" + logindObject = dbus.ObjectPath("/org/freedesktop/login1") + logindInterface = "org.freedesktop.login1.Manager" +) + +type dBusConnector interface { + Object(dest string, path dbus.ObjectPath) dbus.BusObject + AddMatchSignal(options ...dbus.MatchOption) error + Signal(ch chan<- *dbus.Signal) + Close() error +} + +// DBusCon has functions that can be used to interact with systemd and logind over dbus. +type DBusCon struct { + SystemBus dBusConnector +} + +func NewDBusCon(path string) (*DBusCon, error) { + conn, err := dbus.Connect(path) + if err != nil { + return nil, err + } + + return &DBusCon{ + SystemBus: conn, + }, nil +} + +func (bus *DBusCon) Close() error { + return bus.SystemBus.Close() +} + +// InhibitLock is a lock obtained after creating an systemd inhibitor by calling InhibitShutdown(). +type InhibitLock uint32 + +// CurrentInhibitDelay returns the current delay inhibitor timeout value as configured in logind.conf(5). +// see https://www.freedesktop.org/software/systemd/man/logind.conf.html for more details. +func (bus *DBusCon) CurrentInhibitDelay() (time.Duration, error) { + obj := bus.SystemBus.Object(logindService, logindObject) + + res, err := obj.GetProperty(logindInterface + ".InhibitDelayMaxUSec") + if err != nil { + return 0, fmt.Errorf("failed reading InhibitDelayMaxUSec property from logind: %w", err) + } + + delay, ok := res.Value().(uint64) + if !ok { + return 0, errors.New("InhibitDelayMaxUSec from logind is not a uint64 as expected") + } + + // InhibitDelayMaxUSec is in microseconds + duration := time.Duration(delay) * time.Microsecond + + return duration, nil +} + +// InhibitShutdown creates an systemd inhibitor by calling logind's Inhibt() and returns the inhibitor lock +// see https://www.freedesktop.org/wiki/Software/systemd/inhibit/ for more details. +func (bus *DBusCon) InhibitShutdown() (InhibitLock, error) { + obj := bus.SystemBus.Object(logindService, logindObject) + what := "shutdown" + who := "kubelet" + why := "Kubelet needs time to handle node shutdown" + mode := "delay" + + call := obj.Call("org.freedesktop.login1.Manager.Inhibit", 0, what, who, why, mode) + if call.Err != nil { + return InhibitLock(0), fmt.Errorf("failed creating systemd inhibitor: %w", call.Err) + } + + var fd uint32 + + err := call.Store(&fd) + if err != nil { + return InhibitLock(0), fmt.Errorf("failed storing inhibit lock file descriptor: %w", err) + } + + return InhibitLock(fd), nil +} + +// ReleaseInhibitLock will release the underlying inhibit lock which will cause the shutdown to start. +func (bus *DBusCon) ReleaseInhibitLock(lock InhibitLock) error { + err := syscall.Close(int(lock)) + if err != nil { + return fmt.Errorf("unable to close systemd inhibitor lock: %w", err) + } + + return nil +} + +// MonitorShutdown detects the a node shutdown by watching for "PrepareForShutdown" logind events. +// see https://www.freedesktop.org/wiki/Software/systemd/inhibit/ for more details. +func (bus *DBusCon) MonitorShutdown() (<-chan bool, error) { + err := bus.SystemBus.AddMatchSignal(dbus.WithMatchInterface(logindInterface), dbus.WithMatchMember("PrepareForShutdown"), dbus.WithMatchObjectPath("/org/freedesktop/login1")) + if err != nil { + return nil, err + } + + busChan := make(chan *dbus.Signal, 1) + bus.SystemBus.Signal(busChan) + + shutdownChan := make(chan bool, 1) + + go func() { + for { + event, ok := <-busChan + if !ok { + close(shutdownChan) + + return + } + + if event == nil || len(event.Body) == 0 { + log.Printf("failed obtaining shutdown event, PrepareForShutdown event was empty") + + continue + } + + shutdownActive, ok := event.Body[0].(bool) + if !ok { + log.Printf("Failed obtaining shutdown event, PrepareForShutdown event was not bool type as expected") + + continue + } + + shutdownChan <- shutdownActive + } + }() + + return shutdownChan, nil +} diff --git a/internal/pkg/logind/logind.go b/internal/pkg/logind/logind.go new file mode 100644 index 0000000..f8545ae --- /dev/null +++ b/internal/pkg/logind/logind.go @@ -0,0 +1,64 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package logind provides D-Bus logind mock to facilitate graceful kubelet shutdown. +package logind + +import ( + "slices" + "sync" + "syscall" + "time" + + "github.com/godbus/dbus/v5" + "github.com/godbus/dbus/v5/prop" + + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +const ( + logindService = "org.freedesktop.login1" + logindObject = dbus.ObjectPath("/org/freedesktop/login1") + logindInterface = "org.freedesktop.login1.Manager" +) + +// InhibitMaxDelay is the maximum delay for graceful shutdown. +const InhibitMaxDelay = 40 * constants.KubeletShutdownGracePeriod + +type logindMock struct { + mu sync.Mutex + inhibitPipe []int +} + +var logindProps = map[string]map[string]*prop.Prop{ + logindInterface: { + "InhibitDelayMaxUSec": { + Value: uint64(InhibitMaxDelay / time.Microsecond), + Writable: false, + }, + }, +} + +func (mock *logindMock) Inhibit(what, who, why, mode string) (dbus.UnixFD, *dbus.Error) { + mock.mu.Lock() + defer mock.mu.Unlock() + + for _, fd := range mock.inhibitPipe { + syscall.Close(fd) //nolint:errcheck + } + + mock.inhibitPipe = make([]int, 2) + if err := syscall.Pipe(mock.inhibitPipe); err != nil { + return dbus.UnixFD(0), dbus.MakeFailedError(err) + } + + return dbus.UnixFD(mock.inhibitPipe[1]), nil +} + +func (mock *logindMock) getPipe() []int { + mock.mu.Lock() + defer mock.mu.Unlock() + + return slices.Clone(mock.inhibitPipe) +} diff --git a/internal/pkg/logind/logind_test.go b/internal/pkg/logind/logind_test.go new file mode 100644 index 0000000..66f3489 --- /dev/null +++ b/internal/pkg/logind/logind_test.go @@ -0,0 +1,85 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package logind_test + +import ( + "context" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/aenix-io/talm/internal/pkg/logind" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +func TestIntegration(t *testing.T) { + dir := t.TempDir() + + socketPathService := filepath.Join(dir, "system_bus_service") + socketPathClient := filepath.Join(dir, "system_bus_client") + + broker, err := logind.NewBroker(socketPathService, socketPathClient) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + errCh := make(chan error, 1) + + go func() { + errCh <- broker.Run(ctx) + }() + + serviceConn, err := logind.NewServiceMock(socketPathService) + require.NoError(t, err) + + defer serviceConn.Close() //nolint:errcheck + + kubeletConn, err := NewDBusCon("unix:path=" + socketPathClient) + require.NoError(t, err) + + defer kubeletConn.Close() //nolint:errcheck + + t.Log("ready to go") + + d, err := kubeletConn.CurrentInhibitDelay() + require.NoError(t, err) + + assert.Equal(t, 40*constants.KubeletShutdownGracePeriod, d) + + t.Log("acquiring inhibit lock") + + l, err := kubeletConn.InhibitShutdown() + require.NoError(t, err) + + t.Log("monitoring shutdown signal") + + ch, err := kubeletConn.MonitorShutdown() + require.NoError(t, err) + + t.Log("emitting shutdown signal") + + require.NoError(t, serviceConn.EmitShutdown()) + + assert.True(t, <-ch) + + t.Log("releasing inhibit lock") + + require.NoError(t, kubeletConn.ReleaseInhibitLock(l)) + + t.Log("waiting for inhibit lock release") + + assert.NoError(t, serviceConn.WaitLockRelease(ctx)) + + assert.NoError(t, serviceConn.Close()) + assert.NoError(t, kubeletConn.Close()) + assert.NoError(t, broker.Close()) + + cancel() + + assert.NoError(t, <-errCh) +} diff --git a/internal/pkg/logind/service.go b/internal/pkg/logind/service.go new file mode 100644 index 0000000..86b34d9 --- /dev/null +++ b/internal/pkg/logind/service.go @@ -0,0 +1,96 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package logind + +import ( + "context" + "syscall" + + "github.com/godbus/dbus/v5" + "github.com/godbus/dbus/v5/prop" +) + +// ServiceMock connects to the broker and mocks the D-Bus and logind. +type ServiceMock struct { + conn *dbus.Conn + logind logindMock +} + +// NewServiceMock initializes the D-Bus and logind mock. +func NewServiceMock(socketPath string) (*ServiceMock, error) { + var mock ServiceMock + + conn, err := dbus.Dial("unix:path=" + socketPath) + if err != nil { + return nil, err + } + + if err = conn.Auth(nil); err != nil { + return nil, err + } + + if err = conn.Export(dbusMock{}, dbusPath, dbusInterface); err != nil { + return nil, err + } + + if err = conn.Export(&mock.logind, logindObject, logindService); err != nil { + return nil, err + } + + if err = conn.Export(&mock.logind, logindObject, logindInterface); err != nil { + return nil, err + } + + _, err = prop.Export(conn, logindObject, logindProps) + if err != nil { + return nil, err + } + + mock.conn = conn + + return &mock, nil +} + +// Close the connection. +func (mock *ServiceMock) Close() error { + return mock.conn.Close() +} + +// EmitShutdown notifies about the shutdown. +func (mock *ServiceMock) EmitShutdown() error { + return mock.conn.Emit(logindObject, logindService+".PrepareForShutdown", true) +} + +// WaitLockRelease waits for the inhibit lock to be released. +func (mock *ServiceMock) WaitLockRelease(ctx context.Context) error { + pipe := mock.logind.getPipe() + + // no inhibit lock + if len(pipe) == 0 { + return nil + } + + // close the write side of the pipe, other fd to the write pipe is in the kubelet + if err := syscall.Close(pipe[1]); err != nil { + return err + } + + errCh := make(chan error, 1) + + go func() { + // attempt to read from the pipe, as soon as kubelet closes its end, read should return + buf := make([]byte, 1) + _, err := syscall.Read(pipe[0], buf) + + errCh <- err + }() + + select { + case err := <-errCh: + return err + case <-ctx.Done(): + return ctx.Err() + } +} diff --git a/internal/pkg/meta/constants.go b/internal/pkg/meta/constants.go new file mode 100644 index 0000000..8c69687 --- /dev/null +++ b/internal/pkg/meta/constants.go @@ -0,0 +1,30 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package meta + +const ( + // Upgrade is the upgrade tag. + Upgrade = iota + 6 + // StagedUpgradeImageRef stores image reference for staged upgrade. + StagedUpgradeImageRef + // StagedUpgradeInstallOptions stores JSON-serialized install.Options. + StagedUpgradeInstallOptions + // StateEncryptionConfig stores JSON-serialized v1alpha1.Encryption. + StateEncryptionConfig + // MetalNetworkPlatformConfig stores serialized NetworkPlatformConfig for the `metal` platform. + MetalNetworkPlatformConfig + // DownloadURLCode stores the value of the `${code}` variable in the download URL for talos.config= URL. + DownloadURLCode + // UserReserved1 is reserved for user-defined metadata. + UserReserved1 + // UserReserved2 is reserved for user-defined metadata. + UserReserved2 + // UserReserved3 is reserved for user-defined metadata. + UserReserved3 + // UUIDOverride stores the UUID that this machine will use instead of the one from the hardware. + UUIDOverride + // UniqueMachineToken store the unique token for this machine. It's useful because UUID may repeat or be filled with zeros. + UniqueMachineToken +) diff --git a/internal/pkg/meta/internal/adv/adv.go b/internal/pkg/meta/internal/adv/adv.go new file mode 100644 index 0000000..39a3644 --- /dev/null +++ b/internal/pkg/meta/internal/adv/adv.go @@ -0,0 +1,30 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package adv provides common interfaces to access ADV data. +package adv + +// ADV describes implementation which stores tag-value data. +type ADV interface { + ReadTag(t uint8) (val string, ok bool) + ReadTagBytes(t uint8) (val []byte, ok bool) + SetTag(t uint8, val string) (ok bool) + SetTagBytes(t uint8, val []byte) (ok bool) + DeleteTag(t uint8) (ok bool) + ListTags() (tags []uint8) + Bytes() ([]byte, error) +} + +const ( + // End is the noop tag. + End = iota + _ + _ + // Reserved1 is a reserved tag. + Reserved1 + // Reserved2 is a reserved tag. + Reserved2 + // Reserved3 is a reserved tag. + Reserved3 +) diff --git a/internal/pkg/meta/internal/adv/syslinux/syslinux.go b/internal/pkg/meta/internal/adv/syslinux/syslinux.go new file mode 100644 index 0000000..5effba1 --- /dev/null +++ b/internal/pkg/meta/internal/adv/syslinux/syslinux.go @@ -0,0 +1,260 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package syslinux provides syslinux-compatible ADV data. +package syslinux + +import ( + "encoding/binary" + "io" + + "github.com/aenix-io/talm/internal/pkg/meta/internal/adv" +) + +const ( + // AdvSize is the total size. + AdvSize = 512 + // AdvLen is the usable data size. + AdvLen = AdvSize - 3*4 + // AdvMagic1 is the head signature. + AdvMagic1 = uint32(0x5a2d2fa5) + // AdvMagic2 is the total checksum. + AdvMagic2 = uint32(0xa3041767) + // AdvMagic3 is the tail signature. + AdvMagic3 = uint32(0xdd28bf64) +) + +// ADV represents the Syslinux Auxiliary Data Vector. +type ADV []byte + +// NewADV returns the Auxiliary Data Vector. +func NewADV(r io.ReadSeeker) (adv ADV, err error) { + b := make([]byte, 2*AdvSize) + + if r == nil { + return b, nil + } + + _, err = r.Seek(-2*AdvSize, io.SeekEnd) + if err != nil { + return nil, err + } + + _, err = io.ReadFull(r, b) + if err != nil { + return nil, err + } + + adv = b + + return adv, nil +} + +// ReadTag reads a tag in the ADV. +func (a ADV) ReadTag(t uint8) (val string, ok bool) { + var b []byte + + b, ok = a.ReadTagBytes(t) + val = string(b) + + return +} + +// ReadTagBytes reads a tag in the ADV. +func (a ADV) ReadTagBytes(t uint8) (val []byte, ok bool) { + // Header is in first 8 bytes. + i := 8 + + // End at tail plus two bytes required for successful next tag. + for i < AdvSize-4-2 { + tag := a[i] + size := int(a[i+1]) + + if tag == adv.End { + break + } + + if tag != t { + // Jump to the next tag. + i += 2 + size + + continue + } + + length := int(a[i+1]) + i + + val = a[i+2 : length+2] + + ok = true + + break + } + + return val, ok +} + +// ListTags returns a list of tags in the ADV. +func (a ADV) ListTags() []uint8 { + // Header is in first 8 bytes. + i := 8 + + var tags []uint8 + + // End at tail plus two bytes required for successful next tag. + for i < AdvSize-4-2 { + tag := a[i] + size := int(a[i+1]) + + if tag == adv.End { + break + } + + tags = append(tags, tag) + + // Jump to the next tag. + i += 2 + size + } + + return tags +} + +// SetTag sets a tag in the ADV. +func (a ADV) SetTag(t uint8, val string) bool { + return a.SetTagBytes(t, []byte(val)) +} + +// SetTagBytes sets a tag in the ADV. +func (a ADV) SetTagBytes(t uint8, val []byte) (ok bool) { + if len(val) > 255 { + return false + } + + // delete the tag if it exists + a.DeleteTag(t) + + // Header is in first 8 bytes. + i := 8 + + // End at tail plus two bytes required for successful next tag. + for i < AdvSize-4-2 { + tag := a[i] + size := int(a[i+1]) + + if tag != adv.End { + // Jump to the next tag. + i += 2 + size + + continue + } + + // overflow check + if i+2+len(val) > AdvSize-4-2 { + return false + } + + length := uint8(len(val)) + + a[i] = t + a[i+1] = length + + copy(a[i+2:i+2+int(length)], val) + + ok = true + + break + } + + if ok { + a.cleanup() + } + + return ok +} + +// DeleteTag deletes a tag in the ADV. +func (a ADV) DeleteTag(t uint8) (ok bool) { + // Header is in first 8 bytes. + i := 8 + + // End at tail plus two bytes required for successful next tag. + for i < AdvSize-4-2 { + tag := a[i] + size := int(a[i+1]) + + if tag == adv.End { + break + } + + if tag != t { + // Jump to the next tag. + i += 2 + size + + continue + } + + // Save the data after the tag that we will shift to the left by 2 + length + // of the tag data. + start := i + 2 + size + + end := a[AdvSize-4] + + data := make([]byte, len(a[start:end])) + + copy(data, a[start:end]) + + // The total size we want to zero out is the length of all the remaining + // data we saved above. + length := 2 + len(data) + + // Zero each element to the right. + for j := i; j < length; j++ { + a[j] = 0 + } + + // Shift the data. + copy(a[i:], data) + + ok = true + + break + } + + if ok { + a.cleanup() + } + + return ok +} + +// Bytes returns serialized contents of ADV. +func (a ADV) Bytes() ([]byte, error) { + return a, nil +} + +func (a ADV) cleanup() { + a.head() + + a.total() + + a.tail() + + copy(a[AdvSize:], a[:AdvSize]) +} + +func (a ADV) head() { + binary.LittleEndian.PutUint32(a[0:4], AdvMagic1) +} + +func (a ADV) total() { + csum := AdvMagic2 + for i := 8; i < AdvSize-4; i += 4 { + csum -= binary.LittleEndian.Uint32(a[i : i+4]) + } + + binary.LittleEndian.PutUint32(a[4:8], csum) +} + +func (a ADV) tail() { + binary.LittleEndian.PutUint32(a[AdvSize-4:AdvSize], AdvMagic3) +} diff --git a/internal/pkg/meta/internal/adv/syslinux/syslinux_test.go b/internal/pkg/meta/internal/adv/syslinux/syslinux_test.go new file mode 100644 index 0000000..93d77af --- /dev/null +++ b/internal/pkg/meta/internal/adv/syslinux/syslinux_test.go @@ -0,0 +1,378 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:dupl,lll,maligned,scopelint,testpackage +package syslinux + +import ( + "bytes" + "io" + "os" + "reflect" + "testing" +) + +func TestNewADV(t *testing.T) { + f, err := os.Open("testdata/adv.sys") + if err != nil { + t.Errorf("failed to open test adv.sys: %v", err) + } + + //nolint:errcheck + defer f.Close() + + type args struct { + r io.ReadSeeker + } + + tests := []struct { + name string + args args + wantAdv ADV + wantErr bool + }{ + { + name: "valid tags", + args: args{ + r: f, + }, + wantAdv: ADV{165, 47, 45, 90, 142, 155, 111, 208, 1, 7, 116, 101, 115, 116, 32, 109, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221, 165, 47, 45, 90, 142, 155, 111, 208, 1, 7, 116, 101, 115, 116, 32, 109, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221}, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + //nolint:errcheck + defer f.Seek(0, 0) + + gotAdv, err := NewADV(tt.args.r) + if (err != nil) != tt.wantErr { + t.Errorf("NewADV() error = %v, wantErr %v", err, tt.wantErr) + + return + } + + if !reflect.DeepEqual(gotAdv, tt.wantAdv) { + t.Errorf("NewADV() = %v, want %v", gotAdv, tt.wantAdv) + } + }) + } +} + +func TestADV_ReadTag(t *testing.T) { + type args struct { + t uint8 + } + + tests := []struct { + name string + a ADV + args args + wantVal string + wantOk bool + }{ + { + name: "bootonce", + a: ADV{165, 47, 45, 90, 142, 155, 111, 208, 1, 7, 116, 101, 115, 116, 32, 109, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221, 165, 47, 45, 90, 142, 155, 111, 208, 1, 7, 116, 101, 115, 116, 32, 109, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221}, + args: args{ + t: 1, + }, + wantVal: "test me", + wantOk: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if len(tt.a) != 2*AdvSize { + t.Errorf("Test data is invalid, ADV size = %v, want %v", len(tt.a), 2*AdvSize) + } + + gotVal, gotOk := tt.a.ReadTag(tt.args.t) + if gotVal != tt.wantVal { + t.Errorf("ADV.ReadTag() gotVal = %v, want %v", gotVal, tt.wantVal) + } + + if gotOk != tt.wantOk { + t.Errorf("ADV.ReadTag() gotOk = %v, want %v", gotOk, tt.wantOk) + } + + tags := tt.a.ListTags() + if !reflect.DeepEqual(tags, []uint8{tt.args.t}) { + t.Errorf("ADV.ListTags() got = %v, want %v", tags, []uint8{tt.args.t}) + } + }) + } +} + +func TestADV_SetTag(t *testing.T) { + type args struct { + t uint8 + val string + } + + tests := []struct { + name string + a ADV + args args + wantADV ADV + wantOk bool + }{ + { + name: "set test me", + a: ADV{165, 47, 45, 90, 103, 23, 4, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221, 165, 47, 45, 90, 103, 23, 4, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221}, + args: args{ + t: 1, + val: "test me", + }, + wantADV: ADV{165, 47, 45, 90, 142, 155, 111, 208, 1, 7, 116, 101, 115, 116, 32, 109, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221, 165, 47, 45, 90, 142, 155, 111, 208, 1, 7, 116, 101, 115, 116, 32, 109, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221}, + wantOk: true, + }, + { + name: "set test", + a: ADV{165, 47, 45, 90, 142, 155, 111, 208, 1, 7, 116, 101, 115, 116, 32, 109, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221, 165, 47, 45, 90, 142, 155, 111, 208, 1, 7, 116, 101, 115, 116, 32, 109, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221}, + args: args{ + t: 6, + val: "add test", + }, + wantADV: ADV{165, 47, 45, 90, 197, 189, 210, 250, 1, 7, 116, 101, 115, 116, 32, 109, 101, 6, 8, 97, 100, 100, 32, 116, 101, 115, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221, 165, 47, 45, 90, 197, 189, 210, 250, 1, 7, 116, 101, 115, 116, 32, 109, 101, 6, 8, 97, 100, 100, 32, 116, 101, 115, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221}, + wantOk: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if len(tt.a) != 2*AdvSize { + t.Errorf("Test data is invalid, source ADV size = %v, want %v", len(tt.a), 2*AdvSize) + } + + if len(tt.wantADV) != 2*AdvSize { + t.Errorf("Test data is invalid, target ADV size = %v, want %v", len(tt.wantADV), 2*AdvSize) + } + + if gotOk := tt.a.SetTag(tt.args.t, tt.args.val); gotOk != tt.wantOk { + t.Errorf("ADV.SetTag() = %v, want %v", gotOk, tt.wantOk) + } + + if !bytes.Equal(tt.a, tt.wantADV) { + t.Errorf("ADV = %v, want %v", tt.a, tt.wantADV) + } + }) + } +} + +func TestADV_DeleteTag(t *testing.T) { + type args struct { + t uint8 + } + + tests := []struct { + name string + a ADV + args args + wantADV ADV + wantOk bool + }{ + { + name: "delete test me", + a: ADV{165, 47, 45, 90, 142, 155, 111, 208, 1, 7, 116, 101, 115, 116, 32, 109, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221, 165, 47, 45, 90, 142, 155, 111, 208, 1, 7, 116, 101, 115, 116, 32, 109, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221}, + args: args{ + t: 1, + }, + wantADV: ADV{165, 47, 45, 90, 103, 23, 4, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221, 165, 47, 45, 90, 103, 23, 4, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221}, + wantOk: true, + }, + { + name: "delete test", + a: ADV{165, 47, 45, 90, 185, 209, 145, 51, 1, 7, 116, 101, 115, 116, 32, 109, 101, 6, 8, 97, 100, 100, 32, 116, 101, 115, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221, 165, 47, 45, 90, 185, 209, 145, 51, 1, 7, 116, 101, 115, 116, 32, 109, 101, 6, 8, 97, 100, 100, 32, 116, 101, 115, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221}, + args: args{ + t: 6, + }, + wantADV: ADV{165, 47, 45, 90, 142, 155, 111, 208, 1, 7, 116, 101, 115, 116, 32, 109, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221, 165, 47, 45, 90, 142, 155, 111, 208, 1, 7, 116, 101, 115, 116, 32, 109, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221}, + wantOk: true, + }, + { + name: "delete test again", + a: ADV{165, 47, 45, 90, 142, 155, 111, 208, 1, 7, 116, 101, 115, 116, 32, 109, 101, 6, 8, 97, 100, 100, 32, 116, 101, 115, 116, 6, 8, 97, 100, 100, 32, 116, 101, 115, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221, 165, 47, 45, 90, 142, 155, 111, 208, 1, 7, 116, 101, 115, 116, 32, 109, 101, 6, 8, 97, 100, 100, 32, 116, 101, 115, 116, 6, 8, 97, 100, 100, 32, 116, 101, 115, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221}, + args: args{ + t: 6, + }, + wantADV: ADV{165, 47, 45, 90, 197, 189, 210, 250, 1, 7, 116, 101, 115, 116, 32, 109, 101, 6, 8, 97, 100, 100, 32, 116, 101, 115, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221, 165, 47, 45, 90, 197, 189, 210, 250, 1, 7, 116, 101, 115, 116, 32, 109, 101, 6, 8, 97, 100, 100, 32, 116, 101, 115, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221}, + wantOk: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if len(tt.a) != 2*AdvSize { + t.Errorf("Test data is invalid, source ADV size = %v, want %v", len(tt.a), 2*AdvSize) + } + + if len(tt.wantADV) != 2*AdvSize { + t.Errorf("Test data is invalid, target ADV size = %v, want %v", len(tt.wantADV), 2*AdvSize) + } + + if gotOk := tt.a.DeleteTag(tt.args.t); gotOk != tt.wantOk { + t.Errorf("ADV.DeleteTag() = %v, want %v", gotOk, tt.wantOk) + } + + if !bytes.Equal(tt.a, tt.wantADV) { + t.Errorf("ADV = %v, want %v", tt.a, tt.wantADV) + } + }) + } +} + +func TestADV_cleanup(t *testing.T) { + tests := []struct { + name string + a ADV + want ADV + }{ + { + a: make(ADV, 1024), + want: ADV{165, 47, 45, 90, 103, 23, 4, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221, 165, 47, 45, 90, 103, 23, 4, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.a.cleanup() + + if !bytes.Equal(tt.a, tt.want) { + t.Errorf("ADV.cleanup() = %v, want %v", tt.a, tt.want) + } + }) + } +} + +func TestADV_head(t *testing.T) { + tests := []struct { + name string + a ADV + }{ + { + a: ADV{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221, 165, 47, 45, 90, 103, 23, 4, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.a.head() + + // Expecting 165, 47, 45, 90. + if tt.a[0] != 165 { + t.Errorf("head() = %v, want %v", tt.a[0], 165) + } + + if tt.a[1] != 47 { + t.Errorf("head() = %v, want %v", tt.a[0], 47) + } + + if tt.a[2] != 45 { + t.Errorf("head() = %v, want %v", tt.a[0], 45) + } + + if tt.a[3] != 90 { + t.Errorf("head() = %v, want %v", tt.a[0], 90) + } + }) + } +} + +func TestADV_total(t *testing.T) { + tests := []struct { + name string + a ADV + }{ + { + a: ADV{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221, 165, 47, 45, 90, 103, 23, 4, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.a.total() + // Expecting 103, 23, 4, 163. + if tt.a[4] != 103 { + t.Errorf("head() = %v, want %v", tt.a[4], 103) + } + + if tt.a[5] != 23 { + t.Errorf("head() = %v, want %v", tt.a[5], 23) + } + + if tt.a[6] != 4 { + t.Errorf("head() = %v, want %v", tt.a[6], 4) + } + + if tt.a[7] != 163 { + t.Errorf("head() = %v, want %v", tt.a[7], 163) + } + }) + } +} + +func TestADV_tail(t *testing.T) { + tests := []struct { + name string + a ADV + }{ + { + a: ADV{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221, 165, 47, 45, 90, 103, 23, 4, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 191, 40, 221}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.a.tail() + + // Expecting 100, 191, 40, 221. + if tt.a[len(tt.a)-4] != 100 { + t.Errorf("head() = %v, want %v", tt.a[len(tt.a)-4], 100) + } + + if tt.a[len(tt.a)-3] != 191 { + t.Errorf("head() = %v, want %v", tt.a[len(tt.a)-3], 191) + } + + if tt.a[len(tt.a)-2] != 40 { + t.Errorf("head() = %v, want %v", tt.a[len(tt.a)-2], 40) + } + + if tt.a[len(tt.a)-1] != 221 { + t.Errorf("head() = %v, want %v", tt.a[len(tt.a)-1], 221) + } + }) + } +} + +func TestADV_overwrite(t *testing.T) { + buf := make([]byte, 2*AdvSize) + + a, err := NewADV(bytes.NewReader(buf)) + if err != nil { + t.Errorf("NewADV() failed: %s", err) + } + + for range 1024 { + if !a.SetTag(1, "yes") { + t.Errorf("SetTag() failed") + } + } +} + +func TestADV_many_tags(t *testing.T) { + buf := make([]byte, 2*AdvSize) + + a, err := NewADV(bytes.NewReader(buf)) + if err != nil { + t.Errorf("NewADV() failed: %s", err) + } + + for i := uint8(1); i < 255; i++ { + a.SetTag(i, "xa") + } +} diff --git a/internal/pkg/meta/internal/adv/syslinux/testdata/adv.sys b/internal/pkg/meta/internal/adv/syslinux/testdata/adv.sys new file mode 100644 index 0000000000000000000000000000000000000000..d913fc3a6a6a6cf861fb1e7d8b4b53dfadf8dc1b GIT binary patch literal 60416 zcmeFZd0Z368#uhPyV=}qfXF3kHd$^&qKHJ$hWn741Ofu0g@}rZkN_$oxS%X+)YjW- zt5sX;(Q4}fo*W8lJ@BYkwMA>~Vmz?69(d$kYQLZF?|t9@-+$ike#p+u&NI)<^UUtd za}82aMs`MGe6rr42o4AgQb1CX8^m4j;qS-)bg^rS34$SLECih!Ua)^56eE8)x)y@o z9EWPzM!?+}J8^YeFKj#vze{Q}?lFS)d#{bOlXtBVKlID-2X(dMV-`SW!SRQ6o!H;C z5FHE!vMC_LXbFB&$Ji-q`@P>G7?l4a7Fczp=*;Q|ilkc?-(3O@qfJL9dp$vAZ z=I-O&s*NvSsLw1pYkc|AIL?MZ5WXbrEgKK{T+sYo0n>lC-p9fPcNkBh5M$U-<#WKXhjSKxi) zu6r-oQLwQ%bvXpFUqi-mJK_L@H@tgpES_a2gV%9Z}NGyt9R@3 z76@8)=_pkF5UPftKF_CMRQYvIAFk<51lCn#5U3Oz>H|;41*(wU* zB&A2jwjR_7PQ7Idp-xP?ipGHxvD(A#_WsV<~5`c1F#@O6$mMxTBot$)kaK~Qb>y^apL zhidiTx=r)nDf}^j@!Y2Q?^GL)6hShm*XzA;)4x@8r}2nA;GQ|YpZXhaEZ+Mm_Y&WG zX9O3G;6lfd=_`Bt-=*%}sc1F#8#-#ovukX4!(u>jnUre z_8KnNGmyK#X0vJjm+YF}Z17Jjlk7OOj++6K|8ZyotAoj_9UM?gE%#)5M<%DOj2i$q4>hVUx70xY(O&BM2O66N1N8%~ z@;g$G3)ZPycZYPFxHrx;XgBDrd;fNQ*%Pa1H{AyM#=Tu%0jtj#I#M&U6pzi>Y5zBH z)qlZVcU!98XMxU^x9feLmGPf%YxKysUeC9Jl)E!8N;5+fajexQ(h%Z#bUEgT=k z*zGl@p~fv7BgNSLH73Uo9F(woZTk_{5qf3haC5)Z{V>ae*w5b^IvPE9Ym5a)cIx+s zH#xCu&O07ozO2ly?u~w@S@5C_n2_U>mi>d9;pXTFHxmxfKlmALqK81AvOla|iDioOdi6 zq_}UkZ+j?+;J)vE>h4Ee^33WmmRt1930(faN*BkL|A*dH>GNMzYXIk7(8Tq9j2lv>MCO26X+`2ii@Il>sy+5pjdj&Xj zgYnqGbwRVX-}>Lu|KuO;;MPJJRDd5ekqR4chX#U<+~FQsf3Zmi3TDhUXg}9xGIbKY z*s)@TJ}WaLEuDHKdCQncZ>E|IJjN-jkR|3F5=HvcQ6x&gVsvB>0@ElrcJ^K1ifI?O?%BT;-&GG`}f8X6<`+4R5?vO7%b`7852buN1VJ zZcwe5MaO@qk(2nMAzT%Q<=qD?0hwf7U+!XVk zVDb7J@43Ot`Tx*lH_%RweHcJL*Ae3cI%#JxKzr{P8wa@GlY0t%=;aSCH_*#JU0mS` zORo9!KHo-Pvu>+v`)7XsL(6wu<$tCpQ7o|z{ufV}tjwQoqk05wJyPl}-}E!pHb4_c zjb$y=Y8#-;r15F*^&IHh;GVSSzZ&|$ZyI4jgtV?1@{i)T2QYs ze06JR7+I;|ULtQqNBV27ReF{U@#9{!f!B&2`|<{LQj(KBF@I8CM>9t}Wc{sKFp%7C zMhB9gng{kH>xE`)AnD3TPo>`5Of{PKWdU^mrHAG4w)n==)N8oRi7W7o+o@i-@ifQv z=i7I{CfK;0+bO{|L(aqwOO%~W(^Yx)ouwML(VYX0%ecnLDIc0Ex#EZB)DxI``X3}5 zBRFC{po)q82iaTeUUln-zIVnq{y5OL^FMv>8tD7SfxdUW=osjnOSs;5T5IVp`dUBz z#-hV<>Rul^FptLvc1HTT&z~(*IKnK8K3U0q_uFkfmM;&kf}px~h`ZD2Fb*7;P^KJZ zCJ3kw$!~o24fnEGAK@S;FpYR&LN?j{pkEp32nX)x*2c;jBjk-!oEszD8>bBWM?TtQ zJj{ts|B6J*F8{M)$+#Z>y+%L-9YN4ksFnxO5MYd-ro<%aa3eDOoN^568|wgXAQ z{DJ!or(8J%atA@+3224ffC8Ka1x_69|6Tt-=D^_#5NA&QzuT8b2C6lIBS!}Rhb_W? zGZ?pN{r8eVU;Q8D+E{xC+B_1n@dbecFz>j18D^xS@R5Ovsgq4bidnNwin5~e(%G{L zr_Gv5NJS8|>0e}@|C|2v9{-z%AVuWt(z#{RrWTtN-jO~EwMG*(oEzC_MO=U)wa8Rd zrU;)^F>RKj$fTHD!l9FjB4%@a7$;9FDJmSRC@m?PTwbL3xOnocsYQyyY2`D@6_aNb zDoVLQa5=?{qPYswY{kd3XPKtWG8Y954Cvp>mEpEjs{}u2fB=IwKw3z=@xwNqq0h{+ zjouao(6>`L3$h?Y2Vp}+A=J}g<5b!JAh)*PQ1C0aODoi8_KBZw-_<-~DFp3eV*m2E zZl{3g*uOYA_lWz_-jA2}+2Z-ZwE`Ryg>XAM$!S=ildzd#`sE@1R0_lS!OMB`q&8_4 zF8+@^FeyK8;--$kWSB`5*!$+fr4IaU8lQ{QA8)Cr65mFclLq7-Fg)8aigXwHB&?I? zIzfE;!$QmLA9s=G+l)tIm-Zf7)@P&pM!LWdZ(?5lq)i-wBq@j4vG1IY4&uY}vKnP4LwXrWF*GXW%pVJYl63AU{C z)_XVA)#5ar__hdGs8?b=UzjKc#xBY|hR?S< zE~f0$f%**USAg!3=#nK9J0b_Av37Mr!CZN#*attzJLiA^(Aw0PS^|(n5^kzZI{+hI z2S9U$^t1K?B^pJ-nm)6z0PqmaE7Pb>>P{L0$2$Qq>{0@tUrE7F3!pwba88d}k@`fu2wo>PO<5pWeMhqD zj-<+K6PT1MS>>gdxU7~q=p^Ff3I(SCR%JsbzLEK%Fu>~+2r=3jguyF`-%Rt_6RMYIBlpcXcV0wu}}% zk!-(O?@W;yZ0>qV<_!r-OGu`Kbob81Dx^r8MR=Q&nl_M^LW`ir@QPj4I#(J6zX`OT3%R7&y{7xW z7xZ3oZqeHD$1G z>(#NT0JBholAhYwLg6=9G>v`adj<-bj=ek)&ts+o`m6{wC&y#z2^d2a#K8#F176;k zc(KBT1@J3~Re9veZGfHrqI;RtQyas-s=e zH!G&WEdUk?0wAQW-!`n0^L*=~D4Lap;C3fNH-o?oPumMgyWj6pXvb=#gq%ax@F*ckr1sI}rA62<{ z`@lJ{ySXCV7F2}J<_To=D1d-9$^QT-@IDBj#(;xh|HT*5;l|uSAdEALtg;)~3i$LX zgOyL6w^`x5M=UrI^~ajJM}`RmJKQ>Nl2@={l8@ccyGOq3gDutZPhY1!b((^QUTk=h z3ad7Z$#sC?9jG~e74muEkc$<|UiFCQ@N5X~*v5(qH%L-7?d4=@x`nDeX0xRh>keU% z#Y7fovaKh}_pq7!68NHRdfCYWhi^Y$`>?{{lhFvu3@SBc@~P`7Pi?f!Lbc~XTc^Ql zqLWw`oFzkLTFGxQeDmKzx&3sWAtH7hIhnrdrk#~U_$CRmKZ`+}Jw zq>6`i@vz!BfEYDx_@c@P(6m@t8X%y0cws!V$VeR*REW*PggoR8j5nlwj8Vr$i}3pJ zN^BVk&*Cd#GS;VQv0@tlaMPN>4T$SefFNNCv$okduC^bWTHjL(#2CJ+wYHjsW%#Og zqB{lvK@6zxmhj^%p*6hvGr}R?=0UxsP!x(FF6I&HR*PU`s2woP7YK>+K(n>Vguybi zx*7f+H%*<(6IWrS{7V>b9~QF;&Bw-Bd4A|=I~X?yH6xe!zH|7Zpl6V94O)Lj@M|MK zW?Qa?KYa)d%o)6b@f+h79%g+HMS>F2rF>)WtasR7C(!Q>=Tad&*JOW@$obHjWOzoh z;6?3BVXes3SSP%w6>lgZgS>YNbI(>abtj>#Tj?%Km@Q^iSZ7=$ctxFzW4>lz{!WxB z{|-!Fykgx12YSW2T>5&gKYKR|sXKMufDq4oek5GHmF#fQ^X(h62Q=y02k+|cPLT=E zrcCqBuozs?^ibddeda^|ps&qDXmY`i@c0~0vn^dMTv6sfYSVScH zqY!Eezz=?}OdTVppIfH!EORmX;iDZ2QxwU6MBKoSw{r5zW2BQl`-JE8fH$g^JUSY| zk84Tm=Lzzj`!nRXMgX0Od=NqhqA+69{_tSM zfh{-eJsyQ@xL$qwBMQhgRmbcgV7t!jNkG!v>znhxoL-_nDB4e-8#Fds+drENNM1^TN4KVI>gIb!V#9F%0KV>r z?=%kdv)?t}$Caf^JepEF1uxn~Chp}B)ZXJQfKb(MK^<&rijbE)+%J;ji6GJl+j?+C z2OC-@Z698Qe!2v8VA6#p%BW!2qV-_!WCS-xlIS$?#*I1jC3Rccrj6N~)Mqo>SGRVJ zk+FTy^GSmq`h&Uqarm&mq@ip96pr4)51z4viIp4udHA7X34W*)6Pzb>1T-ic3w0+e zvl||p^h}k&5`;GmZOkR@YTqn0Md1hYLG2r)B9v;I{z%Yd2jL-hkeyj7$+fdD7wEo} z2*6FN96#6&`mD~JN|Lx*iofRsUHYa*LY6;CvVNfsPnVtbWm7jfCgeH-y0e8h>-9>V zaf#pe2_O?E{C$OvnrkK=jKx@A*0*YNaRTC1vr>@_qe9-Lb#Np}wT>dPf*Y6IFkf8z_7bK(L;r$*x#T%rTxR$}(#8rr z@ID;kR{jcJGZu?dt$|*@Lh2XI)tyP>e@(~U@N(6>h2!3HhMG!eldu5!TNsoj47AGh zM}vy#JSealTIg77I3STi!lqPEf7#@K9~9tQ6G`uKq|q^wYK5(TbRsL!9WIgiMD5iS zjESPPtWP5#ACtA~Q9(AUDv)zKrBPTg6|3bWe_%zW3?A&Wau|V9?nwNhkb^r&310s=Me+Wu2{RoB^Wg(#4vG1S?o1jNGN~42bl>uZe9MeQL0=(etfwC?fO)rJBixBf zKFyi;f|)ipkr*2j)JTM>&{hfE)$8`&yN3Xl)Rz>zWKBD`uSl*FcI)bhfMCZvHQRXJ zdD|M5FT*UKhE#5MqbCTdbsxj@3KugZin+x(iWdVpd+&Az*46Emws-;GvyHxv4wGmu zEbyh1?lb5iCa;gpS>UL*`{`5N)bmj?kPg#SDGNA=ow#J>2FQ+>>+jNJ86B~XuVG)f6oAq%g28COHO%YPfaT|Vh zJiKtKr8e-q$XmC-fOtnhwquLnuG(teEFRNXjjI?ROE@lhD}MQs+n9V#aw0BHJ?Kl| z&8RK3PfheGg)%u4I_4YwC~bN)5Uwb6N^Q&+&)jXgRSK`e_?qq-H#%r7t?xOEMD8(2 zBqj_n`kn~uO(yA47h1P;{bVfhg8aADTj0Hn;x+wegUi#XnhL~nB`OKLvNwdj=T?{*RVssUcQ1YXzrBncE+ zKuG#$Y;E#q^n=9rt2ck41FODJu830FeZ-DeLHl{)kKHuLGXcC=zxoI+b$T=7Yt~~I@k^dn@;6TC zaL#fv^hDMl+craI73}&1P2WQcxL_%Zy*f;OOa z&8)7Zo!5gxu{J~^t|mVl(i zHDc6BuUCbW#bJxY!K%52epUF4koXj$stbd5yfyLIB@E=(@AK1+?=$*r9?}?88K%jS zyr6Pt*W>lyO;+?D61;#wjm(sYX{esUe)QnO^)KI9b`=aFvcennpps7#_VG?GIxO?1 zqOt0+N6m~IN2WVa_oi|hAfharVa=j3*$ylmbFBdiXo=7rf*$fW9q~Gsp6xW$;8cr! z%17)IBkrK)e3V^Xcf#PY-8)@)|}P#_f-sznhT9} zVPr6B&K;2Yd6jzIa15bXIBaNTLW`tvm8sm5UH!FLq2u%0?g=3K%kZ;(rLZuOMCh|^ zcLkc$=0Vrhw5;>6FFPShp6^nAuquLx7_snL`iupHZ}txSSnshjubHn-nu~n8 zbhPqVLTbWjt&5j=yd-`Ntt>L+=JxP8kErJP0<8rdrMi%~{v{ZEN_W5z3h?hvA>q8U zN@0L4XH-LLDJz)Ex8wo(rB!4WBtFIbXp2J44IfCr1Tt|b)A4a<`>sC*H$a9mZx{k5FYtyO2D@YBEq;)Hs6CS`6>?anyKDggLEyMgwSDR zoMWz2xPt`S#gsw{Ns5gAvF*(;CZ>?dEY3)abQ7?%iLWAavg5~#i#JFdsRbxxma2aS zF>?ESm-#TTnU2~fISXhu6$2d&AkQsA*K zK=L_+l5?TsrB6g^Td28;el<9{8mBs{x`+p!#eV6ctW3jjAYbf-nt4{{mdP!%#g>&b<{hZsUUwu@l{K6SBj*akQ&HI7%x8Z^uL7%(9sCVC zN-q`=u3ppfAPMnu%FJ?N8gAi5wN&8jJ{>O}nWn1FG9dYo#6~;{5QLG4Y3K|IDLD@1 zcH?RrzWwEu2QS(bCX{}ue$hK3XGq=2Qsr*)N_!A{Eb4Di*R^YC90`FAT7Q~OI!EiT z(n&WgNmm&vD3^y?^xf&UAHt%+bUS#hckEZ`hZSX|{_o6{vP35#&lcTnl~N13Ia{>h zTa<3M#xc>|%KbqwzpaOdtQRmmKZ8ICNiWd9BBcVyDhX|Fr5BuJ7PM*>oMslB8@%9T z`6u9dC!I6d|g=X3vi2bW1`N${f<5}JrT@kC4e(V zd}hSv>GjS*SL` z47-9p%dx5hz*1V&QX&A)w^=OcXbiHI&SSI{>dV{ZNZugF+(E?Mw6lu_l}px5D*;hb z&#PX5nX5fMXLUoA0x;PHkt+ZMYksY9rZ3Xh=ilXT%WfiMRBAk6*=zlzaTX)X z`$n(#P^R%^ivw6=zFq#LXiY=aQ=X-oEVS$=H|1k3-qVI{PeV<4!>eWn(+dNRlIyfc zJ*BEDCF~RA`BJ(=BJ1Rviszr8Bgx98bfmXRJ~M$&N1`$a$#7|VDxqI-22navM*ShB zYyxx5T<4O2S8$%wP`}%7xVMaU`hmbabFvfnz^M9XYxo4G5iO(F1?iEfaA{op-bnlz z!PGBOFsoGG`BJU(^udhCdo?I!#GucWzH&f*ZaLI9I<A6v>4Wn)#QAYN0BAjBZ39KIVCc`{W(#vo(W)^73y0F+nplB57jT_ypBAVMt)QH)y>apQFbrU5t=qbrAE$V&b1IH1P2zLAdlkCm9yF$)w0Pt^M2J zRX74RFmbtAIbM!>B4YUPSC_oRM6dQC$CT9U{gTo}@$(b~uk?F2edQ29 z+G$_Z68FSXPyYDBHv-l<$}4$a{kE-hP`W=teDL*j%(Q0{fAcZEy;lYF-TT$Za^cXw z7th=O*|vEj<|6e~gFViByK^f=Zk!z4V4MH)nTurPJ2|H2h>~A&9uacRmpja0qdhF3 z0iBjN=j44&4Ehb!fbOY|)Y+w^Ia*hcyHy~Nbvj!UC>unMT4)Imj8b#DYPi2L$pbCP zLQy6{$fQfX9ykC-;#~DC4Y+44M>Ro2()eejV>XkD#uY5|)sf^TAUW3K9S-}~ohlYX z4ME}(&c6`c)u4R4aY0M7QZ<_WXi1cOHDb}OuD!vdyPj|FY6ugrL80Ci@AX#8ynfpO zk2dd>KJDIb{`6?|2Bs`#bq(hUD&{K>NAg=MMg(uj9AO=+5~d$3k%~3#HFCO}+wMS@ zrLt$09T$$1VEu*DGyDkHR%cJ)qZPeyS?S3@AHBG>BIoq!3ZHgIq9BT2C1A3msC%5T zxFVbG{H!2!$%;&9@LJ{hb>&33b_ngTm3XYoP96_~EfH9Igy_cVNbm{ld>I~mzi*PN zB){B? zQu5erc^~^AF)MPiBYksm)-NwCyyHOxTf1r$haC>h zM;Y&G<`?fvPl>BVQWLMaD)hx7^{aUEW4eB270(#aHTqg}l&bOTIpJMNQUP&bL~n^f zyB=YZBy_T*(FeQIL&wOvV&oDd5#ypX%mz#hdU!ryM}yU_Sy4*P%eFAlm@z5Ie7J65 z+@y%Obn`xD?Pf1B0Mm4r!*n;>S)b2aqFt9RG_7E21nQsZJE33fFa^PcW$h@WBH7J; z!IU^d1Z>GgCW9$Whu-S{{wW;1&h^ZO+zmwT4WCTGIhluSHmK{^ZNHZAWfln45yH-N zX0&ov7dk>fr%Jlu4eKM&7DzUouRa5bC6NxA(={HJ2svFw zc16hdBaL4`2%YXuYhU4qGGyRdI47Z9>5N1{iem>8H>p=br=gG0I02m?=+ctHnBjcA5t2$MRlpF@ zUq=cd18^!ALiqr`;a#)m?a7MZ7q}|Q2fjvMWY^7iY|~U#i9~@W1+Av%qD-2MmPCFP zElVZNkB_#ScLpYxs)^8V!(}->6Ey;MSDk44WKnK>$En;ul(omQtVK*Tfz{en2|GFa zz>R6dn3$#{`S7bA*d@B1iALEx+lpDjKt}K4bBlhjxn2QJa=ZoFiIB~S-zJOMzR0CM zF9AVRTo&IV-R*OP*0m90(ubJ4MBv#MLmV9o!9LfT+?_bHi8w-Yw*0TgT$#=KBpRT~ zSvtDgW44L?yCc^5l+`ZUxr-c?5K1kyn`GuClv_s<9pkRBR|~#GU)fBLEvu*LZ>JMC zquSFl*324PDOd)YY}L<{H_YD42zJeMamR{c@x=4j=?jF6po9&wAteAE2N6^X>d zl$N#%phm9;fB#qLds5b<%HA7dao#(2~2Ajp(D19?gRg6t=i6O4{WN!9&t@i=$& z&-k_0@v=X2sQoj^a#aG3uT0@d{YCU&n11tOeC#nN^ z?!;y@kv*n}*gM<+@pCP!*R?*i;fKy9IRNRM(?my*>Pi~{8T zSpcsu*&XF-0+Bq4rb9b7H#v{jvPjrQ_X^vHecuO8z@SQT6i<1`zu}Bo2q!j3!X^Ty z(xqy?KIlb~M3DzoD%D#5bX?ADPv+r;4TWff(SP)}~xiVaWi(?rgXq8}ZMB@(h^Aw5^6X z;PJX3DsyNEpZw~8+?LE%1`&@vOSqp_wfmb0>6NX7a&kRcI9UKpMq45oV!nLoNpAiM zJXRyI9!|g&TA?AC*g=Lnxf7pVw5-ld1F6I&bzm~&ZUgg)r`cs^%8@c2(ukwo+>B&Jrb~Qm;W%5j+LdOa-ikKena}~UpIOJ zaOoXIY7V#x=Y8`+rz6$*{`vPco%t!9p>{me;P3z+>F4rK)TkxpVN@*4j^U2Fc=Lq5;r8yv zIAhoPp)%rD;SUeA;C&LsrUIpUk8cuqSb%sePp7Fz^T?HbIw&nUH!V4EWinf^r7^kW zTOxYK*9Ea$JdC?}lV#N+Z}piLRFjQfQW_Q!NqZ{dzE>VzD^E>Iyt?14oHbeQC)Ewc zb?K?h{Ki*onsQoPr%tI1E1BdbztR)8h49&8ht9+-&)?hVSpQzC@v}?W@;TX5X}0G( zL#jDz>6$ExuFPT=TXIi;lX7d4*A6sMInCR#$=fX1oyjSgZgmYGinJO2$I%jB= z!T=GQ)>d&Ja$QpK%uIQ);bF?MrJFKEdVQd3R}tP27ZXb#QLgo&f9TdhoN;yT80?rv z5I;&=`sUk!Q(vfDz5vC*>^g6Fo)(=`f~5%y`13!asbW469xV5MJX@J4oh%GjKAr8q zQ~%a384g2=4;45jPVks>r0_Q`Qm50D5V_Kz-wBWw@|$f1;St{+NLrT{li>5|oEBz_ zH12`Ka%4Jp?BJB%joem7#%ZrfUwnRDYLXn*{j9wcbHN&)q%0E8e2gDjA(0rVSMimy zmIN-m(x(l)%J)iUrcbB#mLl*t8>r||EjwsFx=dgdsWyZ*Lveqrw&Yl)P9HrHt6wO; zo+F|e<&tQcYR!W1W34cJQg(4Ybz62JinD}*4Sx7{lpjr9o$p7De-WvFR)3~2`OJa5=RjU_AkR6F#~g^91F7af z$~h1*2U5&|hRuP7&Vh!cLtK2>v~LQhA=5oF+0iR!4oW0O*9gdvHW2l09b z-aLoS>BNw8wOP(~%pAO&lLhp=2ZVlfj@O~hGWPa_uj@Ll5zy%=I~i|f?nvU=h$K1T z6m{ic5{!CUXRB>Y1QIXjEl0HgrQZ*p<5E6L_+FCMpd;^pI`fOUhFXZn%ydao%|gK` z&RI?KInkAlMe&2t$aGE{oUYRIiR)kX3o6sUTUkY%{z4Cnc)TQN2C5fWVkPX$#B~dK zol$(kt>{oR<`9E=l%_4-~xaevH>oxu) z-90$QrK~c~u7+5dBlNlHX&d*H?jZV15>39s?5c@zH_I)Vg>&{M13aQmrncy(7klK*C}Xh6b5{razP!J)>`4%7`ggB3Tklh#$sUB5cG-m8b&cG*l*2rs*zoOxKTp^7+>>jWfnetlmYas9X4#1gW*L*8ege^}xSgwo( zfXvwX69ap^tZ-+q7xQ&BUV?cZjkjODjcAvsXZ#@vjh3j0(0ML6Rs3b~?X(P+^dIlM zRsHm_LV3pIhX@DjUDjsKaiIyz?UzJcU|KsW`bhJB%4XL~;SDc6HqYl-VqbBmQp(fQ zB?^T2HuX@Zo$!?(ib6b1d*yg)JY?of1TiQl4Op+?2V*czj@T^Ra53%yjb-^ z>P^OJ_8aOXSu?EnnKHL(q6?sOi7u*%zbteCann_@d2#xqsE67UQaWAnKQM*#guIdy z=zG}f>F_M>NR%K=d39t#I~M;jv~HcvWOuWP*jtnGdl~8Zwd4 zSt>)q9n566np*M+aw!clU_|e2leg+zLaH;-^pS{`e7+m>8G!FJIGK=<%fYS_~OhDKd7|(?AhEe`!6qgvO>O?LgK~5JY`4 zYkHjDWb$;p;aOqYyuV`Q%DHNT6u1!KWUb<2gyUJzWhK%h!$O2+55mt1)&3?N60J&6S;t$T*F@7{;(EID%EL_u zhF4*{LuiOqX_0T*xY&OIBwctN4e_W#eZ6NB2@Q-(@;M1Z7*FJrrt_+k=E*KHG49RO z@WCuOP4q1GB}D1JPC;{#jusf z%qUdbEYa^n<9pWwHtx@z=Xo=veJ_YM8s9YMFvDfP!*l;(R%+6bDw5?gaE{ z{@3*D7{a|A-kSh`L+F&_-Ge85hcEiHJmpL;V@$c`mxK*Y#FF@e6uyM}Uq36aLwX>a z639$YE*e8$JkzGiIYeL3C(Aj_Q^XTKOBb7?r{qSG9;L@rn}d@`+3+n`JJ<=wJzaVs z++71ttdNnX$Cq6KZFYmywuRl)Nrlf+XO(h~;k$`5J0tq@2L+KHmCNJ|)M}SHp5ECbpQFRHA-t#hBPCW>U50Nri*Q5hZsz zf5x~g(N=VN;wY`7NJ3|o(kUi7wu+uqoq8vFqNFwprr%xpJt-VUiCnMGVebF+c(>Cx z$-}2Cy~O3BAn%io=#@Wwp0?R|=24xc^V6ZhwsgqN1}Rm)w10bmen!TxcPI4YJ#$t! zIVsnlD2oNP{l5da-(Uwf#fZaja^x%eI*dpg6MvMvXxEyCQ-6F!zY252mz>j~#829Q zY^X)K|3RWI-M+g<)jC?a>#M<57h+DE8?fJ5EhKKnhDf2oJY-`H7Hf+{-7Fi|@kQM6 ziW#kL;0EXY?(A1h4s|0gjhVZUJC5L!w?Gjqw(NS9cs}KY0%oFNBUt)gB9*?@8bcG0 zLFORFoow28q(2t54vmcVCGB`ibCl~?V171mESX?J8#f+ah;ZQ-gz8uGL922m{ZnkS zk54!HaB}pxsXh1-kZU*93-rFs1?lq|9+H*^f~ezS_2urU$ubpcfVYah`+cceASX*1P=7RZ=x>Z?tR@iX4hTqA>i)*dV48HgAiBW9wt|^b> zUc}iYE(K$mh+@^@QYF!Fwm5h_+Iad%4qkAVMJGDrsZoK^Rnev-^r}T0a z_+F|iNX1aHHYC98-=tOka9A1^poI}LQCr0w;a#MRm>?&ke-RTKynOo5KPQB$tW)C9 z;$`7oCT>gUwvnl4iPynt!jt{NkQ)e4R9rzofSL2f+$)V0Q~msfHzPVxCx!Jd3sV`_ zq_dZnUP`YrW5ka>@ID@K?|fq-v3EjZ9B+?RLiLLl^20G%1lAcP=FUN}Nlr>~)xk*D z7Z&O#%6-8uajXKbx1alGEa`qe@-jR0_dJkd=g`Yrah1C2LZ9*&bU7;#CO388jM&%| zy}I!?vg%@rRM6O_H!6jwaOR@uswM$idsk~x)fSS1kCZ>f2u^ydd@=-IG_mvN<<2A0 zvc_id@P-Q}VK^_Y#5u_$@XnGxyoz1?V$%|3-jZWvLxa4>Wh#D9M(B&w5wrj< zbr%?!()aPkh8L!xw*5z8cVp=O)agr1-3LMHHEOIU_A3^gMujXmVZ$nH3(W&iT*`D3u&`DM7xoyRnPO~m*=XoMq<{8!Xv7d z95l&Ilp8J&JRK*O;{O**=N;A5{r>U0vJ*1dAi4Jj2unaviGU_RAnbsI4FUy3 zK}Ade5flt4ml%r!tJc*ziq^L7RkflrwAJFQyJ#IZ#DN=Ue)oIM?~h!@$>DGUxu4JH zdET$r;}k`+IHDO>mE9A7)fB7a=BJdO!5oG8-F$aW9vwe`EU3wg9g}t zf)fPdTnVA6C$bv}O$Cu%)uH1U9|)R1g3gzX4-g|oQfmb&BXrtm(Pn%=qx^ULv{1k0 zl}_Gw#)GVdLTT}23EXo;^!jq-(#>ex8R54ze-y8tV z=jHL>&IIsCge>Zt#!}gq?U2F6+;+a!$=ufIoB*GwQ}fw*&Qxm*v!5_l*`=Z}Jdx>` zeV!j`UtLZew31JU%|g72I=q$zZaA!^DuOUE6{(yh=l_Pb#lks|xE}f?L4ZwOFqLay z_$*x)D>#aMqoTH%T1IXSWYc(OF1j*GV@qopT(O?+{T2|=_!2z_^yHry5J6lb5lvHb z6OyLNQSWqOs{i!BAtD;>&kMfnmqT~Wreg>4#-0p_e1paZ*P&;d@qNSA0v>F;1FoBu ztOV17fouq}@oQ;+)B_qfnzFlDZ%?(J{j1=LF==p2_Gej10k!Zygf_Vo3wXB{*oh7F zOeD`bi#7$Mhsm_z>4EsazLzICK|Yr!Nw$}#+l_d3PJ~pYm()4129ab*`JLR4~ zwt#?lh8~YNK{X<7zNn^WR53klDO`h#iE3WPVnc618tF3)r*&WpfUnlOp5w!IcG0)4 zri%p*42DTSjM&m8N4Z`<=4cJx``Bc|S{kT-0RaA!ZUlBL6xVAY;S3U2-S}Omya7fE zOsjAt0$pd2qvIp(3t1heSlC!v2=@c}0N5%5U-ZYxK6lDa#iw8wZFokcklL5bu) z{wAWG1+r|#PWYtO4_o8l{wL3izjXq3Qo@6AL|C66q9ngf7?U zUC|3&T=c*9h8I@zhrJm)q7;_ISQnFNg9JSqXb1b5=5lqx_UIa&?Yff2FxE2x{Y9Tq z3%_>Z>L%F%OX1jKCRw)SohpX88f*>=)H@vul$ETW)8gqKjidh+ z#ZoPG=h?tj7X6)`-$9QEG{E?*4+hx>3zEq&IBAaIL!Os-Zs|NrTD;^{ zy-14uN&8{^EEqP!+-LBPWiVuLGtEJ;?axFBh)vxzV2I-oFD%#*zcXAW@v)L_OAEqp z80R9|bj0$k+6BM4L?kSpY=}J)mgBRXnvhN0+QuXhP54VK5dJ)g-?pZT4mC;4 zaiGeL6DwUgZ)Oq$8cs4cu;1z%Q|@phvFBZj=Hd@eLi(BH{Ar$O#x!|0gf31_3uPmV zSP3G#e2t11P1zON-Wh0p5MZ_Ig96nY5d9eB)3`Q}I_6vQRG@)9E|#maHzo8FZ_!>K zX7Y^8`dMh>`3A`at9A)|Kb42^wgN zuU>}jdUSfB#;%@?P4CiFo?_WAZeUvuqqoPNAqx@^T3#$-YA*H9L-;0v_5u3Cj+ktK zdPhG9KDD!b;yl)wVC(+x`-~C7;($ODRnX@rUvHRVbV`KYDG_w_OrdI$z2w*NRxvHb zP%}R4UZQ(1@pCb3?x2}g)dIG36bd)?r6knvKLHUdBh9qsNDOR97H*is;564$$uk z*pJUuZ24YLGZiH7l@7|OfGF#9F+Dz-o^Acwf%k!-}I zmiXN8=d6iO-{n3s|IZ13+hAF*8Elibps7=$B>*&i7@a`w2jAD45h#OZ(lP-y> zUoc>uHXA^A=B604Ap>tsLof9km~>iS9m}NCg0HkN!X?y6{c0^S(LV`f`J5(nT2pe2 zsX%iqDFd`A&L7JF0MO=qtA^<_5_}o#5ndsZV~7Hc*KfY20;3AYVmiU2cvd%)t_a_a`&K~cg&+2MKAUC*r3N<8q|rrA zqB&*YUdn>(TtIeojp*gk??fMJ=;)_=eZqgy*NHNN0Ij=CJsBNWuGWgZqD=;#S7^3p z((wTwfJ(((d16w>m~usV=*%0@6S!cUh^35BcfMAi|= zs#%Oxa~Q;`zoy=^?I*Lwm`$gWDasBavvKB$Z>f$LfzFLQfzOM4?sE^Vjzo(irTKp& zRF};K4Ajf)BOl7`BS$gE$XLY(+&=P^pCr=fpqOje$l54`D;)5kO`?U#Ia00ouXo}Eoqr{YSn2+O$2Jc zEIorxA+Ea&0&uqkx{>P`F7~UxoduWF2BAEQ8HD8-QsleOyhoD1$n%(#V1rF_0 zAta3FbEAO9j{_;2P|V*-HvmNRj{I$>DIW|ok$&EDD(qu3^Ca8u-;d8u`(u% zR0$i{1U=+PJ%tNATQ@uU$-@nsu!jv4J)Ofe%@CMW*x{GWCz}l(2H~|y8)_%4*-}El zV)Xjhc!~dic^zC?z_uVy$468UOCO3?U7prL3U3Ad48#^$Qg2ccmiP__bZ_oSJr6ya zyJQiMzxa9HfBR{apf6)|4=)&52m*WJ>hJ?#;~N`u_!Z zpM^p-Aa-#ZL8h9iAlVV;aZ(s}U$SJjJuxc2?(#N0ZH-_kKJfa)*bJ~EyRi4H#3!Fl z<$82$7mZ8(5Agc0*PpnMOxL0_-qp{zB*i9Ft9+6|_kgmmbAvAj#=v;~+C<+fSq%Pl znr0sU_yh=<1xa32{W8hG43@Gh)e(*0Dm6&=L4q*3LF0ivI>Limh0fAqNy3c@kj-R$E<5~#-0oG~6-rYVIe!v52A z+0AK7is0C;bTeOO+&KR6Wtx1AFU z?f+Ph93Ap%Q-`VkV0s6tM{QY{zuK#&|30{BhZ9@K)u#e3hGA@6j6=EF&2=6y#&A>P zK$8Q8%jPN>$Mf7LP11Rk78I}LPhb zRAUJo1P7+=gRQ~6%(U%H#uBFfv*YlT z-xSiwhe;E$?6`xx^LBNfFlw&7zz2QI2@qWqAlsLq4K7CDqA>iD1;&9yL@Vj};6Z3I z_M(Yz+BI%`{yETsh9;oP3l=&h2Zd=)AQ4okhN$4a1v~hT{Ow-==AoHig0Yymu_{0k zutFck<0JX>8s(J-=r_MpC#wv)njNHOhx)$(8aaONfMahmrHYTJz{d^%oQEmgz22Ru zK&^KDO84n-zY;Blg(CKNI6=P4I}_DT^o(ukzJQZz$5AIV>6RV4&&t(U>Z4f)$HWtd$MmsI-<1t2y~NlsMP)`Pk5835k_ejr6HXFFHF%>8^%AJAD7NQ zNy)UX7y^})GwEN&+)MG+8M?yM4KO2jfSno~_WW zHZUx{P_Y9H^;`eL2+vhB>1BJkd&e(!P;X#6ZZTy5Jawn>6s3&#>~oeVTidT5T+*bz zS`VP}W&-*Q?N}N(La1BK>Kk55bRSEq5ZUdchhwGnFxYI?9iU2RT|uo3`)2Y2dcPo7 zRXBlm;9Gn+)b5+8rDc22A4^-nF1m!ygKNi69it|m?D zZddiwd8bV3QzqS6Lfua2mci<6VqN!cdd-N&=K$=VkK}unSw9v*XbIl)k$qV`K;!u^Bd)k~>YhwZtg7ZxkQ&(R$>}{~^V}@^$83 z^;&pO5EW9obcMfoj6MQ$7yt)Q-%ILtn%y$rQpodm_PTVfI@H9UK@QG~?al#NK|*$r z#i{B5z^oP!hHWOpZd0Z|g@0)z23ojon_g;89>jL6(``%tF>@kMvio4i>G^`XdI4Ov z$8z9{&2a;ooG`(RW*AoibWt5T?2n#~#jda%6(CxZk*@I1I^YdK7l^T*O^Kc^8Pe3ux{FNp}J5 zT_EEwkbM`(zYB!6rJWFMa4a0UGPLn-U*eB$SDvFe8nJB9QwJ#vgfV!8?xgI zt`&eqX}e3SR-n#UHR^)t752@OVCPFUz6$-{;D>o*u!;)x9%cbjk&i%CK7G#rs3l62 zgU49eYOp8{EVZ^weNXhrfS? zb8wpZFF77d{{D1qw7v?V4&RyOymflUBWCcqn8f@!C~w_^Y$t>&;JIWHFF(XDMQN!8 zvbLEGE8SieWG_%VMoo9TY(@2*&|$3vJY6<^-c;C>B~c30h%^Sl_SO&j-gatvS`n8^ zu3uJTzxtmPZrh&n%&u*v3w7X;Mqn8vRDh`{FoYe;jl6E~;eGw<}I?1iCN*kpAy1+&LodXt+~`TqJfp~5SlHPX>Co-dwa~1j7DPb7Q;*f^WLxa zZOe*T#ui>R`l4SFeo&N@kDgv@ewq(}H4gpCA3~JPA&^bUw1jNr3@xO0Jv1{1yT)A2 z07TiM8&lErN%5XoQAogB(TOfNb!<;$-zU0+7#VxgqwQ1+_iTx#ZQyUO{zVXkPN9Z)R~9Bym~grW&q>IB$Fm=}(ZY0TtMhrJGa9 zz{=RnLXdw=m?gPfWf8Zdvjr*wDf(s}zs;uMNNLr)^qD@ky;UfMtGb_v?bAIqeP<>m zQFwkVKII1!L>vt`F?$oiIAxU4R>$Id&RnP8Q+=+^JQ{rNTBrzfm-g4rm3^xApeY@U zLlBkr@jkiQWG><`$_W+QoC2D=-n5O~)*K`RC0V%7eq#bB_>_S5IS>pZ;iV1xzFew) zF!#uXu_un3vCNKoW~l%Vn_Iti&d(M-C@cR^9VzGC7ys;>AgYW8gC8Ki#e)?N(e09~~`zD;|9R34%bpl0KKi9s+oVBZMU-ddq<;18HR8jik>H0fFcx#6DJv z!OpAcDsB8=*LERp5G4{Mqz31*V z&dF;Qa^qA4ZH(B0UI{0=OVTaQKw`Jw@o*t7!x$~TlZTFR^c2>vm zqVy$)E`>KXLh1wLK=`6!YuHicCh^u8N60|owz8S04Zq9qb!3XQ&iw$nA%$kO`|T06 zKE<-EzjH-CF_BpMOiFCi2G?Xj$!#r8fRczLeU4_qO&zDGOfVTqUjY@ zB0a~dB{l5qk!>?|$iyz>bQUtr^5X_?e8k@_i`k)JwkM1?r7}CTtoF1CdEGD-=rI4t zLCJ6#lFYJ-zn(XUj0^o?19t_qS(crgx$nLfg zmtrl8J-nJ1;~8z}v2n%*p_MHlc^M)Bl)=J|&fB$~x$XS2Yg~7ez$7vygi$gtk$F{i z`YszK+(&yCK`Ph58T2}WJ~xq09=}z@meDLl>!IB9)#Fh?3~j)az%~9}>NxWd?-(}J ziOxN@stKHlh8miHD`;4`Xi$M3?8?N*oVo$UQYsNasgkPQq8&{M{&?UjRU1bM2E`wfp2(;@9-rnqA9LC zLLL&!B)50bH)qh%w(^B&Qa+yyPXvGh;u5PzpJYmr_@&B{U9R~nGrg++LeXtAXplv~ zE^K&Gc^$s&UdxtyFr^9wb zd`2woi}bv(sCv>INE-yP1v?ov;a?hWgkGM)Mg6Sfk1@4LfcqN}UW zn!0IDLZ$%I4x>EuCi=U&(zHl8(c8EvR5*B?e}=f5Y0PG<~3=A%VPi>QAUc;G(y=22N`{dU^ zb(h@~y`9i2iL8+vls2IUf2r3uqJC>g`aeHz*UzA=kLZV?a{qI~0`y@a`tuNx<%9sQJo&rsi2+K9r_jcXVIjf17op}~qq2%U16Zy2WhY#5gIt=mjdKf6Ly zosYrK{xSKAoBPg|)K7+pdp%!0?Ax8i?6NpXd6a7$di*JVb{g#panX1NG+xKr8?l~a zl6YrC_UNgTtX=21=cbqQoC0*?x9hYt$=L|a>|B=OPweDk=RTD8?B?l9W))`ltSL&XxO198YPc=xt<=DY4g~4>uETbpKFLrb#D7+T2Bok#s9E+f5Igh!fb7)5aj|YC7!f=Fq?83pk{OE3 zak2goB%M$K2K58SX7jAk`3yX*qqUV$-IoByAYylWO&&nL%M&$LYuG{y9CnEbQae`o z7k+3mzZN~WkekLh-YS^2qx33UxZR%qJ+)#>l0e+{qLhfmQVRWP)ig{)NxEV~i8`i9 z&CQ<|GqHGH27014t1=~&h|7uAv*vX}+{UqH;LWmD7_ieT*0FgMSujf8T$UZX4mrSwj{%X%<0+2m1kIDK# zByv4%HV=T}oBN|BUQ3J=QgJ=nY0LGiHDD;&P*;k>Jo@yc-lc}w4a<2-vk6_eAY z^@yt1QBw2MSB9=U(BRoL$|MW+3$JygWG*I#{gpvuhmRB2ltOIhO1DxK9(D$csq;^g zW-zfkb$%AWl(9;twxkrRBi34gSQihx+tCB7M!E;ZRT~E0Y4d2CY{eG9i z1RoZgGJ29)!5$||QEA?;2x4@c9%@ci7$8I^W=ci1(9+-ZV()WHRCGm}BEtsjoCUGxl|PY%uLHfI&`Yf%s}bFR(5q_{#vPf8#^?AE3JrHa0ZNL zL;buwV&~0XqCA=h4izl|G+^WDB-sel?+o#%%+lL%3vj z23K6;s^rMzYw%O4f%$9EsId3N-fLOeaL87=B>1w|^uIb%C6ilOXmYBw3dohdX07I;S0x=W>6Du4pp#g zFA=KkPB~-2zvz82%djccnk$}<=(B*3n(uGVm zO-5wN`{M-2Rhld_jSR;8@8jFYTCqbX_xom#&z|kIYzzA3kLp@Q@b7q87Lx^E!hh9g zWeU^jL|?GLaqgqHsD9s^ju=WBFg0z6d{zjW9B}K$MwtZt;v-)0^-u!Kb3z;(#Yi7v zjZSqlo^_KoUZo1RC+C##j~tU}mt$|@+UE){)vd%2{i~nBAL`K3(33-YJ9H>ic;_@N zulXZf%d=)s<2mlTh<@`osgla{rIhqw6fB%u$5{iPbJf9qB3lBJIA4bNCHz2q6n6Ka zdnb(uhp%$nl?cctmwMQ**^!&~MdYR4!eie-tLc9pruxTUh zSJ}0`73elSmi57!GFXvULd0R|a6@}?@xsK4EQCN? z)-kYAfAm#!)KA)sdWb%a)sc#TH-k^%jb8%_b?rCj&*sX#h5*R#_akzu=i>W&OTQ>1 zUHnOSh8PsS?W`NMBc8Lgaq>C@!ywY9oc%w$3INkb}5M4BB)Oag;F_C zDtkOnHXTe)jM(ig=RuGw0ro4DjVu;o`A1XZ$##Z;oww`wc#&~AW@LKeOX)@qRiYUspBRX&w(yPo7 zcNhC2H`xH_`j7ECx@1f$=y1^+eOkTNSn{q#NmaFL3V`K8R;l6Hpzc!MdZ6c3Y zQQJ8Qtx>vMMp0WKd6|= zytCh)yD3(9;7rJ3*?C2jgCpkVL`Qp(HOE8QVUGVDRFnX+hLNR5;vZI0796r?r6mlQa zl~C~Mo^u>4zYS)nAW#MMa2zfX1CBrWNJyU+M_lQ+P9>>W0U;g6dPni+FX-?D!}NF` z|0=*C+ckZ%gcRM<7Ua(Ay(2#1Xey=CiSw1`B<2I0wbcJ)THxew7Eqe)96+uugZzo( zoGlBG-JEUp02C{TYG1XGPQ~WQ&mgne{PP?Jziq2HJ);#}D}tEjHh|U!3ed5`g!n56~P3&h}P!@HhM4*Sfim zcSLB=hHc<3hu^nkfyB?-Y5`-n4(MBjt1J(z&WYx8G@|(ndSkx{Nfy^{XN2ew)+I`| z*a~l!YMCKe+$kkD1YKHBNk2u~3i6?;dH+~t)0{2e1G`<>#3iNiw1f(fg#;6C<6M!D zxv=Q1o+(_wW7ge6M_SSEH-^k*>P57W8C=$L!{bV&8zM2q1PZLVAzd)xaBX`y=)2z- z`m*vJR3VvMJZ91g_XeN2^pL$FA)O&$lt9(qImzpxyu9pFS*`l*p1l=eOVC<5`q?b{ z!%kv8zI{xbT}6a6DfGy=^t$B}u(mfBw|fIVGablXi`m=uv`XpJij7d((Fj-)_8dPE zdK~{Fv^heih2Wl!IFRU#Plp+dK-0KvizLVm{A|`Ubo+W{stFRir2S!jTx=fV0G`f5 z5mFL^R^n#LepXXS9YGG3MCU?8FN;O5M$kRi=zYzLK)}E^(cj{0(U%2iCwemQ1g=a-U3{2p9^1o zYp-kZQT-K-eIM-1KZMTDeCd+PH=Yus`9ADZBID2~*lTU0j9Q;%&vqizt>{Uh?!8R_ zc2PA^hfLO@X~_XoHMw9k2OM0#X^})AtPjP$#C{0eg5oz0RnsQO)`v3pqgmy##uifDCQM$Kynzn!QZqMcbOjUSo zVPWv)lWC_D_!kefoUTtp5057|zLf<{0yCfyxS}C<7rBC(bWy-!M2ox|`G+(j^W0GJ zUlQ<_uUB4dcm68s42#?Re!6qcr+-2UkWk6i109Kypc_jyA`U(cyjDKQ+u?Qtplkt~N zmeJC<@w3MNL+7{oj6i7ZEsr;gN+@AL(_L^^KI0JGo zcy5lOl6*eRj`6u}sP&S0wV*!SdorWvwRZ$Vl;Ju_$;kdk8mea8DlX_RkOm|Pp}fM# zkeI2h236L@Y5r2Q+%72L(3_)+0-5BO*m1*i$lawSh zx`LSKX#{!Z?NSqLn#vab^2)DBSZ}UX)xKO2!HlP)8SXQnf6>zQ_??59pqo)Apk|a@ zwIt1dhZe;VUaR(|96#Zh1K879whwoa>!H- zVNVmK4s(SXzqKKV(%i%(qYKLKMs}-^0<^n}_7_E*F-%#Ar_ahbc%Aw{KW?Y)h(-k! zd_#WlJ5zKL8T!&j$x!%9G!Sl|wZMC}8}Ukho(}@D0-#>c1!lHK9IBj0B}H)JJ#`$Af#q`ml%=J5#|C zli@gcMtHp=`7A-vJ5zR-RPYeQRB(#Cxuz<|GobA6>ZW)&oAo+H)f+2YQKI##DvNe> zvP9+UlDyGz16N-7zD1H5iI@0BQS}`zMY9ML|JL0yHVxSLt%tpnz-4YlZd!Zc_wJ#7SRy(Raf;ujYAgo+70+$VN%wsvZ-LNhf<|G z)%ioclr1qChD=Ql{S0San+E274>NdT-PM5ec}=ITZbiel`l-(*AsL0^%8`z0ex_*MCo#FqMegjPa1sUdLF+~yJLKeLifxy%Lq1wA z8#Zfrev&dM65`zj7usvX>Zm(Qo6v{%C$741y8dy@SEkv`;nF7)a+Cb&>Bdt6^y*Tc z9pdO@Mc)hpRLevky;AnG?@a?||^*AF+*P@}T1$yJ?L; z(g%DRPR-D9?v<9Iw+@A+i=WynUwaAetFDJt0rkG(7-zAnrY05yas=Eafwg|E7- z*nnsAtNd3Dula4_&^uAi^Q{3%W9(O?r_OpciVEbY&;B^wtE|JXB;;#NWc*X*Y;M5w z4NcR}l8)Bkj4{+kdB}1gIG*nL-76*;FFdx$Dimu!$BfC-L|c|8yb6AYD^L2Dk13(1 zD3*?q6tV~h zB89R_P+WTob7kzUCpEG9=xz*rcUGwv7vDutBn)-mJ--ulwxQheofJh&H-K!>p1b1F zV+<#<&=UaqVkUE)!0spkNRL>V(ZFA}SvGA)M;apf1W_klG$@qDKXjS6o??7##|_@& zzB_&z6eEMNx~SI6hG~F%r4;{N@bfHC!@7Hf)boHV`yI~+4&oDJuEskUIU3QLgozN51oU~jlm46 z_5iouHtgqO}nUp@|+7|?VMi@!EIU7XAb&AjlwOtEcAz?a4p zDa+P4LhhgYO;65>$q9no0@<7>yL}*DOHs$-`k%5#Q1+Dyur^CcL;o<1l7cLnqPLEI zYYrl6%}@sO2hI7QlCT)RfO@}py}aBQj?a&Z_M9IjM(4)QN-L4p7c}CK>s(S?OQ3h^i&}q|fxnH@5qydl1}u|Ii1K`Jd$v zA|*1shDiJDdxtizsBFVnu{3Q$GvRO}CGZXnGVx6_B;zXfd+nJTf4 zqv*(|V3@>L>$br~R=kL{?SU-2*+0fDdxc*NZMJ+hRO<_#?~-%Gt8eI;;~igw3gUq23Ok68w^zKzSNVDgUCnc6Mumullh17$+w)!B|9hYtJa9`c;QE5jHPM$y(q7+pQrXlJ<_Njji!A~ z^PUeC`87cEKhF8SAhE4?Ebli`{GkP_+omcfI+;z3V#@lSyk(p(z+Dx{ zKcwQep_alPhf-fDZ6!T-MDBnX=K9c)qDS}AtnULO+3^P9l|pK37%`c>=Di=Kf;gw2 zna=ST{XNZ<^ea^MFOfU7{pHb?A@wYr!%m((6p>oGOm64K zc5Gy|7O*;8mj$4?%BpU7>SMjXd^Q;5?!M9trEX%wfD?Rutue?6p5c!?rTMP_3e%82 zBNs3mvGyf!AHX2{=DBb1Kl{J%;^)L#;kz|S&PP46a$}l*-%M@)Ue{4dV>fD0&>AXq0SKL+^|{L_fy8dS(LvTCw=YS=-6Kz;~F3w%W>;oT;V>j#>h?`|II z29Y6<++9P>pppS671Mwey|^!Gxv$H!HAfOa>o7BxVnZxZU*j`cIzd|GYq5mi_8c6$ z3#oMd)iI2D&PlKq+XqDm4C6HqBzwlrU1V}&Pc)t^wjV$h7s`dYpZbN)B@?y5%tK1B zTrpY)o9k`z5nnZRxV7FO7inSimpKyGY$bK5CFjWAp{!mZBGhyOj6$T>DjrgOm2yQAc_Hq0q@!z8W8cmmaWQ z#kojb0dTmT;v<#DQLW*!u~$oVY!QQan*!n=mbdh}6s=&fqVr2FmvDohzN3RYAas$?_6cWuHy_zt zu4f;?8qQmO7rReeV}Mut8Gva9c%vQ`fz_JwYW=FJq!Prue{lxkz|3Ahm0+XuUem}w zg<4L2t#Cf(hR(k)%~Q#&J?Zu2q)m*_RZXi-V!Wm-p1%?=UL34-qUYwM7@nJr-}IJ1 zQfK!!+jHpfdis}@IrKhZaD{>9$(yf!kiCdTj8KDR$VY0os)@ow4^F~US2Lr@G^S0c zOiSw@QP}e zZ()iHtAgpqQMi_}ltx5p`doF%pPZo@rdO2$;=b;*vP7Rt=z6Oae3N&E>`r0Qc2qFO z99vjTZb<2QWpui`m&s6D2BrZ+rr$8qh{6c{^hES5P7cduj$P_>b{1pY@ec zB8(g+)p)Z88XtTXU7xB&WZC&pr9>76yN0?5w?_#y#R!wAmJm?_QNE*afa>R&qA97h z@d2D*2z3b-Tz9UgWNqe%V8thcr*UWm95FuNvjzzjVwx&2MHkQ|N)Wq!J(@`f-Fk&@ z3kSR^8XzMnu((P*-jd7JC2sAVc)>O~HsPLmzi9Q9*mcQVJ@%WINh=p__fHpH&rxyZ zbuH;+$|i=UvnJD|!3#D)xgM1^*0|qA%O@G{v%GqgUIpx*H?v9dt;@NvIBfMQ;fj22 zzHc=SomLFgU@qM(=e=Mi8>r@rPV`kb@#ZGQ=;-G&=xi&lxL!k}X4qJJ)3u_uys|Z^ zx9vBvyE*0gl5B6RZC}i6@$PSz?Re`_mx-8vub7x5PzfA`qS>&%lTCYR41*2tNI+@F zvc53IDa~17ib1`731VQ+fvE&P#*B!kJ&a|AapV|6#+rG`6rOpp%eEhoNB|hJ)kV4d zz8N%xqaTNUc52E}qwELs=)!Y1FR>E;XhhgP?-hSjWBvX%+B|CcE?)vi(7!HgIig1^ zLC9oq^M1GLVRSk7-ty{kOH-*cq3}3E_+_?I)Natvl8N)o+Av;I5)069fEYrHmS2yc z&xVy@ovi- z*(}j5{IN*F$o4S)H&S^y%zSv}81OpUScLAcXk7yQQ1eer6+l<~TT^46Csn+!Niv5z zQ9^m+1Fp}%tdEtInMnUE51=y%;j*p}*$9C(+#qIC#g4|rS(TaUDhZbKa|&1^nFZFM z)#J4DR47r3V975NieVGy5kYa&L=YQ2^pB+mzj|+z<56gn&GLAf*HluGYTt4uMCrT% z633R2e@dd|T=Ob%PCJxckt;t6KR_(I9J4P8RX0`T%$LhH`VI-YreGZ-m4+{0h; zUnvHrjGANfmonr8srX>XG|flmrouA)x4KLw?H85S4*_2&{;3HwsoGv1|o4JmKeB%?=tU-pIaR97d+sIhBF7^!pzox<540bU*x?o+(sHt;(suQ8& zLX+k>RBb8{;D!k#MeP z?L1`$_G5d1a9!&Vrsx=bvV&ZpEDY``7U{kvmgOQWQ@q?(jLgqWoCBXxo90-^*I#6r zf9hHIu5BMEpYej0m1rBybqDRJ&|cFB+N|m$H7fG(Gse?GL7~gC*P`N81}trRs!{}5 z-AqRA{I%8eO1ill)V1w5g71k@--ut)Uhiz}&mhxOE%gd~X8hYGYUJmyVgnr<%6-Qm z-nM+TdAKyOPt*hA-&ZLtdSUAx0G2Vr5RL&>Sx2bp-F2wZwy2hofET3Ns$sA_$`&Uc z{1bpj+rj9R1Z%hPLvbC?++Z`$SV)jM*g6&bYTlW&1?~MXlLloSq0{1Q;GaGhE%V~0 z^hMdkm&ast?d-Ko|7)w6vvW+<4UXnm+;zsGLYwmu3BWFG{?uG2Bd$|;oY`NTN|2Pa zVL;D}rJ%joibDXCEAzQYTm-LJC-M({YX0_`=8tSt6Cu@|C4ZH3*Q)UVf;c`k7lQ5dvoJNOSO@S= zj-b~SuMCow|H|hn*13>fpq@^wo5v>ZSzXSyih*#K*cDczY_{ioo7Zg5Isdzq{qK@z z^P1(EMy;BgEpg=_18{#M5r?{oPX--fm4QBGAiE6oC<8fVpnWMgJKaKk!Yc#8GSIFR z6rEmpgkJ_eTfea|vH^{0uN|{4H$y6VlMNS{r_%Po?rxi%AF-b=0y#ma)lj~?76$xJa zgL>?9q~ZC&%H?W|ylZ=CNBFP?c}e)tEitk^ zJ@@~(9NTQb&eyuXbV7i+OZ2wMDWD%35q*=BZ$HGWCUK*~@!ediNuq;tGxXCzr5T3m zpxO*4=wL8uGO2^1W|*miW6dyM2gA*^q>PY4e+@XVad*IIWVBrHwgTgxHtB~_O=T?SHfxB5HJ0SjeQ_qRV1i4H#qQ|5k zaRkA36MuuP3+sZ9v3DV9TeQ@p^lMm`{NZ#(7SyL&#_`Xy z`iL#dV`RcHj_k#}1?Q(bcv0=t=qBF<1;At{*nextN)+=yvMMle1{6fvuaF*eI+88Z zaxQ-p{T!Keoq>|e&q@47ZV9sKOr!(&VNNj9c5~d!ahJO*$UdTc^VtQF6l@XZ5lbmdCR#%W!j#Q@T zc)fSd{?+MsA2QrK840qXvekY3ZU!r!Vf|$R)|Sklw@S~M!r-2n!mz>Y*_7OArZXgg z-|$$zYApG1+zD%;TTwQd)_VqZ8{_5e8_P}3e99BjX5&$ zNuJWrT9c7iw0Tb6H7N#PJK& zG;2I*@H3V{R{Opfr_~CZs#te92bvJG@m9I1i9))jXi1~pY!QH0#^>BJy3cyevgm_Z zq;Hd<|4{2@hVWAz8f||xnQTnwbTzr~4v^CFyP!0CBu+Em2yvJ{9Bsv9v-i~!ZL zyDP|fFeoY{n3`vn>F#FDUGx1u2XeUV+IUK;M;zFS$C18eme@t5Ei_J-XpiMr{S938 zo3(XOX~_AeP~QE&tgPTep;vi=TGmQK!N^zk*G#9EW}b}uDpD!jPJ&9`@{_wV19NP+ zPrplbO_-5Ip~l%f;;z-REi0B)tU(PgXF_{|H`0vGFk;YD=P8jCXlUp5NKX#)P-(eJv~2oyXB8 z4;c5%b`yNY@}}E)R?~i2-}t9_pK~kn)yo!3N3~yy^u@)yQhy%z)m~;*m-HL4lyTXn_~p0CeC6! zP}u|@pZQk1nxFi5dhDp$d^n;504^mtXqgkyd+#r4mmM*`vK_;nA4C(kW3>2Kt#T*J zS+Ov({=G8tMJ~{zUZ7hpVPXG$Vw9i$8qT^*8+BI^jHtta8G}8)$kF z8ZKC6mUZW6B{OonL<1--yCKp9gYDFK(SDWvs=_!M3w-CKHZ<0xed;`uC%O@8=#OlY z>aKfpW}mYZz$6WLg%{Awng?vk^)l!#xu0-aK9S09-*kCV)r zi=>gAHOZ0jwGB003%TQYV1*;)<@7v9lt`ljdLnsWNu%0Yr5^d;{;+nUCD@~GbH`|{ zn;~GfOCs-Ask7=F>3XSkzxtn{ZB+x>bD89~wZ@WV|0WGMw4pWS@sd>@h@ehsDAI7G zCrhV#9LrE$$dtVeC#_U&*-j=;gL%^jPmB!bcr<<2#EPuTKsB7(SL>Z;HZ4A7#C!OC z6Q}02w3t~oWvf#swbl0_NP$k5`=}Q?sr@GMW{ev75H$Ipf-|k)a2=@kHq)+|X}1>8 zP)E{tFKu7RRGlofhU2bjxf!R57tCq$k7s$FKqBRyO*~nbRcYloSG8)y;hlo!)xEwu zILMO};zh2@*MDJpRdg}SKdab3mF1f|JMbOnQ^-4x^S<_v*V@IW-U$ASNg#8KKSD7x z*IPT^KV){Ng0Q?>r~RCP)kxp|i0gG0TKDBO=MQNw;~nqFgaAQP#mvenQN>?B-95 zD78~#5uCJRy)tFJx!;|%#;`35Zs9MQ^NCLoixVi;CM@ix-p#@&br-J5h^D!i%WRW= z0#nxe5EFB))XX$$$FE&qO`l~ot#%7`f)z3HoN0RD*46bWbo6h87->DjvZlX{)YH#* zHep74#VFDdu2$OH)+9_E;Y2uVNrFPNJgiA>MEILoILl;Xg`F^5%SULp!(Qp>oDk9t zkEf`-iDa#t{@KRQu5!v+KmDY|<-sH#$jX^A)iH|3a-%p#U7X7UX6biL2((DP!du{1 z_$;(Qt8rqQKN5=1PpD3g8-ukT(W3c5g(BG50>H5QXWrr<%LC$GO`sCb$T% zh;65|!1hj)WaFRPTR@N&XT-EgkI>lt_}QU1TgNWj}t z`j@D3C4PVC&35zUH$RFWPo+=Ef_-7<^qfe}{#@`J9TBa1qkoYy`c4x{tQj*H*dTb+ zEE8AF{)Hw(!)+?yS>taFrHP(t*HM|~+noL5RR0<1(Ou^agKudod7aO|an^$TQDzVuiRy-oD{Kfk&WBD`e)Be4taj=3AD}+KVohyC7eU^ zK)6TkPrSHpYWj5Hmm~9kih`D0Cfl-ZW&f`Z2mH_RC|tqm%9w*Bll9~+m)Hd;t%u5P zj;wW(K0MK^r(!lqIF~a&E{T+v9v6)z7o?Av9#_!deaAnhG#xgW*+e4dXoCr7A$M@X zStuC{JqzKWUo43t%^kKV|iMH-MZtg$k|b>~5DEDFQPVG8Wcg@RodIOhMR&G?X? z3$v)m;PY(A>z&yxt)6li=A-%J}) zlAgMQ%PTQvr9QNs8(yJThR@`5)s`Z7vYj8U=XA}gf-P)V(gtQ5IMUO|U=2(e)Jz3& zgHy8sVt#5GMzSa9lW_BO)8{Wnytg3V5njzAd3`tl?W4j=I9FK)^}>XI$%;#O4UTMVLXld%?_ie>H9%~| z-_-CXh%PFl%~U1B1wj^yAl58XE=%@XhL?KF0w!&{jztQ89qZptWd=L+JYwt-!n)Qo zS4h!|IY%jWlyj_P1gr0(baPKN&F+l2*q;fJ=~FOe8Kimq=xLuOnDV_HUj}m+D*qhl z(a7*6Q#!^03KxCqYG(y}i`cn=9nj0&5SJ|{o<`KuJ#zhAo)4Ag1CP$jij(EB2B%yI zCDsn*&f;`emu2s>ut>dFw!zx$lA6Radpz=Q@w;Va5jmRe z^CuFKP|t|0;DRC&!ZEdVdVwRBlB5f@-%dx^Sf3p?J%XlBGy{)x{b1)Z5T&&Vj?cJg zlxkkM^ZvW2Y!%45UgSz*>?{NBZ2z%U9@_3q)AMp3e-gUCl`{5(no}90blX14sV+09~v`&N4qrWzA^XcqZD7<5OBr%=L?Qtjda!3LK6QDtn zp;RYhP-*nA+~65`N(4qb3P#wUpz@Q4W6^An+jElzM)jlvPnCM1@x38T7q4Y10tu7jtom)$b^#|1!)OLoquRT1wEPMuogC9>ySZQkvB2F z68L0qH%E1(7EH@a$FTGT80?wk7`YS_GWqBBf1F<6TsWxJbX&f2C+Xq&R3>(L(DV?R zPpYnVo!5KD5MzyQtG%5_ob(x#a$q`4$YjJu+wYKOeKhDSA3QI6cAVguXVn@j6Is@5 z`7FD_s>Lj>qzez@>*B??PYYTjq|5%Jz5dC3>B9~4t3>m?6sbG(VNk&{m16lMl8?pACIRU^XiPHFdiGa@Wq6PuB$`RO zr&M}PMO1%|*CKl16d6y6-fUro%VSWMVW6HfxD#wS?y6ZD*t^ubcd4XpsdNFN=n)Z#30Jxf zcswIXWTEnM^9Z{jALIenD&X3__KdxNZ1N8i8~#pC^CJDx-v4cnojMLX3`ZXJva9lV z;R{aNy?S5?!!GFULgbeM>^SdUX2_KSM)+W$)bNcyjvbo5VY?XEotWv9z7Eq zEXyr*hvjL{6$02_D_{9B-IBaKja#p>b$<|ORa3HZJz3|in9~9=qetu%MJCR#u+?h#(6=Z7~M1c#y#qwk^Ck)FuLXuBwdjdFa^iv9Tz zW=ys15hw5Ay)Hc7L>$Y4=6){v^Nd6CdZ%A%fSA=u+%&BQ6F%d>}y zl@VgwTjJ)|ooDV%LCq%x5*=csS@-AW{>AURK3dSp70{wM?_*sdSC7S{*=-%40}QN~ zH1AC=?6+?-17Qr2oJ-WKfDPh>Ljdm>A8eSpAO@Nq+z-zsmnvQ$rjv+0m`kNC(y>q6 z!9&tw=QABq>XwK6FSHn2U6-bQhaNV`T3B z$u#M9!riV6{0Z1y-_mA&=oaV}|1B54=@59=j;$+ENR!6(F{{2Vxntd<4?Ac;-p4vY zBu@K~n5+>CL7Ojz2)^gYtnLVlz|&c-|}mrQ0S>rDPl&ykM&;f z_9#c_2hpHcLc4s)=zzV1JbZ_VT<)xNZ;yqv4@@C$43Q?t}j zPE@f~^q+fZ{~hcU&aj40TYV4(5|57ae3 zJZ_9&HO|h(VS;Ts)oVrhuy+IUVP=iDZO?#o@qYfQJ}**$UX@HM{Q}Le-u(FMhAi8; zC6p6c6r1m#{H1SfKgT2ZZqie&V-xkM^Y(vF+E$gJD?&;^QUqtei|FkmJldR2;~Zd5 zPJ-qHmN3w#oyzMhcV9Biw;Hn2YyCa|QLM zb2Q#zlgC5*Lzk?&ESsp9dV*EKBc`2Q<1P(fEWG+57qjkwFqdXtPzYz`+Makdu?! zF>~<{T~Y6Y;$GXQuhr8}E=Ga*L|!Zv@I~fgEW#giG2vBxBX>3ghW{FB-3!fJ>%JLWOM^s1 z!P0kQO!J6DYaNz;gY9Z1v0~G(1%en-_lBFjq-nR!s5HfBQe7}>P|rQ1@QUeqS4 zE~|TE-7ZU+X5fdq;;PqSpX;f%f#_KK%ct{f@yqeZDY)*q^o5V@prduSeEDY85>&Q; zPNL?EkLywCcEj=R=aG=?@1mc1--qg1A8uH`7A2{OOM2g4H`g{V9}{bwunWUmFrb&j zcb*WvR&{=n7LPRiJ%la(1-s<1*I7-+r7sr~s?*nuAes>st;V3}*X?eWG!c8EldLp| z=D6H_oYZR@B<68h+;B^Z4jth~0>0$Y_w+pZ;cv}|=GOj79=^s><=p+;a7ng!Nyed> z#J(S=`DSX8gh_(4VWoMJTXUSzZ=UIYLqE2+Q!FF1M)0ijdtK;qZa-lPz2%tV8}*X4 zhWfcA_#=Q+OFb6u0Jh)Uo6q-VQ4@5>ms969QZ13!5X4T>!KPK^*HQwF)+4fbvcqX- z`5KdpzT0ArbFcp1M&HMs&9W^`pG6FRD3X3XH`2Y_gA(?RM0y_ju5JTb zS+u(0hVPFQ<&b5RT-U!O`Rm$by17nbuH%|pn7LoTw@tpHPY%qWB@o0?j9gFNI^v+$Qn>=Bbt=na`XZv)PZRREMdU2f@snj8} z&m#q9BrgFW^`-sMR#VUDOa{QEyO!A^KFAaM#xKG~6?Xh^7PIWGO~5<~lSPubG>5CD zfksCBkG5^o-Lq^oGX*JUuX1>G4q|nTf1S%wTfvMJ>(`&9eu^zGtP@KWU`O19oac!0A)`T)jP&Y+2k7DV#igRoZ zqHOfph18w~)ia>@GxmePNO{Fi|3_GN)Ul@mmz|W_=Dcv`oMFJDdJ30PfM(fx=TQ%6 z$uKaLdd61Z@iWtLz?Tbo<0PRkJiUiW#fWeV$V;XrA5Ggxg9myU)2^3Rb~4JN8I}d5 zC{7{)>8f+KMHRDqIMN5fJJ&{;@$rqWrw=y8d~JK&uSohO?a+pG9;D3X&35ZNXARAK z9k|a7nTIWe`v9#QVJ(FF0T+5+lM0@$u{`(P9FXaBkzeO@Mx!hWuBmmx04c#S*V25# zg1=vZZPaW?_;qY@86o^>w^V;+FON{1^H{<>BWQ9ZTzvPpAf5j<0Q()!BWs9f7nv6v zonzbMlH(1d7bIYbxe)L>KEX!6jwVtkA@69SK!ca5$r&2l%Pk4~a?_^?oaM(uu8^I1 zgy$zh*uzrnueG^!@;T9b%ho!vwWk4u{VrOE;52=Vr3%^S@th>F9L&s>2CCr+2q-!)j^Lzk2Mi zr1U=~GXc-`fEUHJYzXN*L+<5e0Y(dJGGgaQXds8+Tw&M_=3_sdv@TvpUk03?g_$`4 z3pl>{*793x_Uj8aGlk9k6;@_{Zh+|TH@5W3nPfZDMEl?SM_3ju2P-OL*RlPs(k%0o z$8@;uhP?U@|KC*ZW92_@{)yYn*(%b;?3Qd>ERZrZdUo`D#}(qSZ^Bu`K4{BuxT9+~ z(QFMHD7Dk)F0^j%j89=FA1aSGf4LNI(+iXZ;wdw&Pb)pLK4W^m|0~)R-jicv@|}>8 z7GCGma>P3S8WkbJzUV#q5rrSSytOpT+;2Q?S{E0y?bo@ua~b~ze}GdgxH(M@0to!# zVOhZ+oUXo^1?ki+-yWz%BEQ9~lO5d`INhRyuW8n5Zl4V&&RJr+odYvdS^Y0^{-Sxf z9%d~NsCQ&QSvsU{h5A~HleKa=5XWtJMW-xp37GEB>vF}Dt-z}8w8U+vWAc&Hz-1fH zKkZ-DrDQ42>ia_G8VAy+a?t~kl|1N*t_=4)5Me7GJf{$@uk3BSaT;!#uSJ9r#uVzv#Jgw5y$KQQ`cS)pPLS*=t z3RaiV5G z&T_{u1&(u?85){bX;n-fX|;4G>{}x0zASoA*rl9#`we4uB|7c1I8o5<3a1qz@hl-f zG#uHhGX+5F!NM4y-HJ1b=k(I%O0I`ElgwSN4BzzU>b;4ct-?pk-4{g8r$ar1_jd%t zK~je>k4^ex>$6|1&;A)E`R#z2K&uA@-f;^jR(C=GWSIoBV?drVl#cq9nW5{h#uGbJbHj zcs|ukyL9^uso5Z`CVd)-kc(&YAW5Y4U&;Q?{87D~^OTy13$ADar|Y&M=oJyK)_**MN2=W^?mx(0-wkA zJcwn|&!P>XT$8|P{mh=G7R%nV>@q_A8^Ib)$H-Oz#C#b|cMiKQntp(s2k;wq)MNF{ zlk9tlcfV8Rx@qix!v`Y@`J#wT`ZlB4_B~Wx&sv3{m9Y^10f(&%Hwh9%Bp##;^*W`2 zhQsL5x#&@K1NbSpG>MXjBPEqw)F%G$amn4`_-rVog6)65^>1*5Z4M>wsOR>x=rl6D9 z>;DiQznh9~{&2;=A~-KNknZ`E@R9C6r^vanPsX3}=KVzsPY5OpgqPbS2VFAp6A;d6 zB08=Jx})bc>+FOo@@{;X^0f2ibkyWzo|o`zBhAK%M(wnxWVfrT;|E3NNE4$a&X`m8 z*s^tli=QR_DJFMoy@a1PpXvLTrk~}n6IkVYzF>)s^Otp31wWNW4EIe6_FgbWpBFhJ zHP29Ak6KO6M8h%L*&eJ;VA;KrJsS(-^)I!Qe74;zlzWIxO= zhJ^oUb`@#>s(?2K|o%K$w zKBmuZdbV9bia+fVQ!~ahG+9baZ30jf*M! z`R@=K!7FhkhWjFk{m(CSN?8udO9TDG72<;Z5mtm}nXmTQ-9v5{B0F)u`X``&GHG)E z^plzBVVvj~5sxuvp-gxoG9IzNN}$E~*-~f?s!3UT6^nFS?Cv+jm~Xs4f$H(2a>g)O zU-^K`YHvVz&US4|y7{DRl^>=1j^4|fI%K|8-lw`uzrrM}32HO&J*=<4&$H@0Wg zV%Fc8F-eg_Hqp>1IQ^vQ;Ia#wPM+Mi{?vNxXg=7eBbxA5T&o%90SPNm1M={wot4Jmz!Y-*j8zJS--o#>-}X<9iJ6Cb05ITVJd| z{9fMU8jqZrCtvrT|LgcyheMtB+bFL+Cy_nx=v-v!o9_^EqL4CD&5NJjh^`JDD}0?j zI4ccudMYY#{><;`pWab(5)sRx4P%Mm8`9GaZ{>OMxbR3xJZ$$!fhaHhOmV_j<{c;* zOR@$}Z-WeZ^qO`OGC_f!yNV{RSl3Ma9<-u(!{FK=YB5brO5|nOFU#Nr0 zIwXIP2Zx~Flzz@MLFk=vMw^cK-eh8KT0 zdFUvoxi3yqF8lVT`*mr9V$f7|<2A&aTcnJ)d+91YT3ldVQBm`|2aVJ>EmEe0n)y-V z1kz`(b7*Y_+NtJ*v(0c}3*7f+38RJxlFqU$Dr5@h&w;Xv7^Nck4SGB2^Dc%>UX-$38!~E&;5!uU)VLVz0y2Sd|7m; zj5^9U3?cX7bBI@&$E%PaC4zJRzHVwVGzVcl47@J?)X{cM`-ZWhMBgAdDKm znKhR!dx@k=U}s%1KkCbdVnO#EdGuQgn#;vo9sQ=0nB88rAfVO{(KBrJSBtuScSx9V z<2Zub10?!V>>6NuL0pOMJkz%N5Y^W=mYgvN`UZFhwwq3+tA6mxwY|#5dGKShY*h{y z`QHD#EBTX?=l-wtWC`hcWaF=J*NY1a`1eHbf$#^yCfwHsPk?p^^}ZgqZv~xdr!3%3 z-6##-o`?v!u-a$;6PIMaSRTeX8Yv&Fa@g1B+q7nWu8w~uksOfdh*J{RwtcUG{kCPq zy|WqMz-(aQ?dO%7kjDl$*S#*{Zs(L}_MnN2%u#)`otJK%ESmO>q+EXMd*Zy}vsm+w zcQ&rv1U5$Eq}_sGp(3&*)2shq?MzbUJO6d1T53^n+1a_8E$gUq%yg}$5*NH z^H&E11_g(V7#TWh^q8?>adJ{}N@`kqMrKwvsV2zHn>IaP zqb(>LOExRi&uKNbwRfyp9JMIwtB$z1xWvT7MPDsmx-@NdT24;h>NV@vt>5rXRAO4r zmM!1^uyxyy+js2zY1i&OKmW3K-~Iy!4;}vXNbk{O$NNs4Jazia*>mSF{C4s8OMmoV zzH;^2^&2;D-Tw2=-GO`eA3Xf)(c{N||6?0`YJXg#@B{up4FW(Q2m-+%1dIS9K`0mnMuRb6EC>VRz<3Z2BESSN5k!J05Dj8LEQkZ~ zU=m0GiC{8F0?8l+q=Gb%4l+O{$O73Q2TTD|K`yz*HVsS%`9K4-pa2wtB2WxUKq)8# z3 zrYpD#XGN$&tq?1KB3z+W6ey-CvK2Z-xuQ^!uBcSZP*f;N6=jM%MXn-Kk)@cWn5l4A zuoOLtF2zE{Vnv5y4|*SbjUGq8LrvCcctn4^x*E zO#B7im_3p49l8$PjGjRcqUX_N1rxtVmrJlADl%@q!%>(5cBKMJ4;#d$fF-GL_oAI( ztx8Q^y!inUz&HT^Qfa`JWT@Qiyme#k?_?SJ;0b?)sG|=o?gSk{2na4zL!kBrpbt@X z0wM%}UevD>NXa(_v}3~mHVB7+2ql$1h;$SLg%99liQjg1if;-COaa-ygHb6U_IHq2 z3BX9bq^P#Gtf;PD(p+Pxla$m{)f)6=b&`srdPz-nWwS(4nk1gshh)rA-{tGT{XS8ZsLRF{>N)~}+_^)-^>ni_qeM3PuOhbfs+ zM)%O?QviceRaReLG^0$StCp12)z#FgB^5Q5rION`#_C9nT_%aq*Hubpl{HHmD|D4* zD`~pw85DX^HA+upG_tWuCcS7@*-qC+4yRO7jzEcyE@I@*0hEH+l*9s1kd;yZ6b6u$ zmo+6WH8w3Xq5|USnQ5^(G1(CZ5hOD!F)h^!vp7g%RPv87DQ7*1r-mga=P{6!=r7ci zXy1>Z7P&#bc|*RUBI#4n=)^QqdACWRM5JsWm;U z+pzxHi1Z(v3ZfG-%d-knK(W5AyndXde#(TB3U)yg?o|zmQYb0A3{G|G2t!%mK~R=ZmpeY}2%uEgNa_v64Kbwc#zRjpNAV1dBL4JJ#k{pecmNn=~5@ISjzi~lh9p;pS z0w?dH(smF-8iR4-YDKt25gghiQPfZID$%#V%1ZioO8WdDGf<)2xY*dRZGb^t8a$%O zj|~DOhy(VjP+PzZt}iK0nKKaOG5sDl&qU2*REjqHs1wb7f(3XD?boH&FaqP+jt+ql}no}RB79)D+R_?+EjdT4fZpEmrN)p3z+$j!?p~E`*LVfa1 zthB5tP!d&~USXhCvhU7v!UF>nvBU~=S;Tq)V~0C#Tj7cY z#E(fTOCF(&0Hdk9LfdALL5wmYI3z51MDXaq;_}g@#iNQt{Y%U1XM_Z$O3H$0>dDQ4 zCHV!vW+)Z)QLeG{sM@HOaTG)noVaK#Gyq=ZvWD_X9aA>0sKRNYqBbw204^`x2CJVV z)z6T~P$V-mEmN&*!dqoXoo+^j{>T#F4siZ#CEHx?_li~*{RdD|M^sj-pu8D%uhNxF z*;i~p<{PTls*8#%rzRAG9L>b?RAW_fLLU+zm6$wk$r3{gqN|4G1zXWg@#82zRZDy; zzz0er$%llO6mFD?}+naECVkkIj5x()CQJfAX*+OJ2Kk1r!sm|{jB7Y;K(>W1FX6w9S8-) zKcQRzi2cSWPd3~#01xr~f;Wz^;v`^g)KiWkebP7Zl>;M2O!^j*=h>MEoD_Q-M|KWS zml7u@sUmr@=aYCu#ON7X6fAS#43MB@*P0z$(=sUqMnY1C&u(03Rt)N(3T zDkI=6tqhF&LJDf7f3?;;z>%>7^a|p1{#WF-eZ-HYN7LANzySK807@cx_o?oHu3;}$ z^@5H09*CaQ%F+g8Zj*!0oQ5#v9`$Ejxr1WlZP0kLd?!LT;G+%7PGF!!-QY+m3j}r_+dLgcb_XpXe zvm@$FYo$7b91x%l!QSK--1Tl@7oy5GxR27x8Rm)DGnwjMv`icG{T&<;4;5+AwO5o8 zY|!eV8YYp<;~#5eQ3LQv>r@R&8-St4q&dn~C>`C?djLnG2L#>$MtX=A){>WwwoJo} z^bO}iCRjr8>!q>T+t_L&{#z6$TjMfmfHPl9{pltfZXAGfqojLPy|fQn+R&@Ja_22OS!^L&NbNVE>*PH2zOEd^P~~(F`P#b+)dE{S8sp=t|&!4t?sRPnLk%n5Nv3Lz|e$WsU)v$Z; z@2#v&GjFr&2YGVvzAx5LTRLL(Yh%A&SwnJ z?`yX!-{$8Gz)MCtP)4|~)Zhy;e!e=8_rh3}tu)ep&W-FnBwey73xL5e&&RP@*YuAK=AlUH<%m zjV~NPw-iuwu5s6$G{RTffb<(jWb5Z3>Lb`JO+^ClKHVMYv}4XbW#i9>gi+qJH3yat zzy(_WB_VA1U;w_>c-|s9cT5<7DlM;?t&Bh?Xcc`|ainC3qZzeVHH5Xj-FT_=PA?j( z6(syxiuS_&TH$EEGJ^S+hB-%rD+b_tEze^w8~P2vVQs+8OUff&{FIZKstCSTOU?TA z$W|j-q={WPxOKrcYjG)UkAq!Q#-rF>sNL2*7SnC4TD~FvwRRyh5_PWf~ z(7iN4srTrYDe~}|Vd?`d9`v(vSUR%`O?1q8oa9IW{h5{fllOu zsi8UucGIE(NN5Az4v>T2SJ6S$W@QBDMpXMj&8E-+^stsU{9JidaG#=%ke=3ZSpCWf zmmi}z=QUTe2H+Tt%LCrF5$e1CQpN{~NlSH=tcTaNykifQy^1U?RdA-Xz8CH&h#Q+O z@%@6{WvY)--6?+U0y=1prs^jujXvA80`EUcZ{AJRqT`E|LqNJA;DL*B0%#z2C}<(~ z{uOzg>ojgUP4k$c7>)l+)iA5aST;sE>}%8d?!9rOr44su9|;@I2-XUG9w{Su0u5BF zCJ5(1-?9Alg0xmbb3iqK-_{65t0n}wan!y3=QJr5sy4|*=+}<9g8#HC;lITCAnb0C z#yd{aSTVq)YGaVS%3-pkjjq3{JWBll0>VQyOfO=XXt{^3rKyLByh5WY!e^fr&U>pI z7ChF%%csax?Wk7p{+_Z8GeLDLo1d5dgXUAt0Q#NQo%E1|BZp16`ipcW!_OhYfmj<> z{zLkub7sd1sw=pE4-p_w{+0H3Ov)S=NO-<>4=CYbXFx2JfWLE{#!QH&cP zCCuLBfWx3@;1RO=%L8dz&MUogIN*fQK38dEY>m!1ewU3;9zgdR|9G8$X#hUedegGm z&}Yb?oqrWq4x_I%kgFOFOh`dfRl`mbwRB0N(#Tz=p(|BJmkRA9@$G!~0p=nNdV&~M z&l`YtZ3at8hRAwt*z@f;5^@*i$FID+QyJm5KpWSS;P7w+fD{7U+dvQ3ynRktZ_KtBijm7_5c zEmnnd`NL7T<~#L0>T$X{0=c5`AWy<5b?PMHGi56#Ex!`;6?s@JuHUiXR}QL%eNh#` zG-y1(_ccxsus3kky{a3nw^hTGE!s))YszrCXi8;Rjxr*6IEs0RxUBx>9-PZk9~Fc< zsfSU9zk@tCKO1XqlLuX*K_97l1wsG&v@2D^+^F%MtT4)r8ef^_A2@)|(kLdWB8a^j zj$9StK1GX`ZzXyo(Ox`K>+$rtvKRiQwM@CLoIqKL41`JAeL3J4El>KVa)SHc7#Ub6 zYv}o^qZB$B3E8hTXe~L`oY4qfhzJ#NkDg5TYn@1jOHc5>n|ta3CubTt6PmB3-xyFv zjQmpz=3bYJql_>*5#LCD$3@zb!u!ezZX7cB2k`^88C2n5W|YIO{Nj{?sE0=`(99|% z|36PFtGucVr_MSgxyfDUYJ?LsK?#~|P4{3Z>Nin_fHeWb(YgSmQ;x=Mljc$I05x9A zkvuHj-U}_70G(erMLRM5ma@(5i<7#S(yRgL8e{d)fQD19F-4Lin}+j2)h52f@jK=r z0Ao_!3fq)zfhVClBHCFSV^32KPts`xYee}jcc;$Wv@t(dbb7Xy4lfJsHpt7tQ)az?YYMYZ0thm0uprNWL z3aQ(OI2F}bRF>*Sg0jl;f*I9@f{I8)wyIuwwzN)HM3h=&|25VZR1ka1Pt!`J@vZfM z=+YC-(@KeV1qHOJcv*FMOSDHel{Oq;=;~$@v1L(|O8Ph*C4NSkEO%xvQbIf_ zswO-Q)i|MB?OIEhmPmtjM_hG8WrnIGl^Qa7u`0z!-^y48AG=YqvZG>>3XSs9#LDn21+E6|jn^Cij{!G4{T|g~1%qUaY!T*w1v}I&jU7nx{ zpH@vNsjCA4P9UkcG^MB(B$d@wmsO?~Rq5R8*fYo;-!dZao8sH<6~yB*vuE9aPkp|t zbaPiKs;kE4`iJ7m@Zx-Rg8xOXY6yRTyiM$MFDGWV6(@)czO}KQ`KVOfAiG;ByEFr2 ztw>3)$zaYvvEri2D6HDIySyOEqZ(o}esi5sR;_AaR+fWND@NAUM0yZQN}JMMcNC^+ z3iLJWe6mAy+M3A71o+J32tC!ivbnF&D64aGDEF0pOFz*D+BXh`PZyU z+dBGV=&EERqd=jbcWU%j4`OC>O#(h%k9X7)rL!~2OPpNm%a+L&x};a0tL>0dKtp|X zZJn-KU!|&>t1M~kh<)ZdV|uN=?q+IsTKX#5UEg)B3tMG1^9xF-2ODd9!VP#8ANMb^n&%*g}GpiU8MfKBCi^?%!g@=B$suph@gQw@l zWtQpdn#-7#3td!USUr_dUS1hrQm+GJlAVZ>$|;kg?o}D8^&-E+pu~V(N>945OkX#+ ztdZhbQjw5olvc;cs*0Lw#ogkO{!P_ISP4U2R#KcE*R;&#W8^{UW=itd!uf$V{&6Ke zjm(kbV`H+5iqrM8yGx3GsPEb~tyR~+Wat{oWXdN0vH#cJm3TFg?CW1=frKR+I3h$% zngjwQ2(BPI0s=#jbp!zc37dq)5EeBcC@LZ%3L*%2QE@~E1XmaZM8y#_jOf4&Y8r7| zz!4P{M`i@(wY=j!_q==VKXBjl$vJgSb^5FC`>NAb`Bin5)(K%sw9IvNqT^!2WjpZj zp-CxXrb{1b{9|cMp{22McCkoe8t5r8)zc7!(gqw%gP3PD-N|Z_LgUt|n+bB6OkyWx z8g^!c5oE@hI`yQug?@Kp69e>3CCu(E$%1i$@c=EJ7!yF)K#-CZvRa{8osLV+*B!FR zjOWhbkwJQ>R!Q+=5A_v!5`6y(D~e&z#+b`#nJz2=JELje_eh%56A4d`jWix0akDBD zeH1C76Z6v@q?-&o(gkVAI_}f``Bz;?1ATe#RI^=t!bk}!f{9Y6w4>@oj^l-(@Ns%D zGm2{A<}C9_;-0PsXTk!B3u!m^ahXhbezrnGx%@kPSVL-}t8?A}uSged#a+de;doKd z)GAI)q%0?>i*m;RkAh~_mqz3_nYifo33|l^q?C<9FTy_Cn7Xmdg{n9~o%(TVk|t9U zzp%8T_lh8sBq2$1GnqA09kghp)Exc5BLB=>VIjrz_}uhDp{gvB7@4#yI<~yag>}d* zHp#t7FsdVxQ(CB{N>d3Q-!yBbb-k=5ns(+CwgjC;M>(Y}71zR>iQ)3NDxQShlJ06G zDgUX}qb9~oMK^hBgihy$%Z5C*Q)8rB0|#mtLHgoW@DGv1r@Hbt<}zYvEUDBy4&haq z1UQtY7?004uVrpv#}ukMv=wUU196dp^IB3u@uR%BJ_}LZyzV(Jtt_967lC5lC~rJ= zh+$f*u2fGuCT+|KPlybIM6OYJbX03b*Rn417Q0nxaz(ttuQr6oFcUbco7(b9OHnoeW%7%<_vOsbvVAF`1J0fS8&RnUNIU z`%tAjMltJC+c4WFjNlQu%P%lS%&j)`N5E%-iZ?Uiui_<3EOr|=u{(oW!-ctOVq{xF z4H<1M684g6ZTjM5NWwa|j<|U3iWo92Ho(NJlx4~`FO{DU6TH)lNzR!VBW?GNSC?l* z%1ph;5(WQfQFL^iPOp||)WXNaeDz^jY-(P2r%qQ@i$RT!u{%Xc zG^d!UhzkPtFX&h#9dnu2YiTsX4Nc|Mn<;Y4YAAxI2yfts_pEd<#Mdm8A~~9TRk=w3 z|FCIdnqiz($|m=(3|maROp&fb%Y=JFB#TWxkpmru0359d zDll^L1^VMRB0B1hE?ys;$8}UARl)4-g?NzSgbJmAJ3Za}NT7JI9oO zxNIYna&?6ZB9|!0&(K-ygmwp0 zF+**UqT1;Duj%}9FZ*1Ix~6s}j+#o6ov%sQ^QE<3;EdpeMcdnb;vE7?fgg0MFVbKPWI&KqX%< z8C4bQ_vaon(MG-%!-*cG9zSh&tP#fu3_pj*{4RI_tDE192Q+8-UTjq%EL&6k-Z?BQ z5t@~7T_!+j-!?g@Tqq6GckgRFhpkHqk658)XG z&p($?@tYXN#PC)O?^OiWtUVKA_#}pDG5n<>pp!8vVJyofWRt?fXnheYElyhubZ=lH zR@!f8iHWmK6o3kISYthC;Us!@5*33XZk4E#kCBW+0p0$|L=2`q1T(y##EOMtScGek zZ)LesmgAuu8=P0+xB5p!PGVSrBe6K@s7bsB4luP&j1L!!VU-ws#o(tTv&QDL$dliw zkgt4QgE3eW1|eby6O9r9Kejf#lFZJs+qGfKGn)-q#hMR39~uu;3kEMeBtcRlt> z2dzfx{r)pY`ie!_Pv&yy%gZzuXRZ0*kX@!QY6_yRKJ6EH@|eKLR3vwU{* zd&7W~Ldg6%;n9O-MVrssXm3BKdYg7_Z-0;dAI-(0*tunzYhL~1@gK3Lp4C6lGR`p8 zZ2Xw?BxPYj@K}9GghQ96 z@TVK=$%)r(OIDfQ7MFGOA&c&-&pBjzrC@a`FKw%KAn|65ZG*M;j-E97qIC~n-RsI{ zuhieGqRzi#VjAJ;OlT(29gcU6`d{+OkW1no{m%Q%e;Z%3F-LgnS6|Y_WT~53d%j+MynsKtm!+)$`tZj zwZj3l#k)V1s+Lo;_p@`z3y~*IvatzaP9%P=C{lmM8 zrxVw$PfFr-I!B)MuPF=;3fpdQ>ynM|%8oa*H3ujYk)+WRYc^@?u3ZcjFYVr3JlQsO zxoCBlhR5)CTl3x+M0aY8@a>p$IlD<&wn;}C+?O7BlA>&=9hN4nj1^t1x?xcK{j?jB zMh|}bF{w=PFpZv?eEe3AYe@Cw5_bM;gXsD5c8unFzC5wS&&torQLE8r`qvAt9FsxM zSA*+jVZGvvkJB|}W=->g58DkO##Xb8gax>}LO#K3#VBqG)nQ#RI}2&rOzBMEHVcrxcW@T(V_c zSR(PpgHijV7Q?XXq2173gXj!o$LRaT{;x03yY~3fffDx4hr;1%qRF6{lk*j{o!)qB zBuqZn;`*TM_k%*k?o_%f)oAtYP<-mgxo3U_PeKma(0}L(&Zd~Gd}+sAc_8TM&He8} zKW<#q6i#*@T#@J;N>aPOA&C9U*`hPWUihY=v9EqV`*n>u%j(3OH>3O?#$I&Oo`1+l zev)$NMO59;=7dp4akYZI)5cU&Y`@N`F`WEl=)}?WNB_88eYV8&p2l?4wwv*#KbMxU zAKV?R{k7;&OzKa!PVwg58A@{*KX8j1MV~r*|K2au>8N@2!P$F1CDBh!cBqyPJ^7=c zXKqMC)aJ8=N=f|JE4R=}7XEs(Tg~Uq9-Y&^0i&XK@AD7e7?%E7y>XRqLoVg)yDfJT zoCj9KU7GiMDC6w9&1WgcZ{FLtKb(B%&Ycsv^U+Eh*`XMkK0CU~@kE}r&zK*mOPzPG z|Mc$gpBRm~ceX6@8?Ox`@6{%6Ozf02X=XIfULdFcARIfH@k^y5;7*!i!KFhJjK2*UwQrzzHGcMPz-2Ca~wP$Dd+WTCnV|Ypr-}pe)d3%4FE1y&F`qAj%7RA9& zPbc;U8ov|7KFe{|hx>won3$aWIx zd2^pLh)1Xt6B<@mQf*)Ikmes?>l7q)>T z1_2mE3<7&^8~9?d8iOzl)?pyWU?T=O7;ML&5QBXfbYP&w;4TJ_FnEf=C;X0|o}V80cZZ#(;-?_CSb%1qO>Tu)@F=`z#;7REj~$jN(iT6f>gp zFvy?LT|7gz9D^zhYQA_dFk`?N29r~$vx5XGB zF$g3#)zU9+=*?Jx)8&0}11D|!#fN|8o&QCC`Hc1s47xDr#h`D7IgUU2&muniV6al9 z|1b32|Bbs8_e2Fo4Fj1N)oUjrucMnJn_$Pd?z~Kun9mZ_Y$w7sH-D} zLM7`u#m&wFzzacE_ypQbkq}o{z|e>jM(eqwls9A{XLYhnD&dM02+7SqFwn!rj-#g_ zgfej;G1~s_e_|);L9}@?>L;p;?2g;wbv8XfL&?8*&wO(#mcMDha(**ag zd!0;%O*XPrO^^j!ytN|9G{_}sf9^6GiV@0MI7ZZlRo*a6;D(+P<9ir^D2$GI6B`iQ zEy%4YhGCBh;tTy=NTLUxKE_igJbfib)TM|YOic0SauM{*s%c((H=^$)_Cm6}32P~{ zrg_!3@su;o)9xdLQOv~ne=TG!Wyv%z@dm+!1I%XszuioZ~rawxhU79$jCEG;H8$Ex7{2jB06IVZo&W#< literal 0 HcmV?d00001 diff --git a/internal/pkg/meta/internal/adv/talos/talos.go b/internal/pkg/meta/internal/adv/talos/talos.go new file mode 100644 index 0000000..2dd715d --- /dev/null +++ b/internal/pkg/meta/internal/adv/talos/talos.go @@ -0,0 +1,230 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package talos implements modern ADV which supports large size for the values and tags. +package talos + +import ( + "bytes" + "crypto/sha256" + "encoding/binary" + "fmt" + "io" + "slices" + + "github.com/siderolabs/gen/maps" + "github.com/siderolabs/gen/xslices" + + "github.com/aenix-io/talm/internal/pkg/meta/internal/adv" +) + +// Basic constants configuring the ADV. +const ( + Length = 256 * 1024 // 256KiB + DataLength = Length - 40 + Size = 2 * Length // Redundancy +) + +// Magic constants. +const ( + Magic1 uint32 = 0x5a4b3c2d + Magic2 uint32 = 0xa5b4c3d2 +) + +// Tag is the key. +// +// We use a byte here for compatibility with syslinux, but format has space for uint32. +type Tag uint8 + +// Value stored for the tag. +type Value []byte + +// ADV implements the Talos extended ADV. +// +// Layout (all in big-endian): +// +// 0x0000 4 bytes magic1 +// 0x0004 4 bytes tag +// 0x0008 4 bytes size +// 0x000c (size) bytes value +// ... more tags +// -0x0024 32 bytes sha256 of the whole block with checksum set to zero +// -0x0004 4 bytes magic2 +// +// Whole data structure is written twice for redundancy. +type ADV struct { + Tags map[Tag]Value +} + +// NewADV loads ADV from the block device. +func NewADV(r io.Reader) (*ADV, error) { + a := &ADV{ + Tags: make(map[Tag]Value), + } + + if r == nil { + return a, nil + } + + buf := make([]byte, Length) + + _, err := io.ReadFull(r, buf) + if err != nil { + return nil, err + } + + if err = a.Unmarshal(buf); err == nil { + return a, nil + } + + // try 2nd copy + _, err = io.ReadFull(r, buf) + if err != nil { + return nil, err + } + + return a, a.Unmarshal(buf) +} + +// Unmarshal single copy from the serialized representation. +func (a *ADV) Unmarshal(buf []byte) error { + magic1 := binary.BigEndian.Uint32(buf[:4]) + if magic1 != Magic1 { + return fmt.Errorf("adv: unexpected magic %x, expecting %x", magic1, Magic1) + } + + magic2 := binary.BigEndian.Uint32(buf[len(buf)-4:]) + if magic2 != Magic2 { + return fmt.Errorf("adv: unexpected magic %x, expecting %x", magic2, Magic2) + } + + checksum := slices.Clone(buf[len(buf)-36 : len(buf)-4]) + + copy(buf[len(buf)-36:len(buf)-4], make([]byte, 32)) + + hash := sha256.New() + hash.Write(buf) + actualChecksum := hash.Sum(nil) + + if !bytes.Equal(checksum, actualChecksum) { + return fmt.Errorf("adv: checksum mismatch: %x, expecting %x", checksum, actualChecksum) + } + + data := buf[4 : len(buf)-36] + + for len(data) >= 8 { + tag := binary.BigEndian.Uint32(data[:4]) + if tag == adv.End { + break + } + + size := binary.BigEndian.Uint32(data[4:8]) + + if uint32(len(data)) < size+8 { + return fmt.Errorf("adv: value goes beyond the end of the buffer: tag %d, size %d", tag, size) + } + + value := data[8 : 8+size] + + a.Tags[Tag(tag)] = Value(value) + + data = data[8+size:] + } + + return nil +} + +// Marshal single copy of ADV. +func (a *ADV) Marshal() ([]byte, error) { + buf := make([]byte, Length) + + binary.BigEndian.PutUint32(buf[0:4], Magic1) + binary.BigEndian.PutUint32(buf[len(buf)-4:], Magic2) + + data := buf[4 : len(buf)-36] + + for tag, value := range a.Tags { + if len(value)+8 > len(data) { + return nil, fmt.Errorf("adv: overflow %d bytes", len(value)+8-len(data)) + } + + binary.BigEndian.PutUint32(data[0:4], uint32(tag)) + binary.BigEndian.PutUint32(data[4:8], uint32(len(value))) + copy(data[8:8+len(value)], value) + + data = data[8+len(value):] + } + + hash := sha256.New() + hash.Write(buf) + copy(buf[len(buf)-36:len(buf)-4], hash.Sum(nil)) + + return buf, nil +} + +// Bytes marshal full representation. +func (a *ADV) Bytes() ([]byte, error) { + marshaled, err := a.Marshal() + if err != nil { + return nil, err + } + + return append(marshaled, marshaled...), nil +} + +// ReadTag to get tag value. +func (a *ADV) ReadTag(t uint8) (val string, ok bool) { + b, ok := a.ReadTagBytes(t) + + val = string(b) + + return +} + +// ReadTagBytes to get tag value. +func (a *ADV) ReadTagBytes(t uint8) (val []byte, ok bool) { + val, ok = a.Tags[Tag(t)] + + return +} + +// ListTags to get list of tags. +func (a *ADV) ListTags() (tags []uint8) { + return xslices.Map(maps.Keys(a.Tags), func(t Tag) uint8 { return uint8(t) }) +} + +// SetTag to set tag value. +func (a *ADV) SetTag(t uint8, val string) (ok bool) { + return a.SetTagBytes(t, []byte(val)) +} + +// SetTagBytes to set tag value. +func (a *ADV) SetTagBytes(t uint8, val []byte) (ok bool) { + size := 20 // magic + checksum + + for _, v := range a.Tags { + size += len(v) + 8 + } + + oldVal := a.Tags[Tag(t)] + + size += len(Value(val)) - len(oldVal) + + if size > DataLength { + return false + } + + a.Tags[Tag(t)] = Value(val) + + return true +} + +// DeleteTag to delete tag value. +func (a *ADV) DeleteTag(t uint8) (ok bool) { + _, ok = a.Tags[Tag(t)] + + delete(a.Tags, Tag(t)) + + return +} diff --git a/internal/pkg/meta/internal/adv/talos/talos_test.go b/internal/pkg/meta/internal/adv/talos/talos_test.go new file mode 100644 index 0000000..f683e2d --- /dev/null +++ b/internal/pkg/meta/internal/adv/talos/talos_test.go @@ -0,0 +1,92 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package talos_test + +import ( + "bytes" + "slices" + "sort" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/aenix-io/talm/internal/pkg/meta/internal/adv" + "github.com/aenix-io/talm/internal/pkg/meta/internal/adv/talos" +) + +func TestMarshalUnmarshal(t *testing.T) { + a, err := talos.NewADV(bytes.NewReader(make([]byte, talos.Size))) + assert.Error(t, err) + require.NotNil(t, a) + + const ( + val1 = "value1" + val2 = "value2" + val3 = "value3" + ) + + assert.True(t, a.SetTag(adv.Reserved1, val1)) + assert.True(t, a.SetTag(adv.Reserved2, val2)) + assert.True(t, a.SetTag(adv.Reserved3, val3)) + + b, err := a.Bytes() + require.NoError(t, err) + assert.Len(t, b, talos.Size) + + // test recoverable corruption + for _, c := range []struct { + zeroOut [][2]int + }{ + {}, + { + zeroOut: [][2]int{ + {0, 2}, + }, + }, + { + zeroOut: [][2]int{ + {30, 1000}, + }, + }, + { + zeroOut: [][2]int{ + {8, 4}, + {40, 2}, + }, + }, + { + zeroOut: [][2]int{ + {0, talos.Length}, + }, + }, + } { + corrupted := slices.Clone(b) + + for _, z := range c.zeroOut { + copy(corrupted[z[0]:z[0]+z[1]], make([]byte, z[1])) + } + + a, err = talos.NewADV(bytes.NewReader(b)) + require.NoError(t, err) + require.NotNil(t, a) + + val, ok := a.ReadTag(adv.Reserved1) + assert.True(t, ok) + assert.Equal(t, val1, val) + + val, ok = a.ReadTag(adv.Reserved2) + assert.True(t, ok) + assert.Equal(t, val2, val) + + val, ok = a.ReadTag(adv.Reserved3) + assert.True(t, ok) + assert.Equal(t, val3, val) + + tags := a.ListTags() + sort.Slice(tags, func(i, j int) bool { return tags[i] < tags[j] }) + assert.Equal(t, []uint8{adv.Reserved1, adv.Reserved2, adv.Reserved3}, tags) + } +} diff --git a/internal/pkg/meta/meta.go b/internal/pkg/meta/meta.go new file mode 100644 index 0000000..3cf24ea --- /dev/null +++ b/internal/pkg/meta/meta.go @@ -0,0 +1,345 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package meta provides access to META partition: key-value partition persisted across reboots. +package meta + +import ( + "context" + "fmt" + "io" + "log" + "os" + "sync" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-blockdevice/blockdevice/probe" + + "github.com/aenix-io/talm/internal/pkg/meta/internal/adv" + "github.com/aenix-io/talm/internal/pkg/meta/internal/adv/syslinux" + "github.com/aenix-io/talm/internal/pkg/meta/internal/adv/talos" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// Meta represents the META reader/writer. +// +// Meta abstracts away all details about loading/storing the metadata providing an easy to use interface. +type Meta struct { + mu sync.Mutex + + legacy adv.ADV + talos adv.ADV + state state.State + opts Options +} + +// Options configures the META. +type Options struct { + fixedPath string + printer func(string, ...any) +} + +// Option is a functional option. +type Option func(*Options) + +// WithFixedPath sets the fixed path to META partition. +func WithFixedPath(path string) Option { + return func(o *Options) { + o.fixedPath = path + } +} + +// WithPrinter sets the function to print the logs, default is log.Printf. +func WithPrinter(printer func(string, ...any)) Option { + return func(o *Options) { + o.printer = printer + } +} + +// New initializes empty META, trying to probe the existing META first. +func New(ctx context.Context, st state.State, opts ...Option) (*Meta, error) { + meta := &Meta{ + state: st, + opts: Options{ + printer: log.Printf, + }, + } + + for _, opt := range opts { + opt(&meta.opts) + } + + var err error + + meta.legacy, err = syslinux.NewADV(nil) + if err != nil { + return nil, err + } + + meta.talos, err = talos.NewADV(nil) + if err != nil { + return nil, err + } + + err = meta.Reload(ctx) + + return meta, err +} + +func (meta *Meta) getPath() (string, error) { + if meta.opts.fixedPath != "" { + return meta.opts.fixedPath, nil + } + + dev, err := probe.GetDevWithPartitionName(constants.MetaPartitionLabel) + if err != nil { + return "", err + } + + defer dev.Close() //nolint:errcheck + + return dev.PartPath(constants.MetaPartitionLabel) +} + +// Reload refreshes the META from the disk. +func (meta *Meta) Reload(ctx context.Context) error { + meta.mu.Lock() + defer meta.mu.Unlock() + + path, err := meta.getPath() + if err != nil { + return err + } + + f, err := os.Open(path) + if err != nil { + return err + } + + defer f.Close() //nolint:errcheck + + adv, err := talos.NewADV(f) + if adv == nil && err != nil { + // if adv is not nil, but err is nil, it might be missing ADV, ignore it + return err + } + + legacyAdv, err := syslinux.NewADV(f) + if err != nil { + return err + } + + // copy values from in-memory to on-disk version + for _, t := range meta.talos.ListTags() { + val, _ := meta.talos.ReadTagBytes(t) + adv.SetTagBytes(t, val) + } + + meta.opts.printer("META: loaded %d keys", len(adv.ListTags())) + + meta.talos = adv + meta.legacy = legacyAdv + + return meta.syncState(ctx) +} + +// syncState sync resources with adv contents. +func (meta *Meta) syncState(ctx context.Context) error { + if meta.state == nil { + return nil + } + + existingTags := make(map[resource.ID]struct{}) + + for _, t := range meta.talos.ListTags() { + existingTags[runtime.MetaKeyTagToID(t)] = struct{}{} + val, _ := meta.talos.ReadTag(t) + + if err := updateTagResource(ctx, meta.state, t, val); err != nil { + return err + } + } + + items, err := meta.state.List(ctx, runtime.NewMetaKey(runtime.NamespaceName, "").Metadata()) + if err != nil { + return err + } + + for _, item := range items.Items { + if _, exists := existingTags[item.Metadata().ID()]; exists { + continue + } + + if err = meta.state.Destroy(ctx, item.Metadata()); err != nil { + return err + } + } + + return nil +} + +// Flush writes the META to the disk. +func (meta *Meta) Flush() error { + meta.mu.Lock() + defer meta.mu.Unlock() + + path, err := meta.getPath() + if err != nil { + return err + } + + f, err := os.OpenFile(path, os.O_RDWR, 0) + if err != nil { + return err + } + + defer f.Close() //nolint:errcheck + + serialized, err := meta.talos.Bytes() + if err != nil { + return err + } + + n, err := f.WriteAt(serialized, 0) + if err != nil { + return err + } + + if n != len(serialized) { + return fmt.Errorf("expected to write %d bytes, wrote %d", len(serialized), n) + } + + serialized, err = meta.legacy.Bytes() + if err != nil { + return err + } + + offset, err := f.Seek(-int64(len(serialized)), io.SeekEnd) + if err != nil { + return err + } + + n, err = f.WriteAt(serialized, offset) + if err != nil { + return err + } + + if n != len(serialized) { + return fmt.Errorf("expected to write %d bytes, wrote %d", len(serialized), n) + } + + meta.opts.printer("META: saved %d keys", len(meta.talos.ListTags())) + + return f.Sync() +} + +// ReadTag reads a tag from the META. +func (meta *Meta) ReadTag(t uint8) (val string, ok bool) { + meta.mu.Lock() + defer meta.mu.Unlock() + + val, ok = meta.talos.ReadTag(t) + if !ok { + val, ok = meta.legacy.ReadTag(t) + } + + return val, ok +} + +// ReadTagBytes reads a tag from the META. +func (meta *Meta) ReadTagBytes(t uint8) (val []byte, ok bool) { + meta.mu.Lock() + defer meta.mu.Unlock() + + val, ok = meta.talos.ReadTagBytes(t) + if !ok { + val, ok = meta.legacy.ReadTagBytes(t) + } + + return val, ok +} + +// SetTag writes a tag to the META. +func (meta *Meta) SetTag(ctx context.Context, t uint8, val string) (bool, error) { + meta.mu.Lock() + defer meta.mu.Unlock() + + ok := meta.talos.SetTag(t, val) + + if ok { + err := updateTagResource(ctx, meta.state, t, val) + if err != nil { + return false, err + } + } + + return ok, nil +} + +// SetTagBytes writes a tag to the META. +func (meta *Meta) SetTagBytes(ctx context.Context, t uint8, val []byte) (bool, error) { + meta.mu.Lock() + defer meta.mu.Unlock() + + ok := meta.talos.SetTagBytes(t, val) + + if ok { + err := updateTagResource(ctx, meta.state, t, string(val)) + if err != nil { + return false, err + } + } + + return ok, nil +} + +// DeleteTag deletes a tag from the META. +func (meta *Meta) DeleteTag(ctx context.Context, t uint8) (bool, error) { + meta.mu.Lock() + defer meta.mu.Unlock() + + ok := meta.talos.DeleteTag(t) + if !ok { + ok = meta.legacy.DeleteTag(t) + } + + if meta.state == nil { + return ok, nil + } + + err := meta.state.Destroy(ctx, runtime.NewMetaKey(runtime.NamespaceName, runtime.MetaKeyTagToID(t)).Metadata()) + if state.IsNotFoundError(err) { + err = nil + } + + return ok, err +} + +func updateTagResource(ctx context.Context, st state.State, t uint8, val string) error { + if st == nil { + return nil + } + + _, err := safe.StateUpdateWithConflicts(ctx, st, runtime.NewMetaKey(runtime.NamespaceName, runtime.MetaKeyTagToID(t)).Metadata(), func(r *runtime.MetaKey) error { + r.TypedSpec().Value = val + + return nil + }) + + if err == nil { + return nil + } + + if state.IsNotFoundError(err) { + r := runtime.NewMetaKey(runtime.NamespaceName, runtime.MetaKeyTagToID(t)) + r.TypedSpec().Value = val + + return st.Create(ctx, r) + } + + return err +} diff --git a/internal/pkg/meta/meta_test.go b/internal/pkg/meta/meta_test.go new file mode 100644 index 0000000..917315c --- /dev/null +++ b/internal/pkg/meta/meta_test.go @@ -0,0 +1,98 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package meta provides access to META partition: key-value partition persisted across reboots. +package meta_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/aenix-io/talm/internal/pkg/meta" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +func setupTest(t *testing.T) (*meta.Meta, string, state.State) { + t.Helper() + + tmpDir := t.TempDir() + + path := filepath.Join(tmpDir, "meta") + + f, err := os.Create(path) + require.NoError(t, err) + + require.NoError(t, f.Truncate(1024*1024)) + + require.NoError(t, f.Close()) + + st := state.WrapCore(namespaced.NewState(inmem.Build)) + + m, err := meta.New(context.Background(), st, meta.WithFixedPath(path)) + require.NoError(t, err) + + return m, path, st +} + +func TestFlow(t *testing.T) { + t.Parallel() + + m, path, st := setupTest(t) + + ctx := context.Background() + + ok, err := m.SetTag(ctx, meta.Upgrade, "1.2.3") + require.NoError(t, err) + assert.True(t, ok) + + val, ok := m.ReadTag(meta.Upgrade) + assert.True(t, ok) + assert.Equal(t, "1.2.3", val) + + _, ok = m.ReadTag(meta.StagedUpgradeImageRef) + assert.False(t, ok) + + ok, err = m.DeleteTag(ctx, meta.Upgrade) + require.NoError(t, err) + assert.True(t, ok) + + ok, err = m.SetTag(ctx, meta.StagedUpgradeInstallOptions, "install-fast") + require.NoError(t, err) + assert.True(t, ok) + + assert.NoError(t, m.Flush()) + + assert.NoError(t, m.Reload(ctx)) + + val, ok = m.ReadTag(meta.StagedUpgradeInstallOptions) + assert.True(t, ok) + assert.Equal(t, "install-fast", val) + + m2, err := meta.New(ctx, st, meta.WithFixedPath(path)) + require.NoError(t, err) + + _, ok = m2.ReadTag(meta.Upgrade) + assert.False(t, ok) + + val, ok = m2.ReadTag(meta.StagedUpgradeInstallOptions) + assert.True(t, ok) + assert.Equal(t, "install-fast", val) + + list, err := safe.StateList[*runtime.MetaKey](ctx, st, runtime.NewMetaKey(runtime.NamespaceName, "").Metadata()) + require.NoError(t, err) + + for iter := list.Iterator(); iter.Next(); { + assert.Equal(t, "0x08", iter.Value().Metadata().ID()) + assert.Equal(t, "install-fast", iter.Value().TypedSpec().Value) + } +} diff --git a/internal/pkg/mount/all.go b/internal/pkg/mount/all.go new file mode 100644 index 0000000..e006c19 --- /dev/null +++ b/internal/pkg/mount/all.go @@ -0,0 +1,124 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package mount + +import ( + "bufio" + "fmt" + "log" + "os" + "strings" + "time" + + "golang.org/x/sys/unix" +) + +func unmountWithTimeout(target string, flags int, timeout time.Duration) error { + errCh := make(chan error, 1) + + go func() { + errCh <- unix.Unmount(target, flags) + }() + + timer := time.NewTimer(timeout) + defer timer.Stop() + + select { + case <-timer.C: + return fmt.Errorf("unmounting %s timed out after %s", target, timeout) + case err := <-errCh: + return err + } +} + +// UnmountAll attempts to unmount all the mounted filesystems via "self" mountinfo. +func UnmountAll() error { + // timeout in seconds + const timeout = 10 + + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + + for range timeout { + mounts, err := readMountInfo() + if err != nil { + return err + } + + failedUnmounts := 0 + + for _, mountInfo := range mounts { + if mountInfo.MountPoint == "" { + continue + } + + if strings.HasPrefix(mountInfo.MountSource, "/dev/") { + err = unmountWithTimeout(mountInfo.MountPoint, 0, time.Second) + + if err == nil { + log.Printf("unmounted %s (%s)", mountInfo.MountPoint, mountInfo.MountSource) + } else { + log.Printf("failed unmounting %s: %s", mountInfo.MountPoint, err) + + failedUnmounts++ + } + } + } + + if failedUnmounts == 0 { + break + } + + log.Printf("retrying %d unmount operations...", failedUnmounts) + + <-ticker.C + } + + return nil +} + +type mountInfo struct { + MountPoint string + MountSource string +} + +func readMountInfo() ([]mountInfo, error) { + f, err := os.Open("/proc/self/mountinfo") + if err != nil { + return nil, err + } + + defer f.Close() //nolint:errcheck + + var mounts []mountInfo + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + + parts := strings.SplitN(line, " - ", 2) + + if len(parts) < 2 { + continue + } + + var mntInfo mountInfo + + pre := strings.Fields(parts[0]) + post := strings.Fields(parts[1]) + + if len(pre) >= 5 { + mntInfo.MountPoint = pre[4] + } + + if len(post) >= 1 { + mntInfo.MountSource = post[1] + } + + mounts = append(mounts, mntInfo) + } + + return mounts, scanner.Err() +} diff --git a/internal/pkg/mount/bpffs.go b/internal/pkg/mount/bpffs.go new file mode 100644 index 0000000..2ac458c --- /dev/null +++ b/internal/pkg/mount/bpffs.go @@ -0,0 +1,14 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package mount + +// BPFMountPoints returns the bpf mount points. +func BPFMountPoints() (mountpoints *Points, err error) { + base := "/sys/fs/bpf" + bpf := NewMountPoints() + bpf.Set("bpf", NewMountPoint("bpffs", base, "bpf", 0, "")) + + return bpf, nil +} diff --git a/internal/pkg/mount/cgroups.go b/internal/pkg/mount/cgroups.go new file mode 100644 index 0000000..0214ff1 --- /dev/null +++ b/internal/pkg/mount/cgroups.go @@ -0,0 +1,65 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package mount + +import ( + "path/filepath" + + "github.com/siderolabs/go-procfs/procfs" + "golang.org/x/sys/unix" + + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// ForceGGroupsV1 returns the cgroup version to be used (only for !container mode). +func ForceGGroupsV1() bool { + value := procfs.ProcCmdline().Get(constants.KernelParamCGroups).First() + + return value != nil && *value == "0" +} + +// CGroupMountPoints returns the cgroup mount points. +func CGroupMountPoints() (mountpoints *Points, err error) { + if ForceGGroupsV1() { + return cgroupMountPointsV1() + } + + return cgroupMountPointsV2() +} + +func cgroupMountPointsV2() (mountpoints *Points, err error) { + cgroups := NewMountPoints() + + cgroups.Set("cgroup2", NewMountPoint("cgroup", constants.CgroupMountPath, "cgroup2", unix.MS_NOSUID|unix.MS_NODEV|unix.MS_NOEXEC|unix.MS_RELATIME, "nsdelegate")) + + return cgroups, nil +} + +func cgroupMountPointsV1() (mountpoints *Points, err error) { + cgroups := NewMountPoints() + cgroups.Set("dev", NewMountPoint("tmpfs", constants.CgroupMountPath, "tmpfs", unix.MS_NOSUID|unix.MS_NODEV|unix.MS_NOEXEC|unix.MS_RELATIME, "mode=755")) + + controllers := []string{ + "blkio", + "cpu", + "cpuacct", + "cpuset", + "devices", + "freezer", + "hugetlb", + "memory", + "net_cls", + "net_prio", + "perf_event", + "pids", + } + + for _, controller := range controllers { + p := filepath.Join(constants.CgroupMountPath, controller) + cgroups.Set(controller, NewMountPoint(controller, p, "cgroup", unix.MS_NOSUID|unix.MS_NODEV|unix.MS_NOEXEC|unix.MS_RELATIME, controller)) + } + + return cgroups, nil +} diff --git a/internal/pkg/mount/iter.go b/internal/pkg/mount/iter.go new file mode 100644 index 0000000..86c7b25 --- /dev/null +++ b/internal/pkg/mount/iter.go @@ -0,0 +1,115 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package mount + +// PointsIterator represents an iteratable group of mount points. +type PointsIterator struct { + p *Points + value *Point + key string + index int + end int + err error + reverse bool +} + +// Iter initializes and returns a mount point iterator. +func (p *Points) Iter() *PointsIterator { + return &PointsIterator{ + p: p, + index: -1, + end: len(p.order) - 1, + value: nil, + } +} + +// IterRev initializes and returns a mount point iterator that advances in +// reverse. +func (p *Points) IterRev() *PointsIterator { + return &PointsIterator{ + p: p, + reverse: true, + index: len(p.points), + end: 0, + value: nil, + } +} + +// Set sets an ordered value. +func (p *Points) Set(key string, value *Point) { + if _, ok := p.points[key]; ok { + for i := range p.order { + if p.order[i] == key { + p.order = append(p.order[:i], p.order[i+1:]...) + } + } + } + + p.order = append(p.order, key) + p.points[key] = value +} + +// Get gets an ordered value. +func (p *Points) Get(key string) (value *Point, ok bool) { + if value, ok = p.points[key]; ok { + return value, true + } + + return nil, false +} + +// Len returns number of mount points. +func (p *Points) Len() int { + return len(p.points) +} + +// Key returns the current key. +func (i *PointsIterator) Key() string { + return i.key +} + +// Value returns current mount point. +func (i *PointsIterator) Value() *Point { + if i.err != nil || i.index > len(i.p.points) { + panic("invoked Value on expired iterator") + } + + return i.value +} + +// Err returns an error. +func (i *PointsIterator) Err() error { + return i.err +} + +// Next advances the iterator to the next value. +func (i *PointsIterator) Next() bool { + if i.err != nil { + return false + } + + if i.reverse { + i.index-- + + if i.index < i.end { + return false + } + } else { + i.index++ + + if i.index > i.end { + return false + } + } + + i.key = i.p.order[i.index] + i.value = i.p.points[i.key] + + if i.reverse { + return i.index >= i.end + } + + return i.index <= i.end +} diff --git a/internal/pkg/mount/mount.go b/internal/pkg/mount/mount.go new file mode 100644 index 0000000..9734870 --- /dev/null +++ b/internal/pkg/mount/mount.go @@ -0,0 +1,504 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package mount + +import ( + "bufio" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/siderolabs/go-blockdevice/blockdevice" + "github.com/siderolabs/go-blockdevice/blockdevice/filesystem" + "github.com/siderolabs/go-blockdevice/blockdevice/util" + "github.com/siderolabs/go-retry/retry" + "golang.org/x/sys/unix" + + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/makefs" +) + +// RetryFunc defines the requirements for retrying a mount point operation. +type RetryFunc func(*Point) error + +// Mount mounts the device(s). +func Mount(mountpoints *Points) (err error) { + iter := mountpoints.Iter() + + // Mount the device(s). + + for iter.Next() { + if _, err = mountMountpoint(iter.Value()); err != nil { + return fmt.Errorf("error mounting %q: %w", iter.Value().Source(), err) + } + } + + if iter.Err() != nil { + return iter.Err() + } + + return nil +} + +//nolint:gocyclo +func mountMountpoint(mountpoint *Point) (skipMount bool, err error) { + // Repair the disk's partition table. + if mountpoint.MountFlags.Check(Resize) { + if _, err = mountpoint.ResizePartition(); err != nil { + return false, fmt.Errorf("error resizing %w", err) + } + } + + if mountpoint.MountFlags.Check(SkipIfMounted) { + skipMount, err = mountpoint.IsMounted() + if err != nil { + return false, fmt.Errorf("mountpoint is set to skip if mounted, but the mount check failed: %w", err) + } + } + + if mountpoint.MountFlags.Check(SkipIfNoFilesystem) && mountpoint.Fstype() == filesystem.Unknown { + skipMount = true + } + + if !skipMount { + if err = mountpoint.Mount(); err != nil { + if mountpoint.MountFlags.Check(SkipIfNoDevice) && errors.Is(err, unix.ENODEV) { + if mountpoint.Logger != nil { + mountpoint.Logger.Printf("error mounting: %q: %s", mountpoint.Source(), err) + } + + return true, nil + } + + return false, fmt.Errorf("error mounting: %w", err) + } + } + + // Grow the filesystem to the maximum allowed size. + // + // Growfs is called always, even if ResizePartition returns false to workaround failure scenario + // when partition was resized, but growfs never got called. + if mountpoint.MountFlags.Check(Resize) { + if err = mountpoint.GrowFilesystem(); err != nil { + return false, fmt.Errorf("error resizing filesystem: %w", err) + } + } + + return skipMount, nil +} + +// Unmount unmounts the device(s). +func Unmount(mountpoints *Points) (err error) { + iter := mountpoints.IterRev() + for iter.Next() { + mountpoint := iter.Value() + if err = mountpoint.Unmount(); err != nil { + return fmt.Errorf("unmount: %w", err) + } + } + + if iter.Err() != nil { + return iter.Err() + } + + return nil +} + +// Move moves the device(s). +// TODO(andrewrynhard): We need to skip calling the move method on mountpoints +// that are a child of another mountpoint. The kernel will handle moving the +// child mountpoints for us. +func Move(mountpoints *Points, prefix string) (err error) { + iter := mountpoints.Iter() + for iter.Next() { + mountpoint := iter.Value() + if err = mountpoint.Move(prefix); err != nil { + return fmt.Errorf("move: %w", err) + } + } + + if iter.Err() != nil { + return iter.Err() + } + + return nil +} + +// PrefixMountTargets prefixes all mountpoints targets with fixed path. +func PrefixMountTargets(mountpoints *Points, targetPrefix string) error { + iter := mountpoints.Iter() + for iter.Next() { + mountpoint := iter.Value() + mountpoint.target = filepath.Join(targetPrefix, mountpoint.target) + } + + return iter.Err() +} + +//nolint:gocyclo +func mountRetry(f RetryFunc, p *Point, isUnmount bool) (err error) { + err = retry.Constant(5*time.Second, retry.WithUnits(50*time.Millisecond)).Retry(func() error { + if err = f(p); err != nil { + switch err { + case unix.EBUSY: + return retry.ExpectedError(err) + case unix.ENOENT, unix.ENODEV: + // if udevd triggers BLKRRPART ioctl, partition device entry might disappear temporarily + return retry.ExpectedError(err) + case unix.EUCLEAN: + if errRepair := p.Repair(); errRepair != nil { + return fmt.Errorf("error repairing: %w", errRepair) + } + + return retry.ExpectedError(err) + case unix.EINVAL: + isMounted, checkErr := p.IsMounted() + if checkErr != nil { + return retry.ExpectedError(checkErr) + } + + if !isMounted && !isUnmount { + if errRepair := p.Repair(); errRepair != nil { + return fmt.Errorf("error repairing: %w", errRepair) + } + + return retry.ExpectedError(err) + } + + if !isMounted && isUnmount { // if partition is already unmounted, ignore EINVAL + return nil + } + + return err + default: + return err + } + } + + return nil + }) + + return err +} + +// Point represents a Linux mount point. +type Point struct { + source string + target string + fstype string + flags uintptr + data string + *Options +} + +// PointMap represents a unique set of mount points. +type PointMap = map[string]*Point + +// Points represents an ordered set of mount points. +type Points struct { + points PointMap + order []string +} + +// NewMountPoint initializes and returns a Point struct. +func NewMountPoint(source, target, fstype string, flags uintptr, data string, setters ...Option) *Point { + opts := NewDefaultOptions(setters...) + + p := &Point{ + source: source, + target: target, + fstype: fstype, + flags: flags, + data: data, + Options: opts, + } + + if p.Prefix != "" { + p.target = filepath.Join(p.Prefix, p.target) + } + + if p.Options.ProjectQuota { + if len(p.data) > 0 { + p.data += "," + } + + p.data += "prjquota" + } + + return p +} + +// NewMountPoints initializes and returns a Points struct. +func NewMountPoints() *Points { + return &Points{ + points: make(PointMap), + } +} + +// Source returns the mount points source field. +func (p *Point) Source() string { + return p.source +} + +// Target returns the mount points target field. +func (p *Point) Target() string { + return p.target +} + +// Fstype returns the mount points fstype field. +func (p *Point) Fstype() string { + return p.fstype +} + +// Flags returns the mount points flags field. +func (p *Point) Flags() uintptr { + return p.flags +} + +// Data returns the mount points data field. +func (p *Point) Data() string { + return p.data +} + +// Mount attempts to retry a mount on EBUSY. It will attempt a retry +// every 100 milliseconds over the course of 5 seconds. +func (p *Point) Mount() (err error) { + for _, hook := range p.Options.PreMountHooks { + if err = hook(p); err != nil { + return err + } + } + + if err = ensureDirectory(p.target); err != nil { + return err + } + + if p.MountFlags.Check(ReadOnly) { + p.flags |= unix.MS_RDONLY + } + + switch { + case p.MountFlags.Check(Overlay): + err = mountRetry(overlay, p, false) + case p.MountFlags.Check(ReadonlyOverlay): + err = mountRetry(readonlyOverlay, p, false) + default: + err = mountRetry(mount, p, false) + } + + if err != nil { + return err + } + + if p.MountFlags.Check(Shared) { + if err = mountRetry(share, p, false); err != nil { + return fmt.Errorf("error sharing mount point %s: %+v", p.target, err) + } + } + + return nil +} + +// Unmount attempts to retry an unmount on EBUSY. It will attempt a +// retry every 100 milliseconds over the course of 5 seconds. +func (p *Point) Unmount() (err error) { + var mounted bool + + if mounted, err = p.IsMounted(); err != nil { + return err + } + + if mounted { + if err = mountRetry(unmount, p, true); err != nil { + return err + } + } + + for _, hook := range p.Options.PostUnmountHooks { + if err = hook(p); err != nil { + return err + } + } + + return nil +} + +// IsMounted checks whether mount point is active under /proc/mounts. +func (p *Point) IsMounted() (bool, error) { + f, err := os.Open("/proc/mounts") + if err != nil { + return false, err + } + + defer f.Close() //nolint:errcheck + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + + if len(fields) < 2 { + continue + } + + mountpoint := fields[1] + + if mountpoint == p.target { + return true, nil + } + } + + return false, scanner.Err() +} + +// Move moves a mountpoint to a new location with a prefix. +func (p *Point) Move(prefix string) (err error) { + target := p.Target() + mountpoint := NewMountPoint(target, target, "", unix.MS_MOVE, "", WithPrefix(prefix)) + + if err = mountpoint.Mount(); err != nil { + return fmt.Errorf("error moving mount point %s: %w", target, err) + } + + return nil +} + +// ResizePartition resizes a partition to the maximum size allowed. +func (p *Point) ResizePartition() (resized bool, err error) { + var devname string + + if devname, err = util.DevnameFromPartname(p.Source()); err != nil { + return false, err + } + + bd, err := blockdevice.Open("/dev/"+devname, blockdevice.WithExclusiveLock(true)) + if err != nil { + return false, fmt.Errorf("error opening block device %q: %w", devname, err) + } + + //nolint:errcheck + defer bd.Close() + + pt, err := bd.PartitionTable() + if err != nil { + return false, err + } + + if err := pt.Repair(); err != nil { + return false, err + } + + for _, partition := range pt.Partitions().Items() { + if partition.Name == constants.EphemeralPartitionLabel { + resized, err := pt.Resize(partition) + if err != nil { + return false, err + } + + if !resized { + return false, nil + } + } + } + + if err := pt.Write(); err != nil { + return false, err + } + + return true, nil +} + +// GrowFilesystem grows a partition's filesystem to the maximum size allowed. +// NB: An XFS partition MUST be mounted, or this will fail. +func (p *Point) GrowFilesystem() (err error) { + if err = makefs.XFSGrow(p.Target()); err != nil { + return fmt.Errorf("xfs_growfs: %w", err) + } + + return nil +} + +// Repair repairs a partition's filesystem. +func (p *Point) Repair() error { + if p.Logger != nil { + p.Logger.Printf("filesystem on %s needs cleaning, running repair", p.Source()) + } + + if err := makefs.XFSRepair(p.Source(), p.Fstype()); err != nil { + return fmt.Errorf("xfs_repair: %w", err) + } + + if p.Logger != nil { + p.Logger.Printf("filesystem successfully repaired on %s", p.Source()) + } + + return nil +} + +func mount(p *Point) (err error) { + return unix.Mount(p.source, p.target, p.fstype, p.flags, p.data) +} + +func unmount(p *Point) error { + return SafeUnmount(context.Background(), p.Logger, p.target) +} + +func share(p *Point) error { + return unix.Mount("", p.target, "", unix.MS_SHARED|unix.MS_REC, "") +} + +func overlay(p *Point) error { + parts := strings.Split(p.target, "/") + prefix := strings.Join(parts[1:], "-") + + basePath := constants.VarSystemOverlaysPath + + if p.MountFlags.Check(SystemOverlay) { + basePath = constants.SystemOverlaysPath + } + + diff := fmt.Sprintf(filepath.Join(basePath, "%s-diff"), prefix) + workdir := fmt.Sprintf(filepath.Join(basePath, "%s-workdir"), prefix) + + for _, target := range []string{diff, workdir} { + if err := ensureDirectory(target); err != nil { + return err + } + } + + lowerDir := p.target + if p.source != "" { + lowerDir = p.source + } + + opts := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", lowerDir, diff, workdir) + if err := unix.Mount("overlay", p.target, "overlay", 0, opts); err != nil { + return fmt.Errorf("error creating overlay mount to %s: %w", p.target, err) + } + + return nil +} + +func readonlyOverlay(p *Point) error { + opts := fmt.Sprintf("lowerdir=%s", p.source) + if err := unix.Mount("overlay", p.target, "overlay", p.flags, opts); err != nil { + return fmt.Errorf("error creating overlay mount to %s: %w", p.target, err) + } + + return nil +} + +func ensureDirectory(target string) (err error) { + if _, err := os.Stat(target); os.IsNotExist(err) { + if err = os.MkdirAll(target, 0o755); err != nil { + return fmt.Errorf("error creating mount point directory %s: %w", target, err) + } + } + + return nil +} diff --git a/internal/pkg/mount/mount_test.go b/internal/pkg/mount/mount_test.go new file mode 100644 index 0000000..63e55c9 --- /dev/null +++ b/internal/pkg/mount/mount_test.go @@ -0,0 +1,120 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package mount_test + +import ( + "log" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/siderolabs/go-blockdevice/blockdevice/loopback" + "github.com/stretchr/testify/suite" + "golang.org/x/sys/unix" + + "github.com/aenix-io/talm/internal/pkg/mount" + "github.com/siderolabs/talos/pkg/makefs" +) + +// Some tests in this package cannot be run under buildkit, as buildkit doesn't propagate partition devices +// like /dev/loopXpY into the sandbox. To run the tests on your local computer, do the following: +// +// go test -exec sudo -v --count 1 github.com/aenix-io/talm/internal/pkg/mount + +type manifestSuite struct { + suite.Suite + + disk *os.File + loopbackDevice *os.File +} + +const ( + diskSize = 4 * 1024 * 1024 * 1024 // 4 GiB +) + +func TestManifestSuite(t *testing.T) { + suite.Run(t, new(manifestSuite)) +} + +func (suite *manifestSuite) SetupTest() { + suite.skipIfNotRoot() + + var err error + + suite.disk, err = os.CreateTemp("", "talos") + suite.Require().NoError(err) + + suite.Require().NoError(suite.disk.Truncate(diskSize)) + + suite.loopbackDevice, err = loopback.NextLoopDevice() + suite.Require().NoError(err) + + suite.T().Logf("Using %s", suite.loopbackDevice.Name()) + + suite.Require().NoError(loopback.Loop(suite.loopbackDevice, suite.disk)) + + suite.Require().NoError(loopback.LoopSetReadWrite(suite.loopbackDevice)) +} + +func (suite *manifestSuite) TearDownTest() { + if suite.loopbackDevice != nil { + suite.Assert().NoError(loopback.Unloop(suite.loopbackDevice)) + } + + if suite.disk != nil { + suite.Assert().NoError(os.Remove(suite.disk.Name())) + suite.Assert().NoError(suite.disk.Close()) + } +} + +func (suite *manifestSuite) skipIfNotRoot() { + if os.Getuid() != 0 { + suite.T().Skip("can't run the test as non-root") + } +} + +func (suite *manifestSuite) skipUnderBuildkit() { + hostname, _ := os.Hostname() //nolint:errcheck + + if hostname == "buildkitsandbox" { + suite.T().Skip("test not supported under buildkit as partition devices are not propagated from /dev") + } +} + +func (suite *manifestSuite) TestCleanCorrupedXFSFileSystem() { + suite.skipUnderBuildkit() + + tempDir := suite.T().TempDir() + + mountDir := filepath.Join(tempDir, "var") + + suite.Assert().NoError(os.MkdirAll(mountDir, 0o700)) + suite.Require().NoError(makefs.XFS(suite.loopbackDevice.Name())) + + logger := log.New(os.Stderr, "", log.LstdFlags) + + mountpoint := mount.NewMountPoint(suite.loopbackDevice.Name(), mountDir, "xfs", unix.MS_NOATIME, "", mount.WithLogger(logger)) + + suite.Assert().NoError(mountpoint.Mount()) + + defer func() { + suite.Assert().NoError(mountpoint.Unmount()) + }() + + suite.Assert().NoError(mountpoint.Unmount()) + + // // now corrupt the disk + cmd := exec.Command("xfs_db", []string{ + "-x", + "-c blockget", + "-c blocktrash -s 512109 -n 1000", + suite.loopbackDevice.Name(), + }...) + + suite.Assert().NoError(cmd.Run()) + + suite.Assert().NoError(mountpoint.Mount()) +} diff --git a/internal/pkg/mount/options.go b/internal/pkg/mount/options.go new file mode 100644 index 0000000..882b3c1 --- /dev/null +++ b/internal/pkg/mount/options.go @@ -0,0 +1,143 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package mount + +import ( + "log" + + "github.com/aenix-io/talm/internal/pkg/encryption/helpers" + "github.com/siderolabs/talos/pkg/machinery/config/config" +) + +const ( + // ReadOnly is a flag for setting the mount point as readonly. + ReadOnly Flags = 1 << iota + // Shared is a flag for setting the mount point as shared. + Shared + // Resize indicates that a the partition for a given mount point should be + // resized to the maximum allowed. + Resize + // Overlay indicates that a the partition for a given mount point should be + // mounted using overlayfs. + Overlay + // SystemOverlay indicates that overlay directory should be created under tmpfs. + // + // SystemOverlay should be combined with Overlay option. + SystemOverlay + // ReadonlyOverlay indicates that a the partition for a given mount point should be + // mounted using multi-layer readonly overlay from multiple partitions given as sources. + ReadonlyOverlay + // SkipIfMounted is a flag for skipping mount if the mountpoint is already mounted. + SkipIfMounted + // SkipIfNoFilesystem is a flag for skipping formatting and mounting if the mountpoint has not filesystem. + SkipIfNoFilesystem + // SkipIfNoDevice is a flag for skipping errors when the device is not found. + SkipIfNoDevice +) + +// Flags is the mount flags. +type Flags uint + +// Options is the functional options struct. +type Options struct { + Loopback string + Prefix string + MountFlags Flags + PreMountHooks []Hook + PostUnmountHooks []Hook + Encryption config.Encryption + SystemInformationGetter helpers.SystemInformationGetter + Logger *log.Logger + ProjectQuota bool +} + +// Option is the functional option func. +type Option func(*Options) + +// Check checks if all provided flags are set. +func (f Flags) Check(flags Flags) bool { + return (f & flags) == flags +} + +// Intersects checks if at least one flag is set. +func (f Flags) Intersects(flags Flags) bool { + return (f & flags) != 0 +} + +// WithPrefix is a functional option for setting the mount point prefix. +func WithPrefix(o string) Option { + return func(args *Options) { + args.Prefix = o + } +} + +// WithFlags is a functional option to set up mount flags. +func WithFlags(flags Flags) Option { + return func(args *Options) { + args.MountFlags |= flags + } +} + +// WithPreMountHooks adds functions to be called before mounting the partition. +func WithPreMountHooks(hooks ...Hook) Option { + return func(args *Options) { + args.PreMountHooks = append(args.PreMountHooks, hooks...) + } +} + +// WithPostUnmountHooks adds functions to be called after unmounting the partition. +func WithPostUnmountHooks(hooks ...Hook) Option { + return func(args *Options) { + args.PostUnmountHooks = append(args.PostUnmountHooks, hooks...) + } +} + +// WithEncryptionConfig partition encryption configuration. +func WithEncryptionConfig(cfg config.Encryption) Option { + return func(args *Options) { + args.Encryption = cfg + } +} + +// WithLogger sets the logger. +func WithLogger(logger *log.Logger) Option { + return func(args *Options) { + args.Logger = logger + } +} + +// WithProjectQuota enables project quota mount option. +func WithProjectQuota(enable bool) Option { + return func(args *Options) { + args.ProjectQuota = enable + } +} + +// WithSystemInformationGetter the function to get system information on the node. +func WithSystemInformationGetter(getter helpers.SystemInformationGetter) Option { + return func(args *Options) { + args.SystemInformationGetter = getter + } +} + +// Hook represents pre/post mount hook. +type Hook func(p *Point) error + +// NewDefaultOptions initializes a Options struct with default values. +func NewDefaultOptions(setters ...Option) *Options { + opts := &Options{ + Loopback: "", + Prefix: "", + MountFlags: 0, + PreMountHooks: []Hook{}, + PostUnmountHooks: []Hook{}, + } + + for _, setter := range setters { + setter(opts) + } + + return opts +} diff --git a/internal/pkg/mount/overlay.go b/internal/pkg/mount/overlay.go new file mode 100644 index 0000000..19c8dd9 --- /dev/null +++ b/internal/pkg/mount/overlay.go @@ -0,0 +1,24 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package mount + +import ( + "golang.org/x/sys/unix" + + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// OverlayMountPoints returns the mountpoints required to boot the system. +// These mountpoints are used as overlays on top of the read only rootfs. +func OverlayMountPoints() (mountpoints *Points, err error) { + mountpoints = NewMountPoints() + + for _, target := range constants.Overlays { + mountpoint := NewMountPoint("", target, "", unix.MS_I_VERSION, "", WithFlags(Overlay)) + mountpoints.Set(target, mountpoint) + } + + return mountpoints, nil +} diff --git a/internal/pkg/mount/pseudo.go b/internal/pkg/mount/pseudo.go new file mode 100644 index 0000000..23da458 --- /dev/null +++ b/internal/pkg/mount/pseudo.go @@ -0,0 +1,44 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package mount + +import ( + "os" + + "golang.org/x/sys/unix" + + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// PseudoMountPoints returns the mountpoints required to boot the system. +func PseudoMountPoints() (mountpoints *Points, err error) { + pseudo := NewMountPoints() + pseudo.Set("dev", NewMountPoint("devtmpfs", "/dev", "devtmpfs", unix.MS_NOSUID, "mode=0755")) + pseudo.Set("proc", NewMountPoint("proc", "/proc", "proc", unix.MS_NOSUID|unix.MS_NOEXEC|unix.MS_NODEV, "")) + pseudo.Set("sys", NewMountPoint("sysfs", "/sys", "sysfs", 0, "")) + pseudo.Set("run", NewMountPoint("tmpfs", "/run", "tmpfs", unix.MS_NOSUID|unix.MS_NOEXEC|unix.MS_RELATIME, "mode=755")) + pseudo.Set("system", NewMountPoint("tmpfs", "/system", "tmpfs", 0, "mode=755")) + pseudo.Set("tmp", NewMountPoint("tmpfs", "/tmp", "tmpfs", unix.MS_NOSUID|unix.MS_NOEXEC|unix.MS_NODEV, "size=64M,mode=755")) + + return pseudo, nil +} + +// PseudoSubMountPoints returns the mountpoints required to boot the system. +func PseudoSubMountPoints() (mountpoints *Points, err error) { + pseudo := NewMountPoints() + pseudo.Set("devshm", NewMountPoint("tmpfs", "/dev/shm", "tmpfs", unix.MS_NOSUID|unix.MS_NOEXEC|unix.MS_NODEV|unix.MS_RELATIME, "")) + pseudo.Set("devpts", NewMountPoint("devpts", "/dev/pts", "devpts", unix.MS_NOSUID|unix.MS_NOEXEC, "ptmxmode=000,mode=620,gid=5")) + pseudo.Set("hugetlb", NewMountPoint("hugetlbfs", "/dev/hugepages", "hugetlbfs", unix.MS_NOSUID|unix.MS_NODEV, "")) + pseudo.Set("securityfs", NewMountPoint("securityfs", "/sys/kernel/security", "securityfs", unix.MS_NOSUID|unix.MS_NOEXEC|unix.MS_NODEV|unix.MS_RELATIME, "")) + + if _, err := os.Stat(constants.EFIVarsMountPoint); err == nil { + // mount EFI vars if they exist + pseudo.Set("efivars", NewMountPoint("efivarfs", constants.EFIVarsMountPoint, "efivarfs", unix.MS_NOSUID|unix.MS_NOEXEC|unix.MS_NODEV|unix.MS_RELATIME|unix.MS_RDONLY, "", + WithFlags(SkipIfNoDevice), + )) + } + + return pseudo, nil +} diff --git a/internal/pkg/mount/squashfs.go b/internal/pkg/mount/squashfs.go new file mode 100644 index 0000000..25ebfcc --- /dev/null +++ b/internal/pkg/mount/squashfs.go @@ -0,0 +1,27 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package mount + +import ( + "github.com/freddierice/go-losetup/v2" + "golang.org/x/sys/unix" + + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// SquashfsMountPoints returns the mountpoints required to boot the system. +func SquashfsMountPoints(prefix string) (mountpoints *Points, err error) { + var dev losetup.Device + + dev, err = losetup.Attach("/"+constants.RootfsAsset, 0, true) + if err != nil { + return nil, err + } + + squashfs := NewMountPoints() + squashfs.Set("squashfs", NewMountPoint(dev.Path(), "/", "squashfs", unix.MS_RDONLY|unix.MS_I_VERSION, "", WithPrefix(prefix), WithFlags(ReadOnly|Shared))) + + return squashfs, nil +} diff --git a/internal/pkg/mount/switchroot/switchroot.go b/internal/pkg/mount/switchroot/switchroot.go new file mode 100644 index 0000000..cf7e3f9 --- /dev/null +++ b/internal/pkg/mount/switchroot/switchroot.go @@ -0,0 +1,170 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package switchroot + +import ( + "fmt" + "log" + "os" + "path/filepath" + + "github.com/siderolabs/go-debug" + "golang.org/x/sys/unix" + + "github.com/aenix-io/talm/internal/pkg/mount" + "github.com/aenix-io/talm/internal/pkg/secureboot" + "github.com/aenix-io/talm/internal/pkg/secureboot/tpm2" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// Paths preserved in the initramfs. +var preservedPaths = map[string]struct{}{ + constants.ExtensionsConfigFile: {}, + constants.FirmwarePath: {}, + constants.SDStubDynamicInitrdPath: {}, +} + +// Switch moves the rootfs to a specified directory. See +// https://github.com/karelzak/util-linux/blob/master/sys-utils/switch_root.c. +func Switch(prefix string, mountpoints *mount.Points) (err error) { + log.Println("moving mounts to the new rootfs") + + if err = mount.Move(mountpoints, prefix); err != nil { + return err + } + + log.Printf("changing working directory into %s", prefix) + + if err = unix.Chdir(prefix); err != nil { + return fmt.Errorf("error changing working directory to %s: %w", prefix, err) + } + + var old *os.File + + if old, err = os.Open("/"); err != nil { + return fmt.Errorf("error opening /: %w", err) + } + + //nolint:errcheck + defer old.Close() + + log.Printf("moving %s to /", prefix) + + if err = unix.Mount(prefix, "/", "", unix.MS_MOVE, ""); err != nil { + return fmt.Errorf("error moving /: %w", err) + } + + log.Println("changing root directory") + + if err = unix.Chroot("."); err != nil { + return fmt.Errorf("error chroot: %w", err) + } + + log.Println("cleaning up initramfs") + + if _, err = recursiveDelete(int(old.Fd()), "/"); err != nil { + return fmt.Errorf("error deleting initramfs: %w", err) + } + + // extend PCR 11 with leave-initrd + if err = tpm2.PCRExtent(secureboot.UKIPCR, []byte(secureboot.LeaveInitrd)); err != nil { + return fmt.Errorf("failed to extend PCR %d with leave-initrd: %v", secureboot.UKIPCR, err) + } + + // Note that /sbin/init is machined. We call it init since this is the + // convention. + log.Println("executing /sbin/init") + + envv := []string{ + constants.TcellMinimizeEnvironment, + } + + if debug.RaceEnabled { + envv = append(envv, "GORACE=halt_on_error=1") + + log.Printf("race detection enabled with halt_on_error=1") + } + + if err = unix.Exec("/sbin/init", []string{"/sbin/init"}, envv); err != nil { + return fmt.Errorf("error executing /sbin/init: %w", err) + } + + return nil +} + +func recursiveDelete(fd int, path string) (preserved bool, err error) { + parentDev, err := getDev(fd) + if err != nil { + return false, err + } + + dir := os.NewFile(uintptr(fd), "__ignored__") + //nolint:errcheck + defer dir.Close() + + names, err := dir.Readdirnames(-1) + if err != nil { + return false, err + } + + preserved = false + + for _, name := range names { + p, err := recusiveDeleteInner(fd, parentDev, name, filepath.Join(path, name)) + if err != nil { + return false, err + } + + preserved = preserved || p + } + + return preserved, nil +} + +func recusiveDeleteInner(parentFd int, parentDev uint64, childName, path string) (preserved bool, err error) { + if _, preserved = preservedPaths[path]; preserved { + return preserved, nil + } + + childFd, err := unix.Openat(parentFd, childName, unix.O_DIRECTORY|unix.O_NOFOLLOW, unix.O_RDWR) + if err != nil { + return false, unix.Unlinkat(parentFd, childName, 0) + } + + //nolint:errcheck + defer unix.Close(childFd) + + var childFdDev uint64 + + if childFdDev, err = getDev(childFd); err != nil { + return false, err + } else if childFdDev != parentDev { + return false, nil + } + + preserved, err = recursiveDelete(childFd, path) + if err != nil { + return false, err + } + + if preserved { + // some child paths got preserved, skip unlinking the parent + return preserved, nil + } + + err = unix.Unlinkat(parentFd, childName, unix.AT_REMOVEDIR) + + return false, err +} + +func getDev(fd int) (dev uint64, err error) { + var stat unix.Stat_t + + if err := unix.Fstat(fd, &stat); err != nil { + return 0, err + } + + return stat.Dev, nil +} diff --git a/internal/pkg/mount/switchroot/switchroot_test.go b/internal/pkg/mount/switchroot/switchroot_test.go new file mode 100644 index 0000000..94dca42 --- /dev/null +++ b/internal/pkg/mount/switchroot/switchroot_test.go @@ -0,0 +1,14 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package switchroot_test + +import "testing" + +func TestEmpty(t *testing.T) { + // added for accurate coverage estimation + // + // please remove it once any unit-test is added + // for this package +} diff --git a/internal/pkg/mount/system.go b/internal/pkg/mount/system.go new file mode 100644 index 0000000..5ca34a7 --- /dev/null +++ b/internal/pkg/mount/system.go @@ -0,0 +1,299 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package mount + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "sync" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/maps" + "github.com/siderolabs/go-blockdevice/blockdevice" + "github.com/siderolabs/go-blockdevice/blockdevice/filesystem" + "golang.org/x/sys/unix" + + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime" + "github.com/aenix-io/talm/internal/app/machined/pkg/runtime/disk" + "github.com/aenix-io/talm/internal/pkg/encryption" + "github.com/aenix-io/talm/internal/pkg/partition" + "github.com/siderolabs/talos/pkg/machinery/constants" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +var ( + mountpoints = map[string]*Point{} + mountpointsMutex sync.RWMutex +) + +// SystemMountPointsForDevice returns the mountpoints required to boot the system. +// This function is called exclusively during installations ( both image +// creation and bare metall installs ). This is why we want to look up +// device by specified disk as well as why we don't want to grow any +// filesystems. +func SystemMountPointsForDevice(ctx context.Context, devpath string, opts ...Option) (mountpoints *Points, err error) { + mountpoints = NewMountPoints() + + bd, err := blockdevice.Open(devpath) + if err != nil { + return nil, err + } + + defer bd.Close() //nolint:errcheck + + for _, name := range []string{constants.EphemeralPartitionLabel, constants.BootPartitionLabel, constants.EFIPartitionLabel, constants.StatePartitionLabel} { + mountpoint, err := SystemMountPointForLabel(ctx, bd, name, opts...) + if err != nil { + return nil, err + } + + mountpoints.Set(name, mountpoint) + } + + return mountpoints, nil +} + +// SystemMountPointForLabel returns a mount point for the specified device and label. +// +//nolint:gocyclo +func SystemMountPointForLabel(ctx context.Context, device *blockdevice.BlockDevice, label string, opts ...Option) (mountpoint *Point, err error) { + var target string + + switch label { + case constants.EphemeralPartitionLabel: + target = constants.EphemeralMountPoint + case constants.BootPartitionLabel: + target = constants.BootMountPoint + case constants.EFIPartitionLabel: + target = constants.EFIMountPoint + case constants.StatePartitionLabel: + target = constants.StateMountPoint + default: + return nil, fmt.Errorf("unknown label: %q", label) + } + + part, err := device.GetPartition(label) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } + + if part == nil { + // A boot partitition is not required. + if label == constants.BootPartitionLabel { + return nil, nil + } + + return nil, fmt.Errorf("failed to find device with label %s: %w", label, err) + } + + fsType, err := part.Filesystem() + if err != nil { + return nil, err + } + + partPath, err := part.Path() + if err != nil { + return nil, err + } + + o := NewDefaultOptions(opts...) + + preMountHooks := []Hook{} + + if o.Encryption != nil { + encryptionHandler, err := encryption.NewHandler( + device, + part, + o.Encryption, + o.SystemInformationGetter, + ) + if err != nil { + return nil, err + } + + preMountHooks = append(preMountHooks, + func(p *Point) error { + var ( + err error + path string + ) + + if path, err = encryptionHandler.Open(ctx); err != nil { + return err + } + + p.source = path + + return nil + }, + ) + + opts = append(opts, + WithPostUnmountHooks( + func(p *Point) error { + return encryptionHandler.Close() + }, + ), + ) + } + + // Format the partition if it does not have any filesystem + preMountHooks = append(preMountHooks, func(p *Point) error { + sb, err := filesystem.Probe(p.source) + if err != nil { + return err + } + + p.fstype = "" + + // skip formatting the partition if filesystem is detected + // and assign proper fs type to the mountpoint + if sb != nil && sb.Type() != filesystem.Unknown { + p.fstype = sb.Type() + + return nil + } + + opts := partition.NewFormatOptions(part.Name) + if opts == nil { + return fmt.Errorf("failed to determine format options for partition label %s", part.Name) + } + + if !o.MountFlags.Check(SkipIfNoFilesystem) { + p.fstype = opts.FileSystemType + + return partition.Format(p.source, opts, log.Printf) + } + + return nil + }) + + opts = append(opts, WithPreMountHooks(preMountHooks...)) + + mountpoint = NewMountPoint(partPath, target, fsType, unix.MS_NOATIME, "", opts...) + + return mountpoint, nil +} + +// SystemPartitionMount mounts a system partition by the label. +// +//nolint:gocyclo +func SystemPartitionMount(ctx context.Context, r runtime.Runtime, logger *log.Logger, label string, opts ...Option) (err error) { + device := r.State().Machine().Disk(disk.WithPartitionLabel(label)) + if device == nil { + return fmt.Errorf("failed to find device with partition labeled %s", label) + } + + if r.Config() != nil && r.Config().Machine() != nil { + encryptionConfig := r.Config().Machine().SystemDiskEncryption().Get(label) + + if encryptionConfig != nil { + opts = append(opts, + WithEncryptionConfig(encryptionConfig), + WithSystemInformationGetter(r.GetSystemInformation), + ) + } + } + + opts = append(opts, WithLogger(logger)) + + mountpoint, err := SystemMountPointForLabel(ctx, device.BlockDevice, label, opts...) + if err != nil { + return err + } + + if mountpoint == nil { + return fmt.Errorf("no mountpoints for label %q", label) + } + + var skipMount bool + + if skipMount, err = mountMountpoint(mountpoint); err != nil { + return err + } + + if skipMount { + if logger != nil { + logger.Printf("mount skipped") + } + + return + } + + o := NewDefaultOptions(opts...) + encrypted := o.Encryption != nil + + // record mount as the resource + mountStatus := runtimeres.NewMountStatus(v1alpha1.NamespaceName, label) + mountStatus.TypedSpec().Source = mountpoint.Source() + mountStatus.TypedSpec().Target = mountpoint.Target() + mountStatus.TypedSpec().FilesystemType = mountpoint.Fstype() + mountStatus.TypedSpec().Encrypted = encrypted + + if encrypted { + encryptionProviders := make(map[string]struct{}) + + for _, cfg := range o.Encryption.Keys() { + switch { + case cfg.Static() != nil: + encryptionProviders[cfg.Static().String()] = struct{}{} + case cfg.NodeID() != nil: + encryptionProviders[cfg.NodeID().String()] = struct{}{} + case cfg.KMS() != nil: + encryptionProviders[cfg.KMS().String()] = struct{}{} + case cfg.TPM() != nil: + encryptionProviders[cfg.TPM().String()] = struct{}{} + } + } + + mountStatus.TypedSpec().EncryptionProviders = maps.Keys(encryptionProviders) + } + + // ignore the error if the MountStatus already exists, as many mounts are silently skipped with the flag SkipIfMounted + if err = r.State().V1Alpha2().Resources().Create(context.Background(), mountStatus); err != nil && !state.IsConflictError(err) { + return fmt.Errorf("error creating mount status resource: %w", err) + } + + mountpointsMutex.Lock() + defer mountpointsMutex.Unlock() + + mountpoints[label] = mountpoint + + return nil +} + +// SystemPartitionUnmount unmounts a system partition by the label. +func SystemPartitionUnmount(r runtime.Runtime, logger *log.Logger, label string) (err error) { + mountpointsMutex.RLock() + mountpoint, ok := mountpoints[label] + mountpointsMutex.RUnlock() + + if !ok { + if logger != nil { + logger.Printf("unmount skipped") + } + + return nil + } + + err = mountpoint.Unmount() + if err != nil { + return err + } + + if err = r.State().V1Alpha2().Resources().Destroy(context.Background(), runtimeres.NewMountStatus(v1alpha1.NamespaceName, label).Metadata()); err != nil { + return fmt.Errorf("error destroying mount status resource: %w", err) + } + + mountpointsMutex.Lock() + delete(mountpoints, label) + mountpointsMutex.Unlock() + + return nil +} diff --git a/internal/pkg/mount/unmount.go b/internal/pkg/mount/unmount.go new file mode 100644 index 0000000..e877ada --- /dev/null +++ b/internal/pkg/mount/unmount.go @@ -0,0 +1,77 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package mount + +import ( + "context" + "fmt" + "log" + "time" + + "golang.org/x/sys/unix" +) + +func unmountLoop(ctx context.Context, logger *log.Logger, target string, flags int, timeout time.Duration, extraMessage string) (bool, error) { + errCh := make(chan error, 1) + + go func() { + errCh <- unix.Unmount(target, flags) + }() + + start := time.Now() + + progessTicker := time.NewTicker(timeout / 5) + defer progessTicker.Stop() + +unmountLoop: + for { + select { + case <-ctx.Done(): + return true, ctx.Err() + case err := <-errCh: + return true, err + case <-progessTicker.C: + timeLeft := timeout - time.Since(start) + + if timeLeft <= 0 { + break unmountLoop + } + + if logger != nil { + logger.Printf("unmounting %s%s is taking longer than expected, still waiting for %s", target, extraMessage, timeLeft) + } + } + } + + return false, nil +} + +// SafeUnmount unmounts the target path, first without force, then with force if the first attempt fails. +// +// It makes sure that unmounting has a finite operation timeout. +func SafeUnmount(ctx context.Context, logger *log.Logger, target string) error { + const ( + unmountTimeout = 90 * time.Second + unmountForceTimeout = 10 * time.Second + ) + + ok, err := unmountLoop(ctx, logger, target, 0, unmountTimeout, "") + + if ok { + return err + } + + if logger != nil { + logger.Printf("unmounting %s with force", target) + } + + ok, err = unmountLoop(ctx, logger, target, unix.MNT_FORCE, unmountTimeout, " with force flag") + + if ok { + return err + } + + return fmt.Errorf("unmounting %s with force flag timed out", target) +} diff --git a/internal/pkg/partition/constants.go b/internal/pkg/partition/constants.go new file mode 100644 index 0000000..5345371 --- /dev/null +++ b/internal/pkg/partition/constants.go @@ -0,0 +1,42 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package partition + +// Type in partition table. +type Type = string + +// GPT partition types. +// +// TODO: should be moved into the blockdevice library. +const ( + EFISystemPartition Type = "C12A7328-F81F-11D2-BA4B-00A0C93EC93B" + BIOSBootPartition Type = "21686148-6449-6E6F-744E-656564454649" + LinuxFilesystemData Type = "0FC63DAF-8483-4772-8E79-3D69D8477DE4" +) + +// FileSystemType is used to format partitions. +type FileSystemType = string + +// Filesystem types. +const ( + FilesystemTypeNone FileSystemType = "none" + FilesystemTypeXFS FileSystemType = "xfs" + FilesystemTypeVFAT FileSystemType = "vfat" +) + +// Partition default sizes. +const ( + MiB = 1024 * 1024 + + EFISize = 100 * MiB + BIOSGrubSize = 1 * MiB + BootSize = 1000 * MiB + // EFIUKISize is the size of the EFI partition when UKI is enabled. + // With UKI all assets are stored in the EFI partition. + // This is the size of the old EFISize + BIOSGrubSize + BootSize. + EFIUKISize = EFISize + BIOSGrubSize + BootSize + MetaSize = 1 * MiB + StateSize = 100 * MiB +) diff --git a/internal/pkg/partition/format.go b/internal/pkg/partition/format.go new file mode 100644 index 0000000..d3e7636 --- /dev/null +++ b/internal/pkg/partition/format.go @@ -0,0 +1,103 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package partition provides common utils for system partition format. +package partition + +import ( + "fmt" + + "github.com/siderolabs/go-blockdevice/blockdevice" + + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/makefs" +) + +// FormatOptions contains format parameters. +type FormatOptions struct { + Label string + FileSystemType FileSystemType + Force bool +} + +// NewFormatOptions creates a new format options. +func NewFormatOptions(label string) *FormatOptions { + return systemPartitionsFormatOptions(label) +} + +// Format zeroes the device and formats it using filesystem type provided. +func Format(devname string, t *FormatOptions, printf func(string, ...any)) error { + if t.FileSystemType == FilesystemTypeNone { + return zeroPartition(devname, printf) + } + + opts := []makefs.Option{makefs.WithForce(t.Force), makefs.WithLabel(t.Label)} + printf("formatting the partition %q as %q with label %q\n", devname, t.FileSystemType, t.Label) + + switch t.FileSystemType { + case FilesystemTypeVFAT: + return makefs.VFAT(devname, opts...) + case FilesystemTypeXFS: + return makefs.XFS(devname, opts...) + default: + return fmt.Errorf("unsupported filesystem type: %q", t.FileSystemType) + } +} + +// zeroPartition fills the partition with zeroes. +func zeroPartition(devname string, printf func(string, ...any)) (err error) { + printf("zeroing out %q", devname) + + part, err := blockdevice.Open(devname, blockdevice.WithExclusiveLock(true)) + if err != nil { + return err + } + + defer part.Close() //nolint:errcheck + + return part.FastWipe() +} + +func systemPartitionsFormatOptions(label string) *FormatOptions { + switch label { + case constants.EFIPartitionLabel: + return &FormatOptions{ + Label: constants.EFIPartitionLabel, + FileSystemType: FilesystemTypeVFAT, + Force: true, + } + case constants.BIOSGrubPartitionLabel: + return &FormatOptions{ + Label: constants.BIOSGrubPartitionLabel, + FileSystemType: FilesystemTypeNone, + Force: true, + } + case constants.BootPartitionLabel: + return &FormatOptions{ + Label: constants.BootPartitionLabel, + FileSystemType: FilesystemTypeXFS, + Force: true, + } + case constants.MetaPartitionLabel: + return &FormatOptions{ + Label: constants.MetaPartitionLabel, + FileSystemType: FilesystemTypeNone, + Force: true, + } + case constants.StatePartitionLabel: + return &FormatOptions{ + Label: constants.StatePartitionLabel, + FileSystemType: FilesystemTypeXFS, + Force: true, + } + case constants.EphemeralPartitionLabel: + return &FormatOptions{ + Label: constants.EphemeralPartitionLabel, + FileSystemType: FilesystemTypeXFS, + Force: true, + } + default: + return nil + } +} diff --git a/internal/pkg/partition/format_test.go b/internal/pkg/partition/format_test.go new file mode 100644 index 0000000..d1bcc00 --- /dev/null +++ b/internal/pkg/partition/format_test.go @@ -0,0 +1,157 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package partition provides common utils for system partition format. +package partition_test + +import ( + "bytes" + "io" + "os" + "testing" + + "github.com/siderolabs/go-blockdevice/blockdevice" + "github.com/siderolabs/go-blockdevice/blockdevice/loopback" + "github.com/siderolabs/go-blockdevice/blockdevice/partition/gpt" + "github.com/stretchr/testify/suite" + + "github.com/aenix-io/talm/internal/pkg/partition" +) + +type manifestSuite struct { + suite.Suite + + disk *os.File + loopbackDevice *os.File +} + +const ( + diskSize = 10 * 1024 * 1024 // 10 MiB +) + +func TestManifestSuite(t *testing.T) { + suite.Run(t, new(manifestSuite)) +} + +func (suite *manifestSuite) SetupTest() { + suite.skipIfNotRoot() + + var err error + + suite.disk, err = os.CreateTemp("", "talos") + suite.Require().NoError(err) + + suite.Require().NoError(suite.disk.Truncate(diskSize)) + + suite.loopbackDevice, err = loopback.NextLoopDevice() + suite.Require().NoError(err) + + suite.T().Logf("Using %s", suite.loopbackDevice.Name()) + + suite.Require().NoError(loopback.Loop(suite.loopbackDevice, suite.disk)) + + suite.Require().NoError(loopback.LoopSetReadWrite(suite.loopbackDevice)) +} + +func (suite *manifestSuite) TearDownTest() { + if suite.loopbackDevice != nil { + suite.Assert().NoError(loopback.Unloop(suite.loopbackDevice)) + } + + if suite.disk != nil { + suite.Assert().NoError(os.Remove(suite.disk.Name())) + suite.Assert().NoError(suite.disk.Close()) + } +} + +func (suite *manifestSuite) skipIfNotRoot() { + if os.Getuid() != 0 { + suite.T().Skip("can't run the test as non-root") + } +} + +func (suite *manifestSuite) skipUnderBuildkit() { + hostname, _ := os.Hostname() //nolint:errcheck + + if hostname == "buildkitsandbox" { + suite.T().Skip("test not supported under buildkit as partition devices are not propagated from /dev") + } +} + +func (suite *manifestSuite) TestZeroPartition() { + suite.skipUnderBuildkit() + + bd, err := blockdevice.Open(suite.loopbackDevice.Name(), blockdevice.WithExclusiveLock(true)) + suite.Require().NoError(err) + + defer bd.Close() //nolint:errcheck + + pt, err := gpt.New(bd.Device(), gpt.WithMarkMBRBootable(false)) + suite.Require().NoError(err) + + // Create a partition table with a single partition. + _, err = pt.Add(0, gpt.WithMaximumSize(true), gpt.WithPartitionName("zerofill")) + suite.Require().NoError(err) + + suite.Require().NoError(pt.Write()) + suite.Require().NoError(bd.Close()) + + bd, err = blockdevice.Open(suite.loopbackDevice.Name(), blockdevice.WithExclusiveLock(true)) + suite.Require().NoError(err) + + defer bd.Close() //nolint:errcheck + + fills := bytes.NewBuffer(bytes.Repeat([]byte{1}, 10)) + + parts, err := bd.GetPartition("zerofill") + suite.Require().NoError(err) + + part, err := parts.Path() + suite.Require().NoError(err) + + // open the partition as read write + dst, err := os.OpenFile(part, os.O_WRONLY, 0o644) + suite.Require().NoError(err) + + defer dst.Close() //nolint:errcheck + + // Write some data to the partition. + _, err = io.Copy(dst, fills) + suite.Require().NoError(err) + + data, err := os.Open(part) + suite.Require().NoError(err) + + defer data.Close() //nolint:errcheck + + read := make([]byte, fills.Len()) + + _, err = data.Read(read) + suite.Require().NoError(err) + suite.Require().NoError(data.Close()) + + suite.Assert().True(bytes.Equal(fills.Bytes(), read)) + + suite.Require().NoError(bd.Close()) + + err = partition.Format(part, &partition.FormatOptions{ + FileSystemType: partition.FilesystemTypeNone, + }, suite.T().Logf) + suite.Require().NoError(err) + + // reading 10 times more than what we wrote should still return 0 since the partition is wiped + zerofills := bytes.NewBuffer(bytes.Repeat([]byte{0}, 100)) + + data, err = os.Open(part) + suite.Require().NoError(err) + + defer data.Close() //nolint:errcheck + + read = make([]byte, zerofills.Len()) + + _, err = data.Read(read) + suite.Require().NoError(err) + + suite.Assert().True(bytes.Equal(zerofills.Bytes(), read)) +} diff --git a/internal/pkg/partition/partition.go b/internal/pkg/partition/partition.go new file mode 100644 index 0000000..fcafa68 --- /dev/null +++ b/internal/pkg/partition/partition.go @@ -0,0 +1,121 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package partition provides common utils for system partition format. +package partition + +import ( + "github.com/dustin/go-humanize" + "github.com/siderolabs/go-blockdevice/blockdevice/partition/gpt" + + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// Options contains the options for creating a partition. +type Options struct { + PartitionLabel string + PartitionType Type + Size uint64 + LegacyBIOSBootable bool +} + +// NewPartitionOptions returns a new PartitionOptions. +func NewPartitionOptions(label string, uki bool) *Options { + return systemPartitionsPartitonOptions(label, uki) +} + +// Locate existing partition on the disk by label. +func Locate(pt *gpt.GPT, label string) (*gpt.Partition, error) { + for _, part := range pt.Partitions().Items() { + if part.Name == label { + return part, nil + } + } + + return nil, nil +} + +// Partition creates a new partition on the specified device. +// Returns the path to the newly created partition. +func Partition(pt *gpt.GPT, pos int, device string, partitionOpts Options, printf func(string, ...any)) (string, error) { + printf("partitioning %s - %s %q\n", device, partitionOpts.PartitionLabel, humanize.Bytes(partitionOpts.Size)) + + opts := []gpt.PartitionOption{ + gpt.WithPartitionType(partitionOpts.PartitionType), + gpt.WithPartitionName(partitionOpts.PartitionLabel), + } + + if partitionOpts.Size == 0 { + opts = append(opts, gpt.WithMaximumSize(true)) + } + + if partitionOpts.LegacyBIOSBootable { + opts = append(opts, gpt.WithLegacyBIOSBootableAttribute(true)) + } + + part, err := pt.InsertAt(pos, partitionOpts.Size, opts...) + if err != nil { + return "", err + } + + partitionName, err := part.Path() + if err != nil { + return "", err + } + + printf("created %s (%s) size %d blocks", partitionName, partitionOpts.PartitionLabel, part.Length()) + + return partitionName, nil +} + +func systemPartitionsPartitonOptions(label string, uki bool) *Options { + switch label { + case constants.EFIPartitionLabel: + partitionOptions := &Options{ + PartitionType: EFISystemPartition, + Size: EFISize, + } + + if uki { + partitionOptions.Size = EFIUKISize + } + + return partitionOptions + case constants.BIOSGrubPartitionLabel: + if uki { + panic("BIOS partition is not supported with UKI") + } + + return &Options{ + PartitionType: BIOSBootPartition, + Size: BIOSGrubSize, + } + case constants.BootPartitionLabel: + if uki { + panic("BOOT partition is not supported with UKI") + } + + return &Options{ + PartitionType: LinuxFilesystemData, + Size: BootSize, + } + case constants.MetaPartitionLabel: + return &Options{ + PartitionType: LinuxFilesystemData, + Size: MetaSize, + } + case constants.StatePartitionLabel: + return &Options{ + PartitionType: LinuxFilesystemData, + Size: StateSize, + } + case constants.EphemeralPartitionLabel: + return &Options{ + PartitionType: LinuxFilesystemData, + Size: 0, + } + default: + return nil + } +} diff --git a/internal/pkg/secureboot/database/database.go b/internal/pkg/secureboot/database/database.go new file mode 100644 index 0000000..b63995b --- /dev/null +++ b/internal/pkg/secureboot/database/database.go @@ -0,0 +1,62 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package database generates SecureBoot auto-enrollment database. +package database + +import ( + "crypto/sha256" + + "github.com/foxboron/go-uefi/efi" + "github.com/foxboron/go-uefi/efi/signature" + "github.com/foxboron/go-uefi/efi/util" + "github.com/google/uuid" + + "github.com/aenix-io/talm/internal/pkg/secureboot/pesign" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// Entry is a UEFI database entry. +type Entry struct { + Name string + Contents []byte +} + +// Generate generates a UEFI database to enroll the signing certificate. +// +// ref: https://blog.hansenpartnership.com/the-meaning-of-all-the-uefi-keys/ +func Generate(enrolledCertificate []byte, signer pesign.CertificateSigner) ([]Entry, error) { + // derive UUID from enrolled certificate + uuid := uuid.NewHash(sha256.New(), uuid.NameSpaceX500, enrolledCertificate, 4) + + efiGUID := util.StringToGUID(uuid.String()) + + // Create ESL + db := signature.NewSignatureDatabase() + if err := db.Append(signature.CERT_X509_GUID, *efiGUID, enrolledCertificate); err != nil { + return nil, err + } + + // Sign the ESL, but for each EFI variable + signedDB, err := efi.SignEFIVariable(signer.Signer(), signer.Certificate(), "db", db.Bytes()) + if err != nil { + return nil, err + } + + signedKEK, err := efi.SignEFIVariable(signer.Signer(), signer.Certificate(), "KEK", db.Bytes()) + if err != nil { + return nil, err + } + + signedPK, err := efi.SignEFIVariable(signer.Signer(), signer.Certificate(), "PK", db.Bytes()) + if err != nil { + return nil, err + } + + return []Entry{ + {Name: constants.SignatureKeyAsset, Contents: signedDB}, + {Name: constants.KeyExchangeKeyAsset, Contents: signedKEK}, + {Name: constants.PlatformKeyAsset, Contents: signedPK}, + }, nil +} diff --git a/internal/pkg/secureboot/measure/internal/pcr/bank_data.go b/internal/pkg/secureboot/measure/internal/pcr/bank_data.go new file mode 100644 index 0000000..209d15a --- /dev/null +++ b/internal/pkg/secureboot/measure/internal/pcr/bank_data.go @@ -0,0 +1,100 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package pcr + +import ( + "crypto" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "encoding/hex" + "fmt" + "os" + + "github.com/google/go-tpm/tpm2" + + "github.com/aenix-io/talm/internal/pkg/secureboot" + tpm2internal "github.com/aenix-io/talm/internal/pkg/secureboot/tpm2" +) + +// RSAKey is the input for the CalculateBankData function. +type RSAKey interface { + crypto.Signer + PublicRSAKey() *rsa.PublicKey +} + +// CalculateBankData calculates the PCR bank data for a given set of UKI file sections. +// +// This mimics the process happening happening in the TPM when the UKI is being loaded. +func CalculateBankData(pcrNumber int, alg tpm2.TPMAlgID, sectionData map[secureboot.Section]string, rsaKey RSAKey) ([]tpm2internal.BankData, error) { + // get fingerprint of public key + pubKeyFingerprint := sha256.Sum256(x509.MarshalPKCS1PublicKey(rsaKey.PublicRSAKey())) + + hashAlg, err := alg.Hash() + if err != nil { + return nil, err + } + + pcrSelector, err := tpm2internal.CreateSelector([]int{secureboot.UKIPCR}) + if err != nil { + return nil, fmt.Errorf("failed to create PCR selection: %v", err) + } + + pcrSelection := tpm2.TPMLPCRSelection{ + PCRSelections: []tpm2.TPMSPCRSelection{ + { + Hash: alg, + PCRSelect: pcrSelector, + }, + }, + } + + hashData := NewDigest(hashAlg) + + for _, section := range secureboot.OrderedSections() { + if file := sectionData[section]; file != "" { + hashData.Extend(append([]byte(section), 0)) + + sectionData, err := os.ReadFile(file) + if err != nil { + return nil, err + } + + hashData.Extend(sectionData) + } + } + + banks := make([]tpm2internal.BankData, 0) + + for _, phaseInfo := range secureboot.OrderedPhases() { + // extend always, but only calculate signature if requested + hashData.Extend([]byte(phaseInfo.Phase)) + + if !phaseInfo.CalculateSignature { + continue + } + + hash := hashData.Hash() + + policyPCR, err := tpm2internal.CalculatePolicy(hash, pcrSelection) + if err != nil { + return nil, err + } + + sigData, err := Sign(policyPCR, hashAlg, rsaKey) + if err != nil { + return nil, err + } + + banks = append(banks, tpm2internal.BankData{ + PCRs: []int{pcrNumber}, + PKFP: hex.EncodeToString(pubKeyFingerprint[:]), + Sig: sigData.SignatureBase64, + Pol: sigData.Digest, + }) + } + + return banks, nil +} diff --git a/internal/pkg/secureboot/measure/internal/pcr/bank_data_test.go b/internal/pkg/secureboot/measure/internal/pcr/bank_data_test.go new file mode 100644 index 0000000..7fe3710 --- /dev/null +++ b/internal/pkg/secureboot/measure/internal/pcr/bank_data_test.go @@ -0,0 +1,60 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package pcr_test + +import ( + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "os" + "testing" + + "github.com/google/go-tpm/tpm2" + "github.com/stretchr/testify/require" + + "github.com/aenix-io/talm/internal/pkg/secureboot" + "github.com/aenix-io/talm/internal/pkg/secureboot/measure/internal/pcr" + tpm2internal "github.com/aenix-io/talm/internal/pkg/secureboot/tpm2" +) + +type keyWrapper struct { + *rsa.PrivateKey +} + +func (k keyWrapper) PublicRSAKey() *rsa.PublicKey { + return &k.PrivateKey.PublicKey +} + +func TestCalculateBankData(t *testing.T) { + t.Parallel() + + pemKey, err := os.ReadFile("../../testdata/pcr-signing-key.pem") + require.NoError(t, err) + + block, _ := pem.Decode(pemKey) + require.NotNil(t, block) + + key, err := x509.ParsePKCS1PrivateKey(block.Bytes) + require.NoError(t, err) + + bankData, err := pcr.CalculateBankData(15, tpm2.TPMAlgSHA256, + map[secureboot.Section]string{ + secureboot.Initrd: "testdata/a", + secureboot.Linux: "testdata/b", + secureboot.DTB: "testdata/c", + }, + keyWrapper{key}) + require.NoError(t, err) + + require.Equal(t, + []tpm2internal.BankData{ + { + PCRs: []int{15}, + PKFP: "58f58f625bd8a8b6681e4b40688cf99b26419b6b2c5f6e14a2c7c67a3b0b1620", + Pol: "a1c9d366451c82969238eb82a5282f84b6a3d499e540430ecf792083155225dd", + Sig: "bO4F/T6bio7jLJFpi4GsJHjZDj+H5Pq4stjKA5WhkzBNCmE1gQECeOALUfNJ1RW/HKhwSp7KhGFqqnjyg/eXR3c0pVuYUKuAjZz9NMXS4dAQlSLxtNWMmlX3XDst/UKfxB6Z+m2KluJpF3KeAw03tP9lru6nfzaickOs1UL83IO5QgLkCHpUcSloZcya0xS9ETCNBd5gm8K+c9gc7+CmpFLTo1uTxbBK0Mea+3fn7GAZROHPMBLosvTM5D9vplsWIXXAXSaHr/sj5bxOIR+orCQZOdYY+/8ra4oFVXzHc9kkPP3A6mWzoADKryWWIVKPx/DGLi0ExT2fpCNdUoMOacvD+dqDqjVBhcOwoAZkNqve/W/poqaLlKyFTqlmGmv+08WavtShYmCURa4Mn3UFf49BTVkktxoQ+jTMroyit1uK/ppMSjaPwQp2Dd1pRCY4hcFfLwqryy1zRMT/XmZ2e91MYe40Pr9Tom2ZH0YAigDosBPuP6RHt7IypFIgary3louW1dqNLXW8p38Y91nYDKBWI9x0tVn5ufqtk5wkHnExTjUYkTWU98+p5J7urDIhLuX1mSi57Ekq02f+lVLMs85SHfmMfzZl7l7Xi4npYbW+5xHKiAxLnaXVJCHdW0xiAD0VTLer5Oe5nf7FrjSzS39rXoryKfcHFOIxRT1XQOA=", //nolint:lll + }, + }, bankData) +} diff --git a/internal/pkg/secureboot/measure/internal/pcr/extend.go b/internal/pkg/secureboot/measure/internal/pcr/extend.go new file mode 100644 index 0000000..75ee2c7 --- /dev/null +++ b/internal/pkg/secureboot/measure/internal/pcr/extend.go @@ -0,0 +1,49 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package pcr + +import ( + "crypto" +) + +// Digest implements the PCR extension algorithm. +// +// Each time `Extend` is called, the hash of the previous data is +// prepended to the hash of new data and hashed together. +// +// The initial hash value is all zeroes. +type Digest struct { + alg crypto.Hash + hash []byte +} + +// NewDigest creates a new Digest with the speified hash algorithm. +func NewDigest(alg crypto.Hash) *Digest { + return &Digest{ + alg: alg, + hash: make([]byte, alg.Size()), + } +} + +// Hash returns the current hash value. +func (d *Digest) Hash() []byte { + return d.hash +} + +// Extend extends the current hash with the specified data. +func (d *Digest) Extend(data []byte) { + // create hash of incoming data + hash := d.alg.New() + hash.Write(data) + hashSum := hash.Sum(nil) + + // extend hash with previous data and hashed incoming data + hash.Reset() + hash.Write(d.hash) + hash.Write(hashSum) + + // set sum as new hash + d.hash = hash.Sum(nil) +} diff --git a/internal/pkg/secureboot/measure/internal/pcr/extend_test.go b/internal/pkg/secureboot/measure/internal/pcr/extend_test.go new file mode 100644 index 0000000..d257d6d --- /dev/null +++ b/internal/pkg/secureboot/measure/internal/pcr/extend_test.go @@ -0,0 +1,36 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package pcr_test + +import ( + "crypto" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/aenix-io/talm/internal/pkg/secureboot/measure/internal/pcr" +) + +func TestExtend(t *testing.T) { + t.Parallel() + + hash := pcr.NewDigest(crypto.SHA256) + + assert.Equal(t, make([]byte, 32), hash.Hash()) + + hash.Extend([]byte("foo")) + + assert.Equal(t, + []byte{0x42, 0x48, 0x16, 0xd0, 0x20, 0xcf, 0x3d, 0x79, 0x3a, 0xc0, 0x21, 0xda, 0x47, 0x37, 0x9b, 0xdf, 0x60, 0x80, 0x80, 0xa8, 0x3e, 0xb9, 0x36, 0x4a, 0x7f, 0xbe, 0xb, 0xdf, 0xa8, 0x71, 0x11, 0xd7}, + hash.Hash(), + ) + + hash.Extend([]byte("bar")) + + assert.Equal(t, + []byte{0x63, 0x5c, 0x18, 0xb1, 0x5e, 0xf5, 0xc5, 0xd6, 0xc0, 0x20, 0xe7, 0x23, 0x39, 0xdd, 0xef, 0xd8, 0xb0, 0x5c, 0x4c, 0x4a, 0x44, 0xb3, 0x4e, 0xff, 0x8c, 0xef, 0x22, 0x6f, 0x89, 0x2, 0x77, 0x2}, + hash.Hash(), + ) +} diff --git a/internal/pkg/secureboot/measure/internal/pcr/sign.go b/internal/pkg/secureboot/measure/internal/pcr/sign.go new file mode 100644 index 0000000..75ad45e --- /dev/null +++ b/internal/pkg/secureboot/measure/internal/pcr/sign.go @@ -0,0 +1,37 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package pcr contains code that handles PCR operations. +package pcr + +import ( + "crypto" + "encoding/base64" + "encoding/hex" + "fmt" +) + +// Signature returns the hashed signature digest and base64 encoded signature. +type Signature struct { + Digest string + SignatureBase64 string +} + +// Sign the digest using specified hash and key. +func Sign(digest []byte, hash crypto.Hash, key crypto.Signer) (*Signature, error) { + digestToHash := hash.New() + digestToHash.Write(digest) + digestHashed := digestToHash.Sum(nil) + + // sign policy digest + signedData, err := key.Sign(nil, digestHashed, hash) + if err != nil { + return nil, fmt.Errorf("signing failed: %v", err) + } + + return &Signature{ + Digest: hex.EncodeToString(digest), + SignatureBase64: base64.StdEncoding.EncodeToString(signedData), + }, nil +} diff --git a/internal/pkg/secureboot/measure/internal/pcr/sign_test.go b/internal/pkg/secureboot/measure/internal/pcr/sign_test.go new file mode 100644 index 0000000..850264f --- /dev/null +++ b/internal/pkg/secureboot/measure/internal/pcr/sign_test.go @@ -0,0 +1,45 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package pcr_test + +import ( + "crypto" + "crypto/x509" + "encoding/hex" + "encoding/pem" + "os" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/aenix-io/talm/internal/pkg/secureboot/measure/internal/pcr" +) + +func TestSign(t *testing.T) { + t.Parallel() + + pemKey, err := os.ReadFile("../../testdata/pcr-signing-key.pem") + require.NoError(t, err) + + block, _ := pem.Decode(pemKey) + require.NotNil(t, block) + + key, err := x509.ParsePKCS1PrivateKey(block.Bytes) + require.NoError(t, err) + + digest, err := hex.DecodeString("e9590019f04a00029bb5ac512c3d3dfff0ec0e66418cfb5035e22313af891d81") + require.NoError(t, err) + + signature, err := pcr.Sign(digest, crypto.SHA256, key) + require.NoError(t, err) + + require.Equal(t, + &pcr.Signature{ + Digest: "e9590019f04a00029bb5ac512c3d3dfff0ec0e66418cfb5035e22313af891d81", + SignatureBase64: "Ylam12MOrPQs2m0AsHzRYBjZwYB1B5W0N4Qq62bNjiV4KgQVpwGTnIA0Rgmdaa1bTL+9+7oZ84H1xR0Q248Yd+2P1ZU5KaSysdoi3nlvotRYUq93HQeVjSLe1WUnoZ56EovP47tPuvLqIHmjPYq3V/EVLS6fD3+mXKZr/Q7sdlUjmGtYO5H0rV39C6Oq4Pwk9WJ4oRRKWwCp4KbxOujJ2ANqJl2QdJJA4WSle8+OML+SomelSDCjwt+s+T+0ZUhCY11Els1PtKO55ySU9N67m7wMIAy7aMwF6vbqyRajFDZN8ad7huhXDpwBGBMaEX5ajm2FseUj+h0EYbAm030FwduqZ9WlTMwp9KUx6dK2uOjckKgItBQfVXFoOo8dl4Al9PDktcmuytogI7o1OdzmJAcrb8BiPLLppmNsEgKR+5+poAsSA3Z0dcREiLbvKm10m7mXHGwRg84knZGSrsbHkD9I3ngeOM3JiPLGGCp4nYjBNzKP4jiygTEgEuZ2ueV9PikwlnM5qaDdByIH+0u3LAJubzN2XyI6TGugNRzdvKRIxtl5dSwRoIptiXInN81q6pw2i27YmzvR16tCTxXFRIcHjxpq5Q4KpVohbYhh4kHiWexbqJMpUPoLVEaw+m+Kh7gMvZlud67I6ldRIjDoy/LSdnsXcjpQFkNoF0ZKhX8=", //nolint:lll + }, + signature, + ) +} diff --git a/internal/pkg/secureboot/measure/internal/pcr/testdata/a b/internal/pkg/secureboot/measure/internal/pcr/testdata/a new file mode 100644 index 0000000..5d308e1 --- /dev/null +++ b/internal/pkg/secureboot/measure/internal/pcr/testdata/a @@ -0,0 +1 @@ +aaaa diff --git a/internal/pkg/secureboot/measure/internal/pcr/testdata/b b/internal/pkg/secureboot/measure/internal/pcr/testdata/b new file mode 100644 index 0000000..b433656 --- /dev/null +++ b/internal/pkg/secureboot/measure/internal/pcr/testdata/b @@ -0,0 +1 @@ +bbbb diff --git a/internal/pkg/secureboot/measure/internal/pcr/testdata/c b/internal/pkg/secureboot/measure/internal/pcr/testdata/c new file mode 100644 index 0000000..28924d0 --- /dev/null +++ b/internal/pkg/secureboot/measure/internal/pcr/testdata/c @@ -0,0 +1 @@ +cccc diff --git a/internal/pkg/secureboot/measure/measure.go b/internal/pkg/secureboot/measure/measure.go new file mode 100644 index 0000000..a484777 --- /dev/null +++ b/internal/pkg/secureboot/measure/measure.go @@ -0,0 +1,60 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package measure contains Go implementation of 'systemd-measure' command. +// +// This implements TPM PCR emulation, UKI signature measurement, signing the measured values. +package measure + +import ( + "crypto" + "crypto/rsa" + + "github.com/google/go-tpm/tpm2" + + "github.com/aenix-io/talm/internal/pkg/secureboot" + "github.com/aenix-io/talm/internal/pkg/secureboot/measure/internal/pcr" + tpm2internal "github.com/aenix-io/talm/internal/pkg/secureboot/tpm2" +) + +// SectionsData holds a map of Section to file path to the corresponding section. +type SectionsData map[secureboot.Section]string + +// RSAKey is the input for the CalculateBankData function. +type RSAKey interface { + crypto.Signer + PublicRSAKey() *rsa.PublicKey +} + +// GenerateSignedPCR generates the PCR signed data for a given set of UKI file sections. +func GenerateSignedPCR(sectionsData SectionsData, rsaKey RSAKey) (*tpm2internal.PCRData, error) { + data := &tpm2internal.PCRData{} + + for _, algo := range []struct { + alg tpm2.TPMAlgID + bankDataSetter *[]tpm2internal.BankData + }{ + { + alg: tpm2.TPMAlgSHA256, + bankDataSetter: &data.SHA256, + }, + { + alg: tpm2.TPMAlgSHA384, + bankDataSetter: &data.SHA384, + }, + { + alg: tpm2.TPMAlgSHA512, + bankDataSetter: &data.SHA512, + }, + } { + bankData, err := pcr.CalculateBankData(secureboot.UKIPCR, algo.alg, sectionsData, rsaKey) + if err != nil { + return nil, err + } + + *algo.bankDataSetter = bankData + } + + return data, nil +} diff --git a/internal/pkg/secureboot/measure/measure_test.go b/internal/pkg/secureboot/measure/measure_test.go new file mode 100644 index 0000000..77a2012 --- /dev/null +++ b/internal/pkg/secureboot/measure/measure_test.go @@ -0,0 +1,143 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package measure_test + +import ( + "bytes" + "crypto/rsa" + "crypto/x509" + "encoding/json" + "encoding/pem" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/aenix-io/talm/internal/pkg/secureboot" + "github.com/aenix-io/talm/internal/pkg/secureboot/measure" +) + +const ( + // ExpectedSignatureJSON is pre-calculated signature. + //nolint:lll + ExpectedSignatureJSON = `{"sha256":[{"pcrs":[11],"pkfp":"58f58f625bd8a8b6681e4b40688cf99b26419b6b2c5f6e14a2c7c67a3b0b1620","pol":"88b9f03a2edc4960894b5fa460169282749aff60d0b491ca71a2bde937f63f28","sig":"JBOKtBN2YAvugIqpPK6aulzCOAscdFojCqkawL9xa43tmhCj2wnfbrskym1QLlozYyiD6/6HpEYB91J71vWtTQTNUiJP910+qvcPkREHUdiPdzBTEDXDklvB9ACXfzcEl1lEhqlYfjJ9nxTmapWjOOL/iV/GgamkAtEwT91HE0ITf6ESedSt0+0T9lft6W10i7nO6WPJfQ2WDAZ6tGZUJLpvf36ZPloxRKXCCGkhtI1FYetT2ZJHgamrm6rGWa+IkOwFBydBjdSg20HDqNPFqzPlXLagx1SxXIPlH9Et+ebiFBRhevgD2GDL52wY3XS88G0AuYAlFrkrmx72wNZ7v/f16srXGOUSLwIzVE10z88nS+TYEUugKtIwrj1h165thMwcQONPSj/tL2OKcmM6jpAtIpApZUYOKip+sS5cKBloqhNbVuu6lzexi7THYWpn+Bzlj5zu7VO6yDLb8FaCxgTdoCiR8/q5yzXa8+LBJX1wfdvktDYGdmO0h/nGQYjOfKCSV2WCZLxIiNPedWkUX7hMIggTaaYiaxwYJHIZNKS0N3H1FGkSSFnYNJFHyW0S7BSi8dF9l0VlgXh6HrwSbMyv1avNQiL/pP7CwyA/dyYdsON6neWFRoYfaKRZdrQRKvH4VhjdB4CGLgxB7Pom8BQc8Vi68sAsKLKsNXY8GSs="}],"sha384":[{"pcrs":[11],"pkfp":"58f58f625bd8a8b6681e4b40688cf99b26419b6b2c5f6e14a2c7c67a3b0b1620","pol":"532380d1bfae365cca029f2e4a57601d528a346bfd9feae1e1f3f994260fb56d","sig":"qKilaBxg/goIw3TXfd+kIMqfA1EOcWngv+iH/xUliHLyxzTVHxx4vF4+wVJUNmS7vj10ffU876cySqyV9S9EFSPn3EEHLKza5n4skf++IZj3UvBXRwXTKKnE+iLDvj3r8maEorOD5J9rlv7nZZ95HESNxy0J2lXa0ik+vGKaKqySkyqFF2xM22M9Ko3iGr2bTSK8faGcorORNe/Hjr3GeJWm66Ea1EJvlIR9HjwDdeY+RMF/WEtSYMVkRM6ZYp/j2epTOTWsjUqYECxcWxjydrXm+Sf1ZDrHGURBDVHxpfTDWfrdpQ33tyappjwSnwH2zVLqThG6DEUqXmvn0/uYnSjLlKjRVuJyMotrcHSymot2oPQ/olfReOUAOkQR2TqiFEN5NeYAEghMDzwyYv1Rr5ahOJPcPMpEPnztUem4+kaAlOW4aAhuKcrgoiNle4G68CE0d3SEL3JvF3pyg9zSqai59vN8dd7Gw3vDYETPsOFWL7UJKb0DCCDCEFBN9kSvRScjjsz18NfJ0C5df7VhJC+P0WukxAPxQPvaK99HnD4JPpa9P0lEqubb+bvzZ+YuR6zblLVEt+4AISt/4Td0F8ylyMaz4+Rj+zm+VCCrNz3zqIjbLMQ/k1v6rRd/Yjre8DyQjSh7vD2eZr0BopUmimv5gGXvCranbHaq2jSXI5w="}],"sha512":[{"pcrs":[11],"pkfp":"58f58f625bd8a8b6681e4b40688cf99b26419b6b2c5f6e14a2c7c67a3b0b1620","pol":"2ea15476710c3d5e2ac70b6cee68a6a7619c91f69cf2d1387ad567f3fb5076fe","sig":"2SuhxgPM5hjb//gqud9Lomsjss/UTHz8B6wpcyUQHCroGHxd7OoDfzUd3nuXOf7BRbzDRiwo/sX4cR1CDDer9R4ut6QpQ6BHjPdcPxnH06BZvNTDubmU3gWJA49vZDD3IBnEWp/cMyGtVVbYuidy790EP+D9nBs247Cx4B1cepVcO3T5UHrrXst7oOt7OvTC+j3tPrGedhuEmRI0g+ms3VBeX4fcTFkqsjJmEI8kEz+otc388o9m/Edd5jxj42nf+Fu3RKH6GaLicQOr8Eat2RfMwfnv6ZxLhlwolwb/56DGgOJ0yuNEUuLvNru60KBXPjpTurUVSvYqL7qxrz7LqOOm+TMOgUjDV481GFwma6RvVOaAxlbPKwFwJSeMGqyhZBCOXRUraFfD0vJNwhB+BCaEmgXpFLjhX7IsUwieLq73vy9zRp4x+/fyhj2rnAfNku04JovBy7PYI/1nfjmdBoSvCyPmLCu5pznU61aMgpm1O1VgNdsSEqCu/pej724WGJ8tMjlhPHDgzsPnKu68m/0K+Kh5oYlP4uy04NtHXsif37bhdLmg09NcPmI7UEl+h1CGLNT2jf9tklvtDqVfzLyXnnbIoJFasO542Xx9RwlvAWYgs95jZIMYSpE+8o+Eg+rMyW7H/ej2M87HfQltv+R2Jb5Fm99D5kmrdnoH1vc="}]}` +) + +type rsaWrapper struct { + *rsa.PrivateKey +} + +func (w rsaWrapper) PublicRSAKey() *rsa.PublicKey { + return &w.PrivateKey.PublicKey +} + +func loadRSAKey(path string) (measure.RSAKey, error) { + keyData, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + // convert private key to rsa.PrivateKey + rsaPrivateKeyBlock, _ := pem.Decode(keyData) + if rsaPrivateKeyBlock == nil { + return nil, err + } + + rsaKey, err := x509.ParsePKCS1PrivateKey(rsaPrivateKeyBlock.Bytes) + if err != nil { + return nil, fmt.Errorf("parse private key failed: %v", err) + } + + return rsaWrapper{rsaKey}, nil +} + +func TestMeasureMatchesExpectedOutput(t *testing.T) { + expectedSignatureHex := ExpectedSignatureJSON + + if _, err := exec.LookPath("systemd-measure"); err == nil { + t.Log("systemd-measure binary found, using it to get expected signature") + expectedSignatureHex = getSignatureUsingSDMeasure(t) + } + + tmpDir := t.TempDir() + + sectionsData := measure.SectionsData{} + + // create temporary files with the ordered section name and data as the section name + for _, section := range secureboot.OrderedSections() { + sectionFile := filepath.Join(tmpDir, string(section)) + + if err := os.WriteFile(sectionFile, []byte(section), 0o644); err != nil { + t.Fatal(err) + } + + sectionsData[section] = sectionFile + } + + rsaKey, err := loadRSAKey("testdata/pcr-signing-key.pem") + if err != nil { + t.Fatal(err) + } + + pcrData, err := measure.GenerateSignedPCR(sectionsData, rsaKey) + if err != nil { + t.Fatal(err) + } + + pcrDataJSON, err := json.Marshal(&pcrData) + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, expectedSignatureHex, string(pcrDataJSON)) +} + +func getSignatureUsingSDMeasure(t *testing.T) string { + tmpDir := t.TempDir() + + sdMeasureArgs := make([]string, len(secureboot.OrderedSections())) + + // create temporary files with the ordered section name and data as the section name + for i, section := range secureboot.OrderedSections() { + sectionFile := filepath.Join(tmpDir, string(section)) + + if err := os.WriteFile(sectionFile, []byte(section), 0o644); err != nil { + t.Error(err) + } + + sdMeasureArgs[i] = fmt.Sprintf("--%s=%s", strings.TrimPrefix(string(section), "."), sectionFile) + } + + var signature bytes.Buffer + + sdCmd := exec.Command( + "systemd-measure", + append([]string{ + "sign", + "--private-key", + "testdata/pcr-signing-key.pem", + "--bank=sha256", + "--bank=sha384", + "--bank=sha512", + "--phase=enter-initrd:leave-initrd:enter-machined", + "--json=short", + }, + sdMeasureArgs..., + )...) + + sdCmd.Stdout = &signature + + if err := sdCmd.Run(); err != nil { + t.Error(err) + } + + s := bytes.TrimSpace(signature.Bytes()) + + return string(s) +} diff --git a/internal/pkg/secureboot/measure/testdata/pcr-signing-key.pem b/internal/pkg/secureboot/measure/testdata/pcr-signing-key.pem new file mode 100644 index 0000000..26f0efd --- /dev/null +++ b/internal/pkg/secureboot/measure/testdata/pcr-signing-key.pem @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKQIBAAKCAgEA7qhAkdtZxqkIP79DDGin9eaJBeNlJsClJTcbaXbNfk2QJGT3 +lqo9ErXQQftwYWLGo+kVd8puhnHGPkLW9apT1/ZmUJEFwxV5xws0RllGVPhUga+1 +oubHUqhEiy707S4RrUEMk/o9wqmtnl2hY5FxMeQn2o7xrpcNhm8FtHpvQrT0MsbC +1cS1ytZH/hwPy/QIB9bx+ugOha6wtQBnpgix1BhHC/NwDIYPg+ONpQSCu9gkXVtL +GlKfmjscUANQtBuVKa5NflrjkHw7NAdKYdKpMnmzr0yu6Tn/2oNmUiJAwHz0BXpf +b4Yn8n/IoKJQ5Tv1g6d30wxxpBd0lbwSe9MLRchIDJ5aFRybyRxaPGT17U3yEVzb +V78kIFtocaqkc1ise8remZ0wxHzuolbTZD6oswt7C9jMLvfMAQ7JtENXrpDM//Xz +dRLzyTWKOjhG0YmKKRY6cIrPkugM0PHGCE3RMSH1FmPMrWWBNAMwS0Zba0Wm1b7v +dw5fKeE8txH+IpA3IaE9AytYk0ig98ZgmXmBV0sgxmJ/94scEF+sDg65LIkSEJMz +f6q30UghbJJoP7eKOoDX9KBrR+POEsWm/EcU5jTEQTHMU+qKtj5KD6TUn8R8yi4w +CnyZ7uJLUqm8Ou8MzEZWbrsrbMvrewPDAHn0QQvb2tDtBgn6oH192jpkzckCAwEA +AQKCAgEAkiWcrPU7i+lVMNxqLb4lJPOQ83cmKU4Nk7WkZrgm7PKIk5D1AWGs1rla +GB3m2uxHIncI+3uOpWwk71m1E2nDwFuWmj3E3otXMKnO0Em5RS1xap10SJa0dwyu +NOGDgX8Vuhg8oJ28lmmb9X/25edZ/yhts2yX2ceMs8dnIfdcDOiNJk8LXycAAH+q +RJVgoxAEnvBk7LaQthKdCap+znFCnNRlJY9lDXZHKAgAZI5XlLquwjC21B7GuAb8 +to7hK/o8JPMlZ3w3IPLCuoDAbxk3Hb7jZzU5Y39uC50t2pw5NOcP9A7VRJFOAzV3 +Yc8kZMyL85xpR2e2a7slXNB4LTW3D0zy/fSO63R9cLNrlp+I9p7xDgz1mylo8FoW +T1TyNAWo/gIa7r43Ufp/C0lrWSd5gkz2nMVWiFl8M1lpx7zDSZk8U1sKzFZswmFQ +h5On7kxo14gUdzogb1hrJuEI9Ke52kRb4YMm094LFI/BWQ89QF0NXJ6CSkb2MyWc +f0kyfEMRUbmHi/EfpQlKK2uhOsxXdhZN2VP4nl1Yg0xxv0cLSMcP7DdJPso5VSC/ ++wF8ni7+GMEDtntMEGjuXjq/zypyjptaRKpw4iRqxydUqa0C1PovzDoUDn7eKJBv +2p9evDG8zWenZ7g/VYWy4ZtpwVa3SAXeeuLmdllphPp6n6uweFECggEBAPiSb2IY +O33JRmxQeqrc6cmcv1l4AbedSr3F7X4DC+HZCkG27bZlzMBqbAQichC/pPhx544S +CBxB7e1Qsqjw8LLNea0sQsRbVXMSQpwBqjJ2g4mCN4S/hPQHSeESODPz/OaZrQq5 +iVSckrAlbgFs8HepOLxTQQVVp97m+vrnwlh7SZj/CXXi5QcZ4tHPNTFLLtwCgBUA +X0Ausn9hpeHrud8imzQuPXJRgBMaF0BYRUxj0fjp3sty2ZfjiPjhTrdSCWOiJUSb +onDb6kyYwN/hBEt7JNbSbC7viOM72/TwJQqDn4Bw8C7E0kiy3ZpQiCdyOYWnyDZP +SyDhAM4nHLNYss0CggEBAPXJ+DMmLCPkE5OOElxGU+JIPM28+5jhKn+HcswAS5VV +6tp1m8gIrmgwTpXA9aGi9BjPHzo6y3s1sNYHp3lGr5nSdgQaYuLvBR56FE0RXVP7 +bUHiN//R7a8QTkyYCqdrQNxLAzEARBNZaMIhMPfPXDDevFZWunAxIDXxC9IVRpjS +VjmKwqLuk66uInko4qEn34NiU25x+VpDOjz0fia58VTlL9WrYy0w/QANVlg1JKRq +wjjx4kWnmCQ1qQeagB+xqJtrEc/GK4z6qY/OC0pKZA26t6sXhkmn0zQshmwSAZem +/QtEW3lDmsfAlvEibJUSshXz3ygWz4v21nWocF4gXu0CggEBAPgwXf43686gVSx4 +/sHzaYrgcz5F0JEhACuToJmdORP7vX33xEnGQzYsDEXkjreiYnmeYXE9F9P/EC1P +0dNVHz+oYcFC3DdqaltG9DMIhoN0ScnWttBY2cs+K8oKgwt8phspfdmjfzd4Tg6K +kNfjigYwdHG1Psqwx7iMMDStiyMFlmqo2y1Vqw/4DL0ogxgA1XzfEjvl7zUKazc8 +rIBy+VeOGiFzue6W6aYo+uZIPIkVceVyvf2tYw2BJpY5gHsR8kYE8+kY7Ix7R+nK +62meJsem4RWNbG9AxBD/B5P84z8oRO3d1jMcWko0LYeSuR+JsV1+NS3k5kKh5kfw +TXvVKFECggEACMp4fhvXaFE4AgcK0RIS3f0Hb7Raq1UiV/1YNcOs8GJqS/X45Gar +Fj7kEKceIfHaGSkPTN3deUKqWH1dmBDXJwFIB02KS+OQo05qe3crh11uwvR8XEH9 +5k0G/+ZQOzyyzS5BpvcDeE2yWX8maTaZbYYJ5myjrm+TX1qHubPZGo4rV1OHMpyl +25GO2haERI9Qhzp1EXYyHPBanOOBv5DW+NpZo6LFoVAnPGE9vVnpPZgz6iV8mlEs +N99TdFoqSvfnt+dUc8H6vMgaWHJeJQIUIgmTmCL3QpsmCq+s/yCFvg7S7hw7yVKJ +rqtMusMobwyEIhTe3mgydCcX9I1Zt4Qg4QKCAQAnnNjOMflzEz+TvGNPUyVyzXef +xRs2XBDipX/vPK7L3VvgiXGQuOAq4PYj7NdYHfIiVmFeqz7eY9gMmQyoscMwVPCm +X+FuHepHXfcxUjQwnpFa/lAqpNxspgU09CQXW5hUkIdWblZePtRiGKvlo51CdzU5 +KlOylrIXl6opsApdrgSCduBtuR9uz2Cn3NiaG0Xe2x67VphclNaW+RATU4bUJEMn +9aO8k+wp8CjlJ1xjSrhqIIBHGMmouyK5J0r3S3vRlTCLEjGpGq8+Z9shVkfuDk6N +HOB2KZ1LN68eOYb4eZFKE+l2nGylFsHlOtkagX1IxtVoW+a5vDenGBNd+gC9 +-----END RSA PRIVATE KEY----- diff --git a/internal/pkg/secureboot/pesign/pesign.go b/internal/pkg/secureboot/pesign/pesign.go new file mode 100644 index 0000000..6478da3 --- /dev/null +++ b/internal/pkg/secureboot/pesign/pesign.go @@ -0,0 +1,47 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package pesign implements the PE (portable executable) signing. +package pesign + +import ( + "crypto" + "crypto/x509" + "os" + + "github.com/foxboron/go-uefi/efi" +) + +// Signer sigs PE (portable executable) files. +type Signer struct { + provider CertificateSigner +} + +// CertificateSigner is a provider of the certificate and the signer. +type CertificateSigner interface { + Signer() crypto.Signer + Certificate() *x509.Certificate +} + +// NewSigner creates a new Signer. +func NewSigner(provider CertificateSigner) (*Signer, error) { + return &Signer{ + provider: provider, + }, nil +} + +// Sign signs the input file and writes the output to the output file. +func (s *Signer) Sign(input, output string) error { + unsigned, err := os.ReadFile(input) + if err != nil { + return err + } + + signed, err := efi.SignEFIExecutable(s.provider.Signer(), s.provider.Certificate(), unsigned) + if err != nil { + return err + } + + return os.WriteFile(output, signed, 0o600) +} diff --git a/internal/pkg/secureboot/pesign/pesign_test.go b/internal/pkg/secureboot/pesign/pesign_test.go new file mode 100644 index 0000000..af03402 --- /dev/null +++ b/internal/pkg/secureboot/pesign/pesign_test.go @@ -0,0 +1,63 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package pesign_test + +import ( + "crypto" + stdx509 "crypto/x509" + "os" + "path/filepath" + "testing" + "time" + + "github.com/siderolabs/crypto/x509" + "github.com/stretchr/testify/require" + + "github.com/aenix-io/talm/internal/pkg/secureboot/pesign" + "github.com/siderolabs/talos/pkg/machinery/config/generate/secrets" +) + +type certificateProvider struct { + *x509.CertificateAuthority +} + +func (c *certificateProvider) Signer() crypto.Signer { + return c.CertificateAuthority.Key.(crypto.Signer) +} + +func (c *certificateProvider) Certificate() *stdx509.Certificate { + return c.CertificateAuthority.Crt +} + +func TestSign(t *testing.T) { + currentTime := time.Now() + + opts := []x509.Option{ + x509.RSA(true), + x509.Bits(2048), + x509.CommonName("test-sign"), + x509.NotAfter(currentTime.Add(secrets.CAValidityTime)), + x509.NotBefore(currentTime), + x509.Organization("test-sign"), + } + + tmpDir := t.TempDir() + + signingKey, err := x509.NewSelfSignedCertificateAuthority(opts...) + require.NoError(t, err) + + signer, err := pesign.NewSigner(&certificateProvider{signingKey}) + require.NoError(t, err) + + require.NoError(t, signer.Sign("testdata/systemd-bootx64.efi", filepath.Join(tmpDir, "boot.efi"))) + + unsigned, err := os.Stat("testdata/systemd-bootx64.efi") + require.NoError(t, err) + + signed, err := os.Stat(filepath.Join(tmpDir, "boot.efi")) + require.NoError(t, err) + + require.Greater(t, signed.Size(), unsigned.Size()) +} diff --git a/internal/pkg/secureboot/pesign/testdata/systemd-bootx64.efi b/internal/pkg/secureboot/pesign/testdata/systemd-bootx64.efi new file mode 100644 index 0000000000000000000000000000000000000000..216f5963d31e4f0f70afe60099ccc8d361311bb7 GIT binary patch literal 80896 zcmeEvd3;nw)^>Lm(m=uu2oeRQ(MA(lOw>4u5bcHpZ$n2To4BJ$#B~_b4RK@%c3Qby zufvSsHZHghugi!F0$~dY5J1I%3n(h!0@XAK3PJ#(zvrpCodlfs{k^~MukVj9O5b~{ zPMtb+>YP)jPSvejdZXk2`p=7!WhD;BosXnB=GMG@X9CdyN57=@Ne+kgi^H+4NTt?X zgKH-5bDd6Jcn=uXBM)PObJ~t&#dB zpuU}T?3DFC z&oV2ioA|4m?+o6jTjwl*A=8cTb!&>p|D2%d$kB~%{fXX)V$PQrj0ojsy+Be6D+3q8^DYf*Dvt=RWZmLB2B zMuW3JWzZ`!Bp*`iNOO3vJ8yH|`V#(M8!sDAKGuu_*U|PU+73h)sAgZzW^Z{8IpNj0 zH{Ig9nSBHjHpX=W?GTG#Ep-!j+$ej#-gmukly9Wp_yo+; zjT6Il<1>lIxXcqkkGddcp({~$2Kjz7>>~RxjdT<>;vz@?Xw|fsC3*2d+KS5tc&iVGvUD>!;B3;3?kra6laSmmzR9pg#fhyin#)t9U@Xs^Gj(o3#!HvUC#+EH%bKIC5jjC3!!x zCj!O}c9!oJ->tse>dE@wd?WnE@4B(YW`3sl+wbvspfDCNqPjNtBR#S{NF4|myAHk@ zFm_YZT;liRq@6t!UY&Up5!R0!K1?^ScIibYr#l>zsCgHjiC(){_1gYa*=yNtz^!@D zf~m%F-S}<37V4^7o!3;MUtl(LYhsnSi)=BHb#stQH_y;59YQ@66!fVZebX)Vz7bS1 z9ZGgX5io+70vZ9{K+J4Qkj=YX3J=NH+682kq)Fi!ugEC60A!p+GOU0n%eEjvw8rXk zNq?dp;0aiFdUD0cf0M8QFcQp#E~Vsciw)JoQ#|eqLx-qIC>`)*ml`{ka4Z#{Z$$wJ z%vN$ITQ3Afp^vEuxi!p({H_{;x~u=;a8L#6!BO$iF~!URy?1JOfV4uDq%FK!H@1Sg zP_2#e=wr|*vupU+$sm5>!Sf=UgT2GYPK3N(?Zd;?sNZ8(@+SOqlJTeT$&;Z`v7r=> zOB>Qof}zO7gR3D8_wm>j4G>C)qlQ1RLgWP7#OsW%v3~IlMq?}m4Bc#`MK*`dh5YU~ z2AE9T^x{n#Zi2Xp-pyXJ@`EWbEAhPAXy)HZFf4$XS}QpS*?_(I?I^*zWE_`G9|P#> zTHQ+h4JdT2q>)8+W0SaHwgc8lhjd8;0)u%GoCMvdLn7!nM;bh1lOEo=T@N3SH(Sg&TD3>P@;eFplA?E2^t2FYZ5ZOz=V)gu-oGPC)n(bap8bY-^f-tYwPoBp@kUhB8Nk1OUd2X zMX=p|<51ohb#6ca&&3S^&2Ofq|+XjbERM z-h$_l7!F{QZ91eAddUxyu^}bc9w?X&9D#}fA zI%4PP#upSGnt**B0dt#z8K+>bI1g%g=93tPr(frQ1q`L-?Ta1)6H1MS5@Vwt*%lgr z!TF{piqZyfWg{3P?gt;xWuIf<)s6k)0hmEFZ4ETxRstgFCYAIa2((6cGJQ*^WPW2S zO;YA{x)HsHdg6|cr0LAf+ec$DX$65+LJ1rqas<}bKz(aODh#nSJ}BxEM`&`5?S7*{ z?8WU851>(@5`ANBUhTm*btB}JCi-XH7>O>Jni(+e&uu7?j(B;o5gMa6luV!r5Z6eX zGX)OCMA!%NISbY&6_C)MV{_T))#w}1YP3+J_q?g6jo$35>k_v1a}A9CwGE0}`M<|` zDSE0c82$Df5(^^Hv1+vZ#3^cQ{`EQCn-jc-Z|e+nKLVix}d;e@|s7Ju8Mo0TqM zg3#*PUA<-3od#W%rA_;iLcrW(2}e z!;a7k8)13sVjt^E>D=P4ibKNUL#k8Yr=?E7d<&g2jx2=8Bs9o(J$y8}Pf}TC?46JK zEHq>wpmn3#^E4jObT%qr!71cjws-{s4}sP6S+dqjj^2Qgyv=B&C^S;OKaxB5w_trbU*~YwUlY zC%fVaV80L8^A79Q91rbGEBr>#Q^u~z1j478g(20g8;^OY3>=*yb$gNUoM%20Km?3J z`knw7h?^0fTv@AIR{4=EPlV=Rm+4LK%UNxa11;4ar7M_MyTp+KM|}xr+H5Ask#!nd zpgSl7_55X+31bBP90g|<%trrD)~#h8_|1+Q7b~#xx1K<6*P$zsc_aHb;3?1-H#_cN z^muybtxQGB3-nDwFRa(XgOI>>^yvn&hJdU*n22JJmlE&wuS}5SU(rkma&@A})fRPz zYKYMWP@(}kDXyeK7;A~5EC4qi&mVZI&}cD&#Iai!kp*yn^fc{}H+c_SUR3_cLE@=w zBfgaOPpf`hX0nygkxbuWQ0h?PW94rEaKKomWWl+IwXnOz&$U3JSkPIjk67>vt>JDl zAJ^C&5TtGhQm}vWHe-?^ZELHU$`NVdGkT;`l=G)Zr=#dGTtr{sml$Dj#7#ER;kZWM zB8Qu`fiPL(3JI`ZT!O1^DqVsi?!H7S;y1)KW3|+`Qw(Hc_`+<)Z+y-PjdZIycQsSr zKw&Z#OxGdwt`)t>AS7gpVSmLPy1gwJn1pF(><12cm>qA^svY2409}(QqbeXn1XxBYu^%>tm)wGZaQ#Aq{=St)~zx)VZ;-u)wSYP zft(+%3m7|L9X1^g6mAG;gEs^$Z#*w!*z8%l>)7qqtjavo_LLd;Cj(9}Xe(-2}@mcHc_%oLy< zQy{MR(fL!B@FA>r7`C#Z-e_9MQC^Dn;zA!{OTve^n=m;Frl*2Ad=`eb%@*K|gS4Gq z6n7w!CvIsZ4%R}Q>51{MB=0arFmf5nd7nqmTh9W}G7QzQodN4r&jK=&R+c|-cml8$ z_^G5dtde<;DUC^3M_iTpddxG=&Yy2z6|ToTTkYo^_Ep%|1NL<>u2jETd0pmNhpX82 z2?!zKy)Y-%O;D>x_G#5!@l+n3TGkI@12cNKM9 z0}5CH%)NATj%No**R2U<;yR22lKkUxWLu^fi>hiR#2k-oH0onxLEjxsX!FGVlru6P zgGwbEJvv~%YV$k5;Ys!hA}0a!Ej2iyZ=g1IFS&Iy(Mti?JaDhNPRe1mU2XpABd84l z87Qqw{Q9%>6?%{_3-+>VoA)0ggCL`$1Eu6Mf@0;*L0PdvmFX#UTKu(jsVckFm}q*d zQa7;F%vPncL~-jD}W3@=L zpXcLQjT65~8C_;8JH_Y+rdate|HhHu-;AxS3`Ta^3=Fs)S$IDw%#_n?DHRkA(KzS(QuCoLQr%&o$yqA!R$K$tW1dENwbPN$ zXS+6NcSQr9;CMj|Y|S=5kRW1y1#*m%GP>`5>Tz$`_1=qm`7j%Z#I5!NWkXzNKUDFd zSU$k(5m(rE6GX0kH(p#|-;EL7?Yq&U6Yiqf7)mLDsd&hXi|>WJY*Uxd)#Vd)S*tE9 z)n&1|yo(FPa1E}KH)4nEc{Os;|G!B2|DYId2S>M4M!phEr&?wKY!(K;0VqSEsie1o zU=;_F>;N410{)=CZ>CXK6X*%a7}&;dLM=|df zUtlB$@tD_fI4gh@oVN$8yk$Mmz!QM0>y-u%eWFDsNXyMY8U?9@o-;Hh052Gdb)Tn9 z+^_mpixf&o5GsA3-gqhxKg5BjBW@25;=q$ekZ?T*o}v42i^+KzwT+fHM#FFAn^BcM zcc@B_d?p73)a^F}1`hnU854QoUPCxFZCM~eKz0B$mH{XL&W)D($Gb878IL%y3@e<8 zI77`W4w6=W2aNmRTp=#epJ1uvY+mZScyD*}bREGHP>R_`!yq{ir|Uh2Y#5C>KjV;? zIVcxmC&ht{nT|P`G9@dhJ94Q zb_Xo%e=}^Ug3XFP$PvTJ-vjrCs7JSe!?$b(KUaafqtYR>@=I;(IW6GXV%JXzO7^44 zpd>rmO<~Wm!H*#wgy(=5uUG`|hCv13I7AlnCDDB)?am0+!*)FStz zMmXBV#-M`e3s5xV&$7#TRXMUz7bkv9H1uv(YYft(mndS(ZSal?9FCo5UnZ%wN}TYb zP+AI8mLGlH#Oan+B4|?oLjIBoKsI`ab)Wly=m6C8SWD2ZK^R)@^JEP%rnsA;4II&| z{3N@o*U?EFU>4W|-FWdA*=@sN2{GTqSOP7^B9O5P*sT1|;HF`Cxa}eG>!C6ZHT}?> zSQ_T%n3zlm7-OMSQ#=#IrAUt62(00U3LILsg=Ro}A*>TrR7;|Vn~#uIzy!lFE%VuN zEl6%Ko$Z+*I>C8}jUf3LaVm19vj}sSO)H8pKYD5VJtRe1sqe%>Fb)yB$x?UsKv_WY z#-GqfFZ`-FG(ps2=*)}9P7F!jR)UY$9xG z_g>Pbb_}ED8LMeDTiH}Qp2gte=O;1k4=cOZhi1kq`USCbP7Dsxe)a$i*Y9eo6^u*4 zO#T@JFTntuw>dmL2mv{v%|R?XJT2(Jf@fjfR2KqB(Kk`;w03f49DR{8VorxcIVlTP z387#XKot^xO5^mcxDTc>&elmcFV)Sd4&n66U}RzJQn{|y25iK@2uZ0xXk@BX1o6w= zGI05;9mer9PfvcB>{b9gYaxzb&@NqdazSK8lh!>>7LLSe}S?XlrZ zd6A2>H*)y}6D_}SQU;y!_6^r(?5(7ln5iF~uV!20<>S?GHr`S$JFf7r!GgCuoXhem zCuq24;@SMOTgUukodijQW8k8MPsBq>dHa0k;A~79t39tu{-pNtpc<|pn_gWX>qVTs zS_X#v=3r=!@)>Xsr>0uE`QpTblDtDCPlgRpGMB5*@aR*&Ln@e^SdRu))efu6lRvPS zy-w}&o72WG1_4#@<=lEUV{?Rg0xNJsD`Z!;pGz7Mdnh1qsV-acf$f;=a#FDoE>T7ibh;2~FR?f7Gh?z#i(>2a3_wGRb(6buAL%W|8~w5jf;$>6Y&#>&yg?TwI*WthkKs zDwuIIhUHCr^un(zAlqy8oX>PwAm>XR6LHtX3>A{d*~;ROQv6^^5p-C^PJ}sVXMoaQ zxVpma$GQjB42GB~J7gGlbvXzrtk<=HTN%6EqMLJ+XoTzA=!Ksl{P{I}0p$kde9daz zsxpwn=IVu8D>~`M*LqGJDYR|`+is?t%@_%EJQn~!g4GqKS|gGXiLjj~EUXmPVHLBE z%by@JTgTtK;;L>@iYyf>X_0dSM*7M2y;C z{u*@(e@-JL6yZL2@VP_G^qpu>uzh($%5`qkTNm4%x7lxEaTshsZs8E~GOWAMl$w55 zxtZQk57#B>g-sP5%CU~yw%kY`3~CW=R_HqKAAcK<4{)AUTi$D1pKxuj@_|OmLtJy;tUEWC`x;U@(=$jp18!+*Np(D!uJo<@ z5+t~)H*3{@2ZQOdWd#Z`=x7h$hJGyVKh33uN8*mG%=Ims03anLfb@c2P0WDQHY}$4 z2DFkLOvuu6urB+3z=>%_AtR9g5zR;ag_@eQc7GFE;^G?Lg|q%?It5@gNrIo650?OM?K z!7MdYl`;~gfLNBA$WoF9mbw(B5=JVr56h#_9-46x+lFNkC};kAyJ7J#%@ZdlqJH!` zvndWTl8KOte5SgP{%qYE9WRZ*hJ7LXM#&cZPzX+3%({b}s?wxymlvlj6mx>yK-+cz zg=CZbIkiMr;NK(yIja~L)ph53-MLE4_>k479y~`40PMMDyUxk*cw~ObOeg||POo>U zH6qOksneYslOvdpe3w<}6cx)Ij_?VmZY`76 zUO(%wK4YVIpym*5!37RJi{W_iuZjx-)$g5&)NpM&9zwa1ebPV4#mYm?xyi_|^2h(k zOHsF7n5E=*bU5OlXw@|+6Mc{c5pa&Yg&WQT&i)!s)?rBSOT$3@y?ZWyJ3&5Ivu0!p9~}9){m5C-=Z_Z>{>@unv>+aNka=PVXWiGNM?L1t|kp z>EWh7X^$?2f!WCkEFLyBE(dpsgCtgxix;aQ-JewD!687VUp57@y_{g8Uda1hU4 z!oJOv#RE`VuBtIO8odfbUKhMwUwmI{N zZ{L#djE%L6&0WQ%N=c*yM_IiPK7sb|3|6qE9ShM26(W+Ph2La_s04oAN3fQ$?y?X7 zGFhvt-iaD9>R-K;n7N}2L&%TOGM2RLkQ_*yk@f(9}aAR;*>}*j1@Vxc@hQV15G!=;FNT6#9L<0)RMdQIY(#dPKM8KO@ zOG;L=c57O8-eK`7E3hG=v9>B*PTpfbl*}qGlU~*|z^j%?Mp%8;fVZF@w5F^xy#-d; z$?&lx?a?<;KHQX|RW~A=1wb$~T^iAq5DO|O9M@H>a5f&rckdETnK@7@x4=}Z&~sQ| zL5Oxu)&d`JjwhFrg=+z?W~H3ujTg7wf(-;}5XNqVDz0h#8~SQ|9V<2H=U31EkRYdvdp1H-iNCsBTst6TMfWhmCHeG!_iCE`w*sDH)xff`Id9m`}&SSEbE_Ir3?9SHU^a z{!!<1Zz?f94Psf4Yn5=g;vbba6#GuP9dHq3+NnIIJ2?g(gAe?jwt$>Mxd2?unp!ohp{gy^@2TLZUB8 zSa(t_D=J(UykD<@DDn?RzcF|#iC~qix(Zi@jUo}0v`I|}pQKMrc!wEwSJZR`JgpjQ ztddtd;96K$F(`UHyGyk(BpX_K*`sHuncNHy}KT&+Shx(Wt*dK^0d9%+Mw=-n&|A~6&2b2NGiRRh`Ic7xBq-YO2fj0XSG;(rB!qWg(;3?{m*~+Bg@|hRclyxnyKvqgH(lg%I5Q6ZY>V zlee8@a#xZ(qmHeS&!5QWJq|~zdY(n9@0Qr_#BDRW*9a$P35~(KK}L!!y`Iqh0nLD= zQW##T%ux@Kz^+wNGzzGUP|=5PVBbNmW|GdZKy)TiOz1i52$d-RBZD-6lol*mA=NQ` zC35V{n^NL%b8C%^CofY_MpQhz6>+3>rE3lM_ z$zuTHOW~f1&gDKcIFd8Pa-R__#m%Z1w(!;ma_ZqS2b^o7n6AKZd=_xl2S*dpEnrb; zb9?Q^EJe@2N`5J>;4pC-6N2spIoSW$%u;T&NC$MRkwJpQFmP(82hWuyPr8F$PAj?j zOP0je;N(Or0*O`xO5`MUx1RGYCIiLyQ&9iGZKhnK#%4mvC-QqL?)~6aM}NGwe2@t- zDIa7Mmz<34(c_;)m!V1EsBXkC1w|gXz**m#=wRZ4bdSd{#_><2d?n+n5o}?CfXf*J zmg|mfOy~xOQ@g@;-C2+-p*M%eU}B8b%9 zA74lFDuoj3XS`xu#X5_lG=e!UHqyz83iQG?6VLayW$c+{$ggEGk}kQKQ9B*+Tf4<>&S+ao!KN}=au_DGh7 z2RNoeV~fWmbUc>ohcoQLj>~8oam68jijG)}p%;h+9SgDZ6R-deWSRJ!4|3GzSj*|$ zY+R*>BtDu?>Whu<^VUb#!>?d%m5JJ-uN;ZTrAS7f!5t=WUBGp?DA#|*xo5GdyUj9J z!1QMMM)qkta0sG)@lYZ|Q)PW!r;$(UIWN3l(6M|Qg zUW|B~0C~J(+yRESB%j4Ipp?c-I!=kEJ=b3)N=PKKr1}p9!!< z0MpGfY|Is=%Bo6aEkTx9=58)}FY<-^-;%}^$YWp$!EP}e>2MwUT#O4s34Pn6N)W)} z;qMV-TopbBzj=>eEB;(I>wMxlIG3G-w`cM{031RVTJaY$HS4l_Xo0|PR!>@v$zVC7`@#n+eN_5fH~7J!UW+347+VN1LSJR*4$-qINDyMe1P zyD$v;%uyb0)W3`y^)s*sncMSTMIGEfpFdevhW3s{S#GVD{U~X^4k;*{{fELE6NO(& z6uux)cxxi1GgAEKC~Q7aJ^XDVknaKEFWq1=AKP58Os|+j=D^$@$#sXwO z81DXOD{&Fq@4`LZp1NQ+>FN~Lh43y= zetRNjhmtZ`Lsw~$(ASeugUmWFkupxzX)i1Q6$|!EGVD zZ$ld20z2D}mpy=>6E^U%F^B`Y#1nvv?E>-5dQ*}>QK@iaGObAUB{Dc~4_HN6@VE8w z)bY6Q+Zp%R2?++|0t(hn*|U5=hNlMYtczrpuCmTXmN}A(cz&~V0>vnS-XZI?s?eXZ zPt1{($mUE^{B$#UVa+@>>pW&{K$htDu^Ow{<>-_&;aLlxterRqmuMxYHyFM+qKNiP zxf*vIyQ@TLJ`h{^zih!pzJM4)Bs+DYOIq6FOl`KaWhs7R)cDxM#UP@JYifRj>v;88 zV}r3+CYScaiOUj*f_;GZOJIPgy-4yQzZm%u-hmdcmmwoYd_HhSG3{~q3vV5RY2v;{ z09vU5Xcllb0>eULK&OeDfFo8T=Kjm!z`K|iLA@8RqaVOPm3VXmT?uQ49^sGR9U|z&{T?%pJE|Ox(6B@ZEatP9`_PP7 z5&&c+c6{tTXDm-8*g>-&u~Ha2mrhCy-isYblMpeiPm$7hos85?sJK8kvDsHQzT)!O z1U=^{oP9<%vSP)}1?h|dDoZasUQrOk7I)mk0bCAaqL4|*ZFKm6>1M7N2HHcp1D**W z`Xw~o8ACX03ib+b3?D|t$GzIz2Cw_1cb1A2YsE)rJtkK@w33b9nJRInO5r-qEvPA0 zX5)fZ-3R_+R|MEROW_BFpd^6zSX~LIS#xB~5_qP9B5khdZ{OQpqykY}b@MrUB{`Xs z6iO9WH-bLccI-`3-d4*yAaYAHdzMZZ*#KWOeoAsA-wNrSB}lMK z)qh7Fp8t*(8?VSiqt)u&thLwdVmTsiJA+raL#_dzM6}OB*t=RQeWY8n1M`c(o#uNhO ztmW%aW=VI{A26%}+;hXF<;6|Fn5*L=enqeOjkRJNt}w!~j>RwKb-U?~xp>vanp?oI zj}hS|U`2>RU;Ox0_B%P)WJhPjhh$_yH%+zA+ ztOLNRk7%ijO|M%`EsBj*CEB7|OyjE*%Dtw?n^dqnJq3=$c{KA zvYgFC)_M+elf<#S5^Ust{QOC>d>v?+FUfz6IiRjeL?35u%g0N|ns^UM{`~+C7_Tu= zH=gHyKJfykumNKau`M6XbSr2cJ_3iyl)%2;%aB7`Nx6DwH3a?&auRrhSVc!>Ar^HOTK>F?K9N$x_*AK$FdEu_8s z8g#WSHh^V`%9oIhs?P$koi>r628@mMH&H@YN8js)6mu>M>*hj*b9ol4={kTpQn(;2 zD(86#XGAgqP-?F~He%$h$wYtx_yx_QRrGg^$>JG|zHqYNWp+nq^L>}@op56=&qjH? zu9XE9;;g(Dx`on%VSf(3sZ1|lgNe)p7%sDUvE-B95ZQqzN0r0T5ZQ}6{2y-IZGvN;>G3UXM}0drT77O zI~&2jkOi^s*UXTG>|(NOtdT(d(?->i)zgV!0xPick0Oy^<$pjx;b6{r%?Q3uAh?nU zE>;K**$8_50RhE|2zJ7|R*m~If#AdmAo!UB7+ho&MZvt1>h+tkbf(;1-%ZT7@ zh2R?-!7B=Z4~nXUVqrHR;LA(KTn8u?qk#vRyr-(et;=7CvKXx7svzX4N^H7b67{)_ z^IC~htA6ejVOwwK=BXMrr8}U;5j`J-0q@C=6-djaOma{nvX{?Uz!r*g5**4j_fp zV44){08;Bv0nlawErK)p3W|$8?&wy2QCh(H`JaK52LUyj*99mRk+)Qw0q6C{dTV$gx!0I4Y&e8H` zE^Y0;wBTmJfR%q+qTtiSZ7OSY`IYjXNW?|zo@9!1)qN)KJE?nGs(^7=G}NLs=5kf< z;h7PdNbw#LXGYG$?GeddO)~t2`hw^VQFy+hPUWqKnr@b^QN-_eL z(x1igx-*vTy5Ww2Aa8+0OHtXWFMeg%d!ZA5{>wyk?gDWY$8y{Fz`mry^u#WQzaUH4 zR9p}G667jS4C)F*FN z#9+7P4&6#MaU0+}!Mc&+N%YwPm`~i>XhIoc0}{9;b0S4yIUa1M0dJNDcs!5tpQnES z$IBd-6)@#X;)S1S<@Ee8SiL=lgiJQrl63Db!c||=)9!~MN3Xk$VJQ!oPoJ$>e0mw@I2*pZ` zK8YUpW5-ISAI%u=Z@_kHR2LWXq2=M_-C4C({a;W{zN4{_VlU=~xXUP2i#$S|wdP7) zz>C?k8QaA+2rW85i`<251QpZQ5lRXn7)@ZTbv3K>+JN!Al9T6U?Pep_(`yz5}w!+whEt0LJ2BKh6j!KPbnZAa>1*sqg+ruP3 z5p1j!BNpQ22I);;b5*#>Ir&cs#60H3%3ryJL}7(@Mxk6k!0Q-(b5f=12EKEGH#0;R za6iF_^moAs$$rIH;a&m(2rq90mmDD+C6>Q=EudE>M(rryWI?zBd-D6gMxcsrpkN(t zxEUNO&Mj|M6e4GR>>E^w7X-wy1$b5-Iv52`2Kv7(%l2<6D_HPq7R2IzGe)G!wSWx9 z*E- z90AHpoqqFLHy2he^qbea7^d6>vJGkMMaE#FkgW0>SG&Zaf3w!Lco{EIV;O5)iwO6gvYX>K zHPQbx{O8&DKZLcHQ(2{I0b{RjJx>7^KR~$o*77I6};MP5NWtCrK*ao~gtb73a# zur_8cq)%ikt9X;;WC0wGV??$}ag#IoDCS@&Pru0#^dt~riAc&1hB?-YKzdGV)bzd* zsLPsBhq__^kld}r^_$MM;lCC*gFoxxzm9i=IQ?aWf29*jz4-ftPMltgJts{&AdaFuSMBwnyFfS+5*f4qVDoDhyQf4|{nx zV9Ib2B$^G^O`nc8^XKa|%?q~1$OWj%flQovBxi;}9E@iYtD0w~vL&^mEk?%ZzgU-P za}-OI7!u-TScj9~x;eZOfH(U{&s2nE!PU<1<3aTV_Xf9^c%uZ9tUag5&^+oyJt7(20312?xY&&JST{nfYXz|r<0+ia zXP9IyC-cRB@m|W&1-Oh?V}Pq3GVvtE6p>$2Zx!X$qUo>5_dIdN)^hR+jceT&>cn;L z0r*s%Xe--;*D9CG30+6oB(=6;o1D-Y(0!ULM`&=u92LnoF_+V#iSN)Z=*}(j%*v@x z+bc26;c;;Blq%=NE^I{?I-zAIei|SM-tli(@$yQtTxyS4g53eQr`RD%+6H}n@J-3} z(MSRF$M9k*1x$jpm=EQ0cwIPNW%D9tm}FT#-?_Hq4Ag8dVW9xTAf7opq1ZAwoS{G= zIj>`2fJQ_IK_j8j#3!qtpzv4XlF-~?5Xzb_T(Yh`@A6W#n{$m5QE6C`F+d>l6EG`A z%LwU{c#x>76snoH$QRt^$^>e06P3zPkp?coqY=*nqg>H7u0&WHBgh6M8i~Xp61h^k z51b(uD6now2}DCFi0XA_t~t6(%{EfGEsKH&CDskgg*z>BA*$tqkD7IwvOE{iq&qJ6 z+RLc77#{|CRQ_*Tsh~OD#c((VDdjwGeVrYowBQ$vs<>w@q88~PP|kS(M9qX3sS;S;+bv7v zWVR~0{WX-ZV;G;Qdv=Lv;JsoFy6!Nd9uTvu27xB}N&fI!toAZ`Xb#05iuJ*ZIQ7nV zL7pfqGUzF+0c?njA=n)){vxr;PFK78?5Z)cOK~zH6jj(s@(yz#UQm`|Eb?1Xm!qi5 zlE4ec@M6s!O{ky$j7pU;J{xA$3#vva;vdFXLLEtiL?cJ;3F(O@x)qY~ES-FAbLlpO ziPZvtwZnVhTu}`~a3SUq`X|US@V5hgRizfeX|X__nz5V9kBD|OFCnUd0%Iz>-HsHX zY>YEr`=mV;0!b_?8l@OrEVd~zx@eK_zji+6K(AWw;6AwLgWIGukl);M#(%8e@M2YWEVPgGj$b=P{ho} zQW5zmE=BjsHw9lpg3dTC|EloHacB}el;qF^n<@@x3!etf_S%eD=@nf@pp5FKE`h=w z!D5h+4qiaWzV^aol$z29fxmK*-4MHe$XW70uWojE?YCqrW61h@&8eB4;?} zRJ%d1l^@02Nti$fDDI@Be%wE)_gW zdOP=L&%&@8`(9kEK$=+X`Rbkno#?9W<%rl`-OHga{*tV~OZdw`vgqO?NhOQ1CSU&p zwkf2F#Z0lrt>byqx89aGa9W)opiZomM}~6tdHV|3qF2}`?CpUO!T|9TEPsx0cWgyM*m6ohJMi&RV3-o14QzZ zIVdGkTlpP&ny;WB zKM)_lhO^4bmKFSV*DbB z9TdQ8As}{J?7n7VAGkNP9~{BEx!~iviXVJcQz0(Bb*m!4m&Rs;Vd%?`$wWR+ynxKD z(-sZV@ls0Y1HW?(N?_B++2-)>eyh~!8x7KL0`2ll^XR`Iku7|-DaWx71yG)gb`lrf zIEJd)JlEKg?^H`}reQ?f0~g^yOY|8hQ!>$gk6>?wW5!hz$K6$qQ<=5MA%KUccgA6y zd*odf9GD&QuDe5vU`pb^8R&&qr)UubRUGE^*cSQ@5fxjn|GuBvopBbbPz)C@fnGY> z@D{57N$@f%L{baDmmKm8xiMgxU8V-}PCMydo#BTia$btC&YR)Crl*BQh5wpT7AiA` zxpb?zizB?l85*RogcLdOue0v_tlXS4h!mBFk4>GNjbHENj`-~xbX!9j%fm<8PCg^n z#Tv5GZ#4No6L^oLYNsd!%Y{2YVb0g2fgDnM|Q`dsptWaCoUwp5T4Dd zAr-KPHq{v|4OwU>{c>CZzZR}ccZOcXj_dF-=cK0&*EvJ~FuhPUG2a=QxoHn-Dm#e| zF-PES@!nnT*e(ss^+iYuCk}zE$3gZmvg?Ku3bAO;(y*s2n;Xk=dGrkSKF34-5CIGQ z39nLwG`wyRx~+D*T;qij4e#L(@wcL`V{!ZS2DVxh_02ajrn!s070%qiC{m# z#V%u6XvJo{k{iLX#yH0pa6OTLn+<+r9n3y3+#G99n0%BcjL}dU*%nOp8~w4}A($M% z2{`ZzQ?TvtQjCs{7=}+AXt<7)rp9p0^bwRpv_w$>r0@_}LGdOai+x};;!GBtD-&-s zxr06y(v|wkH)K^xQ@t0bB%D#bIjHgA_hHrRL)R>(9jLE5<_vabCVmcw3Xzd7Um<6K z&_y`Z_yV2>-&T&V$UMheb<&APIQkAec4+&H_ThTV=d9-{s-xbzzHu=Wgu>KzQXk`3 z`1|eQJw800HR;^)aQxO_*5vf?{=T?%YERb2(r`0Peewf>SuYRlM8{C-A6^1w%m6Bw zm0#iWK^db9)G%;}+~@(ATL~jA6wU-vFo$pv*Iog5oVF^>#%?kEJlZ>&G%AWwFZLpV z@nQ5lV?0)Ao4)h@mEI5h$Siu~{MU;{N`Qq>iF#@&QtYG6y_z20BEq8t! z>xH+nD^l|I#k%9gZEnUOtH2cJw_uo91!w^8u#f&eg+ouFrkVk&@@rM)gHI>0f-ulc z64+b8jK)R0RnS7<`|eW+e3g&?YXW1Z6DWtL>mVH;tRkLx5Hb3rl$gsw!aStK`eNv% zFyJl39Ox5HkvWDVQKn3sew%K5Dn3Q^7~b!MF7q(fugZ%}XL$)U*MrfqNt_MKg?W8f zJ-jL@_5+fRiZqpkfWX=$Kz9K2`p{3&51jJ##inV&j(Dx@=(JEzsT$F@RQl0rL3!%a z^LQjOS=#JaZnojq3EGx}16`rhsl3^OAAH#Whb~1QU}>6XS9!RpZzwbNhP*CpMiDN& z6*xjT<+th9&deQqs^}{eM^fMSaFrj3IjBW&_?AOoaarhvZ||7=^k8`W00{6j zE1pu78?aKJ2e53^ipM(t7U}wWuQ!-8IzC&*N!1G_!F0^-0!Hdz02}KA&uo06kf~=9 zXS}RZF|1CPgAC}vPM5miI_P*G!m{3#%~Z5hW#OWxhaig`>%LYUKdXg?VvZ9x>HNzhbAW_OKw@1h>xVSkedS@!7bOxuph9GyFwjVmGLk zj!`_2@FWWUSjnjcryxy=MVzOf=kp|tSeP$UKoaC@3R6~lL+9(^<9L7eVj8O%$Jrw7 zkwVrxBgTj5nanU#-SEEYab2w2f~1I0!bIx|nBTup)A23`aibflKk{Pcf6EJ&M<=tV zJm@vaMzTqM8o>A^K8l~~z$2Adi>v`N>5tN-c(Aw87~YW{OqD0Z9Ind^CWqJM9wiYKKX+~Pn8S1GmDrR*fWOb+t@rx8stew8JlU)bJ00v<_zQuyB zQ4E_PTZ%py?eAT}$$X5G5MDk0v>B}W=Eezq<&($%`dwqyb@vSCp@=KK`>%nszd65P z2$8{nzV{glL-jBn;U*6xO7jM?3%CYE)x~TZT2Y(J3wRmqrv|8DW5Y{R?W23|n zN$a^h+>AtbG5skWIgwgt;G7ubUt2ZQHf*W}J4asNWMp3Ii@xhPfSbgu< zjOy7};8~XazG8z`y$Zn7@kKtI7lTcKZn@0a5G;*O1JQC%Ba*f1p?HekE8CH}Qz8N` zHKv6tQyi7}c1g6H+0qoD_qE6!c+87OeYl{dZ=sOvzeMAj<8a|C9NOb@6@BcDm@RTh z)VE@l6%*h=&O~GNv6tx2=tpn{;b+h+@oWgG=@4(Ette-4qK9JJ0E+KBQ{GpL-@A2`7()?uX{U=|eW)XyT; zy$zAZyoh4|_*1+Ty?~2IhDV^I-P9eI-un#dKa;}v&Radc)y1i((S$uPpigqbbu5If zCAIP+aR;%xq6sE49&(pqv@T~Iom3t7GU#g+{hMLC7yEF8sk&-9hR)im=}sJW%CC5} z6wF}33{~(FRWQVYmjf|97 z`N;}n>3dK#L>ZgE@UVq2I30^D+5V;^_c$)%KyTis&If*)U1`KP$5@(pGZ#K8Ur597 z=Fpe6tk)v;AbX?becFI|nTNq)i1I^R_T&jUm|SG=o^L8(Uk{Gwkn_0x;;}QJa=kBu z0*%Ibbld#alyhAoNeVPSkNB(KNW-WtxRAiepq#wu2AYiHrOF$;zo(>>8)Z7l}j@0%Hz5u+c z8KKxW?+wYmT3mQm?r-b{&fq%}W~73dqF`RZMOYU}7ME<~h2Qw$NC=0!#(ibAdC81n zu%aXA?l|Y=>olX60F9;96W|sqvzxo-x6`}5qLdU zempLSNG)R6zj5anX6*q+AL@X~XRL-w6p_aF@re68ZPCc2^8Uk?us z`$Yq5$1%}piL99ZW z(movGBZu{odHb+sTXQ(@3hf$w)z9g}b?0x*VfG>D!-NSOVUtTz_{nA%qi$rjQW6}0 zXB@?e_S~_f3_a9mwVOr;R^eV3&uykPUd+a2sm3KKj9J9k5f~G1M9RL6^~#hN|0T!0m7zfLKF0IPPSQ%t=| z8e6>L8b&ekhv{Tv=w6;ZJW zYcX@h2TIuk<{7Bd-YoRr)r`Tkz;;BR=b{GmLOBBJJFM5AGJ78lH}9c!fa2^u;p^#C zJv!_7k=*B~RsyEf3`UErP2#zNc)q64L&vk(H!lB*FM59X?sLDq=JT%%xBse3pS$1K!F}u5`clr9@K?pJ zhUJRi?xwoPp#du!yO64lCS3TMTtz2 zF&@Q?I_!4MY*~<}am2UJ@XhMDIEw}u9RN@ID!EDfD|(1BYnGjL5mDV`XC(PQ9%s?fGiQuE7 z7=M~YnzbrhM}c0^++_58ZmUj4V{q1~Ekj%pHrvA}KsTiX@RP-~kg6}=ho``FY3)z* z=x>q~+LFOfpN^6PFzf&)dpufSI<>tjR!cj|vqov_WE4<(1IodrSVPgWV4GUL#ORIB zE`Xndsxh;cQO7ee0zo8l(9MYKh#tr^>T%EnzR#FhfgKN=FQNUC1pC_#IEk9pIld-7 zo3G@v9kEZjD0``lwnzsug%j&f=x4~*H;?o1&1ul&S7Gv<5GsCX8V5LSvAy;2n6Nxs zk7Be6ww03;7KA8UjGX8?R?^w}eK#C9V~;>MEpm4fQRIhSCfIfixQG-<%K=SBxB=QO zy4}t^`4x1Awk84;4A_mhFc=V>PXw~VxB~k)p5(g>!6k?B9z*DTIMNt5^Mem790op1 zylL078C_i9+CL%xlOGBJEyJu zyD^7D3scd;=t@aN-vf1|0(2r8hrKaSi(&8;)n{ns3E1)37;Kcdpnf1(@e&>-*Wee| zwCpJ=K`Pzwn&zVj@CX($jTQ~&PkQ8f%Z!ID0!E_+qntHoo8B^<5gFcb0&buJd6FQl z-;(s5!zRPCS|ikJpS#5(nr76(7aFwck7fH(Gub{_J*@E#)e?8~9p2CcB;;CB7nj?K zl(Tk*K0o%{&{2QzKAb zjKMJ3E_<8?&!(LlcNJ+Zjtz`lL8S5e=q0?uhfEbaQ4p>+B-V zPn@5QbOsEWSZ-YAi+xou!`WCG*N=QZ0fSmYbTF(EBrzkIRcEWJ7d;}YcChL`sG47> zPhLdS6`YG^asWKeimd$SUX-Bk;Ub)*5<^4(MLd{f;&pNnLO-%=0!-9e-FTm=QclGr zX$nn6?E8{fXbm=WIM;K|y_CUmVbQHL-jsoI<4ppMwm+mIO}&gAe2LKMadZhhzl<3x zXzFRfO!<8fIDS=Aa~w-D(WqbtOyaP_p@YtYZ^k-M%=mnveAcxUQ!nyX#oGmY;Wtz6 z^z2V-lb<9Tx#$Ji$nF4+?g1SEa~#5`heH{7k&*lR{w!ntc&M71?FinEa3*peyaE04 zDOkO>q<3GRJe~2#R|w*bU9TTGiW-rHEkx@W);p>n`MTVvtw_l%hvd6x^5XbTn{fbv zws?bn1Z%Hr?R*@>Iqf7pl5b+$wGT&jVjFvd>)WwlSX#JSH$&Jl)2FOd8??K0?7mXx zUVq^szc%QQC;&IBre;dj4yI%CA5P)l0mBySfFFK-q)i}ahaa0(rgn6MG6Tkj6>bUx zCX>%Wp~|%tCB3m%9qRW)>t3_|3RW*=O1Y6Ts%mN{M^M+zumX8Pb}YVhfxed@_)tc! z1AT(9ro?hqt!~QR5AWC!uC3Hh#a8M(k+&J2xCUidqRiQ|BG)FUJvCfa;*Wm{T&LGF z$h>SCV=ZfiInNUOX%+Ls=L=cC%b#_^AN(m2qt6D?X?US z*avu<$*>Qdu-yU5H+q4y)ntUX{Rg=!uE1_e zMEExc)6h}rNinFva5uhUg4bu~s=in#%VQIg4-OQE7D@%EiPC)rJeR-9galD$!MF6Mns&k zm!rAvGi2SrOdPKlZj9Xka@U0hv*uCSqLc^OnrX$T#b>O-ntn^&i&m++7p-efX61|Q z%E|c;fEeg%Cw4&VYtgJ$6eObATVXzH#!-2#{QI7fl5-Llu?Y?Y!rq9UfjcQ{(;W4) z($B13A9T%3&*Ue1ep!p{c2WXlP<#f_lq)p76q-h&nbjIioAQP_6nS7kC6a;X_CDXa@7}XfZGzUot25rTL<$tgO|EiQr7~}uD-l*17axnfJC{O ziOnXV{dt?q<&LttdIWro;|w|lvqdru5=Jk6c;qnGClF_BPd|_mtmwg4*QR8np+LNq zA*Gg*y~R#iBdZTMGc`#p{SQS!H3sFD!l+QI{uhiFu^h}h3<_}g&);axtO^*~6Ka6% zf(0T*CD+a>!5M^F^(X*9XezryYQo2zA%4(kRJ4f2;%o3pfM_?|8lTSCy#xH65wFEB zckUc_YSqjM@8}_4^N+$LO0WYGYzM(Q11w2<gk$ ze4R&*v&FNDiyS4!D&LHqwE*_bI9i2E=!{i6+W5jdu7chax2-MwbmEp6mX@KDUsd$h zouBAA9~H&7U`?zO&UVD>i8GG!xGL?@B^Wt5xn^IKABX8XMMPh<#^leikK!DzR3T=!g=cH>jKBh3&m57{u?*TBIMv z2H)K&H@!*RZzVq@Q{F#Giwt58MGT)@fzp+m*n4&Ks!(mLCzTQF`A07b_K5zBO3KY7 zP@+XrQ6kD(%drj`<;0{sTyT*V87X_VmYbW3%}&@ThPM)bB5^ol>+rkI_xTYxX zuNWKQOFfUI4evUATJfRS-(gR*$S)9>*u~}j@nwcxy!C)#!EUIrzaw~#AG^WO6;bZ1 zA|vP2#@a7%#?shF-{h%aH-$%_4(|LJs`T!`P)i5$oNnV&8LrZu`8$FoT_R zBQ4ejKYqjI{m_ZAC@R-oYP*0YZc*f zTn7$+g6ah9LlOCbx$($7_*$%Tz(rd1l_cb_Zq1RYG7ishs7ovaiz#on5v7$8!fd~W z@6Un6uOQ_h@%g`knO9;}0kgZ;NHtvXY#qC`=_}z0yY{w=tiuvb^~+3(SF2h~RhxI# zCG7I*?J#d(2HKXnBpq;IdpkA`AtUL15hbJ>(pdyZVgs?ImgJ$x(Cu2MR9pBlCe{Yq|uR3;~_UKNq8|nVBou3Rie~o{? zw%BtX+-#dd3{0_kNb2Q`#c=8AjQu#X*wa7uUDXs%FYHxV5AGn{?Tmel#!LVa;bRw1 zJ`?+M>RdWLG*N{VoafGr7``M^?Wx2aJ`#c%y+H6%g8_9}*V>l;)StEuUinpRX}v+9kO&G| zt%z0de%B|)J66C8dB5K?^XzUywC(TxzwhV$yq_)~^6bu>nK^T2=A1LprkOU}vcsH!@h%IyjfHcnRlC{3FV_CYO=G?c3$r)W| z0RTSkddJF%1jR;36R0ao<@j*La?f@Y%7WliwCfs-ETo*A(nYs0gY8dFF}n8R`~R+E zQEY739azuMS(qidx*+V)lamdQfdrxXJ2N!Bbw3IASZfCewptNP<&$8FNX}R(>d?fC zP6odZ58}j=$wu@4^Isy8k{itbePyrR#EAsQLl?tkr!+fOcTTAd9X-VxIuaqGP;wp_ zGG*?%3`X1|c77e+LxyOvp6e^JCS4C(945tN#?t6Xk8Wn!-3$+sPKVC;w{bhFYa&oV zME$>AwC1R;16V-+W{c6Iy0)U!|8BgUQMl!Ig#$u^fn4RWw8+m%Q*eW>uVibqTJCv8 z#W)l9)o*9 zx{ha4*gInGl0kKECJUI0Z$RigA~Z)a`=7^{ROBN{XDYVsXkAO6ZmbeLTCK`pV93~^ zIF%yWV8y$kNvy7z)j|~V<>RPwRYvv@{_V7FpB%u?$;@E zhjKgdByI+Z45ZK<=!z8LJd4`DC9DZx6O6_YtJHT`-t9Lkj7BMpGKkT&>BMg0kA-N? zo+qr~>@1?fiZ+rVOnNz#y{h7919wlqB$Z|YN9+3zJSUz}7pb`EFxX3O;->3) zj0=izqf+I9`?mEXSO-UphToWT2qt@&(&^5N>=rT2K$c=gN5kwhKqij5qzUgh!P%hg zsaY8^`!owR36WvlFEL!4;9^aYtJPY!RbI3eFDIfry&qZ5VAFG%|^e3X8 z>_k1-(|yEv7?@*G5f`&m1bb-l^%GeQ9sFNPHO|e~EFJ%4Zpu*=B?wq8iF_*|4ta1A zX8sR~!HJU@_c}T^-#W*2U~1|=T}}T80>R zj@({hNa!r~%1L)MZVmgOx7SmzfCe=yYbZNW0cYf?EPM(*(#jnyKD+SQ<&H<*Et^!E zH8ixtdJy|aDTr6%C>k=v zSm>wWq+JLv1{#c)1;ryTT#>=u$753G@1hB?IKvf5LBy_@^I4lSvhxzRb|03^hI+mg zY$3^OSbHy#g-c1TQuqt$H_lK_L+#jZL6P=KuGd(>ID7~9I13MPhcTW3a|-&4#78jR zwLsX=h^JUHMg>CydFR^F*)g#A!Eethc5S}3VVdu{tT!7N+mJ29k0OFBq+DHvJvXgq zQzW|MQ;1Yo&RtQe0VswO~U)(1;^L{Wz+k^61gY>~@ z1cUTYLctScV$|nQCk4iZ40%B;AV`uFkvJdBEgp+V)SVTPMd-I5M$#4vF6jcp7K<-6 zE~H2o$Gdo_I4PUum_(kqov7%pc#h3Q-~-(vc58TXVg)bUO>}0_-*gOk&|}1|bz8kH zf=V!y_BrR)kBYJ8?ves(Z51YBG_i@GzYe>isrRt6v#6yaIP8GyFZMW)cCJnKSim*S zfk)J0?IA&y6u)&gfOB&^NOQxDlk^4Z5<-JU5CfRDFp36wvl%J^Whyxv$P_L$dgz1E zOLVTL2C7CtFM=s~43|N=x~+Sc$+YV@JQdiU`F6`?cG!eJHCTJhQSDyTW~+Iste?Q@8?~ zh+ctRc!Np7Nq4b65rL3@3AsmSKPF@*WrnizbbvDLS!^d(a0#h|{qIW5!M7wSXN^?! zjVtmc;w&c;OX8N)g|g_S=|4MCD9HG!f7S(zhp8{KfmB8?o!UvSV{4H&pi`Q*42CYs zl_UH(@^|Y-Mv-f}Fj6Z)1SGm@UWg}P@oqDF($a}Im?I)!2V&9=<($ zdgV?u9U6*m&yS$n(=q=`A~-<--iYpe4noBW=Cj|TtpK*=iL`rBCdIlApcXHBjt;vG z31Eu(a`+as$cEAU+#OiG^N<0a?R~Os^iSZo=CpvMNn6Zh@xu~WVlp@1y=i9t>mL?P zWsjiUsDmS~1^%eA>3C%ig#WErO{ga`FUQ0q!Y4S@Uil1E|qRaqWDCF`Lq;981` z*sveL1duc~Y#CFs=;`axc_Xg2I)F(C}*h2 zjAC317)z7D2(UfOU361KC>mERH2K6xh$q&I*<+LJWE>8NB5n>xJn_D1C3TBgXo`qj zgcEdxpT)-T3?sY)kJF7%rz(o|4PP1cE1JD9FwpL*HVhW(IXC;3T)kj+XmhFV<@Pq! z%i+2ozdHQg-3@;Z;AoAgl)Asgys8OZgAijc>_MGI5(#P)+YEb>tpsK)1ulDo2sJ%^ z?9oJyn$j15{F}%cJ{q@n7)1u=bNC4K;zT0^p@1UYgYhfN2!GMNfecG`=;%a;4mnNm z9bClhgh?7&B##tB=Cd*0$kT zK`=BQ#rscb2uF;Ybv^rsXb^0U;aq`h73w4L+`aerC9!jk+FqJ^1S-Ulx(I@dOOLAC zUYgGPcej^C77p27dc-hf?!D^*+})v%6;`0GVmqLc(*n94R703!#EM|mVIXYWPXs>s zyTDO-o)aV(b|aCdOi1~t&~}cz)%zYDAxVRwpNgwAo&sW zOAW@bq;&mMsE8_Drw+mz6PCm^6f>DE9 z2P(*4osr>LR`**%3%H@IviIQ3kYyOf`UMb}85nWoQ-<#QNEg~mfwJ`??FE3e&sp5V z-(atC-01g!8YNw)RV$`c0-H09a!am z%(FvqbR}usKB&U6)eI;QW=`w>CVZ!I6<5ZYxDAOM)#ju~Iob9(Rt_16vbeWou3T6r zmoo4UFKn}p#cesM;dI=9lTzU*``GblISxWS-u)Uw4;{{pWEl9gbnQq(4G~yLdF%2k z*1o@?ThQ~HT$@Je%g~*FN$A2baZsMrJ7YIw-H-BIg*1_YMGD@)_7;5Qn*^U`cI5u7 zzw>uYy}dqZ|05j;d)Qgjq2fsCqKn=jZ+2x=KbV-*^r~I?{aNp_Qi>ALjkJ>XbrCkg z#`{tU$5pY;7Fd;7aa}K^q7ddYE+s8aZ)wJ>y;oj~q?TsBtNTi3wl-&`IIg{nUs2)T zP~nDC@CP?w|3x?;mKqL_Vh03dV-IDuAg==5=95at>a6|v26rXjL$j+QTAS6pfq9~) z-XMh9dtVv&=M+X!Q6dyvgQ)|5D%vZ!cR3$L&*RZ`D>+02XTX4NddUU@a~XH(msH~P z#!OgjNwpqYbjZG=6Jaict4AW#u${)fV}&?zU}0m7=naCEyc!&K)0j6nRL$ z+_vBzj0KjWNuK9)atp?t>rFJFa!(30vhh_0lkJYjk24i&j+G}d=gj-%WnYe!!&ziS zRs|kwuU76;VCclp*fe^qvGyI;eab#L{`eS(I1Vkt6fINphAi$0)D}9eht-ZYCG(8oUzdDYUc#w z_|PnCFifh>=uljm=2*Fx)QFZt--N71P1n551e(ZKXT6B8qAe+mU{Egu3)qF!I`}fb z!t=1hP!0;rimcaJELxs{46d%!6j;ZQ{63RW)hBpbcO786oZXl!b9vkDCAe-_)-i{! zm_oA!FtQQ7UCq9YafXFOsHV7@_KHcgtd|CESIeRogJs`24E6f92hCg`K!NVddB=Ln zGhM4VYw!-!)^7%cE_6ok&uU?Hj4Mf_0*G@W-y9Yiv=NKc=zcTKVZdeZ`ozaWqY--0 zowgC}V2oBLxC-`cI3WuysEeoB6rWlP$iqXge@3I(PBWOd+j^x47o)hWoQ{f!Wi0kS!yEihp;!6pE+1Gj!81V*S|i_4#!IGat4@f)Zb z5LRni&P)woC}%4#Xt`qSVd3*|McoOE?}MxQO04D{o@ay+_tB{*RyrP;I);CK!at+= zCyReh=AV=JXH+Frq5F9|DKx_FVJHOAx78UrA*=g3HK~*zq!@l4fZ&vMe0Kn4qV*%W z^n}X6omYVgz-x~F7Q7BLzyvuE`+gQAHk~T z5)9D{1pAc&ThFR?Pv<3H9k{Y_@mEmyzNv_1ozXoD1;(%d!c$map(^lm7PyBRXBnKA z4y$E17eMG^4AdJ?s;&9xa@i{6puXb~E#!(`i9C!R?#SDkT`}pM(22OosjD0TA8FC4 zL%Rn=rw+#*51$~o5|2;20k$pEwn)Jo$4D#3imIts(%dK1B9m z1z_M|y#t}*cy!M2=($s%wjY)`Pt_@OV-dix$Tg}6OIqttLi(zaAeuy6BYaaG^RR{FEJ@fWvDwW@ zg{v%SXJAu{-DrP7p|%z{Vb;8j2WL(Eae zIme2ZkPuxwMe_9ZTf8EPfo_|TRo8k|88jG*p}OT)rPf~SSn*qYr}E1n4%r~^Plojq zlyW>82D8A)CVhS8&m9fWS?`= zXMqzDI6i18#M0-^v7f;MJZK5*U&9YCTg5P^@J`oe2xfj-x<{5A+<|+lmS28Y?X}@& zqoueaGo0G}I-_Dzvv?CFxR|~7`8~a#kb)JAB~zDyCN0mBxaR);GLUc|ss{;6@Ev5n z0*_u~evB~t70d_)GYJptJcu>DhPgOzBiI22_Eob~mHyptK>9=`0Ktx}~m3D_3c7+HfckGH=}h`f?1WM?RB%?UVubX~(y%%OD8m zRYdmNou^&=LU<*4(XCwDgj+KuzF#~Y%Rf@x8?p38;0=8Xx*!ZNE3zzLL=6>U!Wp|= z^aQ#r#e{@?C^jyf8}qX=_ueOOP6TFq@8flcB*>-cS^Ei?hM?_xONAi2u0dzx3*JS5 z5ei_N0+@C+2>Zo{B&=M;%6d*B*og}4HwujJ^#Rw@2cS|2Fj4_rsQ}gx;F0N(C+IC^ zC9ff&UDOloQWeZf!u*m_DDMJAIalPfOQ9Y=r_|z^v}xx-S~5Zva|<_zy(B|NM~CQM zoeUY6Tg*gyHTa?-!r3=#VN0cpBRYU4j-x0=rQ}OJRT*8PNn-a`SR5LzEIYC}Ky@u$ zAFw+lu;su|VkI1j7qcXa-U1+~uPdbKmzdF01*EpoJZ7Vd+b4lu;R3=abctOdg%_MZ zra+@-;PW_~0JQPVvBJ-G6?9K!M%R4#2^1GF>Yn?RudI#GDu56?mUf{snr`BSSaBDv zIgUqf&%%V_3__-Q-4(VaH=qSDkp)zi7C0Bbk}X!Vm$mNPeiHGG4a-`>M}en`kD@Ri z)F0pAeuk0BHA4k7n)C^UQA6m)2wT$`Vk|5Iw972aPt|6Bi#*Oq%g;j$vBcK z&^SO9F-s>ACX7gU0EJNv%mG+AYNtYSy^h%-a2H+7dm7d4ON+xC+f5i4mKLqNpOQe96UNpDB0@)QsVhr7O_6?gCN zW$E{bb*vpG5~GASHq?ai#XkAD{G+w9d`2uGB5-p#H3PT>$z%XJY6Us4#@w!=*r?d4 zInlxCZ^H5h6TmCB6%I*x8^#I#B(y9L6kRJRkR|gmUWj092J65!8Lc7>#thctqk&-9 z5G-p9{eU1d=VxU@Wp%7LnOka(hIK5pS&Ulx=20vjEBa%IO!*9c*BglSo8EitgzhW9 zC?zkWRCBIh{@$twK5zNQ6-=ReZ*4pk$ucj<<|;+Jg>l~#D#mVobGo30mz~Yes)S|Z zBQ)B5I@MZfS#p65Hq!e?vS_8_=B?It8#7K%(CeyH>meq^(ssk6!8bPVa&E9)EEU+r zH&Ne>wiI&=5SPit?c9)XwA_%@#*~=O!St!XfDrrm#d6_dPP`^A=F>U9O4ggmrnkP6 z8_B}>EnRX1v5Z4num*&+&W9PCrSdVm$=WMd-~b>(gu73*#>43#S7U@mx;7ty>hNiS zi3nEeZd#1MTq;eZb?+2na8xoBy)djzR%ZcVNF_xDP@xCKmA%z~x1Orx(-V;$ruj0S%r}$MUPAQiLvO{jl zbYcrU6Tc!k_c7UPzLFH8aZ0$?-(?WfnyBd;g2Y{xsPHA2w}DC59jr4}@HO+7e{Hnjel3_fi~a=XfHJ+fc#W#E<%#$J z_v~Y0c>$AHi;RLv(^*}S3lHlLz?tJ&NyBCnNi$dYE{33)>2sxOc!(9dj}@$r_rtOB zY3Q6J?OZ3WZ?A-K#wPPl?xkY|zW`X*FDS?RjP$Mv@iZ=al<1HY{i9a4;t;=BljQ$& zPAh>ew8NF&So{j1cR<$S>enljywU`&7+6T6AS|Whk-}gjR3gWIM%n zflB9@P*of&f2qr0;Pen;l2fhV06A9PWoP$4#&ql#T;|98@sZJ^0dP8Crz%^K)uM~f zRce|FvxrwOr5|T$IYlC+Cjl!2ec^R~NOx`lE99r8YpAZjUp%Fv$5_#O0EQE&v{eyP zA}_?#OwFv4E+Kod#R?u_3%Z{)X%T(sU>NJ|hKb+S^|?&!J>r|k+i4i_CVRx2Qmnfr z!>+*@Yf81&p}csZHpGQ?w55YB#U)}wb{{Gc*>t56hnmNcF8K;;u$;QrgynQkg!BOq zfU^5>C>v*fMLKWC;RGBq-3)eQSncYE^(#EP$$0fi_|5B>dpmQ{Z9S)B`e&xqs=pOm z^W+;fZ3v}47q5_4Pi8*&!vtL-&&uYv*oXryBo~Aphfe}L-+3Jw%ePTEOWMg|GdKIf zIm*ZsO?ziKwgS|c^iJo_%fJzTPr}6tUTNf`=nFi$VO%WwEnp3?I9)|3aVf*tqM$51 ztRpKc}8f2*Fq)gV2;08rK)uS%c(M*!KUCIa530M{$PsnVy&^KiMqi6wi9 zqF*a%OFA|p8@pL9ad34k6mhib z$Oc+O*H^#sZuF^N?LrhYt6u;6VaA)c3^?@${>-rM9FDTQLY_9lG{+)xx zBl2SWl+}6}*Q4);>C?$;16RnFgLG7O$OqA0`&a=g*tnP&7q!)4F2S-EzrpI~grUQZ z{ZlaOkYvq$84cs8=pbZSM;?nme;uurWSj$grJSdjCPMVUB2Q)E2p=cVht@~Wp%{2- z2J}Q3Y<>MQ-h~DT-Z56P#lb4K0?@T-Aim<$;@W(a{#J(guUNrk4Q$=c`pA3pcWo{% zb%6W`HnF^S5xK2=+0e5UlRgU%DaS5wObKoTMc=Wtzbys*0Sk{6oXCox|L&0jUDq&! zted|qnuFJHeF3wqFaXv7T@DID>*IOOn#PqS~e(G8oo_BFW)9}x|g>_M<^F#VLygacKik-k1)?i0^R%2@AghX zz2NzJ!TDP}tkocZgu4F2nl7d#$BMm#ij24wh%^3AcJPo#W7!c*E$q{aF6Q^m@0?tF z*Rkh(?&2n$mY4PoJNzc;b0Fw=?)dvw7Y_gYeDWP94VH2V zQ157X)l3HZ@hp8=@Tx}AekaX2Rp#tkd7x?9+dg=}AZdjL_7 zSfGQEL>VS#bFTk02{T)*{}&X@=XhAFpQk2_yV3q(jfF)dvK5|&Bj9F;71e>8S$Bdb z0LdI62}X!85pZS9gecBLs0+LjcXz>5^7YzvW~XnWGJ@--yo_uzu*I{;H?{fW>(b%Vvm z;4P1(jUqJ1YClV(LF)zrA+UGS9!D#7Pe%7$EFan4x_|hRzbyaeTu);-{P6MvjUkUO z{J`?By^S7M_-?uO3s6s#ME1Zm`)j`O<)+9E_;ycj?KwKMa{V!=3He_{I?7Nt!cZ{J zyd5Io_kCXB_#;>|ZgO?*gR?*FJ28-Tx+Z-V9NBd)(HZ*^?iyD}4|{{TJl@Y!y-x4r ztOJ$p`<$oK$dT-$IOj+;y^i8Iqi1)&dgzs0^Jz&UC(%<^hAbOH7E4QEB@}kWBo0U| z0lGip{8PsKg5RA92v}(Gu!f8B!rcNTi>R{SGO3jTy4X)Y)nnD;#U|QY+duo{y^H$H zz!WxfI0puf9gj4#xCqTWu#-c?^IV8jlMA%~)6&Huqze@<>bV3^97xRn^x}t8x`gq? z3Ql1q*q&V`S>g8pS~*)KU80h3ItHyXe31*$DZhB zB(BwPe6t~vYLK+0R>6G-CWItfKqr-{qz_e65t44#P`g#qDM;$5Qc&Ae(y>TdTdtD+ zBuS1HJkDKm{kSY0Y^h$W$6e4D(0bcVMY$7MH3FIONciWTBfuyH@Li1n{8uH2HuG7M zc0bQ0U~hu=k3ESuQ^tnfL2y0be^cQ5@aTS)uS$9$06n>N2i`e4X(l5W5rB+E)^p)p zjC-L3rY40~|>j;ytVA>T-6|r;xOU_PsMv=-L0Ho|tYswQi!GSNB zeC z>4bqG;57xXWI9lq0F+?HejBwiFbi+#fUyMXGSwmIo_LSc(-*?j%S}e)B_&kFtFCqn zd5NQoJX2aPx|p>Xs&`P9Ex?tk$Sst75s$>+ehRv_gCBFU{Yn9hs{kb}#OqNX7Oe`= z*B@F%TnfsKTPvERGxA#Ndnw`R@W+H^;tgFD+!woA-#y5%x+O=~O>Mn$%s~Su#yh4* z&XO*7uT9x9N8$5#5g$K-1y*6oHQ>iK+jvy5_*0sC;%qB5sw6E9wNYU z1<P~S7GMI9pI(Bj9~o&tYx5xn*hujEPY z<3QOXp>vw6co`{)y-L#I0b2{jn2$K0d*Qg+)?de7#W1$7R(N+{?9s8e@kn%RBLRM` z0Dh+c3a6oCU+y5aG3H!0STR8NL+czcJsr^sB?Fu=HJr;SewuI`$L7)$$L48=b-tP2 z`Bui@9gfXI9Ghnj7`&tN?aaYDOQgH5O=)NgpB8R4j@^ew5@UGF!Z~umov5l^~=1XtC#3Rw$2MBNSB8x6<$rZt%)}3 zdPd*8UE!WIqm_u~L71PxsAI3xB&l>SM#{bn?3;Ol@(7}QQj&;oC5)x8-mPGo&IQ)v zfVC)~yTZe)&|dzK*XoYL4EPdo@@j$eHZ%PVNNjGUA?^`!{i*l|KCptg)J8g`JMbba zqzt3g6RZ_y?GVt_(S0`5#bK`jt@bY!78fWi4*nEa-19gJ#U|o1o}p(@Gd~Hh20P>6 z{P6L-o{Lw<@vsWaY5Nmwn;ic==;Rmd6WP6qi`mXc`oygqEtJg3wGPWXQa^%vqI=YeM zf-!YrU{^PcCe~rN)65JHK_;4;v_Je&QV^e@78|jRTgyY#iF<&U{fezph3if*mtF&R z4Po>vwoX(qE-uO7Hm<&S_}N7S8>70tZ?1HE7iMMmZfEpFL~Vr{U6F$b$8$yATW=T{ z#*A3%6wFV)nr_lor~U3b5C8`)=9uU@H%Q?#ol)$7FdpOhZd6kDOJLA`wrb`k)y!@@ zx*9+=SITz0IW_#dipW(oWx}Eb(SSbN4I>hl)Z6idY{`!FIsGm3l4{jAl~!0){VDO3 z8UdSBqpHBJA7ryIG4@-2*?8gLvm59p!usJIc$H>;&3T3uUr6mIi_X2y8!t;Rgj>Vg|bHDw`5S&oMgR)cynfCTHoLs8-PWP z7Y^_wAj+~PUF!lm#Nw4v<>Z7k$;oi+S(Hq0Wu0oK}o9@m+Xw`q`UJp&+tnIN>GVv}ZhEPE{q~%N#{bmrceYzs(RBqm5 zMHganBt*>;0-Ub^PEr6HN%5snmXe}IXAo?L0{e#Bb{NTuW$lA185c~CJR>2aIsh7x zc%YAtv{AB2y`Y^kJlzPN1C{=jv~FZQ929FU zze$-drbKSl*=+?^;(EgReBa+wyuhI=`+dN#4It?IQO zyiEePD03wo4%xc@>ZK5*@%x@XS(hSLEJJ%Z6=#8Y?KXI9^eO}Jp~J1Wx>?cV>D?dE z0`@$9&7;|fMM`F2`w8wYFaOIqe*ri0WHe;!_=0`2a2K{qarT#)p!XmU5-hFr0vWo6 z1t9A&f1z;w48a3rESYmIHTqpVD?4P7;7TWu4)5gQVdv$X#Gnpt_y)PZDs^Urjua{= zo`Z=luj0fQ^V7n0BWvbVc8b+lf0S?Bf~ifeu%4@d5r~&3;)@Ra4x)>3f}CB%y(k19 zco)iu%>vVt0|EB+!~Xk?TH}jS&hHB0{Dc*t&u%=nd|%2R8m{t`Ih)?XuCE=8>J(A-mm5a=y@h| zoT#zPZaIEnlFhn%vviP`D61FxM7fzvj2=5h%HR*Sit0WV3)a5&pc=$)VYb5hiy_In zU?WHtOB*cI!3C;&Wh|YWbFvnBE2EnsvHDEyo zKH;0nl%WDvKT005r`f(3CTN~tg^nmWfpt8)sv?@5|H|}exZXAC_0SmS;_gh>fQBYH zkfT^PytRk9_~yvmAEkwtu=J%s*IMeug1aX z=9HfBMcu<&zD^C#!jIJO`Hn{`aZqe$_iOpMq72@CL_|9ye0$>i)^R@v+UT(9jN(3@ z$)eu#?JKA@R!~H<;FLcD0xou=3{Z{xG(JF8_bSpQAK z0tCY;!rOr99!vpI>bF|sfiF7bXt`ow=F-8@Dr~~0W^~VYuFuc;P51RV&mYN^gM~>( zvHri0F>V@jcIR~dG_9$uR_)*Dn7I)X!ai_ym1@9x4!#)F{#MHsX(N}uf;Zb8gL73x z1s5iKJz&X`#jEmg3GCG&9I%2BchXaVN6QM|$8AXvP^DBkaF65Ir<}19ASYkU{|bDX zOl=-2WgSr{-r*rmyeifeo72<%fSf^n%sCd8n?M-Q|L zR)+c9UczM#g9i+?;2j-!^wN7x!jvkQn1Wf0hxIuu^v)z__IK8dU(*Coa5LH!r!`HX ziQBcnI_AnD-ioXR>Xdf0r4VwUWF}7-fT+M!>l+n#H~dZxrJ8$=mAe3^W8V>VhQUGA zdP!yj7YemJ1peRg2cU$>Tu)(BMveFh=lZebKd;+a^YD30sduh_OsxI1LJ+Gh)sD71VXQM%p=?b z#pI+5_@|zKnmAFKoYA2#_2#QTqo#H|`3GhcVz#G6rxaS>@n8(5|FPXy$9?sc)}9;0 z@AQ24r35b~GEPL2Qrn zap@1EqvB>^Xpf_hL>A*E`!P6ug{`4|B<>1)$ErCQYE!pUJofKzIUI;sS!@~f zC*NF7%pR#sfdpFenKODqmUGhcVo$iAHJuE^L4bT!)1`P=zlQL0j)nGBpXQ7JJf+(? z>9yd)2+yRi;#23?53Muq=VX9z8^IIe&>{H9vChdlXv=kPFfpIP{7G_;B};q=_2qJ1^ha!J0KRzSbv68iXQJw31Az#zXBWpn^+uS2Vhw=Z~{6V zotMgqnT^U)qw~`7NRQ4NipS9CybL@tqVtC1F+4gi6Ax&78;Vi5f`=P1l+mFrjweeZ zLt7~_TK8v!KX$ct6jwxEik1{cm*EQiPWZ(-BTsZ%LrOY39i2}ME6e{|N#sEOXVJ^P z9dMah()r||l+PjuJ3k%J`NR>(%Rh*pB~zS}5NH!_#{wK$h8sUJ%Od~Ce}(TJAC6d+ zj4~|z^fC}S6_}p@8GwtLKEr`Osf z`KV8^%=SH3HtMai`~&5Y-AFu8KI+*NEAn0ElZSOaGc5m^$brtzA^FdgMV>3me{$3T zz?ESwoj~W|2`OD=k@recwx?kIe^VSfrabaddE~hg*k~eOmqiXnp6#NmJY{HBdCL1K z2-hr2!E7i?fg7_7KQmH7S;Nax7%ST9%Q6wh#8*K?Q?tq<7>mQu+tf&DDjw;P(scGc zQaTimj7Vt)9>XK0!|}+Bl)|Kuwuc+HsG2j*Q!^0+`xvG9a@?(uxP=(e(VR`@1?_<5 zWLpTM9;VYK80|1ax@b|piFV2Li{2l6>B>JW+DMnxAm{qP-o`1D|1slkXXIQ&<6b@U zhzoA0PrD7j@T$rCUTPhA+%s=pgkRjBM$R=O7iKfMBzhqvp3;_IM$PT8fU~g~s<9R} zb}|~nHGbG>RivD2LKR!z44~Gp&sdtaz7XHW?wl5zH6RbA_7GFv@YA{XE*Ffp&e+dV z@jfDF=ck9ohQ0t$BLZ*6j?Om^FX^;0M+d8^0)BI==^X*N#~nCM$+IN zk+O|>(W&tDU6I%M=77OK;0tH#6L1)Oc$fp3zk+mn*}4QQH@_n~yw&kkWY$W#iKBIY zdKkuNi?Jd}-qIxh{5ovHZJ(3}-$B-eWt8o()7Xpi2Etqel+mve1=EO!b@U%G`lYZ< zeTyXx<{B(PK+KjF08Ssf4}_SI=bE&S_uM$nJqV>42buQXRTgc?E>HQcEYg5-UzW6f zSG)ufhvj)?G@5@{()!-n&_`R}$GBh`Sy9M|^J1xI5xfi+ezm^YA%X!yFf9+S)*XxY zo_%xn5|1nL+Vt4O6H30?HXwYwtM$DO*)e|D`p((lcX$DqT<+C<7TzLSG4>N!4vUw7 zmgRYJY?@x&){l$P?B2UK7qgdD6vQ^5e=>~N)QgVB~2;!@mq!uvkohg?m| zlUpQq<1ShvkyR1%qVf!$2dx`MAf@E5$l8HS&RcLWD#5Ra$VEPnETcuanUBPL!^);HD!N)q zXzll6>ORZ2u>pm{R($(^`!mA`8ZO*OTWfd`Wl?Q}@bu$%5Qt1SJopwiOi69PFF$^T z@Vge@>j2q^XDxm&G?w9Sjg(~iBESWbYBL4e2k2bXI|~rKYxAPLYScoAT4NmQV!mp7 zDWHA$X9B`!EXChtfM%N>{0$oQXmu%IS#tto_9wZ{T);X3=@Wcwbs5uGj9Q30d(Jxj z_!k29P52eU_e2W`oY;~vTK*Y1^3O2l8|B71pyYg`)R=*Pvy#3w8>br+fuEvT|FGv8 zGmY6OYs0GAj5r-~KtVrfT7z1WQ|r;sTKprm43t;+&j2LT5FKP(2ndcZsYJTjFea!~ zjht$XmV&f%&d2X&^gbEW3aKTGbOR+g3;gIgA$<5&ty`Nfc!bcd7vDICO!$io++}$d zzJt&zu@Oa(V;I>eHx`L?Ql}5Knn(*7<3Yc%c=nzr*?=625?&1#0NhNZ1f*JyE%8$G zm-RGB9(%9qt;cTI zI5v|sgA{Tfa!j>6lIi0DmTHB%gc=-Cuh2`a*}m(~2VCvdn2Y`@>h{JyndUZ~ z6lb4--g-bO@>H(>WK0s{*%zu9^sL5w=1l*-G8KRz)omW5U=y6gk6iMW`Di^cKJ$Tr z7b8Sj!kIK*<_PJ;(G0}bP!Kuxnn>_jQmzKPmkXZN0Q1*CpC4>cAa^Vn0q5qba$$TSMvL!)pIJ?zy z%wfVLTFuQJ9}LutU+DLT#(V0#kD}s-3?)%+rPvUsPp-k zW~%~?D8p}KjhP-cX@>k}oj1_1v^wB1y@;C+)K%Ab%tqt{!%a>8K*&>@Ym^59{(u?u zgiz}ub4JkBSnCC1UVme7vN^iJ=-UU@oLgf|_1AjLV7))=tCboTddz^Qx~AS!Yt;DB zR%6H*GR!as^H-gBQO;S7Zlndgo}f9!bd}9@PcNBkOh-(=8S*xG{9)ka5sXKN&0wxE zWVA16Rt7x5pjq9x%v|hQ#@?`IZ(|t5DX*Gp0`1F~dRf>T$~947$PlB}v&38DF>=nC zQ=W59RgO^#Y)9u!Y&J&cp5hCR_JwosAD~9%%<@^Ye(FBIWIAq_pI&lqx!YByzsxD0 zS>>8>zM-GYD4p%9DC3tRV?G9Ceg&{-p3izh-jENB8^p{3l12kafRhP1-de*)7z4v@ zR0oh4^3;UF0q9nzPt;h30v?ayZS;l$(9fEL;s(3TY8Ps{g#$7a7)P|Chia)e#6}j8 z96`U&V>VzcWUSa*jI^m}s0r&1cp9pac$S%G1eXOvo`%{SnI01gCYsqmXY4q0HacZi zHF;{hb$akoNABpn3E^?(98Y5{NG+JDG{ZD|RfK66#C(}-qQ)V}v_C%1=~~mRUlyU3OL!<9Iu=B1wAzwMw1kjy3K5KY-}$O-Hl+qx1m?_@$T`HNb@n}55@uuu`ldV?R*vV!CmQzlhKTzYb(fJ;jdT(v5r}6uNIDd3* zyavvYCDnocn)yDMUd{9al+=t_>#3^_`$A@-H)_6+RPnq74)M%*o*%0Y{3rwjzdy?B!GfUPc8V&e;zR9M)u1*a^f0+%{jp1tF_vd@-a)Q-$dNTA^fjw9ByvrdY zNg4rm5w(9$?qYQ9`*Rzrxop0tD5mugGB|QC-DvXABGY71$c^}@T^=5fHNcF-$6FgG ztxgnfoRlcPDW;YtC1OH=T~ns?N;KnaEIxk{+`Km=W+cT-pG3vl`Xnm;o|LGHlJ_Si z0(~t3>_m~*jnzK0rke9WP^$5VDgPD;^PzMWj%w3Ggewxgrq5Fs8pkiSSQJ9ba3Fwb z46YlhmtquMnwE{8=0h@+psA^^Zd}x_oSB*=cuWDq`hq0JxY`N@M7LHc^f)J^RIkzk zCuM{7I?wGTrnJ;sq-(1-MN_JEQK(b~{2_mh-)92TTAz}EO6a&n=olg}G>7HcKnIn$>wZ8!=lNbPGRFWJaT>3ZjjB0m|S+$>ffKp9P`a z290u&9cGnMR759ao-BN0zT#5znIa18`F>j`s{?zf%{3LW$%3fHI}NpZ}@(S4Sll^+UsG^gxLC8_-?NbE<1=xv`RV z6EyZTW4?qu#7K>a^!OW{~;MYJ@?`N z4LaS1X#i|hKk%!M@98-RScitc9%cL1`S1Mx-?Y#AUmT^{ZPV>n0I#Qp_w-CYrl;rd z<9d36$M*ERh2KN<@3bR(dhGxAAkY5(uRQ;NI!He?E!<5qPhjk%v~UMKK8Pewt?ZYV z5iI^u%SmOx6421(7L@?oF9p(snGy?3&cGSASWnTHS78&H!Nv_fkxC|I(m4u2LuJIHbIWwIOsqYjroXGG8s7CN`cn zeT0#?*?wqQlZPjd^(VQ!rmdYN&qLY=;UVj3D3uQvlpE{X4Q>pg`Zy-ZA6;H+6G7>| zyDs2waEm>lH-56lVPHu5D52`cr0}DJvIHe%+vTPI7#Vwr;~<`9T8ZMr9X!++R8tFE z(88Kpqul4gY_rF2#==W6hcO4Qm{jYjEt?n3%^8Xsjy?8u=3>iMIK^q~h;jcp<2#P}cbZ2c+CGv2i~TjC_Z> zEDRW|d%Fo0P7$1vFl8kS#}J=$JY2U!Z4)flSwjSoUK zZ(Ni^>t9Y%Iw*&gU*~N$n*6~KpRia2Vcn^BH~QT*)xr2!+UfCu(OG17nL(rTD+kc(}kDyU>ivm2W~ zW10Nzjj?Xr4dxR-K6ab^=W{twupWZF+C1e@bPG9QOzmr*JOl*heoT|vCRm!s^D71U z=-@c@pd^>ym8dd%ba>q8@MxF`p@lRaf*q!SZ2R179{4r+U{1wQC3=yZk0pn3@7=%T zeD1sK!kRDksvkTN!`{0*bQF&)eRon#-$McJFfqy$1|C>zZ9A>9I@=l!fuQmkLkl5mM5+79PfY&uc^Vgm>cybg8$X2=D+SL%%U~(Otr|QH z*kalAlY9{bK&@8~lgbNHy=(8cd@qos^16K^q2Pcym_Ddiu&L<%VqXKFA9&d@v$W|Aipd_>OZUb>iLDEBPn(-WF5w}4sz~_Jdizo5gB{ZAPcL8q zjL`mwV1W(7tr|LHrh@t9DCP)~Bsn)IYU9q0nou*m2jgUy0S^Z^8)J>U{0XN`EI9p) z!bv5iQ_ITFNlK^&=l9eZuJheh<+HMbO*v;R1S=h9=I7!6>0=F5AG{Fi3+J}s)+!NI zG0Z%-8Jo-pD2GENho5;n3M!#>|w0}QaC8jQsl2D4VEIk!>Op4%vu*TSj84T-%= z$WyG>sNk4-=zcL=jgbCc3}STvOAb4LWRrnW3OKz2S%*=oR%6KTH-prujPtQKM%S#v&X&F)H9+>SDlkm!Jz4oV+=VJ8 zBuH|2;pPgEU1mc+35&aUti8_c=Y~srdKwzl|6BC;=e>zEJ1>ef8}`qDyHCSz!IOFa z?EmRL?03-4zw*CyaZgWQ_zk1D{YIAWQ-@t2)7kf?aIijE z+Aa$Cm&(%9gpvHlFm4X%{ZVE^w_3h#?O4L%vp=$*s-ymlZ8S_w39h4VaKl1RApdmWlk~gLt_)A`7I$4ESVc{}<}@l(R)Xkp$_Yw4S1AU!blx3+!*t2{p@e+i)P#(dTRE} z{HN=$?nyO<8ADK{w@gUH)=Wmk5TnXe+zlfJtMQ%6Goy#>h%Lq}FxF2+!Y~T39x03p zV+6(`iC+4l7ODWrh>yi6StAh@>bqSpRrY_s*4bz~u_0i5SOEVR_tbw~eYH`ae-V|i zB)98ivm^+vBfifuqhZhD@&{9ad1iEQ^5}3ebW`Xjbn-(Us}E{POv1iRGon_Z;g`p!GM8kC># zuapNIYg6?eGW)R_LpinHU=3A@WuOyx@v;2D z|J)s-8QFj#SOpNk`VGg6GEc$};SU8>6B=`Zuwc586?NOss-B z{~`1UU{3iPa)Qtd`z$t7Wx;8W73Qm62vg{i{3Io}x<;g`IrIE;Mez1ftVPkMn&t^q z$r7IBUkuODA!c}$iM7C7QOoL~(j*Mj$$GW&JqgANE$SP6+T~V-$FGzA7Ans!FfrStgjbdM? zlGz8rK0UMvFAll}KVp*njoun)Qt)t!FRUSHlJ&anXNw;t09OEQatYQOV_9$PK znM$o8{2MiV(z$Wd{kiL_cox51W! zzqy#PVdOLN@;sv)|LtdTPNk&tZ*nTd6jt3aq z7;)pqHvW(mfH*GEt{C$n!d&Umj6g6X8kZ*(7+FsZl-ep|_!&d(hbGE+cKgGKFlEG7 zhM&GINqlIHu?$=DPyo(=MuaF=`yjhjESmu%KyO0=LShUAG{VXkC^Ty7!;Op0DMet~ z4EVzgU^=1?1J%n|3_+tF?ofhM2=ie4hfffjP-R)r59Pk(Hi44|bq90B(3Ojg=l7o$ zs=_GtJ`bRJ%<1Z0B1Q&Dkl*oXYk-WMTe}dR*tlatgxuVh(Wsl0C0~(_B>}z|f-9lHe^H`Ff&V zRcjLdKhz%k_e1U3`SJD=^&ZlW^jGbFHAxJ2)%4Tc{OdksX)S5s%HnjxHyX7Q-!U!b zOrKr~4Shxc`WIC3%bN{GHkoG?nc|1b&pE3eW9|v$DuZ+)I^U*guwkLsAH>wcp2P2H zh~v-3I2UbvlAJQ*gZ0%D3Qh-B{o4ewZbHq#GKghkC|GRel^9*PEQFPt^y!<=T}D#2 zT^{&(VZdO;y+67!QczO#1xD&wqN?HD~|w zg-cGj@0Q;W+LLlo>rtCm=4WlaYUu3H#hU{M|MYsxy#rspE%f-o5tR&7VH; zn@-2mYldd|$93iWenn{7tN*iZ{>#@NSTgg1vu(8zK>qfZs((-HlWv2&KTg3P)X$6@_1o0Xar$|deooU* zyL|I$3jSyM8PU(%^z&Z*d{jSo=;to|d_zA!)lXxhs?VXHKi1DI{hXkmQ}lD1e$LU) zYW=)aKbPp|wfcFZe%`8|cj@PY`nf|tU(wI^^z#e-Gzt`cN9gDA`gy8;PSMXw{k%v& zYxMIn{cO=sn=Uu%-?!@L-TL{?Q>#9>>c76?AszV)oTA#de=jLjzpvEKzyA5S-II&{ ztNhMQFYi2Jk%tH0dVOxtaI5vxeleE)R;*sY_;HVhyZ>K=?>is+*7u*^|7R(1GCn#) z(FIGx|7Y9$Pi_C-p@6HbDA_bUM4xynItfF6(h#1FDmp0{fCqd|qN)Es zpZ{M_Ah@tPG%kN!RcXmA9P^SRQR8%4eJIovoIHN~B7`G_7t%yLK2R@FMBK!VhrxW| zcX?wHHYWu8hf19x;eND!i%r)kk z>@|673fB~`ajvOcGk48`HT7$n)-N>$C{mM_N@7A cjd4f%9eH;Y-m&10`a6CY!T;p`KS_cA3;Bo?NdN!< literal 0 HcmV?d00001 diff --git a/internal/pkg/secureboot/secureboot.go b/internal/pkg/secureboot/secureboot.go new file mode 100644 index 0000000..d5e2b3d --- /dev/null +++ b/internal/pkg/secureboot/secureboot.go @@ -0,0 +1,87 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package secureboot contains base definitions for the Secure Boot process. +package secureboot + +// Section is a name of a PE file section (UEFI binary). +type Section string + +// List of well-known section names. +const ( + Linux Section = ".linux" + OSRel Section = ".osrel" + CMDLine Section = ".cmdline" + Initrd Section = ".initrd" + Splash Section = ".splash" + DTB Section = ".dtb" + Uname Section = ".uname" + SBAT Section = ".sbat" + PCRSig Section = ".pcrsig" + PCRPKey Section = ".pcrpkey" +) + +// OrderedSections returns the sections that are measured into PCR. +// +// Derived from https://github.com/systemd/systemd/blob/main/src/fundamental/tpm-pcr.h#L23-L36 +// .pcrsig section is omitted here since that's what we are calulating here. +func OrderedSections() []Section { + // DO NOT REARRANGE + return []Section{Linux, OSRel, CMDLine, Initrd, Splash, DTB, Uname, SBAT, PCRPKey} +} + +// Phase is the phase value extended to the PCR. +type Phase string + +const ( + // EnterInitrd is the phase value extended to the PCR during the initrd. + EnterInitrd Phase = "enter-initrd" + // LeaveInitrd is the phase value extended to the PCR just before switching to machined. + LeaveInitrd Phase = "leave-initrd" + // EnterMachined is the phase value extended to the PCR before starting machined. + // There should be only a signed signature for the enter-machined phase. + EnterMachined Phase = "enter-machined" + // StartTheWorld is the phase value extended to the PCR before starting all services. + StartTheWorld Phase = "start-the-world" +) + +// PhaseInfo describes which phase extensions are signed/measured. +type PhaseInfo struct { + Phase Phase + CalculateSignature bool +} + +// OrderedPhases returns the phases that are measured, in order. +// +// Derived from https://github.com/systemd/systemd/blob/v253/src/boot/measure.c#L295-L308 +// ref: https://www.freedesktop.org/software/systemd/man/systemd-pcrphase.service.html#Description +// +// In the case of Talos disk decryption, happens in machined, so we need to only sign EnterMachined +// so that machined can only decrypt the disk if the system booted with the correct kernel/initrd/cmdline +// OrderedPhases returns the phases that are measured. +func OrderedPhases() []PhaseInfo { + // DO NOT REARRANGE + return []PhaseInfo{ + { + Phase: EnterInitrd, + CalculateSignature: false, + }, + { + Phase: LeaveInitrd, + CalculateSignature: false, + }, + { + Phase: EnterMachined, + CalculateSignature: true, + }, + } +} + +const ( + // UKIPCR is the PCR number where sections except `.pcrsig` are measured. + UKIPCR = 11 + // SecureBootStatePCR is the PCR number where the secure boot state and the signature are measured. + // PCR 7 changes when UEFI SecureBoot mode is enabled/disabled, or firmware certificates (PK, KEK, db, dbx, …) are updated. + SecureBootStatePCR = 7 +) diff --git a/internal/pkg/secureboot/tpm2/keys.go b/internal/pkg/secureboot/tpm2/keys.go new file mode 100644 index 0000000..6c3f43f --- /dev/null +++ b/internal/pkg/secureboot/tpm2/keys.go @@ -0,0 +1,73 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package tpm2 provides TPM2.0 related functionality helpers. +package tpm2 + +import ( + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "os" + + "github.com/google/go-tpm/tpm2" +) + +// ParsePCRSigningPubKey parses a PEM encoded RSA public key. +func ParsePCRSigningPubKey(file string) (*rsa.PublicKey, error) { + pcrSigningPubKey, err := os.ReadFile(file) + if err != nil { + return nil, fmt.Errorf("failed to read pcr signing public key: %v", err) + } + + block, _ := pem.Decode(pcrSigningPubKey) + if block == nil { + return nil, errors.New("failed to decode pcr signing public key") + } + + // parse rsa public key + tpm2PubKeyAny, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + return nil, err + } + + tpm2PubKey, ok := tpm2PubKeyAny.(*rsa.PublicKey) + if !ok { + return nil, errors.New("failed to cast pcr signing public key to rsa") + } + + return tpm2PubKey, nil +} + +// RSAPubKeyTemplate returns a TPM2.0 public key template for RSA keys. +func RSAPubKeyTemplate(bitlen, exponent int, modulus []byte) tpm2.TPMTPublic { + return tpm2.TPMTPublic{ + Type: tpm2.TPMAlgRSA, + NameAlg: tpm2.TPMAlgSHA256, + ObjectAttributes: tpm2.TPMAObject{ + Decrypt: true, + SignEncrypt: true, + UserWithAuth: true, + }, + Parameters: tpm2.NewTPMUPublicParms(tpm2.TPMAlgRSA, &tpm2.TPMSRSAParms{ + Symmetric: tpm2.TPMTSymDefObject{ + Algorithm: tpm2.TPMAlgNull, + Mode: tpm2.NewTPMUSymMode(tpm2.TPMAlgRSA, tpm2.TPMAlgNull), + }, + Scheme: tpm2.TPMTRSAScheme{ + Scheme: tpm2.TPMAlgNull, + Details: tpm2.NewTPMUAsymScheme(tpm2.TPMAlgRSA, &tpm2.TPMSSigSchemeRSASSA{ + HashAlg: tpm2.TPMAlgNull, + }), + }, + KeyBits: tpm2.TPMKeyBits(bitlen), + Exponent: uint32(exponent), + }), + Unique: tpm2.NewTPMUPublicID(tpm2.TPMAlgRSA, &tpm2.TPM2BPublicKeyRSA{ + Buffer: modulus, + }), + } +} diff --git a/internal/pkg/secureboot/tpm2/pcr.go b/internal/pkg/secureboot/tpm2/pcr.go new file mode 100644 index 0000000..fffb9e7 --- /dev/null +++ b/internal/pkg/secureboot/tpm2/pcr.go @@ -0,0 +1,194 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package tpm2 provides TPM2.0 related functionality helpers. +package tpm2 + +import ( + "bytes" + "crypto/sha256" + "fmt" + "log" + "os" + "strings" + + "github.com/google/go-tpm/tpm2" + "github.com/google/go-tpm/tpm2/transport" + + "github.com/aenix-io/talm/internal/pkg/secureboot" +) + +// CreateSelector converts PCR numbers into a bitmask. +func CreateSelector(pcrs []int) ([]byte, error) { + const sizeOfPCRSelect = 3 + + mask := make([]byte, sizeOfPCRSelect) + + for _, n := range pcrs { + if n >= 8*sizeOfPCRSelect { + return nil, fmt.Errorf("PCR index %d is out of range (exceeds maximum value %d)", n, 8*sizeOfPCRSelect-1) + } + + mask[n>>3] |= 1 << (n & 0x7) + } + + return mask, nil +} + +// ReadPCR reads the value of a single PCR. +func ReadPCR(t transport.TPM, pcr int) ([]byte, error) { + pcrSelector, err := CreateSelector([]int{pcr}) + if err != nil { + return nil, fmt.Errorf("failed to create PCR selection: %w", err) + } + + pcrRead := tpm2.PCRRead{ + PCRSelectionIn: tpm2.TPMLPCRSelection{ + PCRSelections: []tpm2.TPMSPCRSelection{ + { + Hash: tpm2.TPMAlgSHA256, + PCRSelect: pcrSelector, + }, + }, + }, + } + + pcrValue, err := pcrRead.Execute(t) + if err != nil { + return nil, fmt.Errorf("failed to read PCR: %w", err) + } + + return pcrValue.PCRValues.Digests[0].Buffer, nil +} + +// PCRExtent hashes the input and extends the PCR with the hash. +func PCRExtent(pcr int, data []byte) error { + t, err := transport.OpenTPM() + if err != nil { + // if the TPM is not available or not a TPM 2.0, we can skip the PCR extension + if os.IsNotExist(err) || strings.Contains(err.Error(), "device is not a TPM 2.0") { + log.Printf("TPM device is not available, skipping PCR extension") + + return nil + } + + return err + } + + defer t.Close() //nolint:errcheck + + // since we are using SHA256, we can assume that the PCR bank is SHA256 + digest := sha256.Sum256(data) + + pcrHandle := tpm2.PCRExtend{ + PCRHandle: tpm2.AuthHandle{ + Handle: tpm2.TPMHandle(pcr), + Auth: tpm2.PasswordAuth(nil), + }, + Digests: tpm2.TPMLDigestValues{ + Digests: []tpm2.TPMTHA{ + { + HashAlg: tpm2.TPMAlgSHA256, + Digest: digest[:], + }, + }, + }, + } + + if _, err = pcrHandle.Execute(t); err != nil { + return err + } + + return nil +} + +// PolicyPCRDigest executes policyPCR and returns the digest. +func PolicyPCRDigest(t transport.TPM, policyHandle tpm2.TPMHandle, pcrSelection tpm2.TPMLPCRSelection) (*tpm2.TPM2BDigest, error) { + policyPCR := tpm2.PolicyPCR{ + PolicySession: policyHandle, + Pcrs: pcrSelection, + } + + if _, err := policyPCR.Execute(t); err != nil { + return nil, fmt.Errorf("failed to execute policyPCR: %w", err) + } + + policyGetDigest := tpm2.PolicyGetDigest{ + PolicySession: policyHandle, + } + + policyGetDigestResponse, err := policyGetDigest.Execute(t) + if err != nil { + return nil, fmt.Errorf("failed to get policy digest: %w", err) + } + + return &policyGetDigestResponse.PolicyDigest, nil +} + +//nolint:gocyclo +func validatePCRBanks(t transport.TPM) error { + pcrValue, err := ReadPCR(t, secureboot.UKIPCR) + if err != nil { + return fmt.Errorf("failed to read PCR: %w", err) + } + + if err = validatePCRNotZeroAndNotFilled(pcrValue, secureboot.UKIPCR); err != nil { + return err + } + + pcrValue, err = ReadPCR(t, secureboot.SecureBootStatePCR) + if err != nil { + return fmt.Errorf("failed to read PCR: %w", err) + } + + if err = validatePCRNotZeroAndNotFilled(pcrValue, secureboot.SecureBootStatePCR); err != nil { + return err + } + + caps := tpm2.GetCapability{ + Capability: tpm2.TPMCapPCRs, + Property: 0, + PropertyCount: 1, + } + + capsResp, err := caps.Execute(t) + if err != nil { + return fmt.Errorf("failed to get PCR capabilities: %w", err) + } + + assignedPCRs, err := capsResp.CapabilityData.Data.AssignedPCR() + if err != nil { + return fmt.Errorf("failed to parse assigned PCRs: %w", err) + } + + for _, s := range assignedPCRs.PCRSelections { + if s.Hash != tpm2.TPMAlgSHA256 { + continue + } + + // check if 24 banks are available + if len(s.PCRSelect) != 24/8 { + return fmt.Errorf("unexpected number of PCR banks: %d", len(s.PCRSelect)) + } + + // check if all banks are available + if s.PCRSelect[0] != 0xff || s.PCRSelect[1] != 0xff || s.PCRSelect[2] != 0xff { + return fmt.Errorf("unexpected PCR banks: %v", s.PCRSelect) + } + } + + return nil +} + +func validatePCRNotZeroAndNotFilled(pcrValue []byte, pcr int) error { + if bytes.Equal(pcrValue, bytes.Repeat([]byte{0x00}, sha256.Size)) { + return fmt.Errorf("PCR bank %d is populated with all zeroes", pcr) + } + + if bytes.Equal(pcrValue, bytes.Repeat([]byte{0xFF}, sha256.Size)) { + return fmt.Errorf("PCR bank %d is populated with all 0xFF", pcr) + } + + return nil +} diff --git a/internal/pkg/secureboot/tpm2/pcr_test.go b/internal/pkg/secureboot/tpm2/pcr_test.go new file mode 100644 index 0000000..3ff8729 --- /dev/null +++ b/internal/pkg/secureboot/tpm2/pcr_test.go @@ -0,0 +1,48 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package tpm2 provides TPM2.0 related functionality helpers. +package tpm2_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + tpm2internal "github.com/aenix-io/talm/internal/pkg/secureboot/tpm2" +) + +func TestGetSelection(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + name string + pcrs []int + expected []byte + }{ + { + name: "empty", + expected: []byte{0, 0, 0}, + }, + { + name: "1, 3, 5", + pcrs: []int{1, 3, 5}, + expected: []byte{42, 0, 0}, + }, + { + name: "21, 22, 23", + pcrs: []int{21, 22, 23}, + expected: []byte{0, 0, 0xe0}, + }, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + actual, err := tpm2internal.CreateSelector(tt.pcrs) + require.NoError(t, err) + + require.Equal(t, tt.expected, actual) + }) + } +} diff --git a/internal/pkg/secureboot/tpm2/policy.go b/internal/pkg/secureboot/tpm2/policy.go new file mode 100644 index 0000000..f6491bc --- /dev/null +++ b/internal/pkg/secureboot/tpm2/policy.go @@ -0,0 +1,79 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package tpm2 provides TPM2.0 related functionality helpers. +package tpm2 + +import ( + "crypto/sha256" + "fmt" + + "github.com/google/go-tpm/tpm2" +) + +// CalculatePolicy calculates the policy hash for a given PCR value and PCR selection. +func CalculatePolicy(pcrValue []byte, pcrSelection tpm2.TPMLPCRSelection) ([]byte, error) { + calculator, err := tpm2.NewPolicyCalculator(tpm2.TPMAlgSHA256) + if err != nil { + return nil, err + } + + pcrHash := sha256.Sum256(pcrValue) + + policy := tpm2.PolicyPCR{ + PcrDigest: tpm2.TPM2BDigest{ + Buffer: pcrHash[:], + }, + Pcrs: pcrSelection, + } + + if err := policy.Update(calculator); err != nil { + return nil, err + } + + return calculator.Hash().Digest, nil +} + +// CalculateSealingPolicyDigest calculates the sealing policy digest for a given PCR value, PCR selection and public key. +func CalculateSealingPolicyDigest(pcrValue []byte, pcrSelection tpm2.TPMLPCRSelection, pubKey string) ([]byte, error) { + calculator, err := tpm2.NewPolicyCalculator(tpm2.TPMAlgSHA256) + if err != nil { + return nil, err + } + + pubKeyData, err := ParsePCRSigningPubKey(pubKey) + if err != nil { + return nil, err + } + + publicKeyTemplate := RSAPubKeyTemplate(pubKeyData.N.BitLen(), pubKeyData.E, pubKeyData.N.Bytes()) + + name, err := tpm2.ObjectName(&publicKeyTemplate) + if err != nil { + return nil, fmt.Errorf("failed to calculate name: %v", err) + } + + policyAuthorize := tpm2.PolicyAuthorize{ + KeySign: *name, + } + + if err := policyAuthorize.Update(calculator); err != nil { + return nil, err + } + + pcrHash := sha256.Sum256(pcrValue) + + policy := tpm2.PolicyPCR{ + PcrDigest: tpm2.TPM2BDigest{ + Buffer: pcrHash[:], + }, + Pcrs: pcrSelection, + } + + if err := policy.Update(calculator); err != nil { + return nil, err + } + + return calculator.Hash().Digest, nil +} diff --git a/internal/pkg/secureboot/tpm2/policy_test.go b/internal/pkg/secureboot/tpm2/policy_test.go new file mode 100644 index 0000000..7a966f5 --- /dev/null +++ b/internal/pkg/secureboot/tpm2/policy_test.go @@ -0,0 +1,51 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package tpm2 provides TPM2.0 related functionality helpers. +package tpm2_test + +import ( + "testing" + + "github.com/google/go-tpm/tpm2" + "github.com/stretchr/testify/require" + + tpm2internal "github.com/aenix-io/talm/internal/pkg/secureboot/tpm2" +) + +func TestCalculatePolicy(t *testing.T) { + t.Parallel() + + policy, err := tpm2internal.CalculatePolicy([]byte{1, 3, 5}, tpm2.TPMLPCRSelection{ + PCRSelections: []tpm2.TPMSPCRSelection{ + { + Hash: tpm2.TPMAlgSHA256, + PCRSelect: []byte{10, 11, 12}, + }, + }, + }) + require.NoError(t, err) + require.Equal(t, + []byte{0x84, 0xd6, 0x51, 0x47, 0xb0, 0x53, 0x94, 0xd0, 0xfa, 0xc4, 0x5e, 0x36, 0x0, 0x20, 0x3e, 0x3a, 0x11, 0x1, 0x27, 0xfb, 0xe2, 0x6f, 0xc1, 0xe3, 0x3, 0x3, 0x10, 0x21, 0x33, 0xf9, 0x15, 0xe3}, + policy, + ) +} + +func TestCalculateSealingPolicyDigest(t *testing.T) { + t.Parallel() + + calculated, err := tpm2internal.CalculateSealingPolicyDigest([]byte{1, 3, 5}, tpm2.TPMLPCRSelection{ + PCRSelections: []tpm2.TPMSPCRSelection{ + { + Hash: tpm2.TPMAlgSHA256, + PCRSelect: []byte{10, 11, 12}, + }, + }, + }, "testdata/pcr-signing-crt.pem") + require.NoError(t, err) + require.Equal(t, + []byte{0xa0, 0xf4, 0x29, 0x7a, 0x6c, 0x1a, 0xc8, 0xcf, 0x88, 0x48, 0x8b, 0xcf, 0x63, 0x20, 0xdc, 0x2e, 0x75, 0xc8, 0x2, 0xa1, 0xb4, 0x62, 0x5a, 0xdc, 0x9a, 0xfb, 0x2a, 0x1a, 0x8a, 0xd2, 0xf0, 0x53}, + calculated, + ) +} diff --git a/internal/pkg/secureboot/tpm2/seal.go b/internal/pkg/secureboot/tpm2/seal.go new file mode 100644 index 0000000..d793ba3 --- /dev/null +++ b/internal/pkg/secureboot/tpm2/seal.go @@ -0,0 +1,139 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package tpm2 provides TPM2.0 related functionality helpers. +package tpm2 + +import ( + "fmt" + + "github.com/google/go-tpm/tpm2" + "github.com/google/go-tpm/tpm2/transport" + + "github.com/aenix-io/talm/internal/pkg/secureboot" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// Seal seals the key using TPM2.0. +func Seal(key []byte) (*SealedResponse, error) { + t, err := transport.OpenTPM() + if err != nil { + return nil, err + } + defer t.Close() //nolint:errcheck + + // fail early if PCR banks are not present or filled with all zeroes or 0xff + if err = validatePCRBanks(t); err != nil { + return nil, err + } + + sealingPolicyDigest, err := calculateSealingPolicyDigest(t) + if err != nil { + return nil, err + } + + primary := tpm2.CreatePrimary{ + PrimaryHandle: tpm2.TPMRHOwner, + InPublic: tpm2.New2B(tpm2.ECCSRKTemplate), + } + + createPrimaryResponse, err := primary.Execute(t) + if err != nil { + return nil, err + } + + defer func() { + flush := tpm2.FlushContext{ + FlushHandle: createPrimaryResponse.ObjectHandle, + } + + _, flushErr := flush.Execute(t) + if flushErr != nil { + err = flushErr + } + }() + + outPub, err := createPrimaryResponse.OutPublic.Contents() + if err != nil { + return nil, err + } + + create := tpm2.Create{ + ParentHandle: tpm2.AuthHandle{ + Handle: createPrimaryResponse.ObjectHandle, + Name: createPrimaryResponse.Name, + Auth: tpm2.HMAC( + tpm2.TPMAlgSHA256, + 20, + tpm2.Salted(createPrimaryResponse.ObjectHandle, *outPub), + tpm2.AESEncryption(128, tpm2.EncryptInOut), + ), + }, + InSensitive: tpm2.TPM2BSensitiveCreate{ + Sensitive: &tpm2.TPMSSensitiveCreate{ + Data: tpm2.NewTPMUSensitiveCreate(&tpm2.TPM2BSensitiveData{ + Buffer: key, + }), + }, + }, + InPublic: tpm2.New2B(tpm2.TPMTPublic{ + Type: tpm2.TPMAlgKeyedHash, + NameAlg: tpm2.TPMAlgSHA256, + ObjectAttributes: tpm2.TPMAObject{ + FixedTPM: true, + FixedParent: true, + }, + Parameters: tpm2.NewTPMUPublicParms(tpm2.TPMAlgKeyedHash, &tpm2.TPMSKeyedHashParms{ + Scheme: tpm2.TPMTKeyedHashScheme{ + Scheme: tpm2.TPMAlgNull, + }, + }), + AuthPolicy: tpm2.TPM2BDigest{ + Buffer: sealingPolicyDigest, + }, + }), + } + + createResp, err := create.Execute(t) + if err != nil { + return nil, err + } + + resp := SealedResponse{ + SealedBlobPrivate: tpm2.Marshal(createResp.OutPrivate), + SealedBlobPublic: tpm2.Marshal(createResp.OutPublic), + KeyName: tpm2.Marshal(createPrimaryResponse.Name), + PolicyDigest: sealingPolicyDigest, + } + + return &resp, nil +} + +func calculateSealingPolicyDigest(t transport.TPM) ([]byte, error) { + pcrSelector, err := CreateSelector([]int{secureboot.SecureBootStatePCR}) + if err != nil { + return nil, fmt.Errorf("failed to create PCR selection: %v", err) + } + + pcrSelection := tpm2.TPMLPCRSelection{ + PCRSelections: []tpm2.TPMSPCRSelection{ + { + Hash: tpm2.TPMAlgSHA256, + PCRSelect: pcrSelector, + }, + }, + } + + pcrValue, err := ReadPCR(t, secureboot.SecureBootStatePCR) + if err != nil { + return nil, err + } + + sealingDigest, err := CalculateSealingPolicyDigest(pcrValue, pcrSelection, constants.PCRPublicKey) + if err != nil { + return nil, fmt.Errorf("failed to calculate sealing policy digest: %v", err) + } + + return sealingDigest, nil +} diff --git a/internal/pkg/secureboot/tpm2/signature.go b/internal/pkg/secureboot/tpm2/signature.go new file mode 100644 index 0000000..3cf5ad5 --- /dev/null +++ b/internal/pkg/secureboot/tpm2/signature.go @@ -0,0 +1,50 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package tpm2 provides TPM2.0 related functionality helpers. +package tpm2 + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// PCRData is the data structure for PCR signature json. +type PCRData struct { + SHA1 []BankData `json:"sha1,omitempty"` + SHA256 []BankData `json:"sha256,omitempty"` + SHA384 []BankData `json:"sha384,omitempty"` + SHA512 []BankData `json:"sha512,omitempty"` +} + +// BankData constains data for a specific PCR bank. +type BankData struct { + // list of PCR banks + PCRs []int `json:"pcrs"` + // Public key of the TPM + PKFP string `json:"pkfp"` + // Policy digest + Pol string `json:"pol"` + // Signature of the policy digest in base64 + Sig string `json:"sig"` +} + +// ParsePCRSignature parses the PCR signature json file. +func ParsePCRSignature() (*PCRData, error) { + pcrSignature, err := os.ReadFile(constants.PCRSignatureJSON) + if err != nil { + return nil, fmt.Errorf("failed to read pcr signature: %v", err) + } + + pcrData := &PCRData{} + + if err = json.Unmarshal(pcrSignature, pcrData); err != nil { + return nil, fmt.Errorf("failed to unmarshal pcr signature: %v", err) + } + + return pcrData, nil +} diff --git a/internal/pkg/secureboot/tpm2/testdata/pcr-signing-crt.pem b/internal/pkg/secureboot/tpm2/testdata/pcr-signing-crt.pem new file mode 100644 index 0000000..60d4744 --- /dev/null +++ b/internal/pkg/secureboot/tpm2/testdata/pcr-signing-crt.pem @@ -0,0 +1,14 @@ +-----BEGIN PUBLIC KEY----- +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA7qhAkdtZxqkIP79DDGin +9eaJBeNlJsClJTcbaXbNfk2QJGT3lqo9ErXQQftwYWLGo+kVd8puhnHGPkLW9apT +1/ZmUJEFwxV5xws0RllGVPhUga+1oubHUqhEiy707S4RrUEMk/o9wqmtnl2hY5Fx +MeQn2o7xrpcNhm8FtHpvQrT0MsbC1cS1ytZH/hwPy/QIB9bx+ugOha6wtQBnpgix +1BhHC/NwDIYPg+ONpQSCu9gkXVtLGlKfmjscUANQtBuVKa5NflrjkHw7NAdKYdKp +Mnmzr0yu6Tn/2oNmUiJAwHz0BXpfb4Yn8n/IoKJQ5Tv1g6d30wxxpBd0lbwSe9ML +RchIDJ5aFRybyRxaPGT17U3yEVzbV78kIFtocaqkc1ise8remZ0wxHzuolbTZD6o +swt7C9jMLvfMAQ7JtENXrpDM//XzdRLzyTWKOjhG0YmKKRY6cIrPkugM0PHGCE3R +MSH1FmPMrWWBNAMwS0Zba0Wm1b7vdw5fKeE8txH+IpA3IaE9AytYk0ig98ZgmXmB +V0sgxmJ/94scEF+sDg65LIkSEJMzf6q30UghbJJoP7eKOoDX9KBrR+POEsWm/EcU +5jTEQTHMU+qKtj5KD6TUn8R8yi4wCnyZ7uJLUqm8Ou8MzEZWbrsrbMvrewPDAHn0 +QQvb2tDtBgn6oH192jpkzckCAwEAAQ== +-----END PUBLIC KEY----- diff --git a/internal/pkg/secureboot/tpm2/tpm2.go b/internal/pkg/secureboot/tpm2/tpm2.go new file mode 100644 index 0000000..89d1d71 --- /dev/null +++ b/internal/pkg/secureboot/tpm2/tpm2.go @@ -0,0 +1,14 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package tpm2 provides TPM2.0 related functionality helpers. +package tpm2 + +// SealedResponse is the response from the TPM2.0 Seal operation. +type SealedResponse struct { + SealedBlobPrivate []byte + SealedBlobPublic []byte + KeyName []byte + PolicyDigest []byte +} diff --git a/internal/pkg/secureboot/tpm2/unseal.go b/internal/pkg/secureboot/tpm2/unseal.go new file mode 100644 index 0000000..736e83c --- /dev/null +++ b/internal/pkg/secureboot/tpm2/unseal.go @@ -0,0 +1,269 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package tpm2 provides TPM2.0 related functionality helpers. +package tpm2 + +import ( + "bytes" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + + "github.com/google/go-tpm/tpm2" + "github.com/google/go-tpm/tpm2/transport" + + "github.com/aenix-io/talm/internal/pkg/secureboot" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// Unseal unseals a sealed blob using the TPM +// +//nolint:gocyclo,cyclop +func Unseal(sealed SealedResponse) ([]byte, error) { + t, err := transport.OpenTPM() + if err != nil { + return nil, err + } + defer t.Close() //nolint:errcheck + + // fail early if PCR banks are not present or filled with all zeroes or 0xff + if err = validatePCRBanks(t); err != nil { + return nil, err + } + + tpmPub, err := tpm2.Unmarshal[tpm2.TPM2BPublic](sealed.SealedBlobPublic) + if err != nil { + return nil, err + } + + tpmPriv, err := tpm2.Unmarshal[tpm2.TPM2BPrivate](sealed.SealedBlobPrivate) + if err != nil { + return nil, err + } + + srk, err := tpm2.Unmarshal[tpm2.TPM2BName](sealed.KeyName) + if err != nil { + return nil, err + } + + // we need to create a primary since we don't persist the SRK + primary := tpm2.CreatePrimary{ + PrimaryHandle: tpm2.TPMRHOwner, + InPublic: tpm2.New2B(tpm2.ECCSRKTemplate), + } + + createPrimaryResponse, err := primary.Execute(t) + if err != nil { + return nil, err + } + + defer func() { + flush := tpm2.FlushContext{ + FlushHandle: createPrimaryResponse.ObjectHandle, + } + + _, flushErr := flush.Execute(t) + if flushErr != nil { + err = flushErr + } + }() + + outPub, err := createPrimaryResponse.OutPublic.Contents() + if err != nil { + return nil, err + } + + if !bytes.Equal(createPrimaryResponse.Name.Buffer, srk.Buffer) { + // this means the srk name does not match, possibly due to a different TPM or tpm was reset + // could also mean the disk was used on a different machine + return nil, errors.New("srk name does not match") + } + + load := tpm2.Load{ + ParentHandle: tpm2.NamedHandle{ + Handle: createPrimaryResponse.ObjectHandle, + Name: createPrimaryResponse.Name, + }, + InPrivate: *tpmPriv, + InPublic: *tpmPub, + } + + loadResponse, err := load.Execute(t) + if err != nil { + return nil, err + } + + policySess, policyCloseFunc, err := tpm2.PolicySession( + t, + tpm2.TPMAlgSHA256, + 20, + tpm2.Salted(createPrimaryResponse.ObjectHandle, *outPub), + ) + if err != nil { + return nil, fmt.Errorf("failed to create policy session: %w", err) + } + + defer policyCloseFunc() //nolint:errcheck + + pubKey, err := ParsePCRSigningPubKey(constants.PCRPublicKey) + if err != nil { + return nil, err + } + + loadExternal := tpm2.LoadExternal{ + Hierarchy: tpm2.TPMRHOwner, + InPublic: tpm2.New2B(RSAPubKeyTemplate(pubKey.N.BitLen(), pubKey.E, pubKey.N.Bytes())), + } + + loadExternalResponse, err := loadExternal.Execute(t) + if err != nil { + return nil, fmt.Errorf("failed to load external key: %w", err) + } + + defer func() { + flush := tpm2.FlushContext{ + FlushHandle: loadExternalResponse.ObjectHandle, + } + + _, flushErr := flush.Execute(t) + if flushErr != nil { + err = flushErr + } + }() + + pcrSelector, err := CreateSelector([]int{secureboot.UKIPCR}) + if err != nil { + return nil, err + } + + policyDigest, err := PolicyPCRDigest(t, policySess.Handle(), tpm2.TPMLPCRSelection{ + PCRSelections: []tpm2.TPMSPCRSelection{ + { + Hash: tpm2.TPMAlgSHA256, + PCRSelect: pcrSelector, + }, + }, + }) + if err != nil { + return nil, fmt.Errorf("failed to retrieve policy digest: %w", err) + } + + sigJSON, err := ParsePCRSignature() + if err != nil { + return nil, err + } + + pubKeyFingerprint := sha256.Sum256(x509.MarshalPKCS1PublicKey(pubKey)) + + var signature string + // TODO: maybe we should use the highest supported algorithm of the TPM + // fallback to the next one if the signature is not found + for _, bank := range sigJSON.SHA256 { + digest, decodeErr := hex.DecodeString(bank.Pol) + if decodeErr != nil { + return nil, decodeErr + } + + if bytes.Equal(digest, policyDigest.Buffer) { + signature = bank.Sig + + if hex.EncodeToString(pubKeyFingerprint[:]) != bank.PKFP { + return nil, errors.New("certificate fingerprint does not match") + } + + break + } + } + + if signature == "" { + return nil, errors.New("signature not found") + } + + signatureDecoded, err := base64.StdEncoding.DecodeString(signature) + if err != nil { + return nil, err + } + + // Verify will only verify the RSA part of the RSA+SHA256 signature, + // hence we need to do the SHA256 part ourselves + policyDigestHash := sha256.Sum256(policyDigest.Buffer) + + verifySignature := tpm2.VerifySignature{ + KeyHandle: loadExternalResponse.ObjectHandle, + Digest: tpm2.TPM2BDigest{ + Buffer: policyDigestHash[:], + }, + Signature: tpm2.TPMTSignature{ + SigAlg: tpm2.TPMAlgRSASSA, + Signature: tpm2.NewTPMUSignature(tpm2.TPMAlgRSASSA, &tpm2.TPMSSignatureRSA{ + Hash: tpm2.TPMAlgSHA256, + Sig: tpm2.TPM2BPublicKeyRSA{ + Buffer: signatureDecoded, + }, + }), + }, + } + + verifySignatureResponse, err := verifySignature.Execute(t) + if err != nil { + return nil, fmt.Errorf("failed to verify signature: %w", err) + } + + policyAuthorize := tpm2.PolicyAuthorize{ + PolicySession: policySess.Handle(), + ApprovedPolicy: *policyDigest, + KeySign: loadExternalResponse.Name, + CheckTicket: verifySignatureResponse.Validation, + } + + if _, err = policyAuthorize.Execute(t); err != nil { + return nil, fmt.Errorf("failed to execute policy authorize: %w", err) + } + + secureBootStatePCRSelector, err := CreateSelector([]int{secureboot.SecureBootStatePCR}) + if err != nil { + return nil, err + } + + secureBootStatePolicyDigest, err := PolicyPCRDigest(t, policySess.Handle(), tpm2.TPMLPCRSelection{ + PCRSelections: []tpm2.TPMSPCRSelection{ + { + Hash: tpm2.TPMAlgSHA256, + PCRSelect: secureBootStatePCRSelector, + }, + }, + }) + if err != nil { + return nil, fmt.Errorf("failed to calculate policy PCR digest: %w", err) + } + + if !bytes.Equal(secureBootStatePolicyDigest.Buffer, sealed.PolicyDigest) { + return nil, errors.New("sealing policy digest does not match") + } + + unsealOp := tpm2.Unseal{ + ItemHandle: tpm2.AuthHandle{ + Handle: loadResponse.ObjectHandle, + Name: loadResponse.Name, + Auth: policySess, + }, + } + + unsealResponse, err := unsealOp.Execute(t, tpm2.HMAC( + tpm2.TPMAlgSHA256, + 20, + tpm2.Salted(createPrimaryResponse.ObjectHandle, *outPub), + tpm2.AESEncryption(128, tpm2.EncryptOut), + tpm2.Bound(loadResponse.ObjectHandle, loadResponse.Name, nil), + )) + if err != nil { + return nil, fmt.Errorf("failed to unseal op: %w", err) + } + + return unsealResponse.OutData.Buffer, nil +} diff --git a/internal/pkg/secureboot/uki/assemble.go b/internal/pkg/secureboot/uki/assemble.go new file mode 100644 index 0000000..62286d2 --- /dev/null +++ b/internal/pkg/secureboot/uki/assemble.go @@ -0,0 +1,84 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package uki + +import ( + "debug/pe" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" +) + +// assemble the UKI file out of sections. +func (builder *Builder) assemble() error { + peFile, err := pe.Open(builder.SdStubPath) + if err != nil { + return err + } + + defer peFile.Close() //nolint: errcheck + + // find the first VMA address + lastSection := peFile.Sections[len(peFile.Sections)-1] + + // align the VMA to 512 bytes + // https://github.com/saferwall/pe/blob/main/helper.go#L22-L26 + const alignment = 0x1ff + + header, ok := peFile.OptionalHeader.(*pe.OptionalHeader64) + if !ok { + return errors.New("failed to get optional header") + } + + baseVMA := header.ImageBase + uint64(lastSection.VirtualAddress) + uint64(lastSection.VirtualSize) + baseVMA = (baseVMA + alignment) &^ alignment + + // calculate sections size and VMA + for i := range builder.sections { + if !builder.sections[i].Append { + continue + } + + st, err := os.Stat(builder.sections[i].Path) + if err != nil { + return err + } + + builder.sections[i].Size = uint64(st.Size()) + builder.sections[i].VMA = baseVMA + + baseVMA += builder.sections[i].Size + baseVMA = (baseVMA + alignment) &^ alignment + } + + // create the output file + args := []string{} + + for _, section := range builder.sections { + if !section.Append { + continue + } + + args = append(args, "--add-section", fmt.Sprintf("%s=%s", section.Name, section.Path), "--change-section-vma", fmt.Sprintf("%s=0x%x", section.Name, section.VMA)) + } + + builder.unsignedUKIPath = filepath.Join(builder.scratchDir, "unsigned.uki") + + args = append(args, builder.SdStubPath, builder.unsignedUKIPath) + + objcopy := "/usr/x86_64-alpine-linux-musl/bin/objcopy" + + if builder.Arch == "arm64" { + objcopy = "/usr/aarch64-alpine-linux-musl/bin/objcopy" + } + + cmd := exec.Command(objcopy, args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + return cmd.Run() +} diff --git a/internal/pkg/secureboot/uki/generate.go b/internal/pkg/secureboot/uki/generate.go new file mode 100644 index 0000000..0a9c959 --- /dev/null +++ b/internal/pkg/secureboot/uki/generate.go @@ -0,0 +1,242 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package uki + +import ( + "crypto/x509" + "encoding/json" + "encoding/pem" + "os" + "path/filepath" + + talosx509 "github.com/siderolabs/crypto/x509" + "github.com/siderolabs/gen/xslices" + + "github.com/aenix-io/talm/internal/pkg/secureboot" + "github.com/aenix-io/talm/internal/pkg/secureboot/measure" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/version" + "github.com/siderolabs/talos/pkg/splash" +) + +func (builder *Builder) generateOSRel() error { + osRelease, err := version.OSReleaseFor(version.Name, builder.Version) + if err != nil { + return err + } + + path := filepath.Join(builder.scratchDir, "os-release") + + if err = os.WriteFile(path, osRelease, 0o600); err != nil { + return err + } + + builder.sections = append(builder.sections, + section{ + Name: secureboot.OSRel, + Path: path, + Measure: true, + Append: true, + }, + ) + + return nil +} + +func (builder *Builder) generateCmdline() error { + path := filepath.Join(builder.scratchDir, "cmdline") + + if err := os.WriteFile(path, []byte(builder.Cmdline), 0o600); err != nil { + return err + } + + builder.sections = append(builder.sections, + section{ + Name: secureboot.CMDLine, + Path: path, + Measure: true, + Append: true, + }, + ) + + return nil +} + +func (builder *Builder) generateInitrd() error { + builder.sections = append(builder.sections, + section{ + Name: secureboot.Initrd, + Path: builder.InitrdPath, + Measure: true, + Append: true, + }, + ) + + return nil +} + +func (builder *Builder) generateSplash() error { + path := filepath.Join(builder.scratchDir, "splash.bmp") + + if err := os.WriteFile(path, splash.GetBootImage(), 0o600); err != nil { + return err + } + + builder.sections = append(builder.sections, + section{ + Name: secureboot.Splash, + Path: path, + Measure: true, + Append: true, + }, + ) + + return nil +} + +func (builder *Builder) generateUname() error { + // it is not always possible to get the kernel version from the kernel image, so we + // do a bit of pre-checks + var kernelVersion string + + if builder.Version == version.Tag { + // if building from the same version of Talos, use default kernel version + kernelVersion = constants.DefaultKernelVersion + } else { + // otherwise, try to get the kernel version from the kernel image + kernelVersion, _ = DiscoverKernelVersion(builder.KernelPath) //nolint:errcheck + } + + if kernelVersion == "" { + // we haven't got the kernel version, skip the uname section + return nil + } + + path := filepath.Join(builder.scratchDir, "uname") + + if err := os.WriteFile(path, []byte(kernelVersion), 0o600); err != nil { + return err + } + + builder.sections = append(builder.sections, + section{ + Name: secureboot.Uname, + Path: path, + Measure: true, + Append: true, + }, + ) + + return nil +} + +func (builder *Builder) generateSBAT() error { + sbat, err := GetSBAT(builder.SdStubPath) + if err != nil { + return err + } + + path := filepath.Join(builder.scratchDir, "sbat") + + if err = os.WriteFile(path, sbat, 0o600); err != nil { + return err + } + + builder.sections = append(builder.sections, + section{ + Name: secureboot.SBAT, + Path: path, + Measure: true, + }, + ) + + return nil +} + +func (builder *Builder) generatePCRPublicKey() error { + publicKeyBytes, err := x509.MarshalPKIXPublicKey(builder.PCRSigner.PublicRSAKey()) + if err != nil { + return err + } + + publicKeyPEM := pem.EncodeToMemory(&pem.Block{ + Type: talosx509.PEMTypeRSAPublic, + Bytes: publicKeyBytes, + }) + + path := filepath.Join(builder.scratchDir, "pcr-public.pem") + + if err = os.WriteFile(path, publicKeyPEM, 0o600); err != nil { + return err + } + + builder.sections = append(builder.sections, + section{ + Name: secureboot.PCRPKey, + Path: path, + Append: true, + Measure: true, + }, + ) + + return nil +} + +func (builder *Builder) generateKernel() error { + path := filepath.Join(builder.scratchDir, "kernel") + + if err := builder.peSigner.Sign(builder.KernelPath, path); err != nil { + return err + } + + builder.sections = append(builder.sections, + section{ + Name: secureboot.Linux, + Path: path, + Append: true, + Measure: true, + }, + ) + + return nil +} + +func (builder *Builder) generatePCRSig() error { + sectionsData := xslices.ToMap( + xslices.Filter(builder.sections, + func(s section) bool { + return s.Measure + }, + ), + func(s section) (secureboot.Section, string) { + return s.Name, s.Path + }) + + pcrData, err := measure.GenerateSignedPCR(sectionsData, builder.PCRSigner) + if err != nil { + return err + } + + pcrSignatureData, err := json.Marshal(pcrData) + if err != nil { + return err + } + + path := filepath.Join(builder.scratchDir, "pcrpsig") + + if err = os.WriteFile(path, pcrSignatureData, 0o600); err != nil { + return err + } + + builder.sections = append(builder.sections, + section{ + Name: secureboot.PCRSig, + Path: path, + Append: true, + }, + ) + + return nil +} diff --git a/internal/pkg/secureboot/uki/kernel.go b/internal/pkg/secureboot/uki/kernel.go new file mode 100644 index 0000000..96e7863 --- /dev/null +++ b/internal/pkg/secureboot/uki/kernel.go @@ -0,0 +1,69 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package uki + +import ( + "bytes" + "encoding/binary" + "errors" + "os" + "strings" +) + +// DiscoverKernelVersion reads kernel version from the kernel image. +// +// This only works for x86 kernel images. +// +// Based on https://www.kernel.org/doc/html/v5.6/x86/boot.html. +func DiscoverKernelVersion(kernelPath string) (string, error) { + f, err := os.Open(kernelPath) + if err != nil { + return "", err + } + + defer f.Close() //nolint:errcheck + + header := make([]byte, 1024) + + _, err = f.Read(header) + if err != nil { + return "", err + } + + // check header magic + if string(header[0x202:0x206]) != "HdrS" { + return "", errors.New("invalid kernel image") + } + + setupSects := header[0x1f1] + versionOffset := binary.LittleEndian.Uint16(header[0x20e:0x210]) + + if versionOffset == 0 { + return "", errors.New("no kernel version") + } + + if versionOffset > uint16(setupSects)*0x200 { + return "", errors.New("invalid kernel version offset") + } + + versionOffset += 0x200 + + version := make([]byte, 256) + + _, err = f.ReadAt(version, int64(versionOffset)) + if err != nil { + return "", err + } + + idx := bytes.IndexByte(version, 0) + if idx == -1 { + return "", errors.New("invalid kernel version") + } + + versionString := string(version[:idx]) + versionString, _, _ = strings.Cut(versionString, " ") + + return versionString, nil +} diff --git a/internal/pkg/secureboot/uki/kernel_test.go b/internal/pkg/secureboot/uki/kernel_test.go new file mode 100644 index 0000000..0b9b750 --- /dev/null +++ b/internal/pkg/secureboot/uki/kernel_test.go @@ -0,0 +1,21 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package uki_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/aenix-io/talm/internal/pkg/secureboot/uki" +) + +func TestKernelVersion(t *testing.T) { + version, err := uki.DiscoverKernelVersion("testdata/kernel") + require.NoError(t, err) + + assert.Equal(t, "6.1.58-talos", version) +} diff --git a/internal/pkg/secureboot/uki/sbat.go b/internal/pkg/secureboot/uki/sbat.go new file mode 100644 index 0000000..ec17ebb --- /dev/null +++ b/internal/pkg/secureboot/uki/sbat.go @@ -0,0 +1,35 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package uki + +import ( + "debug/pe" + "errors" + + "github.com/aenix-io/talm/internal/pkg/secureboot" +) + +// GetSBAT returns the SBAT section from the PE file. +func GetSBAT(path string) ([]byte, error) { + pefile, err := pe.Open(path) + if err != nil { + return nil, err + } + + defer pefile.Close() //nolint:errcheck + + for _, section := range pefile.Sections { + if section.Name == string(secureboot.SBAT) { + data, err := section.Data() + if err != nil { + return nil, err + } + + return data[:section.VirtualSize], nil + } + } + + return nil, errors.New("could not find SBAT section") +} diff --git a/internal/pkg/secureboot/uki/sbat_test.go b/internal/pkg/secureboot/uki/sbat_test.go new file mode 100644 index 0000000..2be8d4b --- /dev/null +++ b/internal/pkg/secureboot/uki/sbat_test.go @@ -0,0 +1,25 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package uki_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/aenix-io/talm/internal/pkg/secureboot/uki" +) + +func TestGetSBAT(t *testing.T) { + t.Parallel() + + data, err := uki.GetSBAT("../pesign/testdata/systemd-bootx64.efi") + require.NoError(t, err) + + require.Equal(t, + "sbat,1,SBAT Version,sbat,1,https://github.com/rhboot/shim/blob/main/SBAT.md\nsystemd,1,The systemd Developers,systemd,254,https://systemd.io/\nsystemd.talos,1,Talos Linux,systemd,254,https://github.com/siderolabs/tools/issues\n\x00", //nolint:lll + string(data), + ) +} diff --git a/internal/pkg/secureboot/uki/testdata/kernel b/internal/pkg/secureboot/uki/testdata/kernel new file mode 100644 index 0000000000000000000000000000000000000000..ebfbfcb36d38aca21306431845e8304fa6709841 GIT binary patch literal 16384 zcmdsedw5jU+3(tuNiu}M4iI9X6gD`hfq)4jX2=8*5@1vUQNl%~N+4N7BqlW3n@fu@ z>12>0ByGQH?adZQwW$4$evjXgNvMzua*?V5TMS?|qSifZKoo@#ggL)=?U^KieR|IG zoIlQUl4oZ2UYB>h>%G74+B^T@)3K~McI(0Vj{4^M_ViD$e7h%`{aI;N`~CO8-LdTL z1mmmp=C_mH9(MYCh@EGB|9*m-1yv;idvymme7oe6ET<-%w(*1Q&bON30K=#jHSQDlE^l!vqm~5 z`L8i%5qC6%Dd7)Yiw3%&KM&@Z<5=7cUFLwBzhiE0`nN3;t4dUD6=R(a48?kp-@X$S zRwEuyd>smXUt;f^ zmN79D0{31r9(NU2&X-MkC1huSlpe+st^>UE0rNH#3T-6(rJ%B)qE^rPNW-)d1! z--hbjxxDMtm60nNEQfjZw2sULFFpLQcFW$*%*N&OczScn<(8o30)Lt7DcB1S7+9X` zAQ=1FxSA0YvZ&EJLKcJZ-^ttLoqVbHOsfBw|MSNF>;iwD7s)Tm+ha%Z=ibJEkum0w zZ?WR1rXIL!*W2?(hg6&YpXLnjzEpD(e~!2Dg?v5-K<;nj-mW+;o(BeCVXl7y0&#yQ z*N+Wj%zYrwo)9a3k|pNepJ)I6K=OTa&vi4R86xQ z$8)(p7Yi6?F|sXoZiOObR;em=mlvs8Wr@NowDMx;VI1R49zrO+SpcLh?%uXZVtkJx z@caki87QV}!pRxZV)$Gw#u&9zQipIIn4^1Xq}#CWv|#!Bwf)?CFl0mlEa3|L7Vt%a ze(6W+PjT;Cq59K&FKJJ)o7djS{f3*@O}>LWE~sO8F53K~xV}ulPQaiN3}W2Tp?=8I zJ4~oe;mQ7SVu8o_5Be}@H>8>I`{f!Y@qj|e=|3i5zGgKBC=EV+ z6@y>9O1m;A&M+rx2e|&>H8lJ8M|$oE_xvaKT~w#=z<je7MRnF3RUgAJLfqxvGK%}J zasR=Zlj_~ZVE2}hH7B)LbgjAF$il!Wd{)K@`I$EC{UpydlF@XlIahwRjY}tu7`~hA zvYR{N?OHua{Sh&c4z<=xkVGf^v$op~Bg+D$9MMEp#w(`FrcqTamV@f1GHqi&Xu@Be{ul#wpnHk&!CH&?!$u-RZXsWxiPp!dmB@1B3qF&3uPUNmn#ObePl$^6r4Bx<850wFg@OML{||Hj5uV<{AAqy^9Vqb}Pk)R1S|eI! z`Y-=G7$xrj5n@cz;|it#gn?@L!FrR${$v5X%`?s8gj)Sd3Om1q{V^V=L_W?2dg3QO z6q1$Zo$+cc_jX>8$26b(#Ql%;kdou;?KU^q5YM!s0;+B7^aO9$bS;+4O%qd{;EZVy zvrRK9*Q7{%g%yDhUtts96a)!TjOXb`N&5Av7z$Xb5X)I)*W!8BHK;y@EwRV1S;QNRtD^GOl;SG@5(m{#M>{+Qz-7WbRK=ocOZitA)1&VKP3J1!!+~^qN@i zA9#n;7zCpj&&N_6%%y|8;SML*6th}^yIfHw7UTZoVkL@?xc)OT!i3&@`WJ{Kh;Hey90OmO&U?bdbgXYdB=Y>t~rcGjd);d9iOYLrc ztF$U-&+K>A?_4T zH_B;Te;DfoVwgMRv4a?s zC&m8+kc3p)g1a%edv-~=rmX^IRwwRC%S#G1mp~$aZN?OB5#MWNn>^fqS)K$!6$+a| zRuWnYKtkB$9T2Hhh}30M`0T{78dUG?>m`L_Pu53P1g8e(4$zhFSM_nGqPGH19!=KnyF1q4KWx z$WUXD2@PAq?LJyk^!v|)%o_kl4SJmGp@kZelo3>FGFx1t7#O7vgircf^J}^})1Bf!a%p1w=^QJ$EaXQ!M znB&w@d`}njjUz8-%(xH10!_4~%1dRT zwv;Qzf(R_a2~(`2c%#tRrk4cf9mr`2w+ZOWg)MB0##&Zey8N^V)&Om?uN@=dj`|>l zF(YiL8r&q8J~G23feSXp8C66rlI-)e zACUwg5bF`auS}VeceRE4U|{PIT|`nn_vSo%6p03DlHVLn!m6UR<)1dm|M-?DxX;xqu;-wD{?;?R?F(E`^5s3V81|WEeFmF{3v1RX z^TL*8(1n@rG)dEOz2{5#65jY@qLu&s3x5P}GNvdV_W=P}@~t0WLoag))=I zOs}$R?GSru>9lgGoi~7g`eDMThm)%Z@<4|m`U79W{tV(x--x>(evMHOw~sy3Mo(w! z5cStRZD%x9-W4->I*+D2k2_i`V|c^TOk)VFDt?f9UlG%-0LJy{Bo1H?44N|o!}YW0 zsX6!sxqHN~sa(%QFh%i-i=Ck#rv*eb`JBoF!n^5jU^v%b6?^XFdOw;$>AM9m<^TpI z?@83F|AGK-0k9ocrV+Fe(XgLpc5e=-Bv53)qc`qH#FdjbsOD83Z&+(yW%GvhNcxJP^jx1$oHBE3Vlb5<=0z4$biyt< z5zp-WF69UE?qlqdPo`>>-2?Kfc6%QJ;?UbCI+&r>tn!t)QPY_E&R@FmLbKM!VuDMn= zwq0n?6Pt(1AM3XtsT_Q)%MFBpmb?q($zskmw`haRJ11}HjDet#?`$}4E;ff2Bj#t!+Uf!45kXo;;_oJU0(@7sb^a9bf(6HMkY@;lDm1+D%xd$wq|OxewczAeG~E8 z6u$LB_*P*vTra~iyy-lR4tq4N|ArehrTG;Fs|=Q=<@m5+fTTMzS= zPoSnFlambAn9TMnOjcSwZ%Oi)R(^W--Bbna$&9^cmIwnVqQSSplsWcQ?k$D(wTPS* z_vS*oEEt8`PUH?GGHNWMhGP3#IArjV#!1}UEg@wQ{Z_Z9*ls=V_Q2Yv^~?p+G1o>S zu-2mHnsLWEWF(hH*!jJD?*zggR^o2s2V#sBMJ6 z4;xbxky|B@0|;7BhVWjs!A&81qtHehNJ26^jqiaLF=KLXLI`w3=5Xm7pax}`Mfxtg z<9?FifMjjXn~WUKQD#r9dKdkZ5gcWzcJ;~rVn?c&f=kRxX%Pqy(#DY(_}}6w@PtP< z^mFgGMk%O#oI#y{`wp!G!l-?;VFM{Ga0IC0q2>{VtTVic6oI%6MU6Dn48A9}G?eZH zUhp)F-J9W1kg19p2~4{SegQ&}+MoNs;3)?Xt!)^OY88;PBmBlBnDyTRq)l)qYg}@qZIsVDh!D z98O=}vrzJMk5|WU9D>@4I?mI5t2Ul{--XdQXP^=@LGAD9PSZxANiqz4Cm5gM+iHA- zZ<}%4(>+d$HzslK=`;vvzts?scS1&{5CM750@%#D``VEs^;>g0(({!U#aM z`74&d5cie^Z1WLhwjX&e&rzYY@CQ~x)ziHKg=~4xTP9pSlSX>F*UCSQv3RfOYRGCMaVft}VQg>QBZP8bCsUGG!bTUDz1ohw%0J#!54s23Eu{IB5W;%3GN_ zf$}KtDUw!EM5NXg+AUa{ad~gM7>3GF9K2V%K@3Xhvme^Uk0@>JvkO43)1;t&1?XTq zAU^1g?Q~(EiM=)jqfFKp!h(q1bN%Tn07Yez>NBWT5DmB5v!M0iq8qbNy5 z0QEoqN!J~`dLK;ybPAv$IXD!;ErdXnL7+r4?jXOhwZrY7aT`ANt)2KDvsK5pf;5<< zrT{t(>oOD`5{5_5K^`A1|5D^q0BaVe%=p}Y0QT>~3z2#q<`N zmS0aCxVg>t9`dRjp3<&l#a-l7K|se49{3MU+j9#Kwz=5ovc{i zkapvg7`cG^KZ^vcpMrg7Ah!F3oIrix0` zvT%JnmBJ7bd=E*T|6Lf=1Z_BGBt>RmEJS?2h3_T9B~T+5gN2}41Y{sND{MetWI|H! zi&*B>@!We@!H~mfS;QTetL}s>p}@%HSI9q-r@Hhxmrfv>;!+DAgu((W4#B2Dvy5$~ z8qgtxkU?~$ne*^%7S&6>1~I~=5+s!qH27w&e?+LDlG28hm@Wm`{jF}#RHFC{<07ma zPdSM8NHqe5mHpiRcWT$)g*~T86jdioL|VOv8ZaGMKB~P4W3W8MB<-+2m4ohAzfYUjxqPbC2NmMGMb zz{WSiY=l#x*aU2RBh1FThy%G`;|WUjz{X@l5iw1ehvsmThiw-A2JsM!P$57Rs>nUk zkqnGKv4fyNj9^#bvoS8-P97E|GFnsK1F@_t_yhDXT7GIP44n?!DX8H{wO6gj;Sv7i z)w4|MP!!X`FC7tkO5m4JfI7q-rz??ax07SL^bPvOvgrH5 zue?oky!08D;8<}O228oMpAUN9bnW8X7R*7B|3j=D7MHd6@W5%@sw8f-L%1DJW}K+1 zhz{H_ja-coVUj8TTF?P{+t&s{Uva!Ar2AtegCs}3=C$9$GuCtp{X@+8bUFr~ z5zZh7z!-}3i5%+nx$m=`;q<8xC2_<|=fT)ce2_==&VQRuTW-vqhm#)ai)cP#>6{y; z{D!97xY#(cnEX0r#O4aR$bcZ^IFo7ytq^va2OIiE+`;~J@I52NLU+BTc#vyd?`~C291KLJVd%Os1=(1T-dPh|58A>BQwL2YGc1&R21ia)@s)&IB$? zGbu50ytV!gm_cvnHC`R0!Vl<^2uV)Dn1<~9`rBy7N4b7Cp12^s={v9|@0F{vFE)w? zT*%CDz7B;77ZmkRsI2%RNE(uBKCr;^4v=f!K%Q-$(WGLeaRRzWHEvXt$XIbbiRK61 z6|i2QYuI>wp{P%G+2oq5B0Lq|P2M!cBI3zYaO|fc&)vi1O-wAK{WSvaz&`-OcT%od zVj(*Mq){Uyb5$4*FY}gy9aa)FhX`gXU*ytx_)N!f?R}oL>#uaAoq|FeMHx7%r2rQD z4}3ip7g;VGrdA>2j4DV#EH{D@Fj#`_A@Uy(punTnsZ%7vKJaU?nd86&`cA18uX_Y1 zn;&YQMz9=BSW-R6Kf?Z68s94xs1DHzviAc@IMf^z0N^ zgo?~JRytAtJKE=bgW?h_A>24l)^MKou?Z_|@j**H70-`p|;rf{H7;O>{^e2Yti)gj`gRfkr zAsdmV@5{R`2;V=6>ZUM%L+QX-7VBT8S}TsBgGayhlPc>Sh-YCS!g{<=AUthO zJ`T<;l+}%dMw`l9J24y2Cec7~9OpYL*K~q)@S&zonx5D(*}h>VRp}j5Cg*RM&9}oC zI65fP>%e0j7*0M7GJz8pe;3!+ldTIox&t&A`%HvQ_`7hpZIllkM{ZV2b}pdADzP#3 ztw(}Y(#v@MD;7~!J z);=1AX>p+l?$d(~kq5JO9Va$?LFtEgUq)|`W=6sCu7e;6O73ro8X#6cq(W8vM?$yZ z!5>kJ>$G8rD0iSEbek>9?FlSFos?<$;uIaax!*$#65nkq+7+S7r~o*s615T7NUn`o z7so>s%;YAB&`eyRz%#;lC+x6Lu{7s*)BMVCSdT}z0HG4PQu{E@oDsVm-Vo<@Sl>~T zM9l7O9f89~F@u~R9xpM?i*SNjR#~ozxAcUU-ZRO34})lVyF#e!`uBUguEFN-k%Y8e zsQv_J*CrAaius*5fjiH~(&K`H-L~*bviGNGsGUJb-IstaM2n{0iRuSZIrKP?BR*L~ z+Pv1{3n4i1uC+3i9v&zY`kQK3@8EBULLHLov+m4f70NB|a#>9)Lzyk*gpSKvw7WLh zScZTS*4jJ54%9?J1*bd4&p8MX#N!nv+VZZ$^pH>wO{yFOaz?VFtujbgKN`-ZF79Y~ z@}wK46ZACY7(p0h3p)utNKa3xalu2-q=TVgEAw5H{r{8ox@F;U&-3l%V4Jm@9kKih z+3Hmgy2Yfj@~$r4g!33EGjwFrRgyy-OE#KK-8kE$CQzB&c;jF(5UthJ+9Vts&6{h^ zdt)<1^*jQ#SOBr`mUB_u`_F9N?|kIcw$Ypc0NI!!tO7ZaedgmIeGYQJTe5os#t(4d zB8vNLFlkx>D04$(^VqSlwmTb!LnGGPoeAo6bIS0W22Y~F7~>iaf*puioDa&d{V}7m zBxA6+7g8XKFAa5;75>P{MOcLM$2h8?{rCuXl(* zZdXS{*iP_(I8)WHku;fy*nTvMDu^Bwji&sdaT-aYd5|kgh4603W<=v&G13%OG=3x+ zUlNVt_=iS*E*hT^jp7WA8WG(AriU7P(T5}th(E=fS_yBMT|ECtPp!iB>9?30IFIK# zXeJI)!)Rpz1cWS?5`-SD`HIYF(0*0qOE@`0&J3X#Oh-|uj*ehAN9VtNkHCm@M8x6Z zK?FLNy5Zs8DvX1e&dDb+B`bnVx1kU+iYqH`D9Cr$Moo}pZ&!@`)Ne@Tn^8aYJq-y3 zlH#~>D;-h+3zx}vXBzm5zCs*Bh94l72^SiHa@dW17mi|mJrSEWoLv7U{Z8ikuxt2L zxIPHK5vqOw453o`Rid$eijwg_C))6+8I2OQfDXud^ML9zuL|1kN4^1!e#L0q8%yEN zcw;v8U=9u(@RSfybKjXTHKloEKa<`5R1_g@Fb#P?mmU>!%~N5gU_}3nM*x=Qo3Zs! zWbNvZvA-qzhN?dD)V{67>bLM)&awIf6{H@=E;12(u^!*GMmww zbO&8+q^l`(<)y2s6doj1&c$VFmeaPT@?y;6b0y9CYK$>kuzfMtpD|2wskV7!MoTGF z-)gTBQUvBNjxc|)*wSC@+t4#ld&9k@#{LX0cTCCJk&(4yYSs>C*8Z4#vS2bD^p#0p zSy`rZVVQ}MW#Z=jiv|9xooE(bK=z%Z6rjew6Kga~qeJK#fmB0wR`5OKYWsvpPyk_i zj7V{c$_bR;WkYAL@dnD`Mq~=6{TX!*Q_JSz@$tD3d{K|NK`_Px3rahLY{^_iw*5>1JxEXj{;$MO=1al%EL2U2lcNPBl4w|W&bF)SCqpL@2sQiJ# z#i@-@pN#j^6yO|_l$Fs=XRmmEb=Y@U%6l>2_*nR57#XsyEcke+C;WtCM<@?P_jXD0 zQ(bJ!$Rv7H9Kn@7UK|j-*-J80zA`fTo!1 z0XOP&g8#fJT$&jy*w+*;$PC7&aa<-iIUA4vR$ub~f7$0GX9cJB5TS>du1CG}W zuVDR9E5&L7eCKp^MG4*_DJ!d3 zQ(Ep;c!_wgq)I8r3t{xy7oNccM1@9SiK%f6Yq7D#Q^yu70zXh~mBv`~6IGRr#jGkS zVa&FCElv>S<#>mTv1GoMdWKb1p=UV0Co$>!WlylE#PW(&Wh`+NTa@$Q+98B1gEdRo!4vXbJ73cbX(s;IK)3GBL3xji!c zc6wE)`; zSz1)a($+FnT{oXCuPCpoC@W!FQKgzO>F%`D(u!1ZBQAgdJs;SGC|STz!o!Ii=9@pu zD~e0vSaHcR&D~3kobu(R#3yN@3#qIJr$5~Zy4 ziBd=g2(Z8;4TvP(|C^w!Dk~`hDK&hGB63!=YE@+gHdCaMoK%`r1aBZuIg`>9H?@9G}m@90It85=o*v2izizN_ur z_ZXXnaWCLEGX8zcr|U6&<`2eoTKc5BCf_~fo(xCMjNH7LvkJs3j?S5L3uit!jpU}P zq(oU=T3k}0JOP?R%~ltcmXRo>D$e-@Ik_{%C@@GZE!WTp(yS>eRh7HaFm-tx%g3y# z%7Z0fw?;geswjo)R-p;nvvzDT8@pT?n|9Y?W$a|!PPnH^89QN06`P4Ch$}(--jxaP zn5yMP3VWICOFPkzk zed6626V#%ziYn!f>}6VMS@FtJwF<9&E~{9ZqTHIU%+D`S3Lcy}GryqlvAl=o=H%a- zs}yP_%Dm;Ol75epo<8;NDO1xN%7VgNWm4LtyV%EfGWIM!L43xhG4>^XFXJ-+*D#4x z5Ee@JHw4O^_IM{6TkGnB!1Xw(Bo#VQIGleZFk)ZIyRxx z?GLoQ(Z)lYO@W#lzm z@#{CnKbMWh+D&O-l1pm4)IPec`U*x%RnfcYC+hc0(Nk@(0*fRooj8}1$8_qy%(=$W zkJjyG*-T1rZe&>Z!pr7u-BYpf#r{}nkrfREOfnn9v&y11n%-9R9k?D2$xz+Eib zgO^zjqx1jIzi+z{Mu}Iy>%n7J-2?*}U~TkR$bc*5O_PtIgD&Y!*QIvX=r>)}^op-k z73(_iCSBC_VW&$nTy+g3BCfjVr~d>1l8~au6L1f=#0p%87wW&Ac8xu{(B;3R|5rAy zT>chU-F9$@EttR1RhKmY;}$(`bJcl>t(fuXBR>d3MK=V*BIy~qnD{t*4h?rtCPeFY zcR}bH*?(SREy1X(7hKPO`>3nVd;Uf0`vu9f3xP5O8jWcX)1K%Ei~T?L=kk7f-sOK! zA9z0@{UOl-@)+v8de4s*L!qV8F&|E`NLDsl8qTCcp4UR52c+N+*z}*ztXYxef8oTI zzy9LX+%JP4>t9@a?AWB0KmK5O|9wYCzwp({jSFAuDp|Mu)&A4QeY$Z=&d`Y!|Il8% Hc>I3>DLZq= literal 0 HcmV?d00001 diff --git a/internal/pkg/secureboot/uki/uki.go b/internal/pkg/secureboot/uki/uki.go new file mode 100644 index 0000000..84c3d19 --- /dev/null +++ b/internal/pkg/secureboot/uki/uki.go @@ -0,0 +1,135 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package uki creates the UKI file out of the sd-stub and other sections. +package uki + +import ( + "fmt" + "log" + "os" + + "github.com/aenix-io/talm/internal/pkg/secureboot" + "github.com/aenix-io/talm/internal/pkg/secureboot/measure" + "github.com/aenix-io/talm/internal/pkg/secureboot/pesign" +) + +// section is a UKI file section. +type section struct { + // Section name. + Name secureboot.Section + // Path to the contents of the section. + Path string + // Should the section be measured to the TPM? + Measure bool + // Should the section be appended, or is it already in the PE file. + Append bool + // Size & VMA of the section. + Size uint64 + VMA uint64 +} + +// Builder is a UKI file builder. +type Builder struct { + // Source options. + // + // Arch of the UKI file. + Arch string + // Version of Talos. + Version string + // Path to the sd-stub. + SdStubPath string + // Path to the sd-boot. + SdBootPath string + // Path to the kernel image. + KernelPath string + // Path to the initrd image. + InitrdPath string + // Kernel cmdline. + Cmdline string + // SecureBoot certificate and signer. + SecureBootSigner pesign.CertificateSigner + // PCR signer. + PCRSigner measure.RSAKey + + // Output options: + // + // Path to the signed sd-boot. + OutSdBootPath string + // Path to the output UKI file. + OutUKIPath string + + // fields initialized during build + sections []section + scratchDir string + peSigner *pesign.Signer + unsignedUKIPath string +} + +// Build the UKI file. +// +// Build process is as follows: +// - sign the sd-boot EFI binary, and write it to the OutSdBootPath +// - build ephemeral sections (uname, os-release), and other proposed sections +// - measure sections, generate signature, and append to the list of sections +// - assemble the final UKI file starting from sd-stub and appending generated section. +func (builder *Builder) Build(printf func(string, ...any)) error { + var err error + + builder.scratchDir, err = os.MkdirTemp("", "talos-uki") + if err != nil { + return err + } + + defer func() { + if err = os.RemoveAll(builder.scratchDir); err != nil { + log.Printf("failed to remove scratch dir: %v", err) + } + }() + + printf("signing systemd-boot") + + builder.peSigner, err = pesign.NewSigner(builder.SecureBootSigner) + if err != nil { + return fmt.Errorf("error initializing signer: %w", err) + } + + // sign sd-boot + if err = builder.peSigner.Sign(builder.SdBootPath, builder.OutSdBootPath); err != nil { + return fmt.Errorf("error signing sd-boot: %w", err) + } + + printf("generating UKI sections") + + // generate and build list of all sections + for _, generateSection := range []func() error{ + builder.generateOSRel, + builder.generateCmdline, + builder.generateInitrd, + builder.generateSplash, + builder.generateUname, + builder.generateSBAT, + builder.generatePCRPublicKey, + // append kernel last to account for decompression + builder.generateKernel, + // measure sections last + builder.generatePCRSig, + } { + if err = generateSection(); err != nil { + return fmt.Errorf("error generating sections: %w", err) + } + } + + printf("assembling UKI") + + // assemble the final UKI file + if err = builder.assemble(); err != nil { + return fmt.Errorf("error assembling UKI: %w", err) + } + + printf("signing UKI") + + // sign the UKI file + return builder.peSigner.Sign(builder.unsignedUKIPath, builder.OutUKIPath) +} diff --git a/internal/pkg/smbios/smbios.go b/internal/pkg/smbios/smbios.go new file mode 100644 index 0000000..f5bb68c --- /dev/null +++ b/internal/pkg/smbios/smbios.go @@ -0,0 +1,27 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package smbios provides access to SMBIOS information. +package smbios + +import ( + "sync" + + "github.com/siderolabs/go-smbios/smbios" +) + +var ( + syncSMBIOS sync.Once + connSMBIOS *smbios.SMBIOS + errSMBIOS error +) + +// GetSMBIOSInfo returns the SMBIOS info. +func GetSMBIOSInfo() (*smbios.SMBIOS, error) { + syncSMBIOS.Do(func() { + connSMBIOS, errSMBIOS = smbios.New() + }) + + return connSMBIOS, errSMBIOS +} diff --git a/pkg/commands/bootstrap.go b/pkg/commands/imported_bootstrap.go similarity index 83% rename from pkg/commands/bootstrap.go rename to pkg/commands/imported_bootstrap.go index 851e81b..a959597 100644 --- a/pkg/commands/bootstrap.go +++ b/pkg/commands/imported_bootstrap.go @@ -1,3 +1,6 @@ +// Code generated by go run tools/import_commands.go --talos-version v1.7.1 bootstrap +// DO NOT EDIT. + // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. @@ -6,6 +9,7 @@ package commands import ( "context" + "errors" "fmt" "os" @@ -18,9 +22,9 @@ import ( ) var bootstrapCmdFlags struct { - configFiles []string // -f/--files recoverFrom string recoverSkipHashCheck bool + configFiles []string } // bootstrapCmd represents the bootstrap command. @@ -36,23 +40,10 @@ This command should not be used when "init" type node are used. Talos etcd cluster can be recovered from a known snapshot with '--recover-from=' flag.`, Args: cobra.NoArgs, - PreRunE: func(cmd *cobra.Command, args []string) error { - if len(bootstrapCmdFlags.configFiles) > 1 { - return fmt.Errorf("command \"bootstrap\" is not supported with multiple --file") - } - nodesFromArgs := len(GlobalArgs.Nodes) > 0 - endpointsFromArgs := len(GlobalArgs.Endpoints) > 0 - for _, configFile := range bootstrapCmdFlags.configFiles { - if err := processModelineAndUpdateGlobals(configFile, nodesFromArgs, endpointsFromArgs, true); err != nil { - return err - } - } - return nil - }, RunE: func(cmd *cobra.Command, args []string) error { return WithClient(func(ctx context.Context, c *client.Client) error { if len(GlobalArgs.Nodes) > 1 { - return fmt.Errorf("command \"bootstrap\" is not supported with multiple nodes") + return errors.New("command \"bootstrap\" is not supported with multiple nodes") } if bootstrapCmdFlags.recoverFrom != "" { @@ -92,7 +83,27 @@ Talos etcd cluster can be recovered from a known snapshot with '--recover-from=' } func init() { - bootstrapCmd.Flags().StringSliceVarP(&bootstrapCmdFlags.configFiles, "file", "f", nil, "specify config file or patch in a YAML file") + bootstrapCmd.Flags().StringSliceVarP(&bootstrapCmdFlags.configFiles, + "file", "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + bootstrapCmd.PreRunE = func(cmd *cobra.Command, + + args []string) error { + nodesFromArgs := len(GlobalArgs.Nodes) > 0 + endpointsFromArgs := len(GlobalArgs.Endpoints) > + + 0 + for _, configFile := range bootstrapCmdFlags. + configFiles { + if err := processModelineAndUpdateGlobals(configFile, + nodesFromArgs, endpointsFromArgs, false); err != + nil { + return err + } + } + return nil + } + bootstrapCmd.Flags().StringVar(&bootstrapCmdFlags.recoverFrom, "recover-from", "", "recover etcd cluster from the snapshot") bootstrapCmd.Flags().BoolVar(&bootstrapCmdFlags.recoverSkipHashCheck, "recover-skip-hash-check", false, "skip integrity check when recovering etcd (use when recovering from data directory copy)") addCommand(bootstrapCmd) diff --git a/pkg/commands/imported_containers.go b/pkg/commands/imported_containers.go new file mode 100644 index 0000000..6057166 --- /dev/null +++ b/pkg/commands/imported_containers.go @@ -0,0 +1,129 @@ +// Code generated by go run tools/import_commands.go --talos-version v1.7.1 containers +// DO NOT EDIT. + +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package commands + +import ( + "context" + "fmt" + "os" + "sort" + "strings" + "text/tabwriter" + + criconstants "github.com/containerd/containerd/pkg/cri/constants" + "github.com/spf13/cobra" + "google.golang.org/grpc" + "google.golang.org/grpc/peer" + + "github.com/siderolabs/talos/pkg/cli" + "github.com/siderolabs/talos/pkg/machinery/api/common" + machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/client" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// containersCmd represents the processes command. +var containersCmd = &cobra.Command{ + Use: "containers", + Aliases: []string{"c"}, + Short: "List containers", + Long: ``, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return WithClient(func(ctx context.Context, c *client.Client) error { + var ( + namespace string + driver common.ContainerDriver + ) + + if kubernetesFlag { + namespace = criconstants.K8sContainerdNamespace + driver = common.ContainerDriver_CRI + } else { + namespace = constants.SystemContainerdNamespace + driver = common.ContainerDriver_CONTAINERD + } + + var remotePeer peer.Peer + + resp, err := c.Containers(ctx, namespace, driver, grpc.Peer(&remotePeer)) + if err != nil { + if resp == nil { + return fmt.Errorf("error getting container list: %s", err) + } + + cli.Warning("%s", err) + } + + return containerRender(&remotePeer, resp) + }) + }, +} + +func containerRender(remotePeer *peer.Peer, resp *machineapi.ContainersResponse) error { + w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + fmt.Fprintln(w, "NODE\tNAMESPACE\tID\tIMAGE\tPID\tSTATUS") + + defaultNode := client.AddrFromPeer(remotePeer) + + for _, msg := range resp.Messages { + sort.Slice(msg.Containers, + func(i, j int) bool { + return strings.Compare(msg.Containers[i].Id, msg.Containers[j].Id) < 0 + }) + + for _, p := range msg.Containers { + display := p.Id + if p.Id != p.PodId { + // container in a sandbox + display = "└─ " + display + } + + node := defaultNode + + if msg.Metadata != nil { + node = msg.Metadata.Hostname + } + + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%d\t%s\n", node, p.Namespace, display, p.Image, p.Pid, p.Status) + } + } + + return w.Flush() +} + +func init() { + containersCmd.Flags().StringSliceVarP(&containersCmdFlags.configFiles, + "file", "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + containersCmd.PreRunE = func(cmd *cobra.Command, args []string) error { + nodesFromArgs := len(GlobalArgs. + Nodes) > 0 + + endpointsFromArgs := len(GlobalArgs.Endpoints) > 0 + for _, configFile := range containersCmdFlags. + configFiles { + if err := processModelineAndUpdateGlobals(configFile, nodesFromArgs, endpointsFromArgs, + false); err != nil { + return err + } + } + return nil + } + + containersCmd.Flags().BoolVarP(&kubernetesFlag, "kubernetes", "k", false, "use the k8s.io containerd namespace") + + containersCmd.Flags().BoolP("use-cri", "c", false, "use the CRI driver") + containersCmd.Flags().MarkHidden("use-cri") //nolint:errcheck + + addCommand(containersCmd) +} + +var containersCmdFlags struct { + configFiles []string +} diff --git a/pkg/commands/dashboard.go b/pkg/commands/imported_dashboard.go similarity index 59% rename from pkg/commands/dashboard.go rename to pkg/commands/imported_dashboard.go index 3f4e126..4bdb035 100644 --- a/pkg/commands/dashboard.go +++ b/pkg/commands/imported_dashboard.go @@ -1,3 +1,6 @@ +// Code generated by go run tools/import_commands.go --talos-version v1.7.1 dashboard +// DO NOT EDIT. + // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. @@ -15,8 +18,8 @@ import ( ) var dashboardCmdFlags struct { - configFiles []string // -f/--files interval time.Duration + configFiles []string } // dashboardCmd represents the monitor command. @@ -27,26 +30,16 @@ var dashboardCmd = &cobra.Command{ Keyboard shortcuts: - - h, : switch one node to the left - - l, : switch one node to the right - - j, : scroll logs/process list down - - k, : scroll logs/process list up - - : scroll logs/process list half page down - - : scroll logs/process list half page up - - : scroll logs/process list one page down - - : scroll logs/process list one page up + - h, <Left> - switch one node to the left + - l, <Right> - switch one node to the right + - j, <Down> - scroll logs/process list down + - k, <Up> - scroll logs/process list up + - <C-d> - scroll logs/process list half page down + - <C-u> - scroll logs/process list half page up + - <C-f> - scroll logs/process list one page down + - <C-b> - scroll logs/process list one page up `, Args: cobra.NoArgs, - PreRunE: func(cmd *cobra.Command, args []string) error { - nodesFromArgs := len(GlobalArgs.Nodes) > 0 - endpointsFromArgs := len(GlobalArgs.Endpoints) > 0 - for _, configFile := range dashboardCmdFlags.configFiles { - if err := processModelineAndUpdateGlobals(configFile, nodesFromArgs, endpointsFromArgs, false); err != nil { - return err - } - } - return nil - }, RunE: func(cmd *cobra.Command, args []string) error { return WithClient(func(ctx context.Context, c *client.Client) error { return dashboard.Run(ctx, c, @@ -59,7 +52,26 @@ Keyboard shortcuts: } func init() { - dashboardCmd.Flags().StringSliceVarP(&dashboardCmdFlags.configFiles, "file", "f", nil, "specify config files or patches in a YAML file (can specify multiple)") + dashboardCmd.Flags().StringSliceVarP(&dashboardCmdFlags.configFiles, + "file", "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + dashboardCmd.PreRunE = func(cmd *cobra.Command, + + args []string) error { + nodesFromArgs := len(GlobalArgs. + Nodes) > 0 + endpointsFromArgs := len(GlobalArgs.Endpoints) > 0 + for _, configFile := range dashboardCmdFlags. + configFiles { + if err := + processModelineAndUpdateGlobals(configFile, nodesFromArgs, endpointsFromArgs, + false); err != nil { + return err + } + } + return nil + } + dashboardCmd.Flags().DurationVarP(&dashboardCmdFlags.interval, "update-interval", "d", 3*time.Second, "interval between updates") addCommand(dashboardCmd) } diff --git a/pkg/commands/imported_disks.go b/pkg/commands/imported_disks.go new file mode 100644 index 0000000..b292328 --- /dev/null +++ b/pkg/commands/imported_disks.go @@ -0,0 +1,168 @@ +// Code generated by go run tools/import_commands.go --talos-version v1.7.1 disks +// DO NOT EDIT. + +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package commands + +import ( + "context" + "fmt" + "os" + "strings" + "text/tabwriter" + + humanize "github.com/dustin/go-humanize" + "github.com/spf13/cobra" + + "github.com/siderolabs/talos/pkg/cli" + "github.com/siderolabs/talos/pkg/machinery/client" +) + +var disksCmdFlags struct { + insecure bool + configFiles []string +} + +var disksCmd = &cobra.Command{ + Use: "disks", + Short: "Get the list of disks from /sys/block on the machine", + Long: ``, + RunE: func(cmd *cobra.Command, args []string) error { + if disksCmdFlags.insecure { + return WithClientMaintenance(nil, printDisks) + } + + return WithClient(printDisks) + }, +} + +//nolint:gocyclo +func printDisks(ctx context.Context, c *client.Client) error { + response, err := c.Disks(ctx) + if err != nil { + if response == nil { + return fmt.Errorf("error getting disks: %w", err) + } + + cli.Warning("%s", err) + } + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + node := "" + + labels := strings.Join( + []string{ + "DEV", + "MODEL", + "SERIAL", + "TYPE", + "UUID", + "WWID", + "MODALIAS", + "NAME", + "SIZE", + "BUS_PATH", + "SUBSYSTEM", + "READ_ONLY", + "SYSTEM_DISK", + }, "\t") + + getWithPlaceholder := func(in string) string { + if in == "" { + return "-" + } + + return in + } + + for i, message := range response.Messages { + if message.Metadata != nil && message.Metadata.Hostname != "" { + node = message.Metadata.Hostname + } + + if len(message.Disks) == 0 { + continue + } + + for j, disk := range message.Disks { + if i == 0 && j == 0 { + if node != "" { + fmt.Fprintln(w, "NODE\t"+labels) + } else { + fmt.Fprintln(w, labels) + } + } + + args := []interface{}{} + + if node != "" { + args = append(args, node) + } + + isReadonly := "" + + if disk.Readonly { + isReadonly = "*" + } + + isSystemDisk := "" + + if disk.SystemDisk { + isSystemDisk = "*" + } + + args = append(args, []interface{}{ + getWithPlaceholder(disk.DeviceName), + getWithPlaceholder(disk.Model), + getWithPlaceholder(disk.Serial), + disk.Type.String(), + getWithPlaceholder(disk.Uuid), + getWithPlaceholder(disk.Wwid), + getWithPlaceholder(disk.Modalias), + getWithPlaceholder(disk.Name), + humanize.Bytes(disk.Size), + getWithPlaceholder(disk.BusPath), + getWithPlaceholder(disk.Subsystem), + isReadonly, + isSystemDisk, + }...) + + pattern := strings.Repeat("%s\t", len(args)) + pattern = strings.TrimSpace(pattern) + "\n" + + fmt.Fprintf(w, pattern, args...) + } + } + + return w.Flush() +} + +func init() { + disksCmd.Flags().StringSliceVarP(&disksCmdFlags.configFiles, "file", + "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + disksCmd.PreRunE = func(cmd *cobra.Command, args []string) error { + nodesFromArgs := len(GlobalArgs. + Nodes) > 0 + endpointsFromArgs := len(GlobalArgs.Endpoints) > + 0 + for _, configFile := range disksCmdFlags.configFiles { + if err := processModelineAndUpdateGlobals(configFile, + nodesFromArgs, + + endpointsFromArgs, + false); err != + + nil { + return err + } + } + return nil + } + + disksCmd.Flags().BoolVarP(&disksCmdFlags.insecure, "insecure", "i", false, "get disks using the insecure (encrypted with no auth) maintenance service") + addCommand(disksCmd) +} diff --git a/pkg/commands/imported_dmesg.go b/pkg/commands/imported_dmesg.go new file mode 100644 index 0000000..b5519f1 --- /dev/null +++ b/pkg/commands/imported_dmesg.go @@ -0,0 +1,75 @@ +// Code generated by go run tools/import_commands.go --talos-version v1.7.1 dmesg +// DO NOT EDIT. + +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package commands + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" + "github.com/siderolabs/talos/pkg/machinery/api/common" + "github.com/siderolabs/talos/pkg/machinery/client" +) + +var dmesgTail bool + +// dmesgCmd represents the dmesg command. +var dmesgCmd = &cobra.Command{ + Use: "dmesg", + Short: "Retrieve kernel logs", + Long: ``, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return WithClient(func(ctx context.Context, c *client.Client) error { + stream, err := c.Dmesg(ctx, follow, dmesgTail) + if err != nil { + return fmt.Errorf("error getting dmesg: %w", err) + } + + return helpers.ReadGRPCStream(stream, func(data *common.Data, node string, multipleNodes bool) error { + if data.Bytes != nil { + fmt.Printf("%s: %s", node, data.Bytes) + } + + return nil + }) + }) + }, +} + +func init() { + dmesgCmd.Flags().StringSliceVarP(&dmesgCmdFlags.configFiles, "file", + "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + dmesgCmd.PreRunE = func(cmd *cobra.Command, args []string) error { + nodesFromArgs := + len(GlobalArgs.Nodes) > + 0 + endpointsFromArgs := len(GlobalArgs.Endpoints) > 0 + for _, configFile := range dmesgCmdFlags.configFiles { + if err := processModelineAndUpdateGlobals(configFile, nodesFromArgs, + + endpointsFromArgs, + + false); err != nil { + return err + } + } + return nil + } + + addCommand(dmesgCmd) + dmesgCmd.Flags().BoolVarP(&follow, "follow", "F", false, "specify if the kernel log should be streamed") + dmesgCmd.Flags().BoolVarP(&dmesgTail, "tail", "", false, "specify if only new messages should be sent (makes sense only when combined with --follow)") +} + +var dmesgCmdFlags struct { + configFiles []string +} diff --git a/pkg/commands/imported_etcd.go b/pkg/commands/imported_etcd.go new file mode 100644 index 0000000..fa379b6 --- /dev/null +++ b/pkg/commands/imported_etcd.go @@ -0,0 +1,480 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package commands + +import ( + "context" + "crypto/sha256" + "fmt" + "io" + "os" + "strings" + "text/tabwriter" + + "github.com/dustin/go-humanize" + "github.com/siderolabs/gen/xslices" + "github.com/spf13/cobra" + snapshot "go.etcd.io/etcd/etcdutl/v3/snapshot" + + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" + "github.com/siderolabs/talos/pkg/cli" + "github.com/siderolabs/talos/pkg/logging" + "github.com/siderolabs/talos/pkg/machinery/api/common" + "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/client" + etcdresource "github.com/siderolabs/talos/pkg/machinery/resources/etcd" +) + +// etcdCmd represents the etcd command. +var etcdCmd = &cobra.Command{ + Use: "etcd", + Short: "Manage etcd", + Long: ``, +} + +// etcdAlarmCmd represents the etcd alarm command. +var etcdAlarmCmd = &cobra.Command{ + Use: "alarm", + Short: "Manage etcd alarms", + Long: ``, +} + +type alarmMessage interface { + GetMetadata() *common.Metadata + GetMemberAlarms() []*machine.EtcdMemberAlarm +} + +func displayAlarms(messages []alarmMessage) error { + w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + node := "" + pattern := "%s\t%s\n" + header := "MEMBER\tALARM" + + for i, message := range messages { + if message.GetMetadata() != nil && message.GetMetadata().GetHostname() != "" { + node = message.GetMetadata().GetHostname() + } + + for j, alarm := range message.GetMemberAlarms() { + if i == 0 && j == 0 { + if node != "" { + header = "NODE\t" + header + pattern = "%s\t" + pattern + } + + fmt.Fprintln(w, header) + } + + args := []interface{}{ + etcdresource.FormatMemberID(alarm.GetMemberId()), + alarm.GetAlarm().String(), + } + if node != "" { + args = append([]interface{}{node}, args...) + } + + fmt.Fprintf(w, pattern, args...) + } + } + + return w.Flush() +} + +// etcdAlarmListCmd represents the etcd alarm list command. +var etcdAlarmListCmd = &cobra.Command{ + Use: "list", + Short: "List the etcd alarms for the node.", + Long: ``, + RunE: func(cmd *cobra.Command, args []string) error { + return WithClient(func(ctx context.Context, c *client.Client) error { + response, err := c.EtcdAlarmList(ctx) + if err != nil { + if response == nil { + return fmt.Errorf("error getting alarms: %w", err) + } + cli.Warning("%s", err) + } + + return displayAlarms(xslices.Map(response.Messages, func(v *machine.EtcdAlarm) alarmMessage { + return v + })) + }) + }, +} + +// etcdAlarmDisarmCmd represents the etcd alarm disarm command. +var etcdAlarmDisarmCmd = &cobra.Command{ + Use: "disarm", + Short: "Disarm the etcd alarms for the node.", + Long: ``, + RunE: func(cmd *cobra.Command, args []string) error { + return WithClient(func(ctx context.Context, c *client.Client) error { + response, err := c.EtcdAlarmDisarm(ctx) + if err != nil { + if response == nil { + return fmt.Errorf("error disarming alarms: %w", err) + } + cli.Warning("%s", err) + } + + return displayAlarms(xslices.Map(response.Messages, func(v *machine.EtcdAlarmDisarm) alarmMessage { + return v + })) + }) + }, +} + +// etcdDefragCmd represents the etcd defrag command. +var etcdDefragCmd = &cobra.Command{ + Use: "defrag", + Short: "Defragment etcd database on the node", + Long: `Defragmentation is a maintenance operation that releases unused space from the etcd database file. +Defragmentation is a resource heavy operation and should be performed only when necessary on a single node at a time.`, + RunE: func(cmd *cobra.Command, args []string) error { + return WithClient(func(ctx context.Context, c *client.Client) error { + if err := helpers.FailIfMultiNodes(ctx, "etcd defrag"); err != nil { + return err + } + + _, err := c.EtcdDefragment(ctx) + + return err + }) + }, +} + +var etcdLeaveCmd = &cobra.Command{ + Use: "leave", + Short: "Tell nodes to leave etcd cluster", + Long: ``, + RunE: func(cmd *cobra.Command, args []string) error { + return WithClient(func(ctx context.Context, c *client.Client) error { + if err := helpers.FailIfMultiNodes(ctx, "etcd leave"); err != nil { + return err + } + + return c.EtcdLeaveCluster(ctx, &machine.EtcdLeaveClusterRequest{}) + }) + }, +} + +var etcdMemberRemoveCmd = &cobra.Command{ + Use: "remove-member ", + Short: "Remove the node from etcd cluster", + Long: `Use this command only if you want to remove a member which is in broken state. +If there is no access to the node, or the node can't access etcd to call etcd leave. +Always prefer etcd leave over this command.`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return WithClient(func(ctx context.Context, c *client.Client) error { + memberID, err := etcdresource.ParseMemberID(args[0]) + if err != nil { + return fmt.Errorf("error parsing member ID: %w", err) + } + + return c.EtcdRemoveMemberByID(ctx, &machine.EtcdRemoveMemberByIDRequest{ + MemberId: memberID, + }) + }) + }, +} + +var etcdForfeitLeadershipCmd = &cobra.Command{ + Use: "forfeit-leadership", + Short: "Tell node to forfeit etcd cluster leadership", + Long: ``, + RunE: func(cmd *cobra.Command, args []string) error { + return WithClient(func(ctx context.Context, c *client.Client) error { + _, err := c.EtcdForfeitLeadership(ctx, &machine.EtcdForfeitLeadershipRequest{}) + + return err + }) + }, +} + +var etcdMemberListCmd = &cobra.Command{ + Use: "members", + Short: "Get the list of etcd cluster members", + Long: ``, + RunE: func(cmd *cobra.Command, args []string) error { + return WithClient(func(ctx context.Context, c *client.Client) error { + response, err := c.EtcdMemberList(ctx, &machine.EtcdMemberListRequest{ + QueryLocal: true, + }) + if err != nil { + if response == nil { + return fmt.Errorf("error getting members: %w", err) + } + cli.Warning("%s", err) + } + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + node := "" + pattern := "%s\t%s\t%s\t%s\t%v\n" + + for i, message := range response.Messages { + if message.Metadata != nil && message.Metadata.Hostname != "" { + node = message.Metadata.Hostname + } + + if len(message.Members) == 0 { + continue + } + + for j, member := range message.Members { + if i == 0 && j == 0 { + if node != "" { + fmt.Fprintln(w, "NODE\tID\tHOSTNAME\tPEER URLS\tCLIENT URLS\tLEARNER") + pattern = "%s\t" + pattern + } else { + fmt.Fprintln(w, "ID\tHOSTNAME\tPEER URLS\tCLIENT URLS\tLEARNER") + } + } + + args := []interface{}{ + etcdresource.FormatMemberID(member.Id), + member.Hostname, + strings.Join(member.PeerUrls, ","), + strings.Join(member.ClientUrls, ","), + member.IsLearner, + } + if node != "" { + args = append([]interface{}{node}, args...) + } + + fmt.Fprintf(w, pattern, args...) + } + } + + return w.Flush() + }) + }, +} + +var etcdStatusCmd = &cobra.Command{ + Use: "status", + Short: "Get the status of etcd cluster member", + Long: `Returns the status of etcd member on the node, use multiple nodes to get status of all members.`, + RunE: func(cmd *cobra.Command, args []string) error { + return WithClient(func(ctx context.Context, c *client.Client) error { + response, err := c.EtcdStatus(ctx) + if err != nil { + if response == nil { + return fmt.Errorf("error getting status: %w", err) + } + cli.Warning("%s", err) + } + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + node := "" + pattern := "%s\t%s\t%s (%.2f%%)\t%s\t%d\t%d\t%d\t%v\t%s\n" + header := "MEMBER\tDB SIZE\tIN USE\tLEADER\tRAFT INDEX\tRAFT TERM\tRAFT APPLIED INDEX\tLEARNER\tERRORS" + + for i, message := range response.Messages { + if message.Metadata != nil && message.Metadata.Hostname != "" { + node = message.Metadata.Hostname + } + + if i == 0 { + if node != "" { + header = "NODE\t" + header + pattern = "%s\t" + pattern + } + + fmt.Fprintln(w, header) + } + + var ratio float64 + + if message.GetMemberStatus().GetDbSize() > 0 { + ratio = float64(message.GetMemberStatus().GetDbSizeInUse()) / float64(message.GetMemberStatus().GetDbSize()) * 100.0 + } + + args := []interface{}{ + etcdresource.FormatMemberID(message.GetMemberStatus().GetMemberId()), + humanize.Bytes(uint64(message.GetMemberStatus().GetDbSize())), + humanize.Bytes(uint64(message.GetMemberStatus().GetDbSizeInUse())), + ratio, + etcdresource.FormatMemberID(message.GetMemberStatus().GetLeader()), + message.GetMemberStatus().GetRaftIndex(), + message.GetMemberStatus().GetRaftTerm(), + message.GetMemberStatus().GetRaftAppliedIndex(), + message.GetMemberStatus().GetIsLearner(), + strings.Join(message.GetMemberStatus().GetErrors(), ", "), + } + if node != "" { + args = append([]interface{}{node}, args...) + } + + fmt.Fprintf(w, pattern, args...) + } + + return w.Flush() + }) + }, +} + +var etcdSnapshotCmd = &cobra.Command{ + Use: "snapshot ", + Short: "Stream snapshot of the etcd node to the path.", + Long: ``, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return WithClient(func(ctx context.Context, c *client.Client) error { + if err := helpers.FailIfMultiNodes(ctx, "etcd snapshot"); err != nil { + return err + } + + dbPath := args[0] + partPath := dbPath + ".part" + + defer os.RemoveAll(partPath) //nolint:errcheck + + dest, err := os.OpenFile(partPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) + if err != nil { + return fmt.Errorf("error creating temporary file: %w", err) + } + + defer dest.Close() //nolint:errcheck + + r, err := c.EtcdSnapshot(ctx, &machine.EtcdSnapshotRequest{}) + if err != nil { + return fmt.Errorf("error reading file: %w", err) + } + + defer r.Close() //nolint:errcheck + + size, err := io.Copy(dest, r) + if err != nil { + return fmt.Errorf("error reading: %w", err) + } + + if err = dest.Sync(); err != nil { + return fmt.Errorf("failed to fsync: %w", err) + } + + // this check is from https://github.com/etcd-io/etcd/blob/client/v3.5.0-alpha.0/client/v3/snapshot/v3_snapshot.go#L46 + if (size % 512) != sha256.Size { + return fmt.Errorf("sha256 checksum not found (size %d)", size) + } + + if err = dest.Close(); err != nil { + return fmt.Errorf("failed to close: %w", err) + } + + if err = os.Rename(partPath, dbPath); err != nil { + return fmt.Errorf("error renaming to final location: %w", err) + } + + fmt.Printf("etcd snapshot saved to %q (%d bytes)\n", dbPath, size) + + manager := snapshot.NewV3(logging.Wrap(os.Stderr)) + + status, err := manager.Status(dbPath) + if err != nil { + return err + } + + fmt.Printf("snapshot info: hash %08x, revision %d, total keys %d, total size %d\n", + status.Hash, status.Revision, status.TotalKey, status.TotalSize) + + return nil + }) + }, +} + +func init() { + etcdCmd.Flags().StringSliceVarP(&etcdCmdFlags.configFiles, "file", + "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + etcdCmd.PreRunE = func(cmd *cobra.Command, + + args []string) error { + nodesFromArgs := len(GlobalArgs. + Nodes, + ) > + 0 + endpointsFromArgs := len(GlobalArgs. + Endpoints) > 0 + for _, configFile := range etcdCmdFlags.configFiles { + if err := processModelineAndUpdateGlobals(configFile, nodesFromArgs, + + endpointsFromArgs, false); err != nil { + return err + } + } + return nil + } + etcdAlarmCmd. + Flags().StringSliceVarP(&etcdCmdFlags.configFiles, + "file", "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + etcdAlarmCmd. + PreRunE = etcdCmd.PreRunE + etcdDefragCmd.Flags().StringSliceVarP(&etcdCmdFlags.configFiles, "file", "f", nil, "specify config files or patches in a YAML file (can specify multiple)") + etcdDefragCmd. + PreRunE = etcdCmd. + PreRunE + etcdForfeitLeadershipCmd. + Flags().StringSliceVarP(&etcdCmdFlags.configFiles, + "file", "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + etcdForfeitLeadershipCmd.PreRunE = etcdCmd.PreRunE + etcdLeaveCmd. + Flags().StringSliceVarP(&etcdCmdFlags. + configFiles, "file", "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + etcdLeaveCmd. + PreRunE = etcdCmd.PreRunE + etcdMemberListCmd. + Flags().StringSliceVarP(&etcdCmdFlags. + configFiles, "file", "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + etcdMemberListCmd.PreRunE = etcdCmd.PreRunE + etcdMemberRemoveCmd.Flags(). + StringSliceVarP(&etcdCmdFlags. + configFiles, "file", + "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + etcdMemberRemoveCmd. + PreRunE = etcdCmd.PreRunE + etcdSnapshotCmd.Flags().StringSliceVarP(&etcdCmdFlags. + configFiles, "file", + "f", + nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + etcdSnapshotCmd.PreRunE = etcdCmd. + PreRunE + + etcdStatusCmd. + Flags().StringSliceVarP(&etcdCmdFlags.configFiles, "file", + "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + etcdStatusCmd. + PreRunE = etcdCmd. + PreRunE + + etcdAlarmCmd.AddCommand( + etcdAlarmListCmd, + etcdAlarmDisarmCmd, + ) + + etcdCmd.AddCommand( + etcdAlarmCmd, + etcdDefragCmd, + etcdForfeitLeadershipCmd, + etcdLeaveCmd, + etcdMemberListCmd, + etcdMemberRemoveCmd, + etcdSnapshotCmd, + etcdStatusCmd, + ) + + addCommand(etcdCmd) +} + +var etcdCmdFlags struct { + configFiles []string +} diff --git a/pkg/commands/imported_events.go b/pkg/commands/imported_events.go new file mode 100644 index 0000000..f575c7a --- /dev/null +++ b/pkg/commands/imported_events.go @@ -0,0 +1,152 @@ +// Code generated by go run tools/import_commands.go --talos-version v1.7.1 events +// DO NOT EDIT. + +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package commands + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "text/tabwriter" + "time" + + "github.com/siderolabs/gen/xslices" + "github.com/spf13/cobra" + + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" + "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/client" +) + +var eventsCmdFlags struct { + tailEvents int32 + tailDuration time.Duration + tailID string + actorID string + configFiles []string +} + +// eventsCmd represents the events command. +var eventsCmd = &cobra.Command{ + Use: "events", + Short: "Stream runtime events", + Long: ``, + RunE: func(cmd *cobra.Command, args []string) error { + return WithClient(func(ctx context.Context, c *client.Client) error { + w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + fmt.Fprintln(w, "NODE\tID\tEVENT\tACTOR\tSOURCE\tMESSAGE") + + opts := []client.EventsOptionFunc{} + + if eventsCmdFlags.tailEvents != 0 { + opts = append(opts, client.WithTailEvents(eventsCmdFlags.tailEvents)) + } + + if eventsCmdFlags.tailDuration != 0 { + opts = append(opts, client.WithTailDuration(eventsCmdFlags.tailDuration)) + } + + if eventsCmdFlags.tailID != "" { + opts = append(opts, client.WithTailID(eventsCmdFlags.tailID)) + } + + if eventsCmdFlags.actorID != "" { + opts = append(opts, client.WithActorID(eventsCmdFlags.actorID)) + } + + events, err := c.Events(ctx, opts...) + if err != nil { + return err + } + + return helpers.ReadGRPCStream(events, func(ev *machine.Event, node string, multipleNodes bool) error { + format := "%s\t%s\t%s\n%s\t%s\t%s\n" + + event, err := client.UnmarshalEvent(ev) + if err != nil { + if errors.Is(err, client.ErrEventNotSupported) { + return nil + } + + return err + } + + var args []interface{} + + switch msg := event.Payload.(type) { + case *machine.SequenceEvent: + args = []interface{}{msg.GetSequence()} + if msg.Error != nil { + args = append(args, "error:"+" "+msg.GetError().GetMessage()) + } else { + args = append(args, msg.GetAction().String()) + } + case *machine.PhaseEvent: + args = []interface{}{msg.GetPhase(), msg.GetAction().String()} + case *machine.TaskEvent: + args = []interface{}{msg.GetTask(), msg.GetAction().String()} + case *machine.ServiceStateEvent: + args = []interface{}{msg.GetService(), fmt.Sprintf("%s: %s", msg.GetAction(), msg.GetMessage())} + case *machine.ConfigLoadErrorEvent: + args = []interface{}{"error", msg.GetError()} + case *machine.ConfigValidationErrorEvent: + args = []interface{}{"error", msg.GetError()} + case *machine.AddressEvent: + args = []interface{}{msg.GetHostname(), fmt.Sprintf("ADDRESSES: %s", strings.Join(msg.GetAddresses(), ","))} + case *machine.MachineStatusEvent: + args = []interface{}{ + msg.GetStage().String(), + fmt.Sprintf("ready: %v, unmet conditions: %v", + msg.GetStatus().Ready, + xslices.Map(msg.GetStatus().GetUnmetConditions(), + func(c *machine.MachineStatusEvent_MachineStatus_UnmetCondition) string { + return c.Name + }, + ), + ), + } + } + + args = append([]interface{}{event.Node, event.ID, event.TypeURL, event.ActorID}, args...) + fmt.Fprintf(w, format, args...) + + return w.Flush() + }) + }) + }, +} + +func init() { + eventsCmd.Flags().StringSliceVarP(&eventsCmdFlags.configFiles, + "file", "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + eventsCmd.PreRunE = func(cmd *cobra.Command, args []string) error { + nodesFromArgs := len( + GlobalArgs. + Nodes) > + 0 + endpointsFromArgs := + + len(GlobalArgs.Endpoints) > 0 + for _, configFile := range eventsCmdFlags. + configFiles { + if err := processModelineAndUpdateGlobals(configFile, nodesFromArgs, endpointsFromArgs, false); err != nil { + return err + } + } + return nil + + } + + addCommand(eventsCmd) + eventsCmd.Flags().Int32Var(&eventsCmdFlags.tailEvents, "tail", 0, "show specified number of past events (use -1 to show full history, default is to show no history)") + eventsCmd.Flags().DurationVar(&eventsCmdFlags.tailDuration, "duration", 0, "show events for the past duration interval (one second resolution, default is to show no history)") + eventsCmd.Flags().StringVar(&eventsCmdFlags.tailID, "since", "", "show events after the specified event ID (default is to show no history)") + eventsCmd.Flags().StringVar(&eventsCmdFlags.actorID, "actor-id", "", "filter events by the specified actor ID (default is no filter)") +} diff --git a/pkg/commands/get.go b/pkg/commands/imported_get.go similarity index 95% rename from pkg/commands/get.go rename to pkg/commands/imported_get.go index 8b0ec36..ad37e19 100644 --- a/pkg/commands/get.go +++ b/pkg/commands/imported_get.go @@ -1,3 +1,6 @@ +// Code generated by go run tools/import_commands.go --talos-version v1.7.1 get +// DO NOT EDIT. + // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. @@ -24,12 +27,12 @@ import ( ) var getCmdFlags struct { - insecure bool - configFiles []string // -f/--files + insecure bool - namespace string - output string - watch bool + namespace string + output string + watch bool + configFiles []string } // getCmd represents the get (resources) command. @@ -52,16 +55,6 @@ To get a list of all available resource definitions, issue 'talosctl get rd'`, return nil, cobra.ShellCompDirectiveError | cobra.ShellCompDirectiveNoFileComp }, Args: cobra.RangeArgs(1, 2), - PreRunE: func(cmd *cobra.Command, args []string) error { - nodesFromArgs := len(GlobalArgs.Nodes) > 0 - endpointsFromArgs := len(GlobalArgs.Endpoints) > 0 - for _, configFile := range getCmdFlags.configFiles { - if err := processModelineAndUpdateGlobals(configFile, nodesFromArgs, endpointsFromArgs, false); err != nil { - return err - } - } - return nil - }, RunE: func(cmd *cobra.Command, args []string) error { if getCmdFlags.insecure { return WithClientMaintenance(nil, getResources(args)) @@ -326,7 +319,22 @@ func CompleteNodes(*cobra.Command, []string, string) ([]string, cobra.ShellCompD } func init() { - getCmd.Flags().StringSliceVarP(&getCmdFlags.configFiles, "file", "f", nil, "specify config files or patches in a YAML file (can specify multiple)") + getCmd.Flags().StringSliceVarP(&getCmdFlags.configFiles, "file", + "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + getCmd.PreRunE = func(cmd *cobra.Command, args []string) error { + nodesFromArgs := len( + GlobalArgs.Nodes) > 0 + endpointsFromArgs := len(GlobalArgs.Endpoints) > 0 + for _, configFile := range getCmdFlags.configFiles { + if err := processModelineAndUpdateGlobals(configFile, nodesFromArgs, + endpointsFromArgs, false); err != nil { + return err + } + } + return nil + } + getCmd.Flags().StringVar(&getCmdFlags.namespace, "namespace", "", "resource namespace (default is to use default namespace per resource)") getCmd.Flags().StringVarP(&getCmdFlags.output, "output", "o", "table", "output mode (json, table, yaml, jsonpath)") getCmd.Flags().BoolVarP(&getCmdFlags.watch, "watch", "w", false, "watch resource changes") diff --git a/pkg/commands/imported_health.go b/pkg/commands/imported_health.go new file mode 100644 index 0000000..b34d580 --- /dev/null +++ b/pkg/commands/imported_health.go @@ -0,0 +1,276 @@ +// Code generated by go run tools/import_commands.go --talos-version v1.7.1 health +// DO NOT EDIT. + +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package commands + +import ( + "context" + "fmt" + "io" + "os" + "time" + + "github.com/cosi-project/runtime/pkg/safe" + "github.com/spf13/cobra" + "google.golang.org/grpc/codes" + + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" + "github.com/siderolabs/talos/pkg/cluster" + "github.com/siderolabs/talos/pkg/cluster/check" + "github.com/siderolabs/talos/pkg/cluster/sonobuoy" + clusterapi "github.com/siderolabs/talos/pkg/machinery/api/cluster" + "github.com/siderolabs/talos/pkg/machinery/client" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + clusterres "github.com/siderolabs/talos/pkg/machinery/resources/cluster" +) + +type clusterNodes struct { + InitNode string + ControlPlaneNodes []string + WorkerNodes []string + + nodes []cluster.NodeInfo + nodesByType map[machine.Type][]cluster.NodeInfo +} + +func (cl *clusterNodes) InitNodeInfos() error { + var initNodes []string + + if cl.InitNode != "" { + initNodes = []string{cl.InitNode} + } + + initNodeInfos, err := cluster.IPsToNodeInfos(initNodes) + if err != nil { + return err + } + + controlPlaneNodeInfos, err := cluster.IPsToNodeInfos(cl.ControlPlaneNodes) + if err != nil { + return err + } + + workerNodeInfos, err := cluster.IPsToNodeInfos(cl.WorkerNodes) + if err != nil { + return err + } + + nodesByType := make(map[machine.Type][]cluster.NodeInfo) + nodesByType[machine.TypeInit] = initNodeInfos + nodesByType[machine.TypeControlPlane] = controlPlaneNodeInfos + nodesByType[machine.TypeWorker] = workerNodeInfos + cl.nodesByType = nodesByType + + nodes := make([]cluster.NodeInfo, 0, len(initNodeInfos)+len(controlPlaneNodeInfos)+len(workerNodeInfos)) + nodes = append(nodes, initNodeInfos...) + nodes = append(nodes, controlPlaneNodeInfos...) + nodes = append(nodes, workerNodeInfos...) + cl.nodes = nodes + + return nil +} + +func (cl *clusterNodes) Nodes() []cluster.NodeInfo { + return cl.nodes +} + +func (cl *clusterNodes) NodesByType(t machine.Type) []cluster.NodeInfo { + return cl.nodesByType[t] +} + +var healthCmdFlags struct { + clusterState clusterNodes + clusterWaitTimeout time.Duration + forceEndpoint string + runOnServer bool + runE2E bool + configFiles []string +} + +// healthCmd represents the health command. +var healthCmd = &cobra.Command{ + Use: "health", + Short: "Check cluster health", + Long: ``, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + err := healthCmdFlags.clusterState.InitNodeInfos() + if err != nil { + return err + } + + if err := runHealth(); err != nil { + return err + } + + if healthCmdFlags.runE2E { + return runE2E() + } + + return nil + }, +} + +func runHealth() error { + if healthCmdFlags.runOnServer { + return WithClient(healthOnServer) + } + + return WithClientNoNodes(healthOnClient) +} + +func healthOnClient(ctx context.Context, c *client.Client) error { + clientProvider := &cluster.ConfigClientProvider{ + DefaultClient: c, + } + defer clientProvider.Close() //nolint:errcheck + + clusterInfo, err := buildClusterInfo(healthCmdFlags.clusterState) + if err != nil { + return err + } + + state := struct { + cluster.ClientProvider + cluster.K8sProvider + cluster.Info + }{ + ClientProvider: clientProvider, + K8sProvider: &cluster.KubernetesClient{ + ClientProvider: clientProvider, + ForceEndpoint: healthCmdFlags.forceEndpoint, + }, + Info: clusterInfo, + } + + // Run cluster readiness checks + checkCtx, checkCtxCancel := context.WithTimeout(ctx, healthCmdFlags.clusterWaitTimeout) + defer checkCtxCancel() + + return check.Wait(checkCtx, &state, append(check.DefaultClusterChecks(), check.ExtraClusterChecks()...), check.StderrReporter()) +} + +func healthOnServer(ctx context.Context, c *client.Client) error { + if err := helpers.FailIfMultiNodes(ctx, "health"); err != nil { + return err + } + + controlPlaneNodes := healthCmdFlags.clusterState.ControlPlaneNodes + if healthCmdFlags.clusterState.InitNode != "" { + controlPlaneNodes = append(controlPlaneNodes, healthCmdFlags.clusterState.InitNode) + } + + healthCheckClient, err := c.ClusterHealthCheck(ctx, healthCmdFlags.clusterWaitTimeout, &clusterapi.ClusterInfo{ + ControlPlaneNodes: controlPlaneNodes, + WorkerNodes: healthCmdFlags.clusterState.WorkerNodes, + ForceEndpoint: healthCmdFlags.forceEndpoint, + }) + if err != nil { + return err + } + + if err := healthCheckClient.CloseSend(); err != nil { + return err + } + + for { + msg, err := healthCheckClient.Recv() + if err != nil { + if err == io.EOF || client.StatusCode(err) == codes.Canceled { + return nil + } + + return err + } + + if msg.GetMetadata().GetError() != "" { + return fmt.Errorf("healthcheck error: %s", msg.GetMetadata().GetError()) + } + + fmt.Fprintln(os.Stderr, msg.GetMessage()) + } +} + +func runE2E() error { + return WithClient(func(ctx context.Context, c *client.Client) error { + clientProvider := &cluster.ConfigClientProvider{ + DefaultClient: c, + } + defer clientProvider.Close() //nolint:errcheck + + state := &cluster.KubernetesClient{ + ClientProvider: clientProvider, + ForceEndpoint: healthCmdFlags.forceEndpoint, + } + + // Run cluster readiness checks + checkCtx, checkCtxCancel := context.WithTimeout(ctx, healthCmdFlags.clusterWaitTimeout) + defer checkCtxCancel() + + options := sonobuoy.DefaultOptions() + options.UseSpinner = true + + return sonobuoy.Run(checkCtx, state, options) + }) +} + +func init() { + healthCmd.Flags().StringSliceVarP(&healthCmdFlags.configFiles, + "file", "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + healthCmd.PreRunE = func(cmd *cobra.Command, args []string) error { + nodesFromArgs := + len(GlobalArgs. + Nodes) > 0 + endpointsFromArgs := len(GlobalArgs. + Endpoints) > 0 + for _, configFile := range healthCmdFlags. + configFiles { + if err := processModelineAndUpdateGlobals(configFile, nodesFromArgs, endpointsFromArgs, + false); err != nil { + return err + } + } + return nil + } + + addCommand(healthCmd) + healthCmd.Flags().StringVar(&healthCmdFlags.clusterState.InitNode, "init-node", "", "specify IPs of init node") + healthCmd.Flags().StringSliceVar(&healthCmdFlags.clusterState.ControlPlaneNodes, "control-plane-nodes", nil, "specify IPs of control plane nodes") + healthCmd.Flags().StringSliceVar(&healthCmdFlags.clusterState.WorkerNodes, "worker-nodes", nil, "specify IPs of worker nodes") + healthCmd.Flags().DurationVar(&healthCmdFlags.clusterWaitTimeout, "wait-timeout", 20*time.Minute, "timeout to wait for the cluster to be ready") + healthCmd.Flags().StringVar(&healthCmdFlags.forceEndpoint, "k8s-endpoint", "", "use endpoint instead of kubeconfig default") + healthCmd.Flags().BoolVar(&healthCmdFlags.runOnServer, "server", true, "run server-side check") + healthCmd.Flags().BoolVar(&healthCmdFlags.runE2E, "run-e2e", false, "run Kubernetes e2e test") +} + +func buildClusterInfo(clusterState clusterNodes) (cluster.Info, error) { + // if nodes are set explicitly via command line args, use them + if len(clusterState.ControlPlaneNodes) > 0 || len(clusterState.WorkerNodes) > 0 { + return &clusterState, nil + } + + // read members from the Talos API + + var members []*clusterres.Member + + err := WithClientNoNodes(func(ctx context.Context, c *client.Client) error { + items, err := safe.StateListAll[*clusterres.Member](ctx, c.COSI) + if err != nil { + return err + } + + items.ForEach(func(item *clusterres.Member) { members = append(members, item) }) + + return nil + }) + if err != nil { + return nil, err + } + + return check.NewDiscoveredClusterInfo(members) +} diff --git a/pkg/commands/imported_image.go b/pkg/commands/imported_image.go new file mode 100644 index 0000000..a768b7f --- /dev/null +++ b/pkg/commands/imported_image.go @@ -0,0 +1,199 @@ +// Code generated by go run tools/import_commands.go --talos-version v1.7.1 image +// DO NOT EDIT. + +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package commands + +import ( + "context" + "fmt" + "os" + "text/tabwriter" + "time" + + "github.com/dustin/go-humanize" + "github.com/spf13/cobra" + + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" + "github.com/siderolabs/talos/pkg/images" + "github.com/siderolabs/talos/pkg/machinery/api/common" + "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/client" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" +) + +type imageCmdFlagsType struct { + namespace string + configFiles []string +} + +var imageCmdFlags imageCmdFlagsType + +func (flags imageCmdFlagsType) apiNamespace() (common.ContainerdNamespace, error) { + switch flags.namespace { + case "cri": + return common.ContainerdNamespace_NS_CRI, nil + case "system": + return common.ContainerdNamespace_NS_SYSTEM, nil + default: + return 0, fmt.Errorf("unsupported namespace %q", flags.namespace) + } +} + +// imagesCmd represents the image command. +var imageCmd = &cobra.Command{ + Use: "image", + Aliases: []string{"images"}, + Short: "Manage CRI containter images", + Long: ``, + Args: cobra.NoArgs, +} + +// imageListCmd represents the image list command. +var imageListCmd = &cobra.Command{ + Use: "list", + Aliases: []string{"l", "ls"}, + Short: "List CRI images", + Long: ``, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return WithClient(func(ctx context.Context, c *client.Client) error { + ns, err := imageCmdFlags.apiNamespace() + if err != nil { + return err + } + + rcv, err := c.ImageList(ctx, ns) + if err != nil { + return fmt.Errorf("error listing images: %w", err) + } + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + fmt.Fprintln(w, "NODE\tIMAGE\tDIGEST\tSIZE\tCREATED") + + if err = helpers.ReadGRPCStream(rcv, func(msg *machine.ImageListResponse, node string, multipleNodes bool) error { + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", + node, + msg.Name, + msg.Digest, + humanize.Bytes(uint64(msg.Size)), + msg.CreatedAt.AsTime().Format(time.RFC3339), + ) + + return nil + }); err != nil { + return err + } + + return w.Flush() + }) + }, +} + +// imagePullCmd represents the image pull command. +var imagePullCmd = &cobra.Command{ + Use: "pull", + Aliases: []string{"p"}, + Short: "Pull an image into CRI", + Long: ``, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return WithClient(func(ctx context.Context, c *client.Client) error { + ns, err := imageCmdFlags.apiNamespace() + if err != nil { + return err + } + + err = c.ImagePull(ctx, ns, args[0]) + if err != nil { + return fmt.Errorf("error pulling image: %w", err) + } + + return nil + }) + }, +} + +// imageDefaultCmd represents the image default command. +var imageDefaultCmd = &cobra.Command{ + Use: "default", + Short: "List the default images used by Talos", + Long: ``, + RunE: func(cmd *cobra.Command, args []string) error { + images := images.List(container.NewV1Alpha1(&v1alpha1.Config{ + MachineConfig: &v1alpha1.MachineConfig{ + MachineKubelet: &v1alpha1.KubeletConfig{}, + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + EtcdConfig: &v1alpha1.EtcdConfig{}, + APIServerConfig: &v1alpha1.APIServerConfig{}, + ControllerManagerConfig: &v1alpha1.ControllerManagerConfig{}, + SchedulerConfig: &v1alpha1.SchedulerConfig{}, + CoreDNSConfig: &v1alpha1.CoreDNS{}, + ProxyConfig: &v1alpha1.ProxyConfig{}, + }, + })) + + fmt.Printf("%s\n", images.Flannel) + fmt.Printf("%s\n", images.FlannelCNI) + fmt.Printf("%s\n", images.CoreDNS) + fmt.Printf("%s\n", images.Etcd) + fmt.Printf("%s\n", images.KubeAPIServer) + fmt.Printf("%s\n", images.KubeControllerManager) + fmt.Printf("%s\n", images.KubeScheduler) + fmt.Printf("%s\n", images.KubeProxy) + fmt.Printf("%s\n", images.Kubelet) + fmt.Printf("%s\n", images.Installer) + fmt.Printf("%s\n", images.Pause) + + return nil + }, +} + +func init() { + imageCmd.Flags().StringSliceVarP(&imageCmdFlags.configFiles, "file", + "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + imageCmd.PreRunE = func(cmd *cobra.Command, args []string) error { + nodesFromArgs := len(GlobalArgs.Nodes) > + 0 + endpointsFromArgs := len(GlobalArgs. + Endpoints) > 0 + for _, configFile := range imageCmdFlags.configFiles { + if err := processModelineAndUpdateGlobals(configFile, nodesFromArgs, + endpointsFromArgs, false); err != nil { + return err + } + } + return nil + } + imageDefaultCmd. + Flags().StringSliceVarP(&etcdCmdFlags.configFiles, + "file", "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + imageDefaultCmd.PreRunE = etcdCmd.PreRunE + + imageListCmd.Flags().StringSliceVarP(&etcdCmdFlags. + configFiles, "file", "f", nil, + "specify config files or patches in a YAML file (can specify multiple)", + ) + imageListCmd. + PreRunE = etcdCmd.PreRunE + imagePullCmd. + Flags().StringSliceVarP(&etcdCmdFlags.configFiles, + "file", + "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + imagePullCmd.PreRunE = etcdCmd.PreRunE + + imageCmd.PersistentFlags().StringVar(&imageCmdFlags.namespace, "namespace", "cri", "namespace to use: `system` (etcd and kubelet images) or `cri` for all Kubernetes workloads") + addCommand(imageCmd) + + imageCmd.AddCommand(imageDefaultCmd) + imageCmd.AddCommand(imageListCmd) + imageCmd.AddCommand(imagePullCmd) +} diff --git a/pkg/commands/imported_kubeconfig.go b/pkg/commands/imported_kubeconfig.go new file mode 100644 index 0000000..6082ce7 --- /dev/null +++ b/pkg/commands/imported_kubeconfig.go @@ -0,0 +1,203 @@ +// Code generated by go run tools/import_commands.go --talos-version v1.7.1 kubeconfig +// DO NOT EDIT. + +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package commands + +import ( + "bufio" + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/mattn/go-isatty" + "github.com/siderolabs/go-kubeconfig" + "github.com/spf13/cobra" + "k8s.io/client-go/tools/clientcmd" + + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" + "github.com/siderolabs/talos/pkg/machinery/client" +) + +var kubeconfigFlags struct { + force bool + forceContextName string + merge bool +} + +// kubeconfigCmd represents the kubeconfig command. +var kubeconfigCmd = &cobra.Command{ + Use: "kubeconfig [local-path]", + Short: "Download the admin kubeconfig from the node", + Long: `Download the admin kubeconfig from the node. +If merge flag is defined, config will be merged with ~/.kube/config or [local-path] if specified. +Otherwise kubeconfig will be written to PWD or [local-path] if specified.`, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return WithClient(func(ctx context.Context, c *client.Client) error { + if err := helpers.FailIfMultiNodes(ctx, "kubeconfig"); err != nil { + return err + } + + var localPath string + + if len(args) == 0 { + // no path given, use defaults + var err error + + if kubeconfigFlags.merge { + localPath, err = kubeconfig.SinglePath() + if err != nil { + return err + } + } else { + localPath, err = os.Getwd() + if err != nil { + return fmt.Errorf("error getting current working directory: %s", err) + } + } + } else { + localPath = args[0] + } + + localPath = filepath.Clean(localPath) + + st, err := os.Stat(localPath) + if err != nil { + if !os.IsNotExist(err) { + return fmt.Errorf("error checking path %q: %w", localPath, err) + } + + err = os.MkdirAll(filepath.Dir(localPath), 0o755) + if err != nil { + return err + } + } else if st.IsDir() { + // only dir name was given, append `kubeconfig` by default + localPath = filepath.Join(localPath, "kubeconfig") + } + + _, err = os.Stat(localPath) + if err == nil && !(kubeconfigFlags.force || kubeconfigFlags.merge) { + return fmt.Errorf("kubeconfig file already exists, use --force to overwrite: %q", localPath) + } else if err != nil { + if os.IsNotExist(err) { + // merge doesn't make sense if target path doesn't exist + kubeconfigFlags.merge = false + } else { + return fmt.Errorf("error checking path %q: %w", localPath, err) + } + } + + r, err := c.KubeconfigRaw(ctx) + if err != nil { + return fmt.Errorf("error copying: %w", err) + } + + defer r.Close() //nolint:errcheck + + data, err := helpers.ExtractFileFromTarGz("kubeconfig", r) + if err != nil { + return err + } + + if kubeconfigFlags.merge { + return extractAndMerge(data, localPath) + } + + return os.WriteFile(localPath, data, 0o600) + }) + }, +} + +func extractAndMerge(data []byte, localPath string) error { + config, err := clientcmd.Load(data) + if err != nil { + return err + } + + merger, err := kubeconfig.Load(localPath) + if err != nil { + return err + } + + interactive := isatty.IsTerminal(os.Stdout.Fd()) + + err = merger.Merge(config, kubeconfig.MergeOptions{ + ActivateContext: true, + ForceContextName: kubeconfigFlags.forceContextName, + OutputWriter: os.Stdout, + ConflictHandler: func(component kubeconfig.ConfigComponent, name string) (kubeconfig.ConflictDecision, error) { + if kubeconfigFlags.force { + return kubeconfig.OverwriteDecision, nil + } + + if !interactive { + return kubeconfig.RenameDecision, nil + } + + return askOverwriteOrRename(fmt.Sprintf("%s %q already exists", component, name)) + }, + }) + if err != nil { + return err + } + + return merger.Write(localPath) +} + +func askOverwriteOrRename(prompt string) (kubeconfig.ConflictDecision, error) { + reader := bufio.NewReader(os.Stdin) + + for { + fmt.Printf("%s [(r)ename/(o)verwrite]: ", prompt) + + response, err := reader.ReadString('\n') + if err != nil { + return "", err + } + + switch strings.ToLower(strings.TrimSpace(response)) { + case "overwrite", "o": + return kubeconfig.OverwriteDecision, nil + case "rename", "r": + return kubeconfig.RenameDecision, nil + } + } +} + +func init() { + kubeconfigCmd.Flags().StringSliceVarP(&kubeconfigCmdFlags.configFiles, + "file", "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + kubeconfigCmd.PreRunE = func(cmd *cobra.Command, args []string) error { + nodesFromArgs := len(GlobalArgs. + Nodes) > 0 + + endpointsFromArgs := len(GlobalArgs. + Endpoints) > 0 + for _, configFile := range kubeconfigCmdFlags. + configFiles { + if err := processModelineAndUpdateGlobals(configFile, nodesFromArgs, endpointsFromArgs, + false); err != nil { + return err + } + + } + return nil + } + + kubeconfigCmd.Flags().BoolVarP(&kubeconfigFlags.force, "force", "F", false, "Force overwrite of kubeconfig if already present, force overwrite on kubeconfig merge") + kubeconfigCmd.Flags().StringVar(&kubeconfigFlags.forceContextName, "force-context-name", "", "Force context name for kubeconfig merge") + kubeconfigCmd.Flags().BoolVarP(&kubeconfigFlags.merge, "merge", "m", true, "Merge with existing kubeconfig") + addCommand(kubeconfigCmd) +} + +var kubeconfigCmdFlags struct { + configFiles []string +} diff --git a/pkg/commands/imported_list.go b/pkg/commands/imported_list.go new file mode 100644 index 0000000..8842173 --- /dev/null +++ b/pkg/commands/imported_list.go @@ -0,0 +1,210 @@ +// Code generated by go run tools/import_commands.go --talos-version v1.7.1 ls +// DO NOT EDIT. + +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package commands + +import ( + "context" + "errors" + "fmt" + "os" + "strconv" + "strings" + "text/tabwriter" + "time" + + humanize "github.com/dustin/go-humanize" + "github.com/spf13/cobra" + + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" + machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/client" +) + +const sixMonths = 6 * time.Hour * 24 * 30 + +var ( + long bool + recurse bool + recursionDepth int32 + humanizeFlag bool + types []string +) + +// lsCmd represents the ls command. +var lsCmd = &cobra.Command{ + Use: "list [path]", + Aliases: []string{"ls"}, + Short: "Retrieve a directory listing", + Long: ``, + Args: cobra.MaximumNArgs(1), + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + if len(args) != 0 { + return nil, cobra.ShellCompDirectiveError | cobra.ShellCompDirectiveNoFileComp + } + + return completePathFromNode(toComplete), cobra.ShellCompDirectiveNoFileComp + }, + RunE: func(cmd *cobra.Command, args []string) error { + if recurse && recursionDepth != 1 { + return errors.New("only one of flags --recurse and --depth can be specified at the same time") + } + + return WithClient(func(ctx context.Context, c *client.Client) error { + rootDir := "/" + + if len(args) > 0 { + rootDir = args[0] + } + + // handle all variants: --type=f,l; -tfl; etc + var reqTypes []machineapi.ListRequest_Type + for _, typ := range types { + for _, t := range typ { + // handle both `find -type X` and os.FileMode.String() designations + switch t { + case 'f': + reqTypes = append(reqTypes, machineapi.ListRequest_REGULAR) + case 'd': + reqTypes = append(reqTypes, machineapi.ListRequest_DIRECTORY) + case 'l', 'L': + reqTypes = append(reqTypes, machineapi.ListRequest_SYMLINK) + default: + return fmt.Errorf("invalid file type: %s", string(t)) + } + } + } + + if recurse { + recursionDepth = -1 + } + + stream, err := c.LS(ctx, &machineapi.ListRequest{ + Root: rootDir, + Recurse: recursionDepth > 1 || recurse, + RecursionDepth: recursionDepth, + Types: reqTypes, + }) + if err != nil { + return fmt.Errorf("error fetching logs: %s", err) + } + + if !long { + w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + fmt.Fprintln(w, "NODE\tNAME") + + defer w.Flush() //nolint:errcheck + + return helpers.ReadGRPCStream(stream, func(info *machineapi.FileInfo, node string, multipleNodes bool) error { + if info.Error != "" { + return helpers.NonFatalError(fmt.Errorf("%s: error reading file %s: %s", node, info.Name, info.Error)) + } + + if !multipleNodes { + fmt.Println(info.RelativeName) + } else { + fmt.Fprintf(w, "%s\t%s\n", + node, + info.RelativeName, + ) + } + + return nil + }) + } + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + defer w.Flush() //nolint:errcheck + + fmt.Fprintln(w, "NODE\tMODE\tUID\tGID\tSIZE(B)\tLASTMOD\tNAME") + + return helpers.ReadGRPCStream(stream, func(info *machineapi.FileInfo, node string, multipleNodes bool) error { + if info.Error != "" { + return helpers.NonFatalError(fmt.Errorf("%s: error reading file %s: %s", node, info.Name, info.Error)) + } + + display := info.RelativeName + if info.Link != "" { + display += " -> " + info.Link + } + + size := strconv.FormatInt(info.Size, 10) + + if humanizeFlag { + size = humanize.Bytes(uint64(info.Size)) + } + + timestamp := time.Unix(info.Modified, 0) + timestampFormatted := "" + + if humanizeFlag { + timestampFormatted = humanize.Time(timestamp) + } else { + if time.Since(timestamp) < sixMonths { + timestampFormatted = timestamp.Format("Jan _2 15:04:05") + } else { + timestampFormatted = timestamp.Format("Jan _2 2006 15:04") + } + } + + fmt.Fprintf(w, "%s\t%s\t%d\t%d\t%s\t%s\t%s\n", + node, + os.FileMode(info.Mode).String(), + info.Uid, + info.Gid, + size, + timestampFormatted, + display, + ) + + return nil + }) + }) + }, +} + +func init() { + lsCmd.Flags().StringSliceVarP(&lsCmdFlags.configFiles, "file", + "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + lsCmd.PreRunE = func(cmd *cobra.Command, args []string) error { + nodesFromArgs := + len(GlobalArgs. + Nodes, + ) > + 0 + endpointsFromArgs := len(GlobalArgs. + Endpoints, + ) > 0 + for _, configFile := range lsCmdFlags. + configFiles { + if err := processModelineAndUpdateGlobals(configFile, nodesFromArgs, + endpointsFromArgs, false); err != nil { + return err + } + } + return nil + } + + typesHelp := strings.Join([]string{ + "filter by specified types:", + "F" + "\t" + "regular file", + "d" + "\t" + "directory", + "l, L" + "\t" + "symbolic link", + }, "\n") + + lsCmd.Flags().BoolVarP(&long, "long", "l", false, "display additional file details") + lsCmd.Flags().BoolVarP(&recurse, "recurse", "r", false, "recurse into subdirectories") + lsCmd.Flags().BoolVarP(&humanizeFlag, "humanize", "H", false, "humanize size and time in the output") + lsCmd.Flags().Int32VarP(&recursionDepth, "depth", "d", 1, "maximum recursion depth") + lsCmd.Flags().StringSliceVarP(&types, "type", "t", nil, typesHelp) + addCommand(lsCmd) +} + +var lsCmdFlags struct { + configFiles []string +} diff --git a/pkg/commands/imported_logs.go b/pkg/commands/imported_logs.go new file mode 100644 index 0000000..035cda4 --- /dev/null +++ b/pkg/commands/imported_logs.go @@ -0,0 +1,269 @@ +// Code generated by go run tools/import_commands.go --talos-version v1.7.1 logs +// DO NOT EDIT. + +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package commands + +import ( + "bufio" + "context" + "fmt" + "io" + "os" + "sync" + + criconstants "github.com/containerd/containerd/pkg/cri/constants" + "github.com/siderolabs/gen/xslices" + "github.com/spf13/cobra" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/peer" + + "github.com/siderolabs/talos/pkg/cli" + "github.com/siderolabs/talos/pkg/machinery/api/common" + "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/client" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +var ( + follow bool + tailLines int32 +) + +var logsCmd = &cobra.Command{ + Use: "logs ", + Short: "Retrieve logs for a service", + Long: ``, + Args: cobra.ExactArgs(1), + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + if len(args) != 0 { + return nil, cobra.ShellCompDirectiveError | cobra.ShellCompDirectiveNoFileComp + } + + if kubernetesFlag { + return getContainersFromNode(kubernetesFlag), cobra.ShellCompDirectiveNoFileComp + } + + return mergeSuggestions(getServiceFromNode(), getContainersFromNode(kubernetesFlag), getLogsContainers()), cobra.ShellCompDirectiveNoFileComp + }, + RunE: func(cmd *cobra.Command, args []string) error { + return WithClient(func(ctx context.Context, c *client.Client) error { + var ( + namespace string + driver common.ContainerDriver + ) + + if kubernetesFlag { + namespace = criconstants.K8sContainerdNamespace + driver = common.ContainerDriver_CRI + } else { + namespace = constants.SystemContainerdNamespace + driver = common.ContainerDriver_CONTAINERD + } + + stream, err := c.Logs(ctx, namespace, driver, args[0], follow, tailLines) + if err != nil { + return fmt.Errorf("error fetching logs: %s", err) + } + + defaultNode := client.RemotePeer(stream.Context()) + + respCh, errCh := newLineSlicer(stream) + + var gotErrors bool + + for data := range respCh { + if data.Metadata != nil && data.Metadata.Error != "" { + _, err = fmt.Fprintf(os.Stderr, "ERROR: %s\n", data.Metadata.Error) + if err != nil { + return err + } + + gotErrors = true + + continue + } + + node := defaultNode + if data.Metadata != nil && data.Metadata.Hostname != "" { + node = data.Metadata.Hostname + } + + _, err = fmt.Printf("%s: %s\n", node, data.Bytes) + if err != nil { + return err + } + } + + if err = <-errCh; err != nil { + return fmt.Errorf("error getting logs: %v", err) + } + + if gotErrors { + os.Exit(1) + } + + return nil + }) + }, +} + +// lineSlicer splits random chunks of bytes coming from nodes into a stream +// of lines aggregated per node. +type lineSlicer struct { + respCh chan *common.Data + errCh chan error + pipes map[string]*io.PipeWriter + wg sync.WaitGroup +} + +func newLineSlicer(stream machine.MachineService_LogsClient) (chan *common.Data, chan error) { + slicer := &lineSlicer{ + respCh: make(chan *common.Data), + errCh: make(chan error, 1), + pipes: map[string]*io.PipeWriter{}, + } + + go slicer.run(stream) + + return slicer.respCh, slicer.errCh +} + +func (slicer *lineSlicer) chopper(r io.Reader, hostname string) { + defer slicer.wg.Done() + + scanner := bufio.NewScanner(r) + + for scanner.Scan() { + line := scanner.Bytes() + line = xslices.CopyN(line, len(line)) + + slicer.respCh <- &common.Data{ + Metadata: &common.Metadata{ + Hostname: hostname, + }, + Bytes: line, + } + } +} + +func (slicer *lineSlicer) getPipe(node string) *io.PipeWriter { + pipe, ok := slicer.pipes[node] + if !ok { + var piper *io.PipeReader + piper, pipe = io.Pipe() + + slicer.wg.Add(1) + + go slicer.chopper(piper, node) + + slicer.pipes[node] = pipe + } + + return pipe +} + +func (slicer *lineSlicer) cleanupChoppers() { + for _, p := range slicer.pipes { + _ = p.Close() //nolint:errcheck + } + + slicer.wg.Wait() +} + +func (slicer *lineSlicer) run(stream machine.MachineService_LogsClient) { + defer close(slicer.errCh) + defer close(slicer.respCh) + + defer slicer.cleanupChoppers() + + for { + data, err := stream.Recv() + if err != nil { + if err == io.EOF || client.StatusCode(err) == codes.Canceled { + return + } + slicer.errCh <- err + + return + } + + if data.Metadata != nil && data.Metadata.Error != "" { + // errors are delivered OOB + slicer.respCh <- data + + continue + } + + node := "" + + if data.Metadata != nil { + node = data.Metadata.Hostname + } + + _, err = slicer.getPipe(node).Write(data.Bytes) + cli.Should(err) + } +} + +func getLogsContainers() []string { + var result []string + + //nolint:errcheck + WithClient( + func(ctx context.Context, c *client.Client) error { + var remotePeer peer.Peer + + resp, err := c.LogsContainers(ctx, grpc.Peer(&remotePeer)) + if err != nil { + return err + } + + result = xslices.FlatMap(resp.Messages, func(lc *machine.LogsContainer) []string { return lc.Ids }) + + return nil + }, + ) + + return result +} + +func init() { + logsCmd.Flags().StringSliceVarP(&logsCmdFlags.configFiles, "file", + "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + logsCmd.PreRunE = func(cmd *cobra.Command, + + args []string) error { + nodesFromArgs := len(GlobalArgs. + Nodes) > + 0 + endpointsFromArgs := len(GlobalArgs.Endpoints) > 0 + for _, configFile := range logsCmdFlags.configFiles { + if err := processModelineAndUpdateGlobals(configFile, + nodesFromArgs, endpointsFromArgs, + false); err != nil { + + return err + } + } + return nil + } + + logsCmd.Flags().BoolVarP(&kubernetesFlag, "kubernetes", "k", false, "use the k8s.io containerd namespace") + logsCmd.Flags().BoolVarP(&follow, "follow", "F", false, "specify if the logs should be streamed") + logsCmd.Flags().Int32VarP(&tailLines, "tail", "", -1, "lines of log file to display (default is to show from the beginning)") + + logsCmd.Flags().BoolP("use-cri", "c", false, "use the CRI driver") + logsCmd.Flags().MarkHidden("use-cri") //nolint:errcheck + + addCommand(logsCmd) +} + +var logsCmdFlags struct { + configFiles []string +} diff --git a/pkg/commands/imported_memory.go b/pkg/commands/imported_memory.go new file mode 100644 index 0000000..76c8201 --- /dev/null +++ b/pkg/commands/imported_memory.go @@ -0,0 +1,177 @@ +// Code generated by go run tools/import_commands.go --talos-version v1.7.1 memory +// DO NOT EDIT. + +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package commands + +import ( + "context" + "fmt" + "os" + "text/tabwriter" + + "github.com/spf13/cobra" + "google.golang.org/grpc" + "google.golang.org/grpc/peer" + + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" + "github.com/siderolabs/talos/pkg/cli" + machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/client" +) + +var verbose bool + +// memoryCmd represents the processes command. +var memoryCmd = &cobra.Command{ + Use: "memory", + Aliases: []string{"m", "free"}, + Short: "Show memory usage", + Long: ``, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return WithClient(func(ctx context.Context, c *client.Client) error { + var remotePeer peer.Peer + + resp, err := c.Memory(ctx, grpc.Peer(&remotePeer)) + if err != nil { + if resp == nil { + return fmt.Errorf("error getting memory stats: %s", err) + } + + cli.Warning("%s", err) + } + + if verbose { + verboseRender(&remotePeer, resp) + } else { + err = briefRender(&remotePeer, resp) + if err != nil { + return err + } + } + + return helpers.CheckErrors(resp.Messages...) + }) + }, +} + +func briefRender(remotePeer *peer.Peer, resp *machineapi.MemoryResponse) error { + w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + fmt.Fprintln(w, "NODE\tTOTAL\tUSED\tFREE\tSHARED\tBUFFERS\tCACHE\tAVAILABLE") + + defaultNode := client.AddrFromPeer(remotePeer) + + for _, msg := range resp.Messages { + node := defaultNode + + if msg.Metadata != nil { + node = msg.Metadata.Hostname + } + + // Default to displaying output as MB + fmt.Fprintf(w, "%s\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n", + node, + msg.Meminfo.Memtotal/1024, + (msg.Meminfo.Memtotal-msg.Meminfo.Memfree-msg.Meminfo.Cached-msg.Meminfo.Buffers)/1024, + msg.Meminfo.Memfree/1024, + msg.Meminfo.Shmem/1024, + msg.Meminfo.Buffers/1024, + msg.Meminfo.Cached/1024, + msg.Meminfo.Memavailable/1024, + ) + } + + return w.Flush() +} + +func verboseRender(remotePeer *peer.Peer, resp *machineapi.MemoryResponse) { + defaultNode := client.AddrFromPeer(remotePeer) + + // Dump as /proc/meminfo + for _, msg := range resp.Messages { + node := defaultNode + + if msg.Metadata != nil { + node = msg.Metadata.Hostname + } + + fmt.Printf("%s: %s\n", "NODE", node) + fmt.Printf("%s: %d %s\n", "MemTotal", msg.Meminfo.Memtotal, "kB") + fmt.Printf("%s: %d %s\n", "MemFree", msg.Meminfo.Memfree, "kB") + fmt.Printf("%s: %d %s\n", "MemAvailable", msg.Meminfo.Memavailable, "kB") + fmt.Printf("%s: %d %s\n", "Buffers", msg.Meminfo.Buffers, "kB") + fmt.Printf("%s: %d %s\n", "Cached", msg.Meminfo.Cached, "kB") + fmt.Printf("%s: %d %s\n", "SwapCached", msg.Meminfo.Swapcached, "kB") + fmt.Printf("%s: %d %s\n", "Active", msg.Meminfo.Active, "kB") + fmt.Printf("%s: %d %s\n", "Inactive", msg.Meminfo.Inactive, "kB") + fmt.Printf("%s: %d %s\n", "ActiveAnon", msg.Meminfo.Activeanon, "kB") + fmt.Printf("%s: %d %s\n", "InactiveAnon", msg.Meminfo.Inactiveanon, "kB") + fmt.Printf("%s: %d %s\n", "ActiveFile", msg.Meminfo.Activefile, "kB") + fmt.Printf("%s: %d %s\n", "InactiveFile", msg.Meminfo.Inactivefile, "kB") + fmt.Printf("%s: %d %s\n", "Unevictable", msg.Meminfo.Unevictable, "kB") + fmt.Printf("%s: %d %s\n", "Mlocked", msg.Meminfo.Mlocked, "kB") + fmt.Printf("%s: %d %s\n", "SwapTotal", msg.Meminfo.Swaptotal, "kB") + fmt.Printf("%s: %d %s\n", "SwapFree", msg.Meminfo.Swapfree, "kB") + fmt.Printf("%s: %d %s\n", "Dirty", msg.Meminfo.Dirty, "kB") + fmt.Printf("%s: %d %s\n", "Writeback", msg.Meminfo.Writeback, "kB") + fmt.Printf("%s: %d %s\n", "AnonPages", msg.Meminfo.Anonpages, "kB") + fmt.Printf("%s: %d %s\n", "Mapped", msg.Meminfo.Mapped, "kB") + fmt.Printf("%s: %d %s\n", "Shmem", msg.Meminfo.Shmem, "kB") + fmt.Printf("%s: %d %s\n", "Slab", msg.Meminfo.Slab, "kB") + fmt.Printf("%s: %d %s\n", "SReclaimable", msg.Meminfo.Sreclaimable, "kB") + fmt.Printf("%s: %d %s\n", "SUnreclaim", msg.Meminfo.Sunreclaim, "kB") + fmt.Printf("%s: %d %s\n", "KernelStack", msg.Meminfo.Kernelstack, "kB") + fmt.Printf("%s: %d %s\n", "PageTables", msg.Meminfo.Pagetables, "kB") + fmt.Printf("%s: %d %s\n", "NFSUnstable", msg.Meminfo.Nfsunstable, "kB") + fmt.Printf("%s: %d %s\n", "Bounce", msg.Meminfo.Bounce, "kB") + fmt.Printf("%s: %d %s\n", "WritebackTmp", msg.Meminfo.Writebacktmp, "kB") + fmt.Printf("%s: %d %s\n", "CommitLimit", msg.Meminfo.Commitlimit, "kB") + fmt.Printf("%s: %d %s\n", "CommittedAS", msg.Meminfo.Committedas, "kB") + fmt.Printf("%s: %d %s\n", "VmallocTotal", msg.Meminfo.Vmalloctotal, "kB") + fmt.Printf("%s: %d %s\n", "VmallocUsed", msg.Meminfo.Vmallocused, "kB") + fmt.Printf("%s: %d %s\n", "VmallocChunk", msg.Meminfo.Vmallocchunk, "kB") + fmt.Printf("%s: %d %s\n", "HardwareCorrupted", msg.Meminfo.Hardwarecorrupted, "kB") + fmt.Printf("%s: %d %s\n", "AnonHugePages", msg.Meminfo.Anonhugepages, "kB") + fmt.Printf("%s: %d %s\n", "ShmemHugePages", msg.Meminfo.Shmemhugepages, "kB") + fmt.Printf("%s: %d %s\n", "ShmemPmdMapped", msg.Meminfo.Shmempmdmapped, "kB") + fmt.Printf("%s: %d %s\n", "CmaTotal", msg.Meminfo.Cmatotal, "kB") + fmt.Printf("%s: %d %s\n", "CmaFree", msg.Meminfo.Cmafree, "kB") + fmt.Printf("%s: %d\n", "HugePagesTotal", msg.Meminfo.Hugepagestotal) + fmt.Printf("%s: %d\n", "HugePagesFree", msg.Meminfo.Hugepagesfree) + fmt.Printf("%s: %d\n", "HugePagesRsvd", msg.Meminfo.Hugepagesrsvd) + fmt.Printf("%s: %d\n", "HugePagesSurp", msg.Meminfo.Hugepagessurp) + fmt.Printf("%s: %d %s\n", "Hugepagesize", msg.Meminfo.Hugepagesize, "kB") + fmt.Printf("%s: %d %s\n", "DirectMap4k", msg.Meminfo.Directmap4K, "kB") + fmt.Printf("%s: %d %s\n", "DirectMap2M", msg.Meminfo.Directmap2M, "kB") + fmt.Printf("%s: %d %s\n", "DirectMap1G", msg.Meminfo.Directmap1G, "kB") + } +} + +func init() { + memoryCmd.Flags().StringSliceVarP(&memoryCmdFlags.configFiles, + "file", "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + memoryCmd.PreRunE = func(cmd *cobra.Command, args []string) error { + nodesFromArgs := + len(GlobalArgs.Nodes) > 0 + endpointsFromArgs := len(GlobalArgs.Endpoints) > 0 + for _, configFile := range memoryCmdFlags.configFiles { + if err := processModelineAndUpdateGlobals(configFile, nodesFromArgs, + endpointsFromArgs, false); err != nil { + return err + } + } + return nil + } + + memoryCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "display extended memory statistics") + addCommand(memoryCmd) +} + +var memoryCmdFlags struct { + configFiles []string +} diff --git a/pkg/commands/imported_mounts.go b/pkg/commands/imported_mounts.go new file mode 100644 index 0000000..167f56e --- /dev/null +++ b/pkg/commands/imported_mounts.go @@ -0,0 +1,71 @@ +// Code generated by go run tools/import_commands.go --talos-version v1.7.1 mounts +// DO NOT EDIT. + +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package commands + +import ( + "context" + "fmt" + "os" + + "github.com/spf13/cobra" + "google.golang.org/grpc" + "google.golang.org/grpc/peer" + + "github.com/siderolabs/talos/pkg/cli" + "github.com/siderolabs/talos/pkg/machinery/client" + "github.com/siderolabs/talos/pkg/machinery/formatters" +) + +// mountsCmd represents the mounts command. +var mountsCmd = &cobra.Command{ + Use: "mounts", + Aliases: []string{"mount"}, + Short: "List mounts", + Long: ``, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return WithClient(func(ctx context.Context, c *client.Client) error { + var remotePeer peer.Peer + + resp, err := c.Mounts(ctx, grpc.Peer(&remotePeer)) + if err != nil { + if resp == nil { + return fmt.Errorf("error getting mount information: %s", err) + } + + cli.Warning("%s", err) + } + + return formatters.RenderMounts(resp, os.Stdout, &remotePeer) + }) + }, +} + +func init() { + mountsCmd.Flags().StringSliceVarP(&mountsCmdFlags.configFiles, + "file", "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + mountsCmd.PreRunE = func(cmd *cobra.Command, args []string) error { + nodesFromArgs := + len(GlobalArgs.Nodes) > 0 + endpointsFromArgs := len( + GlobalArgs.Endpoints) > 0 + for _, configFile := range mountsCmdFlags.configFiles { + if err := processModelineAndUpdateGlobals(configFile, nodesFromArgs, endpointsFromArgs, false); err != nil { + return err + } + } + return nil + } + + addCommand(mountsCmd) +} + +var mountsCmdFlags struct { + configFiles []string +} diff --git a/pkg/commands/imported_netstat.go b/pkg/commands/imported_netstat.go new file mode 100644 index 0000000..7d09e40 --- /dev/null +++ b/pkg/commands/imported_netstat.go @@ -0,0 +1,436 @@ +// Code generated by go run tools/import_commands.go --talos-version v1.7.1 netstat +// DO NOT EDIT. + +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package commands + +import ( + "context" + "errors" + "fmt" + "os" + "strconv" + "strings" + "text/tabwriter" + + criconstants "github.com/containerd/containerd/pkg/cri/constants" + "github.com/spf13/cobra" + + "github.com/siderolabs/talos/pkg/cli" + "github.com/siderolabs/talos/pkg/machinery/api/common" + "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/client" +) + +var netstatCmdFlags struct { + verbose bool + extend bool + pid bool + timers bool + listening bool + all bool + pods bool + tcp bool + udp bool + udplite bool + raw bool + ipv4 bool + ipv6 bool + configFiles []string +} + +type netstat struct { + client *client.Client + NodeNetNSPods map[string]map[string]string +} + +// netstatCmd represents the netstat command. +var netstatCmd = &cobra.Command{ + Use: "netstat", + Aliases: []string{"ss"}, + Short: "Show network connections and sockets", + Long: `Show network connections and sockets. + +You can pass an optional argument to view a specific pod's connections. +To do this, format the argument as "namespace/pod". +Note that only pods with a pod network namespace are allowed. +If you don't pass an argument, the command will show host connections.`, + Args: cobra.MaximumNArgs(1), + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + if len(args) > 0 { + return nil, cobra.ShellCompDirectiveError | cobra.ShellCompDirectiveNoFileComp + } + + var podList []string + + if WithClient(func(ctx context.Context, c *client.Client) error { + n := netstat{ + NodeNetNSPods: make(map[string]map[string]string), + client: c, + } + + err := n.getPodNetNsFromNode(ctx) + if err != nil { + return err + } + + for _, netNsPods := range n.NodeNetNSPods { + for _, podName := range netNsPods { + podList = append(podList, podName) + } + } + + return nil + }) != nil { + return nil, cobra.ShellCompDirectiveError | cobra.ShellCompDirectiveNoFileComp + } + + return podList, cobra.ShellCompDirectiveNoFileComp + }, + RunE: func(cmd *cobra.Command, args []string) error { + req := netstatFlagsToRequest() + + return WithClient(func(ctx context.Context, c *client.Client) (err error) { + if netstatCmdFlags.pods && len(args) > 0 { + return errors.New("cannot use --pods and specify a pod") + } + + findThePod := len(args) > 0 + + n := netstat{ + client: c, + } + + n.NodeNetNSPods = make(map[string]map[string]string) + + if findThePod || netstatCmdFlags.pods { + err = n.getPodNetNsFromNode(ctx) + if err != nil { + return err + } + } + + if findThePod { + var foundNode, foundNetNs string + + foundNode, foundNetNs = n.findPodNetNs(args[0]) + + if foundNetNs == "" { + cli.Fatalf("pod %s not found", args[0]) + } + + ctx = client.WithNode(ctx, foundNode) + + req.Netns.Netns = []string{foundNetNs} + req.Netns.Hostnetwork = false + } + + response, err := c.Netstat(ctx, req) + if err != nil { + if response == nil { + return err + } + + cli.Warning("%s", err) + } + + err = n.printNetstat(response) + + return err + }) + }, +} + +//nolint:gocyclo +func netstatFlagsToRequest() *machine.NetstatRequest { + req := machine.NetstatRequest{ + Feature: &machine.NetstatRequest_Feature{ + Pid: netstatCmdFlags.pid, + }, + L4Proto: &machine.NetstatRequest_L4Proto{ + Tcp: netstatCmdFlags.tcp, + Tcp6: netstatCmdFlags.tcp, + Udp: netstatCmdFlags.udp, + Udp6: netstatCmdFlags.udp, + Udplite: netstatCmdFlags.udplite, + Udplite6: netstatCmdFlags.udplite, + Raw: netstatCmdFlags.raw, + Raw6: netstatCmdFlags.raw, + }, + Netns: &machine.NetstatRequest_NetNS{ + Allnetns: netstatCmdFlags.pods, + Hostnetwork: true, + }, + } + + switch { + case netstatCmdFlags.all: + req.Filter = machine.NetstatRequest_ALL + case netstatCmdFlags.listening: + req.Filter = machine.NetstatRequest_LISTENING + default: + req.Filter = machine.NetstatRequest_CONNECTED + } + + if netstatCmdFlags.verbose { + req.L4Proto.Tcp = true + req.L4Proto.Tcp6 = true + req.L4Proto.Udp = true + req.L4Proto.Udp6 = true + req.L4Proto.Udplite = true + req.L4Proto.Udplite6 = true + req.L4Proto.Raw = true + req.L4Proto.Raw6 = true + } + + if !req.L4Proto.Tcp && !req.L4Proto.Tcp6 && !req.L4Proto.Udp && !req.L4Proto.Udp6 && !req.L4Proto.Udplite && !req.L4Proto.Udplite6 && !req.L4Proto.Raw && !req.L4Proto.Raw6 { + req.L4Proto.Tcp = true + req.L4Proto.Tcp6 = true + req.L4Proto.Udp = true + req.L4Proto.Udp6 = true + } + + if netstatCmdFlags.ipv4 && !netstatCmdFlags.ipv6 { + req.L4Proto.Tcp6 = false + req.L4Proto.Udp6 = false + req.L4Proto.Udplite6 = false + req.L4Proto.Raw6 = false + } + + if netstatCmdFlags.ipv6 && !netstatCmdFlags.ipv4 { + req.L4Proto.Tcp = false + req.L4Proto.Udp = false + req.L4Proto.Udplite = false + req.L4Proto.Raw = false + } + + return &req +} + +func (n *netstat) getPodNetNsFromNode(ctx context.Context) (err error) { + resp, err := n.client.Containers(ctx, criconstants.K8sContainerdNamespace, common.ContainerDriver_CRI) + if err != nil { + cli.Warning("error getting containers: %v", err) + + return err + } + + for _, msg := range resp.Messages { + for _, p := range msg.Containers { + if p.NetworkNamespace == "" { + continue + } + + if p.Pid == 0 { + continue + } + + if p.Id != p.PodId { + continue + } + + if n.NodeNetNSPods[msg.Metadata.Hostname] == nil { + n.NodeNetNSPods[msg.Metadata.Hostname] = make(map[string]string) + } + + n.NodeNetNSPods[msg.Metadata.Hostname][p.NetworkNamespace] = p.Id + } + } + + return nil +} + +func (n *netstat) findPodNetNs(findNamespaceAndPod string) (string, string) { + var foundNetNs, foundNode string + + for node, netNSPods := range n.NodeNetNSPods { + for NetNs, podName := range netNSPods { + if podName == strings.ToLower(findNamespaceAndPod) { + foundNetNs = NetNs + foundNode = node + + break + } + } + } + + return foundNode, foundNetNs +} + +//nolint:gocyclo +func (n *netstat) printNetstat(response *machine.NetstatResponse) error { + w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + node := "" + + for i, message := range response.Messages { + if message.Metadata != nil && message.Metadata.Hostname != "" { + node = message.Metadata.Hostname + } + + if len(message.Connectrecord) == 0 { + continue + } + + for j, record := range message.Connectrecord { + if i == 0 && j == 0 { + labels := netstatSummaryLabels() + + if node != "" { + fmt.Fprintln(w, "NODE\t"+labels) + } else { + fmt.Fprintln(w, labels) + } + } + + args := []interface{}{} + + if node != "" { + args = append(args, node) + } + + state := "" + if record.State != 7 { + state = record.State.String() + } + + args = append(args, []interface{}{ + record.L4Proto, + strconv.FormatUint(record.Rxqueue, 10), + strconv.FormatUint(record.Txqueue, 10), + fmt.Sprintf("%s:%d", record.Localip, record.Localport), + fmt.Sprintf("%s:%s", record.Remoteip, wildcardIfZero(record.Remoteport)), + state, + }...) + + if netstatCmdFlags.extend { + args = append(args, []interface{}{ + strconv.FormatUint(uint64(record.Uid), 10), + strconv.FormatUint(record.Inode, 10), + }...) + } + + if netstatCmdFlags.pid { + if record.Process.Pid != 0 { + args = append(args, []interface{}{ + fmt.Sprintf("%d/%s", record.Process.Pid, record.Process.Name), + }...) + } else { + args = append(args, []interface{}{ + "-", + }...) + } + } + + if netstatCmdFlags.pods { + if record.Netns == "" || node == "" || n.NodeNetNSPods[node] == nil { + args = append(args, []interface{}{ + "-", + }...) + } else { + args = append(args, []interface{}{ + n.NodeNetNSPods[node][record.Netns], + }...) + } + } + + if netstatCmdFlags.timers { + timerwhen := strconv.FormatFloat(float64(record.Timerwhen)/100, 'f', 2, 64) + + args = append(args, []interface{}{ + fmt.Sprintf("%s (%s/%d/%d)", strings.ToLower(record.Tr.String()), timerwhen, record.Retrnsmt, record.Timeout), + }...) + } + + pattern := strings.Repeat("%s\t", len(args)) + pattern = strings.TrimSpace(pattern) + "\n" + + fmt.Fprintf(w, pattern, args...) + } + } + + return w.Flush() +} + +func netstatSummaryLabels() (labels string) { + labels = strings.Join( + []string{ + "Proto", + "Recv-Q", + "Send-Q", + "Local Address", + "Foreign Address", + "State", + }, "\t") + + if netstatCmdFlags.extend { + labels += "\t" + strings.Join( + []string{ + "Uid", + "Inode", + }, "\t") + } + + if netstatCmdFlags.pid { + labels += "\t" + "PID/Program name" + } + + if netstatCmdFlags.pods { + labels += "\t" + "Pod" + } + + if netstatCmdFlags.timers { + labels += "\t" + "Timer" + } + + return labels +} + +func wildcardIfZero(num uint32) string { + if num == 0 { + return "*" + } + + return strconv.FormatUint(uint64(num), 10) +} + +func init() { + netstatCmd.Flags().StringSliceVarP(&netstatCmdFlags.configFiles, + "file", "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + netstatCmd.PreRunE = func(cmd *cobra. + Command, args []string) error { + nodesFromArgs := len(GlobalArgs. + Nodes, + ) > 0 + endpointsFromArgs := + + len(GlobalArgs.Endpoints) > 0 + for _, configFile := range netstatCmdFlags. + configFiles { + if err := processModelineAndUpdateGlobals(configFile, nodesFromArgs, endpointsFromArgs, false); err != nil { + return err + } + } + return nil + } + + netstatCmd.Flags().BoolVarP(&netstatCmdFlags.verbose, "verbose", "v", false, "display sockets of all supported transport protocols") + // extend is normally -e but cannot be used as this is endpoint in talosctl + netstatCmd.Flags().BoolVarP(&netstatCmdFlags.extend, "extend", "x", false, "show detailed socket information") + netstatCmd.Flags().BoolVarP(&netstatCmdFlags.pid, "programs", "p", false, "show process using socket") + netstatCmd.Flags().BoolVarP(&netstatCmdFlags.timers, "timers", "o", false, "display timers") + netstatCmd.Flags().BoolVarP(&netstatCmdFlags.listening, "listening", "l", false, "display listening server sockets") + netstatCmd.Flags().BoolVarP(&netstatCmdFlags.all, "all", "a", false, "display all sockets states (default: connected)") + netstatCmd.Flags().BoolVarP(&netstatCmdFlags.pods, "pods", "k", false, "show sockets used by Kubernetes pods") + netstatCmd.Flags().BoolVarP(&netstatCmdFlags.tcp, "tcp", "t", false, "display only TCP sockets") + netstatCmd.Flags().BoolVarP(&netstatCmdFlags.udp, "udp", "u", false, "display only UDP sockets") + netstatCmd.Flags().BoolVarP(&netstatCmdFlags.udplite, "udplite", "U", false, "display only UDPLite sockets") + netstatCmd.Flags().BoolVarP(&netstatCmdFlags.raw, "raw", "w", false, "display only RAW sockets") + netstatCmd.Flags().BoolVarP(&netstatCmdFlags.ipv4, "ipv4", "4", false, "display only ipv4 sockets") + netstatCmd.Flags().BoolVarP(&netstatCmdFlags.ipv6, "ipv6", "6", false, "display only ipv6 sockets") + + addCommand(netstatCmd) +} diff --git a/pkg/commands/imported_pcap.go b/pkg/commands/imported_pcap.go new file mode 100644 index 0000000..5919aac --- /dev/null +++ b/pkg/commands/imported_pcap.go @@ -0,0 +1,285 @@ +// Code generated by go run tools/import_commands.go --talos-version v1.7.1 pcap +// DO NOT EDIT. + +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package commands + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "os" + "strings" + "syscall" + "time" + + "github.com/gopacket/gopacket" + "github.com/gopacket/gopacket/pcapgo" + "github.com/spf13/cobra" + "google.golang.org/grpc/codes" + + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" + "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/client" +) + +var pcapCmdFlags struct { + iface string + promisc bool + snaplen int + output string + bpfFilter string + duration time.Duration + configFiles []string +} + +// pcapCmd represents the pcap command. +var pcapCmd = &cobra.Command{ + Use: "pcap", + Aliases: []string{"tcpdump"}, + Short: "Capture the network packets from the node.", + Long: `The command launches packet capture on the node and streams back the packets as raw pcap file. + +Default behavior is to decode the packets with internal decoder to stdout: + + talosctl pcap -i eth0 + +Raw pcap file can be saved with ` + "`--output`" + ` flag: + + talosctl pcap -i eth0 --output eth0.pcap + +Output can be piped to tcpdump: + + talosctl pcap -i eth0 -o - | tcpdump -vvv -r - + +BPF filter can be applied, but it has to compiled to BPF instructions first using tcpdump. +Correct link type should be specified for the tcpdump: EN10MB for Ethernet links and RAW +for e.g. Wireguard tunnels: + + talosctl pcap -i eth0 --bpf-filter "$(tcpdump -dd -y EN10MB 'tcp and dst port 80')" + + talosctl pcap -i kubespan --bpf-filter "$(tcpdump -dd -y RAW 'port 50000')" + +As packet capture is transmitted over the network, it is recommended to filter out the Talos API traffic, +e.g. by excluding packets with the port 50000. + `, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return WithClient(func(ctx context.Context, c *client.Client) error { + if err := helpers.FailIfMultiNodes(ctx, "pcap"); err != nil { + return err + } + + if pcapCmdFlags.duration > 0 { + var cancel context.CancelFunc + + ctx, cancel = context.WithTimeout(ctx, pcapCmdFlags.duration) + defer cancel() + } + + req := machine.PacketCaptureRequest{ + Interface: pcapCmdFlags.iface, + Promiscuous: pcapCmdFlags.promisc, + } + + var err error + + req.BpfFilter, err = parseBPFInstructions(pcapCmdFlags.bpfFilter) + if err != nil { + return err + } + + r, err := c.PacketCapture(ctx, &req) + if err != nil { + return fmt.Errorf("error copying: %w", err) + } + + if pcapCmdFlags.output == "" { + return dumpPackets(ctx, r) + } + + var out io.Writer + + if pcapCmdFlags.output == "-" { + out = os.Stdout + } else { + out, err = os.Create(pcapCmdFlags.output) + if err != nil { + return err + } + } + + _, err = io.Copy(out, r) + + if errors.Is(err, io.EOF) || client.StatusCode(err) == codes.DeadlineExceeded { + err = nil + } + + return err + }) + }, +} + +// snapLength defines a snap length for the packet reading. For some reason +// TF_PACKET captures more than the snap length. Tools like tcpdump ignore snaplen entirely and set their own +// (https://github.com/the-tcpdump-group/tcpdump/blob/9fad826b0e487e8939325d62b7a461619b2722eb/netdissect.h#L342) +// so it makes sense to do the same. +const snapLength = 262144 + +func dumpPackets(ctx context.Context, r io.Reader) error { + src, err := pcapgo.NewReader(r) + if err != nil { + if errors.Is(err, io.EOF) { + // nothing in the capture at all + return nil + } + + return fmt.Errorf("error opening pcap reader: %w", err) + } + + src.SetSnaplen(snapLength) + + forEachPacket( + ctx, + gopacket.NewZeroCopyPacketSource(src, src.LinkType(), gopacket.WithPool(true)), + func(packet gopacket.Packet, err error) { + switch err { + case nil: + fmt.Println(packet) + default: + fmt.Println("packet capture error:", err) + } + }, + ) + + return nil +} + +// parseBPFInstructions parses the BPF raw instructions in 'tcpdump -dd' format. +// +// Example: +// +// { 0x30, 0, 0, 0x00000000 }, +// { 0x54, 0, 0, 0x000000f0 }, +// { 0x15, 0, 8, 0x00000060 }, +// +//nolint:dupword +func parseBPFInstructions(in string) ([]*machine.BPFInstruction, error) { + in = strings.TrimSpace(in) + + if in == "" { + return nil, nil + } + + var result []*machine.BPFInstruction //nolint:prealloc + + for _, line := range strings.Split(in, "\n") { + if line == "" { + continue + } + + ins := &machine.BPFInstruction{} + + n, err := fmt.Sscanf(line, "{ 0x%x, %d, %d, 0x%x },", &ins.Op, &ins.Jt, &ins.Jf, &ins.K) + if err != nil { + return nil, fmt.Errorf("error parsing bpf instruction %q: %w", line, err) + } + + if n != 4 { + return nil, fmt.Errorf("error parsing bpf instruction %q: expected 4 fields, got %d", line, n) + } + + result = append(result, ins) + } + + return result, nil +} + +func init() { + pcapCmd.Flags().StringSliceVarP(&pcapCmdFlags.configFiles, "file", + "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + pcapCmd.PreRunE = func(cmd *cobra.Command, + + args []string) error { + nodesFromArgs := len(GlobalArgs. + Nodes, + ) > + 0 + endpointsFromArgs := len(GlobalArgs. + Endpoints) > 0 + for _, configFile := range pcapCmdFlags.configFiles { + if err := processModelineAndUpdateGlobals(configFile, + + nodesFromArgs, endpointsFromArgs, false); err != nil { + return err + } + } + return nil + } + + pcapCmd.Flags().StringVarP(&pcapCmdFlags.iface, "interface", "i", "eth0", "interface name to capture packets on") + pcapCmd.Flags().BoolVar(&pcapCmdFlags.promisc, "promiscuous", false, "put interface into promiscuous mode") + pcapCmd.Flags().IntVarP(&pcapCmdFlags.snaplen, "snaplen", "s", 4096, "maximum packet size to capture") + pcapCmd.Flags().StringVarP(&pcapCmdFlags.output, "output", "o", "", "if not set, decode packets to stdout; if set write raw pcap data to a file, use '-' for stdout") + pcapCmd.Flags().StringVar(&pcapCmdFlags.bpfFilter, "bpf-filter", "", "bpf filter to apply, tcpdump -dd format") + pcapCmd.Flags().DurationVar(&pcapCmdFlags.duration, "duration", 0, "duration of the capture") + pcapCmd.Flags().MarkDeprecated("snaplen", "support of snap length is removed") //nolint:errcheck + + addCommand(pcapCmd) +} + +// forEachPacket reads packets from the packet source and calls the provided function for each packet. fn should not +// store the packet as it will be reused for the next packet. It will also call fn with nil packet and non nill +// error if the error is not known. If the context is canceled, the function will return as soon as +// [gopacket.PacketSource.NextPacket] returns. +// +// This function is more or less direct copy of [gopacket.PacketSource.PacketsCtx] minus the sleeps. +// +//nolint:gocyclo +func forEachPacket(ctx context.Context, p *gopacket.PacketSource, fn func(gopacket.Packet, error)) { + for ctx.Err() == nil { + packet, err := p.NextPacket() + if err == nil { + fn(packet, nil) + + if ctx.Err() != nil { + break + } + + // If we use pooled packets, we need to send them back to the pool + if pooled, ok := packet.(gopacket.PooledPacket); ok { + pooled.Dispose() + } + + continue + } + + // if timeout error -> retry + var netErr net.Error + if ok := errors.As(err, &netErr); ok && netErr.Timeout() { + continue + } + + // Immediately break for known unrecoverable errors + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || + errors.Is(err, io.ErrNoProgress) || errors.Is(err, io.ErrClosedPipe) || errors.Is(err, io.ErrShortBuffer) || + errors.Is(err, syscall.EBADF) || + strings.Contains(err.Error(), "use of closed file") { + break + } + + // Otherwise, send error to the caller + fn(nil, err) + + // and try again if context is not canceled + if ctx.Err() != nil { + break + } + } +} diff --git a/pkg/commands/imported_processes.go b/pkg/commands/imported_processes.go new file mode 100644 index 0000000..1b8f136 --- /dev/null +++ b/pkg/commands/imported_processes.go @@ -0,0 +1,265 @@ +// Code generated by go run tools/import_commands.go --talos-version v1.7.1 processes +// DO NOT EDIT. + +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package commands + +import ( + "context" + "fmt" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/dustin/go-humanize" + ui "github.com/gizak/termui/v3" + "github.com/gizak/termui/v3/widgets" + "github.com/ryanuber/columnize" + "github.com/spf13/cobra" + "golang.org/x/term" + "google.golang.org/grpc" + "google.golang.org/grpc/peer" + + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" + "github.com/siderolabs/talos/pkg/cli" + machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/client" +) + +var ( + sortMethod string + watchProcesses bool +) + +// processesCmd represents the processes command. +var processesCmd = &cobra.Command{ + Use: "processes", + Aliases: []string{"p", "ps"}, + Short: "List running processes", + Long: ``, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return WithClient(func(ctx context.Context, c *client.Client) error { + var err error + + switch { + case watchProcesses: + if err = ui.Init(); err != nil { + return fmt.Errorf("failed to initialize termui: %w", err) + } + defer ui.Close() + + processesUI(ctx, c) + default: + var output string + output, err = processesOutput(ctx, c) + if err != nil { + return err + } + // Note this is unlimited output of process lines + // we arent artificially limited by the box we would otherwise draw + fmt.Println(output) + } + + return nil + }) + }, +} + +func init() { + processesCmd.Flags().StringSliceVarP(&processesCmdFlags.configFiles, + "file", "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + processesCmd.PreRunE = func(cmd *cobra.Command, + + args []string) error { + nodesFromArgs := len(GlobalArgs. + Nodes) > + 0 + endpointsFromArgs := len(GlobalArgs. + Endpoints) > 0 + for _, configFile := range processesCmdFlags.configFiles { + if err := processModelineAndUpdateGlobals(configFile, + nodesFromArgs, endpointsFromArgs, + false); err != + nil { + return err + } + + } + return nil + } + + processesCmd.Flags().StringVarP(&sortMethod, "sort", "s", "rss", "Column to sort output by. [rss|cpu]") + processesCmd.Flags().BoolVarP(&watchProcesses, "watch", "w", false, "Stream running processes") + addCommand(processesCmd) +} + +func processesUI(ctx context.Context, c *client.Client) { + l := widgets.NewParagraph() + l.Border = false + l.WrapText = false + l.PaddingTop = 0 + l.PaddingBottom = 0 + + var processOutput string + + draw := func() { + // Attempt to get terminal dimensions + // Since we're getting this data on each call + // we'll be able to handle terminal window resizing + w, h, err := term.GetSize(0) + cli.Should(err) + // x, y, w, h + l.SetRect(0, 0, w, h) + l.WrapText = false + + processOutput, err = processesOutput(ctx, c) + if err != nil { + l.Text = err.Error() + l.WrapText = true + + ui.Render(l) + + return + } + + // Dont refresh if we dont have any output + if processOutput == "" { + return + } + + // Truncate our output based on terminal size + l.Text = processOutput + + ui.Render(l) + } + + draw() + + uiEvents := ui.PollEvents() + ticker := time.NewTicker(time.Second).C + + for { + select { + case <-ctx.Done(): + return + case e := <-uiEvents: + switch e.ID { + case "q", "": + return + case "r", "m": + sortMethod = "rss" + case "c": + sortMethod = "cpu" + } + case <-ticker: + draw() + } + } +} + +type by func(p1, p2 *machineapi.ProcessInfo) bool + +func (b by) sort(procs []*machineapi.ProcessInfo) { + ps := &procSorter{ + procs: procs, + by: b, // The Sort method's receiver is the function (closure) that defines the sort order. + } + sort.Sort(ps) +} + +type procSorter struct { + procs []*machineapi.ProcessInfo + by func(p1, p2 *machineapi.ProcessInfo) bool // Closure used in the Less method. +} + +// Len is part of sort.Interface. +func (s *procSorter) Len() int { + return len(s.procs) +} + +// Swap is part of sort.Interface. +func (s *procSorter) Swap(i, j int) { + s.procs[i], s.procs[j] = s.procs[j], s.procs[i] +} + +// Less is part of sort.Interface. It is implemented by calling the "by" closure in the sorter. +func (s *procSorter) Less(i, j int) bool { + return s.by(s.procs[i], s.procs[j]) +} + +// Sort Methods. +var rss = func(p1, p2 *machineapi.ProcessInfo) bool { + // Reverse sort ( Descending ) + return p1.ResidentMemory > p2.ResidentMemory +} + +var cpu = func(p1, p2 *machineapi.ProcessInfo) bool { + // Reverse sort ( Descending ) + return p1.CpuTime > p2.CpuTime +} + +func processesOutput(ctx context.Context, c *client.Client) (output string, err error) { + var remotePeer peer.Peer + + resp, err := c.Processes(ctx, grpc.Peer(&remotePeer)) + if err != nil { + return output, err + } + + defaultNode := client.AddrFromPeer(&remotePeer) + + s := []string{} + + s = append(s, "NODE | PID | STATE | THREADS | CPU-TIME | VIRTMEM | RESMEM | COMMAND") + + for _, msg := range resp.Messages { + procs := msg.Processes + + switch sortMethod { + case "cpu": + by(cpu).sort(procs) + default: + by(rss).sort(procs) + } + + var args string + + for _, p := range procs { + switch { + case p.Executable == "": + args = p.Command + case p.Args != "" && strings.Fields(p.Args)[0] == filepath.Base(strings.Fields(p.Executable)[0]): + args = strings.Replace(p.Args, strings.Fields(p.Args)[0], p.Executable, 1) + default: + args = p.Args + } + + node := defaultNode + + if msg.Metadata != nil { + node = msg.Metadata.Hostname + } + + s = append(s, + fmt.Sprintf("%12s | %6d | %1s | %4d | %8.2f | %7s | %7s | %s", + node, p.Pid, p.State, p.Threads, p.CpuTime, humanize.Bytes(p.VirtualMemory), humanize.Bytes(p.ResidentMemory), args)) + } + } + + res := columnize.SimpleFormat(s) + if err != nil { + return res, err + } + + return res, helpers.CheckErrors(resp.Messages...) +} + +var processesCmdFlags struct { + configFiles []string +} diff --git a/pkg/commands/imported_read.go b/pkg/commands/imported_read.go new file mode 100644 index 0000000..ce86b43 --- /dev/null +++ b/pkg/commands/imported_read.go @@ -0,0 +1,85 @@ +// Code generated by go run tools/import_commands.go --talos-version v1.7.1 read +// DO NOT EDIT. + +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package commands + +import ( + "context" + "fmt" + "io" + "os" + + "github.com/spf13/cobra" + + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" + "github.com/siderolabs/talos/pkg/machinery/client" +) + +// readCmd represents the read command. +var readCmd = &cobra.Command{ + Use: "read ", + Short: "Read a file on the machine", + Long: ``, + Args: cobra.ExactArgs(1), + Aliases: []string{"cat"}, + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + if len(args) != 0 { + return nil, cobra.ShellCompDirectiveError | cobra.ShellCompDirectiveNoFileComp + } + + return completePathFromNode(toComplete), cobra.ShellCompDirectiveNoFileComp + }, + RunE: func(cmd *cobra.Command, args []string) error { + return WithClient(func(ctx context.Context, c *client.Client) error { + if err := helpers.FailIfMultiNodes(ctx, "read"); err != nil { + return err + } + + r, err := c.Read(ctx, args[0]) + if err != nil { + return fmt.Errorf("error reading file: %w", err) + } + + defer r.Close() //nolint:errcheck + + _, err = io.Copy(os.Stdout, r) + if err != nil { + return fmt.Errorf("error reading: %w", err) + } + + return r.Close() + }) + }, +} + +func init() { + readCmd.Flags().StringSliceVarP(&readCmdFlags.configFiles, "file", + "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + readCmd.PreRunE = func(cmd *cobra.Command, + + args []string) error { + nodesFromArgs := + len(GlobalArgs. + Nodes) > 0 + endpointsFromArgs := len(GlobalArgs.Endpoints) > 0 + for _, configFile := range readCmdFlags.configFiles { + if err := processModelineAndUpdateGlobals(configFile, nodesFromArgs, + endpointsFromArgs, false, + ); err != nil { + return err + } + } + return nil + } + + addCommand(readCmd) +} + +var readCmdFlags struct { + configFiles []string +} diff --git a/pkg/commands/imported_reboot.go b/pkg/commands/imported_reboot.go new file mode 100644 index 0000000..40dc23d --- /dev/null +++ b/pkg/commands/imported_reboot.go @@ -0,0 +1,114 @@ +// Code generated by go run tools/import_commands.go --talos-version v1.7.1 reboot +// DO NOT EDIT. + +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package commands + +import ( + "context" + "errors" + "fmt" + + "github.com/spf13/cobra" + + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/action" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" + "github.com/siderolabs/talos/pkg/machinery/client" +) + +var rebootCmdFlags struct { + trackableActionCmdFlags + mode string + configFiles []string +} + +// rebootCmd represents the reboot command. +var rebootCmd = &cobra.Command{ + Use: "reboot", + Short: "Reboot a node", + Long: ``, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + if rebootCmdFlags.debug { + rebootCmdFlags.wait = true + } + + var opts []client.RebootMode + + switch rebootCmdFlags.mode { + // skips kexec and reboots with power cycle + case "powercycle": + opts = append(opts, client.WithPowerCycle) + case "default": + default: + return fmt.Errorf("invalid reboot mode: %q", rebootCmdFlags.mode) + } + + if !rebootCmdFlags.wait { + return WithClient(func(ctx context.Context, c *client.Client) error { + if err := helpers.ClientVersionCheck(ctx, c); err != nil { + return err + } + + if err := c.Reboot(ctx, opts...); err != nil { + return fmt.Errorf("error executing reboot: %s", err) + } + + return nil + }) + } + + return action.NewTracker( + &GlobalArgs, + action.MachineReadyEventFn, + rebootGetActorID(opts...), + action.WithPostCheck(action.BootIDChangedPostCheckFn), + action.WithDebug(rebootCmdFlags.debug), + action.WithTimeout(rebootCmdFlags.timeout), + ).Run() + }, +} + +func rebootGetActorID(opts ...client.RebootMode) func(ctx context.Context, c *client.Client) (string, error) { + return func(ctx context.Context, c *client.Client) (string, error) { + resp, err := c.RebootWithResponse(ctx, opts...) + if err != nil { + return "", err + } + + if len(resp.GetMessages()) == 0 { + return "", errors.New("no messages returned from action run") + } + + return resp.GetMessages()[0].GetActorId(), nil + } +} + +func init() { + rebootCmd.Flags().StringSliceVarP(&rebootCmdFlags.configFiles, + "file", "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + rebootCmd.PreRunE = func(cmd *cobra.Command, args []string) error { + nodesFromArgs := len( + GlobalArgs. + Nodes) > 0 + endpointsFromArgs := len(GlobalArgs.Endpoints) > 0 + for _, configFile := range rebootCmdFlags.configFiles { + if err := processModelineAndUpdateGlobals(configFile, nodesFromArgs, + + endpointsFromArgs, + false); err != + nil { + return err + } + } + return nil + } + + rebootCmd.Flags().StringVarP(&rebootCmdFlags.mode, "mode", "m", "default", "select the reboot mode: \"default\", \"powercycle\" (skips kexec)") + rebootCmdFlags.addTrackActionFlags(rebootCmd) + addCommand(rebootCmd) +} diff --git a/pkg/commands/reset.go b/pkg/commands/imported_reset.go similarity index 86% rename from pkg/commands/reset.go rename to pkg/commands/imported_reset.go index 8c1e0ca..99aff59 100644 --- a/pkg/commands/reset.go +++ b/pkg/commands/imported_reset.go @@ -1,11 +1,15 @@ +// Code generated by go run tools/import_commands.go --talos-version v1.7.1 reset +// DO NOT EDIT. + // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. -package talos +package commands import ( "context" + "errors" "fmt" "sort" "strings" @@ -13,7 +17,6 @@ import ( "github.com/siderolabs/gen/maps" "github.com/spf13/cobra" - "github.com/siderolabs/talos/cmd/talosctl/cmd/common" "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/action" "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" @@ -76,6 +79,7 @@ var resetCmdFlags struct { wipeMode WipeMode userDisksToWipe []string systemLabelsToWipe []string + configFiles []string } // resetCmd represents the reset command. @@ -92,7 +96,7 @@ var resetCmd = &cobra.Command{ resetRequest := buildResetRequest() if resetCmdFlags.wait && resetCmdFlags.insecure { - return fmt.Errorf("cannot use --wait and --insecure together") + return errors.New("cannot use --wait and --insecure together") } if !resetCmdFlags.wait { @@ -140,8 +144,6 @@ var resetCmd = &cobra.Command{ } } - common.SuppressErrors = true - return action.NewTracker( &GlobalArgs, action.StopAllServicesEventFn, @@ -179,13 +181,33 @@ func resetGetActorID(ctx context.Context, c *client.Client, req *machineapi.Rese } if len(resp.GetMessages()) == 0 { - return "", fmt.Errorf("no messages returned from action run") + return "", errors.New("no messages returned from action run") } return resp.GetMessages()[0].GetActorId(), nil } func init() { + resetCmd.Flags().StringSliceVarP(&resetCmdFlags.configFiles, "file", + "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + resetCmd.PreRunE = func(cmd *cobra.Command, args []string) error { + nodesFromArgs := + len(GlobalArgs. + Nodes) > 0 + + endpointsFromArgs := len(GlobalArgs. + Endpoints) > 0 + for _, configFile := range resetCmdFlags.configFiles { + if err := + processModelineAndUpdateGlobals(configFile, nodesFromArgs, endpointsFromArgs, + false); err != nil { + return err + } + } + return nil + } + resetCmd.Flags().BoolVar(&resetCmdFlags.graceful, "graceful", true, "if true, attempt to cordon/drain node and leave etcd (if applicable)") resetCmd.Flags().BoolVar(&resetCmdFlags.reboot, "reboot", false, "if true, reboot the node after resetting instead of shutting down") resetCmd.Flags().BoolVar(&resetCmdFlags.insecure, "insecure", false, "reset using the insecure (encrypted with no auth) maintenance service") diff --git a/pkg/commands/imported_restart.go b/pkg/commands/imported_restart.go new file mode 100644 index 0000000..088dd81 --- /dev/null +++ b/pkg/commands/imported_restart.go @@ -0,0 +1,88 @@ +// Code generated by go run tools/import_commands.go --talos-version v1.7.1 restart +// DO NOT EDIT. + +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package commands + +import ( + "context" + "fmt" + + criconstants "github.com/containerd/containerd/pkg/cri/constants" + "github.com/spf13/cobra" + + "github.com/siderolabs/talos/pkg/machinery/api/common" + "github.com/siderolabs/talos/pkg/machinery/client" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// restartCmd represents the restart command. +var restartCmd = &cobra.Command{ + Use: "restart ", + Short: "Restart a process", + Long: ``, + Args: cobra.ExactArgs(1), + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + if len(args) != 0 { + return nil, cobra.ShellCompDirectiveError | cobra.ShellCompDirectiveNoFileComp + } + + return getContainersFromNode(kubernetesFlag), cobra.ShellCompDirectiveNoFileComp + }, + RunE: func(cmd *cobra.Command, args []string) error { + return WithClient(func(ctx context.Context, c *client.Client) error { + var ( + namespace string + driver common.ContainerDriver + ) + + if kubernetesFlag { + namespace = criconstants.K8sContainerdNamespace + driver = common.ContainerDriver_CRI + } else { + namespace = constants.SystemContainerdNamespace + driver = common.ContainerDriver_CONTAINERD + } + + if err := c.Restart(ctx, namespace, driver, args[0]); err != nil { + return fmt.Errorf("error restarting process: %s", err) + } + + return nil + }) + }, +} + +func init() { + restartCmd.Flags().StringSliceVarP(&restartCmdFlags.configFiles, + "file", "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + restartCmd.PreRunE = func(cmd *cobra. + Command, args []string) error { + nodesFromArgs := len(GlobalArgs.Nodes) > 0 + endpointsFromArgs := len(GlobalArgs. + Endpoints) > 0 + for _, configFile := range restartCmdFlags.configFiles { + if err := processModelineAndUpdateGlobals(configFile, + nodesFromArgs, endpointsFromArgs, false); err != nil { + + return err + } + } + return nil + } + + restartCmd.Flags().BoolVarP(&kubernetesFlag, "kubernetes", "k", false, "use the k8s.io containerd namespace") + + restartCmd.Flags().BoolP("use-cri", "c", false, "use the CRI driver") + restartCmd.Flags().MarkHidden("use-cri") //nolint:errcheck + + addCommand(restartCmd) +} + +var restartCmdFlags struct { + configFiles []string +} diff --git a/pkg/commands/imported_rollback.go b/pkg/commands/imported_rollback.go new file mode 100644 index 0000000..c782e3c --- /dev/null +++ b/pkg/commands/imported_rollback.go @@ -0,0 +1,61 @@ +// Code generated by go run tools/import_commands.go --talos-version v1.7.1 rollback +// DO NOT EDIT. + +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package commands + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + + "github.com/siderolabs/talos/pkg/machinery/client" +) + +// rollbackCmd represents the rollback command. +var rollbackCmd = &cobra.Command{ + Use: "rollback", + Short: "Rollback a node to the previous installation", + Long: ``, + RunE: func(cmd *cobra.Command, args []string) error { + return WithClient(func(ctx context.Context, c *client.Client) error { + if err := c.Rollback(ctx); err != nil { + return fmt.Errorf("error executing rollback: %s", err) + } + + return nil + }) + }, +} + +func init() { + rollbackCmd.Flags().StringSliceVarP(&rollbackCmdFlags.configFiles, + "file", "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + rollbackCmd.PreRunE = func(cmd *cobra. + Command, args []string) error { + nodesFromArgs := len(GlobalArgs. + Nodes, + ) > 0 + endpointsFromArgs := len(GlobalArgs.Endpoints) > 0 + for _, configFile := range rollbackCmdFlags. + configFiles { + if err := processModelineAndUpdateGlobals(configFile, nodesFromArgs, endpointsFromArgs, + false); err != + nil { + return err + } + } + return nil + } + + addCommand(rollbackCmd) +} + +var rollbackCmdFlags struct { + configFiles []string +} diff --git a/pkg/commands/imported_root.go b/pkg/commands/imported_root.go new file mode 100644 index 0000000..5fc4695 --- /dev/null +++ b/pkg/commands/imported_root.go @@ -0,0 +1,209 @@ +package commands + +import ( + "context" + "fmt" + "io" + "slices" + "sort" + "strings" + + criconstants "github.com/containerd/containerd/pkg/cri/constants" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/peer" + + "github.com/siderolabs/gen/maps" + _ "github.com/siderolabs/talos/pkg/grpc/codec" // register codec + "github.com/siderolabs/talos/pkg/machinery/api/common" + machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/client" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/formatters" +) + +// completeResource represents tab complete options for `ls` and `ls *` commands. +func completePathFromNode(inputPath string) []string { + pathToSearch := inputPath + + // If the pathToSearch is empty, use root '/' + if pathToSearch == "" { + pathToSearch = "/" + } + + var paths map[string]struct{} + + // search up one level to find possible completions + if pathToSearch != "/" && !strings.HasSuffix(pathToSearch, "/") { + index := strings.LastIndex(pathToSearch, "/") + // we need a trailing slash to search for items in a directory + pathToSearch = pathToSearch[:index] + "/" + } + + paths = getPathFromNode(pathToSearch, inputPath) + + return maps.Keys(paths) +} + +//nolint:gocyclo +func getPathFromNode(path, filter string) map[string]struct{} { + paths := make(map[string]struct{}) + + //nolint:errcheck + GlobalArgs.WithClient( + func(ctx context.Context, c *client.Client) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + stream, err := c.LS( + ctx, &machineapi.ListRequest{ + Root: path, + }, + ) + if err != nil { + return err + } + + for { + resp, err := stream.Recv() + if err != nil { + if err == io.EOF || client.StatusCode(err) == codes.Canceled { + return nil + } + + return fmt.Errorf("error streaming results: %s", err) + } + + if resp.Metadata != nil && resp.Metadata.Error != "" { + continue + } + + if resp.Error != "" { + continue + } + + // skip reference to the same directory + if resp.RelativeName == "." { + continue + } + + // limit the results to a reasonable amount + if len(paths) > pathAutoCompleteLimit { + return nil + } + + // directories have a trailing slash + if resp.IsDir { + fullPath := path + resp.RelativeName + "/" + + if relativeTo(fullPath, filter) { + paths[fullPath] = struct{}{} + } + } else { + fullPath := path + resp.RelativeName + + if relativeTo(fullPath, filter) { + paths[fullPath] = struct{}{} + } + } + } + }, + ) + + return paths +} + +func getServiceFromNode() []string { + var svcIds []string + + //nolint:errcheck + GlobalArgs.WithClient( + func(ctx context.Context, c *client.Client) error { + var remotePeer peer.Peer + + resp, err := c.ServiceList(ctx, grpc.Peer(&remotePeer)) + if err != nil { + return err + } + + for _, msg := range resp.Messages { + for _, s := range msg.Services { + svc := formatters.ServiceInfoWrapper{ServiceInfo: s} + svcIds = append(svcIds, svc.Id) + } + } + + return nil + }, + ) + + return svcIds +} + +func getContainersFromNode(kubernetes bool) []string { + var containerIds []string + + //nolint:errcheck + GlobalArgs.WithClient( + func(ctx context.Context, c *client.Client) error { + var ( + namespace string + driver common.ContainerDriver + ) + + if kubernetes { + namespace = criconstants.K8sContainerdNamespace + driver = common.ContainerDriver_CRI + } else { + namespace = constants.SystemContainerdNamespace + driver = common.ContainerDriver_CONTAINERD + } + + resp, err := c.Containers(ctx, namespace, driver) + if err != nil { + return err + } + + for _, msg := range resp.Messages { + for _, p := range msg.Containers { + if p.Pid == 0 { + continue + } + + if kubernetes && p.Id == p.PodId { + continue + } + + containerIds = append(containerIds, p.Id) + } + } + + return nil + }, + ) + + return containerIds +} + +func mergeSuggestions(a, b, c []string) []string { + merged := append(slices.Clone(a), b...) + + sort.Strings(merged) + + n := 1 + + for i := 1; i < len(merged); i++ { + if merged[i] != merged[i-1] { + merged[n] = merged[i] + n++ + } + } + + merged = merged[:n] + + return merged +} + +func relativeTo(fullPath string, filter string) bool { + return strings.HasPrefix(fullPath, filter) +} diff --git a/pkg/commands/imported_service.go b/pkg/commands/imported_service.go new file mode 100644 index 0000000..5ad31a4 --- /dev/null +++ b/pkg/commands/imported_service.go @@ -0,0 +1,247 @@ +// Code generated by go run tools/import_commands.go --talos-version v1.7.1 service +// DO NOT EDIT. + +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package commands + +import ( + "context" + "fmt" + "os" + "text/tabwriter" + + "github.com/spf13/cobra" + "google.golang.org/grpc" + "google.golang.org/grpc/peer" + + "github.com/siderolabs/talos/pkg/cli" + "github.com/siderolabs/talos/pkg/machinery/client" + "github.com/siderolabs/talos/pkg/machinery/formatters" +) + +// serviceCmd represents the service command. +var serviceCmd = &cobra.Command{ + Use: "service [ [start|stop|restart|status]]", + Aliases: []string{"services"}, + Short: "Retrieve the state of a service (or all services), control service state", + Long: `Service control command. If run without arguments, lists all the services and their state. +If service ID is specified, default action 'status' is executed which shows status of a single list service. +With actions 'start', 'stop', 'restart', service state is updated respectively.`, + Args: cobra.MaximumNArgs(2), + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + switch len(args) { + case 0: + return getServiceFromNode(), cobra.ShellCompDirectiveNoFileComp + case 1: + return []string{"start", "stop", "restart", "status"}, cobra.ShellCompDirectiveNoFileComp + } + + return nil, cobra.ShellCompDirectiveError | cobra.ShellCompDirectiveNoFileComp + }, + RunE: func(cmd *cobra.Command, args []string) error { + action := "status" + serviceID := "" + + if len(args) >= 1 { + serviceID = args[0] + } + + if len(args) == 2 { + action = args[1] + } + + return WithClient(func(ctx context.Context, c *client.Client) error { + switch action { + case "status": + if serviceID == "" { + return serviceList(ctx, c) + } + + return serviceInfo(ctx, c, serviceID) + case "start": + return serviceStart(ctx, c, serviceID) + case "stop": + return serviceStop(ctx, c, serviceID) + case "restart": + return serviceRestart(ctx, c, serviceID) + default: + return fmt.Errorf("unsupported service action: %q", action) + } + }) + }, +} + +func serviceList(ctx context.Context, c *client.Client) error { + var remotePeer peer.Peer + + resp, err := c.ServiceList(ctx, grpc.Peer(&remotePeer)) + if err != nil { + if resp == nil { + return fmt.Errorf("error listing services: %w", err) + } + + cli.Warning("%s", err) + } + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + fmt.Fprintln(w, "NODE\tSERVICE\tSTATE\tHEALTH\tLAST CHANGE\tLAST EVENT") + + defaultNode := client.AddrFromPeer(&remotePeer) + + for _, msg := range resp.Messages { + for _, s := range msg.Services { + svc := formatters.ServiceInfoWrapper{ServiceInfo: s} + + node := defaultNode + + if msg.Metadata != nil { + node = msg.Metadata.Hostname + } + + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s ago\t%s\n", node, svc.Id, svc.State, svc.HealthStatus(), svc.LastUpdated(), svc.LastEvent()) + } + } + + return w.Flush() +} + +func serviceInfo(ctx context.Context, c *client.Client, id string) error { + var remotePeer peer.Peer + + services, err := c.ServiceInfo(ctx, id, grpc.Peer(&remotePeer)) + if err != nil { + if services == nil { + return fmt.Errorf("error listing services: %w", err) + } + + cli.Warning("%s", err) + } + + defaultNode := client.AddrFromPeer(&remotePeer) + + if len(services) == 0 { + return fmt.Errorf("service %q is not registered on any nodes", id) + } + + return formatters.RenderServicesInfo(services, os.Stdout, defaultNode, true) +} + +func serviceStart(ctx context.Context, c *client.Client, id string) error { + var remotePeer peer.Peer + + resp, err := c.ServiceStart(ctx, id, grpc.Peer(&remotePeer)) + if err != nil { + if resp == nil { + return fmt.Errorf("error starting service: %w", err) + } + + cli.Warning("%s", err) + } + + defaultNode := client.AddrFromPeer(&remotePeer) + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + fmt.Fprintln(w, "NODE\tRESPONSE") + + for _, msg := range resp.Messages { + node := defaultNode + + if msg.Metadata != nil { + node = msg.Metadata.Hostname + } + + fmt.Fprintf(w, "%s\t%s\n", node, msg.Resp) + } + + return w.Flush() +} + +func serviceStop(ctx context.Context, c *client.Client, id string) error { + var remotePeer peer.Peer + + resp, err := c.ServiceStop(ctx, id, grpc.Peer(&remotePeer)) + if err != nil { + if resp == nil { + return fmt.Errorf("error starting service: %w", err) + } + + cli.Warning("%s", err) + } + + defaultNode := client.AddrFromPeer(&remotePeer) + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + fmt.Fprintln(w, "NODE\tRESPONSE") + + for _, msg := range resp.Messages { + node := defaultNode + + if msg.Metadata != nil { + node = msg.Metadata.Hostname + } + + fmt.Fprintf(w, "%s\t%s\n", node, msg.Resp) + } + + return w.Flush() +} + +func serviceRestart(ctx context.Context, c *client.Client, id string) error { + var remotePeer peer.Peer + + resp, err := c.ServiceRestart(ctx, id, grpc.Peer(&remotePeer)) + if err != nil { + if resp == nil { + return fmt.Errorf("error starting service: %w", err) + } + + cli.Warning("%s", err) + } + + defaultNode := client.AddrFromPeer(&remotePeer) + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + fmt.Fprintln(w, "NODE\tRESPONSE") + + for _, msg := range resp.Messages { + node := defaultNode + + if msg.Metadata != nil { + node = msg.Metadata.Hostname + } + + fmt.Fprintf(w, "%s\t%s\n", node, msg.Resp) + } + + return w.Flush() +} + +func init() { + serviceCmd.Flags().StringSliceVarP(&serviceCmdFlags.configFiles, + "file", "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + serviceCmd.PreRunE = func(cmd *cobra. + Command, args []string) error { + nodesFromArgs := len(GlobalArgs. + Nodes, + ) > 0 + endpointsFromArgs := len(GlobalArgs.Endpoints) > 0 + for _, configFile := range serviceCmdFlags.configFiles { + if err := processModelineAndUpdateGlobals(configFile, + nodesFromArgs, endpointsFromArgs, false); err != nil { + + return err + } + } + return nil + } + + addCommand(serviceCmd) +} + +var serviceCmdFlags struct { + configFiles []string +} diff --git a/pkg/commands/imported_shutdown.go b/pkg/commands/imported_shutdown.go new file mode 100644 index 0000000..0a8eb2f --- /dev/null +++ b/pkg/commands/imported_shutdown.go @@ -0,0 +1,106 @@ +// Code generated by go run tools/import_commands.go --talos-version v1.7.1 shutdown +// DO NOT EDIT. + +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package commands + +import ( + "context" + "errors" + "fmt" + + "github.com/spf13/cobra" + + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/action" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" + "github.com/siderolabs/talos/pkg/machinery/client" +) + +var shutdownCmdFlags struct { + trackableActionCmdFlags + force bool + configFiles []string +} + +// shutdownCmd represents the shutdown command. +var shutdownCmd = &cobra.Command{ + Use: "shutdown", + Short: "Shutdown a node", + Long: ``, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + if shutdownCmdFlags.debug { + shutdownCmdFlags.wait = true + } + + opts := []client.ShutdownOption{ + client.WithShutdownForce(shutdownCmdFlags.force), + } + + if !shutdownCmdFlags.wait { + return WithClient(func(ctx context.Context, c *client.Client) error { + if err := helpers.ClientVersionCheck(ctx, c); err != nil { + return err + } + + if err := c.Shutdown(ctx, opts...); err != nil { + return fmt.Errorf("error executing shutdown: %s", err) + } + + return nil + }) + } + + return action.NewTracker( + &GlobalArgs, + action.StopAllServicesEventFn, + shutdownGetActorID, + action.WithDebug(shutdownCmdFlags.debug), + action.WithTimeout(shutdownCmdFlags.timeout), + ).Run() + }, +} + +func shutdownGetActorID(ctx context.Context, c *client.Client) (string, error) { + resp, err := c.ShutdownWithResponse(ctx, client.WithShutdownForce(shutdownCmdFlags.force)) + if err != nil { + return "", err + } + + if len(resp.GetMessages()) == 0 { + return "", errors.New("no messages returned from action run") + } + + return resp.GetMessages()[0].GetActorId(), nil +} + +func init() { + shutdownCmd.Flags().StringSliceVarP(&shutdownCmdFlags.configFiles, + "file", "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + shutdownCmd.PreRunE = func(cmd *cobra. + Command, args []string) error { + nodesFromArgs := + + len(GlobalArgs.Nodes) > 0 + + endpointsFromArgs := len(GlobalArgs.Endpoints) > 0 + for _, configFile := range shutdownCmdFlags.configFiles { + if err := processModelineAndUpdateGlobals(configFile, nodesFromArgs, + + endpointsFromArgs, false, + ); err != nil { + return err + } + + } + return nil + } + + shutdownCmd.Flags().BoolVar(&shutdownCmdFlags.force, "force", false, "if true, force a node to shutdown without a cordon/drain") + shutdownCmdFlags.addTrackActionFlags(shutdownCmd) + addCommand(shutdownCmd) +} diff --git a/pkg/commands/imported_stats.go b/pkg/commands/imported_stats.go new file mode 100644 index 0000000..53e5cd2 --- /dev/null +++ b/pkg/commands/imported_stats.go @@ -0,0 +1,130 @@ +// Code generated by go run tools/import_commands.go --talos-version v1.7.1 stats +// DO NOT EDIT. + +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package commands + +import ( + "context" + "fmt" + "os" + "sort" + "strings" + "text/tabwriter" + + criconstants "github.com/containerd/containerd/pkg/cri/constants" + "github.com/spf13/cobra" + "google.golang.org/grpc" + "google.golang.org/grpc/peer" + + "github.com/siderolabs/talos/pkg/cli" + "github.com/siderolabs/talos/pkg/machinery/api/common" + machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/client" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// statsCmd represents the stats command. +var statsCmd = &cobra.Command{ + Use: "stats", + Short: "Get container stats", + Long: ``, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return WithClient(func(ctx context.Context, c *client.Client) error { + var ( + namespace string + driver common.ContainerDriver + ) + + if kubernetesFlag { + namespace = criconstants.K8sContainerdNamespace + driver = common.ContainerDriver_CRI + } else { + namespace = constants.SystemContainerdNamespace + driver = common.ContainerDriver_CONTAINERD + } + + var remotePeer peer.Peer + + resp, err := c.Stats(ctx, namespace, driver, grpc.Peer(&remotePeer)) + if err != nil { + if resp == nil { + return fmt.Errorf("error getting stats: %s", err) + } + + cli.Warning("%s", err) + } + + return statsRender(&remotePeer, resp) + }) + }, +} + +func statsRender(remotePeer *peer.Peer, resp *machineapi.StatsResponse) error { + w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + + fmt.Fprintln(w, "NODE\tNAMESPACE\tID\tMEMORY(MB)\tCPU") + + defaultNode := client.AddrFromPeer(remotePeer) + + for _, msg := range resp.Messages { + sort.Slice(msg.Stats, + func(i, j int) bool { + return strings.Compare(msg.Stats[i].Id, msg.Stats[j].Id) < 0 + }) + + for _, s := range msg.Stats { + display := s.Id + if s.Id != s.PodId { + // container in a sandbox + display = "└─ " + display + } + + node := defaultNode + + if msg.Metadata != nil { + node = msg.Metadata.Hostname + } + + fmt.Fprintf(w, "%s\t%s\t%s\t%.2f\t%d\n", node, s.Namespace, display, float64(s.MemoryUsage)*1e-6, s.CpuUsage) + } + } + + return w.Flush() +} + +func init() { + statsCmd.Flags().StringSliceVarP(&statsCmdFlags.configFiles, "file", + "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + statsCmd.PreRunE = func(cmd *cobra.Command, args []string) error { + nodesFromArgs := len(GlobalArgs. + Nodes) > + 0 + endpointsFromArgs := len(GlobalArgs.Endpoints) > 0 + for _, configFile := range statsCmdFlags.configFiles { + if err := processModelineAndUpdateGlobals(configFile, + + nodesFromArgs, endpointsFromArgs, false, + ); err != nil { + return err + } + } + return nil + } + + statsCmd.Flags().BoolVarP(&kubernetesFlag, "kubernetes", "k", false, "use the k8s.io containerd namespace") + + statsCmd.Flags().BoolP("use-cri", "c", false, "use the CRI driver") + statsCmd.Flags().MarkHidden("use-cri") //nolint:errcheck + + addCommand(statsCmd) +} + +var statsCmdFlags struct { + configFiles []string +} diff --git a/pkg/commands/imported_time.go b/pkg/commands/imported_time.go new file mode 100644 index 0000000..edae956 --- /dev/null +++ b/pkg/commands/imported_time.go @@ -0,0 +1,114 @@ +// Code generated by go run tools/import_commands.go --talos-version v1.7.1 time +// DO NOT EDIT. + +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package commands + +import ( + "context" + "errors" + "fmt" + "os" + "text/tabwriter" + "time" + + "github.com/spf13/cobra" + "google.golang.org/grpc" + "google.golang.org/grpc/peer" + + "github.com/siderolabs/talos/pkg/cli" + timeapi "github.com/siderolabs/talos/pkg/machinery/api/time" + "github.com/siderolabs/talos/pkg/machinery/client" +) + +var timeCmdFlags struct { + ntpServer string + configFiles []string +} + +// timeCmd represents the time command. +var timeCmd = &cobra.Command{ + Use: "time [--check server]", + Short: "Gets current server time", + Long: ``, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return WithClient(func(ctx context.Context, c *client.Client) error { + var ( + resp *timeapi.TimeResponse + remotePeer peer.Peer + err error + ) + + if timeCmdFlags.ntpServer == "" { + resp, err = c.Time(ctx, grpc.Peer(&remotePeer)) + } else { + resp, err = c.TimeCheck(ctx, timeCmdFlags.ntpServer, grpc.Peer(&remotePeer)) + } + + if err != nil { + if resp == nil { + return fmt.Errorf("error fetching time: %w", err) + } + + cli.Warning("%s", err) + } + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + fmt.Fprintln(w, "NODE\tNTP-SERVER\tNODE-TIME\tNTP-SERVER-TIME") + + defaultNode := client.AddrFromPeer(&remotePeer) + + var localtime, remotetime time.Time + for _, msg := range resp.Messages { + node := defaultNode + + if msg.Metadata != nil { + node = msg.Metadata.Hostname + } + + if !msg.Localtime.IsValid() { + return errors.New("error parsing local time") + } + + if !msg.Remotetime.IsValid() { + return errors.New("error parsing remote time") + } + + localtime = msg.Localtime.AsTime() + remotetime = msg.Remotetime.AsTime() + + fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", node, msg.Server, localtime.String(), remotetime.String()) + } + + return w.Flush() + }) + }, +} + +func init() { + timeCmd.Flags().StringSliceVarP(&timeCmdFlags.configFiles, "file", + "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + timeCmd.PreRunE = func(cmd *cobra.Command, + + args []string) error { + nodesFromArgs := len(GlobalArgs. + Nodes, + ) > 0 + endpointsFromArgs := len(GlobalArgs.Endpoints) > 0 + for _, configFile := range timeCmdFlags.configFiles { + if err := processModelineAndUpdateGlobals(configFile, nodesFromArgs, endpointsFromArgs, false); err != nil { + return err + } + } + return nil + + } + + timeCmd.Flags().StringVarP(&timeCmdFlags.ntpServer, "check", "c", "", "checks server time against specified ntp server") + addCommand(timeCmd) +} diff --git a/pkg/commands/imported_version.go b/pkg/commands/imported_version.go new file mode 100644 index 0000000..1f3f6bb --- /dev/null +++ b/pkg/commands/imported_version.go @@ -0,0 +1,139 @@ +// Code generated by go run tools/import_commands.go --talos-version v1.7.1 version +// DO NOT EDIT. + +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package commands + +import ( + "context" + "fmt" + "strings" + + "github.com/spf13/cobra" + "google.golang.org/grpc" + "google.golang.org/grpc/peer" + "google.golang.org/protobuf/encoding/protojson" + + "github.com/siderolabs/talos/pkg/cli" + "github.com/siderolabs/talos/pkg/machinery/client" + "github.com/siderolabs/talos/pkg/machinery/version" +) + +// versionCmdFlags represents the `talosctl version` command's flags. +var versionCmdFlags struct { + clientOnly bool + shortVersion bool + json bool + insecure bool + configFiles []string +} + +// versionCmd represents the `talosctl version` command. +var versionCmd = &cobra.Command{ + Use: "version", + Short: "Prints the version", + Long: ``, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + if !versionCmdFlags.json { + fmt.Println("Client:") + if versionCmdFlags.shortVersion { + version.PrintShortVersion() + } else { + version.PrintLongVersion() + } + + // Exit early if we're only looking for client version + if versionCmdFlags.clientOnly { + return nil + } + + fmt.Println("Server:") + } + + if versionCmdFlags.insecure { + return WithClientMaintenance(nil, cmdVersion) + } + + return WithClient(cmdVersion) + }, +} + +func cmdVersion(ctx context.Context, c *client.Client) error { + var remotePeer peer.Peer + + resp, err := c.Version(ctx, grpc.Peer(&remotePeer)) + if err != nil { + if resp == nil { + return fmt.Errorf("error getting version: %s", err) + } + + cli.Warning("%s", err) + } + + defaultNode := client.AddrFromPeer(&remotePeer) + + for _, msg := range resp.Messages { + node := defaultNode + + if msg.Metadata != nil { + node = msg.Metadata.Hostname + } + + if !versionCmdFlags.json { + fmt.Printf("\t%s: %s\n", "NODE", node) + + version.PrintLongVersionFromExisting(msg.Version) + + var enabledFeatures []string + if msg.Features.GetRbac() { + enabledFeatures = append(enabledFeatures, "RBAC") + } + + fmt.Printf("\tEnabled: %s\n", strings.Join(enabledFeatures, ", ")) + + continue + } + + b, err := protojson.Marshal(msg) + if err != nil { + return err + } + + fmt.Printf("%s\n", b) + } + + return nil +} + +func init() { + versionCmd.Flags().StringSliceVarP(&versionCmdFlags.configFiles, + "file", "f", nil, "specify config files or patches in a YAML file (can specify multiple)", + ) + versionCmd.PreRunE = func(cmd *cobra. + Command, args []string) error { + nodesFromArgs := len(GlobalArgs.Nodes) > 0 + endpointsFromArgs := len(GlobalArgs. + Endpoints) > 0 + for _, configFile := range versionCmdFlags.configFiles { + if err := processModelineAndUpdateGlobals(configFile, nodesFromArgs, endpointsFromArgs, + false); err != nil { + return err + } + } + return nil + } + + versionCmd.Flags().BoolVar(&versionCmdFlags.shortVersion, "short", false, "Print the short version") + versionCmd.Flags().BoolVar(&versionCmdFlags.clientOnly, "client", false, "Print client version only") + versionCmd.Flags().BoolVarP(&versionCmdFlags.insecure, "insecure", "i", false, "use Talos maintenance mode API") + + // TODO remove when https://github.com/siderolabs/talos/issues/907 is implemented + versionCmd.Flags().BoolVar(&versionCmdFlags.json, "json", false, "") + cli.Should(versionCmd.Flags().MarkHidden("json")) + + addCommand(versionCmd) +} diff --git a/tools/import_commands.go b/tools/import_commands.go new file mode 100644 index 0000000..4e9530d --- /dev/null +++ b/tools/import_commands.go @@ -0,0 +1,282 @@ +package main + +import ( + "bytes" + "flag" + "fmt" + "go/ast" + "go/format" + "go/parser" + "go/token" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "path/filepath" + "strings" +) + +var talosVersion = flag.String("talos-version", "main", "the desired Talos version (branch or tag)") + +func changePackageName(node *ast.File, newPackageName string) { + node.Name = ast.NewIdent(newPackageName) +} + +func addFieldToStruct(node *ast.File, varName, fieldType, fieldName string) { + // Variable to track if the variable or type is found + var found bool + + // Inspect nodes to find and modify the target struct declaration or its type + ast.Inspect(node, func(n ast.Node) bool { + switch decl := n.(type) { + case *ast.GenDecl: + if decl.Tok == token.TYPE { + for _, spec := range decl.Specs { + ts, ok := spec.(*ast.TypeSpec) + if !ok || !strings.HasSuffix(ts.Name.Name, "Type") { + continue + } + if typeName := strings.TrimSuffix(ts.Name.Name, "Type"); typeName != varName { + continue + } + + st, ok := ts.Type.(*ast.StructType) + if !ok { + continue + } + + // Add field to the found struct type + field := &ast.Field{ + Names: []*ast.Ident{ast.NewIdent(fieldName)}, + Type: ast.NewIdent(fieldType), + } + st.Fields.List = append(st.Fields.List, field) + found = true + return false // stop searching, type found and updated + } + } else if decl.Tok == token.VAR { + for _, spec := range decl.Specs { + vs, ok := spec.(*ast.ValueSpec) + if !ok || len(vs.Names) != 1 || vs.Names[0].Name != varName { + continue + } + + st, ok := vs.Type.(*ast.StructType) + if !ok { + continue + } + + // Add field to the found struct variable + field := &ast.Field{ + Names: []*ast.Ident{ast.NewIdent(fieldName)}, + Type: ast.NewIdent(fieldType), + } + st.Fields.List = append(st.Fields.List, field) + found = true + return false // stop searching, variable found and updated + } + } + } + return true + }) + + // If the struct or type is not found, create a new variable struct + if !found { + newField := &ast.Field{ + Names: []*ast.Ident{ast.NewIdent(fieldName)}, + Type: ast.NewIdent(fieldType), + } + newStruct := &ast.StructType{ + Fields: &ast.FieldList{ + List: []*ast.Field{newField}, + }, + } + newSpec := &ast.ValueSpec{ + Names: []*ast.Ident{ast.NewIdent(varName)}, + Type: newStruct, + } + newDecl := &ast.GenDecl{ + Tok: token.VAR, + Specs: []ast.Spec{newSpec}, + } + + node.Decls = append(node.Decls, newDecl) + fmt.Println("New struct variable created:", varName) + } +} + +func prependStmtToInit(node *ast.File, cmdName string) { + ast.Inspect(node, func(n ast.Node) bool { + fn, ok := n.(*ast.FuncDecl) + if ok && fn.Name.Name == "init" { + stmt := &ast.ExprStmt{ + X: &ast.CallExpr{ + Fun: &ast.SelectorExpr{ + X: &ast.SelectorExpr{ + X: ast.NewIdent(cmdName + "Cmd"), + Sel: ast.NewIdent("Flags()"), + }, + Sel: ast.NewIdent("StringSliceVarP"), + }, + Args: []ast.Expr{ + &ast.UnaryExpr{ + Op: token.AND, + X: ast.NewIdent(cmdName + "CmdFlags.configFiles"), + }, + ast.NewIdent(`"file"`), + ast.NewIdent(`"f"`), + ast.NewIdent("nil"), + ast.NewIdent(`"specify config files or patches in a YAML file (can specify multiple)"`), + }, + }, + } + fn.Body.List = append([]ast.Stmt{stmt}, fn.Body.List...) + return false + } + return true + }) +} + +func insertInitCode(node *ast.File, cmdName, initCode string) { + anonFuncCode := fmt.Sprintf(`func() { %s }`, initCode) + + initCodeExpr, err := parser.ParseExpr(anonFuncCode) + if err != nil { + log.Fatalf("Failed to parse init code: %v", err) + } + + ast.Inspect(node, func(n ast.Node) bool { + switch x := n.(type) { + case *ast.FuncDecl: + if x.Name.Name == "init" { + if x.Body != nil { + initFunc, ok := initCodeExpr.(*ast.FuncLit) + if !ok { + log.Fatalf("Failed to extract function body from init code expression") + } + + x.Body.List = append(initFunc.Body.List, x.Body.List...) + } + } + } + return true + }) +} + +func processFile(filename, cmdName string) { + content, err := ioutil.ReadFile(filename) + if err != nil { + log.Fatalf("Failed to read the file: %v", err) + } + src := string(content) + + src = strings.ReplaceAll(src, "\"f\"", "\"F\"") + src = strings.ReplaceAll(src, "github.com/siderolabs/talos/internal", "github.com/aenix-io/talm/internal") + + // Create a new set of tokens and parse the source code + fset := token.NewFileSet() + node, err := parser.ParseFile(fset, filename, src, parser.ParseComments) + if err != nil { + log.Fatalf("Failed to parse file: %v", err) + } + + changePackageName(node, "commands") + addFieldToStruct(node, cmdName+"CmdFlags", "[]string", "configFiles") + + initCode := fmt.Sprintf(`%sCmd.Flags().StringSliceVarP(&%sCmdFlags.configFiles, "file", "f", nil, "specify config files or patches in a YAML file (can specify multiple)") + %sCmd.PreRunE = func(cmd *cobra.Command, args []string) error { + nodesFromArgs := len(GlobalArgs.Nodes) > 0 + endpointsFromArgs := len(GlobalArgs.Endpoints) > 0 + for _, configFile := range %sCmdFlags.configFiles { + if err := processModelineAndUpdateGlobals(configFile, nodesFromArgs, endpointsFromArgs, false); err != nil { + return err + } + } + return nil + } + `, cmdName, cmdName, cmdName, cmdName) + + if cmdName == "etcd" { + for _, subCmdName := range []string{"etcdAlarmCmd", "etcdDefragCmd", "etcdForfeitLeadershipCmd", "etcdLeaveCmd", "etcdMemberListCmd", "etcdMemberRemoveCmd", "etcdSnapshotCmd", "etcdStatusCmd"} { + initCode = fmt.Sprintf("%s\n%s", initCode, fmt.Sprintf(` + %s.Flags().StringSliceVarP(&etcdCmdFlags.configFiles, "file", "f", nil, "specify config files or patches in a YAML file (can specify multiple)") + %s.PreRunE = etcdCmd.PreRunE + `, subCmdName, subCmdName)) + } + } + if cmdName == "image" { + for _, subCmdName := range []string{"imageDefaultCmd", "imageListCmd", "imagePullCmd"} { + initCode = fmt.Sprintf("%s\n%s", initCode, fmt.Sprintf(` + %s.Flags().StringSliceVarP(&etcdCmdFlags.configFiles, "file", "f", nil, "specify config files or patches in a YAML file (can specify multiple)") + %s.PreRunE = etcdCmd.PreRunE + `, subCmdName, subCmdName)) + } + } + + insertInitCode(node, cmdName, initCode) + + var buf bytes.Buffer + comment := fmt.Sprintf("// Code generated by go run tools/import_commands.go --talos-version %s %s\n// DO NOT EDIT.\n\n", *talosVersion, cmdName) + buf.WriteString(comment) + + if err := format.Node(&buf, fset, node); err != nil { + log.Fatalf("Failed to format the AST: %v", err) + } + + if err := ioutil.WriteFile(filename, buf.Bytes(), 0644); err != nil { + log.Fatalf("Failed to write the modified file: %v", err) + } + + log.Printf("File %s updated successfully.", filename) +} + +func main() { + flag.Parse() + url := fmt.Sprintf("https://github.com/siderolabs/talos/raw/%s/cmd/talosctl/cmd/talos/", *talosVersion) + + args := flag.Args() + if len(args) == 0 { + fmt.Println("Please provide commands to import") + return + } + + for _, cmd := range args { + srcName := cmd + ".go" + dstName := "pkg/commands/imported_" + srcName + + err := downloadFile(srcName, dstName, url) + if err != nil { + log.Fatalf("Error downloading file: %v", err) + } + + log.Printf("File %s succefully downloaded to %s", srcName, dstName) + + cmdName := strings.TrimSuffix(filepath.Base(srcName), ".go") + if cmdName == "list" { + cmdName = "ls" + } + processFile(dstName, cmdName) + } +} + +func downloadFile(srcName, dstName string, url string) error { + resp, err := http.Get(url + "/" + srcName) + if err != nil { + return err + } + defer resp.Body.Close() + + file, err := os.Create(dstName) + if err != nil { + return err + } + defer file.Close() + + _, err = io.Copy(file, resp.Body) + if err != nil { + return err + } + + return nil +} diff --git a/tools/import_functions.go b/tools/import_functions.go deleted file mode 100644 index 76d6450..0000000 --- a/tools/import_functions.go +++ /dev/null @@ -1,197 +0,0 @@ -package main - -import ( - "bytes" - "flag" - "fmt" - "go/ast" - "go/format" - "go/parser" - "go/token" - "io" - "io/ioutil" - "log" - "net/http" - "os" - "path/filepath" - "strings" -) - -func changePackageName(node *ast.File, newPackageName string) { - node.Name = ast.NewIdent(newPackageName) -} - -func addFieldToStructDecl(node *ast.File, varName string, fieldType, fieldName string) { - ast.Inspect(node, func(n ast.Node) bool { - decl, ok := n.(*ast.GenDecl) - if !ok || decl.Tok != token.VAR { - return true - } - for _, spec := range decl.Specs { - vs, ok := spec.(*ast.ValueSpec) - if !ok || len(vs.Names) != 1 || vs.Names[0].Name != varName { - continue - } - st, ok := vs.Type.(*ast.StructType) - if !ok { - continue - } - field := &ast.Field{ - Names: []*ast.Ident{ast.NewIdent(fieldName)}, - Type: ast.NewIdent(fieldType), - } - st.Fields.List = append(st.Fields.List, field) - return false - } - return true - }) -} - -func prependStmtToInit(node *ast.File, cmdName string) { - ast.Inspect(node, func(n ast.Node) bool { - fn, ok := n.(*ast.FuncDecl) - if ok && fn.Name.Name == "init" { - stmt := &ast.ExprStmt{ - X: &ast.CallExpr{ - Fun: &ast.SelectorExpr{ - X: &ast.SelectorExpr{ - X: ast.NewIdent(cmdName + "Cmd"), - Sel: ast.NewIdent("Flags()"), - }, - Sel: ast.NewIdent("StringSliceVarP"), - }, - Args: []ast.Expr{ - &ast.UnaryExpr{ - Op: token.AND, - X: ast.NewIdent(cmdName + "CmdFlags.configFiles"), - }, - ast.NewIdent(`"file"`), - ast.NewIdent(`"f"`), - ast.NewIdent("nil"), - ast.NewIdent(`"specify config files or patches in a YAML file (can specify multiple)"`), - }, - }, - } - fn.Body.List = append([]ast.Stmt{stmt}, fn.Body.List...) - return false - } - return true - }) -} - -func insertInitCode(node *ast.File, cmdName, initCode string) { - anonFuncCode := fmt.Sprintf(`func() { %s }`, initCode) - - initCodeExpr, err := parser.ParseExpr(anonFuncCode) - if err != nil { - log.Fatalf("Failed to parse init code: %v", err) - } - - ast.Inspect(node, func(n ast.Node) bool { - switch x := n.(type) { - case *ast.FuncDecl: - if x.Name.Name == "init" { - if x.Body != nil { - initFunc, ok := initCodeExpr.(*ast.FuncLit) - if !ok { - log.Fatalf("Failed to extract function body from init code expression") - } - - x.Body.List = append(initFunc.Body.List, x.Body.List...) - } - } - } - return true - }) -} - -func processFile(filename string) { - fset := token.NewFileSet() - node, err := parser.ParseFile(fset, filename, nil, parser.ParseComments) - if err != nil { - log.Fatalf("Failed to parse file: %v", err) - } - - cmdName := strings.TrimSuffix(filepath.Base(filename), ".go") - changePackageName(node, "commands") - addFieldToStructDecl(node, cmdName+"CmdFlags", "[]string", "configFiles") - - initCode := fmt.Sprintf(`%sCmd.Flags().StringSliceVarP(&%sCmdFlags.configFiles, "file", "f", nil, "specify config files or patches in a YAML file (can specify multiple)") - %sCmd.PreRunE = func(cmd *cobra.Command, args []string) error { - nodesFromArgs := len( - GlobalArgs.Nodes) > 0 - endpointsFromArgs := len(GlobalArgs.Endpoints) > - 0 - for _, configFile := range %sCmdFlags.configFiles { - if err := - processModelineAndUpdateGlobals(configFile, - - nodesFromArgs, - endpointsFromArgs, false, - ); err != nil { - return err - } - } - return nil - } - `, cmdName, cmdName, cmdName, cmdName) - - insertInitCode(node, cmdName, initCode) - - var buf bytes.Buffer - if err := format.Node(&buf, fset, node); err != nil { - log.Fatalf("Failed to format the AST: %v", err) - } - - if err := ioutil.WriteFile(filename, buf.Bytes(), 0644); err != nil { - log.Fatalf("Failed to write the modified file: %v", err) - } - - log.Printf("File %s updated successfully.", filename) -} - -func main() { - talosVersion := flag.String("talos-version", "main", "the desired Talos version (branch or tag)") - flag.Parse() - url := fmt.Sprintf("https://github.com/siderolabs/talos/raw/%s/cmd/talosctl/cmd/talos/", *talosVersion) - - args := flag.Args() - if len(args) == 0 { - fmt.Println("Please provide commands to import") - return - } - - for _, cmd := range args { - srcName := cmd + ".go" - dstName := "pkg/commands/imported_" + srcName - - err := downloadFile(srcName, dstName, url) - if err != nil { - log.Fatalf("Error downloading file: %v", err) - } - - log.Printf("File %s succefully downloaded to %s", srcName, dstName) - processFile(dstName) - } -} - -func downloadFile(srcName, dstName string, url string) error { - resp, err := http.Get(url + "/" + srcName) - if err != nil { - return err - } - defer resp.Body.Close() - - file, err := os.Create(dstName) - if err != nil { - return err - } - defer file.Close() - - _, err = io.Copy(file, resp.Body) - if err != nil { - return err - } - - return nil -}