feat(k8s): mvp

DarkflameUniverse · Jan 2, 2024 · 44f7dcd · 44f7dcd
1 parent 1941679
commit 44f7dcd
Show file tree

Hide file tree

Showing 12 changed files with 802 additions and 4 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -20,3 +20,6 @@
 [submodule "thirdparty/magic_enum"]
 	path = thirdparty/magic_enum
 	url = https://github.com/Neargye/magic_enum.git
+[submodule "thirdparty/kubernetes-client-c"]
+	path = thirdparty/kubernetes-client-c
+	url = https://github.com/kubernetes-client/c
diff --git a/dMasterServer/CMakeLists.txt b/dMasterServer/CMakeLists.txt
@@ -1,16 +1,26 @@
 set(DMASTERSERVER_SOURCES
 	"InstanceManager.cpp"
 	"ObjectIDManager.cpp"
-	"Start.cpp"
 )
 
 add_library(dMasterServer ${DMASTERSERVER_SOURCES})
-add_executable(MasterServer "MasterServer.cpp")
-add_compile_definitions(MasterServer PRIVATE PROJECT_VERSION="\"${PROJECT_VERSION}\"")
-
 target_link_libraries(dMasterServer ${COMMON_LIBRARIES})
+
+add_executable(MasterServer "MasterServer.cpp" "Start.cpp")
+add_compile_definitions(MasterServer PRIVATE PROJECT_VERSION="\"${PROJECT_VERSION}\"")
 target_link_libraries(MasterServer ${COMMON_LIBRARIES} dMasterServer)
 
+option(ENABLE_K8S "Whether to enable kubernetes support")
+
+if (${ENABLE_K8S})
+	# See k8s folder
+	find_package(kubernetes CONFIG REQUIRED COMPONENTS kubernetes)
+
+	add_executable(MasterServerK8s "MasterServer.cpp" "StartK8s.cpp")
+	add_compile_definitions(MasterServerK8s PRIVATE PROJECT_VERSION="\"${PROJECT_VERSION}\"")
+	target_link_libraries(MasterServerK8s PRIVATE ${COMMON_LIBRARIES} dMasterServer kubernetes::kubernetes)
+endif()
+
 if(WIN32)
 	add_dependencies(MasterServer WorldServer AuthServer ChatServer)
 endif()
diff --git a/dMasterServer/StartK8s.cpp b/dMasterServer/StartK8s.cpp
@@ -0,0 +1,355 @@
+#include "Start.h"
+#include "Logger.h"
+
+extern "C" {
+#include <kubernetes/config/kube_config.h>
+#include <kubernetes/config/incluster_config.h>
+#include <kubernetes/include/apiClient.h>
+#include <kubernetes/model/v1_daemon_set_spec.h>
+#include <kubernetes/api/CoreV1API.h>
+#include <kubernetes/api/AppsV1API.h>
+}
+
+#include <cstdlib>
+#include <cstdio>
+#include <cerrno>
+
+namespace k8s {
+	using DaemonSet = std::unique_ptr<v1_daemon_set_t, decltype(v1_daemon_set_free)&>;
+	using Deployment = std::unique_ptr<v1_deployment_t, decltype(v1_deployment_free)&>;
+	using PodSpec = std::unique_ptr<v1_pod_spec_t, decltype(v1_pod_spec_free)&>;
+	using Pod = std::unique_ptr<v1_pod_t, decltype(v1_pod_free)&>;
+}
+
+class ApiClient {
+	char* basePath = NULL;
+	sslConfig_t* sslConfig = NULL;
+	list_t* apiKeys = NULL;
+	apiClient_t* k8sApiClient = NULL;
+
+public:
+	ApiClient();
+	~ApiClient();
+
+	k8s::DaemonSet CreateOrReplaceDeamonSet(const std::string& namespace_, k8s::DaemonSet& daemonset);
+	k8s::Deployment CreateOrReplaceDeployment(const std::string& namespace_, k8s::Deployment& deployment);
+	k8s::Pod SpawnPod(const std::string& namespace_, k8s::Pod& pod);
+};
+
+k8s::PodSpec CreatePodSpec(const std::string& entrypoint, int container_port, int host_port, const std::vector<std::string>& args) {
+	v1_pod_spec_t* pod_spec = (v1_pod_spec_t*)calloc(1, sizeof(v1_pod_spec_t));
+	pod_spec->restart_policy = strdup("OnFailure");
+
+	/* set containers for pod */
+	list_t* containerlist = list_createList();
+	v1_container_t* con = (v1_container_t*)calloc(1, sizeof(v1_container_t));
+	con->name = strdup("darkflame-server");
+	con->image = strdup("darkflame-server");
+	con->image_pull_policy = strdup("IfNotPresent");
+
+	/* set command for container */
+	list_t* commandlist = list_createList();
+	char* cmd = strdup(entrypoint.c_str());
+	list_addElement(commandlist, cmd);
+	con->command = commandlist;
+
+	if (!args.empty()) {
+		list_t *arglist = list_createList();
+		for (auto it = args.begin(); it != args.end(); it++) {
+			char *arg1 = strdup(it->c_str());
+			list_addElement(arglist, arg1);
+		}
+		con->args = arglist;
+	}
+
+	list_t* envfromlist = list_createList();
+	char* secretName = strdup("db-secret");
+	char* envPrefix = strdup("MYSQL_");
+	v1_secret_env_source_t* secret = v1_secret_env_source_create(secretName, 0);
+	v1_env_from_source_t* src = v1_env_from_source_create(NULL, envPrefix, secret);
+	list_addElement(envfromlist, src);
+	con->env_from = envfromlist;
+
+	list_t* envlist = list_createList();
+	list_addElement(envlist, v1_env_var_create(strdup("MYSQL_HOST"), strdup("mariadb-service"), NULL));
+	list_addElement(envlist, v1_env_var_create(strdup("MYSQL_USERNAME"), strdup("darkflame"), NULL));
+	list_addElement(envlist, v1_env_var_create(strdup("DLU_CONFIG_DIR"), strdup("/app/config"), NULL));
+	con->env = envlist;
+
+	list_t* portlist = list_createList();
+	list_addElement(portlist, v1_container_port_create(container_port, NULL, host_port, strdup("rakpeer"), strdup("UDP")));
+	con->ports = portlist;
+
+	/* set volume mounts for container  */
+	list_t* volumemounts = list_createList();
+
+	v1_volume_mount_t* volmou = (v1_volume_mount_t*)calloc(1, sizeof(v1_volume_mount_t));
+	volmou->mount_path = strdup("/app/config");
+	volmou->name = strdup("game-config");
+	volmou->read_only = 1;
+	list_addElement(volumemounts, volmou);
+
+	v1_volume_mount_t* volmou1 = (v1_volume_mount_t*)calloc(1, sizeof(v1_volume_mount_t));
+	volmou1->mount_path = strdup("/app/vanity");
+	volmou1->name = strdup("vanity");
+	volmou1->read_only = 1;
+	list_addElement(volumemounts, volmou1);
+
+	v1_volume_mount_t* volmou2 = (v1_volume_mount_t*)calloc(1, sizeof(v1_volume_mount_t));
+	volmou2->mount_path = strdup("/app/res");
+	volmou2->name = strdup("luclient");
+	volmou2->read_only = 1;
+	list_addElement(volumemounts, volmou2);
+
+	v1_volume_mount_t* volmou3 = (v1_volume_mount_t*)calloc(1, sizeof(v1_volume_mount_t));
+	volmou3->mount_path = strdup("/app/resServer");
+	volmou3->name = strdup("res-server");
+	volmou3->read_only = 1;
+	list_addElement(volumemounts, volmou3);
+
+	con->volume_mounts = volumemounts;
+
+	list_addElement(containerlist, con);
+	pod_spec->containers = containerlist;
+
+	/* set volumes for pod */
+	list_t* volumelist = list_createList();
+
+	v1_volume_t* volume = (v1_volume_t*)calloc(1, sizeof(v1_volume_t));
+	volume->name = strdup("game-config");
+	v1_config_map_volume_source_t* config_map = (v1_config_map_volume_source_t*)calloc(1, sizeof(v1_config_map_volume_source_t));
+	config_map->name = strdup("game-config");
+	volume->config_map = config_map;
+	list_addElement(volumelist, volume);
+
+	v1_volume_t* volume1 = (v1_volume_t*)calloc(1, sizeof(v1_volume_t));
+	volume1->name = strdup("vanity");
+	v1_config_map_volume_source_t* config_map1 = (v1_config_map_volume_source_t*)calloc(1, sizeof(v1_config_map_volume_source_t));
+	config_map1->name = strdup("vanity");
+	volume1->config_map = config_map1;
+	list_addElement(volumelist, volume1);
+
+	v1_volume_t* volume2 = (v1_volume_t*)calloc(1, sizeof(v1_volume_t));
+	volume2->name = strdup("luclient");
+	v1_host_path_volume_source_t* host_path = (v1_host_path_volume_source_t*)calloc(1, sizeof(v1_host_path_volume_source_t));
+	host_path->path = strdup("/luclient");
+	volume2->host_path = host_path;
+	list_addElement(volumelist, volume2);
+
+	v1_volume_t* volume3 = (v1_volume_t*)calloc(1, sizeof(v1_volume_t));
+	volume3->name = strdup("res-server");
+	v1_host_path_volume_source_t* host_path2 = (v1_host_path_volume_source_t*)calloc(1, sizeof(v1_host_path_volume_source_t));
+	host_path2->path = strdup("/resServer");
+	volume3->host_path = host_path2;
+	list_addElement(volumelist, volume3);
+
+	pod_spec->volumes = volumelist;
+
+	return std::move(k8s::PodSpec(pod_spec, v1_pod_spec_free));
+}
+
+k8s::DaemonSet CreateDeamonSet(const std::string& name, const std::string& app_label, k8s::PodSpec podSpec) {
+	v1_daemon_set_t* daemonsetinfo = (v1_daemon_set_t*)calloc(1, sizeof(v1_daemon_set_t));
+	daemonsetinfo->api_version = strdup("apps/v1");
+	daemonsetinfo->kind = strdup("DaemonSet");
+
+	daemonsetinfo->metadata = (v1_object_meta_t*)calloc(1, sizeof(v1_object_meta_t));
+	daemonsetinfo->metadata->name = strdup(name.c_str());
+
+	daemonsetinfo->spec = (v1_daemon_set_spec_t*)calloc(1, sizeof(v1_daemon_set_spec_t));
+	list_t* match_labels = list_createList();
+	list_addElement(match_labels, keyValuePair_create(strdup("app.kubernetes.io/name"), strdup(app_label.c_str())));
+	daemonsetinfo->spec->selector = (v1_label_selector_t*)calloc(1, sizeof(v1_label_selector_t));
+	daemonsetinfo->spec->selector->match_labels = match_labels;
+
+	v1_pod_template_spec_t* pod_template_spec = (v1_pod_template_spec_t*)calloc(1, sizeof(v1_pod_template_spec_t));
+	pod_template_spec->metadata = (v1_object_meta_t*)calloc(1, sizeof(v1_object_meta_t));
+	list_t* labels = list_createList();
+	list_addElement(labels, keyValuePair_create(strdup("app.kubernetes.io/name"), strdup(app_label.c_str())));
+	pod_template_spec->metadata->labels = labels;
+	pod_template_spec->spec = podSpec.release();
+
+	daemonsetinfo->spec->_template = pod_template_spec;
+	return std::move(k8s::DaemonSet(daemonsetinfo, v1_daemon_set_free));
+}
+
+k8s::Deployment CreateDeployment(const std::string& name, const std::string& app_label, k8s::PodSpec podSpec) {
+	v1_deployment_t* deployment = (v1_deployment_t*)calloc(1, sizeof(v1_deployment_t));
+	deployment->api_version = strdup("apps/v1");
+	deployment->kind = strdup("Deployment");
+
+	deployment->metadata = (v1_object_meta_t*)calloc(1, sizeof(v1_object_meta_t));
+	deployment->metadata->name = strdup(name.c_str());
+
+	deployment->spec = (v1_deployment_spec_t*)calloc(1, sizeof(v1_deployment_spec_t));
+	deployment->spec->replicas = 1;
+
+	list_t* match_labels = list_createList();
+	list_addElement(match_labels, keyValuePair_create(strdup("app.kubernetes.io/name"), strdup(app_label.c_str())));
+	deployment->spec->selector = (v1_label_selector_t*)calloc(1, sizeof(v1_label_selector_t));
+	deployment->spec->selector->match_labels = match_labels;
+
+	v1_pod_template_spec_t* pod_template_spec = (v1_pod_template_spec_t*)calloc(1, sizeof(v1_pod_template_spec_t));
+	pod_template_spec->metadata = (v1_object_meta_t*)calloc(1, sizeof(v1_object_meta_t));
+	list_t* labels = list_createList();
+	list_addElement(labels, keyValuePair_create(strdup("app.kubernetes.io/name"), strdup(app_label.c_str())));
+	pod_template_spec->metadata->labels = labels;
+	pod_template_spec->spec = podSpec.release();
+
+	deployment->spec->_template = pod_template_spec;
+	return std::move(k8s::Deployment(deployment, v1_deployment_free));
+}
+
+char shiftNumbers(char c) {
+	return c < 'a' ? c - ('a' - 10 - '0') : c;
+}
+
+k8s::Pod CreatePod(const std::string& name, const std::string& app_label, k8s::PodSpec podSpec) {
+	v1_pod_t* pod = (v1_pod_t*)calloc(1, sizeof(v1_pod_t));
+	pod->api_version = strdup("v1");
+	pod->kind = strdup("Pod");
+
+	std::uniform_int_distribution<char> distribution('a' - 10, 'z');
+	std::string pod_name = name + '-';
+	for (int i = 0; i < 5; i++) {
+		pod_name += shiftNumbers(distribution(Game::randomEngine));
+	}
+
+	pod->metadata = (v1_object_meta_t*)calloc(1, sizeof(v1_object_meta_t));
+	pod->metadata->name = strdup(pod_name.c_str());
+
+	list_t* labels = list_createList();
+	list_addElement(labels, keyValuePair_create(strdup("app.kubernetes.io/name"), strdup(app_label.c_str())));
+	pod->metadata->labels = labels;
+
+	pod->spec = podSpec.release();
+	return std::move(k8s::Pod(pod, v1_pod_free));
+}
+
+ApiClient::ApiClient() {
+	int rc = 0;
+	rc = load_incluster_config(&basePath, &sslConfig, &apiKeys);
+	if (0 == rc) {
+		k8sApiClient = apiClient_create_with_base_path(basePath, sslConfig, apiKeys);
+	} else {
+		throw std::runtime_error("Cannot load kubernetes configuration in cluster.");
+	}
+}
+
+k8s::DaemonSet ApiClient::CreateOrReplaceDeamonSet(const std::string& namespace_, k8s::DaemonSet& daemonset) {
+	char* podNamespace = strdup(namespace_.c_str());
+
+	v1_daemon_set_t* created;
+	v1_daemon_set_t* existing = AppsV1API_readNamespacedDaemonSet(k8sApiClient, daemonset->metadata->name, podNamespace, NULL);
+	LOG("read code=%ld", k8sApiClient->response_code);
+	if (k8sApiClient->response_code == 200) {
+		daemonset->metadata->resource_version = existing->metadata->resource_version;
+		created = AppsV1API_replaceNamespacedDaemonSet(k8sApiClient, daemonset->metadata->name, podNamespace, daemonset.get(), NULL, NULL, NULL, NULL);
+	} else {
+		created = AppsV1API_createNamespacedDaemonSet(k8sApiClient, podNamespace, daemonset.get(), NULL, NULL, NULL, NULL);
+	}
+	LOG("code=%ld", k8sApiClient->response_code);
+
+	free(podNamespace);
+	v1_daemon_set_free(existing);
+	return k8s::DaemonSet(created, v1_daemon_set_free);
+}
+
+k8s::Pod ApiClient::SpawnPod(const std::string& namespace_, k8s::Pod& pod) {
+	char* podNamespace = strdup(namespace_.c_str());
+
+	// Kill the pod (if it exists)
+	//int grace_period_seconds = 0;
+	//v1_pod_t* existing = CoreV1API_deleteNamespacedPod(k8sApiClient, pod->metadata->name, podNamespace, NULL, NULL, &grace_period_seconds, NULL, NULL, NULL);
+	//LOG("read code=%ld", k8sApiClient->response_code);
+	// Spawn a new pod
+	v1_pod_t* created = CoreV1API_createNamespacedPod(k8sApiClient, podNamespace, pod.get(), NULL, NULL, NULL, NULL);
+	LOG("code=%ld", k8sApiClient->response_code);
+
+	free(podNamespace);
+	//v1_pod_free(existing);
+	return k8s::Pod(created, v1_pod_free);
+}
+
+k8s::Deployment ApiClient::CreateOrReplaceDeployment(const std::string& namespace_, k8s::Deployment& deployment) {
+	char* podNamespace = strdup(namespace_.c_str());
+
+	v1_deployment_t* created;
+	v1_deployment_t* existing = AppsV1API_readNamespacedDeployment(k8sApiClient, deployment->metadata->name, podNamespace, NULL);
+	LOG("read code=%ld", k8sApiClient->response_code);
+	if (k8sApiClient->response_code == 200) {
+		deployment->metadata->resource_version = existing->metadata->resource_version;
+		created = AppsV1API_replaceNamespacedDeployment(k8sApiClient, deployment->metadata->name, podNamespace, deployment.get(), NULL, NULL, NULL, NULL);
+	} else {
+		created = AppsV1API_createNamespacedDeployment(k8sApiClient, podNamespace, deployment.get(), NULL, NULL, NULL, NULL);
+	}
+	LOG("code=%ld", k8sApiClient->response_code);
+
+	free(podNamespace);
+	v1_deployment_free(existing);
+	return k8s::Deployment(created, v1_deployment_free);
+}
+
+ApiClient::~ApiClient() {
+	free_client_config(basePath, sslConfig, apiKeys);
+	basePath = NULL;
+	sslConfig = NULL;
+	apiKeys = NULL;
+
+	apiClient_free(k8sApiClient);
+	k8sApiClient = NULL;
+	apiClient_unsetupGlobalEnv();
+}
+
+void StartAuthServer() {
+	if (Game::ShouldShutdown()) {
+		LOG("Currently shutting down.  Auth will not be restarted.");
+		return;
+	}
+	LOG("Starting AuthServer");
+	ApiClient apiClient;
+	k8s::PodSpec podSpec = CreatePodSpec("/app/AuthServer", 1001, 1001, {});
+	//k8s::DaemonSet daemonset = CreateDeamonSet("auth-servers", "auth-server", podSpec);
+	k8s::Pod pod = CreatePod("auth-server-pod", "auth-server", std::move(podSpec));
+	apiClient.SpawnPod("default", pod);
+	LOG("Done starting AuthServer");
+}
+
+void StartChatServer() {
+	if (Game::ShouldShutdown()) {
+		LOG("Currently shutting down.  Chat will not be restarted.");
+		return;
+	}
+	LOG("Starting ChatServer");
+	ApiClient apiClient;
+	k8s::PodSpec podSpec = CreatePodSpec("/app/ChatServer", 2005, 0, {});
+	//k8s::Deployment deployment = CreateDeployment("chat-server", "chat-server", podSpec);
+	k8s::Pod pod = CreatePod("chat-server-pod", "chat-server", std::move(podSpec));
+	apiClient.SpawnPod("default", pod);
+	LOG("Done starting ChatServer");
+}
+
+void StartWorldServer(LWOMAPID mapID, uint16_t port, LWOINSTANCEID instanceID, int maxPlayers, LWOCLONEID cloneID) {
+	LOG("Starting WorldServer %d %d %d (port %d)", mapID, instanceID, cloneID, port);
+
+	std::vector<std::string> args = {
+		"-zone",
+		std::to_string(mapID),
+		"-port",
+		std::to_string(port),
+		"-instance",
+		std::to_string(instanceID),
+		"-maxclients",
+		std::to_string(maxPlayers),
+		"-clone",
+		std::to_string(cloneID),
+	};
+
+	ApiClient apiClient;
+	int host_port = (int) port; // for now
+	k8s::PodSpec podSpec = CreatePodSpec("/app/WorldServer", (int) port, host_port, args);
+	std::string name = "world-" + std::to_string(mapID) + "-" + std::to_string(instanceID) + "-" + std::to_string(cloneID);
+	k8s::Pod pod = CreatePod(name, "world-server", std::move(podSpec));
+	apiClient.SpawnPod("default", pod);
+	LOG("Done starting WorldServer");
+}