fix(libsinp): enable multi cri caching + tests

Signed-off-by: Roberto Scolaro <[email protected]>

test/libsinsp_e2e/container/container_cri.cpp

@@ -553,3 +553,120 @@ TEST_F(container_cri, fake_cri_fail_sync) {
 	                     exp_info,
 	                     1 << CT_CONTAINERD);
 }
+
+TEST_F(container_cri, fake_cri_multiple) {
+	auto pb_prefix = LIBSINSP_TEST_RESOURCES_PATH "/fake_cri_falco";
+	auto alt_pb_prefix = LIBSINSP_TEST_RESOURCES_PATH "/fake_cri_multi";
+	std::atomic<bool> done(false);
+	std::atomic<bool> done_alt(false);
+	const std::string alt_cri_container_id = "593f5b76be2a";
+
+	auto runtime = "containerd";
+
+	unlink(fake_cri_socket.c_str());
+	subprocess fake_cri_handle(LIBSINSP_TEST_PATH "/fake_cri/fake_cri",
+	                           {"unix://" + fake_cri_socket, pb_prefix, runtime});
+	pid_t fake_cri_pid = fake_cri_handle.get_pid();
+
+	std::string alt_fake_cri_socket = "/tmp/alt_fake_cri.sock";
+
+	unlink(alt_fake_cri_socket.c_str());
+	subprocess alt_fake_cri_handle(
+	        LIBSINSP_TEST_PATH "/fake_cri/fake_cri",
+	        {"unix://" + alt_fake_cri_socket, alt_pb_prefix, runtime, alt_cri_container_id});
+	pid_t alt_fake_cri_pid = alt_fake_cri_handle.get_pid();
+
+	auto start_time = time(NULL);
+
+	event_filter_t filter = [&](sinsp_evt* evt) {
+		return evt->get_type() == PPME_CONTAINER_JSON_E ||
+		       evt->get_type() == PPME_CONTAINER_JSON_2_E;
+	};
+
+	run_callback_t test = [&](sinsp* inspector) {
+		subprocess handle(LIBSINSP_TEST_PATH "/test_helper", {"cri_container_echo"});
+		handle.in() << "\n";
+		handle.wait();
+		subprocess alt_handle(LIBSINSP_TEST_PATH "/test_helper",
+		                      {"cri_container_echo",
+		                       "593f5b76be2afc23c39aa7eaa29174eac353d32be5e006b710c01aacca4aa05e"});
+		alt_handle.in() << "\n";
+		alt_handle.wait();
+		while(!done && !done_alt && time(NULL) < start_time + 10) {
+			usleep(100000);
+		}
+	};
+
+	captured_event_callback_t cri_callback = [&](const callback_param& param) {
+		sinsp_threadinfo* tinfo = param.m_evt->get_tinfo();
+		EXPECT_TRUE(tinfo != NULL);
+
+		if(tinfo->m_container_id == cri_container_id) {
+			EXPECT_EQ(cri_container_id, tinfo->m_container_id);
+
+			const auto container_info =
+			        param.m_inspector->m_container_manager.get_container(tinfo->m_container_id);
+			EXPECT_NE(container_info, nullptr);
+
+			EXPECT_EQ(sinsp_container_type::CT_CONTAINERD, container_info->m_type);
+			EXPECT_EQ("falco", container_info->m_name);
+			EXPECT_EQ("docker.io/falcosecurity/falco:latest", container_info->m_image);
+			EXPECT_EQ("sha256:8d0619a4da278dfe2772f75aa3cc74df0a250385de56085766035db5c9a062ed",
+			          container_info->m_imagedigest);
+			EXPECT_EQ("4bc0e14060f4263acf658387e76715bd836a13b9ba44f48465bd0633a412dbd0",
+			          container_info->m_imageid);
+			EXPECT_EQ(1073741824, container_info->m_memory_limit);
+			EXPECT_EQ(102, container_info->m_cpu_shares);
+			EXPECT_EQ(0, container_info->m_cpu_quota);
+			EXPECT_EQ(100000, container_info->m_cpu_period);
+			done = true;
+		} else {
+			EXPECT_EQ(alt_cri_container_id, tinfo->m_container_id);
+
+			const auto container_info =
+			        param.m_inspector->m_container_manager.get_container(tinfo->m_container_id);
+			EXPECT_NE(container_info, nullptr);
+
+			EXPECT_EQ(sinsp_container_type::CT_CONTAINERD, container_info->m_type);
+			EXPECT_EQ("falco-2", container_info->m_name);
+			EXPECT_EQ("docker.io/falcosecurity/falco:latest", container_info->m_image);
+			EXPECT_EQ("sha256:4df3aba7463d88aefbab4eb9e241468b0475f5e8c2c138d4cd811ca812975612",
+			          container_info->m_imagedigest);
+			EXPECT_EQ("74d48ff156776f5fc1c625d72163eb539e63967bc87baf9158cdaca218c39465",
+			          container_info->m_imageid);
+			EXPECT_EQ(1073741824, container_info->m_memory_limit);
+			EXPECT_EQ(102, container_info->m_cpu_shares);
+			EXPECT_EQ(0, container_info->m_cpu_quota);
+			EXPECT_EQ(100000, container_info->m_cpu_period);
+			done_alt = true;
+		}
+	};
+
+	before_capture_t setup = [&](sinsp* inspector) {
+		inspector->set_docker_socket_path("");
+		inspector->set_cri_socket_path(fake_cri_socket);
+		inspector->add_cri_socket_path(alt_fake_cri_socket);
+		inspector->set_cri_extra_queries(true);
+	};
+
+	after_capture_t cleanup = [&](sinsp* inspector) {
+		inspector->set_docker_socket_path(default_docker_socket);
+	};
+
+	EXPECT_NO_FATAL_FAILURE({ event_capture::run(test, cri_callback, filter, setup, cleanup); });
+
+	// The fake server had to stay running the whole time in order
+	// for the test to be succesful
+	// Needed to reap the zombine if it exited
+	waitpid(fake_cri_pid, NULL, WNOHANG);
+	EXPECT_TRUE(fake_cri_handle.is_alive());
+
+	waitpid(alt_fake_cri_pid, NULL, WNOHANG);
+	EXPECT_TRUE(alt_fake_cri_handle.is_alive());
+
+	EXPECT_TRUE(done);
+	EXPECT_TRUE(done_alt);
+
+	fake_cri_handle.kill();
+	alt_fake_cri_handle.kill();
+}

test/libsinsp_e2e/fake_cri/fake_cri.cpp

@@ -20,22 +20,27 @@ class FakeCRIServer final : public runtime::v1alpha2::RuntimeService::Service {
 	              ContainerStatusResponse&& cs,
 	              PodSandboxStatusResponse&& ps,
 	              ListContainersResponse&& lc,
-	              const std::string& runtime_name):
+	              const std::string& runtime_name,
+	              const std::string& filter):
 	        m_delay_us(delay_us),
 	        m_container_status_response(cs),
 	        m_pod_sandbox_status_response(ps),
 	        m_list_containers_response(lc),
-	        m_runtime_name(runtime_name) {}
+	        m_runtime_name(runtime_name),
+	        m_filter(filter) {}
 
 	grpc::Status ContainerStatus(grpc::ServerContext* context,
 	                             const ContainerStatusRequest* req,
 	                             ContainerStatusResponse* resp) {
 		usleep(m_delay_us);
+
 		if(CONTAINER_IDS.find(req->container_id()) == CONTAINER_IDS.end()) {
-			std::cout << "CONTAINER NOT FOUND\n";
-			return grpc::Status(
-			        grpc::StatusCode::NOT_FOUND,
-			        "fake_cri does not serve this container id: " + req->container_id());
+			if(m_filter.empty() || (!m_filter.empty() && req->container_id().find(m_filter) != 0)) {
+				std::cout << "CONTAINER NOT FOUND\n";
+				return grpc::Status(
+				        grpc::StatusCode::NOT_FOUND,
+				        "fake_cri does not serve this container id: " + req->container_id());
+			}
 		}
 		resp->CopyFrom(m_container_status_response);
 		resp->mutable_status()->set_id(req->container_id());
@@ -87,6 +92,7 @@ class FakeCRIServer final : public runtime::v1alpha2::RuntimeService::Service {
 	PodSandboxStatusResponse m_pod_sandbox_status_response;
 	ListContainersResponse m_list_containers_response;
 	std::string m_runtime_name;
+	std::string m_filter;
 	static const std::set<std::string> CONTAINER_IDS;
 	static const std::set<std::string> POD_SANDBOX_IDS;
 };
@@ -124,7 +130,7 @@ int main(int argc, char** argv) {
 	if(argc < 3) {
 		fprintf(stderr,
 		        "Usage: fake_cri [--nodelay|--slow|--veryslow] listen_addr pb_file_prefix "
-		        "[runtime_name]\n");
+		        "[runtime_name] [container_id filter]\n");
 		return 1;
 	}
 
@@ -145,6 +151,7 @@ int main(int argc, char** argv) {
 	const char* addr = argv[1];
 	const std::string pb_prefix(argv[2]);
 	const std::string runtime(argc > 3 ? argv[3] : "containerd");
+	const std::string filter(argc > 4 ? argv[4] : "");
 
 	ContainerStatusResponse cs;
 	{
@@ -198,7 +205,7 @@ int main(int argc, char** argv) {
 		}
 	}
 
-	FakeCRIServer service(delay_us, std::move(cs), std::move(ps), std::move(lc), runtime);
+	FakeCRIServer service(delay_us, std::move(cs), std::move(ps), std::move(lc), runtime, filter);
 	FakeCRIImageServer image_service(std::move(is));
 
 	grpc::ServerBuilder builder;

test/libsinsp_e2e/fake_cri/fake_cri_multi_images.pb

@@ -0,0 +1,6 @@
+images {
+    id: "74d48ff156776f5fc1c625d72163eb539e63967bc87baf9158cdaca218c39465"
+    repo_tags: "docker.io/falcosecurity/falco:latest"
+    repo_digests: "docker.io/falcosecurity/falco@sha256:4df3aba7463d88aefbab4eb9e241468b0475f5e8c2c138d4cd811ca812975612"
+    size: 1402153176
+}

test/libsinsp_e2e/fake_cri/fake_cri_multi_listcontainers.pb

@@ -0,0 +1,50 @@
+containers {
+id: "593f5b76be2afc23c39aa7eaa29174eac353d32be5e006b710c01aacca4aa05e"
+pod_sandbox_id: "599ad631db94fef0be7722785e299ba128bd3f7f83a27dd00e4f94974eb5acfa"
+metadata {
+    name: "falco-2"
+    attempt: 0
+}
+state: CONTAINER_RUNNING
+created_at: 1545339739712670450
+image {
+    image: "docker.io/falcosecurity/falco:latest"
+}
+image_ref: "docker.io/falcosecurity/falco@sha256:4df3aba7463d88aefbab4eb9e241468b0475f5e8c2c138d4cd811ca812975612"
+labels {
+    key: "io.kubernetes.container.name"
+    value: "falco"
+}
+labels {
+    key: "io.kubernetes.pod.name"
+    value: "falco-9bzbj"
+}
+labels {
+    key: "io.kubernetes.pod.namespace"
+    value: "default"
+}
+labels {
+    key: "io.kubernetes.pod.uid"
+    value: "893231bb-049a-11e9-9b30-0a583e8b7896"
+}
+annotations {
+    key: "io.kubernetes.container.hash"
+    value: "decd134"
+}
+annotations {
+    key: "io.kubernetes.container.restartCount"
+    value: "0"
+}
+annotations {
+    key: "io.kubernetes.container.terminationMessagePath"
+    value: "/dev/termination-log"
+}
+annotations {
+    key: "io.kubernetes.container.terminationMessagePolicy"
+    value: "File"
+}
+annotations {
+    key: "io.kubernetes.pod.terminationGracePeriod"
+    value: "5"
+}
+}

test/libsinsp_e2e/fake_cri/fake_cri_multi_pod.pb

@@ -0,0 +1,58 @@
+status {
+metadata {
+    name: "falco-9bzbj"
+    uid: "893231bb-049a-11e9-9b30-0a583e8b7896"
+    namespace: "default",
+    attempt: 0
+}
+state: SANDBOX_READY
+created_at: 1545339738831266021
+network {
+    ip: ""
+}
+linux {
+namespaces {
+options {
+    network: NODE,
+    pid: NODE,
+    ipc: POD
+}
+}
+}
+labels {
+    key: "app"
+    value: "falco"
+}
+labels {
+    key: "controller-revision-hash"
+    value: "b5944cc84"
+}
+labels {
+    key: "io.kubernetes.pod.name"
+    value: "falco-9bzbj"
+}
+labels {
+    key: "io.kubernetes.pod.namespace"
+    value: "default"
+}
+labels {
+    key: "io.kubernetes.pod.uid"
+    value: "893231bb-049a-11e9-9b30-0a583e8b7896"
+}
+labels {
+    key: "pod-template-generation"
+    value: "1"
+}
+annotations {
+    key: "kubernetes.io/config.seen"
+    value: "2018-12-20T21:02:18.502551218Z"
+}
+annotations {
+    key: "kubernetes.io/config.source"
+    value: "api"
+}
+}
+info {
+    key: "info"
+    value: "{\"pid\":31353, \"processStatus\":\"running\", \"netNamespaceClosed\":false, \"image\":\"k8s.gcr.io/pause:3.1\", \"snapshotKey\":\"599ad631db94fef0be7722785e299ba128bd3f7f83a27dd00e4f94974eb5acfa\", \"snapshotter\":\"overlayfs\", \"runtime\":{\"runtimeType\":\"io.containerd.runtime.v1.linux\", \"runtimeEngine\":\"\", \"runtimeRoot\":\"\"}, \"config\":{\"metadata\":{\"name\":\"falco-9bzbj\", \"uid\":\"893231bb-049a-11e9-9b30-0a583e8b7896\", \"namespace\":\"default\"}, \"log_directory\":\"/var/log/pods/893231bb-049a-11e9-9b30-0a583e8b7896\", \"dns_config\":{\"servers\":[\"10.96.0.10\"], \"searches\":[\"default.svc.cluster.local\", \"svc.cluster.local\", \"cluster.local\", \"us-east-2.compute.internal\"], \"options\":[\"ndots:5\"]}, \"labels\":{\"app\":\"falco\", \"controller-revision-hash\":\"b5944cc84\", \"io.kubernetes.pod.name\":\"falco-9bzbj\", \"io.kubernetes.pod.namespace\":\"default\", \"io.kubernetes.pod.uid\":\"893231bb-049a-11e9-9b30-0a583e8b7896\", \"pod-template-generation\":\"1\"}, \"annotations\":{\"kubernetes.io/config.seen\":\"2018-12-20T21:02:18.502551218Z\", \"kubernetes.io/config.source\":\"api\"}, \"linux\":{\"cgroup_parent\":\"/kubepods/burstable/pod893231bb-049a-11e9-9b30-0a583e8b7896\", \"security_context\":{\"namespace_options\":{\"network\":2, \"pid\":2}, \"privileged\":true}}}, \"runtimeSpec\":{\"ociVersion\":\"1.0.1\", \"process\":{\"user\":{\"uid\":0, \"gid\":0}, \"args\":[\"/pause\"], \"env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"], \"cwd\":\"/\", \"capabilities\":{\"bounding\":[\"CAP_CHOWN\", \"CAP_DAC_OVERRIDE\", \"CAP_FSETID\", \"CAP_FOWNER\", \"CAP_MKNOD\", \"CAP_NET_RAW\", \"CAP_SETGID\", \"CAP_SETUID\", \"CAP_SETFCAP\", \"CAP_SETPCAP\", \"CAP_NET_BIND_SERVICE\", \"CAP_SYS_CHROOT\", \"CAP_KILL\", \"CAP_AUDIT_WRITE\"], \"effective\":[\"CAP_CHOWN\", \"CAP_DAC_OVERRIDE\", \"CAP_FSETID\", \"CAP_FOWNER\", \"CAP_MKNOD\", \"CAP_NET_RAW\", \"CAP_SETGID\", \"CAP_SETUID\", \"CAP_SETFCAP\", \"CAP_SETPCAP\", \"CAP_NET_BIND_SERVICE\", \"CAP_SYS_CHROOT\", \"CAP_KILL\", \"CAP_AUDIT_WRITE\"], \"inheritable\":[\"CAP_CHOWN\", \"CAP_DAC_OVERRIDE\", \"CAP_FSETID\", \"CAP_FOWNER\", \"CAP_MKNOD\", \"CAP_NET_RAW\", \"CAP_SETGID\", \"CAP_SETUID\", \"CAP_SETFCAP\", \"CAP_SETPCAP\", \"CAP_NET_BIND_SERVICE\", \"CAP_SYS_CHROOT\", \"CAP_KILL\", \"CAP_AUDIT_WRITE\"], \"permitted\":[\"CAP_CHOWN\", \"CAP_DAC_OVERRIDE\", \"CAP_FSETID\", \"CAP_FOWNER\", \"CAP_MKNOD\", \"CAP_NET_RAW\", \"CAP_SETGID\", \"CAP_SETUID\", \"CAP_SETFCAP\", \"CAP_SETPCAP\", \"CAP_NET_BIND_SERVICE\", \"CAP_SYS_CHROOT\", \"CAP_KILL\", \"CAP_AUDIT_WRITE\"]}, \"noNewPrivileges\":true, \"oomScoreAdj\":-998}, \"root\":{\"path\":\"rootfs\", \"readonly\":true}, \"mounts\":[{\"destination\":\"/proc\", \"type\":\"proc\", \"source\":\"proc\"}, {\"destination\":\"/dev\", \"type\":\"tmpfs\", \"source\":\"tmpfs\", \"options\":[\"nosuid\", \"strictatime\", \"mode=755\", \"size=65536k\"]}, {\"destination\":\"/dev/pts\", \"type\":\"devpts\", \"source\":\"devpts\", \"options\":[\"nosuid\", \"noexec\", \"newinstance\", \"ptmxmode=0666\", \"mode=0620\", \"gid=5\"]}, {\"destination\":\"/dev/mqueue\", \"type\":\"mqueue\", \"source\":\"mqueue\", \"options\":[\"nosuid\", \"noexec\", \"nodev\"]}, {\"destination\":\"/sys\", \"type\":\"sysfs\", \"source\":\"sysfs\", \"options\":[\"nosuid\", \"noexec\", \"nodev\", \"ro\"]}, {\"destination\":\"/dev/shm\", \"type\":\"bind\", \"source\":\"/run/containerd/io.containerd.grpc.v1.cri/sandboxes/599ad631db94fef0be7722785e299ba128bd3f7f83a27dd00e4f94974eb5acfa/shm\", \"options\":[\"rbind\", \"ro\"]}], \"annotations\":{\"io.kubernetes.cri.container-type\":\"sandbox\", \"io.kubernetes.cri.sandbox-id\":\"599ad631db94fef0be7722785e299ba128bd3f7f83a27dd00e4f94974eb5acfa\"}, \"linux\":{\"resources\":{\"devices\":[{\"allow\":false, \"access\":\"rwm\"}], \"cpu\":{\"shares\":2}}, \"cgroupsPath\":\"/kubepods/burstable/pod893231bb-049a-11e9-9b30-0a583e8b7896/599ad631db94fef0be7722785e299ba128bd3f7f83a27dd00e4f94974eb5acfa\", \"namespaces\":[{\"type\":\"ipc\"}, {\"type\":\"uts\"}, {\"type\":\"mount\"}], \"maskedPaths\":[\"/proc/acpi\", \"/proc/kcore\", \"/proc/keys\", \"/proc/latency_stats\", \"/proc/timer_list\", \"/proc/timer_stats\", \"/proc/sched_debug\", \"/sys/firmware\", \"/proc/scsi\"], \"readonlyPaths\":[\"/proc/asound\", \"/proc/bus\", \"/proc/fs\", \"/proc/irq\", \"/proc/sys\", \"/proc/sysrq-trigger\"]}}}"
+}

test/libsinsp_e2e/test_helper.cpp

@@ -551,16 +551,12 @@ void custom_container(const vector<string>& args) {
 	}
 }
 
-bool cri_container_set_cgroup() {
+bool _cri_container_set_cgroup(const std::string& id) {
 	std::string cpu_cgroup;
 	if(is_cgroupv2_mounted()) {
-		cpu_cgroup =
-		        "/sys/fs/cgroup/system.slice/"
-		        "aec4c703604b4504df03108eef12e8256870eca8aabcb251855a35bf4f0337f1";
+		cpu_cgroup = "/sys/fs/cgroup/system.slice/" + id;
 	} else {
-		cpu_cgroup =
-		        "/sys/fs/cgroup/cpu/docker/"
-		        "aec4c703604b4504df03108eef12e8256870eca8aabcb251855a35bf4f0337f1";
+		cpu_cgroup = "/sys/fs/cgroup/cpu/docker/" + id;
 	}
 	struct stat s;
 
@@ -599,6 +595,11 @@ bool cri_container_set_cgroup() {
 	return true;
 }
 
+bool cri_container_set_cgroup() {
+	return _cri_container_set_cgroup(
+	        "aec4c703604b4504df03108eef12e8256870eca8aabcb251855a35bf4f0337f1");
+}
+
 void cri_container_simple(char* const exargs[]) {
 	signal(SIGCHLD, SIG_IGN);
 	pid_t pid = fork();
@@ -619,9 +620,15 @@ void cri_container_simple(char* const exargs[]) {
 	}
 }
 
-void cri_container_echo(const vector<string>& args) {
-	if(!cri_container_set_cgroup()) {
-		return;
+void cri_container_echo(const std::vector<std::string>& args) {
+	if(args.size() == 1) {
+		if(!_cri_container_set_cgroup(args.at(0))) {
+			return;
+		}
+	} else {
+		if(!cri_container_set_cgroup()) {
+			return;
+		}
 	}
 
 	char* const exargs[] = {(char*)"/bin/echo", (char*)"-n", nullptr};

userspace/libsinsp/container.cpp

@@ -142,11 +142,13 @@ bool sinsp_container_manager::resolve_container(sinsp_threadinfo* tinfo,
 		create_engines();
 	}
 
+	size_t i = 0;
 	for(auto& eng : m_container_engines) {
 		matches = matches || eng->resolve(tinfo, query_os_for_missing_info);
 		if(matches) {
 			break;
 		}
+		i++;
 	}
 
 	// Also possibly set the category for the threadinfo
@@ -581,12 +583,15 @@ void sinsp_container_manager::create_engines() {
 
 		const auto& cri_socket_paths = cri_settings.get_cri_unix_socket_paths();
 
-		for(const auto& socket_path : cri_socket_paths) {
-			auto cri_engine = std::make_shared<container_engine::cri>(*this, socket_path);
+		size_t engine_index = 0;
+		for(auto socket_path : cri_socket_paths) {
+			auto cri_engine =
+			        std::make_shared<container_engine::cri>(*this, socket_path, engine_index);
 			m_container_engines.push_back(cri_engine);
 			m_container_engine_by_type[CT_CRI].push_back(cri_engine);
 			m_container_engine_by_type[CT_CRIO].push_back(cri_engine);
 			m_container_engine_by_type[CT_CONTAINERD].push_back(cri_engine);
+			engine_index++;
 		}
 	}
 	if(m_container_engine_mask & (1 << CT_LXC)) {