Skip to content

Commit

Permalink
cleanup(libsinsp): add fallback to sandbox container in k8s filterchecks
Browse files Browse the repository at this point in the history 
Includes minor additional cleanups wrt to comments.

Signed-off-by: Melissa Kilby <[email protected]>
  • Loading branch information
@incertum
incertum committed Feb 11, 2024
1 parent bf5d6a8 commit a17eaba
Show file tree
Hide file tree
Showing 4 changed files with 132 additions and 39 deletions.
51 changes: 26 additions & 25 deletions userspace/libsinsp/cri.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,21 @@ template<class api> class cri_interface
*/
grpc::Status get_container_status_resp(const std::string &container_id, typename api::ContainerStatusResponse &resp);

/**
* @brief make request and get PodSandboxStatusResponse
* @param pod_sandbox_id ID of the pod sandbox
* @param resp initialized api::PodSandboxStatusResponse of the pod sandbox
* @return grpc::Status
*/
grpc::Status get_pod_sandbox_status_resp(const std::string &pod_sandbox_id, typename api::PodSandboxStatusResponse &resp);

/**
* @brief get image id info from CRI
* @param image_ref the image ref from container metadata
* @return image id if found, empty string otherwise
*/
std::string get_container_image_id(const std::string &image_ref);

/**
* @brief thin wrapper around CRI gRPC ContainerStats call
* @param container_id container ID
Expand Down Expand Up @@ -265,7 +280,7 @@ template<class api> class cri_interface
/**
* @brief fill out container image information based on CRI response
* @param status `status` field of the ContainerStatusResponse
* @param root Json::Value of status.info() at "info of the ContainerStatusResponse
* @param root Json::Value of status.info() at "info" of the ContainerStatusResponse
* @param container the container info to fill out
* @return true if successful
*/
Expand All @@ -283,7 +298,7 @@ template<class api> class cri_interface

/**
* @brief fill out container environment variables based on CRI response, valid for containerd only
* @param root Json::Value of status.info() at "info of the ContainerStatusResponse
* @param root Json::Value of status.info() at "info" of the ContainerStatusResponse
* @param container the container info to fill out
* @return true if successful
*
Expand All @@ -293,7 +308,7 @@ template<class api> class cri_interface

/**
* @brief fill out extra image info based on CRI response, valid for containerd only
* @param root Json::Value of status.info() at "info of the ContainerStatusResponse
* @param root Json::Value of status.info() at "info" of the ContainerStatusResponse
* @param container the container info to fill out
* @return true if successful
*
Expand All @@ -303,15 +318,15 @@ template<class api> class cri_interface

/**
* @brief fill out extra container info (e.g. resource limits) based on CRI response
* @param root Json::Value of status.info() at "info of the ContainerStatusResponse
* @param root Json::Value of status.info() at "info" of the ContainerStatusResponse
* @param container the container info to fill out
* @return true if successful
*/
bool parse_cri_ext_container_info(const Json::Value &root, sinsp_container_info &container);

/**
* @brief fill out extra container user info (e.g. configured uid) based on CRI response
* @param root Json::Value of status.info() at "info of the ContainerStatusResponse
* @param root Json::Value of status.info() at "info" of the ContainerStatusResponse
* @param container the container info to fill out
* @return true if successful
*
Expand All @@ -336,8 +351,8 @@ template<class api> class cri_interface
bool parse_cri_labels(const typename api::PodSandboxStatus &status, sinsp_container_info &container);

/**
* @brief fill out pod sandbox id
* @param root Json::Value of status.info() at "info of the ContainerStatusResponse
* @brief fill out pod sandbox id, only valid when used w/ ContainerStatusResponse, not PodSandboxStatusResponse
* @param root Json::Value of status.info() at "info" of the ContainerStatusResponse
* @param container the container info to fill out
* @return true if successful
*/
Expand All @@ -347,7 +362,7 @@ template<class api> class cri_interface
/**
* @brief fill out pod sandbox network info
* @param status `status` field of the PodSandboxStatusResponse
* @param root Json::Value of status.info() at "info of the PodSandboxStatusResponse
* @param root Json::Value of status.info() at "info" of the PodSandboxStatusResponse
* @param container the container info to fill out
* @return true if successful
*/
Expand All @@ -364,23 +379,9 @@ template<class api> class cri_interface
bool parse_cri_pod_sandbox_labels(const typename api::PodSandboxStatus &status, sinsp_container_info &container);

/**
* @brief make request and get PodSandboxStatusResponse
* @param pod_sandbox_id ID of the pod sandbox
* @param resp initialized api::PodSandboxStatusResponse of the pod sandbox
* @return grpc::Status
*/
grpc::Status get_pod_sandbox_status_resp(const std::string &pod_sandbox_id, typename api::PodSandboxStatusResponse &resp);

/**
* @brief get image id info from CRI
* @param image_ref the image ref from container metadata
* @return image id if found, empty string otherwise
*/
std::string get_container_image_id(const std::string &image_ref);

/**
* @brief fill in container metadata using the CRI API
* @param key the async lookup key (includes container_id)
* @brief fill in container metadata using the CRI API (`containerd` and `cri-o` container runtimes).
* This is the main CRI parser calling each parse_* helper after making the respective CRI API call(s).
* @param key includes container_id, but container.m_id is used to make the CRI API calls
* @param container the container info to fill
* @return true on success, false on failure
*/
Expand Down
27 changes: 22 additions & 5 deletions userspace/libsinsp/cri.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,7 @@ inline bool cri_interface<api>::parse_cri_base(const typename api::PodSandboxSta
// This is in Nanoseconds(in CRI API). Need to convert it to seconds.
container.m_created_time = static_cast<int64_t>(status.created_at() / ONE_SECOND_IN_NS);
container.m_pod_sandbox_id = container.m_full_id;
// Add the pod sandbox id as label to the container for backward compatibility
container.m_labels["io.kubernetes.sandbox.id"] = container.m_full_id;

return true;
Expand Down Expand Up @@ -231,9 +232,10 @@ inline bool cri_interface<api>::parse_cri_image(const typename api::ContainerSta

if(image_name.empty() || strncmp(image_name.c_str(), "sha256", 6) == 0)
{
/* Retrieve image_name from annotations as backup when image name may start with sha256
or otherwise was not retrieved. Brute force try each schema we know of for containerd and crio container
runtimes. */
/* Retrieve image_name from annotations as backup when image name may (still) start with sha256
* or otherwise was not successfully retrieved. Brute force try each schema we know of for containerd
* and cri-o container runtimes.
*/

if(!root.isNull())
{
Expand Down Expand Up @@ -405,8 +407,13 @@ inline bool cri_interface<api>::parse_cri_env(const Json::Value &root, sinsp_con
template<typename api>
inline bool cri_interface<api>::parse_cri_json_imageid_containerd(const Json::Value &root, sinsp_container_info &container)
{
if(root.isNull())
{
return false;
}

const Json::Value *image = nullptr;
if(!walk_down_json(info, &image, "config", "image", "image") || !image->isString())
if(!walk_down_json(root, &image, "config", "image", "image") || !image->isString())
{
return false;
}
Expand All @@ -428,6 +435,11 @@ inline bool cri_interface<api>::parse_cri_json_imageid_containerd(const Json::Va
template<typename api>
inline bool cri_interface<api>::parse_cri_ext_container_info(const Json::Value &root, sinsp_container_info &container)
{
if(root.isNull())
{
return false;
}

const Json::Value *linux = nullptr;
if(!walk_down_json(root, &linux, "runtimeSpec", "linux") || !linux->isObject())
{
Expand Down Expand Up @@ -480,6 +492,11 @@ inline bool cri_interface<api>::parse_cri_ext_container_info(const Json::Value &
template<typename api>
inline bool cri_interface<api>::parse_cri_user_info(const Json::Value &root, sinsp_container_info &container)
{
if(root.isNull())
{
return false;
}

const Json::Value *uid = nullptr;
if(!walk_down_json(root, &uid, "runtimeSpec", "process", "user", "uid") || !uid->isInt())
{
Expand Down Expand Up @@ -782,7 +799,7 @@ inline bool cri_interface<api>::parse(const libsinsp::cgroup_limits::cgroup_limi
* in k8s filterchecks. Now, we also store the pod sandbox labels in the container.
* While this might seem redundant in cases where multiple containers exist in a pod, considering that the concurrent
* number of containers on a node is typically capped at 100-300 and many pods contain only 1-3 containers,
* it doesn't add significant overhead. Moreover, these extra lookups have always been performed for container ips in the past
* it doesn't add significant overhead. Moreover, these extra lookups have always been performed for container ips in the past
* and therefore are no new additions.
*/
typename api::PodSandboxStatusResponse pod_sandbox_status_resp;
Expand Down
57 changes: 50 additions & 7 deletions userspace/libsinsp/sinsp_filtercheck_k8s.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,10 @@ uint8_t* sinsp_filter_check_k8s::extract(sinsp_evt *evt, OUT uint32_t* len, bool

m_tstr.clear();

// Note: All fields are retrieved from the CRI API calls aka as part of the container engine lookups.
// There is no interaction w/ the Kubernetes Server in any way to retrieve these fields. As alternative explore the new `k8smeta` plugin.
// Comments explain the origin of each field (either ContainerStatusResponse or PodSandboxStatusResponse CRI API call).

switch(m_field_id)
{
case TYPE_K8S_POD_NAME:
Expand Down Expand Up @@ -277,32 +281,71 @@ uint8_t* sinsp_filter_check_k8s::extract(sinsp_evt *evt, OUT uint32_t* len, bool
case TYPE_K8S_POD_LABEL:
case TYPE_K8S_POD_LABELS:
// Requires s_cri_extra_queries enabled, which is the default for Falco.
// Note that m_pod_sandbox_labels, while part of the container, is retrieved from an extra PodSandboxStatusResponse call, not the ContainerStatusResponse CRI API call.
// Note that m_pod_sandbox_labels, while part of the container struct, is retrieved from an extra PodSandboxStatusResponse call, not the ContainerStatusResponse CRI API call.
{
if (m_field_id == TYPE_K8S_POD_LABEL && container_info->m_pod_sandbox_labels.count(m_argname) > 0)
sinsp_container_info::ptr_t sandbox_container_info;
if(container_info->m_pod_sandbox_cniresult.empty()) // more robust check than checking for empty labels
{
// Fallback: Retrieve PodSandboxStatusResponse fields stored in explicit pod sandbox container
sandbox_container_info = m_inspector->m_container_manager.get_container(container_info->m_pod_sandbox_id.substr(0, 12));
}
if (m_field_id == TYPE_K8S_POD_LABEL)
{
m_tstr = container_info->m_pod_sandbox_labels.at(m_argname);
if(sandbox_container_info && sandbox_container_info->m_pod_sandbox_labels.count(m_argname) > 0) // fallback
{
m_tstr = sandbox_container_info->m_pod_sandbox_labels.at(m_argname);
}
else if (container_info->m_pod_sandbox_labels.count(m_argname) > 0)
{
m_tstr = container_info->m_pod_sandbox_labels.at(m_argname);
}
RETURN_EXTRACT_STRING(m_tstr);
}
else if (m_field_id == TYPE_K8S_POD_LABELS)
{
concatenate_container_labels(container_info->m_pod_sandbox_labels, &m_tstr);
if(sandbox_container_info) // fallback
{
concatenate_container_labels(sandbox_container_info->m_pod_sandbox_labels, &m_tstr);
} else
{
concatenate_container_labels(container_info->m_pod_sandbox_labels, &m_tstr);
}
RETURN_EXTRACT_STRING(m_tstr);
}
}
break;
case TYPE_K8S_POD_IP:
// Requires s_cri_extra_queries enabled, which is the default for Falco.
// Note that m_pod_sandbox_labels, while part of the container, is retrieved from an extra PodSandboxStatusResponse call, not the ContainerStatusResponse CRI API call.
m_u32val = htonl(container_info->m_container_ip);
// Note that m_pod_sandbox_labels, while part of the container struct, is retrieved from an extra PodSandboxStatusResponse call, not the ContainerStatusResponse CRI API call.
if(container_info->m_pod_sandbox_cniresult.empty()) // more robust check than checking for 0 in m_container_ip
{
// Fallback: Retrieve PodSandboxStatusResponse fields stored in pod sandbox container
const sinsp_container_info::ptr_t sandbox_container_info = m_inspector->m_container_manager.get_container(container_info->m_pod_sandbox_id.substr(0, 12));
if(sandbox_container_info)
{
m_u32val = htonl(sandbox_container_info->m_container_ip);
}
} else
{
m_u32val = htonl(container_info->m_container_ip);
}
char addrbuff[100];
inet_ntop(AF_INET, &m_u32val, addrbuff, sizeof(addrbuff));
m_tstr = addrbuff;
RETURN_EXTRACT_STRING(m_tstr);
break;
case TYPE_K8S_POD_CNIRESULT:
// Requires s_cri_extra_queries enabled, which is the default for Falco.
// Note that m_pod_sandbox_labels, while part of the container, is retrieved from an extra PodSandboxStatusResponse call, not the ContainerStatusResponse CRI API call.
// Note that m_pod_sandbox_labels, while part of the container struct, is retrieved from an extra PodSandboxStatusResponse call, not the ContainerStatusResponse CRI API call.
if(container_info->m_pod_sandbox_cniresult.empty())
{
// Fallback: Retrieve PodSandboxStatusResponse fields stored in pod sandbox container
const sinsp_container_info::ptr_t sandbox_container_info = m_inspector->m_container_manager.get_container(container_info->m_pod_sandbox_id.substr(0, 12));
if(sandbox_container_info)
{
RETURN_EXTRACT_STRING(sandbox_container_info->m_pod_sandbox_cniresult);
}
}
RETURN_EXTRACT_STRING(container_info->m_pod_sandbox_cniresult);
break;
default:
Expand Down
36 changes: 34 additions & 2 deletions userspace/libsinsp/test/container_engine/container_parser_cri_containerd.ut.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -689,6 +689,38 @@ TEST_F(sinsp_with_test_input, container_parser_cri_containerd)
ASSERT_EQ(get_field_as_string(evt, "k8s.pod.labels"), "app:myapp, example-label/custom_one:mylabel");
ASSERT_EQ(get_field_as_string(evt, "k8s.pod.ip"), "10.244.0.2");
ASSERT_EQ(get_field_as_string(evt, "k8s.pod.cni.json"), "{\"bridge\":{\"IPConfigs\":null},\"eth0\":{\"IPConfigs\":[{\"Gateway\":\"10.244.0.1\",\"IP\":\"10.244.0.2\"}]}}");


//
// Simulate unsuccessful simultaneous PodSandboxStatusResponse lookup and subsequent k8s filterchecks fallbacks
//

container.m_pod_sandbox_cniresult.clear();
container.m_container_ip = 0;
container.m_pod_sandbox_labels.clear();
m_inspector.m_container_manager.replace_container(std::move(container_ptr));

std::shared_ptr<sinsp_container_info> sandbox_container_ptr = std::make_shared<sinsp_container_info>();
sinsp_container_info &sandbox_container = *sandbox_container_ptr;

sandbox_container.m_type = CT_CONTAINERD;
sandbox_container.m_id = "63060edc2d3a"; // truncated id extracted from cgroups for the sandbox container
sandbox_container.m_is_pod_sandbox = true;
res = cri_api_v1alpha2->parse_cri_base(resp_pod_sandbox_container, sandbox_container);
ASSERT_TRUE(res);
res = cri_api_v1alpha2->parse_cri_labels(resp_pod_sandbox_container, sandbox_container);
ASSERT_TRUE(res);
res = cri_api_v1alpha2->parse_cri_pod_sandbox_network(resp_pod_sandbox_container, root_pod_sandbox, sandbox_container);
ASSERT_TRUE(res);
res = cri_api_v1alpha2->parse_cri_pod_sandbox_labels(resp_pod_sandbox_container, sandbox_container);
ASSERT_TRUE(res);
m_inspector.m_container_manager.add_container(std::move(sandbox_container_ptr), nullptr);

ASSERT_EQ(get_field_as_string(evt, "k8s.pod.label.example-label/custom_one"), "mylabel");
ASSERT_EQ(get_field_as_string(evt, "k8s.pod.label[example-label/custom_one]"), "mylabel");
ASSERT_EQ(get_field_as_string(evt, "k8s.pod.labels"), "app:myapp, example-label/custom_one:mylabel");
ASSERT_EQ(get_field_as_string(evt, "k8s.pod.ip"), "10.244.0.2");
ASSERT_EQ(get_field_as_string(evt, "k8s.pod.cni.json"), "{\"bridge\":{\"IPConfigs\":null},\"eth0\":{\"IPConfigs\":[{\"Gateway\":\"10.244.0.1\",\"IP\":\"10.244.0.2\"}]}}");
}

TEST_F(sinsp_with_test_input, container_parser_cri_containerd_sandbox_container)
Expand All @@ -712,7 +744,7 @@ TEST_F(sinsp_with_test_input, container_parser_cri_containerd_sandbox_container)

container.m_type = CT_CONTAINERD;
container.m_id = "63060edc2d3a"; // truncated id extracted from cgroups for the sandbox container
container.m_is_pod_sandbox = true;
container.m_is_pod_sandbox = true;
auto res = cri_api_v1alpha2->parse_cri_base(resp_pod_sandbox_container, container);
ASSERT_TRUE(res);
res = cri_api_v1alpha2->parse_cri_labels(resp_pod_sandbox_container, container);
Expand All @@ -721,7 +753,7 @@ TEST_F(sinsp_with_test_input, container_parser_cri_containerd_sandbox_container)
ASSERT_TRUE(res);
res = cri_api_v1alpha2->parse_cri_pod_sandbox_labels(resp_pod_sandbox_container, container);
ASSERT_TRUE(res);
ASSERT_TRUE(container.m_is_pod_sandbox);
ASSERT_TRUE(container.m_is_pod_sandbox);

//
// Test sinsp filterchecks, similar to spawn_process_container test
Expand Down

0 comments on commit a17eaba

Please sign in to comment.