feat: 配置平台(CMDB) 相关插件支持 云区域:IP #7432 (#7454)

* feat: 配置平台(CMDB) 相关插件支持 云区域:IP #7432

* feat: 配置平台(CMDB) 相关插件支持 云区域:IP #7432

* feat: 配置平台(CMDB) 相关插件支持 云区域:IP #7432

* feat: 配置平台(CMDB) 相关插件支持 云区域:IP #7432

* feat: 配置平台(CMDB) 相关插件支持 云区域:IP #7432

* feat: 配置平台(CMDB) 相关插件支持 云区域:IP #7432

* feat: 配置平台(CMDB) 相关插件支持 云区域:IP #7432

* feat: 配置平台(CMDB) 相关插件支持 云区域:IP #7432

* feat: 配置平台(CMDB) 相关插件支持 云区域:IP #7432

* feat: 配置平台(CMDB) 相关插件支持 云区域:IP #7432

gcloud/utils/cmdb.py

@@ -123,7 +123,7 @@ def get_business_host(username, bk_biz_id, supplier_account, host_fields, ip_lis
     kwargs = {"bk_biz_id": bk_biz_id, "bk_supplier_account": supplier_account, "fields": list(host_fields or [])}
 
     # 带管控区域的主机数据查询
-    if ip_list and bk_cloud_id:
+    if ip_list and bk_cloud_id is not None:
         kwargs["host_property_filter"] = {
             "condition": "AND",
             "rules": [

pipeline_plugins/components/collections/sites/open/cc/base.py

@@ -138,6 +138,76 @@ def cc_get_host_id_by_innerip(executor, bk_biz_id, ip_list, supplier_account):
     return {"result": True, "data": [str(host["bk_host_id"]) for host in host_list]}
 
 
+def cc_get_host_id_by_innerip_and_cloudid(executor, bk_biz_id, ip_str, supplier_account):
+    """根据主机内网 IP 获取主机 ID
+
+    :param executor: API 请求用户身份
+    :type executor: string
+    :param bk_biz_id: 业务 CC ID
+    :type bk_biz_id: int
+    :param ip_list: 主机内网 IP 或 cloudID:IP 列表
+    :type ip_str: string
+    :param supplier_account: 开发商账号
+    :type supplier_account: int
+    :return: 主机 id 列表
+    :rtype: list
+    ["1", "2", "3", ...]
+    """
+    ipv4_list_with_cloud_id, ip_str_without_ipv4_with_cloud_id = get_ip_by_regex_type(
+        IpRegexType.IPV4_WITH_CLOUD_ID.value, ip_str
+    )
+    ipv4_list, _ = get_ip_by_regex_type(IpRegexType.IPV4.value, ip_str_without_ipv4_with_cloud_id)
+    ip_dict = {None: ipv4_list} if ipv4_list else {}
+    [ip_dict.setdefault(int(ipv4.split(":")[0]), []).append(ipv4.split(":")[1]) for ipv4 in ipv4_list_with_cloud_id]
+    # TODO 待优化
+    hosts = []
+    for cloud_id, ipv4s in ip_dict.items():
+        if cloud_id is None:
+            host_fields = ["bk_host_id", "bk_host_innerip"]
+        else:
+            host_fields = ["bk_host_id", "bk_host_innerip", "bk_cloud_id"]
+        host_list = cmdb.get_business_host(
+            executor,
+            bk_biz_id,
+            supplier_account,
+            host_fields,
+            ipv4s,
+            cloud_id,
+        )
+
+        if not host_list:
+            message = f"IP {ipv4s} 在本业务下不存在: 请检查配置, 修复后重新执行 | cc_get_host_id_by_innerip_and_cloudid"
+            logger.error(message)
+            return {"result": False, "message": message}
+
+        if len(host_list) > len(ipv4s):
+            # find repeat innerip host
+            host_counter = Counter([host["bk_host_innerip"] for host in host_list])
+            mutiple_hosts = [innerip for innerip, count in host_counter.items() if count > 1]
+            message = f"IP [{', '.join(mutiple_hosts)}] 在本业务下重复: 请检查配置, 修复后重新执行 | cc_get_host_id_by_innerip_and_cloudid"
+            logger.error(message)
+            return {
+                "result": False,
+                "message": message,
+            }
+
+        if len(host_list) < len(ipv4s):
+            return_innerip_set = set()
+            for host in host_list:
+                if host.get("bk_cloud_id") is not None:
+                    return_innerip_set.add(f"{host['bk_cloud_id']}:{host['bk_host_innerip']}")
+                else:
+                    return_innerip_set.add(host["bk_host_innerip"])
+            absent_innerip = set(ipv4s).difference(return_innerip_set)
+            message = (
+                f"IP [{', '.join(absent_innerip)}] 在本业务下不存在: 请检查配置, 修复后重新执行 | cc_get_host_id_by_innerip_and_cloudid"
+            )
+            logger.error(message)
+            return {"result": False, "message": message}
+        hosts.extend(host_list)
+    return {"result": True, "data": list({str(host["bk_host_id"]) for host in hosts})}
+
+
 def cc_get_host_by_innerip_with_ipv6(
     executor, bk_biz_id, ip_str, supplier_account, is_biz_set=False, host_id_detail=False
 ):
@@ -519,6 +589,23 @@ def get_host_topo(self, executor, biz_cc_id, supplier_account, host_attrs, ip_st
             executor, biz_cc_id, supplier_account, host_attrs, ip_list=None, property_filters=property_filters
         )
 
+    def get_host_list_with_cloud_id(self, executor, biz_cc_id, ip_str, supplier_account):
+        """
+        获取host_list
+        @param executor: executor 执行人
+        @param biz_cc_id: biz_cc_id 业务id
+        @param ip_str: ip_str ip字符串
+        @param supplier_account: supplier_account
+        @return:
+        """
+        # 如果开启IPV6
+        if settings.ENABLE_IPV6:
+            host_result = cc_get_host_by_innerip_with_ipv6(executor, biz_cc_id, ip_str, supplier_account)
+            if not host_result["result"]:
+                return host_result
+            return {"result": True, "data": [str(host["bk_host_id"]) for host in host_result["data"]]}
+        return cc_get_host_id_by_innerip_and_cloudid(executor, biz_cc_id, ip_str, supplier_account)
+
 
 class BaseTransferHostToModuleService(Service, CCPluginIPMixin, metaclass=ABCMeta):
     def inputs_format(self):

pipeline_plugins/components/collections/sites/open/cc/batch_update_host/v1_0.py

@@ -24,7 +24,6 @@
 from pipeline_plugins.base.utils.inject import supplier_account_for_business
 from pipeline_plugins.components.collections.sites.open.cc.base import CCPluginIPMixin, cc_format_prop_data
 from pipeline_plugins.components.utils import chunk_table_data, convert_num_to_str
-from pipeline_plugins.components.utils.sites.open.utils import plat_ip_reg
 
 logger = logging.getLogger("celery")
 get_client_by_user = settings.ESB_GET_CLIENT_BY_USER
@@ -122,12 +121,8 @@ def execute(self, data, parent_data):
         host_property_copy = deepcopy(host_property_custom)
         update_host_message = []
         for host_property_dir in host_property_copy:
-            inner_host_ip = host_property_dir["bk_host_innerip"]
-            # 兼容填写管控区域ID：IP的情况, 只获取对应IP, 判断ipv4
-            if plat_ip_reg.match(inner_host_ip) and ":" in inner_host_ip:
-                inner_host_ip = inner_host_ip.split(":")[1]
-
-            host_result = self.get_host_list(executor, biz_cc_id, inner_host_ip, supplier_account)
+            ip_str = host_property_dir["bk_host_innerip"]
+            host_result = self.get_host_list_with_cloud_id(executor, biz_cc_id, ip_str, supplier_account)
             if not host_result["result"]:
                 data.outputs.ex_data = host_result.get("message")
                 return False

pipeline_plugins/components/collections/sites/open/cc/update_host/legacy.py

@@ -16,19 +16,14 @@
 
 from django.utils import translation
 from django.utils.translation import ugettext_lazy as _
-
+from pipeline.component_framework.component import Component
 from pipeline.core.flow.activity import Service
 from pipeline.core.flow.io import StringItemSchema
-from pipeline.component_framework.component import Component
-
-from pipeline_plugins.base.utils.inject import supplier_account_for_business
-from pipeline_plugins.components.collections.sites.open.cc.base import (
-    cc_format_prop_data,
-    CCPluginIPMixin,
-)
 
 from gcloud.conf import settings
 from gcloud.utils.handlers import handle_api_error
+from pipeline_plugins.base.utils.inject import supplier_account_for_business
+from pipeline_plugins.components.collections.sites.open.cc.base import CCPluginIPMixin, cc_format_prop_data
 
 logger = logging.getLogger("celery")
 get_client_by_user = settings.ESB_GET_CLIENT_BY_USER
@@ -79,8 +74,8 @@ def execute(self, data, parent_data):
         supplier_account = supplier_account_for_business(biz_cc_id)
 
         # 查询主机id
-        ip_list = data.get_one_of_inputs("cc_host_ip")
-        host_result = self.get_host_list(executor, biz_cc_id, ip_list, supplier_account)
+        ip_str = data.get_one_of_inputs("cc_host_ip")
+        host_result = self.get_host_list_with_cloud_id(executor, biz_cc_id, ip_str, supplier_account)
         if not host_result["result"]:
             data.set_outputs("ex_data", host_result["message"])
             return False

pipeline_plugins/tests/components/collections/sites/open/cc/base/test_cc_get_host_id_by_cloudid_innerip.py

@@ -0,0 +1,126 @@
+# -*- coding: utf-8 -*-
+"""
+Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community
+Edition) available.
+Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
+from django.test import TestCase
+from mock import MagicMock, patch
+
+from pipeline_plugins.components.collections.sites.open.cc.base import cc_get_host_id_by_innerip_and_cloudid
+
+
+class CCGetHostIdByCloudIdInnerIpTestCase(TestCase):
+    def setUp(self):
+        self.executor = "executor_token"
+        self.bk_biz_id = "bk_biz_id_token"
+        self.supplier_account = "supplier_account_token"
+        self.ip_str = "1.1.1.1"
+
+    def test__get_business_host_return_empty(self):
+        mock_cmdb = MagicMock()
+        mock_cmdb.get_business_host = MagicMock(return_value=[])
+        with patch("pipeline_plugins.components.collections.sites.open.cc.base.cmdb", mock_cmdb):
+            data = cc_get_host_id_by_innerip_and_cloudid(
+                self.executor, self.bk_biz_id, self.ip_str, self.supplier_account
+            )
+
+        mock_cmdb.get_business_host.assert_called_once_with(
+            self.executor, self.bk_biz_id, self.supplier_account, ["bk_host_id", "bk_host_innerip"], [self.ip_str], None
+        )
+        self.assertFalse(data["result"])
+        self.assertEqual(
+            data["message"], "IP ['1.1.1.1'] 在本业务下不存在: 请检查配置, 修复后重新执行 | cc_get_host_id_by_innerip_and_cloudid"
+        )
+
+    def test__return_host_list_gt_ip_list(self):
+        mock_cmdb = MagicMock()
+        mock_cmdb.get_business_host = MagicMock(
+            return_value=[
+                {"bk_host_innerip": "1.1.1.1"},
+                {"bk_host_innerip": "1.1.1.1"},
+            ]
+        )
+        with patch("pipeline_plugins.components.collections.sites.open.cc.base.cmdb", mock_cmdb):
+            data = cc_get_host_id_by_innerip_and_cloudid(
+                self.executor, self.bk_biz_id, self.ip_str, self.supplier_account
+            )
+
+        mock_cmdb.get_business_host.assert_called_once_with(
+            self.executor, self.bk_biz_id, self.supplier_account, ["bk_host_id", "bk_host_innerip"], [self.ip_str], None
+        )
+        self.assertFalse(data["result"])
+        self.assertEqual(
+            data["message"], "IP [1.1.1.1] 在本业务下重复: 请检查配置, 修复后重新执行 | cc_get_host_id_by_innerip_and_cloudid"
+        )
+
+    def test__return_host_list_lt_cloudid_with_ip_list(self):
+        self.ip_str = "0:1.1.1.1"
+        mock_cmdb = MagicMock()
+        mock_cmdb.get_business_host = MagicMock(return_value=[])
+        with patch("pipeline_plugins.components.collections.sites.open.cc.base.cmdb", mock_cmdb):
+            data = cc_get_host_id_by_innerip_and_cloudid(
+                self.executor, self.bk_biz_id, self.ip_str, self.supplier_account
+            )
+
+        mock_cmdb.get_business_host.assert_called_once_with(
+            self.executor,
+            self.bk_biz_id,
+            self.supplier_account,
+            ["bk_host_id", "bk_host_innerip", "bk_cloud_id"],
+            [self.ip_str.split(":")[1]],
+            0,
+        )
+        self.assertFalse(data["result"])
+        self.assertEqual(
+            data["message"], "IP ['1.1.1.1'] 在本业务下不存在: 请检查配置, 修复后重新执行 | cc_get_host_id_by_innerip_and_cloudid"
+        )
+
+    def test__ip_normal(self):
+        mock_cmdb = MagicMock()
+        mock_cmdb.get_business_host = MagicMock(
+            return_value=[
+                {"bk_host_innerip": "1.1.1.1", "bk_host_id": 1},
+            ]
+        )
+        with patch("pipeline_plugins.components.collections.sites.open.cc.base.cmdb", mock_cmdb):
+            data = cc_get_host_id_by_innerip_and_cloudid(
+                self.executor, self.bk_biz_id, self.ip_str, self.supplier_account
+            )
+
+        mock_cmdb.get_business_host.assert_called_once_with(
+            self.executor, self.bk_biz_id, self.supplier_account, ["bk_host_id", "bk_host_innerip"], [self.ip_str], None
+        )
+        self.assertTrue(data["result"])
+        self.assertEqual(data["data"], ["1"])
+
+    def test__cloudid_with_ip_normal(self):
+        self.ip_str = "0:1.1.1.1"
+        mock_cmdb = MagicMock()
+        mock_cmdb.get_business_host = MagicMock(
+            return_value=[
+                {"bk_host_innerip": "1.1.1.1", "bk_host_id": 1, "bk_cloud_id": 0},
+            ]
+        )
+        with patch("pipeline_plugins.components.collections.sites.open.cc.base.cmdb", mock_cmdb):
+            data = cc_get_host_id_by_innerip_and_cloudid(
+                self.executor, self.bk_biz_id, self.ip_str, self.supplier_account
+            )
+
+        mock_cmdb.get_business_host.assert_called_once_with(
+            self.executor,
+            self.bk_biz_id,
+            self.supplier_account,
+            ["bk_host_id", "bk_host_innerip", "bk_cloud_id"],
+            [self.ip_str.split(":")[1]],
+            0,
+        )
+        self.assertTrue(data["result"])
+        self.assertEqual(data["data"], ["1"])

pipeline_plugins/tests/components/collections/sites/open/cc/base/test_cc_get_host_id_by_innerip.py

@@ -11,9 +11,8 @@
 specific language governing permissions and limitations under the License.
 """
 
-from mock import MagicMock, patch
-
 from django.test import TestCase
+from mock import MagicMock, patch
 
 from pipeline_plugins.components.collections.sites.open.cc.base import cc_get_host_id_by_innerip
 

pipeline_plugins/tests/components/collections/sites/open/cc_test/batch_update_host/test_v1_0.py

@@ -12,15 +12,15 @@
 """
 from django.test import TestCase
 from mock import MagicMock
-
 from pipeline.component_framework.test import (
-    ComponentTestMixin,
-    ComponentTestCase,
+    Call,
     CallAssertion,
+    ComponentTestCase,
+    ComponentTestMixin,
     ExecuteAssertion,
-    Call,
     Patcher,
 )
+
 from pipeline_plugins.components.collections.sites.open.cc.batch_update_host.v1_0 import CCBatchUpdateHostComponent
 
 
@@ -47,7 +47,9 @@ def __init__(self, batch_update_host_return=None):
 )
 
 GET_CLIENT_BY_USER = "pipeline_plugins.components.collections.sites.open.cc.batch_update_host.v1_0.get_client_by_user"
-CC_GET_IPS_INFO_BY_STR = "pipeline_plugins.components.collections.sites.open.cc.base.cc_get_host_id_by_innerip"
+CC_GET_IPS_INFO_BY_STR = (
+    "pipeline_plugins.components.collections.sites.open.cc.base.cc_get_host_id_by_innerip_and_cloudid"
+)
 VERIFY_HOST_PROPERTY = (
     "pipeline_plugins.components.collections.sites.open.cc.batch_update_host.v1_0.verify_host_property"
 )
@@ -126,7 +128,10 @@ def verify_host_property_fail(executor, supplier_account, language, cc_host_prop
     execute_assertion=ExecuteAssertion(success=True, outputs={}),
     schedule_assertion=None,
     execute_call_assertion=[
-        CallAssertion(func=CC_GET_IPS_INFO_BY_STR, calls=[Call("executor", 1, ["1.1.1.1"], 0)],),
+        CallAssertion(
+            func=CC_GET_IPS_INFO_BY_STR,
+            calls=[Call("executor", 1, "1.1.1.1", 0)],
+        ),
         CallAssertion(
             func=BATCH_UPDATE_HOST_SUCCESS_CLIENT.cc.batch_update_host,
             calls=[
@@ -168,7 +173,10 @@ def verify_host_property_fail(executor, supplier_account, language, cc_host_prop
     ),
     schedule_assertion=None,
     execute_call_assertion=[
-        CallAssertion(func=CC_GET_IPS_INFO_BY_STR, calls=[Call("executor", 1, ["1.1.1.1"], 0)],),
+        CallAssertion(
+            func=CC_GET_IPS_INFO_BY_STR,
+            calls=[Call("executor", 1, "1.1.1.1", 0)],
+        ),
         CallAssertion(
             func=BATCH_UPDATE_HOST_FAIL_CLIENT.cc.batch_update_host,
             calls=[