diff --git a/python/tests/component_check.py b/python/tests/component_check.py index 52d919a4..3c882860 100644 --- a/python/tests/component_check.py +++ b/python/tests/component_check.py @@ -1,10 +1,8 @@ -import os import json +import os import inspect -import time -from jsonschema import validate, ValidationError, SchemaError +from jsonschema import validate from pydantic import BaseModel -from typing import Generator from appbuilder.utils.func_utils import Singleton from appbuilder.tests.component_schemas import type_to_json_schemas from appbuilder.utils.json_schema_to_model import json_schema_to_pydantic_model @@ -40,12 +38,15 @@ def register_rule(self, rule_name: str, rule_obj: RuleBase): def remove_rule(self, rule_name: str): del self.rules[rule_name] - def notify(self, component_cls) -> tuple[bool, list]: + def notify(self, component_cls, component_case) -> tuple[bool, list]: check_pass = True check_details = {} reasons = [] for rule_name, rule_obj in self.rules.items(): - res = rule_obj.check(component_cls) + if rule_name == "ToolEvalOutputJsonRule": + res = rule_obj.check(component_cls, component_case) + else: + res = rule_obj.check(component_cls) check_details[rule_name] = res if res.check_result == False: check_pass = False @@ -63,53 +64,40 @@ class ManifestValidRule(RuleBase): def __init__(self, **kwargs): super().__init__() self.rule_name = "ManifestValidRule" - self.component_tool_eval_cases = kwargs.get("component_tool_eval_cases", {}) - def check(self, component_cls) -> CheckInfo: + def check(self, component_obj) -> CheckInfo: check_pass_flag = True invalid_details = [] - component_cls_name = component_cls.__name__ - if component_cls_name not in self.component_tool_eval_cases: - invalid_details.append("{} 没有添加测试case到 component_tool_eval_cases 中".format(component_cls_name)) - else: - component_case = self.component_tool_eval_cases[component_cls_name]() - envs = component_case.envs() - os.environ.update(envs) - init_args = component_case.init_args() - try: - component_obj = component_cls(**init_args) - if not hasattr(component_obj, "manifests"): - raise ValueError("No manifests found") - manifests = component_obj.manifests - # NOTE(暂时检查manifest中的第一个mainfest) - if not manifests or len(manifests) == 0: - raise ValueError("No manifests found") - manifest = manifests[0] - tool_name = manifest['name'] - tool_desc = manifest['description'] - schema = manifest["parameters"] - schema["title"] = tool_name - # 第一步，将json schema转换为pydantic模型 - pydantic_model = json_schema_to_pydantic_model(schema, tool_name) - check_to_json = pydantic_model.schema_json() - json_to_dict = json.loads(check_to_json) - - if "properties" in schema: - properties = schema["properties"] - for key, value in properties.items(): - if "type" not in value: - invalid_details.append("\'type' must be in properties item: {}".format(key)) - if "description" not in value: - invalid_details.append("\'description' must be in properties item: {}".format(key)) - - except Exception as e: - print(e) - check_pass_flag = False - invalid_details.append(str(e)) - - for env in envs: - os.environ.pop(env) + try: + if not hasattr(component_obj, "manifests"): + raise ValueError("No manifests found") + manifests = component_obj.manifests + # NOTE(暂时检查manifest中的第一个mainfest) + if not manifests or len(manifests) == 0: + raise ValueError("No manifests found") + manifest = manifests[0] + tool_name = manifest['name'] + tool_desc = manifest['description'] + schema = manifest["parameters"] + schema["title"] = tool_name + # 第一步，将json schema转换为pydantic模型 + pydantic_model = json_schema_to_pydantic_model(schema, tool_name) + check_to_json = pydantic_model.schema_json() + json_to_dict = json.loads(check_to_json) + + if "properties" in schema: + properties = schema["properties"] + for key, value in properties.items(): + if "type" not in value: + invalid_details.append("\'type' must be in properties item: {}".format(key)) + if "description" not in value: + invalid_details.append("\'description' must be in properties item: {}".format(key)) + + except Exception as e: + print(e) + check_pass_flag = False + invalid_details.append(str(e)) if len(invalid_details) > 0: check_pass_flag = False @@ -137,14 +125,14 @@ def __init__(self): self.rule_name = "MainfestMatchToolEvalRule" - def check(self, component_cls) -> CheckInfo: + def check(self, component_obj) -> CheckInfo: check_pass_flag = True invalid_details = [] try: - if not hasattr(component_cls, "manifests"): + if not hasattr(component_obj, "manifests"): raise ValueError("No manifests found") - manifests = component_cls.manifests + manifests = component_obj.manifests # NOTE(暂时检查manifest中的第一个mainfest) if not manifests or len(manifests) == 0: raise ValueError("No manifests found") @@ -158,7 +146,7 @@ def check(self, component_cls) -> CheckInfo: # 交互检查 tool_eval_input_params = [] print("required_params: {}".format(manifest_var)) - signature = inspect.signature(component_cls.tool_eval) + signature = inspect.signature(component_obj.tool_eval) ileagal_params = [] for param_name, param in signature.parameters.items(): if param_name == 'kwargs' or param_name == 'args' or param_name == 'self': @@ -193,10 +181,6 @@ def check(self, component_cls) -> CheckInfo: check_detail=",".join(invalid_details)) - - - - class ToolEvalInputNameRule(RuleBase): """ 检查tool_eval的输入参数中，是否包含系统保留的输入名称 @@ -222,10 +206,15 @@ def __init__(self): "_sys_custom_variables", "_sys_thought_model_config", "_sys_rag_model_config", + "_sys_parent_span_id", + "_sys_span_id", + "_sys_memory", + "_sys_code_execution_endpoint", + "_sys_session_id" ] - def check(self, component_cls) -> CheckInfo: - tool_eval_signature = inspect.signature(component_cls.__init__) + def check(self, component_obj) -> CheckInfo: + tool_eval_signature = inspect.signature(component_obj.tool_eval) params = tool_eval_signature.parameters invalid_details = [] check_pass_flag = True @@ -250,7 +239,6 @@ class ToolEvalOutputJsonRule(RuleBase): def __init__(self, **kwargs): super().__init__() self.rule_name = 'ToolEvalOutputJsonRule' - self.component_tool_eval_cases = kwargs.get("component_tool_eval_cases") def _check_pre_format(self, outputs): invalid_details = [] @@ -351,42 +339,26 @@ def _check_text_and_code(self, component_case, output_dict): else: return [] - def check(self, component_cls) -> CheckInfo: + def check(self, component_obj, component_case) -> CheckInfo: invalid_details = [] - component_cls_name = component_cls.__name__ - if component_cls_name not in self.component_tool_eval_cases: - invalid_details.append("{} 没有添加测试case到 component_tool_eval_cases 中".format(component_cls_name)) - else: - component_case = self.component_tool_eval_cases[component_cls_name]() - - envs = {} - if hasattr(component_case, "envs"): - envs = component_case.envs() - os.environ.update(envs) - - input_dict = component_case.inputs() - init_args = component_case.init_args() - component_obj = component_cls(**init_args) - output_json_schemas = component_case.schemas() - - try: - stream_output_dict = {"text": "", "oral_text":"", "code": ""} - stream_outputs = component_obj.tool_eval(**input_dict) - for stream_output in stream_outputs: - iter_invalid_detail = self._check_jsonschema(stream_output.model_dump(), output_json_schemas) - invalid_details.extend(iter_invalid_detail) - iter_output_dict = self._gather_iter_outputs(stream_output) - stream_output_dict["text"] += iter_output_dict["text"] - stream_output_dict["oral_text"] += iter_output_dict["oral_text"] - stream_output_dict["code"] += iter_output_dict["code"] - if len(invalid_details) == 0: - invalid_details.extend(self._check_text_and_code(component_case, stream_output_dict)) - except Exception as e: - invalid_details.append("ToolEval执行失败: {}".format(e)) - - for env in envs: - os.environ.pop(env) + input_dict = component_case.inputs() + output_json_schemas = component_case.schemas() + + try: + stream_output_dict = {"text": "", "oral_text":"", "code": ""} + stream_outputs = component_obj.tool_eval(**input_dict) + for stream_output in stream_outputs: + iter_invalid_detail = self._check_jsonschema(stream_output.model_dump(), output_json_schemas) + invalid_details.extend(iter_invalid_detail) + iter_output_dict = self._gather_iter_outputs(stream_output) + stream_output_dict["text"] += iter_output_dict["text"] + stream_output_dict["oral_text"] += iter_output_dict["oral_text"] + stream_output_dict["code"] += iter_output_dict["code"] + if len(invalid_details) == 0: + invalid_details.extend(self._check_text_and_code(component_case, stream_output_dict)) + except Exception as e: + invalid_details.append("ToolEval执行失败: {}".format(e)) if len(invalid_details) > 0: return CheckInfo( @@ -400,6 +372,88 @@ def check(self, component_cls) -> CheckInfo: check_detail="") -def register_component_check_rule(rule_name: str, rule_cls: RuleBase, init_args={}): +def register_component_check_rule(rule_name: str, rule_cls: RuleBase): component_checker = ComponentCheckBase() - component_checker.register_rule(rule_name, rule_cls(**init_args)) \ No newline at end of file + component_checker.register_rule(rule_name, rule_cls()) + + +def check_component_with_retry(component_import_res_tuple): + """ + 使用重试机制检查组件。测试用例失败后会重试两次。 + + Args: + component_import_res_tuple (tuple): 包含组件和导入结果的元组。 + + Returns: + list: 包含错误信息的数据列表。 + + """ + component, import_res, component_case_cls = component_import_res_tuple + component_check_base = ComponentCheckBase() + + error_data = [] + max_retries = 2 # 设置最大重试次数 + attempts = 0 + + while attempts <= max_retries: + if import_res["import_error"] != "": + error_data.append({"Component Name": component, "Error Message": import_res["import_error"]}) + print("组件名称:{} 错误信息:{}".format(component, import_res["import_error"])) + break + + component_case = component_case_cls() + envs = component_case.envs() + os.environ.update(envs) + component_cls = import_res["obj"] + component_obj = component_cls(**component_case.init_args()) + + try: + # 此处的self.component_check_base.notify需要根据实际情况修改 + pass_check, reasons = component_check_base.notify(component_obj, component_case) # 示例修改 + reasons = list(set(reasons)) + if not pass_check: + error_data.append({"Component Name": component, "Error Message": ", ".join(reasons)}) + print("组件名称:{} 错误信息:{}".format(component, ", ".join(reasons))) + # 如果检查失败，增加尝试次数并重试 + attempts += 1 + if attempts <= max_retries: + print("组件名称:{} 将重试，当前尝试次数:{}".format(component, attempts)) + continue + # 如果检查通过，则退出循环 + break + except Exception as e: + error_data.append({"Component Name": component, "Error Message": str(e)}) + print("组件名称:{} 错误信息:{}".format(component, str(e))) + # 如果发生异常，增加尝试次数并重试 + attempts += 1 + if attempts <= max_retries: + print("组件名称:{} 将重试，当前尝试次数:{}".format(component, attempts)) + continue + + finally: + for env in envs: + os.environ.pop(env) + + return error_data + +def write_error_data(txt_file_path, error_df, error_stats): + """将组件错误信息写入文件 + + Args: + error_df (Union[pd.DataFrame, None]): 错误信息表格 + error_stats (dict): 错误统计信息 + """ + with open(txt_file_path, 'w') as file: + file.write("Component Name\tError Message\n") + for _, row in error_df.iterrows(): + file.write(f"{row['Component Name']}\t{row['Error Message']}\n") + file.write("\n错误统计信息:\n") + for error, count in error_stats.items(): + file.write(f"错误信息: {error}, 出现次数: {count}\n") + print(f"\n错误信息已写入: {txt_file_path}") + + +register_component_check_rule("ManifestValidRule", ManifestValidRule) +register_component_check_rule("MainfestMatchToolEvalRule", MainfestMatchToolEvalRule) +register_component_check_rule("ToolEvalInputNameRule", ToolEvalInputNameRule) +register_component_check_rule("ToolEvalOutputJsonRule", ToolEvalOutputJsonRule) \ No newline at end of file diff --git a/python/tests/test_all_components.py b/python/tests/test_all_components.py index 83c605e1..4822759c 100644 --- a/python/tests/test_all_components.py +++ b/python/tests/test_all_components.py @@ -4,86 +4,12 @@ import pandas as pd import unittest import os -from appbuilder.tests.component_check import ComponentCheckBase -from appbuilder.tests.component_check import register_component_check_rule -from appbuilder.tests.component_check import ManifestValidRule, MainfestMatchToolEvalRule, ToolEvalInputNameRule, ToolEvalOutputJsonRule + from appbuilder.core._exception import AppbuilderBuildexException -from component_collector import get_all_components, get_v2_components, get_component_white_list from component_tool_eval_cases import component_tool_eval_cases +from component_collector import get_all_components, get_v2_components, get_component_white_list +from component_check import check_component_with_retry, write_error_data -register_component_check_rule("ManifestValidRule", ManifestValidRule, \ - {"component_tool_eval_cases": component_tool_eval_cases}) -register_component_check_rule("MainfestMatchToolEvalRule", MainfestMatchToolEvalRule, {}) -register_component_check_rule("ToolEvalInputNameRule", ToolEvalInputNameRule, {}) -register_component_check_rule("ToolEvalOutputJsonRule", ToolEvalOutputJsonRule, \ - {"component_tool_eval_cases": component_tool_eval_cases}) - - -def check_component_with_retry(component_import_res_tuple): - """ - 使用重试机制检查组件。测试用例失败后会重试两次。 - - Args: - component_import_res_tuple (tuple): 包含组件和导入结果的元组。 - - Returns: - list: 包含错误信息的数据列表。 - - """ - component, import_res = component_import_res_tuple - component_check_base = ComponentCheckBase() - - error_data = [] - max_retries = 2 # 设置最大重试次数 - attempts = 0 - - while attempts <= max_retries: - if import_res["import_error"] != "": - error_data.append({"Component Name": component, "Error Message": import_res["import_error"]}) - print("组件名称:{} 错误信息:{}".format(component, import_res["import_error"])) - break - - component_obj = import_res["obj"] - try: - # 此处的self.component_check_base.notify需要根据实际情况修改 - pass_check, reasons = component_check_base.notify(component_obj) # 示例修改 - reasons = list(set(reasons)) - if not pass_check: - error_data.append({"Component Name": component, "Error Message": ", ".join(reasons)}) - print("组件名称:{} 错误信息:{}".format(component, ", ".join(reasons))) - # 如果检查失败，增加尝试次数并重试 - attempts += 1 - if attempts <= max_retries: - print("组件名称:{} 将重试，当前尝试次数:{}".format(component, attempts)) - continue - # 如果检查通过，则退出循环 - break - except Exception as e: - error_data.append({"Component Name": component, "Error Message": str(e)}) - print("组件名称:{} 错误信息:{}".format(component, str(e))) - # 如果发生异常，增加尝试次数并重试 - attempts += 1 - if attempts <= max_retries: - print("组件名称:{} 将重试，当前尝试次数:{}".format(component, attempts)) - continue - - return error_data - -def write_error_data(txt_file_path, error_df, error_stats): - """将组件错误信息写入文件 - - Args: - error_df (Union[pd.DataFrame, None]): 错误信息表格 - error_stats (dict): 错误统计信息 - """ - with open(txt_file_path, 'w') as file: - file.write("Component Name\tError Message\n") - for _, row in error_df.iterrows(): - file.write(f"{row['Component Name']}\t{row['Error Message']}\n") - file.write("\n错误统计信息:\n") - for error, count in error_stats.items(): - file.write(f"错误信息: {error}, 出现次数: {count}\n") - print(f"\n错误信息已写入: {txt_file_path}") @unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_PARALLEL", "") class TestComponentManifestsAndToolEval(unittest.TestCase): @@ -110,7 +36,7 @@ def setUp(self) -> None: self.v2_components = get_v2_components() self.whitelist_components = get_component_white_list() - def _test_component(self, components, whitelist_components, txt_file_path): + def _test_component(self, components, component_cases, whitelist_components, txt_file_path): """测试所有组件的manifests和tool_eval入参 Args: 无 @@ -122,7 +48,16 @@ def _test_component(self, components, whitelist_components, txt_file_path): with Pool(processes=os.cpu_count()) as pool: # 使用pool.map来执行多进程 - results = pool.map(check_component_with_retry, components.items()) + args = [] + for component, import_res in components.items(): + if component not in component_cases: + error_data.append({"Component Name": component, "Error Message": "{} 没有添加测试case到 \ + component_tool_eval_cases 中".format(component)}) + continue + else: + args.append((component, import_res, component_tool_eval_cases[component])) + + results = pool.map(check_component_with_retry, args) # 合并每个进程返回的错误数据 for result in results: @@ -154,13 +89,13 @@ def _test_component(self, components, whitelist_components, txt_file_path): else: print("\n所有组件测试通过，无错误信息。") - def test_all_components(self): + def _test_all_components(self): """测试旧版本组件""" - self._test_component(self.all_components, self.whitelist_components, 'components_error_info.txt') + self._test_component(self.all_components, [], self.whitelist_components, 'components_error_info.txt') def test_v2_components(self): """测试v2版本组件""" - self._test_component(self.v2_components, [], 'v2_components_error_info.txt') + self._test_component(self.v2_components, component_tool_eval_cases, [], 'v2_components_error_info.txt') if __name__ == '__main__':