Merge pull request #40 from OpenBMB/format_answer_generation_code

Format answer generation code

.gitignore

@@ -0,0 +1,9 @@
+data/
+data.zip
+*.DS_store
+
+__MACOSX/
+
+run.bash
+
+*.pyc

toolbench/inference/Algorithms/base_search.py

@@ -1,26 +1,33 @@
-class base_search_method:
+from Downstream_tasks.base_env import base_env
 
-    def __init__(self):
+class base_search_method:
+    """For the base tree search method, you need to support the following functions"""
+
+    def __init__(self,llm,io_func: base_env, process_id=0, callbacks = None):
+        """Args:
+            llm: The interface of the LLM 
+            io_func(base_env): Interface to the environment,
+            process_id (int, optional): In multiprocessing annotation, this describes the process id. Defaults to 0.
+            callbacks (_type_, optional): _description_. Defaults to None.
+        """
         pass
 
-    def to_json(self):
+    def to_json(self,answer=False,process=True):
         '''
+        return a json object, 
+        If "answer" = True. must have the following field to make answer annotation
+        If "process" = True. You need provide the full information of the tree searching process
+
         "answer_generation": {
             "valid_data": bool,
             "final_answer": string,
-            "chain": [
-                {
-                    "thought":string,
-                    "action_name":string,
-                    "action_input":string,
-                    "observation": string,
-                }
-            ],
+            "finish_type": enum["give_up","give_answer"]
+            "train_messages": [ [openAI-message] ],
         }
-        as answer data
         '''
         raise NotImplementedError
 
-    def start(self):
+    def start(self, **args):
+        """This is the entry point of the searching process"""
         raise NotImplementedError
 

toolbench/inference/Algorithms/single_chain.py

@@ -5,12 +5,11 @@
 from copy import deepcopy
 
 class single_chain(base_search_method):
-
+    """Implement of CoT method
+    """
     def __init__(self,llm,io_func,extra_prefix="",process_id=0,start_message_list=None):
-        super(single_chain, self).__init__()
-        '''
-        Linearly generate thought, action, action input loop
-        '''
+        """extra_prefix and start_message_list is used in Reflection Algo"""
+        super(single_chain, self).__init__(llm,io_func, process_id, callbacks=None)
         self.io_func = io_func
         self.llm = llm
         self.extra_prefix = extra_prefix
@@ -61,6 +60,9 @@ def to_json(self, answer=False,process=True):
         return json_obj
 
     def to_json_single(self):
+        """parse the last try
+        Though the nodes are formed as a tree, We still know they are actually a chain
+        """
         json_obj = {}
         tree_obj = self.terminal_node[-1].get_chain_result_from_this_node()
         json_obj["chain"] = tree_obj
@@ -73,7 +75,8 @@ def start(self,single_chain_max_step,pass_at=1,answer=1):
             self.forward_args.pop("self")
 
         for i in range(pass_at):
-            print(f"[process({self.process_id})][single_chain]try for the {i+1} time")
+            if self.process_id == 0:
+                print(f"[single_chain]try for the {i+1} time")
             self.tree = my_tree()
             self.tree.root.node_type = "Action Input"
             self.tree.root.io_state = deepcopy(self.io_func)
@@ -87,16 +90,9 @@ def start(self,single_chain_max_step,pass_at=1,answer=1):
                     return 1
         return 0
 
+
     def do_chain(self,now_node,single_chain_max_step):
-        if callable(self.llm):
-            return self.do_chain_react(now_node,single_chain_max_step)
-        else:
-            return self.do_chain_function(now_node,single_chain_max_step)
 
-    def do_chain_function(self,now_node,single_chain_max_step):
-        '''
-        initialize root's self.messages to generate system and user
-        '''
         if self.start_message_list == None:
             system = FORMAT_INSTRUCTIONS_SYSTEM_FUNCTION
             system = system.replace("{task_description}",self.io_func.task_description)
@@ -106,10 +102,12 @@ def do_chain_function(self,now_node,single_chain_max_step):
             user = user.replace("{input_description}",self.io_func.input_description)
             self.tree.root.messages.append({"role":"user","content":user})
         else:
+            """In Reflection Algo, we startswith former trials and reflections, so the caller will give the start messages"""
             self.tree.root.messages = self.start_message_list
 
         now_node = self.tree.root
         while True:
+            # recursively parse message into nodes
             self.llm.change_messages(now_node.messages)
             new_message,error_code,total_tokens = self.llm.parse(functions=self.io_func.functions,process_id=self.process_id)
             self.total_tokens += total_tokens
@@ -168,11 +166,7 @@ def do_chain_function(self,now_node,single_chain_max_step):
                 now_node = temp_node
 
                 if status != 0:
-                    # 0 means normal return
-                    # 1 means there is no corresponding api name
-                    # 2 means there is an error in the input
-                    # 3 represents the end of the generation, and the final answer appears
-                    # 4 means that the model decides to pruning by itself
+                    # return code refers to Downstream_tasks/rapidapi
                     if status == 4:
                         now_node.pruned = True
                     elif status == 1: # hallucination api name

toolbench/inference/Downstream_tasks/rapidapi.py

@@ -25,13 +25,17 @@
 
 # For pipeline environment preparation
 def get_white_list(tool_root_dir):
+    # print(tool_root_dir)
     white_list_dir = os.path.join(tool_root_dir)
     white_list = {}
     for cate in tqdm(os.listdir(white_list_dir)):
+        if not os.path.isdir(os.path.join(white_list_dir,cate)):
+            continue
         for file in os.listdir(os.path.join(white_list_dir,cate)):
             if not file.endswith(".json"):
                 continue
             standard_tool_name = file.split(".")[0]
+            # print(standard_tool_name)
             with open(os.path.join(white_list_dir,cate,file)) as reader:
                 js_data = json.load(reader)
             origin_tool_name = js_data["tool_name"]
@@ -279,14 +283,21 @@ def step(self,**args):
         return obs, code
 
     def _step(self, action_name="", action_input=""):
-        '''
-        Need to return an observation string and status code:
-        0 means normal response
-        1 means there is no corresponding api name
-        2 means there is an error in the input
-        3 represents the end of the generation and the final answer appears
-        4 means that the model decides to pruning by itself
-        '''
+        """Need to return an observation string and status code:
+            0 means normal response
+            1 means there is no corresponding api name
+            2 means there is an error in the input
+            3 represents the end of the generation and the final answer appears
+            4 means that the model decides to pruning by itself
+            5 represents api call timeout
+            6 for 404
+            7 means not subscribed
+            8 represents unauthorized
+            9 represents too many requests
+            10 stands for rate limit
+            11 message contains "error" field
+            12 error sending request
+        """
         if action_name == "Finish":
             try:
                 json_data = json.loads(action_input,strict=False)

toolbench/inference/LLM_rank/rank_candidate.py

@@ -2,7 +2,7 @@
 Evaluate the score of a query corresponding to different candidates
 '''
 
-from Prompts.rank_prompts import LLM_PAIRWISE_RANK_ALLCHAIN_SYSTEM_PROMPT,LLM_PAIRWISE_RANK_SUBFIX_SYSTEM_PROMPT, LLM_PAIRWISE_RANK_USER_PROMPT
+from Prompts.rank_prompts import LLM_PAIRWISE_RANK_SUBFIX_SYSTEM_PROMPT, LLM_PAIRWISE_RANK_USER_PROMPT
 import random
 from Tree.Tree import tree_node
 
@@ -25,71 +25,6 @@ def rank2symmetry(llm_interface, LLM_rank_args, cand1,cand2):
         return 0, query_count1 + query_count2, total_tokens1 + total_tokens2
 
 
-def rank2_allchain(llm_interface,LLM_rank_args, cand1,cand2):
-    '''
-    cand1 first,
-    Return whether the sample on the left is better and the number of queries
-    1. The implementation of this function is related to downstream tasks. It is recommended to only modify the implementation of this function. The required parameters are passed in from the LLM_rank_args parameter upstream of the waterfall.
-    2. This implementation is the sorting algorithm of the multitool-multiapi-ETS algorithm
-    '''
-
-    system_message = LLM_PAIRWISE_RANK_ALLCHAIN_SYSTEM_PROMPT
-    system_message = system_message.replace("{task_description}", LLM_rank_args["task_description"])
-    cand1_des = cand1.get_former_trice_from_this_node()
-    cand2_des = cand2.get_former_trice_from_this_node()
-    system_message = system_message.replace("{candidate_A}",cand1_des)
-    system_message = system_message.replace("{candidate_B}",cand2_des)
-
-
-    llm_interface.change_messages([{"role":"system","content":system_message},
-                                   {"role":"user","content":LLM_PAIRWISE_RANK_USER_PROMPT},
-                                   ])
-    output,error_code,total_tokens = llm_interface.parse(functions=LLM_rank_args["functions"],function_call="none",process_id=LLM_rank_args["process_id"])
-    if output["content"].strip().lower()[-1] == "a":
-        return 1, 1,total_tokens
-    else:
-        return 0, 1,total_tokens
-
-def rank2_allchain_candidate_list(llm_interface,LLM_rank_args, cand1,cand2):
-    '''
-    cand1 first, specially designed for candidates
-    Return whether the sample on the left is better and the number of queries
-    1. The implementation of this function is related to downstream tasks. It is recommended to only modify the implementation of this function. The required parameters are passed in from the LLM_rank_args parameter upstream of the waterfall.
-    2. This implementation is the sorting algorithm of the multitool-multiapi-ETS algorithm
-    '''
-    def node_list_to_former_trice(node_list):
-        output_str_list = []
-
-        for node in node_list:
-            now_node_des_list = []
-            now_node_des_list.append(f"{node['node_type']}: {node['description']}\n")
-            if "observation" in node.keys() and node["observation"] != "":
-                now_node_des_list.append(f"observation: {node['observation']}\n")
-            output_str_list = output_str_list + now_node_des_list
-        now_str = ""
-        for k, cont in enumerate(output_str_list):
-            now_str += f"step_{k+1}: {cont}\n"
-
-        if now_str == "":
-            now_str = "None"
-        return now_str
-
-    system_message = LLM_PAIRWISE_RANK_ALLCHAIN_SYSTEM_PROMPT
-    system_message = system_message.replace("{task_description}", LLM_rank_args["task_description"])
-    cand1_des = node_list_to_former_trice(cand1["cont"])
-    cand2_des = node_list_to_former_trice(cand2["cont"])
-    system_message = system_message.replace("{candidate_A}",cand1_des)
-    system_message = system_message.replace("{candidate_B}",cand2_des)
-
-
-    llm_interface.change_messages([{"role":"system","content":system_message},
-                                   {"role":"user","content":LLM_PAIRWISE_RANK_USER_PROMPT},
-                                   ])
-    output,error_code,total_tokens = llm_interface.parse(functions=LLM_rank_args["functions"],function_call="none",process_id=LLM_rank_args["process_id"])
-    if output["content"].strip().lower()[-1] == "a":
-        return 1, 1,total_tokens
-    else:
-        return 0, 1,total_tokens
 
 def rank2_subfix(llm_interface,LLM_rank_args, cand1,cand2):
     '''
@@ -137,25 +72,6 @@ def sum_based_rankn(llm_interface,LLM_rank_args, candidates):
     return scores, total_querys, total_tokens
 
 
-def quick_sort_rank(candidates):
-    '''
-    LLM quick sort, sort from small to large
-    '''
-    if len(candidates) <= 1:
-        return candidates
-    pos = random.randint(0,len(candidates)-1)
-    left,right = [], []
-    for k in range(len(candidates)):
-        if k == pos:
-            continue
-        out = rank2symmetry(candidates[pos],candidates[k])
-        if out > 0:
-            left.append(candidates[k])
-        else:
-            right.append(candidates[k])
-
-    return quick_sort_rank(left) + [candidates[pos]] + quick_sort_rank(right)
-
 
 if __name__ ==  "__main__":
     random.seed(42)
@@ -176,7 +92,6 @@ def quick_sort_rank(candidates):
         "234",
         "ln(2)"
     ]
-    output = quick_sort_rank(candidates)
     '''
     starting_delta:
     50 -> 42.85%

toolbench/inference/Prompts/ReAct_prompts.py

@@ -1,28 +1,4 @@
 
-PREFIX = """Do the following tasks as best you can. You have access to the following tools:"""
-FORMAT_INSTRUCTIONS = """Use the following format:
-
-Task: the task you must handle
-Thought: you should always think about what to do
-Action: the action to take, should be one of {tool_names}
-Action Input: the input to the action
-Observation: the result of the action
-... (this Thought/Action/Action Input/Observation can repeat N times)
-Thought: I now know the final answer. (or) I give up and retry.
-Final Answer: the final answer to the original input question. (or) I give up and try again.
-Here is the task:
-{task_description}
-{input_description}
-Begin!
-{former_trice}"""
-
-
-
-REACT_DIVERSE_PROMPT = '''There are some former choices.
-**********************************
-{previous_candidate}**********************************
-I will make Action that is different from all of the above.
-'''
 
 
 FORMAT_INSTRUCTIONS_SYSTEM_FUNCTION = """You are AutoGPT, you can use many tools(functions) to do the following task.