Skip to content

Commit

Permalink
perf: output colored table and update model baseline API (#166)
Browse files Browse the repository at this point in the history
  • Loading branch information
@junewgl
junewgl authored Dec 6, 2023
1 parent 96aaabd commit e1fbd2c
Show file tree
Hide file tree
Showing 4 changed files with 280 additions and 162 deletions.
5 changes: 2 additions & 3 deletions dbgpt_hub/baseline/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
==============
"""

from .show_result_api import show_all
from .show_result_api import show_model
from .show_result_api import show_score, show_scores

__all__ = ["show_all", "show_model"]
__all__ = ["show_score", "show_scores"]
227 changes: 164 additions & 63 deletions dbgpt_hub/baseline/baseline.json
View file
Original file line number Diff line number Diff line change
@@ -1,74 +1,175 @@
{
"spider": {
"llama2-7b-hf": {
"base": {
"alpaca":{
"instruction": "I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.\\n\\\"\\n##Instruction:\\ndepartment_management contains tables such as department, head, management. Table department has columns such as Department_ID, Name, Creation, Ranking, Budget_in_Billions, Num_Employees. Department_ID is the primary key.\\nTable head has columns such as head_ID, name, born_state, age. head_ID is the primary key.\\nTable management has columns such as department_ID, head_ID, temporary_acting. department_ID is the primary key.\\nThe head_ID of management is the foreign key of head_ID of head.\\nThe department_ID of management is the foreign key of Department_ID of department.\\n\\n",
"acc": {
"ex":{
"easy": 0.1,
"medium": 0.1,
"hard": 0.1,
"extra": 0.1,
"all": 0.1
},
"em":{
"easy": 0.1,
"medium": 0.1,
"hard": 0.1,
"extra": 0.1,
"all": 0.1
}
"spider": {
"llama2-7b-hf": {
"base": {
"alpaca":{
"instruction": "I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.\\n\\\"\\n##Instruction:\\ndepartment_management contains tables such as department, head, management. Table department has columns such as Department_ID, Name, Creation, Ranking, Budget_in_Billions, Num_Employees. Department_ID is the primary key.\\nTable head has columns such as head_ID, name, born_state, age. head_ID is the primary key.\\nTable management has columns such as department_ID, head_ID, temporary_acting. department_ID is the primary key.\\nThe head_ID of management is the foreign key of head_ID of head.\\nThe department_ID of management is the foreign key of Department_ID of department.\\n\\n",
"acc": {
"ex":{
"easy": 0.1,
"medium": 0.1,
"hard": 0.1,
"extra": 0.1,
"all": 0.1
},
"em":{
"easy": 0.1,
"medium": 0.1,
"hard": 0.1,
"extra": 0.1,
"all": 0.1
}
},
"openai":{
"instruction": "openai-instruction",
"acc": {
"ex":{
"easy": 0.887,
"medium": 0.711,
"hard": 0.575,
"extra": 0.380,
"all": 0.677
},
"em":{
"easy": 0.887,
"medium": 0.711,
"hard": 0.575,
"extra": 0.380,
"all": 0.677
}
}
},
"openai":{
"instruction": "openai-instruction",
"acc": {
"ex":{
"easy": 0.887,
"medium": 0.711,
"hard": 0.575,
"extra": 0.380,
"all": 0.677
},
"em":{
"easy": 0.887,
"medium": 0.711,
"hard": 0.575,
"extra": 0.380,
"all": 0.677
}
}
}
},
"lora":{
"alpaca":{
"instruction": "I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.\\n\\\"\\n##Instruction:\\ndepartment_management contains tables such as department, head, management. Table department has columns such as Department_ID, Name, Creation, Ranking, Budget_in_Billions, Num_Employees. Department_ID is the primary key.\\nTable head has columns such as head_ID, name, born_state, age. head_ID is the primary key.\\nTable management has columns such as department_ID, head_ID, temporary_acting. department_ID is the primary key.\\nThe head_ID of management is the foreign key of head_ID of head.\\nThe department_ID of management is the foreign key of Department_ID of department.\\n\\n",
"acc": {
"ex":{
"easy": 0.1,
"medium": 0.1,
"hard": 0.1,
"extra": 0.1,
"all": 0.1
},
"em":{
"easy": 0.1,
"medium": 0.1,
"hard": 0.1,
"extra": 0.1,
"all": 0.1
}
}
}
},
"qlora": {
"openai":{
"instruction": "I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.\\n\\\"\\n##Instruction:\\ndepartment_management contains tables such as department, head, management. Table department has columns such as Department_ID, Name, Creation, Ranking, Budget_in_Billions, Num_Employees. Department_ID is the primary key.\\nTable head has columns such as head_ID, name, born_state, age. head_ID is the primary key.\\nTable management has columns such as department_ID, head_ID, temporary_acting. department_ID is the primary key.\\nThe head_ID of management is the foreign key of head_ID of head.\\nThe department_ID of management is the foreign key of Department_ID of department.\\n\\n",
"acc": {
"ex":{
"easy": 0.1,
"medium": 0.1,
"hard": 0.1,
"extra": 0.1,
"all": 0.1
},
"em":{
"easy": 0.1,
"medium": 0.1,
"hard": 0.1,
"extra": 0.1,
"all": 0.1
}
}
}
},
"ppo":{}
},
"llama2-7b-chat-hf": {
"ppo": {
"alpaca":{
"instruction": "I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.\\n\\\"\\n##Instruction:\\ndepartment_management contains tables such as department, head, management. Table department has columns such as Department_ID, Name, Creation, Ranking, Budget_in_Billions, Num_Employees. Department_ID is the primary key.\\nTable head has columns such as head_ID, name, born_state, age. head_ID is the primary key.\\nTable management has columns such as department_ID, head_ID, temporary_acting. department_ID is the primary key.\\nThe head_ID of management is the foreign key of head_ID of head.\\nThe department_ID of management is the foreign key of Department_ID of department.\\n\\n",
"acc": {
"ex":{
"easy": 0.1,
"medium": 0.1,
"hard": 0.1,
"extra": 0.1,
"all": 0.1
},
"em":{
"easy": 0.1,
"medium": 0.1,
"hard": 0.1,
"extra": 0.1,
"all": 0.1
}
}
},
"lora": {
"alpaca":{
"instruction": "test",
"acc": {
"ex":{
"easy": 0.887,
"medium": 0.711,
"hard": 0.575,
"extra": 0.380,
"all": 0.677
},
"em":{
"easy": 0.887,
"medium": 0.711,
"hard": 0.575,
"extra": 0.380,
"all": 0.677
}
"openai":{
"instruction": "openai-instruction",
"acc": {
"ex":{
"easy": 0.887,
"medium": 0.711,
"hard": 0.575,
"extra": 0.380,
"all": 0.677
},
"em":{
"easy": 0.887,
"medium": 0.711,
"hard": 0.575,
"extra": 0.380,
"all": 0.677
}
}
}
}
}
},
"bird":{
"llama2-7b-chat-hf": {
"ppo": {
"alpaca":{
"instruction": "I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.\\n\\\"\\n##Instruction:\\ndepartment_management contains tables such as department, head, management. Table department has columns such as Department_ID, Name, Creation, Ranking, Budget_in_Billions, Num_Employees. Department_ID is the primary key.\\nTable head has columns such as head_ID, name, born_state, age. head_ID is the primary key.\\nTable management has columns such as department_ID, head_ID, temporary_acting. department_ID is the primary key.\\nThe head_ID of management is the foreign key of head_ID of head.\\nThe department_ID of management is the foreign key of Department_ID of department.\\n\\n",
"acc": {
"ex":{
"easy": 0.1,
"medium": 0.1,
"hard": 0.1,
"extra": 0.1,
"all": 0.1
},
"em":{
"easy": 0.1,
"medium": 0.1,
"hard": 0.1,
"extra": 0.1,
"all": 0.1
}
}
},
"qlora": {

"openai":{
"instruction": "openai-instruction",
"acc": {
"ex":{
"easy": 0.887,
"medium": 0.711,
"hard": 0.575,
"extra": 0.380,
"all": 0.677
},
"em":{
"easy": 0.887,
"medium": 0.711,
"hard": 0.575,
"extra": 0.380,
"all": 0.677
}
}
}
,
"ppo":{}
},
"llama2-7b-chat-hf": {
}
}
}
}
}
Loading

0 comments on commit e1fbd2c

Please sign in to comment.