Skip to content

Commit

Permalink
redid subject ids
Browse files Browse the repository at this point in the history
  • Loading branch information
Martha authored and Martha committed Dec 8, 2024
1 parent c91fe40 commit ac21db9
Show file tree
Hide file tree
Showing 9 changed files with 2,177 additions and 2,155 deletions.
Binary file modified letterstring/plotting/all_gen.pdf
View file
Binary file not shown.
Binary file modified letterstring/plotting/all_perm_gen.pdf
View file
Binary file not shown.
Binary file modified letterstring/plotting/gpt_all_promptstyles.pdf
View file
Binary file not shown.
Binary file modified letterstring/plotting/gpt_human_gen.pdf
View file
Binary file not shown.
Binary file modified letterstring/plotting/num_gens_webb.pdf
View file
Binary file not shown.
Binary file modified letterstring/plotting/num_gens_webb_human.pdf
View file
Binary file not shown.
44 changes: 33 additions & 11 deletions letterstring/plotting/plotting_gen.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": 60,
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -21,7 +21,7 @@
},
{
"cell_type": "code",
"execution_count": 61,
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -33,14 +33,19 @@
},
{
"cell_type": "code",
"execution_count": 62,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Overall human accuracy is: 0.324, confidence intervals 0.304, 0.344\n"
"ename": "FileNotFoundError",
"evalue": "[Errno 2] No such file or directory: '../problems/gpt_human_alphs/all_prob_1_7_gpt_human_alphs.npz'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[5], line 19\u001b[0m\n\u001b[1;32m 17\u001b[0m npz \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mload(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m../problems/gpt_human_alphs/all_prob_\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mi\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m_14_gpt_human_alphs.npz\u001b[39m\u001b[38;5;124m'\u001b[39m, allow_pickle\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m 18\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m---> 19\u001b[0m npz \u001b[38;5;241m=\u001b[39m \u001b[43mnp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43mf\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m../problems/gpt_human_alphs/all_prob_\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mi\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m_7_gpt_human_alphs.npz\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mallow_pickle\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m 21\u001b[0m prob_dict[\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mnp_\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mi\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m npz[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mall_prob\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39mitem()\n\u001b[1;32m 23\u001b[0m source_1 \u001b[38;5;241m=\u001b[39m []\n",
"File \u001b[0;32m/opt/anaconda3/envs/gpt_analogy/lib/python3.9/site-packages/numpy/lib/npyio.py:427\u001b[0m, in \u001b[0;36mload\u001b[0;34m(file, mmap_mode, allow_pickle, fix_imports, encoding, max_header_size)\u001b[0m\n\u001b[1;32m 425\u001b[0m own_fid \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 426\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 427\u001b[0m fid \u001b[38;5;241m=\u001b[39m stack\u001b[38;5;241m.\u001b[39menter_context(\u001b[38;5;28;43mopen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mos_fspath\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfile\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mrb\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m)\n\u001b[1;32m 428\u001b[0m own_fid \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m 430\u001b[0m \u001b[38;5;66;03m# Code to distinguish from NumPy binary files and pickles.\u001b[39;00m\n",
"\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '../problems/gpt_human_alphs/all_prob_1_7_gpt_human_alphs.npz'"
]
}
],
Expand All @@ -61,9 +66,9 @@
"\n",
"for i in perms:\n",
" if i =='symb':\n",
" npz = np.load(f'../problems/gpt_human_alphs/all_prob_{i}_14_gpt_human_alphs.npz', allow_pickle=True)\n",
" npz = np.load(f'../problems/gen/all_prob_{i}_14_gpt_human_alphs.npz', allow_pickle=True)\n",
" else:\n",
" npz = np.load(f'../problems/gpt_human_alphs/all_prob_{i}_7_gpt_human_alphs.npz', allow_pickle=True)\n",
" npz = np.load(f'../problems/gen/all_prob_{i}_7_gpt_human_alphs.npz', allow_pickle=True)\n",
"\n",
" prob_dict[f'np_{i}'] = npz['all_prob'].item()\n",
"\n",
Expand Down Expand Up @@ -125,13 +130,27 @@
},
{
"cell_type": "code",
"execution_count": 63,
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"177\n",
" subj_id model promptstyle nperms alph prob_type prob_ind \\\n",
"0 178 human human np_10 alph_1 succ 4.0 \n",
"1 178 human human np_10 alph_5 add_letter 9.0 \n",
"2 178 human human np_10 alph_3 attn 9.0 \n",
"3 178 human human np_10 alph_0 fix_alphabet 3.0 \n",
"4 178 human human np_10 alph_2 remove_redundant 7.0 \n",
"\n",
" response_string \n",
"0 gyik \n",
"1 fgrij \n",
"2 aaaa \n",
"3 elgai \n",
"4 vhefg \n",
"Overall human accuracy is: 0.754, confidence intervals 0.734, 0.773\n",
"Overall human accuracy unpermuted is: 0.822, confidence intervals 0.777, 0.868\n",
"Num samples is 276\n"
Expand All @@ -152,9 +171,12 @@
"\n",
"human_df = pd.read_csv('../results_csvs/human_nogen.csv')\n",
"human_df = human_df.fillna('')\n",
"print(max(human_df_gen['subj_id']))\n",
"human_df['subj_id'] = human_df['subj_id']+max(human_df_gen['subj_id'])\n",
"print(human_df.head())\n",
"\n",
"for i in perms:\n",
" npz = np.load(f'../problems/human/all_prob_{i}_7_human.npz', allow_pickle=True)\n",
" npz = np.load(f'../problems/nogen/all_prob_{i}_7_human.npz', allow_pickle=True)\n",
" prob_dict[f'np_{i}'] = npz['all_prob'].item()\n",
"\n",
"source_1 = []\n",
Expand Down
Binary file modified letterstring/plotting/prob_types_prolific_horiz.pdf
View file
Binary file not shown.
Loading

0 comments on commit ac21db9

Please sign in to comment.