forked from jennhu/metalinguistic-prompting
-
Notifications
You must be signed in to change notification settings - Fork 0
/
run_exp2_openai.sh
40 lines (32 loc) · 1.25 KB
/
run_exp2_openai.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
#!/bin/bash
CORPUS="dtfit"
MODEL=$1 # text-curie-001, text-davinci-002, text-davinci-003
KEYFILE="key.txt"
RESULTDIR="results/exp2_word-comparison"
DATAFILE="datasets/exp2/${CORPUS}/corpus.csv"
mkdir -p $RESULTDIR
# Helper function
run_experiment () {
# Capture relevant variables
local EVAL_TYPE=$1
local OPTION_ORDER=$2
# Define variable-dependent file/folder names
OUTFILE="${RESULTDIR}/${CORPUS}_${MODEL}_${EVAL_TYPE}_${OPTION_ORDER}.json"
# Run the evaluation script
echo "Running Experiment 2 (word comparison): model = ${MODEL}; eval_type = ${EVAL_TYPE}; option_order = ${OPTION_ORDER}" >&2
python run_exp2_word-comparison.py \
--model $MODEL \
--model_type "openai" \
--key $KEYFILE \
--option_order ${OPTION_ORDER} \
--eval_type ${EVAL_TYPE} \
--data_file $DATAFILE --out_file ${OUTFILE}
}
# FIRST, run the "direct" model. Option order doesn't matter, so set it to goodFirst.
run_experiment "direct" "goodFirst"
# NEXT, run the other models, crossing prompt method with option order.
for OPTION_ORDER in "goodFirst" "badFirst"; do
for EVAL_TYPE in "metaQuestionSimple" "metaInstruct" "metaQuestionComplex"; do
run_experiment "${EVAL_TYPE}" "${OPTION_ORDER}"
done
done