[ENH] adding simplified glm (#372)

tatsu-lab · Jul 18, 2024 · 19866ff · 19866ff
1 parent aecedce
commit 19866ff
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 0 deletions.
diff --git a/src/alpaca_eval/metrics/glm_winrate.py b/src/alpaca_eval/metrics/glm_winrate.py
@@ -24,6 +24,16 @@
         "regularize_to_baseline_lambda": 0.2,
         "kwargs": {"n_splits": 5},
     },
+    "length_controlled_noreg": {
+        "formula": "np.tanh(std_delta_len) + instruction_difficulty + not_gamed_baseline.astype(float) - 1",
+        "regularize_to_baseline_lambda": None,
+        "kwargs": {"n_splits": 5},
+    },
+    "length_controlled_minimal": {
+        "formula": "np.tanh(std_delta_len) + not_gamed_baseline.astype(float) - 1",
+        "regularize_to_baseline_lambda": None,
+        "kwargs": {"n_splits": 5},
+    },
 }
 DFLT_WEIGHT_PATH = (
     Path(__file__).parent

diff --git a/...ights/weighted_alpaca_eval_gpt4_turbo/length_controlled_v1/baseline_gpt4_1106_preview.csv b/...ights/weighted_alpaca_eval_gpt4_turbo/length_controlled_v1/baseline_gpt4_1106_preview.csv
@@ -173,3 +173,4 @@ SPPO-Gemma-2-9B-It-PairRM,-1.1139152907711427,0.5972758612054220,0.2481716650556
 gemma-2-9b-it-DPO,-1.0421098127771280,0.7544689135365252,0.9950063245939248
 gemma-2-9b-it-SimPO,-1.1421073244366444,0.6125150070394807,1.1709131554933978
 higgs-llama-3-70b-v2,-1.3408055191105048,0.9224458425462844,0.4939211483441316
+example,-1.8366972159920483,0.7446561168325255,-6.8226040527680167