Merge pull request #23 from EleutherAI/categories_across_scale_time

Categories Across Scale & Time
EleutherAI · Jun 4, 2024 · 5fc689a · 5fc689a
2 parents 0c4e010 + 5504999
commit 5fc689a
Show file tree

Hide file tree

Showing 22 changed files with 5,334 additions and 323 deletions.
diff --git a/working_dirs/kyle/results_analysis/calculate_kl.py b/working_dirs/kyle/results_analysis/calculate_kl.py
@@ -0,0 +1,18 @@
+from scipy.stats import entropy
+
+def calculate_kl_divergence(all_memories_and_sample, memories_dataset, duplication_count):
+    all_memories_bootstrap_sample = all_memories_and_sample[all_memories_and_sample["sequence_duplicates"] == duplication_count].sample(frac=1, replace=True)
+    memories_at_duplication_count = memories_dataset[memories_dataset["sequence_duplicates"] == duplication_count]["generation_perplexity"]
+    non_memories_at_duplication_count = all_memories_bootstrap_sample[all_memories_bootstrap_sample["sequence_duplicates"] == duplication_count]["generation_perplexity"]
+
+    # downsample to calculate kl divergence
+    balance_size = min(len(memories_at_duplication_count), len(non_memories_at_duplication_count))
+    memories_at_duplication_count = memories_at_duplication_count.sample(balance_size)
+    non_memories_at_duplication_count = non_memories_at_duplication_count.sample(balance_size)
+
+    # generate kl divergence in perplexity with memories as the true distribution
+    kl_divergence = entropy(memories_at_duplication_count, non_memories_at_duplication_count)
+    return {
+        "Duplicates": duplication_count,
+        "KL Divergence": kl_divergence
+    }
diff --git a/working_dirs/kyle/results_analysis/figure_categories_count_percents.png b/working_dirs/kyle/results_analysis/figure_categories_count_percents.png
diff --git a/working_dirs/kyle/results_analysis/hist.ipynb b/working_dirs/kyle/results_analysis/hist.ipynb
diff --git a/working_dirs/kyle/results_analysis/metrics_analysis.png b/working_dirs/kyle/results_analysis/metrics_analysis.png
diff --git a/working_dirs/kyle/results_analysis/ppl_dup_figure.ipynb b/working_dirs/kyle/results_analysis/ppl_dup_figure.ipynb
diff --git a/working_dirs/kyle/results_analysis/results_analysis.ipynb b/working_dirs/kyle/results_analysis/results_analysis.ipynb
diff --git a/working_dirs/kyle/results_analysis/results_analysis_whole_sequence.ipynb b/working_dirs/kyle/results_analysis/results_analysis_whole_sequence.ipynb
diff --git a/...g_dirs/kyle/results_analysis/scale+time_figures/ppl_figs/kl_divergence_generation_ppl.pdf b/...g_dirs/kyle/results_analysis/scale+time_figures/ppl_figs/kl_divergence_generation_ppl.pdf
diff --git a/...esults_analysis/scale+time_figures/ppl_figs/kl_divergence_generation_ppl_bootstrapped.pdf b/...esults_analysis/scale+time_figures/ppl_figs/kl_divergence_generation_ppl_bootstrapped.pdf
diff --git a/...nalysis/scale+time_figures/recitation_threshold_2/categories_counts_across_time+scale.pdf b/...nalysis/scale+time_figures/recitation_threshold_2/categories_counts_across_time+scale.pdf
diff --git a/...cale+time_figures/recitation_threshold_2/categories_counts_percents_across_time+scale.pdf b/...cale+time_figures/recitation_threshold_2/categories_counts_percents_across_time+scale.pdf
diff --git a/...ures/recitation_threshold_2/categories_counts_percents_across_time+scale_bounded_bars.pdf b/...ures/recitation_threshold_2/categories_counts_percents_across_time+scale_bounded_bars.pdf
diff --git a/...lysis/scale+time_figures/recitation_threshold_2/categories_percents_across_time+scale.pdf b/...lysis/scale+time_figures/recitation_threshold_2/categories_percents_across_time+scale.pdf
diff --git a/...nalysis/scale+time_figures/recitation_threshold_5/categories_counts_across_time+scale.pdf b/...nalysis/scale+time_figures/recitation_threshold_5/categories_counts_across_time+scale.pdf
diff --git a/...cale+time_figures/recitation_threshold_5/categories_counts_percents_across_time+scale.pdf b/...cale+time_figures/recitation_threshold_5/categories_counts_percents_across_time+scale.pdf
diff --git a/...ures/recitation_threshold_5/categories_counts_percents_across_time+scale_bounded_bars.pdf b/...ures/recitation_threshold_5/categories_counts_percents_across_time+scale_bounded_bars.pdf
diff --git a/...lysis/scale+time_figures/recitation_threshold_5/categories_percents_across_time+scale.pdf b/...lysis/scale+time_figures/recitation_threshold_5/categories_percents_across_time+scale.pdf
diff --git a/...s/kyle/results_analysis/scale+time_figures/recitation_threshold_5/histograms_percents.pdf b/...s/kyle/results_analysis/scale+time_figures/recitation_threshold_5/histograms_percents.pdf
diff --git a/...nalysis/scale+time_figures/recitation_threshold_6/categories_counts_across_time+scale.pdf b/...nalysis/scale+time_figures/recitation_threshold_6/categories_counts_across_time+scale.pdf
diff --git a/...cale+time_figures/recitation_threshold_6/categories_counts_percents_across_time+scale.pdf b/...cale+time_figures/recitation_threshold_6/categories_counts_percents_across_time+scale.pdf
diff --git a/...ures/recitation_threshold_6/categories_counts_percents_across_time+scale_bounded_bars.pdf b/...ures/recitation_threshold_6/categories_counts_percents_across_time+scale_bounded_bars.pdf
diff --git a/...lysis/scale+time_figures/recitation_threshold_6/categories_percents_across_time+scale.pdf b/...lysis/scale+time_figures/recitation_threshold_6/categories_percents_across_time+scale.pdf