statistics_generations_with_text_descriptive.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /Users/DKWB2F7/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt \n",
    "import random\n",
    "import pandas as pd \n",
    "import math \n",
    "import json\n",
    "import nltk \n",
    "nltk.download('punkt')\n",
    "from tqdm import tqdm\n",
    "tqdm.pandas()\n",
    "from scipy.stats import ttest_ind"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<textdescriptives.load_components.TextDescriptives at 0x173670150>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import spacy\n",
    "import textdescriptives as td\n",
    "# load your favourite spacy model (remember to install it first using e.g. `python -m spacy download en_core_web_sm`)\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "nlp.add_pipe(\"textdescriptives/all\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_linguistic_feature(text):\n",
    "    D = {}\n",
    "    doc = nlp(text) \n",
    "    D['total_wc'] = doc._.descriptive_stats['n_tokens'] \n",
    "    D['avg_word_len'] = doc._.descriptive_stats['token_length_mean']\n",
    "    D['n_sentences'] = doc._.descriptive_stats['n_sentences']\n",
    "    D['sentence_length_mean'] = doc._.descriptive_stats['sentence_length_mean']\n",
    "    D['ttr'] = doc._.descriptive_stats['proportion_unique_tokens']\n",
    "    D['syllables_per_token_mean'] = doc._.descriptive_stats['syllables_per_token_mean']\n",
    "    D['stopwords_frac'] = doc._.quality.n_stop_words.value *100/D['total_wc'] \n",
    "    D['flesch_reading_ease'] = doc._.readability['flesch_reading_ease']\n",
    "    D['entropy'] = doc._.information_theory['entropy']\n",
    "    D['perplexity'] = doc._.information_theory['perplexity']\n",
    "    D['per_word_perplexity'] = doc._.information_theory['per_word_perplexity']\n",
    "    D['dependency_distance_mean'] = doc._.dependency_distance['dependency_distance_mean']\n",
    "    D['first_order_coherence'] = doc._.coherence['first_order_coherence']\n",
    "    D['second_order_coherence'] = doc._.coherence['second_order_coherence']\n",
    "    return  D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'total_wc': 35,\n",
       " 'avg_word_len': 3.2857142857142856,\n",
       " 'n_sentences': 5,\n",
       " 'sentence_length_mean': 7.0,\n",
       " 'ttr': 0.6571428571428571,\n",
       " 'syllables_per_token_mean': 1.0857142857142856,\n",
       " 'stopwords_frac': 68.57142857142857,\n",
       " 'flesch_reading_ease': 107.87857142857146,\n",
       " 'entropy': 2.022515801318652,\n",
       " 'perplexity': 7.557313737483889,\n",
       " 'per_word_perplexity': 0.1843247253044851,\n",
       " 'dependency_distance_mean': 1.961904761904762,\n",
       " 'first_order_coherence': 0.6005533672869205,\n",
       " 'second_order_coherence': 0.5181362926959991}"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = \"The world is changed. I feel it in the water. I feel it in the earth. I smell it in the air. Much that once was is lost, for none now live who remember it.\"\n",
    "get_linguistic_feature(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to collect values for each key from a list of dictionaries\n",
    "def collect_values(data):\n",
    "    values_dict = {}\n",
    "    for d in data:\n",
    "        for key, value in d.items():\n",
    "            if key not in values_dict:\n",
    "                values_dict[key] = []\n",
    "            values_dict[key].append(value)\n",
    "    return values_dict\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Partwise calculations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## n  = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0', 'prompt', 'human_story', 'author1', 'llm_story'], dtype='object')"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('1_author.csv')\n",
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "author1\n",
       "gemma      1794\n",
       "llama      1774\n",
       "mistral    1800\n",
       "olmo       1797\n",
       "dtype: int64"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby('author1').size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gemma\n",
      "====================================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1794/1794 [12:18<00:00,  2.43it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_wc:\n",
      "  human: 1172.84 ± 366.11\n",
      "  llm: 368.58 ± 34.86\n",
      "  p-value: 0.00000\n",
      "avg_word_len:\n",
      "  human: 3.92 ± 0.24\n",
      "  llm: 4.82 ± 0.27\n",
      "  p-value: 0.00000\n",
      "n_sentences:\n",
      "  human: 126.57 ± 55.14\n",
      "  llm: 29.07 ± 4.91\n",
      "  p-value: 0.00000\n",
      "sentence_length_mean:\n",
      "  human: 9.87 ± 2.51\n",
      "  llm: 12.97 ± 2.06\n",
      "  p-value: 0.00000\n",
      "ttr:\n",
      "  human: 0.36 ± 0.05\n",
      "  llm: 0.57 ± 0.03\n",
      "  p-value: 0.00000\n",
      "syllables_per_token_mean:\n",
      "  human: 1.19 ± 0.05\n",
      "  llm: 1.39 ± 0.08\n",
      "  p-value: 0.00000\n",
      "stopwords_frac:\n",
      "  human: 49.18 ± 6.05\n",
      "  llm: 47.69 ± 3.35\n",
      "  p-value: 0.00000\n",
      "flesch_reading_ease:\n",
      "  human: 96.46 ± 6.05\n",
      "  llm: 75.73 ± 7.56\n",
      "  p-value: 0.00000\n",
      "entropy:\n",
      "  human: 38.44 ± 12.20\n",
      "  llm: 15.50 ± 1.71\n",
      "  p-value: 0.00000\n",
      "perplexity:\n",
      "  human: 13551447155987963025912622930198528.00 ± 516158453813077790092565595367145472.00\n",
      "  llm: 32912107.89 ± 249770737.25\n",
      "  p-value: 0.26641\n",
      "per_word_perplexity:\n",
      "  human: 5100958117911535871877743378432.00 ± 194379954202003993301593805029376.00\n",
      "  llm: 62407.77 ± 413198.83\n",
      "  p-value: 0.26663\n",
      "dependency_distance_mean:\n",
      "  human: 1.93 ± 0.25\n",
      "  llm: 2.40 ± 0.17\n",
      "  p-value: 0.00000\n",
      "first_order_coherence:\n",
      "  human: 0.38 ± 0.05\n",
      "  llm: 0.47 ± 0.05\n",
      "  p-value: 0.00000\n",
      "second_order_coherence:\n",
      "  human: 0.27 ± 0.07\n",
      "  llm: 0.45 ± 0.05\n",
      "  p-value: 0.00000\n",
      "====================================================================================================\n",
      "llama\n",
      "====================================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1774/1774 [04:29<00:00,  6.59it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_wc:\n",
      "  human: 1172.84 ± 366.11\n",
      "  llm: 583.03 ± 225.53\n",
      "  p-value: 0.00000\n",
      "avg_word_len:\n",
      "  human: 3.92 ± 0.24\n",
      "  llm: 4.17 ± 0.26\n",
      "  p-value: 0.00000\n",
      "n_sentences:\n",
      "  human: 126.57 ± 55.14\n",
      "  llm: 40.20 ± 18.00\n",
      "  p-value: 0.00000\n",
      "sentence_length_mean:\n",
      "  human: 9.87 ± 2.51\n",
      "  llm: 15.18 ± 3.28\n",
      "  p-value: 0.00000\n",
      "ttr:\n",
      "  human: 0.36 ± 0.05\n",
      "  llm: 0.41 ± 0.08\n",
      "  p-value: 0.00000\n",
      "syllables_per_token_mean:\n",
      "  human: 1.19 ± 0.05\n",
      "  llm: 1.24 ± 0.07\n",
      "  p-value: 0.00000\n",
      "stopwords_frac:\n",
      "  human: 49.18 ± 6.05\n",
      "  llm: 61.36 ± 4.27\n",
      "  p-value: 0.00000\n",
      "flesch_reading_ease:\n",
      "  human: 96.46 ± 6.05\n",
      "  llm: 86.38 ± 7.42\n",
      "  p-value: 0.00000\n",
      "entropy:\n",
      "  human: 38.44 ± 12.20\n",
      "  llm: 25.67 ± 18.60\n",
      "  p-value: 0.00000\n",
      "perplexity:\n",
      "  human: 13551447155987963025912622930198528.00 ± 516158453813077790092565595367145472.00\n",
      "  llm: 5852669516980846499817407939307510038265994956461792054714842998266017472246068255720377022317136974258211245942124863324291694575006871417290812609364536852656898965504.00 ± inf\n",
      "  p-value: 1.00000\n",
      "per_word_perplexity:\n",
      "  human: 5100958117911535871877743378432.00 ± 194379954202003993301593805029376.00\n",
      "  llm: 2073953149822362452265686499429652426285239066088268336445555386863543482429948768842320075902480199818066850808561519702334334497469215547065509887037419146776150016.00 ± inf\n",
      "  p-value: 1.00000\n",
      "dependency_distance_mean:\n",
      "  human: 1.93 ± 0.25\n",
      "  llm: 3.04 ± 9.40\n",
      "  p-value: 0.00000\n",
      "first_order_coherence:\n",
      "  human: 0.38 ± 0.05\n",
      "  llm: 0.44 ± 0.06\n",
      "  p-value: 0.00000\n",
      "second_order_coherence:\n",
      "  human: 0.27 ± 0.07\n",
      "  llm: nan ± nan\n",
      "  p-value: nan\n",
      "====================================================================================================\n",
      "mistral\n",
      "====================================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1800/1800 [06:41<00:00,  4.49it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_wc:\n",
      "  human: 1172.84 ± 366.11\n",
      "  llm: 811.55 ± 195.35\n",
      "  p-value: 0.00000\n",
      "avg_word_len:\n",
      "  human: 3.92 ± 0.24\n",
      "  llm: 4.34 ± 0.26\n",
      "  p-value: 0.00000\n",
      "n_sentences:\n",
      "  human: 126.57 ± 55.14\n",
      "  llm: 49.25 ± 13.89\n",
      "  p-value: 0.00000\n",
      "sentence_length_mean:\n",
      "  human: 9.87 ± 2.51\n",
      "  llm: 16.90 ± 2.93\n",
      "  p-value: 0.00000\n",
      "ttr:\n",
      "  human: 0.36 ± 0.05\n",
      "  llm: 0.39 ± 0.05\n",
      "  p-value: 0.00000\n",
      "syllables_per_token_mean:\n",
      "  human: 1.19 ± 0.05\n",
      "  llm: 1.27 ± 0.07\n",
      "  p-value: 0.00000\n",
      "stopwords_frac:\n",
      "  human: 49.18 ± 6.05\n",
      "  llm: 58.20 ± 3.71\n",
      "  p-value: 0.00000\n",
      "flesch_reading_ease:\n",
      "  human: 96.46 ± 6.05\n",
      "  llm: 81.88 ± 7.05\n",
      "  p-value: 0.00000\n",
      "entropy:\n",
      "  human: 38.44 ± 12.20\n",
      "  llm: 33.57 ± 8.49\n",
      "  p-value: 0.00000\n",
      "perplexity:\n",
      "  human: 13551447155987963025912622930198528.00 ± 516158453813077790092565595367145472.00\n",
      "  llm: 2047006804070606038560757776384.00 ± 86497053012255762389970683166720.00\n",
      "  p-value: 0.26648\n",
      "per_word_perplexity:\n",
      "  human: 5100958117911535871877743378432.00 ± 194379954202003993301593805029376.00\n",
      "  llm: 1098940079255947397725945856.00 ± 46428910751532763272804564992.00\n",
      "  p-value: 0.26674\n",
      "dependency_distance_mean:\n",
      "  human: 1.93 ± 0.25\n",
      "  llm: 2.71 ± 0.17\n",
      "  p-value: 0.00000\n",
      "first_order_coherence:\n",
      "  human: 0.38 ± 0.05\n",
      "  llm: 0.46 ± 0.05\n",
      "  p-value: 0.00000\n",
      "second_order_coherence:\n",
      "  human: 0.27 ± 0.07\n",
      "  llm: 0.45 ± 0.05\n",
      "  p-value: 0.00000\n",
      "====================================================================================================\n",
      "olmo\n",
      "====================================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1797/1797 [05:51<00:00,  5.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_wc:\n",
      "  human: 1172.84 ± 366.11\n",
      "  llm: 720.87 ± 217.08\n",
      "  p-value: 0.00000\n",
      "avg_word_len:\n",
      "  human: 3.92 ± 0.24\n",
      "  llm: 4.37 ± 0.31\n",
      "  p-value: 0.00000\n",
      "n_sentences:\n",
      "  human: 126.57 ± 55.14\n",
      "  llm: 42.32 ± 16.17\n",
      "  p-value: 0.00000\n",
      "sentence_length_mean:\n",
      "  human: 9.87 ± 2.51\n",
      "  llm: 17.83 ± 3.84\n",
      "  p-value: 0.00000\n",
      "ttr:\n",
      "  human: 0.36 ± 0.05\n",
      "  llm: 0.38 ± 0.09\n",
      "  p-value: 0.00000\n",
      "syllables_per_token_mean:\n",
      "  human: 1.19 ± 0.05\n",
      "  llm: 1.29 ± 0.09\n",
      "  p-value: 0.00000\n",
      "stopwords_frac:\n",
      "  human: 49.18 ± 6.05\n",
      "  llm: 57.97 ± 4.91\n",
      "  p-value: 0.00000\n",
      "flesch_reading_ease:\n",
      "  human: 96.46 ± 6.05\n",
      "  llm: 79.89 ± 9.29\n",
      "  p-value: 0.00000\n",
      "entropy:\n",
      "  human: 38.44 ± 12.20\n",
      "  llm: 29.34 ± 9.45\n",
      "  p-value: 0.00000\n",
      "perplexity:\n",
      "  human: 13551447155987963025912622930198528.00 ± 516158453813077790092565595367145472.00\n",
      "  llm: 14501970882986753391657064272947576832.00 ± 614582507435598813812926437674224451584.00\n",
      "  p-value: 0.31790\n",
      "per_word_perplexity:\n",
      "  human: 5100958117911535871877743378432.00 ± 194379954202003993301593805029376.00\n",
      "  llm: 5638402365080386410658623677005824.00 ± 238951208178693234454991033327419392.00\n",
      "  p-value: 0.31788\n",
      "dependency_distance_mean:\n",
      "  human: 1.93 ± 0.25\n",
      "  llm: 2.89 ± 7.31\n",
      "  p-value: 0.00000\n",
      "first_order_coherence:\n",
      "  human: 0.38 ± 0.05\n",
      "  llm: 0.48 ± 0.06\n",
      "  p-value: 0.00000\n",
      "second_order_coherence:\n",
      "  human: 0.27 ± 0.07\n",
      "  llm: 0.48 ± 0.06\n",
      "  p-value: 0.00000\n",
      "====================================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "human_values = []\n",
    "human_calculation_done = False\n",
    "authors = ['gemma','llama','mistral','olmo']\n",
    "llm_values_1 = []\n",
    "for author in authors:\n",
    "    df_p = df[df['author1']==author]\n",
    "    print(author)\n",
    "    print('='*100)\n",
    "    llm_values = []\n",
    "    human_wc, human_avg_wc, human_sc, human_avg_sl, human_ttr, human_stopwords, human_readability = [],[],[],[],[],[],[]\n",
    "    llm_wc, llm_avg_wc, llm_sc, llm_avg_sl, llm_ttr, llm_stopwords, llm_readability = [],[],[],[],[],[],[]\n",
    "    for index, row in tqdm(df_p.iterrows(), total=df_p.shape[0]):\n",
    "        if not human_calculation_done:\n",
    "            human_values.append(get_linguistic_feature(row['human_story']))\n",
    "        llm_values.append(get_linguistic_feature(row['llm_story']))\n",
    "\n",
    "    human_calculation_done = True \n",
    "    values_dict1 = collect_values(human_values)\n",
    "    values_dict2 = collect_values(llm_values)\n",
    "\n",
    "    # Calculate and print the mean, standard deviation, and p-value for each key\n",
    "    for key in values_dict1.keys():\n",
    "        values1 = values_dict1[key]\n",
    "        values2 = values_dict2[key]\n",
    "        mean1 = np.mean(values1)\n",
    "        std_dev1 = np.std(values1)\n",
    "        mean2 = np.mean(values2)\n",
    "        std_dev2 = np.std(values2)\n",
    "        \n",
    "        # Perform independent t-test with Welch’s correction for unequal variances\n",
    "        t_stat, p_value = ttest_ind(values1, values2, equal_var=False)\n",
    "        \n",
    "        # Print results\n",
    "        print(f\"{key}:\")\n",
    "        print(f\"  human: {mean1:.2f} ± {std_dev1:.2f}\")\n",
    "        print(f\"  llm: {mean2:.2f} ± {std_dev2:.2f}\")\n",
    "        print(f\"  p-value: {p_value:.5f}\")\n",
    "    print('='*100)\n",
    "    llm_values_1 = llm_values_1 + llm_values\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_wc:\n",
      "  human: 1173.79 ± 367.82\n",
      "  llm: 818.63 ± 222.50\n",
      "  p-value: 0.00000\n",
      "avg_word_len:\n",
      "  human: 3.92 ± 0.24\n",
      "  llm: 4.33 ± 0.24\n",
      "  p-value: 0.00000\n",
      "n_sentences:\n",
      "  human: 126.64 ± 54.93\n",
      "  llm: 51.67 ± 16.80\n",
      "  p-value: 0.00000\n",
      "sentence_length_mean:\n",
      "  human: 126.64 ± 54.93\n",
      "  llm: 51.67 ± 16.80\n",
      "  p-value: 0.00000\n",
      "ttr:\n",
      "  human: 0.36 ± 0.05\n",
      "  llm: 0.39 ± 0.05\n",
      "  p-value: 0.00888\n",
      "syllables_per_token_mean:\n",
      "  human: 1.19 ± 0.05\n",
      "  llm: 1.26 ± 0.05\n",
      "  p-value: 0.00000\n",
      "stopwords_frac:\n",
      "  human: 49.19 ± 6.09\n",
      "  llm: 58.56 ± 3.56\n",
      "  p-value: 0.00000\n",
      "flesch_reading_ease:\n",
      "  human: 96.46 ± 6.05\n",
      "  llm: 83.37 ± 5.11\n",
      "  p-value: 0.00000\n",
      "entropy:\n",
      "  human: 38.49 ± 12.25\n",
      "  llm: 33.55 ± 9.36\n",
      "  p-value: 0.01306\n",
      "perplexity:\n",
      "  human: 13498785251096858996956582222233600.00 ± 515155082908459625412432661592408064.00\n",
      "  llm: 590662716993854752423936.00 ± 3007230335504000504299520.00\n",
      "  p-value: 0.26641\n",
      "per_word_perplexity:\n",
      "  human: 5081135752388379806843254341632.00 ± 194002094809484586561740432474112.00\n",
      "  llm: 339562483322898612224.00 ± 1728274215431946895360.00\n",
      "  p-value: 0.26663\n",
      "dependency_distance_mean:\n",
      "  human: 1.93 ± 0.25\n",
      "  llm: 2.67 ± 0.18\n",
      "  p-value: 0.00000\n",
      "first_order_coherence:\n",
      "  human: 0.38 ± 0.05\n",
      "  llm: 0.45 ± 0.03\n",
      "  p-value: 0.00000\n",
      "second_order_coherence:\n",
      "  human: 0.27 ± 0.07\n",
      "  llm: 0.44 ± 0.04\n",
      "  p-value: 0.00000\n"
     ]
    }
   ],
   "source": [
    "values_dict1 = collect_values(human_values)\n",
    "values_dict2 = collect_values(llm_values)\n",
    "\n",
    "# Calculate and print the mean, standard deviation, and p-value for each key\n",
    "for key in values_dict1.keys():\n",
    "    values1 = values_dict1[key]\n",
    "    values2 = values_dict2[key]\n",
    "    mean1 = np.mean(values1)\n",
    "    std_dev1 = np.std(values1)\n",
    "    mean2 = np.mean(values2)\n",
    "    std_dev2 = np.std(values2)\n",
    "    \n",
    "    # Perform independent t-test with Welch’s correction for unequal variances\n",
    "    t_stat, p_value = ttest_ind(values1, values2, equal_var=False)\n",
    "    \n",
    "    # Print results\n",
    "    print(f\"{key}:\")\n",
    "    print(f\"  human: {mean1:.2f} ± {std_dev1:.2f}\")\n",
    "    print(f\"  llm: {mean2:.2f} ± {std_dev2:.2f}\")\n",
    "    print(f\"  p-value: {p_value:.5f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Difference across n (when multiple authors collaborate on the story, compared with the single author"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7165"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(llm_values_1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "n = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "author_list\n",
       "gemma_llama      317\n",
       "gemma_mistral    394\n",
       "gemma_olmo       389\n",
       "gemma_orca       260\n",
       "llama_gemma      391\n",
       "llama_mistral    398\n",
       "llama_olmo       397\n",
       "llama_orca       289\n",
       "mistral_gemma    395\n",
       "mistral_llama    340\n",
       "mistral_olmo     400\n",
       "mistral_orca     288\n",
       "olmo_gemma       397\n",
       "olmo_llama       347\n",
       "olmo_mistral     399\n",
       "olmo_orca        264\n",
       "orca_gemma       370\n",
       "orca_llama       305\n",
       "orca_mistral     363\n",
       "orca_olmo        367\n",
       "dtype: int64"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2 = pd.read_csv('2_author.csv')\n",
    "df_2['author_list'] = df_2['author1']+'_'+df_2['author2']\n",
    "df_2.groupby('author_list').size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0.1', 'Unnamed: 0', 'prompt', 'human_story', 'author1',\n",
       "       'author2', 'part1', 'part2', 'llm_story', 'author_list'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 7070/7070 [1:09:51<00:00,  1.69it/s]   "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_wc:\n",
      "  llm_1: 621.31 ± 249.47\n",
      "  llm_2: 936.15 ± 181.29\n",
      "  p-value: 0.00000\n",
      "avg_word_len:\n",
      "  llm_1: 4.43 ± 0.37\n",
      "  llm_2: 4.45 ± 0.27\n",
      "  p-value: 0.00004\n",
      "n_sentences:\n",
      "  llm_1: 40.22 ± 15.91\n",
      "  llm_2: 60.83 ± 14.19\n",
      "  p-value: 0.00000\n",
      "sentence_length_mean:\n",
      "  llm_1: 15.72 ± 3.61\n",
      "  llm_2: 15.74 ± 2.70\n",
      "  p-value: 0.81757\n",
      "ttr:\n",
      "  llm_1: 0.44 ± 0.10\n",
      "  llm_2: 0.40 ± 0.06\n",
      "  p-value: 0.00000\n",
      "syllables_per_token_mean:\n",
      "  llm_1: 1.30 ± 0.10\n",
      "  llm_2: 1.30 ± 0.07\n",
      "  p-value: 0.03395\n",
      "stopwords_frac:\n",
      "  llm_1: 56.29 ± 6.58\n",
      "  llm_2: 56.69 ± 4.39\n",
      "  p-value: 0.00002\n",
      "flesch_reading_ease:\n",
      "  llm_1: 80.95 ± 8.76\n",
      "  llm_2: 81.20 ± 7.20\n",
      "  p-value: 0.07322\n",
      "entropy:\n",
      "  llm_1: 26.03 ± 13.10\n",
      "  llm_2: 38.42 ± 7.93\n",
      "  p-value: 0.00000\n",
      "perplexity:\n",
      "  llm_1: 1449076862962180353152875207888493519293563325966298079010272959911758332049947410390642156043386853958061026031987783958279938336653975821056201639897017267350692429824.00 ± inf\n",
      "  llm_2: 11317695211048858084704256.00 ± 692498657341208613619761152.00\n",
      "  p-value: 1.00000\n",
      "per_word_perplexity:\n",
      "  llm_1: 513495169265159901376332678284728978921278617468503480628181387554311551555253358891156730225321716491359651799549250442420060530554109771537504852416452269327777792.00 ± inf\n",
      "  llm_2: 7137007858192405757952.00 ± 431788357163128226578432.00\n",
      "  p-value: 1.00000\n",
      "dependency_distance_mean:\n",
      "  llm_1: 2.76 ± 5.95\n",
      "  llm_2: 2.61 ± 0.17\n",
      "  p-value: 0.03821\n",
      "first_order_coherence:\n",
      "  llm_1: 0.46 ± 0.06\n",
      "  llm_2: 0.46 ± 0.05\n",
      "  p-value: 0.00001\n",
      "second_order_coherence:\n",
      "  llm_1: nan ± nan\n",
      "  llm_2: 0.45 ± 0.05\n",
      "  p-value: nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "llm_values_n = []\n",
    "for index, row in tqdm(df_2.iterrows(), total=df_2.shape[0]):\n",
    "    llm_values_n.append(get_linguistic_feature(row['llm_story']))\n",
    "values_dict1 = collect_values(llm_values_1)\n",
    "values_dict2 = collect_values(llm_values_n)\n",
    "\n",
    "# Calculate and print the mean, standard deviation, and p-value for each key\n",
    "for key in values_dict1.keys():\n",
    "    values1 = values_dict1[key]\n",
    "    values2 = values_dict2[key]\n",
    "    mean1 = np.mean(values1)\n",
    "    std_dev1 = np.std(values1)\n",
    "    mean2 = np.mean(values2)\n",
    "    std_dev2 = np.std(values2)\n",
    "    \n",
    "    # Perform independent t-test with Welch’s correction for unequal variances\n",
    "    t_stat, p_value = ttest_ind(values1, values2, equal_var=False)\n",
    "    \n",
    "    # Print results\n",
    "    print(f\"{key}:\")\n",
    "    print(f\"  llm_1: {mean1:.2f} ± {std_dev1:.2f}\")\n",
    "    print(f\"  llm_2: {mean2:.2f} ± {std_dev2:.2f}\")\n",
    "    print(f\"  p-value: {p_value:.5f}\")  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "n= 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "author_list\n",
       "gemma_mistral_olmo     276\n",
       "gemma_olmo_llama       233\n",
       "llama_gemma_mistral    468\n",
       "llama_gemma_olmo       471\n",
       "llama_olmo_gemma       461\n",
       "llama_olmo_mistral     462\n",
       "mistral_gemma_olmo     480\n",
       "mistral_llama_gemma    385\n",
       "mistral_llama_olmo     394\n",
       "mistral_olmo_gemma     471\n",
       "olmo_gemma_llama       382\n",
       "olmo_gemma_mistral     478\n",
       "olmo_llama_gemma       378\n",
       "olmo_llama_mistral     368\n",
       "olmo_mistral_llama     386\n",
       "dtype: int64"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_3 = pd.read_csv('3_author.csv')\n",
    "df_3['author_list'] = df_3['author1']+'_'+df_3['author2'] + '_'+df_3['author3']\n",
    "df_3.groupby('author_list').size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0.1', 'Unnamed: 0', 'prompt', 'human_story', 'author1',\n",
       "       'author2', 'author3', 'part1', 'part2', 'part3', 'llm_story',\n",
       "       'author_list'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_3.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6093/6093 [27:10<00:00,  3.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_wc:\n",
      "  llm_1: 621.31 ± 249.47\n",
      "  llm_3: 989.58 ± 94.92\n",
      "  p-value: 0.0000000000\n",
      "avg_word_len:\n",
      "  llm_1: 4.43 ± 0.37\n",
      "  llm_3: 4.48 ± 0.25\n",
      "  p-value: 0.0000000000\n",
      "n_sentences:\n",
      "  llm_1: 40.22 ± 15.91\n",
      "  llm_3: 65.20 ± 11.34\n",
      "  p-value: 0.0000000000\n",
      "sentence_length_mean:\n",
      "  llm_1: 15.72 ± 3.61\n",
      "  llm_3: 15.51 ± 2.31\n",
      "  p-value: 0.0000368788\n",
      "ttr:\n",
      "  llm_1: 0.44 ± 0.10\n",
      "  llm_3: 0.40 ± 0.04\n",
      "  p-value: 0.0000000000\n",
      "syllables_per_token_mean:\n",
      "  llm_1: 1.30 ± 0.10\n",
      "  llm_3: 1.30 ± 0.07\n",
      "  p-value: 0.9910348132\n",
      "stopwords_frac:\n",
      "  llm_1: 56.29 ± 6.58\n",
      "  llm_3: 56.06 ± 3.66\n",
      "  p-value: 0.0095144892\n",
      "flesch_reading_ease:\n",
      "  llm_1: 80.95 ± 8.76\n",
      "  llm_3: 81.17 ± 6.87\n",
      "  p-value: 0.1069458150\n",
      "entropy:\n",
      "  llm_1: 26.03 ± 13.10\n",
      "  llm_3: 40.30 ± 4.75\n",
      "  p-value: 0.0000000000\n",
      "perplexity:\n",
      "  llm_1: 1449076862962180353152875207888493519293563325966298079010272959911758332049947410390642156043386853958061026031987783958279938336653975821056201639897017267350692429824.00 ± inf\n",
      "  llm_3: 10162588300539433320448.00 ± 515430681320392314847232.00\n",
      "  p-value: 1.0000000000\n",
      "per_word_perplexity:\n",
      "  llm_1: 513495169265159901376332678284728978921278617468503480628181387554311551555253358891156730225321716491359651799549250442420060530554109771537504852416452269327777792.00 ± inf\n",
      "  llm_3: 7280851370518150144.00 ± 365969115407234105344.00\n",
      "  p-value: 1.0000000000\n",
      "dependency_distance_mean:\n",
      "  llm_1: 2.76 ± 5.95\n",
      "  llm_3: 2.59 ± 0.14\n",
      "  p-value: 0.0170995915\n",
      "first_order_coherence:\n",
      "  llm_1: 0.46 ± 0.06\n",
      "  llm_3: 0.46 ± 0.04\n",
      "  p-value: 0.0900976944\n",
      "second_order_coherence:\n",
      "  llm_1: nan ± nan\n",
      "  llm_3: 0.45 ± 0.05\n",
      "  p-value: nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "llm_values_n = []\n",
    "for index, row in tqdm(df_3.iterrows(), total=df_3.shape[0]):\n",
    "    llm_values_n.append(get_linguistic_feature(row['llm_story']))\n",
    "values_dict1 = collect_values(llm_values_1)\n",
    "values_dict2 = collect_values(llm_values_n)\n",
    "\n",
    "# Calculate and print the mean, standard deviation, and p-value for each key\n",
    "for key in values_dict1.keys():\n",
    "    values1 = values_dict1[key]\n",
    "    values2 = values_dict2[key]\n",
    "    mean1 = np.mean(values1)\n",
    "    std_dev1 = np.std(values1)\n",
    "    mean2 = np.mean(values2)\n",
    "    std_dev2 = np.std(values2)\n",
    "    \n",
    "    # Perform independent t-test with Welch’s correction for unequal variances\n",
    "    t_stat, p_value = ttest_ind(values1, values2, equal_var=False)\n",
    "    \n",
    "    # Print results\n",
    "    print(f\"{key}:\")\n",
    "    print(f\"  llm_1: {mean1:.2f} ± {std_dev1:.2f}\")\n",
    "    print(f\"  llm_3: {mean2:.2f} ± {std_dev2:.2f}\")\n",
    "    print(f\"  p-value: {p_value:.10f}\")  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "n=4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "author_list\n",
       "gemma_mistral_olmo_llama    455\n",
       "gemma_olmo_llama_mistral    447\n",
       "llama_gemma_mistral_olmo    478\n",
       "llama_gemma_olmo_mistral    475\n",
       "llama_olmo_gemma_mistral    473\n",
       "llama_olmo_mistral_gemma    470\n",
       "mistral_gemma_olmo_llama    457\n",
       "mistral_llama_gemma_olmo    475\n",
       "mistral_llama_olmo_gemma    468\n",
       "mistral_olmo_gemma_llama    453\n",
       "olmo_gemma_llama_mistral    457\n",
       "olmo_gemma_mistral_llama    458\n",
       "olmo_llama_gemma_mistral    471\n",
       "olmo_llama_mistral_gemma    469\n",
       "olmo_mistral_llama_gemma    449\n",
       "dtype: int64"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_4 = pd.read_csv('4_author.csv')\n",
    "df_4['author_list'] = df_4['author1']+'_'+df_4['author2'] + '_'+df_4['author3'] + '_' + df_4['author4']\n",
    "df_4.groupby('author_list').size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6955/6955 [29:07<00:00,  3.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_wc:\n",
      "  llm_1: 621.31 ± 249.47\n",
      "  llm_4: 943.34 ± 70.66\n",
      "  p-value: 0.0000000000\n",
      "avg_word_len:\n",
      "  llm_1: 4.43 ± 0.37\n",
      "  llm_4: 4.49 ± 0.25\n",
      "  p-value: 0.0000000000\n",
      "n_sentences:\n",
      "  llm_1: 40.22 ± 15.91\n",
      "  llm_4: 60.41 ± 9.88\n",
      "  p-value: 0.0000000000\n",
      "sentence_length_mean:\n",
      "  llm_1: 15.72 ± 3.61\n",
      "  llm_4: 15.94 ± 2.23\n",
      "  p-value: 0.0000225347\n",
      "ttr:\n",
      "  llm_1: 0.44 ± 0.10\n",
      "  llm_4: 0.41 ± 0.03\n",
      "  p-value: 0.0000000000\n",
      "syllables_per_token_mean:\n",
      "  llm_1: 1.30 ± 0.10\n",
      "  llm_4: 1.31 ± 0.07\n",
      "  p-value: 0.0000457922\n",
      "stopwords_frac:\n",
      "  llm_1: 56.29 ± 6.58\n",
      "  llm_4: 55.94 ± 3.40\n",
      "  p-value: 0.0000503220\n",
      "flesch_reading_ease:\n",
      "  llm_1: 80.95 ± 8.76\n",
      "  llm_4: 80.25 ± 7.11\n",
      "  p-value: 0.0000001474\n",
      "entropy:\n",
      "  llm_1: 26.03 ± 13.10\n",
      "  llm_4: 37.97 ± 4.07\n",
      "  p-value: 0.0000000000\n",
      "perplexity:\n",
      "  llm_1: 1449076862962180353152875207888493519293563325966298079010272959911758332049947410390642156043386853958061026031987783958279938336653975821056201639897017267350692429824.00 ± inf\n",
      "  llm_4: 677016273764254819203821653196800.00 ± 56456825069994160737149596056158208.00\n",
      "  p-value: 1.0000000000\n",
      "per_word_perplexity:\n",
      "  llm_1: 513495169265159901376332678284728978921278617468503480628181387554311551555253358891156730225321716491359651799549250442420060530554109771537504852416452269327777792.00 ± inf\n",
      "  llm_4: 554023137286623668556028444672.00 ± 46200347847785734961581820739584.00\n",
      "  p-value: 1.0000000000\n",
      "dependency_distance_mean:\n",
      "  llm_1: 2.76 ± 5.95\n",
      "  llm_4: 2.61 ± 0.15\n",
      "  p-value: 0.0333615546\n",
      "first_order_coherence:\n",
      "  llm_1: 0.46 ± 0.06\n",
      "  llm_4: 0.46 ± 0.04\n",
      "  p-value: 0.0000333151\n",
      "second_order_coherence:\n",
      "  llm_1: nan ± nan\n",
      "  llm_4: 0.46 ± 0.04\n",
      "  p-value: nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "llm_values_n = []\n",
    "for index, row in tqdm(df_4.iterrows(), total=df_4.shape[0]):\n",
    "    llm_values_n.append(get_linguistic_feature(row['llm_story']))\n",
    "values_dict1 = collect_values(llm_values_1)\n",
    "values_dict2 = collect_values(llm_values_n)\n",
    "\n",
    "# Calculate and print the mean, standard deviation, and p-value for each key\n",
    "for key in values_dict1.keys():\n",
    "    values1 = values_dict1[key]\n",
    "    values2 = values_dict2[key]\n",
    "    mean1 = np.mean(values1)\n",
    "    std_dev1 = np.std(values1)\n",
    "    mean2 = np.mean(values2)\n",
    "    std_dev2 = np.std(values2)\n",
    "    \n",
    "    # Perform independent t-test with Welch’s correction for unequal variances\n",
    "    t_stat, p_value = ttest_ind(values1, values2, equal_var=False)\n",
    "    \n",
    "    # Print results\n",
    "    print(f\"{key}:\")\n",
    "    print(f\"  llm_1: {mean1:.2f} ± {std_dev1:.2f}\")\n",
    "    print(f\"  llm_4: {mean2:.2f} ± {std_dev2:.2f}\")\n",
    "    print(f\"  p-value: {p_value:.10f}\")  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "n = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "author_list\n",
       "gemma_llama_olmo_mistral_orca    410\n",
       "gemma_orca_llama_mistral_olmo    444\n",
       "llama_gemma_olmo_orca_mistral    448\n",
       "llama_gemma_orca_mistral_olmo    451\n",
       "llama_olmo_orca_gemma_mistral    418\n",
       "mistral_gemma_llama_olmo_orca    433\n",
       "mistral_llama_gemma_orca_olmo    400\n",
       "mistral_olmo_gemma_orca_llama    387\n",
       "mistral_orca_gemma_olmo_llama    390\n",
       "olmo_gemma_orca_mistral_llama    213\n",
       "olmo_llama_mistral_orca_gemma    201\n",
       "olmo_mistral_gemma_llama_orca    202\n",
       "olmo_mistral_llama_orca_gemma    214\n",
       "olmo_mistral_orca_gemma_llama    198\n",
       "orca_mistral_olmo_llama_gemma    411\n",
       "dtype: int64"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_5 = pd.read_csv('5_author.csv')\n",
    "df_5['author_list'] = df_5['author1']+'_'+df_5['author2'] + '_'+df_5['author3'] + '_' + df_5['author4']+'_'+df_5['author5']\n",
    "df_5.groupby('author_list').size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5220/5220 [19:50<00:00,  4.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_wc:\n",
      "  llm_1: 621.31 ± 249.47\n",
      "  llm_4: 859.35 ± 60.68\n",
      "  p-value: 0.0000000000\n",
      "avg_word_len:\n",
      "  llm_1: 4.43 ± 0.37\n",
      "  llm_4: 4.47 ± 0.26\n",
      "  p-value: 0.0000000000\n",
      "n_sentences:\n",
      "  llm_1: 40.22 ± 15.91\n",
      "  llm_4: 55.25 ± 9.03\n",
      "  p-value: 0.0000000000\n",
      "sentence_length_mean:\n",
      "  llm_1: 15.72 ± 3.61\n",
      "  llm_4: 15.88 ± 2.23\n",
      "  p-value: 0.0027719747\n",
      "ttr:\n",
      "  llm_1: 0.44 ± 0.10\n",
      "  llm_4: 0.42 ± 0.03\n",
      "  p-value: 0.0000000000\n",
      "syllables_per_token_mean:\n",
      "  llm_1: 1.30 ± 0.10\n",
      "  llm_4: 1.30 ± 0.07\n",
      "  p-value: 0.6869240220\n",
      "stopwords_frac:\n",
      "  llm_1: 56.29 ± 6.58\n",
      "  llm_4: 55.84 ± 3.58\n",
      "  p-value: 0.0000009142\n",
      "flesch_reading_ease:\n",
      "  llm_1: 80.95 ± 8.76\n",
      "  llm_4: 80.85 ± 7.24\n",
      "  p-value: 0.4521086463\n",
      "entropy:\n",
      "  llm_1: 26.03 ± 13.10\n",
      "  llm_4: 34.69 ± 3.58\n",
      "  p-value: 0.0000000000\n",
      "perplexity:\n",
      "  llm_1: 1449076862962180353152875207888493519293563325966298079010272959911758332049947410390642156043386853958061026031987783958279938336653975821056201639897017267350692429824.00 ± inf\n",
      "  llm_4: 148349059703247328.00 ± 1231924948088857344.00\n",
      "  p-value: 1.0000000000\n",
      "per_word_perplexity:\n",
      "  llm_1: 513495169265159901376332678284728978921278617468503480628181387554311551555253358891156730225321716491359651799549250442420060530554109771537504852416452269327777792.00 ± inf\n",
      "  llm_4: 133231203595628.88 ± 1086916802848175.62\n",
      "  p-value: 1.0000000000\n",
      "dependency_distance_mean:\n",
      "  llm_1: 2.76 ± 5.95\n",
      "  llm_4: 2.61 ± 0.14\n",
      "  p-value: 0.0287590200\n",
      "first_order_coherence:\n",
      "  llm_1: 0.46 ± 0.06\n",
      "  llm_4: 0.46 ± 0.04\n",
      "  p-value: 0.3995521178\n",
      "second_order_coherence:\n",
      "  llm_1: nan ± nan\n",
      "  llm_4: 0.46 ± 0.05\n",
      "  p-value: nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "llm_values_n = []\n",
    "for index, row in tqdm(df_5.iterrows(), total=df_5.shape[0]):\n",
    "    llm_values_n.append(get_linguistic_feature(row['llm_story']))\n",
    "values_dict1 = collect_values(llm_values_1)\n",
    "values_dict2 = collect_values(llm_values_n)\n",
    "\n",
    "# Calculate and print the mean, standard deviation, and p-value for each key\n",
    "for key in values_dict1.keys():\n",
    "    values1 = values_dict1[key]\n",
    "    values2 = values_dict2[key]\n",
    "    mean1 = np.mean(values1)\n",
    "    std_dev1 = np.std(values1)\n",
    "    mean2 = np.mean(values2)\n",
    "    std_dev2 = np.std(values2)\n",
    "    \n",
    "    # Perform independent t-test with Welch’s correction for unequal variances\n",
    "    t_stat, p_value = ttest_ind(values1, values2, equal_var=False)\n",
    "    \n",
    "    # Print results\n",
    "    print(f\"{key}:\")\n",
    "    print(f\"  llm_1: {mean1:.2f} ± {std_dev1:.2f}\")\n",
    "    print(f\"  llm_4: {mean2:.2f} ± {std_dev2:.2f}\")\n",
    "    print(f\"  p-value: {p_value:.10f}\")  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_wc:\n",
      "  llm_1: 621.31 ± 249.47\n",
      "  llm_3: 859.35 ± 60.68\n",
      "  p-value: 0.0000000000\n",
      "avg_word_len:\n",
      "  llm_1: 4.43 ± 0.37\n",
      "  llm_3: 4.47 ± 0.26\n",
      "  p-value: 0.0000000000\n",
      "n_sentences:\n",
      "  llm_1: 40.22 ± 15.91\n",
      "  llm_3: 55.25 ± 9.03\n",
      "  p-value: 0.0000000000\n",
      "sentence_length_mean:\n",
      "  llm_1: 15.72 ± 3.61\n",
      "  llm_3: 15.88 ± 2.23\n",
      "  p-value: 0.0027719747\n",
      "ttr:\n",
      "  llm_1: 0.44 ± 0.10\n",
      "  llm_3: 0.42 ± 0.03\n",
      "  p-value: 0.0000000000\n",
      "syllables_per_token_mean:\n",
      "  llm_1: 1.30 ± 0.10\n",
      "  llm_3: 1.30 ± 0.07\n",
      "  p-value: 0.6869240220\n",
      "stopwords_frac:\n",
      "  llm_1: 56.29 ± 6.58\n",
      "  llm_3: 55.84 ± 3.58\n",
      "  p-value: 0.0000009142\n",
      "flesch_reading_ease:\n",
      "  llm_1: 80.95 ± 8.76\n",
      "  llm_3: 80.85 ± 7.24\n",
      "  p-value: 0.4521086463\n",
      "entropy:\n",
      "  llm_1: 26.03 ± 13.10\n",
      "  llm_3: 34.69 ± 3.58\n",
      "  p-value: 0.0000000000\n",
      "perplexity:\n",
      "  llm_1: 1449076862962180353152875207888493519293563325966298079010272959911758332049947410390642156043386853958061026031987783958279938336653975821056201639897017267350692429824.00 ± inf\n",
      "  llm_3: 148349059703247328.00 ± 1231924948088857344.00\n",
      "  p-value: 1.0000000000\n",
      "per_word_perplexity:\n",
      "  llm_1: 513495169265159901376332678284728978921278617468503480628181387554311551555253358891156730225321716491359651799549250442420060530554109771537504852416452269327777792.00 ± inf\n",
      "  llm_3: 133231203595628.88 ± 1086916802848175.62\n",
      "  p-value: 1.0000000000\n",
      "dependency_distance_mean:\n",
      "  llm_1: 2.76 ± 5.95\n",
      "  llm_3: 2.61 ± 0.14\n",
      "  p-value: 0.0287590200\n",
      "first_order_coherence:\n",
      "  llm_1: 0.46 ± 0.06\n",
      "  llm_3: 0.46 ± 0.04\n",
      "  p-value: 0.3995521178\n",
      "second_order_coherence:\n",
      "  llm_1: 0.45 ± 0.06\n",
      "  llm_3: 0.46 ± 0.05\n",
      "  p-value: nan\n"
     ]
    }
   ],
   "source": [
    "for key in values_dict1.keys():\n",
    "    values1 = values_dict1[key]\n",
    "    values2 = values_dict2[key]\n",
    "    mean1 = np.nanmean(values1)\n",
    "    std_dev1 = np.nanstd(values1)\n",
    "    mean2 = np.mean(values2)\n",
    "    std_dev2 = np.std(values2)\n",
    "    \n",
    "    # Perform independent t-test with Welch’s correction for unequal variances\n",
    "    t_stat, p_value = ttest_ind(values1, values2, equal_var=False)\n",
    "    \n",
    "    # Print results\n",
    "    print(f\"{key}:\")\n",
    "    print(f\"  llm_1: {mean1:.2f} ± {std_dev1:.2f}\")\n",
    "    print(f\"  llm_3: {mean2:.2f} ± {std_dev2:.2f}\")\n",
    "    print(f\"  p-value: {p_value:.10f}\")  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How much each llm behave differently in different part "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compare_values_with_part1(llm_values_1, llm_values_n):\n",
    "    values_dict1 = collect_values(llm_values_1)\n",
    "    values_dict2 = collect_values(llm_values_n)\n",
    "\n",
    "    # Calculate and print the mean, standard deviation, and p-value for each key\n",
    "    for key in values_dict1.keys():\n",
    "        values1 = values_dict1[key]\n",
    "        values2 = values_dict2[key]\n",
    "        mean1 = np.nanmean(values1)\n",
    "        std_dev1 = np.nanstd(values1)\n",
    "        mean2 = np.nanmean(values2)\n",
    "        std_dev2 = np.nanstd(values2)\n",
    "        \n",
    "        # Perform independent t-test with Welch’s correction for unequal variances\n",
    "        t_stat, p_value = ttest_ind(values1, values2, equal_var=False)\n",
    "        \n",
    "        # Print results\n",
    "        print(f\"{key}:\")\n",
    "        print(f\"  llm_1: {mean1:.2f} ± {std_dev1:.2f}\")\n",
    "        print(f\"  llm_n: {mean2:.2f} ± {std_dev2:.2f}\")\n",
    "        print(f\"  p-value: {p_value:.10f}\")  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0', 'prompt', 'human_story', 'author1', 'author2', 'author3',\n",
       "       'author4', 'author5', 'part1', 'part2', 'part3', 'part4', 'part5',\n",
       "       'llm_story'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('5_author.csv')\n",
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['llama', 'olmo', 'mistral', 'gemma', 'orca'], dtype=object)"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "authors = df.author1.unique()\n",
    "authors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "llama\n",
      "====================================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1317/1317 [00:59<00:00, 22.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------------------------------------------\n",
      "Comparision with part-2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1011/1011 [00:44<00:00, 22.69it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_wc:\n",
      "  llm_1: 172.51 ± 19.91\n",
      "  llm_n: 170.88 ± 13.28\n",
      "  p-value: 0.0179200125\n",
      "avg_word_len:\n",
      "  llm_1: 4.32 ± 0.28\n",
      "  llm_n: 4.33 ± 0.30\n",
      "  p-value: 0.2638956300\n",
      "n_sentences:\n",
      "  llm_1: 10.93 ± 2.66\n",
      "  llm_n: 10.34 ± 2.41\n",
      "  p-value: 0.0000000382\n",
      "sentence_length_mean:\n",
      "  llm_1: 16.53 ± 3.64\n",
      "  llm_n: 17.31 ± 3.68\n",
      "  p-value: 0.0000004863\n",
      "ttr:\n",
      "  llm_1: 0.60 ± 0.05\n",
      "  llm_n: 0.61 ± 0.05\n",
      "  p-value: 0.2985939907\n",
      "syllables_per_token_mean:\n",
      "  llm_1: 1.26 ± 0.08\n",
      "  llm_n: 1.26 ± 0.08\n",
      "  p-value: 0.3148460089\n",
      "stopwords_frac:\n",
      "  llm_1: 59.14 ± 5.56\n",
      "  llm_n: 57.20 ± 4.85\n",
      "  p-value: 0.0000000000\n",
      "flesch_reading_ease:\n",
      "  llm_1: 83.11 ± 8.35\n",
      "  llm_n: 82.61 ± 8.50\n",
      "  p-value: 0.1578329165\n",
      "entropy:\n",
      "  llm_1: 6.83 ± 0.98\n",
      "  llm_n: 6.84 ± 0.81\n",
      "  p-value: 0.7674865467\n",
      "perplexity:\n",
      "  llm_1: 1361.46 ± 1311.87\n",
      "  llm_n: 1282.25 ± 1182.96\n",
      "  p-value: 0.1270725640\n",
      "per_word_perplexity:\n",
      "  llm_1: 6.49 ± 5.91\n",
      "  llm_n: 6.24 ± 5.36\n",
      "  p-value: 0.2939786446\n",
      "dependency_distance_mean:\n",
      "  llm_1: 2.63 ± 0.25\n",
      "  llm_n: 2.68 ± 0.25\n",
      "  p-value: 0.0000493092\n",
      "first_order_coherence:\n",
      "  llm_1: 0.44 ± 0.08\n",
      "  llm_n: 0.47 ± 0.08\n",
      "  p-value: 0.0000000000\n",
      "second_order_coherence:\n",
      "  llm_1: 0.44 ± 0.08\n",
      "  llm_n: 0.46 ± 0.09\n",
      "  p-value: 0.0000000000\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comparision with part-3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1091/1091 [00:48<00:00, 22.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_wc:\n",
      "  llm_1: 172.51 ± 19.91\n",
      "  llm_n: 173.35 ± 15.29\n",
      "  p-value: 0.2421826789\n",
      "avg_word_len:\n",
      "  llm_1: 4.32 ± 0.28\n",
      "  llm_n: 4.32 ± 0.32\n",
      "  p-value: 0.8775417410\n",
      "n_sentences:\n",
      "  llm_1: 10.93 ± 2.66\n",
      "  llm_n: 10.60 ± 2.51\n",
      "  p-value: 0.0018780650\n",
      "sentence_length_mean:\n",
      "  llm_1: 16.53 ± 3.64\n",
      "  llm_n: 17.15 ± 3.70\n",
      "  p-value: 0.0000387575\n",
      "ttr:\n",
      "  llm_1: 0.60 ± 0.05\n",
      "  llm_n: 0.59 ± 0.05\n",
      "  p-value: 0.0000000000\n",
      "syllables_per_token_mean:\n",
      "  llm_1: 1.26 ± 0.08\n",
      "  llm_n: 1.26 ± 0.09\n",
      "  p-value: 0.3434504960\n",
      "stopwords_frac:\n",
      "  llm_1: 59.14 ± 5.56\n",
      "  llm_n: 57.80 ± 4.92\n",
      "  p-value: 0.0000000004\n",
      "flesch_reading_ease:\n",
      "  llm_1: 83.11 ± 8.35\n",
      "  llm_n: 82.75 ± 9.00\n",
      "  p-value: 0.3154746061\n",
      "entropy:\n",
      "  llm_1: 6.83 ± 0.98\n",
      "  llm_n: 7.02 ± 0.89\n",
      "  p-value: 0.0000007134\n",
      "perplexity:\n",
      "  llm_1: 1361.46 ± 1311.87\n",
      "  llm_n: 1607.96 ± 1623.86\n",
      "  p-value: 0.0000559105\n",
      "per_word_perplexity:\n",
      "  llm_1: 6.49 ± 5.91\n",
      "  llm_n: 7.69 ± 7.28\n",
      "  p-value: 0.0000115658\n",
      "dependency_distance_mean:\n",
      "  llm_1: 2.63 ± 0.25\n",
      "  llm_n: 2.68 ± 0.27\n",
      "  p-value: 0.0000062269\n",
      "first_order_coherence:\n",
      "  llm_1: 0.44 ± 0.08\n",
      "  llm_n: 0.47 ± 0.08\n",
      "  p-value: 0.0000000000\n",
      "second_order_coherence:\n",
      "  llm_1: 0.44 ± 0.08\n",
      "  llm_n: 0.46 ± 0.08\n",
      "  p-value: nan\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comparision with part-4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 613/613 [00:28<00:00, 21.68it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_wc:\n",
      "  llm_1: 172.51 ± 19.91\n",
      "  llm_n: 174.76 ± 16.18\n",
      "  p-value: 0.0086869790\n",
      "avg_word_len:\n",
      "  llm_1: 4.32 ± 0.28\n",
      "  llm_n: 4.36 ± 0.34\n",
      "  p-value: 0.0124314772\n",
      "n_sentences:\n",
      "  llm_1: 10.93 ± 2.66\n",
      "  llm_n: 10.53 ± 2.58\n",
      "  p-value: 0.0020491446\n",
      "sentence_length_mean:\n",
      "  llm_1: 16.53 ± 3.64\n",
      "  llm_n: 17.45 ± 3.89\n",
      "  p-value: 0.0000011327\n",
      "ttr:\n",
      "  llm_1: 0.60 ± 0.05\n",
      "  llm_n: 0.58 ± 0.05\n",
      "  p-value: 0.0000000000\n",
      "syllables_per_token_mean:\n",
      "  llm_1: 1.26 ± 0.08\n",
      "  llm_n: 1.28 ± 0.09\n",
      "  p-value: 0.0004533130\n",
      "stopwords_frac:\n",
      "  llm_1: 59.14 ± 5.56\n",
      "  llm_n: 58.41 ± 5.09\n",
      "  p-value: 0.0043566721\n",
      "flesch_reading_ease:\n",
      "  llm_1: 83.11 ± 8.35\n",
      "  llm_n: 80.88 ± 10.10\n",
      "  p-value: 0.0000020942\n",
      "entropy:\n",
      "  llm_1: 6.83 ± 0.98\n",
      "  llm_n: 6.99 ± 0.94\n",
      "  p-value: 0.0006340255\n",
      "perplexity:\n",
      "  llm_1: 1361.46 ± 1311.87\n",
      "  llm_n: 1629.76 ± 1723.10\n",
      "  p-value: 0.0006557870\n",
      "per_word_perplexity:\n",
      "  llm_1: 6.49 ± 5.91\n",
      "  llm_n: 7.71 ± 7.63\n",
      "  p-value: 0.0004660569\n",
      "dependency_distance_mean:\n",
      "  llm_1: 2.63 ± 0.25\n",
      "  llm_n: 2.68 ± 0.24\n",
      "  p-value: 0.0000342504\n",
      "first_order_coherence:\n",
      "  llm_1: 0.44 ± 0.08\n",
      "  llm_n: 0.47 ± 0.08\n",
      "  p-value: 0.0000000036\n",
      "second_order_coherence:\n",
      "  llm_1: 0.44 ± 0.08\n",
      "  llm_n: 0.46 ± 0.09\n",
      "  p-value: 0.0000000087\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comparision with part-5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1188/1188 [00:52<00:00, 22.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_wc:\n",
      "  llm_1: 172.51 ± 19.91\n",
      "  llm_n: 172.23 ± 19.33\n",
      "  p-value: 0.7218469276\n",
      "avg_word_len:\n",
      "  llm_1: 4.32 ± 0.28\n",
      "  llm_n: 4.36 ± 0.32\n",
      "  p-value: 0.0007762945\n",
      "n_sentences:\n",
      "  llm_1: 10.93 ± 2.66\n",
      "  llm_n: 9.66 ± 2.40\n",
      "  p-value: 0.0000000000\n",
      "sentence_length_mean:\n",
      "  llm_1: 16.53 ± 3.64\n",
      "  llm_n: 18.66 ± 3.82\n",
      "  p-value: 0.0000000000\n",
      "ttr:\n",
      "  llm_1: 0.60 ± 0.05\n",
      "  llm_n: 0.58 ± 0.06\n",
      "  p-value: 0.0000000000\n",
      "syllables_per_token_mean:\n",
      "  llm_1: 1.26 ± 0.08\n",
      "  llm_n: 1.27 ± 0.09\n",
      "  p-value: 0.0030893840\n",
      "stopwords_frac:\n",
      "  llm_1: 59.14 ± 5.56\n",
      "  llm_n: 58.41 ± 4.70\n",
      "  p-value: 0.0003587606\n",
      "flesch_reading_ease:\n",
      "  llm_1: 83.11 ± 8.35\n",
      "  llm_n: 80.13 ± 9.19\n",
      "  p-value: 0.0000000000\n",
      "entropy:\n",
      "  llm_1: 6.83 ± 0.98\n",
      "  llm_n: 6.87 ± 1.03\n",
      "  p-value: 0.2989780430\n",
      "perplexity:\n",
      "  llm_1: 1361.46 ± 1311.87\n",
      "  llm_n: 1500.89 ± 1792.09\n",
      "  p-value: 0.0278465971\n",
      "per_word_perplexity:\n",
      "  llm_1: 6.49 ± 5.91\n",
      "  llm_n: 7.23 ± 7.99\n",
      "  p-value: 0.0089999056\n",
      "dependency_distance_mean:\n",
      "  llm_1: 2.63 ± 0.25\n",
      "  llm_n: 2.73 ± 0.24\n",
      "  p-value: 0.0000000000\n",
      "first_order_coherence:\n",
      "  llm_1: 0.44 ± 0.08\n",
      "  llm_n: 0.49 ± 0.08\n",
      "  p-value: 0.0000000000\n",
      "second_order_coherence:\n",
      "  llm_1: 0.44 ± 0.08\n",
      "  llm_n: 0.48 ± 0.09\n",
      "  p-value: nan\n",
      "====================================================================================================\n",
      "olmo\n",
      "====================================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1028/1028 [00:44<00:00, 23.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------------------------------------------\n",
      "Comparision with part-2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 805/805 [00:41<00:00, 19.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_wc:\n",
      "  llm_1: 168.01 ± 8.69\n",
      "  llm_n: 197.91 ± 18.93\n",
      "  p-value: 0.0000000000\n",
      "avg_word_len:\n",
      "  llm_1: 4.42 ± 0.30\n",
      "  llm_n: 4.35 ± 0.32\n",
      "  p-value: 0.0000020391\n",
      "n_sentences:\n",
      "  llm_1: 10.23 ± 2.28\n",
      "  llm_n: 13.64 ± 3.60\n",
      "  p-value: 0.0000000000\n",
      "sentence_length_mean:\n",
      "  llm_1: 17.23 ± 3.89\n",
      "  llm_n: 15.38 ± 3.77\n",
      "  p-value: 0.0000000000\n",
      "ttr:\n",
      "  llm_1: 0.62 ± 0.05\n",
      "  llm_n: 0.57 ± 0.06\n",
      "  p-value: 0.0000000000\n",
      "syllables_per_token_mean:\n",
      "  llm_1: 1.28 ± 0.09\n",
      "  llm_n: 1.28 ± 0.09\n",
      "  p-value: 0.0673831901\n",
      "stopwords_frac:\n",
      "  llm_1: 54.66 ± 5.04\n",
      "  llm_n: 57.65 ± 5.86\n",
      "  p-value: 0.0000000000\n",
      "flesch_reading_ease:\n",
      "  llm_1: 80.78 ± 9.01\n",
      "  llm_n: 83.31 ± 9.87\n",
      "  p-value: 0.0000000181\n",
      "entropy:\n",
      "  llm_1: 6.50 ± 0.68\n",
      "  llm_n: 8.15 ± 1.14\n",
      "  p-value: 0.0000000000\n",
      "perplexity:\n",
      "  llm_1: 840.67 ± 661.02\n",
      "  llm_n: 5963.67 ± 7092.64\n",
      "  p-value: 0.0000000000\n",
      "per_word_perplexity:\n",
      "  llm_1: 4.28 ± 3.25\n",
      "  llm_n: 24.43 ± 28.01\n",
      "  p-value: 0.0000000000\n",
      "dependency_distance_mean:\n",
      "  llm_1: 2.66 ± 0.27\n",
      "  llm_n: 2.54 ± 0.29\n",
      "  p-value: 0.0000000000\n",
      "first_order_coherence:\n",
      "  llm_1: 0.47 ± 0.08\n",
      "  llm_n: 0.43 ± 0.07\n",
      "  p-value: 0.0000000000\n",
      "second_order_coherence:\n",
      "  llm_1: 0.47 ± 0.08\n",
      "  llm_n: 0.43 ± 0.08\n",
      "  p-value: 0.0000000000\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comparision with part-3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1269/1269 [01:04<00:00, 19.57it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_wc:\n",
      "  llm_1: 168.01 ± 8.69\n",
      "  llm_n: 194.89 ± 23.64\n",
      "  p-value: 0.0000000000\n",
      "avg_word_len:\n",
      "  llm_1: 4.42 ± 0.30\n",
      "  llm_n: 4.38 ± 0.33\n",
      "  p-value: 0.0003659052\n",
      "n_sentences:\n",
      "  llm_1: 10.23 ± 2.28\n",
      "  llm_n: 12.66 ± 3.47\n",
      "  p-value: 0.0000000000\n",
      "sentence_length_mean:\n",
      "  llm_1: 17.23 ± 3.89\n",
      "  llm_n: 16.36 ± 4.13\n",
      "  p-value: 0.0000002204\n",
      "ttr:\n",
      "  llm_1: 0.62 ± 0.05\n",
      "  llm_n: 0.58 ± 0.06\n",
      "  p-value: 0.0000000000\n",
      "syllables_per_token_mean:\n",
      "  llm_1: 1.28 ± 0.09\n",
      "  llm_n: 1.29 ± 0.09\n",
      "  p-value: 0.4201272425\n",
      "stopwords_frac:\n",
      "  llm_1: 54.66 ± 5.04\n",
      "  llm_n: 57.06 ± 5.65\n",
      "  p-value: 0.0000000000\n",
      "flesch_reading_ease:\n",
      "  llm_1: 80.78 ± 9.01\n",
      "  llm_n: 81.41 ± 9.81\n",
      "  p-value: 0.1094797002\n",
      "entropy:\n",
      "  llm_1: 6.50 ± 0.68\n",
      "  llm_n: 7.96 ± 1.27\n",
      "  p-value: 0.0000000000\n",
      "perplexity:\n",
      "  llm_1: 840.67 ± 661.02\n",
      "  llm_n: 5316.66 ± 6942.37\n",
      "  p-value: 0.0000000000\n",
      "per_word_perplexity:\n",
      "  llm_1: 4.28 ± 3.25\n",
      "  llm_n: 22.02 ± 27.35\n",
      "  p-value: 0.0000000000\n",
      "dependency_distance_mean:\n",
      "  llm_1: 2.66 ± 0.27\n",
      "  llm_n: 2.58 ± 0.34\n",
      "  p-value: 0.0000000001\n",
      "first_order_coherence:\n",
      "  llm_1: 0.47 ± 0.08\n",
      "  llm_n: 0.45 ± 0.08\n",
      "  p-value: 0.0000000000\n",
      "second_order_coherence:\n",
      "  llm_1: 0.47 ± 0.08\n",
      "  llm_n: 0.44 ± 0.08\n",
      "  p-value: 0.0000000000\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comparision with part-4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 823/823 [00:41<00:00, 19.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_wc:\n",
      "  llm_1: 168.01 ± 8.69\n",
      "  llm_n: 192.64 ± 26.60\n",
      "  p-value: 0.0000000000\n",
      "avg_word_len:\n",
      "  llm_1: 4.42 ± 0.30\n",
      "  llm_n: 4.41 ± 0.36\n",
      "  p-value: 0.2852021114\n",
      "n_sentences:\n",
      "  llm_1: 10.23 ± 2.28\n",
      "  llm_n: 12.42 ± 3.65\n",
      "  p-value: 0.0000000000\n",
      "sentence_length_mean:\n",
      "  llm_1: 17.23 ± 3.89\n",
      "  llm_n: 16.58 ± 4.37\n",
      "  p-value: 0.0008346220\n",
      "ttr:\n",
      "  llm_1: 0.62 ± 0.05\n",
      "  llm_n: 0.58 ± 0.06\n",
      "  p-value: 0.0000000000\n",
      "syllables_per_token_mean:\n",
      "  llm_1: 1.28 ± 0.09\n",
      "  llm_n: 1.29 ± 0.10\n",
      "  p-value: 0.0175148781\n",
      "stopwords_frac:\n",
      "  llm_1: 54.66 ± 5.04\n",
      "  llm_n: 56.78 ± 5.55\n",
      "  p-value: 0.0000000000\n",
      "flesch_reading_ease:\n",
      "  llm_1: 80.78 ± 9.01\n",
      "  llm_n: 80.55 ± 10.73\n",
      "  p-value: 0.6194104130\n",
      "entropy:\n",
      "  llm_1: 6.50 ± 0.68\n",
      "  llm_n: 7.86 ± 1.41\n",
      "  p-value: 0.0000000000\n",
      "perplexity:\n",
      "  llm_1: 840.67 ± 661.02\n",
      "  llm_n: 5225.43 ± 6362.26\n",
      "  p-value: 0.0000000000\n",
      "per_word_perplexity:\n",
      "  llm_1: 4.28 ± 3.25\n",
      "  llm_n: 21.65 ± 25.23\n",
      "  p-value: 0.0000000000\n",
      "dependency_distance_mean:\n",
      "  llm_1: 2.66 ± 0.27\n",
      "  llm_n: 2.58 ± 0.27\n",
      "  p-value: 0.0000000000\n",
      "first_order_coherence:\n",
      "  llm_1: 0.47 ± 0.08\n",
      "  llm_n: 0.45 ± 0.08\n",
      "  p-value: 0.0000031533\n",
      "second_order_coherence:\n",
      "  llm_1: 0.47 ± 0.08\n",
      "  llm_n: 0.44 ± 0.08\n",
      "  p-value: 0.0000000001\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comparision with part-5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1295/1295 [01:04<00:00, 20.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_wc:\n",
      "  llm_1: 168.01 ± 8.69\n",
      "  llm_n: 191.41 ± 30.28\n",
      "  p-value: 0.0000000000\n",
      "avg_word_len:\n",
      "  llm_1: 4.42 ± 0.30\n",
      "  llm_n: 4.42 ± 0.34\n",
      "  p-value: 0.8185814127\n",
      "n_sentences:\n",
      "  llm_1: 10.23 ± 2.28\n",
      "  llm_n: 12.19 ± 3.59\n",
      "  p-value: 0.0000000000\n",
      "sentence_length_mean:\n",
      "  llm_1: 17.23 ± 3.89\n",
      "  llm_n: 16.68 ± 4.16\n",
      "  p-value: 0.0010943821\n",
      "ttr:\n",
      "  llm_1: 0.62 ± 0.05\n",
      "  llm_n: 0.57 ± 0.07\n",
      "  p-value: 0.0000000000\n",
      "syllables_per_token_mean:\n",
      "  llm_1: 1.28 ± 0.09\n",
      "  llm_n: 1.29 ± 0.10\n",
      "  p-value: 0.0056727663\n",
      "stopwords_frac:\n",
      "  llm_1: 54.66 ± 5.04\n",
      "  llm_n: 56.69 ± 5.47\n",
      "  p-value: 0.0000000000\n",
      "flesch_reading_ease:\n",
      "  llm_1: 80.78 ± 9.01\n",
      "  llm_n: 80.45 ± 10.20\n",
      "  p-value: 0.3980000201\n",
      "entropy:\n",
      "  llm_1: 6.50 ± 0.68\n",
      "  llm_n: 7.85 ± 1.48\n",
      "  p-value: 0.0000000000\n",
      "perplexity:\n",
      "  llm_1: 840.67 ± 661.02\n",
      "  llm_n: 5325.62 ± 6911.90\n",
      "  p-value: 0.0000000000\n",
      "per_word_perplexity:\n",
      "  llm_1: 4.28 ± 3.25\n",
      "  llm_n: 22.04 ± 27.27\n",
      "  p-value: 0.0000000000\n",
      "dependency_distance_mean:\n",
      "  llm_1: 2.66 ± 0.27\n",
      "  llm_n: 2.59 ± 0.31\n",
      "  p-value: 0.0000000001\n",
      "first_order_coherence:\n",
      "  llm_1: 0.47 ± 0.08\n",
      "  llm_n: 0.45 ± 0.08\n",
      "  p-value: 0.0000001954\n",
      "second_order_coherence:\n",
      "  llm_1: 0.47 ± 0.08\n",
      "  llm_n: 0.45 ± 0.08\n",
      "  p-value: nan\n",
      "====================================================================================================\n",
      "mistral\n",
      "====================================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1610/1610 [01:14<00:00, 21.65it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------------------------------------------\n",
      "Comparision with part-2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1025/1025 [00:49<00:00, 20.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_wc:\n",
      "  llm_1: 177.25 ± 12.61\n",
      "  llm_n: 182.24 ± 12.88\n",
      "  p-value: 0.0000000000\n",
      "avg_word_len:\n",
      "  llm_1: 4.49 ± 0.32\n",
      "  llm_n: 4.39 ± 0.30\n",
      "  p-value: 0.0000000000\n",
      "n_sentences:\n",
      "  llm_1: 11.89 ± 2.53\n",
      "  llm_n: 12.78 ± 2.71\n",
      "  p-value: 0.0000000000\n",
      "sentence_length_mean:\n",
      "  llm_1: 15.54 ± 3.28\n",
      "  llm_n: 14.86 ± 3.10\n",
      "  p-value: 0.0000000822\n",
      "ttr:\n",
      "  llm_1: 0.64 ± 0.04\n",
      "  llm_n: 0.62 ± 0.04\n",
      "  p-value: 0.0000000000\n",
      "syllables_per_token_mean:\n",
      "  llm_1: 1.30 ± 0.09\n",
      "  llm_n: 1.27 ± 0.08\n",
      "  p-value: 0.0000000000\n",
      "stopwords_frac:\n",
      "  llm_1: 53.78 ± 5.07\n",
      "  llm_n: 56.10 ± 4.82\n",
      "  p-value: 0.0000000000\n",
      "flesch_reading_ease:\n",
      "  llm_1: 81.04 ± 8.58\n",
      "  llm_n: 83.99 ± 8.40\n",
      "  p-value: 0.0000000000\n",
      "entropy:\n",
      "  llm_1: 7.29 ± 0.81\n",
      "  llm_n: 7.47 ± 0.84\n",
      "  p-value: 0.0000000313\n",
      "perplexity:\n",
      "  llm_1: 2018.08 ± 1848.89\n",
      "  llm_n: 2511.62 ± 2578.75\n",
      "  p-value: 0.0000001201\n",
      "per_word_perplexity:\n",
      "  llm_1: 9.38 ± 8.09\n",
      "  llm_n: 11.33 ± 11.06\n",
      "  p-value: 0.0000012212\n",
      "dependency_distance_mean:\n",
      "  llm_1: 2.60 ± 0.24\n",
      "  llm_n: 2.55 ± 0.22\n",
      "  p-value: 0.0000000016\n",
      "first_order_coherence:\n",
      "  llm_1: 0.46 ± 0.07\n",
      "  llm_n: 0.44 ± 0.07\n",
      "  p-value: 0.0000000000\n",
      "second_order_coherence:\n",
      "  llm_1: 0.45 ± 0.08\n",
      "  llm_n: 0.43 ± 0.07\n",
      "  p-value: 0.0000000000\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comparision with part-3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 201/201 [00:09<00:00, 21.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_wc:\n",
      "  llm_1: 177.25 ± 12.61\n",
      "  llm_n: 178.09 ± 22.71\n",
      "  p-value: 0.6064777328\n",
      "avg_word_len:\n",
      "  llm_1: 4.49 ± 0.32\n",
      "  llm_n: 4.42 ± 0.34\n",
      "  p-value: 0.0067363213\n",
      "n_sentences:\n",
      "  llm_1: 11.89 ± 2.53\n",
      "  llm_n: 11.61 ± 2.92\n",
      "  p-value: 0.1864784394\n",
      "sentence_length_mean:\n",
      "  llm_1: 15.54 ± 3.28\n",
      "  llm_n: 16.11 ± 3.50\n",
      "  p-value: 0.0319511104\n",
      "ttr:\n",
      "  llm_1: 0.64 ± 0.04\n",
      "  llm_n: 0.61 ± 0.05\n",
      "  p-value: 0.0000000000\n",
      "syllables_per_token_mean:\n",
      "  llm_1: 1.30 ± 0.09\n",
      "  llm_n: 1.29 ± 0.10\n",
      "  p-value: 0.0775318804\n",
      "stopwords_frac:\n",
      "  llm_1: 53.78 ± 5.07\n",
      "  llm_n: 56.43 ± 4.69\n",
      "  p-value: 0.0000000000\n",
      "flesch_reading_ease:\n",
      "  llm_1: 81.04 ± 8.58\n",
      "  llm_n: 81.59 ± 10.07\n",
      "  p-value: 0.4586410853\n",
      "entropy:\n",
      "  llm_1: 7.29 ± 0.81\n",
      "  llm_n: 7.23 ± 1.12\n",
      "  p-value: 0.4763001760\n",
      "perplexity:\n",
      "  llm_1: 2018.08 ± 1848.89\n",
      "  llm_n: 2291.33 ± 2499.77\n",
      "  p-value: 0.1360763194\n",
      "per_word_perplexity:\n",
      "  llm_1: 9.38 ± 8.09\n",
      "  llm_n: 10.35 ± 10.66\n",
      "  p-value: 0.2139176983\n",
      "dependency_distance_mean:\n",
      "  llm_1: 2.60 ± 0.24\n",
      "  llm_n: 2.61 ± 0.21\n",
      "  p-value: 0.6113361542\n",
      "first_order_coherence:\n",
      "  llm_1: 0.46 ± 0.07\n",
      "  llm_n: 0.45 ± 0.07\n",
      "  p-value: 0.1357968579\n",
      "second_order_coherence:\n",
      "  llm_1: 0.45 ± 0.08\n",
      "  llm_n: 0.44 ± 0.08\n",
      "  p-value: 0.2078959992\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comparision with part-4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1518/1518 [01:11<00:00, 21.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_wc:\n",
      "  llm_1: 177.25 ± 12.61\n",
      "  llm_n: 178.82 ± 19.47\n",
      "  p-value: 0.0078541919\n",
      "avg_word_len:\n",
      "  llm_1: 4.49 ± 0.32\n",
      "  llm_n: 4.41 ± 0.32\n",
      "  p-value: 0.0000000000\n",
      "n_sentences:\n",
      "  llm_1: 11.89 ± 2.53\n",
      "  llm_n: 11.74 ± 2.80\n",
      "  p-value: 0.0996951782\n",
      "sentence_length_mean:\n",
      "  llm_1: 15.54 ± 3.28\n",
      "  llm_n: 15.98 ± 3.52\n",
      "  p-value: 0.0003766743\n",
      "ttr:\n",
      "  llm_1: 0.64 ± 0.04\n",
      "  llm_n: 0.61 ± 0.04\n",
      "  p-value: 0.0000000000\n",
      "syllables_per_token_mean:\n",
      "  llm_1: 1.30 ± 0.09\n",
      "  llm_n: 1.28 ± 0.09\n",
      "  p-value: 0.0000001525\n",
      "stopwords_frac:\n",
      "  llm_1: 53.78 ± 5.07\n",
      "  llm_n: 56.21 ± 4.71\n",
      "  p-value: 0.0000000000\n",
      "flesch_reading_ease:\n",
      "  llm_1: 81.04 ± 8.58\n",
      "  llm_n: 82.02 ± 9.09\n",
      "  p-value: 0.0020495947\n",
      "entropy:\n",
      "  llm_1: 7.29 ± 0.81\n",
      "  llm_n: 7.32 ± 1.04\n",
      "  p-value: 0.2791024802\n",
      "perplexity:\n",
      "  llm_1: 2018.08 ± 1848.89\n",
      "  llm_n: 2378.86 ± 2528.46\n",
      "  p-value: 0.0000061063\n",
      "per_word_perplexity:\n",
      "  llm_1: 9.38 ± 8.09\n",
      "  llm_n: 10.82 ± 10.76\n",
      "  p-value: 0.0000257480\n",
      "dependency_distance_mean:\n",
      "  llm_1: 2.60 ± 0.24\n",
      "  llm_n: 2.60 ± 0.23\n",
      "  p-value: 0.8379765755\n",
      "first_order_coherence:\n",
      "  llm_1: 0.46 ± 0.07\n",
      "  llm_n: 0.45 ± 0.07\n",
      "  p-value: 0.0008453254\n",
      "second_order_coherence:\n",
      "  llm_1: 0.45 ± 0.08\n",
      "  llm_n: 0.44 ± 0.08\n",
      "  p-value: nan\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comparision with part-5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 866/866 [00:39<00:00, 21.69it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_wc:\n",
      "  llm_1: 177.25 ± 12.61\n",
      "  llm_n: 178.15 ± 22.69\n",
      "  p-value: 0.2782941172\n",
      "avg_word_len:\n",
      "  llm_1: 4.49 ± 0.32\n",
      "  llm_n: 4.45 ± 0.33\n",
      "  p-value: 0.0009425928\n",
      "n_sentences:\n",
      "  llm_1: 11.89 ± 2.53\n",
      "  llm_n: 11.18 ± 2.68\n",
      "  p-value: 0.0000000002\n",
      "sentence_length_mean:\n",
      "  llm_1: 15.54 ± 3.28\n",
      "  llm_n: 16.64 ± 3.58\n",
      "  p-value: 0.0000000000\n",
      "ttr:\n",
      "  llm_1: 0.64 ± 0.04\n",
      "  llm_n: 0.61 ± 0.05\n",
      "  p-value: 0.0000000000\n",
      "syllables_per_token_mean:\n",
      "  llm_1: 1.30 ± 0.09\n",
      "  llm_n: 1.30 ± 0.10\n",
      "  p-value: 0.9546966327\n",
      "stopwords_frac:\n",
      "  llm_1: 53.78 ± 5.07\n",
      "  llm_n: 56.50 ± 4.72\n",
      "  p-value: 0.0000000000\n",
      "flesch_reading_ease:\n",
      "  llm_1: 81.04 ± 8.58\n",
      "  llm_n: 79.91 ± 9.44\n",
      "  p-value: 0.0033619803\n",
      "entropy:\n",
      "  llm_1: 7.29 ± 0.81\n",
      "  llm_n: 7.29 ± 1.14\n",
      "  p-value: 0.9165784291\n",
      "perplexity:\n",
      "  llm_1: 2018.08 ± 1848.89\n",
      "  llm_n: 2409.30 ± 2609.89\n",
      "  p-value: 0.0000959810\n",
      "per_word_perplexity:\n",
      "  llm_1: 9.38 ± 8.09\n",
      "  llm_n: 10.99 ± 11.18\n",
      "  p-value: 0.0001855808\n",
      "dependency_distance_mean:\n",
      "  llm_1: 2.60 ± 0.24\n",
      "  llm_n: 2.62 ± 0.24\n",
      "  p-value: 0.0575283251\n",
      "first_order_coherence:\n",
      "  llm_1: 0.46 ± 0.07\n",
      "  llm_n: 0.46 ± 0.07\n",
      "  p-value: nan\n",
      "second_order_coherence:\n",
      "  llm_1: 0.45 ± 0.08\n",
      "  llm_n: 0.45 ± 0.08\n",
      "  p-value: nan\n",
      "====================================================================================================\n",
      "gemma\n",
      "====================================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 854/854 [00:39<00:00, 21.50it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------------------------------------------\n",
      "Comparision with part-2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1545/1545 [01:05<00:00, 23.59it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_wc:\n",
      "  llm_1: 172.97 ± 16.47\n",
      "  llm_n: 157.17 ± 36.22\n",
      "  p-value: 0.0000000000\n",
      "avg_word_len:\n",
      "  llm_1: 4.80 ± 0.29\n",
      "  llm_n: 4.76 ± 0.34\n",
      "  p-value: 0.0031048579\n",
      "n_sentences:\n",
      "  llm_1: 12.61 ± 2.65\n",
      "  llm_n: 11.70 ± 3.72\n",
      "  p-value: 0.0000000000\n",
      "sentence_length_mean:\n",
      "  llm_1: 14.20 ± 2.65\n",
      "  llm_n: 14.05 ± 2.78\n",
      "  p-value: 0.1854688309\n",
      "ttr:\n",
      "  llm_1: 0.67 ± 0.04\n",
      "  llm_n: 0.67 ± 0.05\n",
      "  p-value: 0.0201563205\n",
      "syllables_per_token_mean:\n",
      "  llm_1: 1.38 ± 0.10\n",
      "  llm_n: 1.36 ± 0.10\n",
      "  p-value: 0.0036165083\n",
      "stopwords_frac:\n",
      "  llm_1: 46.51 ± 3.97\n",
      "  llm_n: 49.56 ± 4.74\n",
      "  p-value: 0.0000000000\n",
      "flesch_reading_ease:\n",
      "  llm_1: 75.95 ± 8.53\n",
      "  llm_n: 77.13 ± 9.70\n",
      "  p-value: 0.0018982316\n",
      "entropy:\n",
      "  llm_1: 6.97 ± 0.87\n",
      "  llm_n: 6.24 ± 1.59\n",
      "  p-value: 0.0000000000\n",
      "perplexity:\n",
      "  llm_1: 1527.20 ± 1658.08\n",
      "  llm_n: 1210.79 ± 1590.61\n",
      "  p-value: 0.0000060803\n",
      "per_word_perplexity:\n",
      "  llm_1: 7.05 ± 6.83\n",
      "  llm_n: 5.59 ± 6.67\n",
      "  p-value: 0.0000005286\n",
      "dependency_distance_mean:\n",
      "  llm_1: 2.47 ± 0.21\n",
      "  llm_n: 2.46 ± 0.22\n",
      "  p-value: 0.1594212370\n",
      "first_order_coherence:\n",
      "  llm_1: 0.49 ± 0.07\n",
      "  llm_n: 0.47 ± 0.08\n",
      "  p-value: 0.0000078010\n",
      "second_order_coherence:\n",
      "  llm_1: 0.48 ± 0.08\n",
      "  llm_n: 0.47 ± 0.08\n",
      "  p-value: nan\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comparision with part-3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1379/1379 [00:46<00:00, 29.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_wc:\n",
      "  llm_1: 172.97 ± 16.47\n",
      "  llm_n: 124.51 ± 46.13\n",
      "  p-value: 0.0000000000\n",
      "avg_word_len:\n",
      "  llm_1: 4.80 ± 0.29\n",
      "  llm_n: 4.81 ± 0.42\n",
      "  p-value: 0.5782471887\n",
      "n_sentences:\n",
      "  llm_1: 12.61 ± 2.65\n",
      "  llm_n: 9.19 ± 4.12\n",
      "  p-value: 0.0000000000\n",
      "sentence_length_mean:\n",
      "  llm_1: 14.20 ± 2.65\n",
      "  llm_n: 14.32 ± 3.05\n",
      "  p-value: 0.3558719642\n",
      "ttr:\n",
      "  llm_1: 0.67 ± 0.04\n",
      "  llm_n: 0.70 ± 0.07\n",
      "  p-value: 0.0000000000\n",
      "syllables_per_token_mean:\n",
      "  llm_1: 1.38 ± 0.10\n",
      "  llm_n: 1.39 ± 0.13\n",
      "  p-value: 0.0860631585\n",
      "stopwords_frac:\n",
      "  llm_1: 46.51 ± 3.97\n",
      "  llm_n: 49.99 ± 5.20\n",
      "  p-value: 0.0000000000\n",
      "flesch_reading_ease:\n",
      "  llm_1: 75.95 ± 8.53\n",
      "  llm_n: 75.13 ± 12.29\n",
      "  p-value: 0.0646353800\n",
      "entropy:\n",
      "  llm_1: 6.97 ± 0.87\n",
      "  llm_n: 4.95 ± 1.99\n",
      "  p-value: 0.0000000000\n",
      "perplexity:\n",
      "  llm_1: 1527.20 ± 1658.08\n",
      "  llm_n: 703.67 ± 1241.23\n",
      "  p-value: 0.0000000000\n",
      "per_word_perplexity:\n",
      "  llm_1: 7.05 ± 6.83\n",
      "  llm_n: 3.33 ± 5.33\n",
      "  p-value: 0.0000000000\n",
      "dependency_distance_mean:\n",
      "  llm_1: 2.47 ± 0.21\n",
      "  llm_n: 2.48 ± 0.28\n",
      "  p-value: 0.5456303545\n",
      "first_order_coherence:\n",
      "  llm_1: 0.49 ± 0.07\n",
      "  llm_n: 0.47 ± 0.08\n",
      "  p-value: 0.0000268500\n",
      "second_order_coherence:\n",
      "  llm_1: 0.48 ± 0.08\n",
      "  llm_n: 0.47 ± 0.09\n",
      "  p-value: nan\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comparision with part-4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 616/616 [00:21<00:00, 29.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_wc:\n",
      "  llm_1: 172.97 ± 16.47\n",
      "  llm_n: 129.76 ± 47.16\n",
      "  p-value: 0.0000000000\n",
      "avg_word_len:\n",
      "  llm_1: 4.80 ± 0.29\n",
      "  llm_n: 4.83 ± 0.42\n",
      "  p-value: 0.0755270156\n",
      "n_sentences:\n",
      "  llm_1: 12.61 ± 2.65\n",
      "  llm_n: 9.30 ± 4.15\n",
      "  p-value: 0.0000000000\n",
      "sentence_length_mean:\n",
      "  llm_1: 14.20 ± 2.65\n",
      "  llm_n: 14.71 ± 2.91\n",
      "  p-value: 0.0007435376\n",
      "ttr:\n",
      "  llm_1: 0.67 ± 0.04\n",
      "  llm_n: 0.69 ± 0.07\n",
      "  p-value: 0.0000008719\n",
      "syllables_per_token_mean:\n",
      "  llm_1: 1.38 ± 0.10\n",
      "  llm_n: 1.39 ± 0.13\n",
      "  p-value: 0.0102775721\n",
      "stopwords_frac:\n",
      "  llm_1: 46.51 ± 3.97\n",
      "  llm_n: 50.51 ± 5.35\n",
      "  p-value: 0.0000000000\n",
      "flesch_reading_ease:\n",
      "  llm_1: 75.95 ± 8.53\n",
      "  llm_n: 74.11 ± 11.81\n",
      "  p-value: 0.0010324759\n",
      "entropy:\n",
      "  llm_1: 6.97 ± 0.87\n",
      "  llm_n: 5.12 ± 2.04\n",
      "  p-value: 0.0000000000\n",
      "perplexity:\n",
      "  llm_1: 1527.20 ± 1658.08\n",
      "  llm_n: 828.02 ± 1389.55\n",
      "  p-value: 0.0000000000\n",
      "per_word_perplexity:\n",
      "  llm_1: 7.05 ± 6.83\n",
      "  llm_n: 3.86 ± 5.91\n",
      "  p-value: 0.0000000000\n",
      "dependency_distance_mean:\n",
      "  llm_1: 2.47 ± 0.21\n",
      "  llm_n: 2.49 ± 0.24\n",
      "  p-value: 0.1536013747\n",
      "first_order_coherence:\n",
      "  llm_1: 0.49 ± 0.07\n",
      "  llm_n: 0.47 ± 0.08\n",
      "  p-value: 0.0022696028\n",
      "second_order_coherence:\n",
      "  llm_1: 0.48 ± 0.08\n",
      "  llm_n: 0.47 ± 0.09\n",
      "  p-value: nan\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comparision with part-5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 826/826 [00:29<00:00, 27.78it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_wc:\n",
      "  llm_1: 172.97 ± 16.47\n",
      "  llm_n: 133.75 ± 46.90\n",
      "  p-value: 0.0000000000\n",
      "avg_word_len:\n",
      "  llm_1: 4.80 ± 0.29\n",
      "  llm_n: 4.84 ± 0.44\n",
      "  p-value: 0.0223985011\n",
      "n_sentences:\n",
      "  llm_1: 12.61 ± 2.65\n",
      "  llm_n: 9.59 ± 4.29\n",
      "  p-value: 0.0000000000\n",
      "sentence_length_mean:\n",
      "  llm_1: 14.20 ± 2.65\n",
      "  llm_n: 14.86 ± 3.21\n",
      "  p-value: 0.0000047137\n",
      "ttr:\n",
      "  llm_1: 0.67 ± 0.04\n",
      "  llm_n: 0.68 ± 0.06\n",
      "  p-value: 0.0029071703\n",
      "syllables_per_token_mean:\n",
      "  llm_1: 1.38 ± 0.10\n",
      "  llm_n: 1.40 ± 0.14\n",
      "  p-value: 0.0000609947\n",
      "stopwords_frac:\n",
      "  llm_1: 46.51 ± 3.97\n",
      "  llm_n: 50.38 ± 5.32\n",
      "  p-value: 0.0000000000\n",
      "flesch_reading_ease:\n",
      "  llm_1: 75.95 ± 8.53\n",
      "  llm_n: 73.28 ± 12.96\n",
      "  p-value: 0.0000007981\n",
      "entropy:\n",
      "  llm_1: 6.97 ± 0.87\n",
      "  llm_n: 5.31 ± 2.05\n",
      "  p-value: 0.0000000000\n",
      "perplexity:\n",
      "  llm_1: 1527.20 ± 1658.08\n",
      "  llm_n: 955.59 ± 1778.28\n",
      "  p-value: 0.0000000000\n",
      "per_word_perplexity:\n",
      "  llm_1: 7.05 ± 6.83\n",
      "  llm_n: 4.43 ± 7.44\n",
      "  p-value: 0.0000000000\n",
      "dependency_distance_mean:\n",
      "  llm_1: 2.47 ± 0.21\n",
      "  llm_n: 2.50 ± 0.24\n",
      "  p-value: 0.0079918268\n",
      "first_order_coherence:\n",
      "  llm_1: 0.49 ± 0.07\n",
      "  llm_n: 0.48 ± 0.08\n",
      "  p-value: 0.0206210263\n",
      "second_order_coherence:\n",
      "  llm_1: 0.48 ± 0.08\n",
      "  llm_n: 0.47 ± 0.09\n",
      "  p-value: nan\n",
      "====================================================================================================\n",
      "orca\n",
      "====================================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 411/411 [00:18<00:00, 21.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------------------------------------------\n",
      "Comparision with part-2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 834/834 [00:38<00:00, 21.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_wc:\n",
      "  llm_1: 174.45 ± 22.20\n",
      "  llm_n: 175.61 ± 10.42\n",
      "  p-value: 0.3139138563\n",
      "avg_word_len:\n",
      "  llm_1: 4.34 ± 0.29\n",
      "  llm_n: 4.38 ± 0.30\n",
      "  p-value: 0.0233548482\n",
      "n_sentences:\n",
      "  llm_1: 11.52 ± 2.82\n",
      "  llm_n: 11.35 ± 2.55\n",
      "  p-value: 0.2908143716\n",
      "sentence_length_mean:\n",
      "  llm_1: 15.83 ± 3.44\n",
      "  llm_n: 16.22 ± 3.56\n",
      "  p-value: 0.0629179730\n",
      "ttr:\n",
      "  llm_1: 0.62 ± 0.05\n",
      "  llm_n: 0.62 ± 0.04\n",
      "  p-value: 0.2719430899\n",
      "syllables_per_token_mean:\n",
      "  llm_1: 1.27 ± 0.08\n",
      "  llm_n: 1.28 ± 0.08\n",
      "  p-value: 0.6732976554\n",
      "stopwords_frac:\n",
      "  llm_1: 57.61 ± 5.38\n",
      "  llm_n: 56.93 ± 4.77\n",
      "  p-value: 0.0296424228\n",
      "flesch_reading_ease:\n",
      "  llm_1: 83.08 ± 8.54\n",
      "  llm_n: 82.51 ± 8.45\n",
      "  p-value: 0.2674072215\n",
      "entropy:\n",
      "  llm_1: 7.02 ± 1.13\n",
      "  llm_n: 7.17 ± 0.79\n",
      "  p-value: 0.0144951515\n",
      "perplexity:\n",
      "  llm_1: 1824.31 ± 2153.68\n",
      "  llm_n: 1846.71 ± 2054.93\n",
      "  p-value: 0.8611367017\n",
      "per_word_perplexity:\n",
      "  llm_1: 8.53 ± 9.50\n",
      "  llm_n: 8.72 ± 9.15\n",
      "  p-value: 0.7339136686\n",
      "dependency_distance_mean:\n",
      "  llm_1: 2.61 ± 0.23\n",
      "  llm_n: 2.61 ± 0.23\n",
      "  p-value: 0.8192626080\n",
      "first_order_coherence:\n",
      "  llm_1: 0.44 ± 0.07\n",
      "  llm_n: 0.45 ± 0.08\n",
      "  p-value: 0.0541824122\n",
      "second_order_coherence:\n",
      "  llm_1: 0.44 ± 0.08\n",
      "  llm_n: 0.44 ± 0.08\n",
      "  p-value: 0.4003308270\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comparision with part-3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1280/1280 [00:59<00:00, 21.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_wc:\n",
      "  llm_1: 174.45 ± 22.20\n",
      "  llm_n: 178.11 ± 15.62\n",
      "  p-value: 0.0020116818\n",
      "avg_word_len:\n",
      "  llm_1: 4.34 ± 0.29\n",
      "  llm_n: 4.41 ± 0.33\n",
      "  p-value: 0.0000800672\n",
      "n_sentences:\n",
      "  llm_1: 11.52 ± 2.82\n",
      "  llm_n: 10.93 ± 2.53\n",
      "  p-value: 0.0001837492\n",
      "sentence_length_mean:\n",
      "  llm_1: 15.83 ± 3.44\n",
      "  llm_n: 17.03 ± 3.66\n",
      "  p-value: 0.0000000018\n",
      "ttr:\n",
      "  llm_1: 0.62 ± 0.05\n",
      "  llm_n: 0.61 ± 0.04\n",
      "  p-value: 0.0000474865\n",
      "syllables_per_token_mean:\n",
      "  llm_1: 1.27 ± 0.08\n",
      "  llm_n: 1.28 ± 0.09\n",
      "  p-value: 0.0133433593\n",
      "stopwords_frac:\n",
      "  llm_1: 57.61 ± 5.38\n",
      "  llm_n: 57.79 ± 4.94\n",
      "  p-value: 0.5518321509\n",
      "flesch_reading_ease:\n",
      "  llm_1: 83.08 ± 8.54\n",
      "  llm_n: 80.86 ± 9.65\n",
      "  p-value: 0.0000107129\n",
      "entropy:\n",
      "  llm_1: 7.02 ± 1.13\n",
      "  llm_n: 7.14 ± 0.93\n",
      "  p-value: 0.0422323550\n",
      "perplexity:\n",
      "  llm_1: 1824.31 ± 2153.68\n",
      "  llm_n: 1854.83 ± 1797.86\n",
      "  p-value: 0.7954233341\n",
      "per_word_perplexity:\n",
      "  llm_1: 8.53 ± 9.50\n",
      "  llm_n: 8.64 ± 7.91\n",
      "  p-value: 0.8244671872\n",
      "dependency_distance_mean:\n",
      "  llm_1: 2.61 ± 0.23\n",
      "  llm_n: 2.66 ± 0.23\n",
      "  p-value: 0.0000403410\n",
      "first_order_coherence:\n",
      "  llm_1: 0.44 ± 0.07\n",
      "  llm_n: 0.45 ± 0.08\n",
      "  p-value: 0.0041403141\n",
      "second_order_coherence:\n",
      "  llm_1: 0.44 ± 0.08\n",
      "  llm_n: 0.45 ± 0.08\n",
      "  p-value: 0.0032866606\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comparision with part-4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1650/1650 [01:16<00:00, 21.66it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_wc:\n",
      "  llm_1: 174.45 ± 22.20\n",
      "  llm_n: 178.01 ± 14.37\n",
      "  p-value: 0.0020828739\n",
      "avg_word_len:\n",
      "  llm_1: 4.34 ± 0.29\n",
      "  llm_n: 4.43 ± 0.34\n",
      "  p-value: 0.0000000624\n",
      "n_sentences:\n",
      "  llm_1: 11.52 ± 2.82\n",
      "  llm_n: 10.75 ± 2.51\n",
      "  p-value: 0.0000006070\n",
      "sentence_length_mean:\n",
      "  llm_1: 15.83 ± 3.44\n",
      "  llm_n: 17.37 ± 3.89\n",
      "  p-value: 0.0000000000\n",
      "ttr:\n",
      "  llm_1: 0.62 ± 0.05\n",
      "  llm_n: 0.61 ± 0.05\n",
      "  p-value: 0.0004985951\n",
      "syllables_per_token_mean:\n",
      "  llm_1: 1.27 ± 0.08\n",
      "  llm_n: 1.29 ± 0.09\n",
      "  p-value: 0.0000627112\n",
      "stopwords_frac:\n",
      "  llm_1: 57.61 ± 5.38\n",
      "  llm_n: 57.33 ± 4.88\n",
      "  p-value: 0.3410024754\n",
      "flesch_reading_ease:\n",
      "  llm_1: 83.08 ± 8.54\n",
      "  llm_n: 79.95 ± 9.97\n",
      "  p-value: 0.0000000003\n",
      "entropy:\n",
      "  llm_1: 7.02 ± 1.13\n",
      "  llm_n: 7.16 ± 0.87\n",
      "  p-value: 0.0151559324\n",
      "perplexity:\n",
      "  llm_1: 1824.31 ± 2153.68\n",
      "  llm_n: 1863.87 ± 1953.05\n",
      "  p-value: 0.7347935621\n",
      "per_word_perplexity:\n",
      "  llm_1: 8.53 ± 9.50\n",
      "  llm_n: 8.71 ± 8.45\n",
      "  p-value: 0.7241132888\n",
      "dependency_distance_mean:\n",
      "  llm_1: 2.61 ± 0.23\n",
      "  llm_n: 2.68 ± 0.23\n",
      "  p-value: 0.0000000371\n",
      "first_order_coherence:\n",
      "  llm_1: 0.44 ± 0.07\n",
      "  llm_n: 0.46 ± 0.08\n",
      "  p-value: 0.0000003144\n",
      "second_order_coherence:\n",
      "  llm_1: 0.44 ± 0.08\n",
      "  llm_n: 0.45 ± 0.08\n",
      "  p-value: nan\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comparision with part-5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1045/1045 [00:47<00:00, 21.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_wc:\n",
      "  llm_1: 174.45 ± 22.20\n",
      "  llm_n: 177.89 ± 16.57\n",
      "  p-value: 0.0046331080\n",
      "avg_word_len:\n",
      "  llm_1: 4.34 ± 0.29\n",
      "  llm_n: 4.43 ± 0.33\n",
      "  p-value: 0.0000009566\n",
      "n_sentences:\n",
      "  llm_1: 11.52 ± 2.82\n",
      "  llm_n: 10.37 ± 2.39\n",
      "  p-value: 0.0000000000\n",
      "sentence_length_mean:\n",
      "  llm_1: 15.83 ± 3.44\n",
      "  llm_n: 17.97 ± 4.23\n",
      "  p-value: 0.0000000000\n",
      "ttr:\n",
      "  llm_1: 0.62 ± 0.05\n",
      "  llm_n: 0.60 ± 0.05\n",
      "  p-value: 0.0000000012\n",
      "syllables_per_token_mean:\n",
      "  llm_1: 1.27 ± 0.08\n",
      "  llm_n: 1.29 ± 0.09\n",
      "  p-value: 0.0007597915\n",
      "stopwords_frac:\n",
      "  llm_1: 57.61 ± 5.38\n",
      "  llm_n: 57.63 ± 4.75\n",
      "  p-value: 0.9388196546\n",
      "flesch_reading_ease:\n",
      "  llm_1: 83.08 ± 8.54\n",
      "  llm_n: 79.51 ± 9.80\n",
      "  p-value: 0.0000000000\n",
      "entropy:\n",
      "  llm_1: 7.02 ± 1.13\n",
      "  llm_n: 7.17 ± 0.95\n",
      "  p-value: 0.0188298843\n",
      "perplexity:\n",
      "  llm_1: 1824.31 ± 2153.68\n",
      "  llm_n: 1979.02 ± 2309.45\n",
      "  p-value: 0.2276797273\n",
      "per_word_perplexity:\n",
      "  llm_1: 8.53 ± 9.50\n",
      "  llm_n: 9.24 ± 10.04\n",
      "  p-value: 0.2060876578\n",
      "dependency_distance_mean:\n",
      "  llm_1: 2.61 ± 0.23\n",
      "  llm_n: 2.71 ± 0.24\n",
      "  p-value: 0.0000000000\n",
      "first_order_coherence:\n",
      "  llm_1: 0.44 ± 0.07\n",
      "  llm_n: 0.47 ± 0.07\n",
      "  p-value: 0.0000000000\n",
      "second_order_coherence:\n",
      "  llm_1: 0.44 ± 0.08\n",
      "  llm_n: 0.47 ± 0.08\n",
      "  p-value: nan\n",
      "====================================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "for author in authors:\n",
    "    print(author)\n",
    "    print('='*100)\n",
    "    df_1 = df[df['author1']==author]\n",
    "    llm_values_1 = []\n",
    "    for index, row in tqdm(df_1.iterrows(), total=df_1.shape[0]):\n",
    "        llm_values_1.append(get_linguistic_feature(row['part1'])) \n",
    "    \n",
    "    print('-'*100)\n",
    "    print('Comparision with part-2')\n",
    "    df_n = df[df['author2']==author]\n",
    "    llm_values_n = []\n",
    "    for index, row in tqdm(df_n.iterrows(), total=df_n.shape[0]):\n",
    "        llm_values_n.append(get_linguistic_feature(row['part2'])) \n",
    "    compare_values_with_part1(llm_values_1,llm_values_n)\n",
    "    \n",
    "    print('-'*100)\n",
    "    print('Comparision with part-3')\n",
    "    df_n = df[df['author3']==author]\n",
    "    llm_values_n = []\n",
    "    for index, row in tqdm(df_n.iterrows(), total=df_n.shape[0]):\n",
    "        llm_values_n.append(get_linguistic_feature(row['part3'])) \n",
    "    compare_values_with_part1(llm_values_1,llm_values_n)\n",
    "\n",
    "    print('-'*100)\n",
    "    print('Comparision with part-4')\n",
    "    df_n = df[df['author4']==author]\n",
    "    llm_values_n = []\n",
    "    for index, row in tqdm(df_n.iterrows(), total=df_n.shape[0]):\n",
    "        llm_values_n.append(get_linguistic_feature(row['part4']))\n",
    "    compare_values_with_part1(llm_values_1,llm_values_n)\n",
    "\n",
    "    print('-'*100)\n",
    "    print('Comparision with part-5')\n",
    "    df_n = df[df['author5']==author]\n",
    "    llm_values_n = []\n",
    "    for index, row in tqdm(df_n.iterrows(), total=df_n.shape[0]):\n",
    "        llm_values_n.append(get_linguistic_feature(row['part5']))\n",
    "    compare_values_with_part1(llm_values_1,llm_values_n)\n",
    "\n",
    "    print('='*100)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}