Instruct_OPUS_Tatoeba_v1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e40f56c1-ae4a-427e-8ec6-ece0f6bf9e4b",
   "metadata": {},
   "source": [
    "Copyright (c) 2023 by the LEL-A team\n",
    "\n",
    "This software is distributed under the terms of the MIT license\n",
    "which is available at https://opensource.org/licenses/MIT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "dd6a7ef1-9857-4214-b343-d09647e1fb28",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import uuid\n",
    "import pandas as pd\n",
    "import gzip\n",
    "import xmltodict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9ccb2e62-0db3-4cb5-8a3c-b3c774258fa8",
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir -p src_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "154e0957-edae-402a-807f-79d9146a7e9d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File ‘./src_data/OPUS_Tatoeba_v2022_03_03_tmx_de_en.tmx.gz’ already there; not retrieving.\n"
     ]
    }
   ],
   "source": [
    "# -nc option: do not download if available\n",
    "!wget -nc -O ./src_data/OPUS_Tatoeba_v2022_03_03_tmx_de_en.tmx.gz https://object.pouta.csc.fi/OPUS-Tatoeba/v2022-03-03/tmx/de-en.tmx.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f3e26d19-b21e-4af6-aa3f-c27065ef4578",
   "metadata": {},
   "outputs": [],
   "source": [
    "DATASET_NAME_AND_VERSION = \"Instruct_OPUS_Tatoeba_v2022_03_03_de_en_v1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c55d3429-7d8b-4c7e-aa58-690338eb4b7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6f7cef75-baea-40a2-a106-d43b44a6dc09",
   "metadata": {},
   "outputs": [],
   "source": [
    "with gzip.open(\n",
    "    \"./src_data/OPUS_Tatoeba_v2022_03_03_tmx_de_en.tmx.gz\"\n",
    ") as open_file:\n",
    "    tmx_data = xmltodict.parse(open_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "667d6a89-044b-4c76-8473-783975b02f26",
   "metadata": {},
   "outputs": [],
   "source": [
    "# unpack tmx_data to lists\n",
    "\n",
    "de_texts = []\n",
    "en_texts = []\n",
    "\n",
    "for tu in tmx_data[\"tmx\"][\"body\"][\"tu\"]:\n",
    "    de_text = None\n",
    "    en_text = None\n",
    "    for tuv in tu[\"tuv\"]:\n",
    "        lang = tuv[\"@xml:lang\"]\n",
    "        text = tuv[\"seg\"]\n",
    "        assert lang in [\"de\", \"en\"]\n",
    "        text = str(text).strip()\n",
    "        assert len(text) > 0, f\"{len(text)}, {text}\"\n",
    "        if lang == \"de\":\n",
    "            if de_text is None:\n",
    "                de_text = text\n",
    "            else:\n",
    "                assert False, \"duplicate de entry should not happen\"\n",
    "        elif lang == \"en\":\n",
    "            if en_text is None:\n",
    "                en_text = text\n",
    "            else:\n",
    "                assert False, \"duplicate en entry should not happen\"\n",
    "        else:\n",
    "            assert False, \"unknown language\"\n",
    "    assert de_text is not None\n",
    "    assert en_text is not None\n",
    "    \n",
    "    assert isinstance(de_text, str)\n",
    "    assert isinstance(en_text, str)\n",
    "    \n",
    "    assert len(de_text) > 2, de_text\n",
    "    assert len(en_text) > 2, en_text\n",
    "    \n",
    "    de_texts.append(de_text)\n",
    "    en_texts.append(en_text)\n",
    "    assert len(de_texts) == len(en_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f7188f7e-3103-459d-8043-850bc31128ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_pair_example(index):\n",
    "    print(en_texts[index])    \n",
    "    print(de_texts[index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "03c6227e-db4f-494e-b01c-65b9fadb97a9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Muiriel has turned twenty.\n",
      "Muiriel ist jetzt 20.\n"
     ]
    }
   ],
   "source": [
    "print_pair_example(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "186caae2-53c3-4fcd-8c24-65b3d9c54b9f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The wine is in the market.\n",
      "Der Wein ist auf dem Markt.\n"
     ]
    }
   ],
   "source": [
    "print_pair_example(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "4a0b7e59-d324-4ff2-9c0c-eb83c4f2e423",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_synonyms():\n",
    "    \"\"\"Select synonyms from synonyms dict.\"\"\"\n",
    "    return {k: random.choice(v) for (k, v) in synonyms.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1080193a-baae-41bf-be85-506bbeb34688",
   "metadata": {},
   "outputs": [],
   "source": [
    "synonyms = {\n",
    "    \"text_de\": [\"Text\", \"Absatz\", \"Textbereich\", \"Abschnitt\"],\n",
    "    \"text_en\": [\"text\", \"paragraph\", \"text area\", \"section\"],    \n",
    "    \"exakt_de\": [\"exakt\", \"genau\", \"präzise\", \"akkurat\"],\n",
    "    \"exakt_en\": [\"exactly\", \"accurately\", \"precisely\"],\n",
    "    \"ist_de\": [\"ist\", \"lautet\", \"heißt\"],    \n",
    "    \"ist_en\": [\"is\", \"reads\", \"is called\"],\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c7ff219d-7485-492e-9b79-31b608488852",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_templates_to_en = [\n",
    "# de\n",
    "\"\"\"\\\n",
    "Bitte übersetze folgenden {text_de} ins Englische. Antworte nur {exakt_de} mit der Übersetzung.\n",
    "Der {text_de} {ist_de}: {input}\"\"\",\n",
    "\n",
    "\"\"\"\\\n",
    "Gegeben ist folgender {text_de}: {input}\n",
    "Übersetze den {text_de} ins Englische und antworte nur {exakt_de} mit der Übersetzung.\"\"\",\n",
    "\n",
    "\"\"\"\\\n",
    "Gegeben ist der {text_de}: {input}\n",
    "Übertrage den {text_de} in englische Sprache und antworte nur {exakt_de} mit der Übersetzung.\"\"\",    \n",
    "    \n",
    "\"\"\"\\\n",
    "Übersetz den folgenden {text_de} ins Englische und erzeuge nur {exakt_de} die Übersetzung.\n",
    "{input}\"\"\",\n",
    "\n",
    "\"\"\"\\\n",
    "Sag mir, was folgender {text_de} auf Englisch heißt. Gebe mir {exakt_de} nur die Übersetzung.\n",
    "{input}\"\"\",    \n",
    "    \n",
    "# en\n",
    "\"\"\"\\\n",
    "Please translate the following {text_en} into English. Answer only {exakt_en} with the translation.\n",
    "The {text_en} {ist_en}: {input}\"\"\",\n",
    "\n",
    "\"\"\"\\\n",
    "Given is the following {text_en}: {input}\n",
    "Translate the {text_en} into English and answer only {exakt_en} with the translation.\"\"\",\n",
    "\n",
    "\"\"\"\\\n",
    "Given is the following {text_en}: {input}\n",
    "Transfer the {text_en} into English and answer only {exakt_en} with the translation.\"\"\",    \n",
    "\n",
    "\"\"\"\\\n",
    "Translate the following {text_en} into English and create only the exact translation.\n",
    "{input}\"\"\",  \n",
    "    \n",
    "\"\"\"\\\n",
    "Tell me what the following {text_en} is called in English. Give me {exakt_en} only the translation.\n",
    "{input}\"\"\",        \n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f238232d-dd0e-40c1-9f7b-f236c635cb14",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_templates_to_de = [\n",
    "# de\n",
    "\"\"\"\\\n",
    "Bitte übersetze folgenden {text_de} ins Deutsche. Antworte nur {exakt_de} mit der Übersetzung.\n",
    "Der {text_de} {ist_de}: {input}\"\"\",\n",
    "\n",
    "\"\"\"\\\n",
    "Gegeben ist folgender {text_de}: {input}\n",
    "Übersetze den {text_de} ins Deutsche und antworte nur {exakt_de} mit der Übersetzung.\"\"\",\n",
    "\n",
    "\"\"\"\\\n",
    "Gegeben ist der {text_de}: {input}\n",
    "Übertrage den {text_de} in deutsche Sprache und antworte nur {exakt_de} mit der Übersetzung.\"\"\",    \n",
    "    \n",
    "\"\"\"\\\n",
    "Übersetz den folgenden {text_de} ins Deutsche und erzeuge nur {exakt_de} die Übersetzung.\n",
    "{input}\"\"\",\n",
    "\n",
    "\"\"\"\\\n",
    "Sag mir, was folgender {text_de} auf Deutsch heißt. Gebe mir {exakt_de} nur die Übersetzung.\n",
    "{input}\"\"\",    \n",
    "    \n",
    "# en\n",
    "\"\"\"\\\n",
    "Please translate the following {text_en} into German. Answer only {exakt_en} with the translation.\n",
    "The {text_en} {ist_en}: {input}\"\"\",\n",
    "\n",
    "\"\"\"\\\n",
    "Given is the following {text_en}: {input}\n",
    "Translate the {text_en} into German and answer only {exakt_en} with the translation.\"\"\",\n",
    "\n",
    "\"\"\"\\\n",
    "Given is the following {text_en}: {input}\n",
    "Transfer the {text_en} into German and answer only {exakt_en} with the translation.\"\"\",    \n",
    "\n",
    "\"\"\"\\\n",
    "Translate the following {text_en} into German and create only the exact translation.\n",
    "{input}\"\"\",  \n",
    "    \n",
    "\"\"\"\\\n",
    "Tell me what the following {text_en} is called in German. Give me {exakt_en} only the translation.\n",
    "{input}\"\"\",        \n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d1f2877f-b925-41dd-8799-7c24ce99120b",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_texts = []\n",
    "output_texts = []\n",
    "src_langs = []\n",
    "target_langs = []\n",
    "\n",
    "for de_text, en_text in zip(de_texts, en_texts):\n",
    "    s = create_synonyms()\n",
    "\n",
    "    to_en_input = random.choice(input_templates_to_en)\n",
    "    to_en_input = to_en_input.format(**s, input=de_text)\n",
    "    if \"{\" in to_en_input:\n",
    "        print(to_en_input)\n",
    "    input_texts.append(to_en_input)\n",
    "    output_texts.append(en_text)\n",
    "    src_langs.append(\"de\")\n",
    "    target_langs.append(\"en\")\n",
    "    \n",
    "    to_de_input = random.choice(input_templates_to_de)\n",
    "    to_de_input = to_de_input.format(**s, input=en_text)\n",
    "    if (\"{\" in to_de_input) or (\"}\" in to_de_input):\n",
    "        print(to_de_input)\n",
    "    input_texts.append(to_de_input)\n",
    "    output_texts.append(de_text)\n",
    "    src_langs.append(\"en\")\n",
    "    target_langs.append(\"de\")    \n",
    "\n",
    "assert len(input_texts) == len(output_texts) == len(src_langs) == len(target_langs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d126cdb4-2236-4a8e-b97b-f3b75517c299",
   "metadata": {},
   "source": [
    "# Explore Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e98487f3-0706-48ba-bf56-fdfee8825d74",
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_example(index):\n",
    "    print(input_texts[index]) \n",
    "    print()\n",
    "    print(output_texts[index])\n",
    "    print()    \n",
    "    print(f\"from {src_langs[index]} to {target_langs[index]}\")    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "832dd137-9630-42b9-8e52-a1c078c623ab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gegeben ist folgender Text: Lass uns etwas versuchen!\n",
      "Übersetze den Text ins Englische und antworte nur präzise mit der Übersetzung.\n",
      "\n",
      "Let's try something.\n",
      "\n",
      "from de to en\n"
     ]
    }
   ],
   "source": [
    "print_example(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "8587373b-d59f-49be-999f-5d87c24e764e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Translate the following text into German and create only the exact translation.\n",
      "Let's try something.\n",
      "\n",
      "Lass uns etwas versuchen!\n",
      "\n",
      "from en to de\n"
     ]
    }
   ],
   "source": [
    "print_example(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "8c6464c7-1516-4776-a4e2-f0e8e8b966f3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tell me what the following text is called in English. Give me exactly only the translation.\n",
      "Was ist das?\n",
      "\n",
      "What's this?\n",
      "\n",
      "from de to en\n"
     ]
    }
   ],
   "source": [
    "print_example(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "58057042-7f85-4761-b8a4-17b521dc4998",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Please translate the following text into German. Answer only exactly with the translation.\n",
      "The text reads: What's this?\n",
      "\n",
      "Was ist das?\n",
      "\n",
      "from en to de\n"
     ]
    }
   ],
   "source": [
    "print_example(11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "cdab8338-ea06-44fb-a815-4838a5166c24",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sag mir, was folgender Text auf Englisch heißt. Gebe mir akkurat nur die Übersetzung.\n",
      "Der Wein ist auf dem Markt.\n",
      "\n",
      "The wine is in the market.\n",
      "\n",
      "from de to en\n"
     ]
    }
   ],
   "source": [
    "print_example(-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "0fb2a1f3-ca4d-4d18-a11e-b674eff66768",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Please translate the following section into German. Answer only accurately with the translation.\n",
      "The section is called: The wine is in the market.\n",
      "\n",
      "Der Wein ist auf dem Markt.\n",
      "\n",
      "from en to de\n"
     ]
    }
   ],
   "source": [
    "print_example(-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c821d9c-dea6-4c4b-b2b6-d7853d6d43a8",
   "metadata": {},
   "source": [
    "# Save Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "5d391d15-c1f5-4ce7-a286-b56f9d0dbb35",
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir -p instruct_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "ba67d938-c3d9-4583-8374-7e87590679f8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input</th>\n",
       "      <th>output</th>\n",
       "      <th>src_lang</th>\n",
       "      <th>target_lang</th>\n",
       "      <th>uuid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Gegeben ist folgender Text: Lass uns etwas ver...</td>\n",
       "      <td>Let's try something.</td>\n",
       "      <td>de</td>\n",
       "      <td>en</td>\n",
       "      <td>f809c770-a2dc-447b-9e61-7cfcb6a036e3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Translate the following text into German and c...</td>\n",
       "      <td>Lass uns etwas versuchen!</td>\n",
       "      <td>en</td>\n",
       "      <td>de</td>\n",
       "      <td>fbee69e9-9e95-4741-98df-31ffeb14e32b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Übersetz den folgenden Text ins Englische und ...</td>\n",
       "      <td>I have to go to sleep.</td>\n",
       "      <td>de</td>\n",
       "      <td>en</td>\n",
       "      <td>a9dcd022-4a25-48f6-a5a2-804409c1bb84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Translate the following section into German an...</td>\n",
       "      <td>Ich muss schlafen gehen.</td>\n",
       "      <td>en</td>\n",
       "      <td>de</td>\n",
       "      <td>5d7714b2-1e6f-4176-b4ce-42aa15150bbd</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Sag mir, was folgender Text auf Englisch heißt...</td>\n",
       "      <td>What is it?</td>\n",
       "      <td>de</td>\n",
       "      <td>en</td>\n",
       "      <td>156662e8-24b1-4a64-8c45-06a317ccf103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>626249</th>\n",
       "      <td>Bitte übersetze folgenden Abschnitt ins Deutsc...</td>\n",
       "      <td>Ihr Vater ist streng.</td>\n",
       "      <td>en</td>\n",
       "      <td>de</td>\n",
       "      <td>e067d944-0ddb-4c74-8fb4-2942b3dd7399</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>626250</th>\n",
       "      <td>Given is the following section: Der Rotwein ko...</td>\n",
       "      <td>The red wine costs thirty coins.</td>\n",
       "      <td>de</td>\n",
       "      <td>en</td>\n",
       "      <td>1263859f-6b16-402a-a51d-5452b7fb37ac</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>626251</th>\n",
       "      <td>Given is the following section: The red wine c...</td>\n",
       "      <td>Der Rotwein kostet dreißig Münzen.</td>\n",
       "      <td>en</td>\n",
       "      <td>de</td>\n",
       "      <td>f2a23cbc-bf2f-4596-bf5e-5c3f4be8f69e</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>626252</th>\n",
       "      <td>Sag mir, was folgender Text auf Englisch heißt...</td>\n",
       "      <td>The wine is in the market.</td>\n",
       "      <td>de</td>\n",
       "      <td>en</td>\n",
       "      <td>3a9d74d7-7a6c-4ecf-8a9f-d22cd92917ea</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>626253</th>\n",
       "      <td>Please translate the following section into Ge...</td>\n",
       "      <td>Der Wein ist auf dem Markt.</td>\n",
       "      <td>en</td>\n",
       "      <td>de</td>\n",
       "      <td>02ef21b7-b6b9-4e6b-add6-3baf67ff89f9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>626254 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    input   \n",
       "0       Gegeben ist folgender Text: Lass uns etwas ver...  \\\n",
       "1       Translate the following text into German and c...   \n",
       "2       Übersetz den folgenden Text ins Englische und ...   \n",
       "3       Translate the following section into German an...   \n",
       "4       Sag mir, was folgender Text auf Englisch heißt...   \n",
       "...                                                   ...   \n",
       "626249  Bitte übersetze folgenden Abschnitt ins Deutsc...   \n",
       "626250  Given is the following section: Der Rotwein ko...   \n",
       "626251  Given is the following section: The red wine c...   \n",
       "626252  Sag mir, was folgender Text auf Englisch heißt...   \n",
       "626253  Please translate the following section into Ge...   \n",
       "\n",
       "                                    output src_lang target_lang   \n",
       "0                     Let's try something.       de          en  \\\n",
       "1                Lass uns etwas versuchen!       en          de   \n",
       "2                   I have to go to sleep.       de          en   \n",
       "3                 Ich muss schlafen gehen.       en          de   \n",
       "4                              What is it?       de          en   \n",
       "...                                    ...      ...         ...   \n",
       "626249               Ihr Vater ist streng.       en          de   \n",
       "626250    The red wine costs thirty coins.       de          en   \n",
       "626251  Der Rotwein kostet dreißig Münzen.       en          de   \n",
       "626252          The wine is in the market.       de          en   \n",
       "626253         Der Wein ist auf dem Markt.       en          de   \n",
       "\n",
       "                                        uuid  \n",
       "0       f809c770-a2dc-447b-9e61-7cfcb6a036e3  \n",
       "1       fbee69e9-9e95-4741-98df-31ffeb14e32b  \n",
       "2       a9dcd022-4a25-48f6-a5a2-804409c1bb84  \n",
       "3       5d7714b2-1e6f-4176-b4ce-42aa15150bbd  \n",
       "4       156662e8-24b1-4a64-8c45-06a317ccf103  \n",
       "...                                      ...  \n",
       "626249  e067d944-0ddb-4c74-8fb4-2942b3dd7399  \n",
       "626250  1263859f-6b16-402a-a51d-5452b7fb37ac  \n",
       "626251  f2a23cbc-bf2f-4596-bf5e-5c3f4be8f69e  \n",
       "626252  3a9d74d7-7a6c-4ecf-8a9f-d22cd92917ea  \n",
       "626253  02ef21b7-b6b9-4e6b-add6-3baf67ff89f9  \n",
       "\n",
       "[626254 rows x 5 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_df = pd.DataFrame({\n",
    "    \"input\": input_texts, \n",
    "    \"output\": output_texts, \n",
    "    \"src_lang\": src_langs, \n",
    "    \"target_lang\": target_langs,\n",
    "})\n",
    "result_df[\"uuid\"] = [uuid.uuid4() for _ in range(len(result_df))]\n",
    "result_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "6928fda6-5e4c-4509-9952-66763eaed088",
   "metadata": {},
   "outputs": [],
   "source": [
    "result_df.to_csv(f\"./instruct_data/{DATASET_NAME_AND_VERSION}.csv.gz\", index=False, compression=\"gzip\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:EuroInstructProject] *",
   "language": "python",
   "name": "conda-env-EuroInstructProject-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}