Skip to content

Commit

Permalink
added base
Browse files Browse the repository at this point in the history
  • Loading branch information
@huseinzol05
huseinzol05 committed Oct 22, 2024
1 parent b37d9bc commit a4b3a8f
Show file tree
Hide file tree
Showing 7 changed files with 1,783 additions and 0 deletions.
251 changes: 251 additions & 0 deletions session/translation/end-to-end/evaluate/base-en-ms.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,251 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "3560a64c",
"metadata": {},
"outputs": [],
"source": [
"# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/bjn_Latn.dev\n",
"# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/eng_Latn.dev\n",
"# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/ind_Latn.dev\n",
"# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/jav_Latn.dev\n",
"# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/zsm_Latn.dev\n",
"# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/zho_Hans.dev\n",
"# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/tam_Taml.dev"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "019bd464",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"\n",
"os.environ['CUDA_VISIBLE_DEVICES'] = '0'"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "2d713f77",
"metadata": {},
"outputs": [],
"source": [
"with open('eng_Latn.dev') as fopen:\n",
" en = fopen.read().split('\\n')\n",
" \n",
"with open('zsm_Latn.dev') as fopen:\n",
" ms = fopen.read().split('\\n')\n",
" \n",
"en_, ms_ = [], []\n",
"for i in range(len(en)):\n",
" if len(en[i]) and len(ms[i]):\n",
" en_.append(en[i])\n",
" ms_.append(ms[i])"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "076a2a37",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(997, 997)"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(en_), len(ms_)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "14d402ec",
"metadata": {},
"outputs": [],
"source": [
"from tqdm import tqdm\n",
"import requests\n",
"import os\n",
"import json"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "4795de0a",
"metadata": {},
"outputs": [],
"source": [
"!rm -rf base-en-ms\n",
"!mkdir base-en-ms"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "37ac8ece",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
]
}
],
"source": [
"from transformers import AutoTokenizer, T5ForConditionalGeneration\n",
"\n",
"tokenizer = AutoTokenizer.from_pretrained('mesolitica/nanot5-base-malaysian-translation-v2')\n",
"model = T5ForConditionalGeneration.from_pretrained('mesolitica/nanot5-base-malaysian-translation-v2')"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "bc6b27f8",
"metadata": {},
"outputs": [],
"source": [
"_ = model.cuda()"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "3274b3a7",
"metadata": {},
"outputs": [],
"source": [
"all_special_ids = [0, 1, 2]"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "4ed56e3f",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████| 997/997 [03:16<00:00, 5.08it/s]\n"
]
}
],
"source": [
"for i in tqdm(range(len(ms_))):\n",
" filename = os.path.join('base-en-ms', f'{i}.json')\n",
" \n",
" if os.path.exists(filename):\n",
" continue\n",
" \n",
" headers = {\n",
" 'accept': 'application/json',\n",
" 'Content-Type': 'application/json',\n",
" }\n",
" \n",
" input_ids = tokenizer.encode(f'terjemah ke Melayu: {en_[i]}{tokenizer.eos_token}', return_tensors = 'pt')\n",
" outputs = model.generate(input_ids.cuda(), max_length = 1024, num_beams=5, early_stopping=True)\n",
" outputs = [i for i in outputs[0] if i not in all_special_ids]\n",
" r = tokenizer.decode(outputs, spaces_between_special_tokens = False).strip()\n",
"\n",
" with open(filename, 'w') as fopen:\n",
" json.dump({'text': ms_[i], 'r': r}, fopen)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "73e0fe41",
"metadata": {},
"outputs": [],
"source": [
"from sacrebleu.metrics import BLEU, CHRF, TER\n",
"\n",
"chrf = CHRF(word_order = 2)"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "2a764b8d",
"metadata": {},
"outputs": [],
"source": [
"predicted = []\n",
"for i in range(len(ms_)):\n",
" filename = os.path.join('base-en-ms', f'{i}.json')\n",
" with open(filename) as fopen:\n",
" d = json.load(fopen)\n",
" predicted.append(d['r'])"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "f1772a2b",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"chrF2++ = 66.23"
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"score = chrf.corpus_score(predicted, [ms_])\n",
"score"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0e552296",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.10"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Loading

0 comments on commit a4b3a8f

Please sign in to comment.