Skip to content

Commit

Permalink
initial evaluation 2.1
Browse files Browse the repository at this point in the history
  • Loading branch information
@huseinzol05
huseinzol05 committed Dec 6, 2024
1 parent 7b33ebd commit b511e1d
Show file tree
Hide file tree
Showing 6 changed files with 1,134 additions and 6 deletions.
335 changes: 335 additions & 0 deletions session/translation/end-to-end/evaluate/base-2.1-en-ms.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,335 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "3560a64c",
"metadata": {},
"outputs": [],
"source": [
"# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/bjn_Latn.dev\n",
"# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/eng_Latn.dev\n",
"# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/ind_Latn.dev\n",
"# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/jav_Latn.dev\n",
"# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/zsm_Latn.dev\n",
"# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/zho_Hans.dev\n",
"# !wget https://github.com/mesolitica/malaysian-dataset/raw/master/translation/flores200-eval/tam_Taml.dev"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "019bd464",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"\n",
"os.environ['CUDA_VISIBLE_DEVICES'] = '0'"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "2d713f77",
"metadata": {},
"outputs": [],
"source": [
"with open('eng_Latn.dev') as fopen:\n",
" en = fopen.read().split('\\n')\n",
" \n",
"with open('zsm_Latn.dev') as fopen:\n",
" ms = fopen.read().split('\\n')\n",
" \n",
"en_, ms_ = [], []\n",
"for i in range(len(en)):\n",
" if len(en[i]) and len(ms[i]):\n",
" en_.append(en[i])\n",
" ms_.append(ms[i])"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "076a2a37",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(997, 997)"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(en_), len(ms_)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "14d402ec",
"metadata": {},
"outputs": [],
"source": [
"from tqdm import tqdm\n",
"import requests\n",
"import os\n",
"import json"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "4795de0a",
"metadata": {},
"outputs": [],
"source": [
"!rm -rf base-en-ms\n",
"!mkdir base-en-ms"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "37ac8ece",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "62383ba5f6d846a2b79873571a96b13a",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"tokenizer_config.json: 0%| | 0.00/21.0k [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "3146f7ae87a147f7ba53bebf1bfb2eb1",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"tokenizer.json: 0%| | 0.00/1.36M [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "788d6a02976c4f10b2cbe23f36aaa3fb",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"special_tokens_map.json: 0%| | 0.00/2.68k [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "9fb7a59925d348379c921f885b726e1b",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"config.json: 0%| | 0.00/853 [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "7da062dc345045daae694fa651a84b45",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model.safetensors: 0%| | 0.00/990M [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "aeb21db8a9284b15ad9ebf2df181e745",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"generation_config.json: 0%| | 0.00/147 [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"from transformers import AutoTokenizer, T5ForConditionalGeneration\n",
"\n",
"tokenizer = AutoTokenizer.from_pretrained('mesolitica/nanot5-base-malaysian-translation-v2.1')\n",
"model = T5ForConditionalGeneration.from_pretrained('mesolitica/nanot5-base-malaysian-translation-v2.1')"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "bc6b27f8",
"metadata": {},
"outputs": [],
"source": [
"_ = model.cuda()"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "3274b3a7",
"metadata": {},
"outputs": [],
"source": [
"all_special_ids = [0, 1, 2]"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "4ed56e3f",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|█████████████████████████████████████████| 997/997 [06:40<00:00, 2.49it/s]\n"
]
}
],
"source": [
"for i in tqdm(range(len(ms_))):\n",
" filename = os.path.join('base-en-ms', f'{i}.json')\n",
" \n",
" if os.path.exists(filename):\n",
" continue\n",
" \n",
" headers = {\n",
" 'accept': 'application/json',\n",
" 'Content-Type': 'application/json',\n",
" }\n",
" \n",
" input_ids = tokenizer.encode(f'terjemah ke Melayu: {en_[i]}{tokenizer.eos_token}', return_tensors = 'pt')\n",
" outputs = model.generate(input_ids.cuda(), max_length = 1024, num_beams=5, early_stopping=True)\n",
" outputs = [i for i in outputs[0] if i not in all_special_ids]\n",
" r = tokenizer.decode(outputs, spaces_between_special_tokens = False).strip()\n",
"\n",
" with open(filename, 'w') as fopen:\n",
" json.dump({'text': ms_[i], 'r': r}, fopen)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "73e0fe41",
"metadata": {},
"outputs": [],
"source": [
"from sacrebleu.metrics import BLEU, CHRF, TER\n",
"\n",
"chrf = CHRF(word_order = 2)"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "2a764b8d",
"metadata": {},
"outputs": [],
"source": [
"predicted = []\n",
"for i in range(len(ms_)):\n",
" filename = os.path.join('base-en-ms', f'{i}.json')\n",
" with open(filename) as fopen:\n",
" d = json.load(fopen)\n",
" predicted.append(d['r'])"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "f1772a2b",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"chrF2++ = 66.57"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"score = chrf.corpus_score(predicted, [ms_])\n",
"score"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0e552296",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.10"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Loading

0 comments on commit b511e1d

Please sign in to comment.