added translation evaluation

malaya/translation.py

@@ -56,6 +56,14 @@
 1. NLLB-200, MOE, 54.5B, https://tinyurl.com/nllb200moe54bmetrics, zho_Hans-eng_Latn,54.7
 2. NLLB-200, Dense, 3.3B, 17.58 GB, https://tinyurl.com/nllb200dense3bmetrics, zho_Hans-eng_Latn,56.2
 3. NLLB-200, Dense, 1.3B, 5.48 GB, https://tinyurl.com/nllb200dense1bmetrics, zho_Hans-eng_Latn,54.7
+""",
+    'tam_Taml-en': """
+NLLB Metrics, https://github.com/facebookresearch/fairseq/tree/nllb#multilingual-translation-models:
+1. NLLB-200, MOE, 54.5B, https://tinyurl.com/nllb200moe54bmetrics, tam_Taml-eng_Latn,60.8
+""",
+    'tam_Taml-ms': """
+NLLB Metrics, https://github.com/facebookresearch/fairseq/tree/nllb#multilingual-translation-models:
+1. NLLB-200, MOE, 54.5B, https://tinyurl.com/nllb200moe54bmetrics, tam_Taml-zsm_Latn,53.9
 """
 }
 

session/llama3/export.py

@@ -0,0 +1,15 @@
+import os
+
+os.environ['CUDA_VISIBLE_DEVICES'] = ''
+os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'
+
+from transformers.trainer_utils import get_last_checkpoint
+
+latest = get_last_checkpoint("/root/lora-embedding-256-llama3.2-3b-multipack")
+latest
+
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from peft import LoraConfig, get_peft_model, AutoPeftModelForCausalLM, PeftModel
+
+model = AutoPeftModelForCausalLM.from_pretrained(latest)
+model.push_to_hub('mesolitica/lora-embedding-256-llama3.2-3b-multipack', private = True)

session/llama3/lora-256-1b.sh

@@ -0,0 +1,23 @@
+WANDB_PROJECT="lora-embedding-256-llama3.2-1b-multipack" \
+TORCH_DISTRIBUTED_DEBUG="info" \
+torchrun --nproc_per_node 2 \
+-m run-instruction-lora-embedding-multipack \
+--model_name_or_path unsloth/Llama-3.2-1B-Instruct \
+--per_device_train_batch_size 2 \
+--gradient_accumulation_steps 6 \
+--output_dir lora-embedding-256-llama3.2-1b-multipack \
+--bf16 --do_train --do_eval false --num_train_epochs 5 \
+--train_file /home/husein/ssd4/continue-training/packing-4096 \
+--logging_steps 1 \
+--learning_rate 2e-5 \
+--learning_rate 2e-5 \
+--weight_decay 0.01 \
+--block_size 24576 \
+--save_steps 100 \
+--save_total_limit 3 \
+--gradient_checkpointing true \
+--neftune_noise_alpha 5.0 \
+--torch_dtype bfloat16 \
+--rank 256 \
+--ddp_find_unused_parameters false \
+--include_num_input_tokens_seen true

session/llama3/lora-256-3b.sh

@@ -0,0 +1,22 @@
+WANDB_PROJECT="lora-embedding-256-llama3.2-3b-multipack" \
+torchrun --nproc_per_node 4 \
+-m run-instruction-lora-embedding-multipack \
+--model_name_or_path unsloth/Llama-3.2-3B-Instruct \
+--per_device_train_batch_size 3 \
+--gradient_accumulation_steps 6 \
+--output_dir lora-embedding-256-llama3.2-3b-multipack \
+--bf16 --do_train --do_eval false --num_train_epochs 5 \
+--train_file malaysian-llama3.2-24k-language-multipack \
+--logging_steps 1 \
+--learning_rate 2e-5 \
+--learning_rate 2e-5 \
+--weight_decay 0.01 \
+--block_size 24576 \
+--save_steps 100 \
+--save_total_limit 3 \
+--gradient_checkpointing true \
+--neftune_noise_alpha 5.0 \
+--torch_dtype bfloat16 \
+--rank 256 \
+--ddp_find_unused_parameters false \
+--include_num_input_tokens_seen true

session/llama3/lora-embedding-128-8b-multipack.sh → session/llama3/lora-256-8b.sh

@@ -1,5 +1,5 @@
 WANDB_PROJECT=unsloth-Meta-Llama-3.1-8B-Instruct-lora-128-embedding-16k-multipack \
-deepspeed run-instruction-qlora-embedding-multipack.py \
+deepspeed run-instruction-lora-embedding-multipack.py \
 --deepspeed ds_config_zero3.json \
 --model_name_or_path unsloth/Meta-Llama-3.1-8B-Instruct \
 --per_device_train_batch_size 1 \

session/translation/end-to-end/evaluate/README.md

@@ -0,0 +1,5 @@
+# Evaluation
+
+## How do we evaluate Google Translate?
+
+We created Playwright wrapper on top of Google Translate page, and we are not open source the code, but you should able to replicate pretty quick if you are use to Playwright.