<jupyter_start><jupyter_text>Login HuggingFace to Upload Model<jupyter_code># !git config --global credential.helper store

# from huggingface_hub import notebook_login
# notebook_login()<jupyter_output><empty_output><jupyter_text>1. Training<jupyter_code>import torch
torch.cuda.empty_cache()
!nvidia-smi
def print_encoded(encoded_inputs):
    for ts in encoded_inputs:
        print(ts, encoded_inputs[ts])<jupyter_output><empty_output><jupyter_text>1.1 Prepare-Datasets<jupyter_code>import transformers
print(transformers.__version__)
model_checkpoint = "Helsinki-NLP/opus-mt-th-en"
from datasets import load_dataset, load_metric, DatasetDict, Dataset

# raw_datasets = load_dataset("scb_mt_enth_2020", name="then")
metric = load_metric("sacrebleu")
import pandas as pd
def pre_process_from_csv(path, n_row=100000):
    df_5M = pd.read_csv(path, nrows=n_row)
    list_5M = df_5M.to_dict('records')
    list_sub = ['LST_Corpus']*len(list_5M)
    dict_5M = pd.DataFrame({"translation": list_5M, "subdataset": list_sub})
    return dict_5M
raw_datasets = pre_process_from_csv('TEST_dataset_5M.csv', 500000)
raw_datasets['translation']
cut_datasets = DatasetDict()
cut_datasets = Dataset.from_pandas(raw_datasets, split="train+validation").train_test_split(0.1)
cut_datasets
import datasets, random
import pandas as pd
from IPython.display import display, HTML

def show_random_element(dataset, num_examples=5):
    assert num_examples <= len(dataset), "Can't pick more elements than dataset"
    picks = []
    for n_ in range(num_examples):
        pick = random.randint(0, len(dataset)-1)
        while pick in picks: # random inside
            pick = random.randint(0, len(dataset)-1)
        picks.append(pick)

    df = pd.DataFrame(dataset[picks])
    for column, typ in dataset.features.items():
        if isinstance(typ, datasets.ClassLabel):
            df[column] = df[column].transform(lambda i: typ.names[i])
    display(HTML(df.to_html()))
show_random_element(cut_datasets['train'])
cut_datasets['train'][0:2]<jupyter_output><empty_output><jupyter_text>Metric<jupyter_code># metric
metric = load_metric("sacrebleu")
fake_preds = ["hello there", "general kenobi"]
fake_labels = [["hello there"], ["general kenobi"]]
metric.compute(predictions=fake_preds, references=fake_labels)<jupyter_output><empty_output><jupyter_text>1.2 Preprocessing the Data<jupyter_code>from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
# Set Source and Target Language

if 'mbart' in model_checkpoint:
    tokenizer.src_lang = 'th'
    tokenizer.tgt_lang = 'en'
if model_checkpoint in ["t5-small", "t5-base", "t5-larg", "t5-3b", "t5-11b"]:
    model_checkpoint = "translate Thai to English: "
else:
    prefix = ""
max_input_len  = 128
max_target_len = 128

source_lang = 'th'
target_lang = 'en'

def preprocess_function(examples):
#     inputs = [prefix + ex[source_lang] for ex in examples['translation']]
    inputs, targets = [], []
    for ex_ in (examples['translation']):
        ex_target = ex_[target_lang]
        if ex_target is not None:
            targets.append(ex_[target_lang])
            inputs.append(prefix + ex_[source_lang])
            
#     if len(targets) != len(inputs):
#         print(inputs)
#         print()
#         print(targets)

    model_inputs = tokenizer(inputs, max_length=max_input_len, truncation=True) # Pad to longest word (128 char)
#     print(model_inputs['input_ids'])
    
    with tokenizer.as_target_tokenizer():
        try:
            labels = tokenizer(targets, max_length=max_target_len, truncation=True)
        except:
            print('targets: ', targets)
    
    
#     if len(model_inputs['input_ids']) != len(model_inputs['input_ids']):
#         print(model_inputs)
        
    model_inputs['labels'] = labels['input_ids']
    return model_inputs
max_input_len  = 128
max_target_len = 128

source_lang = 'th'
target_lang = 'en'

def preprocess_function(examples):
#     inputs = [prefix + ex[source_lang] for ex in examples['translation']]
    inputs, targets = [], []
    for ex_ in (examples['translation']):
        ex_target = ex_[target_lang]
        if ex_target is not None:
            targets.append(ex_[target_lang])
            inputs.append(prefix + ex_[source_lang])
            
    model_inputs = tokenizer(inputs, max_length=max_input_len, truncation=True) # Pad to longest word (128 char)
    
    with tokenizer.as_target_tokenizer():
        labels = tokenizer(targets, max_length=max_target_len, truncation=True)
    
    model_inputs['labels'] = labels['input_ids']
    return model_inputs
tokenized_datasets = cut_datasets.map(preprocess_function, 
                                      batched=True, 
                                      batch_size=1000, 
                                      num_proc=32, 
                                      remove_columns=["translation", "subdataset"], # Prevent ArrowInvalid Error'
                                     )
tokenized_datasets<jupyter_output><empty_output><jupyter_text>1.3 Fine-tuning the model<jupyter_code>from transformers import AutoModelForSeq2SeqLM, DataCollatorForSeq2Seq, Seq2SeqTrainingArguments, Seq2SeqTrainer

model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint)
model_checkpoint
torch.cuda.empty_cache()
!nvidia-smi
batch_size = 32
model_name = model_checkpoint.split('/')[-1]
args = Seq2SeqTrainingArguments(
    f'{model_name}-finetuned-5k-{source_lang}-to-{target_lang}',
    evaluation_strategy = "epoch",
    learning_rate = 2e-5,
    per_device_train_batch_size = batch_size,
    per_device_eval_batch_size  = batch_size,
#     gradient_checkpointing=True, # For Accelerate Training
    weight_decay =0.01,
    save_total_limit = 3,
    num_train_epochs=1,
    predict_with_generate=True,
    dataloader_num_workers=32, # Multi-tread CPU
    fp16=True,
    eval_accumulation_steps = 10, # Reduce Using GPU Ram When Evaulation
    push_to_hub = False,
)
?Seq2SeqTrainingArguments<jupyter_output><empty_output><jupyter_text>Create Data collator<jupyter_code>data_collator = DataCollatorForSeq2Seq(tokenizer, model = model)
import numpy as np

def postprocess_text(preds, labels):
    preds = [pred.strip() for pred in preds]
    labels= [[label.strip()] for label in labels]
    return preds, labels

def compute_metrics(eval_preds):
    preds, labels = eval_preds
    if isinstance(preds, tuple):
        preds = preds[0]
    print('preds', preds)
    print('labels', labels)
    decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens = True)

    # Replace -100 in the labels as we can't decode them.
    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)

    # Some simple post-processing
    decoded_preds, decoded_labels = postprocess_text(decoded_preds, decoded_labels)
    
    result = metric.compute(predictions=decoded_preds, references=decoded_labels)
    result = {"bleu": result['score']}

    prediction_lens = [np.count_nonzero(pred != tokenizer.pad_token_id) for pred in preds]
    result["gen_len"] = np.mean(prediction_lens)
    result = {k: round(v, 4) for k, v in result.items()}
    return result
trainer = Seq2SeqTrainer(
    model,
    args,
    train_dataset= tokenized_datasets['train'],
#     eval_dataset = tokenized_datasets['test'],
    eval_dataset = tokenized_datasets['test'],
    data_collator = data_collator,
    tokenizer = tokenizer,
    compute_metrics = compute_metrics,
)<jupyter_output>Using amp half precision backend<jupyter_text>1.4 *train time!*<jupyter_code>trainer.train()<jupyter_output>/usr/local/lib/python3.8/dist-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning
  warnings.warn(
***** Running training *****
  Num examples = 449999
  Num Epochs = 1
  Instantaneous batch size per device = 32
  Total train batch size (w. parallel, distributed & accumulation) = 128
  Gradient Accumulation steps = 1
  Total optimization steps = 3516
/usr/local/lib/python3.8/dist-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.
  warnings.warn('Was asked to gather along dimension 0, but all '<jupyter_text>When Loading the model```from transformers import AutoModelForSeq2SeqLMmodel = AutoModelForSeq2SeqLM.from_pretrained("sgugger/my-awesome-model")```  2. Using Translation Model  Using AutoModel (With model selection)<jupyter_code>?AutoModelForSeq2SeqLM.
# metric = load_metric("bleu")
# fake_preds = [["hello"],["there"], ["general"], ['kenobi']]
# fake_labels = [["hello"],["there"], ["general"], ['kenobi']]
# # fake_labels = [["hello there"], ["general kenobi"]]
# metric.compute(predictions=fake_preds, 
#                references=fake_labels)
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
model_pretrain = "Helsinki-NLP/opus-mt-th-en"
model_for_predict = "./opus-mt-th-en-finetuned-5k-th-to-en"
# model_for_predict = "huak95/opus-mt-th-en-finetuned-5k-th-to-en"

model_pt = AutoModelForSeq2SeqLM.from_pretrained(model_pretrain)
model = AutoModelForSeq2SeqLM.from_pretrained(model_for_predict)
tokenizer = AutoTokenizer.from_pretrained(model_pretrain)
!git lfs track "opus-mt-th-en-finetuned-5k-th-to-en/*"

model.push_to_hub('huak95/opus-mt-th-en-finetuned-5k-th-to-en')
# tokenizer.push_to_hub('huak95/opus-mt-th-en-finetuned-5k-th-to-en')
def harry_translate(input_texts, model=model_pt):
    inputs = tokenizer(input_texts['th'], return_tensors = "pt")

    outputs = model.generate(inputs["input_ids"],
                             max_length=40,
                             num_beams=4,
                             early_stopping=True)

    return tokenizer.decode(outputs[0]).replace('<pad> ',''), input_texts['en']
cut_datasets['test']['translation'][i]
datasets_size = len(cut_datasets['test']['translation'])
rand_idx = np.random.randint(datasets_size, size=10)

for i in rand_idx:
    text_ = cut_datasets['test']['translation'][i]
    thai_text = text_['th']
    pred_text_pt, real_text = harry_translate(text_, model_pt)
    pred_text, real_text = harry_translate(text_, model)
    
#     m_score = compute_metrics.compute(predictions=[pred_text], references=[real_text])

    print(f'index: {i}')
    print('thai_text:',  thai_text)
    print('real_text: ', real_text)
    print('pred_text: ', pred_text)
    print('pre_train: ', pred_text_pt)
    print()<jupyter_output>index: 41090
thai_text: ประสาท ใน ระบบ ดังกล่าว
real_text:  parasympathetic nerve
pred_text:  nerves in the system
pre_train:  The nerves in such systems

index: 1685
thai_text: ทำ ให้ ลุกลาม ใหญ่โต
real_text:  make the sparks fly
pred_text:  make a big spread
pre_train:  In the meantime, though, we need to be careful not to allow ourselves to be influenced by the world’s spirit.

index: 769
thai_text: สิบ ปา กว่า มักจะ ไม่ เท่า ตา เห็น
real_text:  the ears believe other people , the eyes believe themselves .
pred_text:  ten pas is usually not visible.
pre_train:  More than ten pails are usually not the same as the eye.

index: 2433
thai_text: สถานี อุตุนิยมวิทยา
real_text:  a weather station
pred_text:  ecology station
pre_train:  Meteorological Station

index: 5311
thai_text: ท่าน จำเป็นต้อง ใส่ จุด ไว้ ด้วย ตอน จบ ประโยค
real_text:  you need a stop at the end of the sentence .
pred_text:  you need to put a point at the end of the sentence.
pre_train:  He needed to put the point at t[...]
StarcoderdataJupyterStructuredCleanDedup