From 9c9d696eeb259cd352e75c9cc71d4f14eced2f43 Mon Sep 17 00:00:00 2001 From: knc6 Date: Mon, 1 Jul 2024 04:50:03 -0400 Subject: [PATCH 1/5] Fix doc. --- .github/workflows/main.yml | 7 +- atomgpt/config.py | 30 - atomgpt/forward_models/train_id_prop.py | 735 -------------- atomgpt/inverse_models/__init__.py | 14 - atomgpt/inverse_models/_utils.py | 14 - atomgpt/inverse_models/dpo.py | 14 - atomgpt/inverse_models/gemma.py | 14 - .../inverse_models/{inf.py => inference.py} | 2 +- atomgpt/inverse_models/inverse_models.py | 4 +- atomgpt/inverse_models/kernels/__init__.py | 14 - .../kernels/cross_entropy_loss.py | 14 - atomgpt/inverse_models/kernels/fast_lora.py | 14 - atomgpt/inverse_models/kernels/geglu.py | 14 - .../inverse_models/kernels/rms_layernorm.py | 14 - .../inverse_models/kernels/rope_embedding.py | 14 - atomgpt/inverse_models/kernels/swiglu.py | 14 - atomgpt/inverse_models/kernels/utils.py | 14 - atomgpt/inverse_models/llama.py | 14 - atomgpt/inverse_models/loader.py | 14 - atomgpt/inverse_models/mapper.py | 14 - atomgpt/inverse_models/mistral.py | 14 - atomgpt/inverse_models/qwen2.py | 14 - atomgpt/inverse_models/tokenizer_utils.py | 14 - atomgpt/scripts/bnbb.py | 11 - atomgpt/scripts/dpo.py | 120 --- atomgpt/scripts/finetune.py | 359 ------- atomgpt/scripts/finetune.py.bak | 359 ------- atomgpt/scripts/finetune1.py | 548 ----------- atomgpt/scripts/finetune1.py.bak_0.12843 | 547 ----------- atomgpt/scripts/finetune1.py.bak_0.139 | 532 ---------- atomgpt/scripts/finetune1.py.bak_0.146 | 384 -------- atomgpt/scripts/finetune1a.py | 384 -------- atomgpt/scripts/finetune2.py | 358 ------- atomgpt/scripts/finetune3.py | 547 ----------- atomgpt/scripts/finetune4.py | 564 ----------- atomgpt/scripts/finetune5.py | 644 ------------- atomgpt/scripts/finetune6.py | 659 ------------- atomgpt/scripts/finetune7.py | 784 --------------- atomgpt/scripts/finetune7a.py.alignn | 877 ----------------- atomgpt/scripts/finetune7a.py.bak | 815 ---------------- atomgpt/scripts/finetune7alignn.py | 907 ------------------ atomgpt/scripts/finetune7b.py | 812 ---------------- atomgpt/scripts/gp2atom_km.py | 171 ---- atomgpt/scripts/gpt.py | 93 -- atomgpt/scripts/gpt2_describer.py | 93 -- atomgpt/scripts/gpt2_describer1.py | 93 -- atomgpt/scripts/gpt2_robo.py | 116 --- atomgpt/scripts/gpt2atom.py | 201 ---- atomgpt/scripts/usloth_gen.py | 137 --- atomgpt/scripts/usloth_prop.py | 406 -------- atomgpt/train_id_prop.py | 716 -------------- atomgpt/train_prop.py | 416 -------- 52 files changed, 5 insertions(+), 13678 deletions(-) delete mode 100644 atomgpt/config.py delete mode 100644 atomgpt/forward_models/train_id_prop.py rename atomgpt/inverse_models/{inf.py => inference.py} (99%) delete mode 100644 atomgpt/scripts/bnbb.py delete mode 100644 atomgpt/scripts/dpo.py delete mode 100644 atomgpt/scripts/finetune.py delete mode 100644 atomgpt/scripts/finetune.py.bak delete mode 100644 atomgpt/scripts/finetune1.py delete mode 100644 atomgpt/scripts/finetune1.py.bak_0.12843 delete mode 100644 atomgpt/scripts/finetune1.py.bak_0.139 delete mode 100644 atomgpt/scripts/finetune1.py.bak_0.146 delete mode 100644 atomgpt/scripts/finetune1a.py delete mode 100644 atomgpt/scripts/finetune2.py delete mode 100644 atomgpt/scripts/finetune3.py delete mode 100644 atomgpt/scripts/finetune4.py delete mode 100644 atomgpt/scripts/finetune5.py delete mode 100644 atomgpt/scripts/finetune6.py delete mode 100644 atomgpt/scripts/finetune7.py delete mode 100644 atomgpt/scripts/finetune7a.py.alignn delete mode 100644 atomgpt/scripts/finetune7a.py.bak delete mode 100644 atomgpt/scripts/finetune7alignn.py delete mode 100644 atomgpt/scripts/finetune7b.py delete mode 100644 atomgpt/scripts/gp2atom_km.py delete mode 100644 atomgpt/scripts/gpt.py delete mode 100644 atomgpt/scripts/gpt2_describer.py delete mode 100644 atomgpt/scripts/gpt2_describer1.py delete mode 100644 atomgpt/scripts/gpt2_robo.py delete mode 100644 atomgpt/scripts/gpt2atom.py delete mode 100644 atomgpt/scripts/usloth_gen.py delete mode 100644 atomgpt/scripts/usloth_prop.py delete mode 100644 atomgpt/train_id_prop.py delete mode 100644 atomgpt/train_prop.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4d74fad..1f73464 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,16 +58,13 @@ jobs: python atomgpt/forward_models/forward_models.py --config_name atomgpt/examples/forward_model/config.json echo 'inverse model' - python atomgpt/examples/inverse_model/run.py + #python atomgpt/examples/inverse_model/run.py coverage run -m pytest coverage report -m -i codecov #codecov --token="85bd9c5d-9e55-4f6d-bd69-350ee5e3bb41" - #train_alignn.py -h - #echo 'Pre-trained models' - #pretrained.py -h - #find . -type f > after_test_files.txt + find . -type f > after_test_files.txt diff --git a/atomgpt/config.py b/atomgpt/config.py deleted file mode 100644 index f0bb917..0000000 --- a/atomgpt/config.py +++ /dev/null @@ -1,30 +0,0 @@ -from typing import Optional -from pydantic_settings import BaseSettings -class TrainingPropConfig(BaseSettings): - """Training config defaults and validation.""" - - benchmark_file: Optional[str] = None - # "AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae" - id_prop_path: Optional[str] = None - prefix: str = "xyz" - model_name: str = "gpt2" - leaderboard_dir: str = ( - "/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard" - ) - batch_size: int = 8 - max_length: int = 512 - num_epochs: int = 500 - latent_dim: int = 1024 - learning_rate: float = 1e-3 - test_each_run: bool = True - include_struct: bool = False - pretrained_path: str = "" - seed_val: int = 42 - n_train: Optional[int] = None - n_val: Optional[int] = None - n_test: Optional[int] = None - train_ratio: Optional[float] = None - val_ratio: float = 0.1 - test_ratio: float = 0.1 - keep_data_order: bool = False - output_dir: str = "temp" diff --git a/atomgpt/forward_models/train_id_prop.py b/atomgpt/forward_models/train_id_prop.py deleted file mode 100644 index c87a66d..0000000 --- a/atomgpt/forward_models/train_id_prop.py +++ /dev/null @@ -1,735 +0,0 @@ -"""Module for fin tuning LLM model for materials chemsitry.""" - -from jarvis.db.figshare import data -import transformers -import torch -import random -from jarvis.db.jsonutils import loadjson, dumpjson -from torch.utils.data import DataLoader, Dataset -import numpy as np -import os -from jarvis.core.atoms import Atoms -import pandas as pd -from sklearn.metrics import mean_absolute_error -import json -from jarvis.db.figshare import get_jid_data -from jarvis.core.atoms import Atoms -from jarvis.analysis.structure.spacegroup import Spacegroup3D -from jarvis.analysis.diffraction.xrd import XRD -from jarvis.core.specie import Specie -import pprint -from collections import defaultdict -from tqdm import tqdm -import time -import json -import zipfile -from typing import Optional -from pydantic_settings import BaseSettings -import csv -import pprint -class TrainingPropConfig(BaseSettings): - """Training config defaults and validation.""" - - id_prop_path: Optional[str] = "robo_desc.json.zip" - prefix: str = "atomgpt_run" - model_name: str = "gpt2" - batch_size: int = 16 - max_length: int = 512 - num_epochs: int = 500 - latent_dim: int = 1024 - learning_rate: float = 1e-3 - test_each_run: bool = True - include_struct: bool = False - pretrained_path: str = "" - seed_val: int = 42 - n_train: Optional[int] = None - n_val: Optional[int] = None - n_test: Optional[int] = None - output_dir: str = "out_temp" - train_ratio: Optional[float] = None - val_ratio: float = 0.1 - test_ratio: float = 0.1 - keep_data_order: bool = True - - -def get_id_train_val_test( - total_size=1000, - split_seed=123, - train_ratio=None, - val_ratio=0.1, - test_ratio=0.1, - n_train=None, - n_test=None, - n_val=None, - keep_data_order=True, -): - """Get train, val, test IDs.""" - if ( - train_ratio is None - and val_ratio is not None - and test_ratio is not None - ): - if train_ratio is None: - assert val_ratio + test_ratio < 1 - train_ratio = 1 - val_ratio - test_ratio - print("Using rest of the dataset except the test and val sets.") - else: - assert train_ratio + val_ratio + test_ratio <= 1 - # indices = list(range(total_size)) - if n_train is None: - n_train = int(train_ratio * total_size) - if n_test is None: - n_test = int(test_ratio * total_size) - if n_val is None: - n_val = int(val_ratio * total_size) - ids = list(np.arange(total_size)) - if not keep_data_order: - random.seed(split_seed) - random.shuffle(ids) - # np.random.shuffle(ids) - if n_train + n_val + n_test > total_size: - raise ValueError( - "Check total number of samples.", - n_train + n_val + n_test, - ">", - total_size, - ) - - # shuffle consistently with https://github.com/txie-93/cgcnn/data.py - # i.e. shuffle the index in place with standard library random.shuffle - # first obtain only valid indices - - # test_size = round(N * 0.2) - - # full train/val test split - # ids = ids[::-1] - id_train = ids[:n_train] - id_val = ( - ids[-(n_val + n_test) : -n_test] - if n_test > 0 - else ids[-(n_val + n_test) :] - ) # noqa:E203 - id_test = ids[-n_test:] if n_test > 0 else [] - return id_train, id_val, id_test - - -def make_id_prop( - benchmark_file="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae.csv.zip", - desc_file="robo_desc.json.zip", - leaderboard_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard", - # leaderboard_dir="/work/03943/kamalch/ls6/Software/atomgpt/jarvis_leaderboard/jarvis_leaderboard/", - output_dir="test_id_prop", -): - print("benchmark_file", benchmark_file) - method = benchmark_file.split("-")[0] - task = benchmark_file.split("-")[1] - prop_name = benchmark_file.split("-")[2] - dataset = benchmark_file.split("-")[3] - temp = dataset + "_" + prop_name + ".json.zip" - temp2 = dataset + "_" + prop_name + ".json" - fname = os.path.join(leaderboard_dir, "benchmarks", method, task, temp) - zp = zipfile.ZipFile(fname) - bench = json.loads(zp.read(temp2)) - dft_3d = data(dataset) - id_tag = "jid" - output_dir = prop_name + "_" + dataset - if "jid" in dft_3d[0]: - id_tag = "jid" - else: - id_tag = "id" - if not os.path.exists(output_dir): - os.makedirs(output_dir) - train_ids = list(bench["train"].keys()) - test_ids = list(bench["test"].keys()) - if "val" in bench: - val_ids = list(bench["val"].keys()) - else: - val_ids = test_ids - print("Saving files in", output_dir) - if ".zip" in desc_file: - zp = zipfile.ZipFile(desc_file) - dat = json.loads(zp.read(desc_file.split(".zip")[0].split("/")[-1])) - - else: - dat = loadjson(desc_file) - - dat2 = {} - for i in dat: - dat2[i["id"]] = i["desc"] - dft_3d2 = {} - for i in dft_3d: - dft_3d2[i[id_tag]] = i - mem = [] - for i in train_ids: - desc = dat2[i] - prop = dft_3d2[i][prop_name] - info = {} - info["id"] = i - info["desc"] = desc - info["prop"] = prop - mem.append(info) - for i in val_ids: - desc = dat2[i] - - prop = dft_3d2[i][prop_name] - info = {} - info["id"] = i - info["desc"] = desc - info["prop"] = prop - mem.append(info) - for i in test_ids: - desc = dat2[i] - prop = dft_3d2[i][prop_name] - info = {} - info["id"] = i - info["desc"] = desc - info["prop"] = prop - mem.append(info) - print("total", len(dft_3d)) - print("test_ids", len(test_ids)) - print("val_ids", len(val_ids)) - print("train_ids", len(train_ids)) - filename = os.path.join(output_dir, "id_prop_llm.json") - filename_config = os.path.join(output_dir, "config.json") - minfo = {} - minfo["n_train"] = len(train_ids) - minfo["n_val"] = len(val_ids) - minfo["n_test"] = len(test_ids) - minfo["id_prop_path"] = os.path.abspath(filename) - minfo["output_dir"] = os.path.abspath(output_dir) - - dumpjson(data=minfo, filename=filename_config) - dumpjson(data=mem, filename=filename) - return output_dir - - -## -os.environ["WANDB_ANONYMOUS"] = "must" -random_seed = 42 -random.seed(random_seed) -torch.manual_seed(random_seed) -np.random.seed(random_seed) -torch.cuda.manual_seed_all(random_seed) -try: - import torch_xla.core.xla_model as xm - - xm.set_rng_state(random_seed) -except ImportError: - pass -torch.backends.cudnn.deterministic = True -torch.backends.cudnn.benchmark = False -os.environ["PYTHONHASHSEED"] = str(random_seed) -os.environ["CUBLAS_WORKSPACE_CONFIG"] = str(":4096:8") -torch.use_deterministic_algorithms(True) -device = "cpu" -if torch.cuda.is_available(): - device = torch.device("cuda") -# device = "cpu" - - -# Define a custom dataset class for regression -class AtomGPTDataset(Dataset): - def __init__( - self, texts=[], targets=[], ids=[], tokenizer="", max_length=128 - ): - self.texts = texts - self.targets = targets - self.tokenizer = tokenizer - self.max_length = max_length - if not ids: - ids = ["text-" + str(i) for i in range(len(texts))] - self.ids = ids - - def __len__(self): - return len(self.texts) - - def __getitem__(self, idx): - inputs = self.tokenizer( - self.texts[idx], - return_tensors="pt", - max_length=self.max_length, - padding="max_length", - truncation=True, - ) - # torch.tensor(inputs*10,dtype=inputs.dtype) - return ( - inputs, - self.ids[idx], - torch.tensor(self.targets[idx], dtype=torch.float32), - ) - - -# Example usage - - -def run_atomgpt(config_file="config.json"): - print("Running AtomGPT prop predictor.") - run_path = os.path.abspath(config_file).split("config.json")[0] - print('PATH', run_path) - config = loadjson(config_file) - config = TrainingPropConfig(**config) - pprint.pprint(config) - id_prop_path = config.id_prop_path - if ".zip" in id_prop_path: - zp = zipfile.ZipFile(id_prop_path) - dat = json.loads(zp.read(id_prop_path.split(".zip")[0])) - elif ".csv" in id_prop_path: - with open(id_prop_path, "r") as f: - reader = csv.reader(f) - dt = [row for row in reader] - - dat=[] - for i in dt: - info={} - info['id']=i[0] - info['prop']=[float(j) for j in i[1:]] # float(i[1]) - with open(os.path.join(run_path,info['id']),"r") as f: - lines=f.read() - info['desc']=lines - dat.append(info) - - else: - dat = loadjson(id_prop_path) - print("len", len(dat)) - prefix = config.prefix - model_name = config.model_name - batch_size = config.batch_size - max_length = config.max_length - num_epochs = config.num_epochs - latent_dim = config.latent_dim - learning_rate = config.learning_rate - test_each_run = config.test_each_run - pretrained_path = config.pretrained_path - seed_val = config.seed_val - include_struct = config.include_struct - n_train = config.n_train - n_val = config.n_val - n_test = config.n_test - train_ratio = config.train_ratio - val_ratio = config.val_ratio - test_ratio = config.test_ratio - output_dir = config.output_dir - keep_data_order = config.keep_data_order - - f = open(os.path.join(config.output_dir, "config.json"), "w") - f.write(json.dumps(config.dict(), indent=4)) - f.close() - - id_train, id_val, id_test = get_id_train_val_test( - total_size=len(dat), - split_seed=seed_val, - train_ratio=train_ratio, - val_ratio=val_ratio, - test_ratio=test_ratio, - n_train=n_train, - n_test=n_test, - n_val=n_val, - keep_data_order=keep_data_order, - ) - - train_texts = [] - train_targets = [] - train_ids_temp = [] - val_texts = [] - val_targets = [] - val_ids_temp = [] - test_texts = [] - test_targets = [] - test_ids_temp = [] - train_info = [] - val_info = [] - test_info = [] - for ii, i in enumerate(dat): - if ii in id_train: - train_texts.append(i["desc"]) - train_targets.append(i["prop"]) - train_ids_temp.append(i["id"]) - train_info.append(i) - if ii in id_test: - test_texts.append(i["desc"]) - test_targets.append(i["prop"]) - test_ids_temp.append(i["id"]) - val_info.append(i) - if ii in id_val: - val_texts.append(i["desc"]) - val_targets.append(i["prop"]) - val_ids_temp.append(i["id"]) - test_info.append(i) - print("test_texts:", len(test_texts)) - print("val_texts example:", val_texts[0]) - print("test_texts example:", test_texts[0]) - - print("Train\n", pd.DataFrame(train_info)) - print("Val\n", pd.DataFrame(val_info)) - print("test\n", pd.DataFrame(test_info)) - - print("total", len(dat)) - print("test_ids", len(id_test)) - print("val_ids", len(id_val)) - print("train_ids", len(id_train)) - # model_name = "mistralai/Mistral-7B-Instruct-v0.1" - # model_name = "gpt2" - if "t5" in model_name: - model = transformers.T5ForConditionalGeneration.from_pretrained( - model_name - ) - else: - model = transformers.AutoModelForCausalLM.from_pretrained( - model_name, - low_cpu_mem_usage=True, - # load_in_8bit=False, - # torch_dtype=torch.float16, - # load_in_8bit=True, - # device_map="auto" - ) - # device = model.device - if "t5" in model_name: - tokenizer = transformers.T5Tokenizer.from_pretrained(model_name) - - else: - tokenizer = transformers.AutoTokenizer.from_pretrained(model_name) - - # batch_size = 16 - # max_length = 128 - # num_epochs = 100 - # learning_rate = 5e-5 - criterion = torch.nn.L1Loss() - # Define example regression data (texts and corresponding numeric targets) - """ - ############################## - ###Fast test### - train_texts = [ - "This is the first example text.", - "Second example is a bit longer than the first one, but still within the max length.", - "Third example is the longest among these three examples. It exceeds the max length and will be truncated.", - "Second example is a bit longer than the first one, but still within the max length.", - ] - train_targets = [10.2, 15.5, 20.1, 15.5] # Example regression targets - val_texts = test_texts = train_texts - val_targets = test_targets = train_targets - train_ids_temp=['a','b','c','d'] - val_ids_temp = test_ids_temp = train_ids_temp - batch_size = 2 - num_epochs = 3 - - ############################## - ############################## - """ - - # Fine-tune the last layer of GPT-2 for regression - # fine_tune_gpt2_regression(train_texts, train_targets, tokenizer) - if tokenizer.pad_token is None: - tokenizer.add_special_tokens({"pad_token": "[PAD]"}) - tokenizer.add_special_tokens({"unk_token": "#"}) - tokenizer.add_special_tokens({"unk_token": "&"}) - tokenizer.add_special_tokens({"unk_token": "@"}) - model.resize_token_embeddings(len(tokenizer)) - model.lm_head = torch.nn.Sequential( - # torch.nn.Linear(model.config.hidden_size, 1), - torch.nn.Linear(model.config.hidden_size, latent_dim), - # torch.nn.Linear( latent_dim,256), - # torch.nn.Transformer(d_model=latent_dim, nhead=1, num_encoder_layers=1, num_decoder_layers=1), - # torch.nn.Linear(latent_dim, latent_dim), - # torch.nn.Linear(latent_dim, latent_dim), - # torch.nn.ReLU(), - # torch.nn.LeakyReLU(), - # torch.nn.Dropout(p=0.2), - # torch.nn.TransformerEncoder(torch.nn.TransformerEncoderLayer(d_model=latent_dim, nhead=4), num_layers=2), - # torch.nn.Linear(256, 1), - torch.nn.Linear(latent_dim, 1), - ) - if pretrained_path != "": - model.load_state_dict(torch.load(pretrained_path, map_location=device)) - # model.lm_head = torch.nn.Sequential(torch.nn.Linear( model.config.hidden_size, 256),torch.nn.SiLU(),torch.nn.Linear( 256, 1) ) - # set_seed(seed) - # set_deterministic() - model.to(device) - if torch.cuda.device_count() > 1: - device_ids = [d for d in range(torch.cuda.device_count())] - model = torch.nn.DataParallel(model, device_ids=device_ids).cuda() - optimizer = transformers.AdamW(model.parameters(), lr=learning_rate) - # Prepare datasets and dataloaders with data collator - # TODO: knc6 change later - train_dataset = AtomGPTDataset( - texts=train_texts, - targets=train_targets, - ids=train_ids_temp, - tokenizer=tokenizer, - max_length=max_length, - ) - test_dataset = AtomGPTDataset( - texts=val_texts, - targets=val_targets, - tokenizer=tokenizer, - ids=val_ids_temp, - max_length=max_length, - ) - val_dataset = AtomGPTDataset( - texts=test_texts, - targets=test_targets, - tokenizer=tokenizer, - ids=test_ids_temp, - max_length=max_length, - ) - train_dataloader = DataLoader(train_dataset, batch_size=batch_size) - val_dataloader = DataLoader(val_dataset, batch_size=batch_size) - test_dataloader = DataLoader(test_dataset, batch_size=batch_size) - steps_per_epoch = len(train_dataloader) - scheduler = torch.optim.lr_scheduler.OneCycleLR( - optimizer, - max_lr=learning_rate, - epochs=num_epochs, - steps_per_epoch=steps_per_epoch, - # pct_start=pct_start, - pct_start=0.3, - ) - # output_dir = prefix + "_out" # + model_name + "_" + dataset + "_" + prop - if not os.path.exists(output_dir): - os.makedirs(output_dir) - best_loss = np.inf - tot_time_start = time.time() - train_history = [] - val_history = [] - for epoch in range(num_epochs): - model.train() - t1 = time.time() - for batch in train_dataloader: - optimizer.zero_grad() - train_loss = 0 - # train_result = [] - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - targets = batch[2].squeeze() - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - # print('train',predictions,targets) - loss.backward() - optimizer.step() - # optimizer.zero_grad() - train_loss += loss.item() - scheduler.step() - train_loss = train_loss / len(train_dataloader) - t2 = time.time() - train_time = round(t2 - t1, 3) - - # total_eval_mae_loss = 0 - # predictions_list = [] - # targets_list = [] - model.eval() - val_loss = 0 - t1 = time.time() - fname = os.path.join(output_dir, "val_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - with torch.no_grad(): - for batch in val_dataloader: - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - ids = batch[1] - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - targets = batch[2].squeeze() - # print('val',predictions,targets) - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - val_loss += loss.item() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - f.write(line) - f.close() - saving_tag = "" - if val_loss < best_loss: - best_loss = val_loss - best_model_name = "best_model.pt" - torch.save( - model.state_dict(), - os.path.join(output_dir, best_model_name), - ) - # print("Saving model for epoch", epoch) - saving_tag = " saving model:" + str(epoch) - val_loss = val_loss / len(val_dataloader) - t2 = time.time() - val_time = round(t2 - t1, 3) - train_history.append(train_loss) - val_history.append(val_loss) - history = os.path.join(output_dir, "history.json") - - dumpjson( - data={"train": train_history, "val": val_history}, filename=history - ) - mae = "" - model.eval() - with torch.no_grad(): - if test_each_run: - t1_test = time.time() - # model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - test_loss = 0 - for batch in test_dataloader: - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - test_loss += loss.item() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - f.write(line) - test_loss = test_loss / len(test_dataloader) - t2_test = time.time() - test_time = round(t2_test - t1_test, 3) - f.close() - df = pd.read_csv(fname) - mae = mean_absolute_error(df["target"], df["predictions"]) - if mae == "": - print( - "Epoch, train loss, val loss, train_time, val_time", - epoch, - train_loss, - val_loss, - train_time, - val_time, - saving_tag, - ) - else: - print( - "Epoch, train loss, val loss, test loss, train_time, val_time, test_time", - epoch, - train_loss, - val_loss, - # mae, - test_loss, - train_time, - val_time, - test_time, - saving_tag, - ) - - model.eval() - fname = os.path.join(output_dir, "test_results_final.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - optimizer.zero_grad() - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - f.close() - tot_time_end = time.time() - tot_time = tot_time_end - tot_time_start - print("tot_time", tot_time) - - -if __name__ == "__main__": - #output_dir = make_id_prop() - output_dir="." - run_atomgpt(config_file=output_dir + "/config.json") - # config_file="config.json" - # ) diff --git a/atomgpt/inverse_models/__init__.py b/atomgpt/inverse_models/__init__.py index ff7129e..9c90797 100644 --- a/atomgpt/inverse_models/__init__.py +++ b/atomgpt/inverse_models/__init__.py @@ -1,17 +1,3 @@ -# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - from .loader import FastLanguageModel from .llama import FastLlamaModel from .mistral import FastMistralModel diff --git a/atomgpt/inverse_models/_utils.py b/atomgpt/inverse_models/_utils.py index a53de42..6a1d3cf 100644 --- a/atomgpt/inverse_models/_utils.py +++ b/atomgpt/inverse_models/_utils.py @@ -1,17 +1,3 @@ -# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - import torch from typing import Union, Optional, List, Any, Callable import warnings diff --git a/atomgpt/inverse_models/dpo.py b/atomgpt/inverse_models/dpo.py index b7c7305..b004f40 100644 --- a/atomgpt/inverse_models/dpo.py +++ b/atomgpt/inverse_models/dpo.py @@ -1,17 +1,3 @@ -# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - try: from transformers.utils.notebook import ( IntervalStrategy, diff --git a/atomgpt/inverse_models/gemma.py b/atomgpt/inverse_models/gemma.py index 5dd2a5a..eaed401 100644 --- a/atomgpt/inverse_models/gemma.py +++ b/atomgpt/inverse_models/gemma.py @@ -1,17 +1,3 @@ -# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - from .llama import * from ._utils import __version__ diff --git a/atomgpt/inverse_models/inf.py b/atomgpt/inverse_models/inference.py similarity index 99% rename from atomgpt/inverse_models/inf.py rename to atomgpt/inverse_models/inference.py index dd54a06..e8135cd 100644 --- a/atomgpt/inverse_models/inf.py +++ b/atomgpt/inverse_models/inference.py @@ -1,4 +1,4 @@ - +"""Module for inference.""" from jarvis.db.jsonutils import loadjson from unsloth import FastLanguageModel import torch diff --git a/atomgpt/inverse_models/inverse_models.py b/atomgpt/inverse_models/inverse_models.py index 4e8f2f1..f27f38f 100644 --- a/atomgpt/inverse_models/inverse_models.py +++ b/atomgpt/inverse_models/inverse_models.py @@ -28,7 +28,7 @@ help="Name of the config file", ) - +# Adapted from https://github.com/unslothai/unsloth class TrainingPropConfig(BaseSettings): """Training config defaults and validation.""" @@ -157,7 +157,7 @@ def formatting_prompts_func(examples): def text2atoms(response): tmp_atoms_array = response.strip("").split("\n") # tmp_atoms_array= [element for element in tmp_atoms_array if element != ''] - print("tmp_atoms_array", tmp_atoms_array) + # print("tmp_atoms_array", tmp_atoms_array) lat_lengths = np.array(tmp_atoms_array[1].split(), dtype="float") lat_angles = np.array(tmp_atoms_array[2].split(), dtype="float") diff --git a/atomgpt/inverse_models/kernels/__init__.py b/atomgpt/inverse_models/kernels/__init__.py index fb49219..94d5a8e 100644 --- a/atomgpt/inverse_models/kernels/__init__.py +++ b/atomgpt/inverse_models/kernels/__init__.py @@ -1,17 +1,3 @@ -# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - from atomgpt.inverse_models.kernels.cross_entropy_loss import fast_cross_entropy_loss from .rms_layernorm import fast_rms_layernorm from .rope_embedding import fast_rope_embedding, inplace_rope_embedding diff --git a/atomgpt/inverse_models/kernels/cross_entropy_loss.py b/atomgpt/inverse_models/kernels/cross_entropy_loss.py index 0acff4c..f6d5a26 100644 --- a/atomgpt/inverse_models/kernels/cross_entropy_loss.py +++ b/atomgpt/inverse_models/kernels/cross_entropy_loss.py @@ -1,17 +1,3 @@ -# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - import triton import triton.language as tl import torch diff --git a/atomgpt/inverse_models/kernels/fast_lora.py b/atomgpt/inverse_models/kernels/fast_lora.py index edce605..8f88434 100644 --- a/atomgpt/inverse_models/kernels/fast_lora.py +++ b/atomgpt/inverse_models/kernels/fast_lora.py @@ -1,17 +1,3 @@ -# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - import torch from atomgpt.inverse_models.kernels.utils import fast_dequantize, QUANT_STATE, get_lora_parameters, matmul_lora diff --git a/atomgpt/inverse_models/kernels/geglu.py b/atomgpt/inverse_models/kernels/geglu.py index 97a25fa..8074357 100644 --- a/atomgpt/inverse_models/kernels/geglu.py +++ b/atomgpt/inverse_models/kernels/geglu.py @@ -1,17 +1,3 @@ -# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - import triton import triton.language as tl import torch diff --git a/atomgpt/inverse_models/kernels/rms_layernorm.py b/atomgpt/inverse_models/kernels/rms_layernorm.py index 6d06dbc..000410b 100644 --- a/atomgpt/inverse_models/kernels/rms_layernorm.py +++ b/atomgpt/inverse_models/kernels/rms_layernorm.py @@ -1,17 +1,3 @@ -# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - import triton import triton.language as tl import torch diff --git a/atomgpt/inverse_models/kernels/rope_embedding.py b/atomgpt/inverse_models/kernels/rope_embedding.py index 87e0178..64c0cb9 100644 --- a/atomgpt/inverse_models/kernels/rope_embedding.py +++ b/atomgpt/inverse_models/kernels/rope_embedding.py @@ -1,17 +1,3 @@ -# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - import triton import triton.language as tl import torch diff --git a/atomgpt/inverse_models/kernels/swiglu.py b/atomgpt/inverse_models/kernels/swiglu.py index 6614e5d..2856651 100644 --- a/atomgpt/inverse_models/kernels/swiglu.py +++ b/atomgpt/inverse_models/kernels/swiglu.py @@ -1,17 +1,3 @@ -# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - import triton import triton.language as tl import torch diff --git a/atomgpt/inverse_models/kernels/utils.py b/atomgpt/inverse_models/kernels/utils.py index 1f2085d..9f56d20 100644 --- a/atomgpt/inverse_models/kernels/utils.py +++ b/atomgpt/inverse_models/kernels/utils.py @@ -1,17 +1,3 @@ -# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - import triton MAX_FUSED_SIZE = 65536 next_power_of_2 = triton.next_power_of_2 diff --git a/atomgpt/inverse_models/llama.py b/atomgpt/inverse_models/llama.py index 0b6b6d7..3d60ca6 100644 --- a/atomgpt/inverse_models/llama.py +++ b/atomgpt/inverse_models/llama.py @@ -1,17 +1,3 @@ -# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - import torch from typing import Optional, Tuple, List, Union from torch.nn.functional import scaled_dot_product_attention diff --git a/atomgpt/inverse_models/loader.py b/atomgpt/inverse_models/loader.py index 93ff812..33c5b2f 100644 --- a/atomgpt/inverse_models/loader.py +++ b/atomgpt/inverse_models/loader.py @@ -1,17 +1,3 @@ -# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - from atomgpt.inverse_models.llama import FastLlamaModel, logger from atomgpt.inverse_models.mistral import FastMistralModel from atomgpt.inverse_models.qwen2 import FastQwen2Model diff --git a/atomgpt/inverse_models/mapper.py b/atomgpt/inverse_models/mapper.py index b4fbe57..3e6c777 100644 --- a/atomgpt/inverse_models/mapper.py +++ b/atomgpt/inverse_models/mapper.py @@ -1,17 +1,3 @@ -# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - __all__ = [ "INT_TO_FLOAT_MAPPER", "FLOAT_TO_INT_MAPPER", diff --git a/atomgpt/inverse_models/mistral.py b/atomgpt/inverse_models/mistral.py index 762ecbc..cdb6175 100644 --- a/atomgpt/inverse_models/mistral.py +++ b/atomgpt/inverse_models/mistral.py @@ -1,17 +1,3 @@ -# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - from atomgpt.inverse_models.llama import * import os from atomgpt.inverse_models._utils import __version__ diff --git a/atomgpt/inverse_models/qwen2.py b/atomgpt/inverse_models/qwen2.py index 76fe31a..fb0b41a 100644 --- a/atomgpt/inverse_models/qwen2.py +++ b/atomgpt/inverse_models/qwen2.py @@ -1,17 +1,3 @@ -# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - from .llama import * from .mistral import FastMistralModel import os diff --git a/atomgpt/inverse_models/tokenizer_utils.py b/atomgpt/inverse_models/tokenizer_utils.py index 1cbe49b..7f29621 100644 --- a/atomgpt/inverse_models/tokenizer_utils.py +++ b/atomgpt/inverse_models/tokenizer_utils.py @@ -1,17 +1,3 @@ -# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - from transformers import AutoTokenizer from transformers.convert_slow_tokenizer import convert_slow_tokenizer from transformers import PreTrainedTokenizerFast diff --git a/atomgpt/scripts/bnbb.py b/atomgpt/scripts/bnbb.py deleted file mode 100644 index ce21e34..0000000 --- a/atomgpt/scripts/bnbb.py +++ /dev/null @@ -1,11 +0,0 @@ -import bitsandbytes -import bitsandbytes as bnb -get_ptr = bnb.functional.get_ptr -import ctypes -import torch -cdequantize_blockwise_fp32 = bnb.functional.lib.cdequantize_blockwise_fp32 -cdequantize_blockwise_fp16_nf4 = bnb.functional.lib.cdequantize_blockwise_fp16_nf4 -cdequantize_blockwise_bf16_nf4 = bnb.functional.lib.cdequantize_blockwise_bf16_nf4 -cgemm_4bit_inference_naive_fp16 = bnb.functional.lib.cgemm_4bit_inference_naive_fp16 -cgemm_4bit_inference_naive_bf16 = bnb.functional.lib.cgemm_4bit_inference_naive_bf16 - diff --git a/atomgpt/scripts/dpo.py b/atomgpt/scripts/dpo.py deleted file mode 100644 index b7c7305..0000000 --- a/atomgpt/scripts/dpo.py +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -try: - from transformers.utils.notebook import ( - IntervalStrategy, - NotebookTrainingTracker, - NotebookProgressCallback, - ) - HAS_NOTEBOOK = True -except: - HAS_NOTEBOOK = False -pass - -DPOTrainer_metrics = [ - "rewards/chosen", - "rewards/rejected", - "rewards/accuracies", - "rewards/margins", - "logps/rejected", - "logps/chosen", - "logits/rejected", - "logits/chosen", -] -set_DPOTrainer_metrics = frozenset(DPOTrainer_metrics) - - -def NotebookProgressCallback_on_train_begin(self, args, state, control, **kwargs): - self.first_column = "Epoch" if args.evaluation_strategy == IntervalStrategy.EPOCH else "Step" - self.training_loss = 0 - self.last_log = 0 - column_names = [self.first_column] + ["Training Loss"] - if args.evaluation_strategy != IntervalStrategy.NO: - column_names.append("Validation Loss") - column_names += [x.replace("/", " / ") for x in DPOTrainer_metrics] - self.training_tracker = NotebookTrainingTracker(state.max_steps, column_names) -pass - - -def NotebookProgressCallback_on_log(self, args, state, control, logs=None, **kwargs): - # Only for when there is no evaluation - if args.evaluation_strategy == IntervalStrategy.NO and "loss" in logs: - values = {"Training Loss": logs["loss"]} - for metric in DPOTrainer_metrics: - values[metric.replace("/", " / ")] = logs[metric] - pass - # First column is necessarily Step since we're not in epoch eval strategy - values["Step"] = state.global_step - self.training_tracker.write_line(values) - pass -pass - - -def NotebookTrainingTracker_write_line(self, values): - """ - Write the values in the inner table. - - Args: - values (`Dict[str, float]`): The values to display. - """ - if self.inner_table is None: - self.inner_table = [list(values.keys()), list(values.values())] - else: - columns = self.inner_table[0] - new_values = {} - for key, value in values.items(): - lowered = key.lower() - if lowered in set_DPOTrainer_metrics: - new_values[lowered.replace("/", " / ")] = value - else: - new_values[key] = value - pass - values = new_values - - self.inner_table[0] = columns - if len(self.inner_table) > 1: - last_values = self.inner_table[-1] - first_column = self.inner_table[0][0] - if last_values[0] != values[first_column]: - # write new line - self.inner_table.append([values[c] if c in values else "No Log" for c in columns]) - else: - # update last line - new_values = values - for c in columns: - if c not in new_values.keys(): - new_values[c] = last_values[columns.index(c)] - self.inner_table[-1] = [new_values[c] for c in columns] - else: - # Edit for evaluation purposes - self.inner_table.append([values[c] if c in values else 0 for c in columns]) - pass - pass -pass - - -def PatchDPOTrainer(): - if HAS_NOTEBOOK: - from transformers.trainer import is_in_notebook - if is_in_notebook(): - # Patch DPO notebook printing - NotebookTrainingTracker.write_line = NotebookTrainingTracker_write_line - from transformers.trainer import DEFAULT_PROGRESS_CALLBACK - DEFAULT_PROGRESS_CALLBACK.on_train_begin = NotebookProgressCallback_on_train_begin - DEFAULT_PROGRESS_CALLBACK.on_log = NotebookProgressCallback_on_log - pass - pass -pass - diff --git a/atomgpt/scripts/finetune.py b/atomgpt/scripts/finetune.py deleted file mode 100644 index ae997ec..0000000 --- a/atomgpt/scripts/finetune.py +++ /dev/null @@ -1,359 +0,0 @@ -"""Module for fin tuning LLM model for materials chemsitry.""" -from jarvis.db.figshare import data -import transformers -import torch -from jarvis.db.jsonutils import dumpjson -import sys -import argparse -from torch.utils.data import DataLoader, Dataset -import numpy as np -import os -from jarvis.core.atoms import Atoms - -# from tqdm import tqdm -import time -import json -import zipfile - -os.environ["WANDB_ANONYMOUS"] = "must" -np.random.seed(42) -torch.manual_seed(42) -torch.cuda.manual_seed_all(42) -torch.backends.cudnn.deterministic = True -torch.backends.cudnn.benchmark = False -os.environ["PYTHONHASHSEED"] = str(42) -os.environ["CUBLAS_WORKSPACE_CONFIG"] = str(":4096:8") -torch.use_deterministic_algorithms(True) - -IGNORE_INDEX = -100 -device = "cpu" -if torch.cuda.is_available(): - device = torch.device("cuda") -parser = argparse.ArgumentParser(description="AtomGPT") -parser.add_argument( - "--benchmark_file", - default="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae", - help="Benchmarks available in jarvis_leaderboard/benchmarks/*/*.zip", -) - - -def get_crystal_string(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.1f}".format(x) for x in lengths]) - + "\n" - + " ".join([str(int(x)) for x in angles]) - + "\n" - + "\n".join( - [ - str(t) + "\n" + " ".join(["{0:.2f}".format(x) for x in c]) - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - - return crystal_str - - -# Define a custom dataset class for regression -class AtomGPTDataset(Dataset): - def __init__( - self, texts=[], targets=[], ids=[], tokenizer="", max_length=128 - ): - self.texts = texts - self.targets = targets - self.tokenizer = tokenizer - self.max_length = max_length - if not ids: - ids = ["text-" + str(i) for i in range(len(texts))] - self.ids = ids - - def __len__(self): - return len(self.texts) - - def __getitem__(self, idx): - inputs = self.tokenizer( - self.texts[idx], - return_tensors="pt", - max_length=self.max_length, - padding="max_length", - truncation=True, - ) - return ( - inputs, - self.ids[idx], - torch.tensor(self.targets[idx], dtype=torch.float32), - ) - - -# Example usage - - -def run_atomgpt( - prefix="sample", - model_name="gpt2", - benchmark_file="AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip", - root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard", - batch_size=16, - max_length=128, - num_epochs=500, - learning_rate=5e-5, -): - # Load pre-trained tokenizer - # tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - # model = GPT2LMHeadModel.from_pretrained("gpt2") - - dft_3d = data("dft_3d") - # root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard" - # benchmark_file="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae.csv.zip" - # benchmark_file = "AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip" - method = benchmark_file.split("-")[0] - task = benchmark_file.split("-")[1] - prop = benchmark_file.split("-")[2] - dataset = benchmark_file.split("-")[3] - temp = dataset + "_" + prop + ".json.zip" - temp2 = dataset + "_" + prop + ".json" - fname = os.path.join(root_dir, "benchmarks", method, task, temp) - zp = zipfile.ZipFile(fname) - bench = json.loads(zp.read(temp2)) - - # train_atoms = [] - # val_atoms = [] - # test_atoms = [] - # train_targets = [] - # val_targets = [] - # test_targets = [] - train_ids = list(bench["train"].keys()) - val_ids = list(bench["val"].keys()) - test_ids = list(bench["test"].keys()) - - # model_name = "mistralai/Mistral-7B-Instruct-v0.1" - # model_name = "gpt2" - model = transformers.AutoModelForCausalLM.from_pretrained(model_name) - - tokenizer = transformers.AutoTokenizer.from_pretrained(model_name) - - # batch_size = 16 - # max_length = 128 - # num_epochs = 100 - # learning_rate = 5e-5 - criterion = torch.nn.L1Loss() - # Define example regression data (texts and corresponding numeric targets) - train_texts = [] - train_targets = [] - train_ids_temp = [] - val_texts = [] - val_targets = [] - val_ids_temp = [] - test_texts = [] - test_targets = [] - test_ids_temp = [] - - for i in dft_3d: - if i[prop] != "na": - atoms = Atoms.from_dict(i["atoms"]) - tmp = get_crystal_string(atoms) - if i["jid"] in train_ids: - train_texts.append(tmp) - train_targets.append(i[prop]) - train_ids_temp.append(i["jid"]) - elif i["jid"] in val_ids: - val_texts.append(tmp) - val_targets.append(i[prop]) - val_ids_temp.append(i["jid"]) - elif i["jid"] in test_ids: - test_texts.append(tmp) - test_targets.append(i[prop]) - test_ids_temp.append(i["jid"]) - """ - ############################## - ###Fast test### - train_texts = [ - "This is the first example text.", - "Second example is a bit longer than the first one, but still within the max length.", - "Third example is the longest among these three examples. It exceeds the max length and will be truncated.", - "Second example is a bit longer than the first one, but still within the max length.", - ] - train_targets = [10.2, 15.5, 20.1, 15.5] # Example regression targets - val_texts = test_texts = train_texts - val_targets = test_targets = train_targets - train_ids_temp=['a','b','c','d'] - val_ids_temp = test_ids_temp = train_ids_temp - batch_size = 2 - num_epochs = 3 - - ############################## - ############################## - """ - - # Fine-tune the last layer of GPT-2 for regression - # fine_tune_gpt2_regression(train_texts, train_targets, tokenizer) - if tokenizer.pad_token is None: - tokenizer.add_special_tokens({"pad_token": "[PAD]"}) - model.resize_token_embeddings(len(tokenizer)) - model.lm_head = torch.nn.Linear( - model.config.hidden_size, 1 - ) # Single output for regression - model.to(device) - optimizer = transformers.AdamW(model.parameters(), lr=learning_rate) - # Prepare datasets and dataloaders with data collator - train_dataset = AtomGPTDataset( - texts=train_texts, - targets=train_targets, - ids=train_ids_temp, - tokenizer=tokenizer, - max_length=max_length, - ) - val_dataset = AtomGPTDataset( - texts=val_texts, - targets=val_targets, - tokenizer=tokenizer, - ids=val_ids_temp, - max_length=max_length, - ) - test_dataset = AtomGPTDataset( - texts=test_texts, - targets=test_targets, - tokenizer=tokenizer, - ids=test_ids_temp, - max_length=max_length, - ) - - # val_dataset = train_dataset - train_dataloader = DataLoader(train_dataset, batch_size=batch_size) - val_dataloader = DataLoader(val_dataset, batch_size=batch_size) - test_dataloader = DataLoader(test_dataset, batch_size=batch_size) - test_dataloader = val_dataloader - steps_per_epoch = len(train_dataloader) - scheduler = torch.optim.lr_scheduler.OneCycleLR( - optimizer, - max_lr=learning_rate, - epochs=num_epochs, - steps_per_epoch=steps_per_epoch, - # pct_start=pct_start, - pct_start=0.3, - ) - print("benchmark_file", benchmark_file) - print("train_data", len(train_texts)) - print("test_data", len(test_texts)) - output_dir = prefix + "_out_" + model_name + "_" + dataset + "_" + prop - if not os.path.exists(output_dir): - os.makedirs(output_dir) - best_loss = np.inf - tot_time_start = time.time() - train_history = [] - val_history = [] - for epoch in range(num_epochs): - model.train() - t1 = time.time() - for batch in train_dataloader: - train_loss = 0 - # train_result = [] - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - predictions = ( - model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - ) - targets = batch[2].squeeze() - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - # print('train',predictions,targets) - loss.backward() - optimizer.step() - scheduler.step() - optimizer.zero_grad() - train_loss += loss.item() - train_loss = train_loss / len(train_dataloader) - t2 = time.time() - train_time = round(t2 - t1, 3) - model.eval() - - # total_eval_mae_loss = 0 - # predictions_list = [] - # targets_list = [] - val_loss = 0 - t1 = time.time() - for batch in val_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze(0) # .squeeze(0) - predictions = ( - model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - ) - targets = batch[2].squeeze() - # print('val',predictions,targets) - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - val_loss += loss.item() - saving_tag = "" - if val_loss < best_loss: - best_loss = val_loss - best_model_name = "best_model.pt" - torch.save( - model.state_dict(), - os.path.join(output_dir, best_model_name), - ) - # print("Saving model for epoch", epoch) - saving_tag = " saving model:" + str(epoch) - val_loss = val_loss / len(val_dataloader) - t2 = time.time() - val_time = round(t2 - t1, 3) - print( - "Epoch, train loss, val loss, train_time, val_time", - epoch, - train_loss, - val_loss, - train_time, - val_time, - saving_tag, - ) - train_history.append(train_loss) - val_history.append(val_loss) - history = os.path.join(output_dir, "history.json") - - dumpjson( - data={"train": train_history, "val": val_history}, filename=history - ) - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze(0) # .squeeze(0) - predictions = ( - model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - f.close() - tot_time_end = time.time() - tot_time = tot_time_end - tot_time_start - print("tot_time", tot_time) - - -if __name__ == "__main__": - args = parser.parse_args(sys.argv[1:]) - benchmark_file = args.benchmark_file - run_atomgpt(benchmark_file=benchmark_file) diff --git a/atomgpt/scripts/finetune.py.bak b/atomgpt/scripts/finetune.py.bak deleted file mode 100644 index ae997ec..0000000 --- a/atomgpt/scripts/finetune.py.bak +++ /dev/null @@ -1,359 +0,0 @@ -"""Module for fin tuning LLM model for materials chemsitry.""" -from jarvis.db.figshare import data -import transformers -import torch -from jarvis.db.jsonutils import dumpjson -import sys -import argparse -from torch.utils.data import DataLoader, Dataset -import numpy as np -import os -from jarvis.core.atoms import Atoms - -# from tqdm import tqdm -import time -import json -import zipfile - -os.environ["WANDB_ANONYMOUS"] = "must" -np.random.seed(42) -torch.manual_seed(42) -torch.cuda.manual_seed_all(42) -torch.backends.cudnn.deterministic = True -torch.backends.cudnn.benchmark = False -os.environ["PYTHONHASHSEED"] = str(42) -os.environ["CUBLAS_WORKSPACE_CONFIG"] = str(":4096:8") -torch.use_deterministic_algorithms(True) - -IGNORE_INDEX = -100 -device = "cpu" -if torch.cuda.is_available(): - device = torch.device("cuda") -parser = argparse.ArgumentParser(description="AtomGPT") -parser.add_argument( - "--benchmark_file", - default="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae", - help="Benchmarks available in jarvis_leaderboard/benchmarks/*/*.zip", -) - - -def get_crystal_string(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.1f}".format(x) for x in lengths]) - + "\n" - + " ".join([str(int(x)) for x in angles]) - + "\n" - + "\n".join( - [ - str(t) + "\n" + " ".join(["{0:.2f}".format(x) for x in c]) - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - - return crystal_str - - -# Define a custom dataset class for regression -class AtomGPTDataset(Dataset): - def __init__( - self, texts=[], targets=[], ids=[], tokenizer="", max_length=128 - ): - self.texts = texts - self.targets = targets - self.tokenizer = tokenizer - self.max_length = max_length - if not ids: - ids = ["text-" + str(i) for i in range(len(texts))] - self.ids = ids - - def __len__(self): - return len(self.texts) - - def __getitem__(self, idx): - inputs = self.tokenizer( - self.texts[idx], - return_tensors="pt", - max_length=self.max_length, - padding="max_length", - truncation=True, - ) - return ( - inputs, - self.ids[idx], - torch.tensor(self.targets[idx], dtype=torch.float32), - ) - - -# Example usage - - -def run_atomgpt( - prefix="sample", - model_name="gpt2", - benchmark_file="AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip", - root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard", - batch_size=16, - max_length=128, - num_epochs=500, - learning_rate=5e-5, -): - # Load pre-trained tokenizer - # tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - # model = GPT2LMHeadModel.from_pretrained("gpt2") - - dft_3d = data("dft_3d") - # root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard" - # benchmark_file="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae.csv.zip" - # benchmark_file = "AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip" - method = benchmark_file.split("-")[0] - task = benchmark_file.split("-")[1] - prop = benchmark_file.split("-")[2] - dataset = benchmark_file.split("-")[3] - temp = dataset + "_" + prop + ".json.zip" - temp2 = dataset + "_" + prop + ".json" - fname = os.path.join(root_dir, "benchmarks", method, task, temp) - zp = zipfile.ZipFile(fname) - bench = json.loads(zp.read(temp2)) - - # train_atoms = [] - # val_atoms = [] - # test_atoms = [] - # train_targets = [] - # val_targets = [] - # test_targets = [] - train_ids = list(bench["train"].keys()) - val_ids = list(bench["val"].keys()) - test_ids = list(bench["test"].keys()) - - # model_name = "mistralai/Mistral-7B-Instruct-v0.1" - # model_name = "gpt2" - model = transformers.AutoModelForCausalLM.from_pretrained(model_name) - - tokenizer = transformers.AutoTokenizer.from_pretrained(model_name) - - # batch_size = 16 - # max_length = 128 - # num_epochs = 100 - # learning_rate = 5e-5 - criterion = torch.nn.L1Loss() - # Define example regression data (texts and corresponding numeric targets) - train_texts = [] - train_targets = [] - train_ids_temp = [] - val_texts = [] - val_targets = [] - val_ids_temp = [] - test_texts = [] - test_targets = [] - test_ids_temp = [] - - for i in dft_3d: - if i[prop] != "na": - atoms = Atoms.from_dict(i["atoms"]) - tmp = get_crystal_string(atoms) - if i["jid"] in train_ids: - train_texts.append(tmp) - train_targets.append(i[prop]) - train_ids_temp.append(i["jid"]) - elif i["jid"] in val_ids: - val_texts.append(tmp) - val_targets.append(i[prop]) - val_ids_temp.append(i["jid"]) - elif i["jid"] in test_ids: - test_texts.append(tmp) - test_targets.append(i[prop]) - test_ids_temp.append(i["jid"]) - """ - ############################## - ###Fast test### - train_texts = [ - "This is the first example text.", - "Second example is a bit longer than the first one, but still within the max length.", - "Third example is the longest among these three examples. It exceeds the max length and will be truncated.", - "Second example is a bit longer than the first one, but still within the max length.", - ] - train_targets = [10.2, 15.5, 20.1, 15.5] # Example regression targets - val_texts = test_texts = train_texts - val_targets = test_targets = train_targets - train_ids_temp=['a','b','c','d'] - val_ids_temp = test_ids_temp = train_ids_temp - batch_size = 2 - num_epochs = 3 - - ############################## - ############################## - """ - - # Fine-tune the last layer of GPT-2 for regression - # fine_tune_gpt2_regression(train_texts, train_targets, tokenizer) - if tokenizer.pad_token is None: - tokenizer.add_special_tokens({"pad_token": "[PAD]"}) - model.resize_token_embeddings(len(tokenizer)) - model.lm_head = torch.nn.Linear( - model.config.hidden_size, 1 - ) # Single output for regression - model.to(device) - optimizer = transformers.AdamW(model.parameters(), lr=learning_rate) - # Prepare datasets and dataloaders with data collator - train_dataset = AtomGPTDataset( - texts=train_texts, - targets=train_targets, - ids=train_ids_temp, - tokenizer=tokenizer, - max_length=max_length, - ) - val_dataset = AtomGPTDataset( - texts=val_texts, - targets=val_targets, - tokenizer=tokenizer, - ids=val_ids_temp, - max_length=max_length, - ) - test_dataset = AtomGPTDataset( - texts=test_texts, - targets=test_targets, - tokenizer=tokenizer, - ids=test_ids_temp, - max_length=max_length, - ) - - # val_dataset = train_dataset - train_dataloader = DataLoader(train_dataset, batch_size=batch_size) - val_dataloader = DataLoader(val_dataset, batch_size=batch_size) - test_dataloader = DataLoader(test_dataset, batch_size=batch_size) - test_dataloader = val_dataloader - steps_per_epoch = len(train_dataloader) - scheduler = torch.optim.lr_scheduler.OneCycleLR( - optimizer, - max_lr=learning_rate, - epochs=num_epochs, - steps_per_epoch=steps_per_epoch, - # pct_start=pct_start, - pct_start=0.3, - ) - print("benchmark_file", benchmark_file) - print("train_data", len(train_texts)) - print("test_data", len(test_texts)) - output_dir = prefix + "_out_" + model_name + "_" + dataset + "_" + prop - if not os.path.exists(output_dir): - os.makedirs(output_dir) - best_loss = np.inf - tot_time_start = time.time() - train_history = [] - val_history = [] - for epoch in range(num_epochs): - model.train() - t1 = time.time() - for batch in train_dataloader: - train_loss = 0 - # train_result = [] - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - predictions = ( - model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - ) - targets = batch[2].squeeze() - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - # print('train',predictions,targets) - loss.backward() - optimizer.step() - scheduler.step() - optimizer.zero_grad() - train_loss += loss.item() - train_loss = train_loss / len(train_dataloader) - t2 = time.time() - train_time = round(t2 - t1, 3) - model.eval() - - # total_eval_mae_loss = 0 - # predictions_list = [] - # targets_list = [] - val_loss = 0 - t1 = time.time() - for batch in val_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze(0) # .squeeze(0) - predictions = ( - model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - ) - targets = batch[2].squeeze() - # print('val',predictions,targets) - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - val_loss += loss.item() - saving_tag = "" - if val_loss < best_loss: - best_loss = val_loss - best_model_name = "best_model.pt" - torch.save( - model.state_dict(), - os.path.join(output_dir, best_model_name), - ) - # print("Saving model for epoch", epoch) - saving_tag = " saving model:" + str(epoch) - val_loss = val_loss / len(val_dataloader) - t2 = time.time() - val_time = round(t2 - t1, 3) - print( - "Epoch, train loss, val loss, train_time, val_time", - epoch, - train_loss, - val_loss, - train_time, - val_time, - saving_tag, - ) - train_history.append(train_loss) - val_history.append(val_loss) - history = os.path.join(output_dir, "history.json") - - dumpjson( - data={"train": train_history, "val": val_history}, filename=history - ) - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze(0) # .squeeze(0) - predictions = ( - model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - f.close() - tot_time_end = time.time() - tot_time = tot_time_end - tot_time_start - print("tot_time", tot_time) - - -if __name__ == "__main__": - args = parser.parse_args(sys.argv[1:]) - benchmark_file = args.benchmark_file - run_atomgpt(benchmark_file=benchmark_file) diff --git a/atomgpt/scripts/finetune1.py b/atomgpt/scripts/finetune1.py deleted file mode 100644 index 85eb1a6..0000000 --- a/atomgpt/scripts/finetune1.py +++ /dev/null @@ -1,548 +0,0 @@ -"""Module for fin tuning LLM model for materials chemsitry.""" - -from jarvis.db.figshare import data -import transformers -import torch -from jarvis.db.jsonutils import dumpjson -import sys -import argparse -from torch.utils.data import DataLoader, Dataset -import numpy as np -import os -from jarvis.core.atoms import Atoms -import pandas as pd -from sklearn.metrics import mean_absolute_error - -# from tqdm import tqdm -import time -import json -import zipfile - -os.environ["WANDB_ANONYMOUS"] = "must" -np.random.seed(42) -torch.manual_seed(42) -torch.cuda.manual_seed_all(42) -torch.backends.cudnn.deterministic = True -torch.backends.cudnn.benchmark = False -os.environ["PYTHONHASHSEED"] = str(42) -os.environ["CUBLAS_WORKSPACE_CONFIG"] = str(":4096:8") -torch.use_deterministic_algorithms(True) -#torch.set_default_dtype(torch.float16) -IGNORE_INDEX = -100 -torch.cuda.empty_cache() - - -device = "cpu" -if torch.cuda.is_available(): - device = torch.device("cuda") -parser = argparse.ArgumentParser(description="AtomGPT") -parser.add_argument( - "--benchmark_file", - default="AI-SinglePropertyPrediction-ead-tinnet_N-test-mae.csv.zip", - # default="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae", - help="Benchmarks available in jarvis_leaderboard/benchmarks/*/*.zip", -) - - -def get_crystal_string_old(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.1f}".format(x) for x in lengths]) - + "\n" - + " ".join([str(int(x)) for x in angles]) - + "\n" - + "\n".join( - [ - str(t) + "\n" + " ".join(["{0:.2f}".format(x) for x in c]) - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - - return crystal_str - - -def get_crystal_string(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.1f}".format(x) for x in lengths]) - + "\n" - + " ".join([str(int(x)) for x in angles]) - + "\n" - + "\n".join( - [ - str(t) + " " + " ".join(["{0:.1f}".format(x) for x in c]) - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - return crystal_str - - -# Define a custom dataset class for regression -class AtomGPTDataset(Dataset): - def __init__( - self, texts=[], targets=[], ids=[], tokenizer="", max_length=128 - ): - self.texts = texts - self.targets = targets - self.tokenizer = tokenizer - self.max_length = max_length - if not ids: - ids = ["text-" + str(i) for i in range(len(texts))] - self.ids = ids - - def __len__(self): - return len(self.texts) - - def __getitem__(self, idx): - inputs = self.tokenizer( - self.texts[idx], - return_tensors="pt", - max_length=self.max_length, - padding="max_length", - truncation=True, - ) - return ( - inputs, - self.ids[idx], - torch.tensor(self.targets[idx], dtype=torch.float32), - ) - - -# Example usage - - -def run_atomgpt( - prefix="ss", - model_name="gpt2", - benchmark_file="AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip", - root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard", - batch_size=8, - max_length=512, - num_epochs=500, - latent_dim=512, - learning_rate=1e-3, - # learning_rate=1e-3, - test_each_run=True, - # learning_rate=5e-5, -): - # Load pre-trained tokenizer - # tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - # model = GPT2LMHeadModel.from_pretrained("gpt2") - - # dft_3d = data("dft_3d") - # root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard" - # benchmark_file="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae.csv.zip" - # benchmark_file = "AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip" - print("benchmark_file", benchmark_file) - method = benchmark_file.split("-")[0] - task = benchmark_file.split("-")[1] - prop = benchmark_file.split("-")[2] - dataset = benchmark_file.split("-")[3] - temp = dataset + "_" + prop + ".json.zip" - temp2 = dataset + "_" + prop + ".json" - fname = os.path.join(root_dir, "benchmarks", method, task, temp) - zp = zipfile.ZipFile(fname) - bench = json.loads(zp.read(temp2)) - dft_3d = data(dataset) - id_tag = "jid" - if "jid" in dft_3d[0]: - id_tag = "jid" - else: - id_tag = "id" - - # train_atoms = [] - # val_atoms = [] - # test_atoms = [] - # train_targets = [] - # val_targets = [] - # test_targets = [] - train_ids = list(bench["train"].keys()) - val_ids = list(bench["val"].keys()) - test_ids = list(bench["test"].keys()) - - # model_name = "mistralai/Mistral-7B-Instruct-v0.1" - # model_name = "gpt2" - if "t5" in model_name: - model = transformers.T5ForConditionalGeneration.from_pretrained( - model_name - ) - else: - model = transformers.AutoModelForCausalLM.from_pretrained( - model_name, - low_cpu_mem_usage=True, - #load_in_8bit=False, - #torch_dtype=torch.float16, - #load_in_8bit=True, - # device_map="auto" - ) - # device = model.device - if "t5" in model_name: - tokenizer = transformers.T5Tokenizer.from_pretrained(model_name) - - else: - tokenizer = transformers.AutoTokenizer.from_pretrained(model_name) - - # batch_size = 16 - # max_length = 128 - # num_epochs = 100 - # learning_rate = 5e-5 - criterion = torch.nn.L1Loss() - # Define example regression data (texts and corresponding numeric targets) - train_texts = [] - train_targets = [] - train_ids_temp = [] - val_texts = [] - val_targets = [] - val_ids_temp = [] - test_texts = [] - test_targets = [] - test_ids_temp = [] - - for i in dft_3d: - if i[prop] != "na": - atoms = Atoms.from_dict(i["atoms"]) - tmp = get_crystal_string(atoms) - if i[id_tag] in train_ids: - train_texts.append(tmp) - train_targets.append(i[prop]) - train_ids_temp.append(i[id_tag]) - elif i[id_tag] in val_ids: - val_texts.append(tmp) - val_targets.append(i[prop]) - val_ids_temp.append(i[id_tag]) - elif i[id_tag] in test_ids: - test_texts.append(tmp) - test_targets.append(i[prop]) - test_ids_temp.append(i[id_tag]) - """ - ############################## - ###Fast test### - train_texts = [ - "This is the first example text.", - "Second example is a bit longer than the first one, but still within the max length.", - "Third example is the longest among these three examples. It exceeds the max length and will be truncated.", - "Second example is a bit longer than the first one, but still within the max length.", - ] - train_targets = [10.2, 15.5, 20.1, 15.5] # Example regression targets - val_texts = test_texts = train_texts - val_targets = test_targets = train_targets - train_ids_temp=['a','b','c','d'] - val_ids_temp = test_ids_temp = train_ids_temp - batch_size = 2 - num_epochs = 3 - - ############################## - ############################## - """ - - # Fine-tune the last layer of GPT-2 for regression - # fine_tune_gpt2_regression(train_texts, train_targets, tokenizer) - if tokenizer.pad_token is None: - tokenizer.add_special_tokens({"pad_token": "[PAD]"}) - model.resize_token_embeddings(len(tokenizer)) - model.lm_head = torch.nn.Sequential( - torch.nn.Linear(model.config.hidden_size, latent_dim), - torch.nn.Linear(latent_dim, latent_dim), - #torch.nn.Linear(latent_dim, latent_dim), - torch.nn.Linear(latent_dim, 1), - ) - # model.lm_head = torch.nn.Sequential(torch.nn.Linear( model.config.hidden_size, 256),torch.nn.SiLU(),torch.nn.Linear( 256, 1) ) - model.to(device) - if torch.cuda.device_count() > 1: - device_ids = [d for d in range(torch.cuda.device_count())] - model = torch.nn.DataParallel(model, device_ids=device_ids).cuda() - optimizer = transformers.AdamW(model.parameters(), lr=learning_rate) - # Prepare datasets and dataloaders with data collator - train_dataset = AtomGPTDataset( - texts=train_texts, - targets=train_targets, - ids=train_ids_temp, - tokenizer=tokenizer, - max_length=max_length, - ) - val_dataset = AtomGPTDataset( - texts=val_texts, - targets=val_targets, - tokenizer=tokenizer, - ids=val_ids_temp, - max_length=max_length, - ) - test_dataset = AtomGPTDataset( - texts=test_texts, - targets=test_targets, - tokenizer=tokenizer, - ids=test_ids_temp, - max_length=max_length, - ) - - # val_dataset = train_dataset - train_dataloader = DataLoader(train_dataset, batch_size=batch_size) - val_dataloader = DataLoader(val_dataset, batch_size=batch_size) - test_dataloader = DataLoader(test_dataset, batch_size=batch_size) - val_dataloader = test_dataloader - steps_per_epoch = len(train_dataloader) - scheduler = torch.optim.lr_scheduler.OneCycleLR( - optimizer, - max_lr=learning_rate, - epochs=num_epochs, - steps_per_epoch=steps_per_epoch, - # pct_start=pct_start, - pct_start=0.3, - ) - print("train_data", len(train_texts)) - print("test_data", len(test_texts)) - output_dir = prefix + "_out_" + model_name + "_" + dataset + "_" + prop - if not os.path.exists(output_dir): - os.makedirs(output_dir) - best_loss = np.inf - tot_time_start = time.time() - train_history = [] - val_history = [] - for epoch in range(num_epochs): - model.train() - t1 = time.time() - for batch in train_dataloader: - optimizer.zero_grad() - train_loss = 0 - # train_result = [] - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - # if 't5' in model_name: - # decoder_input_ids = tokenizer("", return_tensors="pt").input_ids.to(device) - # decoder_input_ids = model._shift_right(decoder_input_ids) - # predictions = ( - # model(input_ids = input_ids.to(device),decoder_input_ids=decoder_input_ids).logits.squeeze().mean(dim=-1) - # ) - # else: - # predictions = ( - # model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - # ) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - targets = batch[2].squeeze() - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - #print('train',predictions,targets) - loss.backward() - optimizer.step() - scheduler.step() - # optimizer.zero_grad() - train_loss += loss.item() - train_loss = train_loss / len(train_dataloader) - t2 = time.time() - train_time = round(t2 - t1, 3) - model.eval() - - # total_eval_mae_loss = 0 - # predictions_list = [] - # targets_list = [] - val_loss = 0 - t1 = time.time() - for batch in val_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - targets = batch[2].squeeze() - # print('val',predictions,targets) - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - val_loss += loss.item() - saving_tag = "" - if val_loss < best_loss: - best_loss = val_loss - best_model_name = "best_model.pt" - torch.save( - model.state_dict(), - os.path.join(output_dir, best_model_name), - ) - # print("Saving model for epoch", epoch) - saving_tag = " saving model:" + str(epoch) - val_loss = val_loss / len(val_dataloader) - t2 = time.time() - val_time = round(t2 - t1, 3) - train_history.append(train_loss) - val_history.append(val_loss) - history = os.path.join(output_dir, "history.json") - - dumpjson( - data={"train": train_history, "val": val_history}, filename=history - ) - mae = "" - if test_each_run: - t1_test = time.time() - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - t2_test = time.time() - test_time = round(t2_test - t1_test, 3) - f.close() - df = pd.read_csv(fname) - mae = mean_absolute_error(df["target"], df["predictions"]) - if mae == "": - print( - "Epoch, train loss, val loss, train_time, val_time", - epoch, - train_loss, - val_loss, - train_time, - val_time, - saving_tag, - ) - else: - print( - "Epoch, train loss, val loss, test loss, train_time, val_time, test_time", - epoch, - train_loss, - val_loss, - mae, - train_time, - val_time, - test_time, - saving_tag, - ) - - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - f.close() - tot_time_end = time.time() - tot_time = tot_time_end - tot_time_start - print("tot_time", tot_time) - - -if __name__ == "__main__": - args = parser.parse_args(sys.argv[1:]) - benchmark_file = args.benchmark_file - model_name="meta-llama/Llama-2-7b-hf", - model_name="google/flan-t5-small" - model_name="google/flan-t5-base" - model_name = "facebook/opt-350m" - model_name = "mistralai/Mixtral-8x7B-v0.1" - model_name = "mistralai/Mistral-7B-Instruct-v0.1" - model_name="gpt2" - model_name="gpt2-medium" - run_atomgpt( - model_name=model_name, - benchmark_file=benchmark_file, - num_epochs=500, - batch_size=2 - ) diff --git a/atomgpt/scripts/finetune1.py.bak_0.12843 b/atomgpt/scripts/finetune1.py.bak_0.12843 deleted file mode 100644 index 968ba64..0000000 --- a/atomgpt/scripts/finetune1.py.bak_0.12843 +++ /dev/null @@ -1,547 +0,0 @@ -"""Module for fin tuning LLM model for materials chemsitry.""" - -from jarvis.db.figshare import data -import transformers -import torch -from jarvis.db.jsonutils import dumpjson -import sys -import argparse -from torch.utils.data import DataLoader, Dataset -import numpy as np -import os -from jarvis.core.atoms import Atoms -import pandas as pd -from sklearn.metrics import mean_absolute_error - -# from tqdm import tqdm -import time -import json -import zipfile - -os.environ["WANDB_ANONYMOUS"] = "must" -np.random.seed(42) -torch.manual_seed(42) -torch.cuda.manual_seed_all(42) -torch.backends.cudnn.deterministic = True -torch.backends.cudnn.benchmark = False -os.environ["PYTHONHASHSEED"] = str(42) -os.environ["CUBLAS_WORKSPACE_CONFIG"] = str(":4096:8") -torch.use_deterministic_algorithms(True) -#torch.set_default_dtype(torch.float16) -IGNORE_INDEX = -100 -torch.cuda.empty_cache() - - -device = "cpu" -if torch.cuda.is_available(): - device = torch.device("cuda") -parser = argparse.ArgumentParser(description="AtomGPT") -parser.add_argument( - "--benchmark_file", - default="AI-SinglePropertyPrediction-ead-tinnet_N-test-mae.csv.zip", - # default="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae", - help="Benchmarks available in jarvis_leaderboard/benchmarks/*/*.zip", -) - - -def get_crystal_string_old(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.1f}".format(x) for x in lengths]) - + "\n" - + " ".join([str(int(x)) for x in angles]) - + "\n" - + "\n".join( - [ - str(t) + "\n" + " ".join(["{0:.2f}".format(x) for x in c]) - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - - return crystal_str - - -def get_crystal_string(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.1f}".format(x) for x in lengths]) - + "\n" - + " ".join([str(int(x)) for x in angles]) - + "\n" - + "\n".join( - [ - str(t) + " " + " ".join(["{0:.1f}".format(x) for x in c]) - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - return crystal_str - - -# Define a custom dataset class for regression -class AtomGPTDataset(Dataset): - def __init__( - self, texts=[], targets=[], ids=[], tokenizer="", max_length=128 - ): - self.texts = texts - self.targets = targets - self.tokenizer = tokenizer - self.max_length = max_length - if not ids: - ids = ["text-" + str(i) for i in range(len(texts))] - self.ids = ids - - def __len__(self): - return len(self.texts) - - def __getitem__(self, idx): - inputs = self.tokenizer( - self.texts[idx], - return_tensors="pt", - max_length=self.max_length, - padding="max_length", - truncation=True, - ) - return ( - inputs, - self.ids[idx], - torch.tensor(self.targets[idx], dtype=torch.float32), - ) - - -# Example usage - - -def run_atomgpt( - prefix="ss", - model_name="gpt2", - benchmark_file="AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip", - root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard", - batch_size=8, - max_length=512, - num_epochs=500, - latent_dim=512, - learning_rate=1e-3, - # learning_rate=1e-3, - test_each_run=True, - # learning_rate=5e-5, -): - # Load pre-trained tokenizer - # tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - # model = GPT2LMHeadModel.from_pretrained("gpt2") - - # dft_3d = data("dft_3d") - # root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard" - # benchmark_file="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae.csv.zip" - # benchmark_file = "AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip" - print("benchmark_file", benchmark_file) - method = benchmark_file.split("-")[0] - task = benchmark_file.split("-")[1] - prop = benchmark_file.split("-")[2] - dataset = benchmark_file.split("-")[3] - temp = dataset + "_" + prop + ".json.zip" - temp2 = dataset + "_" + prop + ".json" - fname = os.path.join(root_dir, "benchmarks", method, task, temp) - zp = zipfile.ZipFile(fname) - bench = json.loads(zp.read(temp2)) - dft_3d = data(dataset) - id_tag = "jid" - if "jid" in dft_3d[0]: - id_tag = "jid" - else: - id_tag = "id" - - # train_atoms = [] - # val_atoms = [] - # test_atoms = [] - # train_targets = [] - # val_targets = [] - # test_targets = [] - train_ids = list(bench["train"].keys()) - val_ids = list(bench["val"].keys()) - test_ids = list(bench["test"].keys()) - - # model_name = "mistralai/Mistral-7B-Instruct-v0.1" - # model_name = "gpt2" - if "t5" in model_name: - model = transformers.T5ForConditionalGeneration.from_pretrained( - model_name - ) - else: - model = transformers.AutoModelForCausalLM.from_pretrained( - model_name, - low_cpu_mem_usage=True, - #load_in_8bit=False, - #torch_dtype=torch.float16, - #load_in_8bit=True, - # device_map="auto" - ) - # device = model.device - if "t5" in model_name: - tokenizer = transformers.T5Tokenizer.from_pretrained(model_name) - - else: - tokenizer = transformers.AutoTokenizer.from_pretrained(model_name) - - # batch_size = 16 - # max_length = 128 - # num_epochs = 100 - # learning_rate = 5e-5 - criterion = torch.nn.L1Loss() - # Define example regression data (texts and corresponding numeric targets) - train_texts = [] - train_targets = [] - train_ids_temp = [] - val_texts = [] - val_targets = [] - val_ids_temp = [] - test_texts = [] - test_targets = [] - test_ids_temp = [] - - for i in dft_3d: - if i[prop] != "na": - atoms = Atoms.from_dict(i["atoms"]) - tmp = get_crystal_string(atoms) - if i[id_tag] in train_ids: - train_texts.append(tmp) - train_targets.append(i[prop]) - train_ids_temp.append(i[id_tag]) - elif i[id_tag] in val_ids: - val_texts.append(tmp) - val_targets.append(i[prop]) - val_ids_temp.append(i[id_tag]) - elif i[id_tag] in test_ids: - test_texts.append(tmp) - test_targets.append(i[prop]) - test_ids_temp.append(i[id_tag]) - """ - ############################## - ###Fast test### - train_texts = [ - "This is the first example text.", - "Second example is a bit longer than the first one, but still within the max length.", - "Third example is the longest among these three examples. It exceeds the max length and will be truncated.", - "Second example is a bit longer than the first one, but still within the max length.", - ] - train_targets = [10.2, 15.5, 20.1, 15.5] # Example regression targets - val_texts = test_texts = train_texts - val_targets = test_targets = train_targets - train_ids_temp=['a','b','c','d'] - val_ids_temp = test_ids_temp = train_ids_temp - batch_size = 2 - num_epochs = 3 - - ############################## - ############################## - """ - - # Fine-tune the last layer of GPT-2 for regression - # fine_tune_gpt2_regression(train_texts, train_targets, tokenizer) - if tokenizer.pad_token is None: - tokenizer.add_special_tokens({"pad_token": "[PAD]"}) - model.resize_token_embeddings(len(tokenizer)) - model.lm_head = torch.nn.Sequential( - torch.nn.Linear(model.config.hidden_size, latent_dim), - # torch.nn.Linear(latent_dim, latent_dim), - torch.nn.Linear(latent_dim, latent_dim), - torch.nn.Linear(latent_dim, 1), - ) - # model.lm_head = torch.nn.Sequential(torch.nn.Linear( model.config.hidden_size, 256),torch.nn.SiLU(),torch.nn.Linear( 256, 1) ) - model.to(device) - if torch.cuda.device_count() > 1: - device_ids = [d for d in range(torch.cuda.device_count())] - model = torch.nn.DataParallel(model, device_ids=device_ids).cuda() - optimizer = transformers.AdamW(model.parameters(), lr=learning_rate) - # Prepare datasets and dataloaders with data collator - train_dataset = AtomGPTDataset( - texts=train_texts, - targets=train_targets, - ids=train_ids_temp, - tokenizer=tokenizer, - max_length=max_length, - ) - val_dataset = AtomGPTDataset( - texts=val_texts, - targets=val_targets, - tokenizer=tokenizer, - ids=val_ids_temp, - max_length=max_length, - ) - test_dataset = AtomGPTDataset( - texts=test_texts, - targets=test_targets, - tokenizer=tokenizer, - ids=test_ids_temp, - max_length=max_length, - ) - - # val_dataset = train_dataset - train_dataloader = DataLoader(train_dataset, batch_size=batch_size) - val_dataloader = DataLoader(val_dataset, batch_size=batch_size) - test_dataloader = DataLoader(test_dataset, batch_size=batch_size) - val_dataloader = test_dataloader - steps_per_epoch = len(train_dataloader) - scheduler = torch.optim.lr_scheduler.OneCycleLR( - optimizer, - max_lr=learning_rate, - epochs=num_epochs, - steps_per_epoch=steps_per_epoch, - # pct_start=pct_start, - pct_start=0.3, - ) - print("train_data", len(train_texts)) - print("test_data", len(test_texts)) - output_dir = prefix + "_out_" + model_name + "_" + dataset + "_" + prop - if not os.path.exists(output_dir): - os.makedirs(output_dir) - best_loss = np.inf - tot_time_start = time.time() - train_history = [] - val_history = [] - for epoch in range(num_epochs): - model.train() - t1 = time.time() - for batch in train_dataloader: - optimizer.zero_grad() - train_loss = 0 - # train_result = [] - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - # if 't5' in model_name: - # decoder_input_ids = tokenizer("", return_tensors="pt").input_ids.to(device) - # decoder_input_ids = model._shift_right(decoder_input_ids) - # predictions = ( - # model(input_ids = input_ids.to(device),decoder_input_ids=decoder_input_ids).logits.squeeze().mean(dim=-1) - # ) - # else: - # predictions = ( - # model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - # ) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - targets = batch[2].squeeze() - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - #print('train',predictions,targets) - loss.backward() - optimizer.step() - scheduler.step() - # optimizer.zero_grad() - train_loss += loss.item() - train_loss = train_loss / len(train_dataloader) - t2 = time.time() - train_time = round(t2 - t1, 3) - model.eval() - - # total_eval_mae_loss = 0 - # predictions_list = [] - # targets_list = [] - val_loss = 0 - t1 = time.time() - for batch in val_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - targets = batch[2].squeeze() - # print('val',predictions,targets) - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - val_loss += loss.item() - saving_tag = "" - if val_loss < best_loss: - best_loss = val_loss - best_model_name = "best_model.pt" - torch.save( - model.state_dict(), - os.path.join(output_dir, best_model_name), - ) - # print("Saving model for epoch", epoch) - saving_tag = " saving model:" + str(epoch) - val_loss = val_loss / len(val_dataloader) - t2 = time.time() - val_time = round(t2 - t1, 3) - train_history.append(train_loss) - val_history.append(val_loss) - history = os.path.join(output_dir, "history.json") - - dumpjson( - data={"train": train_history, "val": val_history}, filename=history - ) - mae = "" - if test_each_run: - t1_test = time.time() - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - t2_test = time.time() - test_time = round(t2_test - t1_test, 3) - f.close() - df = pd.read_csv(fname) - mae = mean_absolute_error(df["target"], df["predictions"]) - if mae == "": - print( - "Epoch, train loss, val loss, train_time, val_time", - epoch, - train_loss, - val_loss, - train_time, - val_time, - saving_tag, - ) - else: - print( - "Epoch, train loss, val loss, test loss, train_time, val_time, test_time", - epoch, - train_loss, - val_loss, - mae, - train_time, - val_time, - test_time, - saving_tag, - ) - - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - f.close() - tot_time_end = time.time() - tot_time = tot_time_end - tot_time_start - print("tot_time", tot_time) - - -if __name__ == "__main__": - args = parser.parse_args(sys.argv[1:]) - benchmark_file = args.benchmark_file - model_name="meta-llama/Llama-2-7b-hf", - model_name="google/flan-t5-small" - model_name="google/flan-t5-base" - model_name = "facebook/opt-350m" - model_name = "mistralai/Mixtral-8x7B-v0.1" - model_name = "mistralai/Mistral-7B-Instruct-v0.1" - model_name="gpt2" - run_atomgpt( - model_name=model_name, - benchmark_file=benchmark_file, - num_epochs=300, - batch_size=16 - ) diff --git a/atomgpt/scripts/finetune1.py.bak_0.139 b/atomgpt/scripts/finetune1.py.bak_0.139 deleted file mode 100644 index 8306582..0000000 --- a/atomgpt/scripts/finetune1.py.bak_0.139 +++ /dev/null @@ -1,532 +0,0 @@ -"""Module for fin tuning LLM model for materials chemsitry.""" - -from jarvis.db.figshare import data -import transformers -import torch -from jarvis.db.jsonutils import dumpjson -import sys -import argparse -from torch.utils.data import DataLoader, Dataset -import numpy as np -import os -from jarvis.core.atoms import Atoms -import pandas as pd -from sklearn.metrics import mean_absolute_error - -# from tqdm import tqdm -import time -import json -import zipfile - -os.environ["WANDB_ANONYMOUS"] = "must" -np.random.seed(42) -torch.manual_seed(42) -torch.cuda.manual_seed_all(42) -torch.backends.cudnn.deterministic = True -torch.backends.cudnn.benchmark = False -os.environ["PYTHONHASHSEED"] = str(42) -os.environ["CUBLAS_WORKSPACE_CONFIG"] = str(":4096:8") -torch.use_deterministic_algorithms(True) - -IGNORE_INDEX = -100 -device = "cpu" -if torch.cuda.is_available(): - device = torch.device("cuda") -parser = argparse.ArgumentParser(description="AtomGPT") -parser.add_argument( - "--benchmark_file", - default="AI-SinglePropertyPrediction-ead-tinnet_N-test-mae.csv.zip", - # default="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae", - help="Benchmarks available in jarvis_leaderboard/benchmarks/*/*.zip", -) - - -def get_crystal_string_old(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.1f}".format(x) for x in lengths]) - + "\n" - + " ".join([str(int(x)) for x in angles]) - + "\n" - + "\n".join( - [ - str(t) + "\n" + " ".join(["{0:.2f}".format(x) for x in c]) - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - - return crystal_str - - -def get_crystal_string(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.1f}".format(x) for x in lengths]) - + "\n" - + " ".join([str(int(x)) for x in angles]) - + "\n" - + "\n".join( - [ - str(t) + " " + " ".join(["{0:.1f}".format(x) for x in c]) - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - return crystal_str - - -# Define a custom dataset class for regression -class AtomGPTDataset(Dataset): - def __init__( - self, texts=[], targets=[], ids=[], tokenizer="", max_length=128 - ): - self.texts = texts - self.targets = targets - self.tokenizer = tokenizer - self.max_length = max_length - if not ids: - ids = ["text-" + str(i) for i in range(len(texts))] - self.ids = ids - - def __len__(self): - return len(self.texts) - - def __getitem__(self, idx): - inputs = self.tokenizer( - self.texts[idx], - return_tensors="pt", - max_length=self.max_length, - padding="max_length", - truncation=True, - ) - return ( - inputs, - self.ids[idx], - torch.tensor(self.targets[idx], dtype=torch.float32), - ) - - -# Example usage - - -def run_atomgpt( - prefix="ss", - model_name="gpt2", - benchmark_file="AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip", - root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard", - batch_size=16, - max_length=512, - num_epochs=500, - latent_dim=512, - learning_rate=1e-3, - # learning_rate=1e-3, - test_each_run=True, - # learning_rate=5e-5, -): - # Load pre-trained tokenizer - # tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - # model = GPT2LMHeadModel.from_pretrained("gpt2") - - # dft_3d = data("dft_3d") - # root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard" - # benchmark_file="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae.csv.zip" - # benchmark_file = "AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip" - print("benchmark_file", benchmark_file) - method = benchmark_file.split("-")[0] - task = benchmark_file.split("-")[1] - prop = benchmark_file.split("-")[2] - dataset = benchmark_file.split("-")[3] - temp = dataset + "_" + prop + ".json.zip" - temp2 = dataset + "_" + prop + ".json" - fname = os.path.join(root_dir, "benchmarks", method, task, temp) - zp = zipfile.ZipFile(fname) - bench = json.loads(zp.read(temp2)) - dft_3d = data(dataset) - id_tag = "jid" - if "jid" in dft_3d[0]: - id_tag = "jid" - else: - id_tag = "id" - - # train_atoms = [] - # val_atoms = [] - # test_atoms = [] - # train_targets = [] - # val_targets = [] - # test_targets = [] - train_ids = list(bench["train"].keys()) - val_ids = list(bench["val"].keys()) - test_ids = list(bench["test"].keys()) - - # model_name = "mistralai/Mistral-7B-Instruct-v0.1" - # model_name = "gpt2" - if "t5" in model_name: - model = transformers.T5ForConditionalGeneration.from_pretrained( - model_name - ) - else: - model = transformers.AutoModelForCausalLM.from_pretrained( - model_name, - # load_in_8bit=True, - # device_map="auto" - ) - # device = model.device - if "t5" in model_name: - tokenizer = transformers.T5Tokenizer.from_pretrained(model_name) - - else: - tokenizer = transformers.AutoTokenizer.from_pretrained(model_name) - - # batch_size = 16 - # max_length = 128 - # num_epochs = 100 - # learning_rate = 5e-5 - criterion = torch.nn.L1Loss() - # Define example regression data (texts and corresponding numeric targets) - train_texts = [] - train_targets = [] - train_ids_temp = [] - val_texts = [] - val_targets = [] - val_ids_temp = [] - test_texts = [] - test_targets = [] - test_ids_temp = [] - - for i in dft_3d: - if i[prop] != "na": - atoms = Atoms.from_dict(i["atoms"]) - tmp = get_crystal_string(atoms) - if i[id_tag] in train_ids: - train_texts.append(tmp) - train_targets.append(i[prop]) - train_ids_temp.append(i[id_tag]) - elif i[id_tag] in val_ids: - val_texts.append(tmp) - val_targets.append(i[prop]) - val_ids_temp.append(i[id_tag]) - elif i[id_tag] in test_ids: - test_texts.append(tmp) - test_targets.append(i[prop]) - test_ids_temp.append(i[id_tag]) - """ - ############################## - ###Fast test### - train_texts = [ - "This is the first example text.", - "Second example is a bit longer than the first one, but still within the max length.", - "Third example is the longest among these three examples. It exceeds the max length and will be truncated.", - "Second example is a bit longer than the first one, but still within the max length.", - ] - train_targets = [10.2, 15.5, 20.1, 15.5] # Example regression targets - val_texts = test_texts = train_texts - val_targets = test_targets = train_targets - train_ids_temp=['a','b','c','d'] - val_ids_temp = test_ids_temp = train_ids_temp - batch_size = 2 - num_epochs = 3 - - ############################## - ############################## - """ - - # Fine-tune the last layer of GPT-2 for regression - # fine_tune_gpt2_regression(train_texts, train_targets, tokenizer) - if tokenizer.pad_token is None: - tokenizer.add_special_tokens({"pad_token": "[PAD]"}) - model.resize_token_embeddings(len(tokenizer)) - model.lm_head = torch.nn.Sequential( - torch.nn.Linear(model.config.hidden_size, latent_dim), - # torch.nn.Linear(latent_dim, latent_dim), - torch.nn.Linear(latent_dim, latent_dim), - torch.nn.Linear(latent_dim, 1), - ) - # model.lm_head = torch.nn.Sequential(torch.nn.Linear( model.config.hidden_size, 256),torch.nn.SiLU(),torch.nn.Linear( 256, 1) ) - model.to(device) - optimizer = transformers.AdamW(model.parameters(), lr=learning_rate) - # Prepare datasets and dataloaders with data collator - train_dataset = AtomGPTDataset( - texts=train_texts, - targets=train_targets, - ids=train_ids_temp, - tokenizer=tokenizer, - max_length=max_length, - ) - val_dataset = AtomGPTDataset( - texts=val_texts, - targets=val_targets, - tokenizer=tokenizer, - ids=val_ids_temp, - max_length=max_length, - ) - test_dataset = AtomGPTDataset( - texts=test_texts, - targets=test_targets, - tokenizer=tokenizer, - ids=test_ids_temp, - max_length=max_length, - ) - - # val_dataset = train_dataset - train_dataloader = DataLoader(train_dataset, batch_size=batch_size) - val_dataloader = DataLoader(val_dataset, batch_size=batch_size) - test_dataloader = DataLoader(test_dataset, batch_size=batch_size) - test_dataloader = val_dataloader - steps_per_epoch = len(train_dataloader) - scheduler = torch.optim.lr_scheduler.OneCycleLR( - optimizer, - max_lr=learning_rate, - epochs=num_epochs, - steps_per_epoch=steps_per_epoch, - # pct_start=pct_start, - pct_start=0.3, - ) - print("train_data", len(train_texts)) - print("test_data", len(test_texts)) - output_dir = prefix + "_out_" + model_name + "_" + dataset + "_" + prop - if not os.path.exists(output_dir): - os.makedirs(output_dir) - best_loss = np.inf - tot_time_start = time.time() - train_history = [] - val_history = [] - for epoch in range(num_epochs): - model.train() - t1 = time.time() - for batch in train_dataloader: - optimizer.zero_grad() - train_loss = 0 - # train_result = [] - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - # if 't5' in model_name: - # decoder_input_ids = tokenizer("", return_tensors="pt").input_ids.to(device) - # decoder_input_ids = model._shift_right(decoder_input_ids) - # predictions = ( - # model(input_ids = input_ids.to(device),decoder_input_ids=decoder_input_ids).logits.squeeze().mean(dim=-1) - # ) - # else: - # predictions = ( - # model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - # ) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - targets = batch[2].squeeze() - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - # print('train',predictions,targets) - loss.backward() - optimizer.step() - scheduler.step() - # optimizer.zero_grad() - train_loss += loss.item() - train_loss = train_loss / len(train_dataloader) - t2 = time.time() - train_time = round(t2 - t1, 3) - model.eval() - - # total_eval_mae_loss = 0 - # predictions_list = [] - # targets_list = [] - val_loss = 0 - t1 = time.time() - for batch in val_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - targets = batch[2].squeeze() - # print('val',predictions,targets) - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - val_loss += loss.item() - saving_tag = "" - if val_loss < best_loss: - best_loss = val_loss - best_model_name = "best_model.pt" - torch.save( - model.state_dict(), - os.path.join(output_dir, best_model_name), - ) - # print("Saving model for epoch", epoch) - saving_tag = " saving model:" + str(epoch) - val_loss = val_loss / len(val_dataloader) - t2 = time.time() - val_time = round(t2 - t1, 3) - train_history.append(train_loss) - val_history.append(val_loss) - history = os.path.join(output_dir, "history.json") - - dumpjson( - data={"train": train_history, "val": val_history}, filename=history - ) - mae = "" - if test_each_run: - t1_test = time.time() - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - t2_test = time.time() - test_time = round(t2_test - t1_test, 3) - f.close() - df = pd.read_csv(fname) - mae = mean_absolute_error(df["target"], df["predictions"]) - if mae == "": - print( - "Epoch, train loss, val loss, train_time, val_time", - epoch, - train_loss, - val_loss, - train_time, - val_time, - saving_tag, - ) - else: - print( - "Epoch, train loss, val loss, test loss, train_time, val_time, test_time", - epoch, - train_loss, - val_loss, - mae, - train_time, - val_time, - test_time, - saving_tag, - ) - - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - f.close() - tot_time_end = time.time() - tot_time = tot_time_end - tot_time_start - print("tot_time", tot_time) - - -if __name__ == "__main__": - args = parser.parse_args(sys.argv[1:]) - benchmark_file = args.benchmark_file - run_atomgpt( - # model_name="gpt2", - model_name="google/flan-t5-small", - # model_name="meta-llama/Llama-2-7b-hf", - benchmark_file=benchmark_file, - num_epochs=300, - ) diff --git a/atomgpt/scripts/finetune1.py.bak_0.146 b/atomgpt/scripts/finetune1.py.bak_0.146 deleted file mode 100644 index 380e837..0000000 --- a/atomgpt/scripts/finetune1.py.bak_0.146 +++ /dev/null @@ -1,384 +0,0 @@ -"""Module for fin tuning LLM model for materials chemsitry.""" -from jarvis.db.figshare import data -import transformers -import torch -from jarvis.db.jsonutils import dumpjson -import sys -import argparse -from torch.utils.data import DataLoader, Dataset -import numpy as np -import os -from jarvis.core.atoms import Atoms - -# from tqdm import tqdm -import time -import json -import zipfile - -os.environ["WANDB_ANONYMOUS"] = "must" -np.random.seed(42) -torch.manual_seed(42) -torch.cuda.manual_seed_all(42) -torch.backends.cudnn.deterministic = True -torch.backends.cudnn.benchmark = False -os.environ["PYTHONHASHSEED"] = str(42) -os.environ["CUBLAS_WORKSPACE_CONFIG"] = str(":4096:8") -torch.use_deterministic_algorithms(True) - -IGNORE_INDEX = -100 -device = "cpu" -if torch.cuda.is_available(): - device = torch.device("cuda") -parser = argparse.ArgumentParser(description="AtomGPT") -parser.add_argument( - "--benchmark_file", - default="AI-SinglePropertyPrediction-ead-tinnet_N-test-mae.csv.zip", - #default="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae", - help="Benchmarks available in jarvis_leaderboard/benchmarks/*/*.zip", -) -def get_crystal_string_old(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.1f}".format(x) for x in lengths]) - + "\n" - + " ".join([str(int(x)) for x in angles]) - + "\n" - + "\n".join( - [ - str(t) + "\n" + " ".join(["{0:.2f}".format(x) for x in c]) - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - - return crystal_str - -def get_crystal_string(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.1f}".format(x) for x in lengths]) - + "\n" - + " ".join([str(int(x)) for x in angles]) - + "\n" - + "\n".join( - [ - str(t) + " " + " ".join(["{0:.1f}".format(x) for x in c]) - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - return crystal_str - - -# Define a custom dataset class for regression -class AtomGPTDataset(Dataset): - def __init__( - self, texts=[], targets=[], ids=[], tokenizer="", max_length=128 - ): - self.texts = texts - self.targets = targets - self.tokenizer = tokenizer - self.max_length = max_length - if not ids: - ids = ["text-" + str(i) for i in range(len(texts))] - self.ids = ids - - def __len__(self): - return len(self.texts) - - def __getitem__(self, idx): - inputs = self.tokenizer( - self.texts[idx], - return_tensors="pt", - max_length=self.max_length, - padding="max_length", - truncation=True, - ) - return ( - inputs, - self.ids[idx], - torch.tensor(self.targets[idx], dtype=torch.float32), - ) - - -# Example usage - - -def run_atomgpt( - prefix="ss", - model_name="gpt2", - benchmark_file="AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip", - root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard", - batch_size=16, - max_length=512, - num_epochs=500, - latent_dim=512, - learning_rate=5e-5, -): - # Load pre-trained tokenizer - # tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - # model = GPT2LMHeadModel.from_pretrained("gpt2") - - #dft_3d = data("dft_3d") - # root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard" - # benchmark_file="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae.csv.zip" - # benchmark_file = "AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip" - method = benchmark_file.split("-")[0] - task = benchmark_file.split("-")[1] - prop = benchmark_file.split("-")[2] - dataset = benchmark_file.split("-")[3] - temp = dataset + "_" + prop + ".json.zip" - temp2 = dataset + "_" + prop + ".json" - fname = os.path.join(root_dir, "benchmarks", method, task, temp) - zp = zipfile.ZipFile(fname) - bench = json.loads(zp.read(temp2)) - dft_3d = data(dataset) - id_tag="jid" - if "jid" in dft_3d[0]: - id_tag="jid" - else: - id_tag="id" - - # train_atoms = [] - # val_atoms = [] - # test_atoms = [] - # train_targets = [] - # val_targets = [] - # test_targets = [] - train_ids = list(bench["train"].keys()) - val_ids = list(bench["val"].keys()) - test_ids = list(bench["test"].keys()) - - # model_name = "mistralai/Mistral-7B-Instruct-v0.1" - # model_name = "gpt2" - model = transformers.AutoModelForCausalLM.from_pretrained(model_name) - - tokenizer = transformers.AutoTokenizer.from_pretrained(model_name) - - # batch_size = 16 - # max_length = 128 - # num_epochs = 100 - # learning_rate = 5e-5 - criterion = torch.nn.L1Loss() - # Define example regression data (texts and corresponding numeric targets) - train_texts = [] - train_targets = [] - train_ids_temp = [] - val_texts = [] - val_targets = [] - val_ids_temp = [] - test_texts = [] - test_targets = [] - test_ids_temp = [] - - for i in dft_3d: - if i[prop] != "na": - atoms = Atoms.from_dict(i["atoms"]) - tmp = get_crystal_string(atoms) - if i[id_tag] in train_ids: - train_texts.append(tmp) - train_targets.append(i[prop]) - train_ids_temp.append(i[id_tag]) - elif i[id_tag] in val_ids: - val_texts.append(tmp) - val_targets.append(i[prop]) - val_ids_temp.append(i[id_tag]) - elif i[id_tag] in test_ids: - test_texts.append(tmp) - test_targets.append(i[prop]) - test_ids_temp.append(i[id_tag]) - """ - ############################## - ###Fast test### - train_texts = [ - "This is the first example text.", - "Second example is a bit longer than the first one, but still within the max length.", - "Third example is the longest among these three examples. It exceeds the max length and will be truncated.", - "Second example is a bit longer than the first one, but still within the max length.", - ] - train_targets = [10.2, 15.5, 20.1, 15.5] # Example regression targets - val_texts = test_texts = train_texts - val_targets = test_targets = train_targets - train_ids_temp=['a','b','c','d'] - val_ids_temp = test_ids_temp = train_ids_temp - batch_size = 2 - num_epochs = 3 - - ############################## - ############################## - """ - - # Fine-tune the last layer of GPT-2 for regression - # fine_tune_gpt2_regression(train_texts, train_targets, tokenizer) - if tokenizer.pad_token is None: - tokenizer.add_special_tokens({"pad_token": "[PAD]"}) - model.resize_token_embeddings(len(tokenizer)) - model.lm_head = torch.nn.Sequential(torch.nn.Linear( model.config.hidden_size, latent_dim),torch.nn.Linear( latent_dim, latent_dim), torch.nn.Linear( latent_dim, 1) ) - #model.lm_head = torch.nn.Sequential(torch.nn.Linear( model.config.hidden_size, 256),torch.nn.SiLU(),torch.nn.Linear( 256, 1) ) - model.to(device) - optimizer = transformers.AdamW(model.parameters(), lr=learning_rate) - # Prepare datasets and dataloaders with data collator - train_dataset = AtomGPTDataset( - texts=train_texts, - targets=train_targets, - ids=train_ids_temp, - tokenizer=tokenizer, - max_length=max_length, - ) - val_dataset = AtomGPTDataset( - texts=val_texts, - targets=val_targets, - tokenizer=tokenizer, - ids=val_ids_temp, - max_length=max_length, - ) - test_dataset = AtomGPTDataset( - texts=test_texts, - targets=test_targets, - tokenizer=tokenizer, - ids=test_ids_temp, - max_length=max_length, - ) - - # val_dataset = train_dataset - train_dataloader = DataLoader(train_dataset, batch_size=batch_size) - val_dataloader = DataLoader(val_dataset, batch_size=batch_size) - test_dataloader = DataLoader(test_dataset, batch_size=batch_size) - test_dataloader = val_dataloader - steps_per_epoch = len(train_dataloader) - scheduler = torch.optim.lr_scheduler.OneCycleLR( - optimizer, - max_lr=learning_rate, - epochs=num_epochs, - steps_per_epoch=steps_per_epoch, - # pct_start=pct_start, - pct_start=0.3, - ) - print("benchmark_file", benchmark_file) - print("train_data", len(train_texts)) - print("test_data", len(test_texts)) - output_dir = prefix + "_out_" + model_name + "_" + dataset + "_" + prop - if not os.path.exists(output_dir): - os.makedirs(output_dir) - best_loss = np.inf - tot_time_start = time.time() - train_history = [] - val_history = [] - for epoch in range(num_epochs): - model.train() - t1 = time.time() - for batch in train_dataloader: - train_loss = 0 - # train_result = [] - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - predictions = ( - model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - ) - targets = batch[2].squeeze() - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - # print('train',predictions,targets) - loss.backward() - optimizer.step() - scheduler.step() - optimizer.zero_grad() - train_loss += loss.item() - train_loss = train_loss / len(train_dataloader) - t2 = time.time() - train_time = round(t2 - t1, 3) - model.eval() - - # total_eval_mae_loss = 0 - # predictions_list = [] - # targets_list = [] - val_loss = 0 - t1 = time.time() - for batch in val_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze(0) # .squeeze(0) - predictions = ( - model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - ) - targets = batch[2].squeeze() - # print('val',predictions,targets) - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - val_loss += loss.item() - saving_tag = "" - if val_loss < best_loss: - best_loss = val_loss - best_model_name = "best_model.pt" - torch.save( - model.state_dict(), - os.path.join(output_dir, best_model_name), - ) - # print("Saving model for epoch", epoch) - saving_tag = " saving model:" + str(epoch) - val_loss = val_loss / len(val_dataloader) - t2 = time.time() - val_time = round(t2 - t1, 3) - print( - "Epoch, train loss, val loss, train_time, val_time", - epoch, - train_loss, - val_loss, - train_time, - val_time, - saving_tag, - ) - train_history.append(train_loss) - val_history.append(val_loss) - history = os.path.join(output_dir, "history.json") - - dumpjson( - data={"train": train_history, "val": val_history}, filename=history - ) - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze(0) # .squeeze(0) - predictions = ( - model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - f.close() - tot_time_end = time.time() - tot_time = tot_time_end - tot_time_start - print("tot_time", tot_time) - - -if __name__ == "__main__": - args = parser.parse_args(sys.argv[1:]) - benchmark_file = args.benchmark_file - run_atomgpt(benchmark_file=benchmark_file,num_epochs=300) diff --git a/atomgpt/scripts/finetune1a.py b/atomgpt/scripts/finetune1a.py deleted file mode 100644 index 64ccf1b..0000000 --- a/atomgpt/scripts/finetune1a.py +++ /dev/null @@ -1,384 +0,0 @@ -"""Module for fin tuning LLM model for materials chemsitry.""" -from jarvis.db.figshare import data -import transformers -import torch -from jarvis.db.jsonutils import dumpjson -import sys -import argparse -from torch.utils.data import DataLoader, Dataset -import numpy as np -import os -from jarvis.core.atoms import Atoms - -# from tqdm import tqdm -import time -import json -import zipfile - -os.environ["WANDB_ANONYMOUS"] = "must" -np.random.seed(42) -torch.manual_seed(42) -torch.cuda.manual_seed_all(42) -torch.backends.cudnn.deterministic = True -torch.backends.cudnn.benchmark = False -os.environ["PYTHONHASHSEED"] = str(42) -os.environ["CUBLAS_WORKSPACE_CONFIG"] = str(":4096:8") -torch.use_deterministic_algorithms(True) - -IGNORE_INDEX = -100 -device = "cpu" -if torch.cuda.is_available(): - device = torch.device("cuda") -parser = argparse.ArgumentParser(description="AtomGPT") -parser.add_argument( - "--benchmark_file", - default="AI-SinglePropertyPrediction-ead-tinnet_N-test-mae.csv.zip", - #default="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae", - help="Benchmarks available in jarvis_leaderboard/benchmarks/*/*.zip", -) -def get_crystal_string_old(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.1f}".format(x) for x in lengths]) - + "\n" - + " ".join([str(int(x)) for x in angles]) - + "\n" - + "\n".join( - [ - str(t) + "\n" + " ".join(["{0:.2f}".format(x) for x in c]) - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - - return crystal_str - -def get_crystal_string(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.1f}".format(x) for x in lengths]) - + "\n" - + " ".join([str(int(x)) for x in angles]) - + "\n" - + "\n".join( - [ - str(t) + " " + " ".join(["{0:.1f}".format(x) for x in c]) - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - return crystal_str - - -# Define a custom dataset class for regression -class AtomGPTDataset(Dataset): - def __init__( - self, texts=[], targets=[], ids=[], tokenizer="", max_length=128 - ): - self.texts = texts - self.targets = targets - self.tokenizer = tokenizer - self.max_length = max_length - if not ids: - ids = ["text-" + str(i) for i in range(len(texts))] - self.ids = ids - - def __len__(self): - return len(self.texts) - - def __getitem__(self, idx): - inputs = self.tokenizer( - self.texts[idx], - return_tensors="pt", - max_length=self.max_length, - padding="max_length", - truncation=True, - ) - return ( - inputs, - self.ids[idx], - torch.tensor(self.targets[idx], dtype=torch.float32), - ) - - -# Example usage - - -def run_atomgpt( - prefix="ss", - model_name="gpt2", - benchmark_file="AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip", - root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard", - batch_size=16, - max_length=512, - num_epochs=500, - latent_dim=512, - learning_rate=5e-5, -): - # Load pre-trained tokenizer - # tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - # model = GPT2LMHeadModel.from_pretrained("gpt2") - - #dft_3d = data("dft_3d") - # root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard" - # benchmark_file="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae.csv.zip" - # benchmark_file = "AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip" - method = benchmark_file.split("-")[0] - task = benchmark_file.split("-")[1] - prop = benchmark_file.split("-")[2] - dataset = benchmark_file.split("-")[3] - temp = dataset + "_" + prop + ".json.zip" - temp2 = dataset + "_" + prop + ".json" - fname = os.path.join(root_dir, "benchmarks", method, task, temp) - zp = zipfile.ZipFile(fname) - bench = json.loads(zp.read(temp2)) - dft_3d = data(dataset) - id_tag="jid" - if "jid" in dft_3d[0]: - id_tag="jid" - else: - id_tag="id" - - # train_atoms = [] - # val_atoms = [] - # test_atoms = [] - # train_targets = [] - # val_targets = [] - # test_targets = [] - train_ids = list(bench["train"].keys()) - val_ids = list(bench["val"].keys()) - test_ids = list(bench["test"].keys()) - - # model_name = "mistralai/Mistral-7B-Instruct-v0.1" - # model_name = "gpt2" - model = transformers.AutoModelForCausalLM.from_pretrained(model_name) - - tokenizer = transformers.AutoTokenizer.from_pretrained(model_name) - - # batch_size = 16 - # max_length = 128 - # num_epochs = 100 - # learning_rate = 5e-5 - criterion = torch.nn.L1Loss() - # Define example regression data (texts and corresponding numeric targets) - train_texts = [] - train_targets = [] - train_ids_temp = [] - val_texts = [] - val_targets = [] - val_ids_temp = [] - test_texts = [] - test_targets = [] - test_ids_temp = [] - - for i in dft_3d: - if i[prop] != "na": - atoms = Atoms.from_dict(i["atoms"]) - tmp = get_crystal_string(atoms) - if i[id_tag] in train_ids: - train_texts.append(tmp) - train_targets.append(i[prop]) - train_ids_temp.append(i[id_tag]) - elif i[id_tag] in val_ids: - val_texts.append(tmp) - val_targets.append(i[prop]) - val_ids_temp.append(i[id_tag]) - elif i[id_tag] in test_ids: - test_texts.append(tmp) - test_targets.append(i[prop]) - test_ids_temp.append(i[id_tag]) - """ - ############################## - ###Fast test### - train_texts = [ - "This is the first example text.", - "Second example is a bit longer than the first one, but still within the max length.", - "Third example is the longest among these three examples. It exceeds the max length and will be truncated.", - "Second example is a bit longer than the first one, but still within the max length.", - ] - train_targets = [10.2, 15.5, 20.1, 15.5] # Example regression targets - val_texts = test_texts = train_texts - val_targets = test_targets = train_targets - train_ids_temp=['a','b','c','d'] - val_ids_temp = test_ids_temp = train_ids_temp - batch_size = 2 - num_epochs = 3 - - ############################## - ############################## - """ - - # Fine-tune the last layer of GPT-2 for regression - # fine_tune_gpt2_regression(train_texts, train_targets, tokenizer) - if tokenizer.pad_token is None: - tokenizer.add_special_tokens({"pad_token": "[PAD]"}) - model.resize_token_embeddings(len(tokenizer)) - model.lm_head = torch.nn.Sequential(torch.nn.Linear( model.config.hidden_size, latent_dim),torch.nn.Linear( latent_dim, 1) ) - #model.lm_head = torch.nn.Sequential(torch.nn.Linear( model.config.hidden_size, 256),torch.nn.SiLU(),torch.nn.Linear( 256, 1) ) - model.to(device) - optimizer = transformers.AdamW(model.parameters(), lr=learning_rate) - # Prepare datasets and dataloaders with data collator - train_dataset = AtomGPTDataset( - texts=train_texts, - targets=train_targets, - ids=train_ids_temp, - tokenizer=tokenizer, - max_length=max_length, - ) - val_dataset = AtomGPTDataset( - texts=val_texts, - targets=val_targets, - tokenizer=tokenizer, - ids=val_ids_temp, - max_length=max_length, - ) - test_dataset = AtomGPTDataset( - texts=test_texts, - targets=test_targets, - tokenizer=tokenizer, - ids=test_ids_temp, - max_length=max_length, - ) - - # val_dataset = train_dataset - train_dataloader = DataLoader(train_dataset, batch_size=batch_size) - val_dataloader = DataLoader(val_dataset, batch_size=batch_size) - test_dataloader = DataLoader(test_dataset, batch_size=batch_size) - test_dataloader = val_dataloader - steps_per_epoch = len(train_dataloader) - scheduler = torch.optim.lr_scheduler.OneCycleLR( - optimizer, - max_lr=learning_rate, - epochs=num_epochs, - steps_per_epoch=steps_per_epoch, - # pct_start=pct_start, - pct_start=0.3, - ) - print("benchmark_file", benchmark_file) - print("train_data", len(train_texts)) - print("test_data", len(test_texts)) - output_dir = prefix + "_out_" + model_name + "_" + dataset + "_" + prop - if not os.path.exists(output_dir): - os.makedirs(output_dir) - best_loss = np.inf - tot_time_start = time.time() - train_history = [] - val_history = [] - for epoch in range(num_epochs): - model.train() - t1 = time.time() - for batch in train_dataloader: - train_loss = 0 - # train_result = [] - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - predictions = ( - model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - ) - targets = batch[2].squeeze() - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - # print('train',predictions,targets) - loss.backward() - optimizer.step() - scheduler.step() - optimizer.zero_grad() - train_loss += loss.item() - train_loss = train_loss / len(train_dataloader) - t2 = time.time() - train_time = round(t2 - t1, 3) - model.eval() - - # total_eval_mae_loss = 0 - # predictions_list = [] - # targets_list = [] - val_loss = 0 - t1 = time.time() - for batch in val_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze(0) # .squeeze(0) - predictions = ( - model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - ) - targets = batch[2].squeeze() - # print('val',predictions,targets) - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - val_loss += loss.item() - saving_tag = "" - if val_loss < best_loss: - best_loss = val_loss - best_model_name = "best_model.pt" - torch.save( - model.state_dict(), - os.path.join(output_dir, best_model_name), - ) - # print("Saving model for epoch", epoch) - saving_tag = " saving model:" + str(epoch) - val_loss = val_loss / len(val_dataloader) - t2 = time.time() - val_time = round(t2 - t1, 3) - print( - "Epoch, train loss, val loss, train_time, val_time", - epoch, - train_loss, - val_loss, - train_time, - val_time, - saving_tag, - ) - train_history.append(train_loss) - val_history.append(val_loss) - history = os.path.join(output_dir, "history.json") - - dumpjson( - data={"train": train_history, "val": val_history}, filename=history - ) - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze(0) # .squeeze(0) - predictions = ( - model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - f.close() - tot_time_end = time.time() - tot_time = tot_time_end - tot_time_start - print("tot_time", tot_time) - - -if __name__ == "__main__": - args = parser.parse_args(sys.argv[1:]) - benchmark_file = args.benchmark_file - run_atomgpt(benchmark_file=benchmark_file,num_epochs=300) diff --git a/atomgpt/scripts/finetune2.py b/atomgpt/scripts/finetune2.py deleted file mode 100644 index 88433b7..0000000 --- a/atomgpt/scripts/finetune2.py +++ /dev/null @@ -1,358 +0,0 @@ -"""Module for fin tuning LLM model for materials chemsitry.""" -from jarvis.db.figshare import data -import transformers -import torch -from jarvis.db.jsonutils import dumpjson -import sys -import argparse -from torch.utils.data import DataLoader, Dataset -import numpy as np -import os -from jarvis.core.atoms import Atoms - -# from tqdm import tqdm -import time -import json -import zipfile - -os.environ["WANDB_ANONYMOUS"] = "must" -np.random.seed(42) -torch.manual_seed(42) -torch.cuda.manual_seed_all(42) -torch.backends.cudnn.deterministic = True -torch.backends.cudnn.benchmark = False -os.environ["PYTHONHASHSEED"] = str(42) -os.environ["CUBLAS_WORKSPACE_CONFIG"] = str(":4096:8") -torch.use_deterministic_algorithms(True) - -IGNORE_INDEX = -100 -device = "cpu" -if torch.cuda.is_available(): - device = torch.device("cuda") -parser = argparse.ArgumentParser(description="AtomGPT") -parser.add_argument( - "--benchmark_file", - default="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae", - help="Benchmarks available in jarvis_leaderboard/benchmarks/*/*.zip", -) - - - -def get_crystal_string(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.1f}".format(x) for x in lengths]) - + "\n" - + " ".join([str(int(x)) for x in angles]) - + "\n" - + "\n".join( - [ - str(t) + " " + " ".join(["{0:.2f}".format(x) for x in c]) - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - - - return crystal_str -# Define a custom dataset class for regression -class AtomGPTDataset(Dataset): - def __init__( - self, texts=[], targets=[], ids=[], tokenizer="", max_length=128 - ): - self.texts = texts - self.targets = targets - self.tokenizer = tokenizer - self.max_length = max_length - if not ids: - ids = ["text-" + str(i) for i in range(len(texts))] - self.ids = ids - - def __len__(self): - return len(self.texts) - - def __getitem__(self, idx): - inputs = self.tokenizer( - self.texts[idx], - return_tensors="pt", - max_length=self.max_length, - padding="max_length", - truncation=True, - ) - return ( - inputs, - self.ids[idx], - torch.tensor(self.targets[idx], dtype=torch.float32), - ) - - -# Example usage - - -def run_atomgpt( - prefix="ss", - model_name="gpt2", - benchmark_file="AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip", - root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard", - batch_size=16, - max_length=128, - num_epochs=500, - learning_rate=5e-5, -): - # Load pre-trained tokenizer - # tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - # model = GPT2LMHeadModel.from_pretrained("gpt2") - - dft_3d = data("dft_3d") - # root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard" - # benchmark_file="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae.csv.zip" - # benchmark_file = "AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip" - method = benchmark_file.split("-")[0] - task = benchmark_file.split("-")[1] - prop = benchmark_file.split("-")[2] - dataset = benchmark_file.split("-")[3] - temp = dataset + "_" + prop + ".json.zip" - temp2 = dataset + "_" + prop + ".json" - fname = os.path.join(root_dir, "benchmarks", method, task, temp) - zp = zipfile.ZipFile(fname) - bench = json.loads(zp.read(temp2)) - - # train_atoms = [] - # val_atoms = [] - # test_atoms = [] - # train_targets = [] - # val_targets = [] - # test_targets = [] - train_ids = list(bench["train"].keys()) - val_ids = list(bench["val"].keys()) - test_ids = list(bench["test"].keys()) - - # model_name = "mistralai/Mistral-7B-Instruct-v0.1" - # model_name = "gpt2" - model = transformers.AutoModelForCausalLM.from_pretrained(model_name) - - tokenizer = transformers.AutoTokenizer.from_pretrained(model_name) - - # batch_size = 16 - # max_length = 128 - # num_epochs = 100 - # learning_rate = 5e-5 - criterion = torch.nn.L1Loss() - # Define example regression data (texts and corresponding numeric targets) - train_texts = [] - train_targets = [] - train_ids_temp = [] - val_texts = [] - val_targets = [] - val_ids_temp = [] - test_texts = [] - test_targets = [] - test_ids_temp = [] - - for i in dft_3d: - if i[prop] != "na": - atoms = Atoms.from_dict(i["atoms"]) - tmp = get_crystal_string(atoms) - if i["jid"] in train_ids: - train_texts.append(tmp) - train_targets.append(i[prop]) - train_ids_temp.append(i["jid"]) - elif i["jid"] in val_ids: - val_texts.append(tmp) - val_targets.append(i[prop]) - val_ids_temp.append(i["jid"]) - elif i["jid"] in test_ids: - test_texts.append(tmp) - test_targets.append(i[prop]) - test_ids_temp.append(i["jid"]) - """ - ############################## - ###Fast test### - train_texts = [ - "This is the first example text.", - "Second example is a bit longer than the first one, but still within the max length.", - "Third example is the longest among these three examples. It exceeds the max length and will be truncated.", - "Second example is a bit longer than the first one, but still within the max length.", - ] - train_targets = [10.2, 15.5, 20.1, 15.5] # Example regression targets - val_texts = test_texts = train_texts - val_targets = test_targets = train_targets - train_ids_temp=['a','b','c','d'] - val_ids_temp = test_ids_temp = train_ids_temp - batch_size = 2 - num_epochs = 3 - - ############################## - ############################## - """ - - # Fine-tune the last layer of GPT-2 for regression - # fine_tune_gpt2_regression(train_texts, train_targets, tokenizer) - if tokenizer.pad_token is None: - tokenizer.add_special_tokens({"pad_token": "[PAD]"}) - model.resize_token_embeddings(len(tokenizer)) - model.lm_head = torch.nn.Sequential(torch.nn.Linear( model.config.hidden_size, 256),torch.nn.Linear( 256, 1) ) - #model.lm_head = torch.nn.Sequential(torch.nn.Linear( model.config.hidden_size, 256),torch.nn.SiLU(),torch.nn.Linear( 256, 1) ) - model.to(device) - optimizer = transformers.AdamW(model.parameters(), lr=learning_rate) - # Prepare datasets and dataloaders with data collator - train_dataset = AtomGPTDataset( - texts=train_texts, - targets=train_targets, - ids=train_ids_temp, - tokenizer=tokenizer, - max_length=max_length, - ) - val_dataset = AtomGPTDataset( - texts=val_texts, - targets=val_targets, - tokenizer=tokenizer, - ids=val_ids_temp, - max_length=max_length, - ) - test_dataset = AtomGPTDataset( - texts=test_texts, - targets=test_targets, - tokenizer=tokenizer, - ids=test_ids_temp, - max_length=max_length, - ) - - # val_dataset = train_dataset - train_dataloader = DataLoader(train_dataset, batch_size=batch_size) - val_dataloader = DataLoader(val_dataset, batch_size=batch_size) - test_dataloader = DataLoader(test_dataset, batch_size=batch_size) - test_dataloader = val_dataloader - steps_per_epoch = len(train_dataloader) - scheduler = torch.optim.lr_scheduler.OneCycleLR( - optimizer, - max_lr=learning_rate, - epochs=num_epochs, - steps_per_epoch=steps_per_epoch, - # pct_start=pct_start, - pct_start=0.3, - ) - print("benchmark_file", benchmark_file) - print("train_data", len(train_texts)) - print("test_data", len(test_texts)) - output_dir = prefix + "_out_" + model_name + "_" + dataset + "_" + prop - if not os.path.exists(output_dir): - os.makedirs(output_dir) - best_loss = np.inf - tot_time_start = time.time() - train_history = [] - val_history = [] - for epoch in range(num_epochs): - model.train() - t1 = time.time() - for batch in train_dataloader: - train_loss = 0 - # train_result = [] - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - predictions = ( - model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - ) - targets = batch[2].squeeze() - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - # print('train',predictions,targets) - loss.backward() - optimizer.step() - scheduler.step() - optimizer.zero_grad() - train_loss += loss.item() - train_loss = train_loss / len(train_dataloader) - t2 = time.time() - train_time = round(t2 - t1, 3) - model.eval() - - # total_eval_mae_loss = 0 - # predictions_list = [] - # targets_list = [] - val_loss = 0 - t1 = time.time() - for batch in val_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze(0) # .squeeze(0) - predictions = ( - model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - ) - targets = batch[2].squeeze() - # print('val',predictions,targets) - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - val_loss += loss.item() - saving_tag = "" - if val_loss < best_loss: - best_loss = val_loss - best_model_name = "best_model.pt" - torch.save( - model.state_dict(), - os.path.join(output_dir, best_model_name), - ) - # print("Saving model for epoch", epoch) - saving_tag = " saving model:" + str(epoch) - val_loss = val_loss / len(val_dataloader) - t2 = time.time() - val_time = round(t2 - t1, 3) - print( - "Epoch, train loss, val loss, train_time, val_time", - epoch, - train_loss, - val_loss, - train_time, - val_time, - saving_tag, - ) - train_history.append(train_loss) - val_history.append(val_loss) - history = os.path.join(output_dir, "history.json") - - dumpjson( - data={"train": train_history, "val": val_history}, filename=history - ) - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze(0) # .squeeze(0) - predictions = ( - model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - f.close() - tot_time_end = time.time() - tot_time = tot_time_end - tot_time_start - print("tot_time", tot_time) - - -if __name__ == "__main__": - args = parser.parse_args(sys.argv[1:]) - benchmark_file = args.benchmark_file - run_atomgpt(benchmark_file=benchmark_file,num_epochs=300) diff --git a/atomgpt/scripts/finetune3.py b/atomgpt/scripts/finetune3.py deleted file mode 100644 index 968ba64..0000000 --- a/atomgpt/scripts/finetune3.py +++ /dev/null @@ -1,547 +0,0 @@ -"""Module for fin tuning LLM model for materials chemsitry.""" - -from jarvis.db.figshare import data -import transformers -import torch -from jarvis.db.jsonutils import dumpjson -import sys -import argparse -from torch.utils.data import DataLoader, Dataset -import numpy as np -import os -from jarvis.core.atoms import Atoms -import pandas as pd -from sklearn.metrics import mean_absolute_error - -# from tqdm import tqdm -import time -import json -import zipfile - -os.environ["WANDB_ANONYMOUS"] = "must" -np.random.seed(42) -torch.manual_seed(42) -torch.cuda.manual_seed_all(42) -torch.backends.cudnn.deterministic = True -torch.backends.cudnn.benchmark = False -os.environ["PYTHONHASHSEED"] = str(42) -os.environ["CUBLAS_WORKSPACE_CONFIG"] = str(":4096:8") -torch.use_deterministic_algorithms(True) -#torch.set_default_dtype(torch.float16) -IGNORE_INDEX = -100 -torch.cuda.empty_cache() - - -device = "cpu" -if torch.cuda.is_available(): - device = torch.device("cuda") -parser = argparse.ArgumentParser(description="AtomGPT") -parser.add_argument( - "--benchmark_file", - default="AI-SinglePropertyPrediction-ead-tinnet_N-test-mae.csv.zip", - # default="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae", - help="Benchmarks available in jarvis_leaderboard/benchmarks/*/*.zip", -) - - -def get_crystal_string_old(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.1f}".format(x) for x in lengths]) - + "\n" - + " ".join([str(int(x)) for x in angles]) - + "\n" - + "\n".join( - [ - str(t) + "\n" + " ".join(["{0:.2f}".format(x) for x in c]) - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - - return crystal_str - - -def get_crystal_string(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.1f}".format(x) for x in lengths]) - + "\n" - + " ".join([str(int(x)) for x in angles]) - + "\n" - + "\n".join( - [ - str(t) + " " + " ".join(["{0:.1f}".format(x) for x in c]) - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - return crystal_str - - -# Define a custom dataset class for regression -class AtomGPTDataset(Dataset): - def __init__( - self, texts=[], targets=[], ids=[], tokenizer="", max_length=128 - ): - self.texts = texts - self.targets = targets - self.tokenizer = tokenizer - self.max_length = max_length - if not ids: - ids = ["text-" + str(i) for i in range(len(texts))] - self.ids = ids - - def __len__(self): - return len(self.texts) - - def __getitem__(self, idx): - inputs = self.tokenizer( - self.texts[idx], - return_tensors="pt", - max_length=self.max_length, - padding="max_length", - truncation=True, - ) - return ( - inputs, - self.ids[idx], - torch.tensor(self.targets[idx], dtype=torch.float32), - ) - - -# Example usage - - -def run_atomgpt( - prefix="ss", - model_name="gpt2", - benchmark_file="AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip", - root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard", - batch_size=8, - max_length=512, - num_epochs=500, - latent_dim=512, - learning_rate=1e-3, - # learning_rate=1e-3, - test_each_run=True, - # learning_rate=5e-5, -): - # Load pre-trained tokenizer - # tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - # model = GPT2LMHeadModel.from_pretrained("gpt2") - - # dft_3d = data("dft_3d") - # root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard" - # benchmark_file="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae.csv.zip" - # benchmark_file = "AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip" - print("benchmark_file", benchmark_file) - method = benchmark_file.split("-")[0] - task = benchmark_file.split("-")[1] - prop = benchmark_file.split("-")[2] - dataset = benchmark_file.split("-")[3] - temp = dataset + "_" + prop + ".json.zip" - temp2 = dataset + "_" + prop + ".json" - fname = os.path.join(root_dir, "benchmarks", method, task, temp) - zp = zipfile.ZipFile(fname) - bench = json.loads(zp.read(temp2)) - dft_3d = data(dataset) - id_tag = "jid" - if "jid" in dft_3d[0]: - id_tag = "jid" - else: - id_tag = "id" - - # train_atoms = [] - # val_atoms = [] - # test_atoms = [] - # train_targets = [] - # val_targets = [] - # test_targets = [] - train_ids = list(bench["train"].keys()) - val_ids = list(bench["val"].keys()) - test_ids = list(bench["test"].keys()) - - # model_name = "mistralai/Mistral-7B-Instruct-v0.1" - # model_name = "gpt2" - if "t5" in model_name: - model = transformers.T5ForConditionalGeneration.from_pretrained( - model_name - ) - else: - model = transformers.AutoModelForCausalLM.from_pretrained( - model_name, - low_cpu_mem_usage=True, - #load_in_8bit=False, - #torch_dtype=torch.float16, - #load_in_8bit=True, - # device_map="auto" - ) - # device = model.device - if "t5" in model_name: - tokenizer = transformers.T5Tokenizer.from_pretrained(model_name) - - else: - tokenizer = transformers.AutoTokenizer.from_pretrained(model_name) - - # batch_size = 16 - # max_length = 128 - # num_epochs = 100 - # learning_rate = 5e-5 - criterion = torch.nn.L1Loss() - # Define example regression data (texts and corresponding numeric targets) - train_texts = [] - train_targets = [] - train_ids_temp = [] - val_texts = [] - val_targets = [] - val_ids_temp = [] - test_texts = [] - test_targets = [] - test_ids_temp = [] - - for i in dft_3d: - if i[prop] != "na": - atoms = Atoms.from_dict(i["atoms"]) - tmp = get_crystal_string(atoms) - if i[id_tag] in train_ids: - train_texts.append(tmp) - train_targets.append(i[prop]) - train_ids_temp.append(i[id_tag]) - elif i[id_tag] in val_ids: - val_texts.append(tmp) - val_targets.append(i[prop]) - val_ids_temp.append(i[id_tag]) - elif i[id_tag] in test_ids: - test_texts.append(tmp) - test_targets.append(i[prop]) - test_ids_temp.append(i[id_tag]) - """ - ############################## - ###Fast test### - train_texts = [ - "This is the first example text.", - "Second example is a bit longer than the first one, but still within the max length.", - "Third example is the longest among these three examples. It exceeds the max length and will be truncated.", - "Second example is a bit longer than the first one, but still within the max length.", - ] - train_targets = [10.2, 15.5, 20.1, 15.5] # Example regression targets - val_texts = test_texts = train_texts - val_targets = test_targets = train_targets - train_ids_temp=['a','b','c','d'] - val_ids_temp = test_ids_temp = train_ids_temp - batch_size = 2 - num_epochs = 3 - - ############################## - ############################## - """ - - # Fine-tune the last layer of GPT-2 for regression - # fine_tune_gpt2_regression(train_texts, train_targets, tokenizer) - if tokenizer.pad_token is None: - tokenizer.add_special_tokens({"pad_token": "[PAD]"}) - model.resize_token_embeddings(len(tokenizer)) - model.lm_head = torch.nn.Sequential( - torch.nn.Linear(model.config.hidden_size, latent_dim), - # torch.nn.Linear(latent_dim, latent_dim), - torch.nn.Linear(latent_dim, latent_dim), - torch.nn.Linear(latent_dim, 1), - ) - # model.lm_head = torch.nn.Sequential(torch.nn.Linear( model.config.hidden_size, 256),torch.nn.SiLU(),torch.nn.Linear( 256, 1) ) - model.to(device) - if torch.cuda.device_count() > 1: - device_ids = [d for d in range(torch.cuda.device_count())] - model = torch.nn.DataParallel(model, device_ids=device_ids).cuda() - optimizer = transformers.AdamW(model.parameters(), lr=learning_rate) - # Prepare datasets and dataloaders with data collator - train_dataset = AtomGPTDataset( - texts=train_texts, - targets=train_targets, - ids=train_ids_temp, - tokenizer=tokenizer, - max_length=max_length, - ) - val_dataset = AtomGPTDataset( - texts=val_texts, - targets=val_targets, - tokenizer=tokenizer, - ids=val_ids_temp, - max_length=max_length, - ) - test_dataset = AtomGPTDataset( - texts=test_texts, - targets=test_targets, - tokenizer=tokenizer, - ids=test_ids_temp, - max_length=max_length, - ) - - # val_dataset = train_dataset - train_dataloader = DataLoader(train_dataset, batch_size=batch_size) - val_dataloader = DataLoader(val_dataset, batch_size=batch_size) - test_dataloader = DataLoader(test_dataset, batch_size=batch_size) - val_dataloader = test_dataloader - steps_per_epoch = len(train_dataloader) - scheduler = torch.optim.lr_scheduler.OneCycleLR( - optimizer, - max_lr=learning_rate, - epochs=num_epochs, - steps_per_epoch=steps_per_epoch, - # pct_start=pct_start, - pct_start=0.3, - ) - print("train_data", len(train_texts)) - print("test_data", len(test_texts)) - output_dir = prefix + "_out_" + model_name + "_" + dataset + "_" + prop - if not os.path.exists(output_dir): - os.makedirs(output_dir) - best_loss = np.inf - tot_time_start = time.time() - train_history = [] - val_history = [] - for epoch in range(num_epochs): - model.train() - t1 = time.time() - for batch in train_dataloader: - optimizer.zero_grad() - train_loss = 0 - # train_result = [] - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - # if 't5' in model_name: - # decoder_input_ids = tokenizer("", return_tensors="pt").input_ids.to(device) - # decoder_input_ids = model._shift_right(decoder_input_ids) - # predictions = ( - # model(input_ids = input_ids.to(device),decoder_input_ids=decoder_input_ids).logits.squeeze().mean(dim=-1) - # ) - # else: - # predictions = ( - # model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - # ) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - targets = batch[2].squeeze() - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - #print('train',predictions,targets) - loss.backward() - optimizer.step() - scheduler.step() - # optimizer.zero_grad() - train_loss += loss.item() - train_loss = train_loss / len(train_dataloader) - t2 = time.time() - train_time = round(t2 - t1, 3) - model.eval() - - # total_eval_mae_loss = 0 - # predictions_list = [] - # targets_list = [] - val_loss = 0 - t1 = time.time() - for batch in val_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - targets = batch[2].squeeze() - # print('val',predictions,targets) - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - val_loss += loss.item() - saving_tag = "" - if val_loss < best_loss: - best_loss = val_loss - best_model_name = "best_model.pt" - torch.save( - model.state_dict(), - os.path.join(output_dir, best_model_name), - ) - # print("Saving model for epoch", epoch) - saving_tag = " saving model:" + str(epoch) - val_loss = val_loss / len(val_dataloader) - t2 = time.time() - val_time = round(t2 - t1, 3) - train_history.append(train_loss) - val_history.append(val_loss) - history = os.path.join(output_dir, "history.json") - - dumpjson( - data={"train": train_history, "val": val_history}, filename=history - ) - mae = "" - if test_each_run: - t1_test = time.time() - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - t2_test = time.time() - test_time = round(t2_test - t1_test, 3) - f.close() - df = pd.read_csv(fname) - mae = mean_absolute_error(df["target"], df["predictions"]) - if mae == "": - print( - "Epoch, train loss, val loss, train_time, val_time", - epoch, - train_loss, - val_loss, - train_time, - val_time, - saving_tag, - ) - else: - print( - "Epoch, train loss, val loss, test loss, train_time, val_time, test_time", - epoch, - train_loss, - val_loss, - mae, - train_time, - val_time, - test_time, - saving_tag, - ) - - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - f.close() - tot_time_end = time.time() - tot_time = tot_time_end - tot_time_start - print("tot_time", tot_time) - - -if __name__ == "__main__": - args = parser.parse_args(sys.argv[1:]) - benchmark_file = args.benchmark_file - model_name="meta-llama/Llama-2-7b-hf", - model_name="google/flan-t5-small" - model_name="google/flan-t5-base" - model_name = "facebook/opt-350m" - model_name = "mistralai/Mixtral-8x7B-v0.1" - model_name = "mistralai/Mistral-7B-Instruct-v0.1" - model_name="gpt2" - run_atomgpt( - model_name=model_name, - benchmark_file=benchmark_file, - num_epochs=300, - batch_size=16 - ) diff --git a/atomgpt/scripts/finetune4.py b/atomgpt/scripts/finetune4.py deleted file mode 100644 index ab31042..0000000 --- a/atomgpt/scripts/finetune4.py +++ /dev/null @@ -1,564 +0,0 @@ -"""Module for fin tuning LLM model for materials chemsitry.""" - -from jarvis.db.figshare import data -import transformers -import torch -from jarvis.db.jsonutils import dumpjson -import sys -import argparse -from torch.utils.data import DataLoader, Dataset -import numpy as np -import os -from jarvis.core.atoms import Atoms -import pandas as pd -from sklearn.metrics import mean_absolute_error - -# from tqdm import tqdm -import time -import json -import zipfile - -os.environ["WANDB_ANONYMOUS"] = "must" -np.random.seed(42) -torch.manual_seed(42) -torch.cuda.manual_seed_all(42) -torch.backends.cudnn.deterministic = True -torch.backends.cudnn.benchmark = False -os.environ["PYTHONHASHSEED"] = str(42) -os.environ["CUBLAS_WORKSPACE_CONFIG"] = str(":4096:8") -torch.use_deterministic_algorithms(True) -#torch.set_default_dtype(torch.float16) -IGNORE_INDEX = -100 -torch.cuda.empty_cache() - - -device = "cpu" -if torch.cuda.is_available(): - device = torch.device("cuda") -parser = argparse.ArgumentParser(description="AtomGPT") -parser.add_argument( - "--benchmark_file", - default="AI-SinglePropertyPrediction-ead-tinnet_N-test-mae.csv.zip", - # default="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae", - help="Benchmarks available in jarvis_leaderboard/benchmarks/*/*.zip", -) - - - - -def get_crystal_string_1225(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.2f}".format(x) for x in lengths]) - + "\n" - +" ".join([str(int(x)) for x in angles]) - + "\n" - + "\n".join( - [ - str(t) +" "+" ".join(["{0:.2f}".format(x) for x in c]) - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - #extra=atoms.composition.reduced_formula - #crystal_str+=" "+extra - return crystal_str - -def get_crystal_string(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.2f}".format(x) for x in lengths]) - + "\n" - +" ".join([str(int(x)) for x in angles]) - + "\n" - + "\n".join( - [ - str(t) +" "+" ".join(["{0:.3f}".format(x) for x in c]) - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - extra=atoms.composition.reduced_formula - crystal_str+="\n"+extra - return crystal_str - - -# Define a custom dataset class for regression -class AtomGPTDataset(Dataset): - def __init__( - self, texts=[], targets=[], ids=[], tokenizer="", max_length=128 - ): - self.texts = texts - self.targets = targets - self.tokenizer = tokenizer - self.max_length = max_length - if not ids: - ids = ["text-" + str(i) for i in range(len(texts))] - self.ids = ids - - def __len__(self): - return len(self.texts) - - def __getitem__(self, idx): - inputs = self.tokenizer( - self.texts[idx], - return_tensors="pt", - max_length=self.max_length, - padding="max_length", - truncation=True, - ) - return ( - inputs, - self.ids[idx], - torch.tensor(self.targets[idx], dtype=torch.float32), - ) - - -# Example usage - - -def run_atomgpt( - prefix="ss", - model_name="gpt2", - benchmark_file="AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip", - root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard", - batch_size=8, - max_length=512, - num_epochs=500, - latent_dim=1024, - learning_rate=1e-3, - # learning_rate=1e-3, - test_each_run=True, - #learning_rate=5e-5, - pretrained_path="", -): - # Load pre-trained tokenizer - # tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - # model = GPT2LMHeadModel.from_pretrained("gpt2") - - # dft_3d = data("dft_3d") - # root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard" - # benchmark_file="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae.csv.zip" - # benchmark_file = "AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip" - print("benchmark_file", benchmark_file) - method = benchmark_file.split("-")[0] - task = benchmark_file.split("-")[1] - prop = benchmark_file.split("-")[2] - dataset = benchmark_file.split("-")[3] - temp = dataset + "_" + prop + ".json.zip" - temp2 = dataset + "_" + prop + ".json" - fname = os.path.join(root_dir, "benchmarks", method, task, temp) - zp = zipfile.ZipFile(fname) - bench = json.loads(zp.read(temp2)) - dft_3d = data(dataset) - id_tag = "jid" - if "jid" in dft_3d[0]: - id_tag = "jid" - else: - id_tag = "id" - - # train_atoms = [] - # val_atoms = [] - # test_atoms = [] - # train_targets = [] - # val_targets = [] - # test_targets = [] - train_ids = list(bench["train"].keys()) - val_ids = list(bench["val"].keys()) - test_ids = list(bench["test"].keys()) - - # model_name = "mistralai/Mistral-7B-Instruct-v0.1" - # model_name = "gpt2" - if "t5" in model_name: - model = transformers.T5ForConditionalGeneration.from_pretrained( - model_name - ) - else: - model = transformers.AutoModelForCausalLM.from_pretrained( - model_name, - low_cpu_mem_usage=True, - #load_in_8bit=False, - #torch_dtype=torch.float16, - #load_in_8bit=True, - # device_map="auto" - ) - # device = model.device - if "t5" in model_name: - tokenizer = transformers.T5Tokenizer.from_pretrained(model_name) - - else: - tokenizer = transformers.AutoTokenizer.from_pretrained(model_name) - - # batch_size = 16 - # max_length = 128 - # num_epochs = 100 - # learning_rate = 5e-5 - criterion = torch.nn.L1Loss() - # Define example regression data (texts and corresponding numeric targets) - train_texts = [] - train_targets = [] - train_ids_temp = [] - val_texts = [] - val_targets = [] - val_ids_temp = [] - test_texts = [] - test_targets = [] - test_ids_temp = [] - - for i in dft_3d: - if i[prop] != "na": - atoms = Atoms.from_dict(i["atoms"]) - tmp = get_crystal_string(atoms) - if i[id_tag] in train_ids: - train_texts.append(tmp) - train_targets.append(i[prop]) - train_ids_temp.append(i[id_tag]) - elif i[id_tag] in val_ids: - val_texts.append(tmp) - val_targets.append(i[prop]) - val_ids_temp.append(i[id_tag]) - elif i[id_tag] in test_ids: - test_texts.append(tmp) - test_targets.append(i[prop]) - test_ids_temp.append(i[id_tag]) - print("test_texts:",test_texts[0]) - """ - ############################## - ###Fast test### - train_texts = [ - "This is the first example text.", - "Second example is a bit longer than the first one, but still within the max length.", - "Third example is the longest among these three examples. It exceeds the max length and will be truncated.", - "Second example is a bit longer than the first one, but still within the max length.", - ] - train_targets = [10.2, 15.5, 20.1, 15.5] # Example regression targets - val_texts = test_texts = train_texts - val_targets = test_targets = train_targets - train_ids_temp=['a','b','c','d'] - val_ids_temp = test_ids_temp = train_ids_temp - batch_size = 2 - num_epochs = 3 - - ############################## - ############################## - """ - - # Fine-tune the last layer of GPT-2 for regression - # fine_tune_gpt2_regression(train_texts, train_targets, tokenizer) - if tokenizer.pad_token is None: - tokenizer.add_special_tokens({"pad_token": "[PAD]"}) - tokenizer.add_special_tokens({"unk_token": ""}) - model.resize_token_embeddings(len(tokenizer)) - model.lm_head = torch.nn.Sequential( - torch.nn.Linear(model.config.hidden_size, latent_dim), - # torch.nn.Linear(latent_dim, latent_dim), - torch.nn.Linear(latent_dim, latent_dim), - torch.nn.Linear(latent_dim, 1), - ) - if pretrained_path!="": - model.load_state_dict(torch.load(pretrained_path,map_location=device)) - # model.lm_head = torch.nn.Sequential(torch.nn.Linear( model.config.hidden_size, 256),torch.nn.SiLU(),torch.nn.Linear( 256, 1) ) - model.to(device) - if torch.cuda.device_count() > 1: - device_ids = [d for d in range(torch.cuda.device_count())] - model = torch.nn.DataParallel(model, device_ids=device_ids).cuda() - optimizer = transformers.AdamW(model.parameters(), lr=learning_rate) - # Prepare datasets and dataloaders with data collator - train_dataset = AtomGPTDataset( - texts=train_texts, - targets=train_targets, - ids=train_ids_temp, - tokenizer=tokenizer, - max_length=max_length, - ) - val_dataset = AtomGPTDataset( - texts=val_texts, - targets=val_targets, - tokenizer=tokenizer, - ids=val_ids_temp, - max_length=max_length, - ) - test_dataset = AtomGPTDataset( - texts=test_texts, - targets=test_targets, - tokenizer=tokenizer, - ids=test_ids_temp, - max_length=max_length, - ) - - # val_dataset = train_dataset - train_dataloader = DataLoader(train_dataset, batch_size=batch_size) - val_dataloader = DataLoader(val_dataset, batch_size=batch_size) - test_dataloader = DataLoader(test_dataset, batch_size=batch_size) - val_dataloader = test_dataloader - steps_per_epoch = len(train_dataloader) - scheduler = torch.optim.lr_scheduler.OneCycleLR( - optimizer, - max_lr=learning_rate, - epochs=num_epochs, - steps_per_epoch=steps_per_epoch, - # pct_start=pct_start, - pct_start=0.3, - ) - #scheduler = torch.optim.lr_scheduler.StepLR( - # optimizer, - # step_size=30, - # ) - print("train_data", len(train_texts)) - print("test_data", len(test_texts)) - output_dir = prefix + "_out_" + model_name + "_" + dataset + "_" + prop - if not os.path.exists(output_dir): - os.makedirs(output_dir) - best_loss = np.inf - tot_time_start = time.time() - train_history = [] - val_history = [] - for epoch in range(num_epochs): - model.train() - t1 = time.time() - for batch in train_dataloader: - optimizer.zero_grad() - train_loss = 0 - # train_result = [] - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - # if 't5' in model_name: - # decoder_input_ids = tokenizer("", return_tensors="pt").input_ids.to(device) - # decoder_input_ids = model._shift_right(decoder_input_ids) - # predictions = ( - # model(input_ids = input_ids.to(device),decoder_input_ids=decoder_input_ids).logits.squeeze().mean(dim=-1) - # ) - # else: - # predictions = ( - # model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - # ) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - targets = batch[2].squeeze() - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - #print('train',predictions,targets) - loss.backward() - optimizer.step() - scheduler.step() - # optimizer.zero_grad() - train_loss += loss.item() - train_loss = train_loss / len(train_dataloader) - t2 = time.time() - train_time = round(t2 - t1, 3) - model.eval() - - # total_eval_mae_loss = 0 - # predictions_list = [] - # targets_list = [] - val_loss = 0 - t1 = time.time() - for batch in val_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - targets = batch[2].squeeze() - # print('val',predictions,targets) - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - val_loss += loss.item() - saving_tag = "" - if val_loss < best_loss: - best_loss = val_loss - best_model_name = "best_model.pt" - torch.save( - model.state_dict(), - os.path.join(output_dir, best_model_name), - ) - # print("Saving model for epoch", epoch) - saving_tag = " saving model:" + str(epoch) - val_loss = val_loss / len(val_dataloader) - t2 = time.time() - val_time = round(t2 - t1, 3) - train_history.append(train_loss) - val_history.append(val_loss) - history = os.path.join(output_dir, "history.json") - - dumpjson( - data={"train": train_history, "val": val_history}, filename=history - ) - mae = "" - if test_each_run: - t1_test = time.time() - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - t2_test = time.time() - test_time = round(t2_test - t1_test, 3) - f.close() - df = pd.read_csv(fname) - mae = mean_absolute_error(df["target"], df["predictions"]) - if mae == "": - print( - "Epoch, train loss, val loss, train_time, val_time", - epoch, - train_loss, - val_loss, - train_time, - val_time, - saving_tag, - ) - else: - print( - "Epoch, train loss, val loss, test loss, train_time, val_time, test_time", - epoch, - train_loss, - val_loss, - mae, - train_time, - val_time, - test_time, - saving_tag, - ) - - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - f.close() - tot_time_end = time.time() - tot_time = tot_time_end - tot_time_start - print("tot_time", tot_time) - - -if __name__ == "__main__": - args = parser.parse_args(sys.argv[1:]) - benchmark_file = args.benchmark_file - model_name="meta-llama/Llama-2-7b-hf", - model_name="google/flan-t5-base" - model_name = "facebook/opt-350m" - model_name = "mistralai/Mixtral-8x7B-v0.1" - model_name = "mistralai/Mistral-7B-Instruct-v0.1" - model_name="google/flan-t5-small" - model_name="google-t5/t5-small" - model_name="gpt2" - run_atomgpt( - model_name=model_name, - benchmark_file=benchmark_file, - #num_epochs=300, - #pretrained_path="xyz_out_google/flan-t5-small_tinnet_N_ead/best_model.pt", - #pretrained_path="ss_out_google/flan-t5-small_tinnet_N_ead/best_model.pt", - prefix="xyz1", - batch_size=16 - ) diff --git a/atomgpt/scripts/finetune5.py b/atomgpt/scripts/finetune5.py deleted file mode 100644 index bfa7d28..0000000 --- a/atomgpt/scripts/finetune5.py +++ /dev/null @@ -1,644 +0,0 @@ -"""Module for fin tuning LLM model for materials chemsitry.""" - -from jarvis.db.figshare import data -import transformers -import torch -from jarvis.db.jsonutils import dumpjson -import sys -import argparse -from torch.utils.data import DataLoader, Dataset -import numpy as np -import os -from jarvis.core.atoms import Atoms -import pandas as pd -from sklearn.metrics import mean_absolute_error -from describe import atoms_describer -# from tqdm import tqdm -import time -import json -import zipfile - - -### -#from robocrys import StructureCondenser, StructureDescriber - - -def get_robo(structure=None): -#structure = Structure.from_file("POSCAR") # other file formats also supported - -# alternatively, uncomment the lines below to use the MPRester object -# to fetch structures from the Materials Project database -# from pymatgen import MPRester -# structure = MPRester(API_KEY=None).get_structure_by_material_id("mp-856") - - condenser = StructureCondenser() - describer = StructureDescriber() - - #condensed_structure = condenser.condense_structure(structure) - #description = describer.describe(condensed_structure) - description = describer.describe(structure) - print(description) - return description -## -os.environ["WANDB_ANONYMOUS"] = "must" -np.random.seed(42) -torch.manual_seed(42) -torch.cuda.manual_seed_all(42) -torch.backends.cudnn.deterministic = True -torch.backends.cudnn.benchmark = False -os.environ["PYTHONHASHSEED"] = str(42) -os.environ["CUBLAS_WORKSPACE_CONFIG"] = str(":4096:8") -torch.use_deterministic_algorithms(True) -#torch.set_default_dtype(torch.float16) -IGNORE_INDEX = -100 -torch.cuda.empty_cache() - -device = "cpu" -if torch.cuda.is_available(): - device = torch.device("cuda") -scale=torch.tensor(100) #.to(device) -parser = argparse.ArgumentParser(description="AtomGPT") -parser.add_argument( - "--benchmark_file", - default="AI-SinglePropertyPrediction-PBE_gap-halide_peroskites-test-mae.csv.zip", - #default="AI-SinglePropertyPrediction-ead-tinnet_N-test-mae.csv.zip", - # default="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae", - help="Benchmarks available in jarvis_leaderboard/benchmarks/*/*.zip", -) - - - - -def get_crystal_string_1225(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.1f}".format(x) for x in lengths]) - + "\n" - +" ".join([str(int(x)) for x in angles]) - + "\n" - + "\n".join( - [ - str(t) +" "+" ".join(["{0:.2f}".format(x) for x in c]) - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - extra=str(atoms.num_atoms)+"\n"+atoms.composition.reduced_formula - #crystal_str+=" "+extra - extra+="\n"+crystal_str - return extra - #return crystal_str - -def get_crystal_string(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.2f}".format(x) for x in lengths]) - + "\n" - +" ".join([str(int(x)) for x in angles]) - + "\n" - + "\n".join( - [ - str(t) +" "+" ".join(["{0:.3f}".format(x) for x in c]) - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - #extra=str(atoms.num_atoms)+"\n"+atoms.composition.reduced_formula - #crystal_str+=" "+extra - #extra+="\n"+crystal_str - #return extra - #extra=atoms.composition.reduced_formula - #crystal_str+="\n"+extra+"\n"+atoms.spacegroup()+"\n" - #return crystal_str -def get_crystal_string_t(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.2f}".format(x) for x in lengths]) - + "#\n" - +" ".join([str(int(x)) for x in angles]) - + "@\n" - + "\n".join( - [ - str(t) +" "+" ".join(["{0:.3f}".format(x) for x in c])+"&" - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - #extra=str(atoms.num_atoms)+"\n"+atoms.composition.reduced_formula - #crystal_str+=" "+extra - #extra+="\n"+crystal_str - #return extra - #extra=atoms.composition.reduced_formula - #crystal_str+="\n"+extra+"\n"+atoms.spacegroup()+"\n" - crystal_str = atoms_describer(atoms)+"\n*\n"+crystal_str - return crystal_str - -# Define a custom dataset class for regression -class AtomGPTDataset(Dataset): - def __init__( - self, texts=[], targets=[], ids=[], tokenizer="", max_length=128 - ): - self.texts = texts - self.targets = targets - self.tokenizer = tokenizer - self.max_length = max_length - if not ids: - ids = ["text-" + str(i) for i in range(len(texts))] - self.ids = ids - - def __len__(self): - return len(self.texts) - - def __getitem__(self, idx): - inputs = self.tokenizer( - self.texts[idx], - return_tensors="pt", - max_length=self.max_length, - padding="max_length", - truncation=True, - ) - #torch.tensor(inputs*10,dtype=inputs.dtype) - return ( - inputs, - self.ids[idx], - torch.tensor(self.targets[idx], dtype=torch.float32), - ) - - -# Example usage - - -def run_atomgpt( - prefix="ss", - model_name="gpt2", - benchmark_file="AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip", - root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard", - batch_size=8, - max_length=512, - num_epochs=500, - latent_dim=512, - learning_rate=1e-3, - # learning_rate=1e-3, - test_each_run=True, - #learning_rate=5e-5, - pretrained_path="", -): - # Load pre-trained tokenizer - # tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - # model = GPT2LMHeadModel.from_pretrained("gpt2") - - # dft_3d = data("dft_3d") - # root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard" - # benchmark_file="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae.csv.zip" - # benchmark_file = "AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip" - print("benchmark_file", benchmark_file) - method = benchmark_file.split("-")[0] - task = benchmark_file.split("-")[1] - prop = benchmark_file.split("-")[2] - dataset = benchmark_file.split("-")[3] - temp = dataset + "_" + prop + ".json.zip" - temp2 = dataset + "_" + prop + ".json" - fname = os.path.join(root_dir, "benchmarks", method, task, temp) - zp = zipfile.ZipFile(fname) - bench = json.loads(zp.read(temp2)) - dft_3d = data(dataset) - id_tag = "jid" - if "jid" in dft_3d[0]: - id_tag = "jid" - else: - id_tag = "id" - - # train_atoms = [] - # val_atoms = [] - # test_atoms = [] - # train_targets = [] - # val_targets = [] - # test_targets = [] - train_ids = list(bench["train"].keys()) - test_ids = list(bench["test"].keys()) - if 'val' in bench: - val_ids = list(bench["val"].keys()) - else: - val_ids = test_ids - print("total",len(dft_3d)) - print("test_ids",len(test_ids)) - print("val_ids",len(val_ids)) - print("train_ids",len(train_ids)) - # model_name = "mistralai/Mistral-7B-Instruct-v0.1" - # model_name = "gpt2" - if "t5" in model_name: - model = transformers.T5ForConditionalGeneration.from_pretrained( - model_name - ) - else: - model = transformers.AutoModelForCausalLM.from_pretrained( - model_name, - low_cpu_mem_usage=True, - #load_in_8bit=False, - #torch_dtype=torch.float16, - #load_in_8bit=True, - # device_map="auto" - ) - # device = model.device - if "t5" in model_name: - tokenizer = transformers.T5Tokenizer.from_pretrained(model_name) - - else: - tokenizer = transformers.AutoTokenizer.from_pretrained(model_name) - - # batch_size = 16 - # max_length = 128 - # num_epochs = 100 - # learning_rate = 5e-5 - criterion = torch.nn.L1Loss() - # Define example regression data (texts and corresponding numeric targets) - train_texts = [] - train_targets = [] - train_ids_temp = [] - val_texts = [] - val_targets = [] - val_ids_temp = [] - test_texts = [] - test_targets = [] - test_ids_temp = [] - - for i in dft_3d: - if i[prop] != "na": - atoms = Atoms.from_dict(i["atoms"]) - tmp = get_crystal_string_t(atoms) - if i[id_tag] in train_ids: - train_texts.append(tmp) - train_targets.append(i[prop]) - train_ids_temp.append(i[id_tag]) - elif i[id_tag] in test_ids: - test_texts.append(tmp) - test_targets.append(i[prop]) - test_ids_temp.append(i[id_tag]) - elif i[id_tag] in val_ids: - val_texts.append(tmp) - val_targets.append(i[prop]) - val_ids_temp.append(i[id_tag]) - print("test_texts:",len(test_texts)) - print("test_texts:",test_texts[0]) - """ - ############################## - ###Fast test### - train_texts = [ - "This is the first example text.", - "Second example is a bit longer than the first one, but still within the max length.", - "Third example is the longest among these three examples. It exceeds the max length and will be truncated.", - "Second example is a bit longer than the first one, but still within the max length.", - ] - train_targets = [10.2, 15.5, 20.1, 15.5] # Example regression targets - val_texts = test_texts = train_texts - val_targets = test_targets = train_targets - train_ids_temp=['a','b','c','d'] - val_ids_temp = test_ids_temp = train_ids_temp - batch_size = 2 - num_epochs = 3 - - ############################## - ############################## - """ - - # Fine-tune the last layer of GPT-2 for regression - # fine_tune_gpt2_regression(train_texts, train_targets, tokenizer) - if tokenizer.pad_token is None: - tokenizer.add_special_tokens({"pad_token": "[PAD]"}) - tokenizer.add_special_tokens({"unk_token": "#"}) - tokenizer.add_special_tokens({"unk_token": "&"}) - tokenizer.add_special_tokens({"unk_token": "@"}) - model.resize_token_embeddings(len(tokenizer)) - model.lm_head = torch.nn.Sequential( - #torch.nn.Linear(model.config.hidden_size, 1), - torch.nn.Linear(model.config.hidden_size, latent_dim), - #torch.nn.Transformer(d_model=latent_dim, nhead=1, num_encoder_layers=1, num_decoder_layers=1), - # torch.nn.Linear(latent_dim, latent_dim), - #torch.nn.Linear(latent_dim, latent_dim), - #torch.nn.ReLU(), - #torch.nn.LeakyReLU(), - #torch.nn.Dropout(p=0.2), - #torch.nn.TransformerEncoder(torch.nn.TransformerEncoderLayer(d_model=latent_dim, nhead=4), num_layers=2), - torch.nn.Linear(latent_dim, 1), - ) - if pretrained_path!="": - model.load_state_dict(torch.load(pretrained_path,map_location=device)) - # model.lm_head = torch.nn.Sequential(torch.nn.Linear( model.config.hidden_size, 256),torch.nn.SiLU(),torch.nn.Linear( 256, 1) ) - model.to(device) - if torch.cuda.device_count() > 1: - device_ids = [d for d in range(torch.cuda.device_count())] - model = torch.nn.DataParallel(model, device_ids=device_ids).cuda() - optimizer = transformers.AdamW(model.parameters(), lr=learning_rate) - # Prepare datasets and dataloaders with data collator - train_dataset = AtomGPTDataset( - texts=train_texts, - targets=train_targets, - ids=train_ids_temp, - tokenizer=tokenizer, - max_length=max_length, - ) - val_dataset = AtomGPTDataset( - texts=val_texts, - targets=val_targets, - tokenizer=tokenizer, - ids=val_ids_temp, - max_length=max_length, - ) - test_dataset = AtomGPTDataset( - texts=test_texts, - targets=test_targets, - tokenizer=tokenizer, - ids=test_ids_temp, - max_length=max_length, - ) - - # val_dataset = train_dataset - train_dataloader = DataLoader(train_dataset, batch_size=batch_size) - val_dataloader = DataLoader(val_dataset, batch_size=batch_size) - test_dataloader = DataLoader(test_dataset, batch_size=batch_size) - val_dataloader = test_dataloader - steps_per_epoch = len(train_dataloader) - scheduler = torch.optim.lr_scheduler.OneCycleLR( - optimizer, - max_lr=learning_rate, - epochs=num_epochs, - steps_per_epoch=steps_per_epoch, - # pct_start=pct_start, - pct_start=0.3, - ) - #scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10, eta_min=0.001) - #scheduler = torch.optim.lr_scheduler.StepLR( - # optimizer, - # step_size=30, - # ) - print("train_data", len(train_texts)) - print("test_data", len(test_texts)) - output_dir = prefix + "_out_" + model_name + "_" + dataset + "_" + prop - if not os.path.exists(output_dir): - os.makedirs(output_dir) - best_loss = np.inf - tot_time_start = time.time() - train_history = [] - val_history = [] - for epoch in range(num_epochs): - model.train() - t1 = time.time() - for batch in train_dataloader: - optimizer.zero_grad() - train_loss = 0 - # train_result = [] - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - # if 't5' in model_name: - # decoder_input_ids = tokenizer("", return_tensors="pt").input_ids.to(device) - # decoder_input_ids = model._shift_right(decoder_input_ids) - # predictions = ( - # model(input_ids = input_ids.to(device),decoder_input_ids=decoder_input_ids).logits.squeeze().mean(dim=-1) - # ) - # else: - # predictions = ( - # model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - # ) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - targets = batch[2].squeeze() - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - #print('train',predictions,targets) - loss.backward() - optimizer.step() - scheduler.step() - # optimizer.zero_grad() - train_loss += loss.item() - train_loss = train_loss / len(train_dataloader) - t2 = time.time() - train_time = round(t2 - t1, 3) - model.eval() - - # total_eval_mae_loss = 0 - # predictions_list = [] - # targets_list = [] - val_loss = 0 - t1 = time.time() - for batch in val_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - targets = batch[2].squeeze() - # print('val',predictions,targets) - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - val_loss += loss.item() - saving_tag = "" - if val_loss < best_loss: - best_loss = val_loss - best_model_name = "best_model.pt" - torch.save( - model.state_dict(), - os.path.join(output_dir, best_model_name), - ) - # print("Saving model for epoch", epoch) - saving_tag = " saving model:" + str(epoch) - val_loss = val_loss / len(val_dataloader) - t2 = time.time() - val_time = round(t2 - t1, 3) - train_history.append(train_loss) - val_history.append(val_loss) - history = os.path.join(output_dir, "history.json") - - dumpjson( - data={"train": train_history, "val": val_history}, filename=history - ) - mae = "" - if test_each_run: - t1_test = time.time() - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - t2_test = time.time() - test_time = round(t2_test - t1_test, 3) - f.close() - df = pd.read_csv(fname) - mae = mean_absolute_error(df["target"], df["predictions"]) - if mae == "": - print( - "Epoch, train loss, val loss, train_time, val_time", - epoch, - train_loss, - val_loss, - train_time, - val_time, - saving_tag, - ) - else: - print( - "Epoch, train loss, val loss, test loss, train_time, val_time, test_time", - epoch, - train_loss, - val_loss, - mae, - train_time, - val_time, - test_time, - saving_tag, - ) - - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - f.close() - tot_time_end = time.time() - tot_time = tot_time_end - tot_time_start - print("tot_time", tot_time) - - -if __name__ == "__main__": - args = parser.parse_args(sys.argv[1:]) - benchmark_file = args.benchmark_file - model_name = "facebook/opt-350m" - model_name = "mistralai/Mixtral-8x7B-v0.1" - model_name="google/flan-t5-small" - model_name="google/flan-t5-base" - model_name = "mistralai/Mistral-7B-Instruct-v0.1" - model_name="google-t5/t5-small" - model_name="xlnet/xlnet-base-cased" - model_name="afmck/testing-llama-tiny" - model_name="EleutherAI/gpt-neo-125m" - model_name="openai-community/gpt2-medium" - model_name="meta-llama/Llama-2-7b-hf" - model_name="stas/tiny-random-llama-2" - model_name="ahxt/llama2_xs_460M_experimental" - model_name="gpt2" - run_atomgpt( - model_name=model_name, - benchmark_file=benchmark_file, - #num_epochs=300, - #pretrained_path="xyz_out_google/flan-t5-small_tinnet_N_ead/best_model.pt", - #pretrained_path="ss_out_google/flan-t5-small_tinnet_N_ead/best_model.pt", - prefix="xyzt", - batch_size=16, - latent_dim=1024, - num_epochs=5000, - #batch_size=16 - ) diff --git a/atomgpt/scripts/finetune6.py b/atomgpt/scripts/finetune6.py deleted file mode 100644 index 6323ac1..0000000 --- a/atomgpt/scripts/finetune6.py +++ /dev/null @@ -1,659 +0,0 @@ -"""Module for fin tuning LLM model for materials chemsitry.""" - -from jarvis.db.figshare import data -import transformers -import torch -import random -from jarvis.db.jsonutils import dumpjson -import sys -import argparse -from torch.utils.data import DataLoader, Dataset -import numpy as np -import os -from jarvis.core.atoms import Atoms -import pandas as pd -from sklearn.metrics import mean_absolute_error -from describe import atoms_describer - -# from tqdm import tqdm -import time -import json -import zipfile - - -### -# from robocrys import StructureCondenser, StructureDescriber - - -def get_robo(structure=None): - # structure = Structure.from_file("POSCAR") # other file formats also supported - - # alternatively, uncomment the lines below to use the MPRester object - # to fetch structures from the Materials Project database - # from pymatgen import MPRester - # structure = MPRester(API_KEY=None).get_structure_by_material_id("mp-856") - - condenser = StructureCondenser() - describer = StructureDescriber() - - # condensed_structure = condenser.condense_structure(structure) - # description = describer.describe(condensed_structure) - description = describer.describe(structure) - print(description) - return description - - -## -os.environ["WANDB_ANONYMOUS"] = "must" -random_seed = 42 -random.seed(random_seed) -torch.manual_seed(random_seed) -np.random.seed(random_seed) -torch.cuda.manual_seed_all(random_seed) -try: - import torch_xla.core.xla_model as xm - - xm.set_rng_state(random_seed) -except ImportError: - pass -torch.backends.cudnn.deterministic = True -torch.backends.cudnn.benchmark = False -os.environ["PYTHONHASHSEED"] = str(random_seed) -os.environ["CUBLAS_WORKSPACE_CONFIG"] = str(":4096:8") -torch.use_deterministic_algorithms(True) -IGNORE_INDEX = -100 -# torch.cuda.empty_cache() -device = "cpu" -if torch.cuda.is_available(): - device = torch.device("cuda") -parser = argparse.ArgumentParser(description="AtomGPT") -parser.add_argument( - "--benchmark_file", - default="AI-SinglePropertyPrediction-PBE_gap-halide_peroskites-test-mae.csv.zip", - # default="AI-SinglePropertyPrediction-ead-tinnet_N-test-mae.csv.zip", - # default="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae", - help="Benchmarks available in jarvis_leaderboard/benchmarks/*/*.zip", -) - - -def get_crystal_string_1225(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.1f}".format(x) for x in lengths]) - + "\n" - + " ".join([str(int(x)) for x in angles]) - + "\n" - + "\n".join( - [ - str(t) + " " + " ".join(["{0:.2f}".format(x) for x in c]) - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - extra = str(atoms.num_atoms) + "\n" + atoms.composition.reduced_formula - # crystal_str+=" "+extra - extra += "\n" + crystal_str - return extra - # return crystal_str - - -def get_crystal_string(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.2f}".format(x) for x in lengths]) - + "\n" - + " ".join([str(int(x)) for x in angles]) - + "\n" - + "\n".join( - [ - str(t) + " " + " ".join(["{0:.3f}".format(x) for x in c]) - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - # extra=str(atoms.num_atoms)+"\n"+atoms.composition.reduced_formula - # crystal_str+=" "+extra - # extra+="\n"+crystal_str - # return extra - # extra=atoms.composition.reduced_formula - # crystal_str+="\n"+extra+"\n"+atoms.spacegroup()+"\n" - # return crystal_str - - -def get_crystal_string_t(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.2f}".format(x) for x in lengths]) - + "#\n" - + " ".join([str(int(x)) for x in angles]) - + "@\n" - + "\n".join( - [ - str(t) + " " + " ".join(["{0:.3f}".format(x) for x in c]) + "&" - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - # extra=str(atoms.num_atoms)+"\n"+atoms.composition.reduced_formula - # crystal_str+=" "+extra - # extra+="\n"+crystal_str - # return extra - # extra=atoms.composition.reduced_formula - # crystal_str+="\n"+extra+"\n"+atoms.spacegroup()+"\n" - crystal_str = atoms_describer(atoms) + "\n*\n" + crystal_str - return crystal_str - - -# Define a custom dataset class for regression -class AtomGPTDataset(Dataset): - def __init__( - self, texts=[], targets=[], ids=[], tokenizer="", max_length=128 - ): - self.texts = texts - self.targets = targets - self.tokenizer = tokenizer - self.max_length = max_length - if not ids: - ids = ["text-" + str(i) for i in range(len(texts))] - self.ids = ids - - def __len__(self): - return len(self.texts) - - def __getitem__(self, idx): - inputs = self.tokenizer( - self.texts[idx], - return_tensors="pt", - max_length=self.max_length, - padding="max_length", - truncation=True, - ) - # torch.tensor(inputs*10,dtype=inputs.dtype) - return ( - inputs, - self.ids[idx], - torch.tensor(self.targets[idx], dtype=torch.float32), - ) - - -# Example usage - - -def run_atomgpt( - prefix="ss", - model_name="gpt2", - benchmark_file="AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip", - root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard", - batch_size=8, - max_length=512, - num_epochs=500, - latent_dim=512, - learning_rate=1e-3, - # learning_rate=1e-3, - test_each_run=True, - # learning_rate=5e-5, - pretrained_path="", -): - # Load pre-trained tokenizer - # tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - # model = GPT2LMHeadModel.from_pretrained("gpt2") - - # dft_3d = data("dft_3d") - # root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard" - # benchmark_file="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae.csv.zip" - # benchmark_file = "AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip" - print("benchmark_file", benchmark_file) - method = benchmark_file.split("-")[0] - task = benchmark_file.split("-")[1] - prop = benchmark_file.split("-")[2] - dataset = benchmark_file.split("-")[3] - temp = dataset + "_" + prop + ".json.zip" - temp2 = dataset + "_" + prop + ".json" - fname = os.path.join(root_dir, "benchmarks", method, task, temp) - zp = zipfile.ZipFile(fname) - bench = json.loads(zp.read(temp2)) - dft_3d = data(dataset) - id_tag = "jid" - if "jid" in dft_3d[0]: - id_tag = "jid" - else: - id_tag = "id" - - # train_atoms = [] - # val_atoms = [] - # test_atoms = [] - # train_targets = [] - # val_targets = [] - # test_targets = [] - train_ids = list(bench["train"].keys()) - test_ids = list(bench["test"].keys()) - if "val" in bench: - val_ids = list(bench["val"].keys()) - else: - val_ids = test_ids - print("total", len(dft_3d)) - print("test_ids", len(test_ids)) - print("val_ids", len(val_ids)) - print("train_ids", len(train_ids)) - # model_name = "mistralai/Mistral-7B-Instruct-v0.1" - # model_name = "gpt2" - if "t5" in model_name: - model = transformers.T5ForConditionalGeneration.from_pretrained( - model_name - ) - else: - model = transformers.AutoModelForCausalLM.from_pretrained( - model_name, - low_cpu_mem_usage=True, - # load_in_8bit=False, - # torch_dtype=torch.float16, - # load_in_8bit=True, - # device_map="auto" - ) - # device = model.device - if "t5" in model_name: - tokenizer = transformers.T5Tokenizer.from_pretrained(model_name) - - else: - tokenizer = transformers.AutoTokenizer.from_pretrained(model_name) - - # batch_size = 16 - # max_length = 128 - # num_epochs = 100 - # learning_rate = 5e-5 - criterion = torch.nn.L1Loss() - # Define example regression data (texts and corresponding numeric targets) - train_texts = [] - train_targets = [] - train_ids_temp = [] - val_texts = [] - val_targets = [] - val_ids_temp = [] - test_texts = [] - test_targets = [] - test_ids_temp = [] - - for i in dft_3d: - if i[prop] != "na": - atoms = Atoms.from_dict(i["atoms"]) - tmp = get_crystal_string_t(atoms) - if i[id_tag] in train_ids: - train_texts.append(tmp) - train_targets.append(i[prop]) - train_ids_temp.append(i[id_tag]) - elif i[id_tag] in test_ids: - test_texts.append(tmp) - test_targets.append(i[prop]) - test_ids_temp.append(i[id_tag]) - elif i[id_tag] in val_ids: - val_texts.append(tmp) - val_targets.append(i[prop]) - val_ids_temp.append(i[id_tag]) - print("test_texts:", len(test_texts)) - print("test_texts:", test_texts[0]) - """ - ############################## - ###Fast test### - train_texts = [ - "This is the first example text.", - "Second example is a bit longer than the first one, but still within the max length.", - "Third example is the longest among these three examples. It exceeds the max length and will be truncated.", - "Second example is a bit longer than the first one, but still within the max length.", - ] - train_targets = [10.2, 15.5, 20.1, 15.5] # Example regression targets - val_texts = test_texts = train_texts - val_targets = test_targets = train_targets - train_ids_temp=['a','b','c','d'] - val_ids_temp = test_ids_temp = train_ids_temp - batch_size = 2 - num_epochs = 3 - - ############################## - ############################## - """ - - # Fine-tune the last layer of GPT-2 for regression - # fine_tune_gpt2_regression(train_texts, train_targets, tokenizer) - if tokenizer.pad_token is None: - tokenizer.add_special_tokens({"pad_token": "[PAD]"}) - tokenizer.add_special_tokens({"unk_token": "#"}) - tokenizer.add_special_tokens({"unk_token": "&"}) - tokenizer.add_special_tokens({"unk_token": "@"}) - model.resize_token_embeddings(len(tokenizer)) - model.lm_head = torch.nn.Sequential( - # torch.nn.Linear(model.config.hidden_size, 1), - torch.nn.Linear(model.config.hidden_size, latent_dim), - # torch.nn.Linear( latent_dim,256), - # torch.nn.Transformer(d_model=latent_dim, nhead=1, num_encoder_layers=1, num_decoder_layers=1), - # torch.nn.Linear(latent_dim, latent_dim), - # torch.nn.Linear(latent_dim, latent_dim), - # torch.nn.ReLU(), - # torch.nn.LeakyReLU(), - # torch.nn.Dropout(p=0.2), - # torch.nn.TransformerEncoder(torch.nn.TransformerEncoderLayer(d_model=latent_dim, nhead=4), num_layers=2), - # torch.nn.Linear(256, 1), - torch.nn.Linear(latent_dim, 1), - ) - if pretrained_path != "": - model.load_state_dict(torch.load(pretrained_path, map_location=device)) - # model.lm_head = torch.nn.Sequential(torch.nn.Linear( model.config.hidden_size, 256),torch.nn.SiLU(),torch.nn.Linear( 256, 1) ) - # set_seed(seed) - # set_deterministic() - model.to(device) - if torch.cuda.device_count() > 1: - device_ids = [d for d in range(torch.cuda.device_count())] - model = torch.nn.DataParallel(model, device_ids=device_ids).cuda() - optimizer = transformers.AdamW(model.parameters(), lr=learning_rate) - # Prepare datasets and dataloaders with data collator - train_dataset = AtomGPTDataset( - texts=train_texts, - targets=train_targets, - ids=train_ids_temp, - tokenizer=tokenizer, - max_length=max_length, - ) - val_dataset = AtomGPTDataset( - texts=val_texts, - targets=val_targets, - tokenizer=tokenizer, - ids=val_ids_temp, - max_length=max_length, - ) - test_dataset = AtomGPTDataset( - texts=test_texts, - targets=test_targets, - tokenizer=tokenizer, - ids=test_ids_temp, - max_length=max_length, - ) - - # val_dataset = train_dataset - train_dataloader = DataLoader(train_dataset, batch_size=batch_size) - val_dataloader = DataLoader(val_dataset, batch_size=batch_size) - test_dataloader = DataLoader(test_dataset, batch_size=batch_size) - val_dataloader = test_dataloader - steps_per_epoch = len(train_dataloader) - scheduler = torch.optim.lr_scheduler.OneCycleLR( - optimizer, - max_lr=learning_rate, - epochs=num_epochs, - steps_per_epoch=steps_per_epoch, - # pct_start=pct_start, - pct_start=0.3, - ) - # scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10, eta_min=0.001) - # scheduler = torch.optim.lr_scheduler.StepLR( - # optimizer, - # step_size=30, - # ) - print("train_data", len(train_texts)) - print("test_data", len(test_texts)) - output_dir = prefix + "_out_" + model_name + "_" + dataset + "_" + prop - if not os.path.exists(output_dir): - os.makedirs(output_dir) - best_loss = np.inf - tot_time_start = time.time() - train_history = [] - val_history = [] - for epoch in range(num_epochs): - model.train() - t1 = time.time() - for batch in train_dataloader: - optimizer.zero_grad() - train_loss = 0 - # train_result = [] - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - # if 't5' in model_name: - # decoder_input_ids = tokenizer("", return_tensors="pt").input_ids.to(device) - # decoder_input_ids = model._shift_right(decoder_input_ids) - # predictions = ( - # model(input_ids = input_ids.to(device),decoder_input_ids=decoder_input_ids).logits.squeeze().mean(dim=-1) - # ) - # else: - # predictions = ( - # model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - # ) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - targets = batch[2].squeeze() - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - # print('train',predictions,targets) - loss.backward() - optimizer.step() - scheduler.step() - # optimizer.zero_grad() - train_loss += loss.item() - train_loss = train_loss / len(train_dataloader) - t2 = time.time() - train_time = round(t2 - t1, 3) - model.eval() - - # total_eval_mae_loss = 0 - # predictions_list = [] - # targets_list = [] - val_loss = 0 - t1 = time.time() - for batch in val_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - targets = batch[2].squeeze() - # print('val',predictions,targets) - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - val_loss += loss.item() - saving_tag = "" - if val_loss < best_loss: - best_loss = val_loss - best_model_name = "best_model.pt" - torch.save( - model.state_dict(), - os.path.join(output_dir, best_model_name), - ) - # print("Saving model for epoch", epoch) - saving_tag = " saving model:" + str(epoch) - val_loss = val_loss / len(val_dataloader) - t2 = time.time() - val_time = round(t2 - t1, 3) - train_history.append(train_loss) - val_history.append(val_loss) - history = os.path.join(output_dir, "history.json") - - dumpjson( - data={"train": train_history, "val": val_history}, filename=history - ) - mae = "" - if test_each_run: - t1_test = time.time() - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - t2_test = time.time() - test_time = round(t2_test - t1_test, 3) - f.close() - df = pd.read_csv(fname) - mae = mean_absolute_error(df["target"], df["predictions"]) - if mae == "": - print( - "Epoch, train loss, val loss, train_time, val_time", - epoch, - train_loss, - val_loss, - train_time, - val_time, - saving_tag, - ) - else: - print( - "Epoch, train loss, val loss, test loss, train_time, val_time, test_time", - epoch, - train_loss, - val_loss, - mae, - train_time, - val_time, - test_time, - saving_tag, - ) - - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - f.close() - tot_time_end = time.time() - tot_time = tot_time_end - tot_time_start - print("tot_time", tot_time) - - -if __name__ == "__main__": - args = parser.parse_args(sys.argv[1:]) - benchmark_file = args.benchmark_file - model_name = "facebook/opt-350m" - model_name = "mistralai/Mixtral-8x7B-v0.1" - model_name = "google/flan-t5-small" - model_name = "google/flan-t5-base" - model_name = "mistralai/Mistral-7B-Instruct-v0.1" - model_name = "google-t5/t5-small" - model_name = "xlnet/xlnet-base-cased" - model_name = "afmck/testing-llama-tiny" - model_name = "EleutherAI/gpt-neo-125m" - model_name = "openai-community/gpt2-medium" - model_name = "meta-llama/Llama-2-7b-hf" - model_name = "stas/tiny-random-llama-2" - model_name = "ahxt/llama2_xs_460M_experimental" - model_name = "gpt2" - run_atomgpt( - model_name=model_name, - benchmark_file=benchmark_file, - # num_epochs=300, - # pretrained_path="xyz_out_google/flan-t5-small_tinnet_N_ead/best_model.pt", - # pretrained_path="ss_out_google/flan-t5-small_tinnet_N_ead/best_model.pt", - prefix="xyzt6", - batch_size=16, - latent_dim=1024, - num_epochs=5000, - # batch_size=16 - ) diff --git a/atomgpt/scripts/finetune7.py b/atomgpt/scripts/finetune7.py deleted file mode 100644 index 7d151bc..0000000 --- a/atomgpt/scripts/finetune7.py +++ /dev/null @@ -1,784 +0,0 @@ -"""Module for fin tuning LLM model for materials chemsitry.""" - -from jarvis.db.figshare import data -import transformers -import torch -import random -from jarvis.db.jsonutils import dumpjson -import sys -import argparse -from torch.utils.data import DataLoader, Dataset -import numpy as np -import os -from jarvis.core.atoms import Atoms -import pandas as pd -from sklearn.metrics import mean_absolute_error - -# from describe import atoms_describer -import json -from jarvis.db.figshare import get_jid_data -from jarvis.core.atoms import Atoms -from jarvis.analysis.structure.spacegroup import Spacegroup3D -from jarvis.analysis.diffraction.xrd import XRD -from jarvis.core.specie import Specie -import pprint -from collections import defaultdict - -from tqdm import tqdm -import time -import json -import zipfile -from transformers import GPT2Config, GPT2Model, GPT2Tokenizer - -device = "cpu" -if torch.cuda.is_available(): - device = torch.device("cuda") -parser = argparse.ArgumentParser(description="AtomGPT") -parser.add_argument( - "--benchmark_file", - default="AI-SinglePropertyPrediction-PBE_gap-halide_peroskites-test-mae.csv.zip", - # default="AI-SinglePropertyPrediction-ead-tinnet_N-test-mae.csv.zip", - # default="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae", - help="Benchmarks available in jarvis_leaderboard/benchmarks/*/*.zip", -) - - -def atoms_describer( - atoms=[], - xrd_peaks=5, - xrd_round=1, - cutoff=4, - take_n_bomds=2, - include_spg=True, -): - """Describe an atomic structure.""" - if include_spg: - spg = Spacegroup3D(atoms) - theta, d_hkls, intens = XRD().simulate(atoms=(atoms)) - # x = atoms.atomwise_angle_and_radial_distribution() - # bond_distances = {} - # for i, j in x[-1]["different_bond"].items(): - # bond_distances[i.replace("_", "-")] = ", ".join( - # map(str, (sorted(list(set([round(jj, 2) for jj in j]))))) - # ) - dists = defaultdict(list) - elements = atoms.elements - for i in atoms.get_all_neighbors(r=cutoff): - for j in i: - key = "-".join(sorted([elements[j[0]], elements[j[1]]])) - dists[key].append(j[2]) - bond_distances = {} - for i, j in dists.items(): - dist = sorted(set([round(k, 2) for k in j])) - if len(dist) >= take_n_bomds: - dist = dist[0:take_n_bomds] - bond_distances[i] = ", ".join(map(str, dist)) - fracs = {} - for i, j in (atoms.composition.atomic_fraction).items(): - fracs[i] = round(j, 3) - info = {} - chem_info = { - "atomic_formula": atoms.composition.reduced_formula, - "prototype": atoms.composition.prototype, - "molecular_weight": round(atoms.composition.weight / 2, 2), - "atomic_fraction": (fracs), - "atomic_X": ", ".join( - map(str, [Specie(s).X for s in atoms.uniq_species]) - ), - "atomic_Z": ", ".join( - map(str, [Specie(s).Z for s in atoms.uniq_species]) - ), - } - struct_info = { - "lattice_parameters": ", ".join( - map(str, [round(j, 2) for j in atoms.lattice.abc]) - ), - "lattice_angles": ", ".join( - map(str, [round(j, 2) for j in atoms.lattice.angles]) - ), - # "spg_number": spg.space_group_number, - # "spg_symbol": spg.space_group_symbol, - "top_k_xrd_peaks": ", ".join( - map( - str, - sorted(list(set([round(i, xrd_round) for i in theta])))[ - 0:xrd_peaks - ], - ) - ), - "density": round(atoms.density, 3), - # "crystal_system": spg.crystal_system, - # "point_group": spg.point_group_symbol, - # "wyckoff": ", ".join(list(set(spg._dataset["wyckoffs"]))), - "bond_distances": bond_distances, - # "natoms_primitive": spg.primitive_atoms.num_atoms, - # "natoms_conventional": spg.conventional_standard_structure.num_atoms, - } - if include_spg: - struct_info["spg_number"] = spg.space_group_number - struct_info["spg_symbol"] = spg.space_group_symbol - struct_info["crystal_system"] = spg.crystal_system - struct_info["point_group"] = spg.point_group_symbol - struct_info["wyckoff"] = ", ".join(list(set(spg._dataset["wyckoffs"]))) - struct_info["natoms_primitive"] = spg.primitive_atoms.num_atoms - struct_info[ - "natoms_conventional" - ] = spg.conventional_standard_structure.num_atoms - info["chemical_info"] = chem_info - info["structure_info"] = struct_info - line = "The number of atoms are: " + str( - atoms.num_atoms - ) # +"., The elements are: "+",".join(atoms.elements)+". " - for i, j in info.items(): - if not isinstance(j, dict): - line += "The " + i + " is " + j + ". " - else: - # print("i",i) - # print("j",j) - for ii, jj in j.items(): - tmp = "" - if isinstance(jj, dict): - for iii, jjj in jj.items(): - tmp += iii + ": " + str(jjj) + " " - else: - tmp = jj - line += "The " + ii + " is " + str(tmp) + ". " - return line - - -os.environ["WANDB_ANONYMOUS"] = "must" -random_seed = 42 -random.seed(random_seed) -torch.manual_seed(random_seed) -np.random.seed(random_seed) -torch.cuda.manual_seed_all(random_seed) -try: - import torch_xla.core.xla_model as xm - - xm.set_rng_state(random_seed) -except ImportError: - pass -torch.backends.cudnn.deterministic = True -torch.backends.cudnn.benchmark = False -os.environ["PYTHONHASHSEED"] = str(random_seed) -os.environ["CUBLAS_WORKSPACE_CONFIG"] = str(":4096:8") -torch.use_deterministic_algorithms(True) -IGNORE_INDEX = -100 -# torch.cuda.empty_cache() - - -def get_crystal_string_1225(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.1f}".format(x) for x in lengths]) - + "\n" - + " ".join([str(int(x)) for x in angles]) - + "\n" - + "\n".join( - [ - str(t) + " " + " ".join(["{0:.2f}".format(x) for x in c]) - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - extra = str(atoms.num_atoms) + "\n" + atoms.composition.reduced_formula - # crystal_str+=" "+extra - extra += "\n" + crystal_str - return extra - # return crystal_str - - -def get_crystal_string_t(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.2f}".format(x) for x in lengths]) - + "#\n" - + " ".join([str(int(x)) for x in angles]) - + "@\n" - + "\n".join( - [ - str(t) + " " + " ".join(["{0:.3f}".format(x) for x in c]) + "&" - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - crystal_str = atoms_describer(atoms) + "\n*\n" + crystal_str - return crystal_str - - -class AtomGPTDataset(Dataset): - def __init__( - self, texts=[], targets=[], ids=[], tokenizer="", max_length=128 - ): - self.texts = texts - self.targets = targets - self.tokenizer = tokenizer - self.max_length = max_length - if not ids: - ids = ["text-" + str(i) for i in range(len(texts))] - self.ids = ids - - def __len__(self): - return len(self.texts) - - def __getitem__(self, idx): - inputs = self.tokenizer( - self.texts[idx], - return_tensors="pt", - max_length=self.max_length, - padding="max_length", - truncation=True, - ) - return ( - inputs, - self.ids[idx], - torch.tensor(self.targets[idx], dtype=torch.float32), - ) - -config = GPT2Config.from_pretrained("gpt2") -class ForcePredictor(torch.nn.Module): - def __init__(self, gpt2_model): - super(ForcePredictor, self).__init__() - self.gpt2 = gpt2_model - self.linear = torch.nn.Linear(config.n_embd, 1) # Assuming force is a 3D vector - - def forward(self, input_ids): - outputs = self.gpt2(input_ids) - last_hidden_states = outputs.last_hidden_state - force_pred = self.linear(last_hidden_states[:, -1, :]) - #print("force_pred",outputs.keys()) - return force_pred - - -class AtomGPTPredictorLMhead(torch.nn.Module): - def __init__( - self, model_name=None, n_out=1, latent_dim=1024, tokenizer="" - ): - super(AtomGPTPredictorLMhead, self).__init__() - self.model_name = model_name - self.n_out = n_out - self.latent_dim = latent_dim - if "t5" in model_name: - model = transformers.T5ForConditionalGeneration.from_pretrained( - model_name - ) - else: - #config = GPT2Config.from_pretrained("gpt2") - #model = GPT2Model(config) - model = transformers.AutoModelForCausalLM.from_pretrained( - model_name, - low_cpu_mem_usage=True, - # load_in_8bit=False, - # torch_dtype=torch.float16, - # load_in_8bit=True, - # device_map="auto" - ) - model.resize_token_embeddings(len(tokenizer)) - self.config = model.config - model.lm_head = torch.nn.Sequential( - torch.nn.Linear(model.config.hidden_size, latent_dim), - torch.nn.Linear(latent_dim, n_out), - ) - self.model = model - - def forward(self, input_ids): - #outputs = self.model(input_ids) - if "t5" in model_name: - outputs = self.model(input_ids, decoder_input_ids=input_ids) - else: - outputs = self.model(input_ids) - return outputs - - -class AtomGPTPredictorHiddenFeats(torch.nn.Module): - def __init__(self, model_name=None, n_out=1, tokenizer=""): - super(AtomGPTPredictorHiddenFeats, self).__init__() - if "t5" in model_name: - model = transformers.T5ForConditionalGeneration.from_pretrained( - model_name - ) - else: - model = transformers.AutoModelForCausalLM.from_pretrained( - model_name, - low_cpu_mem_usage=True, - # load_in_8bit=False, - # torch_dtype=torch.float16, - # load_in_8bit=True, - # device_map="auto" - ) - model.resize_token_embeddings(len(tokenizer)) - self.model = model - self.config = self.model.config - self.global_out = torch.nn.Linear(self.config.n_embd, n_out) - - def forward(self, input_ids): - outputs = self.model(input_ids) - print('outputs',outputs.keys()) - last_hidden_states = outputs.last_hidden_state - pred = self.linear(last_hidden_states[:, -1, :]) - return pred - - -def run_atomgpt( - prefix="ss", - model_name="gpt2", - benchmark_file="AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip", - root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard", - batch_size=8, - max_length=512, - num_epochs=500, - latent_dim=512, - learning_rate=1e-3, - # learning_rate=1e-3, - test_each_run=True, - # learning_rate=5e-5, - pretrained_path="", -): - # Load pre-trained tokenizer - # tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - # model = GPT2LMHeadModel.from_pretrained("gpt2") - - # dft_3d = data("dft_3d") - # root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard" - # benchmark_file="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae.csv.zip" - # benchmark_file = "AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip" - print("benchmark_file", benchmark_file) - method = benchmark_file.split("-")[0] - task = benchmark_file.split("-")[1] - prop = benchmark_file.split("-")[2] - dataset = benchmark_file.split("-")[3] - temp = dataset + "_" + prop + ".json.zip" - temp2 = dataset + "_" + prop + ".json" - fname = os.path.join(root_dir, "benchmarks", method, task, temp) - zp = zipfile.ZipFile(fname) - bench = json.loads(zp.read(temp2)) - dft_3d = data(dataset) - id_tag = "jid" - if "jid" in dft_3d[0]: - id_tag = "jid" - else: - id_tag = "id" - - # train_atoms = [] - # val_atoms = [] - # test_atoms = [] - # train_targets = [] - # val_targets = [] - # test_targets = [] - train_ids = list(bench["train"].keys()) - test_ids = list(bench["test"].keys()) - if "val" in bench: - val_ids = list(bench["val"].keys()) - else: - val_ids = test_ids - print("total", len(dft_3d)) - print("test_ids", len(test_ids)) - print("val_ids", len(val_ids)) - print("train_ids", len(train_ids)) - # model_name = "mistralai/Mistral-7B-Instruct-v0.1" - # model_name = "gpt2" - # device = model.device - if "t5" in model_name: - tokenizer = transformers.T5Tokenizer.from_pretrained(model_name) - - else: - tokenizer = transformers.AutoTokenizer.from_pretrained(model_name) - #print('Non tokenizer format') - #tokenizer = GPT2Tokenizer.from_pretrained(model_name) - config = GPT2Config.from_pretrained("gpt2") - model = GPT2Model(config) - if tokenizer.pad_token is None: - tokenizer.add_special_tokens({"pad_token": "[PAD]"}) - tokenizer.add_special_tokens({"unk_token": "#"}) - tokenizer.add_special_tokens({"unk_token": "&"}) - tokenizer.add_special_tokens({"unk_token": "@"}) - model.resize_token_embeddings(len(tokenizer)) - model=ForcePredictor(model) - #model=AtomGPTPredictorHiddenFeats(model_name=model_name, tokenizer=tokenizer) - #model = AtomGPTPredictorLMhead(model_name=model_name, tokenizer=tokenizer) - #tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - # batch_size = 16 - # max_length = 128 - # num_epochs = 100 - # learning_rate = 5e-5 - criterion = torch.nn.L1Loss() - # Define example regression data (texts and corresponding numeric targets) - train_texts = [] - train_targets = [] - train_ids_temp = [] - val_texts = [] - val_targets = [] - val_ids_temp = [] - test_texts = [] - test_targets = [] - test_ids_temp = [] - - for i in tqdm(dft_3d): - if i[prop] != "na": - atoms = Atoms.from_dict(i["atoms"]) - tmp = get_crystal_string_t(atoms) - if i[id_tag] in train_ids: - train_texts.append(tmp) - train_targets.append(i[prop]) - train_ids_temp.append(i[id_tag]) - elif i[id_tag] in test_ids: - test_texts.append(tmp) - test_targets.append(i[prop]) - test_ids_temp.append(i[id_tag]) - elif i[id_tag] in val_ids: - val_texts.append(tmp) - val_targets.append(i[prop]) - val_ids_temp.append(i[id_tag]) - print("test_texts:", len(test_texts)) - print("test_texts:", test_texts[0]) - """ - ############################## - ###Fast test### - train_texts = [ - "This is the first example text.", - "Second example is a bit longer than the first one, but still within the max length.", - "Third example is the longest among these three examples. It exceeds the max length and will be truncated.", - "Second example is a bit longer than the first one, but still within the max length.", - ] - train_targets = [10.2, 15.5, 20.1, 15.5] # Example regression targets - val_texts = test_texts = train_texts - val_targets = test_targets = train_targets - train_ids_temp=['a','b','c','d'] - val_ids_temp = test_ids_temp = train_ids_temp - batch_size = 2 - num_epochs = 3 - - ############################## - ############################## - """ - - # Fine-tune the last layer of GPT-2 for regression - # fine_tune_gpt2_regression(train_texts, train_targets, tokenizer) - if pretrained_path != "": - model.load_state_dict(torch.load(pretrained_path, map_location=device)) - # model.lm_head = torch.nn.Sequential(torch.nn.Linear( model.config.hidden_size, 256),torch.nn.SiLU(),torch.nn.Linear( 256, 1) ) - # set_seed(seed) - # set_deterministic() - model.to(device) - if torch.cuda.device_count() > 1: - device_ids = [d for d in range(torch.cuda.device_count())] - model = torch.nn.DataParallel(model, device_ids=device_ids).cuda() - optimizer = transformers.AdamW(model.parameters(), lr=learning_rate) - # Prepare datasets and dataloaders with data collator - train_dataset = AtomGPTDataset( - texts=train_texts, - targets=train_targets, - ids=train_ids_temp, - tokenizer=tokenizer, - max_length=max_length, - ) - val_dataset = AtomGPTDataset( - texts=val_texts, - targets=val_targets, - tokenizer=tokenizer, - ids=val_ids_temp, - max_length=max_length, - ) - test_dataset = AtomGPTDataset( - texts=test_texts, - targets=test_targets, - tokenizer=tokenizer, - ids=test_ids_temp, - max_length=max_length, - ) - - # val_dataset = train_dataset - train_dataloader = DataLoader(train_dataset, batch_size=batch_size) - val_dataloader = DataLoader(val_dataset, batch_size=batch_size) - test_dataloader = DataLoader(test_dataset, batch_size=batch_size) - val_dataloader = test_dataloader - steps_per_epoch = len(train_dataloader) - scheduler = torch.optim.lr_scheduler.OneCycleLR( - optimizer, - max_lr=learning_rate, - epochs=num_epochs, - steps_per_epoch=steps_per_epoch, - # pct_start=pct_start, - pct_start=0.3, - ) - # scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10, eta_min=0.001) - # scheduler = torch.optim.lr_scheduler.StepLR( - # optimizer, - # step_size=30, - # ) - print("train_data", len(train_texts)) - print("test_data", len(test_texts)) - output_dir = prefix + "_out_" + model_name + "_" + dataset + "_" + prop - if not os.path.exists(output_dir): - os.makedirs(output_dir) - best_loss = np.inf - tot_time_start = time.time() - train_history = [] - val_history = [] - for epoch in range(num_epochs): - model.train() - t1 = time.time() - for batch in train_dataloader: - optimizer.zero_grad() - train_loss = 0 - # train_result = [] - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - # if 't5' in model_name: - # decoder_input_ids = tokenizer("", return_tensors="pt").input_ids.to(device) - # decoder_input_ids = model._shift_right(decoder_input_ids) - # predictions = ( - # model(input_ids = input_ids.to(device),decoder_input_ids=decoder_input_ids).logits.squeeze().mean(dim=-1) - # ) - # else: - # predictions = ( - # model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - # ) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - # decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - targets = batch[2].squeeze() - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - # print('train',predictions,targets) - loss.backward() - optimizer.step() - scheduler.step() - # optimizer.zero_grad() - train_loss += loss.item() - train_loss = train_loss / len(train_dataloader) - t2 = time.time() - train_time = round(t2 - t1, 3) - model.eval() - - # total_eval_mae_loss = 0 - # predictions_list = [] - # targets_list = [] - val_loss = 0 - t1 = time.time() - for batch in val_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - # decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - targets = batch[2].squeeze() - # print('val',predictions,targets) - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - val_loss += loss.item() - saving_tag = "" - if val_loss < best_loss: - best_loss = val_loss - best_model_name = "best_model.pt" - torch.save( - model.state_dict(), - os.path.join(output_dir, best_model_name), - ) - # print("Saving model for epoch", epoch) - saving_tag = " saving model:" + str(epoch) - val_loss = val_loss / len(val_dataloader) - t2 = time.time() - val_time = round(t2 - t1, 3) - train_history.append(train_loss) - val_history.append(val_loss) - history = os.path.join(output_dir, "history.json") - - dumpjson( - data={"train": train_history, "val": val_history}, filename=history - ) - mae = "" - if test_each_run: - t1_test = time.time() - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - # decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - t2_test = time.time() - test_time = round(t2_test - t1_test, 3) - f.close() - df = pd.read_csv(fname) - mae = mean_absolute_error(df["target"], df["predictions"]) - if mae == "": - print( - "Epoch, train loss, val loss, train_time, val_time", - epoch, - train_loss, - val_loss, - train_time, - val_time, - saving_tag, - ) - else: - print( - "Epoch, train loss, val loss, test loss, train_time, val_time, test_time", - epoch, - train_loss, - val_loss, - mae, - train_time, - val_time, - test_time, - saving_tag, - ) - - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - # decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - f.close() - tot_time_end = time.time() - tot_time = tot_time_end - tot_time_start - print("tot_time", tot_time) - - -if __name__ == "__main__": - # box = [[2.715, 2.715, 0], [0, 2.715, 2.715], [2.715, 0, 2.715]] - # coords = [[0, 0, 0], [0.25, 0.2, 0.25]] - # elements = ["Si", "Si"] - # Si = Atoms(lattice_mat=box, coords=coords, elements=elements) - # tmp=atoms_describer(Si) - # print(tmp) - # import sys - # sys.exit() - args = parser.parse_args(sys.argv[1:]) - benchmark_file = args.benchmark_file - model_name = "facebook/opt-350m" - model_name = "mistralai/Mixtral-8x7B-v0.1" - model_name = "google/flan-t5-small" - model_name = "google/flan-t5-base" - model_name = "mistralai/Mistral-7B-Instruct-v0.1" - model_name = "xlnet/xlnet-base-cased" - model_name = "afmck/testing-llama-tiny" - model_name = "EleutherAI/gpt-neo-125m" - model_name = "meta-llama/Llama-2-7b-hf" - model_name = "stas/tiny-random-llama-2" - model_name = "ahxt/llama2_xs_460M_experimental" - model_name = "google-t5/t5-small" - model_name = "openai-community/gpt2-medium" - model_name = "gpt2" - run_atomgpt( - model_name=model_name, - benchmark_file=benchmark_file, - # num_epochs=300, - # pretrained_path="xyz_out_google/flan-t5-small_tinnet_N_ead/best_model.pt", - # pretrained_path="ss_out_google/flan-t5-small_tinnet_N_ead/best_model.pt", - prefix="xyzt6", - #batch_size=5, - batch_size=16, - latent_dim=1024, - num_epochs=5000, - # batch_size=16 - ) diff --git a/atomgpt/scripts/finetune7a.py.alignn b/atomgpt/scripts/finetune7a.py.alignn deleted file mode 100644 index 8fcfed0..0000000 --- a/atomgpt/scripts/finetune7a.py.alignn +++ /dev/null @@ -1,877 +0,0 @@ -"""Module for fin tuning LLM model for materials chemsitry.""" - -from jarvis.db.figshare import data -import transformers -import torch -import random -from jarvis.db.jsonutils import dumpjson -import sys -import argparse -from torch.utils.data import DataLoader, Dataset -import numpy as np -import os -from jarvis.core.atoms import Atoms -import pandas as pd -from sklearn.metrics import mean_absolute_error - -# from describe import atoms_describer -import json -from jarvis.db.figshare import get_jid_data -from jarvis.core.atoms import Atoms -from jarvis.analysis.structure.spacegroup import Spacegroup3D -from jarvis.analysis.diffraction.xrd import XRD -from jarvis.core.specie import Specie -import pprint -from collections import defaultdict - -from tqdm import tqdm -import time -import json -import zipfile -from transformers import GPT2Config, GPT2Model, GPT2Tokenizer -from jarvis.core.atoms import Atoms - -#from alignn.graphs import Graph - -#from alignn.pretrained import get_figshare_model - -import torch - -device = "cpu" -if torch.cuda.is_available(): - device = torch.device("cuda") - - -#model_alignn = get_figshare_model() -#model_alignn.to(device) - - - -def get_val(model, g, lg): - activation = {} - def getActivation(name): - # the hook signature - def hook(model, input, output): - activation[name] = output.detach() - return hook - h = model.readout.register_forward_hook(getActivation("readout")) - out = model([g, lg]) - h.remove() - return activation["readout"][0] - -def get_alignn_feats(model_alignn='',atoms=''): - g, lg = Graph.atom_dgl_multigraph(atoms) - x = get_val(model_alignn, g.to(device), lg.to(device)) - return x - -parser = argparse.ArgumentParser(description="AtomGPT") -parser.add_argument( - "--benchmark_file", - default="AI-SinglePropertyPrediction-PBE_gap-halide_peroskites-test-mae.csv.zip", - # default="AI-SinglePropertyPrediction-ead-tinnet_N-test-mae.csv.zip", - # default="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae", - help="Benchmarks available in jarvis_leaderboard/benchmarks/*/*.zip", -) - - -def atoms_describer( - atoms=[], - xrd_peaks=5, - xrd_round=1, - cutoff=4, - take_n_bomds=2, - include_spg=True, -): - """Describe an atomic structure.""" - if include_spg: - spg = Spacegroup3D(atoms) - theta, d_hkls, intens = XRD().simulate(atoms=(atoms)) - # x = atoms.atomwise_angle_and_radial_distribution() - # bond_distances = {} - # for i, j in x[-1]["different_bond"].items(): - # bond_distances[i.replace("_", "-")] = ", ".join( - # map(str, (sorted(list(set([round(jj, 2) for jj in j]))))) - # ) - dists = defaultdict(list) - elements = atoms.elements - for i in atoms.get_all_neighbors(r=cutoff): - for j in i: - key = "-".join(sorted([elements[j[0]], elements[j[1]]])) - dists[key].append(j[2]) - bond_distances = {} - for i, j in dists.items(): - dist = sorted(set([round(k, 2) for k in j])) - if len(dist) >= take_n_bomds: - dist = dist[0:take_n_bomds] - bond_distances[i] = ", ".join(map(str, dist)) - fracs = {} - for i, j in (atoms.composition.atomic_fraction).items(): - fracs[i] = round(j, 3) - info = {} - chem_info = { - "atomic_formula": atoms.composition.reduced_formula, - "prototype": atoms.composition.prototype, - "molecular_weight": round(atoms.composition.weight / 2, 2), - "atomic_fraction": (fracs), - "atomic_X": ", ".join( - map(str, [Specie(s).X for s in atoms.uniq_species]) - ), - "atomic_Z": ", ".join( - map(str, [Specie(s).Z for s in atoms.uniq_species]) - ), - } - struct_info = { - "lattice_parameters": ", ".join( - map(str, [round(j, 2) for j in atoms.lattice.abc]) - ), - "lattice_angles": ", ".join( - map(str, [round(j, 2) for j in atoms.lattice.angles]) - ), - # "spg_number": spg.space_group_number, - # "spg_symbol": spg.space_group_symbol, - "top_k_xrd_peaks": ", ".join( - map( - str, - sorted(list(set([round(i, xrd_round) for i in theta])))[ - 0:xrd_peaks - ], - ) - ), - "density": round(atoms.density, 3), - # "crystal_system": spg.crystal_system, - # "point_group": spg.point_group_symbol, - # "wyckoff": ", ".join(list(set(spg._dataset["wyckoffs"]))), - "bond_distances": bond_distances, - # "natoms_primitive": spg.primitive_atoms.num_atoms, - # "natoms_conventional": spg.conventional_standard_structure.num_atoms, - } - if include_spg: - struct_info["spg_number"] = spg.space_group_number - struct_info["spg_symbol"] = spg.space_group_symbol - struct_info["crystal_system"] = spg.crystal_system - struct_info["point_group"] = spg.point_group_symbol - struct_info["wyckoff"] = ", ".join(list(set(spg._dataset["wyckoffs"]))) - struct_info["natoms_primitive"] = spg.primitive_atoms.num_atoms - struct_info[ - "natoms_conventional" - ] = spg.conventional_standard_structure.num_atoms - info["chemical_info"] = chem_info - info["structure_info"] = struct_info - line = "The number of atoms are: " + str( - atoms.num_atoms - ) # +"., The elements are: "+",".join(atoms.elements)+". " - for i, j in info.items(): - if not isinstance(j, dict): - line += "The " + i + " is " + j + ". " - else: - # print("i",i) - # print("j",j) - for ii, jj in j.items(): - tmp = "" - if isinstance(jj, dict): - for iii, jjj in jj.items(): - tmp += iii + ": " + str(jjj) + " " - else: - tmp = jj - line += "The " + ii + " is " + str(tmp) + ". " - return line - - -def set_seed(): - os.environ["WANDB_ANONYMOUS"] = "must" - random_seed = 42 - random.seed(random_seed) - torch.manual_seed(random_seed) - np.random.seed(random_seed) - torch.cuda.manual_seed_all(random_seed) - try: - import torch_xla.core.xla_model as xm - - xm.set_rng_state(random_seed) - except ImportError: - pass - torch.backends.cudnn.deterministic = True - torch.backends.cudnn.benchmark = False - os.environ["PYTHONHASHSEED"] = str(random_seed) - os.environ["CUBLAS_WORKSPACE_CONFIG"] = str(":4096:8") - torch.use_deterministic_algorithms(True) - - -def get_crystal_string_t(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.2f}".format(x) for x in lengths]) - + "#\n" - + " ".join([str(int(x)) for x in angles]) - + "@\n" - + "\n".join( - [ - str(t) + " " + " ".join(["{0:.3f}".format(x) for x in c]) + "&" - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - crystal_str = atoms_describer(atoms) + "\n*\n" + crystal_str - return crystal_str - - -class AtomGPTDataset(Dataset): - def __init__( - self, - texts=[], - targets=[], - ids=[], - extra_feats=[], - tokenizer="", - max_length=128, - ): - self.texts = texts - self.targets = targets - self.tokenizer = tokenizer - self.max_length = max_length - self.extra_feats = extra_feats - if not ids: - ids = ["text-" + str(i) for i in range(len(texts))] - self.ids = ids - - def __len__(self): - return len(self.texts) - - def __getitem__(self, idx): - inputs = self.tokenizer( - self.texts[idx], - return_tensors="pt", - max_length=self.max_length, - padding="max_length", - truncation=True, - ) - if self.extra_feats: - feats = self.extra_feats[idx] - inputs = torch.cat(inputs, feats) - return ( - inputs, - self.ids[idx], - torch.tensor(self.targets[idx], dtype=torch.float32), - ) - - -class ForcePredictor(torch.nn.Module): - config = GPT2Config.from_pretrained("gpt2") - - def __init__(self, gpt2_model): - super(ForcePredictor, self).__init__() - self.gpt2 = gpt2_model - self.linear = torch.nn.Linear( - config.n_embd, 1 - ) # Assuming force is a 3D vector - - def forward(self, input_ids): - outputs = self.gpt2(input_ids) - last_hidden_states = outputs.last_hidden_state - force_pred = self.linear(last_hidden_states[:, -1, :]) - # print("force_pred",outputs.keys()) - return force_pred - - -class AtomGPTPredictorLMhead(torch.nn.Module): - def __init__( - self, model_name=None, n_out=1, latent_dim=1024, tokenizer="" - ): - - super(AtomGPTPredictorLMhead, self).__init__() - # random_seed = 42 - # random.seed(random_seed) - # torch.manual_seed(random_seed) - # np.random.seed(random_seed) - # torch.cuda.manual_seed_all(random_seed) - # try: - # import torch_xla.core.xla_model as xm - # xm.set_rng_state(random_seed) - # except ImportError: - # pass - # torch.backends.cudnn.deterministic = True - # torch.backends.cudnn.benchmark = False - # os.environ["PYTHONHASHSEED"] = str(random_seed) - # os.environ["CUBLAS_WORKSPACE_CONFIG"] = str(":4096:8") - # torch.use_deterministic_algorithms(True) - - self.model_name = model_name - self.n_out = n_out - self.latent_dim = latent_dim - if "t5" in model_name: - model = transformers.T5ForConditionalGeneration.from_pretrained( - model_name - ) - else: - # config = GPT2Config.from_pretrained("gpt2") - # model = GPT2Model(config) - model = transformers.AutoModelForCausalLM.from_pretrained( - model_name, - low_cpu_mem_usage=True, - # load_in_8bit=False, - # torch_dtype=torch.float16, - # load_in_8bit=True, - # device_map="auto" - ) - model.resize_token_embeddings(len(tokenizer)) - self.config = model.config - model.lm_head = torch.nn.Sequential( - torch.nn.Linear(model.config.hidden_size, latent_dim), - torch.nn.Linear(latent_dim, n_out), - ) - self.model = model - - def forward(self, input_ids): - # outputs = self.model(input_ids) - if "t5" in model_name: - outputs = self.model(input_ids, decoder_input_ids=input_ids) - else: - outputs = self.model(input_ids) - return outputs - - -class AtomGPTPredictorHiddenFeats(torch.nn.Module): - def __init__(self, model_name=None, n_out=1, tokenizer=""): - super(AtomGPTPredictorHiddenFeats, self).__init__() - if "t5" in model_name: - model = transformers.T5ForConditionalGeneration.from_pretrained( - model_name - ) - else: - model = transformers.AutoModelForCausalLM.from_pretrained( - model_name, - low_cpu_mem_usage=True, - # load_in_8bit=False, - # torch_dtype=torch.float16, - # load_in_8bit=True, - # device_map="auto" - ) - model.resize_token_embeddings(len(tokenizer)) - self.model = model - self.config = self.model.config - self.global_out = torch.nn.Linear(self.config.n_embd, n_out) - - def forward(self, input_ids): - outputs = self.model(input_ids) - print("outputs", outputs.keys()) - last_hidden_states = outputs.last_hidden_state - pred = self.linear(last_hidden_states[:, -1, :]) - return pred - - -def run_atomgpt( - prefix="ss", - model_name="gpt2", - benchmark_file="AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip", - root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard", - batch_size=8, - max_length=512, - num_epochs=500, - latent_dim=512, - learning_rate=1e-3, - # learning_rate=1e-3, - test_each_run=True, - # learning_rate=5e-5, - pretrained_path="", -): - # Load pre-trained tokenizer - # tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - # model = GPT2LMHeadModel.from_pretrained("gpt2") - - # dft_3d = data("dft_3d") - # root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard" - # benchmark_file="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae.csv.zip" - # benchmark_file = "AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip" - print("benchmark_file", benchmark_file) - method = benchmark_file.split("-")[0] - task = benchmark_file.split("-")[1] - prop = benchmark_file.split("-")[2] - dataset = benchmark_file.split("-")[3] - temp = dataset + "_" + prop + ".json.zip" - temp2 = dataset + "_" + prop + ".json" - fname = os.path.join(root_dir, "benchmarks", method, task, temp) - zp = zipfile.ZipFile(fname) - bench = json.loads(zp.read(temp2)) - dft_3d = data(dataset) - id_tag = "jid" - if "jid" in dft_3d[0]: - id_tag = "jid" - else: - id_tag = "id" - - # train_atoms = [] - # val_atoms = [] - # test_atoms = [] - # train_targets = [] - # val_targets = [] - # test_targets = [] - train_ids = list(bench["train"].keys()) - test_ids = list(bench["test"].keys()) - if "val" in bench: - val_ids = list(bench["val"].keys()) - else: - val_ids = test_ids - print("total", len(dft_3d)) - print("test_ids", len(test_ids)) - print("val_ids", len(val_ids)) - print("train_ids", len(train_ids)) - # model_name = "mistralai/Mistral-7B-Instruct-v0.1" - # model_name = "gpt2" - # device = model.device - if "t5" in model_name: - tokenizer = transformers.T5Tokenizer.from_pretrained(model_name) - - else: - tokenizer = transformers.AutoTokenizer.from_pretrained(model_name) - # print('Non tokenizer format') - # tokenizer = GPT2Tokenizer.from_pretrained(model_name) - # config = GPT2Config.from_pretrained("gpt2") - # model = GPT2Model(config) - if tokenizer.pad_token is None: - tokenizer.add_special_tokens({"pad_token": "[PAD]"}) - tokenizer.add_special_tokens({"unk_token": "#"}) - tokenizer.add_special_tokens({"unk_token": "&"}) - tokenizer.add_special_tokens({"unk_token": "@"}) - # model.resize_token_embeddings(len(tokenizer)) - # model=ForcePredictor(model) - # model=AtomGPTPredictorHiddenFeats(model_name=model_name, tokenizer=tokenizer) - set_seed() - model = AtomGPTPredictorLMhead( - model_name=model_name, tokenizer=tokenizer, latent_dim=latent_dim - ) - # tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - # batch_size = 16 - # max_length = 128 - # num_epochs = 100 - # learning_rate = 5e-5 - criterion = torch.nn.L1Loss() - # Define example regression data (texts and corresponding numeric targets) - train_texts = [] - train_targets = [] - train_ids_temp = [] - val_texts = [] - val_targets = [] - val_ids_temp = [] - test_texts = [] - test_targets = [] - test_ids_temp = [] - train_feats=[] - val_feats=[] - test_feats=[] - for i in tqdm(dft_3d): - if i[prop] != "na": - atoms = Atoms.from_dict(i["atoms"]) - tmp = get_crystal_string_t(atoms) - feat=[]#get_alignn_feats(atoms=atoms) - if i[id_tag] in train_ids: - train_texts.append(tmp) - train_targets.append(i[prop]) - train_ids_temp.append(i[id_tag]) - train_feats.append(feat) - elif i[id_tag] in test_ids: - test_texts.append(tmp) - test_targets.append(i[prop]) - test_ids_temp.append(i[id_tag]) - test_feats.append(feat) - elif i[id_tag] in val_ids: - val_texts.append(tmp) - val_targets.append(i[prop]) - val_ids_temp.append(i[id_tag]) - val_feats.append(feat) - print("test_texts:", len(test_texts)) - print("test_texts:", test_texts[0]) - """ - ############################## - ###Fast test### - train_texts = [ - "This is the first example text.", - "Second example is a bit longer than the first one, but still within the max length.", - "Third example is the longest among these three examples. It exceeds the max length and will be truncated.", - "Second example is a bit longer than the first one, but still within the max length.", - ] - train_targets = [10.2, 15.5, 20.1, 15.5] # Example regression targets - val_texts = test_texts = train_texts - val_targets = test_targets = train_targets - train_ids_temp=['a','b','c','d'] - val_ids_temp = test_ids_temp = train_ids_temp - batch_size = 2 - num_epochs = 3 - - ############################## - ############################## - """ - - # Fine-tune the last layer of GPT-2 for regression - # fine_tune_gpt2_regression(train_texts, train_targets, tokenizer) - if pretrained_path != "": - model.load_state_dict(torch.load(pretrained_path, map_location=device)) - # model.lm_head = torch.nn.Sequential(torch.nn.Linear( model.config.hidden_size, 256),torch.nn.SiLU(),torch.nn.Linear( 256, 1) ) - # set_seed(seed) - # set_deterministic() - model.to(device) - if torch.cuda.device_count() > 1: - device_ids = [d for d in range(torch.cuda.device_count())] - model = torch.nn.DataParallel(model, device_ids=device_ids).cuda() - optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate) - # optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate) - # optimizer = transformers.AdamW(model.parameters(), lr=learning_rate) - # Prepare datasets and dataloaders with data collator - train_dataset = AtomGPTDataset( - texts=train_texts, - targets=train_targets, - ids=train_ids_temp, - tokenizer=tokenizer, - max_length=max_length, - extra_feats=train_feats, - ) - val_dataset = AtomGPTDataset( - texts=val_texts, - targets=val_targets, - tokenizer=tokenizer, - ids=val_ids_temp, - max_length=max_length, - extra_feats=val_feats, - ) - test_dataset = AtomGPTDataset( - texts=test_texts, - targets=test_targets, - tokenizer=tokenizer, - ids=test_ids_temp, - max_length=max_length, - extra_feats=test_feats, - ) - - # val_dataset = train_dataset - train_dataloader = DataLoader(train_dataset, batch_size=batch_size) - val_dataloader = DataLoader(val_dataset, batch_size=batch_size) - test_dataloader = DataLoader(test_dataset, batch_size=batch_size) - val_dataloader = test_dataloader - steps_per_epoch = len(train_dataloader) - scheduler = torch.optim.lr_scheduler.OneCycleLR( - optimizer, - max_lr=learning_rate, - epochs=num_epochs, - steps_per_epoch=steps_per_epoch, - # pct_start=pct_start, - pct_start=0.3, - ) - # scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10, eta_min=0.001) - # scheduler = torch.optim.lr_scheduler.StepLR( - # optimizer, - # step_size=30, - # ) - print("train_data", len(train_texts)) - print("test_data", len(test_texts)) - output_dir = prefix + "_out_" + model_name + "_" + dataset + "_" + prop - if not os.path.exists(output_dir): - os.makedirs(output_dir) - best_loss = np.inf - tot_time_start = time.time() - train_history = [] - val_history = [] - for epoch in range(num_epochs): - model.train() - t1 = time.time() - for batch in train_dataloader: - optimizer.zero_grad() - train_loss = 0 - # train_result = [] - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - # if 't5' in model_name: - # decoder_input_ids = tokenizer("", return_tensors="pt").input_ids.to(device) - # decoder_input_ids = model._shift_right(decoder_input_ids) - # predictions = ( - # model(input_ids = input_ids.to(device),decoder_input_ids=decoder_input_ids).logits.squeeze().mean(dim=-1) - # ) - # else: - # predictions = ( - # model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - # ) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - # decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - targets = batch[2].squeeze() - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - # print('train',predictions,targets) - loss.backward() - optimizer.step() - scheduler.step() - # optimizer.zero_grad() - train_loss += loss.item() - train_loss = train_loss / len(train_dataloader) - t2 = time.time() - train_time = round(t2 - t1, 3) - model.eval() - - # total_eval_mae_loss = 0 - # predictions_list = [] - # targets_list = [] - val_loss = 0 - t1 = time.time() - for batch in val_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - # decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - targets = batch[2].squeeze() - # print('val',predictions,targets) - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - val_loss += loss.item() - saving_tag = "" - if val_loss < best_loss: - best_loss = val_loss - best_model_name = "best_model.pt" - torch.save( - model.state_dict(), - os.path.join(output_dir, best_model_name), - ) - # print("Saving model for epoch", epoch) - saving_tag = " saving model:" + str(epoch) - val_loss = val_loss / len(val_dataloader) - t2 = time.time() - val_time = round(t2 - t1, 3) - train_history.append(train_loss) - val_history.append(val_loss) - history = os.path.join(output_dir, "history.json") - - dumpjson( - data={"train": train_history, "val": val_history}, filename=history - ) - mae = "" - if test_each_run: - t1_test = time.time() - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - # decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - t2_test = time.time() - test_time = round(t2_test - t1_test, 3) - f.close() - df = pd.read_csv(fname) - mae = mean_absolute_error(df["target"], df["predictions"]) - if mae == "": - print( - "Epoch, train loss, val loss, train_time, val_time", - epoch, - train_loss, - val_loss, - train_time, - val_time, - saving_tag, - ) - else: - print( - "Epoch, train loss, val loss, test loss, train_time, val_time, test_time", - epoch, - train_loss, - val_loss, - mae, - train_time, - val_time, - test_time, - saving_tag, - ) - - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - # decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - f.close() - tot_time_end = time.time() - tot_time = tot_time_end - tot_time_start - print("tot_time", tot_time) - - -if __name__ == "__main__": - # box = [[2.715, 2.715, 0], [0, 2.715, 2.715], [2.715, 0, 2.715]] - # coords = [[0, 0, 0], [0.25, 0.2, 0.25]] - # elements = ["Si", "Si"] - # Si = Atoms(lattice_mat=box, coords=coords, elements=elements) - # tmp=atoms_describer(Si) - # print(tmp) - # import sys - # sys.exit() - args = parser.parse_args(sys.argv[1:]) - benchmark_file = args.benchmark_file - model_name = "facebook/opt-350m" - model_name = "mistralai/Mixtral-8x7B-v0.1" - model_name = "google/flan-t5-small" - model_name = "google/flan-t5-base" - model_name = "mistralai/Mistral-7B-Instruct-v0.1" - model_name = "xlnet/xlnet-base-cased" - model_name = "afmck/testing-llama-tiny" - model_name = "EleutherAI/gpt-neo-125m" - model_name = "meta-llama/Llama-2-7b-hf" - model_name = "stas/tiny-random-llama-2" - model_name = "ahxt/llama2_xs_460M_experimental" - model_name = "google-t5/t5-small" - model_name = "openai-community/gpt2-medium" - model_name = "gpt2" - run_atomgpt( - model_name=model_name, - benchmark_file=benchmark_file, - # num_epochs=300, - # pretrained_path="xyz_out_google/flan-t5-small_tinnet_N_ead/best_model.pt", - # pretrained_path="ss_out_google/flan-t5-small_tinnet_N_ead/best_model.pt", - prefix="xyzt6", - # batch_size=5, - max_length=512, - # max_length=256, - batch_size=16, - latent_dim=1000, - # latent_dim=1024, - num_epochs=5000, - # batch_size=16 - ) - import sys - - sys.exit() - latent_dims = [ - 128, - 256, - 512, - 800, - 1024, - 1200, - 1500, - 2048, - 2500, - 3000, - 3500, - 4000, - ] - for i in latent_dims: - prefix = "lat_lat_" + str(i) - print(prefix) - run_atomgpt( - model_name=model_name, - benchmark_file=benchmark_file, - prefix=prefix, - batch_size=16, - latent_dim=i, - num_epochs=150, - ) - max_lengths = [128, 256, 512, 640, 768, 896, 1000] - for i in max_lengths: - prefix = "max_lengt_" + str(i) - print(prefix) - run_atomgpt( - model_name=model_name, - benchmark_file=benchmark_file, - prefix=prefix, - batch_size=16, - max_length=i, - num_epochs=150, - ) diff --git a/atomgpt/scripts/finetune7a.py.bak b/atomgpt/scripts/finetune7a.py.bak deleted file mode 100644 index d59d880..0000000 --- a/atomgpt/scripts/finetune7a.py.bak +++ /dev/null @@ -1,815 +0,0 @@ -"""Module for fin tuning LLM model for materials chemsitry.""" - -from jarvis.db.figshare import data -import transformers -import torch -import random -from jarvis.db.jsonutils import dumpjson -import sys -import argparse -from torch.utils.data import DataLoader, Dataset -import numpy as np -import os -from jarvis.core.atoms import Atoms -import pandas as pd -from sklearn.metrics import mean_absolute_error - -# from describe import atoms_describer -import json -from jarvis.db.figshare import get_jid_data -from jarvis.core.atoms import Atoms -from jarvis.analysis.structure.spacegroup import Spacegroup3D -from jarvis.analysis.diffraction.xrd import XRD -from jarvis.core.specie import Specie -import pprint -from collections import defaultdict - -from tqdm import tqdm -import time -import json -import zipfile -from transformers import GPT2Config, GPT2Model, GPT2Tokenizer - -device = "cpu" -if torch.cuda.is_available(): - device = torch.device("cuda") -parser = argparse.ArgumentParser(description="AtomGPT") -parser.add_argument( - "--benchmark_file", - default="AI-SinglePropertyPrediction-PBE_gap-halide_peroskites-test-mae.csv.zip", - # default="AI-SinglePropertyPrediction-ead-tinnet_N-test-mae.csv.zip", - # default="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae", - help="Benchmarks available in jarvis_leaderboard/benchmarks/*/*.zip", -) - - -def atoms_describer( - atoms=[], - xrd_peaks=5, - xrd_round=1, - cutoff=4, - take_n_bomds=2, - include_spg=True, -): - """Describe an atomic structure.""" - if include_spg: - spg = Spacegroup3D(atoms) - theta, d_hkls, intens = XRD().simulate(atoms=(atoms)) - # x = atoms.atomwise_angle_and_radial_distribution() - # bond_distances = {} - # for i, j in x[-1]["different_bond"].items(): - # bond_distances[i.replace("_", "-")] = ", ".join( - # map(str, (sorted(list(set([round(jj, 2) for jj in j]))))) - # ) - dists = defaultdict(list) - elements = atoms.elements - for i in atoms.get_all_neighbors(r=cutoff): - for j in i: - key = "-".join(sorted([elements[j[0]], elements[j[1]]])) - dists[key].append(j[2]) - bond_distances = {} - for i, j in dists.items(): - dist = sorted(set([round(k, 2) for k in j])) - if len(dist) >= take_n_bomds: - dist = dist[0:take_n_bomds] - bond_distances[i] = ", ".join(map(str, dist)) - fracs = {} - for i, j in (atoms.composition.atomic_fraction).items(): - fracs[i] = round(j, 3) - info = {} - chem_info = { - "atomic_formula": atoms.composition.reduced_formula, - "prototype": atoms.composition.prototype, - "molecular_weight": round(atoms.composition.weight / 2, 2), - "atomic_fraction": (fracs), - "atomic_X": ", ".join( - map(str, [Specie(s).X for s in atoms.uniq_species]) - ), - "atomic_Z": ", ".join( - map(str, [Specie(s).Z for s in atoms.uniq_species]) - ), - } - struct_info = { - "lattice_parameters": ", ".join( - map(str, [round(j, 2) for j in atoms.lattice.abc]) - ), - "lattice_angles": ", ".join( - map(str, [round(j, 2) for j in atoms.lattice.angles]) - ), - # "spg_number": spg.space_group_number, - # "spg_symbol": spg.space_group_symbol, - "top_k_xrd_peaks": ", ".join( - map( - str, - sorted(list(set([round(i, xrd_round) for i in theta])))[ - 0:xrd_peaks - ], - ) - ), - "density": round(atoms.density, 3), - # "crystal_system": spg.crystal_system, - # "point_group": spg.point_group_symbol, - # "wyckoff": ", ".join(list(set(spg._dataset["wyckoffs"]))), - "bond_distances": bond_distances, - # "natoms_primitive": spg.primitive_atoms.num_atoms, - # "natoms_conventional": spg.conventional_standard_structure.num_atoms, - } - if include_spg: - struct_info["spg_number"] = spg.space_group_number - struct_info["spg_symbol"] = spg.space_group_symbol - struct_info["crystal_system"] = spg.crystal_system - struct_info["point_group"] = spg.point_group_symbol - struct_info["wyckoff"] = ", ".join(list(set(spg._dataset["wyckoffs"]))) - struct_info["natoms_primitive"] = spg.primitive_atoms.num_atoms - struct_info[ - "natoms_conventional" - ] = spg.conventional_standard_structure.num_atoms - info["chemical_info"] = chem_info - info["structure_info"] = struct_info - line = "The number of atoms are: " + str( - atoms.num_atoms - ) # +"., The elements are: "+",".join(atoms.elements)+". " - for i, j in info.items(): - if not isinstance(j, dict): - line += "The " + i + " is " + j + ". " - else: - # print("i",i) - # print("j",j) - for ii, jj in j.items(): - tmp = "" - if isinstance(jj, dict): - for iii, jjj in jj.items(): - tmp += iii + ": " + str(jjj) + " " - else: - tmp = jj - line += "The " + ii + " is " + str(tmp) + ". " - return line - - -def set_seed(): - os.environ["WANDB_ANONYMOUS"] = "must" - random_seed = 42 - random.seed(random_seed) - torch.manual_seed(random_seed) - np.random.seed(random_seed) - torch.cuda.manual_seed_all(random_seed) - try: - import torch_xla.core.xla_model as xm - - xm.set_rng_state(random_seed) - except ImportError: - pass - torch.backends.cudnn.deterministic = True - torch.backends.cudnn.benchmark = False - os.environ["PYTHONHASHSEED"] = str(random_seed) - os.environ["CUBLAS_WORKSPACE_CONFIG"] = str(":4096:8") - torch.use_deterministic_algorithms(True) - - -def get_crystal_string_1225(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.1f}".format(x) for x in lengths]) - + "\n" - + " ".join([str(int(x)) for x in angles]) - + "\n" - + "\n".join( - [ - str(t) + " " + " ".join(["{0:.2f}".format(x) for x in c]) - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - extra = str(atoms.num_atoms) + "\n" + atoms.composition.reduced_formula - # crystal_str+=" "+extra - extra += "\n" + crystal_str - return extra - # return crystal_str - - -def get_crystal_string_t(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.2f}".format(x) for x in lengths]) - + "#\n" - + " ".join([str(int(x)) for x in angles]) - + "@\n" - + "\n".join( - [ - str(t) + " " + " ".join(["{0:.3f}".format(x) for x in c]) + "&" - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - crystal_str = atoms_describer(atoms) + "\n*\n" + crystal_str - return crystal_str - - -class AtomGPTDataset(Dataset): - def __init__( - self, texts=[], targets=[], ids=[], tokenizer="", max_length=128 - ): - self.texts = texts - self.targets = targets - self.tokenizer = tokenizer - self.max_length = max_length - if not ids: - ids = ["text-" + str(i) for i in range(len(texts))] - self.ids = ids - - def __len__(self): - return len(self.texts) - - def __getitem__(self, idx): - inputs = self.tokenizer( - self.texts[idx], - return_tensors="pt", - max_length=self.max_length, - padding="max_length", - truncation=True, - ) - return ( - inputs, - self.ids[idx], - torch.tensor(self.targets[idx], dtype=torch.float32), - ) - - -class ForcePredictor(torch.nn.Module): - config = GPT2Config.from_pretrained("gpt2") - - def __init__(self, gpt2_model): - super(ForcePredictor, self).__init__() - self.gpt2 = gpt2_model - self.linear = torch.nn.Linear( - config.n_embd, 1 - ) # Assuming force is a 3D vector - - def forward(self, input_ids): - outputs = self.gpt2(input_ids) - last_hidden_states = outputs.last_hidden_state - force_pred = self.linear(last_hidden_states[:, -1, :]) - # print("force_pred",outputs.keys()) - return force_pred - - -class AtomGPTPredictorLMhead(torch.nn.Module): - def __init__( - self, model_name=None, n_out=1, latent_dim=1024, tokenizer="" - ): - - super(AtomGPTPredictorLMhead, self).__init__() - #random_seed = 42 - #random.seed(random_seed) - #torch.manual_seed(random_seed) - #np.random.seed(random_seed) - #torch.cuda.manual_seed_all(random_seed) - #try: - # import torch_xla.core.xla_model as xm - # xm.set_rng_state(random_seed) - #except ImportError: - # pass - #torch.backends.cudnn.deterministic = True - #torch.backends.cudnn.benchmark = False - #os.environ["PYTHONHASHSEED"] = str(random_seed) - #os.environ["CUBLAS_WORKSPACE_CONFIG"] = str(":4096:8") - #torch.use_deterministic_algorithms(True) - - self.model_name = model_name - self.n_out = n_out - self.latent_dim = latent_dim - if "t5" in model_name: - model = transformers.T5ForConditionalGeneration.from_pretrained( - model_name - ) - else: - # config = GPT2Config.from_pretrained("gpt2") - # model = GPT2Model(config) - model = transformers.AutoModelForCausalLM.from_pretrained( - model_name, - low_cpu_mem_usage=True, - # load_in_8bit=False, - # torch_dtype=torch.float16, - # load_in_8bit=True, - # device_map="auto" - ) - model.resize_token_embeddings(len(tokenizer)) - self.config = model.config - model.lm_head = torch.nn.Sequential( - torch.nn.Linear(model.config.hidden_size, latent_dim), - torch.nn.Linear(latent_dim, n_out), - ) - self.model = model - - def forward(self, input_ids): - # outputs = self.model(input_ids) - if "t5" in model_name: - outputs = self.model(input_ids, decoder_input_ids=input_ids) - else: - outputs = self.model(input_ids) - return outputs - - -class AtomGPTPredictorHiddenFeats(torch.nn.Module): - def __init__(self, model_name=None, n_out=1, tokenizer=""): - super(AtomGPTPredictorHiddenFeats, self).__init__() - if "t5" in model_name: - model = transformers.T5ForConditionalGeneration.from_pretrained( - model_name - ) - else: - model = transformers.AutoModelForCausalLM.from_pretrained( - model_name, - low_cpu_mem_usage=True, - # load_in_8bit=False, - # torch_dtype=torch.float16, - # load_in_8bit=True, - # device_map="auto" - ) - model.resize_token_embeddings(len(tokenizer)) - self.model = model - self.config = self.model.config - self.global_out = torch.nn.Linear(self.config.n_embd, n_out) - - def forward(self, input_ids): - outputs = self.model(input_ids) - print("outputs", outputs.keys()) - last_hidden_states = outputs.last_hidden_state - pred = self.linear(last_hidden_states[:, -1, :]) - return pred - - -def run_atomgpt( - prefix="ss", - model_name="gpt2", - benchmark_file="AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip", - root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard", - batch_size=8, - max_length=512, - num_epochs=500, - latent_dim=512, - learning_rate=1e-3, - # learning_rate=1e-3, - test_each_run=True, - # learning_rate=5e-5, - pretrained_path="", -): - # Load pre-trained tokenizer - # tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - # model = GPT2LMHeadModel.from_pretrained("gpt2") - - # dft_3d = data("dft_3d") - # root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard" - # benchmark_file="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae.csv.zip" - # benchmark_file = "AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip" - print("benchmark_file", benchmark_file) - method = benchmark_file.split("-")[0] - task = benchmark_file.split("-")[1] - prop = benchmark_file.split("-")[2] - dataset = benchmark_file.split("-")[3] - temp = dataset + "_" + prop + ".json.zip" - temp2 = dataset + "_" + prop + ".json" - fname = os.path.join(root_dir, "benchmarks", method, task, temp) - zp = zipfile.ZipFile(fname) - bench = json.loads(zp.read(temp2)) - dft_3d = data(dataset) - id_tag = "jid" - if "jid" in dft_3d[0]: - id_tag = "jid" - else: - id_tag = "id" - - # train_atoms = [] - # val_atoms = [] - # test_atoms = [] - # train_targets = [] - # val_targets = [] - # test_targets = [] - train_ids = list(bench["train"].keys()) - test_ids = list(bench["test"].keys()) - if "val" in bench: - val_ids = list(bench["val"].keys()) - else: - val_ids = test_ids - print("total", len(dft_3d)) - print("test_ids", len(test_ids)) - print("val_ids", len(val_ids)) - print("train_ids", len(train_ids)) - # model_name = "mistralai/Mistral-7B-Instruct-v0.1" - # model_name = "gpt2" - # device = model.device - if "t5" in model_name: - tokenizer = transformers.T5Tokenizer.from_pretrained(model_name) - - else: - tokenizer = transformers.AutoTokenizer.from_pretrained(model_name) - # print('Non tokenizer format') - # tokenizer = GPT2Tokenizer.from_pretrained(model_name) - # config = GPT2Config.from_pretrained("gpt2") - # model = GPT2Model(config) - if tokenizer.pad_token is None: - tokenizer.add_special_tokens({"pad_token": "[PAD]"}) - tokenizer.add_special_tokens({"unk_token": "#"}) - tokenizer.add_special_tokens({"unk_token": "&"}) - tokenizer.add_special_tokens({"unk_token": "@"}) - # model.resize_token_embeddings(len(tokenizer)) - # model=ForcePredictor(model) - # model=AtomGPTPredictorHiddenFeats(model_name=model_name, tokenizer=tokenizer) - set_seed() - model = AtomGPTPredictorLMhead(model_name=model_name, tokenizer=tokenizer,latent_dim=latent_dim) - # tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - # batch_size = 16 - # max_length = 128 - # num_epochs = 100 - # learning_rate = 5e-5 - criterion = torch.nn.L1Loss() - # Define example regression data (texts and corresponding numeric targets) - train_texts = [] - train_targets = [] - train_ids_temp = [] - val_texts = [] - val_targets = [] - val_ids_temp = [] - test_texts = [] - test_targets = [] - test_ids_temp = [] - - for i in tqdm(dft_3d): - if i[prop] != "na": - atoms = Atoms.from_dict(i["atoms"]) - tmp = get_crystal_string_t(atoms) - if i[id_tag] in train_ids: - train_texts.append(tmp) - train_targets.append(i[prop]) - train_ids_temp.append(i[id_tag]) - elif i[id_tag] in test_ids: - test_texts.append(tmp) - test_targets.append(i[prop]) - test_ids_temp.append(i[id_tag]) - elif i[id_tag] in val_ids: - val_texts.append(tmp) - val_targets.append(i[prop]) - val_ids_temp.append(i[id_tag]) - print("test_texts:", len(test_texts)) - print("test_texts:", test_texts[0]) - """ - ############################## - ###Fast test### - train_texts = [ - "This is the first example text.", - "Second example is a bit longer than the first one, but still within the max length.", - "Third example is the longest among these three examples. It exceeds the max length and will be truncated.", - "Second example is a bit longer than the first one, but still within the max length.", - ] - train_targets = [10.2, 15.5, 20.1, 15.5] # Example regression targets - val_texts = test_texts = train_texts - val_targets = test_targets = train_targets - train_ids_temp=['a','b','c','d'] - val_ids_temp = test_ids_temp = train_ids_temp - batch_size = 2 - num_epochs = 3 - - ############################## - ############################## - """ - - # Fine-tune the last layer of GPT-2 for regression - # fine_tune_gpt2_regression(train_texts, train_targets, tokenizer) - if pretrained_path != "": - model.load_state_dict(torch.load(pretrained_path, map_location=device)) - # model.lm_head = torch.nn.Sequential(torch.nn.Linear( model.config.hidden_size, 256),torch.nn.SiLU(),torch.nn.Linear( 256, 1) ) - # set_seed(seed) - # set_deterministic() - model.to(device) - if torch.cuda.device_count() > 1: - device_ids = [d for d in range(torch.cuda.device_count())] - model = torch.nn.DataParallel(model, device_ids=device_ids).cuda() - optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate) - #optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate) - #optimizer = transformers.AdamW(model.parameters(), lr=learning_rate) - # Prepare datasets and dataloaders with data collator - train_dataset = AtomGPTDataset( - texts=train_texts, - targets=train_targets, - ids=train_ids_temp, - tokenizer=tokenizer, - max_length=max_length, - ) - val_dataset = AtomGPTDataset( - texts=val_texts, - targets=val_targets, - tokenizer=tokenizer, - ids=val_ids_temp, - max_length=max_length, - ) - test_dataset = AtomGPTDataset( - texts=test_texts, - targets=test_targets, - tokenizer=tokenizer, - ids=test_ids_temp, - max_length=max_length, - ) - - # val_dataset = train_dataset - train_dataloader = DataLoader(train_dataset, batch_size=batch_size) - val_dataloader = DataLoader(val_dataset, batch_size=batch_size) - test_dataloader = DataLoader(test_dataset, batch_size=batch_size) - val_dataloader = test_dataloader - steps_per_epoch = len(train_dataloader) - scheduler = torch.optim.lr_scheduler.OneCycleLR( - optimizer, - max_lr=learning_rate, - epochs=num_epochs, - steps_per_epoch=steps_per_epoch, - # pct_start=pct_start, - pct_start=0.3, - ) - # scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10, eta_min=0.001) - # scheduler = torch.optim.lr_scheduler.StepLR( - # optimizer, - # step_size=30, - # ) - print("train_data", len(train_texts)) - print("test_data", len(test_texts)) - output_dir = prefix + "_out_" + model_name + "_" + dataset + "_" + prop - if not os.path.exists(output_dir): - os.makedirs(output_dir) - best_loss = np.inf - tot_time_start = time.time() - train_history = [] - val_history = [] - for epoch in range(num_epochs): - model.train() - t1 = time.time() - for batch in train_dataloader: - optimizer.zero_grad() - train_loss = 0 - # train_result = [] - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - # if 't5' in model_name: - # decoder_input_ids = tokenizer("", return_tensors="pt").input_ids.to(device) - # decoder_input_ids = model._shift_right(decoder_input_ids) - # predictions = ( - # model(input_ids = input_ids.to(device),decoder_input_ids=decoder_input_ids).logits.squeeze().mean(dim=-1) - # ) - # else: - # predictions = ( - # model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - # ) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - # decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - targets = batch[2].squeeze() - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - # print('train',predictions,targets) - loss.backward() - optimizer.step() - scheduler.step() - # optimizer.zero_grad() - train_loss += loss.item() - train_loss = train_loss / len(train_dataloader) - t2 = time.time() - train_time = round(t2 - t1, 3) - model.eval() - - # total_eval_mae_loss = 0 - # predictions_list = [] - # targets_list = [] - val_loss = 0 - t1 = time.time() - for batch in val_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - # decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - targets = batch[2].squeeze() - # print('val',predictions,targets) - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - val_loss += loss.item() - saving_tag = "" - if val_loss < best_loss: - best_loss = val_loss - best_model_name = "best_model.pt" - torch.save( - model.state_dict(), - os.path.join(output_dir, best_model_name), - ) - # print("Saving model for epoch", epoch) - saving_tag = " saving model:" + str(epoch) - val_loss = val_loss / len(val_dataloader) - t2 = time.time() - val_time = round(t2 - t1, 3) - train_history.append(train_loss) - val_history.append(val_loss) - history = os.path.join(output_dir, "history.json") - - dumpjson( - data={"train": train_history, "val": val_history}, filename=history - ) - mae = "" - if test_each_run: - t1_test = time.time() - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - # decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - t2_test = time.time() - test_time = round(t2_test - t1_test, 3) - f.close() - df = pd.read_csv(fname) - mae = mean_absolute_error(df["target"], df["predictions"]) - if mae == "": - print( - "Epoch, train loss, val loss, train_time, val_time", - epoch, - train_loss, - val_loss, - train_time, - val_time, - saving_tag, - ) - else: - print( - "Epoch, train loss, val loss, test loss, train_time, val_time, test_time", - epoch, - train_loss, - val_loss, - mae, - train_time, - val_time, - test_time, - saving_tag, - ) - - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - # decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - f.close() - tot_time_end = time.time() - tot_time = tot_time_end - tot_time_start - print("tot_time", tot_time) - - -if __name__ == "__main__": - # box = [[2.715, 2.715, 0], [0, 2.715, 2.715], [2.715, 0, 2.715]] - # coords = [[0, 0, 0], [0.25, 0.2, 0.25]] - # elements = ["Si", "Si"] - # Si = Atoms(lattice_mat=box, coords=coords, elements=elements) - # tmp=atoms_describer(Si) - # print(tmp) - # import sys - # sys.exit() - args = parser.parse_args(sys.argv[1:]) - benchmark_file = args.benchmark_file - model_name = "facebook/opt-350m" - model_name = "mistralai/Mixtral-8x7B-v0.1" - model_name = "google/flan-t5-small" - model_name = "google/flan-t5-base" - model_name = "mistralai/Mistral-7B-Instruct-v0.1" - model_name = "xlnet/xlnet-base-cased" - model_name = "afmck/testing-llama-tiny" - model_name = "EleutherAI/gpt-neo-125m" - model_name = "meta-llama/Llama-2-7b-hf" - model_name = "stas/tiny-random-llama-2" - model_name = "ahxt/llama2_xs_460M_experimental" - model_name = "google-t5/t5-small" - model_name = "openai-community/gpt2-medium" - model_name = "gpt2" - run_atomgpt( - model_name=model_name, - benchmark_file=benchmark_file, - # num_epochs=300, - # pretrained_path="xyz_out_google/flan-t5-small_tinnet_N_ead/best_model.pt", - # pretrained_path="ss_out_google/flan-t5-small_tinnet_N_ead/best_model.pt", - prefix="xyzt6", - # batch_size=5, - batch_size=16, - latent_dim=1024, - num_epochs=5000, - # batch_size=16 - ) - import sys - sys.exit() - latent_dims=[128,256,512,800,1024,1200,1500,2048,2500,3000,3500,4000] - for i in latent_dims: - prefix='lat_lat_'+str(i) - print(prefix) - run_atomgpt(model_name=model_name,benchmark_file=benchmark_file,prefix=prefix,batch_size=16,latent_dim=i,num_epochs=150) - diff --git a/atomgpt/scripts/finetune7alignn.py b/atomgpt/scripts/finetune7alignn.py deleted file mode 100644 index 9e02bb0..0000000 --- a/atomgpt/scripts/finetune7alignn.py +++ /dev/null @@ -1,907 +0,0 @@ -"""Module for fin tuning LLM model for materials chemsitry.""" - -from jarvis.db.figshare import data -import transformers -import torch -import random -from jarvis.db.jsonutils import loadjson,dumpjson -import sys -import argparse -from torch.utils.data import DataLoader, Dataset -import numpy as np -import os -from jarvis.core.atoms import Atoms -import pandas as pd -from sklearn.metrics import mean_absolute_error - -# from describe import atoms_describer -import json -from jarvis.db.figshare import get_jid_data -from jarvis.core.atoms import Atoms -from jarvis.analysis.structure.spacegroup import Spacegroup3D -from jarvis.analysis.diffraction.xrd import XRD -from jarvis.core.specie import Specie -import pprint -from collections import defaultdict - -from tqdm import tqdm -import time -import json -import zipfile -from transformers import GPT2Config, GPT2Model, GPT2Tokenizer -from jarvis.core.atoms import Atoms - -#from alignn.graphs import Graph - -#from alignn.pretrained import get_figshare_model - -import torch - -device = "cpu" -if torch.cuda.is_available(): - device = torch.device("cuda") - - -#model_alignn = get_figshare_model() -#model_alignn.to(device) - - - -def get_val(model, g, lg): - activation = {} - def getActivation(name): - # the hook signature - def hook(model, input, output): - activation[name] = output.detach() - return hook - h = model.readout.register_forward_hook(getActivation("readout")) - out = model([g, lg]) - h.remove() - return activation["readout"][0] - -df_afeats=pd.DataFrame(loadjson("feats.json")) - -def get_alignn_feats(jid=''): - return df_afeats[df_afeats['id']==jid]['pred'].values[0] - -def get_alignn_feats_1(model_alignn='',atoms=''): - g, lg = Graph.atom_dgl_multigraph(atoms) - x = get_val(model_alignn, g.to(device), lg.to(device)) - return x - -parser = argparse.ArgumentParser(description="AtomGPT") -parser.add_argument( - "--benchmark_file", - #default="AI-SinglePropertyPrediction-PBE_gap-halide_peroskites-test-mae.csv.zip", - # default="AI-SinglePropertyPrediction-ead-tinnet_N-test-mae.csv.zip", - default="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae", - help="Benchmarks available in jarvis_leaderboard/benchmarks/*/*.zip", -) - - -def atoms_describer( - atoms=[], - xrd_peaks=5, - xrd_round=1, - cutoff=4, - take_n_bomds=2, - include_spg=True, -): - """Describe an atomic structure.""" - if include_spg: - spg = Spacegroup3D(atoms) - theta, d_hkls, intens = XRD().simulate(atoms=(atoms)) - # x = atoms.atomwise_angle_and_radial_distribution() - # bond_distances = {} - # for i, j in x[-1]["different_bond"].items(): - # bond_distances[i.replace("_", "-")] = ", ".join( - # map(str, (sorted(list(set([round(jj, 2) for jj in j]))))) - # ) - dists = defaultdict(list) - elements = atoms.elements - for i in atoms.get_all_neighbors(r=cutoff): - for j in i: - key = "-".join(sorted([elements[j[0]], elements[j[1]]])) - dists[key].append(j[2]) - bond_distances = {} - for i, j in dists.items(): - dist = sorted(set([round(k, 2) for k in j])) - if len(dist) >= take_n_bomds: - dist = dist[0:take_n_bomds] - bond_distances[i] = ", ".join(map(str, dist)) - fracs = {} - for i, j in (atoms.composition.atomic_fraction).items(): - fracs[i] = round(j, 3) - info = {} - chem_info = { - "atomic_formula": atoms.composition.reduced_formula, - "prototype": atoms.composition.prototype, - "molecular_weight": round(atoms.composition.weight / 2, 2), - "atomic_fraction": (fracs), - "atomic_X": ", ".join( - map(str, [Specie(s).X for s in atoms.uniq_species]) - ), - "atomic_Z": ", ".join( - map(str, [Specie(s).Z for s in atoms.uniq_species]) - ), - } - struct_info = { - "lattice_parameters": ", ".join( - map(str, [round(j, 2) for j in atoms.lattice.abc]) - ), - "lattice_angles": ", ".join( - map(str, [round(j, 2) for j in atoms.lattice.angles]) - ), - # "spg_number": spg.space_group_number, - # "spg_symbol": spg.space_group_symbol, - "top_k_xrd_peaks": ", ".join( - map( - str, - sorted(list(set([round(i, xrd_round) for i in theta])))[ - 0:xrd_peaks - ], - ) - ), - "density": round(atoms.density, 3), - # "crystal_system": spg.crystal_system, - # "point_group": spg.point_group_symbol, - # "wyckoff": ", ".join(list(set(spg._dataset["wyckoffs"]))), - "bond_distances": bond_distances, - # "natoms_primitive": spg.primitive_atoms.num_atoms, - # "natoms_conventional": spg.conventional_standard_structure.num_atoms, - } - if include_spg: - struct_info["spg_number"] = spg.space_group_number - struct_info["spg_symbol"] = spg.space_group_symbol - struct_info["crystal_system"] = spg.crystal_system - struct_info["point_group"] = spg.point_group_symbol - struct_info["wyckoff"] = ", ".join(list(set(spg._dataset["wyckoffs"]))) - struct_info["natoms_primitive"] = spg.primitive_atoms.num_atoms - struct_info[ - "natoms_conventional" - ] = spg.conventional_standard_structure.num_atoms - info["chemical_info"] = chem_info - info["structure_info"] = struct_info - line = "The number of atoms are: " + str( - atoms.num_atoms - ) # +"., The elements are: "+",".join(atoms.elements)+". " - for i, j in info.items(): - if not isinstance(j, dict): - line += "The " + i + " is " + j + ". " - else: - # print("i",i) - # print("j",j) - for ii, jj in j.items(): - tmp = "" - if isinstance(jj, dict): - for iii, jjj in jj.items(): - tmp += iii + ": " + str(jjj) + " " - else: - tmp = jj - line += "The " + ii + " is " + str(tmp) + ". " - return line - - -def set_seed(): - os.environ["WANDB_ANONYMOUS"] = "must" - random_seed = 42 - random.seed(random_seed) - torch.manual_seed(random_seed) - np.random.seed(random_seed) - torch.cuda.manual_seed_all(random_seed) - try: - import torch_xla.core.xla_model as xm - - xm.set_rng_state(random_seed) - except ImportError: - pass - torch.backends.cudnn.deterministic = True - torch.backends.cudnn.benchmark = False - os.environ["PYTHONHASHSEED"] = str(random_seed) - os.environ["CUBLAS_WORKSPACE_CONFIG"] = str(":4096:8") - torch.use_deterministic_algorithms(True) - - -def get_crystal_string_t(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.2f}".format(x) for x in lengths]) - + "#\n" - + " ".join([str(int(x)) for x in angles]) - + "@\n" - + "\n".join( - [ - str(t) + " " + " ".join(["{0:.3f}".format(x) for x in c]) + "&" - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - crystal_str = atoms_describer(atoms) + "\n*\n" + crystal_str - return crystal_str - - -class AtomGPTDataset(Dataset): - def __init__( - self, - texts=[], - targets=[], - ids=[], - extra_feats=[], - tokenizer="", - max_length=128, - ): - self.texts = texts - self.targets = targets - self.tokenizer = tokenizer - self.max_length = max_length - self.extra_feats = extra_feats - if not ids: - ids = ["text-" + str(i) for i in range(len(texts))] - self.ids = ids - - def __len__(self): - return len(self.texts) - - def __getitem__(self, idx): - inputs = self.tokenizer( - self.texts[idx], - return_tensors="pt", - max_length=self.max_length, - padding="max_length", - truncation=True, - )['input_ids'].squeeze() - feats=torch.empty(1) - if self.extra_feats: - #print('fts',self.extra_feats[idx]) - feats = torch.tensor(np.array(self.extra_feats[idx]),dtype=torch.float32) - #print('feats',feats.shape) - #print('inputs',inputs,type(inputs)) - #inputs = torch.cat((inputs, feats),0) - return ( - inputs, - self.ids[idx], - feats, - torch.tensor(self.targets[idx], dtype=torch.float32), - - ) - - -class ForcePredictor(torch.nn.Module): - config = GPT2Config.from_pretrained("gpt2") - - def __init__(self, gpt2_model): - super(ForcePredictor, self).__init__() - self.gpt2 = gpt2_model - self.linear = torch.nn.Linear( - config.n_embd, 1 - ) # Assuming force is a 3D vector - - def forward(self, input_ids): - outputs = self.gpt2(input_ids) - last_hidden_states = outputs.last_hidden_state - force_pred = self.linear(last_hidden_states[:, -1, :]) - # print("force_pred",outputs.keys()) - return force_pred - - -class AtomGPTPredictorLMhead(torch.nn.Module): - def __init__( - self, model_name=None, n_out=1, latent_dim=1024,n_feats=0, tokenizer="" - ): - - super(AtomGPTPredictorLMhead, self).__init__() - # random_seed = 42 - # random.seed(random_seed) - # torch.manual_seed(random_seed) - # np.random.seed(random_seed) - # torch.cuda.manual_seed_all(random_seed) - # try: - # import torch_xla.core.xla_model as xm - # xm.set_rng_state(random_seed) - # except ImportError: - # pass - # torch.backends.cudnn.deterministic = True - # torch.backends.cudnn.benchmark = False - # os.environ["PYTHONHASHSEED"] = str(random_seed) - # os.environ["CUBLAS_WORKSPACE_CONFIG"] = str(":4096:8") - # torch.use_deterministic_algorithms(True) - - self.model_name = model_name - self.n_out = n_out - self.latent_dim = latent_dim - self.n_feats=n_feats - if "t5" in model_name: - model = transformers.T5ForConditionalGeneration.from_pretrained( - model_name - ) - else: - # config = GPT2Config.from_pretrained("gpt2") - # model = GPT2Model(config) - model = transformers.AutoModelForCausalLM.from_pretrained( - model_name, - low_cpu_mem_usage=True, - # load_in_8bit=False, - # torch_dtype=torch.float16, - # load_in_8bit=True, - # device_map="auto" - ) - model.resize_token_embeddings(len(tokenizer)) - self.config = model.config - model.lm_head = torch.nn.Linear(model.config.hidden_size, latent_dim) - self.out = torch.nn.Linear(latent_dim, n_out) - if self.n_feats>0: - self.feature_layer = torch.nn.Linear(self.n_feats, self.latent_dim) - self.model = model - - def forward(self, input_ids,feats=[]): - # outputs = self.model(input_ids) - if "t5" in model_name: - outputs = self.model(input_ids, decoder_input_ids=input_ids) - else: - outputs = self.model(input_ids) - if self.n_feats>0: - feature_embedding = self.feature_layer(feats) - outputs+=feature_embedding - out=self.out(outputs,self.n_out) - return out - - -class AtomGPTPredictorHiddenFeats(torch.nn.Module): - def __init__(self, model_name=None, n_out=1, tokenizer=""): - super(AtomGPTPredictorHiddenFeats, self).__init__() - if "t5" in model_name: - model = transformers.T5ForConditionalGeneration.from_pretrained( - model_name - ) - else: - model = transformers.AutoModelForCausalLM.from_pretrained( - model_name, - low_cpu_mem_usage=True, - # load_in_8bit=False, - # torch_dtype=torch.float16, - # load_in_8bit=True, - # device_map="auto" - ) - model.resize_token_embeddings(len(tokenizer)) - self.model = model - self.config = self.model.config - self.global_out = torch.nn.Linear(self.config.n_embd, n_out) - - def forward(self, input_ids): - outputs = self.model(input_ids) - print("outputs", outputs.keys()) - last_hidden_states = outputs.last_hidden_state - pred = self.linear(last_hidden_states[:, -1, :]) - return pred - - -def run_atomgpt( - prefix="ss", - model_name="gpt2", - benchmark_file="AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip", - root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard", - batch_size=8, - max_length=512, - num_epochs=500, - latent_dim=512, - learning_rate=1e-3, - # learning_rate=1e-3, - test_each_run=True, - # learning_rate=5e-5, - pretrained_path="", -): - # Load pre-trained tokenizer - # tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - # model = GPT2LMHeadModel.from_pretrained("gpt2") - - # dft_3d = data("dft_3d") - # root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard" - # benchmark_file="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae.csv.zip" - # benchmark_file = "AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip" - print("benchmark_file", benchmark_file) - method = benchmark_file.split("-")[0] - task = benchmark_file.split("-")[1] - prop = benchmark_file.split("-")[2] - dataset = benchmark_file.split("-")[3] - temp = dataset + "_" + prop + ".json.zip" - temp2 = dataset + "_" + prop + ".json" - fname = os.path.join(root_dir, "benchmarks", method, task, temp) - zp = zipfile.ZipFile(fname) - bench = json.loads(zp.read(temp2)) - dft_3d = data(dataset) - id_tag = "jid" - if "jid" in dft_3d[0]: - id_tag = "jid" - else: - id_tag = "id" - - # train_atoms = [] - # val_atoms = [] - # test_atoms = [] - # train_targets = [] - # val_targets = [] - # test_targets = [] - train_ids = list(bench["train"].keys()) - test_ids = list(bench["test"].keys()) - if "val" in bench: - val_ids = list(bench["val"].keys()) - else: - val_ids = test_ids - print("total", len(dft_3d)) - print("test_ids", len(test_ids)) - print("val_ids", len(val_ids)) - print("train_ids", len(train_ids)) - # model_name = "mistralai/Mistral-7B-Instruct-v0.1" - # model_name = "gpt2" - # device = model.device - if "t5" in model_name: - tokenizer = transformers.T5Tokenizer.from_pretrained(model_name) - - else: - tokenizer = transformers.AutoTokenizer.from_pretrained(model_name) - # print('Non tokenizer format') - # tokenizer = GPT2Tokenizer.from_pretrained(model_name) - # config = GPT2Config.from_pretrained("gpt2") - # model = GPT2Model(config) - if tokenizer.pad_token is None: - tokenizer.add_special_tokens({"pad_token": "[PAD]"}) - tokenizer.add_special_tokens({"unk_token": "#"}) - tokenizer.add_special_tokens({"unk_token": "&"}) - tokenizer.add_special_tokens({"unk_token": "@"}) - # model.resize_token_embeddings(len(tokenizer)) - # model=ForcePredictor(model) - # model=AtomGPTPredictorHiddenFeats(model_name=model_name, tokenizer=tokenizer) - set_seed() - model = AtomGPTPredictorLMhead( - model_name=model_name, tokenizer=tokenizer, n_feats=256,latent_dim=latent_dim - ) - # tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - # batch_size = 16 - # max_length = 128 - # num_epochs = 100 - # learning_rate = 5e-5 - criterion = torch.nn.L1Loss() - # Define example regression data (texts and corresponding numeric targets) - train_texts = [] - train_targets = [] - train_ids_temp = [] - val_texts = [] - val_targets = [] - val_ids_temp = [] - test_texts = [] - test_targets = [] - test_ids_temp = [] - train_feats=[] - val_feats=[] - test_feats=[] - for i in tqdm(dft_3d): - if i[prop] != "na": - atoms = Atoms.from_dict(i["atoms"]) - tmp = get_crystal_string_t(atoms) - feat=get_alignn_feats(i[id_tag]) - if i[id_tag] in train_ids: - train_texts.append(tmp) - train_targets.append(i[prop]) - train_ids_temp.append(i[id_tag]) - train_feats.append(feat) - elif i[id_tag] in test_ids: - test_texts.append(tmp) - test_targets.append(i[prop]) - test_ids_temp.append(i[id_tag]) - test_feats.append(feat) - elif i[id_tag] in val_ids: - val_texts.append(tmp) - val_targets.append(i[prop]) - val_ids_temp.append(i[id_tag]) - val_feats.append(feat) - print("test_texts:", len(test_texts)) - print("test_texts:", test_texts[0]) - """ - ############################## - ###Fast test### - train_texts = [ - "This is the first example text.", - "Second example is a bit longer than the first one, but still within the max length.", - "Third example is the longest among these three examples. It exceeds the max length and will be truncated.", - "Second example is a bit longer than the first one, but still within the max length.", - ] - train_targets = [10.2, 15.5, 20.1, 15.5] # Example regression targets - val_texts = test_texts = train_texts - val_targets = test_targets = train_targets - train_ids_temp=['a','b','c','d'] - val_ids_temp = test_ids_temp = train_ids_temp - batch_size = 2 - num_epochs = 3 - - ############################## - ############################## - """ - - # Fine-tune the last layer of GPT-2 for regression - # fine_tune_gpt2_regression(train_texts, train_targets, tokenizer) - if pretrained_path != "": - model.load_state_dict(torch.load(pretrained_path, map_location=device)) - # model.lm_head = torch.nn.Sequential(torch.nn.Linear( model.config.hidden_size, 256),torch.nn.SiLU(),torch.nn.Linear( 256, 1) ) - # set_seed(seed) - # set_deterministic() - model.to(device) - if torch.cuda.device_count() > 1: - device_ids = [d for d in range(torch.cuda.device_count())] - model = torch.nn.DataParallel(model, device_ids=device_ids).cuda() - optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate) - # optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate) - # optimizer = transformers.AdamW(model.parameters(), lr=learning_rate) - # Prepare datasets and dataloaders with data collator - train_dataset = AtomGPTDataset( - texts=train_texts, - targets=train_targets, - ids=train_ids_temp, - tokenizer=tokenizer, - max_length=max_length, - extra_feats=train_feats, - ) - val_dataset = AtomGPTDataset( - texts=val_texts, - targets=val_targets, - tokenizer=tokenizer, - ids=val_ids_temp, - max_length=max_length, - extra_feats=val_feats, - ) - test_dataset = AtomGPTDataset( - texts=test_texts, - targets=test_targets, - tokenizer=tokenizer, - ids=test_ids_temp, - max_length=max_length, - extra_feats=test_feats, - ) - - # val_dataset = train_dataset - train_dataloader = DataLoader(train_dataset, batch_size=batch_size) - val_dataloader = DataLoader(val_dataset, batch_size=batch_size) - test_dataloader = DataLoader(test_dataset, batch_size=batch_size) - val_dataloader = test_dataloader - steps_per_epoch = len(train_dataloader) - scheduler = torch.optim.lr_scheduler.OneCycleLR( - optimizer, - max_lr=learning_rate, - epochs=num_epochs, - steps_per_epoch=steps_per_epoch, - # pct_start=pct_start, - pct_start=0.3, - ) - # scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10, eta_min=0.001) - # scheduler = torch.optim.lr_scheduler.StepLR( - # optimizer, - # step_size=30, - # ) - print("train_data", len(train_texts)) - print("test_data", len(test_texts)) - output_dir = prefix + "_out_" + model_name + "_" + dataset + "_" + prop - if not os.path.exists(output_dir): - os.makedirs(output_dir) - best_loss = np.inf - tot_time_start = time.time() - train_history = [] - val_history = [] - for epoch in range(num_epochs): - model.train() - t1 = time.time() - for batch in train_dataloader: - optimizer.zero_grad() - train_loss = 0 - # train_result = [] - input_ids = batch[0].squeeze() # .squeeze(0) - #input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - # if 't5' in model_name: - # decoder_input_ids = tokenizer("", return_tensors="pt").input_ids.to(device) - # decoder_input_ids = model._shift_right(decoder_input_ids) - # predictions = ( - # model(input_ids = input_ids.to(device),decoder_input_ids=decoder_input_ids).logits.squeeze().mean(dim=-1) - # ) - # else: - # predictions = ( - # model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - # ) - feats=batch[2] - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - # decoder_input_ids=input_ids.to(device), - feats=feats.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model( - input_ids.to(device), - feats=feats.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - targets = batch[-1].squeeze() - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - # print('train',predictions,targets) - loss.backward() - optimizer.step() - scheduler.step() - # optimizer.zero_grad() - train_loss += loss.item() - train_loss = train_loss / len(train_dataloader) - t2 = time.time() - train_time = round(t2 - t1, 3) - model.eval() - - # total_eval_mae_loss = 0 - # predictions_list = [] - # targets_list = [] - val_loss = 0 - t1 = time.time() - for batch in val_dataloader: - with torch.no_grad(): - #input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - input_ids = batch[0].squeeze() # .squeeze(0) - feats=batch[2] - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - feats=feats.to(device), - # decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model( - input_ids.to(device), - feats=feats.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - targets = batch[-1].squeeze() - # print('val',predictions,targets) - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - val_loss += loss.item() - saving_tag = "" - if val_loss < best_loss: - best_loss = val_loss - best_model_name = "best_model.pt" - torch.save( - model.state_dict(), - os.path.join(output_dir, best_model_name), - ) - # print("Saving model for epoch", epoch) - saving_tag = " saving model:" + str(epoch) - val_loss = val_loss / len(val_dataloader) - t2 = time.time() - val_time = round(t2 - t1, 3) - train_history.append(train_loss) - val_history.append(val_loss) - history = os.path.join(output_dir, "history.json") - - dumpjson( - data={"train": train_history, "val": val_history}, filename=history - ) - mae = "" - if test_each_run: - t1_test = time.time() - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - #input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - input_ids = batch[0].squeeze() # .squeeze(0) - feats=batch[2] - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - # decoder_input_ids=input_ids.to(device), - feats=feats.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - - else: - predictions = ( - model( - input_ids.to(device), - feats=feats.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - ids = batch[2] - targets = batch[-1].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - t2_test = time.time() - test_time = round(t2_test - t1_test, 3) - f.close() - df = pd.read_csv(fname) - mae = mean_absolute_error(df["target"], df["predictions"]) - if mae == "": - print( - "Epoch, train loss, val loss, train_time, val_time", - epoch, - train_loss, - val_loss, - train_time, - val_time, - saving_tag, - ) - else: - print( - "Epoch, train loss, val loss, test loss, train_time, val_time, test_time", - epoch, - train_loss, - val_loss, - mae, - train_time, - val_time, - test_time, - saving_tag, - ) - - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - #input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - input_ids = batch[0].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - feats=feats.to(device), - # decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model(input_ids.to(device),feats=feats.to(device)).logits.squeeze().mean(dim=-1) - ) - ids = batch[1] - targets = batch[-1].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - f.close() - tot_time_end = time.time() - tot_time = tot_time_end - tot_time_start - print("tot_time", tot_time) - - -if __name__ == "__main__": - # box = [[2.715, 2.715, 0], [0, 2.715, 2.715], [2.715, 0, 2.715]] - # coords = [[0, 0, 0], [0.25, 0.2, 0.25]] - # elements = ["Si", "Si"] - # Si = Atoms(lattice_mat=box, coords=coords, elements=elements) - # tmp=atoms_describer(Si) - # print(tmp) - # import sys - # sys.exit() - args = parser.parse_args(sys.argv[1:]) - benchmark_file = args.benchmark_file - model_name = "facebook/opt-350m" - model_name = "mistralai/Mixtral-8x7B-v0.1" - model_name = "google/flan-t5-small" - model_name = "google/flan-t5-base" - model_name = "mistralai/Mistral-7B-Instruct-v0.1" - model_name = "xlnet/xlnet-base-cased" - model_name = "afmck/testing-llama-tiny" - model_name = "EleutherAI/gpt-neo-125m" - model_name = "meta-llama/Llama-2-7b-hf" - model_name = "stas/tiny-random-llama-2" - model_name = "ahxt/llama2_xs_460M_experimental" - model_name = "google-t5/t5-small" - model_name = "openai-community/gpt2-medium" - model_name = "gpt2" - run_atomgpt( - model_name=model_name, - benchmark_file=benchmark_file, - # num_epochs=300, - # pretrained_path="xyz_out_google/flan-t5-small_tinnet_N_ead/best_model.pt", - # pretrained_path="ss_out_google/flan-t5-small_tinnet_N_ead/best_model.pt", - prefix="xyzt6", - # batch_size=5, - max_length=512, - # max_length=256, - batch_size=16, - latent_dim=1000, - # latent_dim=1024, - num_epochs=5000, - # batch_size=16 - ) - import sys - - sys.exit() - latent_dims = [ - 128, - 256, - 512, - 800, - 1024, - 1200, - 1500, - 2048, - 2500, - 3000, - 3500, - 4000, - ] - for i in latent_dims: - prefix = "lat_lat_" + str(i) - print(prefix) - run_atomgpt( - model_name=model_name, - benchmark_file=benchmark_file, - prefix=prefix, - batch_size=16, - latent_dim=i, - num_epochs=150, - ) - max_lengths = [128, 256, 512, 640, 768, 896, 1000] - for i in max_lengths: - prefix = "max_lengt_" + str(i) - print(prefix) - run_atomgpt( - model_name=model_name, - benchmark_file=benchmark_file, - prefix=prefix, - batch_size=16, - max_length=i, - num_epochs=150, - ) diff --git a/atomgpt/scripts/finetune7b.py b/atomgpt/scripts/finetune7b.py deleted file mode 100644 index d43c4cf..0000000 --- a/atomgpt/scripts/finetune7b.py +++ /dev/null @@ -1,812 +0,0 @@ -"""Module for fin tuning LLM model for materials chemsitry.""" - -from jarvis.db.figshare import data -import transformers -import torch -import random -from jarvis.db.jsonutils import dumpjson -import sys -import argparse -from torch.utils.data import DataLoader, Dataset -import numpy as np -import os -from jarvis.core.atoms import Atoms -import pandas as pd -from sklearn.metrics import mean_absolute_error - -# from describe import atoms_describer -import json -from jarvis.db.figshare import get_jid_data -from jarvis.core.atoms import Atoms -from jarvis.analysis.structure.spacegroup import Spacegroup3D -from jarvis.analysis.diffraction.xrd import XRD -from jarvis.core.specie import Specie -import pprint -from collections import defaultdict - -from tqdm import tqdm -import time -import json -import zipfile -from transformers import GPT2Config, GPT2Model, GPT2Tokenizer - -device = "cpu" -if torch.cuda.is_available(): - device = torch.device("cuda") -parser = argparse.ArgumentParser(description="AtomGPT") -parser.add_argument( - "--benchmark_file", - default="AI-SinglePropertyPrediction-PBE_gap-halide_peroskites-test-mae.csv.zip", - # default="AI-SinglePropertyPrediction-ead-tinnet_N-test-mae.csv.zip", - # default="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae", - help="Benchmarks available in jarvis_leaderboard/benchmarks/*/*.zip", -) - - -def atoms_describer( - atoms=[], - xrd_peaks=5, - xrd_round=1, - cutoff=4, - take_n_bomds=2, - include_spg=True, -): - """Describe an atomic structure.""" - if include_spg: - spg = Spacegroup3D(atoms) - theta, d_hkls, intens = XRD().simulate(atoms=(atoms)) - # x = atoms.atomwise_angle_and_radial_distribution() - # bond_distances = {} - # for i, j in x[-1]["different_bond"].items(): - # bond_distances[i.replace("_", "-")] = ", ".join( - # map(str, (sorted(list(set([round(jj, 2) for jj in j]))))) - # ) - dists = defaultdict(list) - elements = atoms.elements - for i in atoms.get_all_neighbors(r=cutoff): - for j in i: - key = "-".join(sorted([elements[j[0]], elements[j[1]]])) - dists[key].append(j[2]) - bond_distances = {} - for i, j in dists.items(): - dist = sorted(set([round(k, 2) for k in j])) - if len(dist) >= take_n_bomds: - dist = dist[0:take_n_bomds] - bond_distances[i] = ", ".join(map(str, dist)) - fracs = {} - for i, j in (atoms.composition.atomic_fraction).items(): - fracs[i] = round(j, 3) - info = {} - chem_info = { - "atomic_formula": atoms.composition.reduced_formula, - "prototype": atoms.composition.prototype, - "molecular_weight": round(atoms.composition.weight / 2, 2), - "atomic_fraction": (fracs), - "atomic_X": ", ".join( - map(str, [Specie(s).X for s in atoms.uniq_species]) - ), - "atomic_Z": ", ".join( - map(str, [Specie(s).Z for s in atoms.uniq_species]) - ), - } - struct_info = { - "lattice_parameters": ", ".join( - map(str, [round(j, 2) for j in atoms.lattice.abc]) - ), - "lattice_angles": ", ".join( - map(str, [round(j, 2) for j in atoms.lattice.angles]) - ), - # "spg_number": spg.space_group_number, - # "spg_symbol": spg.space_group_symbol, - "top_k_xrd_peaks": ", ".join( - map( - str, - sorted(list(set([round(i, xrd_round) for i in theta])))[ - 0:xrd_peaks - ], - ) - ), - "density": round(atoms.density, 3), - # "crystal_system": spg.crystal_system, - # "point_group": spg.point_group_symbol, - # "wyckoff": ", ".join(list(set(spg._dataset["wyckoffs"]))), - "bond_distances": bond_distances, - # "natoms_primitive": spg.primitive_atoms.num_atoms, - # "natoms_conventional": spg.conventional_standard_structure.num_atoms, - } - if include_spg: - struct_info["spg_number"] = spg.space_group_number - struct_info["spg_symbol"] = spg.space_group_symbol - struct_info["crystal_system"] = spg.crystal_system - struct_info["point_group"] = spg.point_group_symbol - struct_info["wyckoff"] = ", ".join(list(set(spg._dataset["wyckoffs"]))) - struct_info["natoms_primitive"] = spg.primitive_atoms.num_atoms - struct_info[ - "natoms_conventional" - ] = spg.conventional_standard_structure.num_atoms - info["chemical_info"] = chem_info - info["structure_info"] = struct_info - line = "The number of atoms are: " + str( - atoms.num_atoms - ) # +"., The elements are: "+",".join(atoms.elements)+". " - for i, j in info.items(): - if not isinstance(j, dict): - line += "The " + i + " is " + j + ". " - else: - # print("i",i) - # print("j",j) - for ii, jj in j.items(): - tmp = "" - if isinstance(jj, dict): - for iii, jjj in jj.items(): - tmp += iii + ": " + str(jjj) + " " - else: - tmp = jj - line += "The " + ii + " is " + str(tmp) + ". " - return line - - -def set_seed(): - os.environ["WANDB_ANONYMOUS"] = "must" - random_seed = 42 - random.seed(random_seed) - torch.manual_seed(random_seed) - np.random.seed(random_seed) - torch.cuda.manual_seed_all(random_seed) - try: - import torch_xla.core.xla_model as xm - - xm.set_rng_state(random_seed) - except ImportError: - pass - torch.backends.cudnn.deterministic = True - torch.backends.cudnn.benchmark = False - os.environ["PYTHONHASHSEED"] = str(random_seed) - os.environ["CUBLAS_WORKSPACE_CONFIG"] = str(":4096:8") - torch.use_deterministic_algorithms(True) - - - -def get_crystal_string_t(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.2f}".format(x) for x in lengths]) - + "\n" - + " ".join([str(int(x)) for x in angles]) - + "\n" - + "\n".join( - [ - str(t) + " " + " ".join(["{0:.3f}".format(x) for x in c]) - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - crystal_str = atoms_describer(atoms) + "\n*\n" + crystal_str - return crystal_str - -def get_crystal_string_t1(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.2f}".format(x) for x in lengths]) - + "#\n" - + " ".join([str(int(x)) for x in angles]) - + "@\n" - + "\n".join( - [ - str(t) + " " + " ".join(["{0:.3f}".format(x) for x in c]) + "&" - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - crystal_str = atoms_describer(atoms) + "\n*\n" + crystal_str - return crystal_str - - -class AtomGPTDataset(Dataset): - def __init__( - self, texts=[], targets=[], ids=[], tokenizer="", max_length=128 - ): - self.texts = texts - self.targets = targets - self.tokenizer = tokenizer - self.max_length = max_length - if not ids: - ids = ["text-" + str(i) for i in range(len(texts))] - self.ids = ids - - def __len__(self): - return len(self.texts) - - def __getitem__(self, idx): - inputs = self.tokenizer( - self.texts[idx], - return_tensors="pt", - max_length=self.max_length, - padding="max_length", - truncation=True, - ) - return ( - inputs, - self.ids[idx], - torch.tensor(self.targets[idx], dtype=torch.float32), - ) - - -class ForcePredictor(torch.nn.Module): - config = GPT2Config.from_pretrained("gpt2") - - def __init__(self, gpt2_model): - super(ForcePredictor, self).__init__() - self.gpt2 = gpt2_model - self.linear = torch.nn.Linear( - config.n_embd, 1 - ) # Assuming force is a 3D vector - - def forward(self, input_ids): - outputs = self.gpt2(input_ids) - last_hidden_states = outputs.last_hidden_state - force_pred = self.linear(last_hidden_states[:, -1, :]) - # print("force_pred",outputs.keys()) - return force_pred - - -class AtomGPTPredictorLMhead(torch.nn.Module): - def __init__( - self, model_name=None, n_out=1, latent_dim=1024, tokenizer="" - ): - - super(AtomGPTPredictorLMhead, self).__init__() - #random_seed = 42 - #random.seed(random_seed) - #torch.manual_seed(random_seed) - #np.random.seed(random_seed) - #torch.cuda.manual_seed_all(random_seed) - #try: - # import torch_xla.core.xla_model as xm - # xm.set_rng_state(random_seed) - #except ImportError: - # pass - #torch.backends.cudnn.deterministic = True - #torch.backends.cudnn.benchmark = False - #os.environ["PYTHONHASHSEED"] = str(random_seed) - #os.environ["CUBLAS_WORKSPACE_CONFIG"] = str(":4096:8") - #torch.use_deterministic_algorithms(True) - - self.model_name = model_name - self.n_out = n_out - self.latent_dim = latent_dim - if "t5" in model_name: - model = transformers.T5ForConditionalGeneration.from_pretrained( - model_name - ) - else: - # config = GPT2Config.from_pretrained("gpt2") - # model = GPT2Model(config) - model = transformers.AutoModelForCausalLM.from_pretrained( - model_name, - low_cpu_mem_usage=True, - # load_in_8bit=False, - # torch_dtype=torch.float16, - # load_in_8bit=True, - # device_map="auto" - ) - model.resize_token_embeddings(len(tokenizer)) - self.config = model.config - model.lm_head = torch.nn.Sequential( - torch.nn.Linear(model.config.hidden_size, latent_dim), - torch.nn.Linear(latent_dim, n_out), - ) - self.model = model - - def forward(self, input_ids): - # outputs = self.model(input_ids) - if "t5" in model_name: - outputs = self.model(input_ids, decoder_input_ids=input_ids) - else: - outputs = self.model(input_ids) - return outputs - - -class AtomGPTPredictorHiddenFeats(torch.nn.Module): - def __init__(self, model_name=None, n_out=1, tokenizer=""): - super(AtomGPTPredictorHiddenFeats, self).__init__() - if "t5" in model_name: - model = transformers.T5ForConditionalGeneration.from_pretrained( - model_name - ) - else: - model = transformers.AutoModelForCausalLM.from_pretrained( - model_name, - low_cpu_mem_usage=True, - # load_in_8bit=False, - # torch_dtype=torch.float16, - # load_in_8bit=True, - # device_map="auto" - ) - model.resize_token_embeddings(len(tokenizer)) - self.model = model - self.config = self.model.config - self.global_out = torch.nn.Linear(self.config.n_embd, n_out) - - def forward(self, input_ids): - outputs = self.model(input_ids) - print("outputs", outputs.keys()) - last_hidden_states = outputs.last_hidden_state - pred = self.linear(last_hidden_states[:, -1, :]) - return pred - - -def run_atomgpt( - prefix="ss", - model_name="gpt2", - benchmark_file="AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip", - root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard", - batch_size=8, - max_length=512, - num_epochs=500, - latent_dim=512, - learning_rate=1e-3, - # learning_rate=1e-3, - test_each_run=True, - # learning_rate=5e-5, - pretrained_path="", -): - # Load pre-trained tokenizer - # tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - # model = GPT2LMHeadModel.from_pretrained("gpt2") - - # dft_3d = data("dft_3d") - # root_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard" - # benchmark_file="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae.csv.zip" - # benchmark_file = "AI-SinglePropertyPrediction-optb88vdw_bandgap-dft_3d-test-mae.csv.zip" - print("benchmark_file", benchmark_file) - method = benchmark_file.split("-")[0] - task = benchmark_file.split("-")[1] - prop = benchmark_file.split("-")[2] - dataset = benchmark_file.split("-")[3] - temp = dataset + "_" + prop + ".json.zip" - temp2 = dataset + "_" + prop + ".json" - fname = os.path.join(root_dir, "benchmarks", method, task, temp) - zp = zipfile.ZipFile(fname) - bench = json.loads(zp.read(temp2)) - dft_3d = data(dataset) - id_tag = "jid" - if "jid" in dft_3d[0]: - id_tag = "jid" - else: - id_tag = "id" - - # train_atoms = [] - # val_atoms = [] - # test_atoms = [] - # train_targets = [] - # val_targets = [] - # test_targets = [] - train_ids = list(bench["train"].keys()) - test_ids = list(bench["test"].keys()) - if "val" in bench: - val_ids = list(bench["val"].keys()) - else: - val_ids = test_ids - print("total", len(dft_3d)) - print("test_ids", len(test_ids)) - print("val_ids", len(val_ids)) - print("train_ids", len(train_ids)) - # model_name = "mistralai/Mistral-7B-Instruct-v0.1" - # model_name = "gpt2" - # device = model.device - if "t5" in model_name: - tokenizer = transformers.T5Tokenizer.from_pretrained(model_name) - - else: - tokenizer = transformers.AutoTokenizer.from_pretrained(model_name) - # print('Non tokenizer format') - # tokenizer = GPT2Tokenizer.from_pretrained(model_name) - # config = GPT2Config.from_pretrained("gpt2") - # model = GPT2Model(config) - if tokenizer.pad_token is None: - tokenizer.add_special_tokens({"pad_token": "[PAD]"}) - tokenizer.add_special_tokens({"unk_token": "#"}) - tokenizer.add_special_tokens({"unk_token": "&"}) - tokenizer.add_special_tokens({"unk_token": "@"}) - # model.resize_token_embeddings(len(tokenizer)) - # model=ForcePredictor(model) - # model=AtomGPTPredictorHiddenFeats(model_name=model_name, tokenizer=tokenizer) - set_seed() - model = AtomGPTPredictorLMhead(model_name=model_name, tokenizer=tokenizer,latent_dim=latent_dim) - # tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - # batch_size = 16 - # max_length = 128 - # num_epochs = 100 - # learning_rate = 5e-5 - criterion = torch.nn.L1Loss() - # Define example regression data (texts and corresponding numeric targets) - train_texts = [] - train_targets = [] - train_ids_temp = [] - val_texts = [] - val_targets = [] - val_ids_temp = [] - test_texts = [] - test_targets = [] - test_ids_temp = [] - - for i in tqdm(dft_3d): - if i[prop] != "na": - atoms = Atoms.from_dict(i["atoms"]) - tmp = get_crystal_string_t(atoms) - if i[id_tag] in train_ids: - train_texts.append(tmp) - train_targets.append(i[prop]) - train_ids_temp.append(i[id_tag]) - elif i[id_tag] in test_ids: - test_texts.append(tmp) - test_targets.append(i[prop]) - test_ids_temp.append(i[id_tag]) - elif i[id_tag] in val_ids: - val_texts.append(tmp) - val_targets.append(i[prop]) - val_ids_temp.append(i[id_tag]) - print("test_texts:", len(test_texts)) - print("test_texts:", test_texts[0]) - """ - ############################## - ###Fast test### - train_texts = [ - "This is the first example text.", - "Second example is a bit longer than the first one, but still within the max length.", - "Third example is the longest among these three examples. It exceeds the max length and will be truncated.", - "Second example is a bit longer than the first one, but still within the max length.", - ] - train_targets = [10.2, 15.5, 20.1, 15.5] # Example regression targets - val_texts = test_texts = train_texts - val_targets = test_targets = train_targets - train_ids_temp=['a','b','c','d'] - val_ids_temp = test_ids_temp = train_ids_temp - batch_size = 2 - num_epochs = 3 - - ############################## - ############################## - """ - - # Fine-tune the last layer of GPT-2 for regression - # fine_tune_gpt2_regression(train_texts, train_targets, tokenizer) - if pretrained_path != "": - model.load_state_dict(torch.load(pretrained_path, map_location=device)) - # model.lm_head = torch.nn.Sequential(torch.nn.Linear( model.config.hidden_size, 256),torch.nn.SiLU(),torch.nn.Linear( 256, 1) ) - # set_seed(seed) - # set_deterministic() - model.to(device) - if torch.cuda.device_count() > 1: - device_ids = [d for d in range(torch.cuda.device_count())] - model = torch.nn.DataParallel(model, device_ids=device_ids).cuda() - optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate) - #optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate) - #optimizer = transformers.AdamW(model.parameters(), lr=learning_rate) - # Prepare datasets and dataloaders with data collator - train_dataset = AtomGPTDataset( - texts=train_texts, - targets=train_targets, - ids=train_ids_temp, - tokenizer=tokenizer, - max_length=max_length, - ) - val_dataset = AtomGPTDataset( - texts=val_texts, - targets=val_targets, - tokenizer=tokenizer, - ids=val_ids_temp, - max_length=max_length, - ) - test_dataset = AtomGPTDataset( - texts=test_texts, - targets=test_targets, - tokenizer=tokenizer, - ids=test_ids_temp, - max_length=max_length, - ) - - # val_dataset = train_dataset - train_dataloader = DataLoader(train_dataset, batch_size=batch_size) - val_dataloader = DataLoader(val_dataset, batch_size=batch_size) - test_dataloader = DataLoader(test_dataset, batch_size=batch_size) - val_dataloader = test_dataloader - steps_per_epoch = len(train_dataloader) - scheduler = torch.optim.lr_scheduler.OneCycleLR( - optimizer, - max_lr=learning_rate, - epochs=num_epochs, - steps_per_epoch=steps_per_epoch, - # pct_start=pct_start, - pct_start=0.3, - ) - # scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10, eta_min=0.001) - # scheduler = torch.optim.lr_scheduler.StepLR( - # optimizer, - # step_size=30, - # ) - print("train_data", len(train_texts)) - print("test_data", len(test_texts)) - output_dir = prefix + "_out_" + model_name + "_" + dataset + "_" + prop - if not os.path.exists(output_dir): - os.makedirs(output_dir) - best_loss = np.inf - tot_time_start = time.time() - train_history = [] - val_history = [] - for epoch in range(num_epochs): - model.train() - t1 = time.time() - for batch in train_dataloader: - optimizer.zero_grad() - train_loss = 0 - # train_result = [] - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - # if 't5' in model_name: - # decoder_input_ids = tokenizer("", return_tensors="pt").input_ids.to(device) - # decoder_input_ids = model._shift_right(decoder_input_ids) - # predictions = ( - # model(input_ids = input_ids.to(device),decoder_input_ids=decoder_input_ids).logits.squeeze().mean(dim=-1) - # ) - # else: - # predictions = ( - # model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - # ) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - # decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - targets = batch[2].squeeze() - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - # print('train',predictions,targets) - loss.backward() - optimizer.step() - scheduler.step() - # optimizer.zero_grad() - train_loss += loss.item() - train_loss = train_loss / len(train_dataloader) - t2 = time.time() - train_time = round(t2 - t1, 3) - model.eval() - - # total_eval_mae_loss = 0 - # predictions_list = [] - # targets_list = [] - val_loss = 0 - t1 = time.time() - for batch in val_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - # decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - targets = batch[2].squeeze() - # print('val',predictions,targets) - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - val_loss += loss.item() - saving_tag = "" - if val_loss < best_loss: - best_loss = val_loss - best_model_name = "best_model.pt" - torch.save( - model.state_dict(), - os.path.join(output_dir, best_model_name), - ) - # print("Saving model for epoch", epoch) - saving_tag = " saving model:" + str(epoch) - val_loss = val_loss / len(val_dataloader) - t2 = time.time() - val_time = round(t2 - t1, 3) - train_history.append(train_loss) - val_history.append(val_loss) - history = os.path.join(output_dir, "history.json") - - dumpjson( - data={"train": train_history, "val": val_history}, filename=history - ) - mae = "" - if test_each_run: - t1_test = time.time() - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - # decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - t2_test = time.time() - test_time = round(t2_test - t1_test, 3) - f.close() - df = pd.read_csv(fname) - mae = mean_absolute_error(df["target"], df["predictions"]) - if mae == "": - print( - "Epoch, train loss, val loss, train_time, val_time", - epoch, - train_loss, - val_loss, - train_time, - val_time, - saving_tag, - ) - else: - print( - "Epoch, train loss, val loss, test loss, train_time, val_time, test_time", - epoch, - train_loss, - val_loss, - mae, - train_time, - val_time, - test_time, - saving_tag, - ) - - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - # decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - f.close() - tot_time_end = time.time() - tot_time = tot_time_end - tot_time_start - print("tot_time", tot_time) - - -if __name__ == "__main__": - # box = [[2.715, 2.715, 0], [0, 2.715, 2.715], [2.715, 0, 2.715]] - # coords = [[0, 0, 0], [0.25, 0.2, 0.25]] - # elements = ["Si", "Si"] - # Si = Atoms(lattice_mat=box, coords=coords, elements=elements) - # tmp=atoms_describer(Si) - # print(tmp) - # import sys - # sys.exit() - args = parser.parse_args(sys.argv[1:]) - benchmark_file = args.benchmark_file - model_name = "facebook/opt-350m" - model_name = "mistralai/Mixtral-8x7B-v0.1" - model_name = "google/flan-t5-small" - model_name = "google/flan-t5-base" - model_name = "mistralai/Mistral-7B-Instruct-v0.1" - model_name = "xlnet/xlnet-base-cased" - model_name = "afmck/testing-llama-tiny" - model_name = "EleutherAI/gpt-neo-125m" - model_name = "meta-llama/Llama-2-7b-hf" - model_name = "stas/tiny-random-llama-2" - model_name = "ahxt/llama2_xs_460M_experimental" - model_name = "google-t5/t5-small" - model_name = "openai-community/gpt2-medium" - model_name = "gpt2" - run_atomgpt( - model_name=model_name, - benchmark_file=benchmark_file, - # num_epochs=300, - # pretrained_path="xyz_out_google/flan-t5-small_tinnet_N_ead/best_model.pt", - # pretrained_path="ss_out_google/flan-t5-small_tinnet_N_ead/best_model.pt", - prefix="xyzt6", - # batch_size=5, - batch_size=16, - latent_dim=1024, - num_epochs=5000, - # batch_size=16 - ) - import sys - sys.exit() - latent_dims=[128,256,512,800,1024,1200,1500,2048,2500,3000,3500,4000] - for i in latent_dims: - prefix='lat_lat_'+str(i) - print(prefix) - run_atomgpt(model_name=model_name,benchmark_file=benchmark_file,prefix=prefix,batch_size=16,latent_dim=i,num_epochs=150) - diff --git a/atomgpt/scripts/gp2atom_km.py b/atomgpt/scripts/gp2atom_km.py deleted file mode 100644 index 0bee973..0000000 --- a/atomgpt/scripts/gp2atom_km.py +++ /dev/null @@ -1,171 +0,0 @@ -from typing import * -import pandas as pd -from transformers import GPT2Config, GPT2ForSequenceClassification, GPT2TokenizerFast, TrainingArguments, Trainer -from sklearn.metrics import mean_absolute_error, mean_squared_error -from jarvis.core.atoms import Atoms -from jarvis.io.vasp.inputs import Poscar -from jarvis.db.figshare import data -import torch,json -import numpy as np -from describe import atoms_describer -from sklearn.model_selection import train_test_split -import ast -import os -# torch.cuda.is_available = lambda : False -import argparse -from multiprocessing import Pool -from tqdm import tqdm - -# if output dir not exists, create it -if not os.path.exists('output'): - os.mkdir('output') - -parser = argparse.ArgumentParser() -parser.add_argument('--prop', type=str, required=True) -parser.add_argument('--modelname', type=str, default='gpt2') -parser.add_argument('--random_state', type=int, default=0) -parser.add_argument('--dataset_name', type=str, default='dft_3d_2021') - -args = parser.parse_args() - -prop = args.prop -modelname = args.modelname -random_state = args.random_state -dataset_name = args.dataset_name - -output_dir=f'output/{modelname}_{dataset_name}_{prop}' - -print('imports done') -print('torch.cuda.is_available',torch.cuda.is_available()) - -#%% -def process_data(i): - atoms = i['atoms'] - lattice_mat = np.round(np.array(atoms['lattice_mat']), decimals=4) - coords = np.round(np.array(atoms['coords']), decimals=4) - i['atoms'] = Atoms(lattice_mat=lattice_mat, elements=atoms['elements'], coords=coords, cartesian=atoms['cartesian']) - i['atoms'] = json.dumps(atoms_describer(i['atoms'])) - return i - -print('prop',prop,flush=True) - -# prop = 'exfoliation_energy' -df_csv = f'{dataset_name}_described.csv' -if os.path.exists(df_csv): - df = pd.read_csv(df_csv)[['atoms',prop]] -else: - dat = data(dataset_name) - - pool = Pool() - dd = [] - for result in tqdm(pool.imap(process_data, dat), total=len(dat)): - dd.append(result) - - df = pd.DataFrame(dd) - # df = df.set_index(df.columns[0]) - df = df.replace('na', '') - df = df.replace('',None) - df.to_csv(df_csv) - -# replace all values of "na" with numpy nan -df = df.dropna(subset=[prop]) - -# random split into train and test -train_dd, test_dd = train_test_split(df, test_size=0.2, random_state=random_state) -train_ids, test_ids = train_dd.index, test_dd.index -n_train, n_test = len(train_dd), len(test_dd) -print(n_train, n_test) - -# use the 'atoms' and 'prop' column to create a dataframe with 'text' and 'label' columns -print('MAD of test set',np.abs(df.loc[test_ids,prop]-df.loc[test_ids,prop].mean()).mean()) - -text = df['atoms'] -label = df[prop].apply(lambda x: [x]) - -train_df = pd.DataFrame({'text':text.loc[train_ids],'label':label.loc[train_ids]}) -test_df = pd.DataFrame({'text':text.loc[test_ids],'label':label.loc[test_ids]}) - -print('df created') - - -config = GPT2Config.from_pretrained( - modelname, - # 'gpt2-medium', - pad_token_id=50256, # eos_token_id - num_labels=1, -) -tokenizer = GPT2TokenizerFast.from_pretrained( - config.model_type, - padding=True, - truncation=True, - pad_token_id=config.pad_token_id, - pad_token="<|endoftext|>", # eos_token -) -tokenizer.pad_token -model = GPT2ForSequenceClassification(config) - - -print('model loaded') - - -def tokenize(df: pd.DataFrame, tokenizer: GPT2TokenizerFast) -> List[Dict[str, Any]]: - tokenized_df = pd.DataFrame( - df['text'].apply(tokenizer).tolist() - ) - return ( - pd.merge( - df, - tokenized_df, - left_index=True, - right_index=True, - ) - .drop(columns="text") - .to_dict("records") - ) - -train_ds = tokenize(train_df, tokenizer) -test_ds = tokenize(test_df, tokenizer) - -print('tokenized') - -def compute_metrics(pred): - labels = pred.label_ids - predictions = pred.predictions - return { - "mae": mean_absolute_error(labels, predictions), - #"mse": mean_squared_error(labels, predictions), - } - -training_args = TrainingArguments( - report_to="none", - evaluation_strategy="steps", - max_steps=1000, - eval_steps=50, - # per_device_train_batch_size=16, - # per_device_eval_batch_size=128, - metric_for_best_model="mse", - greater_is_better=False, - learning_rate=5e-5, - # going to delete all of this - output_dir=output_dir, - save_strategy="no", -) - -trainer = Trainer( - model=model, - args=training_args, - train_dataset=train_ds, - eval_dataset=test_ds, - tokenizer=tokenizer, - compute_metrics=compute_metrics -) - -print('trainer loaded') - -trainer.train() -# save model -trainer.save_model(f'{output_dir}/final_{modelname}_{dataset_name}_{prop}') -# save scores -scores = trainer.evaluate() -with open(f'{output_dir}/scores_{modelname}_{dataset_name}_{prop}.json','w') as f: - json.dump(scores,f) diff --git a/atomgpt/scripts/gpt.py b/atomgpt/scripts/gpt.py deleted file mode 100644 index 87cc2a8..0000000 --- a/atomgpt/scripts/gpt.py +++ /dev/null @@ -1,93 +0,0 @@ -#mean_absolute_error: 54.10120434782608 -import json -import numpy as np -from transformers import AutoTokenizer, AutoModelForSequenceClassification -from transformers import AutoTokenizer, GPT2Model -from sklearn.linear_model import LinearRegression -from sklearn.model_selection import train_test_split -from sklearn.metrics import mean_absolute_error, mean_squared_error -import torch -from jarvis.core.atoms import Atoms -from jarvis.io.vasp.inputs import Poscar -from jarvis.db.figshare import data -from sklearn.ensemble import RandomForestRegressor -import matplotlib.pyplot as plt - -# Load JSON data -def load_data_from_json(json_path): - with open(json_path, 'r') as file: - data = json.load(file) - return data - -# Preprocess data and convert text to embeddings -tag = 'formation_energy_peratom' -def preprocess_data(dat,prop='',model='gpt2'):#, model_name): - #tokenizer = AutoTokenizer.from_pretrained(model_name) - tokenizer = AutoTokenizer.from_pretrained(model) - #model = AutoModelForSequenceClassification.from_pretrained(model_name) - model = GPT2Model.from_pretrained(model) - - embeddings = [] - labels=[] - print(model) - for entry in dat: - try: - text=Poscar(Atoms.from_dict(entry['atoms'])).to_string() - #text = entry['text'] - inputs = tokenizer(text, return_tensors="pt") - with torch.no_grad(): - output = model(**inputs) - #print(output.keys(),output['past_key_values']) - emb = output.last_hidden_state.mean(dim=1).numpy().flatten() - #print('emb',emb,emb.shape) - embeddings.append(emb) - labels.append(entry[prop]) - #labels.append(entry['exfoliation_energy']) - #embeddings.append(output.last_hidden_state.mean(dim=1).numpy()) - #embeddings.append(output.last_hidden_state.mean(dim=1).numpy()) - except Exception as exp: - print ('exp',exp,text,len(text)) - pass - - embeddings = np.vstack(embeddings) - #labels = np.array([entry['exfoliation_energy'] for entry in dat]) - return embeddings, labels - -# Main function -def main(): - dat = data('dft_3d') - dd=[] - prop = 'formation_energy_peratom'#'exfoliation_energy' - #prop = 'exfoliation_energy' - for i in dat: - if i[prop]!='na': #[0:10] - dd.append(i) - #dd=dd[0:10] - print('dd',len(dd)) - X, y = preprocess_data(dd,prop=prop)#, model_name) - - # Split the data into training and testing sets - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) - - # Initialize and fit a linear regression model - regression_model = RandomForestRegressor() #LinearRegression() - regression_model.fit(X_train, y_train) - - # Predict using the test set - y_pred = regression_model.predict(X_test) - - # Evaluate the model - mse = mean_squared_error(y_test, y_pred) - mae = mean_absolute_error(y_test, y_pred) - print("mean_absolute_error:", mae) - plt.plot(y_test, y_pred,'.') - plt.savefig('plot.png') - plt.close() - #print("Mean Squared Error:", mse) - -if __name__ == "__main__": - main() -#info=[{"text":"Ram is a good boy","target":1},{"text":"Ravan is bad boy","target":0}] -#embeddings, labels = preprocess_data(info,"gpt2") -#print('embeddings',embeddings,embeddings.shape) -#print('labels',labels,labels.shape) diff --git a/atomgpt/scripts/gpt2_describer.py b/atomgpt/scripts/gpt2_describer.py deleted file mode 100644 index 3d96aa7..0000000 --- a/atomgpt/scripts/gpt2_describer.py +++ /dev/null @@ -1,93 +0,0 @@ -#mean_absolute_error: 64.72426134969325 -import json -import numpy as np -from transformers import AutoTokenizer, AutoModelForSequenceClassification -from transformers import AutoTokenizer, GPT2Model -from sklearn.linear_model import LinearRegression -from sklearn.model_selection import train_test_split -from sklearn.metrics import mean_absolute_error, mean_squared_error -import torch -from jarvis.core.atoms import Atoms -from jarvis.io.vasp.inputs import Poscar -from jarvis.db.figshare import data -from sklearn.ensemble import RandomForestRegressor -import matplotlib.pyplot as plt -from chemnlp.utils.describe import atoms_describer -# Load JSON data -def load_data_from_json(json_path): - with open(json_path, 'r') as file: - data = json.load(file) - return data - -# Preprocess data and convert text to embeddings -tag = 'formation_energy_peratom' -def preprocess_data(dat,prop='',model='gpt2'):#, model_name): - #tokenizer = AutoTokenizer.from_pretrained(model_name) - tokenizer = AutoTokenizer.from_pretrained(model) - #model = AutoModelForSequenceClassification.from_pretrained(model_name) - model = GPT2Model.from_pretrained(model) - - embeddings = [] - labels=[] - print(model) - for entry in dat: - try: - text=json.dumps(atoms_describer(atoms=Atoms.from_dict(entry['atoms']))) #Poscar(Atoms.from_dict(entry['atoms'])).to_string() - #text = entry['text'] - inputs = tokenizer(text, return_tensors="pt") - with torch.no_grad(): - output = model(**inputs) - #print(output.keys(),output['past_key_values']) - emb = output.last_hidden_state.mean(dim=1).numpy().flatten() - #print('emb',emb,emb.shape) - embeddings.append(emb) - labels.append(entry[prop]) - #labels.append(entry['exfoliation_energy']) - #embeddings.append(output.last_hidden_state.mean(dim=1).numpy()) - #embeddings.append(output.last_hidden_state.mean(dim=1).numpy()) - except Exception as exp: - print ('exp',exp,text,len(text)) - pass - - embeddings = np.vstack(embeddings) - #labels = np.array([entry['exfoliation_energy'] for entry in dat]) - return embeddings, labels - -# Main function -def main(): - dat = data('dft_3d') - dd=[] - prop = 'formation_energy_peratom'#'exfoliation_energy' - prop = 'exfoliation_energy' - for i in dat: - if i[prop]!='na': #[0:10] - dd.append(i) - #dd=dd[0:10] - print('dd',len(dd)) - X, y = preprocess_data(dd,prop=prop)#, model_name) - - # Split the data into training and testing sets - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) - - # Initialize and fit a linear regression model - regression_model = RandomForestRegressor() #LinearRegression() - regression_model.fit(X_train, y_train) - - # Predict using the test set - y_pred = regression_model.predict(X_test) - - # Evaluate the model - mse = mean_squared_error(y_test, y_pred) - mae = mean_absolute_error(y_test, y_pred) - print("mean_absolute_error:", mae) - plt.plot(y_test, y_pred,'.') - plt.savefig('plot.png') - plt.close() - #print("Mean Squared Error:", mse) - -if __name__ == "__main__": - main() -#info=[{"text":"Ram is a good boy","target":1},{"text":"Ravan is bad boy","target":0}] -#embeddings, labels = preprocess_data(info,"gpt2") -#print('embeddings',embeddings,embeddings.shape) -#print('labels',labels,labels.shape) diff --git a/atomgpt/scripts/gpt2_describer1.py b/atomgpt/scripts/gpt2_describer1.py deleted file mode 100644 index 502269d..0000000 --- a/atomgpt/scripts/gpt2_describer1.py +++ /dev/null @@ -1,93 +0,0 @@ -import json -import numpy as np -from transformers import AutoTokenizer, AutoModelForSequenceClassification -from transformers import AutoTokenizer, GPT2Model -from sklearn.linear_model import LinearRegression -from sklearn.model_selection import train_test_split -from sklearn.metrics import mean_absolute_error, mean_squared_error -import torch -from jarvis.core.atoms import Atoms -from jarvis.io.vasp.inputs import Poscar -from jarvis.db.figshare import data -from sklearn.ensemble import RandomForestRegressor -import matplotlib.pyplot as plt -from chemnlp.utils.describe import atoms_describer -# Load JSON data -def load_data_from_json(json_path): - with open(json_path, 'r') as file: - data = json.load(file) - return data - -# Preprocess data and convert text to embeddings -tag = 'formation_energy_peratom' -def preprocess_data(dat,prop='',model='gpt2'):#, model_name): - #tokenizer = AutoTokenizer.from_pretrained(model_name) - tokenizer = AutoTokenizer.from_pretrained(model) - #model = AutoModelForSequenceClassification.from_pretrained(model_name) - model = GPT2Model.from_pretrained(model) - - embeddings = [] - labels=[] - print(model) - for entry in dat: - try: - text=json.dumps(atoms_describer(atoms=Atoms.from_dict(entry['atoms']))) #Poscar(Atoms.from_dict(entry['atoms'])).to_string() - #text = entry['text'] - inputs = tokenizer(text, return_tensors="pt") - with torch.no_grad(): - output = model(**inputs) - #print(output.keys(),output['past_key_values']) - emb = output.last_hidden_state.mean(dim=1).numpy().flatten() - #print('emb',emb,emb.shape) - embeddings.append(emb) - labels.append(entry[prop]) - #labels.append(entry['exfoliation_energy']) - #embeddings.append(output.last_hidden_state.mean(dim=1).numpy()) - #embeddings.append(output.last_hidden_state.mean(dim=1).numpy()) - except Exception as exp: - print ('exp',exp,text,len(text)) - pass - - embeddings = np.vstack(embeddings) - #labels = np.array([entry['exfoliation_energy'] for entry in dat]) - return embeddings, labels - -# Main function -def main(): - dat = data('dft_3d') - dd=[] - prop = 'formation_energy_peratom'#'exfoliation_energy' - prop = 'exfoliation_energy' - model = "databricks/dolly-v2-3b" - for i in dat: - if i[prop]!='na': #[0:10] - dd.append(i) - #dd=dd[0:10] - print('dd',len(dd)) - X, y = preprocess_data(dd,prop=prop,model=model)#, model_name) - - # Split the data into training and testing sets - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) - - # Initialize and fit a linear regression model - regression_model = RandomForestRegressor() #LinearRegression() - regression_model.fit(X_train, y_train) - - # Predict using the test set - y_pred = regression_model.predict(X_test) - - # Evaluate the model - mse = mean_squared_error(y_test, y_pred) - mae = mean_absolute_error(y_test, y_pred) - print("mean_absolute_error:", mae) - plt.plot(y_test, y_pred,'.') - plt.savefig('plot.png') - plt.close() - #print("Mean Squared Error:", mse) - -if __name__ == "__main__": - main() -#info=[{"text":"Ram is a good boy","target":1},{"text":"Ravan is bad boy","target":0}] -#embeddings, labels = preprocess_data(info,"gpt2") -#print('embeddings',embeddings,embeddings.shape) -#print('labels',labels,labels.shape) diff --git a/atomgpt/scripts/gpt2_robo.py b/atomgpt/scripts/gpt2_robo.py deleted file mode 100644 index 04ea53a..0000000 --- a/atomgpt/scripts/gpt2_robo.py +++ /dev/null @@ -1,116 +0,0 @@ -#mean_absolute_error: 64.72426134969325 -import json -import numpy as np -from transformers import AutoTokenizer, AutoModelForSequenceClassification -from transformers import AutoTokenizer, GPT2Model -from sklearn.linear_model import LinearRegression -from sklearn.model_selection import train_test_split -from sklearn.metrics import mean_absolute_error, mean_squared_error -import torch -from jarvis.core.atoms import Atoms -from jarvis.io.vasp.inputs import Poscar -from jarvis.db.figshare import data -from sklearn.ensemble import RandomForestRegressor -import matplotlib.pyplot as plt -from chemnlp.utils.describe import atoms_describer -# Load JSON data - -from pymatgen.core.structure import Structure -from robocrys import StructureCondenser, StructureDescriber - - -def get_robo(structure=None): -#structure = Structure.from_file("POSCAR") # other file formats also supported - -# alternatively, uncomment the lines below to use the MPRester object -# to fetch structures from the Materials Project database -# from pymatgen import MPRester -# structure = MPRester(API_KEY=None).get_structure_by_material_id("mp-856") - - condenser = StructureCondenser() - describer = StructureDescriber() - - #condensed_structure = condenser.condense_structure(structure) - #description = describer.describe(condensed_structure) - description = describer.describe(structure) - print(description) - return description - -def load_data_from_json(json_path): - with open(json_path, 'r') as file: - data = json.load(file) - return data - -# Preprocess data and convert text to embeddings -tag = 'formation_energy_peratom' -def preprocess_data(dat,prop='',model='gpt2'):#, model_name): - #tokenizer = AutoTokenizer.from_pretrained(model_name) - tokenizer = AutoTokenizer.from_pretrained(model) - #model = AutoModelForSequenceClassification.from_pretrained(model_name) - model = GPT2Model.from_pretrained(model) - - embeddings = [] - labels=[] - print(model) - for entry in dat: - try: - text=get_robo(Atoms.from_dict(entry['atoms']).pymatgen_converter()) #Poscar(Atoms.from_dict(entry['atoms'])).to_string() - #text=json.dumps(atoms_describer(atoms=Atoms.from_dict(entry['atoms']))) #Poscar(Atoms.from_dict(entry['atoms'])).to_string() - #text = entry['text'] - inputs = tokenizer(text, return_tensors="pt") - with torch.no_grad(): - output = model(**inputs) - #print(output.keys(),output['past_key_values']) - emb = output.last_hidden_state.mean(dim=1).numpy().flatten() - #print('emb',emb,emb.shape) - embeddings.append(emb) - labels.append(entry[prop]) - #labels.append(entry['exfoliation_energy']) - #embeddings.append(output.last_hidden_state.mean(dim=1).numpy()) - #embeddings.append(output.last_hidden_state.mean(dim=1).numpy()) - except Exception as exp: - print ('exp',exp,text,len(text)) - pass - - embeddings = np.vstack(embeddings) - #labels = np.array([entry['exfoliation_energy'] for entry in dat]) - return embeddings, labels - -# Main function -def main(): - dat = data('dft_3d') - dd=[] - prop = 'formation_energy_peratom'#'exfoliation_energy' - prop = 'exfoliation_energy' - for i in dat: - if i[prop]!='na': #[0:10] - dd.append(i) - #dd=dd[0:10] - print('dd',len(dd)) - X, y = preprocess_data(dd,prop=prop)#, model_name) - - # Split the data into training and testing sets - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) - - # Initialize and fit a linear regression model - regression_model = RandomForestRegressor() #LinearRegression() - regression_model.fit(X_train, y_train) - - # Predict using the test set - y_pred = regression_model.predict(X_test) - - # Evaluate the model - mse = mean_squared_error(y_test, y_pred) - mae = mean_absolute_error(y_test, y_pred) - print("mean_absolute_error:", mae) - plt.plot(y_test, y_pred,'.') - plt.savefig('plot.png') - plt.close() - #print("Mean Squared Error:", mse) - -if __name__ == "__main__": - main() -#info=[{"text":"Ram is a good boy","target":1},{"text":"Ravan is bad boy","target":0}] -#embeddings, labels = preprocess_data(info,"gpt2") -#print('embeddings',embeddings,embeddings.shape) -#print('labels',labels,labels.shape) diff --git a/atomgpt/scripts/gpt2atom.py b/atomgpt/scripts/gpt2atom.py deleted file mode 100644 index bdd0b34..0000000 --- a/atomgpt/scripts/gpt2atom.py +++ /dev/null @@ -1,201 +0,0 @@ -from typing import * -import pandas as pd -from transformers import GPT2Config, GPT2ForSequenceClassification, GPT2TokenizerFast, TrainingArguments, Trainer -from sklearn.metrics import mean_absolute_error, mean_squared_error -from jarvis.core.atoms import Atoms -from jarvis.io.vasp.inputs import Poscar -from jarvis.db.figshare import data -import torch,json -import numpy as np -torch.cuda.is_available = lambda : False -# Mondegreen fun -# 0. is the misheard version -# 1. is the real version -# regression task -dat = data('dft_3d') -dd=[] -prop = 'formation_energy_peratom'#'exfoliation_energy' -prop = 'dfpt_piezo_max_dielectric' -prop = 'exfoliation_energy' -for i in dat: - if i[prop]!='na': #[0:10] - atoms=i['atoms'] - lattice_mat = np.round(np.array(atoms['lattice_mat']),decimals=4) - coords = np.round(np.array(atoms['coords']),decimals=4) - atoms=Atoms(lattice_mat=lattice_mat,elements=atoms['elements'],coords=coords,cartesian=atoms['cartesian'],props=atoms['props']) - i['atoms']=atoms.to_dict() - dd.append(i) - #dd=dd[0:10] -#dd=dd[10:22] -n_train=int(len(dd)*.8) -n_test=len(dd)-n_train -train_dd=dd[0:n_train] -test_dd=dd[-n_test:] - -train_df = pd.DataFrame([ - {"text": "Money for nothin' and chips for free", "label": [0.]}, - {"text": "Money for nothin' and your chicks for free", "label": [1.]}, - - {"text": "Every time you go away, you take a piece of meat with you", "label": [0.]}, - {"text": "Every time you go away take a piece of me with you", "label": [1.]}, - - {"text": "Sue Lawley", "label": [0.]}, - {"text": "So lonely", "label": [1.]}, - - {"text": "We built this city on sausage rolls", "label": [0.]}, - {"text": "We built this city on rock 'n' roll", "label": [1.]}, - - {"text": "Saving his life from this warm sausage tea", "label": [0.]}, - {"text": "Spare him his life from this monstrosity", "label": [1.]}, - - {"text": "See that girl, watch her scream, kicking the dancing queen", "label": [0.]}, - {"text": "See that girl, watch that scene, dig in the dancing queen", "label": [1.]}, - - {"text": "Excuse me while I kiss this guy", "label": [0.]}, - {"text": "Excuse me while I kiss the sky", "label": [1.]}, - - {"text": "Dancing queen, feel the beat from the tangerine", "label": [0.]}, - {"text": "Dancing queen, feel the beat from the tambourine", "label": [1.]}, - - {"text": "Sweet dreams are made of cheese", "label": [0.]}, - {"text": "Sweet dreams are made of these", "label": [1.]}, - - {"text": "Calling Jamaica", "label": [0.]}, - {"text": "Call me when you try to wake her", "label": [1.]}, - - {"text": "Or should I just keep chasing penguins", "label": [0.]}, - {"text": "Or should I just keep chasing pavements", "label": [1.]}, - - {"text": "All the lonely Starbucks lovers", "label": [0.]}, - {"text": "Got a long list of ex-lovers", "label": [1.]}, - - {"text": "I can see clearly now, Lorraine is gone", "label": [0.]}, - {"text": "I can see clearly now, the rain is gone", "label": [1.]}, - - {"text": "Gimme Gimme Gimme a man after midnight, take me to the doctors at the break of the day", "label": [0.]}, - {"text": "Gimme Gimme Gimme a man after midnight, take me through the darkness to the break of the day", "label": [1.]}, - - {"text": "Poppadom Peach", "label": [0.]}, - {"text": "Papa don’t preach", "label": [1.]}, - - {"text": "It doesn’t make a difference if we’re naked or not", "label": [0.]}, - {"text": "It doesn’t make a difference if we make it or not", "label": [1.]}, - - {"text": "I'm farting carrots", "label": [0.]}, - {"text": "I'm 14 carat", "label": [1.]}, - - {"text": "Then I saw her face, now I'm gonna leave her", "label": [0.]}, - {"text": "Then I saw her face, now I'm a believer", "label": [1.]}, - - {"text": "I want to hold your ham", "label": [0.]}, - {"text": "I want to hold your hand", "label": [1.]}, - - {"text": "Kicking your cat all over the place", "label": [0.]}, - {"text": "Kicking your can all over the place", "label": [1.]}, -]) - - -test_df = pd.DataFrame([ - {"text": "Blue seal in the sky with diamonds", "label": [0.]}, - {"text": "Lucy in the sky with diamonds", "label": [1.]}, - - {"text": "Here we are now, in containers", "label": [0.]}, - {"text": "Here we are now, entertain us", "label": [1.]}, - - {"text": "Let's pee in the corner, let's pee in the spotlight", "label": [0.]}, - {"text": "That's me in the corner, that's me in the spotlight", "label": [1.]}, - - {"text": "I remove umbilicals", "label": [0.]}, - {"text": "I believe in miracles", "label": [1.]}, - - {"text": "I like big butts in a can of limes", "label": [0.]}, - {"text": "I like big butts and I cannot lie", "label": [1.]}, -]) - - -mem=[] -for i in train_dd: - info={} - text=Poscar(Atoms.from_dict(i['atoms'])).to_string() - #text=(Atoms.from_dict(i['atoms'])).composition.reduced_formula - #text=json.dumps(i['atoms']) - info['text']=text - info['label']=[i[prop]] - mem.append(info) -train_df = pd.DataFrame(mem) - -mem=[] -for i in test_dd: - info={} - text=Poscar(Atoms.from_dict(i['atoms'])).to_string() - #text=(Atoms.from_dict(i['atoms'])).composition.reduced_formula - #text=json.dumps(i['atoms']) - info['text']=text - info['label']=[i[prop]] - mem.append(info) -test_df = pd.DataFrame(mem) - -config = GPT2Config.from_pretrained( - "gpt2", - pad_token_id=50256, # eos_token_id - num_labels=1, -) -tokenizer = GPT2TokenizerFast.from_pretrained( - config.model_type, - padding=True, - truncation=True, - pad_token_id=config.pad_token_id, - pad_token="<|endoftext|>", # eos_token -) -tokenizer.pad_token -model = GPT2ForSequenceClassification(config) - -def tokenize(df: pd.DataFrame, tokenizer: GPT2TokenizerFast) -> List[Dict[str, Any]]: - tokenized_df = pd.DataFrame( - df.text.apply(tokenizer).tolist() - ) - return ( - pd.merge( - df, - tokenized_df, - left_index=True, - right_index=True, - ) - .drop(columns="text") - .to_dict("records") - ) - -train_ds = tokenize(train_df, tokenizer) -test_ds = tokenize(test_df, tokenizer) - -def compute_metrics(pred): - labels = pred.label_ids - predictions = pred.predictions - - return { - "mae": mean_absolute_error(labels, predictions), - #"mse": mean_squared_error(labels, predictions), - } - -training_args = TrainingArguments( - report_to="none", - evaluation_strategy="steps", - max_steps=100, - eval_steps=10, - metric_for_best_model="mse", - greater_is_better=False, - # going to delete all of this - output_dir="kaggle", - save_strategy="no", -) - -trainer = Trainer( - model=model, - args=training_args, - train_dataset=train_ds, - eval_dataset=test_ds, - tokenizer=tokenizer, - compute_metrics=compute_metrics -) - -trainer.train() diff --git a/atomgpt/scripts/usloth_gen.py b/atomgpt/scripts/usloth_gen.py deleted file mode 100644 index 41fa37f..0000000 --- a/atomgpt/scripts/usloth_gen.py +++ /dev/null @@ -1,137 +0,0 @@ -from unsloth import FastLanguageModel -import torch -from datasets import load_dataset -from trl import SFTTrainer -from transformers import TrainingArguments - -max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally! -dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+ -load_in_4bit = ( - True # Use 4bit quantization to reduce memory usage. Can be False. -) - -# 4bit pre quantized models we support for 4x faster downloading + no OOMs. -fourbit_models = [ - "unsloth/mistral-7b-bnb-4bit", - "unsloth/mistral-7b-instruct-v0.2-bnb-4bit", - "unsloth/llama-2-7b-bnb-4bit", - "unsloth/llama-2-13b-bnb-4bit", - "unsloth/codellama-34b-bnb-4bit", - "unsloth/tinyllama-bnb-4bit", -] # More models at https://huggingface.co/unsloth - -nm = "unsloth/mistral-7b-bnb-4bit" -nm = fourbit_models[-2] -nm = fourbit_models[0] -model, tokenizer = FastLanguageModel.from_pretrained( - model_name=nm, # Choose ANY! eg teknium/OpenHermes-2.5-Mistral-7B - max_seq_length=max_seq_length, - dtype=dtype, - load_in_4bit=load_in_4bit, - # token = "hf_...", # use one if using gated models like meta-llama/Llama-2-7b-hf -) - - -model = FastLanguageModel.get_peft_model( - model, - r=16, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128 - target_modules=[ - "q_proj", - "k_proj", - "v_proj", - "o_proj", - "gate_proj", - "up_proj", - "down_proj", - ], - lora_alpha=16, - lora_dropout=0, # Supports any, but = 0 is optimized - bias="none", # Supports any, but = "none" is optimized - use_gradient_checkpointing=True, - random_state=3407, - use_rslora=False, # We support rank stabilized LoRA - loftq_config=None, # And LoftQ -) - -alpaca_prompt = """Below is a description of a superconductor material.. - -### Instruction: -{} - -### Input: -{} - -### Output: -{}""" - -EOS_TOKEN = tokenizer.eos_token # Must add EOS_TOKEN - - -def formatting_prompts_func(examples): - instructions = examples["instruction"] - inputs = examples["input"] - outputs = examples["output"] - texts = [] - for instruction, input, output in zip(instructions, inputs, outputs): - # Must add EOS_TOKEN, otherwise your generation will go on forever! - text = alpaca_prompt.format(instruction, input, output) + EOS_TOKEN - texts.append(text) - return { - "text": texts, - } - - -dataset = load_dataset( - "json", data_files="alpaca_Tc_supercon.json", split="train" -) -dataset = dataset.map( - formatting_prompts_func, - batched=True, -) - -trainer = SFTTrainer( - model=model, - tokenizer=tokenizer, - train_dataset=dataset, - dataset_text_field="text", - max_seq_length=max_seq_length, - dataset_num_proc=2, - packing=False, # Can make training 5x faster for short sequences. - args=TrainingArguments( - per_device_train_batch_size=2, - gradient_accumulation_steps=4, - warmup_steps=5, - overwrite_output_dir=True, - # max_steps = 60, - learning_rate=2e-4, - fp16=not torch.cuda.is_bf16_supported(), - bf16=torch.cuda.is_bf16_supported(), - logging_steps=1, - optim="adamw_8bit", - weight_decay=0.01, - lr_scheduler_type="linear", - seed=3407, - output_dir="outputs", - num_train_epochs=10, - report_to="none", - ), -) - -trainer_stats = trainer.train() -model.save_pretrained("lora_model_m") -# alpaca_prompt = Copied from above -FastLanguageModel.for_inference(model) # Enable native 2x faster inference -inputs = tokenizer( - [ - alpaca_prompt.format( - "Below is a description of a superconductor material.", # instruction - "The chemical formula is YCI The Tc_supercon is 6.483. The spacegroup is 12. Generate atomic structure description with lattice lengths, angles, coordinates and atom types.", # input - "", # output - leave this blank for generation! - ) - ], - return_tensors="pt", -).to("cuda") - -outputs = model.generate(**inputs, max_new_tokens=512, use_cache=True) -# outputs = model.generate(**inputs, max_new_tokens = 128, use_cache = True) -print("xyz", tokenizer.batch_decode(outputs)) diff --git a/atomgpt/scripts/usloth_prop.py b/atomgpt/scripts/usloth_prop.py deleted file mode 100644 index af02399..0000000 --- a/atomgpt/scripts/usloth_prop.py +++ /dev/null @@ -1,406 +0,0 @@ -from sklearn.metrics import mean_absolute_error -import pandas as pd -from unsloth import FastLanguageModel -import torch -from datasets import load_dataset -from trl import SFTTrainer -from transformers import TrainingArguments -import re -import os -import json -import zipfile -from jarvis.core.atoms import Atoms -from jarvis.db.figshare import data -from jarvis.db.jsonutils import loadjson, dumpjson -from jarvis.analysis.structure.spacegroup import Spacegroup3D -from jarvis.analysis.diffraction.xrd import XRD -from jarvis.core.specie import Specie -from collections import defaultdict - - -def atoms_describer( - atoms=[], - xrd_peaks=5, - xrd_round=1, - cutoff=4, - take_n_bomds=2, - include_spg=True, -): - """Describe an atomic structure.""" - if include_spg: - spg = Spacegroup3D(atoms) - theta, d_hkls, intens = XRD().simulate(atoms=(atoms)) - # x = atoms.atomwise_angle_and_radial_distribution() - # bond_distances = {} - # for i, j in x[-1]["different_bond"].items(): - # bond_distances[i.replace("_", "-")] = ", ".join( - # map(str, (sorted(list(set([round(jj, 2) for jj in j]))))) - # ) - dists = defaultdict(list) - elements = atoms.elements - for i in atoms.get_all_neighbors(r=cutoff): - for j in i: - key = "-".join(sorted([elements[j[0]], elements[j[1]]])) - dists[key].append(j[2]) - bond_distances = {} - for i, j in dists.items(): - dist = sorted(set([round(k, 2) for k in j])) - if len(dist) >= take_n_bomds: - dist = dist[0:take_n_bomds] - bond_distances[i] = ", ".join(map(str, dist)) - fracs = {} - for i, j in (atoms.composition.atomic_fraction).items(): - fracs[i] = round(j, 3) - info = {} - chem_info = { - "atomic_formula": atoms.composition.reduced_formula, - "prototype": atoms.composition.prototype, - "molecular_weight": round(atoms.composition.weight / 2, 2), - "atomic_fraction": (fracs), - "atomic_X": ", ".join( - map(str, [Specie(s).X for s in atoms.uniq_species]) - ), - "atomic_Z": ", ".join( - map(str, [Specie(s).Z for s in atoms.uniq_species]) - ), - } - struct_info = { - "lattice_parameters": ", ".join( - map(str, [round(j, 2) for j in atoms.lattice.abc]) - ), - "lattice_angles": ", ".join( - map(str, [round(j, 2) for j in atoms.lattice.angles]) - ), - # "spg_number": spg.space_group_number, - # "spg_symbol": spg.space_group_symbol, - "top_k_xrd_peaks": ", ".join( - map( - str, - sorted(list(set([round(i, xrd_round) for i in theta])))[ - 0:xrd_peaks - ], - ) - ), - "density": round(atoms.density, 3), - # "crystal_system": spg.crystal_system, - # "point_group": spg.point_group_symbol, - # "wyckoff": ", ".join(list(set(spg._dataset["wyckoffs"]))), - "bond_distances": bond_distances, - # "natoms_primitive": spg.primitive_atoms.num_atoms, - # "natoms_conventional": spg.conventional_standard_structure.num_atoms, - } - if include_spg: - struct_info["spg_number"] = spg.space_group_number - struct_info["spg_symbol"] = spg.space_group_symbol - struct_info["crystal_system"] = spg.crystal_system - struct_info["point_group"] = spg.point_group_symbol - struct_info["wyckoff"] = ", ".join(list(set(spg._dataset["wyckoffs"]))) - struct_info["natoms_primitive"] = spg.primitive_atoms.num_atoms - struct_info["natoms_conventional"] = ( - spg.conventional_standard_structure.num_atoms - ) - info["chemical_info"] = chem_info - info["structure_info"] = struct_info - line = "The number of atoms are: " + str( - atoms.num_atoms - ) # +"., The elements are: "+",".join(atoms.elements)+". " - for i, j in info.items(): - if not isinstance(j, dict): - line += "The " + i + " is " + j + ". " - else: - # print("i",i) - # print("j",j) - for ii, jj in j.items(): - tmp = "" - if isinstance(jj, dict): - for iii, jjj in jj.items(): - tmp += iii + ": " + str(jjj) + " " - else: - tmp = jj - line += "The " + ii + " is " + str(tmp) + ". " - return line - - -def get_crystal_string_t(atoms): - lengths = atoms.lattice.abc # structure.lattice.parameters[:3] - angles = atoms.lattice.angles - atom_ids = atoms.elements - frac_coords = atoms.frac_coords - - crystal_str = ( - " ".join(["{0:.2f}".format(x) for x in lengths]) - + "\n" - + " ".join([str(int(x)) for x in angles]) - + "\n" - + "\n".join( - [ - str(t) + " " + " ".join(["{0:.3f}".format(x) for x in c]) - for t, c in zip(atom_ids, frac_coords) - ] - ) - ) - - # crystal_str = atoms_describer(atoms) + "\n*\n" + crystal_str - return crystal_str - - -def make_alpaca_json_gen(dataset=[], prop="Tc_supercon"): - alpaca_prompt = """Below is a description of a material.. - - ### Instruction: - {} - - ### Input: - {} - - ### Output: - {}""" - - mem = [] - all_ids = [] - for i in dataset: - if i[prop] != "na": - atoms = Atoms.from_dict(i["atoms"]) - info = {} - info["instruction"] = ( - "Below is a description of a superconductor material." - ) - info["input"] = ( - "The chemical formula is " - + atoms.composition.reduced_formula - + " The " - + prop - + " is " - + str(round(i[prop], 3)) - + ". The spacegroup is " - + i["spg_number"] - + "." - + " Generate atomic structure description with lattice lengths, angles, coordinates and atom types." - ) - info["output"] = get_crystal_string_t(atoms) - mem.append(info) - return mem - - -def make_alpaca_json_pred( - dataset=[], prop="Tc_supercon", id_tag="jid", ids=[] -): - alpaca_prompt = """Below is a description of a material.. - - ### Instruction: - {} - - ### Input: - {} - - ### Output: - {}""" - all_ids = [] - mem = [] - for i in dataset: - if i[prop] != "na" and i[id_tag] in ids: - atoms = Atoms.from_dict(i["atoms"]) - info = {} - info["instruction"] = ( - "Predict " + prop + " property of this material" - ) - info["input"] = get_crystal_string_t(atoms) - info["output"] = str(round(i[prop], 2)) - mem.append(info) - all_ids.append(i[id_tag]) - return alpaca_prompt, mem, all_ids - - -benchmark_file = ( - "AI-SinglePropertyPrediction-PBE_gap-halide_peroskites-test-mae.csv.zip" -) -root_dir = "/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard" -method = benchmark_file.split("-")[0] -task = benchmark_file.split("-")[1] -prop = benchmark_file.split("-")[2] -dataset = benchmark_file.split("-")[3] -temp = dataset + "_" + prop + ".json.zip" -temp2 = dataset + "_" + prop + ".json" -fname = os.path.join(root_dir, "benchmarks", method, task, temp) -zp = zipfile.ZipFile(fname) -bench = json.loads(zp.read(temp2)) -dft_3d = data(dataset) -id_tag = "jid" -if "jid" in dft_3d[0]: - id_tag = "jid" -else: - id_tag = "id" - -# train_atoms = [] -# val_atoms = [] -# test_atoms = [] -# train_targets = [] -# val_targets = [] -# test_targets = [] -train_ids = list(bench["train"].keys()) -test_ids = list(bench["test"].keys()) -if "val" in bench: - val_ids = list(bench["val"].keys()) -else: - val_ids = test_ids -print("total", len(dft_3d)) -print("test_ids", len(test_ids)) -print("val_ids", len(val_ids)) -print("train_ids", len(train_ids)) -alpaca_prompt, train_data, train_ids = make_alpaca_json_pred( - dataset=dft_3d, prop=prop, id_tag=id_tag, ids=train_ids -) -alpaca_prompt, test_data, test_ids = make_alpaca_json_pred( - dataset=dft_3d, prop=prop, id_tag=id_tag, ids=test_ids -) -dumpjson(data=train_data, filename="train_data.json") -dumpjson(data=test_data, filename="test_data.json") -model_path = "lora_model_train" - -max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally! -dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+ -load_in_4bit = ( - True # Use 4bit quantization to reduce memory usage. Can be False. -) - -# 4bit pre quantized models we support for 4x faster downloading + no OOMs. -fourbit_models = [ - "unsloth/mistral-7b-bnb-4bit", - "unsloth/mistral-7b-instruct-v0.2-bnb-4bit", - "unsloth/llama-2-7b-bnb-4bit", - "unsloth/llama-2-13b-bnb-4bit", - "unsloth/codellama-34b-bnb-4bit", - "unsloth/tinyllama-bnb-4bit", -] # More models at https://huggingface.co/unsloth - -nm = "unsloth/mistral-7b-bnb-4bit" -nm = fourbit_models[-2] -# nm = fourbit_models[0] -model, tokenizer = FastLanguageModel.from_pretrained( - model_name=nm, # Choose ANY! eg teknium/OpenHermes-2.5-Mistral-7B - max_seq_length=max_seq_length, - dtype=dtype, - load_in_4bit=load_in_4bit, - # token = "hf_...", # use one if using gated models like meta-llama/Llama-2-7b-hf -) - - -model = FastLanguageModel.get_peft_model( - model, - r=16, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128 - target_modules=[ - "q_proj", - "k_proj", - "v_proj", - "o_proj", - "gate_proj", - "up_proj", - "down_proj", - ], - lora_alpha=16, - lora_dropout=0, # Supports any, but = 0 is optimized - bias="none", # Supports any, but = "none" is optimized - use_gradient_checkpointing=True, - random_state=3407, - use_rslora=False, # We support rank stabilized LoRA - loftq_config=None, # And LoftQ -) - - -EOS_TOKEN = tokenizer.eos_token # Must add EOS_TOKEN - - -def formatting_prompts_func(examples): - instructions = examples["instruction"] - inputs = examples["input"] - outputs = examples["output"] - texts = [] - for instruction, input, output in zip(instructions, inputs, outputs): - # Must add EOS_TOKEN, otherwise your generation will go on forever! - text = alpaca_prompt.format(instruction, input, output) + EOS_TOKEN - texts.append(text) - return { - "text": texts, - } - - -dataset = load_dataset("json", data_files="train_data.json", split="train") -dataset = dataset.map( - formatting_prompts_func, - batched=True, -) - -trainer = SFTTrainer( - model=model, - tokenizer=tokenizer, - train_dataset=dataset, - dataset_text_field="text", - max_seq_length=max_seq_length, - dataset_num_proc=2, - packing=False, # Can make training 5x faster for short sequences. - args=TrainingArguments( - per_device_train_batch_size=2, - gradient_accumulation_steps=4, - warmup_steps=5, - overwrite_output_dir=True, - # max_steps = 60, - learning_rate=2e-4, - fp16=not torch.cuda.is_bf16_supported(), - bf16=torch.cuda.is_bf16_supported(), - logging_steps=1, - optim="adamw_8bit", - weight_decay=0.01, - lr_scheduler_type="linear", - seed=3407, - output_dir="outputs", - num_train_epochs=3, - report_to="none", - ), -) -trainer_stats = trainer.train() -model.save_pretrained(model_path) - -model_x, tokenizer = FastLanguageModel.from_pretrained( - model_name=model_path, # YOUR MODEL YOU USED FOR TRAINING - max_seq_length=max_seq_length, - dtype=dtype, - load_in_4bit=load_in_4bit, -) -FastLanguageModel.for_inference(model_x) # Enable native 2x faster inference - -# alpaca_prompt = You MUST copy from above! - -f = open("sloth_prop.csv", "w") -f.write("id,target,prediction\n") -for ii, i in enumerate(test_data): - inputs = tokenizer( - [ - alpaca_prompt.format( - "Predict " - + prop - + " property of this material", # instruction - i["input"], # input - "", # output - leave this blank for generation! - ) - ], - return_tensors="pt", - ).to("cuda") - - outputs = tokenizer.batch_decode( - model_x.generate(**inputs, max_new_tokens=64, use_cache=True) - )[0].split("### Output:\n")[-1] - floats = [float(j) for j in re.findall(r"\b\d+\.\d+\b", outputs)] - print(test_ids[ii], ",", i["output"], ",", floats[0]) - line = ( - str(test_ids[ii]) - + "," - + str(i["output"]) - + "," - + str(floats[0]) - + "\n" - ) - f.write(line) - # print(test_ids[ii], ",",i["output"].split("## Output:\\n")[1].split("")[0], ",",tokenizer.batch_decode(outputs)) -f.close() -df = pd.read_csv("sloth_prop.csv") -print("mae", mean_absolute_error(df["target"], df["prediction"])) diff --git a/atomgpt/train_id_prop.py b/atomgpt/train_id_prop.py deleted file mode 100644 index 78bcd03..0000000 --- a/atomgpt/train_id_prop.py +++ /dev/null @@ -1,716 +0,0 @@ -"""Module for fin tuning LLM model for materials chemsitry.""" - -from jarvis.db.figshare import data -import transformers -import torch -import random -from jarvis.db.jsonutils import loadjson, dumpjson -from torch.utils.data import DataLoader, Dataset -import numpy as np -import os -from jarvis.core.atoms import Atoms -import pandas as pd -from sklearn.metrics import mean_absolute_error -import json -from jarvis.db.figshare import get_jid_data -from jarvis.core.atoms import Atoms -from jarvis.analysis.structure.spacegroup import Spacegroup3D -from jarvis.analysis.diffraction.xrd import XRD -from jarvis.core.specie import Specie -import pprint -from collections import defaultdict -from tqdm import tqdm -import time -import json -import zipfile -from typing import Optional -from pydantic_settings import BaseSettings - - -class TrainingPropConfig(BaseSettings): - """Training config defaults and validation.""" - - id_prop_path: Optional[str] = "robo_desc.json.zip" - prefix: str = "atomgpt_run" - model_name: str = "gpt2" - batch_size: int = 16 - max_length: int = 512 - num_epochs: int = 500 - latent_dim: int = 1024 - learning_rate: float = 1e-3 - test_each_run: bool = True - include_struct: bool = False - pretrained_path: str = "" - seed_val: int = 42 - n_train: Optional[int] = None - n_val: Optional[int] = None - n_test: Optional[int] = None - output_dir: str = "out_temp" - train_ratio: Optional[float] = None - val_ratio: float = 0.1 - test_ratio: float = 0.1 - keep_data_order: bool = True - - -def get_id_train_val_test( - total_size=1000, - split_seed=123, - train_ratio=None, - val_ratio=0.1, - test_ratio=0.1, - n_train=None, - n_test=None, - n_val=None, - keep_data_order=True, -): - """Get train, val, test IDs.""" - if ( - train_ratio is None - and val_ratio is not None - and test_ratio is not None - ): - if train_ratio is None: - assert val_ratio + test_ratio < 1 - train_ratio = 1 - val_ratio - test_ratio - print("Using rest of the dataset except the test and val sets.") - else: - assert train_ratio + val_ratio + test_ratio <= 1 - # indices = list(range(total_size)) - if n_train is None: - n_train = int(train_ratio * total_size) - if n_test is None: - n_test = int(test_ratio * total_size) - if n_val is None: - n_val = int(val_ratio * total_size) - ids = list(np.arange(total_size)) - if not keep_data_order: - random.seed(split_seed) - random.shuffle(ids) - # np.random.shuffle(ids) - if n_train + n_val + n_test > total_size: - raise ValueError( - "Check total number of samples.", - n_train + n_val + n_test, - ">", - total_size, - ) - - # shuffle consistently with https://github.com/txie-93/cgcnn/data.py - # i.e. shuffle the index in place with standard library random.shuffle - # first obtain only valid indices - - # test_size = round(N * 0.2) - - # full train/val test split - # ids = ids[::-1] - id_train = ids[:n_train] - id_val = ( - ids[-(n_val + n_test) : -n_test] - if n_test > 0 - else ids[-(n_val + n_test) :] - ) # noqa:E203 - id_test = ids[-n_test:] if n_test > 0 else [] - return id_train, id_val, id_test - - -def make_id_prop( - benchmark_file="AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae.csv.zip", - desc_file="robo_desc.json.zip", - leaderboard_dir="/wrk/knc6/AFFBench/jarvis_leaderboard/jarvis_leaderboard", - # leaderboard_dir="/work/03943/kamalch/ls6/Software/atomgpt/jarvis_leaderboard/jarvis_leaderboard/", - output_dir="test_id_prop", -): - print("benchmark_file", benchmark_file) - method = benchmark_file.split("-")[0] - task = benchmark_file.split("-")[1] - prop_name = benchmark_file.split("-")[2] - dataset = benchmark_file.split("-")[3] - temp = dataset + "_" + prop_name + ".json.zip" - temp2 = dataset + "_" + prop_name + ".json" - fname = os.path.join(leaderboard_dir, "benchmarks", method, task, temp) - zp = zipfile.ZipFile(fname) - bench = json.loads(zp.read(temp2)) - dft_3d = data(dataset) - id_tag = "jid" - output_dir = prop_name + "_" + dataset - if "jid" in dft_3d[0]: - id_tag = "jid" - else: - id_tag = "id" - if not os.path.exists(output_dir): - os.makedirs(output_dir) - train_ids = list(bench["train"].keys()) - test_ids = list(bench["test"].keys()) - if "val" in bench: - val_ids = list(bench["val"].keys()) - else: - val_ids = test_ids - print("Saving files in", output_dir) - if ".zip" in desc_file: - zp = zipfile.ZipFile(desc_file) - dat = json.loads(zp.read(desc_file.split(".zip")[0].split("/")[-1])) - - else: - dat = loadjson(desc_file) - - dat2 = {} - for i in dat: - dat2[i["id"]] = i["desc"] - dft_3d2 = {} - for i in dft_3d: - dft_3d2[i[id_tag]] = i - mem = [] - for i in train_ids: - desc = dat2[i] - prop = dft_3d2[i][prop_name] - info = {} - info["id"] = i - info["desc"] = desc - info["prop"] = prop - mem.append(info) - for i in val_ids: - desc = dat2[i] - - prop = dft_3d2[i][prop_name] - info = {} - info["id"] = i - info["desc"] = desc - info["prop"] = prop - mem.append(info) - for i in test_ids: - desc = dat2[i] - prop = dft_3d2[i][prop_name] - info = {} - info["id"] = i - info["desc"] = desc - info["prop"] = prop - mem.append(info) - print("total", len(dft_3d)) - print("test_ids", len(test_ids)) - print("val_ids", len(val_ids)) - print("train_ids", len(train_ids)) - filename = os.path.join(output_dir, "id_prop_llm.json") - filename_config = os.path.join(output_dir, "config.json") - minfo = {} - minfo["n_train"] = len(train_ids) - minfo["n_val"] = len(val_ids) - minfo["n_test"] = len(test_ids) - minfo["id_prop_path"] = os.path.abspath(filename) - minfo["output_dir"] = os.path.abspath(output_dir) - - dumpjson(data=minfo, filename=filename_config) - dumpjson(data=mem, filename=filename) - return output_dir - - -## -os.environ["WANDB_ANONYMOUS"] = "must" -random_seed = 42 -random.seed(random_seed) -torch.manual_seed(random_seed) -np.random.seed(random_seed) -torch.cuda.manual_seed_all(random_seed) -try: - import torch_xla.core.xla_model as xm - - xm.set_rng_state(random_seed) -except ImportError: - pass -torch.backends.cudnn.deterministic = True -torch.backends.cudnn.benchmark = False -os.environ["PYTHONHASHSEED"] = str(random_seed) -os.environ["CUBLAS_WORKSPACE_CONFIG"] = str(":4096:8") -torch.use_deterministic_algorithms(True) -device = "cpu" -if torch.cuda.is_available(): - device = torch.device("cuda") -# device = "cpu" - - -# Define a custom dataset class for regression -class AtomGPTDataset(Dataset): - def __init__( - self, texts=[], targets=[], ids=[], tokenizer="", max_length=128 - ): - self.texts = texts - self.targets = targets - self.tokenizer = tokenizer - self.max_length = max_length - if not ids: - ids = ["text-" + str(i) for i in range(len(texts))] - self.ids = ids - - def __len__(self): - return len(self.texts) - - def __getitem__(self, idx): - inputs = self.tokenizer( - self.texts[idx], - return_tensors="pt", - max_length=self.max_length, - padding="max_length", - truncation=True, - ) - # torch.tensor(inputs*10,dtype=inputs.dtype) - return ( - inputs, - self.ids[idx], - torch.tensor(self.targets[idx], dtype=torch.float32), - ) - - -# Example usage - - -def run_atomgpt(config_file="config.json"): - print("Running AtomGPT prop predictor.") - config = loadjson(config_file) - config = TrainingPropConfig(**config) - id_prop_path = config.id_prop_path - if ".zip" in id_prop_path: - zp = zipfile.ZipFile(id_prop_path) - dat = json.loads(zp.read(id_prop_path.split(".zip")[0])) - else: - dat = loadjson(id_prop_path) - print("len", len(dat)) - prefix = config.prefix - model_name = config.model_name - batch_size = config.batch_size - max_length = config.max_length - num_epochs = config.num_epochs - latent_dim = config.latent_dim - learning_rate = config.learning_rate - test_each_run = config.test_each_run - pretrained_path = config.pretrained_path - seed_val = config.seed_val - include_struct = config.include_struct - n_train = config.n_train - n_val = config.n_val - n_test = config.n_test - train_ratio = config.train_ratio - val_ratio = config.val_ratio - test_ratio = config.test_ratio - output_dir = config.output_dir - keep_data_order = config.keep_data_order - - f = open(os.path.join(config.output_dir, "config.json"), "w") - f.write(json.dumps(config.dict(), indent=4)) - f.close() - - id_train, id_val, id_test = get_id_train_val_test( - total_size=len(dat), - split_seed=seed_val, - train_ratio=train_ratio, - val_ratio=val_ratio, - test_ratio=test_ratio, - n_train=n_train, - n_test=n_test, - n_val=n_val, - keep_data_order=keep_data_order, - ) - - train_texts = [] - train_targets = [] - train_ids_temp = [] - val_texts = [] - val_targets = [] - val_ids_temp = [] - test_texts = [] - test_targets = [] - test_ids_temp = [] - train_info = [] - val_info = [] - test_info = [] - for ii, i in enumerate(dat): - if ii in id_train: - train_texts.append(i["desc"]) - train_targets.append(i["prop"]) - train_ids_temp.append(i["id"]) - train_info.append(i) - if ii in id_test: - test_texts.append(i["desc"]) - test_targets.append(i["prop"]) - test_ids_temp.append(i["id"]) - val_info.append(i) - if ii in id_val: - val_texts.append(i["desc"]) - val_targets.append(i["prop"]) - val_ids_temp.append(i["id"]) - test_info.append(i) - print("test_texts:", len(test_texts)) - print("val_texts example:", val_texts[0]) - print("test_texts example:", test_texts[0]) - - print("Train\n", pd.DataFrame(train_info)) - print("Val\n", pd.DataFrame(val_info)) - print("test\n", pd.DataFrame(test_info)) - - print("total", len(dat)) - print("test_ids", len(id_test)) - print("val_ids", len(id_val)) - print("train_ids", len(id_train)) - # model_name = "mistralai/Mistral-7B-Instruct-v0.1" - # model_name = "gpt2" - if "t5" in model_name: - model = transformers.T5ForConditionalGeneration.from_pretrained( - model_name - ) - else: - model = transformers.AutoModelForCausalLM.from_pretrained( - model_name, - low_cpu_mem_usage=True, - # load_in_8bit=False, - # torch_dtype=torch.float16, - # load_in_8bit=True, - # device_map="auto" - ) - # device = model.device - if "t5" in model_name: - tokenizer = transformers.T5Tokenizer.from_pretrained(model_name) - - else: - tokenizer = transformers.AutoTokenizer.from_pretrained(model_name) - - # batch_size = 16 - # max_length = 128 - # num_epochs = 100 - # learning_rate = 5e-5 - criterion = torch.nn.L1Loss() - # Define example regression data (texts and corresponding numeric targets) - """ - ############################## - ###Fast test### - train_texts = [ - "This is the first example text.", - "Second example is a bit longer than the first one, but still within the max length.", - "Third example is the longest among these three examples. It exceeds the max length and will be truncated.", - "Second example is a bit longer than the first one, but still within the max length.", - ] - train_targets = [10.2, 15.5, 20.1, 15.5] # Example regression targets - val_texts = test_texts = train_texts - val_targets = test_targets = train_targets - train_ids_temp=['a','b','c','d'] - val_ids_temp = test_ids_temp = train_ids_temp - batch_size = 2 - num_epochs = 3 - - ############################## - ############################## - """ - - # Fine-tune the last layer of GPT-2 for regression - # fine_tune_gpt2_regression(train_texts, train_targets, tokenizer) - if tokenizer.pad_token is None: - tokenizer.add_special_tokens({"pad_token": "[PAD]"}) - tokenizer.add_special_tokens({"unk_token": "#"}) - tokenizer.add_special_tokens({"unk_token": "&"}) - tokenizer.add_special_tokens({"unk_token": "@"}) - model.resize_token_embeddings(len(tokenizer)) - model.lm_head = torch.nn.Sequential( - # torch.nn.Linear(model.config.hidden_size, 1), - torch.nn.Linear(model.config.hidden_size, latent_dim), - # torch.nn.Linear( latent_dim,256), - # torch.nn.Transformer(d_model=latent_dim, nhead=1, num_encoder_layers=1, num_decoder_layers=1), - # torch.nn.Linear(latent_dim, latent_dim), - # torch.nn.Linear(latent_dim, latent_dim), - # torch.nn.ReLU(), - # torch.nn.LeakyReLU(), - # torch.nn.Dropout(p=0.2), - # torch.nn.TransformerEncoder(torch.nn.TransformerEncoderLayer(d_model=latent_dim, nhead=4), num_layers=2), - # torch.nn.Linear(256, 1), - torch.nn.Linear(latent_dim, 1), - ) - if pretrained_path != "": - model.load_state_dict(torch.load(pretrained_path, map_location=device)) - # model.lm_head = torch.nn.Sequential(torch.nn.Linear( model.config.hidden_size, 256),torch.nn.SiLU(),torch.nn.Linear( 256, 1) ) - # set_seed(seed) - # set_deterministic() - model.to(device) - if torch.cuda.device_count() > 1: - device_ids = [d for d in range(torch.cuda.device_count())] - model = torch.nn.DataParallel(model, device_ids=device_ids).cuda() - optimizer = transformers.AdamW(model.parameters(), lr=learning_rate) - # Prepare datasets and dataloaders with data collator - # TODO: knc6 change later - train_dataset = AtomGPTDataset( - texts=train_texts, - targets=train_targets, - ids=train_ids_temp, - tokenizer=tokenizer, - max_length=max_length, - ) - test_dataset = AtomGPTDataset( - texts=val_texts, - targets=val_targets, - tokenizer=tokenizer, - ids=val_ids_temp, - max_length=max_length, - ) - val_dataset = AtomGPTDataset( - texts=test_texts, - targets=test_targets, - tokenizer=tokenizer, - ids=test_ids_temp, - max_length=max_length, - ) - train_dataloader = DataLoader(train_dataset, batch_size=batch_size) - val_dataloader = DataLoader(val_dataset, batch_size=batch_size) - test_dataloader = DataLoader(test_dataset, batch_size=batch_size) - steps_per_epoch = len(train_dataloader) - scheduler = torch.optim.lr_scheduler.OneCycleLR( - optimizer, - max_lr=learning_rate, - epochs=num_epochs, - steps_per_epoch=steps_per_epoch, - # pct_start=pct_start, - pct_start=0.3, - ) - # output_dir = prefix + "_out" # + model_name + "_" + dataset + "_" + prop - if not os.path.exists(output_dir): - os.makedirs(output_dir) - best_loss = np.inf - tot_time_start = time.time() - train_history = [] - val_history = [] - for epoch in range(num_epochs): - model.train() - t1 = time.time() - for batch in train_dataloader: - optimizer.zero_grad() - train_loss = 0 - # train_result = [] - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - targets = batch[2].squeeze() - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - # print('train',predictions,targets) - loss.backward() - optimizer.step() - # optimizer.zero_grad() - train_loss += loss.item() - scheduler.step() - train_loss = train_loss / len(train_dataloader) - t2 = time.time() - train_time = round(t2 - t1, 3) - - # total_eval_mae_loss = 0 - # predictions_list = [] - # targets_list = [] - model.eval() - val_loss = 0 - t1 = time.time() - fname = os.path.join(output_dir, "val_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - with torch.no_grad(): - for batch in val_dataloader: - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - ids = batch[1] - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - targets = batch[2].squeeze() - # print('val',predictions,targets) - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - val_loss += loss.item() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - f.write(line) - f.close() - saving_tag = "" - if val_loss < best_loss: - best_loss = val_loss - best_model_name = "best_model.pt" - torch.save( - model.state_dict(), - os.path.join(output_dir, best_model_name), - ) - # print("Saving model for epoch", epoch) - saving_tag = " saving model:" + str(epoch) - val_loss = val_loss / len(val_dataloader) - t2 = time.time() - val_time = round(t2 - t1, 3) - train_history.append(train_loss) - val_history.append(val_loss) - history = os.path.join(output_dir, "history.json") - - dumpjson( - data={"train": train_history, "val": val_history}, filename=history - ) - mae = "" - model.eval() - with torch.no_grad(): - if test_each_run: - t1_test = time.time() - # model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - test_loss = 0 - for batch in test_dataloader: - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - test_loss += loss.item() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - f.write(line) - test_loss = test_loss / len(test_dataloader) - t2_test = time.time() - test_time = round(t2_test - t1_test, 3) - f.close() - df = pd.read_csv(fname) - mae = mean_absolute_error(df["target"], df["predictions"]) - if mae == "": - print( - "Epoch, train loss, val loss, train_time, val_time", - epoch, - train_loss, - val_loss, - train_time, - val_time, - saving_tag, - ) - else: - print( - "Epoch, train loss, val loss, test loss, train_time, val_time, test_time", - epoch, - train_loss, - val_loss, - # mae, - test_loss, - train_time, - val_time, - test_time, - saving_tag, - ) - - model.eval() - fname = os.path.join(output_dir, "test_results_final.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - optimizer.zero_grad() - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - f.close() - tot_time_end = time.time() - tot_time = tot_time_end - tot_time_start - print("tot_time", tot_time) - - -if __name__ == "__main__": - output_dir = make_id_prop() - run_atomgpt(config_file=output_dir + "/config.json") - # config_file="config.json" - # ) diff --git a/atomgpt/train_prop.py b/atomgpt/train_prop.py deleted file mode 100644 index 40272c3..0000000 --- a/atomgpt/train_prop.py +++ /dev/null @@ -1,416 +0,0 @@ -#!/usr/bin/env python -"""Module to train properties.""" -import transformers -from atomgpt.data.dataset import data_from_benchmark_file, data_from_id_prop -from atomgpt.config import TrainingPropConfig -import os -import json -import zipfile -import torch -from jarvis.db.figshare import data -import time -import pandas as pd -from sklearn.metrics import mean_absolute_error -import random -import numpy as np -import os -from jarvis.db.jsonutils import loadjson, dumpjson -import sys -import argparse -import pprint - -device = "cpu" -if torch.cuda.is_available(): - device = torch.device("cuda") - -parser = argparse.ArgumentParser(description="AtomGPT") -parser.add_argument( - "--config_file", - default="config.json", - help="Config file", -) - - -def set_seed(random_seed=42): - os.environ["WANDB_ANONYMOUS"] = "must" - # random_seed = 42 - random.seed(random_seed) - torch.manual_seed(random_seed) - np.random.seed(random_seed) - torch.cuda.manual_seed_all(random_seed) - try: - import torch_xla.core.xla_model as xm - - xm.set_rng_state(random_seed) - except ImportError: - pass - torch.backends.cudnn.deterministic = True - torch.backends.cudnn.benchmark = False - os.environ["PYTHONHASHSEED"] = str(random_seed) - os.environ["CUBLAS_WORKSPACE_CONFIG"] = str(":4096:8") - torch.use_deterministic_algorithms(True) - - -def run_atomgpt(config_file=""): - print("Running AtomGPT prop predictor.") - config = loadjson(config_file) - config = TrainingPropConfig(**config) - benchmark_file = config.benchmark_file - id_prop_path = config.id_prop_path - prefix = config.prefix - model_name = config.model_name - leaderboard_dir = config.leaderboard_dir - batch_size = config.batch_size - max_length = config.max_length - num_epochs = config.num_epochs - latent_dim = config.latent_dim - learning_rate = config.learning_rate - test_each_run = config.test_each_run - pretrained_path = config.pretrained_path - seed_val = config.seed_val - include_struct = config.include_struct - n_train = config.n_train - n_val = config.n_val - n_test = config.n_test - train_ratio = config.train_ratio - val_ratio = config.val_ratio - test_ratio = config.test_ratio - keep_data_order = config.keep_data_order - output_dir = config.output_dir - print("configs", pprint.pprint(config.dict())) - set_seed(random_seed=seed_val) - if "t5" in model_name: - model = transformers.T5ForConditionalGeneration.from_pretrained( - model_name - ) - else: - model = transformers.AutoModelForCausalLM.from_pretrained( - model_name, - low_cpu_mem_usage=True, - # load_in_8bit=False, - # torch_dtype=torch.float16, - # load_in_8bit=True, - # device_map="auto" - ) - # device = model.device - if "t5" in model_name: - tokenizer = transformers.T5Tokenizer.from_pretrained(model_name) - - else: - tokenizer = transformers.AutoTokenizer.from_pretrained(model_name) - - if tokenizer.pad_token is None: - tokenizer.add_special_tokens({"pad_token": "[PAD]"}) - tokenizer.add_special_tokens({"unk_token": "#"}) - tokenizer.add_special_tokens({"unk_token": "&"}) - tokenizer.add_special_tokens({"unk_token": "@"}) - model.resize_token_embeddings(len(tokenizer)) - model.lm_head = torch.nn.Sequential( - torch.nn.Linear(model.config.hidden_size, latent_dim), - torch.nn.Linear(latent_dim, 1), - ) - if benchmark_file is not None: - ( - train_dataloader, - val_dataloader, - test_dataloader, - ) = data_from_benchmark_file( - benchmark_file=benchmark_file, - leaderboard_dir=leaderboard_dir, - tokenizer=tokenizer, - max_length=max_length, - batch_size=batch_size, - include_struct=include_struct, - ) - elif id_prop_path is not None: - train_dataloader, val_dataloader, test_dataloader = data_from_id_prop( - id_prop_path=id_prop_path, - tokenizer=tokenizer, - max_length=max_length, - split_seed=seed_val, - n_train=n_train, - n_val=n_val, - n_test=n_test, - train_ratio=train_ratio, - val_ratio=val_ratio, - test_ratio=test_ratio, - keep_data_order=keep_data_order, - batch_size=batch_size, - include_struct=include_struct, - calc_desc=False, - ) - else: - raise ValueError("Provide id_prop_path or benchmark_file") - - val_dataloader = test_dataloader # for now - if pretrained_path != "": - model.load_state_dict(torch.load(pretrained_path, map_location=device)) - model.to(device) - if torch.cuda.device_count() > 1: - device_ids = [d for d in range(torch.cuda.device_count())] - model = torch.nn.DataParallel(model, device_ids=device_ids).cuda() - criterion = torch.nn.L1Loss() - optimizer = transformers.AdamW(model.parameters(), lr=learning_rate) - steps_per_epoch = len(train_dataloader) - scheduler = torch.optim.lr_scheduler.OneCycleLR( - optimizer, - max_lr=learning_rate, - epochs=num_epochs, - steps_per_epoch=steps_per_epoch, - # pct_start=pct_start, - pct_start=0.3, - ) - # print("train_data", len(train_texts)) - # print("test_data", len(test_texts)) - # output_dir = prefix + "_out_" + model_name - if not os.path.exists(output_dir): - os.makedirs(output_dir) - best_loss = np.inf - tot_time_start = time.time() - train_history = [] - val_history = [] - for epoch in range(num_epochs): - model.train() - t1 = time.time() - for batch in train_dataloader: - optimizer.zero_grad() - train_loss = 0 - # train_result = [] - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - targets = batch[2].squeeze() - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - # print('train',predictions,targets) - loss.backward() - optimizer.step() - scheduler.step() - # optimizer.zero_grad() - train_loss += loss.item() - train_loss = train_loss / len(train_dataloader) - t2 = time.time() - train_time = round(t2 - t1, 3) - model.eval() - - # total_eval_mae_loss = 0 - # predictions_list = [] - # targets_list = [] - val_loss = 0 - t1 = time.time() - for batch in val_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - targets = batch[2].squeeze() - # print('val',predictions,targets) - loss = criterion( - predictions.squeeze(), targets.squeeze().to(device) - ) - val_loss += loss.item() - saving_tag = "" - if val_loss < best_loss: - best_loss = val_loss - best_model_name = "best_model.pt" - torch.save( - model.state_dict(), - os.path.join(output_dir, best_model_name), - ) - # print("Saving model for epoch", epoch) - saving_tag = " saving model:" + str(epoch) - val_loss = val_loss / len(val_dataloader) - t2 = time.time() - val_time = round(t2 - t1, 3) - train_history.append(train_loss) - val_history.append(val_loss) - history = os.path.join(output_dir, "history.json") - - dumpjson( - data={"train": train_history, "val": val_history}, filename=history - ) - mae = "" - if test_each_run: - t1_test = time.time() - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - - else: - predictions = ( - model( - input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - f.write(line) - t2_test = time.time() - test_time = round(t2_test - t1_test, 3) - f.close() - df = pd.read_csv(fname) - mae = mean_absolute_error(df["target"], df["predictions"]) - if mae == "": - print( - "Epoch, train loss, val loss, train_time, val_time", - epoch, - train_loss, - val_loss, - train_time, - val_time, - saving_tag, - ) - else: - print( - "Epoch, train loss, val loss, test loss, train_time, val_time, test_time", - epoch, - train_loss, - val_loss, - mae, - train_time, - val_time, - test_time, - saving_tag, - ) - - model.eval() - fname = os.path.join(output_dir, "test_results.csv") - f = open(fname, "w") - f.write("id,target,predictions\n") - for batch in test_dataloader: - with torch.no_grad(): - input_ids = batch[0]["input_ids"].squeeze() # .squeeze(0) - if "t5" in model_name: - predictions = ( - model( - input_ids.to(device), - decoder_input_ids=input_ids.to(device), - ) - .logits.squeeze() - .mean(dim=-1) - ) - else: - predictions = ( - model(input_ids.to(device)).logits.squeeze().mean(dim=-1) - ) - ids = batch[1] - targets = batch[2].squeeze() - if len(ids) == 1: - targets = [targets] - predictions = [predictions] - # ids=[ids] - for ii, jj, kk in zip(targets, predictions, ids): - # print(kk,ii.cpu().detach().numpy().tolist(),jj.cpu().detach().numpy().tolist()) - line = ( - str(kk) - + "," - + str(round(ii.cpu().detach().numpy().tolist(), 3)) - + "," - + str(round(jj.cpu().detach().numpy().tolist(), 3)) - + "\n" - ) - # f.write("%s, %6f, %6f\n" % (kk, ii.cpu().detach().numpy().tolist(), jj.cpu().detach().numpy().tolist())) - # print(line) - f.write(line) - f.close() - tot_time_end = time.time() - tot_time = tot_time_end - tot_time_start - print("tot_time", tot_time) - - -if __name__ == "__main__": - # box = [[2.715, 2.715, 0], [0, 2.715, 2.715], [2.715, 0, 2.715]] - # coords = [[0, 0, 0], [0.25, 0.2, 0.25]] - # elements = ["Si", "Si"] - # Si = Atoms(lattice_mat=box, coords=coords, elements=elements) - # tmp=atoms_describer(Si) - # print(tmp) - # import sys - # sys.exit() - args = parser.parse_args(sys.argv[1:]) - config_file = args.config_file - # "AI-SinglePropertyPrediction-PBE_gap-halide_peroskites-test-mae.csv.zip" - # "AI-SinglePropertyPrediction-Tc_supercon-dft_3d-test-mae.csv.zip" - # id_prop_path = ( - # "/wrk/knc6/Software/mini_alignn/alignn/alignn/examples/sample_data" - # ) - # "AI-SinglePropertyPrediction-ead-tinnet_N-test-mae.csv.zip" - # "AI-SinglePropertyPrediction-exfoliation_energy-dft_3d-test-mae" - # args.benchmark_file - model_name = "facebook/opt-350m" - model_name = "mistralai/Mixtral-8x7B-v0.1" - model_name = "google/flan-t5-small" - model_name = "google/flan-t5-base" - model_name = "mistralai/Mistral-7B-Instruct-v0.1" - model_name = "google-t5/t5-small" - model_name = "xlnet/xlnet-base-cased" - model_name = "afmck/testing-llama-tiny" - model_name = "EleutherAI/gpt-neo-125m" - model_name = "openai-community/gpt2-medium" - model_name = "meta-llama/Llama-2-7b-hf" - model_name = "stas/tiny-random-llama-2" - model_name = "ahxt/llama2_xs_460M_experimental" - model_name = "gpt2" - run_atomgpt( - config_file=config_file, - ) From f219da8f9b822310265d763c6df33ad5f2fe7ff5 Mon Sep 17 00:00:00 2001 From: knc6 Date: Mon, 1 Jul 2024 05:03:39 -0400 Subject: [PATCH 2/5] Fix doc. --- dev-requirements.txt | 102 +++++++++++++++++++++++++++++++++++++++++++ setup.py | 2 +- 2 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 dev-requirements.txt diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 0000000..6989dc3 --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,102 @@ +accelerate==0.31.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +alignn==2024.4.20 +annotated-types==0.7.0 +ase==3.23.0 +async-timeout==4.0.3 +attrs==23.2.0 +autopep8==2.3.1 +bitsandbytes==0.43.1 +black==24.4.2 +certifi==2024.6.2 +cffi +chardet==3.0.4 +charset-normalizer==3.3.2 +click==8.1.7 +contourpy==1.2.1 +cycler==0.12.1 +datasets==2.20.0 +dgl==1.1.1 +dill==0.3.8 +docstring_parser==0.16 +eval_type_backport==0.2.0 +filelock +flake8==7.1.0 +fonttools==4.53.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gmpy2 +huggingface-hub==0.23.4 +idna==3.7 +importlib_resources==6.4.0 +jarvis-tools==2024.4.30 +Jinja2 +joblib==1.4.2 +kiwisolver==1.4.5 +lmdb==1.4.1 +markdown-it-py==3.0.0 +MarkupSafe +matplotlib==3.9.0 +mccabe==0.7.0 +mdurl==0.1.2 +mpmath +multidict==4.7.6 +multiprocess==0.70.16 +mypy-extensions==1.0.0 +networkx +numpy==1.26.4 +packaging==24.1 +pandas==2.2.2 +pathspec==0.12.1 +peft==0.11.1 +pillow==10.3.0 +platformdirs==4.2.2 +psutil==6.0.0 +pyarrow==16.1.0 +pyarrow-hotfix==0.6 +pycodestyle==2.12.0 +pycparser +pydantic==2.7.4 +pydantic-settings==2.3.3 +pydantic_core==2.18.4 +pydocstyle==6.3.0 +pyflakes==3.2.0 +Pygments==2.18.0 +pyparsing==2.4.7 +python-dateutil==2.9.0.post0 +python-dotenv==1.0.1 +pytz==2024.1 +PyYAML +regex==2024.5.15 +requests==2.32.3 +rich==13.7.1 +safetensors==0.4.3 +scikit-learn==1.5.0 +scipy==1.13.1 +sentencepiece==0.2.0 +shtab==1.7.1 +six==1.16.0 +snowballstemmer==2.2.0 +spglib==2.4.0 +sympy +threadpoolctl==3.5.0 +tokenizers==0.19.1 +tomli==2.0.1 +toolz==0.12.1 +torch==2.2.2 +torchdata==0.7.1 +tqdm==4.66.4 +transformers==4.41.2 +triton==2.2.0 +trl==0.8.6 +typing_extensions +tyro==0.8.4 +tzdata==2024.1 +urllib3==2.2.2 +xformers==0.0.25.post1 +xmltodict==0.13.0 +xxhash==3.4.1 +yarl==1.9.4 +zipp==3.19.2 + diff --git a/setup.py b/setup.py index aef9d0e..1d5a8f4 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ "sentencepiece" ], - scripts=["atomgpt/train_prop.py"], + # scripts=["atomgpt/train_prop.py"], long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/usnistgov/atomgpt", From 38f3bdab3336ac6dc49888809f07bfe448c39b4a Mon Sep 17 00:00:00 2001 From: Kamal Choudhary Date: Mon, 1 Jul 2024 05:19:51 -0400 Subject: [PATCH 3/5] Add files via upload --- atomgpt/data/schematic.jpeg | Bin 0 -> 219540 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 atomgpt/data/schematic.jpeg diff --git a/atomgpt/data/schematic.jpeg b/atomgpt/data/schematic.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..cadf99c70023493558289e3022776fe80c412c54 GIT binary patch literal 219540 zcmb4r2V7H2^XNeYEEEyxO+`dH0qIReKtV-7K%_(nMF~ZE3C*j5bd_ELDgsIs0)!4y zLJu{Bj`Yx#gkF51-A})J-~a#K!tdhx(U>a%C4sA;KBpFYEIhL-Le zJv}`o4I>l7Ii|Dc=+BXm66ueVog^bWd5-Ed)w%!g@~ax4J#lQB>>4QvEpUvMgp`)# zS1lk(RFJ6Fo~}oQoPzAcG1B8CCyBs4%x@@S_c$3j#fe}203|6AdxrE3k!wf$i1mM~ z+3fusAmQgtt|0D}(Uev>Y_VQglA|9Q8ruQZRj-u9jIIh&=Lk8Me_44$Q4~~YX=xLvN z@48EP<3Rr2simC0T^`9H>?Tc&)}Spa)w646IDSQvYo_ulLPqY#!Qe+%>evqE*$wLA z?gIdYGwiIlUd}Y@LHD@&2L$s_*+GXyWEy&lfIH$OZXG?m2aD`-QU+nnAaaE>(r763 zp`VrYuBRvKk^xBLw9({Xgcs}dKq)W?qzR4Im zl9RSw^P4JrN>M5-ou(=O29RVpIZ1ddoIvvkL_R*{k`*5~^cWzOZX>wgy;rrbJq2Y3 z$F$OG0J{n0pp;P90zN}um77()K!I+Jjpse4xx-~{?{saS6gYD%SlBV2?lGV!6!ScTy=&5r_m;M zx7=I6+to9AjsYCDs4|1|+|+wMJd*7BUnY}MR8Hr@67B;Z(+4`eY*Ux!mkT=KeR3ae z=pY}Xlc03^IteY8@#$;G`+%4X#|z=Kfdi{%qM$;<>wr~Zxn5*f=Z@MZ(YB|!<_nRU z53HfwIH{<}cfI_XfFnwfn#1>igY@QUYb>4ZU02CkfTk@cW=9dT{0o4cN2=tn4LcH| z=X@@0dWz4XJp`qHBrfXVlVC;gwzs%6{yA9lrWn4UzGM1l%>z&WtR9~TFJu0c;^JQX zgEYmS=H13+qmfOT=e1aQwPoY<#PyM0TU9PY4;VPr2e!`kZnNSH6??hH56)|A^U#IL z(jV`#vYs92ba0hQ2<16X$?*$FtXs6dMr!zF=#2&aywGdZ;IN6Y%G_nAWoZz5U9@-{ zGNr_qB(|Nv?4{7V&q{Fzq3^O*MQZRQC;uls`~C9l1Zt)QDJDu&bl!u7`<1qNKMEgH zyVZTDIl9N6khs1*tEDxy>q|o2-P<{mZ!zAj*i%@Z7khh zS~F%PG|wxZC%gzHn$Xrd^4hEBvFC295%}KfmQS6_THgk32Sc*k5=3ufL@950pcc%& z%X6MuKx8t(_cseknD{QX%PT8b-xq=H!WfI|4sn!d|M$%^t5y458S}ViWW^zd?PrjZ z|NFyET0p>{`-yM10Lc$d&b zB-1D_<|+Unw^V93Cu}|5=VBLaH6IuOoLF}5ItHk<=UoFR5(oewJ0tpd6xcQwn_jWSn*jF2dI_eC~jZdV>sV!i%MiPaDH zFp&R>LJ|Zv99TKIye!d|m+0T&n9XrjoU>s=AaZSqCx3M?@3&EiP4Tj~EK$a00>dJY z*S1%M9Rq>w=FY{1#d?XC8IEnQkQNn6pehw-0SO!ZxyBSj3yEhUy@}KF%T>&9n(NVB z3%>i}oSMa>?L2|2boWAeko@oZHlBT+dle$B3<+L!ysLBCuWxV_jmf+A$3EI)B(waI z@ewS#VV8s)%)Xj0Gg}v(ljh(de0#E{woZAftdQ)A*8nR!PM(@GwrL>g>7q_v6c!o1y5>DnOv&AwuPUv#+>l+k7pO!B_~ z`JlluU-5o;zi3v)WeBKt%$NZ}o989tWz#rJ|HTfQ6GhIas4` zpMj)HpvS2E2ZjC^q`xg;3}PSh>|=ROFS5!W(vGgwHmWz9SZ*Lq1U-31+?f9<3bY{} z&dvT+{a|I>PG9q7HpJWD?8@f0sO=j1l$CfHAI*M^_ihH`?awNbs44NLTp0+>pR&4~ z+Tjp_lXo7l{tP88wkG`kKN9y^v?zI@Tg@VJX;=HfBtZ7X|=l*2hc)p3{bMcQ*XtrT2(w{7=Nz%T41HSR%*e zow-JLCdMnLCG}=FWt;nG{Lk}XKIqRXRc0uI%>r3^FOnZ%XuX9sEuvr|}RG4+U!w@z9QLZgj3AEH~Unu=ToVf1&N9+E@BA2-C_dppeE7<#C zmHd8t51~Pv5TON(`!isG<+mR%EW;<>%8L83svwVV#bODhf+K!n~n9oILYa z>`pGZRQhM#C5T^HR4i=_BhBstVI;|pvAqKJ$6W;g6n?v&u3sj}Xmm&H9rZIZ2zCQK z&`rlcm-2Y>2Myd*+Q}cjy39<7vDtFNTSTRUtt33eg8pi?5W76-@e9x~c)V}rvGTda z@$3XwrnK2tucRR&KXPB!NWWPT8-?hK&kcEHWb~>^CwvD5FB-E6W?0UPd0_5Id-(om zg09j{C8^^!eA*E%R@J-DClysHa+ZD4ssp~e*zFdkFc zPE2O*%jJB=CPnTT3pdRy&X<`pH?O_}=WXY>?!NdgXOE6|w^P^hge*vuKYE~NmDBkP zDA{JYo(ry~Ps`01<{E_AC0lvr1iu-%^&d_4TXhS|pmCqaQf!r&7qE} zebW8Q33U0U{rB?exZ*{c@*moFTkYPj>uj##c8Eh;m=Nh-w4zVX{w9^EOZ2gh691g0$ zlCkJHg;}>#`HvQREp3IMNZ!ug#AW2QKM;F4N?2?g7w-7O6gN3`>*Yba_R5bix!U=h z6=xn8tCX~Q=?O+fJ3MSk;l-03V^`H)ccCrOYUzkzwr7@Xv73KLJL*8OS)=p9?-q~x z2Q|H&Bb>LRF9J9JCmcATd=59=w(Iz{A_Hq3ZC83GJ+ugdEkI-sLz}z2En91lNVh3> zYDRLU-gynl!tLG`i;_}LWOV3el6LM6deW|W%FG(RHcViaQeriFlrdSBssV+1?4%V= z{4;iuMXWmW+!JjdW&SP(x# zV5XEAXR&MB7^FSX+uIwu_C93UZ9slOJ{v-1jjwPNHYe0aF5g!ealog>*K~MDi}#rG z*j8@*0u*C<`-hY47m7vy9Os}%a%hlL=P5{XJIj5iAlTgapTlB422 zB3@c~eUsZCSr8iePKDW1;RWt*vA;D)GKag`=G%h)%axER;iTW~dbd<2<^M*!-we|; zqNyLV_J@|^n)un8E=|9wzrh&|25n@^#;!E0f$x=9Jpb;Z-tkO6ttN=mK1L!D!&|-B z5BI*C+%@e5a562UjAH9uz}Nm;@c-}1dDf&$t-ER{YWQ|@(#+%9=oc+_U$iaOcPiM$ z73QXMNcKIhuw0tI=bzBEkX_LZ^EsVYk=ATBAJ#?sfIu*sy{&}Y8`mCVO) zi4o6afD<%v?LpOl=ev}yLh&QRE>*w-g%`2H)AP+6Ti5<%-~PFwhp|>n+I*@z&1odh zMaY@2WZP6(S=?S-j~>Vrl;NZH`B*IOG$XWIiAx(&pj*f*w_V;%{p*L~wZ1k*BT#48 zm!iZ0%+Q$kYp>|B9**5aKjH~XfloAH)eb&qt!hv;lbBYyb8?4dW7WPmt1jgDhEJE{ zE^F^-vh|A@ADJPa_ipEZ*NLKKoV2`g6Ae?!B!WQ(kC+8*hTS6n_6a$1l?qayk2px_ zn?7!>l~)0OyN6%ERRnuqp=O4L+!4`!&BBFGN;Xl@BLrvSs(0w>y#01GCM&P2+iz}$ zkICoQwJ1d94~4})O;f}Tv~DuuhY%g@>(J;p!lSp#mFe|TLs_`WZ}-c>T;W?ZEs~Bn zMa3DExPPmMy#0D*zct({{oZc$jOw5_9lV0`1gfCFb=)gDZ?@2}*KND?AZtBRhe~#f zajF?szr$f)vtksJ07r5`MZz@!u8nJD|InQva zHfExkI_~+M=Z!Ak0$Z0hj~=oY@=_rr?M}DZDLi(mqjVc>x|O zMq5?q@VR1V=^2sn&`DK}ZVV87ZX-Dq!=|G=xMK@LnAn3v=~3ml`8)Yvnsg}1>!Ox8 z4`V7(X-X=IS>e~KL-hKE71K3ME>AXTUb+PN$W$nB zt;B=Jl5Dl|r`KL=Gy|-Jp zfp}cuwaWJbyFET-AKKfA-1hHS8NekmAw9!7zQ&j3a^QaEJsRlC5`vG9R%9f#&&{o6 zj#<-$FZhzPntrx>*I9i7FqznfE!cTSFBfoWM};rV+l0c;-K&GI8#+~oS2TRW7kD_$ zd1DH_?hjDOI58$8*rHtgwcrm2E8RCXv8x3T;-(tdRMoxy#SPG9ukcJ#HPuY{s;J-{ zfXnNyNn~!~FD{CElr&ee=6YI{)^p6j^@e0lkyO9I+i|J)N2?6b_-+CeHQ7=qtc6z7 zM)P@^m}~i~1^x{%-g@=Oe2(}QmxrX^6@34uKWNaz))~#pu}4momszIN%-8-HNNA>M zgI!rt3K=Gc}vPO&5XBEg4h)w}Jm>VcSA~f{WsH>5_g}srNDcy!2N-);kaD8gKHd&wp!{ zrO47z@?0~x@j3yY2E#7-%No+O2}QNAW))Yaxf{^Y4eO;gGM+dZky<*;Z`VgTuIdQz zFEz=)>2iXTJBmo-KA)C;e?EA-?f+ z&=`@4WTI3}u>;Spd;dtnzt)Z+v1p%94RcOa0M(7nwP~SPOnk?z$82vHL7F*BC{fl@ zozpwfM=AH~h}_m?AEj*a1lCXh$nmUKkPP%a5()B57tVnV0wg-;VvEktHRJ7Oypw9o zvJzF2&VHP?)!!}$v*F`Ea!5COlyL&Ug-gBF?3Z57z(JwN&{4jsBQgJ4GqprV&v`j5 z`T~1?Y~4aWH&NXH77F3O+2dK@OvDX+Y1(CeNqCXiJEN$e(f$}X?i!w@_rra+wfRUQ zNqv%0$D-_c;QT8q>nkz{xXudPP7Wd3nM~`nbH7&GVJ0zzDL#$RM?4B@0(6gGYGQwN z*|zcJFn&F2n5SlWNwqu|`}F>b&En;H$LRhg@6!7CNZ-Sn0trs$n(}ym&=o&F@C*Qg?N=a!vqyx~Eoln=Fsx_Q{=TT% z2S43_vBcSorcvOeiJw9U3-&8`S}rRpb$EZVd#0%1EX#Q+wLC!XLlDZEw+qS2w$#S1 zfF7E6%##k)17S8Le*x-9Hy?P3S0+T99VRicl5q=Mpoe>KrZ-NlB(OZ_qEpY;5;UB_ z6W7;FP~<0g9ls%AtZU`2zaWbbOYH`K$i+VKJ<{*6mR4zWWgR$xl6<~A@+X??$%D(? z2gs0fpfRC$ZzZaNR?2)5hpvOG=Z(3$oy2}LO@U(i7SN)f(Ux}g9Gras}o;#J8B_BJ$g&n99@(LQ}npQ4mn`6ul0NL)ZCT$j*yWqT^-?R_{TngT7AR z`g44Mbdg(lZ0pSdR8dBwF8Rg znAkMlMyZ~Sauf+sX%xsHQdW<7Q(As~*Yp+Gha8h+z4R{g1i|t`#Gaf`l}5IVV3n30 zlNOxi`xcAyd%*qTRhzO6b~tF0e_|zig5S%T8~GAUhX4}BIXB|jW66uLe< znoBGETqMl+j-{nt`-k*_;qq+VypHlKk**;nC9Nu}U{HvPt5->Zr*NQ3|+d(zC42o4q2v%V?|J$;k7)f`=VR(rRoRT~HADR`s62jNKPGNF}RdAR14 zKAA`{W-Mb!BsYvsS|YL^H1m&TLr!EqJ(a7TPLX(J~gAVWoP-Omp41XLb`G7*!sO zhV+;rSo#@zbVl;9Pl&7FSlmf5s0OZEJAYv36O)P~iw^)KdJ&C0%e9ML2Qnb{!d&b_Yg68(bFso=BGrmotmk+=R&z?7eiS7dfM#l{qN|>XzFrX< zw}9|{)I_|HKIg@Z;-p)Iyyc0sXX+dDlBaY#J0l(!GLxBJnuS@E3tj_FTWH3Ot>;1b zl4*$tt79Nxez|K>@H8lMW<70y)e7sDWiGmYOo=yUg`OUEbpzN00+oa4*;*HpfyjX2Ow`+{u^h z>FN36c4AU#!Oap99?(g_;4VV(;%v^SYnoJUnzE!Zra;~#%V`PhGJ9+CA)wn{(Fg|s z{wH8FG;t(I?iEU*rYSLZj21T6MnC7+tHR?m*f>|)AaRda^whZ>m>9E;I9w-FI4@1v z_4NF0;z)nrmr^$jFPsMe?+50aAN<__z%fs5^30L00KhV#eR+d8#(ufDW5+p-)UFSn zS*=vJDU=>bVu8a0km^(cs`i#JR$iA7J*r?`?p(&F+6D8q$<`bUgSvC3Al}UMiKn_q z4J*6$Wdyi-iC>hmhAO~XyhP`!gk6xccoTdmzY@MN`K51%c5qzpOzHiCO@mfMV+yoC zTCM1~P_~J4pKTV8ZqJN0UfeHam)rnUGp46glAmRu-Pc7I^5%q*i$Mlw@EmR^sYUp- z^V6nPyE$j^z>ql^r~BfzD!KCErK%92&WlN`>ZNwzXM_E#;GU1U| z`RB@{jB`#B7Z-RV(u*>C0GgtR%Amz-012Ks*gbog>utyS!5ny@#o!f&VeLPw4*DEV z)SHuiS`Iq0F3{C^Ov7GBd2?8?&GvmK+%|OpjP8`T&4Qj@t=M(CA%Qv5JK~kC@~NXE z5Y>w5o1D#Jr!*F03W29~jIritTE0Pfs?_uy6;I!xUVI1@YNU0idD7m(0B!(lp9%rY6 zzgwQBj`u)Uo$|K`f<0RIdP8?=3W(ZlQw+z=9Ly07J%g!dlFkF)gD$~?ifsh47s8GL zz#)PNIq^UcttOpN1stblyu+=BklJSvAA=7TLfi&EA7@}x<2F_Y5sz5xPR|Fl1Ec|8 zHg{O7hty9w`Tp&1LNN5S* zq~Gbuj%Yt7xk!-nTyxwHYXBgUAV@s?qI(8hh*F8~OYi>JOdY!dYEAO5PUuS?>p>K} z0q#k++0-wg_EbB#wKXKfw3I9+5D!Q+^$<@rj1zLK7LoT!YLoc}%5Azy18!oFuz@eC zI!Jee&wNd*g0kXuAYDwG?Pt=u;GX-vg_%8gh`6VO{OdlkG=voCs2Zlgc_1A)hk@w{ z#=X~xoF^`;@0CYL;S49@X9G*J&KE=qv;68xE%MJYdfe-79@*IbY0K*8q& zrhWjR(jETzav-%j)obdQ+mC75#xchSr9YFr85dJ}cFTQ9;@0HA_e6*TALbJ@Ev-Te zu`=AEIh?^iEXYo~j@!Nc61T&4nS08@l5ctAmAo;Uk*+W;s|Vcy{%|?cCuY-V=ws;( zWXLX?C3I=JD{{ULV$m`3;d99jr-l6tgd(8DTo(fR$T#qH`p)tLl)YK9_}JN5lwGM4 zVGEo>^Z|FU1uH=uPGhENg*-}Ms-@Tdp`{rlC!!vmpQ-N=43Q-;{X31k{QTym*>dazx zN~Z}i+9+LV;8r;Wr~J_59vjhr(tIeU zIXAX0#`=;&2Y-4=RS_9zy^&$~fzMad488=`rY>i6CFr(S(lFcEVtEX$hJ34M6rkv) ztC0q}*0|@W0ozs|39&aVF5Wv-K4}i5tfhC|(j0vxHt4xw9F%I~2;EP9ZJmOwp!JqT zXC*E(4u&CyIc11hs?MZlP0F0<1j0a{ceoKWVG*5SfkhaPARa;aRvh44H#}Angac5` ziLJIh^6s_A81;UZdPx-1M@aC;7|ocz&9E9q7_0Jaz^$jmXoSS_)^NT=-GQwD_4SGg zq7gaL{nhp!k7=mw2ZA@R?V*k-tRsx?Ar777 zUK1+Fy1EkA)z}FDCLo!!@Y2qq+vnczVTc>bR6+~0U@T!}xi`-zZVG06$aOdP64j#0 z;QUv}T*;7rLi#@8Kll7qSXNtzKGmy^{rg(V87F%cWbgnNGO z!ko!Vv`c3pk8Qg$p}nmZbho>YYe}e08j0&-!p)gms-O$9@nkMBVJgnu!o^vb{Q>@Q zAz3>c{nLvWHMoo&3to;9s#x?f?FjLOEa40)sW#ri%-)YWhT~nKTGEIsCA?<*bLoQ> z#@n+Qz`yFvB^EY?wbZa72&b*`6UHUj#e9cOIy+=%zj}TLpW7;*Yq%ThDDewO%KL{g zA~A`HP$#5@WprBH4&NB#IEY`X)#tKURp^+@)tOIL{pcx%2Q$8?VGBEv77KZSR&(7p z@cVz*vk&;E_gRP&@DR*g<%_|{EN_0vN|&HhwW4u$U{W*JiBzeho#QGT9BK#NdQdCC zN}CRZGI1)l&yFuVKa+QLs!qqa_M(K!dUs(+e|k{~!)iHvexn!H%*{4F8S=WTWCfG| z4;Msr8;c`;TG;cZlr~_qqjgHU@kVTh6;!4K6m!Yu>$FBR^UCy@-aZR-7&Z4_wP}d- zr+g(Ma~2&4!+$mEAAW)O+F?oYf`kJwBhnS>eHJ*$_nZGtT|%U`H9+32efGiB!3W6pLcdbi)V1Z&P+8TGm|;NA5K|S(t*vO>Jgc z2?D?+;}>Cxi_5sHnCd+^`yiw{^SG|mu!J$M8 zbx>+Xf3Y>@=eR=4c)(pufn8q~Ywb!le5NdYSj%YeH-ZR2lw5o zfA<&U&sPd0!=nqdto7*f6b3C5&u-#R7D@#`-q$oU-Ya<9H{KZe{6W?!0aEcrw~Bx_xEEuvr_uDIfqB@kBiu;6D4{%4Dke^<-vmIn@sdG6N!FWA~*Dg8D%$UU(QrTREk43 zAL0eR3^BMeT#pcqbb=dbKgDE3*e!T%Z|ia|wsS&58Rn_ksq z!hrxvTlo4CZx@7%Uf=-|`&3gV$@FYGzD8L3Ey{sb&s9Z(a$-EeGq?Lu#6t>&>_VRo z{C;GmRGFrus;m+L=)GVJ0=Sjw18M(yWo9=ngu!R__&7y=&RYL?=?jo-;8)w=OF5{W zT~%XZzL(bklUS)bVCSTCf$ti-)DT^B4neLQwx;0fte{SO|B9r~Q`E@j-u(AHI*NA- z0>&epAOCSUbs*&My6|d)-8fYVoO;+lhfJOrQgF=tg9vaJ2%(!<-y)rqz4j;Kf9vrc zeQfauAsTzYT>t+gH@$Bon+r0Yk(|E?7f!X4SFDTjd@NZH}MjKjkplXJ#b26hi_1pEGn zVF~WQffO$3M-^^yP&c|Q{y0QYe^D!)Tih@|^B>ZGX;Hhi`Zx8gV(uT{STOTNto-A! z@9ry${`RFl)>Xgph`K;;!{nZ8Y*HJ2o8I)@$KbAzno!XSaY0`1PPK@6S6yY)%4+IE z+*!%!Wwm_Lm%zbcExG~unxr8UonyG`{?+oz9eX~sGW`SLj5LVG`QN1f*1S~mHrjFJ z5b#l6m1S*t=Z&0^04M26_Ur%ppyp zG0nWe-90+~XbJN8?{wKk2|^wVL*+)5ZSt+c4EAC#KMzR`>CuVOgJ7k}xkJzw z9Kv2m?XY$ZlMcn18=u5Kah{iGF0D{+(VR4}zHIO+5^VX%4m+kQ#&mr2QZ3?b zL^SiP?5*JBIh1D~3e4Q!>h*@WDC)ywF7;RKQpbmlEf4M>lH1WHRjJJhf9;t7{21*t z%OmE~fV;@b1h5VfX_#!OADxxW15BQ^ngL7+qJ024DRNJy%uq0jO2Yzn{N9%(NITq>m~IZKo739OHaBvZ~xRUhuRPyZ@Xjh zd;WgET$d>=g1u<;%j=+0!R9!vX09UsLh-*gZBO*jU|=wR=dW^ANAu0JL}YA>rN{{t zT%L>Z&j9@*;b`(wsN*?s3R@zJ87 z`>u(s7Q>!_;+wNBlbmg$PlAino4ZeL2F{=<`4H<}r z%8C&mlf?3$?C=a#-d~xk89y6fDH!!tlPaOYy9c_i`WfD)(Bs?`^U$$nSU<#M40TDg zq1AlG5?Xi}06dcUc|zVF7;^Ng(DTvG{y_X@ufX_C2?A0{&o(EoNN%(!Za`@n?~_#W zs8pwAlp+BnUH8^-+yO-jD&|MNubXE0 zmpEyRo%>3cLo8Vwm;iYz|cw_wV0WNc?I1^6-}MYm!keNXyNi*LKlq=~c5u z&LQ=Mu>nn2cOoJ9AhlAivmw(u?YD1mN)6EWv7vdltl&4i=eZneL#4l|(+;8D*-LJzujteoQ5ACpRD7_s$fQ;Qnv;kfBvg$wpvw^soSK_TBX~*d@O8twj zct9vaFKe~BGZ98D_4&|6&65f431|oig_*x|MEZ}KB3(4kt^W|Lr4aqr)jhk2l=(w3 zF(9w+dm5@UTb9Ikey5Dlwcnku?NPcmm91&V_n94X0f*b+vj=7)>|~Bp^kh zfuJHbrwo}H86f?zt7X{zZ}R_c{5}cf$SrE>2DjZG2SmpL3EQx?mR0G;e=?CApIsbX z+=G!T(oi!Wo-_VQv)|+R+|)S!#lQL6_?BUe>hz-&qrIu!318&&@T? zlLXfAN7(We(yl+kNBnh7=4{)eXZiSB=8ZVszv7YnR#f^?&#e=NAd*lXt(LL<86gR)=3&#| zc>}Dd=xVTDJ5cSEdgrq3i1?8M*BTBBciKRIJwX0_-47w;8PMw7m(Up1$JuZ%$~)$X z=JXF-YitKj>36q!EXZ-QT> zf0DsG#79swwcLIc_*AaRF)2mT&`iswb=9i;OtX33S{t5K1)kLlzy8470nHn@Smym| z(P*&&8X}T6EvWagSi-T>*0?oMPFyB%p0`OkVL-yti1(!j?>PipsN9`vCIKz@i)b%1 zSL%<~Be$GZz{aJ4eTKTX{s_M;s)QWw60p>Y_vgc%O?7Y0+s<6h6(*bU>~Ev@8wWdH z-0HP9MkPF*A?w|&m^)gMiC`%klxrSq;q(i5RO;OaEf8t3lw3h%UtlYAJQMu%RC3-L znuqw>I7&%i*HI|SIq@}EQr*@?pqt8S+l+y!d@<6<(7dzl`gHZzmk!M6tp$Tr84Q6n z1!3_;6I>owL0dtiva(SIiGwjA!P(@GOp5;A1O=E-A6t zK$BZK94ZFVt&?G*{t<1{pfX6!%WD1Jx#`cFn2Ef>yYaxm)^wV@ZPPL2yKmMN8iU%( z?P%)*V@Z@f62-AoCA;Z+3BrHMvN{7hL+jI+Lr(+6fC@5OD)j$Du-D>uGi9VSTy5r$ zR5Y~AiwM0&e1ts{%3oyKmhkD_smq^-d-iW`;;z#l){_VyOg71PanL+~{VD0FcPynZ zRo2X|Rq*-#y?1Jp>4g%lwdr)8mb*pq%V!VO#NzsC`{JjFL2gbcgXkPgzP3zI+~c6* zeM9@Y;@wlAb~LJi0P?yWLJ&}JFdTwj`BW6Rq~Pw(_i(>0;!#mUMJdk_%>86i}A zTir+KG7VFUr5rI~c^Hn))T9w&nm0w|Jy^Nl+nkv)_p(zfmG4!&Sp}IZ5}N1Y)tJEYV=_ZgBczPV|Xf2;9dPx4*NCi9-|Q=Se3c)N>eg(sNfd2 zLZYXX7N*`?B8ymSOabYr3*mG3-p& zO~56zg;s00g|nn_S)rpJO1@y=C#Q{PUNu|) z?5%(E143Jrs)g>d)-b%{FIU#IMft zf1Y4+<5uv5D^M~ZTvg7%3_fnXP zl{XGeKa%r%ojz9wms|T!AS3Rwmrs{v>^?U9eON%I7v!QIU3Mj8N58ed>X4aC8uG+# zNJJb~+E}zQbqGAhZo%z2{E}mKsa5dxeqISp5$P>W1?WTE0_qtt#FdnaTf5`JvKOsR z>3&`h8tx;ZB8Kb+faF>iYegSy`L*g<>*=mH$7m?;5hKFiWp7Vh-C(mzt!=$_2qe=9 za?v2F)w-jfMO2GCLeqP8L?ubvwWyyzJr$ZL7}M>@A7Rso(u?s0QZsEhSyRcsy1GkJ zmsK4~xM^J*9bb8>gmEgWc0Yj-*E8nOy>O<^oYJVUxyE1rp5AGP#z#@=W2xs{i}E-n@%a%V;+=loIJ3aC zsZe6XEJh~bgMFwn{O17jf%qJ@$;5p~AsEzh-6Z5&Z(d}!f}H7ip+{+s@%f4@xU`|7 zimT9}h5*q!-49VvWu3_tyMG8H(J>tgpIka6=A#RB`3jJHJ~tGEvU;M{o#1M^DIS#p zkwV{GLQ8z?_XSQ_p*nt+Jcd%ZhCiEYIpjaC(sJ9q^hd@A@2A%L4?cf|rb`(@0s^Xw z{CGsscEZe2&CRa@q?Nc{C)mGf%6)0s8Bw91^MTd0@d16k?W#b}qYJ!I?risJ_#=f4 z&GXQ%(#yGkANScd6TT=f3m@rJ{+AV6-^OVlj@w8Gynp&qO)6Sl)ZLQSkCQ2%=gKOG zAyg>dJe37A=%NlvSbvzLcK=~LL@4ISyxHInES{l~U8WY91~OT9k(WlT(R(;*vb!e- z!hHdqhe)#xSat9SCT=Jb0Pc(TJ-lHU>Sw#f|7BFRq(N6CvQ$bV>0`K}gjRb45NMGO>m&K*Bp>>`1%BzL@jx%H!X7{m;3|Fn|`Kh)}mnXl~aMN+kozFfiIQ7$DOWivmKs7o@IX`p+%fEw%z8#f#VDg0_F$a;EPh#$hFG1iFW%LiQpa;`}(Nqb@tNIZ7PI=MZY_T&#Q2FFZ%d72TZ{ zzW!BjFAE}ONPq7KM;U$P0WNdreDmvAi!K9QQs(#2MCo{y`*}RdZLEaHHtu$6oXguu8s;&5JAJcRkaaD;gkb4yDoz zFGuqkx`Eey0m72r^iIM@PS4xnsdfoWch_M?drF{j3M|kmt|A!hWa4TdIMQGEEfLyl znvo3}ge0F{o*|e(^x=Y-90wO`j0gX^IyXG9#N+3k%bgLScb|P^mYZ5EeB_^F*rdN8 zt{b>IU^?`n!ULh~@g;{UyEm~FY*DO9mEH>{y90f#ij8<7jg>KR9^gwXni7k@>@Y9a zVlPp+B`#@PuJMu8CRa{~*)XqV{9#{EX3r^p3?KghJP4wRvJ|ZB+l!^Y&Gh?&t;frx z&m3yE1#Qax!{?w7Ss4A!8{@*48G^?Nug66N#-$TH0;fb9&U~Kg%ah(7Kl$>aXkH5I za8sQ)K`;U9>n{VK|Y=*0ow=gFsCVfN2NsY@V`wJ7& zB_{Lch1Jc~G0fHo#!z{8v?cF09$dq5N9t{+>2RLrtudI~+ztq?A$4L5Dpz9?(WTm_XZyDDme0I~#40A5oWy|9IjUOMHcu~Ssy>;Z3(P_N8DSg*q z)w~&0{!Fb&o}R{C=h-#Hk->O6tF*4%%>)gVJ1u$(|aP%AKFC*KD{a%U&Kq*>bW?a zs&xcDx1E9*41LX#HaKb2z2}Mc3YoO0gklgT#RW%<6kPG8B0F>B$#XSxjDq1GtX^pt z+RmA`JB>c5mXr^Fr6SS*AsDzU+`;A~Q)&{n{Bw9)@VQ70P3JlMXhA%~)1CsDq~}cB zIJuvsk5#KWHV17(zY0}8gNK|OvM`r4vb;;6DXOZ0LcvRoaaDhA97msAv5N~aY(||zX zwLJ?iJ?hQ0njDD3Lcs=bP>a{$=M^Ml=!y!;$j1S)Bn+)z{YxUsUq4MkKqgB5!d6+%Z z*q#lX0!T@v#<+bCtdf4WnbrbO=gA(W#mws(m6V3Lnl{>QGs7aX@gn70-Xe!H@bA@0 zl~iue9T{!gfDmoC_TzsiI)HkLyC#1@Y%f$)QIb!!sCo=1eE*-wrN>3RZrvt0yF-p8 z^rtFHx&DujgQ#XkR21rwG(aOB^|&y*cEmES(f9u@-t~sG8nn%-Ya#W$BJmf7t`oDU zV=Fc7eS&AxUOkxGl+;?6~BI8=Mwh~rN<_h zCA3r^HhSf=hrDVaa z7tY|chuyhqS!Lyaq3B=4ZrpUI=NPAS4sB)=(b|~fh$j;)>J!vSYh*!!1D%7Vjefjd z_qnYIW{wS7%i`I%Q&q3Os%*ezbF6(sUcI@i(>cVLh%d4ckFN^_oP%^b>d}}bjY$LU1U2*TYV&u)G zm=J~FW1F*8p3ZlW-M2OBJDeSj=VWFJhOc1aUP@eef3X^2k7+P6an`qE ziGTTBVI*%6L}nf=o$I?xYp|rJKjfAe=|gY}U+2!cv7Gx_`DNdkU0`yXnX1+iJovoGxpd8s#KfF2Od$Ye6Gs-gCX1yd#7*=)146!>|gK3sA zv7v~}YYpS!*shpQsv;}35h3toF_s+Gw9IX*4mz+trq;d%RVFXb@{<6%(a{en5sj%! zt>jbv)Z>i~Po20_lNu{pJ;2eKU2#r8USj6eR~(pOEHmA!jrgJ%$4I}%IlWB!=&xLT z#EF06j})XdH+?w;Z7p+rz#kQPX~2SQ_Thu9m`w?zz&Hu*lc(S-R_vd8!Y!02zgMS5 zI^#L)8SE|=r5E3sqBU)P5|{B7DiXv1z8Ps4D(WQoYGs0#lJ1IYr~=fKcO`m_cip3J zLT2c!nvTGwim~)8dllF5T$OkMq}gbcO12X1)AIC>b?!kQboD!_8@u>pS$Zz!Vb;1bvpw|GSz%fUXUNv# zxxS6fwj})@=H5Cej_7F{#X`^ocXxLPkl;>mUEFnXcMa|i!QGaH#hu^|3AQ-F1B-jG zTz>C&<$J%X`^UXix9V1J)$Gh!sWUUD`|0kdd$NXt*4gWH#;J`d)!k54qii!U^!p1& zETjsDi3$I~RW7z@u~&%Stc#@2uDpO)aqY@~?#41}paEA!S8my%aoW?GC%?j(n=CBs zyp3lQYc#E;UkZjXJQ7om1;WUoZc(6b3k~-9wA3Ty{MK(AYo~6}aBPcMVYw0ukp0pJ zZ|tHHKEc`u5jraKNG5xw6jkcC8>-2aMb{!}e;8E{un~Mp?A4iM)??}nCdg=RiRpG? zTnxBv`XkoGd8T>snw3_(#pIPax@6Z8Y}=uUTqAxTE3P=7AaVCK^AmAWd|+UYchl_( zFfR~`7MC?~CD}SzP<;>&NemOeEcApSp0MIo$UE|rf7!1L0dXZ2W-}o_?z(VzBBJOd z%V5>ZI67`}TJ2FQPmBP(az;|G&fTexgYW3W>z4eE(UV3rv5&;Gu8bT=uu2*{-4N}j zbcyS7OF^S1&egn#xV$2?x2#X;&mwvSDHpJEYqJvupVdc@0u!a4Sp z*k}folG8&^L7^o^DxIRxLeX3^+n3^KI{~eKa4QX}zp@nu`*+D@>OVh;%YjyqEfi_U z&wZ!C`9gm7B?uEapF5Sw(pXSAmn{xC3PU2hXV#w)_?Ve%Ik4rvn#%IdJ$PK_iQv)*`Lmh_%nZ_EvN;4G-tI=&((pTi#td7FCG+# zYZg@Ux-a(8fYedj%~#LPafzP8NoD_+@$lTFrE>*QI)|AzW#BBy5V{dtDH~=6&XD8M z_};l@7B$OU(|`~7SZ=E+#|l@SyXAysMsa!Xpd`I<@op=1WN;5pkM-Vl9Z4h1sIB=! zJA;(>jbViGSv3b^SwvGMawM~jWqRLhN0I0{BNl_26 zVy~KuPB~eMxq8WK0NL}?>g?AGD-NIJgXXE)UJj2{1I zdEQ$4$?4qzk#A=-8Gr6dt068_miZ`z3tnOV>iRG?qTix3G;k2xQNPELbP^yG)DvJh zNJ34Y=wZ5Ff;$#}>Td>@e%r)p=u^6CQICzsynK+=K!!fKJ0-XGLn^!XE9IvfJNO(J zEv_gX?3F}p&cdMaE@-X*5489mRPrH_Xz9T*EUcg)*T+-ehEY$imf$;Gq^X6GI&BFD zcr$iZ)iDIhMb$<6m{Cch?C2wM+7;6G^~Nbn1l$)y30SBB=8Q%(3^o;I$ckvP4A#P< z3U@;o(jG@A5#OYxF{Bkof@mp10QLJsl>cRBYv;l3BVzL;*{~{j%5T!PA*u&N?4n*} zofw(jVoJ4h5j~&vssQm*KZaC^{=r?7P=_a`k(*z;(OOE@WU+vvK_Q0z!DRptK?bX( z6ZXZ9S-#~|2m%2XEjjI?sg!tF5Dvkmn^dcxHnQ8*lca9XyLQH6wK4a=T#@)&6Fbk+ z>dfv_lgw$I-dr)Z8_r|7BUW3(8%8!u_TsdkWQ~RT!x@WOkjgj8;*T)mW6?N5Tpe9Z zOQgri|0p$vel1U)(8R~K1y)IO*9B`d5wq5`2Kh+ou6g}cCyZ1*jHDy)#;9>txh4%K z<_&hys_~BZcWGm2w(V2{eo3V~Mu4J4%PUo7KsEY@9#76c{ZxsaDz5Hg_BcK8{qcCJ zfE%HPjFhYaOHeByXztX;*@#GS66BATQE<>+I1s3mDA#iTeszQtI63^k^6 zNcn!(cr(*{{*(m>5yoz9Ro0=9|DzaOxU_G-L!vagmyxQ6(~)Q;8jt#MP)yV@-6_5Y zrzxTadU{F7we4s%9;*A(8$_0GUI~ZB+lGC8aTA^_P1Kq7^E6sx)I+zj-46bv?6@_l zT-Wv&h;1%07gp#pL?g13y`XrwFQC$BuC9=eGO1m;4V5udtZVPZ;AwOhW0M>Y`jtaG zx{v9AFYNuX{F<%(C||1@ zX*u45SkRlL))HMd*94Z1fAm~quS}^oQxmwHu5-tzd!pA9A~YG8`Y3*hByCpb&!7Jf zZYiZ6huuBGc~EYF*C$#mcc`#=)SkF$I09oy*72Ub%hb ze3E30|B6p?N$kDEs;=8Tid8>kKr$eIrwv?Zz|QH2Yl^5}P~@;kvwuGn#de44g1Cmj zJ-cm|#FAub-}ky`1CUSRpP0R_=*h{eq&#`%ex}wh?;CUT7j;moTb3SuWI+4Ed>-&c z8GHZuW_0xYuTa`JFSuHFKi{w#om^W;d2hCtN{K%O_QUhYw2a|@+00EY==@IK zR+xY*V&A`JWI5okni^Upe~q)qfa1}b|6W zEK@?EMK#Z9C?QOAW<>}MEN@ckm?TfJy>rKKGd>lQvf>5(h@0%nU}$BUvbBWa_@&MEVn-8l~sIn zMo3L3g*w2{?qFa{trN!VxowECHKIEfFBJu@F$c~m)Je@tMM@)ax+{FT)V|^-8$7Lx zlTqADANr;~%1Ob)lD16L5;5L_JY%SjJfq9_<=Z>sk{Ymt#Yt^sy;5{WyLBeN)~A8J zPFZ5aUSt4zTfFj6tZG%w471K$OY^cMVzasMxF4R#*Ym#~so=@rU;49nw+VV(>zf}3=e-F8zWU6J7*{;5S*4t5F@+_NCvPvNKEQBrh;F4H-S8B846D|ShC*g{jfIcP zx#&Ifjg{CcjVTHeeI8Lvcg!t7R<1wV!{u+jpRJwFbB-q|Q|gkzJt?4X%X97?wWzTp z&(Cw6e@y;;17iJnSBZ!!u8RVfnDW9RWng2(_&89br#2kqysvd(@evTAQTJKCL` z1Vzf6=l>;O);WAFK=~w1Vc`_Eu%fM(extG*R0amrl5)XG1|GzWgMQJIe9T=bWkoC+_bD5%~Swc5@|>ZeG8H_ zSl6j*4<#`+DN_jGvE3(NaPdiiZ0MvtZf4KnUjm(krds?fZEgZ`QvWnWdS&RP*5njA z{DV{Mux2UB5hgT(+~q}pbRdOl6^>V6$rL=(RV(@1$7cnaT39hM@?3- zc!e8FN;bMv7!O)ay{UKkqFI-`?DLF zf>%cKUa+nj7s2{>PT zu1aXmDvn#UQCNSL8Mu*{vIiMQoY0Ra`3J{!OQaQi;}@v=(-&tb#>1|2xY+%DQLsK5 zD87{Pb?YV|OmSEW5U@C)Ya}H`c~n3Pdwqk%X~Hjs@|4Y$5odPcOGdRpgbfg^d@TdC!MF+`CRQQF;PX|QN zC_5GghyrdMHnaivhigKv`#8HC8I3GWBD7Umb^q}azsJB{0F=1t=Iv4SHv!>-_QYcn z#~se#lB2dqT_ew1r+thZs+c33Mxk|0UjC~OHR_$d@a%+O492QQ-gk>N!Mn*N_ZY*4;z&Zt7U}Npi0Io&kPCe5JPM0Pmeil zQOWk|uU$C(I`U@fZROe$rSG`Xw2Ycyhw zft|4Yf@;CO)vnb}hB6=3f1H&Q6~QIfe&*`R{!BIzi3G&&3(ej zt%QwnOMI<$x^eDdEdj))35!0H6?};(ZAFmgUKl5k9CO0-{VsCuFcNuHPPXCqL3?1K zek0}kn|UxPdM9c-9$w(AfA|yxS{I*HaNChCDg~(mSZxvhr~qaIO|BP<8Z!ySZSiv9 z)fXlZ2!1+vBH~60+5Mf8`r1%2D{e}bUj}N;6}Kc}Pb^Y%8D$p}inb+UODs`SbHGpB zWglRPNCnu^w4!(VLmODf6!cmzMH zY6U8>7a*Aa#XM1bCure8yr0@d#`}Iu=bHnXSLVTsM}cgp@RJ>g zvAPKK`pFTWAQPeCxGPIM0bephgpfe>`m6X48NRo%!fhTAx3wOlR8k9=cYW zAzechGGu4!Rh6|Jm+FE_I3VC%_xo={A**K+5nEj6sF5{df?a4ran8|NeMRkSe^5lx zF4vM{enG81Gx_KVsSFZpb^+uF7z*Eh=wMZ0VwWXJqZ`{?v7ipve@6BnI|i5XE|wPc`10f!V{U?6Ghd9OpmUna21!9F&NaRnJ^?2i-#4F?58*CUF@Y7e{btByd0+Nm?1c|>^V$6K z8j!kQKz0YjRwpKN`~4~ui(c*aec&swmd7;|%^~C=D2=_ii;bIG4yneb4}9f(GHHzS z(6j*IV#%Lx{fOPY9qY|s;K&$n#38kk za*_Z2{7+JgjzFb_fv2x_U-Pv8_Zi&(4@TNXkK5#StGGG84IQJQZ;N=y zAYNuC1bCX3M(=zqHqUcX7|%S~6!G&+?64*gNBJP9P*9Cpm{hT?`9fWbK``C-W*#CoD8}g^PeATM# z?A{S`O8*O5GZka!N9R7JNi3I0&GVlTWmW)`f0lCR@qePb3b_3TXWFN6!2gCVzM6|! zzziQa-%NgBK-)#v_lv>{8NVRm;YEWJ{qV%n1o7U8aWQ`Y5DBCl_5dRYfh07J$X>^X z2NdXbdZyI0Y->nzcP@s>qBVTGSy*tD5ztYYwhB(vyzPRV`Mj?#$eW=bKcSstf!oHLPr~r{agS==2zB9R`>j_Zd`nwHeDD)Uq9_ zB{eeHCQ#I6->CWTG2Tr`-y-}dEs>RwbRgdE$GCm+@N2lYl*~UvhJ0z?2HB8S@19msEBjm+j&zPlcu|E{R;P6=tGVBX zs#0*v8ecQgI0ALGzdR1@OR>ao6C)S|^bOgV{LZH=*U8;st7c8kQlYEt?w_a6YyaSsJ|=CjuPa{Ps%cBF!5Z5e{hraA3#XyBCb{6BeV>eqgPgh~ zpEtZB{&t?CUwf_+9>AJ8|KAS3b#lCM7sd`rOd+RT>ZbAlp`)8$C`!8i;9r}mZ}(}S zYp6Q{nMqjGOa**lwlsCrep?=pFEjem7J4TBLl3`_a%C=W@SKPfeEgs51t!k#t>41u z#&?ABvrhpMiLi^>e{k($fURKWzz-1qNnRsu7j%7qvau6w1%15*+#@sgXlOCSXI2@E=HXQ`S z9h2?sX`dpiEXY9LY8-tK69gk*DQjMa4F9&c?AOJHRwfN(Vnk)@47|kpMtT`L9+LZ0 zXEHieuuhNZlUugSy%yxqrgr&3Ko6uR;Zw6dbQ`Z8`>dRtnoX3s`@C!U<9_P`JzXe3 zLT8p|^ifvH^0IZvT)vwtlNPJU6leB~0GLq5&3LtqQp?&941qrLYrG_yZ&qkm#XCCj zkCIgB=K2ckDGDRUy@kG#>b>ud2<(VDTVhf0&jG{Q&6_&4{)pJ>&IkA&j(IYl6ZYJw z5=R7D%0sx-#C;T%Lc>&zR4B^2vf@s^H1@g9Hl2Pi@*8&`e~Wnff5ozOLizDC`et{2VOR^isa+(!5m|2bJO!p3-36`)3UF>3@s;DG z7)rv-juxD0l4)FHstJT1dRI4)j6|DY0Np8n9!V!kM#P;0erh_zy0`GWhP1a*l_MZug{tvG~Dj-pM=@ zN^$)Z!=@r@X>OwgX!gqP|5(&38k0=~18l?5C6iwmtP#=3HIv%`7Imb42&~c5+1JEFEA~aP-#Rd16r5?FWkcp(Lr~Md#idlLIF3Cb zyu@uXs5BEhzwVN^*z(=TfJGJegOo^nZH%`4jaTa$l0(kIf#_^DFbC=+Ib^+!m9xBF zxz?i~A?ntu{g$g|L-Oa9qR^G^lruO3e>#d&XIL`g1|k7-5*&6wIu;L81VTowP@_L| zxI9l1Q)mYZe;jnLOFnFxj~xmr{HOt&?VcmcHna;5S=aYA)v%k1a5lw>Ni>#(m8YE* zERZN;dfDR@?1oHKk#Ql7_(iv}*qogxaMf64S$#0WD3#0Ern}bVXKHNd4$;yhpjgXE_bj&rM*SE80xy`?C-9dmr*9&jAEyxu#~1f z-a|a6t7nq|#oB}9fdo2@mx|sfQ^-R;d9sBN;5fr=q<%|x?R0Yla|16<#%8>Xe#w3g z4=;p(9;0TzHB*CqazMl0!zYs1D8WdVjCTnA9pW}73$(FcrWyf}UDo|sg6g=PJZW6! z$VG9Sz0HoDN%N>0NHY3VDa)fW$xe*Iipa)NjBkH3F6z9N& zsFHnEERc5;siI2n-6)>}J2~7S?;k*_jP$~5iTh=;<(zhGa#1;Y7s)z%Rmkpi=~F>V z6EZap4QTAVU&?L2H?9)&;Y>W~f3~LgJ?Ff-E^OSd zPtNngXYb;jRAp7o3g{}>tb({auQzQ1P%S8Ro150PS{ohZ-~yHdrnPtBKjzo92D)e0 zy}&hjRGi|hJUMXZw>Rqbw5=T%s!-=X~-s z558Dl(kd;)KAk@hQ-FX(g67M)1=8AZ-S%YDGABsO3s)je0yp5(;KrO%1tMJk==2Wi%fp z%{BXbx;&|3>$(TVnBG&+N!0kk2p#1|jWj>J1Gxmn&)=7L=B{`*YQ$oO`Z7AeBm&zu z&X?>SD$>m4SYDGd3bH5NDaJ3V$G0=+c{K0MA08Q~O3Ow8Mpg`db>g@z2&lwFDNU;7 z*2CiZ#_0w9dfU46s(9(NK)e#Fs)$*2#S#68u1cDr8@`{7>&$@6$nWp#r8)RItJTND z#L?}D7?a@ytlG$n4N(!4?zyYRX_O!M^g!5WR6=*}QtzP& z7V)EYEr)9OU4o>P<-d3}ybI_1skx-qQLy;={vz^kO{z0n75(?v=&+4}p2?aFg5PRL z5n6Sr{c5Pgo~YvkBN%o?j8)|pA{joIdFabG03#?VQ^dYz@hg*qS8X|X@k@6e_1MIO z$E|tDfn8B0sN7wo#v9uu@@%F%Itk$s<%hJiocwpsz0L_j-qM;(GU6(fhtZ%HNKZ!U z(Xl;Z!AzfVH#%%Z7X}Rt^k=!mB||@2(LI_E4gAb<{W@^FQtEYve}-1A=Y$uM0@5SL zFfI06)JKnzBpFsaUURpuND%+IY=(@j$K7-iE~CdVX2ygX4`{YjG96Y#N?Z&HIl#JK z!j?vbwdG3D?j0%6QhW>BjYi3a*q(BSyn)=+$M|xO?b3btklxD^QJ&)RXwCV*8@SPL2hGm_fO6mF7L&4 zjSE&l@17-W{ABBMwXu$b#?WobtzNRmIY-&;{)-eb8Y<6UyLF=*sSKzM*qRlUHXGYZ zvBv}bdw`iLt?{!`al@=k{IGCBM#}{)>D9hQ6B(lydO zL>F}JH$6r#kYQ}?8>}{h7N#_Tt_2cK+H?i!*yi7KU#3X%58HCfRqsK%ZRk4S=zGzNJ)UC^%_#E3}_Cn>a@mW{IPgT_+A!M!Mf>x{EZ|KN1J-M?Vbu4hh)i{sl- z17qPrN#@ctgeJZiTau3+ZGiL=X0i1e z3SIRt-&uHG+gFy#vAm0oOF3Pl=+L>u9XfP>PxYj3TN~{#Z2Tnw5CA3+t7izJ^dJZl z^UOJ_d))e;UK{_@`>nbyFlSQbFykwx8M*v0jNVPcKRXJ=+{x*f2X1V57sTu~SDA($ zg;cZG^5B@#xcp@9diPv{mhl}VRM+**J09NHbTqn}@X<|5W1;=7vFA9Tr^L63vUn4>wh#Wl*f^lP?=rt#JOORu3NgeK|ujPFlKsW^xp34xpbz*I=z-k}}cq zeI!{U?0v%ym}TcjPVHy6d)o=W)Jt~aDa$?-XU82@(oFZ7%Ipk8~k{cVCGu{Nk@vM z8E=*_n&q7~{hy6>omK+iVCf14k)x& zun`smXIGriC^SX58Ai=Wo_wN~y3?zcdaWtN1fl zXg4G<<9ocop26HEXR1I!krf4FYGLM#GCn1m(`BBO@0WzGE_z^f3aQ#(D!PPtw?-9w zD44lEkXMsY=LBg&Ch*l!2Z}prKenL2^-5G@57EuO{|a%!@Y3I~g{}ESe@-0f4j59S z8N#!ojJj}dQ#Uu;f? z^S3gyOxci2<|?swJ5PT>+du_Hr>j0+uy4K8UI}88ZI=-9qbLnwv5&uEG zAE>i4Oxu3J=!eiHsl~fznzowy!+W8JM=0B{FvvCFi&zR<0te`&1vXg}V9c=@SO;YfoIu9y;ubeY)9sz2M4VUE{$h&7J1TdII6_Q;pT%GYN@+{Ky(du|O07fNsUPDi zJa3gOI{Pc5f0+O9ihKf+22W`?tc-#pn#xU5Y*}$BKk6iY?P;Ky zhhlnbM1y51N_m-8#96qAFyPc4TVUe{`3u z1vg`h++WekIXHa!M4qi9p5U*{@zfP03bribp2{gm+2O-!cSC(C)%e=K1`JHp-YMO- zY;i@>NE?VOJ2hs>*pIM zRsKwX5ie?y(HLAz23Y-SUW45Wlj>4^S|3@{i8*RVqM|2|F@O(sR9Xw;L}`e2bW?Zx0sr=9_(#So(oqeEHwQ3jqaEyppLyXrfl?vSY!) z!e=Y7o+L+iM^G>u&OYu5*|OEzDY!yWH-@&(olB`e%4*FuGCq3w{=~%mN$Wh@iuFvp z4RJq0U5{jmL52f4pLDvLHKpT~My7Iy6`?djMRcRu=oW7{s)^ec)%jDLY;C*m8s=R)BM2vJI##-Lc@9X=LYrljNt7AoygR22OMw8RiB>|@ry zFEc@+)t;dRDNC-=4W$iUNZt?OB+t7fcLmn%U_Pc;vWfZY-as?iih|dWWAgmDQJg{ivz) z6bya$XyhHpbXjSU0k=UJ!%wgRZKM1X_gW~Fj~#!Lc#?qr-4Ib_w_BX~zPde+z~wuo zYE_y~E|Hh3N}#`&2qyzJ5!AsS?yC8_YF1d5PIRlu0zhN*XX9$NRM8)^8$XfoV-B$8 z`aI!<6>C|=-aG!5t+%7`wd`G(qlYpt{$f?z2NbE`%dFy6(`kQX*Vbg%Sh1(mjU1v7 zT$ekC5#{37nOLZ^q9t!aw?-iSvg-y~v&3|bw*K|M_$!pXzty>y-zW^mOR@Zv%>A6W znwK|mhGsPpZ)fCL+3nM!!=XEhPg6&?Ml?!Kv_P-1+?v(It*{f8p=mUwxtUV8PSeXU z?*fvD!nIo-*Gn;|l?l`}>eim5!9N%d{s@KWeITs2?v~Jla_^;c0DRv8Aa3CZ9gg-C zdPB?K2r(afP6bNql{KxkwDCVQOScF>{_5x{T2I)$E7Knb*vnX`>ElZUvDc9X-240A zK)D|0ZK$DLbtsCLx5u}9R0Na!2Fbs3D&#xMzPKksnBHi^hf3f@oc%wot9U<{Tn932 zWarZWgB-xIHp{wUDpZe^*-DQ{kJ0*fe~DCLBX@Pw_a98EF@ z97c}9yX+^P+!*O!^_NWUErv5kDE@=Vp8qQ&;)gp9Z;6Ls>FG`Y~cCutr*czAZddQif+r-w5P*<9E~#86@!8X z3qiRHs@Ui1F-)E5(v5ErJ+d_Xsu978h4M!U zcqFV$*i|PPrDN#&@f_o69|{ucPHHz+G?1Qs(C={*vH-$rHYX(%gmPLT$(cJhNmWU&Df-%Ywc;tcRe{I{vO1S~h zBIR&UC4F=`-%Y>s*!l=r9&oP2dff`%X6TQO0 zQDrgp0Hoevb(7gM_sd(h5;my4%?kb%;TIup3%kh@zJ zRC~RMc|bq#F=^kqFLAmsrFFkBApZaet^{qLj3FKlX+^cCEXH}US~Ke&Y_{tAHI3Jg zFNz#!Os}=11H-Hpk}qnUz7;Ui9c%cv4Zy+m+ z3~YRa)_snA3owSE?eFdfEcs7gc%O0lo~ipNb!p=Huu-!vW=m1iSGaz-du*6Pl9=+hd#(%U*o@RTxVvV8?O=(*pV)&K z%BQ&UetR%xm`?P4REywli`v|Pmq0FukNvO@0)g7B;7|T8+3Bg`=`SO~gH%&Ihb&_j z^*D)mIts5;Rez++0BuPHdyH$?$&BW_xPEvl^^$-0RDGBhO&0!#b|=%0CCHvY?&Z?X z%+3gd#~EE!QCMOnXOXdQa&T*Z}Lin{%TW6;8yK2qd@Vsey z%7+bnD!sKfw8B9a;8Lo?@|qM3_Wse8>aiTMNP*!gKh|47#*`KKjzi0qd3H$PlprEY zdbufa+(_eHT|JJ61A(?cXT4)5LH87g4!vh>LWkb{#}r1QPLQHOqn$={p?)IOyVP)AhzeD}Zrns3xC6AhU^>Y)kTU&0` zxbuHp6z;I1G@cMj!RmQ$&dy52LAMEvU}Le5ON68Ihbnz(Cr6I(W0t38hG4rm{-u8V zp#i_dx;?1=9P9J+0vX-~R4gz^%5V^c#%Ow;tlu<%dmrMQAusvbqx? zAvyBFoAMd|ZDIj^<+?szPPI{DT%TKWkXHpEV{J68{K@e2Ubn^HFYRp4E}A(>X-hyB znu0X0vM@}{O&nqv_a{*59}6{l>gvI;)8g9$*5iD>*Hohq+-q{yCe$8|AwyZGtr{dADT7ZgxHvt$u2!>%iX6;^hRaav;}?h4RK#-EhXw zuv+^%d)0v_4%V(GhIN&e`46=w*dgdP`kPdw?D7M|=x>Qv#h!I-p6Y_J7XFkUV%8@~ zKttu@WXt__Hg=s9{#$1Oe2xUWuXGTo|KobMbfMbYwG>M#>M0cw6~%h`lJ=t~YF8+6 z9Q*ldd}VvxBl+nsR`irP18vzJ4AiY@NJk?#Fy@_(oE+ln;BUN#OqGiJqYxH;`SR6bQHGRt zx^pkYb#@T&-OFCDRT*mqNbJ@se)^Mo!D7w6c4s?@PJKcKvHS~N$K=U=mMfz_ zLHevrZw-%XGHF@6`p;I-q(f<n~iev0gc5Qtnbk+7{iU534qFZh!>|0%m*^b%K z1<0%Fj(lU##C%KWKe(T}OLE(+0j}p-)*PFH3Y8>1*u-BLe(sX?Jqdi)O6t>D=V_d}mlKql}SIS-&h^5@#@o3i)3MU@e4TONMi+wTN@1-k;C8lMlM3%^k3NdqpyAqE`-2@dnJaw`xLVIa3 zbI4qr*?qY`Q4ySUWjdOccZgo~O6!oTa8zrZwp)ZT>5^=8)A;~*S#_A`>o=`Ann3Q!N^SELTTraWJMIj} zCjV3hNiIZ0%8w87NhV7UNrMKo|6O#r=RR$7sh1b&K*${u{+lEXp3Mm#{Vy`k2V121 zn8>l4xPX+N=aK*4#qq5Q1Uh!RH0_xG|5tIy7x#E-%B~u#QV2A7A&bPYR1_XzD1#Cx zM9#fE!9}>5LV*KD8eC=;pP;<1Vd9@T($O+0yr1tj(_v%S1lEJH!7@Gn|8w9VU#jH| z|K9Z_Kwr}m>{l^%O(-{&?vB6KKMONfgysjGz({!%bovXA1SjSJ#5j^BKD=xH;Bej; zl$@O%aC-PW1_(jLl*-P9qXw0iOGnS1ox$Wz{~zsO9lxR5XwrlbGe4N+p($tLA2st979&DC5`-UFtMo@a zObXOfvCrPCj_ZJy4fGmD=>c_KKWAHm0wvPXE{q8DNPgc|7+}r9#tS<|ClQ%4BmxV| zUiEHyBZn-$?{j6kdFw`@*S1RgboH7kI3Kl|mIUEKmAhLEWSn{os3JNa3 z|EYdNWTZ~q#AVL&`_Q6hV8kiNlyj#9WH_khO@enFxP=;&RvPz?6?ATp`+44N^ma;; z_Bu*P1Tcczq=Q8SYU60KhI8=7#h+~|;w_Px}_cN>+$@+sd`op+SN?iJ+X9>MwENi)3d<{sxsmDJYu zi!_u#Ucq~g@5D^Ii=TU5Neo^=Hm@DYb0NXf#7nf&c2{|pUB04`|KR2_uROM*sDo5# z;wS}ldPw+yweBCaeZ{^qW;ru-b3a8&X-A65IcCXk0RO?M8-qh$V|DIi8&fs9MbBeY zO^6ap{CtLP&3vxL>2hc{f{jh4@8u-FjDIfc=wZB?t=^&hediIYbG}_$nqjY^^QARI zm^lY~W)6*SWKsPa|FLr30bdOD+)A1|yc!OMHksA8JrXy#*!C~_&bw>V?YAbU&vY~L z;or6K_qb^g-bvFvGjV^2ee4*?vQX*qRkHdAm$37KQdD_ZUv;kt@mUl&lY}^Z-=s(s z8rMZwCD14y{ABB`Jb}_%Kwp8NH2##u{7X?P2Czn*jt*C@t<9!bzcya?Ex=j6bjarq zMQ1~~Z><7klQ2Tj-vIu(p%xgx_fzS{kx&a*OCNm#_LDle*$xR+Ugr^Wy?m$F7TthI z{uSjcKv`oE(K*U_`D^n3#@<^8#r1VV6?$EeP@Zjzq+!}4%39ccy zG_Ju31WB+=|K4wEYVNI>`D5<;-KwdYI$gU?_t~er&X7$z!wykDWU4Ei zGO1=Tq|bu2kY>zz%tz}7%uN$uv#QtcQ+d@9jcavcX0@Haxfq{s(Y+t1z=!x%fjl@S zjgj{~xZ;_l?XkeI?VC0_*JdIwI{A}Yw52_5%w`s`CYHy5R;Q&d=Y;)r$Hb_ka|os= zV0^natr>FIz@jR+E!34bO;wiAp@!>SoQ(;V9g{xytXI5;KVCJHr9HlH);H=uGECKg za_nFB;hicNUR`({F9y46`CV;v^La%7g>irCo9}VeKR>(u=SyR9@fs5mH3N3+7Maxs zc@^hgQ~kjt^cX$)@56ilY}U+cay2PqWQ-Yp*!zO}RkCm94Y1+!;cyWYt2|#rH-|$h zR^N|bWf8k)i%TfKgX)^M&l66`YcnVZtL7{{`49P=Avt|IAUs*~;H#bd1rwjyo#bfx zY_bWBhxU}t8+oL`bx+;|?=UI+iDlBMMgvP;fO5fYdp@Ev_YaS{Mhl)$R32BsZpvm4 zazWD|49_f>;kn;&U0Tc>?Kgk&K7KZ0Owsgov`Lkyj}=_cTKm>a%d)PLi6(kkdedys z_S#yn>~Ud>6=wk~0RZ+DZ%4*Sq+QI2ZNsQO-cjKV$g{^w7y+HtVmeRN46i2KbHf+L zvvh|%`0;y1POjl91fRQ8ndxZlIxN3o)~Ayw2aK{t)MS`UUAl$El=PnBJ5$b+8O$`$ z%4~5l!_G)X@mL8((W^{}gTJZR?3;Qm>7S@|SA9Xgo7oY;(Lw4hZtn&Z^i>4H**QWGl(qdIp#zW#He$+kDM;vhL<|QB|a7835kPoXkgRH57mzfeN-JjUCP&t zL7-FubuJGL!cs0Q2fivk=$yYB&T8e!cl&AmKu!QFPbGa43uSJTsk|#~!zJr>Yf5x{ zCa$1Qc^6Vai}-rqE9Wa=onOUYf=`Z_I2BOzbOY3mik8Y~&U>an(&@4no5$hW;2&F;J9T{Rp8fvkO-u(G z_rn^Tmoz8z#Ij{HLi=Ww#QDgxKNeBD#8EhOl%Qj~wm=HM_spD@q#T3&+cEWhE5l}) zZ<}4WE_fEfFYThK{j68P@5|ErW_O(7$jQ3bMcT9%Mv}g~sTm)_AW;bL!(KGJpm;_9 z$AL*0^%K-TXTwtK<=94;qX}ZhhiXWFDyu4>E0=3nLKHt51*!z&yK;*>cv$WKRA9T0yK8Z&O z`c#v*+>r1#dBy(<@A4X&74zvQw|tO z8B<~;3u+#?t!t5bFwm>-Cl!fCtc~6{0?&%A?O-|`VT$FWJ#VX-62FlNPG!61jQ9S9 zu}Z^q8WSY=RbF>yi@r`}S<@S`l7#P1FZaj07^97g zdUI_Y^+zoO^jRq>hYS1I!XFg$nXbIACcN*TOBe(6l*M-%eG=bFNVM?a(MhsoN|UJXTDFD0Q|KUA;$c7qpQJ-mo9~z;YNYxm+MzF>(qveccGlZZts0 zXlFAXuWT~$gweYSe?Nfn}fYXjaOkO$}>YbVBV^t)Yb{jf^I^NU| zK*ZC#dAlm6usjoa$BngC`k=kRbOdkxE)P}$K?1j zorj9|xKyq?)K$p7oO@ z9A%t+Jr|QYo1O})$ZlrXL(I|9o*6B%|>`(_OdsrY{bA_?7#YkRF z11FREXGEs+^ENaxy5`#ZeRXzds4@*SP76=VP>|8GGmfYD@!@QzdVD#IKkhsRCNcqe z1UX9-{N=Wxi%-uabZriYur6~3m%-9K12(CH*YVu^5=u!?naAwCFVC)6{TctL0!XhV z6CQ3U@mZWsNZwKg58J5mLs$T@_t(Qwv~V5ikG8s*4OV2jf_Dvlg3WbEV=T&&Al<-7 zaT@K~^e!7J7H=Dan5a4f%(`0Gws;)ux)MEH7Al7Z(5qht+CN(I6!?29ME%X+&REV{ zWAc1af3xL;jq2aO@dkK;4_B(8=!7fHv1{hp=OgfyiLZOtV)6~Vv7-fBC}155JtqW@nFSLnrk{1?%`y%D^z5`XNmC`E1a^azL|E}J3GwlU*y53Ywl^s{P8N_ZP;Z2fAtj zZ4GH=mI6KNNpQ(K6Q{!e)n*M@YnS2z5WdMS!HFeJuKZ88CH9-8K+(IM{KYTk!{D1$ zUDJUD)bxHqs+%7j<#cpL7P??|I*IoC+K(*5YlU;QiKqq&EtUhcz+^1|@1(+KP)o#A z=%Xy;Vcl%laW6MKTgOy4BIVt3ge6( zYfHyfE|ZOYIu*9q!p5)Q&V6fJg!ZWq48|~jVFu+69_7}}Z9Y`9w>(z@kLzDq2ly&4 z@8>WsBi3b~uA}ZHF79ng>8QA_9Y@1JX%6=g|H!%(genv+j*4ME#(AEtO)BS@Q*vnZ z1BeKAV7f^@$T;$(I7tD{uG#>GWKYk*bvYp#2Lq@flIeue3NK4Xy=-xVkgZUt-YtMS zw;PLnr`bo~AHcGD!U^FLp=pz&DTsPw{8DdTVLkjMLh+}W*~j8`$88la2%0&UZ+@{M zv&$Vpc?%<6J}+MqXYCi!TKiF!7!#|n($Pz8_KwJt68ODapI(Gjk7}VV-pg>JSI_qe z$lc0kiJd+ACNL7`&?#`z?k}3Ft#6OYiC?z09xOwk-4gIg?S-m)v@7$4XR0B8S|laG z&*+&fH=zuqE|YPYXfSPB+`VEn;xLfq{i#B_K~~F%(Yz(0GLY6>cVl0Q+~V*r3?6jB ztE}~3tXtmMZ_IHk=G4=5*=hmKTV-*zw*0?ctvl;S{p_psKc#Na*AyZ*^U+n;Nq#g; z>+IQdIyqF2m-Z*D6DLOBoTbI5cj6JSN(iM+ajM5}Ccs>Sr9t-ay|8@#x{o|*T~Huz zq>S#k)pE70A%Ym={q?G3`f$-Bi}9g+r9tug+pv<>&j%nWTfr6-c$!qxW=F&3?1{+o z%SBj-B{Y=h;g+H984)EkSn}R_rPCkzG!AzhpDUbZ`|E`?fmS4YVd4{J^AZW(jN}@2Hjk3&Y%^yE7=jexpL1db{uZ zAWHJ%v`4pLqcU%HyIs~khGl=E%-t_))dyL+`us#x;^FG<2OK<_xvZd!ctn{gs^D3H z0rMX~BPFq}Q*WK&1ffUsm;fSuhEuI3HWv=y@-NJ7qQH>==E{J$;5$eE@QLx#`#(BI zx<|w}9$Qm>W1Zy!9(pF)OgZv*-qS6af^qw&dQ3~r5SPpC%?<6gZsPW{S;-%{iPr93 zXg)bdOJ^`Q{ZE;!oV$*rwQY=vLn(l^bnw;!@ry@y)@g4cVy-ufl9swY(Ld*~GtZKzTOK&mEFn3}PJD!O9wiqp&hAdNx& zKCdt^U#{CVWN;%Jzq!dSy;!mm>nNaw_yqeS^R!j3U^%J*Yul!7Kfzpt$EnDQFCZ?1 zT}Wfr(8QEauzrI8*%nA^|);-8Ont&$z5PfMrF3$0V ze@W5cTac4$bXu4eZ5LI*)#`+v1~0~=^Rjo(bZ+$(k~om8*ONeB z(L9QiB>Jds1RNiP)QDouKT(vrdBKQW_jpK^II0D!WPMALn>OC0J@n3F@f*%>!lMKK zR3PQp3g9yxSX!W2pYWKHqnYb-Y zU-3*@6C_E7bPMY)Z~*oCupBo>Zzv+tq2R!5PfFWL7*J-!Sk`EcK~3=T5Nuww(c<-6 z$}e}WXv!Yk<}iQM>_~dSnRLP~aUiAmxT2E|z25CPPI!Av#Pac4k7?zUOCsk;)risW zgf{N?A2*VyGDd|~BdN7(D1;E8R+#2Cui`DNWtO^2g^W21?VkW%%2kF4t4qsg)%6I= zQA?Q@C%83Wj@SaWE71*jQsJfGyp`Vnvx+Lt z;y+9Gqk+{O6qfU18UuXwxntxSl-1TOjtq;&_@0Pj^VbX4spI{6RGN=d(_1o%xOL*x z#b-|}`Vi)WeK~fD4dB$y8yWTiC0M;Q)1?4OB^w2btfC!9m=~`zZY-E32g)px?gt;El_aa*L(|2|DR0*{zFKtUxWixqjX9n57R^P4}2YPPL5# zIxB?dScf;!buM5c7-RbSV2iA@{eTvdPgl(od3(VLnZL>g+uSrxuCh0LN&2ULZGtw= znF|i~E+3##CCaF`25mJsY}8V;oV#7u`qKTQ|l)*LRhfOcX3)$Q8Jm&)o)jw z5Fn!%|K1h2Vq;pbH} z{c*p4Zgeh2tldNjxto(cz1}GY&xkRper%0IS#??^YQHulc_OHmTK{X;Cjrzwbjuu< zmK2Sq7~l~(p}&Cb4;f~r)*6#Enmwp@JZa*!j3>^DMXL+lA)|0wtZey0YC3D)U<_f> zfS!1E&KhS7-?lG%)bYaUg^jUskSt61kI)e68f0%khscdp*ql`7=AkD|{gD;TktVVaM=H2+D$OF7xV+(k<|1am%?TN@^jle(=df==$+pb$xxbj<3Mbq0OLs z>X=a%K=H%(AT_T{{{k{8Ej-81%+i+p!Rbm2D|q%xV=6||;U`dY&U8*e3^<|CsQxcZ z0-69(-2f|#=)FiP9}5mALpeQjQad2NbA>Ax-FX&|2ity|YsoU0yz|97`Y{U1(k1YZ znoZ?%GG+hShnrIxa@J*D1ug=$G>ipbG=^Ut`|7v#;FT51}FO!poot0L56>0GAVS$ynSFsY2K=t85TyGI;2o+)+`z0mpHJxYv*v!fxf}v<@)fmR^@rtxj+oU2d0hM)K+sC zhaue{8E#T6K@%mfYY0vFH;2LA8|AunY{9TpuTaQ=zE+iOgY3K)Ch9%g60-uVG05Fl zI9G=k;frGGVU76sx?s7rp#*;C54i}cq+|Ht(nb%gWGtTu7^w}>thvvqtzD6NdnLJ+TWwU)KsdW&D z?nI#FuIXkmJ=T-auJ=i_s-|Qm@rz1CZ38B~n$=r=gQ(&@cl)pzWTOE@@Dt21O;$oS zz@~$?*ycLRN;QuChQp!a=}9tqvz@ku7R-W$23iqL$3W4!VofYVXP`yKRYsGw&bSfQ z1jFx#2mGA~!6-_^;I`?`ej;@#PF$z1%qzVU{P6nt8rQ&99?gCYrAtMfQS1f5fPiffBi zl8I>q^t^MoA*6%TW zX7z1?fynqCwn(N&Lba@E8{|dimrrz-qQ+ z?Ws0qt=;aP8D!nUDyx8d{#E(Nb#pw^k4WLv%}w-W5Z`v=L`w+uppYsT4NrGZ5Ym_0 zriY8aPk_`s(caml+F_VLucL%1NKxVcGvShoK!*30gjf{#<=oeB+?S$z0ZIeu?3C0dEMVfMhi2_vk*xK? z@50tR{;NMqoCW9wi|S-bkePcUwh17+nqPghWLiYG=2`tK3`AoN>#-X6T^C9%62r%~m0 z!*?F&;NpG~AF1R8EksP4ekE!eFL9o>q8UO}rQ6mV;@3S&;ynYjYar0pX#vUp!gzD3 zj>$a|1a-KUVyo0SP<;qPHGjDrEy!r-%N<*qmV7hr7hoQowiI19Ya!FuEKChizfLG6 z+hdar{Nd@iE}0`o+yDajqskfbj}_t>Ode)^8|&uQv%|JE>b#HQv}FV1-WQD8;44vZ zVsmB4A2jH6Q$NV78+KT{ifroA8j!ksOit=_p39f$Katm3)#-MQy~FTY*;i7JkI7r< zzZE012=VpFeSr$=UzqrfyQ5&kf$B-k_{W^w-+Rq{0OjM?ioC;ozi_+DUWI34-?EW` zK_|x+Hd)cAtBtdyc;_!@$?=A6H2$e80W_n~WQxPq4rc zGABeaG<+z)+-*BrL2W5%_cztA!l!RO!Z1K0gDj8Du}(8Dvs4q`^#TvbG*(d5VoW}1 z)$Y$0aC3w3Q$RmEqBTq9WyXBrstyEtXn)P5vsxACjV=2Q#rpYH*Wa~u)8n3wJQ$Z$ z)z|fzc0v%lYy#>dS(BAYv$*FrG#8QJINE{tn%7uMdF|S_+}ut z$KYt!09}Xm%WrFLV7bsW&7g=n#lZI1umcr6#bsS&Tq}9tdW^Nwnbq%`t+Gl&Q3YTN zQ{kZ^>%Ng?g_mbik;GM>>30U%WWvQsyj=6k1hS;BXDS0=TiPQC;rhw5UlhsED3CLs z-1MYTY(HS#1Rw8fi}{cmtV-O^ z{cU_@8T9&)D#c=&srLp<@7!wha?xy_@s7(JNi4qCm@?@a%I$^gq?OJRz+fbNSXV*O zm{K8pCQ)=+|6yTeW{z9?9*G6m=?Yl1x^SEO9qlpD^}Z*0T;FVKt1^V}j#CxR2OV4E z+uDHfCP;%Sagv!XN#;Iz1sV)EjqgMgH^u*cx$4Dqr;7P@`Be_$KR}Oi(}6;pA3fgO zZ=x0~8C!#7QJ{56rl8^}!uiXb-i99AdfsA9731>;;TmOWC`SrARz*AbFrzNVOK?;M z1tBcp0jkbQ&Mf}IM6KR?3eZ_DVnoKnT$ri5Y2z{nTMe257W~1L`qZeDv0wR`Qx%^oBjAl!+)QuQK1a$m(2w zo^_ABxCW5?3V5(z?OM#0{H9wP7Xnsqyyvyo>w309<1tKQ9G9-vdwma<^W1cQ55PRh zSvFLt!a*TReH{FKj`rNs`u%{s2;JxsocN62?%sNN;Qm;bwJaE!5Wj3V-}(`3llS!b zXA2XQ20_2PN~lzgI_2`_+QV93$MA=W58bs_`MQtFieKT^%8dzX)KwwJI<27B_zsIO z$QOfgwE~ujzOI@~t>iV%ewE>LTq)ZrMuQ2npk=ASfT7KxsfMT-&jU{Uy8Zgj9AQvf zT1(XR$FE3rwkZ0PUGSv2e4AsDp7Fvdoeao3La{9$TT&m=SUOpTk*ss_c~w->Wfy{` zJ!-a-7?^ok*g6;j2Ye@uaym(Z(zQ4{oZw=kgxEeeg?4!BI{98&t8U8uTngxCeZwRF z926a&M|FV*j=$l#;_*~@*Y!Hi`MV0LELeaL=#(%*fsy(s-&RY^4#^bD(KWZu`u`caH9zj~c^_sy~Q>VCh^O z;O%i>;M8QLcN6OcQAT<*U$6DPTBFWfj~k9^-pILco4-D4$dqBxd$#g)Uj?!zWiwV! zid@h_h^K~{rvqaH|BNm1M>wLwfC{J9P1hT)bU?Go~i^0 z+G%I87ry3V3Q`!Eoe-0cJOzVci#1_K+dS!YP=dFYA+Q1rx^mH2Y41&GwUq~~=)a{s zihe_l?HpI)f%4{_3t5gD?vFIXv68?d4hiKt-SQVE3u)N+)B|$y7Bt+?NDrK+5QYw` z?7AvDKJoAYN6>#YR>iPDs@uqHl~w*h!EXxorGGj)Us1TwXvG3dOpvio|HCkvP_e^a zKJInSOJ+sbD$V|VH%CkE+omBNzSkxHNtlyK zUb5$@9V>;lzGv}4xgbimXw4VLX#|rHOG-pe`|GFe)n*X{ju680y2fd}dm@$$t4JGW z-heMwH$nWJhC_X#r;FhI_o%x0AwY{3* z#`dSwC|W7se)+M!(lk7BV3^&1Ik3$*DHr41t$Cu`D}yj&qz&UF42AV(uM&OuFqV)l z5)m8TWb-7Q{ZmlOj=s%v(B#s&l0+Y*g^^k^84I1Z)y$OF0?bCTJ0zUT)4&a?3`SUmldc{Qv=@IUo_y4N=G&7Iis?msnzp3o~ z03131#KjiUzv$YGzZKfn6y}wS-x}{hecfApqai9e7 zoe_>i8al9};m}&K{8@~G0v`goBET+13JfhqtzFS(rWN)TL`#4#w_xslw#}B&)=#P; z?u7x~$vCPPdh!n^R`w&nVet5NL$Gfc7fHI4T+hB_#PD|&9q6n_@%=veUzpf+z|R{F zg?*w$gM-+w1S)#}YEeXD)~$sYPg^?k(VFP|4a&$KnhU5wW1lf%3CbgalQBMg|6dJ| zqA}6a%<6L`J$OpxPxCjF6OvR)xYhV#Y&Cyj2s%!uUfP0K%xkP|M36WM$W)zcnD>KS zU#j{gclXB}FTF8xtH>Hur`H2XMyv)iC&h>(2?$9Kn-j|%n@c#X1OmOU;wZGejcDy= zN-N4k)pEIJKNd7lsxBA5<1S%R`Ltn&f8HQ1&c$e3vh)LS>XLW$^aI`pGSy({+Hpkon6dQoi(_PHavyHsM21~?S495YbjTmJRD0eyjKrf5rgV& zWabahzF{7%X~E!3##qkRar!Kr*+}0=cv^LNz>Y>62g8iXYVD3TNozUGdOZ>kf)mIm z4=-MK&oFTdT`)XTv3bqGWDPdu!Bhy(BcZ?qDqMDHvPq7_7oQzjHdBL-V`T`n{1Yax zh&lLXSE=rW)hGF^RFpJFp;$f$_n8a_|5mp*VQDI*^EeOhNg}AQ|5SZ|RGAjb(6O>l zqp>?+vT(-N=ZMo-s`_`Zpe;TT^~3SU7gv!s{pPN1JA1p`=hRp`!>ZVLcI(?N# zr%RSguU(UMODx89(beo9s7H|&9otgJ|HSQC55a=9>vS@v6TF|eg?t{fd<=@&GO=cA z+88lRj#gB%7(MyYh?pry#I24Cgn##fd{&bVpLpZ8EHK4@tozSBOkTZXE{e%)>Kk&( zC)XqvsQr4n|KgL|L>ra<&_(QmQ}!;%rwfy|^YS}yO_lMg0#ogVcu?AnBxn64Z^JTn zGte5hX8Av^mH=yyh3k#CFYQ+han`CHZlD{dO6Z~JAL&z};o;WI>+5~5)A)g!DZ-)6 zrv@X$1OU)fJ}%8RDM!nXYq z>AKGNIb$bm0#ti>L<~~zHeTPNq4qq9(H zC+75USlDxxNTR?0!{W;S?=7zW@?{0CH0=+;R3BUbHHP03DooTXTB2a<)CC1O5Oz~q z{$$ZMtaH81IRdMz>Z<8%yW;|;hU_|YFE_1v+SHDaO1q0~sjL>;)JWeUJ6Ymy=daM0 zHk5rS2QylRm9>6VmJn}(z`_Acq<*T)hM^hXRGLkAkbugPqtab!iAY0MzmKP={m}U|9TG9BNAFLpb$-U`i)WLEM z8og|9H9KWgvj8#(l?$)hQeQ!L@b@-(<>Mknc5tytZF783B=K>S+HdYr7nJ6neP_U;=U__fbBUu% zYEFFvMNNF;^a(5U<9I+$TDD>o=rDEq{Emu+K}XQvhn0n^HiQu0qCO}J6C7a=W{xU| z*h$XwJyX%+H7G9Po-%+xXBb{%Q_@i`|3U5ATsih)_aU!cpH=4^lb@Y)T3_S-^88RS z@0Lkmr&OU4gP{zO!#Z`OpgaupGThc~e1t zmXKn-Pp8`C^Oe-qnJBBSR+%0E>=kEx5IQuP%*McVE`aW!OVi#1(Y;){dPryr)6e^% zdtVWOe5UWECq*KLNV5~e+&px3VsOn^vx8yNtXaiYn!_Y0rQ=9fQ_$V$@6UUy4@On# zbz0xoZ>=rS=I27u*AJw+{7jWl6rSeeP8*Fn#~oT=H5vgbzmk{f{dP zGuf_+PT1g+Cp5Q33Q@2tIecxt^Kxq_fKJ;lIf|wgZz^YxP-B79{d@LUop%`KR>!;; zFN!fZsD;8VmHov#j=hKv8)NM5!SPn&jjrJxnH-NX07C}U(suK61=5sP)CSa_O_l`zYkVCW_0{RPPzGbi+JWWm(Xk&7vq-!JKr z-mY;y7Cw7P5go2v^9HVTKp!n5-Cq+k3B1=O#(JXF>6tOg+Gs;2y8?C}&YEQD=g-*+ zp|`cW*qf&O5J5z;EX}!;2VH5l|7qLI%pR+oZ3PPKK2u9PXqWTceRd|BFx3IN!!#0 zN%ZiLf=R{IW)b#+nd=t%ghAM$vXm&d<0$F5c2X}9!t_a>`%S$i_3#J7QLl=45F3MW zc=TQFCK#u3g<;}|z*J;(wc#4w8zMM-eLJ;T#7liSwcKZ_yjFfcK9)c72|!z?j6tfS z;UIa>`EH56{H%=--$p%o&8i4~uZ!`)o;lO7Qs&$L>Vh4#&IlU0 zYwpoj^R%BL&%ZGRN!}l4ik1ZYQY=#M^RG~=Htk)gT4Fc-|5Q=HhM>n3b((R(j?WWV z-t&*XmLAt%8dtazL2K?ZlJnE61TAR7U(=_i66N0C95b?PeV+g3tx zrEwju&HR>WLZ%x@v<$_0KYC@YFQCUFsB*Z=nHylOV3_Mrf%<^!vr$?D2I|-?2#EQo zQ;f#mS)n(b-W9Y_eSX8`S;QsW>&##3tE*Ns2<7lTIR{iAz(OO%{^JxpZJythA93~b zs?>ps{bPOH%da_i`MArrva0kRns}=)USNnM)15T0Xx zLM%ibl+63BCSin0T#?buz1E>-dXrDZqlvSm4KLb))&6ph^Y8lC|9LwFZ=iIFKkvJ}SDlKc zFnnrWidP-~%{xod=w30WCqP7IYQ__O^PDmyGrp3r)i;`^x8Q_CEm0xl*ZsoLE#2MU zP*?PqV3(_PvmR-$dyF7&z~UP@+Dnj<{-CEomZ#|m7frGSt*%vH&uh2$_SV~$;I#`h zjc}oFM@WZi&&I$;mC5DK7TW^0+@pbo8R&FU*s%?JEV1*N=)yJ<(9PyE(qHo>E^neKyQdbSQ3i?gvujU=-ZT znEJPFDb+8iZd4dV_C%-OTB_cE3GIOC!gYIp@(#OVQ50Us8mjaw_96&h@Xly!rfO#NI%P?jYXI7aaGdC;6*w0k3=*2e#Hz zRt6WNDkUwjjLUu}8GPL(`QW50DYj8z=baK#Hc3=oa1+SeSy-SF!$f==S&I$JpP8?7 z%+I_~O`plsa~AJGa1o|T2i<%8eR!?zRCW(sp-FKQDJD~)&?d3ve&JH3^2Uz^QK0Tl zS_km{iFw+f)(^JqFAU2f)T`oMlU`T)Sl}j^5UOfG$8ROl)LU3@4H#_pE08qr&H{h^ zGV&BK-N0{=;ihr_3&hGJ@)u@8UDUAE)~L!8Q6_o z^>L1yjGv6z+w+w-z9D6`*Z-Du75D{e zJ-A~Mt=7P3C3Q(ig;0l^qd)FFdv%bbwG(`>arGsS-Kq2`nityRJ`K?;fu<^G;9~=1 zgK}sz@}8qhfBtV=4o|iDK#9&OZ`tbr_wKUSs-Wl3 z`Qf{UF{x#r??}?a$x0 z)X00dHQDW%ZgXiPTcbq$2Q&i!%cBSk>w_r4TT#)z>DW14hT#e?~K?|vl z6a2~EyMrGU2fmJVLi%a#DK`)FGZWjFkSxl@lxzQ9gG?IC{LC85tWvE3=TuD;5K|7@N-$~t8CM&qPz)R26@3MPbQZ$nks2@go(Bu7)Yv?2+_41TKPf9|z3BQ=1{vOEv zOCw#&Sal_$fNk^}{{2>GWy0^L;zk?HR4l#L`7onwykDno+dTx$7q8A=yWB8O6OXVd zPyM52pt&al+x{2)>4`cd7vrnd97EpS>{7{Z5J&<=~>MmMeC2x^^uI z(!YRr3p0T}I;Ld~$2x3H)gQ3|4IOrmY9svUqKrvx-#oGHTY0K;@SNB5)8QM#CIyCd z(Ks1j4V`8x4{ECc%6ISpkudJzyc-1nK?j*}4iI6DvUTCI6&F*HK_&uLhjs^CVJ5F_ zjrW>SE6$dy9<6oX!x(CAycB>jcQLVojSn`;pYj$xX7v4i-d4s%JVF(TjFAt0$C4Zf zdlM6_4gKq)!{VSSVf|%-9&IYk355tfHl$}dl{T;y@hhABE~)2TQ~Pzb$_(h$ zbaK>?-K7Nyq?{+S{hfQt()hbVO&#o|&xv#zv7BWxDC$$X<@dOyl#m<4yEA}lNMS9} zg%PvK^Pdssscw@=!4zjcc$B4Y3#aR*!!58nr97TjP>G!B8VS-lauj>hIXJ7w*?v<@ z|7E!Pir=srq}yQ!My*fQ;DuQ*CMj?I%*P6m!@baW8Fiz_6MSa?;cGqAy&0DM;pIeZ z>O$=B)Yhl}$uLT?b`p)jbmXGad{xU!vSu|%igsFv-iE6dSc?B!pm-N?lnlAXU4J^GMew2;5lE4`QvA3geF ziyMFDMdy@2eo8P>epwuU?ns6Q6rWufi^*BEQocH@=vq1hXKSPsdyk>QvLS-;vUDtn zUH$=0{R9g$1%=XeTB%9Jp4b=fJK|vH(-Mm544Z`;dF4kNOoIlh{^~z$e(Z7Ees{*f zkuJhsuD+7{^|M&hIg6UCi@^T)wFCkp(|Ph!f}|yL&N?KyDD#tk+m)u zpevAF&D>-VAppaZ23Eb$Gq(@xFpaWR&#HTTM?e`0e48Qla$xFa2OLf2P+)!VkfgB%J+*b-QCLUO4y(xoJAEPq(^=8Adh@*5dBbqGhg|rs z!UC6|C~$$4+r?LQu;N+#)b9MuWqITA(`*WTm2b85r^P(XBwF7-W$3i!)OJ-W85h&4 zgjg1P6l1(td|}ha_tXcaRFHIp4FfqKC#zC+Za=wnX%nL5nbJHd#Da4>$S34$?~2tt zs#ea&EE_Z0)9vdOIOt)Fmi>thcKR3IAH|n3f+NmO0yfIV84C~QSmu5LydAWDf+Ma+ zeFZYmJh6+1aMt-}#hiGJahgH4CS^AHs|0ld5+tQq9l^-q`X9Yy;YZ2x^|M6w2jgWk z>~c^kqi!h~;+VfI(oI`BnIG5%n>gW-W&-ThT|ArhS%QJ28MU(E+a1FdqpDXiHKbji z;*#q2W1`{8(SniDdIUoF8P{F*UNXHK>?dIWV%)3PV&4PG;T4W`uai)_)-kuM9P^gH zwBM11wX#eHO-wqDlRal$|MA*wL7ouvei6QoF zEFl#a3!u-zM;*Xt^gfmGs?LO=mqBzZ-+KP3Yi~p59`|6Ab)^R^#hoV03e_k)2VUh4%zIRVvPcgeDK|^=@cSV?wMFiBjzC*y$Zf&DCPRAK z%cLKJ2x}|g1BwRp$Z=nw+X}+-@5lay(V||fdsgbJpsHlIf4X++E$Hopv>Q5K^ixa-Q52lF~~lJ7vh)yD=U$ z?}Y5T{sgbjalZ5TAikTByB#L8_^D>pP!M+=lWXG{dMakcUl>}pKX)+vOwL(JNO%I& zy9LUfKhpZw4@){fzn~}+=8`B`y>3y(m(5+y@$qr{zLD(~b_s?1ID6C-(7SFI4dU)N zz!G|42v_wj&iF67i|}fBet!6hMT5Wc-WZM~zsS;B zTm+vFb6M}w^ojPMGvJly54kfCQ`}b>4i_s5FTpX;A^ndmZGc(itl{^YMFdNz@ZKU1QB&6@H0X0F+#{C0Izle4Q7G{G~PNxHNP(@wEwu!Wel z=(`@Cj|vCj*@E&rOhZ)gSxsoYo4Jv)IPPL_-Ph8`aaxG}b_$!}nev(xE<$)jpM z2!Ckc?5mCQL7>}3$lPYgWyR!gkUk+<#T(@or%XjY-eyj6wEe@_Wm&U3cVQuItRTkC zcNksFVKC%lA;i1sE``O$*pOpS2WLNY`t^bCFzRPt#pN3;bPfD9pro}x;_;)Z=wBF4 z7s{XWU+=oeNMYw^8sx+ulus<8VSBDRJF9Hyx!8s%D;Yr?#&c76F}p--?w;M+Ph$|MS@g=bsaA%h31L~JQC9le?oI_1W6bJindK??+`QDp&E9bDJRy2&+ zNYNN|p4!s=$jAw7gb6x#H!?f6uY+8C6R;c*l)GeFt!f1XJo%I#;e2*wdV<4)S{z2m zS3;SJ_1UU^hjm_UrK&O{9_K#Nue|u2wX}P=dS#IH)+FiKm|{(+A%})1QLab1#f%3jUgQCm z-5;j(l5g-dDkVKvyYVhAoT%j%%Lu^*Iu_gzz9gnBClAN}!rWVi#nnXXf{nXtpz+|6 z#@#hIp&N(b!QBbgxI=*8PLM_d!677#2ZFma?rs5se#3X3Gc$MQo^zl3V}8u8>Zf<@ zlHRpns1z&>52Cn2#|9NWdOr50K=i(R48aiEoU6u6m|HF3mJM#Z!JId)f z9s;A&7^Opfv4B0+eAe2m>r$0=pYHHue*^PgaoJ}at(GWh=eeMcC*u`x9L*rA7u_T( z)F+yaqp{bUhacGEiHTh>RIbEqbmz1Kt!P!h!mb41%RdgYIu4n}utJ|RL+rt$CAmHt zT2=DA{{TFG(giKAkf4-6(G?ZP@r|nZn?GDkpHrlE2pw)hz z|Eu9VsGsh2@<&+U8>{#Ry~Jc2hn(g#+CLtpp5;4QPBS+mZ&;k`zO>ak0@v|>W7uxr z8wX6|NzlVCx2mN}Y?w4gM2E?i%x@|~hMbMJM;8BHXn3;}EKF7&cG9x<2vK9ldO8RW zcd5ll!3Xb_epcdugJG{O^}%g(`=vKQk5K3>Em1@Xc=Eq$b4 zJR0&##8KyyMH}IA;!6vYC(Ci0MtN&zz@;4eH+BN5uHe*9Hz)n-{l&N~a=&2|ZcOprD0xHdju8CrA%|zib)IZM;4z3kFz`m$K*uCda+q>u+b69szq%U z(TE7YC-_MEaUEr$$E$tn)P5HY!UHFxrxPtW%?cwcPriCY(63e9Z*5lRQ(Ji6E;u+GCO@FmWu#3b!lT^bJ zjXTB^)H#@(uSL$yH0Ep;X#1&UI!P8_LJzPkJnRDDcYO1xdr`>8(R{V-&Fp93Q=ad8 z3*W0Nr`z=o!4-qh66La-fsHmal~iYCXDNO&?IX+M4^aFt`)sUIlZ!zpGU};Dh1FKz zAIXJzTaf}Wu^OO>2V?;V40=Hvfj6*o6NPE}-`SiVwos%!@GN=|oAQj){2o@Q7pG^m zu6+v~Jt$Ey#7?2;dh~n4a>RE`0E#xC zuyjX!uOKu{A!09iGWT-2p?53aJ865v{@anvOI~wFF=t97{}*P;a$MlW7b1t&Kda5o zVRz74buf(5-A$dE3!%(StN#0Z^{IE`%2eN{LRwchOAJqT*20$-dx3u4^mKCj54FzE zJ5uA>M}!pHMEhQLEEs6mdXQRZ>8nR0t<~EBKaxSTRTr2i=zmHJVjIAAc@w16=b_YD zfJ9xd;Xd1*kPDTHL1fKi&{C4ArwsCzzwd1};yE$XX}*wXC2#J7;6+wYLoLy#?Dlz@ie2p|j*G7FzaLi2~Pp*KHi$k;((dy7G0r z=?O&s-og!OY+E1D)H{y(GIqo@s)&)+Vc&*5czv*!ofB+zF>VbOxqI0lEi9$nx2V1K z218xE^AZB)D`fi?OGG3>#AGUX_=H*_;Cy>Qkgm5;cWMZI&dQ7KvrfNG1w|?asW9GQ zaYva*f*)(PRS}&1y89M)0%C@P7Y&@{ZPpoWx%OVMwu`d)u>~!A^2Ff(9PvqsiuyQKiv;7mO<1s3DC2u;dB8>Xb`*dB zg!^r{)CLr7r~=fo-v=QE6PENIg7;X?&3A5z)o0PS|9(Qn9~a=gl37!S30yae2N7l= zq8%>7iF@O)ZNvE zx~wrsrntl8N83<`BU7{#+(a4UUB4HkuHxdy+-xcGqy1GL*HAyi%`N|Z#wwKzsN;ru zxnW9t!_p-e{vs3BHS!I~f8QVA?AwH0)uLcNb!0*>`zQ3Tai##ac1) z>PsJM77M47{QNQl=D?t}(6bMG4)YG=HG;p_?g;slh~HUHJ)f>i-hP~=VbM|jigw4} zsNvFK3GdR}QgV6P{uPj4@9tWmR^~UcHe28}@4aMqE&a2Amqy z<_WR?WPymQRW+OIlz34*x)(MSpDTA~0}Ct+1~8z6iAU$eo&K@if>!J6CdphtOs&4! zeJ=EHWN{ku=PcQeZ(}tM5?3sx8K(I258z#vO4*VZPFNBPnJP4Q;YRK5$;tG4cH?An5bGB40VmW<)+1 z$I?T~&PNBky0~X%VHT6{EUw9PDLKviSJo0ohNPnd5(Sz|qpsaI5qHQTI(h03h)?Il z%a5&DaRU%amx-2>%j~-x@3(hau7-4iYmAWEm#>WWR52o8buru!_Eo$hS#=W3)F8Tp z7$=Wz9lI{LGqJB_wtZe5yyn^`-qMmx$FpGgoy>7RI%18jTJ_hqn~qRZ^+4-T-UPiS zsG0FS3C_?6sL!-p^2FIeD8}iATVs8BwjYfCU|fJzi+;psrn-qz!UN%Mk1Rp>@zBdv zY@Y;*yRiQQu%*76wiD7KftFkj2q1OF1qqu|6Ep&*>`^lr<@Z&_~>P%b!!y!UFJj#{tk zI6`=6Ff6V^gx_n9K+zqH4muljh~`FlKYKTkzbKM=BEg(0k zD4#hJGA&o0uf;kmAr08Z0xPL9f~9sYt5vgPK<1wv{{Y^eB(IwqMFo^9S~98(FKp&= zmq_zCRG&G<=Z;J0j_wZE4~8sdIeXHy>$rO+B(H>&-3*?yw35MQbNMM@P1js5=b}Y8 zue6CQfcj}OWVGut;P5)Wm91uQ(6O7BvICi5`h|wLx661v$M~E;^CTKoWH0Haa2R%4 zUGTKqldk=ODbuZ%lPdDsp=*x_Y~5_a9SsbC?rntXe#Gv4eG4HrI1~iwP?fl}D>-A; z*$dW(=4-+a=sLb83Y(B&C%QZLs0w&nBS3tWqXWvP1#pzI;KSlK7L^KQT7-CpfQtxU zaZs~d{*=f~)sM6g-ciZ0BE+i6ta7S1c1v@&;OD~x%)B&eSJTuhh^f1y2j;uk6{chd zNnbPSQw{0@g3gQhghiy@Yt7NK@l#s*;(a*lX5;JKkCU6YJcO)Uw;!Mz$iZwj6E~Rn z5vPr-?ouV+EwdEF3W#A%=2h3R-O^Px7Iq#!J3Wlx7Z{d%1V4L^y!=}>Fhh~9dl(Az zRj!IU$fnF{FOgXFk4qK4ZwG0rW1~p`4@mTGRH{oqKgExuT(Cx>&CWc(>_%`vT#W23 z9__)znmu`V`4=7av@%?jv|gLV6kB-A4yeqzl~;jCR{z-6aM4-FzPYV>TmS=;^tet1;#X97!vGL>tU; zeg{9tcs(eSz)Z|jY==yQ&%JnP4$*1m#xBLFrROjdn_04$br5V!2IMvk-zePWmy_3z z??bTkbhR1d!MnK+W(0a%IhSN zrD_nKmD7y2r~pCi>GVMEXPYpVx}7!)Tt2i!m7uyd+0 zsIL}c4+-rw6nVqunGlx52AG%1=G%?vwy?Udb7<6?1N}lG33JO7)OJqQht{AC6!9^D zy~5)xP_xBT$-9yd)BiGzQgbI{utxv>KssPZxmh#&c{anCW{Y&OxjRsUl6^GXqtmuK z<&o}$Xli|u?nzuD6UFL-FOS{kaHz6uZ8lc#LkuR*R&h-`;`^hm>P_v-^k2EKwbrmF z80+Drxm!dt$=P~y;EprPgdTaYw|z?1R_N-DU-b<77ST(2w(6S@;h8K49K-6np&g1Umt8rF}_8*gm&el04Fu z+`ie1nO%-j@kxIbbPN0{`g`L*D|y1O6yW4fQls2)0B&8fuI#I%~M$x6wW}F**nBFWaepvf< zoT1qz4C|2^XW?{qemfw-7Mn-L)-;CImdhVdGw}ioebJ=RJDO&}y?~(AT500ZRE_^I zI+M5zqc~UKYz@rlbPCeF%%BuF)sx=Vg_-r38DScAwKn}?rQ7S`OY93w>nPAtvni#U z#ZKRQ;b_wQQkSf-o)0#}c^CY{{wEd&B73%MU}*-o3C4@sh)W{8%qH`FufU;Dr`D!& zec$Oc;<4L-#oPUgqxf_EJ$AqxqUu`4xfVDbQ0vI2*aAIM6uDl>0w+X8DGuheOmnBK zSMPx_llpspyPP~S_XWh9dfa#?B}dfZ;YhtlMdZ78!TDt zS$)&nLs(!wY6D>$;Mp<6w(|?o)7Y`qBhzgBuZ7|9Rcr`ZZGumjLeJQ z@%ZyZ_>+3<=H{wE8|kqkj`RD4g;HdnwEHpMf_(H0NssgLf0d?0W~Ahb4Jtijmvg&r z2n!QkJVd#j=wY-c+lAk=+6qAOL@@V-17ITr-=bW_IDm$oS~PFm9E}HyCef3di|%F3 z+}$7I8+u1f>zkeLI9$Hvi-s+QyS0s@sn#>lhAovn;1rliZO|;Jx8H#tyazI+iT&~OLJHI%6^Qu{fv`&F(&+j$3EPiKN-r{*meynPV z>Y>kX(kJbD53a&==y_b3Tq!m?&&<3>j zt2Z^?ey$K{|Na1(7u*<3wk&)2y>P&qauBVY4HhO3#Q2=mocKzYh6kth^siT^SK?lo z!m`Mp66d@C8I8uI0v<^z9#GQ7?W0MMlJRVw@O|zTBLb%hsFm-c>E1OcY#E>1NNJs< zSR`kA7$!L9NKM4nMttU&0^x%PALhEI@B`CK?crLZL?XKdJT@0`Ryc}6yS$X?3WWR3 zI!7~3GMcOX`mF739Cb}be3N*g=OT=Yu(@kl@I_GW=L5=cb~t)Nds}77{Jm*hvSD5F ziY|PGuEPg(Jevhx!A@lucluD0e2c?`y8T=$w5wNYUy%?ojSpLhgtQ9qN0X^9tv}An za{b^1$D~j-d+epQ>y8#@)inGe5RAg`-wM;}PG8QJ(MVbq<*>z(*V!zvsBv-8VfebX zkgGH?n#VgXn8#>k8e#V(9Pz8$Us-ztv*p|hnT#Lb&P8Q|DB)a_c)8SZJ?ltam|5*1 zGWA(jJuN@9W2-YDi^A?ywccO9@&9e$}Rn4A362h!P zC)%+nIbY_=l%^^z_u>8?rt zKK-WqB-Hr+l%n>fZ1ke`U9=lNQ?9F(*E|wVO!?o_JGi zPWe^{P`-@MmyN&eT`>z;yDhQ*sc}rw<-8BH^&PHYIQG`ZdeSxY=5mkRUe$i%DJgJT zxwrU^jhex zj?hgKthp&)a1Sf3A-9gxmHaSJ%ksI=(9vySNOOEiWcj|TZ4a$d>Lmx<+GWqBDj~6wwFlz^u&S_ zcZ7|c-rD$jBOdbBv+xD*?;SQ)k!(x#wqSfwyte-fPD?1JYikZJzCi7WyP5Xu4i-2J zzq6{_;)#qE?(e!^Sx?nW#mlqK{B9s$9m!9@jNOPk2?$==Ac4PpO_-9^>l#k#|#G#92I&pi1Xi90am*ZxhsLe1_9?MmB=5DSMQzZMKstiD1!4ugs= zQ8dlPnys`|{Me;R&>YIk{>Vm22yvk~*;`h~0C zA;D=&#_YM=^|l4(ae-+zJ+(wU#}FTvx`kVo7Vplam_J!Q8S*zyjFS5l0U{r@Zz2NT z&M1O@YvInPVFtcH>2}$ybgnV>$y7h}7`cz?6CMe(@=2^%i+^H`RQ%s=ldB zU?ZoJ#{JuoM}+Z#jqe*>BxV}iqhM;cBiFTC=Fe2#!Ld;rngx9R1!;GEEFfWNLx`W# zaG*p&2%SaQU&JodGly|;%A=@g^H8Jr$uXl+q+CkvVzEiLxLuvsc%P=#h zrnlB>ZvJarTJTjso4#W*eI$R}Z0OpE{ul1&Rp}BZ=#dtny57=IDE*vjG*@kyG?W#Q z^S8>ynOJXr$;jqq@5BJGTS>{tuk8NF?-D{MXg?6-+3zm?4*<8lEjc-7?cHf52}HKF z=oWa(86OpW?M%YH=x9tb;o{nerX4y@ZeB$x>;*iMG5=I@j~ijS4z zQ$NlQPOTgs&b3Kb%cC#an0A@kvSes-mp@EmbIAVf&Ncbo*1Z23o7Asx z+-4Di78XQpiQ#Y=#(3IaJtsIySc>ya%lpe@xKD|WQ}YS+;|>&`rtf+<=QO)#tt-4k(25P^Xh!hzT!T=iOqsh>?Cjdd#}@v(t9y+e}q0Egz< zmfm)-O*8wzgjqaIp+nI_uV4A*!<7Uy$VyucpTS?!)TWMz&GJ@yo8-`JB|Ju3v4!Sv zPG}i_o7(DN08|s_>}!xdoailQ94if;BiwS)EvYfu?5XR>J!m;F(|Pwm#lnkq9M9b^ z@XgqwhfF*^#-azJ>NCm!(p_;&&=zT-?>hyr&9>6u!yG`O6?W_oj=7E&9>fjt%ami- zMT1zP<|MKUmGorF)t;{f%Ec~?`Zk3QapWN!?gA^WT{OKSTg7+iu^Q&6A{Nmpw7ohtE(0{8@)^u`qJ&cQL#*9 z?>@uw7jMer6?JD$G`>{7X$)o`Z8&zZhdV4B@8xjg3JZT|(NAlc7Xh8iCDlEst5^QQ zlI?G1|LoFF?ku>;jEF|Kl`UsN_5atdOpw{{f79U-Eabi5p*A?X|37l^Rb;o zzW#H%x})OmK%50v8lpu!Wy6p1&T2hrt6i8+XXwm}m$Baq}e z`ra`Y`Oe}bX!Q3fMHq_x)D&e(_>3@;^(mn&3&Pb8+KE0Pw| z+-dq@GN#yR$Dtn-Rl`1M8aa2Is^JxWzcjImAKySf;Y9%Rcu)4PUKWDFdRB2*>Rlzs zT}~i0EFRtZ-F0{^*tj_-2$F4yd)+RPD8l2#;wFT0HwATueEp^excKLb0jIb!9}}5O z)I9;wf&>nk5UW^lN3vUgw^S0YkIaRZcLq*_RVR6kn9DmwZhi0@6;2^Tm6P`2v^@09 zK>c)w1)PRPR8t!zF#nlRXMHS9T%wwM?wr)0IgwT2o=fPQnXl%x%d*2O{vq^SWr4NR zNtpZZTMfSfPw){D?Sy)jqx+E9kE%MK9U|?QBCwj7Y76|!A)fLTgz@WFOPs?Tf{BZV z?h(fv*bnXHLfbd*?G6VqwMu{|`bG@oYJzvsEQQTqz}Ias%~=p=KtdHCY0g!L2_F`i zH9d}Icq1Y0Yv~+aTYw$b+9K`jY*ZCwd!CZFnex~uT_HbI<#iPWGT0|VF%b?eJ!M0x0`RqmU7pvq@cEF^b z2cH`MAyYp%`Tp$H4cgGecg4*&M1JvoW{ejUG5>SGQyA5aY%%j^iP*R=mR5R!3T@9;paBIT_MxO5rOk@`u}^b;k_M;clI?E^$r0_>4DmMHJ2lG0xHNn zuzV&Ukw&}n#6v=^G{$D?Eo5}Xj+_1L$Mh^PO~u{LyT4xvu~)q7?E*5Y9HV&4QS)Bo zuw~uHfCT4lrW=F`O;wAdg}mvwo>OKVTD0!e1um!B)PhLP^R|8zN{9qTbrNLh^&Ik}YAE7vQ{#PmXm*p6OGxtBS(n`~iW^GV%^9S;uDT z+}m=v3<>4F7aY0aiRNdY>TE{*e^p4`zKV*tAQEMnv-0dQhw=R2`LxF|H^j zeR37Qa=CB3dqekWR(ShomG5+k1A2a50C=<&s8FIcergYHGGaPi?o$=Y$;2I$tAXDJ z^?Igq2nf<9>KjcS(s({Q>b%%16KTltK6!}VG@-I4NQx!|?j>aC8^+_)!8RwcQn>^D zZz}4=@Pva8(T+VeLr8ZKRQ8>>ns%=}4_1RsaXzm0UK*7@W?u8$dwGv)e5sk8@xpj@ z+%J|}Fzj_mlZKUu_O+8sH)`hEC2=1KscxY&!1)8xFBZWu`+^#l7Kf>u`cVoim89}` zb$>Reae)yd_-P$hEm~b?b<+0kmZb4kfVHt>i_2=(t@s}$x+l%1mFL-Nz;a@=zd_&4 ze));9Q4p>DwW2V1mAV@vs=fA4Psxcf7)xseUjJcAkgo6PlAgz>>ugsNc2 zN~0ABp(?Z$QAH}7Sv~fx)C*--BJDcJ*QZ%lZO{39whjBKckleG$ncwqKTL5mud^B( zF};Sr8}_X*+1OFM%cQGncBCFO9)5D7tNK7uT(-I2#Q7I^*ir2mq@1FA7gtMLkXfm{d+qT zt)s`%%rMuXf18P7T_+JoVL2j0Z_(cc4+tH8@Z}J=ZMA-AtLWfzG#@0>>ECEM(@gdL z4k7=%i2#Bin+`}w)j$=f!g_5(_h=qjhi?-Qur#SrpWpS)D=~p&*~#d8+3!#y92_6s z+u|<0{+>mt?b8}zE*o88zG`L2yTfKE>IJ0LIGv*`yOmuXPDni&oHJ~&+ew%sGFR3RCr~Sd}p_=&XSerBK;cVscCgq7b@yi zW1AlsyalmA?1(#W#Y2nk-2Vm-=bmx8fuyR|)!nnz&FwK7L=w`BknkVk$A1^QD&%@b z9#2;%4++-{+LAHw?G-MPQ()+NZ5L5{M;!+Z!NswoWTdt?DdB$Uy9l6(%pGT_5LhI* z(9`0MTQ^wkJ~K#a+m$mDalx5c`PBTYwBzH0$v=RcnULR_^{G}L+_aXPw$dwepu7@u zP9^Lr$3xhMn-#n2Tr{OM?sPH{+q&LuDEVxC7E`q(|h9D;Ldl)7z(UY#yug$K8 zcE?kQ6(G%r+ITNyMHT>W68oP0AhjRtYrG=E=O?_SnWzn#s!`M=?BeFHJ{r3* zaKTWH3~qX1vf$cHx4}2FB*>a}wiCy4afWH~CORf0K#DX>SG6huZ;(E(~z6;Kw ze5+D_ME%@!7=V_F2XMUk&4DNS+b_K%4v7z#eD@F!wiYcO_7&5HI977F_!kk*+Q3lC z{G<30rsH%z;3U1CJqp%FNo+kiQhWwUwcF;l%3OUIvLB~+1}$O(WHxxlt?`OWuxVWe zBPxzkjnQOhd#wK1s{P-)P?8{WX*hc|_v+dueWBRGSZ0_czdj3MI0Fu|9v`q8aAlchfCU}AaptZax1(>lIGFDc~ zf?r(`l5@v@xI3O!sD7mU@;dx9Pqgr&I#1@RbnZi`dgHD5vkD?flvpfTTHiiQ3ecC8aQeM=hw<4Lk=m>l(G!!_`J$Y*HYuynxgXYs+eC8=F0db`lxre& zRCRPac`?HCQ#n0cc$Y!WD&NC%2xn-ynx;aMw7I}^1_JH<7AgU!y-_FXQ>aOAp>eX!B1uDMlpv;HWDhvZ0I5?@#7yjHJu8B{pQ z7sdIIi1A{5m#@~nGuVdmAAqILT3XWU(`Pq*k#>Y8aU$c~fKd8vP`0P}m4+X)jUgrj^HU-BpEo{>6ifW;kx zv3Q6mSkAG})-&}gCqcUc<<6i?K}MX^o6#viE^9fp2da4Or@VAzn@MSR~&oQH$z`YU>)Y-5l)eZEJC=lLG!Q| zXKS#Xoz~-6_v56rcr}XBhf~eRQ+`UcJl@E(U2tRDeFEP}hvKKNrKj^ZH^|Snfin$4 zi&_1 zmvFngoG1CZg4L(XkD(Q_fe`sOfsN8Zbs?h;(mQ+q@$^j)WJHcj$321J>o<|_vaF2w znct=CxgZiPFgBZ`_gg+!KIxT6Ii%*)<=t_}mVO{yeA!$kGkS0MvdMhZKt*G9rcAcO zz34v-!47nxeI#$aH98S4%#>Rbu7CVo^^^HY?MI*v%iw zYIm&|Jf}0gSmul~tDy<{X}$6+uP@9r|KdxH?Hm|p>#2n&^rjpxS=1I2=DawuD2ou-(Y0{;8_6=8w^1*^^PRVrZ)0191YX7U=kpOPjhMC( zaj2DADthHhjWr#Dd0=O+U)lr%Gm^CI5G`0O7c;xO?B0|k^+jk4W`iPvJjCRrNF zJ}k3^?vNuf(MY-S>h%z7iQ-}hzuQ#r7T^b&>X5dIInWlI+huZu=^d`C!<^>CA5~}5 z!$}W6uqU}3Q*o{xT z=<1wRbHE?pFx43&Dxp3b@)!L2^j!T;#i*q=Fxntu!!!DujxCPPhe;z;kuwu!zA9%v zUCgHN9431z{r2zJo|sro@j-Wtl1x}L<#`cjto@}+a@qW}DSEDu_jsoX$p0qDTMJ;C z;w|6|)XfFB;3w!(uFF}CBI?*AW~8^y0d2U*_6Gdx z2}^S;@%A$vwoPqU$(`*@sMZSRJVI2TF1@7-NdrZ%G{UD-Y!~4U);vtu_zQiK-8^E z-aHRP_CcE`^^8vU6k|EEUY4-X6&4idqgAMkmvhzIBf-ih+$0(=fWF+M(?vutgD`B} zJi?vTUjbq0-c-=%rUAEO*<2J#8z9l>w(&HPqlU7m{AYD|JmZrOt7ZtiRp11?p3S4T zdlk9*ZanLz@Xjz_RP8-F46>Jx-xP$)p(f>o{puv}3>c971O>ZG44uv5b0fq#2 zt~gWnqZQ|Y1j}IRe*kQc{4Yj}b07TbxV*pkjhgZ6C(LdM{{Q`?e-`==;EQ~Sy*#}% zZpaIB0w*B=8To}f0qs9n696v<5)pub_k}aTJ&ce(wfJ17p?eafeX(KwpZo|&(tQ5_ z7RkT_2F`HQpuI(+r*$LQaty6(&S+7-$M0WHQCXY*_#;B*2#0y5OT08=C&gG5_VY>p z$5W&w-;t*{NcRnSjf^E@fG zvQR2j660{%2^1zyRKE&&>mK5Byj(R(p#ad7?r56%oqi4F1p3vY@L>ed(AUE%7;l(NE7q1JsIKCr~e@iJxn=q3^li+tx9uj2D10W&ZM-b6&4 zcuc7|@iqyM(v5HlC4pW@M$YXn+VHK1JK8Z*RRZh$-*mps`^Y9rb<7@Q!i{hmkMXcb zfIAXK8zF1PCVAh$5vR>I6`t}8R2nwopNqqOc%mzc*0*Rd|I9zd(TCWr zQ{cALL&e5HRp(|Ay_yrgi7M{4X&Xp#L<96!lZj}n14vZ{pm2v-zquDrq`YJ8}f3jH7vE{M&*bKkZ@9rVcG z+DY^~h2dkWgp%kiBkoXCzvS)dWx^ACoMGqz=LsF&H@s#7c53FYL<$nVkqjHzD6gX5HR6P}Z5 zah}JXDzI7vWxrz2=J5CD@YAE4Fp1f)` zvERHWSFf!f<&SbCCa%16u9^hxZ45RVDfBmm#}u)WhInI=%wmsbQ9mZ|5+Z~P(?1-j znTTjHCb!lgXL&!+vwk?QsyoGC)qM2&qg|{NI6+sKB@YTr>;r3@qn(jxuH0Z^Oaxvg z#o<#q3OAtDD=_~9@PA8{OUQ_fBD)>TPtTBuS4mPi(W`RD#7;a&PxChv9)(M>*D1 zfAjf6rHo<+Gni({9MzG1xlviV5{LLg3MC9coU{$+$XJu^95KNWp7{h6NWnbp`>**v ztE{8WEOf*|?NI^GaQolIsnRiAfP$1THDlJy*S2{~St+YZH7uP?OV#G0J6{luXx$nC=_0JPdA_YR@rXee@(I)_Io8O@cR3{>vtoCKFF?| zFQ{#*$rk{hYT@TYrbJG$9WjOWFu+_S&Lf1l7FTJn0u!OrXqanyB1BPKDo9ZeTR(fY zm+RjX)+e-ECixJER_?pZ?!JI88H#QUo8Zcm!okZLzT^OIb)`dD1eavzP zzI}(z%0@7Ygl+W98BIIhB%h&0lH1WNJA&GgAIcd%q8~_o`I~(XL5>q@==;Ki=Mxw_ z^);rSA&-tA{lh(pqa01&)ED9@z$HHN_?KZ6e6=Z&Rx)@Bd!h1Na38~$g&a_z&Bg$f zZ??t<%@fPbT6Uzx9}ghAakGyxMaMyr!*?C^llW0A=IA?>G6jswR(J$?Gg98o@*PsE z3d+lV&wZ%iiBgppGVzXNaDBFXtyLM_Oyc#?b6O$J3+DmGg2bqRb?`?|T5Z(WN-r;; z6E=MV#xMrjnJ|`n&M0p{ml{?sEn{IfQYCj5OJuEZo8%8MXh?`(paYg`T}{BCVy?_b zd;t01SxHU$iUj-D->8*^eAS-I+yukUi!8I^ZX;$5Xw*D>R4-Q$4q#)3(C)`qzLBWp zyod2qZBBX*1Y95P>5I^@2z!|_7pETk(8@}?Cus;b3Gw+CPWu1?v@9ComJ+E1GVDe2 zMXNyjPp`eAZ5e3=!VfJC5AZCPyicT8o+RkaIzNEi4Re$t73Xl&`Dn7WEl}{^NkpKO zsXgEhX1*JUqDKYbp*98(sWFmRw<0staxrOR0)RT>?Ap`cTHgbtkK00}7fIo-CZB(!~%>P8&*EF-65aJeyE<*qs`hXInyI*%(Y;8N%>;7<~GyDZ<9%aak zY{qNmMC^_7>AM@lpk+ zR%0ZSohZMi`tnyT3ipA*M(VR^^$9y&edNJ>*jAYWxYJv$3|aB!D=JFI^&~o+m7g(E z)QP^vvdP<$V#-#AMhgS%U#*je6<{A-wu5q_DL|IL0R*)+XvtTSkace?;?=Z zUI_TT1**S}s&aZzpmtmm_|8W>G^o?ZZ%FiU?U)hc)oTO`F5DaUr&H>zkj~ZCOMCK* zngHC(^jI@O_w+1QSEMFAiF?i0Dh#j^#nI1^$txLBT^_2kx`|TO%*DnG)O=8aiWWIu z)`8r3OHs^?Z>I3yV(XIrYwfFgCCWlyL5+oUSHVc8dcCM-tn z=Wy}r{(_Bc=r$)&m0<~o%*Z4_mdMvF2H}@=fhkdCB+QQ;T^4cIh@IK<>MaZBw`T=4 z!ezPTD+(fH6qpPYPc~|^smlJ361{laV~<*i2^+FZsrqkBQWkpTj>vc(<~g7smdRrZ zt@(vGS6BdYgcQgY^-U~bHTFejZk?TzZM+A}RjbupU{Ll<9&pP&1Tjx+*$O>MGs6| zIvUIcl4^IO;#FyuZ>e)geexpDiY`Q8%AX$r3sO}d>qcZ16_<${)Q^yPGsFlu*hBDO~bFGh4RC|;fB*rUZZ<= zChV&(YABslc&xl8L2!ehq@HmD3g629JWV<5t##_h(_hlj2VnzCe)#Ef8pJ|LD0bmX zzK376a}#_<*r-)N6v(Z{B>Rst!TZ`;OIwynr>2CwR8-8Ei_UWBD7mtD9S5Xu3f|dB98}NSs!6-HAP=<(( zw@P%`>bSm?dgih`P8D7fI%81^J!>9@-rBqhFWgKkmW1{q37h;eV@&_?)ystzi-Ci; zsjZSp^#j1+hfMN+#*8r`31llvtDwG0D%-o)x~jOFemr>g`UfBuk}o}$+_J$E|A@%@siM~hD84?XwEDd1vu=nRkiGsBfi|AW86jDmH2{z@rpX}FGF>z$ zfHEm!h=0vUhLg}9n5XO5;PX5^%06_M1hQ z+K}WzFYZnrGoKcVKA`AHSJD`;r(t?6ycL_jHXq$b(WOMXoRDO?fxRXd5&8$kjJgzhjIl~&zs3bWYN!Xx{bo|IDsystwxpu>YV3%} ziu#?c<^B4@CO|riP`6Ftlc^#S?p|yHy#dYov!Jvr+bgHyr-}AMVdfXumrt$O4wUZd z9{S^*a0DN+eYDKa>M^&?bB~!X~CZB=kOIJ4Gj|N{>3gw}c6NuY;wiVYPs7 zQ4vB>?^azwd1rFjb2$w%29|Q0QYnkajL|1pvGy}OS5(_|lGo`7K@{TS1(a-Ee+(f7{5MTT6hNHYOF)na!fUqXC*kkn1rgMU_PM5XO#r&U0nEU7hv);_3RVbK=L2yRAaDeHGy+OmAC*mnlbQU^Y8J3 zjx;h?)XsOG@8P@sFFg)($D8#$%nWMAK1^$(xa{XI4k$!F&KD(wicX}d+YQ1hNvZ#) ziz?73r7+80j#Ti|+w=8O9>!%LU=B9zqHdWZC~eRtjM3Bo#I^7t{(G8<9p4}D`! z(KWxKu#+44+XBQBLo}XgLvo*mkd9+8(YgF+B6QLG=zdDkfo%_^p1l^>rDC(iyvnkmPoT@ZX(;zJb%G2MG`Ft+T<@p0Kfbb!C0&)yyQ%!Qf$z8 zXBibDA>5+tMED541=~H-*~2-2P#&G%YXUU)g%8)0D*N%1q1Ssai4g8}=q4P>T=&7h zbb#O>MMr?#1rw5jtZ=mClm{Piqf{iqN~8C7nIa$`CMYvOEyh*sE;b;Q{T0@uhOXRz zwLk(zaG=z%*-P=eV%Sf!5&i=x_$X0axf#Pl^vabaN)8CcJhw_Ti_P2Z!}>*NP5H*{ zbA)U>J;<715=HE0hEZ*VMKo|MA6JYP=wPWmACHUq+HP2PP*~^oD)Kz_JAJ5N{UJpn zyIg2TpGN|a`AR#-bGK#Ugl!^@jUt1iz+&zHpzS??nhe5r(Jvt+v_R;ghhBxyJ0uW# z$3`b~q=SHfl!VZG2SFti6%gqvDq`rMpfnK>5fBwnq)3yZm;XQiIdjh3Gjs2pJ9lr{uNBB6*YXH{Cw~PHyvl zez)7EU`USm)YuThKVznire2M;svR_!Ch+k0WRPEa^_tJnGJV=wsrurP@1CLA5B4Cm zqq;vPIKXDXntkI!B#$6fXaDMYO8khagz2%tnJ>u69ltewf@R*&KFdqVi8KvLI{XI| zSsEWwQMD0B%_lUODcWpDV_me&5=tJU{xSygK<9f=7Xzl`GL`_AMrux(95JDf*H*v(Ar^Xi28-TO<0HouqwM0x6FgO>k4b4;m2~Y z&oV`Ev16arYRbZLjRf}eH`wTTBcoS<5rFmySC%F#jXrOB!etrW(v@qtjKTQ#&>L7V zhI=(P^p&%w*7XR1C~Ygun^B_<&;$@;!o&V$TN5c`ku&J$4Om4k{D zo8L07c4HVjKG-}$A$b<0H>KpQt;|wH|iFum+hDTId2q+?`z-tD;`lQ&c|O!pw~RaFfjS=Jr4z(?lq9=e9*ZdAhAv;2g}MnN<$eZK-6{S zB+IZxF{q!1DW66Q+^N!jmtnkr=LVN==k<@Q=>btk3!cFK2dV8Ib=ept@{CjbnAZ;to`4I zaR0ydBEre2FGm5!$32iFE>w>Uwo~!b9vOv{F(ZtRq~S4FVsJ70AsvYW48FrA+FgW_ zEbd>Q@1(xxdY=30j%m7rdRK zkmL!PwntVU|7gydzuE;F&4nwM&@D1V$K?Y$KbvPxn_Da`k4Me5PbSVo+9lpysuk|J zSmjKJ;$a#;WX~$=?WmCFNF``WR!m1R z_0b@66=>=^75RN39;o4GTe3{nX>?yJ4Jd`Crz||nD80Obw>_P9{MZ~qZtn}sT3vU} zM`UZY>FX&q$@Gp&LU*xtR8j8p(+%9x^{JwIOe)7JnIGRYexH`Ona^OS@1g2iofv5} z&u`=H7e86#u&3^?{qZAZGpg!Djd*oNik{nI(M&6S6ybRjhY2Xh1qrqAs%kJcenaT$ zGC%N>*thuZO>&MbKQLf0){`#IG8U0X(2!a|owkXM840`e^f)$*ao?Lfu+&M9Z+^bA z+xZ2|>)oyHHbB8ef~;bRLPA0JGdlD>5iiCGN8c;p8l^HSw_Yb*?N4J>53j*|Y`tDi z0$E#qYjGG65+E-BsX`QW-VcIQ{vOU3;4 z+qveeF+Rdm>ioBc_ud-a57YjC)k&F+OU6zR$-id|Q|(5jx6wv7_C)>xPxJF1%<1L* z8Z|l=yhkLU=-bDbY+%~3@Pw-EcS|g4)ZAmMWYI)zZ%oHR2UQsg;W_T}S!XstThyNu zyLfXd>2hBte&XF|aE0eOcveWbYTHjZQh*wxxWIVb1J|NdRyLl8AhGa0Gk$?pgFeVxrPF|ZI~HWX`T;%s@EI!#gM@pyh2~j?_KNc*DR=RDW*3&6}4CW$;xA3 zH?c*d_@C3kF-E>BM_3a$)2vnce5?zK1$Ayb(xH-L4j80|U;S$s&3=Q8NfA}ZmG!mP zRQiw5q1!*ef$P}n^>I8bA$S8;6y{l42X*_p>KmWL+wRq1$Yl6oK55=M&J0SM)Wc+J zF6K_Q|M1zrImMOPodi6HQk-AgW08_;5pj3fwzz7mYbQUtD{%Q%N>tJTSj57Blap$! zKapwoeslZVH=lX2dCYfk*7L8sx5LyIb5;=C1lE;F{>=46WImrH+*5=qn+<0jh*Bj4`6{lUmw zo92@I52*hK&}6L8>H}QjdwN$4lS}y_wIa3y99m&Vyh#j&wQ>3trKzmbh0A^6NX0}M zMMfIXc~a=*XwkOYoPT^mT5R_vW=Wg(a|};yK1_Xs$U8qu&7I!_iKWx+cLn2^XtbBg(Da4>wrtz*|{+`DT*{la` zj@$gHlQ7sc22|K$=AX#eA(PLhu2v*z05|o)klmIDZZl`4NmxS5b21|=4;KGyoAaStju z$DebD9KAhqnDHrhRH$J{C_)ACMp##bU8dK<#96S>A8b*_8;8dd*+x6(SK`0LV7ko9 zDb-Z^w0~A(R!Slp<`j&`D2mx!s+|Up#SXHc>5qYh(}OL8+8>N-AMDKrMr^IL2(;%LM*}<3&^T^W4^lX-(n-jc9SrI;aBnjl^UtL=_Kll4WbLC3xAM7)hVpBEsDSNJ!A~F;LYD1)<~g}0w_8W7 zTaz`|Y}I~>vAvzN>e6f=BO>QQ9yX0z_|sA+2N%O;?YbT=N@SF5%G{fDlS%D*Y86py zN@Al2lPvFbGNFPixII0hs`|$=lfHcyE1k`XmoiL_A8MW!+GS)7(jMA}XrWNTO`aP< zK5`G-7|W-}&#H_D3oDhRnh(0E);=_FFMLG|Ra}1BHCXq0nW3b~(M$@Czx{#rflf={ zi$uZ#;^rx9?}))$hBc_?1AXw8Jae97*F!`Z4u_M+#8p)ON6&&F)C%V zlqAj3VmAstD%YHuH_;vRneOM6%#y6vF@RIH%XZhFQqXPoEJRuHI+V}Lo|V8Gno#&l z0)QCLYMdGDO={%2L1}35*$EGjVD6UZa-t_ydw*h~mk1+CY3_;2pRnN!1QV2x5UN|f`R-#$DJKaQ zgb=HK7X+TnzhDoxpT>K&|Mr%YT4U^v15YZaRxY>evsjlaW<5I8M-dIsMjo`l7;Q6{ zFh?D1XAJe#3xDnQ^K21@6QNr%(hptgh2dJ6`6p<)(uT`9v7<~l$scuMgVjSeOD}12 znd-J9AOB*)zME(fN8j$fPCNQv57ZsPbtboG^%w%TrJH)vH`cQMd@#RiZfiEW_L-$C zn8||w+1_z%H%GUEwP9^a?-xGppEd&SZlp%uaH`*o68^%|mc9J@U!>;j(vLI9rSILLNl2(W zOJye|N&X&wjTT4!l6uWU9P|XtQCph(?8~j|SBeY5gBMx8+T-DzP-t6y2lTe-E@zj- zk#U4uPTnja98IjdFwkF*8NPyVqdanHQrRLSDPSt*;93VBBKOY`5SAIguJK=SnfdP8bjrA3FDO`PqYHFs~RTvR| zL;M6WBm?u>IV_VpNIJIPo+}CdXecS~^}6{=ES>dzUfsia=mp?M>dGDb8bd<3@fsBi zwC|6(@X;JDajkU^qKw^(DtBqV$MC;g;r^$aUz+MaK*g20X2kews4RP!>hPSNl=IRb z_a9HYI4)>J_LUjb)N+lFU7HSY<7(C)>sKGC6=l{>Jm2^wIqJ5KB0^duDXaSDOrMG7 z5QN_23bmA+{F{Vo-8WB~G|tG01DX;``cJL0&B^PuLXV$B#f~njNyqhDVwP<{(-^yG zf{CL;GRBM*L3cs_PT_LZg1oWy=sI(;Q%BKx;yH^1^-;{>sOFmc5!t_^v!+&f#UzoM z@j`UG&I-WPN!?~whZrEFYZ>M|f&v1dlP=Lk+Zat>(_qc<@V3#vBEBUf=u$9UB~AbA zj%b6iAE#9@on8RGATcCZ0z&ZnCu@cwGWZ2;jE$85U9Md#lzDLi__*K4Z=pofIaS8BT+w`ih>clI8-RR z&7Aw0AS*05+&S#Ed%#prP&;fG(s)%Au>m%C2OU__=kvz*gTy2Svg*75x)c%S66LPE)tb7t7jQzpwxF=iu8!_e|w3kYN;@FFMAsm_3 zQ0m_0K#C@XUIg^6l5^K<8%08?Gd0q*jbbI}4|jh#u|YGByen&x^S+RAAuuah)PpjV z2(QjdEiz>j%pIOSdCGq?3EJg#e^G4y`<5$@JwrX!C0J1vMAH1YC(m3#uo+CFK$Umg zlcYuFE(RUn?L?Qcb?z=|CDQslwKyHic>6d&D`mgt7n9UEUN>rRsZ0JZ<<9ME*GmDPL?W(Ez|H$TivactCSTm z9Qh_fy4+pOXaAB+EgE;5o3cdAa=EO1Rhhsyo7t|QxBkKy%lK3N@#D)ztnnI5x*PT^ z0Yaa(-9z%G8L#Xp^2#u|RcXF?YAZC$AD}x@Isi!loHAHK{div8m@dR)Ux!x?x;W9Pb2;#+Q#Uqfj zxv|o@1AQgs2`bxS#)<2Bs+!Vmun>N}K6PH^S4njyX~7xAfirtNMfWT zY>TOA)mImzew|C*baEIG;AU4AONs3B(+YEoOU;6!Rg-56^f_D#(P?kT5%*@e8HAqP zePDS+dMM}!|Gdg{XKCPo`)+q$mX-n>$?BFyI~g$pf;|eI2embNNv2!KUU{`;9MZi+ zwJ^A*?tUw@F>2M0#IuNs1po=9K*1zG|~9 zPf3b9PgTG{03+Q+XKN-H+x=hd5+a$cY8Pn489jA|@dXb=-<+6$B6 z{KT()Yjaw)dQG>~MB~K){3L3^bdPR*os~eo&Uh(AEGue+s57PW+%_ozYnx1D<5Xm8 zsCep5M_RwOo%}@euvJL~mHCC>^01_bqBWaPD3|Hv6I$h2-oG}5^~yikOHR>xlokd5 zj=owx)jL%!Z!*PpuG5zX0aJ|a_R5w$Clyir(iq#O55L;KFZ{9u2xh9Uw9TrpvLU3O5XuA-;N^RALPulWiEDnlZclDKZ zTvtGPl3tkxDToa)Fo!?6NO%7YM_9Dt1Gu3)WDg(rz%bEwSG{x-e1TfgSzYi8S-vsx z@1@19zZqxd1{e_ibl;=0L4`--gVcUudmSKYU^cuA=PSlGO=&X9awa^g0+yHOo;q>ofzNr~G0`2!JoHYxg6huH4ea+^(=v7s z&5vx&NMkh|{|6`?FYw!@TVsQvaviEF$v%G#UCttewJ_hN`eBV@RnCBjISajybCuyc zp`U8~OPxW|%o7SI8NL#a-<2Lo(5Q?a;Hn_xxAp4=^35~@i&Tpet;mQ`>fz&?=M>R1IQYcV!F-9{bdbUqmOi(E=*@?)X z*pl6=X)uV<$Ile%vx;9Gx9IfNR|b18r?zaaOriy;An1?aWaMeW3OYL8$Ag6hY^P1e z7hRyodDMg1&)qPTQh!>J<%ZylQ5^A@~fzlk#a(*Qo z-GrM7s;vE~D}}^)-QD_+xA>}}NFKL5Kw-bnE6|e?|E#Tz7a9H2fW~A38B@bd{qv}5 zA@dswR3-6|DiK6ZtxpOA(4wtl5yUH;xDe&6wTxL5uV8;ypl@2t0fA{L1kj*we>
p#!`C&CiwoIq@DP;c;7993gqvwo4kxksHF()$jZ)$ir?zO;?a>bqW3-Df4|v zxoA@v^)Q_k2ZOPjn4SR=IWQjF55F zoB*G3#C6PAx}14*Q2ZaVAu-Rrf7BBpo7Mqj@TiHR6suA*cq5(Vp<9_Ykm-24$)HFP z+R2NaZNDDx{U3jsOfBsjv8i}3;u`x{4{VvIEM~)#N-4;!vhGlNkb7|lw% z=R3j=+mEySHVN%M?32h8_tlSd+&nnF%$Dv@=HeG33MMVjN>;OFdGu3Olc7CCWTxW9 z)N}kJJ?SVFI_sOBD@XxJkQ!&UZKGR`Rjls!o!(0L-Kmu_cvapo&oYyGb!Pd2$A@9d z?}+SYnXg5Ee!F1xYvWytu?8$e{|J5l3DGi!4#-^7ZT`v9t>K#j*#B4|1vsBtLtEB| zze6{zO`xEa$#gkkmHXTgEn(y@N+c zmREVOs%8`DfGQy*9P59<^pp)0Z`E{I_(Q-hhC2>M&x90w`CMyZE#O|rkF&MEy!=kq zjq;!it$E{aASv~JrkV;nu@Ut;<26pUd1lWo=H&O~*8DpWN9qXggB9B6_=(S5+(pCr zbbRY&n*O=gVo6hL>`~QI7unmMO8MVv*dd2C&%@@6BQYs{#Om7EuG4@_=-5s@nFwr>Txj z!I++4BfLP&J?#=TT|FpzTX5TKeFUNBCset*hvKb=tmd`1!awK5etd+#k?RWLYQ=D9 z#GUB>&Bc+*YScdB_n2$_G+&_%r6-?5l`g~e!}jjpo8$%13(9=UznU3K99Nh-EeCa9 zLUsEvI{Mk)B){%w>4UL%Y7k{5Jmvk$0v2Kd=*x7;$%`x9Nm!MnR2qn>5`2?>Dcqi5 zpPc!{CX*?Zugo|E`^f!!b^*OB)6$Jg#Z#PjJugpM1%!#4d+Xq3n;$KGn}DY|6ZQhNE=rgHiHC_@S(n; zET-nHSN$_zcdnYs>xnqND^C5hv>(w|8NxabiW^aAO4c{3tpdzMFD6?fX=6St`nKSY zA1%Q=2xN}?WBq|e)RY}-qj6i|jpYs@tB7Z}YE`RwXtVnL?+=gd3BRNF7Ae=$*ncN~ zGqzsYWijuqxlUe1Ka7!P2TI$vw9C}qnIeP}t(6eVfU^8Rpl0u)07TW<2I(&g()y)9 zf52rn18Xq2rzIh2ZpCA5zv~uqyi76a<0%W2@kyd+=C7x1*E;;PC8j@ho~v6%NNn}I zs}r{!Ma4AsKG+C5M413z#X6sFQp5eP$lWN_jPy$jV|%nuw+W%r5YYQE%nrT?K|(6+t+hK->MHOw`*df_Y$3nO8>n_ZHZ}0#fNR>_Qi})OUVJX zd&+9WiD&IjZ_nEg{#!RDPKN-d-T&_+{;T&t2LJ!mgy~V0PWhizFp0jx8!2feWfJPZ%|A2FT9T*7ncd~8)G<(#NoLSY+9Q7LB( zh?o^qyz+o+xg>RrEMI$@CXS6WE0%?+mdXh4yKjuWFhaM&{Hz5IQ^kZ`!%`!YGe15! zJZP?U`orT(d!AM*kqzI!f9Ulnlg+N?STb7W)Xl( zmWs?!#1`1lefjiB%% z?eH8ye0RgC6Ev|5&W5yrtqFgN%*PTWzoDamm@eY?cPZhg<7IElZm9lTN?rfq1*GZf z*xIArFDTNIW8M9o7>9Nt=1aVC$m9o8X@jfKXYXFj!233jUVV=K9Yq@MQw`_*Id{gRysZ355S|p8D?0}y4R6Htzpi;SE6KN)LJRAKk8R1AjGN1e zCF%(VTI8wH2{6=32%;SJBhuXUq*bS8?9=(+{od*u3&7UC%n^dqvhDenPC_mSSAPyU|Ae#sK8Wq|YN7n5^lj89eWW9u4;@cFFPQVJohx z#vU%(s-o1`COv2$iKym!Z%#AWJRz1{S(7Dj;PvRhsd*x|OsHS_3y3lUsgXOvWip9*dW?Y-eZvAo8P99U zcyfJvt~24|~_eiYXStz1ktzdl@lsdUq{V z(x`Y(r0SV!CJJ%@T7!j-q{|s!PEK8oJ*fjLNvAY|=>Tn&<2aI$Q;})|f=Fg=k-(Q? zu=;dBD~-S9&bC?VfHz_(8~cp!%La?>jEA(#u3QXD@4!6|pe$j|QdrVO0Q8fnmINLC@SGo!mTFC7G(~5T&3BLI`g1a@yQNw7C+yRGFWXHG z9JyMmt?kg;t$Z3U{414S0{`=dd`(H3c=w2pk~KcKO^$3JwGCk^JhpQ0-_?+0~J^n8${h!tb;hD2~5ht`4?JF)>gtNYei{59#%&YAFSO7b0nDPj2< zV|f|x?>ed1{xY3r$sX{<*RB6@>~FkxCFlfugkzc&%%$*`zCXbvnM{khEG~`*UIzyW zcp?c+jmcHK0ym=i!-(`M=v-6A4?2!Rw&jFZxnwm zO&yG@)fbd=gUI3NWPHv{HxgyHV1(uJ1HMP0+NdXqUzEM2X zD8|RqyFx?%{gs&htRkU_E2H3w489dy>kf<(o@mp0fxB+B|MlOLe0>`lt$v)Bk(t~>?V)ga_lM8G zSiQmF=dfG@OE0M-We~_FAA5*X#KQlDH zeXUC#s3=TP?dM8&4HoVI)SOlBc`8mq?1bKTz*h#I$gHubN}MTg9{uQI`wR%_%Tg5# zw0~+z`}szQr19!n)l#3r!!-RVtly)Gl$kOHnX+9)ZMTdh3nn|Yo`z2;?$tS>A--0x zwmpum{cO}Ir5 z*0mf@jf)D4o!2!dRo_~Aqk8U_cXE^B&K)dJ7GLd7154<}BZ%jRRtwq`KgUsdByZYM zz5}VZIGiys?+cAAX!kHXeuk~7imM4y;Va3D^BvLX+WW@G_M9vLQ7Eb(sM|9S`v(+a ze2$Q|gvJ=B$gmm+eX2mcHM9AI_kG>`fV_kW)__pH(ybpf?3@XsDwX-kx|Vx@!Evrh z2~?L@86BE1E#Bpqmzgk@U9Wg95@_Kz`btW{b1+A~M3CG|Po{y}O{I{_hTvt`&Tq6q z)QmCj*Zho?wrEN!&&;O(0Vvq%A)%BrL!LIA4c}{u+~$ap2U7n=`;Zl%5vkXeoRj-C zTzDw=-Fa5|D=>rUPK|0a3?J`4A>eE0vH$hqu>XW_^$U&$XWrCLu?dm7jH9oraVH0XIh-|$ zG8xJnbDz^m_)Jf8I+R}AxkHur`9Q2E5!%6d*9DbbZ!b)CCTnf@N-{3C3%I^}oRaNC z!`@b39Jik_u8+?-TnD`7_1*`$)j`2h13*SDDh*X!_k&Z|9AQ-J6SxCm>GW(8EceV{ zh(0s*-x|LtA@&N5n~`55jyj8idK)e4HLTS&1m|mSt;;<{mN+M`FX#Tia#5s8r^8UF zCkpU)DSyR4odNOVcHDLlSVLKNos8b^L2BP9O%WlHZF(zIQ1Pu6<~5fppkw9&re5Ymd(++UtGmuBK$<7@zz2@NoFj z-U>hpO4%@9(q=`qLezU^ri5&7HZLJx==jbJtEoJHJwxuCPejg(#S+)_?0>!jSzqlPO z??^JTaY&>XO}ae-Sn15M%xE`T&3u|{&?xf_B2*$}v7r0YR19VOFFzQNp+RLQf|w$b zkR7NxytfYcl9ZDBF77>?=SnJ(QD@{8RC@j&itZ-k3-&qIwu@!_trC$W4PrP)U^62BmI)YT(1g=S zS}T*0HbWaoIIxyRM!aH3n1qhxt0mdJc8iie!BaDx>gd)g;Ahy5_>HJuyA&wUgk>-{ z86&+HqKh)`<760u-x`$Do=K;jU3-@O)wUE820?G^@7bE-E%*M|0t)1 zGmI6z?F^n9y5~Cn|54u5V}R6|WDIGru#n;YNDTa8R=(3X6uP^5eCUC(W>XwZ(yOBS z8g-$kUjdi(G4R#T2i3sRgrRz0@^9S^bm9wKu*y|BGGT54LLs2jHO>%XRM&9KX6FHm z+;y-Xb&~b*)dx_?++VQW?MH?!7?A=6T$@$GsjTjQJ1WP?y;(mCeY-Zy=R;qO+WSqn z7XQb7dyy=x(0Hk($0e(1H&F&`W+;$PYA_rtqYpNQc661+M!5G ztVdtlMRDAbpmmfQnTk*)V1XA@-$IE}tc$rY7lnEV^} zT+bawdCc$G_^*RD@+=jp8(maAxpt#vTnh!i#H$sEMKQ@=jppXcdJsUBa=Z?k6sPxMz|Wg;mT9=X+3*qZ z)t~M@G4u-vi9jmR>A78ROy~ZUd~_)=5E~x9n8IIj;QH*q(oHzDE_Yi8xZ=hYqy-cA zMA-4C<+NBs#a(rzD0Pf!avUaC2Bbxst&$j+5$cO%$6=Ypg`t?CRXVBp{QF%kDMxJ0Wagb0>nK zQyPQ3{jp7IMFcbwVfn`!F~*P=(0Y_aUa~G9XFn(@7Nx-SS{FGv9((0G(HNLx{rKG+ zmBfUW3qFuA4yL-H)r>9u=7MCr-%V--B%k1^l$IO|p&za*cA0V?)2Vi~;|uqKOhr)t z9+3s5nmMlTssS-9I5FjMp%s++vL(wnIjcEve-cciXmXe(`Odoe1io2bCTG_>&BZ|_ ze3Ya<49%pIeKSMPgUKZ`Z6K0hFmfVwAG@##Q?BoVwO;PiKafN*1wP>GQOevHz4rYO zjmoSTJp*z~x-J<*O9qW};6NGjP&))f`B>0Vq(I|(#CSz$@im)6%jEPy z3l4dD{R&((VczR)0%QEd8RCfkCLhg-loI2yAM-LT38y%XPwi74Z%geEV@O=6;N}yv zNnUyFb*_d8^aCU4y7L6HP7=D0;eXUarFZ#zRVhff|EYT7vQ;Tn1ysjQ|6|zyt-xTolB>-7ow15>(# zj6rP~`}}My+)BYlU+^vlbKeulL#%y1B0o{W*Q-wpsX*lt319Rc?~i=2lfYvr2wi^s z{?XbmQ6h*1ER(&UR3RvlUt};i_L<(7p;wztu4TnBS!Wdsk3d;wpgYI^Ah1=bvamjW z6-%X_pA$YEG4^ZZoW9_tdxbyfhZUGPOp-;_pTb7MC^5cOi7p?{P3s|%Q76&%mJfV# z(OY@#G=_=eNCsW0So;0WhrLc8ZIa{29C3)~MjL5OuKpFMsNn$>lkbI-&rG>Vs5kUJaEHW6!UrHNMw= zvaPxo&mdGq`rLjdgH5Sgz1dY>j-!y~h7_IdiFT=Ps|B+RWU}9y3b5!5bj)H@1emQ)nHUo$95j zaVrGnev{cJO5G^RB4Kt$fd0`R(TqFyHm)SneJlLL?c|)*((+E{x@ z1`?4PNKuBu8!)I>*cyBHEp(plmLz4DIw*mehg&XNg~puD z&0LwluSbYMy69opg5&P}P*@VV4ky^OuJQNTiDSZjU%pve`UkXTGVY~NMZf48xke4_ zy?e;rJNv=?E3+hxB11UBNx2|vdC|pOV>1ZtbfkCg`!dOER9n)GgF)u_=EGT@X9+2V z#+QvI(jo1Hvr5Olo)89G)VsV|l-|cuzS@;4F%ljfal`J5wa;X1rIFX#KxC@HO*4fFnt=uwFGi zzJN|8zPm*|;0aT~!uTFxgPt_N(~rIXH#VM?olHA)Z(*7qF9J-h6MkN@a5HS;F!5PG z*m>HE{if$QZ2!A4{&pzU9(iKgH7^Wn-{A6neb8lYhfTynXAhWvk?v=gT4Nmy)>i|mV1SU-(``YxS0mA-B~<})Bn-q^_J*YY zF*^Xgb6t}^vdG+Ey8C@-_0M0Bc?%FK1aiwnkOO)kSNgBT^-rn(nj{V}O_X7K1YxRTbztUoQgo=-3Hk>+=e|s9h*Sv9JL)sX>HDVPK=zK=I)zAC}!@epR5%N7Fv9 z)#w3XjoUSOJm?0gh83(`gyR)@M1HZA8&#!epEBoITt+3R{RWJKt+NdXeyDOX%ga`+ zdD+YiUuiZ{8_YL3o_O*yPhrd6-C{a=6Ryw_-it zc8z@=uI5lo^@;j3Z|7#Pl0{o4mtVww^1U#ji!^%GYHcaf3e1U!Fw8MVe)ddHA<2T} zaq2m~zJ&Uz!`-%YHe5LFoT%SGH>rd&1zM&HsU+UlpIyeJUN}0f)TJ~hzaXRS z#vd1S!>;{OBuqWgv?44-weZ`E&dy&OsZ?zPk02@yt=?_NcAvZ3Tcv)|xf*bEr;E%E z{w}83??MA!FfjI<6u~1z@yXd%QG@9sMi`+vbT;&a-uI@wbKu)vwl%;Rd$)@sLoRO6 zZ1m=keZ7f||8uwFJ{9wX`2|Lnw=6jLSKpMY&SE`t<#%~I2#K=f%**Jh0nv6WEjd+C zTam6F-G^o9Akzz_fmJ%TCfO|2u$M11pnnyflH|Mm%o^Dnr*-A1e;_z)&LuLvvNT_F0$1MF?a0qM zUau!okS-Q4tB8hmjNR?V2L{XKBj41S&8-(V__&T~|gWRt)Lp<&iKw5FcrKj6l!s$v+^O>ZJ&6p2XiOU=5zzy)$AB z=a3CYlCfr2wjX;vnrwuB=ba`3D*j#eUcAB5)xI*bg_16ev91?tbZt;X$#XzCQ{S&j z=_j$4pL^x(2{Jbl)1nJFZvJ82t&2s|^DTcdG`8P5L()0~bW%Oyqr~An4&E5Yop%1` z@7iI}L#pw?Shkt@FdTn6`)!R;+Ekl#ry5VtMnX5Bk_6#PY!?$*)xkBAmF1?al7xWS z%MECYu}>xve)Lo(_sGhEmA?Esb1!Seq#GGtna^ED$qB=~?m%p9vhX^ZW_6JZ!(@VQfrO(3KOG40` zc=cUR2wh5<%~dzPsd}Z-I=f|X)llU~fp~52l2ZxXzl{B>t8nW!pCx|%qMXi z{j%P$WLMVeYaBi3DC?79tWWHGAn>8Cw-YfuOvj#+hE(&k?-kGogdqm++z|oy7YrCRBWt$*6uGryL;zHI%)#_#F&bV-%$UO-D&=DiWw^ zxbL=zwdibIeN9^P)gi;pXA}VG3tIHGfzU76?zw0@earX1ech7qfYztP5^!SPT<2Rq zTd(x=h=2EUKWrByLTJHq&h+ zFQX)P(X}l_9-Cf@Dy)Mp8B7i%E}29*3vB>cd_Vb~yHCO&c=k9Wm*icj`)#+XyW_8C ziHY1mNWR%aBWZW?*snXTd&{00#@qf;IC0t}@HBrR*IQwax+Jm4Oc2mR5uV{RS~f&W zGt!t59U2485W0gcK@#_trr%P2una=SbB9PNf`^H}F?q@o5nJ@gP$6z0G{KI5{N%?= z$9-W449zg9XoXpW>7f#gWylCvb=8zj89`SVd%XS4P z!4k&ho$PK2$4P#b0KOUU0B=dwj+{`-}z1D*{%}all!t#dpaUqqq*L0^f zGw9N#h(|>KE@d4vP^4NmR}}%_H%%6&qq14pMN5dwb!Ac`kb#YU z5PWi^!PtE8W`PU>tC3eXmO4?%6u9BN!R=TX%HqX`Ve$9lCQ23r^&zYm78D0f=+Q9< zX2{*P4R3vk=W~kWT$A69M}8zYxc5lCD#_|*-*_J$L_OWpVQ|^!fJQv;)RR4qeBqP- z%+WV?hf%5eW%*X< zxBrW{vkr>u>H7QtGw9$hK?VkQ_rTyVxVyW%C%6;b-G|^ZK>`GK2p&90AXpM0kOTt! zmgm{3x1O!qt=)S6*;`%Jr@C+5y4Bsc>)h_spKmhWy9Sl)#5a>>L=VN~YRjFbf>9T| z?|CdFx)G;Sj^35?eu>y;j6<27Yirt&P7NI)_|dFsc1-hA{95BI)_5}c zvyc=Er70zEx!Kos$=4&o*?^9$mTHMt!=h3Jt`U6ZxD=h|Ku5C&##?} zKh0jHD+AO#`^<>);(2~xW6EP1DXCkb7e<1&yrT|IzF{aO35JM|XoBcFF+H|YKyl_A zPA0j-b1zJm2N}pPQ72;{*vbTvBE&ZD3dI(d?R*%deqcSq?=R`w>CdkZO~8_DR4xG(_o?mOiJ;&(**GvgK%EUzi6W9a=`0^>so#+eZS{V?6o> zq8e}jwEgyjDvbSPDH_~oM9K&bkBs2q&3j}*VSI>{okEubB~k<$Xm{7I{tEl67JMAh zN=KbgVU@m|m9~h=d)=tsmn?FW*?L0WQ8D+p_~E_x%0$==d*3@k0U~ zl@B_2F{m8;yn+6z6NO2F^63>#1ihJVdHmY*!Lhz z{%VmF%Aq5o3z6t06%k(Y06diMdvV$}fRtq;KeFEWuFLu^l#z?#X~bh3I*35a%irmW z#`+XEXniQXiwD1xy>!K@dj55broSa%1tIL12g`w|(RIvY*%@+ZCM9E{RJA>sQsSd* z^vshZPEbA+AGPXFDdy0I!pNzrPOj2)#)!HUCp`Zjs92~@b8D?lXgYX_M{4}Mf$?); zG?#V&oJ?~Dt^;Z^8p9;_Z^Zb-dttSVf+AC})&b^0`k%9y zdorLC%vnc9OC!4fyBd4X7x4SPvbnfXBH{|SX}7haP$mHPw(Q#XLp5L8(gBOAV`g=2 zJU91mV6mV;^k0!Zk)Hr}ovllF!=UOdufY^1K8^rF#u6(`q|NR*3Js`Mh;zhtAn@-% zy0Q~fYv;Fo^*ey7vc8RI_GT>X=Rr$+f$+}TQ}VO-$& z4x>t1U4#?;i*dME?8e<@fAM%Z-tu#Y^7X+j&BReje+%#pDy}@LT?2KL*}j3J^3hgE zj^!s?|K^8k?`12k&#FI&hL@L?4AO14uFL<$*dQxU=_GucB8wX!m()($94;};M4{xQ zY7*a!2Ed>|C9~|K+NUJTI8`**&av|BxQ$A`S$BibDP#`{r;FEHBwjdw-wPmFe{MFD^{r>@YkkCgKIPa-`V$OP;odj+>`~x63Jp~)e zM&{`08@v>X`Mu|kx{qzqP}Uk*O))Tfn^40-1BhQ37bbr(D?9aPeb>KyGX^>@RYQp|XIY2#mXH`f{4M z;S_m!DtJE23-QrPQKj9mnV-`3cV+w!9}8YN&VCR$wk-I`GkceJ{C}^Q$f8j=ftFHW zxxLH>S1^=c({8_0imOUM>d;UPcDtHCe>r{xhFK6qCw)8v_uB$#=d-ljWpw zAk4sk*|}E^i>}w3NDlx25i)C>^c%U%6h~-QPE1_o@lpPrL{xD`5r_i8eV7bBD;CXG zac+;4teI`xD6lUc7ia~l@r%!eC0C40cmhswnZ@G-mR8Gy-jIA+; zd>+pb!#-v=Ru!1Up7KLNxT{4OCsqY~xhg(BVOU#7#G&M95op{n0nRs5c{sVwEUkWB0w>6HZf^^oO*YeW( zsq@kK?&tvk0xEoFtyyNvKYKKvQi7`LpP0O&L;g_x0~i-QeDmK*`4?UTOI=#sc(r?E z;@I)|lyeCRRDw>6Rf?(q?F?0Y-vEBL$lXT zgzRif>ToEB>y$42l1t^>{|t|GzD&u_tlRD{~nJpU(Yk_AK!YJ1_cb_F4S+ z?vu!T&w&5`xsVNQ=#h}FRxlNS(K><*ymw4Dc~ggi`LnIk{aoJe#_mcd-9&t1{}{0{ zN5m9oUaxP_YMtL7s89a+K}(Y@LdlY{hJ~scAnFHl14;X7*khyGvB^~ZsVFOtPBO(V zDqU}=4xfA?xkm8H2o}9rNvpyQ?jT>YenOP!PUUS%#^(@%Upkx_fd^Jws;xi&Sjpav z{1j~xNhP0x1P}eP}I^&$w7Rujj5Gh&HrdbqGvnGnTVe>-F zuo%~~Od6Cb-R>!dV+^y}KoOJ)VyRma>8ZjY1+tHsJu( zPT~TC#u=k02qnkGV(J~4`N6BA%%nR|vN((@^KZB>`o|5p1WC5p2Nm4Idhirwxw-Q- zx}t;ci?`pp@cBd26qP&)*Yo&{9=1#RY`(oSq)2>5aps5?wyJ{3mQN2RzJ(~p)ZbeE zmA!$s;{>N6mcGufDR<-faZ(uUZ8H97h z_E_XD_~*jSTvDK z%BCfg=(lRm^`iNEV7m;nhYH%c*j5jXqL7v@zD2%7pNSg`N+E+-rF^4jcAPPiLrFIS>=kwE9-HdGs-=!26()_#%&&(}_a@rAqg;$%+fFnXxAwIKp+l z-YZ}Zc<)t^&r7GN->T}afMYTsgq#|`q*)-wndv}go)BJz6s0100=u(H#1+;^s(e?c z_PoOr@Q-zOn{wrshXxNBQxHerNtAE$5R0JEi5AQ;ywJ2TAl86BW6E}%MI+fMA|EMV4?TaE$2wjPhabF4 zFJ7fPQVA2Orvwn_KFH%$<64)D>1e_@KSmRpd|zymbiP9nW!+QjWLRLDzc-_bBx8Y69Xr|6p}}Ky3XX*$>khmwk=L7*#{TO z6ko=skTLF~5KYQT?;Z4m);TG((gvO1j49pZ%3V%V(RBZiDb#y|;}M-xg1~-7@xGwp zqiE;&h*xpxEod-y?FEs1anYZ9&v*Z7zrSwguhwX(e5@M-Dq{+fU5m_*l|jcYt7Lb1T|Xq#{B`?NT~q}7i?37s zy~3jc;@WUDy>-077jLJAmI)6+JZry5JJO(Eupm(tt(gX4`S2ty>k7#fm95hdfDcNJ z-^*nD_Ir_2`tDAFV_3yw#&=AXP7MxTSD7b3Ly$EQ%Bq&wt3LDB6_|(ViH#(z!Fza| zmVK*3diimK5QrpN+=+SWc_2;U1~Xi5;hX$28wUAJovwom)q4`UmG+QvR({p(-Z{Dq z9IbYqG4)97l4TmY1cC+MT(3~l72~P1QoVW)f+1MV0t=c9G(u8*pGwd>>w-`0|(nE0JI_bT1o|RjI#<@*20LqbEe&i zby6h&EUY!(KTs?^bMpDEx2mBwGu{z-U#Cuueae;t^-8S=@}%>I`kKB&D|Znsg1|FG zM!cG&H9czmG;|%VB*f};L+GcLME`|MP8pg2U1$HQv-JDL^ogyirH5HPvJ)Uo8RZg0 zFMZ}~#H%SuW}V1PMOAQ@EypR!P_?$TX1I@v0 zr<78>?l+zM3^)n<Mb*+DX=Co2{)as3Io653cv^b`dlv;z=5R%TFydFZaRE z%-3<~=_@I*f2$cElBjhwvstf8zjr|TS+w5whB(m*oU*VWv@`sjAQG8er1#_v{np*x zt9`p#;Hp@I|9+JZ+H}QyLv38kDL%y6zDmuKogT%L`G!FE*Sb$t5_W)fjm)AjqfzmC z{)9suh)N16VIOmF$D(ZXC%LeC{+3b|=5QO-SHhrTv3ooZu~HVJ)1b4z@mva0)r+V- z9Ey!HU$+eb_3)zQ79SF>;!yi)vijt=C#{n|=nUi6B<-q6IV4Z=W)RkMGyiHxASM95 zSoozqnfW^D&1Q1`W+QTzAY|4V-oeh2=zXLEZ_*!Y>yrrNQXl;5tn5}sV4<*wrLQWhIJANB30VE%_BqI2}w zd8AX+XdH%%N*9|2Wxs;D>;iEaNC;z+h^@v_oeg+S3%BHYMy1k}>i0BxP zo6BAXpXfpB(rB7dL(mc0T4V&>dWdIDk}v&KpY}c`Dr$FMUBr7m%@T^Wkr}&f#2|^n zFLm2hj~jE=KQY+y&`F ztzb=(qI;Rr4nn z*sZ=n?&(XSATO_Zfa_|A=z3z9a9P+Fh?&K3pe2 zY+xU}$Qnx;<(oRAB*kuJ50;OU0h;@>#_4vV<`HDD6Os5@TJ&G7$B+%!)@KzoQfOQNEji$9 zq1sbR77~6S5D=mFku@ypHYcd`k?=H=GTN3SIs-AMR?g2&P0AWhiaco$_m&chV5(5b zRTSB@gscqO>I%dz7VGNqQpKEhW2*F<9E}R1!j%N7knCk}GuAjAvV^5fX8MeBQvv}( zids4W);Pjk6GE9srmQb^+Flxb!!iCYShGO~=f8il3mBAalZIvl9H2B&Mt%@-h*~Bf zQlS(Px>D!$WXd>~Me|=Re@Cd($fDVh^d`3|zYFg9>LwtT`;WcK86j+9Oo-hfssUP> zr1BQhR*2mAMv7jTLF-Qd(XJ+R7**TMbXAoSk1Y-A(H)!uG8Uk};d{k_bICM4q+V?Vb#C6IrNCuC`6Yv>_W(Sdc)+G_dE38k|cXTqb&ce2bV^Vbs z4yJx_tO-S`5eS87M-;8>QnpD9<-nM&#?Qk)gcgmG-s|`BQi%2??j!d8g5t+&Ivd%9 z<$FG3CUCBvpop<3=5-moH(V=8=g)@l_|T=9aO|JY%t$H~8Grx;{XZVG!KmYSA6%G2 zUNIpf^h1QMinXI%Zj$hsiqUVqLv4ozOxSiwEUcpB33(_;2)>MY8myUR85#VhS_qEZ zHYiwtRy+K3KSJ_lyjLW|H@SOpQYWa=>L7&=Noiz+H~&0yA;e1S2NvW(`rvsWnW=LF z%U?1Fsco?{)B#rnBCnq0Vzvk$a&VO)b#RHLLtk+Yo;g~==yzZ;U?4nT^F-2OfWTdqE&;S;OgNPE*s6a?bM+N0iR;wPp_KM_^v<@M`^dnCp z5!qg~tcRL~6_HPtd@#34j;0QwNIxfX=JGYZyP84t<|V2{6gBU6uyIQa()1mz90`F_ zl~JAE#tV)v<=|)DtkHHUzvA^T=P{Q;rHrqp+(&W?d`O2?JW<6>3&ep<;gvE_&=FLm z-^V-{1@t3}Md-nx(H94{NlJOS!rgDM)3H0G2{KcM8ojm0__(atc$938&?wlN*}}du zm2iNRJZ!PtxL0er5^-q*mSuFg^DV@&`cPX)2_h{{so*({99h_GRB9i`B|q_O)C*Y- z`h3eCOsOh_SLfw_!~7=88QT(;SHfz%QJ){MUo*!_74tEWn+q3OJ@DDR`hD|-O z5#jo`9oWA1Q^Q!U+WX|STWG3LObrS7%H|+;qsT;-#y^rf&!K#;=1|{2%zwV>qeqj8 zsu|?X@W%VmPs?_^=uZ-*@{V86k#&`f;vHO1i=VaJ!+yoA?H_j{lEMM*fT~*tkPC$}DnG>EX_ZPQ(xO1GIHBlc)OO2GMecn{|8`TV(79e)o`PK&;CgID(QoQWo65v z0c{Xvlv9&bdG3e_O%7l^kTu~Hcwu<*AGev5pUXA=m8blHWH^-+`aAaO%CgV!2kHMb zbjLGIx2lPkn%}2rQ8NqPR-koL1Y$oPdtqDT%+9c2ZD9j1nqF=WzS}ZowiWos0+Fxb5YUyfPRg zC8wSW-DH(~P9pI>_)PBzgCt9AwQ;C*+Wz=KQWeGuaV!aabX zX#MkQNLuB~uFQ6$D(0Qbr}A8ja$P9ivQ8QHD!K2=HCWy0xaInaP&|jAUku5p+h}vP zh*%@4I;=U=y63xBV8s$47E#{ZswGx$?a#P1`JJUtbva1k>=)Zuc)UqwX75(ylL3PD z^;cu03^M*t7m8zK=FLBD$$ckfGV8FbYE<&>kvB5-j3{PtWTN{nF}3P?C0xD_2&RRc zA&923dBwW_!qBl{jUZ+jm9Z`T*&NDCHHo*pz6R-;C9^{>m|amMXO%DuGpei!a?{u+ z#+f8$6`xk9`Enn+s-Urq1WT@5MX}GEohHn3x05QPRH=u6YMEn%O+?hj2lSK-?Pmp( z4QOX!3$^~;0uv5aDINPfi{6@%lqFJm33Y>@^*tZ&58bw!dlBiUmFDFsL&g;zi9VW4S@ebkmi4RlUkNrH#R?iBKW-cq=I1r zvGsRQmS3CEv5WHDciElMls8B_t`G|i6*py*v<&-QM*@fm?iXS}z;Z8cBjTT63*eJ9Hqz=?k78hiCcl^V0rYR+UNnIeKz1xP&M*eY%>!cC8WYAqu zeLX=$jPx10rFLC&_b)cMvn8XI#d?X0X0QoaY}mQdZ>{DMeOieR2vSa1)Z{&hcZBE^ z;N_aPBqH*gufuQ8`6@ADGIx<$FW_1ceMXi@lL)E=7O(0)UlL>vOp3nB%M-hzqBdto z5Q&)!kaHObNp45auP!}Rh~{4TME2z7_i|I6CWmXG?}~Ug{iLEL_x!D;0iT<2hn}KP zUBdNGj*S|HKc<8`L^j@IiZ+YEN{dv~dancCn`U^PkrGhmAYD29Po6pg-lx){&By^ z)uCfdrJCF8(c_xy>LHcfvgyuthEl7KpQZ?+%4dRJzavfUcOCwH^Aho%F%QliSSUyo z2L+41e5(l812Rg4)9>;D-(@)2mAL$Z9Cyf|T_n;Jn7V9K8m&q71@`q#W9&tfZ-eWVV*B5JP3n!!XN+EAC-e(GT4z$l-TDFG?aw|v57gi7`v!$d4r$3q@J z1pe3MLlZIGW?>`$JmxdG;BUY#INLlJRrXH?WsPJFjz;?2S&Zu?UcP7E(*5oA=oX?IZZ?;0A@B%$DA zv)c}blz%UvE#^l7#SHpKIo3b24{;^^5$VSp{4ryB*oBeLv6ey~lNdit`sr=m$G}gQ zqOZ=s2!}h>OcI&VDVFjnVKR{qAHRN*Zw0Wa8Lu#%GQdFb`+AOP-!bR?ezq2vu*EjPS+D(GD=a^rdR>@YMK$O~BV- z7|2QL?<7WDm|V70?l<{MJ_yvxKs0CSZV+sZFTJvHYoQTiL4)(vA}0oc3sjyUm>nbNJOxK>C44V9S)#c8DyBwh@W1)G?fHa z68=uHb>Dn^jTV0z{t};gN|5U}A%2MV*B7dwmXy$iV~Mn#xRyE_mf{s@bV$KBWfqga zFcV1?C|amc78+fENxyf3u0Lz-t4qJ!?pofv>>K8(zT>((TBIwny>;KGajEk|+FLul z=F2p?^o`rfPnlo>|0G^zO)#s)E*6srD|KLkEjAoR=H@P-+Zmh7S)~5mVqcpz+DM$I z$!ySHA=eSaxnkVlckfUGpD7k9A=LJf&aqc%3lXM%*Ta0Y (R(At3u-Wf-p(N@{r z%pd;)2x7OVpOq+*TpGEJ)>?@9SVMCO>m|JSrg3g~itH(y{AE)Ivi5)C!BurSm$8qv z5)~R{L#Y2;sSre1DKMCc*Vz5@}a}?0ye6{i$*A z3w8b8rt01c^UwvNTFQEq&*j(i)tCtM9e=oH>kV1>VLvcmaM&qA;w5A^ z--|C$XZlZvau?srK##`nV_~@~u`B5gmoIT-cO~9o%YLb$)x5I=bqwAUkr z$4!4KJ4t9o^Cs+kn$ev8Bwz!xz&bP^dk)bvj^eftOM#vMnkb9VCyNZY1QW+%E_4T1 zn+tDzZcr;+EaJ5N1TZ94&}cI2oPiy+5N@dKvUt{ zvDC3FB-Vvg@~Yxn(9ee-vlRRYN#~xb?4nv#D%o@MtH~8p#V{N;ZKd{__}WjE05cM9 z9E#u^;Zq;yi(6B0ixbJsJdEn#*fAiwOT=Am8)J3moi)*D?jw z&Y;WOhK?Y1zGV5v(^@Z5qlXNZ%`O(!C{q!#jy5bM(@J?WED4%`PYkHNtjOt!U5~9A zdO-F^bLvy5oW-%l)Mj~we|8FQKFcr~;IuX8xX-xZ=HI&rb_=a3-&S|;HBSH5{~8A{ zF)<{K*fkbhXGsmo*5aKUGNz`)w4d3FW>w6pF}E8DWk4RA3@_GGn}#k|IulbY6$4-K z-ikE~YemSaFuLJgD~4!mCDZx%IGQXaGHgpbt4gn5?Go)UBW+=gS)9#l-5r12aPR*C zTr^`ZMLIKm+`BaU*J;n9w`Oi#3xaIcM3NLQa%&=s5C=|oXB@|PgaL(mni=v6!xO~_ z4mL6p)`KwZ%4HG(Vf8nqKG9(ZgIQ*HUsgs`TKpo$yI69}2EZ(GEYfkT7(7xTzT6&YEU?rUEsD8(-k-8M@15x)P~-?A$M7 zVe%L3inC?CzZB{U&X%u6LnA!ikWoBv@fu>a=sUkjMc)dajxxX=)>Qc4?lJ$r?L`0o zyi3s`YPxKn43CsIj}g|{^jiUsNff4Mr4lou5pzIa>8)))BOgoOZltuR`V(-m8D1YC zLe+W?GBz|5TVAZceZZsZy~8at$sG;qK_6XvP4(7PnjK_>Yoe8OPa(cyq0t}I@JB|4 zYGN5|-iOuCS1l0h8sJm!ieWXjVsMDdPA-4@ngREhPWy*z>?1i--RmdJ8&!#p;GT2c zBW%v3p>5{OSUkZE5O5$ynX_%-*1209iv;s%W>vq4!yc&|2TA?!kWx5d-w0uEgIVaSq z1yxNf;HG9J$gnuRVyT2`XR(Y-^@FlP5_CAdQ^1z=n0tS-(Ezcoz9znoHD~{3@$8lZ z*&#fW!NL=@<^I5@+o@6y$jD(FO5V)_F|d#`gL6wxO!8sesJ8g3^Ud@LJzGU{AP~ynQMde`Y!r>)tV6^3>si zP7E9a=g)U!ALMcT6~3z9Aby5@fii-S<+kXmMxE+w1IM=%lFuJV>+dn{%Z0dZ@+`qJ zzZCsAkL2$Q9<#p0!&^7a2o}9-9>aeCTGa=qp_|@w+;?1++L(!1h!vEb!$BV(EgKwH zh!exJ2gY0BobgPdk({|-cP|PrH*c-8)LmNgC> zru$JdOjVh9BWP_1yoQ@_#Ir3E6HFQLP_@~P9V^4UzN0B((Sere(QL^G&sbxx_RF8eT8Mnu)Ty!?+tm91u%H-p$WPHI+;vSFVG)$c* zR7in?NtR?PyD>EW7SxEsqQvQ*+J`|H^*Q!qKV#Lj3iXu(zIX(OR?wW!$aYo!rKg)A z?pu;w!=+|XPAcr*+7*!^<;+v1NR~0_mtdd(m7wrewy&tXK(bAyt>jve={nh3G@}R; zW2i`jlD!;mpkQ-u@Yn)lTxgq4VsQIm_9i8yV!?Z05?71NWG3b&?kol{38 z$QfQ3LuYr&AR;R+4YfR{r04^-zAq5pD{Q#pX9XUDf90SpDce7*K%(<(52o7}mZ7MA zNB$2W#aWS3W}$bOx#aB^JZ~PZ&LlFqBbG!wZ06rYvKz7*RE0Ox*5mTWo=xY_CsdsL zERsnEfjyOw!k_u;>V*A@#zgT7O0w+w{!{12%@CHa_tC}-YP}Zrbd}j81h#4IDf;7si6+L+gDoKd zHqwsoNvFHVL2C#&YC@BcSkJ%o0IGe7_+u3Ru?EK^5i6}qCj84DbrZ$ZZ)^P6zx+6k zlXnuY$vB@tI}Mu1JTEn|r987rC+B0yr@F1hnXqsom@}xeL>ujOB@=7wF(=>WJ5a=4 z7B8A%U6i8}dRmF|+ru>e7RiO)*oLqOfj~B04oB2xYssg}kDDxsFN)SXPCOrFx74Z8 zXENDuN43Gtu>v2r?#rk_M&(&yb{7UD6-V)1N3pV6sU_-TyWhn7PCb-MatexatLl0M zDAkuJidSCX`)HURu=ME>VC4|>ITebKeAf|zaEe0T7_DWeu5adA5=6s5qrr$@Ufxlg zFx!&)VSs8kTMti;T1E5$XU9{=Q4t(hXO-6%|rn8`F=HXr&J?1J!`av0i1^z~Q?#sf|bdm(kSL zEV({RLmQ~y$P-1OY^^SrdD=~u`egG}7UI@g5`3Y~2p|sL1?#qj!Zviq5!~+`vhNc5 zVFRvO&~N2-wVPx>Q-kcokCM0?7Q{zST+TQy`fD7cJA2A|N! zt<|Ib?_BvIZA9&kPukMkBqp7+0|O~2710Y~@VQC>_?E(yZ-!Rf-$b**4V%s(?(V9< z`ttHT{lO3|p=Y?bG+vWnqfl`{TkuSC5aQ0{f^vIokIN(wcBBC>&#zhV)sg5-Bu6R* z!N!!-L@y(G%VDXvty%&5@-P=N;>o@myRm#K^(Q@ZS%g8Im}aTjoLiCTUDy8P+(tZt zMlUOUlZ{q(qPUSv*xg1sSa#Oe^%cWOaI_rNvO1ymq<38MhV#6RnLG?W(-B@rO!=SC zW=AcWs>$S-qvSlq)uRh1VYZCAHqqhbmADK9neTH8Q;In=Moo#n7O27|q09h}R{gAF ziB>r_*EtP?G?zj3TN{v~wqliq?%90q5y-KJ~Z zo>J~?_3@YZCC6{5&POc$m;x=Uk23ntAeF?WZ=mNhYeZk^JkP>BE=xXzt)J9)pRK1n z#FA8915_%O!Vt6%5m+ljIFa+NN+Cy_87rwyC>$L*!%J+a{ zDxB|WK^y5;V~Oh|9x93^XAybIYYwgna*-CbRbQgD_DSaw78`JZDW?HF=vP{jDXL#y zsECiRPSm{}>+NSwuh+t}6JSiaIH)=^2*p!`O6%w6O#9P~iIY=dtC(|2>vbIK=L5I>{F~d*()JbTeY+q{K-yHl zI@I78AwykKYahAtvM+U4kvm!mK9C3z=hRJ{^3tW-phvha*utnJ<~Sl8soXm5+u@ir zelD*}#)mpvyvVS(&JQO&FiIGAQ%$`Qw1$s;cLE_JRK=I={Oe8MA>rnYhy}El^7zJe zUEfx7dm0-Xhj6WrXPk7i%=<{l-Q?+;Y;KaWHyAF3~=(o1tv$wO<*kh9$ zdJSMPhlSPPb-bZ#opc2SYv6{F&+&lSt5lglWEM}o8UYfy_#g-a9U)hu@=Zo#0+pl1 z6%_SqA}8x~(cg?}cW4r(&!{1iH@+BCp=Y%&6!Yqe78VtMhgSQ(rvxPku~929sk90w zE3ts$A3) z`kHXRx`;6s3i?R2lFMK(9}`0rIl>Hbai(h%Vo<#)Hf{5A?uPO&tY#{fD-PqDZiPDo zn{Zl3m0zO%5SenRlXW zg@3LkfZK@!u~$B?rHtkd9l;%0YNJTNsKSa}f?U+>pI*HE%zd7X6(-^I4?sc)3^y=p z^DALh>;K%8Fa@SUHbahvK-gDrk_ZZIWa-_PH0@DY>G!Bjg>0>qIA$Tx-5QJQ%0R1R z1L3|xFsRKCWw`QQ`%nhrn9q|ppYcJ>^q9QKQH-#FK^mWu+2^{KIk<7q$ghCftQNHX zw-URJ)l4OhYUKF6vCY+x?5OmsZsuTwdSS6UVHr-Sn!GJkNfytK9->#IHc!Wqd{z$mc-52O-?CwN(eO+E zFZ*N8tLYE+mhI(N+*VxizDGhXOmP4!j{mG`+0f!us{5R4mWz4$hppT?s7`J3gV)~B(S zSG`%7|2BsxxZe8M5i(@zGOA`Jq++w92*iAIv3_@1oc8g7E#2;HgJ}`o0+&}Nk_>=ktcxRkuT%xcY@>w)GZ{PQmIPdLC_0C!_*C6p^v0TR0}IHNL3 zLG07bDBi2LE_p*VpuojogFbtj&za(C^pm<0a{`!4{8fd?Z;A!DzOd3%`~;d}5Qfhn zS<;sU*s#{VMskD(*%|!tni1YiqiP=^$+YHGSIg2$>zwPxLYek;KL((g0_-WUR;u?G zD`SeF2|rh=&a5&k9lk_+vQI0(O)cbQW9K=y>(9~_(Z72h>7pBVvWawxxl9L26%HD5 z?VTr;^=B#@6wOspEVT;Jd=_j+X$zahP54#9D(s4W7QxiMRbur_1}Pse2?#^6kXdkS zl`Q=6ky$8YGsRW;DtOH*@Umb>+-L3`eD!9W3DY2(Sy#?*gjJ*N;Vq{Go(&((LT0V2 zu=!~D9s*xGDTN?*ud}Sivw83ST((_F@lDTuUy+X zYd&?AeW03vo;Z_w$S<~KX4kobBFs<#VP@%YbEV@$X&m9xPYrFSZ%Q0Hr~Fvmr3Oas zJ0q;PNr)*&`FnJD7e4edQYPCS5{v7P)ycc@_6qO*PD->*)xb=$be2-4ECSvYs%cy_ zoioa^buaN~-}C}}p$AR`C;1Q@8S)%Ims5hBpSc6)x-ZrB^v#A|Ig}U3thkJvE-klA z(fXIYJBAyrtL52U6PVi=q;wXf{HnT_A&QpMU%X?bW?y9Li_JuRYm)sd1QXI4VWKn2 z*Vs{rUGIj5skO44WJFX{?XKRK7V?sDK281NkJ~l5N3oEdYy$|JEPHTvnAyf_ol{P` zKE%+9!4A6DlF^S!&cJkr1$9;cVSGlFl{y`b&mB%-fN7AK>EMv{t*!I2e$xVGLkpEV zr{qgT)ISFHW;?B-Vrj<%uWO{RzXRPBYO`-O7~ld=kk(PWs1yg3*weQQ5=%C7ElWr> zrD4X|Hi3x1s`&t~Mm)TWCSGVacfQomcb*C{7>N-T1U5k?0JYRJ7>T(=C$uUF#j4jx z1j(R(MFMXo>D+m*^ba7K;k)}^R#STDP=`!nFP8`R;Zn4b+E6baiP15u^=@$t(o?nWu(#otgQx+$pJ7@d}29!uFMe+{K54T-OSmmTg7VaOR^iRB*G7!$wUfW_ZP=jev+Wr z{I&wrG*~17$}fB}6*-6*NuI_kR!pwhJGcyho9K}cM_HLFD`U7ZTNwkDF%bQ$=Dxq` zHy=I+bNHp{xeE1!kae&*_b2L4T&`G=pLGWN-u6e{6g<(X-TX|Nr96EOZ`x-G7GLT> zA7;iV;!zO(-pV=b*WK@=n7+x-YFqjh+l(3m(;JgT%!P32^c^mgGHR`#=$+rDS?)Xa zzxv02>B$!R%tq+z?VuU(Ay8{S{^?b90;bWXi@{8=8APLt>W|q#+2~)9mwLjDD8*2THPIw?%oL^!%qcxtBMwek#A2XICfQ5L#}}imxbo*aq6raO*ped?v-M3lG$e^ z!~eC;?D7#OfU@?A5$D8+T9af=J<Q)p%33%E~C6qkJQPJ?A^jWFE-T_M*O zx9w)8M~#LldamNdR0jX%j^qncBJM=`(t224%6ga0xc4h$i?e$R_=QllwP1nWz1_<9 zwl4ZNZjW%0dNVCTXhBiO#6rLOg|924`fS0yN36CwB-j;%J?gTvr z{}@h#%-7xpg)7-axAZraqR!TY2~3;{j(lV@hF4@`aDYYz98mdYD@7pRG4dQdtZO7O_-I0>H)4! zd{;6DWV9jR27bdmp<#_UB$C6G{hS_~>6ri^xG^7rVi&%f};|aHRd@9pC2rp~7 zkHltCNnfn9c~_gvh*c6VmJ8YJn$INJa9c)MKM08A9WG^c?6#IizbM&MaL*vK@zG=w z5bI|jn7(p^B*t`zM5%*qOU;-vFYgd*=P@otL{@u9f)HRIY%Rk(RGVYo?4hZ#|7kL$ zj3i_JuM^=3**C!{4o9?LG>7kF&WCb>p*U<>7A#}@+s0#TY^X<~27_cI@;`UvT5}j` zAvIIbFpXUr8b9q&bfJD5pt?j{t7(Tt1W5vcB#+;FyZhdGyZd(cY@hz4`gB+I z>C;uWZq@zn_i4B&pLy=8;Ss2D%0ofAt}@sAaK7=mi_ zuA2Ew#69_kY0?>yF#zIL0T=jQ?Cwx$OGMsU1vBf4v5>mOLxFI`0%9#2h%N8Dv68PF z>65oI^+rk@YjmNU9sskc-rPIk9P`=A`hMB$>SK6QJhF(5`@6k~`_oS?L-#}So^gUx zoJ0Jz7IlhlMcDBSg-eHlt925&0|eyWtddQ)YLV=KGGR7I2Q9L7G$Mv!WLaD}Ij(@r z=Yke*?lvXrSd1FDf}PhVwh_*4&K+;EEbjJpN^tJlOtFauPUU^4mDl{XbS_jV+`8s% zA|W#jxZFeU+dkGW8+#7u=-={;q|^(3Xrk_&e6iqoA4a&ck>Hqb@RW`oU*4)@9bZh< z9>d&{yy%SKnA&<$EzlOxo2=XM(?jX|a%W@8Q>*;<13s~+=5Utvje-)7$>X^Z zPmya6m0NV+US#{5WUTqw?xb3N$9EkCBX>EYs+qA@*jFpNJ}K?YzPP%(c_U|Ho=-?- z+gmmNI0yTJ$l!HzXfkpM`)D?bGB5oC%@Ef!`4PI`D?@8OTOYF+Qv`tS95&a&ThvMtybQ;=iX= z-aC~dKgl+QTF|iTtJ?%;#n;{{d7zOT4{+kwB$ceoF3Qq;Ur+J}&!}XK80N0&iIijP z(5C*_YPJukjS?0HQ*tgbDzvn4RYo_N^rQo@Q4b1Y3Nrb%^vk#sFQ|Mc=uL*PoCPI< zg3Bp*zwOF=gLT<5FLIJM47PnoXGZHDR``viW_7ODL7xKVf+|NoExWuFa!re9Jsz8f zyut8p4p8{2+kv^rs4Q-6kK&i4(3keHIE7?X041fQjFkxEP>=h*vb-}7$mUkL3U*8Z zpa~TjRG^NsEt;)#QpF9Kf^TtXz7}9vB~vBmCgPBNc@ib=$?XH_1>h)OM9pE83A)te zCD8g)VRV*JiRjo$n*2WUA{}ZPuP=ZAzLt>Te77t+Aq~6Jh!!~l$_R*S@+bL{`<9`Q zn1WRGbHvYd!k8lo&~wfN+D!B$N4|?RFkvN4%cq(!kh7~VxGU*GH2AJyhE`w{wHxc8 zq`NB=B64_2UhVYOSAd{`g#M=j#{6FF?J7)JYjW}tV$pKlC59=WvYuLr;_YoP^Ogy+ z>K2~}fTm*lK7%2#dmN>ZNA5fQH?>6Ws)waq3w9>s<6AO=8og#BO4HPK;g4>_BPfFV2C zR+wXw+MX%L&j>U;lu_*&Yj;c(NGI>)J6HbLwU>sk4wLYz!*vM`bx;S`{0gtjJt#b| z%!BWv2xxTX8q4{6rR+}3To=Ih!+{D z?GU3XrmSo00H_k%WFc|kIWI)*8Hd2PrBcEcG$VmGRr7A;{*B|s->AX}#;VXHe}h^; zT`=X)Vr-F(o4nFSgP%yO-s+30#ObTyvK{|N>+?_r9RYr0=tqqDHJRWVIYsZR#P9M@ z^n{#EN!9h%jVq9vXzr=a_MrwXy7+6d@j`gz+ZGOMA*RJ;LO+R{4zu&=XBZ4r6yC3iPLoX`A#F({Jr9(~XmW=V zBQZl+7T;bu)769fH4X-)S~@OO3udlBKrUdCu$8`M#ECLKL(19VYtG=cqWDHvUwomW zTWXm%zoXg!EoTOhU9*qF$WsBU$Wj=z)@%ZA zHTf&pQB{+@`47wc-fl;?>G*OZ1d<;$xKZt+cr=F{Oz!S6s-E>q_GSL-D8L&jQU$%F z?p?)T-5p9Ex@(wtS4SW_D)>8zA>FD}Sy5|%PNJS7(l)7Uqnre#Sj`rV)(d;$a5azt zpES}fuOpjXx4=f8t9nw68R!{jz~a}WjkmXoTtT&Q98jrZJxVQH>kRnnHEeZMMKR(Q+0<78oELJ_BtP7ZH_ZF_W;y$iLAG||SWOsIXmwlz^W z=jdEysl>5qr`H#0xH zx`}%gzxW3grpmh<<&k2Sq@Ma$k?-YHh8Ks`rjPM&3XlFqj^~Zm4vpkBqxWVz>lxq{ z1%`N4kNusz2@^l#60X7T|Lxb6YOc~gHYAabv;@uGLYtiNswk~LenUwcOxfe$&mm1G zjub0qv)F<`y|eSjuU*&4*u9CRsa#zP3e7yWhnXdT77KfTDHK0vCwvZK`54%R)-*$% zfroB_l1jiqYXDhWoYnr;Nx)%DG@~H7(lkxpQllHPV@RTZPM0y9mi@Rg7~aJlA-2E~ z-_e6@uTE{7y5Dh77O*l_7b+j3&nb|?2uKglA9y$?UZbO;8^TO^fmZ_wN{(n;PXJ#hE#9M2gQ)aZR-1fp;EUXfjjFFjG^@LqI?K|a)r z`HrH!ryi)nJ@kWGpl($uNpb_`rg%S8X((A)_4r`<)uM?2q{~yd=ecwlAo+8Ju~tE? zfa|Z`m^^1DU#Mkh$GqmTnM)SFPw0ZLsm3K$yjm?^KmJ4oeht+2){KjBQw9^kv>6t2 zZ**`db704`mQeaeE2uSQulyA^cp;fZ4+eKtmQ*~r#JiKP>PajQ9Ry&pR_4UlHXjHRCqaWNG zm-q+3D>MxQ5)&uWtjb{ihG`O%AQ;h2bOaf7ACgpJjt5{S#1%o)%?i0ng^va#@{;Ac zCllF8V~0Pbi1+u{rZ$d!nC-e!^q8)2i>-O@GH?WB;*2BrBaDD!77>V22QWP*^tQ^S zgS=>y$#t`n_ai0!=u zZw_t8mT{`n%5NOYPcBTd*XvVsJcs4EOcbJlgNNm6xB0QpS{~IOj(Iov4y}`_XThhv zu7xLefUf;i2S z|57-7Scn2xJxUS2D)sND`G^m;PZxSBFaS(xbCxOd@sCxjD$i8Zr^}0$s9}|ZCJ^|O zG<{1`oPaGIE!K!ti8w-Gghi@+G?D~OTyL_mJ|{6ZTdxc0BueP*P~k1=FD4J` z@BQ8uyZrA3L^!i&??WzJDX#jI$Wr{{Lg#H$=*9n55U_sE5Ob@4lo^F&r~pOdta^Ta zQy45AOU9Vk{RfCsg`|s>CI|fk$k2aD+pTQ=Fb7OGFM5B(j=UL?tPv6|KbL4O7 z54@YRyK0v1t7i{!+?ylaEa}(~+(S0itoZ-pg0v)+YWV0ojr872gv!M~0A!zC!f=ba z0Qk&eS&llT^9S!F^04V63lP)Z<0orMG7PWDc|Nd?v#p4^Du;nRj*c47{skkXLxiNn zndej`KNLR`p81NP>o@rXs8hT4L&r*jf_NhFfp>}wmu$isu%XufmU%Sl@>OT+>m2*9YR2^2Jt9hB3Lr?YhZ_Vg>tK-CJ ziOQn2=c5c`xdhHTF%I0X8-;L*)}Q2vv@!%p655n78p|mHC;=^EVQufv6_R_wCV1Mv zu6a6+ISR7Q6PKQdpbOIsyadS#)kE&aD40G~cvew%x&&NO7nGE$WKvVAL2uR=*R~>& zs4tg$iXmB_ZHR~KECZgo9|EhFx07z8oc~~D_jc5`CSMub@RHW1qhDB;a{EaGD-|GG zg{!;|qc?B?VPYjDXyC1)jp7z+No4Ndd$U=Ec@uc9wP?jbZt(^0@l;Kvr~e9>i4oxx zQjJHSHA&r4i|zYN4Caf9n^3z>U4zMud`cr%>v|fGyfv)VPJ&xcCE`BU!(dp>eapO} zKhS^m(k?$?P-TKavV5`oKeQw?RP`9V6xcv<)o*90dciQMG| zSzxi1zk)i$P*rhJx$S<~SDOnOzgsBHkc%e}XI9ot%75WfUyqxg_vZE=0D8p571Im6 zb$b#JW2Y9X$P)hsr#Vl99^t*=FQ;;{N|Uso@!QK!&X3HzzOEfIUfcF%I^2DD?~Z3YK#`%* z@LA-$*g`K2wG#%Plmn9TW3rZcMC|VwvUu6nx!eo=+B{$NQYNlut~q z5J3k7MXOQ`zIqqBspK8b9OntxYNM80CX^PB2<<%#pm-~C1h!lF7y~nvv3+^C6LuKZ-Kj_F}=>N_lUPZt&z~W;xwq|H964={-}vYO2pgQwmGS zR7$R4HKRDYR-C=tprUyVZ39wS0Tro0k_~_B3EJAD%>n(jY>-ZPpy-SzU%{XF#$K*aIazl&_DR?Qkl;oYO z8YBfwG>ttdOpMM@B2kaNuP`2E-NpV75T%39PWjt3809@42M9W5}+jG!s_bAYNix_wB#8CZ{&xF&u!DyW4%yq^zxi9BgegOM=OSoizj@|Oz-8H}Z!X>%dvtYkKMrz?BR5I* ziJ$jtB-)hQn6WGKcevYU))a=|X{7r#3g>W7=;8^J)8w2)5_L(GniFT@B<)vYv|q&9 zx}6es9`4Ii&@g!8D7@}gg%wTnN$a|N=LRSV?yz(5itrDk?q0|_B2aSKNqO()xe)%s zYWPe>t$iAc8(k{wZL-2Y->^pAFmPJu30Aq3tNkRHDpEY>vm;_3agUqPL<@l^y6k>9 zL1OPX>l93lB}1=7RW+kDjO@NCk{w2*Bti^4yy|2{v0-lx?yGGvkTmq?7!LCD-LsTq zk1He~CAI|GIgv>rHs0-&ge+xn1(Oj8x1zKhc(v;Ubtb3u3S+5yOhhz#zFO zf+r_yO;={~($%U6TZdjjE#3iQ%WMW%6E4*slFGP)C&kTzz3*b z1S2m=AuS~Q5ph4zUx}lnPY61*nK=L{n_{SW%a5&O)$n)B`t(f1ehRl1k}tN~v|7nV zK^Bl5)v-q^>nh~2(~1MN_LQUpi=+VlS~AGk#vW0!%}@WTka?n+oP~G3Y`{ULXx?V0 zch!>E@UHqu}h=K}SsRo^m^KRaBVLDGFNW#vZ=Od73xj1I;;YXc{?-8^x9V*jm+4Z|2@ zUfCW^lDmdcxy|TfxQe^ETxSZsiY(Zuj1F-qDX7^fQpl%RVbJ8=RA|o)jk{=`@6I6j zQYCcO(`PIuH)5rHFJ-x9i6MaPv>X57!DjzSeroL5iP8r^{11gYocws6s6r@i=7 zEh~TrqCm^HH{lmN$s8gvP4%mEW17^Ng!zXzt3F)cLYA-bYVh1HWXX>*qhjkbMBl8I z+dNF0?+EO4l3z@BbE96HxWCkiD$H<5AnZ-NB73c&0d4IpTrG}BJ(JaMQKan;bonZHqnZJ~8eM9niMb&%%=s2{Vc{+Kii{J4TT2x~Ks!fvf zG`Sr{>eMn@45)Zq0EHr~j&e>sab_LzCmTsE%F2>)YlJh%DE1Z>U&WOd9YYXJwNd{n z0zcm05$+`1?=zKVbFHWL=XS)rW{R$1294)7PnVpt z36@7!V2=OJqL5|PP2pBFm1Z^Kd!_+YLl{u+$(IAT#sv0udM}Ia*ejSg=X7HBfC;Ue+H6 znG}>VIeKAGOq-ZzYQ@*2!S*tV&sCLeb+1`tm6F_yyHZZw!9Szuop?-q8^)<%1>>wvC;ltBmE>D&b!IiD`bj4O9GgZl5fdqGz(hR^>lKDz!cj(1$68hb6kXgkxMUrO^9I#ft10|FzG&zNN`=02lx=9^R2 zgw2%knW55DYQtX^FkIZ>0cB`)*#gn7ESUsL>(xNX(d~1nPN3y5+GEr8>PTeARsh3eMCx+{M<9 zv%@dbt)s5cs#HDQ8>yzIN*P9?ZtT6+0bH%J>ONLe2Jk31eHxX2Olt}C)h{Ks63FI{Igb6Lt57}hLuFmrpZilQkQr~SN<%RN4 zgAe6t;Xj>dEoUk!mQxp@F3ZWEW^x-84OY*L$Tn2O4*6 zlPIf3x`?tia@`gF){wm)A*29eYiNZf>t|tg`&KxZdb6vKuL=U4q1ACsptH&`9jZwq# z5Z4+CMtwHT?u-aRbf8IqOlu(wHG!dHBn@;hf@jHeb&px?P8CD`oi5yUz_HhSACQ zj5zs*#cY0C4EFZjIg^TI`K8wV^~1Tw9U6=0>!%zGyHRe$Xh{A$^P(osNd+2 zAnT7e;Mo?mhR90l3udUMlxb;;ztBK-OP(kUB`mk18x|2DBL;qx#yGf8b%P)z;@oWP zU%FH|JxS&{EK9P=At@Gxhbd5z^R*o<7j}&VK=p`vOOT%MZ7BBnEr)n?OtNbMZitV; zx$t2%fkvA)4(Z|Xrebet-)z$P^@D%}o>&gM^O;aXlyI~VnUdF(@};gK9LPtUs;@CA zxIYG~<#GcAFnJJ8KQ<1I+topan8VoKwrEvaw@)8{e%PoTn*IC-7=8Apu!0;n!g=nz zz0AGC0KYm=9VV{szEyv1(9=M%z|{C7?~3cC;qV&)?{mkb1y;OpX{maD!IKhc&p-(0 zOOz`LB5tEeAp+F9Sr>FuEJGD6_WX_?{y4>j7ObC&jl~%)l6=5OLT5f)t(TAuaE=M) zYr+;GdPh}jb`DAB5yAGf6hyP@LL*AMpxJuc-8F)K5P$HU=GDR8+bbwbJKvgL+gZ)> zSjm-Q`8PA=VlR5Xv7nDcU;bA`TKpVb-U~g~?8mw2&;P3j|9eZmMQBN+pcj6*V-h4k z@GDB_YM;@ze6d5O%6i@ve3@ts-gB;tZI!P9eu%t29@l3RorA8_0(zA7Qb*g|McekAyl)97ukT~S7)TBio=_0c`hBgLLgsR5>6f~$5I8r zchAaMOV5A+k8kT))s6&5Zl%y+c@!K)#@t)Zw<~WU`_TTWVB@E-UjEq{ORL}4t8GqfTHn)rm*n$Tts!|=j~it zig77Vwe?ifbta^@>$5zh@KM;!^Ok6Af`c${6Kf={k!B67H>ZPaAUUy?#o`PHI7BscCOjVLkJxeMJoCOc55PNtT>x|`(bcJf-|1;CH0BJKGztuzzXOB|wF zap(BaZ%xMsbE2eDXDt~!fmWYKtk3jjLxdu>!5(@<)~D^_M>agNn)xq%Bu#snPe+XB zdLp3|Th76sHNLvyw#Y&~bVls?bs?J}#oJ_PVf(Cd^Eazi0c3c@mj>6ForGY=PJq3W zeHU5oYO|!Lw&0g|qn_-<-o?&x6*-3(E0WYFY6tC`UoNWXG;zYXxPC^Wz9Ptsy$ zD}{S{_A&y4>aqpj8Ij%Onq)HcexTP$9%IW!=!Th(6aQa;z zuYQX*>`EP0W7ybk(z}DXOAVC?zUEKb4UZ}Z1pBxg{_qs^6v(%q>8gMDjX@ts!+?b& z8n(Y1-?LbUf;hkv>~_gLc1*Ot^-@fFVfj}-D#hYHfL=3m3`P-AqX{^;C3ez$dhHaO zqT0-*Em{jj#YnMq$N|D;R9C2Kc1DyhdG}qF*)jA<^p>;EEXuDQ6!-MTlG_ z8?P7P=AyLo`>!Z_RrfITe2(4_<*t%XyzD^>HEufna2rURa0C76x?ZE6#_ZG;IB(2T z0|Xn)l2khDHWg!Ho_0@=vlulwli-mnc{%*Lph&Ke#qLnBU+IAr5l4D#lP~n@*(p3a zSvZ^6@xxnOi5pwDh^Ec!skATcsAn8e+{ffA$s^^&HHXDIRoNm6Nz^jO5C?_9<%wCFrBUMl z>H|#r$zIL1e3pHsvJkL9suLt$PujS(p*7x~DVGVg=kngXQsi%lMw-kyJqlunV9N%I z;I|7cJ739bO<_1+P?eEn#q-DOR~)JM*~>?sR5!ssGIzkm@~ASa@EOtonB;<))cQ@o zQ@Uc1U2Se;9)+B;YDr54#;0+IP4d@5uXs!=otED=s@oM9`swp!@U3ZoV z&_y%o?`*4d-J$AB_W>_xsS5em%=0oEbzZ^gjFmF+$&?Ez*~bWkXl?z}hJyn4S>$;w z4{v7g%{l?WUpzu!vk6)5fylHR;9}Ie?J1R_D82jehmECQ2^%PZvwjY+TfnOjfB1Aw ziNC)bk2L6x;WOev-<~UEC!%<_f_?r!R{Wn^{LnYm z)uldZf&~^Toyh)we)0cY|NmbN)ucbHcQvivp?gzb!>GdR!hGPQMMIuL=r)^a+gH+$zQ5&i2U?MtH8oR|ie&cM;wsekubmiPsXj&4T(1!d&uV!_g(elK}g(3dBL#F!JLWr~&$s z2bP(Lf#zli!TaZ^Hv5^uZ&gN&OhMj7&Pkd0H>A0&AOV7vCyVC9yvDD@&0);X!52=> ziCyFSZ*nOj%hos2RH1sn+>;j!ZMA{O>ws9fv8Em;~tS zqg>n;EZf>7D#w^reQ7$WbR)P%CpLK71!eR^0le1CJp?@Y(i1*53Jz}B1`ey&ELn6G zW7shJ{Bm*X0#ovL$8n#}5TV9fIlZa z;L;a+1hZipJbVcVe&l8nw^+{X)%=$`4*a^gC=ccv(S@@I9&slqq6k zjgpf%qdwPv?~ScZY8Y~9)7TtKdkhgCjPILHVQqMfpR-a@~^WA%!p zSPN<}JVbx$&hA5U$y?|%7zdwq-k%t_zf0k_qlw`(YY5afYsS5#j3#sQ;r8hrPA(Uk zcQn_sxv|zP2ebVHJjxoVQzu7G_7c#$LZ#6!B!?V*guPn`-lGOC~qU@Cl+UBYK&NG z;{=1VPVN$I#a3c`uz>dELxgHK8Mm8a)EpzC3$N3Gfu#=PRPm^AxiYKW^YK+S*B6FYDH^#lejJpcA6H;D0y^ z8|lsxKMQi~^JB)9-AzVv{TU|z6HXNhx6{hz2It@sWgU;G^n!epgX!jgXdx|~bjCaA z0NT%fz53xe#osL#<6ilWK5cYVwnU^le_{}8XTsKR4Hp>{NKt`6gb6;@5uQdZL>ZYj z5X}Q;PK*PF=N5$p8Nr4{UKQVClx(p~_!OP(Oc>HS-GGI1fNd@Bo`OC!@ z*0C+fwU&gpvE5>nVzRU~@lvN`c5|u?lZ5IBCh`&WWeCiZ_PNfsvIvhMypfJ3%`s7` zBV*TcAsz+`oGQ5e3g+eQSsj7|?U^nvMh{Z0;R#g0y}0zYrIws_eA=am1V!*d>~@ik zPvM=BaUx?t!pKVAaVe^*77-t3FW!DOgHWC0-KB(@s~d&RC^fPRTvbHbC$5A1JI|kB zml`kB_YY9iz}=8~$SEjV`^dNpz*2Q0Cy9p4inmq~6s!PJ3Nlh8k?4phw8GtuI@Hs2 zN}*a?Ic1e9#Ja#G7PCOxAy)ZD17fSfCwF2|qJ6)djI!onh7Cu)(+KBkyiM}cFL)*J zvMo7KV5Uj%_rhjN=6chcHw42?W>itiN&VPOdoM&5*oAm0X(ydS*>Y^&6;KFFOz}oT zzUU7)9SypxD5d-5Y9YRa1^^F0faROL1+e|}6_SIYbw)#&yxG%lEfuH+hAs}veU6r| zE4_{7SKB&YNS z8lt=Nm34Rb)Y8#vkaPl;&D?(4;h%nO>=5CD%&qd!?SskJ{e#=pDx$en*@(a@ zT9|c-hV18GCC%??!l;8_-vag@WAUwR1I9#9WU^jQ?g(Xj9u%Ge(CyD3z1dio%0{v% zCVu6Pxmjrfr-oBEr+@I|8F3h(kDX7$mP$3Su|@v^)nswmgi868Nh;qKJj-29RyM~t z13u}TshHcQj3=_bUt0Z*v0uHTdmV3hPvtOgw+3gbMJVPH^6lH?#(YxGcT>;-$iH3T zLbm$)}QX#K%mhgrWhYL^MVEFf+-fX!Qa7n~e8qM;X(bb|2 zd>tlW^+Up6CPtYS4J2vqnRB4m6=cbW3ehDK;cph1ZEXsE2XxT#yay(3J14&jsJbL? zbu~5dVVU4Wl2a{v2#8eL5>55EUir7HyQp|PJI0%Fkwg9mYEQb{jrN!ybUc@IM$a*C z-oMr?KWWeAy!c3VqBE!U9%Q0BWmSRAJy83zIiRwOW6CCWp9I_0&wW9UQHMFo8hi!m zbW{G6?uuj2W9uS_W}0D$OY1qmZZgLKU&-m)W*veYFfmG0T0oHn)chuw)8zQO9&$^O z8de_?Z*|^ZBvTPXUzN9BQ~FMpw*Yml(VJDqV;r*0egrhxt56na8t_U0JFILIDmxm` z&nY!T_U=2sxh((CBW7fq#or9lx%&u`o3UqhPh&F`&M>v{r3@De=hMMhjXU*1H(a>V z`Xz~y#3+UD-;{+#SAOGLH~5ulkoj=wDI`X&=P;}w<;F?cxzvl!ODttFG`B5lvXIADFFrQzW z70Dfq7BY8%xkmc{_NDbRVQXUd)e-8=y7Y6h!`R|4Dl&9Ya1!e3(UHKH(aM^QHt<8`-GHYbAcc z^XH>2FtD~2Ar2ki<)T9Dv2avFM{fJ=*;VRZnEPbVs%%lrgE8oZqDqD% zffHyh6XgPy(Xnnv15&bR_bBfiO{euL7uY;)=^_w25;7kzX(cq{=YE$1adt4&V za^6s40Ycv8fF_CgeQ^FX_qA%s2baV*_RmnM?mJpnk*8rq-sDbj@4ee!e%_64{{+eU zF<1nMkp|y+2P+h`fYd?~|n~`8u=S&Z%z?oAPH8Z3(Q@z&-%w1wo#*vx& zkI~}!RZ4{gB+Z=yUF|b^3JHH4iwf*gdqEY`A7c^i4BF(;b&W@?LPbnDv4|@BPkEym znNts{PFS<1^3*9z&-Ch^4DJ{{4Ji4Ey>ozR{R7NA_w7Mrqz#w-MIOF}54cl;q-lgW3P3DjSd@@my~*7FT<9C zgzPuQcXQ=sW9RU%to>zWMB}HXP3Xbk;~;)f+eFm3ZqZW{JKw`a4QDNdYe1`dd)BK^ z*t!8QYygMvUV-X$3-8t}9Xq2psmTzn6Nmbu`EWO}YFfre0XUM^AW#SIR)}42iE)oU zzSwv@(_Na$>}BC<)Zf&tozC}%VvHjWKey#oG9A=I1VkHONGyk*vthigj2G&-sV&T) zfBO-_<2f-Cds#rI%LomsoHl5+a8lK}t^%N-n8m0h3A*a_aMMf$lMbaSbyAc0tg=kb z(eEMz$mN**fO89uN@XFXzuw8u93?iUkuqxO_{?=z}BwF+`9b9=8BFOC&jN%*^jj&s>%vTk5q+@1bJoJ8HB!wmbx#d4vW`c z6P%Sdo&Kaf@GT3;39VNT)I%L@;rU30^_HSljpa2e`;7O(QwJrFyqq0Cfk-5+(4-l+ zRK zokiAaBfE)cky6q1xDB9$xf4TrdL7v6a#S!gk`yY7cfW6ThnNn-BZwf6UH7ZRBu%W- zo_CznfTGd#RA4`G6I|wEP`%jDdR;Bpsq_aX0h;0pAw&_Hwvh>s!Z z=n+V0oBZ6^B|Spf(x&t-qJz7=iGjCLQ;nkKhArPlZi{pUhggZ|hv`Zss&j$m zO}EAL7oB_>gjL25oRe21ja9^5)XMNpkyt`mu7Zj(CMnhvpo@wU98v#<=B zf)Rx=4QC*o6sj!r#eR$~kT1YQS{3r~{E_+({SNRiLSp+=0o9X&Q@dUFJPh>d<}%Ft z93%%zg|7NIbLKXFQ%8!Ir}A4QTzV>o?paout`YEeQOU8Gq=tTU2hh%%`2FY#<=-%I zt9L@wQkUO<6D|dIxrmOcGP7gkAV!{06=b1DNhO$PcGFea3yRI z1>_0nms>SArc;Qs5Iq&fVPPgk4D(eKLKCac$eg-&q*X#u8ODa>g>Rt*c)K1N9jH*n9A*b$ANYViu#QHKl6x;H`_zZkwV z0~1IlsTS91u=y`_xw zW+k4F;l*|{kVMHx2dM7o0EAvKYG@PSzWq-3m54_EOXjf?%{T#R*%A;O>K^~FZ3NE%OfB%sDfFsOL zGu?x5*)JJZ%bW)qF`B`bu2OMfu3fYZj6a4c36B)44hZB<)>=|9b$UmU#q z(^!+b^#x4xbefJjSIFN~)6lM)Q}2L7D1VO@vkf%?T7o-eeS~<;(2>DQG!%?=Nto52 z-@@LdT;IGLOUoqDGmag>qHsg`?UH`4%|)eRm{D?LuVbQsxnZQ=HGb#7gwJS|ShyNq zKg7afuyImhGW8*_1Cw&lgi~MDOhY8u!|hCjmD9OyEZ0lfq*;!D<%!aTK1CAU+F#r? zXns4AInqw0uMhTFf+m8eaRCT4`{&E6NVKCkk6EUx}at~19FOsSsN{KRl=`XPfM51Uud?VKANul3$c z^~%DZx~co<36o`I$Q7Zsc$d}F*kV16z0mpU;d!OV1V>cbyiH@o7^p;FyI$sF|kX zJwWfB$uF$gd6&?aVTVg(wjWUn^69m+RYEgLK^id32sG!+_nsK%IfT4OoJONMkGYv!8pd z2>5K$Ac&Q>{RZ=!k|9c#GuhZ~s10zuR)q;=TZvsmoLWV0m~^R>$kz-NbqLTxSz7o( za2%ELygXnDQTa{~kfqS+*pt4OS;vPSG8)$w23U+lK_j5^&}=i6neqt8nZX8QM2r#BqIN#;kAm0i9a7oc7a19Nd`M$v zFwkNW3Z=8AMO7kTMgVe54jf;p1U12hj)@j}2eDe4JmAlOZ_emAd^wN=#Do6$2+i7B zKp9BAl5>gM*4bYvE(V4QoyVO8?}Nwzm}t*r8UN|}PtoyY?|%`TrriPTgpPX$roK6I ziu%*IE~_pJcm~5Tbw#vT9lOFg!8SWb1k?X+qCnUZFGBy)yt>0Cpw(CPIWDkJ;Rn=%jPev`` zE9-cZ11rysOfotvhN5u&18ktu7`3jUs)4pqlM>Sf3037ZUc;;92*(lG=fror6rDM2 z*sY$lE=GT{ffC5J*YM&h<0C=lgMFRCj3Tq$1pDX$ly6n~Kcnp{weiu%?wY|gN~QX* zk=gSFZ|wLMH3g>v@=NJ80;s}SubtqX>3yiop9NrSHkU1u)2`R>f=|pllvZ@-e4*+Smo_c)erapICB5zn7m(_4bOofsVfZyGT??z#{OGZF>vI07|co`Ny>(eHiQ z=ZF~IG$JC=+yr$XAob9Iut$7(lepeA3y2STZ?0fDRMma6JCucVGgM>MZ8fP>xN8z2(#ZK?zK?J` zV-b;a_th&%4yB4v&WoVcWDJ2v_1@w!51f-?FEUi08YtF(s#UX!V-pHz*&UkMz-3ajXP=F3GO6~J89er?(Pss z8gJYsSQ>W?5IndBf;+*31W5=X5JG;#A2U<;-kEy$)vJ2*$DG==rPe-DyUtm)>#X&C zUv*BtoH*pnVZ$)ijxSzP=d4~vbK+z$jJ}K)GNI-Dv=vQy zrzOLpIFKljR#y0y?+=K;U* z-v^RIl_<5~z=iWOYqP9`;`BC}T5Ft?4y``P>Dme6=f#Sxhr_cRwT#zn!l4eVJ%y)I zQzCL|L8qeioiyy_CEyK?IEEZ+OPBuki?R)hC93MhfX`7_E1=vy7PWP~WQ& z(IcmOOrO5bEuuH506wyNCQ9ob(q4>eJ)8zZ(`v{WzUc1;9KMO0=BDBEmF7A!Rn%Et zlrW>iv1!a{2CLtF#Xnz8v~6;tO^#7)D23W*5P?0wy5(y+$qs3Hao%i9b`^9I)MBcGR?Xqb*e5Y zw+@w0DOaPgnh%Y3eBiZ}!qf*gdAOaj$_cAV${pHu$8qWWo>%daLesF|!xX8ucJdU2H z?i~_Yz`fnb5pc}D`Y)mcXWb_<~yj3J9t5P%kUERrRy-jMc zRSnul2$fP*aCLKKgW)Dr1bP^jCA-4w9y)(CPIY2?Z4DpD$@F*jT`$j)qqUB7xobV` zuI;D(ZX$Nhd3GWdJ?el4@xIRB;M%jd<4Fq3kQb(bs!qcx9bK3h%34M~7S9a~1mj3NQ@VtQT@)^v5)fW7V=tDq*pn zT4FO@U#oj-t4ObQSNf?Sm}8Q&5cLi&F5whYCu-P;QwWK;-*v0x(Tq`&TO2vpHRM+m zNOQpGwR|ONjBJsY$L42LQ;|Vf$v#m(0}4ixFAE%7J^Vzr-0pbu)Tu|rlj_K6bK?&0 z6INI5o3Jas*Gf;ay*mKr#frMBTxm%tRMzj6f8#Cs@sjtZG5eL#lx*!`(bCmtmpjL{ z(D&#_lCav()Z%>|L+W^-1fs%%dG_o#TE_N zyN0t(a=$3-=?Vu%_0NFe+_rq%UzjXlgaR%@?>_5P=Vh0uTXRoQ6V+3-sM=qc^ zxopkp4>(9wzo#kMSMCin)&hhhp<)G;4&t!T9ojTW*HS*=_WcOxa9EF(@`XJ#ueAZb zS5>@kx%$%Nd{H+HaobM>Sv&}=-J7pHnW0$@G%3=SDF~Lh@;=y~7|TfEVEkW)v_*Db z6q0{@uLb;Z3-i7D`1Q5X-_Mi(0p8xd{P~>cpX1@1Pk$Tl-evs<5dPm8Dc2Z@?;>e8 zUHaZN`Hi>4KnJKs>2B(%fV1mX25m$d#btU5rN7HV-=9r>W(@!ZK{y+=qg4H+8v54H z_}o~Y<%!?W5ZAq-;dkjKWDMNLT?Od-StsT(*v1|Eu*}wkU`8Ke=H576+U_Vy)n7EmLE8>@OB76LsX9ZvjSxT}I7kNiC zKkA)VbLIOCzJ+_2cU&5K5Ij{ro<9E%;Ab(j^S3)!{3iQ{`|Gf5aC#)+_RIG7 zsOMEV0Stn2(-Er7J~Gb!ae)`ZW5330!|W`ysjdl9yzc%|=^gKt>ll6Rx@e$Q!q_P+ zt`vm(Jgm={s$X0EaEaC}4V5#6KJB3Q(8pEFT+L!#RM_;~Tz~y9k)k-*JB3Id$_GHf z>0j0Hu!Qmdqq5|bzhawuhNvf^tO9uNziNy@18YP#0baTc0zT8sqKen9OtaKf9*9(R z{*xzyiZA`Mt}pH?ULyszo49x8Z+_(YK%_6n64%MKl-RX7Ws-xmWs@cAQz^$FPSWow z%SmP^{3%*#fmV0a#k>Xv+`Z;Woq>9{Q+lNR5##f;Dxo{x;5&ubiNQc&ErDY|77JP0 zab)5u!0Jr#)ynv2#!?3J+zI>lP5~qiKQVR#I6u}Jr0V4{#p|!GZJ;Tj!QeYv^#$5Z zW?ZSNqUe16#G6bF^+_11n#%^eQNm&^xGLo|7e(~dop@6IQ*m1nR^DcO``Wv5;z!ht z%I#3zj8aud+YLpIurx6GC9wrgWSk+A+<@fZXI6R8}fmy1V;^7yaDeF&9 zAcb**$1JiZRX_vsgr(ekd3Pw?N*@Ww$9+5HN_Tf7a!USk{ z#hzk$tG3jQ`s~(ROdHiZBDpq@r>E2}a17=Vm&-dJm9OHK&UJD0T@p@@=3J9qf9LS; zH6=@25-08D$R4_@fouhO2-#1 z)PzW+QQIc?8;Bo<)A68eLVBZ08+T82Iq}fh{Aq z#X>mQ&SO5<(7|y021LVom~5je9YgblLQ*N4*{|XxC`LIu_1#X;aIlT%i^Q*zTKYKC z0@8TjtyB0yAG%dyi#kI{P&Lmmae)$Fk2K`w29JOgrOx$O7+=evWwK>UX<$^+_zH=0!WjJysI7O_%q;_#APD z2EK z2J=z;nEaaguXfvbR~&ExO=fd@HgfzMeR6DUwbc-&=+fy(3>m2+L3xJaLK|Tel!zS# zViZVQQ2wqk@fgh|!SDqB^R~n^0ts+u@C6eUclMg^s5{6Qh+^y!tVG!buRcD$3sm~u zA^S(YvR4}m!%a%DRxh*3{U0i@Jkfi-tIxH!ijPrJ3KbR})fF+nNPXR%G7cZk za*kUR(tSeoNrMzlt3o-yoV90V)w+WQ&SI@gvA2-!U|6g1R`^B=?AJIUe&JM^2i8y( zgVi7Vz`O1+9R5F=>p#)>{P!e7tK#XhI6CxOKw1jCy);bn!K1oPaK4h3hr;5-nZh_N zM6J69<WLpw7cf}w;?r9!mOi~rSC(&smRQZjt5LgMN7GU8iIUSe zdwSz?ydkjH@G@NTSx1B$g?L21WemLoxeg=AR0`2gRDB~TCY!4)Pkry?uReLTQAEp^ z9FC7XKjUjqY(biF?l3DX3_vsKcJdWFNvg9KG2uh`9n~L_E6>JU(c%OVGL(cAwS&$Z zOm6}5sc$ki9B;9(~j%uPj&o{p3k5NG!m7>U;l zoDy1wY|a~L1Jd8b>W>hC1AaOzR$GhmWS}*a{yl1;H)_f(1VfQxQ8@Wh2j=kaBy1iZ zu+N`TF!ex0->*pXV76sfGY?5(Rvn^Ne~u~JJhcqRD9(yR(-y1PT=F0UxAw{tu5e(~ zx#rB^VhBLHdzuoYBw{m%)NI`keYp;%im3!Hv7XsdD8aHXtfap%w#5eoAP3-S1p=Ru z3L|ht)dnzp(2H%CLE+8O*`qt@E_Wf5LM_Akm2=ZqF-c5><7sj#mFz)IOaM~pde@2! zPMQj}U7Pe1jhlm}Qlx{}z0Y?)0X%$KV^U2xJO!6Qc~pNyJp&xbCV-5hzxPcj(KUcY z7|&I2$$e2>*znZk6rYd`iA8cp`J?C=BZ-BQERhNBMA(x&Tx(<<_l+N)e~kDW==i0( zu;pZv10Aqsyz}x^j*T`BTfaXktwRcxbkkI~zyzBAz(SOH@dSFT=V*?@ks)6?60}Wg z9fGDqAU0bOV%X-AI2zunYD}hVN-F>q-m8S6$La3?+v-aoevl`xFzpRvl z+;LV!&&!v1TVJt8V-|8Y@5O)K6F5Hnv2Kq|=ybCivFZV#;Nm2NV%Z(9 z`Sc_g4|>TwXG+xWZpcKi&z>6^%oJzK576Nbbp-TBk&L`ih$DeV9s6}kYr2ImIXtG> zqD9)V(n=y#5WrHNgyL{Ni60*&rc|X#O=w%%)<#v=kTh*{$9f?W9Ld=D#;8fttGTzK z1VKDl?ukKKxu&vUbRy+2i5R@V+I8+@dq4!G-vE>w-=9A~4sUp;g z0I3?uQ(0d<_=2KsOB0C^hM!{dHu9lE|7v?f1^_;N!~gzGhCL?0%~T9lAO%8;obLKX zI_2v_naUN_Us%QA1I9w_6+coyvX#m+Ti!OAIKWy3`ERy|@D@^ls!niIPLsqq z2tbHVARc&eZWeT-eTowDreU5G9&x>_iIzye`8l>{F8! z3qh>@X~CTy93pWwV$l0{uiw0Z++KX{hAMhV*J4Jl@Y(0PlGkDE87zH87r2;ygL6TO zBkk7gg55xeW$2zC<)U;@Z~vYd@6PU$%^1BW8#!wpREwzIRt`i!Awd~)>!Vqnn(!T7 zyb9ZuYHTSvS!zmO{uku@f(eILR`GC0!s%I23rtZNRuRz`brx%Z}HMIa`P{WzrgX)B3A}JK+8qb|jVWJ9B!qmpI1S zQGk?ON4joB8Kh01cpQG}UQ{Koh~?Wi&zt$(Ouz46kchUTPZ(!Zf(Eb=i-p37Kl~E{ zsdyewL%-ODsguuS&O2A5^B>kRcz3^F|NZw5|HsxlzcONB=8vk%@e?du`0E(q_=xudQ}N5@mFLJccp;gwxY{aIGNAy#-I1Le(ksK&k&9UcGxFKdU28H3>cpdv<&)(}HepeJFM@83LU{E^R$ z1GY-oCoEcujCT}aO~NHvGozU^!0=j&^-FqT8Ua01?$pmDRiv zJDfzA^ufaEgcJN9J^zURy0pQMxV|kl6t2D0Tj0O?r6v{B{7*q)fopDJJ&r!YNyto# zA2APc5q6{)@K^-^Jj!wDK}_xb9$+s6$haI{{YI`awTr5 zg=;mulI*&T-WxvqJjy_xU}e!ZkuN?I`gT{bW~EOh4Ao=eQ|}{NvE0OCp6sYcQncZ7sd4|}Pa>YHeN=fX zULQVQ7BD20!ijdQ_m}bu+^NF>Nb6-am={?z8+P zU_Txn(5onO*;0%7Y{L>llh`(&`O_n2O6mk@m|KKP$Mmg+2p&;71IPALE zIXYsB8!<4QiM+(H)4kosr-ZqLD}|r2Np}nAaHzb`*soPymxifw@GQaW&9- zC~)a~A-8c2w6IvOEuIM(o?>gH3;7n*@KcB1#ex`{|3mPpts%7j{c8b3dc0H=odI!? z_;n*4_Fqtst6WB~n{H-5%A`oXFBN7I?xz4?bAo9pR$QqsVFolQFf zz#7MCn&+*;QneO8CMxdOUVz|L;XBMcC!1f%V-Fp^~G(DJAXHd`$4%Re!3*Z z;)G!j9}U)~ofpw4o2vGPF)Ie|7f0<9?Mk%GTwf2|L>MUsF|MD-JIS+N>?GCgg6Biv z84GKkiB`NaDQMbtcI>e5dxtQw;m7CJbw5(5W=-gGDg5}eR>TT?H8y6* z>QfkF3-FK(NN$2ewXLVGlU~m!CiF=EzNy9kZ9Rc)((UJI-29H4hR4V4Lm;$tFQ_QS zxtevEh7C8Nr@emtIKw#nHy3gai^+f2LJW|P| z*3sRXfkD)muM5nTZkapH1d^$s^RA}e0;-+mM^=ZY;Dt~eb2^ii7{*DM-i~=-sEo37 zsRAQn(CZN#y58|hPYxavZAd1o_M)q<*YSc)DO_i)i6UlGIXLV^WUBZ= z+46bo;6n$6#{ej1`wOC9D8~_FyNjfHw>dEIe7o1)~FmX5yl}#Mj zTSq)^ui^fbH4$tZA_ptDm%4iOFm^hg+sFl=_EVCiqqi)G>auHA8p0Qf|ejRy#`*5E=IN(*y_$y|T6HK&~ z0Tdnweom2i+d@EOgcv*E{4{$5zcN<{H3CX-0vEffmF_VfzYwQVpTc)LBi~b;gF+tv z4%g(L7{?H$OO6ekVWuVZp6`I4nZUTkn{~un0mgCD(pIxgmHs@ZJ4P@$|I!(Nz5JVB z5#BD^IEPceUQm0-M~~7nv&^)J#qKb()eAEBjwtIzX|J`%x);8qp1rB`n~|b#R0#en z3dJwqI6BL%+i#oub~0DGIE=&3tK~nK?jgC~+j1asCvXJy-Pge=xabGO*IYH8yKp|| ziTyiVQk3x5@uBa{#?k*+$xc|AVSeTEQb#Mar_RuMiU(R522kQL8CiN!;$hSI0o=|1 z1AO>=ANoJmuNfV3QL!4$-{W~DadTNYX-T}CImiGPrrW-`uG0(({KNCLlQf!Or+QB@*hCp*l+9a-w^pS^{Ari zxUI`13QgRLd?`uip$35&irNr)r;WZDfv2FPj(La&1DaNVNw!)o$7&4vbRKnG=w+0J z!wDQ}KrJwH3XO_XopQWBh)8*S!6<{7{w$8; z)d7bZ^hxWd7*1=IAe^mW5NMnm67Q~vJH2=~AYY%*6eN_aC`W^h?T>@&yKGxB7sTfj zn|fg7gPtH)Jkai-@p)b{Q9}*WPgMuAh7Uq@%`Lo#X2ouCY4_-00@t_~9N(?f&WJ9Jouk z6%sqvIvF4@P6Co#^2o&rl%79T)eV4-=YpQ~t|}Khl8{vhC!v|*DMBphGX~6cIW>$} zA+$d1-{0CS2{JfbwCE)Q6_@mkf;88CL6ZZF%5=?3vQa{$RIdj1mX1wzNGxN#tUWU> z+$f$IGq>D(||TDgbr-XgC)9vWN01tUXV1DI51l!iY@A?J~e^(8&Gt0OEF$e2RUt?&VEAs8T3w2I7FI#*5-mAPUL#T@l_`?uCn5$ERK z=`y>>b%*Q>%Iqt-WR7_!Nw8*QO@IGZ_(^K^6>?r+_|GS$XKCC{B821?{!=6&IkfUM zOsjIDIYGl!5ihx|r3&HnfM2?d$>-d(km3T}0Y3_}AS&)1s{T037v)D1Vc-5mLhsbD zjt|Dq`3KzM1AtV9wg){4igIEisL;`=NP5JQ>&xh-h+J{vN^FF_o|U*Ouc& zg$V=Uog8XE641qFdhejjUS=80$?sb@GiQSaWQ-UXywEWPHi-l3j-O&BgtiAF2n{9E zm$s=Zyp#ROV>Z*O<4iS$?N4&Dt65*#<_W*c2zIWTr+Y!&+$wKtK+32wyDv($#TU0i zngAKnD9578Wk8_cpax|x3@JI;TU zItQ`TS z3}@4d29A$p7nQ!kcn3LiV-x;1v;kO&G8f!-2xN@kH^wK$_f-FaTfH1I|QQ+q`qajj0*;eH>kX%Tny z-+>9OuBup+i+YJX99=A3G)87aWWhuHqURFrCg556&yoF8s>t0~Ykc@Bfty2-H&e57 zK?Gpt>utXszPH3e^My2nw9r;qS|&t%mY6&CGr1Q@4CXO-d>68u$5!mP&9R!Qt3y|5 z(x_ReGOIf4v`_&peS9LI3t$PZnp9eeAP6#Bi=V{-PL)h{!#SvwC_*zzbnuG&v!ERZQs4M z%}Jucd@TgFN`*a~%Wu!}F|`urQ%{gGVZ+Pt+!sEd%Z!%^cnw>CNii33*%LRW*~v)^ ziQ9{4_Fn{yq}ESJV5_tg9NBrZuLzH{6A)+d;*1ViMn&eKX3B*rG@*mxHzRWjuLh?( z=lhi;nSFlSU7?=*LPqumCp!;og>OT{#gLR)Lfuj#E*C=m9%HdjV0?;(jRH6>hsNUQ zEptU2zvNq;6)P|8)K-6Mdc9jjSWx4XjI_$sBbnS4+{u`a%XEOE} z!PV>GH6VrAlXuUbTUAZr37@E9Sx1yKcz4*S1m6FMV6!y)53ni4-99Hwwz+0N7rbk!R0)pSi ze$Wx%1;7YkivEkszMOIgpWI3_F}(-sRBopZCz8pnyWZf&XGXk(DlpCfC*L!5iPxmK zx%jgN((!(RGQ4<-Fp_;^YuCb7?!QYDbqUN)Ka+e2SpKz~0;REHm-~|GZI5>j7!`vB zGH*jVc4$clFSj9-jBy*>ouqp2_eqTp=fmY@-9tb-*n`U`m%8rIhbpc}&H&ufzh}-F zQ^((>4Loyg?>jfls*&pces9OQ6@D7ITs3_8_+Hd>o|@WgLFgS(TI*gNX2dfCj#U@ z`yT-7Nkg*m1+owCc82px+xU*2MdoLMzI98L#Y7{1|K8gjxuP^@u}6Aut6RCYs`S$~ zdEJ6nwl5y`(j?ibsxKe9x6)orvBmQ$qL1i$Jr&7)GL*iY;XUgc{Mv&AQz!}u{6SM* zM9c4;n&6z(KTXtHMv-vFPkL{Q%Nn1N3sK-7fs$~+nMgKq)EY?ueWrTA7%&a?Kyha? zbpFvoaa6q@uL}|xCLRH>pQX7m2x0!p8~f^<-@*bd@B-v!vU1(!_wH26MJ)AO^aN!y zS~1F4m;N4zl4lu;?#{q1Pyt&6oMf7xl3xXpFfKr1Pz1@xyJt^$Ezf)%qy5xL_9RN8 zb(@}(!WS~XzJ&*~Q6Q;8sfK7=nlmR$wLR&7#A|5oM*0UR z!tXj~8v$TIe{8!$_wxcY>I!;AQ`RZGdUisLLs0SKx+CQ4i%jh4mz1%jy!vywau}~y z@%>k%igLJD-U`o3OIUj%vXfJ%mS(8W|5=L@bJvbCq&j3D5WAz{j9sEG2CvPz#>kPGQk7Su&Pgm8u4A`1qpnBO(&3jH2lqqJKh%nmEes%D zaa36`{wF++%8u-R2Gsyi{h8NOtn3ZW3fE43EB-2#Tx@_zX@!x)_=~R&2v)Q`R~5oh zLhhwnl}^Wp4`j7(-Xb6+CV&RE3MTw0KUKf^PlL{~SPjG+NE8u~Y6#!-nDZ{ZRRCuu z;DxugS1M*T0s!=J$KK*G+d#Z|O8CJv6Z%ZI4UOL|36K>*QY);#;|nNg6kATE6r{(* z#3xBeAp~TvVbo9ZhQg5szK!MTRKaV*1&SmDLe6%_(FndBrM7BsefpO1;ebgs%tsmsrmpwyy_G#l1~q8kL6VAzIQdmGwt{>=?vz6iFfjTZ zD`?5$j+W%aQc&2hJ$s&&25XR=oSv50$L_g1M!&3*-AU~01i6APX~38PHsazm{I4D1 z%x*L@VV6oycY=WG2e%;XB!JVI4YqQ)7jE2l z3)PzW{B&1h!_2xum4?=issnHOjH|VldWLT_KH^e?vH{T2e1B>4l^|8z|NJ z0%U{+wS~eqM#zb(&wl~93fV#zr&AK5%03M9cTSP>r*0GNR37$gKS;iNhP(^%fYFiH zEK-K4c!VwWCNLIygpX!%Cg?Jz_uwyxGnE#@!AzX)G?{t#d2X8i=9h3GUJ!*ri59hx z^Vp);F4yN!R>F>62?(qG~yp@hzcEG)UViduA9I2%KssqbVs0AQ`Qu~&C)`B?G4oCkcpew^sCN3`MPZ4T| zTNpQNt?(EfbFbI%Yk#?;mT4W7QbY8mh})(xs1hSfc5;ryqW&CfYZ`h?e2aP(S3`8k zSKd@x?gOS)>t~K$rh=os5`cB~8y?T{nI3BuA zW$0D_()4F$FWf5uMd@;7dgYnZ7*jYxqSfGT;X??IL%D%qH}W_K5(yW{C|-z-{2Kq3$ zHcJJze(~m~i8HXsbT7LWDa8H8){rwD*Eg5ypXVyml<`PI1zta=a z#V|aKGH^>iSRS7GzP_CJ%r$?_EBeHU>CMylh{Df@1!p*5rLdP?-&6m!kb6o9*M9ol zw{=fA{|}N#hS{`4?d$(bQbAPP9)wjG8>r%hO4vc-2Mp;hHrdzOmyhBb!?6 z3S;LV(0SYT0|%3{;vp=R5TE51ia)kK$5}TJZEdf;Dt>_4Ch^n7wHxn;f_h@x;{O1t z{K>nK{l_;C*mnfsqr&AZRmmNaGtpr3U@&8G)S7Y)1>Gs0onlYOraCdpc4rn6r(9pg zpQt(3;b`@fDNAV0`WF`=N~SBOJKOF$7AJe(lc&>J!<6=Jp(6JZ`Nb z4B%7-Cf)I)T3u`*fci7wMu0%`35wJuBnT8aGw#_j$-vg|%zuD`aAcx3nRtrr!>gk& zF_A-;mW{6abX}7RV6$OK>7AM{3tA#6nh;r64Qnm(3z8w*2_+ zb2Fj_*ygiCaKP?B?d$^~ps$x$@x(^d}7lyKZp#sRPLle+6eM156}Um>1)Pf=+Hzv^vMmTM>o2v zRs%d%#AifrN4z@ESIrtXY#I$7LRs%dyeZ&Y+5`3XKSLKBq@5y@wcj!jmsEcH{xA5b z=zl6qiz8!p6wlrcCq~JJ_Pq@20mVq-e;|6#(6r}1kE;^{ZfaT*G)af_AG$)tRu$G-jOr=wdDIO0J8#*AumvHwLtehP2R#?=p20 zH~6C;I7k{-pfCVj(2$L<_CvDA8is+XIytIKWAxJMe6#gk>g(V)v#8J9F_Mc1>}=_d z2X>T|oZkxVQcIa;P=uE={58)^B7!$K6az=6rB0a0)Rg`l{{6f|jfC*iiM)^eROws> zSLMRg@GrI0y;r>AL>jhy5DFd=Gf-^cL6J@v`Bkw*6I^jB-e^Oap6 zCt1?*;aXZGU?>7CoaevUN3(^HY}0)xSjhhkv(iDo;{X<}l>znFucwJjKAPjZEG+S) zY^VmpU;hWd-MQswPpST>a)V-TU7^$mfNHSu{Se%bI0&xhTgs*|C(cSm#I@X@8<@S? zwO_rAiMj{HOsW_csla2}vwMTj{m_z$0w93twjwp*x=@~K@}sTZo~e4FI%R2;KudLO zuqYs(b{T@=3AKw0EN%<-fGtA04mFxeM6_53?IXm=v?EQsuuvnFHmL(80Ire=Q_CJX zFdxPOhgJeQFboz53;|q!J{+|Q%Bn~L^FBkf%R{Z$^!heygg0qu zVGAahul)=YiV*%tez)^oGx8x&ywK?@jt4S318?p*v2Pxd6v}ep@OS{g3R~assr{F_ z^_2#SWl0#{h94J>oMg*3m7oW64a$Nh#9La8(m!)WNy3ymIV;Oxgppil&=ndWBSFmu z<;zNMcVc@vfa?cx<)HK{rK{-qY=B!WAk{hVj*C}M=Fz?{`h)({a!YsTp+CGETvdwCDowIOhTV5+_lvH3ODT~>Q1d6EpsndS zR;VDadj>Pkwy0?JN*b>G=kPgyVA(6HXM%35zzeAO`+!S8ie81>*QAg<=u zWW2&L8Y4|+tnrb*5$c5V1e`fut=3d8$40(;!|&07fX55BFcxihzoKO&emj{*%pc~{ zo~*!Y|5dg(DHSm*_{cW+?aWk+MBsb>7=0?aP#}8mJ$eDJ6v1HbE&#p$(Z(Bm1A7~- zdn(}<63h8q8Yu3!(E+aI2v&Ye7%&8xjO!Z!}+Sea#yS=^y}wz8II1; ze@)QH;zccAy-lV!x~jZJAqsyoI>w=@%X)H`W6gmS% z+OsDYU*?^60PsADZnXlWf>Hg`Jjy$UZo@b_IE(QyI>tl01ga&Y0!=c;~y6ArXzmnME6QCiZaYcDy~k zUmjhgv-7RVna}KY=xb*Ou?od~dSAIe#WyslGr>Gc_40*hh?lmPKtVn*ZWQV0;vdv( zqy^e7_ycT1fYymZ!tCo)p(G{;<%peT_q}ua&j^r)e}-PX%0*LMJGss%-q?#)5k}08 z5#dl~IW{?t4Li;?B`mFGQlT~-nazQ0jVxB)vsQbT@#~9-Bpzxoy>rTu&T6w|{(hEZAv<4o+SS>d?I~@h z$Qu2|DtUN&)O|qj5U{YvCz-u|Mbn}tHay2_2+b_ETwDkK%Ef0NXm>L z!2n}8t-o)R`{pDgXVi-L!S7fcIHI?PC(qf_X`jT^jxOAgOum3^nUgG2rY z9(Qjf^y0(;cWE=xnj+J5mx$Jjo1%Dud{*NKoM2Ztg;hSbyV5@R#3&20ZT*@OAfVdc z3>vJ5dxV;ERs)g!UsG*A4e9tm4TpOiG<*G;&hf}VU#(R8w^Cm!5I}9N?lJgBqAWXs z+(56;3~fY-*!QSm7qGaCiKAK*K<-qk*?N$FWT46n%mO+^cYWek3T(+gf2VX8LTu8x z5rp|!F($lWecCNqrz1z69jI`etEe*2V2SGL;V8{VSXj+pVbhY3E7_N@uqoduJ`fWD z8Lx~e4pGrqgSmif>^Eh;$Mc73U5y7Y?B(=ztIK~xUoiSkQ;dAUR_VA zR>~!1D_Q*zjUiy7hs_p6v!`a_E&t(oAp+%@)DzSFl+{sg0(n!{g6>){Y^!^H&>zaL zgzhRpGre*!;hRPLz^Zsoq9nK0hhg-f_i3PZto%Z8O?VRN)!B%rl9hHtgS{~ip#@}% z1K+@hm4WJ-PASbYV2fhE6?P*{e+e?{f?e_T1T@wxy!+Tz^J}q-cHfHy{zGQ2Yo}-C zPqM^-x?4JAy+Syzo95)UBxGdY`|?_6Fr8WyX=>j&0=+&81#DZi(NsZP!Uz~=Tt7K| z8`SFUBmocKn!Qvq3Ua1~*DD-PJ0vTFXzh}FVFSq{{PJN`#fQ0aq_kiWf1D_Y$Igc# z3TTN9d44=n)W{iNO5>KzF5)~_mYCS~3o)^d7op4c)#~dOM|6Z$bY6vz4N%nXmrvn` z?8W#54f{4r&k{h(JxQKA@;4(*7>%D!3EC~c)81q-7fLvX#Ti%T6F&qisgsuB&k|M% z#3RSCU?#lF*ckcty}))Tx2wxqW%=pZu>^!*kd6ebiOa!_0xHdY{|^vwBcl1lH%+97 zeyx5_VqP8TnCNj}G|8mCq*Z@JucvW`<*ZJ0D)<)1TbuM2sVm);>^qjZL+=;KzoQf5 zJb`HjC2u!`k+EKwxo6~_=j4M#GWBh~|LRALldGf&vL@u_sMXB37VPZaemkT36qaC7 z>XG7kv$ubc0^&sidx0D9u&51?QKJ+kw)YHhK3}T8KfJiiiy%dXTxcP}06pZiPxyC1 z9pRFpyQ9R%RUqC_((6TLOrAaIydmjLna?O{P<2cz=wj~~Xt;@!%%}=NI(^6q6vtW| zamDMs@ZY%ETaIct{`t}7g|M`#*-~%)0B(;;K=uLWNp6snLizwjR!(1Nr!%)mxvgk{ z;!7IB+F2YcqPNX-sI`XOFUxzd;UzX!JJyg_(xnnirT+CE-(p5}@XgZ%&wUa)x(Ys3 z=3r}+e=F+%E%kci<(*!EhIORkt(k@#D3s!>$$NIZ_*4VZxqVF=n!sCzP)9xfYXH-;%y2bjI2 ze8!aku?GYJq=si;d#{gB*eBLX)~=Dp8a6}rhV1qeAOJJ5oQF-xLPu}kFZOM5{OFVP zJ%Q_54)#ojvM|xfmULQ8ZCbad%!!f{z(RHKIk|AB`VM^%@PhqN{Dp}~i?-}7hHAG4 z0Es%zRzGBNqSCW8m@nS=tG^g)0T8hbMs!)2UloI+xRIf)0a=o}5L|YMyl7ZE0(5ps z_Kst*RY`>cB5^V!yTIK9HfGIxnqe;~wD5I#$C`?LRzP_;DzQwgW&THAMuzzlKKq|; zl6Soo+=b_qgp4RdpO^AX{s36WtW7OkD>87K^mfxJFqWY1iIJ%T-j3$ku*f(Td&P6V zBvqQI_M84BgBqoxoQ71Mcy-6c7xd1u7?!)Cj|5l`>6?edw=R9^e7h)hfjK>%_)0AH z@LNDO-dbA@()0H?dLe}wBw?OqZc+TAJiNZWtjCidmHqyENp*^|FPod6borpN?<@rqsl51M8ip%^1ha1pl@i&K0BW7 zGmfN5$Q&O9lw_CReG1$rJfE8w6T1DRs<|%KzG2OK^Wz4%^?xmI ze)Uni!7)qNIFM#cECvMti2b#@_x{WOTu@g3uzNzhi~BHmI-O+A$Ax_sE>#s-zrXwU z{LIsVpaqfi2#rerirYB@IJ|vv^r~YbN@n#lVmwTr4(RxcN@R}R5vanpki-QiEvHF_ zTd$@7YyBDT>87u^@TvFGTB>p;g{nbCQf`mMt(ToF20x{nyIPsBpwWuBVHo7M$W-cp zbu@lSDNyg_s0?jzgFwvDNy+2TzD=t~f2$R#PND#vl%!c8oQWzJX)7uUT~y9)EnQEw z+rlQ5#`*>|XeonJ7)=ueVOF@s=g?LlmI@pWcfS6~Q);@1;b=9=3mdV>%V4 z4E~od1FORkJn@1HiSA2O80P2jBo7P=>U-VDOi%U+-?V@|Q1TuzaVP8K%`xg($(c~slOu%Jd|A{iED|$5^JNF7UzDLNRu&?6&1>o*(tHTomNTsS zJ3*8TZT@??Ta*A4FwCnQX#?!|VqOo58*;2Nn46t@F2DFoUZc&|wA(D3Bv)&Um=?Oh z>}U7_N-C=gG56S+iv9{OO_GYbDz;PLzmOH~eR>jtS1aw|0c4WW*0`|0P17Wn8S4*W z-g?k|EyA!R@B7qSZ-OEIM=80b^a-z%<1dcke}EE3CaV`t^f{;2WlpZ31yPr^8}2Ji zueevbe>W*v+$@AJzyH8qhs!Br_6`d<)9%<0z=CONt0MDAb_FsH;8)=JtIzgTIrIh< z@RpH;{1GvSiOI=tL=1^RnrT^CV8m~MDQndNu~|9hFsaLco&CY_=qDts6jT6UDf9od z{~vS2t^77PnziQS&&GOqx`FBhrjp>t#fA$6YpO3;r}@4QZxqIKEW^IB%QwvmkD;ji zA*^les^4Sb?Gs{)yg&o3ack>bSW@O3o_`=x*+Bw3Qgf83n#B|R80ATQdSz``{8Pm? z7BR$+XTSnzvf+Ui^{&+xH*)B%Eer13MSk+S`qLcHVk9A7>3$^`tQq#8MBmn=8vdD{i}`T5rw`252HvGq zz=^pvh9Ub|l2TZf>HMHznKKXVst33aTqQW7;Ao*;>bgxEdQF^?$rrE4p8$|sWv+3o zzdD3wF;GDGQealVBD~x=qLq6J=5B!31&rjUp-BbqY2|E+((?$#Pu%Cw6h2=4Thniz z#CwwtZi=qf{GlA~`~Ho&R>AII_xNr_2u%zV6rEbl%79Q8N_gu1gy(s2ij|MDw6W`8 zR=z`dY#h#jKorj^`o>ZusuM*7sjF=z3#_Fv{1%|)=qE`u<8RpWLhVmpDi>0eZi*fn zcx}Q4yOo@ylmQPa@<7V)U1ax`IEH*}@8;B9@FAAmJW_cr?2w>aSQS+cAd2d}93svt z&|yl{&s&~9z{NL3wY>fh6{)^wDSundQ0wBhKy=KdsD8P;-@@g*F7ng}1I0Zotk*_|ljKBC@E z9a6&`!|S(9fDkkvy`%n~jJmsQa+B)rd@iM@@lGtX!LcT@aY2GZ5x+U*j;zQ4HF0@( zPvcy25U?`1axQ46JEFspJ*;+2+t1?^b>$+U0v{L%(H6>veT}>GvqOmkpL>8x?Y+*c|fxZS7g^O1VKf2ZUOxkcFcb%mPQJ z>$5#7WkBT}lMhCKT6hTud9FABB^z*g@`e5sfVOxTdZQtp7-fD1WOH{o^=buEpHs_I zjkgH?Fd!+gaB+F8{793Yy3Dl06a&4UVduhSf9VUfJeV^r{|Yg3C}(YqbN|+&XiOOv z5b^g3$@GtF?ax7_&R*T!m5HU+B*BjE17|;~JhJABb^qGUP}Mm_JBv0r+a4cXy}-b3 zkJ~%szPR_Punkcv5aT?dDZr^;z)x;49j{iB&`$BhlC&?M^>b9gj%2Je=o8r2o;oD% zBbO!H50DF~nf5-%mzf0TnZWn?wf%dE!Ve#G zDL!qM8fuKmX`aA+A^A_^5p&>HHuu>*7 zAagM)_5FZmqS7x*9uYiZ+s}JmTH^f9tTqJE;>Er7v1dzv>L>f@j@-^6Sf2w&6C!l+ ziyBj6&bQM9z1w~_H~wVN3MI%6et$oxhL0j`4-*bA+DW>TY82;LvNz~B8*eRNm|(fW z9M<)vdA_w^YRxv8I;te%U5T&T2Y! z50yK#CHd#3?W>CGSBG0@24S+nLuji{eI=Cycav3x6lrepZYbb*=YEFbd6~6o0!1gd z+|}+44*wKppY1UcbQ`D#7U_MHJ{c~d^iu|iFqF0Y`DLpLFa}$}wC&J#lFldmb1g>WU8WSb znv;Ia?=HXRTf4ko9<|^Rx@a4rWH3fl7)r5RIemWo#xvx4?$dd@x+xw_=%{6Y=qA9!?njb;Z0&mZA=^v{`nYNk`GlRbFV_Z- zY5R$$eyu1ACz2^oq$&rhR};qoeX3A!gIVRvqkms*RSGs>=Jiq!cr|n11eCz4h%U0q z4GHof46)6{W)`h*6n$iX#BS`&mu`4UpFW+j>U-Ly^#xx+zaHVo4mXUZoZe0Icf^RP zr5=A@l5M%$@_tH8O4b`#ib(~s>+C@B246BvUy|LEr4=i-qN*iFtCCK}N8Nw*wY&eZ z+Q-LldXm*&lkL5wCh*k0K+j)|r%oN+87LdP5_`1rH-u4~;R%D`v;+t^mK5C4w7IKZ zqUF&;VxxHE`n=R&4gx+Z6ze6SjCbrMNeeAZFAr>`mG^fS2iSgTJB+Es92Rmz)wW^+T(Kgf`v0P3#iT#?gbTOG|lr;#cYV0wIYq z5vcMcZ5nV5DV30a@VbjMKgBf&;*fZdQO%I@6aXch-Kq(J(Xo_7BTyQcw*j0Fdy-*< zt=XX747gs_+ zC5A*pgRdX3C+3@H*P?6HQ6`>XJr`yb zNy-%8m!=yqDmf?$?uUySU&@^1)~oi7`o%9t0jLmGdBM|_CSiDU(A!D%W*BeoaD}qL zKuZm0T4pXvv7i#yXI3^vLF&te05FUm?}gK8W6FOb^+m{U(bza$4=<>U&L8|BqKtMa zr|m#yBMB4ka)AOb8Uaxa#@y_4GcnX)VRHeeA!-ronn$sBWeTFb(Iyo7p4@WH%ma*= zO4)SclwS`d%UQL+>aM3vPJ{6zNwGpeQY}+lDTJ0H&KponVRC_=oH^Nw9;JfA1kyPg zC~_}%FhgjOB}cMsf8rP3CvexD8PQj^C~ndUd#p9BQv7!SFE8<0IhU#L9%RKm6kt5j zJ`4hO^JzA%J}}g@TI6*AMQI`=KX{&?%aF-KOGg&i-nen)!L;l895OiZ`F3o;%h)!) z8Dkw2WKYDHw`s68xS^iM_U*q(`?)# zG8!lM<7<+rgaABA%$L6DcQpTmd-8|=pZ;&Ib(cP@dkz+(_Wa4f!;jzo)F{0D_1Ng* zUBg^;hGx>mcyL7jt;YRsF{vR_&*mffq@X0g>gYD*!Eu;GS`;%CKvXI+`Pwm5a1I1u zBbL`(q#KdKvQ~F;i|GNME!qG|Wu~5A<*u9T2sAJF2JiNg<0wX!$B|U&Sn4~s0&IOR zClaeEj*UAE2?W9~_e@yIvS#1Zxthu?vQ7eFiOfWq3Xq)zs=s{j-~QV8)kB>uFXJ{f z?>RT>=NRN&vR22msKdhCfqj4A-t^0vgdSBAojUHPe!Az37IM@gWhoD$7;w`W-WHW% zuO~XG1oEiKnW84{$X7pH32h`2=a!kKMn?-AX@t|b+Db7b=E29tYo z#1h~gWVULT$Vms(AzxnQb6?D0ATIqvAA0@wmfifXcV9jRcrA*S^~SpicvJ_1_|xV$ z5mGu12aCc`;K+-B?H0Kn7mlU^)W@fiH}sU3A6R5x^xdX07R#@`3L=Z^$SMT0p8-65 zru<$PGKd8e$Dr-Mo9*8nfo80d033NW#V6j5XWU-)J|$E5@)K(9rM;yJu=Nn@BaC{uCEH`XHrB&#+Ak9qGU9E zwy{3pcelQ3Ft8>-L-N?}GUNZqlb$q|Nl1yxfGBj>ag5A{Ynesj8$Neqs}HRzQKU#1 z5!9;n@&)1zB%F^_Auhq6xz4Y2vTJf7(aNWwT%v037J@k49degAhYA?H626(+1F&~C z1*41@lt4|3=?jYTSet`S11-r~*=8j}-Lf#!j z`y(M>l4|A#TFzR3zN&xe-mCf1)1UVu|32L2{P{BxyhxPl#Z}=&#iU#xQeSk&#mEg= z$kOdxJt-|2V3(z0&g6SZs1%F$;U(=LHMD~)FM{;TZzDuJ(|(%Y)**FF5+(3l(dlQR+&Z=U56b%fyM}$vf%$7$n_Ebf@e>KR)gne&49k zNg|8WShfc%2YQr;ylu3=z*tTE$(}oQ@zX*z)ZkdCl31D2<;?N`dU=rVXPPlt0nHC4 zYua^fR$h#=+);gP&-;W~)M1f2(Ho7P4z7^B6f|IAtlE$E`x3vqn4ZEA_Wd6K7Cp>C z6*z4R(#mdneQY?R$O!QgPf$w5g(h{^T9x4s6XInr9@c#OTeH>Q`XsT(Bzi-fh=G{Q ziu$Tv4Hp9gMjae~j>ZhGn(C9#{Q(ZrK0jyH&2n22eXghw$jJyk2M-Wi0S6ySk+BK1 zBP$R?^y$m?KUNC;Pii@m*e!(Oq@!Vqhah= z_BG2|-IbkA!Off`VZ9NUe(v5oQcZOG4!vp?GRZPwqM&v&CDw*7w1#x?MtREJ@QV*~ zKobkHRNTHy1$5hn0fq~NlW4DYfg#m!gi%*}z0nO4z#DpY42!k!df#$7u#F8qesul6 zH~V9C>e49UBwv;jhXg#)CedloqOy7t=j&;?0&KXVV1@6WkNtn&wI8lO_B(mF@ejcN z-(SnPHAc1RJ@wBt34gk)ow7oqwW)-yc0Kx^F}Hy7wF8-LBK{Wo;04)n3ukUaAtlwH z0fF_&#r7vOEVo+qhqg>B*X}-tv$FmJY~M(>$Ir&prsUuvx}yYC%;FdyvPp*x+H|Yw zC#onA{ilDS@mSm;8gd=dC{zexo;5hJOe%xG(gJn_EDZDkRrAVg(|P4o)5B*DUNsZGH&J5 zay;q8Nm3m!kw>x^o41TIreNVIZL~eat3DAx6HI-Q)fBDwWR;1arr85*yP|64;M}ki zy$I)0yt#WtKa<0umYNVin12APZzu#rg!5Cx{JF9-|HIb*51??o8x;NuDQLs>lKDLb ztiu-#E^D=+)=g??Q#2FwU)&709-8z7!_;9b2iF{p>s?g=Y$;f8LQ--&M;1Bm{i9XBg^o}mS|C!r?_D6$t zwbGaX8@{0;KrL(Y*Sqt}@5|Y|fbvP}0+|zzWMB7Q4wevRa~npT{Zy|ta#!PnhF1#Q zb5JAt`6OvrJx%!S*YG?xj#Q!d;B55~pAzM)Qz&T*oJ8eaVp*F>_yN18C|+WKFOhqg zo`ip}j-y8^ZHXxC@Yz#RcY`^6hXa+imxEcp{>eJ&v%1|zu?dPT+KHv%2XxqNVWVmB zj_vnIhmag-)D4qq``>qqm!IEd*&Vh$>b`1;ZkibfF;r-NmMdYMwfgy6bJ;KQqHhaz zi}vhGO&Zt<-~3ressmx{NgT4EC44goC;CF#gPT5)q0iFDw&2;%iRrIYMe8! z$qUz`T-~0wnMfHmt3{7b0MPkFAt|8NQ9NQG*Yj)%sni*)0dS;L2uJ-?Ym!IyxM&Wr>tOJn!L0uxkl7*)aZdL)Zd}=Cx3p)OB>5=B+`P7`Eew zgZ09wG!u2CPGk0WUDUL|eOVTt!T$hYDs!bdXsA?@4;4I@rjMq(kJ!b`Dx}MZfr-2Z z;cj5ha7W|9oi^CZ9CU|2;8WaHHhD!EO}7EP;EZ<}S_x(>69>iRW0bWi!DY3N(DhJ$ zi50R4A0=?9fO)O44=;JkXtn>6<L@&!r*uqAH1?_a^+zyYwAx9E$dfWT-s8vs#vAj8imPO4WipNf zOjLI1Q`++EbTd)JS(-ytR#u7)3oI@dknfmc?C(>r1+{r~HO}Se|z)h@7j-lfQ@_iez_g~19;!8iay8t>rRyv$w;2MEGz}>x$)EWLf3!b8st&(r#I~9z&SPUs zwCPCmuya&d)5Fc>e;DPS)&0*`<-I|?Z_+C6YqDGEuKEYKfj>QD)~Y{o%lIlbaZ}|( z?avC;;cP~7PwGn47T<9Gf75WUG~Dk~8tTz+wp(#xEu3Xb<8!|ryL;LYA7aC&c0k+! zGD)$9TO@yfuN>ZoIJGrK$iMQ?A_neFIqdJ1Cwxh*P?dIm_-ga&<+~3Nc79*vZ*~1B zdAjlYjn?^Jx2kV9eB2Aa9_JrFd^Pj`pYeZxA2g{yogDlg-BWlGWXtmSl0ux}j|M7c zfo6Q9NxAKb$rG&`ntiUUaqXDDX2jWRmwhf2O}|OM-#Qo8L&0G!Z&QCO_HGvS+x-A5 z-Imf`L;3&af0#knL>-?${r-Y6-9zdhz&vVxhxqZV*x!I>Us7&VquqY*-#cXd2atO6 zW9Rv{aYrXdtfdLeL#P|A$+Jo>b+J0<D{yfqNT<)`1eh8qKy>X7Zt)lg~Px~JwOp|rmTv-a&z z* z{-{#^EzRyR0F?wue)n7;2}6Y?lJH zax3Lfk8iIa(zN1}pb$}0I14#POyC{dxMo^kO_P?AC zaR48KTTkAUC~iF*RoiJTFLNcA_WT5Nj1`g?*}dCt8TnoM&24=E`p^9~&UNEnu{*^s z#5~U4cJH{(XAe6OXZX&cpgk6Hg+EzvYOF#u@FK)ZoR`>1>g)jrema)@HOJP=a+r&! zvB0XW&2*`#luY%VAD0{7h&ADyUWIjLw}6>r+0uE5o1C#V*K^_FQ*^ZmUQ=KLF%*^CB-=d6VSuz;4!WO^|Z#4*OcT2vxjQA z3Q4t+r4H~)HC6o<0`}|!5vITil~Jjjw*&yJp@p&y4w9y9(WLJA4~$De>BL;j_TwDO zX71KW+uz9|$Q%+9CJrhk_luqh`fwy`YA$^{mfS@CzGLiRp1=`)nx;D_#Sjz&<`)^F zaUF08N&5U^yJ^C6KT(~~;82{7oidKiFf*5X{`Gz9Dp%C~#Pf4jsJnSawCBP)y~nF2 zM}WI|aX`h_;tWN5Mpxs)-9QG(Wr&dA=O2ReWedfg^-cUVf$<_~rRD6E-I5Ffx_Q;y z5%X5(zbJRl|Kk}3_z!o~O$1p<^mo=19^D};+HJ&vs1!d?s z^_j2~`~7nA>!8KqZ7!X@y;h9u1Xg%7;mn@7FCSa(FWjb#*L>{|y0Z_5f$ z?6%w5xElu-8$0mxBG!vMy5n$Oa$FufneC@fBpqJf5$N)h@DwUgq0k4bpO%XrCq|MA zn(}jZn3=+g0$CKR@AO=fnVzn$34OM9|ng9VY*f@c=>Fm^l;D z6_&}*FLMS=#UG0DLznmrv(d$gp4jd6=fRIs&?b2nbtq0kZ6$w9(&Vf~F)f~3sZQX2 z>P?=}U7K)Fz29zUw->7iVKTz&RuH##xA1$w_hP5zPAX zdPT=JdBWgf4swrAup{{EK>v9(AM`~85&Bl^k3V*S0+gZrp_)3AeZT2{4QM;SIEU_4 zYU@_c@|9lM1r2}*17eqUA0hj1%Qp3#Eil|$m zl1yWPB$+I|Fe1f{1mq~s!P@4_k+>}Xs9lpdwokm6%O^(L-N6YO?o*e3GP1A#n3Ns; z?UOBiCWnN*W`5QDz2p_?5Ua~1TqbdTWAmn?|49rb*dxlM!q!=3a`Ki?`DJlAp34Oj zx&x;Vqj7QF-ygp@`DMv&mtLtLG-Y7{hYl?$R~XvU!|B9ql{z|X&Db|XV8FW0^eHW_7TECz|e)DM!=A>;7F4W4NfHe9S%MTqX2%1r;$5 zA)YLlwwWgEMoBN>Hk~;1q?FHnw$xZ+PuVdOA{6U6(3_N^k4olYJghT^z>el)#`i;! zQCB>mScE~7;tlO$DX3Dl@%wMSm^Bd{O5{4D>zCF(_8Qu#<2C>oTkgORyA>qb^m_Eg zZ1y5NE)78HiH&EluA>R!L#cw697y|Y4r1A(#LCpbw`DIB1BL^S zfLaA@Y^=2firCy!s%GoeTS#hPh-_eDQQf9mZH&QG9KCbp%BW<^MkKS@7mFTZ8+mf` z10%&}yqfNGxjMH+h4@Dr4PlAcK$hy+Y~Fg*&W4WcuYe-6Sp*8t%u5(fR;MQO#_^l$X;w0LfRxOd>w#I_31*l33Ivelt=# zuDb;aaC%#j!t~<|Cexw~K&mFMd!~N{$goycKxvvGOPRN7NBk#rMcw{C#=2!S?28vVp%a;qF^dbs~9qlm_RE5mva+ zALCqhI!6EpZCy}gjzZT8T3{ducIo%-#%p}Cd$#ngWOzQG6+>JSND@Zf8LMm2F7MQ6 z-mPEBV)IUUsc2UdNR-vpT2nH6Yt8x?%cfWBLU6-w3=M!Xlfhg&o(Oi3&6>>KVQ#)( z`l@1OLb>3>qfBy*b{p2I(k1$*?e#Is6rs8|zh#xD0?z$n@U|Ff1tio-T*w44TSU)mv` zp_v?}AHpFkB$UWoOexgim^Yzgj)e-2S0q0km z=i-JXCX!<{IpI6hTq*`zJa!b{p_h0F93eFf00A(f0gcv#H{}nW?>%LpDpZnm1{UZd zr}bukGJr{}MSjv1+K4J5``B#$oL#8=Xg0TH_Pk0NYBhrq>DaSxE{fTr&(>@* z7$B^fD=}K?14h+{pJz1ff*?Rh(tx9VR2By@L7$AmLw=6Ro$d4xB(eze=yw$B{51W8 zI(NFA-xg-8UHV9hn{<(V!JmjUNe7!Ayd=q#Z&~-Zix*W;Jx(V0=aFSx++a%hrL_aV zu|ZuglecznwJ+)F-q1a{a^?#tVy?XZ4rS1eiiRW$r|k#nv-dyT8a&oUjC(i`Lg8=Y zi!24nAdbvE9)R_P>4D-hKE9*Gx6po~ZUMY6mSc(UI7lz>*w1+({I#oYQqyrPGGsMR z^B>^31OMp9oit>Cl*(Q2GTMRw<657Q5ef*Pi8M6_x?uY}=5x5?$ias50hJLZnmHdB zFt`d=7(d8occ`<^X>5uG_|g`a!lK>|MX&EMWz|pWUR>NFTZ+s{x5V@5%4CZg9sr+a z#C^SdA3r-Df#9564MasyD*ajTI;oQMwNxZR1= zef+c}o@r-=*)|<&{A5c!y^MP$n@|P>UjYEKzelZluptp4kN8|NmgWLB$N=5@ zs>IF@@1(xq@qs=D=k#Kb(v$ckt8R{{-`;jU<;RUEO9r5UlFIz^$_0Us4O~R1+an!2 zxkfIWldQUdTt4K3P#5>LachgaU9_=U`5Lq_z`}i{W2<1W62nO9EGhTme3P;{Fq1Q$ zhF!c=7MFgj9HW*qhe8Bw$&Wcu78EC})hgUQ;M*l|P?5zKpNaR=0=CK2H}b@~s=+!k zir$zr=$$oaKB9W&IlSax3p-8a&8)H%6qk@leWhhch@BRFXdTZYyT7)WBK1lT6M4w= zP2wU6L(1>TyqV6?c{hs0tAj7BdAO0TK#j_D&_j?$Pdy#X=xiDP$FduD0WwSy5tXp5 ztzb|G8yZ;uAH%=^mH`w;PYsKw2IIMvViHp-aksgQ9ixVc! zU~XeL)i{N4v`@2QZx**J!D}R%MPSBJLNuxP!l2v6|8(39bDwzggFMa@4Gu7Ki%c;| z))ZaUk?8&A`97(K%CmiW+4HrNG0OG;XbZa{il%AH3tI&Y+7xc!QKgc#aj6u>-P6)+ zwklN-^r;$nUecCYM2I!72qiVOCB}XEG&Ec{99}!efxOd1j8bg6l~?Os>}hTiY!5pVN|)J?GWO?r=` zo?J2tdFEou4_xeqDqY@!ZP>$tlvQ)!1iHRdkCHdNR8YX${8`)c>uG!`fgBR~OaG^G zx_GTpBoot{_nb?$LYFFG2@a-EOAJpsg!h{wNegoozR0_p-BICjA;*PY`4$0^@-*Dt1JGq<+!`~m69e#bR72qflq+-s) zG+bPTnq1|GLQ zkC>S0|ug%)x! zT!8x&*b-F+8F*?4)i*C%Yf~&;sfp2koAwTn5GOlDT)S5ph5Hc28Mn|xt}w7ocL%{! zCdr}GXUsEg&LAMwUZ!W%T%oQHp+H9QP8G27l;fmpk6uUqnHX4&xpQj06EkUYlc|pX z2cTKmg@!H|tPr&|J^XsX@%(+kSI2{|B&Q|U{U0N@ZwloC757S(_WIAq@luJm8Mg@V zzAr0Z|6}|RZ^S<))B~ENj2D8m0F3yJSW1!-S?kHeV6I#S~^F*<~_3UEy z>O(MzB;8+Se+zf#Pd%M@YZ?9}Zgl{B!#Mh%)-->K%bL-jvv8RkrAlV6_5Td<#T$c2 z{?hy>2xKEm%w1PN?oU5{Y;iMd^V10D#t|2j7m`^+4l}Jm4hxf&?GHbEcJ;s($FfoU zo%VInM1D)xlj`$QUtD_YKg)np8k$zWLm0^1a{LyKoruc@yzysL-L`L$?3uft=<>5f z+8n9Uv$?;;^A|wLr4LZcmf75oR)!oEvw7v<0A&IV{`AFqE)^pxy{ZY~6xofQ`{p%H zIyZqj;yQC(n{+PxF^d~Y7?{T;GG`I~u*cd;X2h&>JSp^>#;fd{O@wmb)sp=?tMo@h zE1GwE#A47W`c7B$CX+*=FB5w7`0lej84{gk5Bl4qwwYz8$G1*z zfBv#^F@j&x&RU4qmT?qNtNVB}+6;vieCgf@x%w?2+Da2&=Jbbh{U6|7{RK&!%^ATz zYSFhO50`8hkdpo){(<)U?pTn@x#gxzA!&9ZSCY|voO;G6f>#@bX^LH#m?7~^1unro zPXkZMSM*-;72LP%@LAau`uvzJb{U{+%}W55X#YO+mdd8Dt2?WC|9&{|6<~oDq$M!o z{Twv?oajX<*oLD};P+jl1aYnMMoAfWuY}$N6@BS3zu4D`1DJxI^9=Xb1A$cL1_LM& z&(I|q|IGwK?AGvV=6w*mm9)&aagDx>{Hg^j#f_wl52HhaG;kWlA(m$39S#*T8}SS7 zZp;fb5lRv{H_F;IFhNnx?rPXlr>`YSiY_~%AfGAIRGZdPA@+RAX|7#qlvgyaC2wNvG5*lWKN=`-kijKbDN(llWN-%>Wl%#X{&PcR7;SFfAH)|i4mog!-+M`jBGlp* z?jkH0g=_)o#)B~&7uesRXi$6OFTfWwpPYl}`G?sJwBU{ea{|1W99n~F2BcO91wJJk z&1YF1%k4_oVH0$hT(9`Bnpg1JGd5A5gLC3)O9`8tA&cgytd@-sWT2&4HPzc zC6$|zVsg{5a$O@UN`(-M?+?f57qIhYQ7IRXmTra#@0>wf4nscSfER&b zzWh+S%bZ|(3#WmxJs-fSte7vS{dPvD9G4CcU4rhC)G#NO-NEkj5K2e&IlGC3a>$Bs zS|y5aqR0A)>|K&Xr6ZsYvL@LtQ0GzB>*_xl^_w?YNX9*hE(?`sG<8tfN`EJEXx4LP z6W#|ou#{KoRS!a(!6oX>E8$N@q^f9{I^veU8XIurSoxp{46Q^z??)4)_(6LipLTza zhbpv3r8-4~xpJ^O)pqF-$q{UHi(-7Dq6WM1#0!lE>ckX;tVp6@x=c|^?OotQhu zO$Je`-L;$!wwWTF_=!D$pkXrIfl8il@(_h29Y@;^r62OpoJc&z2=#8BDCp z(4AQIp+>KsQlIiKBT6zGPGCxc@=JD%|@D6eTi9^u}9IH=XIdJJMq5 zi}{>}Jp;C9)MwF;4qYQEGjoG!Wq$%hPB-E;FFNagC)+H*0;9Oj=sXT+NC_)uF!fop z7Kp`I>TzseR4anXG|Gya$+1slIRL%65eKa{lz$!dPVwUy>1C<RJ0p`q>KqF>c-UM9^vb`E$UKT$n*ng`!$g*WOOYzoEs$q51L?YsIsSw0QE7j zIz&WpjIemNP)ME71&263oe$tsV=+Te39Fx4CyR%ERcylCI#I{x{u7d|h+WCydytjPoO8HF8_{4@>;x1h& zHS{HOxRhc*idr`TvJSbUv}@XdOh^Q&ACqjzXC-+D$DFY+?Rn+|GSBR}Ns|%KE{a{I zGR4uuJn}SGV7KZ3gqz$;XV048ND#Fq{GJ!qo+Kj>X|(cI-lbYYERv9bCpcu7){7_m zPr1_)RrmL~N|>rI&s`{R0y(u**Z!|H{bz5P7FiV7*fw|H_z`(;-w(GJH{C44)Ob1rKnty(NM-TPVropc znH~uEg?X#Hs933c;*HV{dDI}oSCr-XR6RPTrNNZXWc{m|XZ_iO>q3olSESF}F4YQd z>Xsu!u?5Oj)0slkImEgYxa`jl@I`tG{-0s9pSWc{2-6nE7Lpf8P^$ z=ub~gUR2vg@yUIsP|acPhYo<5?fc5)#T<)Wrj>ISZ0XfKO>z0OEH}0F-+!AsTV{TW zrKq_Z5Op7zKDUP|fY?6Tj=nWU8(0t_xGyd=+(m;RT_bS7HkW~anN+H$P zs@M&J9{!RMQPCtD;9@y+x_~6NGaB~+gh$;g#93ZvGkn}k=r|KIxh{In6zB;?yngrf zg^SBIX~Mx3^(O;fcGjwE3gzujIbnoxbW=?qtZ8q?d(lY4kH?bk{_{a|K7qEa0+9{h zv1Id%IiCp=0d_k+iTzbc@>rwssGc87Cl1mjS9hJngQQiwJ}S8ui=3=uggSPij*FG{YNtAOmR3Ac`D?paanH`9U!}HTp7@AbuDmtt zJ^-2Lz(NX91K!R36IJ69rt)Eew_-Uids6g6h5T~r;=XIVp1lEe4k^HQaLh*~o-5}f z;v$Qk;Zwg{`cNm`^A!DH_h|n&eB!{z163WR0#~8h0$Zon)iy+7hU8?!xwr}cS_<54 ze@VlkVxs(8iqEU5ZZw*OSC!*H;$r5SwaT^cVz4uWVELdk0DwkVv*D3;XG2Rm@w%XJ zX`cI{4l0Wyz0ts4cVDTgVfWUrViVT^a^g7hC|)ZTurU+e(XlqiR{-zJB;D*tS;_q6 z1?le{%(lz-b=E01Gg|@lEz?RP1SpXv$3Y6|Yb9|8e6Z#xs&{u4hVC`adeih1Z8h}<#6{AopsBvRBTszu%#{A^0f@mnPl%<_fBUxJ#3B|9rw>bh zRNG?+p{hRc5TzF7rq^-BVdkTJ6U+jrC15#2_&=;Cqr$m48Nw_tBUEWB6UL6I+{LCO z9Dw>NHDKawA*o23tY2fCxU2R1Qp2(Dabe;bReiK>w0zxFH?T|nzL;;mAA&8$dJ?5q z-H8*371U9M9!kfR7l83DIV&giMo7-u`Y+kcCuR~*Lnp#zzJ6KZn`F))(<;Eoj!YPF z5*3d@;B6J#ENWr`icr^ZCwVJ*ZmMDkT%XpHtFj1Dr)Br(+1%yE{BP8Kbx>SQ_vXML zg9aNM0)xA|4KTO_cemi~5Q4kA1{<8gf&~k10fIxYpuq_qLK3pP-?#f)soLGz-M_Z( zRGl;3_ui@Q>F(2g`tN!ZYJ`rmxKIj`l{naXv3?J2C+0^677#nmrBDE4Qs zJgI9w#1ouTEcnbKbAnHoQ;@G0utNTarx#B@NFVDlrS#Ft>8Oe=m8a9+X*E0!^v9lq zVuxYAK?#+zz${$^13z_-Ek2caFVIg3)5)Mz-%S5GV(Qg*DgLLM{|AAwyZcgIgZqHG zkP|4u|4X#B;6Px`T^?J89M~W8{^qgoi}=xtiV9fWry5*6W3x{#n^Z{Bb(tL{(fLQx zQ3aSpUAYkhthMF_%($+|P!4Wl5Eg;^K&|$UV#z3S<*Y#b65m@_T!T{F81V01X74nl z`H1kIY~#CN!iTBZnz)N%>hgFPb&x?8+!6v2k6WNHs;^ft0L(+J*g6z#Viq7qeB=_^{rH+^mofp!M3Fs_ zmHgeDZU*E;n2XH*`3&2mT86zR!H9uce~Q>j0bVRLRLo;cRTV4*(wa~FQZSRDi0obx z#a~plj%&y_eT{_Qi!mp77wU>Oyt;=zi4(G(IcR}YpU_Sa2{i{QI9iv00AiQ; zbU0!5gzr)GgyA+7fJF*s3#v;KtU~ax4ps-FN7zUx1u70}VxIfFF17UK?IpnCgHL0% zLF=_OW72ze4H^4npb$Rn0$Xya3{NQ1kxr7jCzM3)m{<(7)gM$6R@hBZn>}4iD#ul$ zSr1_3!gKuLPp@PU4ipJhM|P-i7~Q5lq8Qf$ilEm;i?OL^Gw6!Pji<``B118hOw;EL z_48zLOTI`~{xq2`XocP@D>a1*2NW-;Pc%{i*m!40d}Q?mV1 zkoDozrXJ^N_q|=;jPs>cE97(+XUjfgx|)_i2|O({*i9DZ#F>rKf!qE|`BRW#dUD@>bM2D{nb{}mQ_n-98oroU^M_eQA=VH8|kHxa=#3sF(>4o5#Seyse71!4Zm!& zQzfER1~XI`0C;#l7#s$ByH&XJ_!ZamkpDA4qT28xRo#m6)C++vi!!DH@Wbl4C}3G1=@xFKbZq-hqH`2-i}cPoH-mFMM5=jb%}5 z=e!JNKpLcBzV3IE9vgjL(Kkg|_+7+a-rV=mE{-AqNI@l4ttC4?f_;}#T(K7AjT@FX z5`*5txF?NEA(YR-y>!weuv5>v#CU#xqoa5%hvP zJhfS($mQRzO-|p(^I(m5w2$iy`&)$X!8qBcal})xY$DOoosAnbP%w)qBzPWh_lEH_ zFyRM>w@+^hb}uM2pWgmM?)!83Q}ZETj9N}nuscah+uEuQW{&XBZdXnYdmOFvf-xP` z4^>d6JmYw22wwIiC$E;hB{J4_sIR3`OkeZRHHAC5ODJP0pz8|2Ics4Q@{Ak+gff@T zT1)n|;%VZGktW;8nR{sA5ag5)RcghvC_}k?$a?z+^NwRxBlD@mx&2c~H!7 znTRO07y65+v!6FozimaN}df@>^b&gZ(lt_!m$0(r1B`Jvp z12I5VIMc{4;{Us*hndgIvfr=BXD_;AymEa}+TB%u$LG>+^M+Ek5Uuc6s)gHnr3Mhr z0$NyAY(1%Rq-3OzG<#WfbM}?Q{hvnt%iL$ z*I7WW!H59tyHL=T_&Ne-v{n@D%_7PMRC0+*)Iw2T!6qeSSvugo|e!Snq(jl4daKQx!-F!88Jf z_GBgm584WCWJQSHiNU4RyEtRs!I{G+tQ%q6Dcgt624s@A@>AKNLq$>bYDRH3Fz?r9HSC4m3N&5#%TVx0!6WWJo+7di z0j=D&v+qEmtMd#22duMe3-jI9h{eHqx<*Uo#AZF#65Zekc|Klva6VY z13+3hj8rfiMFabFl!g*myeq(vSDkDu6Ad>hEw9^xd$+3$9!U#W0gul_=;!%wwJ%<(4l#U*FT^alpOX=9XrNO9QWN zXODHG+^e!tuuA19aKv`-{wOCa_TC^ZY#iH_=EM9ZvB?HIAiXk+2gKB zw#Td;kcH@|LW@;LmjAUPshP*<36PQ*ERNBE`9puw$x>ZY442>cW71xDCTH9Vx@@~K zD+4**pus7>5P7R*wkliYE2N;b2~3`si$Cw(d{@{p(&e2+@YP*g7qw>%OvJKwPrl}R zp+h(v!vSLM3VOmC7tK{VYZIGdlD zI%88gM^h%yH$OshZJXl$$&7dnxKu~4*pb{F1mW2YHC_h4l{+LIHuVJHSLEK_E4^?x z)A&|ZPXbYv0ii}Uy<7Ze4>9_gr6x33;UDuI@;0Yc?I2(EbtZ`-N{>}l<(Pb^j-E_> z=s5VrLAOjbDiN|asJiqaIoXC~k~pWcLW4mA(%^AL&psO&uRRgVJ~s(9LE48U_`NCg zOMu7|&oG3C3Dp)%V)I%p@!xZYS%{Iiv)%8?8}WM*U;>g^pdKUgELJE)l<)y@#PZrZ zAanmyGrCe*Eh9ja`oIp2TUs6k&(y>`nLS%`w%T)+`=cCj6rKzO5mu^VB&3_EYf6%X zoVQeO?J+x)UYj{#L!r;M?dPRr;pD@`OxmyR_<-bTK*d?kS5ptim7UueLyMmRQS>bh z1U3u)_)#rhs)bhg5zo18PrU-Jdfn)Zz9O27-0TorQIG|-g=4kk2R9`xf6ollN?kV% zRaH8VgjE?Fel5ekwNwK}vHI!1sXy8`-9)#L4Z4c1KERVc1#gU70RS|lBoRn3S4Jhm z)->60!bT1s28tGxEF{eDd8tMsn7C&~Ek}m2s4(}7-cbR3GsHugI zyM&Iu2c~0rCDv(I6AUIlRVG>zPoBRzDSWgkdM}O+rVRkh3L@8Gd}&E(2p1L1#T!B!YR0^n29NfG>=;8GN3N2tPJ>-tn@HBI?E_eID#pau89o|2B`3s?Lz5{ zt_GsIfFg)6n*W z#$h>HeLP)^@ki*gnG1*lCJSpQw7!+ipg;zoIros-?OFsmY+#KeoZeh?OWsC9!rZIN z4hm;&OhQ?!EhG-8(D-Reo7R(5a4X5c)xP8;0^-86%xnBqg9KE$t;JgQfPoG7ek7QE z?n!_Q<4E=FGYx%o*SNN9M%u`rEN!1XigzMq?e=-j7%l9lm1%OWRfvW1enh0Gm-A)x zpysBmL+mm5hxfS+jJtJ`Xw43r&1we%q_>PPAzV+btui$D1t^EMOnKfksit zjLXGQMhBf&l+?Ooj)hDt1Y<(+fKbQpTu^dO*|cHdj8WgvMmih=KnzdnS1%HCy1<~= z{h%m%YX!A|Y9HhXB_PKUy>|Eq@Q>*+(&@=gG25$gO-Vyj0%#@#Q$9*tR<154BUvVy zgqE9LYMcWl)Q@T#q0?uk0o{&ZK%T~?(M4v>{A2#?R>gZhNW^8)7hiBZ>*y3ZvX31m zNW1Wwa0UY(_d7xtnB+5))Pci@EY1Aj5IW*!fGSh#fG5*5lmT(f=@){@>bMhnLf7a4vTtAq zdZOB1XC#+IN@DVUx=d?2=Y#XO6xwcM{sXW>HEYQ~o1yr=!h+hy!=C5%3jDxwE z8Xy(w#p-Z32TEV;c6Y54Xi}Ji`@5RRx>#lI10|1l%-$`Vna?xr-xe0WTmw4mE32%F zrLGaRVq#*S$M1j4L?S8Hgq?Kt4+|T;)s@y)TaObMh2$QJt|Sg$KyNOI#<#4 z{G;%^QRBYWhAJEru!DTO$BC4YnTB?kISZ&ji5Ra2muev6`s(7L>f)M7l@p2)KbkEkf#@8Wu`6&m&tD+T&Ja|WHE;J>=Kw`~U%);Pq ztHpjfKaW}y?Cc_egH9Y7ASbW~TZ7*&Mp{}+epb}SZ5vp43uRo%ce6Z>ohJ8$)ys{54CBE5kdLZH)GHP^=ULsBI{K)o zCY|=jYTmNQl|8Fp1f9!<+64vS6l$wsD^`>0A^PnEt>j}@*-jaSzIY|d7kH4~vW1f{YL#vQBh1uswsdZ()HCdJs8$MsoQZB|OYNKTUq zOOkLCPgBQguvURpy;+x*@?5)+7d8aleM}DV`?%`sN~0f?+80mc%xwoJ)QciKFD`n= z=5Y#-x_y~A_#X_=q&jpAFKihEtk?Gwn3y2dY&sbhTIu*yd@sI>HOUOJ6x&Rd~XT14B(@m0|g< zXSD(!JubprEQ$ucoPPQAz=9kLtff?#vbzDTjXwTtLkJ9&FZ1dIvXh`>S=9xDOg>kG z*`MB%9yI*;bZH;;mCz)GbUosTY5QzT-(=E%>Kp3=PHvoxV)(?0(jEt4_K-jE*d-A* zOk3t-HL$2Fr13w`CI%VQDD@Sg9Q-+fs_3rQZYT>_#OQ8lqWN}M0s&zkeZP+1-*=i8 zG<45V$~;5&CzKH?6Lr2gC>>L)KB@(hg}HEHq8c0P$xos~ut+=FCywr)9jlui_7@!c zc~#(5d}9oK^1@8y{n5==oFynsP)ZwqZjLHJ%t$&#e1RW0^E1#fVnlcbV`vA110{#@ z`#`qWTJhoX;7VC^>`e?X$`yI-tuT0NmCGW`cL-%xd3Lf0%PaMSyvi*f#|Hpdv1r== z`~{_DR5X`MdiDgs-^k$Y>VL$v z*FRtEy6uf1&ZJ@ZC<-)V=fu2fx=XCFbz$JS0lZkNCI(V4yskq-t44rLC&@sv!Xn}H zxZXaiLq9YNi9V_js;e#tPZK(p)?1MQ+!mE!>6g$XlTuyQ$xokC>6%%fEw074-c}+b zL*p-g1#$pgP>_a0?+kYsv(& zb3LM$ci_{?`+)v;S_=@W*S9ns7#$lYslnPBqp4B!bzTLB2edtR^isP_mu5rh*X^xrjzM_Q1Ts&&5 zE*yA204mLwSq9dG!YdWXne$ig2(3VIVI+86;eZ%>gDMnZ^>F!j1Fom1d1pNV<(LeS z;m3Xv_@pZe8b%^|85-N(zB?HP#kdVuthq?`l&&gsm57|YPWiG_U_FIBs&gZ#(lH2Y zgy^+}yTgDsO3>!u9kw^R8bUAUwW3*yWu-s?w@TS2=AAh#HGgzr1v+CmNGSqrb`MkV zG>bE;O}d~liswdel&X7p*EjgG_~6R-Nui_l$@vBJ795`X%e_BN9;53ECEJf$KJY9P z5QS&G>LaPv*|ccqp4_gC*~HOKk3=jf%X@7ZOiWh@A(Z8^jB0MkiG-zjb$dkC{aVRxV7u(j9FtWXCIon@LxGYe>nR=JL5BhhdR zGBa)pBjXlI;*}NdXfterk7kx6Y@M_n-5t3N0Bk}F_(0gH%T*Zi#ctEA+hyJ3FF;A; z73;NoxU&m83PG0}C&tYCvr#G})teIEL4l($l*vD5$5CbaUqHu{9?b-V7ovnuBQS<% z8;az&a^ovA=nY%=3xy9b#FVwr>{tF(Qph$QT#c(kL8>H3w^%0z4KgVL=sH)n&VZ&* zmBvS5Lt#~rl(K0XVaBI2Eg}s{z07YkqseSmH(7EAj34j?QmZS*T9UJy<562l)l(9P zXI4!sM62te#%qKup^v?_f{Xj{SP-U+3qmaOw4`H(<*Y6W3z3mJ!4*c@F2vlSFq zl!C74!DDV%788nj04)oxZD6WV$61V3vFz8x75-8WIj5PZsLcl0_+m2$ESF#~55)ye zxH*%=2?%Z0qmrQf>zP{&1=Mbd{!X%CRvPSMX zjR$rY&dVA}%BmWF(CRrV3MgR(U1Qt7fXL4eDj|ipGdpgw8d(b^@`FtR=Q++jJB%XM zWnD!f>ogXCn6!Za>HgdK_dzRsGUu!WVH(UV_NYS|bIS00(7QyR#E{m`UoNIMnH80*3d@1B_yf8am;<~AF z$v9^YQrfl7yK44fD5tJj0x=#IY8pDm;)dqVCaKMv))^T?wur7N=*1Y-2*?=>Z;LZ( zJe*jQBQjBxJ((ZbS0*hRCTVHqLw7KF@%W!$SKPf7KaGV^X)3G7IYOcp9r*DgjGL{7 zDv_l@4ttVBp4F4Qo6){di(RwFOZRQ3+wPv z9vi0L*CA%Gg^!n)89GW&_k&Z&(8uifN#+#`ALwZjhk8VXsKktX@e4N%aot9Gu{XWl zL^g^i7o7f<5AIvca;uy9@6b9szykUQ9DUN~@+H*fZ-1Phqs)H9=jb9qd-VC>2^WEi z0=c|WObq%V-sGs9_={CGrn})q5?AW4Q6QV=YRWs~>h1g|vZ=wfG2#+;1#4Utx;J4i zX*#-;9Q6lRRyQBcGW?D?8@}iu)z=X4+rL8N5npWx!->JQ=OMg}qR=u0ooD!?q+vF{}GwcIaIYB#(lkIVm zbW5h2<+mz3o~|!!3M0VNl}dx`uC3moXQ&Xn$MKJHyaGpbJwjPX9>l=WPnzdZ)&#hm z%&OWD8D4A^Xy_#lC_0dXy`5%^!7MT!fR-p0ju91Iq6{ffZbY)Z!q0kYg(#?%|rD|=`<`g{v&cu zX5VV`P(Q6}?uI?-f!C8-ULT_7$%Xt06-f+YkC>Gi7kw&DDXR@<>?xH3MabI5yBvy1 zDw9jL(hT0D#cD)?T!u2UiaJ#JQe53Ze#R~BRgmRrWD2^^muyBq z1p`~12_%D5gdit6UBACC+^K?{(Th<1nM5vbTN5=0#w9kM+EMJK78`+XP zbCj8bJuPmGePLsf;rD)ntar|{S*8+mj4g#g#rfVZONx7`tNsDXxbI61uyNRV(G#4e zB}~+oQY611bqiGr|MbitqYmd=BFy2xXk}&K@v)!O!f`12*$}R7+8q^hDmL0_$rBKO z3Oo81i<0ZCI|~S;**U~##3jyxV^nc?g%7?IOAY=USt=8SNyM!2A0kGm$op3Fn#WH4 z%nav)cKBc#r_)S0y3a+oQTKf<0G}W(bWV_*&8EiC(WkZs?cH0ws&a&R)!=`9!@u7r zeS}<&;R;3-VXyuE$SNXxggo4fP?1Wpt-5CC_g)gEEuEjWuj;wz0PtY= zI}R)QL`T~q>z!o>VMVgD763%iT)wAv z+<}^(DX<5jl8YCz^6wkkwIW1zB|(Od`p*QewdZ5(}!aVGZOZz3_Kzo)mfDM zzQan+m+=el61on!`Z~T=*QkN9rIlBg9}s448R^I1>uh0EU?4V@LWf-u=|bGDx4Bob zCWuUrS>@$gGVJ;jMEqbW~3L=xQK%TQ|i_KVVxmn7>K zYE6f1kAMsTwjsDpr*d<@5;ZGPh~Uy*ss1Y?{Qeko8mnZ8(8T$J6uxti>5l0w9azV- ztTszm*HbS41{}EQ2wxmD`U_}s`i9D5&x;*pxIvmcfS46k;KmqMJtmMPRH`|5Z@n{i zzkUCz!!Nx%{@l#mZlqr8-ClP=_o2TwAUF|5q_fJSD!Ax|l7d2?JJf6(Tm+& z@9~MK`ITVO1^fhBgIv-~@-<9Ocp7KYrcZI!|Dspf$6h#;Tw4hmM^&+2$`YLo%oS%= z0L+tWL=)Kw+NNJ+0$ryh$zlhMg?% zleKom|5^$I%Acv=m|Rm%;XkKsJ^8gBdD_wlg$P{xL-`rov35TdnHB}&a)!`r77;J7 zeFc*yw89{y>=$Dk52TfqZ@5bHuHmtA-vHyjHN?>w!qto49SQ9+v~K;JefI|8yj0ua z(6DlPnXuAm)*EW$^USJ`4ATNt9R1C7oqlm?H}>B z19;cMA9T$0caS2`ut~Ws2L3|`I5xkbaayfGo+VH#9Y_ElhZDtnwEsAy1F zO23tkf?4syDXhmTHaM3XKn`4cS(J-g3~4&zXQaoMQ}(I%E7^Jf7eK0*27$#9(Es0y z=itiHKxPSg33sriSaNek3D6*a{Os~W61r|}x2{p&4Rq;rh1^_quQ}wk_(!vVke=&5 z0SAvapY9HGPCs+4yfbh7|2SEo>cHdZ0z3G*8m_SWF#aI;Ayo=-|>uZ zKi@W*k8bNX^uz=EC7ub|-Sd1UXHqu6=AE#{J)XNOg}0eA zzVjD=|GrZt(tVrv`XPR2S1!req{W`laN5;#Lv*JePN3vA#=+EIVX>#Ne?f=(nP@kC z=Dd2Emb!rAC&WSyG5#S^vFEenuDB9AVf|kKo{93e`X|~>`8z|iu^>|KKguj#p|yAx zArvJZ;gdSsPv9O|^3S`k@XWQIeC|#RhHZn2V;dM!vpOEacSr@?FX*U#2T(THu0<;} z(DL}PyF3Ihz-*1}_qDTke*3c7NAj+l4taO&emBR5+9WEH=9(MiU%_Mf(@lCJ7yZ;I zkj!yC2)S2-uXo8V8}zmVW4SBp)*UMC5lYCoeEhF51|ELIQfSv4j$M#lPzq!fZ1Dif zjVIv0c|-_Zo@!V9B)v0?@h225)AlWflO%(}qCf8-WyAx4zzH@YQ>YVLezF>@&I{=- zJ)ln}9m0@GxbIP;`0S{T=gj-Yqodh2T*l*Lv+NT6;Ch^X9^Cex4!_tp6 zDb3GU32_XVb`N2^RE5hzCd1{@d3bpRz78R(TTRqAONL#B$cb|JHW_+6$*Qju%;JV5 z>GP3+MjHx#AHJuqF4qY0SXlzX0G)`Gg zjKxP5yc$WTcYBvP8{!Lp-YcCGeGRr$|E5F{&X+sMgQ_n!tX1Rd0nN6mP#Gm{PtrW$ z{?mLQ@lk3wh-ul*U@E)C$?$@q$NA%sp!yAaOfkXs)mD<#_lQrn6yvW^ZOGx-vujbr zj%zJFB8D8P0+fFN4Pt{)yRLKwtzH2C$SMZMvFNs4k`h7NkzgkGdA8BTqlPZyMEB05 zzD`$K7FvkYhXx|~4bEwJD(hf@W_|Dlg5VreNoq));^AD9Y~G8bx5sy)Oloo6SJJiO z+Qw8F3LW8ZVI!LTgwR(Lw~@!U#qPI)PRY|_EAPxjIKdVJJF*Wue>41e!9Qu}UCl`X zrnKSl#iCdB4jlZ}n(e*_j;yE^u=Qogn!78}bN8uEWl@d=Ln+3CLf(wDn2cHneU3V? zo6`G=rT&uF@XOe1;`snh3*+2;#@*ipLHa`+8HDlM!DvckHtxuw=ny__gY_BQKCz*9 zTXta7spXQ|5en7Elp#GFCycD0c%XsZm$+lz-xRGM)ZZAoQVo_hz2cNIbXk28`0VkZ za`7@TC?uhyAbtzW|5mqR!^)(`NEtbEVYEsG88m+I#uD|>K0O2#Q$+5$L*tA9G4?g9 zvIDH&y|r{%{rQZ!p^vTj8|&Zy?Vu?vG@b4KajfIWHd7{Qs?hhKU}1=BdW|ZeDqpy( zM_WPATc4uUAvDdRJ>M7bnW!BFP$<72p1`V#k*#9JA<;tv9KHDLR%rfi{)(EG-qy6` z#IW8!VXZP-ox}ML`LARUZm#fK(kq+}uDCEc|5LXgd*@EER{?&4=MUzDS?i7#JPUXC zn<<&uQE_nFw^}3B*`@SJslfJL29r}(Sw7@XY&ir*_ zVX?Wa%9hS(nQ%@jb`968UA|%W4!hlhJAtIB_KbrNpTJkLBb^DpSHE?9j90Q9?q1n> zccWz1C*_1c&CIMGCG3*Pp7=jJV|Cl5KTTvt`}=!%j-JLE$vSGGe(;n|TO#Hd80rkp z;Z%HyJN!!X@}RTy@~-K7(4cKc4)yl z*wVrUD8qS559r4-*w7QRvWAeNExXzTnli`ws#Jw|iXF+=dVjfvt5uK^%)yS`$(8SvA(_JHrkyhnhvvGJ{3znS;e%KV5WU zv*2G|UUt`*$h#m2n4j?`tGRncw#fmwUmM%cf;%aj4Zz6;hhYYkp}>yxfA=FnbaqS{ z>cci@g+7h}auXX_74K`za>cRC`#+Sa^|hK~$n7s`vt*KJD6jtlz6(K5bha)(oWt3h zO%m{AWJeC-S_--15|-wl3@@xYOtq*boCp9~6#(5V0EUqwL-A0zbWPG75w{w?!3_?eEV4}D-+=74nzSo}; z{JxU}FAhB;&V#>2i`7+3JSD$%Z(;b0Aj#ZU1%#Bg%j!+e1qZ+55E=UyvE}dt8&trL2##+o4adCG3$l`LAljecfE=j;Rx2 z8^K~Bsx?jl>5%7B-BD+;(VNDY-Oa3QLA$;qYKhkRL==_72#KVkC?~5-)AiB2KBWBY z_Ia0N!jgEWdv9TCf9WG-XY2_jU!20*mtuL?8sm3CJe+FML;3cJWgtUbe1O#8n z;SK8AH_?Z1V*iMkCBoCB3pkIR!aJonB&Kx1p(a?7c=wDITZn>7)YFW;Dg$;$1I>2+}+{9_w7k3X47LQ_+;e zHMcU^s2g!pr6>{dy&ZiY*VXwXH~iM4mdoqrpSB*9 z{JPEfO;_=<3~=6r3KNk}TSf)6vMyF+BHo)*`7tf~Jy1ICCGj)Jq}FSPD`IS`t}~sd zfvH>cnt6tsnJI~cR4$G} zzvY0MMRfPJ3tM(dg3R)v?B%f*hL4i>EZfC`hn{xWkR&93BTUbaXAAb8 zmOM;*#9us(k9P#s%(A4IuMgmHG-dNelF z4;mWwy&n4(MSuDuzDK7hfBEW=u|k0k?e{rb(I|JYkR~Gv(mj9+HJ^X-8gS{rtipK+ z*}_&=P&fTpqUQ2^FkfBj)c_qW!Ls6>=U-?podH;PQnGAy;*BOA|$q%TCN zmKn{&>c`Gywj`l6N@E@UagR}On3&RDb&c_Axhw_0mFhG2J^1YiL%ONpZ?1UdYmnWa z#Ts8>nCsW79X7G8vh^)=5?+kAy5i3!wQ6TYV_Z`|x>fI2I>Sw!g9`lJc61PnO`Q}Rst8)KfDJvF#oY-@p{t}*-FG$e4{XmyS zaE*G^XZ!boi09Yw;@iEemy7Oz7X-1?G&}*^FYD6<(v*d_m&4k$sx?D#owSlOaRXY= zwJQGtB0jkuEO7>tvQbiM85FP#KIHb85_p{()+LO)=<~kaC;mBN^mV}N?1mWFNpt_F zzCE!AB38WUo@=yw2QePz*%*(x6OD01XZkcEyfs8x8p6Z#Mt3NLCRJF;D$Q2o_|q;~ z(KBh*>kKuM%fpUQ_HUO27}Q^P*#gwFljX8}s#r+nD`khdjCSQY@<0sE4f}D|*?uy+ zV6XjM6E*e)#m+r~$)8T@eEe!@L*MuMQnW(mjXzLCDL4nuPo~PxkI$Q2N$qDG`1)LT zv;H*A3T~L})3-HeYiZ2-qCa_7ci^l0f0FUZ@|6yy=#w^AME8w*&Dz!#gGvXliQ@TV zGSe_&%t%9n|Ne~#BQBr#Ypu%$=A7f`1K)4V^d14UoQ^Be7(2X+8l>wSwpYl8qqKr1 z@>ok~qI=8d^wP_^_Ul?o{Na=z(hX;jKhdj2KLq{-&^f{^hAQRES)BBd58Ou)hvABX zj!Jv&3|gE_tQ}S1S1Vrcg%8JU?YcZnM5o_d-}sjFs=~C;0e0 z&|ahsb>1$Yks6v-oU!p{p?@UWzLqykqp`=IuxxN)yciXREVmO7S?VPkp)lnP3#w<3 zA&U)q)+W*31q<0sj@td9Av=;Z50D=`^N~_!GmgvAfq@y+R4g&wW51&pB>h7C1+c}x zjok_%bfMZjk>=P}!I?^9V0~Z2wtZnAj{D+RlA* zoXvn)T(Lbb^(4Zk+jjQ@JDH*e;cxOoqiwuLW_yk{@a>b$Mn%y3K|1*x zkB>edGwI$9TX)J!$gj`IQz@c$A;bMJjhpq%c1Z6~@MohqR>Q?YUrD#$MXa@NwQt2| zegE{Ysv}saIH3p{{Ps zx|9iiHgP+q-5xZ13RYn+!mt5<>^#E?4M6`PLyuPY%jC=Z_Ppjw9bsdCvad`#zn%f} zJ$?#$%>N2s-|TC96%{JtgaxY4P!4$?rZ^WY!5%m4Utg%a27Y(r9Lz<%VE-2|yyAJS zpRf~q>u$!zZ=x>Jz{wa9dF3-U!U8V*jIGW@$%wXVPVr})Z5IO`6SjW-n@Y^&vj_d( zXVYl>C9Cl-V5%T1&qQ-1K@o%iIBy4BJj%Av`5 zpAiRH!?%4&2Fkvi?Bi6~qIKgh#xt1)Z{uJ7TBY7(?~E!cY1>V#8tF~im>qkRK_Z3F zQ$feC-<5-X9KicQUTT;1s>6tsq5JzJMEKpG)MJg`3IQn8}0)SBG zCy0j3^p(U}kMl4sP0hf8qhdzM{B*azTGYW`z$s>8=yt)tihvCtEkCX+D%16_UKx#2 z2@KkK%GatzDNXBbr8~N_BJ2zN>hT?a4)9+wIi#@1N4% zRe)jX_1GdRvn>Ng^S{f#8lIBJZlF&q)N0Sj58?BoH^y-papj=nffXA0Z(n{w`R>B{ zi87tPVP2wxyt^&IR_=CpZ$#kWtYe1SEqC17ps4WzXDq%hS;)_eEa_v-{d(S79xqx`qlEwl z6BWud`3i*+rToq`1~Q`DrIUBCOz%S*9m3*Sz$<8z$oHKje`_<+m1u$g9~25ds`#v< zkm&WWe1C?-r~o?uxPu9PW5M zBkQRPPerGJ(a>+NOQW%yGEc-WH$R~d4*Kl+EsXVV_q-$l-22XVkf$63Q*Ti?)qx%= z+LER=?5>?$1yJy=e(Ch==`|?@1(D$Hp4dHD^!v9>w0-;w;B~7S9W(a&Y4Yl~-b+;i zTm^7f{uYQ_lF9wjY`s3%@r&KAd+F}`3728oc*3F5pVir&E{pRX4w~m2=^;FP<6Im| z4VFDSQjb9?mGLIqtWl&d7b{X^13r_t{*;H=wY;NpbaD|R zOXYX(8{oe`^?}`df3;l)m98I<_*^G+{gm4bF1zT{^fPDfZp<`YndChEKgcK@AbT{C zM$wy#E&Y8YPe-})B!ZbfAg}t-!y)7fesYE#%I;6_b}(fN{1;%WLMB^d`%4#ny=*Nj zGz^@%4?siyvzPmHYDN1T#hC|`wwf!fjMil$E3V&0ITmVm45=>x#;GeuC548`hdOkV zr<@DCY+-e4Zxd?vU{ca&S+>?pWpF+a*t>(<;Q!I$viLTR*nl)@SnG*E1kh>A^+XuJ z(YW*E>n7h?IlW-W+{$d&RB%Kakf^$DEf3rsyQD6igfYQfRf7rWH<(kGUpwRls8efWJ8@LwpLWP2(%L^P8IkBUF2DP_1i8nx zh*%3lZ?nNfpSE#5II3wptH2W#t_RL_2XVs4BlVe@v?ed{;%+1sAltxHicw*|9u-KOwA$|4Q z7bo+oxhUa#mEb#q)+r*JJf6fnsHl?{jYe>KYh#THkQtLavRqhFcKs1kjN8=4XtPs=i}+VvOE1?QrL2m(G=G(q|4MpQrB3FVOk74oJT6KokVSwQx1>`if)v4P zKcnSrFEZ{a@Un*`IZv8ti5Vh@WI&lzs%6;(urPj=djC{I(&PoeU8SmSl!XHm!HG!W zM7YDiuZmQ4%Cj1x@$-E7^Dm67e)d0aASoa6LnSVs6ME~OT^|qzvf@3M@(Gp)PbhmIe?(B|rdga(OSU zn)&Db2+gphQAxFw6UR+QEP1r7MH1`DK~kf9WqfV`2ZPA&DZ5&+7Vwf zosC84ai2`B^@>H@LFQ$O&$*iE^^~^uidCuZo3-+6vce}Ii8>Dl10l!d4-21+4qwem z^c9FUb_a@XC3ln~j!yqW@s+&mrE*g+o?E{4`7erhJ>vXNtJ%P(UbQ(?3-ZJ!?JBQ% z9QGK!^cFE+-a-O!8+Q1M}+jTG4^48dHO&9V$aiw>ZX093SqfE58E?dtSn?D}O7eaD%6U2I3O9^sGQg zHcT3&beSu1kJvt&WK2K-MC&OI7ZmD?4Nfe&OhWi*|5|P%7|~Hw6zF^umdgjbypBg~tOdvhDMP*4P~A66C2;6m z9?86t>)&Q-A3Nup9Fx5qJ){YG_x`tVnRn`KQeCwt;3QX1RAU#yhOXQsH9mx2Y^>)! z>0s=f^;&Xcjx>`zT}=YT6=S~iCZC<&lobt)IrV!4YZM^LVn_gRo1sO?jf$J?s2%HJ zs;{z)&_X74)M7ElfTxves^E84X0=b~ne5faEVoJ}&3G)m3wxq`hGgSUj7p-jCnx)? zJMEjD5GBmaJ92CyXkX$LWmz#aOTNEI7M_^9h@U|QI&ieC!w=4<|8A%Gx0E6T_L}v&BB6)xz zGvp*W3zAiGaL74F$vNjBLGlnJClOSTj3OXN#((_Iz5l!SzH{z5cdfVn_153rtGjx4 z_1?R8s9jyVc9n2SqHsi{pxw+vcq$9A{d4Y*4RQ^ckpkSf29{7!*^Rp7F|xaBs+^&) zErX}Ja~PYCmL8_(MVFWkvDFYtouBRb^Whv+$};crTXiK8WqGhQRuSsDxP!M}b}?@v zhKCf({gYg2mv2kyeQdsWZ`j(6m|A2r)&-tY_kC1g^lO{pC!9nOa$4E*%2c2m3#9BXRgKjVpUzL{BW{y}nOj5@J8^Hq*eBw|pz8PKAK^6(rrr6H6R~Ik! znlV;BBW?zN$`o7Ll7$)>XW^S|!n>^&1(wmamcV*C${CG}H*@IY_RP7*_3z`$ql|F% zx06*ahm+6#6tqZxyhj8=Q{_DvcXKSII6iQ>h_p5FvF24@b1H7=iy2buly{72JoqIq zNBGFzLp@p8Zs1DWID%g`vjrS9C>gPy;ag<(QRuOK2;s#0>k)4fKimtcbG#`O!g8dRU7S#P2nn`vw)A%)Zjf=94G91*>=CedW74iv@J=6+vey@JJAQ8URjObfNxRdvA7v=8ZjPdIKJ(i ze8Ds&*Eq{A085OaOMgF8vwY0Y{o~G&y7z_7hbJBB*)xs&t*$>aKPBl_7|_8$bn%>{X)UNQS6~cY7#43b1#pJYDK{t5n2?2 zB`_^kR6}aIVT`&JN{7Cdmi4O0SJHC)tyl#~GJqCL^3>pmgt&@^udVj* znj7r$KbC8?R|LKs!I}>Tu0=uUJnu{GAp^tgPUeEL)E@B0_ZxCzC)2iAM3*38EFN4m!i+LtX;Q9K zV*(NbMaBahm5$^{Eq1{~Fm0XGyG9V2&PdCA+EcO(!_jGXbK?Iyja+0}x%&d^PNg>`Q*r)%j z5b3ixiuEL%rDt*N>Eg8n#~Ia8TmWV;_A>{YI@ILm$;%T)*Gd62gU+AfkdQwhOU8gF zn2DBq(PA0^5;6-|7f4Ti{cjPC_22o#T($Yi8mhQ3VesqIvi=hH1A71~D}`e5f-<7O z#5w+LjMT?b2IrXUcNDhGP!KT>h3t$%OcLIIs4xJJ3k^_y>Ug+m8G)yEHy*;6aH9Sv zOYRH$@g3>+NJY7dUW;m|!`|H!RUFig7mNOx5c!ohR%-VFmB#Sd@ZUP);bhNKE;+Qa zFgI-#53%U8!(Yr_hyr&ICkE}%(HP<6U;%`HDyu=~i%+h4wg(}X2fR1Y)A_dv-Dl-+ozNA;u zpRQh3`q-*CBd@Pr9&aoYMT72^{k#YNMjPkQ`pYfuQ(sv9AczSkaCwOr+tN?4!bC`m zbH%vs)Dt&dO&^3+mN=o?uu}xS$46+Ul3vi^LSl7^lKwUPWAb_7;FJB$;I-OD*_=G}s-IhK%%f$FLkZomRF2i5`BUUp;F_ zl5O3J@A$Bm5{#fLDzz}0OlD`5=W)PX0M=kRBNrM76Bf3S znD&mZxJCXQSNxGwCc+utu%0;#S04W14kyn2y{XtRkFJ9TEwbKMyEOEO9GUOjw3)Hbe0u zfad0uyFU+W9d8-#Uqx*I>faq6Zv#isE}xChF5|BVp7i}Frdx)=$mHPyfilOFf-W2D zv941Bm3TG_YLt(7;|Bs^+Ks0r?zK`e5o~HE5mN%OJ)KYD{vzouaoCv~9VYoC$Q5cz zb|kJQ)7|D5JP!2}x2%k^QO8`={mR@49`7HpCXU{c(iFdMQe?@ZFsdFjSxKkGJ9hc^ zOi8bOZW=LkH^0NIb-pV z))BB);IT$JDSnw|B$?sMPLgZ|F+u~!7#Ei1I6A1pIp-qX1g<_9DcwJLnQc{o7LOBD zeO4Ew2ODN))IVT)`C98ok#os?-tuHsnU|fqr*OBri3wI8s}YnE_A0pNzBw4g4%bT1EsD?4E!WSyORigUH<8gDo?)x!mOZb9nP|UDb8-%A+$0 zm-_vGy%?F12+8y0S`?0R_D^Z;?Nr;EGp;O@AAGQ5_@QZA3jj3p2dz9f`a&KXmRPTq zas)dov2jm*qy?JNwS~`A3RLWnPqLca_g3}Z-LMr)UsZ8U5Dz-z9AokueO2TdYOtX* zCAwsk<~MX%_dZVfRU|iU!tAAXBi>2v49gQ@gOBYahm`YQcq`u-=>@jbVmUp%DBb;8BCyrEm7}dy^Rqek z|7APp$6}%O-vE_pj1Oa);oksR*UL8oxQB|t0q97I8>KSMpc}zJZ!qJq&b5)_lL;!` z$-S5J>z@!0_`mj3)|9rMf(w2F;;)q*%ac%J*ca>_Owq26*=LJFj+xEx4Y#V@b=xbA z=W87PX94BT9C785IvJj|{1N^qrm>Y}>&bY33U_{RYickV!QL$KO?q1m69;-I^Z4>ZPk4YPx>hjOEtxQx(P zmO(M3!5acr7KMPJfeTIIZQ6bS>J?dNDV8bD2-U@%$rx2v!Gx^1p;l@&=Vf^ZIVR~M zC)cgQD^)4d5Ca-2r0vTszM9b)TsU^x<5h>dj>x!mOVwYHzz@KiB+0X>tF3TVOUZ86 z6qY02%tM#**S8h4rPUfcPFWHDMMYCr5H=L*ItD7eQAx9YW#vnXn4O8mt0&H` zU))fUIa+r`ue#b-6*ouq9UNl(G^AeiE1yqY5hRk8XPP~#SLMo|K}u~@JIAgY5XI*g z6Jjyzum*J&d+IjNRz*mQDWMHzAW8?>&arLw027_gcHjgRP9kdMSuMf^breq*GNd>M zl9vtkNb!>ktf8jBLch26oKEWI>_K8<{>sLwiXr1A(San>uCp{lcbL^W)Q(T{^bCe} z_OQ~X%MH@xjq8jxT-1^}$fR`1VBp{>3|y80(dK-&NR>~XY>ZDV-RmT1g;7tFp$3@VAlpQXlZBz%PvX{% zq+M3{47757oc}mc!rht3bUoFJbsbFMl=9zhY?GZ5P90 zsN(0K+KY=#xZ_gfM6Pz{wK@0!ej%gM2h&amzJxjvX8HynWE9>HC9ISNmj*zHq8C}O zoPshf(Bdm!n{VMBjZ#Z2Mj~e(-2p-4H{Fox_qD{ z@D?dYKbhylFwoo`%LfuF-}bqEe2k;{jy!N#xh9o4K|Xk+g8KC5i5g@+eys>@AQ(au z;h~}5oLn4LiShSG$!4KZw5wxcRZ)*c2=~LYz*Z%RdXShLDVS!>IIv{>z&wO;s)|s~ ziI*&ai71kRM^o20Fgz4CUJ;9s9}GZihk3cP7f=(g?2Weq ziI)Ps@8x-MVUS5z0QdlnuHdpg!5VkPyhew2L)6`~5#RgE(xx+9FZfM}eD-Xa3Q zg6lGabjA&h^7lNGy(!aF5_a;#)3r)U<|@`o<~2^@ljy-D)_O{?c%i|-4Z=mTDE?}5 zF4kT8`<&A;1qMk~DqKRW6b-@}4rPC9qY?=s`%1l`aD3c&$wKYaX(X`l_W|EWmd`px z>D#`ihlDGozWqmNe~aOyC9!8EhEj+qH|cJjdvbI&#Unh(t$eQ;+HD*bqaxE?VnIX! zi~$#0)RS1tmg}&1OvT?bObLmO+X=(uOw{t}9dX+u5q|Q>K^p_aq}FA+U&uZUM*4io z^eup5Vf2oO%A)#tF6k9tfF33RTSAy-e{>w5g0mb7%q}H%pAlRv7AT?Zaxt9nQED4KHHLI*JxWpuY8HVP3b zHe2nk-w{w;9`gM|W}kzP+r2RUR6;&vHK)hT%YLKXzOpG~Ge@ubiB%wo%FJfO0D~J` zU+@y5QG|2W%w2} z7U9dz_tnHhA&BH3ak@fHEeuSR@q_ingS^c4)TN?tX-&~3%UsrJS1}A4MHhq zB`FiBYzA4l7;zSuRDDm(L3o<7!p0gfmqWB=gK-+T%{gF`Yz^nDRxMKUu!sLN@Rh57xwJ?>C#pVJeCpYB!=l z3DWDSLs3pr#ObuEK+=#}t)TlxJ#kO=R5lcMut>Pr$sFNaqQi<^F+NWzUxjk2M|nBK zG1hYthF6QQGv@c>Q}CH2$4hepL{|tBZH;K+pqmI87UtY4j|)zKe@zi>Rc3{F@IYW6 z4^DPDFhgSw=z!;=VdA>WNmOItR)0@=2)3<>Y@paLvO5WseJJ%xIq@~_ugEy2^!O^R zg13((-nN&L)K)=w=VrfT3x|H>Rx=3f6-g7yYqwOKz)zH3#m|OG)X6Fq(KfPYa)%mw zuD+)aPU5D`NOg|hC-l3d9!hHYf>ZZe?YLTRmhubaxY#G#lS}IjT>i^Ok@t~8R zg(~2mET8&M@BJ%@9hZF&%5N71&PpbzLJQdpJt-%@d6SS=`BqN-dujQqPi8SDd!V@Mkv`%Do`gaC^l3&inBi~M|SkF-}SCKw$E ziiar}q9-8A6z*GN4ue*+?1IcP<8Jgi6Ymmt|Vm|^z&=CW@_;c|Q_CRuc} zXZ!#PM6+Y8-Oi@%VC7PEp~p*WZt$?XEPP{_NHOGt%Em)O9)s|zw&=}fPEvS;MROXV zTmDf+vsn!&AbFwSOZ@8;nk=!7U_;>vCFa13IFcvwGEG5QTsjd^I*v?~!FA3t@nqy7 z`D7ou?J-{X^9M5Rqmu#IGji0m#`!%7#5KR3hT7 zQS(1On!IugC!H-$d5upwUttL1Qr)kL9K)r9z4xTvY6i*xM}!Sta08S{GePa)q1%Bu)grl{zHHCgfmBKIY}X4Bg9yi3v5WtFSS3kIdc;;LwrYclPitE%jE z78#womkd4m+KN{s)?tLzFuQ4a;dhLVqO6q?0>#r23i8~ME(U}YR?LyTZZJ?tBDtCo zS51hjpcSW%r^UNS=;ov#L{Y<}3Qb|S!L0^FN)H;yS?~Z=O2AN8)yn1pS?%gT+p_-e0 z4Zn~JDoJb64_8?`f~W`h8ECWo4{1Pw+ttYEpsMIu0laxx18}h@AotM?>=nmV%=yvP zp!(UH8XtvG2D9x)OocQzRqQDhIAp!CXWq_=OTIJw0gu zz_2{7+>`M%n`x*~8k~IZfO+zF(g{JjQ?(KI2!F~p-y^2^fv;=^lInK~ z(vdO=k|PO&(&mjv3RQ~`_U-`24GW{iJau_MN`>4MQ~*@K$bT)%DeJib?dLn9EO_qx zmz(@`GpRFB{<-7f$4AG0m6DJrgjpm5*dhCBknO;X)zM5ovid?>3b{#+={h zhCCg#sJV-FIB1&gEvLz26|3+iH7$E+$t$4>Ue`HHdG6)kx5u=|9s5GoJre@py}h}K zj=5^;rLSDCp$_C%jZK3d@eiR!R!~Mc4GJz9MVymf#~#l@CMVRA&`|t zSep1VE*5xFL@~bs7!!olkU3R&WlaF>FQFGd3`4%rEBppTF`385(pvdxv&QMAZhtF% zEa~n?*74KGzjHQNpPZ3{nkQfuTMPJRiX<@Q-P6yqw;AF|H$%s7lfhMW#V%j-ym)kj zM0@#qJ?T_qw1;u>z2-|I%s=la`~XRQ`nQ%jfv#TF(QBQu(mklC{CHxhC)Un|`hF2s zU)ke5$6)poRacT=0BnZ1;m}z{qn_0$3j^cth%i!Yd}(~mTK>#IYy|uu2aiCkUdbc^ z^nj9pb6h(UZ`7gSTvdBB4Iksf&P@B2rd4}gqmLypO<0$Fi~{c1-{vqXTwC?biS;?@ z`89t?zpG>Wliz^;#*YN4j|0Mf@L3oXV$^I^yU@<~Z04I;AJHDk7=N}RNHup%8}NbC zS4Tc?gO+XM%e=FK@VLP?BVXIh>2xOi#~#3%Whoex;YnT0o8qKw7%HHNroC^t(17P) zm?#z8Ok$hF{bm|!J(3U^o5A^xf|Yfhc~~q8hapMaD#TH?uu~lrN89xv-9i4C)1?A1N zN*qyIPNtb_=7vRA2p7K>XsOGKe;j<*)`>`!0*yn1+b%+JNlak?u+UGP0>u`fJ}RXa z7yw1{lEL(g{=HkIIfNC&n=adnl|X3oW;{aas|b6!gUW!oLj#no z`NcFzz*ud2K5Fw_Q$4#&%CZhy=!p!`NgPI5N(cXG#*fj1Ew2{E_pEW?Q3DKF5wJgY zIJG}c?IqrBTyLe+5Gx*BS%e8DM$nSnIY$ll5oQLoeKK&&o1*F>Olyny~EtNr>%xqFu<3!LTTSvXJJd4cz@x=I7T(U&(#0D*nqFX@$I5~mAw(NGK z3^3&Y!h68Dbsy>1Tp7C$2ZfTt_NdDc+#&p)Lj1^roThTYsA8$j;6%5O0-d%KjZ;za zBIMW%%s`q=_62-uQ|`@X2~BdOuZWf$UV;m#kTtl}u0E^ELyTxE6=tk zHtOJ-ECjg~dm3D@GY$!r&BPdd1Q%1M)xPR3|K{-HQ!~MmWcZ%qj?kOeBqVO_SCEOx zm6efU!?^(me0BLeD>ET23Imq|4V#S7cy1&7Ub5w;mcm+9>yAtcI<^QmXrhq|5uYp; zJk}#{TQ`v|f|136?$uwTMBC@!3fWW%{hRgDt4Y1T1{HTkjPUD=88a^R0~XeqA&}CH zJPUrFi4xqYCI(bT*!!|S`aU&xX=+F&5%w-Nv{d>|Na9#!62!Upp_`0(vu0d30AR+o zOWQJOfd+d_t9&KOw@=a}NQe^DVif|-uLQ~y#!1f>Y5X)@gc%I>fmZJ~VI7s-=J3-7vmM-nKn(c7_!ZF4^rDRVnbuM?P1} z9+lSkQ=@0q3~k)E%a+wmv)w6T`n59DL5ys%CU)LdqC@2{mNOyvZvbDw$;d|T!K1Ik z|AA`lSPBBkK36f)pASDDcm6K}=19<|qP-$K9B={Fk05=}zaMmEfj(a}G3rTVOQ1l} z-+({EblYFU^l!kcz3nF7JW(|_`h0@=cer2fOHpL*w43Gm&;okU@8c0#sTfToR-~)K zL6BBE3x;dy~v2oze1GYla z!g84^!Ku@Be3+Z|TYIF#!z#BY2hw^o)U@RE9a?+u!9ZG&r|&vbrLF~sfs$o0qKdA| zqf`SwL+C}6Zfp(8MJG*9DMqOtkd1AJt!2XEpy8m0%0=AiR)VWrk-0$doZgp%yf1k?nZR+g2?6Z6Ull+E50 zkaR7i<8b1`$>YZ~J#0W!*O|q=Tn^GaQ;1Iw zjhS&%f%IX;TxJ~=v~X#q*X(LF1w}*n@~(^fYD>mCi|r1N?3TpJ5tsR)(nd(#xs+x$ zC&WJo2(1JsQg4)@ZU*hcun8#B|LA2W1YDOXqa=NNIqvbRfmaITj{B73DUEE;MvP#( z{|YdonZeOU^44{J((Z!A?%9+mctgrOxPq!q;||MTaxy$^ev6=vcmOpX@Qm}WJ55p| zx%4D+;Hj9r;6g&FDlYLH5O*UGXod$!LYFyTVnUa{5D`(eV6;@V$%`tuFn#`P23pdx zsFtZiyj14=1&{iH1DRtNu73&`Km-cDavtr~1X9we` zwp*}@xB4l#ct$WX7!Hm_SlDHe-iQeuR)1|Ls5-fvcdVA7;L`ytCq4PU@F{5o_J=Wx8;y|IZH11iiP$SRaaQjsq~J zZJuCE+W>HJu?g_;v2d|*umFE-+mJIs?@4PW;4o51nY#w%_de99pI$q}WriWm67S2n zG#s&52)ND2{+o%LJCcA~Yb#B1U{nU$N;$`;J=&6b2>Q2^pJB-V3tKXDaFc^=&+=`@ zci`+s513|2P-AXfe5Eiv<|X)3C9iCFKxcT#Tgjc) z9^lPP&Xcdke07nsZ-_o48Q5KmJ}@Xo$~|ETI=Wy8JHHNReH(UBBC|{z8ADz@Jz$g# zZ`WC}{b5)d0&3?}#x-`&HMwegZN-&uHY+2aH5dbadZZuP9zXUjVzTa8?&l!U$S8vdQ6ke(Dz4HF?Wrht8%^kZ!YhVy zchUPFe9_(hzbwyWf01>@t)|UIyQ5;`QVVIrgks`%zwybz1x+PiY#5P5nUxvg^6)S6 zqNcK=GR!KhBSBBld-Br{{8_G-HqsBEBcSiJ7j&+E&6_uH^zsp%J^K48y}qra>5DBf zGF77D`ftNpDxUk!+I+hA1x-mGwz8LWuQyznZP}`eyQ?5+6Z>dL#>eAPg0en9+03i& z0z7D2!s6HZcJR7ZNORj4Wvo%W+T;RuM%hs+n=z}k}sf~Pk|lIsQ31AU>mwm+=tFKCWYDR|xc^>6Uc0m<@@>q;jIef&aupNPcCSF z12Fzb3?_Ka&g2jZ29F#ir6>53a(!2BKHOn|XY0v8Ph#)%o!?V7*Mq8L`t@a-42Exf z7(j}2o62-!7czFCSbx%9>tFjh=Z#;t^`G40^38NaxkQ>&8vz)riacusi z^Sc?0z$}_p`3>NzGI;sws^Af^dqqbf?dE;07jDkAt9lIFx%Oaf(bAYu(bx_m+00?4 z@D2<+k0C*0l~;?e=ODLLIVQXxYm5&NkOmF{(q)kRLcPQ)dH+*eQd&Wm>3)TOa(RSn z*tdMiP;)qVmDuBM@bleQ2M;|D$30p0b!W9br`i5hzN(1o4f#*od&8{b!_L!F6?87> zwmm%p=`~-FC!gFIeWY*f>?y2YvbM{_`)R%VvdE0z=<{jM%3dJ z&eTkq(Cz}IZ$$GkWK4_X&=C)+vl$}T2hLZ_2dS!3a>a@3P*vT$?~S_bN%(V0Yg}tf zvv`Jd#Jl*ImaROSdv+<&TqSvIjPVUQIU;OGo%xis3izQv%J#ro@K%4BA(xH5Fat@m zR>D=e2ho#dh6sfer9NC(Pm;}Rlvf4`DyXeZ{^&$L-HLcGJCJ5g7t5+f#u6c7VoQ{w z{n;_4`V~y$aII5Q6tY`+D3?uc$hwug*r~(@Eas?P^$>dkQM)jb?pqHUDZdeajLye= z*Jb6Jl5W!vnqcacd{Hj9ZV8zhwFE2$T3bBNDB zwz}H$-|6Njq&hEE5Swtd?vea);#-N!&U5cI4<+PFdV2i=!v)_Wzp5v_LeE%Z@{r6;+#xNqZalEvoe+2sa4_F&HDVi$6E@d(V*Un*pXo@ z-w1?l{ZifXgyXIZwceJ8AAPb8LK*((lRZ^Fuym^RH$e9m0^g;u{GZ9I{5^4W@8 zr5aqL*e!kbgK9EEjaTRZm>DOrJ@(3CAG6Ntg2y&EQ6E_dl9y05{|!iFx~^N@==9?s z-z3WMrQXWc6nZFMm^!a;-@z!p&&f% z4#Dc5;|MQqil{KqT7>G0UoS~HnVKL7{w{q?h0_)#S96zFAL0I|wV{=GE)@}cZB za#VF8_1mGOvd%%4zb#c^pT<~TQgY@iHtyVOCoL{j-@BNWdnNg)`);-4-Rra_H>Zm2 z=fWnX=6cqO?|wDv>y;$?D#X4UL$)dJarUAg1TrpNPFkBQi(9F?7A zDRx9RjZf$Al#(@!ytk31hy|-c1UI%S1sBFc6O-Rcyo!g5ls-m!>q)GJLqM|u{-t+* z0}{5ry=hQ={~NHBy`H_C>R@f8@P78A?u_UNe-61|~oU z6!0Nz(!c6AAnQY#O+kLSKb*-FPnzhQi%=_-E|`)?rl40_caTU$yB8Q=Hef&$-B!f~ zj+7~Q{BY`sCkh;{BLjM<1FDm0!C(sT)yaVLE0~B(hlOChRs11YFWXB;+@a?*Wcpra zSK~#-{?vAkp>1}iiJQ|NPw+)sL)#>A8c83b#}~=^Tj?IwZN;c8jiXcy^YblPT1l0E z10Lub2g+bF8+@-kxR+3dl2D(?;x*4bR4_ToR1xNTdd=!&Fds&Y%X28R5iB!EXgbVL z*Q5PX(%Qf5CWM}g9hEShnaK;$K_D2V7D!Zc`DsXt1>AGYK}H7YwwxI z;`Y38{KrF%*10#Ikv@Ki@2CG9tOt3Q$OusiT8{~c1LsiE8Ubk}(RJhUZMMOn`Ud_` zYgDLyMdp|KMuEZkDB1;Za(tP7L*HhM%eZ#G%xIg&*vIYBJrgR+T7kl0KBEdxg?aw8 zvGWJN&VG>!ntt3J`*`>`<7IYQ<%z%r=>(lShevLLBd-utIUYpL9zZ^;)a?P<)v5MwlH`!sI9u8HACyu=9KYh^z| zZ!R_EgW67=oEF*w)PhRH{Pd$OyBO1QQplrIw4U8!TH|8*?*981aP7AT{$)N|rD3ZC z2v`j$H+(C-l)anH(BtZeFFQtoOZ8X1az(rL+q|vx@vzX4V@}a=xA=+P6*qp=T=d~f zgIM@`BD9TV_AuM)Hj#(B-Cz5r}F+H-2EQua+D+LD`T?UnCXCK`>@2*$)}~IwE_k341GkK3H8?TH#+^jm0^Q%&+LJ% zMwvw2y|Rw7B23NqYG}!f>jJM(OoX8pZ-}-7Yh=Yi;kLo~Ik?cLuCz|Wt)HItt`c^5Xrl$qvq4JJu+dn%iMrB{BLUwCkJ%A4k8-1VkQUHz#_ z7-tjk8!B^L2d$la%f=4j$sl_p1k$KT&w&fBQrX%oWajF7@Fm98+`TCLf|1fqhp zHm>>}KK+H4Cg$)^S>)~AQr^s5vtZ1R9mxq zV&zjF1_^3<%t+eHP~+9?i!WPms9wU>D>xSqecMfRoId~TT>vo|z9)(@9+{MJ<0ihJ zwSdl^9fQR_5ICtiwQNY^^HWz-pC`jux2u1lNfz*I-dCffm;(_f#{50fyEP!c@I1}o zjdfyq#h&osNF0H-s5tcobjmV*3p(yjyCD`^<$inDMwiYn4X&yx^D)Z^q27$)7#!l3 zW*FjLXm_--B+>cAZ=uV?$U@@x#1iy9F~X6TZU(;I6}MCY_IFF)g}uK<`+gq(yBE~0 zlF)Lo^JoFhz(>y)&ff7CJsEeiF8sx`9Daqxe3k%|5Kic$kUm%&3*xFa%)=hX~)eGCXo!$>$$k}y4tbxD@+>sXd!kvR1KAuw#lNJ4$NoB!gf zR_XI%62Ge<+g5!S#duDfoGPf$ zPV??O@wX-8m&`~^JlKEHYgQg?Bf|qBY%66RvU$0rH59EosIA+preCh1&1D54>MP56 zsIM8hG#$kQdZ;VVD`r;2q^1ucT4E6g;c{hoe&Wefs~i0dD1Fj`$ydE6?M-EC9-j!f(2Ke=%o@iNxl@A&H0R|pbyyA`6?6{*HF(o(*P z)5=7EWF6wY)#Wy7xCuum^eF2$*>LK z9dRH*zvYTijkT%bpV2vp(I-3inK`ZfDprWkoPLM!jVjr zMC*J5ytt5YINGkxAky%w>zyeXI@BX{R4G&$B&~w%Ri(!sHvmuR2m(imN@T`xHGS~- zF3I~lSi(FM$aZ4srfDA+h`f+ zSjtrs&3_@u|1E<5HiLg3n^Y1A@!>cqt8Z;~bguUEm(4v~uBMB!X}WZW)FTSORXTXj zmJ$Bt9{+AvZew z$7IDzJI0>lmXpNe9=%&)vF;o^;C$!lIWIcJFbr!qY(8h1edq}vmX6iw*A^q2r9SlF zkhDzZChC%S7Icp{oN|EJ0Dd?G03!~9L3cxj0PvcG;)0OIU`0vpEb40_N{-)A8 zZkpT;;9ornL`$e=GA&F-_o~Bn2e>Uj_5V14pNPz*^Kb$*S{e)=D`alu!^m&TilWEr z&Ap6u`<85MH1id;K?5e{u1fGX;IB5es0+S2WP*(;@q1u`4VO3!)XrlNd7beks_M}% zF&6F@r%~{u$p^Lz{G{Tdk3akD{fhc-ex+C2ZTfVY=F&6fshC84!TD*Is_9^Ev-g#F zSL)O0W7{9RpQEsJD{G#EHGn$*fHo&~cS4m98qrZqJ?)U*UM%i~6Y1ab2 zD-MJ4GjH2aCOlmMZU!e!xdH3Y-JlbRJ%fz3RN;*R;G2%qu78h_223S1-1xz+`^j@H zy(8Q2PdlsJEGwz^!k{abb$f@FZNu8483`Jey<-ICv9$!!OJ3AFV)-MnZEs1&Bhw#8 zd&203ed)rC!5FS(%6(sq#Z*PF)p)tn!@rjj^qTp05`Oec85ct(`7O&U&l`W$np&W~ zX><)$9*~mPmT(i_rGhTCo6n6`&_#VYd@0I4cU6i6<||!#P5x;5dXgV;Yg>>%yxV%4 zn&KnTZoKYqFWUd=<>t!%0{84WyId|A&BF1LgXSNS$Tp*g>b#=Qxw%QSJ4+iTPP=gTEn)44klqB9#=%%5U)bl6XMUm!aW{vl(V}+Jk28n&OszG&3(nCHfT>SZA89uBL$>A225qvpuhkzL<7( z%QtuS`?p%7|70Bk6B62R&t=yaC$}^f(ia1Fnvh&Piy?ieuzHt-hfh`2WokA_%UDqG z*D{N#T7BiP0=iC{H~3z?yl)z)UC@-iGFP~yc71ZW>s8Lf7d%i*(ZHkL@WG#wUB{-2 z)7AMEw(z-ac<4i8GQeMU9)rpH6M%>A@x#ph^}Z(ehB-p&-6fq0giiKpIz4gB*;79x3nLSaG4yV5y&uuW@R^@Os{gdCKD>mxu!l_UIrT_f?ScWZXd<=%}S^F-H0G@#L*0zBDCZ8gI|A zF)iDP`nGE z5h=83Z0(ugu!{R;3xj*#g(n@khv*XtIUdwEnO{sk_qxKL4)9x0)e|rMqk#GeUJ2{Z zEXjr&1OMV1m^k~>Y^)oMQ&1OPaYfz7OjbOo_y$TDv;pyd1?x}r?4QRnG<*Nt{-!I; zJMu;tRNu;mQnzS#`leTBqSbzGMEzz%-HoID{z}Aw+!?b^!V~mA4*1zl{@*?RPZ9Lg zMI^x3<NzY|I*_4yX24afIY9LJwtj*bPOf0)#y(JqBbV@lSH~z z)-V2l0ltEPyG+sUzWe8KnE7szKdxMGF*DwHIJg*JE?AiDZ?0VM?qepsfs7bWF3b<5 z1f(@s1ZB(!G=*d-S_F~oLeZ+PJ%uP2;ZCeg)39grP-BI)y2z)i$@yHL z7|4uRV~+)A6)f)xRf%lqRauJ5;C4q3(+%TvhOmcoDdplukMUsFW;S~nwv79KiDg%MSn#x zXz{R6{a7-B*|O|kEzwGY;o&U*x}o+k^OdFI4-^VZBf`=1}r9r~)i8m;7)V z>nEY)O(nMY3=Xjq`NGDhi?87n_9@D$y=KYQ=fhaX7pZq~mb>X&wOdtO{U05#v#8N| zcdFsA`Kkl#UiA#RKKfN!W!hXbD6i^#wlPN3jw7M zA`*e(`0%aQ~iu_ljl$8k)N}SwZdzOTghJXKTzW`xI5sr&sLbQ=ET8emDuLl zH%6})=|Z@jJgKW|PeF8l`K?$Rk+iv)j;(6xCL zTI|R5o)7I~W zBDpqk-O&X-4(DE{Pw}({UW)r_ggHJIY4Z>-K2P11^G&!`zkgO+vG8PLBpe}*RA6fa zX1G2SSapj?XWbGD1xD@`R_Y-_6@dRk)Lyr;|&FuZKfELK7=n$5#Niy zao*;=yKHdWBXn!y$U94rqGZ+BQ_N2x5ZiRK{1u1ze(@VX{FEs!esMw5RZ-Nb?_7|U z8+JG1ImKhIxaoe8fff!~`yuA;1WPg+Wn1#)53b|`VklYo^}1L zRKctq+UyM0t#D_J6~P%paaRGVmX}`*a6XE=Rt0^@WFZf>f2AlGHQRSndoJ5k-pc}@ z!~s@@B!$Y(?%@bU*9l^+WJ*bkG9_E@e0+DF{DN`^eumw; zj&#qVd>|sK>5K4h)e28~=OIe_c>Phcn(@rF9)7OEz4hZG{F9)_cenhQ?>hRoI=Z&} zcut@4aem-s*5RQjhNS_SHWIEHr9T4N#;6r}f{*Ig1zfga%)bF>VZ;f-=ScbK5OO6^ zO}!B3f{){esba_u*|ChJ#%ajs87vN*Z-+gO{||R>0aZt{t&1+)ec`rXE!gLR`&kS8|U7;-#%x&@y5GvywQtM-LtEk>gri@ zR#(mXz8Bn|Cpg7ZgS{d98~3C(xMk^+(AUroD1OZam(pG2xrm7;X3gxRqPX; zxvy;cWe!^GZn{HTQqA~W8MimkMWp8Z_9!e}U2~sUk`#VUzP^8fl~0LPKZ9APWfB{l z`CSHIggnUqhJB79?;FcdBvOuwrp%s@3pm9UeW_0a28rc_u3~6J0?s7Lc#k)6FnXA? z=RiFF_nTBeY7Nh2Q~ERv!XMc+D2gOqwMKdHlAW5j-2mhs=h_-UuE$GnUy%3My=@Q- zw>ElZ>V|D$UoC7;m)^$xJf0hTDI=ZwDC%#vv*s>SK|xeBqd|?QGpxH!Y}~EI&?;CQ zA=YzMN(f*tlW$ zqqt$c{_cCvIrI;^JjT=c{_>b5Z$4*-{skDn6c2nbMS(LFI9?HTY34}yQUWEyg6 z)LyL+NMr@1LvDb>M6IB7=XT;q`x-PqoR}v18TRC;Z;Na8*q)jPxZ)*^Ly76O-WKj| zVmWV4($^cn0@$DWcHK?+&;LmqljIj9y4WzkrqEW6S=B@$PZ5SKO&gmX%wzj1Esj+_ z=v`cyL0WEFtclVP7^z$ye(0D?{%kxIuIx+K<&*mM$8-fD(i!W0prYs?xYHgV86bF` zRkS>^JmQb99wv_g1OCB&X$Zt|Og1-PW}!mtbO#KKSKhv08{}Fu9uKfdpXo|r=?gY2 z%O}B*QtWo}B0Zafi7%`>AE93rFU|`{O&NyPM{mt2N>JGj-n&@+Mq1|4C(_=aZlHK* zEB-nPtJ?ec&EzQZ+Q}PQ-juhvIy?6ebr78>lwzI6_%G)bI>rvR7L0jyq&4znw0ICbvDKh<=M;- zd{r6^)+(0f#4RJ_^g)VBadyE%38P|USsYU}RcX>vdu;OIm`AgVxG;kCas(r{(gS=X zWRxmbu0mE6B}{C41uXbp3NIuSL~F3MkBZgtreFC0=JLK6L)W*9Q&N}&LIX&SmJd`+JGr ze%(OpipqG^-T(YZI1A}*s%JdAPJSru?x1*6KDexjN}@-~-0~8a-Q}Igv124tl`Iwq zTCD8D{R*6c7hiuYH)$OTjmM-_eTHcrLx=2nop5^TVV^mCxgp23K!TXk>Zp?LfDN~Ns_sZiU_(bkmk5_3fW+VwGB zL3j(KoB-53H^VjqOW-K zo*qs$Kg~y$9oY?3gb0~9zKTsyV%2I_?OAQf+-VWOS=150L8Z|;`~s1GHKO*8MR2)z z_YFKLg98w(LHeDI=3oT)g3muZkbGSl?u$;`#^|~WpNxg#2n#)CM1a%bFfXN8NQSXl zeusx!)8TMIB1E)Igy#%~IAex}4XsSqfVPZ=M=ZD|ZHC$I4cLfBR^E1=wTt-s0dt3| z3_WUpRvFaRAI_J5> z-5?Tr28Mr#u72tNov*U(&7;w31B2fapJfe50VgE4(`e1frtzOi6UPW$mrW`aUf`W( z^J`|7|B(MV%^k4{^U2>NUu{f4Qa|sd=o>bDWJ2O19Q~B1W}P*U*yzSQRm6}w=vjdL z>@(xhBue2pFU7N1Lef+nwKm|N2_5KuzpK|L`98BHahnQihWE8`@u~6x0fle)1ynpQ z2Y>Zzk`dy0jNwJyVggqKk&y2Ii#rN6Whl0*2 z+ggpe?D1aT*Zt`>bdmOCOfz8g|5A`C_zO_yN*i}W)14T6b(*~TVJJ>H$haUet~;gM z3r^DmqW~DetHo}e`y~N7jB48d7ii|Izg2FxyFfv$q{=YBH|zkKY9 z9S)RZgQtn{JKH)(xGpzx+BVeuz$_#3{oW5d z_eg72jH;ctA7?FfKYUI1CvqlJC#+cPI(csxUmVyfjgMdfN;p9n>aO#Zs8%s}{Ye9} z>Lax3tf~`L{^rf#FyWjkSt>%ZVO-mSemEm6D z;UBuT+#jW|D3h@>B#z-*41Fjv0v^%jh0`PnSkgX2qJ^7QjHl z^Zgfog=HPa=|LC5yHWrNW&OLdMd2I)Y8+F(LD;M#YA`unx`2S^g|aWcjY0-wxVoP4Gf_K%c3(L{1` zDT(6I_SPOGySGe;j=-&T)}o342a5UKrgPJEJ#}@A-L&8h-F~F56i)VT%DxK15mrcZ zmi~(-NHcKZVJ(u$(&XDo0Nx6%@PRxu{xpNq6Wk=@Gm-lKzZQAZ>Yk*ljlTB`d}oC} zFfqrdkRg9b!4o$*Lch=`2*fRg55;n$%?V{0`Wp99e`s0E`Le7vL6;>VSklYIbH38g z)Dq-^{;4ZiEQp@gXor}EVul>3TpQrI?ml6Wesz?)J?X{DfplO~ejhnZy_6DR zdSf~0n*Q3W52E`Sv-g+H%!h_d+H(fXgvF?_ZhR1r>gH!- z(&QzpWaBg}7=uKd$Lq9DTB?E+)R0q_|?NfPIO7)Q^^PZ7BVC3SrFk$lt$Dy)^L5yO)w6 z6&KuX7wpGoKsZ+)VLgAXxRkTVy#Sq9nSr}84U0QeJrnC)E5z{40C_v_=k}zuPxIv+jaS$eBGVS>D%0L ztOYxL(|_-4hK)BelN_T0Rke=KzxRwRd%Z`+6@;{w%1+9grx!f=ku64Z-<=Wm?FEh( z(FZAG)2NuTl*!g{-?0KJxG4eWvRh-haWFAV&VRk4^x2oZkqqm1oKj3xr zqM4)9NE8!rU4a($T2*>0)|DxOhdMoZg9m8ndFbfcT4TjbeixMs1|KHQb%yxbc4b%} zY8y?I#Xogd`BNxT){OkbKjr&8iuA>xx>z}mTk(6MUc^Yd9dUN(bax#F+ewtqPxw6Og-^dh3*cw@Ikd? zK{yQOQip3Z*b0&R!YpogCvQdFVOV8+fAaTt8}TcGD}QG^^~=pYvxJY-o!vWL8%Bm3 zCQp(VXRm`1AWur`uU^L&&OJ%`7ZJR$K4bLU3$K%>CB8%adZqi;&}ePY=`R3XyW7xt zfuQ&c|HhD@HHGA%SOJ|uo?VamAo_i{y8Q!|awJ++`rA`uDpUIjsFPZNm2OJXp&@ds zWz`9f1}6HT->49nhDbY-gt{JvAS{ZrLZ!}Iq|>0^_Hmmh2uoysOscCUUT$H7o31rG zg0PSNWk4VMnUtCOjI7(a^c(|P8~1<_2tSr8G$T6Vx+$fv7~A=4Y(AcyCR)~8@*6LK zFvrCkc~`4_!ibZOp4?eAYtAQKTKack<3BjyWM34LlXb@jgZY1)&dy-kLQviAvEDpS znHG}|oZVYzsuOk%gzI=irS+hKL>64l!|hNA2=2CjS_d;#um;@cH| ziG5-kzSrID@7yb5{*r7=ZajWZ96f_CEBdCAAD6E21N7oA02hrHF%F*0rIveoGFi=T zqyuKo)#b#2Za)^$RRZD@W4e;`un~Y$PM8|CtxGNXr&e(>^+TN+30b~REy|Mg4O3|Q zD9x}V?w*n_<2YW64J4BPlm$uDNNO0!>?SkNic7c4_GwEU2N@2@_+s9TFX%Tp z!@5`7@7zT&c>-q9dggj;7-d zx!^%!yqO7!V!B0~C#X$5LCfybR0W>)@jwS%;R*~*q&dindF*&SU}Fz`YQGXzRq1x$ zN7&$&{6>Gci#3+?vKcxEXaHrZcLcmp=Lj@E&lq>uuPw)e%TPzGb(1YB7nF0*c$jl+ zNcrdG>FRq!dS6#{QQ8zMrkXvALSSQf4uSL!)GJ=Y=RZD*Iw=YgV|2XUd;O8(>*XMq z2g|AVy`|>(N63eskGoH%t_UHc4UHekc00TJF}Aj#!89QZSofwcKd1xo4J&WtD~5?jkRYpzpfmpZnyj0^dUx_PDIA61Q?{vhI8Hm)<7Xe>RK2#TENlFY zk9V8!M4A_{skOh@HB317WLqftZC4?bG}kGfB2R(6A`vpLN*_%8bw!fWD{V)k%8p##Bfji}>5)bRVb6{jeYoo{48-d0fdFi zx>7e3IJ9R-eX(k~!&9yzlD)oFtD48b1QqNpPOo)PJIZ_f5&`tXxdl5k`s9K&Q$lm77+#9m9tvuKAkSF z`(B`+@CCqvs=M%M!3yr3Ve8Wg;>|`Sm0UeWy^5ZO=V&h-^3PHFe*tFn{NQ~KC!L;G z>o1?ZG)Js5X;-fLw6I_xOzlzpER7SRmF8Pnq>A`LQ^emO4wO?F0%8mq#Fl*`R&hKc zBOPg{%6cdU2W+)Ye9W05U0iM36MMdz)E^wfe(YC%eiPX8vZvHCQK+)9vpz-SE}Mzq zj$t?kFnszbf0^+HiuY3E^YC+@VnOySC;@7bxo5DVE5P0Q-RgZV-4cEPm@eBy<^90> zZ1$)U`l22^MrmeMh;{4l_mA`MvpbIz53(&zgbw3zQ#@Fj8uGMhzt@RmB(o@-ZREW# zlwIl@sp73Y^7qtGXQYez42jZ9-7}EGRi*9cR0J#e`dG%<2xt>w25%0W>){lTZW|sV zWtJB!cuhMMn0Q-wFshk&Res5Na(s+?s9HJg4=4SO-g;M_jI4@idxFk=haido=IALP z1+d-f8qN}0`9hd0I9WfwAvbx3BdPP)(~LX|T?dPIzIU~c6vjmpL{cikukt9FHgo?A zpwsj0cWH6Nc3tm{xxx3FgLbZi^{LFVigMA=tBY2WF=nm8u+Z4=XmNfzewY`4JjzyP zd${P^NK0Jd5I%zXi!|_R%bNu=6gsKwc3m&p!z&=gdvT47?*onDR5S9grBC=*$gexD@I5eFrFQV;3iWg1w2rJ9bIuJ|rCls?Bjwa$^N= z3MIL8na?#xci3B#(BAt5O#HE;{1*DkC3n=Azme*s=p1mW$r6_#HS*J~f~R$w>PlwsCyjw3}E^w|@rXLhX3Clq=`jV~cA%S7&1 zm|PhXi+PruDye}~&IRik`&@4uvOwP0bN;%@f`%y3JbLiTu6@HbY_pRthVq9|a7LjP$>9e1IbiYQAnl=`(KbY)JJ}R`cRAnI)(t|;Q!wOR`uj?B?xz)ozlM;gQgZd=Ttmy?{S9;Z& z)kp$C;De1nI=v37~mj* zT$un1=Q~iW;Tg`NFZ$g4arbtp&G+zw=k|LyXjhZ*3fpP!&2-?u5DHyyMCcLi8!NQdO?iUNlT)`FL02h7yi5)@__wbnS}<&A*yMDC z*HYe?>ZT|hj`T=e&1K|xV}F}^EBJG<9E$VO?-(g&6>ewYg~AKYU*+B!1Lqwl^4i!1 zuq{dPksIi_DjEoXge}U-b^#L6NJP>wK*k+0goR@HV=7bF;ufgvlevNXumw&tQgjL< znS8CWQX&l^`)CT>lv|}~ne@8t_7&QTwN9dkj?HOU1ke~ zYka5-O$bU=$naT2aU=$xW_vi?TqqhKne=RoKkbrG*_Y_2smbZ|hXB8K>^FzoeZ4Dr zfmi=pbX;!p^Uf94p8Ny<0%)`_4t5Bjs~kt_%s^ZqSh&NcbyH-8;ylmxU&$#gv=1SK zaJigFicRMga31gu#TioYC9gVYU z$>^*2#g>$jM3O-E{RQc?p%ei@7;hcI#Zf#?UFadqX&;T#Bp>jV6#C3e&Ngk6mCLm}>{&tFl z2*dV^%Hk~;iS|=+l0d4hAMe4rhY01rV>GPkpc&TubdOAAUMk7C-{-CxUUf(X5z$@Bf zA}KIbh$zUFrmVHeuhxk3A!Z(3!w;#H#$(8{dC;Z4QZ<%!krWm2yf?;T~60jkcPWQL!l~W%@7|bC(^z9UdKQ| zpa!p!yCOzGBE}lu>HTu!rso22D!Wedf=T8vo=EMo2V6Gud)-LEZBmj3kW?!g-{vu% zq`h(<-c%Aqc&N=2)AzD|BGUnw&Qt$Zg3!o6dPR7Joj#@0MuE`x<2a|~EN`xf>AGOC zfSq^B&QYai0TiTAW;wA-#2B6CPPiY2e?D|*wED%Omb4w&&9#eRTG~x*|R8^KUj*V>?N2YktWUfKm9nfS& zIug${O&QmBj-Faxvx>LzwT!i&S9QlfAKu}#S)98m>bgFJbID5k&2uFlR`XTMx_L1- z4qO|L1Me;buybFrgeV(kZ0JE`cRurpYRoVWz`nI-d)1D*jNeC6M%DQwA zE34`B8P7(orO62)H6S`=Zz|mOpQN1z<8y>Wk*HOMM!JcL%1fK4TyO}WPOhcyxO!#g z0buKYybLvoQpxkXpMM(hqxyxuS<_zt|IPY{nE;IIN7PXgPN(Rwx;N8BCBrG7oGFd= z_#cvlyjmW)6y~~p9QH+}xhxItI~FFF<mtdlIvdP3-@xbJR& zHkZy+CF5eq4o*L5q^%nykkbz&=E?-b?g(01<fC;@}OCK(i zIoT0ktX9&~)nnXR^KDo4-F1C>GMrO7#u(NrIx%rLiDDh{4ABZpoE)A_Rl2rJ3naf& z_K>Mw+(~cSNL#Zh5$@XT&TP2rwnQ2$0@yV7%&NesB;TAygP96fTku9rOM9AiJRe{_ z@Jp7+#=YGj0e*Rbhdc6{aQTW$h37-8?t1b^aND8eSH%H2~GrPN@cRv=#BG&F(jn}6-XmVVad11 ziiXEwTs6%8@%7(z>gF)cr7$cPO;hQ1oKvelha|Udqr_yNxS*u4TypGVMKjD&bfn?P zEtI03|20saV=)v7=13B{Q>`(C>=*G!3h864GJa^Z!#WhX(%kZbhwWPJgI0KBceu)N znXjVf7fii5aODLwN%f3k{r7SzYipA7CUe;JnBY={&i8DY>5?Vpb+qLtxfCH6J*9Pr ziZ(}RDmK3;#^{|#x#BaXUR+>ZkkugLbCQfy^@@HaT^$lL)xTT*3p%V?pEduPVWAPM zHou^BDnR#S3y~Yu5J|^m1Oe<7Fs{a7YTNTd+28(vhER9J29Klp+s}%iKfXi3;%EjF z&xC0{yN=mWkaWKC0+jlKKQQ+;cT^78wQLosV5G2zOBU2ig+WBxKBI6T zPlhdlO$km3o=!|6m^j>WONasFI0r_QZvzk^-(HY$o8d9(^$qAH86 zj$<(fKeDRM22<86&_(M#!2DwcLqSxmgH&gGO84g-58$#%A4+JSd;_yQCQANT<7)ve zOL!Fm1lKdx)g)po%w!8IFXufuOJ}$y(5N0ih2olzCgd~iTRu5(;Z`;+u^@p0HF}>o z#e(Bn?m@e!u}~xdc)Lb7$GpeEK$fSeK_^VezGyN#7uZM@o-&zqf z1GQi-LB=cola6Ip##9}MTZSmZ&KXTbi*TQ(WsRrD$=J)eEH_$g3 zi3{a&@S#GN6od(@W3VgayJ>hsBt{v~g}{(vl;#~b5g6#P6%&_vM{H^#!L#*i5do5T z@|MqdL}bX+T$iBwLwURG%JFg!)PC*ZJEhVT5+G*^BSI+AJY4meHJ% zySCJhZt>2$`9r!**zEeQ)PsN-J{*ZiC2ZQ0q7sEyQ*jc;pV033d<}Jy zluCxfN1yjuZinp?29Yclm5`R`E{1GO(~e9N*D>UBy~S6Y{0pGi*2?|n;QY2%YxC0w zl-GrQn@dMW?*#Y)-q+8}n7TUt^3u6w?F!DoTo@)B`S|Bp|37D{Yoc?FyVTNC%Q=z_ z$S}3mFChFscShb7liS*NtK)V5f)^5*c*o81De)I=-+$~B?vyw{Zo~tf1glTXVy8ZS zrSmXV+v|HX8t^Ld%8&5i2g~H9epdH8hW{Uk^?^mgws+DN6eh-(Hko+-@n45q>mwV7 z*$M$L-utiBoqm0aahDWKYxB2HIa*r)7|QK69+G3uTfI7?s{d_Cxn?}or+=_0?ZYOU zh%fo^?wrcZa52gVA8(V)MsWsl!eyT=MZ|9H|gk3CH~(LRd=TJinx$FZc~7-zg{l+3Rb}Q zot|YVbdu_h@BPrnmBpfP|t3xUf)%c@{?1e-r1CchG-lPGYP*gRT&hUkFu6?7OC zwDWi#4%#VQJO<20hIdASpf%-MwufS zYek0uUb2!xjdEU34|h+Br5W!kvi;XzS|8}NPITnZYqcT4qoag=WNj5hm}vy~q?%^E zBPnF*TJ(6?j7 zDv=}gG0HgEwv%KACW^ve_HU|rxwX`uHCN^N!Gi7#>W#7r^O--*Y3vt?|*W@u4=o{0Qjz2iIMAV zc^IB1ZfcmzIsM7cW{hduc^cIPg$W#HW$l_+L_Lf{uZ?RPUKYuDDX5!aWK;Ntw8Zf# ze;(&nCgw_$+WFfLp7h-i#BlBq?U$rbjoUN=B~M}M5P@!uyTUQCs?fd46Qs~ZoH9@=YC zPGzkw%Pp3C2wf{{3S{t7l9+(p@@^@V{os4#<3h53jA(e)N#+|A9mVk|`$1oE`yBK0 zKFj#kHe`5gvKP@M8~=wNDuQ5^I(5@#p`0Nc3qjkMK?23_2ckUH@6+?{ z5SJS`82K#%4}XN=9G6Eg{caUK+tRv>#`T#mY6EJHXT#=D6~&J!oxVg9+ttxrU#1xB z86q@450EbVcYybX7{Na)Oez^uue_NBz5~V`et{E--eHnK3-*gau((6UJc< z1NPNJaRgv*3lc``Mr?;8w2=M`XMvAL`N#-(WsW}ZQ^LAv8O3qnTmo(bJ{+L~fZ~jU ziGzs_!w&{le6$C=f(a0I0F=$J?2aseWh8Sy7D)1RDfMt4ujX|NfyB(r5RWRvVh zc`#_cV2Y96DwdGEdXbu4dl2&Aeo;-DFQ-H!#mzQB^NXl#z5;zJlUS7Iu7iJG!L(

WMP+661UO>*(E5qv$#cN_cy(gRgONY5jF?%r?4zAn; zdiuA@bi(*I;nBt)gkjCa72Omf7GILPTbuT9Y2&^D(sv81)@6;6F^>Vf(0y@?5NCuL z2WP!;!73pLWh#$MQR3%w<7@f|`lt}0hN7J`*!-%z>0f{ZUd?})`_25Z26kG$BieUd zgaPsOpJYufnSbHw=O^}!=>E?_;@*NEtNIwd+(uya|M}d%xdh)x)tj-tM23y7R58p~$B(#+i{K`9bn?WfA5B&kEI_d)?Hn z+Ko`$$6{@CAQYD;!5L$3r^E%WN7;!*dKhEwMSR{t$SBc*)n@>e2mh1uX#|`^SR&Q@ z-pjgn^u$(6p)JUo2#lBuijYiQEqim?6imL!)JryrQOLFeY%EG-0P+-ReQQDp5h~sC zkZ7549`%xqheihOzz2S_luJg zsbbY}AteDYB)=lvmlQ(wT>lgo`Iy44Uj{m{f3$y;rP}L>one=RgnN&MEi-xYNcc>Y za(7peP|C}chMo7MOWVXEf8dfxe}Cb+T6M0xIfnv76B9oy6hZ-6-%~#bzfq{jM6$y& zm{o<}s(-3IpoqjkFj3lzA0&J+#Yl(UNK_Q#>&Rzni&?^dzwwn%d$;Ke{`WTB=dMC) z8$&`M^RF1M|7W&EKYjh%`!xF-?lUe4bF2GEZWfd~N)c50*s!p6uSbj8u*cn~q8IqP z8uO$UyaKU006MHEMn&h^o~rZx1z}x8ag@b+Cs({#h_q=D+sU~I=@t6w%Qm_7j`cT` zm6DAG;RhT?%AV)shlGI{n8qs33gh6aLmyhbc4~w!7^s9uX7D|}oO1*|D|)wYf!Xcz6uVe2>gu*$Nl$-*nywp}LzqU5_|g{ml3*;Sb)c}Y$U zKnvK_d9{35)%J(>cD!oH_L2I0#o-#W4r=xxh85}Ul7K?Uj^&NVB09;Y zvbb;tL|jjnrqwEH%_>s*^aG!;^|U}rorARU*EE3yD}+m)PEFjRPTQqtlw8!ysP+!Ff!K``6s&F;*?%!qnbBn5iFPjY{*8Y>AWX%BbkJTM~ zb&aZyPTEF=i_8Dw0NLkulrgKUHqcoFrQN^iWx!|hp{^1w&0H#;R=-%nZHgdO3hO$ipcVr=wM*)?D741B#D7>b& z0ag?=s3zhK0ADzKxMtP#2Bm#z*c(-xB>B4Dmrkh!%}fTuxM+2`BmCY>&d>5MK&qly z143~J&hL91`xj_a7+=sL)fes1ngipRUb+++N^94o)+ps%MQJHV9_r0x(SieZ&rmI`Cd1|OVwZklT;XK_WSK0aoiU~; z3;?pyxocR~^Dz@G>SnMj3w1GtBNgDoL30D&!|49UY7raitl&Mq@kXI(A-38j?$*R; zsr|4hb?_YXFCK=7wEX?4QIXH$X(QErQ|S%5oIMjpA|P=Lu4+~|M`!|F&PlteU2>Mu znySpBCe5j~3*YXZeE#{GiV^+-EKAfNW!DWpHULCL`Co1?yKmE4zCbm?DuB^QpfQPAskALr zoI=-wMQh|qR~TB~Kh>fNC^TxJ%AND`ZOnQzS2~`uBOi1x+xi9_{ak|S{Fc0>wqMne z`qu^3{3(cP2X5mG7snlN{zQcA1?zwHbNyYZ$x&=7Q=gDHHPd+kJFPoA%!kbi2%vNz zW_Z;Fs+m11Pb5KIc?EPHy_C!cJ&-$mKogbR2rX|poyFQ;L8*c;_Q)q&SlYw$``IrO zlk8U;o6t{DukY{t7G(KbHarZDmZ(e_5ZSHlyIap2GHxV~;m~K*Zi_QvXN|^9B5Y;JN}rf=3@n;NE6$kDzpor` zED(cApq-jf1q@k4_ga}HRaqyX3RM1_D!&<9!R*l&(XnO~B|3zhuD8(bq3nz{5*92) zf>hUhS~z9}R3kQ1_T?i!Zj^orh50^f>JJ=_YE7yfzqKB*|w_6j+ZlqHsxaG5Vo{ z23Vv8K(GOF@z#{1MTv3NIsZ zA$|rfk+x%C)X9K!hH!F*!euGw`lqB5rI*pmJ^)k02+WHt!q?z4W7Q<__o3LyU9D(! z^|SiK-w#?g3~!>g#Zw^6_-f%1$ zev4Z2x(w;LUdxw?<)4p9*-Msmy$nO=6^A>gwgZHbr_KIdr@x8*X-VYN9n&hTTSRLq zd?yLB4Eg^4ZAXARRS{BQ>rN&w%@)mMLg(+#R~OBy|9B$?9V*r!^xGoM?>rJVUdGnw zw}sT2LMT=EDB8`ljE^ny$6%mXxdV>7!H(8m!sMt`jZ`-k!U=u$9Z=35lOGOx!}2EZ zG?GjDA0il)sD!Be`}k6i41)(RiJKW`6eUEeDW0M_AzD~x%j$SNVq%8yv-d1xlW-)D z5~nIbL?Hr8Aqtpx=22tZ9eZo)TNPYv#hh*U)KT_5wdXT~L+WnET41@CTL zr7&vUamOO+MYj1)Jo&^B8*XCa$v$==5U`m2 zIUxDZKg=s79bZ|YQ~-$65E-(ldu_ zPG&`NuoGPWT0f6TIeR4e6*#AXd*nM2vd&Ol^xyW|>r(w&0v@-Q8 zesuIKpBgv1{+k4OqkI%kyX{XWB&FUqemsJ-6i^B{HA)1Fq5?DlD~`-q=46Y0jS{Cj z5_}QKddOT7a54xIR-Ih`Jgy)q1)SG5z;j7TT!6@6T*R>E2Ji)uAy9Tukww7Tk0WeA zK(8D&j!+@g03$Yzkd^!apsd%FLD&q3jT$(L0LIm^WN~pkOK(cKj1tG}0Zg%hAs0Ma zooa|-%B}v{+Fx^Xrj-t>JQv~0NBMG+ z(Z5Md`e1{f<|ry9s3l3FeV>f-^sHbJ%Uq8l+R zT&w0!(_aHv{y({?dcq*@u_mpD^vw2u%pF{IA5c4it)^adT8 zthMA$7~6$ZE{@6jJcdvHw20gO#r4x3CLuAJ270-T{`$o)0&6}!)fZ=+A%}UJUC~=F zO6ikjOZu)@*wOlo@!3ze<|gQ>mclAJjH)e)`NYyntVuZ;Sfq)ZZcYun6ot&ruUfZj3$sfS>L^&jz}pgv(N0Qd?a0 z@RoMW?g@Gtl9SY(Dkwh5%KpA0G-BUaJ(#F&Fw9VWxun7z{m`HlJR;nWJ{`#1W&DBy ziR0tFq<@p^(qYo zI^)8Qnu}E=Ez((u+fg-d?)?28?~PammJ5KdDONk@6GJVvWcWbY-FQM5v*_8;?Ew%^o&>gIl*NMOuA& zBp%b}Zv9O-FtvYsmVvb3`BO4wSOC#ukxK8mek@GylM;MPx_O1U7F)=QKk=*IEqcky z)95Cu-$g`tGRf#@+v|kAKbtxCiOr-M7a+V>G9~osU=l;j%Ut~nK&vV;MdyHXMaSTw zW0*kyQz_S0fc0Xad=gC&P0`rp@`3tA{7oM9D4_rgnpE07L(pUz5|8M+U8^^w@dtT_ ziLHdjK@$rw<4d6*H4%)h$>!(63aJ<*M_Cqrkig!~_JzuS(q@Cy_) zP3wb_8u}(!jx=rd(}U6LnZEqKh_}P<68_F4#PoZAPoK7eYmLy46rphX5m@H`j6yt= z^MN>8NI0ncoWl+CjUcfxZ3gg`Y5F(FyDBwTJ?=`LTlheF`C6C~JK>!aQP~ysC^yku z;jfolo=WKa!te|wEp3?@dCbkBeoS68)3Cbdg4R`DECa(x+S}Qjx#SQBN}QJYAlx8y zbv*urnggcM1A}9TccRKM?+bcj>WG+Brd3I4ybmF||JSLNHGkR757U*o#pKR14_|1r zY)toR6BKoHX|fYO{UL;D{iHad^?<<9dXY-?eKN@3v4ZC7NUYaXSR`2=3r~e<-1G;` zv5qI)FQPYl7fe;b&V96mz$$sCL~W(wVxlhlVnig8x1Bw83Q`$pnu?9x@kx8Eqbhfy zTKIUH8>3d*j;m{KYuFlxzyC+@Us-DSdy%`nlN#oJu31I>A?JAQYZ)fi!zXI?O(VkS z%6>0GB`$4SZdRsjiV3G-&%-t<)EIF|rC-L0hZye;hD=XKJs;Vg%mN1+U;a^U)vpJ- zG*#1Y&j&xj0A(O?TAzk3gIH)1_cCl+nPlR)R4WY&NIx{y4(L$6EL^zVaEATFIrs~p zjq-Bw5+qhgVyXryZtzmKo61{r`?lR3^m9vjvsdxBlX!Ztsh>9gt$4x@U27&Da-#h2 z84l;(HujeMnNKbHzuCWaa>R~m%kYsE6(j{g3y}Q7WVk85MoCoxLx795M+;uF{%sN|08a`W97ZFV}UF*>Y_6I8eM0HM<;YleqU9IuF#4 zo&Vgp9IH2+wd0fbWl;GiGW-p0RjR;C)628s1l?^nh>QIsi6f7 znNfid1cU$qDG^8rML>!asR~FW^xiuGA%K9?0P`~U-TUsWckg;@{&;J>xBocj`@Zw- zz4qSgoVC~f_W6Fl;&+-Q66K9BC)s>exC-KaEV91nPFmk5)4s z`X|cJ1COKwP~$ms!&pU8BKHwCC*qrXRfb%EyMYURoz6((kIFGAf?F>pyG(tF!mEnE zj;<}Kl=J4v%yb7E!-BF;uVJ>UAV=GUA88nkp|Xp%MS+jCC+9XBm#%2ED*|OeULjH` zK;Z&_fttzn7di$uIR#>pH1>nVzDwaG7Qp$wca(Vfjb+ZM=u*=m0N0okIN!DAJ*{(P z)zo>e(6@~j@~TS-{{S1>z-##09o8q^WDOdIn+zMaEpgSqFDfF%U}zvrB}L0Rlp5)- zGM?qH3cM7xV8}&iEnJT=x2Mof6EsUtx*97HN6n=A-o{%`dHX)JRjUw55*ID;+ghtN ztXy4xmB~4{T{ua?xUotabcgH3>>f4(3hY-@9VsDHcn+=%3SRf?6G=MdvlU8B%COj8 zZMcJZE_7TRGv{#{6Wa2M=v&UH35zSC2`(v{1lR%Bt3|?l=dDl6#!L5E{R+?Fvz_#R zU0)LyKq)(q*qQCxEI&81$q{IN($jYPi;$7R(UeD*jRom#0w>O@4*x9gY2U{suJT}G zRGt{GQU8;^)@9D!558s)Q`R3<6J9l?YF&ntN;+SaCQMG(jvm&q-P2KSHAAn4wfVQ%ypJvlnh8X~Bhb!1YMDPQ`El^C(srar)0AYsqq6wkl!s|KD* ziidz6y(FKLFfgu_QJNTW)gL*dbkwfDqB}^PT9#8YFOo>hkc!5k&Gb1E`WI?>pQF=|XbL*=I7fet5H^&!h;~g=7 z@rUXM{&nz~*%f`o|J$2rl;$U(mORN681%{=l^~wt-sPs7ZK6b|p{ET5s_H+&0f(Z+ z^Y9*Np@O&LfC6}w0Y^#QM~^2Z0ewGB+F6R}>7KS^z%JUNs*;%M%#o|%eQb3+RJWZw zfSU!(pk&n5FBe43WWqlRJrz?*7W?hHrQx6tb@X(rmrZ-Th}m?5U7JJ{8B7BI^maR% zoXr-W|0+UJk!{VuK;7vbiui5TPUPMl;bd;~bF3S&_%4SDH}eMaMzW1*jj6@4Rv|R!mMU zaQ78PhHK}(|LZ{wGkQ35pkim7!Y?wC9HDIO&&>x}SBF|iBDpU=AlzQ^!ay@lml9&^ z=W{%Rd(NVEOjCpQYs5iMf{SN#^!B}wXmEub!juTbpk0>@Z_DTmZMhOOh?U` znzuauG9$jDXON?6%+c=_hld2bx=rd66vci~__M~?hV#H#dj}P!&^ssjqzWdRNzuM8 z2ERu+M91VR@2l{7g~eG2j4_c>6~Y`4R?b&s5Eu^?+vYDpc!~oZQ=ce*ol})}} z#64B>(JrBCxnluz&v*tpPKZ@U)8tCem%Dql?9UPfFXpS3OCGXkJ)^u{jk*tIsK$i> z5&_xV7g#V4&t5M=E&Y0NI{9>30-VFLHyA*n)GWz?awM+K{q|imc-@w#RSIXz%>*h* zvc;gx{Y;P2p zDF+Ty-Swk2^BGX_a=@EIUqBKgy!Ug0fdiJTE}<|n(1~+@fHP*q%w=NxO&rTHqgxR3 zyU~H=V_b!+?cmX9rMCi$;OeNf>)!m?pRL@xTaAhd+h7%ayy2pOvXj%EI_CWH-JZ0m z?neEZ!R$SV`#zHK1C%VBJVGgSlN_~U%*jmm*W z2_Tq`SrkV%FLpyw0g_Qd7yz$-_LnVU0=DfLzgCluY1E9e%$RE(+{8u}c87=@jvMpY zsn0bUuDD^{mx92|LljxaIwtc>Hm?N~nvhy>%qK5wB4XP5%057VK-PD~kDco3>Ni=y zUom;sxM9Nr2WjQ~0x1Dyt_ChzkrJJeh9+G{^V%z4B)8Ho-D?4w+I3c@wM?2@l&(h* z8#^q&f@yLUN%02-q3?y}T^iekkiWK%(zadvq_Lr8Ae1bojW_b<7V26@VIpBOKQALn z1y4-fs-AJWI!}HRzkSRLbCG^ZN`?j=h?8YZO;2RAMq~%Qaw-1dj!t*|0bzhA)O1_^ z6bMWQ?fh#HWQnd8_uM3nlMXpd&~!|l0Qm2WaVBum!Eq3i)XB)Da#5vEi>e%QVp?*V zMV1XHjv5Lxn)Br%l`SBupgoy$olvdv4kk?Bt=PNEm1U~z5y6S}BL=iS} zsMfR`C2w{xxn^YKKB;jZWE(l;wZZ7vf8zJH{sGIkZT@imb8766B$}peY%uH;Y!0Ds zm;We8*2eItJpS~Vyh{f5N zcJT%v{CWeuJ!wu}3dQB4hiq>fF6DIRR^2G8g?4d(Pg!;7A&lrAby2X&y(t-4)x|dt zeVM7_ItLB_9bJ4CU%uCo}7A#H{TZAex@N!0z zv7l-1mb4d<8SFdFl`Btr`=K_-Cp2~#5@P=3g6PXe@R!!g;b#P!wyEnsW_-$@q$ZMC z`&Oea#(U(=#07ugM~KnBtA4DC(Uz zM&Pxb6~=%K8#$2zxNPSqv%$mc<|3F+ngjg9;k1LDm8YJTnW7k~j>jV-2-C*QT1dA% zW=B$Seh(?$cUaRG9kiZ8laLgWt9?2aPy}ywuZ~zW)J66lj^j_}?ZC`LoDn#b7v0ET zb;p`#`u;@RIuW-#Q{0&MbwdK5G1N-m3QLBqR69a9M@5N0=$je6_ubdH1XSMp@fi2w zq}C69&CDSw0^(4AhPjbBpizP}pYk8wO4-3rxfjP;UMVxQa(%ZTH4I)ylBneR<>6^$ z)}oo`<$W9X@TD?RR=X&5@C0)ve=su9s;o=h5?~H`;Ip}D<(|7g6!JXuM3%Z>I-KW1T+60Wb;rE?i?>E;$61{Ant{mTnlmb=e4dzAQwa~>G(@|wA^5*6(CN z<#zz-hE}KS$)W{t_$s)&0)Y1*s2ZrlG1e|nG4LP~pS}YikGJ}-fg}^)G zAc;yz1gy3uMX8E_0TOkHuh9F;@w3e_{SZ6LD_Dv&Uj^A)rLCl{1P7Il3XHWEjK&BA9lRy2}lW#b~xCfF}z zO*W}@3b~dyAdC;`H%bH5(Kw^Zj}&_y10{dEpxZT7IIX`cB7UUz#gIy7g^Ic0si{bz+a zCf;i{>T9bVk;`h4#Yd#I^h?H+o9#+{)7ku$w9EZu+CzKuw3ka@u9TdUY>WJuhst_0 zcdAs`(zznQY#+HVLqHepTDYqQj?>tqgdPpCWBRPG?m6bK(4=s%n}>|UM>)qwsiN9N z9^4VN`pAfhs51qrA)Aakfnd1M=7d8oQn1}J8^kN#JD5T$gsbIjhOuwDjyh@S{OXk! z%wObX{@g4}*&*#5ej)Ah051dUtTyZ<50Oi_f)vKHzFkFXq`?f&&){j&4}e>xXZV!c zN6qa&9z1H<{#k5!ePJYMo8gTpPfzB*zn|thI<%i$=B1t2D#7+tW-{5&cFO?GyUpB7 z;BdoMLTmt196Z<<2s8)Kca>f^h5f1bYSZ#_jX$n-q_FjR`F9_%eF zT8MvMN&k9OcrpxDD9+q>!|MN!#ZaSX`>159&kRBQzQw~3X0h3Wj(0q|%D!^SQ{3~` z$<}+{UAD)OktxSe@ro}-TyE<3+FiO^b{fX3b3K-AKIgu8R_Em?8HU+}1NB2PDwS%% zmEg`OJu5elx6M#0uw2{yN)AWooh_WImALV^wYAmly-;hU!$WIEt%0}i__EmUG0fQZ z@)JJN7tGA(9Yx*K$&1-1-(*W;5Y@J8H8N+P`YGxqGvPlQ+7&jrwtSSCSa`R0Bbc`~ zYpGta0Js*%EHN9y?hOmZ<>q(lDYQQtr=KlpU$KjGj)H)l?jau|b!s-T^eW%Y6V3xv)<6FKd<~l;yAeyd zlo_LS)fQ`q8?Pp~)t%cD-YniN^RkWc3z80p$Rx1h+=tg1|3{%HRu+-SOO4nTeu6~+ z{)8Y59ve8r3vB-LHU3PiEFa_0t8GHQ=m|pKAd2u|c~olY8$5geNc$jYcqYt>`Mt(S z*?L$!YQ%GN9}&{UsQOnjhd)p}bceljg8U2H!#^G1o^^gSs!!^SY@_3oPlvHP{B-5c zf3H$&lfvZD5wy*T=FQo(id$Qe6ZxE z&wP7IQlk@oCN%@$jU4H6n*y3lBtu^IYkfi%bF{r?ui-_Qppr&BYD4x#n0=>S*$Xs& zGp$0Zv^s3^cTYaQNk^$E`}VSUPe!>e#)#fo)j;4Gl1h=jedg(Sl@#ISgpd6NJ<&H6 zwSBwyQP=u|t1=h-X($6 z_Ojon&|2zk0mEE=lgmfg^df?@U2xA974OWvoIng|Z{KmTTZo$R4(qP7i`012;=+&k z#p%U9(S>PQwtwM0YYU8KDmhQ9p!b%?XM)cP{pB;?iOk=JIUO#TD9?~~tY&Xp539Px z9Qze1xVC|Cu0o&~jl&PxYw11GTXB<;(?3qO)J-vZnq@}k=4#R!lCA%C81Rovht!aQ z>_3#>sqlE#+l2q9OCTm(Fh&ni>)QXO%={ahHsgdhtZ^e6dXklkedw=z$zk`YP n71QtYpIQ7|_}8Ip9#}=l&7A*dhWsyyjBBSjXCrUSZ)5)ff-|KK literal 0 HcmV?d00001 From ed46ed3c604699c6bb113ebd7da40f88e812d04a Mon Sep 17 00:00:00 2001 From: Kamal Choudhary Date: Mon, 1 Jul 2024 05:23:12 -0400 Subject: [PATCH 4/5] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 2717b96..5cb3b85 100644 --- a/README.md +++ b/README.md @@ -25,5 +25,7 @@ python atomgpt/inverse_models/inverse_models.py --config_name atomgpt/examples/i (Documentation development is in progress...) +![AtomGPT layer schematic](https://github.com/usnistgov/atomgpt/blob/develop/atomgpt/data/schematic.jpeg) + From 12e052e5477d76f38b86cfb06058dd24bce9822f Mon Sep 17 00:00:00 2001 From: Kamal Choudhary Date: Mon, 1 Jul 2024 05:45:52 -0400 Subject: [PATCH 5/5] Update README.md --- README.md | 40 +++++++++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 5cb3b85..c95dd5d 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,61 @@ # AtomGPT: atomistic generative pre-trained transformer for forward and inverse materials design -Large language models (LLMs) such as generative pretrained transformers (GPTs) have shown potential for various commercial applications, but their applicability for materials design remains underexplored. In this work, AtomGPT is introduced as a model specifically developed for materials design based on transformer architectures, demonstrating capabilities for both atomistic property prediction and structure generation. This study shows that a combination of chemical and structural text descriptions can efficiently predict material properties with accuracy comparable to graph neural network models, including formation energies, electronic bandgaps from two different methods, and superconducting transition temperatures. Furthermore, AtomGPT can generate atomic structures for tasks such as designing new superconductors, with the predictions validated through density functional theory calculations. This work paves the way for leveraging LLMs in forward and inverse materials design, offering an efficient approach to the discovery and optimization of materials. +Large language models (LLMs) such as [ChatGPT](https://openai.com/chatgpt/) have shown immense potential for various commercial applications, but their applicability for materials design remains underexplored. In this work, AtomGPT is introduced as a model specifically developed for materials design based on transformer architectures, demonstrating capabilities for both atomistic property prediction and structure generation tasks. This study shows that a combination of chemical and structural text descriptions can efficiently predict material properties with accuracy comparable to graph neural network models, including formation energies, electronic bandgaps from two different methods, and superconducting transition temperatures. Furthermore, AtomGPT can generate atomic structures for tasks such as designing new superconductors, with the predictions validated through density functional theory calculations. This work paves the way for leveraging LLMs in forward and inverse materials design, offering an efficient approach to the discovery and optimization of materials. +Both forward and inverse models take a config.json file as an input. Such a config file provides basic training parameters, and an `id_prop.csv` file path similar to the ALIGNN (https://github.com/usnistgov/alignn) model. See an example here: [id_prop.csv](https://github.com/usnistgov/atomgpt/blob/develop/atomgpt/examples/forward_model/id_prop.csv). ## Forward model example (structure to property) +Forwards model are used for developing surrogate models for atomic structure to property predictions. It requires text input which can be either the raw POSCAR type files or a text description of the material. After that, we can use Google-T5/ OpenAI GPT2 etc. models with customizing langauage head for accomplishing such a task. The description of a material is generated with [ChemNLP/describer](https://github.com/usnistgov/jarvis/blob/master/jarvis/core/atoms.py#L1567) function. If you turn [`convert`](https://github.com/usnistgov/atomgpt/blob/develop/atomgpt/forward_models/forward_models.py#L277) to `False`, you can also train on bare POSCAR files. + ``` python atomgpt/forward_models/forward_models.py --config_name atomgpt/examples/forward_model/config.json ``` ## Inverse model example (property to structure) +Inverse models are used for generating materials given property and description such as chemical formula. Currently, we use Mistral model, but other models such as Gemma, Lllama etc. can also be easily used. After the structure generation, we can optimize the structure with ALIGNN-FF model (example [here](https://colab.research.google.com/github/knc6/jarvis-tools-notebooks/blob/master/jarvis-tools-notebooks/ALIGNN_Structure_Relaxation_Phonons_Interface.ipynb) and then subject to density functional theory calculations for a few selected candidates using JARVIS-DFT or similar workflow (tutorial for example [here](https://pages.nist.gov/jarvis/tutorials/). Note that currently, the inversely model training as well as conference requires GPUs. + ``` python atomgpt/inverse_models/inverse_models.py --config_name atomgpt/examples/inverse_model/config.json ``` # Google colab/Jupyter notebook - -[![Open in Google Colab]](https://github.com/knc6/jarvis-tools-notebooks/blob/master/jarvis-tools-notebooks/atomgpt_example.ipynb) +Examples for running AtomGPT is given in the [notebook](https://colab.research.google.com/github/knc6/jarvis-tools-notebooks/blob/master/jarvis-tools-notebooks/atomgpt_example.ipynb) +[![Open in Google Colab]](https://colab.research.google.com/github/knc6/jarvis-tools-notebooks/blob/master/jarvis-tools-notebooks/atomgpt_example.ipynb) [Open in Google Colab]: https://colab.research.google.com/assets/colab-badge.svg - -(Documentation development is in progress...) +For other notebook example, see [here](https://github.com/JARVIS-Materials-Design/jarvis-tools-notebooks) ![AtomGPT layer schematic](https://github.com/usnistgov/atomgpt/blob/develop/atomgpt/data/schematic.jpeg) +# Referenes: + +1. [AtomGPT: Atomistic Generative Pretrained Transformer for Forward and Inverse Materials Design](https://pubs.acs.org/doi/full/10.1021/acs.jpclett.4c01126) +2. [ChemNLP: A Natural Language Processing based Library for Materials Chemistry Text Data](https://github.com/usnistgov/chemnlp) + + +How to contribute +----------------- + +For detailed instructions, please see [Contribution instructions](https://github.com/usnistgov/jarvis/blob/master/Contribution.rst) + + +Correspondence +-------------------- + +Please report bugs as Github issues (https://github.com/usnistgov/atomgpt/issues) or email to kamal.choudhary@nist.gov. + + +Funding support +-------------------- + +NIST-MGI (https://www.nist.gov/mgi) and CHIPS (https://www.nist.gov/chips) + +Code of conduct +-------------------- +Please see [Code of conduct](https://github.com/usnistgov/jarvis/blob/master/CODE_OF_CONDUCT.md)