From fe8f5c0e693001fadd58a8d0c38ac2185cd9d05f Mon Sep 17 00:00:00 2001
From: skpig <479469418@qq.com>
Date: Sun, 21 Nov 2021 14:15:13 +0800
Subject: [PATCH 1/5] Add multi node training in CycleMLP
---
image_classification/CycleMLP/README.md | 53 +-
.../CycleMLP/main_multi_node.py | 575 ++++++++++++++++++
.../CycleMLP/run_train_multi_node.sh | 8 +
3 files changed, 632 insertions(+), 4 deletions(-)
create mode 100644 image_classification/CycleMLP/main_multi_node.py
create mode 100644 image_classification/CycleMLP/run_train_multi_node.sh
diff --git a/image_classification/CycleMLP/README.md b/image_classification/CycleMLP/README.md
index ac65489d..5abfb60f 100644
--- a/image_classification/CycleMLP/README.md
+++ b/image_classification/CycleMLP/README.md
@@ -126,6 +126,7 @@ python main_multi_gpu.py \
## Training
+### Training with single GPU
To train the CycleMLP model on ImageNet2012 with single GPUs, run the following script using command line:
```shell
sh run_train.sh
@@ -140,11 +141,9 @@ python main_single_gpu.py \
-data_path='/dataset/imagenet' \
```
-
-
-
+### Training with multi-GPU
Run training using multi-GPUs:
-
+
```shell
@@ -160,9 +159,55 @@ python main_multi_gpu.py \
-data_path='/dataset/imagenet' \
```
+
+
+### Training with multi-node
+PaddleVit also supports multi-node distributed training under collective mode.
+
+Suppose you have 2 hosts (denoted as node) with 4 gpus on each machine.
+Nodes IP addresses are `192.168.0.16` and `192.168.0.17`.
+
+Then some lines of `run_train_multi_node.sh` should be modified:
+```shell
+CUDA_VISIBLE_DEVICES=0,1,2,3 # number of gpus
+
+-ips= '192.168.0.16, 192.168.0.17' # seperated by comma
+```
+Run training script in every node:
+```shell
+sh run_train_multi.sh
+```
+
+
+It is possible to train with multi-node even when you have only one machine
+
+1. Install docker and paddle. For more details, please refer to
+ [here](https://www.paddlepaddle.org.cn/documentation/docs/zh/install/docker/fromdocker.html).
+
+2. Create a network between docker containers.
+ ```shell
+ docker network create -d bridge paddle_net
+ ```
+3. Create multiple containers as virtual hosts/nodes. Suppose creating 2 containers
+with 2 gpus on each node.
+ ```shell
+ docker run --name paddle0 -it -d --gpus "device=0,1" --network paddle_net\
+ paddlepaddle/paddle:2.2.0-gpu-cuda10.2-cudnn7 /bin/bash
+ docker run --name paddle1 -it -d --gpus "device=2,3" --network paddle_net\
+ paddlepaddle/paddle:2.2.0-gpu-cuda10.2-cudnn7 /bin/bash
+ ```
+ Noted:
+ 1. One can assign same gpu device to different containers. But it may occur OOM
+ since multiple models will run on the same gpu.
+ 2. One should use `-v` to bind PaddleViT repository to container.
+4. Modify `run_train_multi_node.sh` as described above and run the training script on every container.
+
+ Noted: One can use `ping` or `ip -a` bash command to check containers' ip addresses.
+
+
## Visualization Attention Map
**(coming soon)**
diff --git a/image_classification/CycleMLP/main_multi_node.py b/image_classification/CycleMLP/main_multi_node.py
new file mode 100644
index 00000000..5e2d3482
--- /dev/null
+++ b/image_classification/CycleMLP/main_multi_node.py
@@ -0,0 +1,575 @@
+# Copyright (c) 2021 PPViT Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""CycleMLP training/validation using multiple GPU """
+
+import sys
+import os
+import time
+import logging
+import argparse
+import random
+import numpy as np
+import paddle
+import paddle.nn as nn
+import paddle.nn.functional as F
+import paddle.distributed as dist
+from datasets import get_dataloader
+from datasets import get_dataset
+from utils import AverageMeter
+from utils import WarmupCosineScheduler
+from utils import get_exclude_from_weight_decay_fn
+from config import get_config
+from config import update_config
+from mixup import Mixup
+from losses import LabelSmoothingCrossEntropyLoss
+from losses import SoftTargetCrossEntropyLoss
+from losses import DistillationLoss
+from cyclemlp import build_cyclemlp as build_model
+
+
+def get_arguments():
+ """return argumeents, this will overwrite the config after loading yaml file"""
+ parser = argparse.ArgumentParser('Swin')
+ parser.add_argument('-cfg', type=str, default=None)
+ parser.add_argument('-dataset', type=str, default=None)
+ parser.add_argument('-batch_size', type=int, default=None)
+ parser.add_argument('-image_size', type=int, default=None)
+ parser.add_argument('-data_path', type=str, default=None)
+ parser.add_argument('-ngpus', type=int, default=None)
+ parser.add_argument('-pretrained', type=str, default=None)
+ parser.add_argument('-resume', type=str, default=None)
+ parser.add_argument('-last_epoch', type=int, default=None)
+ parser.add_argument('-eval', action='store_true')
+ parser.add_argument('-amp', action='store_true')
+ parser.add_argument('-ips', type=str, default="127.0.0.1")
+ arguments = parser.parse_args()
+ return arguments
+
+
+def get_logger(filename, logger_name=None):
+ """set logging file and format
+ Args:
+ filename: str, full path of the logger file to write
+ logger_name: str, the logger name, e.g., 'master_logger', 'local_logger'
+ Return:
+ logger: python logger
+ """
+ log_format = "%(asctime)s %(message)s"
+ logging.basicConfig(stream=sys.stdout, level=logging.INFO,
+ format=log_format, datefmt="%m%d %I:%M:%S %p")
+ # different name is needed when creating multiple logger in one process
+ logger = logging.getLogger(logger_name)
+ fh = logging.FileHandler(os.path.join(filename))
+ fh.setFormatter(logging.Formatter(log_format))
+ logger.addHandler(fh)
+ return logger
+
+
+def train(dataloader,
+ model,
+ criterion,
+ optimizer,
+ epoch,
+ total_epochs,
+ total_batch,
+ debug_steps=100,
+ accum_iter=1,
+ mixup_fn=None,
+ amp=False,
+ local_logger=None,
+ master_logger=None):
+ """Training for one epoch
+ Args:
+ dataloader: paddle.io.DataLoader, dataloader instance
+ model: nn.Layer, a ViT model
+ criterion: nn.criterion
+ epoch: int, current epoch
+ total_epochs: int, total num of epochs
+ total_batch: int, total num of batches for one epoch
+ debug_steps: int, num of iters to log info, default: 100
+ accum_iter: int, num of iters for accumulating gradients, default: 1
+ mixup_fn: Mixup, mixup instance, default: None
+ amp: bool, if True, use mix precision training, default: False
+ local_logger: logger for local process/gpu, default: None
+ master_logger: logger for main process, default: None
+ Returns:
+ train_loss_meter.avg: float, average loss on current process/gpu
+ train_acc_meter.avg: float, average top1 accuracy on current process/gpu
+ master_train_loss_meter.avg: float, average loss on all processes/gpus
+ master_train_acc_meter.avg: float, average top1 accuracy on all processes/gpus
+ train_time: float, training time
+ """
+ model.train()
+ train_loss_meter = AverageMeter()
+ train_acc_meter = AverageMeter()
+ master_train_loss_meter = AverageMeter()
+ master_train_acc_meter = AverageMeter()
+
+ if amp is True:
+ scaler = paddle.amp.GradScaler(init_loss_scaling=1024)
+ time_st = time.time()
+
+ for batch_id, data in enumerate(dataloader):
+ image = data[0]
+ label = data[1]
+ label_orig = label.clone()
+
+ if mixup_fn is not None:
+ image, label = mixup_fn(image, label_orig)
+
+ if amp is True: # mixed precision training
+ with paddle.amp.auto_cast():
+ output = model(image)
+ loss = criterion(image, output, label)
+ scaled = scaler.scale(loss)
+ scaled.backward()
+ if ((batch_id +1) % accum_iter == 0) or (batch_id + 1 == len(dataloader)):
+ scaler.minimize(optimizer, scaled)
+ optimizer.clear_grad()
+ else: # full precision training
+ output = model(image)
+ loss = criterion(output, label)
+ #NOTE: division may be needed depending on the loss function
+ # Here no division is needed:
+ # default 'reduction' param in nn.CrossEntropyLoss is set to 'mean'
+ #loss = loss / accum_iter
+ loss.backward()
+
+ if ((batch_id +1) % accum_iter == 0) or (batch_id + 1 == len(dataloader)):
+ optimizer.step()
+ optimizer.clear_grad()
+
+ pred = F.softmax(output)
+ if mixup_fn:
+ acc = paddle.metric.accuracy(pred, label_orig)
+ else:
+ acc = paddle.metric.accuracy(pred, label_orig.unsqueeze(1))
+
+ batch_size = paddle.to_tensor(image.shape[0])
+
+ # sync from other gpus for overall loss and acc
+ master_loss = loss.clone()
+ master_acc = acc.clone()
+ master_batch_size = batch_size.clone()
+ dist.all_reduce(master_loss)
+ dist.all_reduce(master_acc)
+ dist.all_reduce(master_batch_size)
+ master_loss = master_loss / dist.get_world_size()
+ master_acc = master_acc / dist.get_world_size()
+ master_train_loss_meter.update(master_loss.numpy()[0], master_batch_size.numpy()[0])
+ master_train_acc_meter.update(master_acc.numpy()[0], master_batch_size.numpy()[0])
+
+ train_loss_meter.update(loss.numpy()[0], batch_size.numpy()[0])
+ train_acc_meter.update(acc.numpy()[0], batch_size.numpy()[0])
+
+ if batch_id % debug_steps == 0:
+ if local_logger:
+ local_logger.info(
+ f"Epoch[{epoch:03d}/{total_epochs:03d}], " +
+ f"Step[{batch_id:04d}/{total_batch:04d}], " +
+ f"Avg Loss: {train_loss_meter.avg:.4f}, " +
+ f"Avg Acc: {train_acc_meter.avg:.4f}")
+ if master_logger and dist.get_rank() == 0:
+ master_logger.info(
+ f"Epoch[{epoch:03d}/{total_epochs:03d}], " +
+ f"Step[{batch_id:04d}/{total_batch:04d}], " +
+ f"Avg Loss: {master_train_loss_meter.avg:.4f}, " +
+ f"Avg Acc: {master_train_acc_meter.avg:.4f}")
+
+ train_time = time.time() - time_st
+ return (train_loss_meter.avg,
+ train_acc_meter.avg,
+ master_train_loss_meter.avg,
+ master_train_acc_meter.avg,
+ train_time)
+
+
+def validate(dataloader,
+ model,
+ criterion,
+ total_batch,
+ debug_steps=100,
+ local_logger=None,
+ master_logger=None):
+ """Validation for whole dataset
+ Args:
+ dataloader: paddle.io.DataLoader, dataloader instance
+ model: nn.Layer, a ViT model
+ criterion: nn.criterion
+ total_epoch: int, total num of epoch, for logging
+ debug_steps: int, num of iters to log info, default: 100
+ local_logger: logger for local process/gpu, default: None
+ master_logger: logger for main process, default: None
+ Returns:
+ val_loss_meter.avg: float, average loss on current process/gpu
+ val_acc1_meter.avg: float, average top1 accuracy on current process/gpu
+ val_acc5_meter.avg: float, average top5 accuracy on current process/gpu
+ master_val_loss_meter.avg: float, average loss on all processes/gpus
+ master_val_acc1_meter.avg: float, average top1 accuracy on all processes/gpus
+ master_val_acc5_meter.avg: float, average top5 accuracy on all processes/gpus
+ val_time: float, validation time
+ """
+ model.eval()
+ val_loss_meter = AverageMeter()
+ val_acc1_meter = AverageMeter()
+ val_acc5_meter = AverageMeter()
+ master_val_loss_meter = AverageMeter()
+ master_val_acc1_meter = AverageMeter()
+ master_val_acc5_meter = AverageMeter()
+ time_st = time.time()
+
+ with paddle.no_grad():
+ for batch_id, data in enumerate(dataloader):
+ image = data[0]
+ label = data[1]
+
+ output = model(image)
+ loss = criterion(output, label)
+
+ pred = F.softmax(output)
+ acc1 = paddle.metric.accuracy(pred, label.unsqueeze(1))
+ acc5 = paddle.metric.accuracy(pred, label.unsqueeze(1), k=5)
+
+ batch_size = paddle.to_tensor(image.shape[0])
+
+ master_loss = loss.clone()
+ master_acc1 = acc1.clone()
+ master_acc5 = acc5.clone()
+ master_batch_size = batch_size.clone()
+
+ dist.all_reduce(master_loss)
+ dist.all_reduce(master_acc1)
+ dist.all_reduce(master_acc5)
+ dist.all_reduce(master_batch_size)
+ master_loss = master_loss / dist.get_world_size()
+ master_acc1 = master_acc1 / dist.get_world_size()
+ master_acc5 = master_acc5 / dist.get_world_size()
+
+ master_val_loss_meter.update(master_loss.numpy()[0], master_batch_size.numpy()[0])
+ master_val_acc1_meter.update(master_acc1.numpy()[0], master_batch_size.numpy()[0])
+ master_val_acc5_meter.update(master_acc5.numpy()[0], master_batch_size.numpy()[0])
+
+ val_loss_meter.update(loss.numpy()[0], batch_size.numpy()[0])
+ val_acc1_meter.update(acc1.numpy()[0], batch_size.numpy()[0])
+ val_acc5_meter.update(acc5.numpy()[0], batch_size.numpy()[0])
+
+ if batch_id % debug_steps == 0:
+ if local_logger:
+ local_logger.info(
+ f"Val Step[{batch_id:04d}/{total_batch:04d}], " +
+ f"Avg Loss: {val_loss_meter.avg:.4f}, " +
+ f"Avg Acc@1: {val_acc1_meter.avg:.4f}, " +
+ f"Avg Acc@5: {val_acc5_meter.avg:.4f}")
+ if master_logger and dist.get_rank() == 0:
+ master_logger.info(
+ f"Val Step[{batch_id:04d}/{total_batch:04d}], " +
+ f"Avg Loss: {master_val_loss_meter.avg:.4f}, " +
+ f"Avg Acc@1: {master_val_acc1_meter.avg:.4f}, " +
+ f"Avg Acc@5: {master_val_acc5_meter.avg:.4f}")
+ val_time = time.time() - time_st
+ return (val_loss_meter.avg,
+ val_acc1_meter.avg,
+ val_acc5_meter.avg,
+ master_val_loss_meter.avg,
+ master_val_acc1_meter.avg,
+ master_val_acc5_meter.avg,
+ val_time)
+
+
+def main_worker(*args):
+ # STEP 0: Preparation
+ config = args[0]
+ dist.init_parallel_env()
+ last_epoch = config.TRAIN.LAST_EPOCH
+ world_size = dist.get_world_size()
+ local_rank = dist.get_rank()
+ seed = config.SEED + local_rank
+ paddle.seed(seed)
+ np.random.seed(seed)
+ random.seed(seed)
+ # logger for each process/gpu
+ local_logger = get_logger(
+ filename=os.path.join(config.SAVE, 'log_{}.txt'.format(local_rank)),
+ logger_name='local_logger')
+ # overall logger
+ if local_rank == 0:
+ master_logger = get_logger(
+ filename=os.path.join(config.SAVE, 'log.txt'),
+ logger_name='master_logger')
+ master_logger.info(f'\n{config}')
+ else:
+ master_logger = None
+ local_logger.info(f'----- world_size = {world_size}, local_rank = {local_rank}')
+ if local_rank == 0:
+ master_logger.info(f'----- world_size = {world_size}, local_rank = {local_rank}')
+
+ # STEP 1: Create model
+ model = build_model(config)
+ model = paddle.DataParallel(model)
+
+ # STEP 2: Create train and val dataloader
+ dataset_train, dataset_val = args[1], args[2]
+ dataloader_train = get_dataloader(config, dataset_train, 'train', True)
+ dataloader_val = get_dataloader(config, dataset_val, 'test', True)
+ total_batch_train = len(dataloader_train)
+ total_batch_val = len(dataloader_val)
+ local_logger.info(f'----- Total # of train batch (single gpu): {total_batch_train}')
+ local_logger.info(f'----- Total # of val batch (single gpu): {total_batch_val}')
+ if local_rank == 0:
+ master_logger.info(f'----- Total # of train batch (single gpu): {total_batch_train}')
+ master_logger.info(f'----- Total # of val batch (single gpu): {total_batch_val}')
+
+ # STEP 3: Define Mixup function
+ mixup_fn = None
+ if config.TRAIN.MIXUP_PROB > 0 or config.TRAIN.CUTMIX_ALPHA > 0 or config.TRAIN.CUTMIX_MINMAX is not None:
+ mixup_fn = Mixup(mixup_alpha=config.TRAIN.MIXUP_ALPHA,
+ cutmix_alpha=config.TRAIN.CUTMIX_ALPHA,
+ cutmix_minmax=config.TRAIN.CUTMIX_MINMAX,
+ prob=config.TRAIN.MIXUP_PROB,
+ switch_prob=config.TRAIN.MIXUP_SWITCH_PROB,
+ mode=config.TRAIN.MIXUP_MODE,
+ label_smoothing=config.TRAIN.SMOOTHING)
+
+ # STEP 4: Define criterion
+ if config.TRAIN.MIXUP_PROB > 0.:
+ criterion = SoftTargetCrossEntropyLoss()
+ elif config.TRAIN.SMOOTHING:
+ criterion = LabelSmoothingCrossEntropyLoss()
+ else:
+ criterion = nn.CrossEntropyLoss()
+ # only use cross entropy for val
+ criterion_val = nn.CrossEntropyLoss()
+
+ # STEP 5: Define optimizer and lr_scheduler
+ # set lr according to batch size and world size (hacked from official code)
+ linear_scaled_lr = (config.TRAIN.BASE_LR *
+ config.DATA.BATCH_SIZE * dist.get_world_size()) / 512.0
+ linear_scaled_warmup_start_lr = (config.TRAIN.WARMUP_START_LR *
+ config.DATA.BATCH_SIZE * dist.get_world_size()) / 512.0
+ linear_scaled_end_lr = (config.TRAIN.END_LR *
+ config.DATA.BATCH_SIZE * dist.get_world_size()) / 512.0
+
+ if config.TRAIN.ACCUM_ITER > 1:
+ linear_scaled_lr = linear_scaled_lr * config.TRAIN.ACCUM_ITER
+ linear_scaled_warmup_start_lr = linear_scaled_warmup_start_lr * config.TRAIN.ACCUM_ITER
+ linear_scaled_end_lr = linear_scaled_end_lr * config.TRAIN.ACCUM_ITER
+
+ config.TRAIN.BASE_LR = linear_scaled_lr
+ config.TRAIN.WARMUP_START_LR = linear_scaled_warmup_start_lr
+ config.TRAIN.END_LR = linear_scaled_end_lr
+
+ scheduler = None
+ if config.TRAIN.LR_SCHEDULER.NAME == "warmupcosine":
+ scheduler = WarmupCosineScheduler(learning_rate=config.TRAIN.BASE_LR,
+ warmup_start_lr=config.TRAIN.WARMUP_START_LR,
+ start_lr=config.TRAIN.BASE_LR,
+ end_lr=config.TRAIN.END_LR,
+ warmup_epochs=config.TRAIN.WARMUP_EPOCHS,
+ total_epochs=config.TRAIN.NUM_EPOCHS,
+ last_epoch=config.TRAIN.LAST_EPOCH,
+ )
+ elif config.TRAIN.LR_SCHEDULER.NAME == "cosine":
+ scheduler = paddle.optimizer.lr.CosineAnnealingDecay(learning_rate=config.TRAIN.BASE_LR,
+ T_max=config.TRAIN.NUM_EPOCHS,
+ last_epoch=last_epoch)
+ elif config.scheduler == "multi-step":
+ milestones = [int(v.strip()) for v in config.TRAIN.LR_SCHEDULER.MILESTONES.split(",")]
+ scheduler = paddle.optimizer.lr.MultiStepDecay(learning_rate=config.TRAIN.BASE_LR,
+ milestones=milestones,
+ gamma=config.TRAIN.LR_SCHEDULER.DECAY_RATE,
+ last_epoch=last_epoch)
+ else:
+ local_logger.fatal(f"Unsupported Scheduler: {config.TRAIN.LR_SCHEDULER}.")
+ if local_rank == 0:
+ master_logger.fatal(f"Unsupported Scheduler: {config.TRAIN.LR_SCHEDULER}.")
+ raise NotImplementedError(f"Unsupported Scheduler: {config.TRAIN.LR_SCHEDULER}.")
+
+ if config.TRAIN.OPTIMIZER.NAME == "SGD":
+ if config.TRAIN.GRAD_CLIP:
+ clip = paddle.nn.ClipGradByGlobalNorm(config.TRAIN.GRAD_CLIP)
+ else:
+ clip = None
+ optimizer = paddle.optimizer.Momentum(
+ parameters=model.parameters(),
+ learning_rate=scheduler if scheduler is not None else config.TRAIN.BASE_LR,
+ weight_decay=config.TRAIN.WEIGHT_DECAY,
+ momentum=config.TRAIN.OPTIMIZER.MOMENTUM,
+ grad_clip=clip)
+ elif config.TRAIN.OPTIMIZER.NAME == "AdamW":
+ if config.TRAIN.GRAD_CLIP:
+ clip = paddle.nn.ClipGradByGlobalNorm(config.TRAIN.GRAD_CLIP)
+ else:
+ clip = None
+ optimizer = paddle.optimizer.AdamW(
+ parameters=model.parameters(),
+ learning_rate=scheduler if scheduler is not None else config.TRAIN.BASE_LR,
+ beta1=config.TRAIN.OPTIMIZER.BETAS[0],
+ beta2=config.TRAIN.OPTIMIZER.BETAS[1],
+ weight_decay=config.TRAIN.WEIGHT_DECAY,
+ epsilon=config.TRAIN.OPTIMIZER.EPS,
+ grad_clip=clip,
+ apply_decay_param_fun=get_exclude_from_weight_decay_fn([
+ 'absolute_pos_embed', 'relative_position_bias_table']),
+ )
+ else:
+ local_logger.fatal(f"Unsupported Optimizer: {config.TRAIN.OPTIMIZER.NAME}.")
+ if local_rank == 0:
+ master_logger.fatal(f"Unsupported Optimizer: {config.TRAIN.OPTIMIZER.NAME}.")
+ raise NotImplementedError(f"Unsupported Optimizer: {config.TRAIN.OPTIMIZER.NAME}.")
+
+ # STEP 6: Load pretrained model / load resumt model and optimizer states
+ if config.MODEL.PRETRAINED:
+ if (config.MODEL.PRETRAINED).endswith('.pdparams'):
+ raise ValueError(f'{config.MODEL.PRETRAINED} should not contain .pdparams')
+ assert os.path.isfile(config.MODEL.PRETRAINED + '.pdparams') is True
+ model_state = paddle.load(config.MODEL.PRETRAINED+'.pdparams')
+ model.set_dict(model_state)
+ local_logger.info(f"----- Pretrained: Load model state from {config.MODEL.PRETRAINED}")
+ if local_rank == 0:
+ master_logger.info(
+ f"----- Pretrained: Load model state from {config.MODEL.PRETRAINED}")
+
+ if config.MODEL.RESUME:
+ assert os.path.isfile(config.MODEL.RESUME+'.pdparams') is True
+ assert os.path.isfile(config.MODEL.RESUME+'.pdopt') is True
+ model_state = paddle.load(config.MODEL.RESUME+'.pdparams')
+ model.set_dict(model_state)
+ opt_state = paddle.load(config.MODEL.RESUME+'.pdopt')
+ optimizer.set_state_dict(opt_state)
+ local_logger.info(
+ f"----- Resume Training: Load model and optmizer from {config.MODEL.RESUME}")
+ if local_rank == 0:
+ master_logger.info(
+ f"----- Resume Training: Load model and optmizer from {config.MODEL.RESUME}")
+
+ # STEP 7: Validation (eval mode)
+ if config.EVAL:
+ local_logger.info('----- Start Validating')
+ if local_rank == 0:
+ master_logger.info('----- Start Validating')
+ val_loss, val_acc1, val_acc5, avg_loss, avg_acc1, avg_acc5, val_time = validate(
+ dataloader=dataloader_val,
+ model=model,
+ criterion=criterion_val,
+ total_batch=total_batch_val,
+ debug_steps=config.REPORT_FREQ,
+ local_logger=local_logger,
+ master_logger=master_logger)
+ local_logger.info(f"Validation Loss: {val_loss:.4f}, " +
+ f"Validation Acc@1: {val_acc1:.4f}, " +
+ f"Validation Acc@5: {val_acc5:.4f}, " +
+ f"time: {val_time:.2f}")
+ if local_rank == 0:
+ master_logger.info(f"Validation Loss: {avg_loss:.4f}, " +
+ f"Validation Acc@1: {avg_acc1:.4f}, " +
+ f"Validation Acc@5: {avg_acc5:.4f}, " +
+ f"time: {val_time:.2f}")
+ return
+
+ # STEP 8: Start training and validation (train mode)
+ local_logger.info(f"Start training from epoch {last_epoch+1}.")
+ if local_rank == 0:
+ master_logger.info(f"Start training from epoch {last_epoch+1}.")
+ for epoch in range(last_epoch+1, config.TRAIN.NUM_EPOCHS+1):
+ # train
+ local_logger.info(f"Now training epoch {epoch}. LR={optimizer.get_lr():.6f}")
+ if local_rank == 0:
+ master_logger.info(f"Now training epoch {epoch}. LR={optimizer.get_lr():.6f}")
+ train_loss, train_acc, avg_loss, avg_acc, train_time = train(
+ dataloader=dataloader_train,
+ model=model,
+ criterion=criterion,
+ optimizer=optimizer,
+ epoch=epoch,
+ total_epochs=config.TRAIN.NUM_EPOCHS,
+ total_batch=total_batch_train,
+ debug_steps=config.REPORT_FREQ,
+ accum_iter=config.TRAIN.ACCUM_ITER,
+ mixup_fn=mixup_fn,
+ amp=config.AMP,
+ local_logger=local_logger,
+ master_logger=master_logger)
+
+ scheduler.step()
+
+ local_logger.info(f"----- Epoch[{epoch:03d}/{config.TRAIN.NUM_EPOCHS:03d}], " +
+ f"Train Loss: {train_loss:.4f}, " +
+ f"Train Acc: {train_acc:.4f}, " +
+ f"time: {train_time:.2f}")
+ if local_rank == 0:
+ master_logger.info(f"----- Epoch[{epoch:03d}/{config.TRAIN.NUM_EPOCHS:03d}], " +
+ f"Train Loss: {avg_loss:.4f}, " +
+ f"Train Acc: {avg_acc:.4f}, " +
+ f"time: {train_time:.2f}")
+
+ # validation
+ if epoch % config.VALIDATE_FREQ == 0 or epoch == config.TRAIN.NUM_EPOCHS:
+ local_logger.info(f'----- Validation after Epoch: {epoch}')
+ if local_rank == 0:
+ master_logger.info(f'----- Validation after Epoch: {epoch}')
+ val_loss, val_acc1, val_acc5, avg_loss, avg_acc1, avg_acc5, val_time = validate(
+ dataloader=dataloader_val,
+ model=model,
+ criterion=criterion_val,
+ total_batch=total_batch_val,
+ debug_steps=config.REPORT_FREQ,
+ local_logger=local_logger,
+ master_logger=master_logger)
+ local_logger.info(f"----- Epoch[{epoch:03d}/{config.TRAIN.NUM_EPOCHS:03d}], " +
+ f"Validation Loss: {val_loss:.4f}, " +
+ f"Validation Acc@1: {val_acc1:.4f}, " +
+ f"Validation Acc@5: {val_acc5:.4f}, " +
+ f"time: {val_time:.2f}")
+ if local_rank == 0:
+ master_logger.info(f"----- Epoch[{epoch:03d}/{config.TRAIN.NUM_EPOCHS:03d}], " +
+ f"Validation Loss: {avg_loss:.4f}, " +
+ f"Validation Acc@1: {avg_acc1:.4f}, " +
+ f"Validation Acc@5: {avg_acc5:.4f}, " +
+ f"time: {val_time:.2f}")
+ # model save
+ if local_rank == 0:
+ if epoch % config.SAVE_FREQ == 0 or epoch == config.TRAIN.NUM_EPOCHS:
+ model_path = os.path.join(
+ config.SAVE, f"{config.MODEL.TYPE}-Epoch-{epoch}-Loss-{train_loss}")
+ paddle.save(model.state_dict(), model_path + '.pdparams')
+ paddle.save(optimizer.state_dict(), model_path + '.pdopt')
+ master_logger.info(f"----- Save model: {model_path}.pdparams")
+ master_logger.info(f"----- Save optim: {model_path}.pdopt")
+
+
+def main():
+ # config is updated by: (1) config.py, (2) yaml file, (3) arguments
+ arguments = get_arguments()
+ config = get_config()
+ config = update_config(config, arguments)
+
+ # set output folder
+ if not config.EVAL:
+ config.SAVE = '{}/train-{}'.format(config.SAVE, time.strftime('%Y%m%d-%H-%M-%S'))
+ else:
+ config.SAVE = '{}/eval-{}'.format(config.SAVE, time.strftime('%Y%m%d-%H-%M-%S'))
+
+ if not os.path.exists(config.SAVE):
+ os.makedirs(config.SAVE, exist_ok=True)
+
+ # get dataset and start DDP
+ dataset_train = get_dataset(config, mode='train')
+ dataset_val = get_dataset(config, mode='val')
+ config.NGPUS = len(paddle.static.cuda_places()) if config.NGPUS == -1 else config.NGPUS
+ dist.spawn(main_worker, args=(config, dataset_train, dataset_val, ), nprocs=config.NGPUS, ips=arguments.ips)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/image_classification/CycleMLP/run_train_multi_node.sh b/image_classification/CycleMLP/run_train_multi_node.sh
new file mode 100644
index 00000000..8a6ad31f
--- /dev/null
+++ b/image_classification/CycleMLP/run_train_multi_node.sh
@@ -0,0 +1,8 @@
+CUDA_VISIBLE_DEVICES=0 \
+python main_multi_node.py \
+-cfg='./configs/cyclemlp_b1.yaml' \
+-dataset='imagenet2012' \
+-batch_size=8 \
+-data_path='/dataset/imagenet' \
+-ips='172.18.0.2, 172.18.0.3, 172.18.0.4, 172.18.0.5' # the ips should be replaced
+#-amp
From e50a112a3cbe5a0dc4de6c52f8edb2ea21f9d4a8 Mon Sep 17 00:00:00 2001
From: skpig <479469418@qq.com>
Date: Sun, 21 Nov 2021 21:44:15 +0800
Subject: [PATCH 2/5] change README in CycleMLP
---
image_classification/CycleMLP/README.md | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/image_classification/CycleMLP/README.md b/image_classification/CycleMLP/README.md
index 5abfb60f..0c1df47c 100644
--- a/image_classification/CycleMLP/README.md
+++ b/image_classification/CycleMLP/README.md
@@ -196,13 +196,13 @@ with 2 gpus on each node.
docker run --name paddle1 -it -d --gpus "device=2,3" --network paddle_net\
paddlepaddle/paddle:2.2.0-gpu-cuda10.2-cudnn7 /bin/bash
```
- Noted:
- 1. One can assign same gpu device to different containers. But it may occur OOM
- since multiple models will run on the same gpu.
- 2. One should use `-v` to bind PaddleViT repository to container.
+ > Noted:
+ > 1. One can assign same gpu device to different containers. But it may occur OOM since multiple models will run on the same gpu.
+ > 2. One should use `-v` to bind PaddleViT repository to container.
+
4. Modify `run_train_multi_node.sh` as described above and run the training script on every container.
- Noted: One can use `ping` or `ip -a` bash command to check containers' ip addresses.
+ > Noted: One can use `ping` or `ip -a` bash command to check containers' ip addresses.
From e743b45982b9be38740669fee462e34b3b774578 Mon Sep 17 00:00:00 2001
From: skpig <479469418@qq.com>
Date: Tue, 23 Nov 2021 11:52:18 +0800
Subject: [PATCH 3/5] Add Multi_Node_Training folder
---
.../Multi_Node_Training/README.md | 1 +
.../Multi_Node_Training/config.py | 153 ++++++
.../configs/vit_base_patch16_224.yaml | 21 +
.../configs/vit_base_patch16_384.yaml | 14 +
.../configs/vit_base_patch32_224.yaml | 21 +
.../configs/vit_base_patch32_384.yaml | 21 +
.../configs/vit_large_patch16_224.yaml | 14 +
.../configs/vit_large_patch16_384.yaml | 14 +
.../configs/vit_large_patch32_384.yaml | 14 +
.../Multi_Node_Training/datasets.py | 187 ++++++++
.../Multi_Node_Training/droppath.py | 60 +++
.../Multi_Node_Training/main_multi_node.py | 379 +++++++++++++++
.../run_train_multi_node.sh | 8 +
.../Multi_Node_Training/transformer.py | 437 ++++++++++++++++++
.../Multi_Node_Training/utils.py | 120 +++++
15 files changed, 1464 insertions(+)
create mode 100644 image_classification/Multi_Node_Training/README.md
create mode 100644 image_classification/Multi_Node_Training/config.py
create mode 100644 image_classification/Multi_Node_Training/configs/vit_base_patch16_224.yaml
create mode 100644 image_classification/Multi_Node_Training/configs/vit_base_patch16_384.yaml
create mode 100644 image_classification/Multi_Node_Training/configs/vit_base_patch32_224.yaml
create mode 100644 image_classification/Multi_Node_Training/configs/vit_base_patch32_384.yaml
create mode 100644 image_classification/Multi_Node_Training/configs/vit_large_patch16_224.yaml
create mode 100644 image_classification/Multi_Node_Training/configs/vit_large_patch16_384.yaml
create mode 100644 image_classification/Multi_Node_Training/configs/vit_large_patch32_384.yaml
create mode 100644 image_classification/Multi_Node_Training/datasets.py
create mode 100644 image_classification/Multi_Node_Training/droppath.py
create mode 100644 image_classification/Multi_Node_Training/main_multi_node.py
create mode 100644 image_classification/Multi_Node_Training/run_train_multi_node.sh
create mode 100644 image_classification/Multi_Node_Training/transformer.py
create mode 100644 image_classification/Multi_Node_Training/utils.py
diff --git a/image_classification/Multi_Node_Training/README.md b/image_classification/Multi_Node_Training/README.md
new file mode 100644
index 00000000..839e3dfa
--- /dev/null
+++ b/image_classification/Multi_Node_Training/README.md
@@ -0,0 +1 @@
+# Multiple Node Training
diff --git a/image_classification/Multi_Node_Training/config.py b/image_classification/Multi_Node_Training/config.py
new file mode 100644
index 00000000..a7273699
--- /dev/null
+++ b/image_classification/Multi_Node_Training/config.py
@@ -0,0 +1,153 @@
+# Copyright (c) 2021 PPViT Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Configuration
+
+Configuration for data, model archtecture, and training, etc.
+Config can be set by .yaml file or by argparser(limited usage)
+
+
+"""
+import os
+from yacs.config import CfgNode as CN
+import yaml
+
+_C = CN()
+_C.BASE = ['']
+
+# data settings
+_C.DATA = CN()
+_C.DATA.BATCH_SIZE = 256 #256 # train batch_size for single GPU
+_C.DATA.BATCH_SIZE_EVAL = 8 #64 # val batch_size for single GPU
+_C.DATA.DATA_PATH = '/dataset/imagenet/' # path to dataset
+_C.DATA.DATASET = 'imagenet2012' # dataset name
+_C.DATA.IMAGE_SIZE = 224 # input image size: 224 for pretrain, 384 for finetune
+_C.DATA.CROP_PCT = 0.875 # input image scale ratio, scale is applied before centercrop in eval mode
+_C.DATA.NUM_WORKERS = 2 # number of data loading threads
+
+# model settings
+_C.MODEL = CN()
+_C.MODEL.TYPE = 'ViT'
+_C.MODEL.NAME = 'ViT'
+_C.MODEL.RESUME = None
+_C.MODEL.PRETRAINED = None
+_C.MODEL.NUM_CLASSES = 1000
+_C.MODEL.DROPOUT = 0.1
+_C.MODEL.DROPPATH = 0.1
+_C.MODEL.ATTENTION_DROPOUT = 0.1
+
+# transformer settings
+_C.MODEL.TRANS = CN()
+_C.MODEL.TRANS.PATCH_SIZE = 32
+_C.MODEL.TRANS.EMBED_DIM = 768
+_C.MODEL.TRANS.MLP_RATIO= 4.0
+_C.MODEL.TRANS.NUM_HEADS = 12
+_C.MODEL.TRANS.DEPTH = 12
+_C.MODEL.TRANS.QKV_BIAS = True
+
+# training settings
+_C.TRAIN = CN()
+_C.TRAIN.LAST_EPOCH = 0
+_C.TRAIN.NUM_EPOCHS = 300
+_C.TRAIN.WARMUP_EPOCHS = 3 #34 # ~ 10k steps for 4096 batch size
+_C.TRAIN.WEIGHT_DECAY = 0.05 #0.3 # 0.0 for finetune
+_C.TRAIN.BASE_LR = 0.001 #0.003 for pretrain # 0.03 for finetune
+_C.TRAIN.WARMUP_START_LR = 1e-6 #0.0
+_C.TRAIN.END_LR = 5e-4
+_C.TRAIN.GRAD_CLIP = 1.0
+_C.TRAIN.ACCUM_ITER = 2 #1
+
+_C.TRAIN.LR_SCHEDULER = CN()
+_C.TRAIN.LR_SCHEDULER.NAME = 'warmupcosine'
+_C.TRAIN.LR_SCHEDULER.MILESTONES = "30, 60, 90" # only used in StepLRScheduler
+_C.TRAIN.LR_SCHEDULER.DECAY_EPOCHS = 30 # only used in StepLRScheduler
+_C.TRAIN.LR_SCHEDULER.DECAY_RATE = 0.1 # only used in StepLRScheduler
+
+_C.TRAIN.OPTIMIZER = CN()
+_C.TRAIN.OPTIMIZER.NAME = 'AdamW'
+_C.TRAIN.OPTIMIZER.EPS = 1e-8
+_C.TRAIN.OPTIMIZER.BETAS = (0.9, 0.999) # for adamW
+_C.TRAIN.OPTIMIZER.MOMENTUM = 0.9
+
+# misc
+_C.SAVE = "./output"
+_C.TAG = "default"
+_C.SAVE_FREQ = 10 # freq to save chpt
+_C.REPORT_FREQ = 100 # freq to logging info
+_C.VALIDATE_FREQ = 100 # freq to do validation
+_C.SEED = 0
+_C.EVAL = False # run evaluation only
+_C.AMP = False # mix precision training
+_C.LOCAL_RANK = 0
+_C.NGPUS = -1
+
+
+def _update_config_from_file(config, cfg_file):
+ config.defrost()
+ with open(cfg_file, 'r') as infile:
+ yaml_cfg = yaml.load(infile, Loader=yaml.FullLoader)
+ for cfg in yaml_cfg.setdefault('BASE', ['']):
+ if cfg:
+ _update_config_from_file(
+ config, os.path.join(os.path.dirname(cfg_file), cfg)
+ )
+ print('merging config from {}'.format(cfg_file))
+ config.merge_from_file(cfg_file)
+ config.freeze()
+
+def update_config(config, args):
+ """Update config by ArgumentParser
+ Args:
+ args: ArgumentParser contains options
+ Return:
+ config: updated config
+ """
+ if args.cfg:
+ _update_config_from_file(config, args.cfg)
+ config.defrost()
+ if args.dataset:
+ config.DATA.DATASET = args.dataset
+ if args.batch_size:
+ config.DATA.BATCH_SIZE = args.batch_size
+ if args.image_size:
+ config.DATA.IMAGE_SIZE = args.image_size
+ if args.data_path:
+ config.DATA.DATA_PATH = args.data_path
+ if args.ngpus:
+ config.NGPUS = args.ngpus
+ if args.eval:
+ config.EVAL = True
+ config.DATA.BATCH_SIZE_EVAL = args.batch_size
+ if args.pretrained:
+ config.MODEL.PRETRAINED = args.pretrained
+ if args.resume:
+ config.MODEL.RESUME = args.resume
+ if args.last_epoch:
+ config.TRAIN.LAST_EPOCH = args.last_epoch
+ if args.amp: # only during training
+ if config.EVAL is True:
+ config.AMP = False
+ else:
+ config.AMP = True
+
+ #config.freeze()
+ return config
+
+
+def get_config(cfg_file=None):
+ """Return a clone of config or load from yaml file"""
+ config = _C.clone()
+ if cfg_file:
+ _update_config_from_file(config, cfg_file)
+ return config
diff --git a/image_classification/Multi_Node_Training/configs/vit_base_patch16_224.yaml b/image_classification/Multi_Node_Training/configs/vit_base_patch16_224.yaml
new file mode 100644
index 00000000..eff0fc29
--- /dev/null
+++ b/image_classification/Multi_Node_Training/configs/vit_base_patch16_224.yaml
@@ -0,0 +1,21 @@
+DATA:
+ IMAGE_SIZE: 224
+ CROP_PCT: 0.875
+MODEL:
+ TYPE: ViT
+ NAME: vit_base_patch16_224
+ TRANS:
+ PATCH_SIZE: 16
+ EMBED_DIM: 768
+ MLP_RATIO: 4.0
+ DEPTH: 12
+ NUM_HEADS: 12
+ QKV_BIAS: true
+TRAIN:
+ NUM_EPOCHS: 300
+ WARMUP_EPOCHS: 3
+ WEIGHT_DECAY: 0.3
+ BASE_LR: 0.003
+ WARMUP_START_LR: 1e-6
+ END_LR: 5e-4
+ ACCUM_ITER: 2
diff --git a/image_classification/Multi_Node_Training/configs/vit_base_patch16_384.yaml b/image_classification/Multi_Node_Training/configs/vit_base_patch16_384.yaml
new file mode 100644
index 00000000..04cdfaee
--- /dev/null
+++ b/image_classification/Multi_Node_Training/configs/vit_base_patch16_384.yaml
@@ -0,0 +1,14 @@
+DATA:
+ IMAGE_SIZE: 384
+ CROP_PCT: 1.0
+MODEL:
+ TYPE: ViT
+ NAME: vit_base_patch16_384
+ TRANS:
+ PATCH_SIZE: 16
+ EMBED_DIM: 768
+ MLP_RATIO: 4.0
+ DEPTH: 12
+ NUM_HEADS: 12
+ QKV_BIAS: true
+
diff --git a/image_classification/Multi_Node_Training/configs/vit_base_patch32_224.yaml b/image_classification/Multi_Node_Training/configs/vit_base_patch32_224.yaml
new file mode 100644
index 00000000..8b0516d2
--- /dev/null
+++ b/image_classification/Multi_Node_Training/configs/vit_base_patch32_224.yaml
@@ -0,0 +1,21 @@
+DATA:
+ IMAGE_SIZE: 224
+ CROP_PCT: 0.875
+MODEL:
+ TYPE: ViT
+ NAME: vit_base_patch32_224
+ TRANS:
+ PATCH_SIZE: 32
+ EMBED_DIM: 768
+ MLP_RATIO: 4.0
+ DEPTH: 12
+ NUM_HEADS: 12
+ QKV_BIAS: true
+TRAIN:
+ NUM_EPOCHS: 300
+ WARMUP_EPOCHS: 3
+ WEIGHT_DECAY: 0.3
+ BASE_LR: 0.003
+ WARMUP_START_LR: 1e-6
+ END_LR: 5e-4
+ ACCUM_ITER: 2
diff --git a/image_classification/Multi_Node_Training/configs/vit_base_patch32_384.yaml b/image_classification/Multi_Node_Training/configs/vit_base_patch32_384.yaml
new file mode 100644
index 00000000..5aa3e6f1
--- /dev/null
+++ b/image_classification/Multi_Node_Training/configs/vit_base_patch32_384.yaml
@@ -0,0 +1,21 @@
+DATA:
+ IMAGE_SIZE: 384
+ CROP_PCT: 1.0
+MODEL:
+ TYPE: ViT
+ NAME: vit_base_patch32_384
+ TRANS:
+ PATCH_SIZE: 32
+ EMBED_DIM: 768
+ MLP_RATIO: 4.0
+ DEPTH: 12
+ NUM_HEADS: 12
+ QKV_BIAS: true
+TRAIN:
+ NUM_EPOCHS: 300
+ WARMUP_EPOCHS: 3
+ WEIGHT_DECAY: 0.3
+ BASE_LR: 0.003
+ WARMUP_START_LR: 1e-6
+ END_LR: 5e-4
+ ACCUM_ITER: 2
diff --git a/image_classification/Multi_Node_Training/configs/vit_large_patch16_224.yaml b/image_classification/Multi_Node_Training/configs/vit_large_patch16_224.yaml
new file mode 100644
index 00000000..23ac9b37
--- /dev/null
+++ b/image_classification/Multi_Node_Training/configs/vit_large_patch16_224.yaml
@@ -0,0 +1,14 @@
+DATA:
+ IMAGE_SIZE: 224
+ CROP_PCT: 0.875
+MODEL:
+ TYPE: ViT
+ NAME: vit_large_patch16_224
+ TRANS:
+ PATCH_SIZE: 16
+ EMBED_DIM: 1024
+ MLP_RATIO: 4.0
+ DEPTH: 24
+ NUM_HEADS: 16
+ QKV_BIAS: true
+
diff --git a/image_classification/Multi_Node_Training/configs/vit_large_patch16_384.yaml b/image_classification/Multi_Node_Training/configs/vit_large_patch16_384.yaml
new file mode 100644
index 00000000..c8c01a6a
--- /dev/null
+++ b/image_classification/Multi_Node_Training/configs/vit_large_patch16_384.yaml
@@ -0,0 +1,14 @@
+DATA:
+ IMAGE_SIZE: 384
+ CROP_PCT: 1.0
+MODEL:
+ TYPE: ViT
+ NAME: vit_large_patch16_384
+ TRANS:
+ PATCH_SIZE: 16
+ EMBED_DIM: 1024
+ MLP_RATIO: 4.0
+ DEPTH: 24
+ NUM_HEADS: 16
+ QKV_BIAS: true
+
diff --git a/image_classification/Multi_Node_Training/configs/vit_large_patch32_384.yaml b/image_classification/Multi_Node_Training/configs/vit_large_patch32_384.yaml
new file mode 100644
index 00000000..6b7f15aa
--- /dev/null
+++ b/image_classification/Multi_Node_Training/configs/vit_large_patch32_384.yaml
@@ -0,0 +1,14 @@
+DATA:
+ IMAGE_SIZE: 384
+ CROP_PCT: 1.0
+MODEL:
+ TYPE: ViT
+ NAME: vit_large_patch32_384
+ TRANS:
+ PATCH_SIZE: 32
+ EMBED_DIM: 1024
+ MLP_RATIO: 4.0
+ DEPTH: 24
+ NUM_HEADS: 16
+ QKV_BIAS: true
+
diff --git a/image_classification/Multi_Node_Training/datasets.py b/image_classification/Multi_Node_Training/datasets.py
new file mode 100644
index 00000000..e207f9ba
--- /dev/null
+++ b/image_classification/Multi_Node_Training/datasets.py
@@ -0,0 +1,187 @@
+# Copyright (c) 2021 PPViT Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Dataset related classes and methods for ViT training and validation
+Cifar10, Cifar100 and ImageNet2012 are supported
+"""
+
+import os
+import math
+from paddle.io import Dataset, DataLoader, DistributedBatchSampler
+from paddle.vision import transforms, datasets, image_load
+
+class ImageNet2012Dataset(Dataset):
+ """Build ImageNet2012 dataset
+
+ This class gets train/val imagenet datasets, which loads transfomed data and labels.
+
+ Attributes:
+ file_folder: path where imagenet images are stored
+ transform: preprocessing ops to apply on image
+ img_path_list: list of full path of images in whole dataset
+ label_list: list of labels of whole dataset
+ """
+
+ def __init__(self, file_folder, mode="train", transform=None):
+ """Init ImageNet2012 Dataset with dataset file path, mode(train/val), and transform"""
+ super(ImageNet2012Dataset, self).__init__()
+ assert mode in ["train", "val"]
+ self.file_folder = file_folder
+ self.transform = transform
+ self.img_path_list = []
+ self.label_list = []
+
+ if mode == "train":
+ self.list_file = os.path.join(self.file_folder, "train_list.txt")
+ else:
+ self.list_file = os.path.join(self.file_folder, "val_list.txt")
+
+ with open(self.list_file, 'r') as infile:
+ for line in infile:
+ img_path = line.strip().split()[0]
+ img_label = int(line.strip().split()[1])
+ self.img_path_list.append(os.path.join(self.file_folder, img_path))
+ self.label_list.append(img_label)
+ print(f'----- Imagenet2012 image {mode} list len = {len(self.label_list)}')
+
+ def __len__(self):
+ return len(self.label_list)
+
+ def __getitem__(self, index):
+ data = image_load(self.img_path_list[index]).convert('RGB')
+ data = self.transform(data)
+ label = self.label_list[index]
+
+ return data, label
+
+
+def get_train_transforms(config):
+ """ Get training transforms
+
+ For training, a RandomResizedCrop is applied, then normalization is applied with
+ [0.5, 0.5, 0.5] mean and std. The input pixel values must be rescaled to [0, 1.]
+ Outputs is converted to tensor
+
+ Args:
+ config: configs contains IMAGE_SIZE, see config.py for details
+ Returns:
+ transforms_train: training transforms
+ """
+
+ transforms_train = transforms.Compose([
+ transforms.RandomResizedCrop((config.DATA.IMAGE_SIZE, config.DATA.IMAGE_SIZE),
+ scale=(0.05, 1.0)),
+ transforms.ToTensor(),
+ transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]),
+ #transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+ ])
+ return transforms_train
+
+
+def get_val_transforms(config):
+ """ Get training transforms
+
+ For validation, image is first Resize then CenterCrop to image_size.
+ Then normalization is applied with [0.5, 0.5, 0.5] mean and std.
+ The input pixel values must be rescaled to [0, 1.]
+ Outputs is converted to tensor
+
+ Args:
+ config: configs contains IMAGE_SIZE, see config.py for details
+ Returns:
+ transforms_train: training transforms
+ """
+
+ scale_size = int(math.floor(config.DATA.IMAGE_SIZE / config.DATA.CROP_PCT))
+ transforms_val = transforms.Compose([
+ transforms.Resize(scale_size, 'bicubic'), # single int for resize shorter side of image
+ transforms.CenterCrop((config.DATA.IMAGE_SIZE, config.DATA.IMAGE_SIZE)),
+ transforms.ToTensor(),
+ transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]),
+ #transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+ ])
+ return transforms_val
+
+
+def get_dataset(config, mode='train'):
+ """ Get dataset from config and mode (train/val)
+
+ Returns the related dataset object according to configs and mode(train/val)
+
+ Args:
+ config: configs contains dataset related settings. see config.py for details
+ Returns:
+ dataset: dataset object
+ """
+ assert mode in ['train', 'val']
+ if config.DATA.DATASET == "cifar10":
+ if mode == 'train':
+ dataset = datasets.Cifar10(mode=mode, transform=get_train_transforms(config))
+ else:
+ mode = 'test'
+ dataset = datasets.Cifar10(mode=mode, transform=get_val_transforms(config))
+ elif config.DATA.DATASET == "cifar100":
+ if mode == 'train':
+ dataset = datasets.Cifar100(mode=mode, transform=get_train_transforms(config))
+ else:
+ mode = 'test'
+ dataset = datasets.Cifar100(mode=mode, transform=get_val_transforms(config))
+ elif config.DATA.DATASET == "imagenet2012":
+ if mode == 'train':
+ dataset = ImageNet2012Dataset(config.DATA.DATA_PATH,
+ mode=mode,
+ transform=get_train_transforms(config))
+ else:
+ dataset = ImageNet2012Dataset(config.DATA.DATA_PATH,
+ mode=mode,
+ transform=get_val_transforms(config))
+ else:
+ raise NotImplementedError(
+ "[{config.DATA.DATASET}] Only cifar10, cifar100, imagenet2012 are supported now")
+ return dataset
+
+
+def get_dataloader(config, dataset, mode='train', multi_process=False):
+ """Get dataloader with config, dataset, mode as input, allows multiGPU settings.
+
+ Multi-GPU loader is implements as distributedBatchSampler.
+
+ Args:
+ config: see config.py for details
+ dataset: paddle.io.dataset object
+ mode: train/val
+ multi_process: if True, use DistributedBatchSampler to support multi-processing
+ Returns:
+ dataloader: paddle.io.DataLoader object.
+ """
+
+ if mode == 'train':
+ batch_size = config.DATA.BATCH_SIZE
+ else:
+ batch_size = config.DATA.BATCH_SIZE_EVAL
+
+ if multi_process is True:
+ sampler = DistributedBatchSampler(dataset,
+ batch_size=batch_size,
+ shuffle=(mode == 'train'))
+ dataloader = DataLoader(dataset,
+ batch_sampler=sampler,
+ num_workers=config.DATA.NUM_WORKERS)
+ else:
+ dataloader = DataLoader(dataset,
+ batch_size=batch_size,
+ num_workers=config.DATA.NUM_WORKERS,
+ shuffle=(mode == 'train'))
+ return dataloader
diff --git a/image_classification/Multi_Node_Training/droppath.py b/image_classification/Multi_Node_Training/droppath.py
new file mode 100644
index 00000000..25b8d5ff
--- /dev/null
+++ b/image_classification/Multi_Node_Training/droppath.py
@@ -0,0 +1,60 @@
+# Copyright (c) 2021 PPViT Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Droppath, reimplement from https://github.com/yueatsprograms/Stochastic_Depth
+"""
+
+import paddle
+import paddle.nn as nn
+
+
+class DropPath(nn.Layer):
+ """DropPath class"""
+ def __init__(self, drop_prob=None):
+ super(DropPath, self).__init__()
+ self.drop_prob = drop_prob
+
+ def drop_path(self, inputs):
+ """drop path op
+ Args:
+ input: tensor with arbitrary shape
+ drop_prob: float number of drop path probability, default: 0.0
+ training: bool, if current mode is training, default: False
+ Returns:
+ output: output tensor after drop path
+ """
+ # if prob is 0 or eval mode, return original input
+ if self.drop_prob == 0. or not self.training:
+ return inputs
+ keep_prob = 1 - self.drop_prob
+ keep_prob = paddle.to_tensor(keep_prob, dtype='float32')
+ shape = (inputs.shape[0], ) + (1, ) * (inputs.ndim - 1) # shape=(N, 1, 1, 1)
+ random_tensor = keep_prob + paddle.rand(shape, dtype=inputs.dtype)
+ random_tensor = random_tensor.floor() # mask
+ output = inputs.divide(keep_prob) * random_tensor #divide is to keep same output expectation
+ return output
+
+ def forward(self, inputs):
+ return self.drop_path(inputs)
+
+
+#def main():
+# tmp = paddle.to_tensor(np.random.rand(8, 16, 8, 8), dtype='float32')
+# dp = DropPath(0.5)
+# out = dp(tmp)
+# print(out)
+#
+#if __name__ == "__main__":
+# main()
diff --git a/image_classification/Multi_Node_Training/main_multi_node.py b/image_classification/Multi_Node_Training/main_multi_node.py
new file mode 100644
index 00000000..bb0462b2
--- /dev/null
+++ b/image_classification/Multi_Node_Training/main_multi_node.py
@@ -0,0 +1,379 @@
+# Copyright (c) 2021 PPViT Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""ViT training/validation using multiple GPU """
+
+import sys
+import os
+import time
+import logging
+import argparse
+import random
+import numpy as np
+import paddle
+import paddle.nn as nn
+import paddle.nn.functional as F
+import paddle.distributed as dist
+from datasets import get_dataloader
+from datasets import get_dataset
+from transformer import build_vit as build_model
+from utils import AverageMeter
+from utils import WarmupCosineScheduler
+from config import get_config
+from config import update_config
+
+
+parser = argparse.ArgumentParser('ViT')
+parser.add_argument('-cfg', type=str, default=None)
+parser.add_argument('-dataset', type=str, default=None)
+parser.add_argument('-batch_size', type=int, default=None)
+parser.add_argument('-image_size', type=int, default=None)
+parser.add_argument('-data_path', type=str, default=None)
+parser.add_argument('-ngpus', type=int, default=None)
+parser.add_argument('-pretrained', type=str, default=None)
+parser.add_argument('-resume', type=str, default=None)
+parser.add_argument('-last_epoch', type=int, default=None)
+parser.add_argument('-eval', action='store_true')
+parser.add_argument('-amp', action='store_true')
+parser.add_argument('-ips', type=str, default="127.0.0.1")
+arguments = parser.parse_args()
+
+
+log_format = "%(asctime)s %(message)s"
+logging.basicConfig(stream=sys.stdout, level=logging.INFO,
+ format=log_format, datefmt="%m%d %I:%M:%S %p")
+
+# get default config
+config = get_config()
+# update config by arguments
+config = update_config(config, arguments)
+
+# set output folder
+if not config.EVAL:
+ config.SAVE = '{}/train-{}'.format(config.SAVE, time.strftime('%Y%m%d-%H-%M-%S'))
+else:
+ config.SAVE = '{}/eval-{}'.format(config.SAVE, time.strftime('%Y%m%d-%H-%M-%S'))
+
+if not os.path.exists(config.SAVE):
+ os.makedirs(config.SAVE, exist_ok=True)
+
+# set logging format
+logger = logging.getLogger()
+fh = logging.FileHandler(os.path.join(config.SAVE, 'log.txt'))
+fh.setFormatter(logging.Formatter(log_format))
+logger.addHandler(fh)
+logger.info(f'config= {config}')
+
+
+def train(dataloader,
+ model,
+ criterion,
+ optimizer,
+ epoch,
+ total_batch,
+ debug_steps=100,
+ accum_iter=1,
+ amp=False):
+ """Training for one epoch
+ Args:
+ dataloader: paddle.io.DataLoader, dataloader instance
+ model: nn.Layer, a ViT model
+ criterion: nn.criterion
+ epoch: int, current epoch
+ total_epoch: int, total num of epoch, for logging
+ debug_steps: int, num of iters to log info, default: 100
+ accum_iter: int, num of iters for accumulating gradients, default: 1
+ amp: bool, if True, use mix precision training, default: False
+ Returns:
+ train_loss_meter.avg
+ train_acc_meter.avg
+ train_time
+ """
+ model.train()
+ train_loss_meter = AverageMeter()
+ train_acc_meter = AverageMeter()
+ if amp is True:
+ scaler = paddle.amp.GradScaler(init_loss_scaling=1024)
+ time_st = time.time()
+
+ for batch_id, data in enumerate(dataloader):
+ image = data[0]
+ label = data[1]
+
+ if amp is True:
+ with paddle.amp.auto_cast():
+ output = model(image)
+ loss = criterion(output, label)
+ scaled = scaler.scale(loss)
+ scaled.backward()
+
+ if ((batch_id +1) % accum_iter == 0) or (batch_id + 1 == len(dataloader)):
+ scaler.minimize(optimizer, scaled)
+ optimizer.clear_grad()
+ else:
+ output = model(image)
+ loss = criterion(output, label)
+ #NOTE: division may be needed depending on the loss function
+ # Here no division is needed:
+ # default 'reduction' param in nn.CrossEntropyLoss is set to 'mean'
+ #
+ #loss = loss / accum_iter
+ loss.backward()
+
+ if ((batch_id +1) % accum_iter == 0) or (batch_id + 1 == len(dataloader)):
+ optimizer.step()
+ optimizer.clear_grad()
+
+ pred = F.softmax(output)
+ acc = paddle.metric.accuracy(pred, label.unsqueeze(1))
+
+ batch_size = image.shape[0]
+ train_loss_meter.update(loss.numpy()[0], batch_size)
+ train_acc_meter.update(acc.numpy()[0], batch_size)
+
+ if batch_id % debug_steps == 0:
+ logger.info(
+ f"Epoch[{epoch:03d}/{config.TRAIN.NUM_EPOCHS:03d}], " +
+ f"Step[{batch_id:04d}/{total_batch:04d}], " +
+ f"Avg Loss: {train_loss_meter.avg:.4f}, " +
+ f"Avg Acc: {train_acc_meter.avg:.4f}")
+
+ train_time = time.time() - time_st
+ return train_loss_meter.avg, train_acc_meter.avg, train_time
+
+
+def validate(dataloader, model, criterion, total_batch, debug_steps=100):
+ """Validation for whole dataset
+ Args:
+ dataloader: paddle.io.DataLoader, dataloader instance
+ model: nn.Layer, a ViT model
+ criterion: nn.criterion
+ total_epoch: int, total num of epoch, for logging
+ debug_steps: int, num of iters to log info
+ Returns:
+ val_loss_meter.avg
+ val_acc1_meter.avg
+ val_acc5_meter.avg
+ val_time
+ """
+ model.eval()
+ val_loss_meter = AverageMeter()
+ val_acc1_meter = AverageMeter()
+ val_acc5_meter = AverageMeter()
+ time_st = time.time()
+
+ with paddle.no_grad():
+ for batch_id, data in enumerate(dataloader):
+ image = data[0]
+ label = data[1]
+
+ output = model(image)
+ loss = criterion(output, label)
+
+ pred = F.softmax(output)
+ acc1 = paddle.metric.accuracy(pred, label.unsqueeze(1))
+ acc5 = paddle.metric.accuracy(pred, label.unsqueeze(1), k=5)
+
+ dist.all_reduce(loss)
+ dist.all_reduce(acc1)
+ dist.all_reduce(acc5)
+ loss = loss / dist.get_world_size()
+ acc1 = acc1 / dist.get_world_size()
+ acc5 = acc5 / dist.get_world_size()
+
+ batch_size = paddle.to_tensor(image.shape[0])
+ dist.all_reduce(batch_size)
+
+ val_loss_meter.update(loss.numpy()[0], batch_size.numpy()[0])
+ val_acc1_meter.update(acc1.numpy()[0], batch_size.numpy()[0])
+ val_acc5_meter.update(acc5.numpy()[0], batch_size.numpy()[0])
+
+ if batch_id % debug_steps == 0:
+ logger.info(
+ f"Val Step[{batch_id:04d}/{total_batch:04d}], " +
+ f"Avg Loss: {val_loss_meter.avg:.4f}, " +
+ f"Avg Acc@1: {val_acc1_meter.avg:.4f}, " +
+ f"Avg Acc@5: {val_acc5_meter.avg:.4f}")
+
+ val_time = time.time() - time_st
+ return val_loss_meter.avg, val_acc1_meter.avg, val_acc5_meter.avg, val_time
+
+
+def main_worker(*args):
+ # 0. Preparation
+ dist.init_parallel_env()
+ last_epoch = config.TRAIN.LAST_EPOCH
+ world_size = paddle.distributed.get_world_size()
+ local_rank = paddle.distributed.get_rank()
+ logger.info(f'----- world_size = {world_size}, local_rank = {local_rank}')
+ seed = config.SEED + local_rank
+ paddle.seed(seed)
+ np.random.seed(seed)
+ random.seed(seed)
+ # 1. Create model
+ model = build_model(config)
+ model = paddle.DataParallel(model)
+ # 2. Create train and val dataloader
+ dataset_train, dataset_val = args[0], args[1]
+ dataloader_train = get_dataloader(config, dataset_train, 'train', True)
+ dataloader_val = get_dataloader(config, dataset_val, 'test', True)
+ total_batch_train = len(dataloader_train)
+ total_batch_val = len(dataloader_val)
+ logging.info(f'----- Total # of train batch (single gpu): {total_batch_train}')
+ logging.info(f'----- Total # of val batch (single gpu): {total_batch_val}')
+ # 3. Define criterion
+ criterion = nn.CrossEntropyLoss()
+ # 4. Define optimizer and lr_scheduler
+ scheduler = None
+ if config.TRAIN.LR_SCHEDULER.NAME == "warmupcosine":
+ scheduler = WarmupCosineScheduler(learning_rate=config.TRAIN.BASE_LR,
+ warmup_start_lr=config.TRAIN.WARMUP_START_LR,
+ start_lr=config.TRAIN.BASE_LR,
+ end_lr=config.TRAIN.END_LR,
+ warmup_epochs=config.TRAIN.WARMUP_EPOCHS,
+ total_epochs=config.TRAIN.NUM_EPOCHS,
+ last_epoch=config.TRAIN.LAST_EPOCH,
+ )
+ elif config.TRAIN.LR_SCHEDULER.NAME == "cosine":
+ scheduler = paddle.optimizer.lr.CosineAnnealingDecay(learning_rate=config.TRAIN.BASE_LR,
+ T_max=config.TRAIN.NUM_EPOCHS,
+ last_epoch=last_epoch)
+ elif config.scheduler == "multi-step":
+ milestones = [int(v.strip()) for v in config.TRAIN.LR_SCHEDULER.MILESTONES.split(",")]
+ scheduler = paddle.optimizer.lr.MultiStepDecay(learning_rate=config.TRAIN.BASE_LR,
+ milestones=milestones,
+ gamma=config.TRAIN.LR_SCHEDULER.DECAY_RATE,
+ last_epoch=last_epoch)
+ else:
+ logging.fatal(f"Unsupported Scheduler: {config.TRAIN.LR_SCHEDULER}.")
+ raise NotImplementedError(f"Unsupported Scheduler: {config.TRAIN.LR_SCHEDULER}.")
+
+ if config.TRAIN.OPTIMIZER.NAME == "SGD":
+ if config.TRAIN.GRAD_CLIP:
+ clip = paddle.nn.ClipGradByGlobalNorm(config.TRAIN.GRAD_CLIP)
+ else:
+ clip = None
+ optimizer = paddle.optimizer.Momentum(
+ parameters=model.parameters(),
+ learning_rate=scheduler if scheduler is not None else config.TRAIN.BASE_LR,
+ weight_decay=config.TRAIN.WEIGHT_DECAY,
+ momentum=config.TRAIN.OPTIMIZER.MOMENTUM,
+ grad_clip=clip)
+ elif config.TRAIN.OPTIMIZER.NAME == "AdamW":
+ if config.TRAIN.GRAD_CLIP:
+ clip = paddle.nn.ClipGradByGlobalNorm(config.TRAIN.GRAD_CLIP)
+ else:
+ clip = None
+ optimizer = paddle.optimizer.AdamW(
+ parameters=model.parameters(),
+ learning_rate=scheduler if scheduler is not None else config.TRAIN.BASE_LR,
+ beta1=config.TRAIN.OPTIMIZER.BETAS[0],
+ beta2=config.TRAIN.OPTIMIZER.BETAS[1],
+ weight_decay=config.TRAIN.WEIGHT_DECAY,
+ epsilon=config.TRAIN.OPTIMIZER.EPS,
+ grad_clip=clip,
+ #apply_decay_param_fun=get_exclude_from_weight_decay_fn(['pos_embed', 'cls_token']),
+ )
+ else:
+ logging.fatal(f"Unsupported Optimizer: {config.TRAIN.OPTIMIZER.NAME}.")
+ raise NotImplementedError(f"Unsupported Optimizer: {config.TRAIN.OPTIMIZER.NAME}.")
+
+ # 5. Load pretrained model / load resumt model and optimizer states
+ if config.MODEL.PRETRAINED:
+ if (config.MODEL.PRETRAINED).endswith('.pdparams'):
+ raise ValueError(f'{config.MODEL.PRETRAINED} should not contain .pdparams')
+ assert os.path.isfile(config.MODEL.PRETRAINED + '.pdparams') is True
+ model_state = paddle.load(config.MODEL.PRETRAINED+'.pdparams')
+ model.set_dict(model_state)
+ logger.info(f"----- Pretrained: Load model state from {config.MODEL.PRETRAINED}")
+
+ if config.MODEL.RESUME:
+ assert os.path.isfile(config.MODEL.RESUME+'.pdparams') is True
+ assert os.path.isfile(config.MODEL.RESUME+'.pdopt') is True
+ model_state = paddle.load(config.MODEL.RESUME+'.pdparams')
+ model.set_dict(model_state)
+ opt_state = paddle.load(config.MODEL.RESUME+'.pdopt')
+ optimizer.set_state_dict(opt_state)
+ logger.info(
+ f"----- Resume Training: Load model and optmizer states from {config.MODEL.RESUME}")
+
+ # 6. Validation
+ if config.EVAL:
+ logger.info('----- Start Validating')
+ val_loss, val_acc1, val_acc5, val_time = validate(
+ dataloader=dataloader_val,
+ model=model,
+ criterion=criterion,
+ total_batch=total_batch_val,
+ debug_steps=config.REPORT_FREQ)
+ logger.info(f"Validation Loss: {val_loss:.4f}, " +
+ f"Validation Acc@1: {val_acc1:.4f}, " +
+ f"Validation Acc@5: {val_acc5:.4f}, " +
+ f"time: {val_time:.2f}")
+ return
+
+ # 6. Start training and validation
+ logging.info(f"Start training from epoch {last_epoch+1}.")
+ for epoch in range(last_epoch+1, config.TRAIN.NUM_EPOCHS+1):
+ # train
+ logging.info(f"Now training epoch {epoch}. LR={optimizer.get_lr():.6f}")
+ train_loss, train_acc, train_time = train(dataloader=dataloader_train,
+ model=model,
+ criterion=criterion,
+ optimizer=optimizer,
+ epoch=epoch,
+ total_batch=total_batch_train,
+ debug_steps=config.REPORT_FREQ,
+ accum_iter=config.TRAIN.ACCUM_ITER,
+ amp=config.AMP)
+ scheduler.step()
+
+ logger.info(f"----- Epoch[{epoch:03d}/{config.TRAIN.NUM_EPOCHS:03d}], " +
+ f"Train Loss: {train_loss:.4f}, " +
+ f"Train Acc: {train_acc:.4f}, " +
+ f"time: {train_time:.2f}")
+ # validation
+ if epoch % config.VALIDATE_FREQ == 0 or epoch == config.TRAIN.NUM_EPOCHS:
+ logger.info(f'----- Validation after Epoch: {epoch}')
+ val_loss, val_acc1, val_acc5, val_time = validate(
+ dataloader=dataloader_val,
+ model=model,
+ criterion=criterion,
+ total_batch=total_batch_val,
+ debug_steps=config.REPORT_FREQ)
+ logger.info(f"----- Epoch[{epoch:03d}/{config.TRAIN.NUM_EPOCHS:03d}], " +
+ f"Validation Loss: {val_loss:.4f}, " +
+ f"Validation Acc@1: {val_acc1:.4f}, " +
+ f"Validation Acc@5: {val_acc5:.4f}, " +
+ f"time: {val_time:.2f}")
+ # model save
+ if local_rank == 0:
+ if epoch % config.SAVE_FREQ == 0 or epoch == config.TRAIN.NUM_EPOCHS:
+ model_path = os.path.join(
+ config.SAVE, f"{config.MODEL.TYPE}-Epoch-{epoch}-Loss-{train_loss}")
+ paddle.save(model.state_dict(), model_path + '.pdparams')
+ paddle.save(optimizer.state_dict(), model_path + '.pdopt')
+ logger.info(f"----- Save model: {model_path}.pdparams")
+ logger.info(f"----- Save optim: {model_path}.pdopt")
+
+
+def main():
+ dataset_train = get_dataset(config, mode='train')
+ dataset_val = get_dataset(config, mode='val')
+ config.NGPUS = len(paddle.static.cuda_places()) if config.NGPUS == -1 else config.NGPUS
+ dist.spawn(main_worker, args=(dataset_train, dataset_val, ), nprocs=config.NGPUS, ips=arguments.ips)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/image_classification/Multi_Node_Training/run_train_multi_node.sh b/image_classification/Multi_Node_Training/run_train_multi_node.sh
new file mode 100644
index 00000000..bc3893be
--- /dev/null
+++ b/image_classification/Multi_Node_Training/run_train_multi_node.sh
@@ -0,0 +1,8 @@
+CUDA_VISIBLE_DEVICES=0,1 \
+python main_multi_node.py \
+-cfg='./configs/vit_base_patch16_224.yaml' \
+-dataset='imagenet2012' \
+-batch_size=32 \
+-data_path='./dataset/imagenet' \
+-ips='172.18.0.2, 172.18.0.3' # the ips should be replaced
+#-amp
diff --git a/image_classification/Multi_Node_Training/transformer.py b/image_classification/Multi_Node_Training/transformer.py
new file mode 100644
index 00000000..384cacf5
--- /dev/null
+++ b/image_classification/Multi_Node_Training/transformer.py
@@ -0,0 +1,437 @@
+# Copyright (c) 2021 PPViT Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Implement Transformer Class for ViT
+"""
+
+import copy
+import paddle
+import paddle.nn as nn
+from droppath import DropPath
+from config import get_config
+
+
+class Identity(nn.Layer):
+ """ Identity layer
+ The output of this layer is the input without any change.
+ Use this layer to avoid using 'if' condition in forward methods
+ """
+ def __init__(self):
+ super(Identity, self).__init__()
+
+ def forward(self, x):
+ return x
+
+
+class PatchEmbedding(nn.Layer):
+ """Patch Embedding and Position Embedding
+
+ Apply patch embedding and position embedding on input images.
+
+ Attributes:
+ patch_embddings: impl using a patch_size x patch_size Conv2D operation
+ position_embddings: a parameter with len = num_patch + 1(for cls_token)
+ cls_token: token insert to the patch feature for classification
+ dropout: dropout for embeddings
+ """
+
+ def __init__(self,
+ image_size=224,
+ patch_size=16,
+ in_channels=3,
+ embed_dim=768,
+ dropout=0.):
+ super().__init__()
+ n_patches = (image_size // patch_size) * (image_size // patch_size)
+
+ self.patch_embedding = nn.Conv2D(in_channels=in_channels,
+ out_channels=embed_dim,
+ kernel_size=patch_size,
+ stride=patch_size)
+
+ self.position_embeddings = paddle.create_parameter(
+ shape=[1, n_patches+1, embed_dim],
+ dtype='float32',
+ default_initializer=paddle.nn.initializer.TruncatedNormal(std=.02))
+
+ self.cls_token = paddle.create_parameter(
+ shape=[1, 1, embed_dim],
+ dtype='float32',
+ default_initializer=paddle.nn.initializer.Constant(0))
+
+ self.dropout = nn.Dropout(dropout)
+
+ def forward(self, x):
+ cls_tokens = self.cls_token.expand((x.shape[0], -1, -1))
+ x = self.patch_embedding(x)
+ x = x.flatten(2)
+ x = x.transpose([0, 2, 1])
+ x = paddle.concat((cls_tokens, x), axis=1)
+
+ embeddings = x + self.position_embeddings # tensor broadcast
+ embeddings = self.dropout(embeddings)
+ return embeddings
+
+
+class Attention(nn.Layer):
+ """ Attention module
+
+ Attention module for ViT, here q, k, v are assumed the same.
+ The qkv mappings are stored as one single param.
+
+ Attributes:
+ num_heads: number of heads
+ attn_head_size: feature dim of single head
+ all_head_size: feature dim of all heads
+ qkv: a nn.Linear for q, k, v mapping
+ scales: 1 / sqrt(single_head_feature_dim)
+ out: projection of multi-head attention
+ attn_dropout: dropout for attention
+ proj_dropout: final dropout before output
+ softmax: softmax op for attention
+ """
+ def __init__(self,
+ embed_dim,
+ num_heads,
+ qkv_bias=True,
+ dropout=0.,
+ attention_dropout=0.):
+ super().__init__()
+ self.num_heads = num_heads
+ self.attn_head_size = int(embed_dim / self.num_heads)
+ self.all_head_size = self.attn_head_size * self.num_heads
+
+ w_attr_1, b_attr_1 = self._init_weights()
+ self.qkv = nn.Linear(embed_dim,
+ self.all_head_size*3, #weights for q, k, and v
+ weight_attr=w_attr_1,
+ bias_attr=b_attr_1 if qkv_bias else False)
+
+ self.scales = self.attn_head_size ** -0.5
+
+ w_attr_2, b_attr_2 = self._init_weights()
+ self.out = nn.Linear(embed_dim,
+ embed_dim,
+ weight_attr=w_attr_2,
+ bias_attr=b_attr_2)
+
+ self.attn_dropout = nn.Dropout(attention_dropout)
+ self.proj_dropout = nn.Dropout(dropout)
+ self.softmax = nn.Softmax(axis=-1)
+
+ def _init_weights(self):
+ weight_attr = paddle.ParamAttr(initializer=nn.initializer.KaimingUniform())
+ bias_attr = paddle.ParamAttr(initializer=nn.initializer.KaimingUniform())
+ return weight_attr, bias_attr
+
+ def transpose_multihead(self, x):
+ new_shape = x.shape[:-1] + [self.num_heads, self.attn_head_size]
+ x = x.reshape(new_shape)
+ x = x.transpose([0, 2, 1, 3])
+ return x
+
+ def forward(self, x):
+ qkv = self.qkv(x).chunk(3, axis=-1)
+ q, k, v = map(self.transpose_multihead, qkv)
+
+ attn = paddle.matmul(q, k, transpose_y=True)
+ attn = attn * self.scales
+ attn = self.softmax(attn)
+ attn_weights = attn
+ attn = self.attn_dropout(attn)
+
+ z = paddle.matmul(attn, v)
+ z = z.transpose([0, 2, 1, 3])
+ new_shape = z.shape[:-2] + [self.all_head_size]
+ z = z.reshape(new_shape)
+ # reshape
+ z = self.out(z)
+ z = self.proj_dropout(z)
+ return z, attn_weights
+
+
+class Mlp(nn.Layer):
+ """ MLP module
+
+ Impl using nn.Linear and activation is GELU, dropout is applied.
+ Ops: fc -> act -> dropout -> fc -> dropout
+
+ Attributes:
+ fc1: nn.Linear
+ fc2: nn.Linear
+ act: GELU
+ dropout1: dropout after fc1
+ dropout2: dropout after fc2
+ """
+ def __init__(self,
+ embed_dim,
+ mlp_ratio,
+ dropout=0.):
+ super().__init__()
+ w_attr_1, b_attr_1 = self._init_weights()
+ self.fc1 = nn.Linear(embed_dim,
+ int(embed_dim * mlp_ratio),
+ weight_attr=w_attr_1,
+ bias_attr=b_attr_1)
+
+ w_attr_2, b_attr_2 = self._init_weights()
+ self.fc2 = nn.Linear(int(embed_dim * mlp_ratio),
+ embed_dim,
+ weight_attr=w_attr_2,
+ bias_attr=b_attr_2)
+ self.act = nn.GELU()
+ self.dropout1 = nn.Dropout(dropout)
+ self.dropout2 = nn.Dropout(dropout)
+
+ def _init_weights(self):
+ weight_attr = paddle.ParamAttr(
+ initializer=paddle.nn.initializer.XavierUniform()) #default in pp: xavier
+ bias_attr = paddle.ParamAttr(
+ initializer=paddle.nn.initializer.Normal(std=1e-6)) #default in pp: zero
+ return weight_attr, bias_attr
+
+ def forward(self, x):
+ x = self.fc1(x)
+ x = self.act(x)
+ x = self.dropout1(x)
+ x = self.fc2(x)
+ x = self.dropout2(x)
+ return x
+
+
+class EncoderLayer(nn.Layer):
+ """Encoder Layer
+
+ Encoder layer contains attention, norm, mlp and residual
+
+ Attributes:
+ hidden_size: transformer feature dim
+ attn_norm: nn.LayerNorm before attention
+ mlp_norm: nn.LayerNorm before mlp
+ mlp: mlp modual
+ attn: attention modual
+ """
+ def __init__(self,
+ embed_dim,
+ num_heads,
+ qkv_bias=True,
+ mlp_ratio=4.,
+ dropout=0.,
+ attention_dropout=0.,
+ droppath=0.):
+ super().__init__()
+ w_attr_1, b_attr_1 = self._init_weights()
+ self.attn_norm = nn.LayerNorm(embed_dim,
+ weight_attr=w_attr_1,
+ bias_attr=b_attr_1,
+ epsilon=1e-6)
+
+ self.attn = Attention(embed_dim,
+ num_heads,
+ qkv_bias,
+ dropout,
+ attention_dropout)
+ self.drop_path = DropPath(droppath) if droppath > 0. else Identity()
+
+ w_attr_2, b_attr_2 = self._init_weights()
+ self.mlp_norm = nn.LayerNorm(embed_dim,
+ weight_attr=w_attr_2,
+ bias_attr=b_attr_2,
+ epsilon=1e-6)
+
+ self.mlp = Mlp(embed_dim, mlp_ratio, dropout)
+
+ def _init_weights(self):
+ weight_attr = paddle.ParamAttr(initializer=nn.initializer.Constant(0.0))
+ bias_attr = paddle.ParamAttr(initializer=nn.initializer.Constant(1.0))
+ return weight_attr, bias_attr
+
+ def forward(self, x):
+ h = x
+ x = self.attn_norm(x)
+ x, attn = self.attn(x)
+ x = self.drop_path(x)
+ x = x + h
+
+ h = x
+ x = self.mlp_norm(x)
+ x = self.mlp(x)
+ x = self.drop_path(x)
+ x = x + h
+
+ return x, attn
+
+
+class Encoder(nn.Layer):
+ """Transformer encoder
+
+ Encoder encoder contains a list of EncoderLayer, and a LayerNorm.
+
+ Attributes:
+ layers: nn.LayerList contains multiple EncoderLayers
+ encoder_norm: nn.LayerNorm which is applied after last encoder layer
+ """
+ def __init__(self,
+ embed_dim,
+ num_heads,
+ depth,
+ qkv_bias=True,
+ mlp_ratio=4.0,
+ dropout=0.,
+ attention_dropout=0.,
+ droppath=0.):
+ super(Encoder, self).__init__()
+ # stochatic depth decay
+ depth_decay = [x.item() for x in paddle.linspace(0, droppath, depth)]
+ layer_list = []
+ for i in range(depth):
+ encoder_layer = EncoderLayer(embed_dim,
+ num_heads,
+ qkv_bias=True,
+ mlp_ratio=4.,
+ dropout=0.,
+ attention_dropout=0.,
+ droppath=depth_decay[i])
+ layer_list.append(copy.deepcopy(encoder_layer))
+ self.layers = nn.LayerList(layer_list)
+
+ w_attr_1, b_attr_1 = self._init_weights()
+ self.encoder_norm = nn.LayerNorm(embed_dim,
+ weight_attr=w_attr_1,
+ bias_attr=b_attr_1,
+ epsilon=1e-6)
+
+ def _init_weights(self):
+ weight_attr = paddle.ParamAttr(initializer=nn.initializer.Constant(0.0))
+ bias_attr = paddle.ParamAttr(initializer=nn.initializer.Constant(1.0))
+ return weight_attr, bias_attr
+
+ def forward(self, x):
+ self_attn = []
+ for layer in self.layers:
+ x, attn = layer(x)
+ self_attn.append(attn)
+ out = self.encoder_norm(x)
+ return out, self_attn
+
+
+class VisualTransformer(nn.Layer):
+ """ViT transformer
+
+ ViT Transformer, classifier is a single Linear layer for finetune,
+ For training from scratch, two layer mlp should be used.
+ Classification is done using cls_token.
+
+ Args:
+ image_size: int, input image size, default: 224
+ patch_size: int, patch size, default: 16
+ in_channels: int, input image channels, default: 3
+ num_classes: int, number of classes for classification, default: 1000
+ embed_dim: int, embedding dimension (patch embed out dim), default: 768
+ depth: int, number ot transformer blocks, default: 12
+ num_heads: int, number of attention heads, default: 12
+ mlp_ratio: float, ratio of mlp hidden dim to embed dim(mlp in dim), default: 4.0
+ qkv_bias: bool, If True, enable qkv(nn.Linear) layer with bias, default: True
+ dropout: float, dropout rate for linear layers, default: 0.
+ attention_dropout: float, dropout rate for attention layers default: 0.
+ droppath: float, droppath rate for droppath layers, default: 0.
+ """
+ def __init__(self,
+ image_size=224,
+ patch_size=16,
+ in_channels=3,
+ num_classes=1000,
+ embed_dim=768,
+ depth=12,
+ num_heads=12,
+ mlp_ratio=4,
+ qkv_bias=True,
+ dropout=0.,
+ attention_dropout=0.,
+ droppath=0.,
+ train_from_scratch=False,
+ config=None):
+ super(VisualTransformer, self).__init__()
+ # create patch embedding with positional embedding
+ self.patch_embedding = PatchEmbedding(image_size,
+ patch_size,
+ in_channels,
+ embed_dim,
+ dropout)
+ # create multi head self-attention layers
+ self.encoder = Encoder(embed_dim,
+ num_heads,
+ depth,
+ qkv_bias,
+ mlp_ratio,
+ dropout,
+ attention_dropout,
+ droppath)
+
+ # classifier head (for training from scracth)
+ if train_from_scratch:
+ w_attr_1, b_attr_1 = self._init_weights()
+ w_attr_2, b_attr_2 = self._init_weights()
+ self.classifier = nn.Sequential(
+ nn.Linear(config.MODEL.TRANS.HIDDEN_SIZE,
+ config.MODEL.TRANS.HIDDEN_SIZE,
+ weight_attr=w_attr_1,
+ bias_attr=b_attr_1),
+ nn.ReLU(),
+ nn.Dropout(config.MODEL.DROPOUT),
+ nn.Linear(config.MODEL.TRANS.HIDDEN_SIZE,
+ config.MODEL.NUM_CLASSES,
+ weight_attr=w_attr_2,
+ bias_attr=b_attr_2),
+ nn.Dropout(config.MODEL.DROPOUT),
+ )
+ else:
+ # classifier head (for finetuning)
+ w_attr_1, b_attr_1 = self._init_weights()
+ self.classifier = nn.Linear(embed_dim,
+ num_classes,
+ weight_attr=w_attr_1,
+ bias_attr=b_attr_1)
+
+ def _init_weights(self):
+ weight_attr = paddle.ParamAttr(
+ initializer=paddle.nn.initializer.KaimingUniform())
+ bias_attr = paddle.ParamAttr(
+ initializer=paddle.nn.initializer.KaimingUniform())
+ return weight_attr, bias_attr
+
+ def forward(self, x):
+ x = self.patch_embedding(x)
+ x, attn = self.encoder(x)
+ logits = self.classifier(x[:, 0]) # take only cls_token as classifier
+ return logits
+
+
+def build_vit(config):
+ model = VisualTransformer(image_size=config.DATA.IMAGE_SIZE,
+ patch_size=config.MODEL.TRANS.PATCH_SIZE,
+ in_channels=3,
+ num_classes=config.MODEL.NUM_CLASSES,
+ embed_dim=config.MODEL.TRANS.EMBED_DIM,
+ depth=config.MODEL.TRANS.DEPTH,
+ num_heads=config.MODEL.TRANS.NUM_HEADS,
+ mlp_ratio=config.MODEL.TRANS.MLP_RATIO,
+ qkv_bias=config.MODEL.TRANS.QKV_BIAS,
+ dropout=config.MODEL.DROPOUT,
+ attention_dropout=config.MODEL.ATTENTION_DROPOUT,
+ droppath=config.MODEL.DROPPATH,
+ train_from_scratch=False,
+ config=config)
+ return model
diff --git a/image_classification/Multi_Node_Training/utils.py b/image_classification/Multi_Node_Training/utils.py
new file mode 100644
index 00000000..44800527
--- /dev/null
+++ b/image_classification/Multi_Node_Training/utils.py
@@ -0,0 +1,120 @@
+# Copyright (c) 2021 PPViT Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""utils for ViT
+
+Contains AverageMeter for monitoring, get_exclude_from_decay_fn for training
+and WarmupCosineScheduler for training
+
+"""
+
+import math
+from paddle.optimizer.lr import LRScheduler
+
+
+class AverageMeter():
+ """ Meter for monitoring losses"""
+ def __init__(self):
+ self.avg = 0
+ self.sum = 0
+ self.cnt = 0
+ self.reset()
+
+ def reset(self):
+ """reset all values to zeros"""
+ self.avg = 0
+ self.sum = 0
+ self.cnt = 0
+
+ def update(self, val, n=1):
+ """update avg by val and n, where val is the avg of n values"""
+ self.sum += val * n
+ self.cnt += n
+ self.avg = self.sum / self.cnt
+
+
+
+def get_exclude_from_weight_decay_fn(exclude_list=[]):
+ """ Set params with no weight decay during the training
+
+ For certain params, e.g., positional encoding in ViT, weight decay
+ may not needed during the learning, this method is used to find
+ these params.
+
+ Args:
+ exclude_list: a list of params names which need to exclude
+ from weight decay.
+ Returns:
+ exclude_from_weight_decay_fn: a function returns True if param
+ will be excluded from weight decay
+ """
+ if len(exclude_list) == 0:
+ exclude_from_weight_decay_fn = None
+ else:
+ def exclude_fn(param):
+ for name in exclude_list:
+ if param.endswith(name):
+ return False
+ return True
+ exclude_from_weight_decay_fn = exclude_fn
+ return exclude_from_weight_decay_fn
+
+
+class WarmupCosineScheduler(LRScheduler):
+ """Warmup Cosine Scheduler
+
+ First apply linear warmup, then apply cosine decay schedule.
+ Linearly increase learning rate from "warmup_start_lr" to "start_lr" over "warmup_epochs"
+ Cosinely decrease learning rate from "start_lr" to "end_lr" over remaining
+ "total_epochs - warmup_epochs"
+
+ Attributes:
+ learning_rate: the starting learning rate (without warmup), not used here!
+ warmup_start_lr: warmup starting learning rate
+ start_lr: the starting learning rate (without warmup)
+ end_lr: the ending learning rate after whole loop
+ warmup_epochs: # of epochs for warmup
+ total_epochs: # of total epochs (include warmup)
+ """
+ def __init__(self,
+ learning_rate,
+ warmup_start_lr,
+ start_lr,
+ end_lr,
+ warmup_epochs,
+ total_epochs,
+ cycles=0.5,
+ last_epoch=-1,
+ verbose=False):
+ """init WarmupCosineScheduler """
+ self.warmup_epochs = warmup_epochs
+ self.total_epochs = total_epochs
+ self.warmup_start_lr = warmup_start_lr
+ self.start_lr = start_lr
+ self.end_lr = end_lr
+ self.cycles = cycles
+ super(WarmupCosineScheduler, self).__init__(learning_rate, last_epoch, verbose)
+
+ def get_lr(self):
+ """ return lr value """
+ if self.last_epoch < self.warmup_epochs:
+ val = (self.start_lr - self.warmup_start_lr) * float(
+ self.last_epoch)/float(self.warmup_epochs) + self.warmup_start_lr
+ return val
+
+ progress = float(self.last_epoch - self.warmup_epochs) / float(
+ max(1, self.total_epochs - self.warmup_epochs))
+ val = max(0.0, 0.5 * (1. + math.cos(math.pi * float(self.cycles) * 2.0 * progress)))
+ val = max(0.0, val * (self.start_lr - self.end_lr) + self.end_lr)
+ return val
From caf5f7dd3eb1fb8111974948b00f3b6eddca25c2 Mon Sep 17 00:00:00 2001
From: skpig <479469418@qq.com>
Date: Thu, 9 Dec 2021 23:26:08 +0800
Subject: [PATCH 4/5] Add README
---
.../Multi_Node_Training/README.md | 56 ++++++++++++++++++
.../Multi_Node_Training/README_cn.md | 45 ++++++++++++++
image_classification/ViT/vit.png | Bin 119352 -> 0 bytes
3 files changed, 101 insertions(+)
create mode 100644 image_classification/Multi_Node_Training/README_cn.md
delete mode 100644 image_classification/ViT/vit.png
diff --git a/image_classification/Multi_Node_Training/README.md b/image_classification/Multi_Node_Training/README.md
index 839e3dfa..61e17b1c 100644
--- a/image_classification/Multi_Node_Training/README.md
+++ b/image_classification/Multi_Node_Training/README.md
@@ -1 +1,57 @@
# Multiple Node Training
+English | [简体中文](./README_cn.md)
+
+PaddleVit also supports multi-node distributed training under collective mode.
+
+Here we provides a simple tutorial to modify multi-gpus training scripts
+to multi-nodes training scripts for any models in PaddleViT.
+
+This folder takes ViT model as an example.
+
+## Tutorial
+For any models in PaddleViT, one can implement multi-node training by modifying
+`main_multi_gpu.py`.
+1. Just add arguments `ips='[host ips]' ` in `dist.spawn()`.
+2. Then run training script in every host.
+
+## Training example: ViT
+Suppose you have 2 hosts (denoted as node) with 4 gpus on each machine.
+Nodes IP addresses are `192.168.0.16` and `192.168.0.17`.
+
+1. Then modify some lines of `run_train_multi_node.sh`:
+ ```shell
+ CUDA_VISIBLE_DEVICES=0,1,2,3 # number of gpus
+
+ -ips= '192.168.0.16, 192.168.0.17' # seperated by comma
+ ```
+2. Run training script in every host:
+ ```shell
+ sh run_train_multi.sh
+ ```
+
+##Multi-nodes training with one host
+It is possible to try multi-node training even when you have only one machine.
+
+1. Install docker and paddle. For more details, please refer
+ [here](https://www.paddlepaddle.org.cn/documentation/docs/zh/install/docker/fromdocker.html).
+
+2. Create a network between docker containers.
+ ```shell
+ docker network create -d bridge paddle_net
+ ```
+3. Create multiple containers as virtual hosts/nodes. Suppose creating 2 containers
+with 2 gpus on each node.
+ ```shell
+ docker run --name paddle0 -it -d --gpus "device=0,1" --network paddle_net\
+ paddlepaddle/paddle:2.2.0-gpu-cuda10.2-cudnn7 /bin/bash
+ docker run --name paddle1 -it -d --gpus "device=2,3" --network paddle_net\
+ paddlepaddle/paddle:2.2.0-gpu-cuda10.2-cudnn7 /bin/bash
+ ```
+ > Noted:
+ > 1. One can assign one gpu device to different containers. But it may occur OOM since multiple models will run on the same gpu.
+ > 2. One should use `-v` to bind PaddleViT repository to container.
+
+4. Modify `run_train_multi_node.sh` as described above and run the training script on every container.
+
+ > Noted: One can use `ping` or `ip -a` bash command to check containers' ip addresses.
+
diff --git a/image_classification/Multi_Node_Training/README_cn.md b/image_classification/Multi_Node_Training/README_cn.md
new file mode 100644
index 00000000..c879d46f
--- /dev/null
+++ b/image_classification/Multi_Node_Training/README_cn.md
@@ -0,0 +1,45 @@
+#多机多卡分布式训练
+
+简体中文 | [English](./README.md)
+
+PaddleViT 同样支持Collective多机多卡分布式训练。
+
+##教程
+对于每一个模型，用户可以直接通过修改对应模型文件夹下的`main_multi_gpu.py`
+以进行多机训练。
+1. 在`dist.spawn()`里加入`ips='[host ips]' `。
+2. 在每个主机上运行代码。
+
+##样例：ViT
+这个文件夹提供了分布式训练ViT模型的代码和shell脚本。
+假设你有2台主机，每个主机上有4张显卡。主机的ip地址为`192.168.0.16`和`192.168.0.17`。
+
+1. 修改shell脚本`run_train_multi_node.sh`的参数
+ ```shell
+ CUDA_VISIBLE_DEVICES=0,1,2,3 # number of gpus
+
+ -ips= '192.168.0.16, 192.168.0.17' # seperated by comma
+ ```
+2. 在每个主机上运行脚本代码。
+ ```shell
+ sh run_train_multi.sh
+ ```
+##单机上运行分布式训练
+如果仅有一台主机，同样可以通过docker实现单机上的分布式训练。
+1. 安装docker和paddle。[这里](https://www.paddlepaddle.org.cn/documentation/docs/zh/install/docker/fromdocker.html)
+可以下载paddlepaddle提供的docker镜像。
+2. 创建docker容器间网络。
+ ```shell
+ docker network create -d bridge paddle_net
+ ```
+3. 创建多个docker容器作为虚拟主机，假设我们创建2个容器，并分别分配2个GPU。
+ ```shell
+ docker run --name paddle0 -it -d --gpus "device=0,1" --network paddle_net\
+ paddlepaddle/paddle:2.2.0-gpu-cuda10.2-cudnn7 /bin/bash
+ docker run --name paddle1 -it -d --gpus "device=2,3" --network paddle_net\
+ paddlepaddle/paddle:2.2.0-gpu-cuda10.2-cudnn7 /bin/bash
+ ```
+ > 注意:
+ > 1. 可以将同一个GPU同时分配给多个容器，但是这可能会产生OOM错误，因为多个模型将同时运行在这个GPU上。
+ > 2. 使用`-v`挂载PaddleViT所在的目录。
+
diff --git a/image_classification/ViT/vit.png b/image_classification/ViT/vit.png
deleted file mode 100644
index a6929f74047cd663b33b407d1dd47fe706d27022..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001
literal 119352
zcmeFZg%*3l9bchA1v3qyPp6p#=u^rVkbx_(U%-Un8{C@U~70K|!GkdY*$FogKX${X~64
z8Hrk2YuG4j5K3KOIr5$L<_Mug4^<4mO2i4wcA-MP^v7nHq|8yQe}hxxfnSplr|Pe$
zTcJXkIH8PQ7HMQ;=k{c5(>f??4&ScN8zfK$%0EMQY{wN32c)+e&eGdDQrrAy0`cB$?IzdD>n8VB{S1g+TH8x
zY47Xnt2V;NS2l=lZwWA_Qr%^9Q%pv{v?|7G;wIA4U^Ku!EZCbsQ!q$i?+x&S3;Y02
zAvypI2KbE%{0L=&|M@M1Rwl%s`)~SwA1I(GC@v2CRy4FVHny_+YHc5HX&(y=1u|7q
zvsaUr;xx3jWY9OVHZW#zwzTf*Q)?@t-{a~V
zSUcGBkdXYI=&yhOxlUte)4yl3vit9C0XN9_`y0j&3`~rFjSZa2{rgi+Ia6n23pF8A
zOTaw99K4^{n7IEq{{MXQ_l*BJQ`yegR?ylKIMJT>Z~OlDC!cGhX;irWXh>Fn%y`Aps@lH%DnO&Pu{q
z0m$UueyE&Tw>r7LKfUa8C9tS{g1j)$h4$Nt;C}`Y`FM$)W?rpCt)yJ7xG};#jm%+!Mw}t+9s-yivRaWKW)z0`N7HlcS9oK_a73_OQQSt=yK~D*8k^X-V1rbC-`<8;{To&9?KKr
z-$Ses0eAl%GJfFy*ad^(1^Qn$6zdRx{At4hOBX}_?`dhYo)G_K*GMm4Sh;mu6g3LT
ze_IPMINtwS8!*KGZ}9(5gKuzBX#*CN5!IW>7QWb9&)FS}BQ}I~*^X1ax~?^!9lSQ5
zJ=WKuz6vq=6)6x;r%s|-ZxuJj`zZDO`}d>UZq7t@>-3|=dfEQ-n;RtBud&N-5fEab
zF0uT$TCZ>>C9^*Jcj5DLLJ)8{O1(TBGB!G&XWun+w<+TLGm+o)cTu&<&sGM3u}MvQuKIi187Di#bb
zH#<2z-oHGaa#^b9_zaDtup4&=Ag;?4NhL9(@_9Vj1UlJw^!~u&ixCb+B_|V)CL7bT
z3r559hvfF_$Q6&VIOtu@e_^_&bUd1?6Yzzezd2pCrmV19=3Feu@F+Uty~|@9y_=lmeyPFJ)QZ|E9Of*
z;0{jailZ%J>3-KAO6O^A16AmEQ;qNyp(>^|(+l`OvgIu;)LE}uDN1C~TQ1bF<$V`c
z#*|5u;pg`%1q@cF-yJY-^YnPyz8G6&>KR^CU5QA@yX1JaC-24+OJZ!kP*aF>IispO
z&!F98)LpAqX|Qje#^nUkp8@mXWz=q}DRnwsVViN8pQ2YSODz?RAgIe0{&8Rq1&==d
zjY1~%>*L*Kgr!7Q{cb-5@KTKtXQERrQ*8Z;>duF@wlD+hTd^`4O;xWar$Bf}E>I=oGYjL?)7^*j$p(&uzb$&~_j`+)lYz+xgB!a;5HAgwex{g-0j9ZnS
zHO81$y(;&)2^X=tHw;g;-)%R|>1ei_(`&;-$3Ck;I3bhwtK7Mdh0w^#{9$NVTzv2_4Cc5^&#_@foS1`8LDzK(0JkCeoz;;$I~2iFTr4ya@+Hb=A)fd
z6p8;aa3i3_hr84E*SeV^+Bvq0Genbe!X|O#(j4*&8}}l#55^WFXWGtPY$L$9FdeXF=
z0OXy_6KOh?Kk)i;%O_v`RUayB((NajRL8WWed2x>U6pRTN48E$Hv*O-`%FppC~kft
z)nR%23(azq1KP}bzfx2-z3z2a_Q(sT!@-qQx>lG~Ud@J9sV%(rdIM22-&7sPUNn>g9eymH
z9mekS<%d%s#VOmnJhhY|nQKe>azzX}aDNvGKJX~)SV*lnFx>gu6*t+o#}h7XC}SNW
zUC-y*L_QOHR5f;AEG5`cMy&=S|F>8m!jBRv@;gD`+hF*i%(W<~o0?%K1I6WSMSNxy
z8ys$5S$)V-*es*{;Ls#`LohO2DgmpO@Vt#x9WnW}(&~0S!~oLV9!*>H;JbU%RjeMe
zmLSL-sL{_`K$f~d>w>y@?I1o*V=J&yeQ$Gw31V
zq!BP=+j;SDa>}r3P!GP}O4<#y52LCUD$UJIms2nUl@zv@j=h7wyT9Bu-s}tGEMYF5
znutOMF)X*aYZfULk^qj6d0?TvQp9?;O#1;Rj!}1I6vD+D4zj)Uqv>Q2uD}d;?0$gu
zem&*Y7TERkenCchVGJBZiUTVRtzlp2L_%g_?bnH}CCs_~96Bo69&WU`%}u-06`od^
zbze3@?pwDSN&}9LQ<;$(^|BzZFx+{}5|@r9PI$OT{w+8^%WfG_wLTbhQ(C-FF
z;lZ5@)jA(bO2$`eO{YZKJ90+6*Kk3QS@fcw!5&cVe(#KN>0yxGkLxq(eYG^72v?6|
zzmgps)~L@DLo-c!Sh?tHVUyyC{;tNHo;c|YOxf=Z8YN<{6#
z*Y?Y;ft=06kMA(u8{bhes^!!f_9L}oB5cc5dvYS8LHqi%TP>lK;0WsGO4W|ge=y(3
zTqDx5>0B^2mMPyPwlvqpONAMoNH7^l6|OQK!{jc+O3%p<(NWv|@g?xW3+liS^4^!{
zLAzylYb--|vOw0_itjS)5jK#Q@RY7I19?(eg+H0^!TRu$4UX6|7VZ?>Il(n*jUzLn
z_c5v}v=WBBT|kSeU9eV3=yhcrE3jvUfTIXmDUXV5O&(H@GxO!w(bUxC?pXb-=gx4F
z>v3pfrjCK7!@+cApyl+>sJ1-ZPnI`%xbIs>MW`)Z!w`on^dz!$T#vs^2W5!Mr1W=&
z%CZX8@x9!2=j$X|9vCm>rwYz>ca%szryWL^eYMQ+d}&lz>?0RTcJAW{ruhpZmld@EwnykVzOp6
z?y5g9IuZy#z%e*ns4afhYSo&XmQ3a0_7Ei&7o+ee_V2C**5%&S!__{T28s^l8&p`z
zHzhxW-25Y4#Sy>I?ZRRvi)T@%)e;KgA+k74jb`imW-j+VXqAwQqy)0&n2pQbcJ|rJ
ztB1_#eYRX^tu*XEr~{=Mwo)&+!-AT--s3$pOI1zW7#=f%%LAD@vs392#F7Pv-qVeEz|q>f69(EUYKE8f3m_IIg2H
z8@8zpZt^-)tXki%7#?P!)~5&gOb_~%H7bVutzrk)lFw+A&+h8ccfjdnsqu#Qy*na^
zFOWhl!+fK#lf{kctkc!8m`YeRB{JnqEu)}B>gZJF=k6Hat3_D$WuNP(5}58&h-cmn
z^%2Q9pW>m27uZwjpOg>u(~GbmGulH0ZIz{dmOAF@WOM_3f|YkDCVS&q4~^846mG80
zlq?SXpGwvtD76IVvlTOY6~sC=Vfptzc=|biWAia&y)=@!u20UY3}l5*aQP4*UGt6#
z>25QC{{z!Lp8s3K5c$vJ@A#+p!mK*RowJBp4bXkI24c*~VZuIgLgWVbfipg>$5o&+
z-4m)1AOkl@35nP$PN`XEp@E~A6MrKEn!E3Q
z2gj*QMJ+Oo5PR}*m;)Urrl2vvd5053BamrN{C+UYr0|NB$}|7HoW;2Ugz>xC4nhN5
zJE}Oju8I!Lq>cGa{Z{aU9Fwb~!Lpgop!?^U@+LR_E5&b}JyH4qaWnT04
zE_2L=+$E~Zze&#q?ieZ-zUI~$r=LToXM%783VRpu=u^s~Za?h4#||{(iyneXk7v!N
ze`diX#|F1QYrMWZJE$SRk}wRK2ql>;;;R;_GrGNJV)veqN!3*=u^MLKgVwr84F)^Cs{sQDZ7K2571jq=PWcu!pNBA4pLyG^v&!;i^u47@u!GPqWZSzleM_2PLqkwd
zp^M#Wa`*v%%{`8koKn9Zl}|+^{-V=YyZSggIlGiR475pFWi0SLaobIhE;;bzTM>3T8FN%n5Y{c%zwX;V%
zNd2)%@Q@~N%_x)zi)Hvd@z9{B;qu*46p<2pG7;49f;jaoGt@O
zoScVI_3EZ*b;P2W;2c2pFHNpiZVFkVl|w_@H!-bd0#$a)`!_Z2UkT4T;}K6N@?{S4
zw5(;l+Z0>|;^5FI_IQ!s3@ni=<24XTLU5lx*4HMH$mm@l$+O_Q
zN@BhTDhciKnXYeNCyP}XJ0BRXlqGP+JM|8bz4!U*KwgOtYuFTl5}jyy-1`pICZ%<7
z5Y?iwg4B@Nu|h%<+0YKcBcMIr;lxN9a@o~xw+`#}=llqL6J&u^@Jr#u{UpcPE$>IQ
z>L0f##)ZUY`C`fJ*5N~p=!p-8X$%KmRRLscRu2+ad4uQCY_!7dumY>dcG9q*;>~9`
z^A+h==xRY&?2gcJx*4YPZ^$O@D6HMo5{;ABA6xL)tx86oI>o6aI8(3rjJ-(%(s0LT
z>Zda+UsQbKZ!&joecM?|`c)xveVq8O1}3P#<*=~q2!oG)24kD3CPNuJbZ=Q($k$7Ss5>`pSnNlAC1r4&4
zpeg8?$a2oU+Yt{xD)}Duqpiq#&vWFU0evwC+xi98ukz!AiO#G`<$Rl_bKfr9cz!s)
zwY=3i?Hq2A>5>FT3xm;fOAt=hJEReLKhx0CbE%}#+2Iek=2T`V_O{OI9dDr^5u0#X
zrw8-f&bIidKL6@3qEkOjG353XQ@a<$5po}&$nyOh7<_GQ_Hl-6v%-36VpAAyjHSeu
zvjjyd5mKxp4_2>ZykuUXTK1;Okdcl`e=t|N-WWTS+2G-=omHDOo{?p}v|3KCo(+No
z4mv(U4V3N=v&k-Y^fx6l{)x;RY9lLal0T|)okcpkg;^T>vD`1K2+<$*U*%$beo%X!
z9e(O>q9`HsrLuFE*PPxnyd(r^=<_rxGf};jul52BF>MBneHXuD&gon-2Axp5>os1uxLh`@!miQj+p^
zMReUM$O-0qg$idAxmh4zPV?1p1f-4+7_JNHV6FfyYN9lJ1l6VYmzG+AH}VugN7{Gk$|uX_`2}k!>j8ixZz|
z=%qHV9x?J
z8nn>hQv3Y9MmcfBt2;t|qK~hiqZ`0ZplHLxS3Z#5i=2G)(&Kvp+5BICd`&Y==j*DtupI&mkZ4MrklVVd)^xNnAh>Np|J_E2)re*3qP!TE)Z99?gA
zQBq_A-fMW%GWGV6W{5K4vN()zLygX=c#M9f1|PKb#(tEJw?#86IQe$#7*zNv>`>Gs
zOqGHV$a!xW*Rq0VN;*p}1=Ltrd=pH}eepCn8SQxPf@IHXJ~u_kOoWmp1$@f2>70T;
ze&3=e0(&W3V%qgJ)-=?52|)_KiZvu)BE
zk0aNGXaQqD*7SghAY{Pv0kT>H#H_aiGyI+y8-eXGE6(uka(e(9EsnyAUJwoiEV@<+
zbv3*RN@bfi$D65y(o^xi_kd`qqQelIdgxQ##h^VD9^LPD!{ZYCA5bXSPBI#u^5@fxQie
zL>7vZQ;E~BgIB>JB51^xFHEX0GFx)5jz@jyu=OrkFL!`54!IwzliWFn;Hiz8ugp-M
z;XOZ{Zrc(8!j1)1b?s<=$zG7Ox;$m|4IGQ{TsOQ=Z$A@@P!4jlo8JdS^2v{MOm0Ds
zDAnmh-whGdK2*EOyQ8eeBKPMnfn1XfKVIZ#u^am+Sj5^TqGV45c7@e~-z6qVnG={;
zVqUzJtVqN~O%I>>-pkta;wyEs>DYg!qCb7g=Jm{k1&QB}p9S+eqbo
zA8QQz1AHO|2_qE(Hl5I`6-X_q4f0#bG4XrzOl{G}KsNj;qXE?Cu$vhKEA>syMHT$B
z0Z))^(pmm@Sc6vU#KcFcVs((02i_3+n#EfPH=xddHj-`<8$+1exDz%&G%w}b$UxMbF_ns-{HB-w(cE1XDX5EcN6H|{S^Mm
z$-SZPNs*_wzoiUet~knSQ502Cs1ApBLn=>!!PaofmMyY-^z9vCnObEahE`#+t)49W
zjB_#-v4^22b*|+`qIE-+zqx7yfi=SaYv@C9M&1IwJK%dh2}5$&|t=3tTu^l
zp2HvY-l$gFa21Ev&uH0t00z!N4!?(hFtS*bYGWUq#Ct{^YB!hV%|ho
z2J}{Y{e)i1=VqJNhAWVZ%58cJaA)tKtAzT4!q6Qd*B@`0fGFn
zZyFZZq1(k3!vW~kO!b+9$wP7OOWzXvs*N=A?I|(b(m`?5c%;=iE?lG+3`&QG+}y5M
zQsjKBg5hgssr;e%F{DPFg=u@Bjg7n2jieCKbyxYL>=o@@=5uGd=3K(tT^sy}*2Xnv
zf|F)x9f>zp`Vj(KU&{0!(XYh!Xk!z&TSZA0T&}wJG?557OCG6=O-v$uFLJEq;Nuyp
z6)G&gl1{V{>$2M14D)L3ZB~5A8$f#|yeY2=ah;fD07+w8ko!1)9UEaab}f6-Rs>hpzKN=$KYh~3D#wg337lb%lXObku4D05XPNt*k64zdVNrDbqc
za>c45ixL}a$InLvG=i2jP69UxjSLvzrINnR~yzRpWwPMd8cj~(^`ZnDE;?cp1qy@GG6TZOT7X>1<4C=IXd4jt?(X=`1)
z6l9pp%20WcUj3*YV)&qCwPSBxNQit3>@a!H4sbiDVGr!;2CUv*_fWY`dfNPnQgfd!
zGN1W#YOKKg^ogRUcY6glk}VO?8TY`HjCvMZ>aq=
zuiW|w=U{hMtA7Dc3#m{plDtkNEJctL9yebL8nrN0&n?7N3al>HCrOo5&HpWn~Y?$
znS=bd;~UD(hjsE}Q`cQ_Ttokt4}C|*$4u*XzxW78xSreiXB&BMi>P?iGgZafq_zRQ`}9^?W(T
zw{YS(cCzf+xhZ(vZ`i|_RqpKKBEHn{4RWdnnGk9)t=VbD$qSBzT7fK~M7?S-WqU}n
z+GK=Gx9$6l+izCjKbws;xLUiR37bSLm4N?HE8#6mZFe>MAm+^{KbQ3{8SXby*y#;e
zz$07#o{k-+R$~(-mH0^jlU{=ifV0usd*E{QZV(jmO-u(8Si@SPNn6o}<5i-`#qYF=
zyx&A@pCw6FU9OqV+g+6$aXIST;PX)aCg@(aRN+-yAFlJ%r%a;h-RgEXF2Vo7$9o;3
zC2~3x)p;?eESBrG>(J@}pn=`%-_I#U$qEbfJHx*#>maKvcmVa0f=LVrFvh*>q7L|yxRu`d27WkJwU
z=7ievATACdM8v$G?(?*p9c}j~^Oz)lY5uh+fSt0>dH@GIMy<(2Tf;eF6W3Icx=zXO
zPYbDn#i$cYqg=E%c3)kv)gO*%yc$a|xpT};_6Vx&GImI+-1vpjGdaLD-O4|>H{eZ3kBlHwhs+s3p
zX;5WwuW+Ajiox+IC&AuAL7t;C2#qdxQ^w9nio6=SCe9(JSq$@)>;m8{B!=SYp;l0+
z|K~nL!G@Fo@Epyc^K`kH()0D%Dd^UD;u=nFy^6a#7>z>W-gbLyB~6q0G-1qWZTI>;
zXO8gzXHWMeRQ2G4d|Iw?>FhOURh97|H#;J}fshW_pNke)x;8Z60LmBzki-5O`2gwW
z?;63bl>&xQMmbgvpd}YpQ%>Z*o(l6!O)*L|N&VF6CG}2PILkSW2oF`27K`~Nt07o1
zzA(s~!t-jd|8O|oXYXJKQpa)a*1P;|Utio0$qAa_-Zc5EB3}ShU9?WCtN6C+t=+6WLC3ZebOnviEbHH8O)%6n0w6ye7^ci0)WyT;(2>}
zgTvtBu$aWssaMTGbVq=d$uIrz>ZjG3dtA%8InL7?4P`F&)I7g-;b1=ve{xZxi_#Qv
zFlRvIM2>m5J{Ta`7XBcUx~HkR^Q-BD<={Kfrl#*}Z?n{Mkr2lK`zK3nUA
zm~VBf2VMtCkqf6@sZInRrwQd^l|}=M)TUvZ+6-br=z&y@YQ@;89E`HtnHP=Oa-BF~
zLB530m$K?+r`y}&f%Kd&fZBmrLDDi>t5AWw;O;{u%3q6QofNG41K=a^KY4uL)f_}g
z*WfqZo5%^&N&W_y|4c|fQ4bOzE9maz-V3e)bT-zVDV1(piy<@Ym!itp9}=3Mx|@`l
zLMdVu3EvTxue&5e!G~;nc`*M?
z*yafaBDn?&+HdX4xac9O0_u#dZe+e#>$dy8w4xm)ht6U19FefC!8lsmqq$0KJoGcM
z`ZIurN^lrd7kPXz)>4`8D!+eQOKg`MJCG}Sd$N&*7AJEpI@0y0C!<8^I2!)#8pWbW
zlt6{ocg
zVGa-nquwnH0mY~ut@_p|s$JC*Yfj0{jq9{;gi&2u1mKna40s9TUO1S}
zql<0s&F^_!m1Ye`l34-vkZ>S_uN|FULkZZI9h+i!^b=%k=%>~K5dRqaLPfFg(8A9J
z8wO5;7z5O3JFmsd-R*$lfkDL;^Y4|l4A&ve{@yZ`+eJYY1ENPwQ?b%OxW4RRvp>QZ
zaPv(kbA2%HiGK1)G0UcYB7VF%u4d`b$Z2vo9CSNvr384e8r@xQbF{Nh{)dCxL(y!)
z8#WOHM;zZpprv&fnj=!MkF5tUd?uhXs9_k3=`@ksYd0q)B}0Uk4ID^#i4W{%h-
zXjm5GW3`4c@Kt7)9_kHN3GaE`KP+T8e9my@90Q~tqHZh0k$=9gUXWdaV7?%Lu?GoU
zZVzFlQ!(J|0P$%-`vr#ZduIJH4!3hfDyM^lNEj|Ty=L9?qqX`hx_*AKL>$dz^%q(1
za_=xaHiQ&*6KmOMX;|gicBqH9^q45C!^O>Qek|v)Lvy4-fPlvpeOaA_?@yBgd?ovvoUe(eXF^NOKrV6(
zN8kv!i7JC$K%roG^)#6$sZbz;pTue&-}3tMSoLL$;2y{`dS++Ym$qr-tNf%gc(s`p
z{oPekSj{OspRZLLHd!(+-#J(AtbaFlbPhh-d~w|Ro+f)jw@;=Suxt6XxLux~CT@_>
zL_Kn}Eoawg8{}JM`6|9WP4dfX7I(kn{IbenRXmRlYH(h1_P8F!apb1k_F;qdx!-Zr
zc8p+eKtKq$bnTmq%-?D9s8q+$kYoTdA%kImcrBJ4z^qN?h$26S8?|G1!^qX)vYO@2
z7D}gaDvN}_zgO7t0-k*W;1DGNX@k9Xp?&Wegh%jEp^ZP7^+fyQ)?Cut2bs1KQ$g5w
z_NF&pbsNVRADhv+QDO^hEj{D2rc?$xW3X~T3EgC`T<%84;oY26=p~%GFASD#DO8I{
z>9I5!FEkj7ui2-T^G=_uW{-(~u%E;Pzh#%qPu4b`=6i)6^V$8&xiUy(0_nI9E6?3~
zZs&1f^LISGeXa$kZI3dZ6)$hY**P7LsDNxL=3;wDRF+sLawsP-BG;vQXTctY#TpU!
zqq9Kh*|JRX$3n*@{n#?$ADI4(B!V0mZzAlH-w`}$BB$2CNS-8gc)Wk=n5Dd`)k`GS
zY`ylHcIIiXkq`e^fmE`ZaarqAYi)*{)M(KB)M}gZG}9dSv0O(2SCytHwrbS@Q}>Bk
zkN^r&;#S3)TCukwn%F^s!}h`U?C9@@u$B#ZvfTV-H3%d9h)N&rClH?;3ts?1h2%z0
zaGqS&=gBO=zzR~~&1G*sq*(XI+x?_Z&N!hGo%yYEAXtZ!mDXl-Mjb7#^`JrS#CStE
zn*dtKHaW*oA&?fMXuc9EyCYrjrBSQegOa#IG*%gO5@LFbjWVD?m{MYWTcoV}m
zI{CnSw|pt~GND}j{>Im+`^0ckjM@-LWUWGa{>lpgHPI(6y~tqd)V7weMHq{k9*y=pUZa><8j+v7gV`@6v4=vo+)|#Q$?xiEjRD_$J56BCp9#?VF}q%U
z4W#jA%J3r%2b4NP>)is9N~u{dblpFIY!u#G-WT4Vd>>7*3xd)IWb2PYH!S%Wkrwm$
z^d_$EnPRa2dPrQ5Rni#a72TuCV(-hvO^Wcf)ZF|J)jX@+b=&;~dQeHLW(=Z4bmaR$0(kLwDlkOrql&Nsn`NW_%5WqPn2qKhsIPrs
ze%#nKxQWgm%$Ap9be>^?A+dP?PC>oFM&CJS&qllLAUlq;A+ewV=}p->{JJ{%RT$$)
zQ7ervC1YG`pzNSd?f$nmi!rXTJr*->4NqkS<_AhQhBy--iAzJjgaRW%iypR;qDE6eGOue(DxKrS`UF+@Hj?u8bJsJf2}fS5rBBZ<
z01{&~hj2e7%NMs$Yc2_B1G}|XEat1l^CaSoR$5#Z?6M*vWE(b3pD@8w%m
zJpRPr1GKbwCeAB=RxzrSC@3hI(kB&Pz9umnU+EnJ@%P#yY+m%EWIUat|Dc90mFd|7
z3{Y|Pk|jS}0P$F4AceAAXjf#^hNs>6++e=Zt^goGWxO8J&o3|d+c57d@g913>GKAQU5u~z4!7bcRwdF)_8}RElBOtP!cUcA6`-p2
z=Gfh%^x4c0!&X3WMsXum{)ZAThT1Uxr+R|ZRK
zxO3?&5?RlCe(t+ABTv+pOrQxy$fy+ADV-r|+ML(uo8Z2p`boPGkguDs9%gd}M7*0ZsLp
z6=<_gjzi$+FA1JC-k~r
zScuY1)_jQQb{+HR_D4PwMK&(h@i$eAK+8ajCSE$Y}
zn=}Kzk3%l8M%ubK9l3xX_-n4xu$JCcY~4BR&-AI|9$*f+$N)XGJ|>dH{7A}JGH+K8
zvd^`W+L#GO3tlIh;a4B0?pSKYK|rBiLsAQ5D98Met7@mvm2G?)ar8^w_ka%Q+a5Z&
z>MMus*7##6hp_j1Iar``!E2DLo5E`%!2)N}Z_k44DDlLp-gW<}Ndb@#+O!W=iu^Z7
zuk}F=FR)lMo-&!k6{j<{e>ZD2LJX}{+n+P2@0N|}wFA^q|E>Qy{XBnF24!0-T
z-r__iMODT_H&Rb$C6zu^+q=7zt*+P6fU6!#<*+jX^zRFcq$X2L-5AM}{y5~$9QK$l
zcV}58T8#=6Qi()o3LHoE`X3HlCf{iRO&309WK-PVZbDYp=6(!{k-;$o=>8wIO9Qkw
zg>>@V7jhs&+{E=aq0S(VWN;rJMyGS?LUqRI>8Anevv^c;3E1~~!|6PWu802p*tz~V
z2HxOM_G%U%3@^M$ptqq`U@KK|hLL<}bw7<|@cruA@lO&~^db=s5d$td_uFRQMQ?)c
zuLV}~-l0X>WLu~2YyddhaW}^CI#^E{$2EdipKEnbV*B_4vAyx!f<6`_{!r*%tw1ST
zqSzhO^#!6}z>QTxJc=7fH?ccx(J60gP5q*OSlQNiUpAT!E{(DU1sCGaV)HGUGw@ur
z8?NX$VSkI&xyt9Nj3sNpJBP8^f<1u7hx8$Kofc<6r;hRLy}t(d@@orBwl8evqbIA#
zZXzd*LOJ{3pe~2~N#W+#3{K0UmdBIkiP-AR>T?9XCHbCr1aF5hMu@#5AdKxJc9^f5
zk&PzSWQ~@~aukj)vu+9(y}bGl7T$3D5h%PeLcWc=6Sxp_&j&zOQ%rS50v
z?7$x5d%2TwEVsK8=J#EwhwoAUjvCowMK*Dj|+EDCK3_K
z;ke~e=B(!2rK(KC=!2*8dm{+>RPgYvqwA+bXJdOcV_E$8y{QfS9es@^vqz%KCth>U
z*>mid9JnH=rn(C+9z$m?hPJlJaAm-IFHE9eGTP`VM|I-jQf?(z`LO(O?XK$tne|`V
zIkkenC*`^ORz-7Fw?CbU<8yq!h;CciUzQ-I(G>e=JSlYX-0!fpo&z>tb(*-6dp~wE
zk|)#n`C=@2S)s*6cc1MjL+)}H@vmy@mfWk>c}~=BJX2soLA>O-tH9&=!Dl3QS$^po
zst>#>nMdL6v`>%YGlj$(j??Ni8Z=Uk|u*JQdtf0LVF-uCknWESkap*;Nb%+fqD^E~+h#zt{Im
zXY@VGP^$ManR_z&q|>z4{I@eDF?pB2y>u8HnRK#Jd4kVudn6kPQt1Iuh{OTiLBh^_
zbk5{gH#eYs@2tF$!%%=nT-)_<88M(N@gvo3TCvu{apC3`lS!vF?NE6Uf
zlCQqj?r~uEk7>nEVU-OO11W)9r>NAhi}MStl$tq(bDgjq6o>G
zh>A4`W!|tR&g5Q{JDKfDjWb>8o!*JdZLFMF+d-HRRr)9kYoboYD4N-tjNen8SZ5TY
z)o^jdaKb06Q6>#5s1UEiFk4V5TU@IjIcyrk*ZPa}uXPeZ9QnaeWPiFa=4h!=vB~lH
zAkvi(=$f)lPbC#aDp;zSBaU;-5y~(Fvr;PDE0d_Rr*??nO>g-gNcrkKNOsb&;Pm6d
z77~>_h;d|dUZkx}v$OH`^ppp5U2jS;ic7vGT(UWce*tQj7
zlZyE(@u4bHM?1sZe~qIx>KA)v}usmp|c=Lwmix@MTd
ztnX$KaV^I&awpvC+}*Fo!n$ZI)EYsHl_l_n;^TWW?81AIH%Vz`K~bA
zJfZGSy;vqK)nC8e{O~8H7U}hf>cu(&V1ru#%$i`e+_WEY-O5fl;$~w=Kx`)_5
z@(`7V02ys^_1N8xjX@4^g?32BaO_SLakP6e(3Jp1?z_;zwZA|LD;Y3}Ss*zx1}G$E
z=QBNOh1{TzR!eG}_IpGwmu=3gC6BkKZ&M8saF~T|PL^o_L@pu2<3YSaCUm0-;5B>C
zs#$d44I8i-l6XM!>k!n5JZVZaak}vqbEFvvsdTiSr+1SMxa^)eDoYjCSbx$y+0h*U
zueb%|pqv8$67_cJhvJei81y($h|prtD2=ly<%mYa1HEGs0KMkROD-Yt0)Ahc&=rVG
zIQYAt1?c4i(jv^x(+vPUiRN{8-Jh>6sIiz=Kpt*}!5J%s!eAJ4zhTh53^Y^R2kIX2
zY)IZFhO`t_5WMQT77m$cpVG|(6eM(M+S~6JmR69w>OXGnjnf)QT`iaOYH_af+ne@&
zM9eJkz=F=?|J%X3W(@Zm69aOuf|!ALdOVD>1gHbQbA5@R`Jxy~*+Brapr%zR8L!sK
z!oZMldU?8sN}*OQ)rgCi&GNSGJXC7XI?EDPj#_WCzt|dZ4Z(BcYzSFW#FLiQu9LPT
zb6031c&LB6bI}%$rt(&zN|%J+9;Hi5Q3fe{KeKQP7|`PzElgToI5vKc{uotWvg9Zx
z!~0EX()btMKn8sp5(JdhY}ORBu4TT~{>+mrv}FP3>iMd;`|DWaA-U6z5+|R#_dh@h
zer!lRq@a14?@YywmWym0_Ir^}r=G7m@Urhe08fye;hH
zC&C!3(v_+Nr#OY0keyK~tn%gPEbWzA-FgBeu{5Qkb{+nZpGj54gXUZ5q`SWvx?_QO
zOJ1t$L9Dur%YQU20}yQre)s+FCeNBKpu>r`P_InJCov4)u~_2KD+!$aZ;?iqT4S|N`_wr
z5Cy;bmPNN81_ku{!iJ@XFG|#_MQGIl*e?zsHujB3vWY|}bVd89Q@MQC(4w)8-`+l<
zK1fWN5ubL)(F@+;>-xouh}~cQnTP);c~$#OLryw(R+dN9w-mEY;&WQY>2O_jQ#v?=
zN}AQ>jyJn+Kl5%HuVhAaN#!M8q^qhX)kH-Y$Pb
zUw{_}xD~ne5b!S1Y3M$=xr@EVes+Ahq5|6{x$xZesB
zaUSiWTxM~X9^c(AsMnJ?kb9qa?U+YgBcCVP`KHT&EdGJ1tm`g=wyF#((%^N9Knk7+
zbF-N6P%>+={UJ!&Gn|mmO0_NXjDHPKZp_+-rMAj2`Q90g0UZCN?7}Q}F4*pbl?-NpB;5>#8r!rZLSI{70FwCIV^D
zEU#^lOQV*e3}jvV(zl{qY0zV_{&+XV&si5eV)G?QCP2;9cE5u+
z-{erte-EI38R&Fs;lEQd)Q7&9)qMJdQWzNt0ROD1@`sOA2~oCVExzk7iDS0C
zF*B8KRf?t6gjTYx#qU@xb{ZVGoE;|ji1{n713MioKm_6OCB2{&pEhZVBgA5oo=Ss$&{0k|hAAV6SQnmBLtz$GlbW;w46C))uae$V3LeDK24{FFG!HY%^a
z&s{X5rUpJk(W|fNj(NFJjFAUi4yS4UU!t3}9mtKMV}GLVKxC)OT*fzOQ^xE*2whcDVI@kD_JeUuCKPV&g-+nn-
zSEA`0JBAbOe&fjJ1Q6crpD)j_C9}WK5!XCvM^O*`2U|sPhXmT^Aqioj+^{U!!
z)<3IP=y?N8oI|CREkL@}Iv
z3P34*$G*BKY4vH59RQtU3}{uD4SHbr=PGk*tXEgyB=_$~-{ReO3RDtE+&js@uABHxeq{
z4T91g(jeU>C7^V7hjfE-3`+DZ{B-;@sHt9RJiw^bM{($t-0o$yU%dL
z`N51Fb&KSuYj`wOeCR-;`OoRHPSaeOJxJTlF;orCE5ifc+_4_Ny!R&yW1yO{m8P9G
zD{_pHF^lQjI*OZ?e9T+8edM)C{&@GBKIH2lexlX2JY0khO<^o`t;tWrKhf`lfYqj(
zebn?jLc7IQg@5-xN7?c!y~~Rg8oxakf1wY7V6H|s&}&}O^J)b?}I|CVz-LU~tAj5itpzOuh8n%CNhhp&gvmU=v@biJX(1QwDDuxB4R$ZZ=CM&*whIM;K!^x<+jJX8YqRT>{Z+A7_+;$Ot3wgJo}FnnBh
z$)kxcP3+vRu4rECNql?2Gf;Q16yIRCCFKW)np#^DL4%LOB>v9$i=6oKU3`i;xw+oX
zWd7R%W51kO%hpG7VEE{=T<|(DK&S7a!Jt!jr1(Q4^PwSysCuk29hMqBoy?)&
zqgqXcIZbqz#oGrG)*%f+_=MdZGCX*b)gj58X!VkdxfeyLuVgf~&VhIhMTpXrPB;fuvq?9y`xaEM+)_f}6nk
zoncvS(N#dnqUScHqR9jvrt?SS8$*>F#bDoC>r{hW%^i31$m*c6+9e7^6okfzqjJwQ
zO2}PwuOCx>wk!P(eVIVikQw|-)vd@Z@qV{y>Lu*iIcC-LuGQ})zaSs(60R+dKkSMB
zq+)f%k=zj}a;To;chkqkE2kD)PPs1UiWf;L(p&KtOY$*1q|QTC@T=|8*y_y(-{cH$
z+sEgmv*hnG6|l7z>bL6WzOO~Jw7MMG(55q20sKkSlRj06igAHt{*k9};lwoV-W<0w
zz}1oRw`qcC%SoJ?NOXDNU;?=*6F@rHR`V8GjsVKxik4B#hgVy2-gtLCtZ8<9PCWKa
zSV5!q+FM)gIkFQZP%0csogxZFIxpM}ck*ewjrD(*W}rq5S%R3l13-{sV-wzK7+hQt
z+L7`It>0_uT{DgM;!Zdl}(xWexs=?NcTi(k4?i`TNr)lXat){A71qvjd^{i!A
z7pZVhaEnxQP*X^5ztlTw5AdiLj)B7oMc{wC8|Nl={K^+K>J@olWOPRZTaw7%`?NEJ
zPx!#*z$&d*-zc=(S)0paUnpgR*GMgEd$HHSn9I(@w-}hB_9f+=G3kR#{umffPgb`H
z46cVIenMWWikMT?*r}NEncqhoJ<3E%o_ox0?4^l4K0YzDrbLnnbZ_2XQ5t_iQM~&}
zN7{|7ijj_}`
zA;E&fQhL`Y#CSOF@N_Q`m0SVq?LP~|U^%E6>iy6R@&4J^p7pbn99F}w%y(58e48AE
za0V$3+e~@I6ESL1@$tj0kFq^p)<mN5y32n~?
zjSqq$qnk4Jmy~&}sEp;da|W>+d@Pe#T>e62ykS!Py{&vjW4J%LhHUN8OPk6@@zh!I
zt5u4l6ZPH%cuA)V|2S(Fm5iC~q>y@HPauCFk28G>_6RqK0mwtQp3w43lyE|xZNdow
z5!~yWX`59X=JYV)3FfXlM#*EAz^4+t3z|lY;=42L@5*quh-Q|Hj`Q1Gn`s5bMt$7A
zXwnVjPPV^lGw)mM?WNXM``ro2r^@v30mrfP#SN7dgK=>)BS)D|Y=dEtfLf<98?{75
zWjtkib}A2kw3bg
z#wYNgWddz{GyG8miioeqy`wV51zTLQc`Adu#Dt?vJ7!OfDKz5?qjWm=`Wqv(fmB3s
zV18AGu;Ti{`C~aQ7|*j7j+@o*b6(lAE#!PZa|2Dkm*SzSTA!H_u4y9=al+YuWOO=ONKj3Cq(1S;aBZ)}^M#ia@zzBmzot>7
zkfhlY_n^&*3q5sQl1H=MOYc=XHI=4so>!h`>aU2s_!>@6$Lvp^D;BPJs`Gtr%&v<-
zqiQXf0Mwu6iS%USiJubJ`-=?%C*%)Y$u%evLlQ`Yptz8Xl}|zl=c^nZ?92ze#m%8N
zyFxl0+g4f86Jh`OZ=sWqd^^Caw7Dir=Gs*&oXBiQKl#FC`>pBLY?_AhJ4#)(h*zHn9tcy6
z2R@6%FlnJRcx^?jyRI^j7Zt6P5*VSR*2?IYA@qEFHch%bJC0oTJ9VKbw^XYxVWLn`
z^7bLtX#OGAQl&0f+2p436RY(C8LdXPf71<}_*b%y2!lVka5BP#1$MQn#awWP(I&B9
zp5s6yfhsSZ7MjT3H-E0q4VG#d&eu>a{QF>@LTcJPx_;XO*Tsi`(F2-5iOn09yEiOy=*jgs8z
z_*EQIGL5_D7ol~3+yYZu4iP{7e+2-`r!a?0&E$1fzuxVQLe~hxoNYib_#q9_Tjyz0o$rc;mjhe3ZAd
zwY2Tg@NjLH)Z%!u@cjBA)?_u2H=e(>IIdpgd+k7l$nA#V8)?UbQ&9;YK?WF`vZIZH5*{i_y>^0vTQMoF~b=ScYS
z=VjXh5S5En0eQHo&?1WWdRzk&uM8^`aO5zMz`m2^y<+^QApy+uXFhd{t^*2^Y&zf{
zsmYh4fsu*xvVCO_R>dl>XC*pJ^P*VTOOd}~?g?)K_za-9;s~@vE~vd6Zj4>CXfiGl0FxV>sGq?)q@4s3riL^VRdD!>nZi`J1%vNoX)XEkbHjev#
z+dc|S4BN(=&+73X3TQkAMr1TF6c*bjw5lCUY#DAo(C;knk|mb?`h2=Hvw2T#{{@(p
z1)o+Yll*-LIll=01Z2v~O)uli?l(_V-W~{_wl;`l?vI&Q*VmH3r0abq%&U|Qbk6$SL#T_R9~c!IWu)|um_Hse9Xn!~#^Igc)#ANHKW
zVyZ+TogMoRPX^p%act+u`%6ld-HFpr^WBy1o1%!$&(5LOBFdo0(UaSf*Ti$x_HHUo
z2jI+REAuEMmR)c)h_>>+*>3dTOn?`(|x$hfT
z$FjwcBWt@(IRlUd9ssi6|70GPa!Hi9dd=Fpebiof^wd#bb%jy`0+7?LZKd_!S8_}S
zbuz&t3o^{vmY9Lg{X?dZUoH>}0GqxEW)2$%_CrTLvm@{
zlAUm*)7V7c*;V6L5H;OiBNc^2cNFbvDPJm=mOH%Rv
zM-1RxV}_<+AAbPP^|ZA+BL2NtX#bXgT=a(4gWYb=r>fapvq%BsEw3Kq-jS@+Ig4^G
z2Nw;_f4rQJDI&g|u=bDdzkqUpl7;1XR*=_@9*U^cdPTAWXG_3tb2~ZPHO(v0OuCM1
z1G>fIvZd*lS^c1*E+UWPeu2}jW4~|ZyT4b9AXFBXdi96s^jTI&AVZ@AGXX-2@ct6d}gvVtvfWAYNdcZt5P57;h}g1M)4LUskg
zKVNr+!g_atcDBxH$?hl_K=`PpNGO}1pAbqHbKoElBb&k@4ft+(M+FUvx1k2MzlWOf
zz4E0w2uJ#-PO8+5OIW=KIIT11kEDonMu0Ga8F1%5m+dE^|H)9^LD&eg6`B?A19(06
zU3IJY?rZ|T-`I2#jzL3Fsiu$)>wOb!q&W&m66@C}Pqa5v5aTb>Q*CNm6{Ytf|H>q}
zrDxhVxS>ViH#gxtbSDN(F;6})ir5Vx?LW|i$`WpGwzz?>974hUg_K}p!@yb
zQY2UyN>^7`F;ft-P$35no-8MdKGv4O2Ml?3mn^I|c2|Fd-lXLk^~FRmLMTOIf;z<4
zAd--$_Uvd$k50LOBrP7V6SUU}OZnvqEeGb3_ecR56%DAkZ=~dy^;#MEf`;P-9yREF
zB0qtY^m1qXYowMd`IA58M%6HXEH<^ADB|~CsCTNxN-u`@{kb$X0
z;X^|P5rCwNC=v?>kb54p%VH_3x?pt#obbMye`5WClp1n(HZIR9SDLkZef#kq03yX-
zPUy(urPiYXS5t!bC^)Oh`8@yQtdk4vSm+E0eaYk3(h(@MlvcQ2k{R5EU6&VQ
zQtJB4hAzh}f|kF&Gd~ixlEBMpN0EPJ9{52&ZU3;?4@cNPS!Z7R6&RJSZf)Tm+BTTC
z0Z~~mAl;sGa}8Sm8L5&~!rYBtNdxJUJu&KDi0k>T7&~n@Ib;x4TMQ0)QdR)Yco4B?
zf!Fs`MZ&2}?Cys+};Kk*PY8VLPg
zCc(gjYH4ZVuBUfN3+T*mYZKJ5l>cTK()-&;9m|>nL{|1y2$f`v=yBFxrB#&R2bdTT
z3v-N5E_2pwPzzTxKZ$6H|2e*Sg4B7zV&fXF(wF>eoB8h-3txv66qg5YdHR$
zxUI3Fta;%$c8(!X+aHu|R_n`OscP4hj`%>|p7+QSt=LUEpV0{Fz2wyQw%=RSq1PRT
zJ@rObuIF7Qt@`|r4>Aa2xzxh=1d}WBqe=ACs%%uD(@Axo^`$lRGH7uk@FF3^wc_nZ
z9SRI40H(ey?U}3I_|k1%6tNpecI5_@{2aX{
zL7`q%K}pIu@MFA4^t^A0_4dxxVCE=%Utc5?K(C#I|4+SRn*vO}TpFE=ooUpR?2@%I
zt_%kpB8Bu9!A5sQS&)C2fl+Z-drX(i6zInSdF=l5$xrv@9n44elvGX|;YFHg5g?@z
zqq3A*F8vH^|1*MsYc>B3O0%YNy|8H~&);`I5NcF#`DCpJYdED0HXG6*%otN3J#i?7
zD&W-x!rxN1KiEq>F(evg{SupT_+O7XH5DRNRJ++#C9x6&R1EtkO={*>pNlVeu80`oNk9-
zPzY+(YxjAx*bO5nb38reD@~-XWnA7y=#jH+S;fvf5lLj-JsqN)qr|J97JdXg5)81L+>GdGX
z^Ma3jB0W{|1=cPXzfBMR=`h|Q1i`fTdqax4P>Cx3*R8x%sJysY;>A>fvEENw7rR^>
zVqz-*8MtwR&WGp$TL1iCKP;cA_Ej^6F|<`Cqe>DEMdwULG~-Gai^s9oE#%kt%XdKN
z!jHY^3N?K-6TAO#FFrMZg?}~U8KLn8IJ2%TPY?2W@}L9o(Ka2h;{J|K;GNFpLlNR(
zyty5jE0g9qR=V_>R*@xEnIV>XLNPLDwmfIoX5OGqnCiz!!O%+tfn%I$W)$G
z(rQPEq&bNLi_l+l)w4<}-^t)c?52HNi&rCks)09^ox>1BuPW_DP852trv1?dPv0p`gRw1ln@ccb|+
zB33U&@w{HbI0LUzlnRt4{NErq{kUV(NBnywg6<(quygLmMzOqkfku%sn01-c-as{?3SE$&1L
z84UTce7WovkM;2>912|^jKdUHH6BW$F>rs1{0Mk)LdjT)I2||nJ-df3_oqa;#AXok
zuMgwh=Cstr7{BB$HvF#U@M@ZklHYo}i9n@F;q5Fd#|kUIn5}?D?WT7N3?)ho35@v5
z&2+Tm_11cuRdxAu8<6?ehnnGsb+%x)NZQ_L51kOGBmJE%Ng(maF?bw5CUf73Tno6V
zK`wTUi7_7@9;z-7gWn(4(0&kuuYWbyfB&wO={&l6c>2JlMD=l;LRKo}#ralZlls5O=0Rv~YfF3)RrA9#LtUVN#cy0#dKM=z&QY4U3E
zQ{52|4`c!^%xF5F8;!|e5|-U}hatjOP>yd&fJi>>wVC29aII80Sm1HmQiF-AydO6`
zGMgVwP-^sl?IdhhH0%C3%dAU7g+3z`w~UtoOOKZlgC%(WT>R|jCy)7bNzz>zLMqk#
z_k~EW-6Vsaqj0^}hupwRx}eJLysPiJ82ZSGx@9nX?FcZ?Fu?d!1e;1Gb89$bz~?Q{
zVys_MN+-Pg{EU_c$l~p9FFrLKfk8EzSgz@C`ZG$YXD>i8L~zHfx&Dh|DXV{W#^ny?
zq)@d~vl`g5hJ3m@UZDkMs;A825d?8y2JW)o)rZYqyWjdl!0SN<5SQ~op(vfb*IH_4
z5$4^3@nJL@_I!I(78DG1v@`R+l|H+lgr3wRO&wCDW>H3kt7tA@+iL>$I|`M
z!!rd`F(?%3qG=QMUoE#`ht9lT(L55z*H^+t=`Po8b;?AjEql3nF7=`dmsOyN`91Tr
zvWJ2qGj>+irfuuCs_NEh75m(H-G$^u8=Vml7H%(z#hPa9IyChdZbZBpwQs&tm0*%%
z7-6<~Hv;j;1dz>kask>)v>*hyH)NNS1qyz`A>PZ(dM)_rvv5zMajbM%Z6Bm
zx`m_3h+gT^o7QAI+hq5#T22!yseYxXes7|6?Qz*|p`5S&wxM}zMCHXubIbz2>n^Se
zO?&_v(RU&2po#cfw6uE3jnAg}1JgvK9es=&E%)LJh3>RR!%i;Vk6h0nskG4oGI&g3
zeDR>Hv5R$^8_TItv$5Gfs_PBFD*Fs1G7sddIMTQm9eD4K6CIG%&!+;@0H4*KNUKAZ
zBh#nIBT*;E5U_k*q{SFnj=!v9AmjWE9hp2uli>Mt=<(IBv^}efHOo}2CA}(t
zBLXG=UQRe5(Ts#(>y|m#Z4PDuTAm`%d=~=WnM&ra%yy_CPpJe3TAg#@AT&ls{Sgb$
z7XgU@o5m4%u1$cRi0F9xw
zHHzMl-i8<;D7ywIQvt|zX9x2fU-&<92Oz%e9)Yt0%M*gU4Xn}#VQ<95oTx3&=a6sP
z7@#D|AXoin%I>az4?uDwJcGLk<|k5K9mf*4Ehl1vl?s9r$;Owhay
z!)HxLIv}woGJ@+e2gb{B99Fu8^b7bxy-(fyo}apR7-R-;=^h14DoQz$JQ7OI#PP54
zDL+&Q1@AwMHZ5of`U*-3*^-jjv;hpH5R1N^g#FW4T=(}dYEx6wLbYts5Q=tx%QoyS
z;MHIeG6|4_58$i7wR%gE*rb&-KDXLxYq<>mG$3uKyzQ)k3`E9dzzB5>jNQ848K-uU
zxfi+J#=h(V;NQj@SjivPe!=0-+1|cK^DSVxIz30mW`ZU0V*T%}=lnY+@gbEEyi3|N
zp*z5a#L!UM-#!LS7DG{(i2OF4*JWM*rGKIO^{Fijnk}?2^eL3jBs?x%9Hfx<4_Msi
z>R~K`L6az={i&G(80_laQ8Um`1L|Mp(h9&?2V<@(zyM(u4FQiqk3_QB?UMa&!s7z>I{9I?R$83Td$3xKIE{Eb(5nF-1F
zvj;|$hJ}*U&jWDMG34wg<(@iRLA+iETiWRq-GMUXY0QiN76&XzJ7(rexkc50DVg=Nw9;2%Uvv)}4|A444vBp%H#&!@SsqrYhxQhK#+*rSclBp-~fUWlemCm>4K7i5F
zk6#8L#eplvl0@TBeSLnnsgJuO6@KS?d`TfN&UTeLIdMF$l6<_4fz!mkA~IJ7c3yjxX;qEMA0Skos1$(AgI3
zwjId-_(obbh|ks)CM4Jt_Xr5F_ORkn1%WTCc!-B|$Swe6RXygi1YAuJ4N
zgIM;M^|n9vG|=(RmZ@awedlquejCHs`$NF(?G9_!VBoPy~StQbWe1%4l+NP-zas>XPYK|_nRv(IH8
zP}Z1|;f?sh^abxQ)bNoFndAfBJeI8-BgFg5n4y>_||d*lgP>O42WPz-Rk2I7mQ13i%?cm$Z1@GR4vtToDoYX}u8Ly9r|Oq9a0lev1+LjMNPbOTJ$=H3v;uYr6@LxeY!(>U
z(AD`cuXbh@*UB0PDB_bDz%LL+lVVsDllqb3mct1Nb30P32&TMb{}`1Fn}x285PkQ)
zqYF{78RXzTbyfz-v4=c;n~A(~E2m5BH61B^AR>wbj!j1iKfP
zmAHcy-Y1l37oYlh`>T?w8uuTzp=E9~K}G~^*asb00l$SE#iPDIao*1^NZcgUA;}6R
zQ7sdDS%UXJOC7sHws*YH2zG)tDJa%{t$U85*T_-FsW038O7U
ze<7ZM63})-U?m{ChJ!iqjmwiu)AZc(YKMoEp;_h^6HWrM7r4*ZJszcj`p)cTu|Kx&
zVWJpuhpA-*HL8-DYqtEhXkze5{R)+CDB@^z?U6{+gBX19e1IL@cmO@o*~KsM6ZbM2
zs%ff$t;RgI4hkw>-fz!>y$~0ToqO~bRuBw2HV`KKTrxn+U!%5t9sW;EE
zs!ezigCbzmWbe$>`L4Oan@s!Nvjis@{)1oNgZMqKwvQV3@`R!ef;y_mT?g@>QC3c~
z61tB56gWp9qGS=!DEr?ZRG8ia{KD|=
zTC810#)L;k9eHb^okjouZQ5ZWDKvdUt-;3YJ3-5~f~1%jDUGNwbuqONw)!u#2cIW^1W%)osmgvlY
zolP*yLXh@3xBUmHSSr~}%@^&e3`y3rpYap_({KEv;lG1~EK$hAwY?15tVc{EYm;QN
zcQsIF{1+zh1PvhEA+W3<-AzXg|Ko@MS3=l-hQMY@)5TW}{nz!9d?58^a0fMW4yZwT
zvA^4Gf$_0@Fc6YpSIfl>cfW)p=-fa>d+7Mb1q-&3kAMNt&FMzyK8-@QDC839CAbBG
z`M|SD%I#u53t-V>K*EL6xWCw(attq-4pMwyOaQR)5e0DltGJc$wnzxa*~x%A>}~8?SHma{{Z@;Y+?b=5un7{2Gm~r
zi+v5r{->HoPEKy3OovBfamury=>bE(Bd^t>bIoaajW)^QgS%i}Y#uJU`H)7+UuNdzN
z!QNSJ_lqfb^Jy+lGEvWaLXwb5o4Wy{1nxhL&7UUOL{4Wt~qx*nEDn>wa{^Y&kKU3@-1Hueb8)a)S
zS)^RI8L7as{uhLxH=-JZcfahve-_3AhTro}0*vzWKsTh+;(lGizXDR{7&ssg!NNg<
zUV;`~`o)cQ?Z@Pr$@)~Zrv>8|+D7;990bD;qgtjdX|>oys-E!?vmP#AD*m%(o#iL#
zHw_7Dpx65VCW2>s)66Nmwgx?*L?5u3u^aw#>;Id~eDHn1X?P02MhW=(**bC|t2co0
zqzpO{Zzgc0nV+C$FbCnE4aOm}%8|&Ad{9}BEr^yr|BBK!T$m__Lm+9Xlmd5x765Fu
z%3GD(g3GOs53#w_UJrMWF*KbEv}6&;wjd&GZEacAWDZ+h&*?rbHfbv!zjS;d)!bAql9)=f;wARFgh6!&}NFIXVDMs>R-`yM5vb;
zibsx4ebPc?ibzjS=RbqQRPsFlm|bBx+9L`M-7~RBqO{}Z$^jtDHdk`!W~Y+eRr$d1qd?NI$du25TBuhah{f-t2>ty~V!S5RZ7
zbg8r*8^Y_c8KK@-Oj2wS4ZSw*e@3U6i?AZz{a4<+f`vqR4Njbq0~lm9G<$1xy8}LC
zD@i7=W6R@}Wd--twyl#BhX&kpw3Dwm#ubx?Y>N*kYw$qP7yyhyL;wvc6U=`5oms`!
z!v-u`gHWEwxSnli{n{8gI9h5QAI%Xr1Eat(;X3zgTc{lrsSba78l9hlwoEJm~WStf?JF|tvyW{tTy9>5PAg&@kj!R$V67K+gU!Pk(1L<)nT
z${~As=zu52+IVOmCp&BIt(Lm`b1&;RP1Y;E6t9w!>Ya|#>D~hG;a4B`%yNVSke+~G
zfc)};?M}i~Xh*WObu<8RdMUHV;SmWkstmB3Vv39DNk8oXrpQAPpBBAgAOfw6LCkb<
zlh&_zDeWH*P+*
zG-*EPZb=``3bE@W>7j^xE4ABlx5qX#9qVBP=&Gd>AEb^w?w1lS=DHXPnW6#h!fHw?EESYUn(l
z;=EgNv6A*hE#^7?>YH>Iip%)Iz?PLJME<4eJ8m9+yti#oP}V}DoVv&wVa$;~>aTos{;!u$l8^e$_O@4YYcy4nSs!Q6sD(0t;$teX_?Qy=Ls5ij$*~chhZP0Xf#A5lP*u-wstc
z1pwXig7h==o);7dBa@+n$vomktU!CxT(B;Sj?x`Tz$M(<=3gS?ig9(d)Z`L->^4%|
zMhMrw)FxN{B*+(@Vu=#rLy$o6-JCAf5}z_+wDTW&vTZ0^cTMrOhQVDhT^1t&rbNo$
z^o6dqz3y*nCaIy+L}j$+Ui6?bBl22AsrynMgKe1pSet%of*2DVP1%_BrWqv)1?hfq
z1~_yrG`e3`9^RD#b0dp)c+4ox!J47)huN;xfgegy5#CSveJEc7E8r|MhJ#RlM!Uy=|+I+*BpjrURQs#yQ=>KqT{<&{)
z(y1?rz7A*RGMvsw|H^OR2S|47wdoJ-M?_^m>)tqd^E1Xz>
zpHePW{ect`L3&vXLIbj50yRDB>HyYXKPDSE7qPaFCXL>lOCSW>p(A}_T|q)piym$_
zff2+~CPS)M(rK{$vB?fq31xCxqv3zX%Bi-65>s^S2fGR^il5#-yw7)$rIUof&jthT
zj?{4jgf&_xGT~ZF&pmqk`9w}0r2o5L7S;xnoy}B`<9Aws$VV!P3?~+m-UWHh1L!;}
zx_$uzqnBUcm>NP%Qya@jw7?NoubHscTQB#f<(X7NVJ%h?y0Pjpz4z?EM5_RxmbNFl
zkGwB=)#%j9WR8W~Jg^0X+#f%t&sRbNENvfx)hUY^X88s(wS8XZBOkI9H$^g@t9#T#Gh5u+p-wlZJ|1y^2mFX={nej)H4lwrmKB@NVR
zB>KGier}05rnxmEB-&OyKq_WscBO!r>PTG8aJY1m4MN;+U$NkmdSo76!kKf
z*D03?sS0bg(($`Ns{}BoWxb0PFW`c=L;XOWlB2PL@_Za0B_!mR;eie
zOHa^8pc)lhOqWIQiybaSJ{48u*4m86ppeRUWPv(`U~6WN4kRB&s~XQ|oiQuLjF=dH
z=BfUiX!ekRGFU-qipp>l*#zI+5TlIJzy64z0fO?-Y2i}yG;9g>P@p;Zm1IB}C_JKE
z_r%1bguxjo=Y|+L>j>?_9l;@Eixwl$mtQV4d4xG8sTUB#U-`YH2*yVy$C$x{*y-rP
zh%pFz@kXl5D%foiw@$Z&Zw53*blUabWV#{il&G|PZinM>kVa8wxbHxYpuCP3Q9b1X%&g;Uw66-Gs;tQZIn+jo`NfbvemF2+3
zN(R%8W1B>D``@CwB6NY{og49dtyeTufllQ4SNKpoDuJ-^o;3|$$pJtm!D?sxu(
zN3SaLpbNHS+%rKW6Von#`VL!m(m!4RNjk^No{dSyte&IL*eFqFELD*uIua%N=20Po
zZeQio9JeW7n@U1?M%WH0_tAvC>U8L4AXV(*(e!@F!b%xIhTc6zp7+xFKNlhV;jpd<
zuTEH)v~puNwW&tw+xky$zw@oUu~!{(Qd!o1Tl
z+Spn_Dr6Pa2bv1j?<5H7pu2o3)Z>6Y$M+3NG98$l$^j4|QI)a22c&ql##ul@N<3?k
zs7Pm-u40m@KXZ3b=kOA5i=%`|TQ)x0+%)N(**qfDnqAHhPmvjtE8eA7<#tq1k@m9}
zH+JiKGf9Q;PQ6fGDS^OYkSJpW)z3<49rC{O##jayT9a9ue&HzpW-?SOD{rYtH^z-*
z@h3raHwt@i03U2Z>HY0xN>ka02zs9rgF)ar(L?-)w)A>Cy)bV|7~sAl4^A^rWU318
z>)_S|boy-I5rXkPyb~0}UDYY)Jd8(TyM8dK@wQ!Zl-Mcr0kSN~D
zAEDvm=L9B;a^Sc2w3{na3^2YQ3Ahp~K^UJ6~(Rd+9V$k6~))6pq
z2)qWObrR@QK5J`h7Xa!r>mt3(49ms4fZK0uP+12H#7;d3oG3nk$EpZge+q;4WddlO
z`kXx36xCfAGQVUhaVan!hw-Qi&SEw*^FwbUq67xrF`|2Y4geZfkKqLiM>H}@0Tlm6
z5Dn>TZp&0Kd@Dse>+8A51krLYV&7cUW@@^fKG8Lvu!OP5?r#hz;DunfujRn-w|F{0*Th9xRq&BhrT)`<;U0xy5l7my$l~5l|L)-b
z^EZ7cF#o3#T+#V=Xa1kRT|FT>8^koWFaPf8DdmK5-uz#Zfr_!R-~&$nexbKvm54lIgx6%dTm^ut>}|M3yLBQTELS0%$!UQ>a2J`y%NidZO-
z7!z7KIgSR-OU$d2dw2jABJSy#rrKXvaZ)~=TQi!-tampe
zwOrPLTh``ruYTJqNCHSK>+9=hc#K!N=YTh1zj^lj)P1*RO6tz}gjVfP`x$HQMneQ>
z-^QL)8t}mSB@ej$D}YvHeK6{%SyK<78F==~ZIG@YR0GfmzbNL*NWvoHo~a1byMcl(
z$n)+>2mpk60Jl*9O?c_ijBp=lwg>FIK$9R7kCJf&UO}n)!zjZ9nF|_N=^s%Bc9Nuc7p0s?gN1m8g*dHMd*uUu=Q7#|@-AN8QF@JB3)6?-MKgpw2A8z*0KXW#2Vnv>9r8OzXkqUqFkLWOkf^s_p#c$jx`+k<&e_AvX+W~rXvo4?
zl>>I2qQG>EO{-@PdN;knj;k(QHuzriqnuFfgrPS*YYCcf5ciOB|;c=53m
z_9|`l1?d!cyW}sx%F+?K>VnLU3mU4dY>!4~Gfp}8JyR$DA_6ypXfQ#e%(`G->J+FdVxj9mRn;ei
z;;RB6a=AAaQ)hKP3r|dhyQ`CGw@XXco3la=6>#Il0rlkxZ3RJ)QfJ7S0Xl{;z@I>b
zrtAfv)jDx7{4TIp-!AX_3FgOUz3mlN)Q{?*-pR>$05_5$
zW_F;6LQ|lM)_@7|Yv&Ptsd(=AI@emyS5CtPj)TA0@Vn3fx#4~O{XGPm49XoirD>v8
zR;y?!v&FoBID>AXAWxzbUTb$<#Rk=tx>zPnwP8F)n!
z@gaCQ=aWbfT-~y_zc09R>s;XT>-;bPeh&!*Z6G$k(0+Uzv!1S;y8bE@^8VA*k??e!
znx)gq5-hAIWS;obG8c#)<7*U1$qA(Xwd*Rt=5IV&5z4Ft+ipC!3J?wRi@fg+D=h7p
zZ-;of&N`boR2aB#|<0AK2B-1`(Br!sUE77?w2ccfCO
zoE4FVp1*vAhJkIW(fMF9n-Z{NyPt~#lg8JE`O5Sp(9V#C-k;2Bm9=`DUv@#Ut%Io}
zE00Aa-gA=|fS72e&DZPRBbW`Ael-BOC9AznCukIQ<}a;mF9@hVw=*Ni2O!~~Y(Y18%R4rfueDfDjW1xcTBE@D9MbSI`DVl18bXMTJSjH-Js9la4=yYFqX=Y+g6SF93o^@O{1`#)@Jp1vR3UftlLN-!Ek7U
zvaOUWVL&?3oRgDlB3vNt$GN=c_i
zKk4W1@2;`an%xu7(zzuL>lSE%{HEIRKod;9P{I0_#wrB)T~X*&Zz)-?-q2t&O-cPF
zZ$@k8Co=45)sssJ-dGSs)JJ?6lomz+0iOUw6E3qTHajLdDi-NzioCAD-&`y_5`rMF>Fo$QEbQ&cXRuKm@XFCE4V(Jq@VEVf6T{WT>QH8glspSa9STh+@TclMGG!AW8Bv1k+bz{
zyE!@rj`^;tMOh)8%2HAPHelU~tkFH)9nV=CtF-~z
z{$y8-E4j*b(!8JCzQZ4h3J+90VhvR$@zXW>V8c>SkJA0j?i7L;`%`#W%Qx6(eYpXJ
z%p$zNZL@)MIyKo&5z|J8Ew^9LpzHU}MoOan`WYs-?IsPM$EI1msr3VxY(-%sD|9)~
zD`a|)6jJ128DW-Ry>vPn?f&(>>*&niSp2C{Ib&+CR%|5o15JDLEE3n*AXFvQn~wL*=4%-lpLOBJFdDw?XP`3c2wCbtwgEZ9
zEh0TM1a~<%ou^VC;~~f*lgc0v_2tDhw%LV0l3_H|Dt{->K>092X-mC4NSAVNG~==&
zL|%MAKXJqmUhbTiR~{`yb=aK>ZWycj$lY?~p#_IscNeMWTbZqkUO|8}7D%K8es
zZ!!<>$Sd-o=VB;m5ON^qC`o^Zmn>5Tw4)L2Z^8(CG?Hu=A@q|UJiGm>KZ1Y^Y4Y5F
zu}x<2=6r7h88a7|?Gym-6%EoT`mcgD>MYY%AmQ;yFakQ>d!un+p~P5w)>$sz?9G~`
z;8cqQ(5sd8`Uo7xlRZAm8Visf!1MG!*7cj8lJZQHid*tTsajjg7!tvlb}Ip=flx&Oed*?Z6InYGsQJ}*hf&~GamZVdkxtT@AA
z%!4x^qDkP52@&w3TrO92b2#{$Blde%KiyY|$wkfPdAN9S7%T%hKhIf}z5y^K)RIP-
zANfaO5Od5nN7oCYe@vYS+CBEm?@Wh#5uItm!JhN$3?coa477xOddSBYFJ|MbH#@0j
z$|8PZYC;AX0y#RRIwi7`&Eghy$)t5e_7nDM0&brhvG#O*J@|RXW?5ScI_T2BM$-|=
z2V4$8wUC2JJ*JkU5ahge_*97{h4Nh@@bh{ZrG4JEvxEFjJ1af_;vLvHIhDncSPsYv
zZ--ci`4*D2FsiMbAx+qW;7K>SqN=!yPs|?s=0VHMYIvNir&3KcOH23<&0Xd*|X31n>N
z-4DC_FgQE&xnFa+Xv3+sl!2u(*(Pe0!nOht1yYuSi+sXq=U*=QnE{Nw67Un{bOC*=
zv=OyRE;d`i7D+6Zob^6O9vMGf!H1&I6I19k$tk!KB4C>j!5cb7F!YKb7%dqpCE=GV
zlV>2fZmrR|DD=Swd+W@rivJYr|GMU4JU{WgJCny@GpF?acv08*qtA+n&I~z)hOuqY
zk7Fjm7|v{U@&kw5XPIy}t@7&p9co#qRl|VUDDBp6Jb@^(toZO?-pxL@%|@5vGM#Hr
z9eTn+U!O{YLs*J0O}xuXcVpm*F00#3KH&n`oldjb-T?HkmT0v8Q>Tr;Q|>j=hY-F=
z4iqC45%Bts-Vl~z3ilwED;E%HcgQjCl2@T~g}Q#{l+q3|e$PpJcd@sRf2?ell3O?F
zzdwrD+q#kMzwf_g&Hm*zHOt?AMn$Q90vU2SjC+xK5g+<$IB@Y+)F6LghJv^~W$oqh
z89YtUAu{5UT|!TS8#x(~H-+Bh@p&9*bN8~UX?xi4mYA|f5iT8j!9~{6(9(q$nMg@)
zJafrQ|GpMtZ)5tkZLTQs;*WmV^J8~&jP1HqtC!$t%*gI1qACMIDdQJsN5=wyG8RYV
zQ`#xkler2Q(1SA?PHK|UM55B4q{-xryctL@Pzun4=T&>M8H)l@#)
zHM-r!$_pxrg<iY3c=w&9}17}T7(q9@@_V8OttLtnJ
znN^3TXmc^>og9
zxf+Q9f8(y8=A-XkFU&X32mYa2L)K(7yS7};ez{RNyoob*8e#ep-(>lZfXV-FW>hw}
zs8n@S%8VD+D;@0K0`hH}!MH>O0eg2*{>{p$ISJWu|0`it?N`8Sv$lK$w!!77R4#Qn
z2#Gfhfgx%Mm13p_xz1)m1Dk4l5B&2lCW%#9Q!re)?dcOK(h9nO+fKG7v7RS3pdAZN=1sa9eim1i*)IH`Bad+H
zrLdT5AYVjN;U5%zZaLF%jQSg`EoxvVULn)*BzfDHZkL`jeT%8~z9a@Zo_jB;egK2y
zb&rFa{1S`H9>d3{R0(6k#k3F44%2$O%U^uZ;@(nrbgv`%#t6Gex{A(6wPe?$OpNyJHcAZKw&^a7)-{J*YRxULWitUu-?l
ze1kvij7F=BoTC>b|!lwBzlUe
zCM1)Sxnfa~gK%Yo8wN_Xqk49I!U#8#g-SDt9yHKin?&(}v3ZbJ;~ggQ~7N3bXj?EuoapGFXef?B=ztuU;)3-wu2ZrCXu
zG}Dyge5xVbEaz*4WSZ`0SYLaxLhW`zJX|9#Jpswrlu2%kL##KDfgM}_k*L#6GELxEfSbxC<>KqGZT#}I+rX8=A*&&gLEQ}ujV`e`DcTm{h7Hvi
z9f;*$YX+qcS?d>!He)1-N+khFty%J$HsOb|fWvsYL;~H}m*W6d8h+H8u^k6zI2h2X
zfNqiHVaX3{X;0ampOMy}e=&C06-dlreUT^^Y*&3jzYw*uc5on62`0PucB8v+zp{
zqyl^?%vP6xKi>tBJ@9ZMbt04zv~P8iK_!*Q6`dtFK~(9wB*CR$
zCq&b^TvzDHJkwOcu}k;;>tx-MfO&5W&RZTYy{mZj4gI6$_XKpCY>kTPCw
zI9C8Xh9m6{mpfM*$~U`x{!Zmcr^r7h)S&xd_HvMlrwI+mYiLqvQ`a2XV53A@PaS~P
zhAzpw+$j13X8J*Ih~R`eP?ASU(v@k=i$;p4e=iDe?juGGm$nMlLZnbiIY^dfg?A%L
zlBuO577_*uC}jfD);7solJVWxGEkX98HAHrl!tvE(l54s*S_jh{Y$aSMq^?n#7D$s
zOH#pw+$9l23xi{J^e-fofn$&^EE;xwsuPTRmKZ(-E#t=FwvzUuqZq+MfSvOp2O&Z{
z1BWiQRCI!;*U~g83|-Hb@pQz_ay<1eR!9f0q_Q+v$RL!^v)ol~`0#EV-(+sqw&^l;IZBiXh|irY^DC>7YM8fy=HRU*5~#MLTykhh6H;W?=OuB
z{d#zKX+1UBSPslrlK7Q!Zz35!<(B>oJm1_;G&(H6p~&^*0$9*RRkB>qJv>T(vnbRa
zT-BU(G82z)D~ruao4=N8YlK>-=$2D|YTOu1Q#tGNEhV;H{C)T@pPyf{~+YO
zH8fQXFy`+rzgTy=KZ*=WRTpoDx?*PxUg9jh&+2zpC+_+&;Fg+J_?V$O|9RehJz~g3
zTrsyFWZUp1D(|G5A^bDI0(N8~{)eNKM`;xhhs@fbq(*F0Cb;&#i_@I2K!CT^%=twUunAYtW#D`aDbfhixN`HrIu47Vyq113zpX
z_nO1Q%X?}lZ?Q|LuO6z&=I=T41o7g4buv`1@cf`SnNlhyc#40bEs9
zcQlXho5er4_VZ1jr_Lm*TUpTyl`2FE@nul-`E*ec!=sAnRqgz^>f`ZUi7u%4aa2O;80ky
z(iBYmEB&Js87Kx=`(fL#+kwx?YxP+F@}^ozXmMYK6N8
z=Sq8h!&BQL6NZb+?9S+HwHD4?JSC*~R~A*s86EMGqkpSB?F>oikGq;{ymDjOXo7kc
zMx{0%yvj^TI0VJ|E+U3{vt1
zqQ&o-GIM9U6~|)(rb_$1MN|rC1lk|>G2Evy3uO?gWv8X&PLD5wC#!a}+1yUa!_`hH
z*0TQ^(k`@o`MxdR=UwXh$OVFP|9kI<{ZrbIx{QqKfMCX&%!y=6JI#%Vx9OfkpVcZN
zpTEYY|GrFtDug>rw+oKK#F_|=2|?Q7SMhy*NHd~Cz{3fo^^dWmG`gVv(4+qx34;-&
z^*vr$+Iv|eTdFDC(N8j51ubtbCwEQ6g~5nb=+#1++=yEUA)_jPw7^2syZpVmxn%hU
zD`~^iD;9NFni84}+qsC`q=ZS&5fwrbnVx%W$vab+OGIm}S+Bz>HM8Chj!vf|%!Dpe
z_?>eNqvx;g5fsf-wV;8D{h61*U-7~pHQQzQblN>asX><4?ZL!F_wb>pN_t2)UQ$N$
znq2nbZqx4SANfKNR*4;1zS@G0=UiSlaciIJF%f?m$oDeZR}S*ctEE5JV$*fwJV`sk
zacQdc9jqp{rkqya>j&(z7`KBXju>Rz>)qXL5zy1&C6^dvR5xnm$WdRMyJXRZLa1bG+drH5f`yBwNa>V
z3@;3o`Zt=FhxJWf;@wq)UpY|Lh6rB-;8x!r%(4
zc!5c%s^$t!<#HW9sb37~%w#7B=b~nf+!4&@IOxmIr)Px{N4_XHA;h!P?{*xSxiXmZIUKIi+v*XlT6yK7P9u`c?>PFrb-Q~AqL{h|KV~H
z3n#ZoPmRAj}nF>H&DVyqNuqOUcYF(j91%Ow=}ooO~9s=8^CH7Ppi
z$Df=RXjCvm91kaVguI8gL9MkzE<`v-*fr^g;
zQ$wyG)mpw@{*)Fpab^B9BGP)L-{pP8;3L}mP;vR#?N#LtY-0-O@j`~vC6S3K3mG7Q
z>N)!Qi_^hN>ZnOuM;l)?1@g)Lwk~ktX%uo9X=UmPxbptq
zI1r=8WZ*ub64si*ZmSz<<(_>$+i|bW$1qs=qh?{_MFLFC;W^~6@#xB?e7!rS5p?mh
zmE?C`K`!Fe;T(p_%;$8vl$
zoqkmxooBNccwjk8>U1G^8=OKjf{r+YmDRLv*dUfYr)>{3YUz9Z`shC9Yi||LYI`6VBw`d^F#g6put3wV
zSOZIk;0s~eTo51Gp>RR0lfxTdAcD-KMPUZqwDWk(I(up%)iz-(3{qR6nAWwB2;l8?
zc0^y{!*M+xKL!rs>oLR`lKi-O4Iug{M?AHp%MfL{%Y-D2u^?
zvGXCrH6pnR59$zA!-2C8DG>{1y~O;v4j<~zEQVBn)AO$hzY^f|TgK*~|zS2MK
zqG7)VUPQz|0foeBx$sLo$Q*^KBOHNoyGYe3PchwBFw25>{!o6_D5Q{Jy8>@`sA+mr
z6v=SgDK3?C9bN;>lcJ1NRAk{$17Lh74(=v;S?pXNeE{Kv(oYOkds
z^?<)BIhu34zGk<^`O&qhLObjwOe?@MYD{Ro(Zt`5^7VvM4D%u_Ot}wo3}TnM9KC7f
z8B?FtNFVYph%X<_zKpLdi`aeG9OsV*&LQQ25DIayV_F1R#ZhXVz6Mmk=v}5T?EbG-
zY=b^iL)B5t99v424YNNz&qnEX7NZ+6Jj0@&;g#)iBacB9d$s|^i;*p#_+XpJ^l|_3
zD$gDwgwz^LI;SvHTTS51{!)HVK#Vs^r8cgV6`a^4CaUa3Y*IhkPj6S2MvVl3f#}z?
z9XfzI*&yC3LU#~qBxRol{&2t(6S{3NcgQU(kLmv9+Rc&`%_)W$UZr*~K0JD$*Dgbh
z*zTEb03=eHc9s0}tMgRaQn*Uw&Z8&NrC3F3*~cCnjjm%IPr_?y)W4gQQJ9MPN=lic)UM^oY$6tVu%j2
zH)3rq)&0n}%M~4JbA5Z~{Tr}ryvx9k>=(rjAM4|{vj3CaFo=QfY1+lVPq%l=jpVO(fb
zW+$<>%aKRk^f#V}p%9`q2Zzw>9_0GCmacN7CKi*8(jZ>N{5AIKg7FU()qlH1n$6(H
z=5<(Kr?Kl&UZiZwBIfSlU18Nx9nq(RMjmrY9w2bBMwcFKw}p{oRVBrc1DMwHtW8TD
z3JgL_Z+!12{G#*Zx30c{>$oQ7mOGG%^!@_h^(}dSzy19rTYM#&@d_|F3cv=~LBO<{
z*|Y88@MdN`>^`ege`He#A%JoomR$F66XRFQjK~ldsQ8RCR2Js+Cp*sfg8dOulMsyv
zE)J~l>&*
zht)W;$ID(>t;9%y6IIs|ZG%#7EE70bc<%zl_-&2O`w)gyVQyJ|Q|iR}zyUN+-yLsj8?XCQc6D
zgkb9AVc2q0I(WmP{NiML#53Ov{%-Gk>g=-G>U0-fZu_{=+#fEmdpwR;*v`B%9MsXe
zFGN6MnwuT94;*Nj(pF`5L>AxaQZLM>R%nWu;6UcBHNPQib2ula$ZSc?;CJP_5_g}j
zRKufb(2i=gd+>959>7WCHD={-`v}9dWaVY1a{rsEXlc|iaun*J^%3P`$Dr@Y1z&EY
z0Ix*@s?p|+QlFS`BZ+laVzIaf8~T*d6v`CH+CG=p$Kbv4E=7<1Gw_OOIfTj*)kT(m(EWnS-Q@d_teuo=nY7D*>A@
zl*jgdIzO35V_r|ltu4xyH?TfEKkzH3KI>EQ$=d6
zyyrz<7o#UOu1=Xa+WwE-RWy+T8^M)%cm7ydGy#54q0G7cuCTaG>8%))s$xs`-0x##
z@}ngrqdBMSJdD`*rx2Z3`M4K4lO3`ladAw`w=u=57PUe6>dkJ!L9auU$n33r2pW~J
zG}Ib4<<*@lSj^wy2Is}ii0g89rM}RqqcIq~HeT9%etn!;6>;Zta|+F3>IBEFB@7wl
zI%fK%MPi`5(5T?vf9D-7NsKxId9rY#g5=sb3nk(VhLersrAPicj0-2tvdGo>&k+``
z97eeI?so9X2@E5=Ni32w9tY22=kr4)4w~6*Rv}ug?f$i1bx$#05(0O*LHq%geo^iZ
z+L^zV!3#^Ado;*pCCK9CU}#vWC7xKw%Cb-wHn;^l@y63mW&5cLJx6lu(5mASuS!UL
z=2w(T-*PW4ddg-54jqs`q{Qq0z?Kf|ycBRZ$s2&
z@}JLFb$&Ko8N>Yu;fo{GQcH!0CH;@VQyXjnl4#5{+dv*^bYMWx*N3cwD97$*WVXcmA#vV^fDEU$e&3mg&eD9Rk8SHHo
z|GM7&rod}+*2)|n+H8XszlsvXC`!J-AZESlw-U0x3e^IyB%+3{rpn=
z?^k7oLNR_i&1TZd?ck)dVeo`trF*e+d_a!mj-^5XQ1_izVYc$8f4IC%M<1Mr&m37?KmlFpXvD|!jF0;G;eAd1
zrf{rGWEICDyJAL{XRdcT@NCZe_u2m4(83Q1(pIQ5qzp}>Qp}4{JcCj{cNIg+M4mIcO-w2%_C}H#OBF?}p+`+I
zm~gqhR}ci(Pjpz%>}EweM9zb0b_te2MbOm+1qN;PoW?~>4NLQMvFm6Y21Xo-H5W?_Ul5UIL6c168kv+$qhwu+RC>CJH