From 7687be9e6de703bfc5c8c48127e91ee4ffa54962 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jarl=20S=C3=A6ther?= <60541573+jarlsondre@users.noreply.github.com> Date: Thu, 7 Nov 2024 16:02:29 +0100 Subject: [PATCH] Gpu monitoring (#237) * add gpu utilization decorator and begin work on plots * add decorator for gpu energy utilization * Added config option to hpo script, styling (#235) * Update README.md * Update README.md * Update createEnvVega.sh * remove unused dist file * run black and isort to fix linting errors * remove redundant variable * remove trailing whitespace * fix issues from PR * fix import in eurac trainer * fix linting errors * update logging directory and pattern * update default pattern for gpu energy plots * fix isort linting * add support for none pattern and general cleanup * fix linting errors with black and isort * add configurable and dynamic wait and warmup times for the profiler * remove old plot * move horovod import * fix linting errors --------- Co-authored-by: Anna Lappe <153988542+annaelisalappe@users.noreply.github.com> Co-authored-by: Matteo Bunino <48362942+matbun@users.noreply.github.com> --- src/itwinai/cli.py | 66 ++++- src/itwinai/torch/distributed.py | 228 +++++++++++------- src/itwinai/torch/monitoring/monitoring.py | 176 ++++++++++++++ src/itwinai/torch/monitoring/plotting.py | 140 +++++++++++ .../torch/profiling/communication_plot.py | 40 ++- src/itwinai/torch/profiling/profiler.py | 78 ++++-- src/itwinai/torch/trainer.py | 74 +++--- use-cases/eurac/config.yaml | 4 +- use-cases/eurac/plots/comm_plot.png | Bin 30280 -> 0 bytes use-cases/eurac/plots/communication_plot.png | Bin 0 -> 31555 bytes use-cases/eurac/plots/gpu_energy_plot.png | Bin 0 -> 28368 bytes use-cases/eurac/trainer.py | 6 +- 12 files changed, 647 insertions(+), 165 deletions(-) create mode 100644 src/itwinai/torch/monitoring/monitoring.py create mode 100644 src/itwinai/torch/monitoring/plotting.py delete mode 100644 use-cases/eurac/plots/comm_plot.png create mode 100644 use-cases/eurac/plots/communication_plot.png create mode 100644 use-cases/eurac/plots/gpu_energy_plot.png diff --git a/src/itwinai/cli.py b/src/itwinai/cli.py index fc42d94f..9bb6b27e 100644 --- a/src/itwinai/cli.py +++ b/src/itwinai/cli.py @@ -19,19 +19,63 @@ app = typer.Typer(pretty_exceptions_enable=False) +@app.command() +def generate_gpu_energy_plot( + log_dir: str = "scalability_metrics/gpu_energy_data", + pattern: str = r"gpu_energy_data.*\.csv$", + output_file: str = "plots/gpu_energy_plot.png", +) -> None: + """Generate a GPU energy plot showing the expenditure for each combination of + strategy and number of GPUs in Watt hours. + + Args: + log_dir: The directory where the csv logs are stored. Defaults to + ``utilization_logs``. + pattern: A regex pattern to recognize the file names in the 'log_dir' folder. + Defaults to ``dataframe_(?:\\w+)_(?:\\d+)\\.csv$``. Set it to 'None' to + make it None. In this case, it will match all files in the given folder. + output_file: The path to where the resulting plot should be saved. Defaults to + ``plots/gpu_energy_plot.png``. + + """ + import matplotlib.pyplot as plt + + from itwinai.torch.monitoring.plotting import gpu_energy_plot, read_energy_df + + log_dir_path = Path(log_dir) + if not log_dir_path.exists(): + raise ValueError( + f"The provided log_dir, '{log_dir_path.resolve()}', does not exist." + ) + + if pattern.lower() == "none": + pattern = None + + gpu_utilization_df = read_energy_df(pattern=pattern, log_dir=log_dir_path) + gpu_energy_plot(gpu_utilization_df=gpu_utilization_df) + + output_path = Path(output_file) + output_path.parent.mkdir(parents=True, exist_ok=True) + + plt.savefig(output_path) + print(f"\nSaved GPU energy plot at '{output_path.resolve()}'.") + + @app.command() def generate_communication_plot( - log_dir: str = "profiling_logs", - pattern: str = r"profile_(\w+)_(\d+)_(\d+)\.csv$", - output_file: str = "plots/comm_plot.png", + log_dir: str = "scalability_metrics/communication_data", + pattern: str = r"(.+)_(\d+)_(\d+)\.csv$", + output_file: str = "plots/communication_plot.png", ) -> None: """Generate stacked plot showing computation vs. communication fraction. Stores it + to output_file. Args: - log_dir: The directory where the csv logs are stored. Defauls to + log_dir: The directory where the csv logs are stored. Defaults to ``profiling_logs``. pattern: A regex pattern to recognize the file names in the 'log_dir' folder. - Defaults to ``profile_(\\w+)_(\\d+)_(\\d+)\\.csv$``. + Defaults to ``profile_(\\w+)_(\\d+)_(\\d+)\\.csv$``. Set it to 'None' to + make it None. In this case, it will match all files in the given folder. output_file: The path to where the resulting plot should be saved. Defaults to ``plots/comm_plot.png``. """ @@ -45,13 +89,17 @@ def generate_communication_plot( log_dir_path = Path(log_dir) if not log_dir_path.exists(): - raise IOError( + raise ValueError( f"The directory '{log_dir_path.resolve()}' does not exist, so could not" f"extract profiling logs. Make sure you are running this command in the " - f"same directory as the logging dir." + f"same directory as the logging dir or are passing a sufficient relative" + f"path." ) - df = create_combined_comm_overhead_df(logs_dir=log_dir_path, pattern=pattern) + if pattern.lower() == "none": + pattern = None + + df = create_combined_comm_overhead_df(log_dir=log_dir_path, pattern=pattern) values = get_comp_fraction_full_array(df, print_table=True) strategies = sorted(df["strategy"].unique()) @@ -67,7 +115,7 @@ def generate_communication_plot( output_path.parent.mkdir(parents=True, exist_ok=True) plt.savefig(output_path) - print(f"\nSaved computation vs. communication plot at '{output_path.resolve()}'") + print(f"\nSaved computation vs. communication plot at '{output_path.resolve()}'.") @app.command() diff --git a/src/itwinai/torch/distributed.py b/src/itwinai/torch/distributed.py index 559ba2b0..42a5cf06 100644 --- a/src/itwinai/torch/distributed.py +++ b/src/itwinai/torch/distributed.py @@ -37,6 +37,9 @@ class TorchDistributedStrategy(DistributedStrategy): #: Defaults to False. is_initialized: bool = False + # Provides the name of the strategy for logging purposes etc. + name: str + @property def is_main_worker(self) -> bool: """Checks if local worker has global rank equal to zero. @@ -46,12 +49,14 @@ def is_main_worker(self) -> bool: """ if not self.is_initialized: raise UninitializedStrategyError( - "Strategy has not been initialized. Use the init method.") + "Strategy has not been initialized. Use the init method." + ) return self.global_rank() == 0 @abc.abstractmethod def init(self) -> None: """Initializes the chosen distributed backend""" + # @abc.abstractmethod # def distributed_engine( # self, model: nn.Module, optimizer: Optimizer, @@ -61,8 +66,10 @@ def init(self) -> None: @abc.abstractmethod def distributed( - self, model: nn.Module, optimizer: Optimizer, - lr_scheduler: Optional[LRScheduler] = None + self, + model: nn.Module, + optimizer: Optimizer, + lr_scheduler: Optional[LRScheduler] = None, ) -> Tuple[nn.Module, Optimizer, Optional[LRScheduler]]: """Setup model, optimizer and scheduler for distributed.""" @@ -109,7 +116,8 @@ def device(self) -> str: """ if not self.is_initialized: raise UninitializedStrategyError( - "Strategy has not been initialized. Use the init method.") + "Strategy has not been initialized. Use the init method." + ) return f"cuda:{self.local_rank()}" def set_device(self): @@ -119,18 +127,24 @@ def set_device(self): torch.cuda.set_device(self.local_rank()) def create_dataloader( - self, dataset: Dataset[T_co], batch_size: Optional[int] = 1, + self, + dataset: Dataset[T_co], + batch_size: Optional[int] = 1, shuffle: Optional[bool] = None, sampler: Union[Sampler, Iterable, None] = None, batch_sampler: Union[Sampler[List], Iterable[List], None] = None, - num_workers: int = 0, collate_fn: Optional[_collate_fn_t] = None, - pin_memory: bool = False, drop_last: bool = False, + num_workers: int = 0, + collate_fn: Optional[_collate_fn_t] = None, + pin_memory: bool = False, + drop_last: bool = False, timeout: float = 0, worker_init_fn: Optional[_worker_init_fn_t] = None, - multiprocessing_context=None, generator=None, - *, prefetch_factor: Optional[int] = None, + multiprocessing_context=None, + generator=None, + *, + prefetch_factor: Optional[int] = None, persistent_workers: bool = False, - pin_memory_device: str = "" + pin_memory_device: str = "", ): """Create a distributed DataLoader by using ``DistributedSampler`` as random sampler. @@ -271,22 +285,22 @@ def create_dataloader( https://pytorch.org/docs/stable/data.html#multi-process-data-loading .. _Dataset Types: https://pytorch.org/docs/stable/data.html#dataset-types - """ + """ if not self.is_initialized: raise UninitializedStrategyError( - "Strategy has not been initialized. Use the init method.") + "Strategy has not been initialized. Use the init method." + ) if batch_sampler is not None: - print( - "WARNING: batch_sampler is ignored by TorchDistributedStrategy" - ) + print("WARNING: batch_sampler is ignored by TorchDistributedStrategy") if self.is_distributed: if sampler is None: sampler = DistributedSampler( - dataset, num_replicas=self.global_world_size(), + dataset, + num_replicas=self.global_world_size(), rank=self.global_rank(), - shuffle=shuffle + shuffle=shuffle, ) elif not isinstance(sampler, DistributedSampler): raise RuntimeError( @@ -294,14 +308,20 @@ def create_dataloader( ) # shuffle and batch_sampler must be unset return DataLoader( - dataset=dataset, batch_size=batch_size, sampler=sampler, - num_workers=num_workers, collate_fn=collate_fn, - pin_memory=pin_memory, drop_last=drop_last, timeout=timeout, + dataset=dataset, + batch_size=batch_size, + sampler=sampler, + num_workers=num_workers, + collate_fn=collate_fn, + pin_memory=pin_memory, + drop_last=drop_last, + timeout=timeout, worker_init_fn=worker_init_fn, multiprocessing_context=multiprocessing_context, - generator=generator, prefetch_factor=prefetch_factor, + generator=generator, + prefetch_factor=prefetch_factor, persistent_workers=persistent_workers, - pin_memory_device=pin_memory_device + pin_memory_device=pin_memory_device, ) @abc.abstractmethod @@ -359,11 +379,12 @@ class TorchDDPStrategy(TorchDistributedStrategy): """ #: Torch distributed communication backend. - backend: Literal['nccl', 'gloo', 'mpi'] + backend: Literal["nccl", "gloo", "mpi"] - def __init__(self, backend: Literal['nccl', 'gloo', 'mpi']) -> None: + def __init__(self, backend: Literal["nccl", "gloo", "mpi"]) -> None: super().__init__() self.backend = backend + self.name = "torch-ddp" def init(self) -> None: """Initializes the distributed process group and the distributed @@ -375,8 +396,7 @@ def init(self) -> None: which is already initialized. """ if not distributed_resources_available(): - raise RuntimeError( - "Trying to run distributed on insufficient resources.") + raise RuntimeError("Trying to run distributed on insufficient resources.") if self.is_initialized: raise DistributedStrategyError("Strategy was already initialized") dist.init_process_group(backend=self.backend) @@ -409,15 +429,18 @@ def init(self) -> None: # return model_engine def distributed( - self, model: nn.Module, optimizer: Optimizer, + self, + model: nn.Module, + optimizer: Optimizer, lr_scheduler: Optional[LRScheduler] = None, find_unused_parameters: bool = False, - **kwargs + **kwargs, ) -> Tuple[nn.Module, Optimizer, Optional[LRScheduler]]: """Setup model, optimizer and scheduler for distributed.""" if not self.is_initialized: raise UninitializedStrategyError( - "Strategy has not been initialized. Use the init method.") + "Strategy has not been initialized. Use the init method." + ) if torch.cuda.is_available(): # device = self.dist_lrank() model = model.to(self.device()) @@ -425,7 +448,7 @@ def distributed( model, device_ids=[self.device()], output_device=self.device(), - find_unused_parameters=find_unused_parameters + find_unused_parameters=find_unused_parameters, ) else: dist_model = model @@ -440,7 +463,8 @@ def global_world_size(self) -> int: """ if not self.is_initialized: raise UninitializedStrategyError( - "Strategy has not been initialized. Use the init method.") + "Strategy has not been initialized. Use the init method." + ) return dist.get_world_size() def local_world_size(self) -> int: @@ -452,7 +476,8 @@ def local_world_size(self) -> int: """ if not self.is_initialized: raise UninitializedStrategyError( - "Strategy has not been initialized. Use the init method.") + "Strategy has not been initialized. Use the init method." + ) return torch.cuda.device_count() def global_rank(self) -> int: @@ -464,7 +489,8 @@ def global_rank(self) -> int: """ if not self.is_initialized: raise UninitializedStrategyError( - "Strategy has not been initialized. Use the init method.") + "Strategy has not been initialized. Use the init method." + ) return dist.get_rank() def local_rank(self) -> int: @@ -475,14 +501,16 @@ def local_rank(self) -> int: """ if not self.is_initialized: raise UninitializedStrategyError( - "Strategy has not been initialized. Use the init method.") + "Strategy has not been initialized. Use the init method." + ) return dist.get_rank() % torch.cuda.device_count() def clean_up(self) -> None: """Destroys the current process group.""" if not self.is_initialized: raise UninitializedStrategyError( - "Strategy has not been initialized. Use the init method.") + "Strategy has not been initialized. Use the init method." + ) if torch.cuda.is_available(): dist.barrier() dist.destroy_process_group() @@ -500,7 +528,8 @@ def allgather_obj(self, obj: Any) -> List[Any]: # https://pytorch.org/docs/stable/distributed.html#collective-functions if not self.is_initialized: raise UninitializedStrategyError( - "Strategy has not been initialized. Use the init method.") + "Strategy has not been initialized. Use the init method." + ) res = [None] * self.global_world_size() dist.all_gather_object(res, obj) return res @@ -521,7 +550,8 @@ def gather_obj(self, obj: Any, dst_rank: int = 0) -> Optional[List[Any]]: # https://pytorch.org/docs/stable/distributed.html#collective-functions if not self.is_initialized: raise UninitializedStrategyError( - "Strategy has not been initialized. Use the init method.") + "Strategy has not been initialized. Use the init method." + ) if self.global_rank() == dst_rank: res = [None] * self.global_world_size() dist.gather_object(obj, res, dst=dst_rank) @@ -533,7 +563,8 @@ def gather(self, tensor: torch.Tensor, dst_rank: int = 0) -> Optional[List]: # https://pytorch.org/docs/stable/distributed.html#collective-functions if not self.is_initialized: raise UninitializedStrategyError( - "Strategy has not been initialized. Use the init method.") + "Strategy has not been initialized. Use the init method." + ) # Ensure that the tensor is on the correct device (CUDA) tensor = tensor.to(self.device()) @@ -541,8 +572,10 @@ def gather(self, tensor: torch.Tensor, dst_rank: int = 0) -> Optional[List]: dist.gather(tensor, dst=dst_rank) return - res = [torch.zeros_like(tensor, device=self.device()) - for _ in range(self.global_world_size())] + res = [ + torch.zeros_like(tensor, device=self.device()) + for _ in range(self.global_world_size()) + ] dist.gather(tensor, gather_list=res, dst=dst_rank) @@ -561,11 +594,12 @@ class DeepSpeedStrategy(TorchDistributedStrategy): """ #: Torch distributed communication backend. - backend: Literal['nccl', 'gloo', 'mpi'] + backend: Literal["nccl", "gloo", "mpi"] - def __init__(self, backend: Literal['nccl', 'gloo', 'mpi']) -> None: + def __init__(self, backend: Literal["nccl", "gloo", "mpi"]) -> None: super().__init__() self.backend = backend + self.name = "deepspeed" def init(self) -> None: """Initializes the distributed process group and the distributed @@ -577,18 +611,18 @@ def init(self) -> None: already initialized. """ import deepspeed + self.deepspeed = deepspeed if not distributed_resources_available(): - raise RuntimeError( - "Trying to run distributed on insufficient resources.") + raise RuntimeError("Trying to run distributed on insufficient resources.") if self.is_initialized: raise DistributedStrategyError("Strategy was already initialized") # https://github.com/Lightning-AI/pytorch-lightning/issues/13567 - ompi_lrank = os.environ.get('OMPI_COMM_WORLD_LOCAL_RANK') - os.environ['OMPI_COMM_WORLD_LOCAL_RANK'] = os.environ.get( - 'LOCAL_RANK', ompi_lrank + ompi_lrank = os.environ.get("OMPI_COMM_WORLD_LOCAL_RANK") + os.environ["OMPI_COMM_WORLD_LOCAL_RANK"] = os.environ.get( + "LOCAL_RANK", ompi_lrank ) # https://deepspeed.readthedocs.io/en/latest/initialize.html#training-initialization @@ -598,10 +632,12 @@ def init(self) -> None: self.set_device() def distributed( - self, model: nn.Module, optimizer: Optional[Optimizer] = None, + self, + model: nn.Module, + optimizer: Optional[Optimizer] = None, lr_scheduler: Optional[LRScheduler] = None, model_parameters: Optional[Any] = None, - **init_kwargs + **init_kwargs, ) -> Tuple[nn.Module, Optimizer, Optional[LRScheduler]]: """Setup model, optimizer and scheduler for distributed.""" if not self.is_initialized: @@ -615,7 +651,7 @@ def distributed( optimizer=optimizer, lr_scheduler=lr_scheduler, dist_init_required=True, - **init_kwargs + **init_kwargs, ) return distrib_model, optimizer, lr_scheduler @@ -627,7 +663,8 @@ def global_world_size(self) -> int: """ if not self.is_initialized: raise UninitializedStrategyError( - "Strategy has not been initialized. Use the init method.") + "Strategy has not been initialized. Use the init method." + ) return dist.get_world_size() def local_world_size(self) -> int: @@ -639,7 +676,8 @@ def local_world_size(self) -> int: """ if not self.is_initialized: raise UninitializedStrategyError( - "Strategy has not been initialized. Use the init method.") + "Strategy has not been initialized. Use the init method." + ) return torch.cuda.device_count() def global_rank(self) -> int: @@ -651,7 +689,8 @@ def global_rank(self) -> int: """ if not self.is_initialized: raise UninitializedStrategyError( - "Strategy has not been initialized. Use the init method.") + "Strategy has not been initialized. Use the init method." + ) return dist.get_rank() def local_rank(self) -> int: @@ -662,14 +701,16 @@ def local_rank(self) -> int: """ if not self.is_initialized: raise UninitializedStrategyError( - "Strategy has not been initialized. Use the init method.") + "Strategy has not been initialized. Use the init method." + ) return dist.get_rank() % torch.cuda.device_count() def clean_up(self) -> None: """Destroys the current process group.""" if not self.is_initialized: raise UninitializedStrategyError( - "Strategy has not been initialized. Use the init method.") + "Strategy has not been initialized. Use the init method." + ) # deepspeed.sys.exit() # disabled as it kills the execution def allgather_obj(self, obj: Any) -> List[Any]: @@ -685,7 +726,8 @@ def allgather_obj(self, obj: Any) -> List[Any]: # https://pytorch.org/docs/stable/distributed.html#collective-functions if not self.is_initialized: raise UninitializedStrategyError( - "Strategy has not been initialized. Use the init method.") + "Strategy has not been initialized. Use the init method." + ) res = [None] * self.global_world_size() dist.all_gather_object(res, obj) return res @@ -706,7 +748,8 @@ def gather_obj(self, obj: Any, dst_rank: int = 0) -> Optional[List[Any]]: # https://pytorch.org/docs/stable/distributed.html#collective-functions if not self.is_initialized: raise UninitializedStrategyError( - "Strategy has not been initialized. Use the init method.") + "Strategy has not been initialized. Use the init method." + ) if self.global_rank() == dst_rank: res = [None] * self.global_world_size() dist.gather_object(obj, res, dst=dst_rank) @@ -718,7 +761,8 @@ def gather(self, tensor: torch.Tensor, dst_rank: int = 0) -> Optional[List]: # https://pytorch.org/docs/stable/distributed.html#collective-functions if not self.is_initialized: raise UninitializedStrategyError( - "Strategy has not been initialized. Use the init method.") + "Strategy has not been initialized. Use the init method." + ) # Ensure that the tensor is on the correct device (CUDA) tensor = tensor.to(self.device()) @@ -726,8 +770,10 @@ def gather(self, tensor: torch.Tensor, dst_rank: int = 0) -> Optional[List]: dist.gather(tensor, dst=dst_rank) return - res = [torch.zeros_like(tensor, device=self.device()) - for _ in range(self.global_world_size())] + res = [ + torch.zeros_like(tensor, device=self.device()) + for _ in range(self.global_world_size()) + ] dist.gather(tensor, gather_list=res, dst=dst_rank) @@ -738,6 +784,10 @@ def gather(self, tensor: torch.Tensor, dst_rank: int = 0) -> Optional[List]: class HorovodStrategy(TorchDistributedStrategy): """Horovod distributed strategy class.""" + def __init__(self): + super().__init__() + self.name = "horovod" + def init(self) -> None: """Initializes the Horovod distributed backend. @@ -747,12 +797,12 @@ def init(self) -> None: already initialized. """ if not distributed_resources_available(): - raise RuntimeError( - "Trying to run distributed on insufficient resources.") + raise RuntimeError("Trying to run distributed on insufficient resources.") if self.is_initialized: raise DistributedStrategyError("Strategy was already initialized") import horovod.torch as hvd + self.hvd = hvd self.hvd.init() @@ -761,39 +811,38 @@ def init(self) -> None: self.set_device() def distributed( - self, model: nn.Module, optimizer: Optional[Optimizer] = None, + self, + model: nn.Module, + optimizer: Optional[Optimizer] = None, lr_scheduler: Optional[LRScheduler] = None, - **optim_kwargs + **optim_kwargs, ) -> Tuple[nn.Module, Optimizer, Optional[LRScheduler]]: """Setup model, optimizer and scheduler for distributed.""" if not self.is_initialized: raise UninitializedStrategyError( - "Strategy has not been initialized. Use the init method.") + "Strategy has not been initialized. Use the init method." + ) model.to(self.device()) # Scale learning rate # https://github.com/horovod/horovod/issues/1653#issuecomment-574764452 lr_scaler = 1 - if optim_kwargs.get('op') == self.hvd.Adasum: + if optim_kwargs.get("op") == self.hvd.Adasum: lr_scaler = self.hvd.local_size() - elif optim_kwargs.get('op') == self.hvd.Average: + elif optim_kwargs.get("op") == self.hvd.Average: lr_scaler = self.hvd.size() for g in optimizer.param_groups: - g['lr'] *= lr_scaler + g["lr"] *= lr_scaler self._broadcast_params(model, optimizer) distOptimizer = self.hvd.DistributedOptimizer( - optimizer, - named_parameters=model.named_parameters(), - **optim_kwargs + optimizer, named_parameters=model.named_parameters(), **optim_kwargs ) return model, distOptimizer, lr_scheduler - def _broadcast_params( - self, model: nn.Module, optimizer: optim.Optimizer - ) -> None: + def _broadcast_params(self, model: nn.Module, optimizer: optim.Optimizer) -> None: """Broadcasts variables from root rank to all other processes. Args: @@ -813,7 +862,8 @@ def global_world_size(self) -> int: """ if not self.is_initialized: raise UninitializedStrategyError( - "Strategy has not been initialized. Use the init method.") + "Strategy has not been initialized. Use the init method." + ) return self.hvd.size() def local_world_size(self) -> int: @@ -825,7 +875,8 @@ def local_world_size(self) -> int: """ if not self.is_initialized: raise UninitializedStrategyError( - "Strategy has not been initialized. Use the init method.") + "Strategy has not been initialized. Use the init method." + ) return self.hvd.local_size() def global_rank(self) -> int: @@ -837,7 +888,8 @@ def global_rank(self) -> int: """ if not self.is_initialized: raise UninitializedStrategyError( - "Strategy has not been initialized. Use the init method.") + "Strategy has not been initialized. Use the init method." + ) return self.hvd.rank() def local_rank(self) -> int: @@ -848,14 +900,16 @@ def local_rank(self) -> int: """ if not self.is_initialized: raise UninitializedStrategyError( - "Strategy has not been initialized. Use the init method.") + "Strategy has not been initialized. Use the init method." + ) return self.hvd.local_rank() def clean_up(self) -> None: """Shuts Horovod down.""" if not self.is_initialized: raise UninitializedStrategyError( - "Strategy has not been initialized. Use the init method.") + "Strategy has not been initialized. Use the init method." + ) self.hvd.shutdown() def allgather_obj(self, obj: Any) -> list[Any]: @@ -920,6 +974,10 @@ class NonDistributedStrategy(TorchDistributedStrategy): is_distributed: bool = True is_distributed: bool = False + def __init__(self): + super().__init__() + self.name = "non-distributed" + def init(self) -> None: """If CUDA is available set CUDA device, and do nothing more. @@ -941,20 +999,24 @@ def device(self) -> str: """ if not self.is_initialized: raise UninitializedStrategyError( - "Strategy has not been initialized. Use the init method.") + "Strategy has not been initialized. Use the init method." + ) if torch.cuda.is_available(): return super().device() return "cpu" def distributed( - self, model: nn.Module, optimizer: Optional[Optimizer] = None, + self, + model: nn.Module, + optimizer: Optional[Optimizer] = None, lr_scheduler: Optional[LRScheduler] = None, - **kwargs + **kwargs, ) -> Tuple[nn.Module, Optimizer, Optional[LRScheduler]]: """Do nothing and return model, optimizer and scheduler.""" if not self.is_initialized: raise UninitializedStrategyError( - "Strategy has not been initialized. Use the init method.") + "Strategy has not been initialized. Use the init method." + ) if torch.cuda.is_available(): model = model.cuda() return model, optimizer, lr_scheduler diff --git a/src/itwinai/torch/monitoring/monitoring.py b/src/itwinai/torch/monitoring/monitoring.py new file mode 100644 index 00000000..9857ccb7 --- /dev/null +++ b/src/itwinai/torch/monitoring/monitoring.py @@ -0,0 +1,176 @@ +import functools +import time +from multiprocessing import Manager, Process +from pathlib import Path +from typing import Any, Callable, Dict, List + +import pandas as pd +import pynvml +from pynvml import nvmlDeviceGetHandleByIndex, nvmlInit + +from itwinai.torch.trainer import TorchTrainer + +logging_columns = [ + "sample_idx", + "utilization", + "power", + "local_rank", + "node_idx", + "num_global_gpus", + "strategy", + "probing_interval", +] + + +def probe_gpu_utilization_loop( + node_idx: int, + num_local_gpus: int, + num_global_gpus: int, + strategy_name: str, + log_dict: Any, + stop_flag: Any, + probing_interval: int = 2, + warmup_time: int = 5, +) -> None: + """Logs the GPU utilization across all availble GPUs on a single node. Is meant to + be called by multiprocessing's Process and expects variables to be shared using + a multiprocessing.Manager object. Logs utilization into `log_dict` until + stop_flag.value is set to True. + + Args: + node_idx: The index of the compute node that the function is called by, used + for logging purposes. + num_local_gpus: Number of GPUs on the current compute node. + num_global_gpus: Number of GPUs on all nodes combined. + strategy: Which distributed strategy is being used, e.g. "ddp" or "horovod". + log_dict: Dictionary for storing logging data on. Should be managed by a + multiprocessing.Manager object. + stop_flag: Shared value telling the function when to stop logging. Should be + managed by a multiprocessing.Manager object. + probing_interval: How long to wait between each time a read of the GPU + utilization is done. + warmup_time: How long to wait before logging starts, allowing the training to + properly start before reading. + + """ + + if not set(logging_columns).issubset(set(log_dict.keys())): + missing_columns = set(logging_columns) - set(log_dict.keys()) + raise ValueError( + f"log_dict is missing the following columns: {missing_columns}" + ) + + nvmlInit() + time.sleep(warmup_time) + + sample_idx = 0 + while not stop_flag.value: + for idx in range(num_local_gpus): + handle = nvmlDeviceGetHandleByIndex(idx) + utilization_rates = pynvml.nvmlDeviceGetUtilizationRates(handle) + + gpu_util = utilization_rates.gpu + power = pynvml.nvmlDeviceGetPowerUsage(handle) + power = power / 1000 # mW -> W + + log_dict["sample_idx"].append(sample_idx) + log_dict["utilization"].append(gpu_util) + log_dict["power"].append(power) + log_dict["local_rank"].append(idx) + log_dict["node_idx"].append(node_idx) + log_dict["num_global_gpus"].append(num_global_gpus) + log_dict["strategy"].append(strategy_name) + log_dict["probing_interval"].append(probing_interval) + + sample_idx += 1 + + time.sleep(probing_interval) + + +def measure_gpu_utilization(method: Callable) -> Callable: + """Decorator for measuring GPU utilization and storing it to a .csv file.""" + + def write_logs_to_file(utilization_logs: List[Dict], output_path: Path) -> None: + dataframes = [] + for log in utilization_logs: + if len(log) == 0: + continue + dataframes.append(pd.DataFrame(log)) + + log_df = pd.concat(dataframes) + log_df.to_csv(output_path, index=False) + print(f"Writing GPU energy dataframe to '{output_path}'.") + + @functools.wraps(method) + def measured_method(self: TorchTrainer, *args, **kwargs) -> Any: + gpu_probing_interval = 1 + warmup_time = 5 + + strategy = self.strategy + strategy_name = strategy.name + + local_rank = strategy.local_rank() + global_rank = strategy.global_rank() + num_global_gpus = strategy.global_world_size() + num_local_gpus = strategy.local_world_size() + node_idx = global_rank // num_local_gpus + + output_path = Path( + f"scalability_metrics/gpu_energy_data_{strategy_name}_{num_global_gpus}.csv" + ) + output_path.parent.mkdir(exist_ok=True, parents=True) + + gpu_monitor_process = None + manager = None + stop_flag = None + data = None + + # Starting a child process once per node + if local_rank == 0: + + # Setting up shared variables for the child process + manager = Manager() + data = manager.dict() + for col in logging_columns: + data[col] = manager.list() + stop_flag = manager.Value("i", False) + + gpu_monitor_process = Process( + target=probe_gpu_utilization_loop, + kwargs={ + "node_idx": node_idx, + "num_local_gpus": num_local_gpus, + "num_global_gpus": num_global_gpus, + "strategy_name": strategy_name, + "log_dict": data, + "stop_flag": stop_flag, + "probing_interval": gpu_probing_interval, + "warmup_time": warmup_time, + }, + ) + gpu_monitor_process.start() + + local_utilization_log = {} + try: + result = method(self, *args, **kwargs) + finally: + if local_rank == 0: + stop_flag.value = True + grace_period = 5 # extra time to let process finish gracefully + gpu_monitor_process.join(timeout=gpu_probing_interval + grace_period) + + # Converting the shared log to non-shared log + local_utilization_log = {key: list(data[key]) for key in data.keys()} + manager.shutdown() + + global_utilization_log = strategy.gather_obj(local_utilization_log, dst_rank=0) + if strategy.is_main_worker: + output_dir = Path("scalability_metrics/gpu_energy_data") + output_dir.mkdir(exist_ok=True, parents=True) + output_path = output_dir / f"{strategy_name}_{num_global_gpus}.csv" + + write_logs_to_file(global_utilization_log, output_path) + + return result + + return measured_method diff --git a/src/itwinai/torch/monitoring/plotting.py b/src/itwinai/torch/monitoring/plotting.py new file mode 100644 index 00000000..554f5dc4 --- /dev/null +++ b/src/itwinai/torch/monitoring/plotting.py @@ -0,0 +1,140 @@ +from pathlib import Path +from re import Match, Pattern, compile +from typing import Optional, Tuple, Union + +import matplotlib +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import seaborn as sns +from matplotlib.axes import Axes +from matplotlib.figure import Figure +from scipy.constants import hour as SECONDS_IN_HOUR + +matplotlib.use("Agg") + + +def read_energy_df(pattern: Optional[str], log_dir: Path) -> pd.DataFrame: + """Read files matching the given regex pattern from directory and converting them + into a Pandas DataFrame. If pattern is None, we assume a match on all files. + Expects that the existence of ``log_dir`` is handled before calling this function. + + Args: + pattern: The regex string used to match files. + log_dir: The directory to search for files in. + + Raises: + ValueError: If no matching files are found in the given logging directory. + """ + + pattern_re: Optional[Pattern] = None + if pattern is not None: + pattern_re = compile(pattern) + + # Load and concatenate dataframes + dataframes = [] + for entry in log_dir.iterdir(): + match: Union[bool, Match] = True + if pattern_re is not None: + match = pattern_re.search(str(entry)) + + if not match: + continue + + print(f"Loading data from file: '{entry}' when creating energy DataFrame") + df = pd.read_csv(entry) + dataframes.append(df) + + if len(dataframes) == 0: + if pattern is None: + error_message = f"Unable to find any files in {log_dir.resolve()}!" + else: + error_message = ( + f"No files matched pattern, '{pattern}', in log_dir, " + f"{log_dir.resolve()}!" + ) + raise ValueError(error_message) + + return pd.concat(dataframes) + + +def calculate_aggregated_energy_expenditure( + gpu_utilization_df: pd.DataFrame, +) -> pd.DataFrame: + """Calculates the total energy expenditure in Watt hours for each strategy and + number of GPUs. Expects that the existence of the appropriate DataFrame columns is + handled before calling this function. + + Returns: + pd.DataFrame: A DataFrame containing the total expenditure in Watt hours for + each strategy and number of GPUs, with the columns ``strategy``, + ``num_global_gpus`` and ``total_energy_wh``. + """ + energy_data = [] + + grouped_df = gpu_utilization_df.groupby(["strategy", "num_global_gpus"]) + for (strategy, num_gpus), group in grouped_df: + + if len(group["probing_interval"].unique()) != 1: + raise ValueError( + f"probing_interval must have the same value for each strategy and " + f"number of GPUs, but was heterogeneous for strategy: {strategy} " + f"and number of GPUs: {num_gpus}." + ) + + probing_interval = group["probing_interval"].iloc[0] + total_energy_wh = group["power"].sum() * probing_interval / SECONDS_IN_HOUR + energy_data.append( + { + "strategy": strategy, + "num_global_gpus": num_gpus, + "total_energy_wh": total_energy_wh, + } + ) + return pd.DataFrame(energy_data) + + +def gpu_energy_plot(gpu_utilization_df: pd.DataFrame) -> Tuple[Figure, Axes]: + """Makes an energy bar plot of the GPU utilization dataframe, showing the total + energy expenditure for each strategy and number of GPUs in Watt hours. + """ + required_columns = {"strategy", "power", "num_global_gpus", "probing_interval"} + if not required_columns.issubset(set(gpu_utilization_df.columns)): + missing_columns = set(required_columns) - set(set(gpu_utilization_df.columns)) + raise ValueError( + f"DataFrame is missing the following columns: {missing_columns}" + ) + sns.set_theme() + + energy_df = calculate_aggregated_energy_expenditure(gpu_utilization_df) + + strategies = energy_df["strategy"].unique() + unique_gpu_counts = np.array(energy_df["num_global_gpus"].unique()) + + fig, ax = plt.subplots() + x = np.arange(len(unique_gpu_counts)) + + bar_width = 1 / (len(strategies) + 1) + static_offset = (len(strategies) - 1) / 2 + for strategy_idx, strategy in enumerate(strategies): + dynamic_bar_offset = strategy_idx - static_offset + strategy_data = energy_df[energy_df["strategy"] == strategy] + + # Ensuring the correct spacing of the bars + strategy_num_gpus = len(strategy_data["num_global_gpus"]) + + ax.bar( + x=x[:strategy_num_gpus] + dynamic_bar_offset * bar_width, + height=strategy_data["total_energy_wh"], + width=bar_width, + label=strategy, + ) + + ax.set_xlabel("Num GPUs") + ax.set_ylabel("Energy Consumption (Wh)") + ax.set_title("Energy Consumption by Strategy and Number of GPUs") + ax.set_xticks(x) + ax.set_xticklabels(unique_gpu_counts) + ax.legend(title="Strategy") + + return fig, ax diff --git a/src/itwinai/torch/profiling/communication_plot.py b/src/itwinai/torch/profiling/communication_plot.py index 23dec5a0..285a62da 100644 --- a/src/itwinai/torch/profiling/communication_plot.py +++ b/src/itwinai/torch/profiling/communication_plot.py @@ -1,6 +1,6 @@ from pathlib import Path -from re import Pattern, compile -from typing import Any, List, Tuple +from re import Match, Pattern, compile +from typing import Any, List, Optional, Tuple, Union import matplotlib import matplotlib.pyplot as plt @@ -85,7 +85,7 @@ def create_stacked_plot( fig, ax = plt.subplots() # Creating an offset to "center" around zero - static_offset = len(strategy_labels) / 2 - 0.5 + static_offset = (len(strategy_labels) - 1) / 2 for strategy_idx in range(len(strategy_labels)): dynamic_bar_offset = strategy_idx - static_offset @@ -142,15 +142,21 @@ def create_stacked_plot( return fig, ax -def create_combined_comm_overhead_df(logs_dir: Path, pattern: str) -> pd.DataFrame: +def create_combined_comm_overhead_df( + log_dir: Path, pattern: Optional[str] +) -> pd.DataFrame: """Reads and combines all files in a folder that matches the given regex pattern - into a single DataFrame. The files must be formatted as csv files. + into a single DataFrame. The files must be formatted as csv files. If pattern is + None, we assume a match on all files. Raises: ValueError: If not all expected columns are found in the stored DataFrame. ValueError: If no matching files are found in the given logging directory. """ - re_pattern: Pattern = compile(pattern) + re_pattern: Optional[Pattern] = None + if pattern is not None: + re_pattern = compile(pattern) + dataframes = [] expected_columns = { "strategy", @@ -159,8 +165,11 @@ def create_combined_comm_overhead_df(logs_dir: Path, pattern: str) -> pd.DataFra "name", "self_cuda_time_total", } - for entry in logs_dir.iterdir(): - match = re_pattern.search(str(entry)) + for entry in log_dir.iterdir(): + match: Union[bool, Match] = True + if re_pattern is not None: + match = re_pattern.search(str(entry)) + if not match: continue @@ -168,15 +177,22 @@ def create_combined_comm_overhead_df(logs_dir: Path, pattern: str) -> pd.DataFra if not expected_columns.issubset(df.columns): missing_columns = expected_columns - set(df.columns) raise ValueError( - f"Invalid data format! File at '{match.string}' doesn't contain all" + f"Invalid data format! File at '{str(entry)}' doesn't contain all" f" necessary columns. \nMissing columns: {missing_columns}" ) dataframes.append(df) + if len(dataframes) == 0: - raise ValueError( - f"No matching files found in '{logs_dir.resolve()}' for pattern '{pattern}'" - ) + if pattern is None: + error_message = f"Unable to find any files in {log_dir.resolve()}!" + else: + error_message = ( + f"No files matched pattern, '{pattern}', in log_dir, " + f"{log_dir.resolve()}!" + ) + raise ValueError(error_message) + return pd.concat(dataframes) diff --git a/src/itwinai/torch/profiling/profiler.py b/src/itwinai/torch/profiling/profiler.py index 78c740e7..7ff43665 100644 --- a/src/itwinai/torch/profiling/profiler.py +++ b/src/itwinai/torch/profiling/profiler.py @@ -2,18 +2,12 @@ import functools from pathlib import Path -from typing import Any, Callable, Iterable +from typing import Any, Callable, Iterable, Tuple import matplotlib import pandas as pd from torch.profiler import ProfilerActivity, profile, schedule -from itwinai.torch.distributed import ( - DeepSpeedStrategy, - HorovodStrategy, - NonDistributedStrategy, - TorchDDPStrategy, -) from itwinai.torch.trainer import TorchTrainer # Doing this because otherwise I get an error about X11 Forwarding which I believe @@ -44,17 +38,59 @@ def gather_profiling_data(key_averages: Iterable) -> pd.DataFrame: ) return pd.DataFrame(profiling_data) + def adjust_wait_and_warmup_epochs( + training_epochs: int, wait_epochs: int, warmup_epochs: int + ) -> Tuple[int, int, int]: + """Validates if the given wait and warmup epochs are compatible and if not, + adjusts them so they fit. The largest one is iteratively decreased until + a compatible value is reached. + + Returns: + int: The resulting number of epochs for doing active profiling + int: The resulting number of wait epochs, possibly adjusted + int: The resulting number of warmup epochs, possibly adjusted + """ + active_epochs = training_epochs - wait_epochs - warmup_epochs + if active_epochs > 0: + return active_epochs, wait_epochs, warmup_epochs + + # This can probably be done with a simple math expression, but this was + # simpler to implement and won't really cause much overhead anyway... + while active_epochs <= 0: + if wait_epochs > warmup_epochs: + wait_epochs -= 1 + else: + warmup_epochs -= 1 + active_epochs = training_epochs - wait_epochs - warmup_epochs + + if wait_epochs < 0 or warmup_epochs < 0: + raise ValueError( + f"Unable to adjust wait and warmup epochs to accomodate the" + f"given number of training epochs. Was given the following values: " + f"Training epochs: {training_epochs}, wait epochs: {wait_epochs}" + f", warmup epochs: {warmup_epochs}" + ) + print( + f"Warning: adjusted the given wait and warmup epochs for the profiler - " + f"wait epochs: {wait_epochs}, warmup epochs: {warmup_epochs}." + ) + return active_epochs, wait_epochs, warmup_epochs + @functools.wraps(method) def profiled_method(self: TorchTrainer, *args, **kwargs) -> Any: + active_epochs, wait_epochs, warmup_epochs = adjust_wait_and_warmup_epochs( + training_epochs=self.epochs, + wait_epochs=self.profiling_wait_epochs, + warmup_epochs=self.profiling_warmup_epochs, + ) profiler = profile( activities=[ProfilerActivity.CUDA, ProfilerActivity.CPU], with_modules=True, schedule=schedule( - # skip_first=1 - wait=1, - warmup=2, - active=100, + wait=wait_epochs, + warmup=warmup_epochs, + active=active_epochs, ), ) profiler.start() @@ -66,16 +102,7 @@ def profiled_method(self: TorchTrainer, *args, **kwargs) -> Any: profiler.stop() strategy = self.strategy - if isinstance(strategy, NonDistributedStrategy): - strategy_str = "non-dist" - elif isinstance(strategy, TorchDDPStrategy): - strategy_str = "ddp" - elif isinstance(strategy, DeepSpeedStrategy): - strategy_str = "deepspeed" - elif isinstance(strategy, HorovodStrategy): - strategy_str = "horovod" - else: - strategy_str = "unk" + strategy_name = strategy.name global_rank = strategy.global_rank() num_gpus_global = strategy.global_world_size() @@ -83,19 +110,18 @@ def profiled_method(self: TorchTrainer, *args, **kwargs) -> Any: # Extracting and storing the profiling data key_averages = profiler.key_averages() profiling_dataframe = gather_profiling_data(key_averages=key_averages) - profiling_dataframe["strategy"] = strategy_str + profiling_dataframe["strategy"] = strategy_name profiling_dataframe["num_gpus"] = num_gpus_global profiling_dataframe["global_rank"] = global_rank - profiling_log_dir = Path("profiling_logs") + profiling_log_dir = Path("scalability_metrics/communication_data") profiling_log_dir.mkdir(parents=True, exist_ok=True) - filename = f"profile_{strategy_str}_{num_gpus_global}_{global_rank}.csv" + filename = f"{strategy_name}_{num_gpus_global}_{global_rank}.csv" output_path = profiling_log_dir / filename - print(f"Writing profiling dataframe to {output_path}") + print(f"Writing communication profiling dataframe to '{output_path}'.") profiling_dataframe.to_csv(output_path) - strategy.clean_up() return result diff --git a/src/itwinai/torch/trainer.py b/src/itwinai/torch/trainer.py index fa73289a..d8cbdf7e 100644 --- a/src/itwinai/torch/trainer.py +++ b/src/itwinai/torch/trainer.py @@ -4,7 +4,6 @@ import sys from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union -import horovod.torch as hvd import lightning as L import matplotlib.pyplot as plt import numpy as np @@ -114,7 +113,9 @@ def __init__( metrics: Optional[Dict[str, Metric]] = None, checkpoints_location: str = "checkpoints", checkpoint_every: Optional[int] = None, - name: Optional[str] = None + name: Optional[str] = None, + profiling_wait_epochs: int = 1, + profiling_warmup_epochs: int = 2 ) -> None: super().__init__(name) self.save_parameters(**self.locals2params(locals())) @@ -137,6 +138,8 @@ def __init__( os.makedirs(self.checkpoints_location, exist_ok=True) self.checkpoint_every = checkpoint_every self.profiler = None + self.profiling_wait_epochs = profiling_wait_epochs + self.profiling_warmup_epochs = profiling_warmup_epochs @property def strategy(self) -> TorchDistributedStrategy: @@ -158,17 +161,17 @@ def device(self) -> str: def _detect_strategy(self, strategy: str) -> TorchDistributedStrategy: if strategy is None or not distributed_resources_available(): print("WARNING: falling back to non-distributed strategy.") - dist_str = NonDistributedStrategy() + strategy_obj = NonDistributedStrategy() elif strategy == 'ddp': - dist_str = TorchDDPStrategy(backend='nccl') + strategy_obj = TorchDDPStrategy(backend='nccl') elif strategy == 'horovod': - dist_str = HorovodStrategy() + strategy_obj = HorovodStrategy() elif strategy == 'deepspeed': - dist_str = DeepSpeedStrategy(backend='nccl') + strategy_obj = DeepSpeedStrategy(backend='nccl') else: raise NotImplementedError( f"Strategy '{strategy}' is not recognized/implemented.") - return dist_str + return strategy_obj def _init_distributed_strategy(self) -> None: if not self.strategy.is_initialized: @@ -222,6 +225,31 @@ def _loss_from_config(self) -> None: "create_model_loss_optimizer method for more flexibility." ) + def get_default_distributed_kwargs(self) -> Dict: + """Gives the default kwargs for the trainer's strategy's distributed() method.""" + + if isinstance(self.strategy, DeepSpeedStrategy): + # Batch size definition is not optional for DeepSpeedStrategy! + distribute_kwargs = dict( + config_params=dict( + train_micro_batch_size_per_gpu=self.config.batch_size + ) + ) + elif isinstance(self.strategy, HorovodStrategy): + import horovod as hvd + distribute_kwargs = dict( + compression=( + hvd.Compression.fp16 if self.config.fp16_allreduce + else hvd.Compression.none + ), + op=hvd.Adasum if self.config.use_adasum else hvd.Average, + gradient_predivide_factor=self.config.gradient_predivide_factor + ) + else: + distribute_kwargs = {} + + return distribute_kwargs + def create_model_loss_optimizer(self) -> None: """ Instantiate a torch model, loss, optimizer, and LR scheduler using the @@ -248,26 +276,7 @@ def create_model_loss_optimizer(self) -> None: self._loss_from_config() # IMPORTANT: model, optimizer, and scheduler need to be distributed - - # First, define strategy-wise optional configurations - if isinstance(self.strategy, DeepSpeedStrategy): - # Batch size definition is not optional for DeepSpeedStrategy! - distribute_kwargs = dict( - config_params=dict( - train_micro_batch_size_per_gpu=self.config.batch_size - ) - ) - elif isinstance(self.strategy, HorovodStrategy): - distribute_kwargs = dict( - compression=( - hvd.Compression.fp16 if self.config.fp16_allreduce - else hvd.Compression.none - ), - op=hvd.Adasum if self.config.use_adasum else hvd.Average, - gradient_predivide_factor=self.config.gradient_predivide_factor - ) - else: - distribute_kwargs = {} + distribute_kwargs = self.get_default_distributed_kwargs() # Distributed model, optimizer, and scheduler ( @@ -375,7 +384,7 @@ def execute( if self.logger: self.logger.destroy_logger_context() - # self.strategy.clean_up() + self.strategy.clean_up() return train_dataset, validation_dataset, test_dataset, self.model def _set_epoch_dataloaders(self, epoch: int): @@ -527,13 +536,12 @@ def train(self): # Checkpointing current best model worker_val_losses = self.strategy.gather(val_loss, dst_rank=0) if self.strategy.is_main_worker: - avg_loss = torch.mean( - torch.stack(worker_val_losses) - ).detach().cpu() - if avg_loss < best_loss: + avg_loss = torch.mean(torch.stack(worker_val_losses)).detach().cpu() + if avg_loss < best_loss and self.checkpoint_every is not None: ckpt_name = "best_model.pth" self.save_checkpoint( - name=ckpt_name, epoch=epoch, loss=avg_loss) + name=ckpt_name, epoch=epoch, loss=avg_loss + ) best_loss = avg_loss if self.test_every and epoch_n % self.test_every == 0: diff --git a/use-cases/eurac/config.yaml b/use-cases/eurac/config.yaml index 8912e898..47b7101f 100644 --- a/use-cases/eurac/config.yaml +++ b/use-cases/eurac/config.yaml @@ -6,7 +6,7 @@ tmp_stats: /p/scratch/intertwin/datasets/eurac/stats experiment: "drought use case lstm" run_name: "alps_test" -epochs: 5 +epochs: 3 random_seed: 1010 lr: 0.001 batch_size: 256 @@ -57,6 +57,8 @@ rnn_training_pipeline: strategy: ${strategy} epochs: ${epochs} random_seed: ${random_seed} + profiling_wait_epochs: 0 + profiling_warmup_epochs: 0 logger: class_path: itwinai.loggers.LoggersCollection init_args: diff --git a/use-cases/eurac/plots/comm_plot.png b/use-cases/eurac/plots/comm_plot.png deleted file mode 100644 index b406b1a08737cb91521ed856bc7e8bd2934b9724..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 30280 zcmeFZcUV`TH_ZQyb^HZk8Ai($VNiLM%nVRjon2nBZ}Nbn`>7sZLXSJV!2^t zWo=?RnsAa0*Q7M9DHOVkK3?A6QL(F=Sl(>&gpfD68OLZK9H74VnMzA+Rq{55O8Z)trU#q07xskXK@uJbS5XlZV%PRo(+ zy?MGJL8oS@#jf-F_xP(#$<*ZG^(R=&W!*hI$g8Z>b(=8Ur| z-^P_WxN+*^(PExe)N4i1eh>mEOkvgMDr@Wu8HS)bnO2X2yo zcp!A&{^1DjKR*YZlqvJ$R}DQUR~tsP?)uypAFF;Q)qXs~-P5zqPcPw%oBQENZ*Ok$ zr**}wXx&f*!x4+GnF-n%7JailWg>NpB2rFHFDzQ;w8N#JK8cMryF0?a^NKj`S-|b~ z?NeK5)|-1ZCHxG#7CYQhTqVi-RjcIf+m~Tssq4Jo-=i~~9Sh%eP(Ml6Wp z+6XyEM&Zn56TiUdZG0C#;HCnPg^YEVU#WA}+D)D%%bGjvcLpw3%W3Xc_&S>WIRC?9HP3hhZGL`pv3GFjFWR0HqY$#h8T@|HPro(u`kC?W@_H{Z zrYf=J4;-#btNUxCQ?hLbIH}FqlPAr;X6m{wZsYMS4XX}!7QEVc?J2jqZgpM?v&VK< ziv-o|?LTp;GlhruHzeFKt&iJz;>KWnQ_WIO`4(SfDcsuhbtS8B-1f1w+if4hF9znGcM zthZ8p8@JkpmUQ!~#y6JzpKT`pd{7$7*Xb#g@Pyu7^pT&J(sBtG(~u1-}o zMnyH}24k8@4UH%Ba_}d=#FwkrE!ipO;?d|}nN9cO21LmEM-?pRM?B+g ziMmoDs+B&t$&+#Z?oo@cF?)op!e)gO0*`9)YNb}!M%Q5R&;^{ddnWDYBEPg`%iJio zuj3I7-7r6O*NG$N?`+&w{gm5p+wZ?$8nzzNiqc9mjX!;7Blq6DiYM(X8n_c_tKGMm zsiqnqO0#TNJZAB=@)2W;pxv+&)h94;e$Djuty@FY^_(@gY}w*4KV;Y0(eXN5W_w_U zAHSZ9b^Dt2Qv;u#mO1?JJd|zS^XT%I7YTheV%1?(<(h?TLsk-)D6G`qe*4Yc%S(M} zVV1dhGp48mj?_ek$g5wy8kc;fg0D|kTxtum&*#I&by13C zU%S75fAvDhx_@qRfT6{4SV+uzCNsmLYFRI%+u3|N-EMKN-D~peQoI_k>v+Ltn%nN% z`CkPdON?w@{_wycMlJrXkdtspk;mHrG1mn75Yh0KjOAlXJ!4&ECJ7_8(Y{=o$xnow zqNY7EA4}UC7yT<;oQa35oD#v3>hav&lS@x3!m*eR!9S?n#E7{AOlR5@61! zUu`;KKjx|=Tu}Vm&Yhfyc%047PEPdn6Oj@vw}U-ci;BWW(n>j2604LCE=J|mX||x_ z*DpMH@W9H-;isg8f`S6ufdgCu0!qQb!QE3oe;(XPdzYU1&-n79B8d$fHZYhgDn3>) zYSGoz&9>#9q^@&WoJ=K=zOUwrmTc>8M#lJD`|$`fS3kbRr>{kXg)amL>vF0mt?}=< zbP$iE@j^v#-Zq9CGBSRAF5kuw+hdf%Kly}-INs3WX~}c+b8=q%d`0Xkn`K+>+(4oS zgL&~IMrW>;1jW#2E^~dEOH1?3Yy66fT2Gc;cfEPidF~08Mp|#r;9%U$%`)!^7Q`I2 z^OYeCds%T0#a_&U%XCN74+h>!jI(KTYF2Ju-nL!4V#)&xRApsT z^B1hrb{#PeL-AoOGZDUeZbM7t=eMi5KYgTH6$oCC3mUBHRd&2wB?Sf zq_>7j@|(A$OPsm4rP^)PZPlprdv)?0R1pB?mZNWQb4WhgGdJQI8DrV*XEWZtnfF4b zxc!($@FIt9?w<&}X*Y(r@ENLwg$0*d+)ejI1VDMS`ncNOszkFAZ}D*+jYp3j^|$3+ zabKEpqW~*;-*VdHJZKT}{_bY=BKN1M7Oh&&GvC)1ef-F!5S$yM)1oKhIQ5v-lUa;9 z-cwQKr^ly?u;IRqJE7D!GTLQvJJXxVhsX2_^~t&}@(}Yw24*~w8_gH0j*@fqi$r5?W~aWBnjR>@!#Ud0)u<@ck;|eNHUO`^Z`}{1wqAjP*3-n%L#H1aJN? z;>&-7(GnJ#+ov8q+H!Q(;Gl35W8?Hldt`&2t7>_G;Q2fU!MZmFyXAsTP%!}!j0J%qw&T^V*a&+%u_MfTal`lpPeG%wS=-|3yshY+7nC;+KQtO)LJv{ zdH9;rf971j{uCa z`rJfcLZ0Ka#>8TUtLXWPU>%#mM#ahgI#vGMZ;9%T3akQY2`uY8`-K&E-d z>O9?g09ixVr}njkL0y7QUG89gd|<#4liU-wrm@>~f2#L<84|Q?yNW;?m;v;mN+lCJ z=EhJ=j80ArBF>-oO$GdKo0Gnq*$y=;P7X9E0UD(lmhRi>d2W1cB)=uBIW4w-y1l~e z!~KiEkB?5+StQ157sxE$8+-Vj_OJYo@OlEF3bBQUoxSJY)Z(|>c9TE6?g`o$za+~N z(ZhJK@z`XQWC#b#w{_M9Nl6FG112BLO*3<_v$H#ol@`z8BVGOdFUHG<2#of3{`|AIPF>HJl~?1ZKXOB@x}LeWhew%#NF?ClK)4@Gp9eof zUyVXfNo%XlmoL-xdB7yXB6%bo+uyv|59F(rd-E1>Mfh}EAWL7Hk=kvVNp(NI*1BB# zO;n`^zq2uCCMM=Vm8xD#uu=2#UHa?eeUT6i+eSh=5y1iTp0PgVR90(i6DltQ6#TMphf16VwJGIF zgX_}xtu`m?hZl5G0{K%ug@`Sm;m?wmmag_;3yB5D-z74~OI3>3T9THP1%~3TO>CDL z8yh=1J@Pdx@Pys4Y1_!FGwUbb(>W%erF{RL2hm02W^LxIa-_Vp&n`hz)2g;G>fuF% z0e8>E_N&UMwhrB+qjG7bK4*DpA4lwK$3<^@EU>ts*(p|lM>=NX`pdd#Hy z0If~2|62C}Su9Yx=4>H_wBT+*?= zIg?jR;7EOGOy=56cmS7?C)G3>d3klRXKt0zA>K(p<&yE48f-Fob9H)HXKZrvaLD3K zrMGWSWm|P6&yJ!{nCuC0jk=IE<})0$_qvr;wVtcak!h-wRAq(Ovf&%XD?N@^be5de z_5;;Ql*}#7n{)PG1GegHdhlApgY(Ll7o3+no72qhy$A|YO}LPCFCpaVHB$6bF5C9k zDvz`+rW#kR*2&!w7aeZ{}+*$3S4`&C;_|;=i-98r}G`_W{s3_XF>QZyQ zb1bRyO=?8#P19y(W_mr3oqIkaIX2rf{)AmYH_f6|-ly~1x0f`Vcl8Y`W+iOcWyr!UPcEPRT(`2HgSCGXx@L}M5n2IB2_BEB(-%$bv%8{#zf2G#hh zw!C?K%cbTF%fzA^xcS#|fk#b?+>JhgCP0z+`qmn~`k<2;Y1h6xQI+1`^%OL%4MdUl z8-s1`=*J!9&qv>!d*mc{{kk10y&eBQtnsoUw!o}b{evYn+&O!g1jAKm4*iwa^IU2j zyPm%N^muV8MF| zg-uPGH1G3rOxH}Y^6|;jyk8!sx8$Wgy&nA6YJg2LMOMfzQoJnMeWwv%_U-8CpxbDM*uJ&Fe}K_4L$Www)9Z}_W-kh?*}1tGRH4Wmffsai%yl}p;+lZ z9eG<@o{PEWOZ)M~15yYVeG(|h%sh_SpoIXcc~1{>ZONZJlIR&Cs7Br55}q}rbE%toz=kW-$Y z8d9Fn0{kFe)-OwJJGJoOA1Y0G-I$h+Y=IS{QB$oGN(&8Y0amn+#wupJQ_*j<$=c(eiJ9l z0}HXNsHlrt_SpvU6g&<~kcgw&=G{kC0*{(M*)O?8C-hTAXys+(%8 zNI-j}LdY}Fv#MsFdBBK9jO5SrNl=ins!Dy?mzNWM^o;}h^}V~ZY3;9tpMGH}+?r|C z<$mtbp6EZ;Z8F8I#5zv2*G$ySv+1j58*0sIxUsHz1AUDFvxmiAq6UtrFmh^Zom8h5UK_~h$+FepKTt7cQgfiJ3=>makeT-Gtrx^9b;O@`< zq4Ogm0nKS$d-v`g8y~O1QYmlI&aw&*b8K*7U#6zt9KB>a(xwB}QK`s%TV1qrBz=SY z{OoLBi0e|--Oal{53uWvgf#E)21giUJ5UcupW9y-8@XZYfv}mERgntJC`QlW?jnKM zLy;VjR_9R5Tyzg9E+B7Vrc+ek;}$BL4{3_}3&)fZP1fwhGU7Cbjv z%*@*_C)#rG%9lOkqkb29aF(#iJco`C8*uBI+qhKD|M=*ccKydk`b0SgkzVHjr>`Mm zuDU3P)WD6`rkW^^&ky4o`m4h|FxjF&C=dMn{+=`4adFO;Ol>4t#o4j0IwWyu0u3fH z<_H?5ZMj*qc2%Q4ip$Gf7Dgp!XJ<{Y1~}_ma_v({Y{Xqi4u|4vt*&3Ej-Kqw8Oc{J zd&aB0IM&f(Ss=K)gw+y3Ru~8lf4}Lz8cE0L;kvq5_448m1;v+kc%CCMW7VovG3Q_X zfr%Cg1}D6+0pt659{2yO?dFfoIt4Bw5(PRrH=>bl$#E!C#tR#0N;V!}vs(BTR>&y{ zcX~xUWailA%u+iZ^DO!S!gR992`PWq*hNZu6 zW;~&?^?)o{%C(`Abku4DJP9HDSuow;`e%Q#&hcS9#JQa+w1AnzQ4NzQtDu9o})Uq zTHJXrCXrN^MF&VCjjxwW9`8q}FMdiw(B`50wU*HwCBeIcK2cmjg5{f zg3m_=iacuG9EAzx{@_6s)@}T$+cYMv*|uET>2Xd@PDfl8Zi39xB~KJnovrWHnu?k^ zT}I?RR9|tA7*F;GTIN?hKV+bia6w39`m1%HOF0u8m{L%3NIUVU{x^|KjpE3s%DA?( z)VD6r(T>C~kL~>R1BnHMjQE6_*vDp|o)?9_DhM8^Gd{Qx6qt#2?at}|JzlodYAmoc z+X1yntPdc3tvrXcS=;H!NmUeCDlf%XnYZOOEd;QWMF;xO^g08FAg~+>A7pAF&Pn@x z2@y-&w4KiwBETHxhH~6t%QoTi7edO2oGMtd+nAUVFyW%q6Lp(2uZN=Y5wz(&5AwHf zq+q!ofq{)sg$ym4nb%%Hw?s9J(%TqXhm>XR=v_R_E5j+610+%)*i=R-hSD=IMDQ0Z zD2XjEq}c2EKvp^hrN@j*Cp-0=7gIcvx!GXj8&db~*IV<=-(a;j9|GqAJv}`VXU8-a z!9QI=EhLyXKb*rbgy=2^pb>z`eaXVCG3ixlLqo&Fc!gMC%a%RI^~r%C_o{3r`&F&y zkyd5%^YbH^T^FjpPEEB~24Wq^ZDp4~k^u&#ri{PfDzfeiP9C1mP=sY6^%%I@${q0S zLp)u%0<2>Li@<{y|G>9Q->Tp}dbGadx|35b2{94*&huKS>IoN`CUE9Rjb%)OL*&ws zDKFt{5#Ujjv#h#g*U-=&WMgCVpO4pg^R9DXz-$iWa=AH1{U|y7=+#Dj3l9&E!vJp$ z6U}CcB1vtASa&NI00n@z8BDSz*v#Zvd(cUxDru&w%9k#6oXMEBymjjqIZXNN*$)Pj zmhJg@rS0wQHuF>F)-5OzWQ{&OIYd%Mce-ZEWq)$-BYD%oS-gi1RnheBXcP^ zSIXsbX~hA|3EsJNG&BKVZ%wM|afHVS6q+VvB!(K_>@RY8crJ9-zPPAJlgH!!{e!>( z#g{@L#BrL=+j*8&kL!xKEMyktBg0t+fEaUk4)31mtBKB!pmPX0z&ssz%<^aklJd&s zlN(wbE938;>TGA6www>LYdb_98$A<~#hb*(`y~TNHs}ypw8XHlJV5`$S3a9#HVcB~ zeoMb&hp|PVsB?C-)9hG8Nr{xTwuZ)YC|4n_-zqvzRX9R0j6kAz+u74|W#i_}g8s4x zY9yJR4NbphzL`xkZp}`!&qFPs3{W8l9Wp;bWjg(v!2VWk%z&2`@BU{a(K~b z+SMij-^9ekzUg+?WB#(!E)BEu^MS}o2HB3&30duQVhby?6^pOtYdzqzGGY#|t_K2Z z0G=+h#+sizKYr(733?qG$`-!xkw*}5)Rz?nuMEQwrc{zfa2)U_L;DsrKw2M_oEzrb zcr*@5%mYm+4W{6hyIt8`Yy7>vU+$NDB&=5MHL#?azi;0@`fc07P3vO9Gmi{jM-~82ibz`vcm4MNwu$id~KK4o3IIaC4SohkXlXg#@KaaxFiJYI2sc=k#(%4-rDk>VC z->smGz)=hEGX4W9=BJt(WxyVRWnFKjrAHvSn%@}Q0|CJb_UY%(YvUpKBqtY_G@zAp z=Ta1&lM|NlJYdgfi%xrmmwiZw4DZUvQPGrg&h%g~HH#gO& zi~-ZgyfwS7BdgsKF;-xgCs1j4k+lpHzt6M2hbL|O4G$gVob7qBpam40xN*}aW#m$x zk9|jsE5)I=9r0R~ySzvksL@=MfQyZjl!}dfZ&z1y&MMt~{``4yzbnF>(DCD8nS&e( z!NM4lT3{#M=1ue*`lH8Z#o}E<5)=}Cv>WW-vuBS=nAC3`>79ML%Xx|HlTcHP+j8y6 zh!qKr^#xUN966!_=}=|b`ucUn$NMCdY5@?J=;y)AIeZJ>xm*tp5{wIbw##BRdBni9 zh@Y~^A4jJ9no=Tx$s+JzxM4zx!4uF*zZ&i4=626we#wA!R^VYRt60x^xl(Ol14iMS z=a(18y+D~e098eGi-+O@q)G7660^kb%+7+Iyq{9ozCCVeTy4~WkLEVrNVk};mw;jfZFQ8 z!uBe-ScH#L2t{gsd^)VHISZs>pt~R@_~pwN)2i2Z^5=(7TC-TRqz|o`Uz%*tL->zo zI%(p6jV0~J(b+Vo#`$!^QlB|MEuJ}%ULijY#eJ<8cP6Q?{gox?YisM%W0tx)nb-Ep z4Tuci8>^*Ji>SBAw!Aj$TYu@uE@sEw?D8)vNzs9+*?XH#^gOPq6QZMK{_Fvr@v+Ea z-_I}Wl8t2KLPT{C!IYPm7SqgIj)GLz_|pTDnF%oeiJL!P5;UbB@y zZ>UN}(4Ln|Ge77&EK5S3%~xz;lPMb=eL2ZoADru1`zWx}GOs43d}0n6P7qa!ph@-F zHS4zk>11{G_C^6R3y?5!;cU*jrnc zGWJQhM_np%M`ax7Fg+a4c=ARR)%O5r#GkfU7r=70h<>;d0u3>H>S7b~1Nx6;fx4&# zOOR?(6Cp|C-H9cAI9m_=3?{FC1fBfO(y}rcrB5jqrR`&)ydW~24GWhMN>#BY%l!`* zckBoP6ChEL`oc41%CW8)__MRS`#@;b%-NCf_2o2}TCSXqQ-cgwzQZaKj;gjFrBn^d zl&-~e7Sqz8X~lE%^zHiaPW z*7=VXU7wy>-MDddS(1#2(1{aTvO%2GSoK8tH}M!)8R^+qHC0uLG`HV&w2a`n4z=Z- zEINwQk7|rhHbg5+V(Dn*J7pCe^8~SzhH(m>GedF4P4fF(cc3L;%n&{EBS3cZ(MI5* zh0UV{kw$rXpeR*P@FTp+qcoF#BMhwKL_;)tg4n^Y8jtRR+`$ZP?fl(6%za(M!wJNi z@!`Q%h9Tf_n9xk>rgq+XBzI3VkpqQn4yxB*HfZ^60)eH+AkHd7xE~NBFB@u>770Q$YC*M}5JwWnc zml8mBOjY`K0gfJ!lq{yNc_cIa$alWI0{V}_=sH=gC;WQ4K#DM@nDMx5A8g4`ICbh& zwbXJQX(9e-SRpJ?*n`vv%EB2unT@8%mkzStqy>tgtCKaAQm9m zXL$1g6!M|4d&MHiAy=OPDkHQDTy$HZM$SjqqZB;Kc>J2UWx--PDfY3lS17#`Hc_); zu25aljH}?lit~P}e;%2D5GbGk!yC!9vSKmFDWOauaR8HL;Ldj4N|afzP(2ZJj3fp# zU@Q!RRrCf2u__~hLV7WU&Li8Xg2eH7n_NoDE5njsG4myF(`;74sezzW(FxtS*aX-sCeHI;J>cVM4jzZE_P{kznJEUmsS&J>?8E=K%YV zDtI&4Q#ztueNSHdCQgC}Av<-QMkN5Tc<}-71rr!#0G1UB&t~N`|d-mG$-WJBs({2>Y zS{|zuMO3M~|J(i}&T}@z34)U0G1QXNr%%Ig$VbXc4D%D45xZ)PnxN4qdXm|MtU9Hx z^LyXBe&a^{Zp++Ph#oO^BWR@~hwHx!#25$$d7`d})dp+hLxsVv1ZME zn~^gQQEh{9c*Ut4QGytaaln!Z<+*I(uxqTRA`zrcFBTw4|4LU0$u!p=n+R_i7q|)d zUW5st({&t5Q%U?yq5H>=P=FF+1heLB+t|yWpTB}UNM5Xlet3!1PYtywJKYaHV&Zc^{zVk-kOMS5 zQ~*bP3U$~lemaJ?6Jvq(QS&v+It|E+&=tg-3|aRKzh$-_u3II`$_TDKbtr;lRwHe{ zCfK{T1qC03+*0EsF&KM=`M$89h~Kd{M?18#ZK7xMd1uFa)JbKt`dme@2;pDj^$K*c z9t8{dYhEKzS=rjwfoh*@UtUtjG(3;6GJ)u?B`f+H?i=M>e~p4z9V$Y&AQk2O1tj3n zafvHjFDyfQ;GEK`iWiVHdhw)O<8o-%h@X>zWD9&#hs%-+7H+EVq9h&q=muSk76F3~ zPc;^_Ogi5>@dMsn{BH@*MCK~4gYf;Z*>xJoF`5ear(XMhn#7^+-@lWr^6?R4EMnLd zrz*I}Zjqx=)j{!hj=-eQ#bBq}QYiwrcwA{ZaT+KaAt+>~O5TP|YlA4kJcSi{o)CMV zeA&G~ns|S=%U?$+)Q!V~{!#4Z=jBy#X2gFDi&^QIfjH`A`(OJ&C{2JI5*?$Jf*lM* zZzz(A2~da6uv}yl*1@{9YmFCXMtABQg@~o^m%OmkTe4h#oP`iAs2O>Ow|#>h6hf{k z@SsA9Fpm1>@6)wL%33G}Yr`K}#bAP{?!LRAEt8L;Z%0Xlz z2zESQ{s4jf3G!oE8Fy>T(Oq)x4;Ok&p3F+Yy)-uQIbrBf%eG*8i>J=3)0MtvW%e5; zz8`(WbL$7A<^!)#FRje9^M;0TK0Xh3=8Gv*3|iTAo?Ku*MWNp7e8UXhQw4S9=ON$% zE{&v@2(KZ=^A8^ET<*@&yQ!3EaiBG|Uv!mOS)aG^@5LvRX&%`bN>HiiVqNc^p8K`E z`V|UbD?7Ut0@)k|@Ji@jz51%gPTZn*meK5C7QlZpj;wvCINcvTB8!NPijjVTKtUL2 zn1qJDW-0qTK@ufl5zOo}yoJ&o^`>cSmbE0}Y-*D=kr)MXc$mfCy?Y0%%Qci<#lRV) z1ZNAaih?c7Lliatvr4aorEm@T7-xe) zhYdSl6<*kxpdzoWj&6>REH=27|56D4zxwB2-4%wk%#zI7eiXywkF4j3zWi;cfw$q2 zoptZx@5IZjihE})(44(CY!%d7w<`~+KDXbVx?*{b>%iVp#DJZj%9|9s#%?xe)Bb%# z?d0?0tq){UZFh>_9sAN9W@Ud>XI=FlD?JmGxVyyZNGUX3v4{MB`A?re{`doo6Mr<3 z{ZP&$oJ7FsA(>zYk}(gZ3sEY-p~fKr8zaT4MeV?YOH#m?71o1ns!h}rBgHh)Eg(#y zBSD481!N+hX&@Z>AGdEq0^!)kabA3WdZb}y^hXx}Wr9wQRv%f5l&Sq?>%Fn;I1U^* zXWjkzF_050Cpuq<6yv@343KmF`UJc2ZaIRUlp|ynK{c^bpswVv(UA){GJ2Pn6ws3`g&T+rB<^y2Os5YeWnwFqv5R>w*uf-Pv# z2>GCZValZ=0rWd(6|(E)xFsdm+dEutXbODnPL)djB)Uqw^l7VRxK(ybHZ43M4<0^L zRq{FXe)>nBe^3gq0tW^T5iMfJz;9q2!JS0x*lksS?4(Rg9+Q)4#{9^vfGd%(;lS6E z&l>#v`KV>^ypGp{nxM?HvBO)F?kEYF>#U-%gtD^jj9=QhvCp^bhqk!58{DCBguZP- zRPKVq2-cN|Y~5FvFp*6zC8`g#%285vtW`#p)EKY7Z=R)s-RL?ex|zWz&sj=-Lr zBZi0R;a1+^Rn?fNg67S&3&&Yl#7O~7gc%e@YOt_Y6*IfW0cu475~ttf!uiA_{D2-n ziB%(5AGzLf76kaG@^U#)kM&T#c;~KUI-%+AA?dx;K*G8eV@MHoW< z3Nc~t^g9yl{iT(B6u*Bzq}Jr;>wDX(+omSCz(Rm#ZAI=ori2*HgX~CQYmvg(+L~U} zM%`qBb=u(RAr?_l3)<7Jr*j=}MgK1bsgI@g>%xlQ1(*OGfd&(9c%`0{8YXL49Na;G z{cctA$iFEL9&qD3 z0=va|^ggvY4Db?e2JjOOb0Zgc_s9hIC8LS3xwYwbqpyVrrR7VK0YNWNy@YJ;i{%A>qv)*-d6yGnqN1!!cT>oOa+wj1SGwd zq|-Q#;&llR7(_Xj;_I_9Win4VUO^e=RwH-M5|4|d=T?&ikwmqfg)WFHUGZPCM(qOl>*X1{avaBvZQ`t@#l@1i z{dC)v3{btdhQ@n-#KvX$ha`2Fj@-a_Y3iylkZ>4OA_}80{OrZth2ljmY75>McQ9RuqTkp$EPMxIZf7H#Uzqap{7a870=A10L5{OqYG(@ZTg@`(95qHKV3A{(C z{$#Pb1ZAoXW`l~~Jrwg`Q0uwL2#W_a$x-Z|>^^1z!y@cR$o`{RE(%y9<;|{E|ept@1H)XqaNe78Z~@@J{Bi`bT!cLZQby2kU)T$>TFfgq48( zR}~N7NVnoA(Q8VO;t0&6Dgo_Ie80yW0|wz}0-5dN4o3YHs{$mUmTWSraKl;9yb6JR z3T;5LwzF85S({Gab?;JKvHu~1yHGxS`p4kd&}S%so{wIVyhYFo_jW zY{?t`9<6J=pFiILVlN{5D?F&U6}7mPC!mG%al(j1B%l5EUfDNgJiHl-rL*nQf<43oS)f{VP(y4~SR}l`_#-3L z6}Y;#Fp$QKZ?7chd8W&63jpY z5vc_H#>F38f2U1+A+UJBNDyJOa^1oMgO_RVLk2_CVWq;pY-MFd61lxS$UC?o zo^%hLJ9{=tufT;%H#hTgMR5E;L!ud`KM^F`T$dN`B{;$e0-;xqP5MbC_)nAXc>g<$b~(1!Ux0F5gx3;e# z^6t+Ky95mHLof2{&xRWRhHxr4m4C&jnfdw1EYeuc5}On#lcam|`#STcWC?ISK4E^k z4%Cx32G2ujuEi)52MV5%%=is7z3{_RMcUs$VtxYe4_8fe!gh3kVSz+Luj~FvF8SN{ z!EV|TaPu(`>jT6F?a=9gUP)5!?O|jj)&?wp&hnPcTej4L(=x8=208+);y8Z%c-!Z9WdD?@ldSR@6ZLij+=FDyV{D4X@~e&z!{{TCb@bLe%b&a1pG zhKE}oBBlqr%O@m6FJR!%O7+fOJ`jE{G<1!n8bZ44{r?K^mGG^;&x#&5Z3H0VCE=xc zC>H%+Aix-iV8G8{UG7;NgSy-*$adiJLLlJ6H%*@>hMk9>1SJIma_;K-i@Auwa)Y#U z-d*Me%qDZwBOwt(G@w1utIA+-3QLV|{rdIL|Buy8ZrsMj3MT{+w284IgBicT{NhE+ zfguuCBX9#qBFGcwmp6}T)CW|>Ur z7UD~T|5TM^V;H<%9=`I$xYUPj*YRsF5eFWpvy-T3xC)Ts#zP#0W4(GO#M%O zOwswl!vquHKI=$wghnQG#+1yeL7j-rEa2#fn1|qOOI1SHfd`U}nK>jfKT#=M+N7^K zoNg!uPKNdA88B~wu5#!XB(uKawd~11U4d(fg+Xk%C1d2wh%8BrEaPRVt5~7x5V?2u z4KQ2ghP`;5GVg1`{}znIQzD^a1(m*kf5wL}$gqd7)Y;sVUmf3oGp&KhUQmbA`(fkUVTaH%Q<*?h?r*f;}ba~OdTg#PDga5 z^a94re~}~#ULsYaOP{&1*nre zx}d#4A4*nSNoPz&$aCm{Aom3RR0hHG39K#gG9+hspTpSY!;DA5DgiMsf`ZI-$OCh; zNdgGO8io!e^nC?J7~oi9kOhnbn?SBmp%CXr;YyfOV!^5pC$CDe1IG$n`tS={TG6Z6 zR*fUj0_P%hT%())_S=`3Y)w)B8LE9ivqc*iSEN;t8Th!jJig}{zn;aw@8oR4dvP{$ zt%XZXxFlkFr zpuhq7n3XINgw;wgTWK)qX659xWGaylM86572pD{%B_?cSR%HbD@?)&B=r2KmMc{`i z-0u$X5M8~%iNoD$8_RciV~xbXL5`F86}46d1qIbX)gtCZ)M0u5IoWR&C;dV2dW2=? zo_+AU`~CY#|6naCOeNTzgH`-S5aN*+{)NM&fNO~@aWj*!#(Fv?6)CCv&@ybk6|N#i zL4vMe(l8nzJ4#T5|7@i_J%S5?nVeJ=13<70WnkA~{D}h2ZH^*$6Zoib=C(W6)}upQxANUg;BtqlC><~GcD%wq;rX!O&~;e zw>ZE6D+U$Ja4EWD7G@Lyy|3KqC+~+zVygSHD?EFvf3V%=;Orst;`~#SHBz7|nC1vy z;wm(H7BnXmo5w5bnMA{t^&Eq@$$H+xGJVo@Dfd6E z-(dg0xHJJVps9USWWyTh5tFd{5=RaCRU|^7y*0fQARotwd~tIk3LCUX3v>Wt(+nah zAcByngqtH>7?4G@{Z`=MUXs?Kvy-Bd(DdLkh#QhuCs!r?T)+NS#J6{}=h=VZ>P3QRyS5mkiap=0O?Nc-eu$F@SMw%jCS$liT zU$Rd_>rA8v|o zq=emTUHdCFn&u&REMmzKvhGf`k|6I??^E_#1qJ3&_m?2oEABYGh-|hC>*+<5w91y%Sq7`~y&*dHdR)X+`P7Q{aLv!ldo*dn<=R1T*I?Yy?8Snni8^>LomI zhb3~bu>{y!rtD#8_Vl}V$3m`(GzszQ=rjzvfO&Cj7RCOzS2Gt3#De`P7|f?fDlSa2 z5=q0Xo`(qZka_J*TWqf#|w?T!Xk`7F;IUrJrCs9Z!e6dULe+G&o z@nL@QT+WfcEZ|Qp3TTuu2Iw|vvjB8cemHsG-Mt!44dsWMv)ZA^+b+ypCw|7FWJGcj z4L1Jri9D9Q7KBMOhVx%VUD9-)_Bx|)QiN8O2QUWPT7*EbySt#l^*^Nc=YD6(FEdE7suQZ`-khEX=9$D60HPHx9`k6k6!-5;}-iDgq;&t13PDk*oaMT z5(qj5zY~p>5B~{`9-4u=D?<3;;>Ue8r=Q31lxWO`WR!^L95_!bw8s3Xnpdx0eFViA zpl|?%q#MIaHx56z4j|LtkQmZG=H^C0MXB*-A31@E5$He!aKpY;z$xh^lIa}w-WxF1}K5Wx7khdW+@0g3#DsdPni67R-i?L^m?>dl*(#A8B? zvG3o%uV*8JEwvxnhcrg%x-Qxh%J3h^`KK;gz#_hrFgo}Gp$N@`a;V2@A-?>>lFB5( zbr6x6d7K!0h>->b6=av^uiK$VTs@ENAV5EX)aV31woA+<_mY7DF^UQ&0UNNm;a!UV zbDSFii}WpE0;3#x+BrOIG2lbq40{ZcGRtW;nE+2@g5SwRp`0Y+2FJ3yyL<9q%wDh? zPJ$*gZDODU=K1FW53!(*CAJkp>|^L+#c!>)x|Z}=0id2t8pS*0j4PlXY{FhD*+%Ws zrAsRwsowXV8%gB+T#ls*&__fR{K;0tNoamXd6sCyld zut=K&EQ7L0+%+T_=@(L9hCf4^i){Vz0SXfdB%rRbUxy@dw17Lw$Pz&&azK(L90R!K z(!2jqTE{HgRN%Hz*3kvzY0b5-ao@Hw6Sy%G5O&bcS*)bvf+*Hi5z_vz_ZUyuRKi0t zfsDs}{I~|J+pr&R1l_To+36+mih=d~XEE?-fcUM$Y@I;IC24}Ox|Zn|1VaKI0NJVw zx6+&`O|GBL;Ml4U@b@93s-i?C+8T*EKB(6B!kG{y>o0IL;a4137>b-qRf~|_`HnnU z{YZhy9p0FFL_EZLn>{m+EkyjV!vx(EZ*TqZPvqo)%v)HS?@j2PlPAU0^sZdF0+Z^MYb(Gf_$*lhx15Q?5P6jwP0yHY$yO_K z7#OSlei8`OxrrJDqJ)x`Nm8YubpSjbJvR6P1J?`Yj2LCuL!4ftifzs|B#(gZaWwHR zF)^cAwi>>ZG#K;U&)Pjb%>p7NmwQwDZ7D=VwC zwYBx*)0Mfj3Cz&6wVfUNO>*`dT*`E$DVy^1>RvbULej34O*AO6|9AY!uD#uRC%~Nv zI*~}U#=VjrV#2HeaYrXugNek>2r5K}00Qp9{#qn6!~cTf+k0Cqxv)V=_#y!PNs_>Z zS_w#81F&YGizH^aEiV=p9kIViJfubJW2dKrC`FoMNG~BA#8v35VrOP%<`yLcM4_nR z>`G$#?{k;ag_B4%5riKp?Q`Y72-caH{jlksB5BAtH;dD_=@1JKoYwylKiSjQXY!jU zE2rzy0@}~2;B6C#xj?=<0mAI>d-lYEEz?THu6x8%hD9o0lg_7`8-+$9m|pu$o{{+{ zd^mZ5P2<0!6^q**!9Nm4^~F3T`!38qFtxHumglpv`8-CToA7=D-IARs_F9DZMa)zr zec>qcs`D8CW(nQNBoLnen z86r}W+_I9whQH26*ZwzHYhPJ$22wwvD{#Nv=?GV4P$CcmoO`oDIS_|lEgYKw7ng0K z$-{5`pO<;j>IsR8FyW*Toj89N$bM`8EBY>*KXb?lTWQkg>+Y^ZS~lj;lR`pbn)D*M z9FarM5jf*L3YVc@qqWZ9sWZrAAmn31vjY*sAK0{vLfqo%r>?1%{~;vQ_Lz-WsYc;O zIaf5^J2kW;_Teh*%V6;9+}4gd0?+<8wvGD#Kl^`9BmO`6E)~oJ0R_ZENp@%;t^+5h zh5ZeO{bjM$Sp^9qXj)W6zO6y4q|fL>1k`V4#3#3*@E%iyE){X5p$XbLAyZs?z) zjVD@olDeh~tLPaPYTi59}0N8?7t|(>vk5v-eQM z4-lW2uq|?vGBktlO=dp{Lmbx>N)Far?&&D;N1acBkD3^9yw!&R_v3lTXRd=l9PmNbtO<4lqu z8v7{1s^D{gHR{*+0gOL_mZ`btKkZiIr%%9I(ylMK!~dH>8F`e-54^lGY&YQ1lTD%x zcagns0W63Y`S-IhT}n_^3H*ja7R7ELwgDo%1TYtoQc1u7N390FCSGPua3gli2t(t~ zxPeRie(uB1Z?31P89~~EjtNGI6bk@)aBz?m9Ok&p39|;o{>H|I*>UW9bb$m!5r>KM zz|NgNei#fSyV+zxI@NXPV?*0S4BAgf!wr@S+vrJV&8@4}CcTD32X+N6lsRbeu0^vv z(OZbx10B8r;q3{xIv)t+#?P23WIH<;|F8k_IXKRJpqc|kCL(CX!f6DTSc+ zh{Xt|wNR)7Q78$py>hm_(5bbW8}N;{KpmIBtU{VsptBH?fY=a;Mo!coFjm@rzi6~B zG8YnN1o$5tR_+`7`LhyyB0b<9J|2aF3(W<@!*cb)A@8FIqu5~T5tKw;ur$P{2$ZM> zsn1+zm%b{o!j^FP@aJJiFn7=r#Fq}>FXKaSDe7dijt>t=DS{pF=yt1b($KS-(MZ4Q zPjTgtlro}*<3Iw9a(q!LG>eHQ902OS_VNtrUcuAKOkJxPfbt2})zs@h z&QX10V`EvP!5|QS2HE081gqES{B4~k_`9(OMkvUT0R9wn?2`Z;J`_Q%7x)&;{;xm8 zz1gm9VDP?`Kij<<*;xjagF5JB)%2^HLD)8h7`LE85TzK7Rl>CrsQ}QYu7i5cLJ&=poY*@?ZD??b@`2h{mXgjC}If&h_O~@QUkXYG`0}+jwR+ycB-om!wB2*;| z?nJ$|Wi(n4e9q2@Kl9mh0E`Ezrbv%2>JZXA0|6}@Dgh_p0AU}xe*8EieoyDnE>E&Y z25Kg#7e~-cip|gNc6WiRR>F!QI|;w%S~YNyfrsUG5ZqIE7Uv;RgYJ&RZXDQ6ZzFsU zPgqxMxnSL~1nY9_EV1Pj**@ln0FU7YSUliRA)8ttMCx&3*=AU_tD)c{Y8?z#9N4{z z^xNPHKHs+SPo|&h&zo+m0VhDVG$K1~WMBLK2Wf0V-76$Cns+dnX7w-Lr^pU|*bfqN z98v0FjzA@)28OaO%}gCzOo?L?fyYGMF~LX%9J34tj+n%niA7y^)P+X;W474Y+@J3@ zk-lECY_aIczUr9Sfu7uJt9zCVPC{QI9S8{JS}^kVwQXP7%m!WXpPq1rqGG|`dQEq? ztwA~jmL|RG+24aA>P*Lf(slLpb zFY|p|%jH_?;hgjS-}kxfXYc*usI{3H-cHsijR|-MD z@`DQ;&g-xu+pWc4VlrjV#v$*d1D5T=3`x)w_F-E|C=2{LM)rD8bcMfSx4pnnKBq1;xR^Q7UY9CbIZ_URa4{lx^!C_yrbg45s^7BUrbt4jK5YBxGFVLP# zI3*S*Uc4n?S+iRWpz=r*b+uqL4-bzGkzFF49yB(y8j{^X@)2%M zA&D(7b0&C7$=f(V;R22^WV2W-@~V-tMwtU zct`+;vU~5WC9&gg&1t*%BY_q&hJ$Y=RT%MWFbBG-_)~vGd?m#BMSvFodZN@~LU|w& zt4Y(i!)r2}Nz$T~YZu{1<}cm|j~Ab(*Kl&bjMBwFG_2!{-h|i`iJZH~9 zC-d0kO-`I&_2gs=w*PQs)ML($Y}_PG9r%TY70l9n8T8Mc?%wdaB_^Kv%o~IPUogNd z6f@dQ0BwH!m6Tq5y@^arOFqxHoHktUd+tnrMnhxKCY(;q^;+o(J-_C2-G_VRS2kvNXX~EnXXk zo`t65?$s>ix43&{VFbc5iQork#E(@_)YemFS07k(&}$?ADsWhBAt|Bg+Ir+$n1DG$ zZUPJ1Sl?_tb86rnE05lsLYxIIVRaz-FF~mgo({##w_t)(Kv>^_C4dH^gv~s6djH@> z1-@&aAY7KGpm9z^`Ee{-lK7nl;U1z&0N#Oaw%@nD`5_Go{KTyXuDov+i$<&@sWj^9 z>R^$sXW?GGZ!ix3?hrJs5cec-+AS1mwNMo;b4mC6?zJr zPJJCsHa0frspz$8Es0nLM7(%GK+l_R@P!v);1G=!TRQ$@D&OYJ$$3#e7d-;lTy7O%=%w`4N^q_oLCEGJ|wTgO^OmPGDGqW`3vW z`#+sc@rTUi@~~CX)Z6v9s7LXVso@h~p=uWc@~mik6gB5d@0G{87FxS^r>>qD05Mfm zg_hAXHH!dw3>sdhIPs?q;QRJpTX5R0P>SDF1{`IT zM`XI|4IO6s6&yP)k0J3GNkS#;am}<9;D~3wz=@*cB)U%XW0LUY{5-GgxsD4Rm(N99 zq6QrxfCt-OV{i5)wOYuf=`)|lotYib89bNhB8`gN&Kc5YMK?v#>P-+$6s%7b#kBS9 zQQM-+Y)SW=>i;=!Wnr7WcL0JK7B4JKPuh{mH$gv5`g)NTFX9zVey@p4P z9y6wPRP%#)N|zlhWZ_KrG)^V?ZT-O%I4&tpPFs5IGbfk0?mulo&!#Z(`2Mwc>GVM0 zXZ`FRH*O76zWj~_4qt2Tg_2IJK}I`3g5QwKR9;n#FIW-PFeD{7 zzJO1Fbjt*brNMoz-m{(IW@Sh|RB1bdq3d~G%LOpQ7;H@-r(YClovN#r04f&@qn&5) zP|L~FWH?z`z;543ZqVgCDw)K3MEXZxGA`AS`T?JF@eRwd2l6Nqj{$Po9d}?9*>~~G zzL7i^yS`oSaf8h{D8BJyn9SE~>*}OVlgHmF9L#Ig1K8 z<~x|lh>boZy+i(^=T>wuEDpEqiHDMNHIb`Gly5B}T`6~>f)Zzgpv#PV`ZLcB^!oFl z8BTZ1$$dlEL`so-)E7Yv#QS5&ticsXM;bL?EnM8tOVi7p)ZFjrIXr5`%9S;V{y~!| zDFx-?H=Yl~qRo1*790%606f13Iy<>HS_*)X`K7 zD>Q`z`L}@7sfT_6?OICoFGfIS@onqh{ zx#myrS{Wu5le`jnH|ABf$e}U9+zK@<3snh$!z^kX$R49Q)u0IGI(i0op`;t2Mh

AYV%&ejf+HF@tK|Zl5(&!igQDcb^(3%9!7Hm zYdhb(r2oSg6gLlkfgc&~!Y=;-cZT&f*p6ZGQl~{EWJfoBnK|QFCY6ac9`(k+Mnd%r z`7r8U8A9-ojAs`|73@2t);zTUG~w&`p6tOO?Z|!Pemwy@63q$vTUDRPq&A*hJ?2Yu6A54nkN8o zKaRM`M~D%~DRrSq4Kr&=3GQTCVO!RkzK2YputxkOOH77zChG*SUAbpNiI}=@;ow&1 z;uZ#5I~p8|Eo~F(FYPxI^bRqyUlM>sQHH7vajSvObm!6TudUA)lL9yfr6vC!)7wPX z?mc#F0x&F}L*+nud2m<%r_1^XK$`yK(VdC13c%iSjaaptJ(l>e64$E~=0=aUzS{BR zySm~>kBtSr*LXDs`Bwl*Qj4{)v2MB?{7QR275SkXQDR|}%4rg#jGAcl{o!<}s8`h` zP77&zH$9A(P*q{N9IkSVlbxckxHX2r5?Zn3AKXA)l+*jG1r zRe~pJtOjoPKIXklty__9%UlIGer`<3Ylk1oC#Y7aWN=eVONm12RaM)qFxS^h`@8@# zL}wxzZP*+ZSyn7yG794hQuw&l#9#qlhyGTd=^6d7Y;S1!aT#L*eiecdW9BFUog|VF zJT^9U17m@Q$gMS9lzu2_(qP{V$*|uWBhzjVOfA7s#B zRT`D~=-)u!)Da1J0T###_#yYCE=C|Js*$PjY+K-6LKw)xrgKgPA00pJa)wmPr zSl?Nw8(gMmA2XfTN$ooIX^#=vc1}Df)lZr@D*&_O=wX*$kE>>Fr{clTQqLAewLbhK zd@>Th3@^k6k?O9N1=fav3kX`-i-TC+PZw~Me=Z{r1ssYJd3I5hcpvKGDN88|#6Twg zj#0UpoYpX?)%yN-83;L}LFH)wn9-wQolxUBUr;_1{wFfynan7d8R~c)a7^Yw1MGe4 zDw5`7@vF%`MdG*9<{h0?8xcFZ4fuP5SHWyI*{1_HQv8B z(F+te@-r8hh5Lla>gY{H5FwZwepx|THc-$(O5We&<(?4_7OL)<2o;VcEK%>D$AhJ* zl;H>olf%w5R<~os8s(p&%q@_gyH+kKM53-!aYLj!@6kEUD7PnJD}I@Q{Y>pW*yKZ= z9DsIZ`Hv1=#%VHRm86Ivj?$+5Oeun};<3oN4CGbpr#UAiX4rSE`N=eTz*2m>u=U!m zJ;X*78JJFTJJSSZJC;DcsX!P6eo{(o-sVE8+H}sJ7(CQ4DlrjVxyHoAU=kz)7{MOx z+q6QVknomlAT~tb%Ig9)wRx4TLyJU+Kq)`h37dv(WqzMV>X!MZ?UX)I(0p4izG_D?zw41Tac{(-q2lXoI;vP1Ug2O;!+bd47K-2{q%vgm{KZq zO9&Uv2LqbfLTesI_}AwF`N!Doxh5;a*r8v=8Vo*Vc|6d=s==0J|r3s*X^~fc{fT?er2p$Q2pDVY%a!-|M!0QA6Dau?a6@i4Lj10 R?9hu$uTB`9`PbkNeg(6umzn?o diff --git a/use-cases/eurac/plots/communication_plot.png b/use-cases/eurac/plots/communication_plot.png new file mode 100644 index 0000000000000000000000000000000000000000..c40459d0212fcbd8ff1756e7d6b46cbe587d7e7b GIT binary patch literal 31555 zcmeFaXIPbMwk5jIttbXeUv^gW)>_~CzBkM{<``qXb?5Xcg*6P@ z7$_9V8mi(kRSIP(8-=pu%!=jsOR(d$OZXvfFL%aX&E|srm27LO{UkA8+8du{9Ogb!urnF0%5HqP87{!g!ASmo7#o z+MGghc}+cbSl#Jqf72DGP1TJbf4f$$h&q$7hA5P*4^D~&(B|zrI_#1P;7Kx_)$84=BU@Yln{=VG6(14P;udrUl$J4yLLU@ z>RoIsv=U!VdG)TKW*Pa1qRa|5^7GYA$`br|XYqgQ*QXct$%lM)OG_IhCnu-#FLl~& zYHBJjA< zUA*Y4P3jwHI_>Oata)?wR(9=)r91s3Ipq`-$X}_dp_0z8vaT#yL7~Vp{8?1QD>8a` za%gBM#Xxgaw_SLeHWtI#K7T6NdH z^|H9#&{a3}3!3XXpKy5LH_Gd6o#bca2JZqkdiqAUl8Yk2#>WddGS}{?OR~w+j%e|e ziS}HW)2e^dbvl4YqdMxum5wsQAL5$9>F%??6{hDV>~u11o9vnf%4EX61qnx2#cQ$p zj8BbpS#*3hnQp!Hq0GV4{7%6m+MbOR3fr@6&xO1tOT8cM)8)^6J8Q#oyWp%}bYT3B z)#+O=ofp~4#NxORbmY^odg0WI_2;V-bPdP)#8e;i1v!jaygpzve`;c&sj?y}UiHP# zRPKY5cL$oXqXP|w-4h#?c+9@%I3LZhX{jj@2y{!FaeegUiCI(LobTOD+=JrNtee$S+Q#94?l{l;^2+MWTlpdI=34g? zmC?!?mxh>fXKktOLj^^)c6ODIxHpcr`6@i(IKjMT$3ew4+eCztW*RK#9&xM5i&|8v z8@h`sWLP(;7TygQEmvsU#KaU!_5av6+N1V4fH$Mr#WIpdJ;ux+!=ifFrS1NU-iEYW znO0-HOw&JoVyAne4cVo(ED<$3?Wq(tAC-QwKJw_TwVx_0)vOvFPfrZwZjicl>(<$( z9A`DG*(zn3xfsud`>Kh0nbj|q4mtd)lcS|N^i4Fpy_`Ege@KzjpDO1wZ87v?7ms$z zqeyyF8D{?3&wBBrY(Gkagclx~$|oA;zuYSPvZpRN;;yf+T9A;**vIW3)Kj}pbV?`a zh^mzar>Va_yKA3e-m&%TAI6=25iWm!T2?#7*g2Rrw|bo{7AhO{u8&V;Pi6F`b*Gh2 zo%-tAKU1zS*cTo4!0gAn!(>#`ts28sBARR3i^ED<^3yY0o*lX(V)U`Aw>sYU>C>m9 zQ(a-b5x&iV@$vB(qa-Z@^8}@k17=^IFsjF?n|FMj%MbZEH`T?;ux3qlh|O9xW8+Ba z1-HALr*=ep&SxfaDh0=%2prH!X}7$;ZS3&@>+f+VgM>I{#Fc}D4&UD{Rvk%IbNKO& zF<8p==3QoYYIR(-lkvjq*Qdw&YERHaF8yHZC=JolO4QT5bm?_w8U9+8YA%ewDJLgi zZb-AtHgHWydMscVF*~&?#-yYs%X#Fb zi@%SL53_)QS+l{{rBWg!MvGJoeXO(YMf@mO)^zCR&{Cr;WsoSi0L!Dv1;=} z_NF}d6Kc_?RIoOxusNs|*lAUH?yh)X=@IQesCVz(t4cA64-g;mwi#NO?z^U$sQ2

YUmG&WW{`Z|#7EkFdQ@zo(M%-Hwu49F%D^kKR%;Dh+ZJwVclcr3MjfA$O3xe2 zZ;2#x6u?^>d-}zD8q3pt=GAernn?z+yN=!rnF&70=|0`F+4l0~hofvO_Gzam%15ZI zU2ER;GlPU{${9h)*7y3+r=C4xuTE%u7oO`eca!XttSf_|LKnX5?Kv!~xPq1TMeAFf zdhFfI(hnbwVsVyZ*7?#WDozxBX>BXWnp*H|`plRnRo8rf`+{V(-AodXMqJQ>mEab# zwwtmX_7u-^o!X&xgCPL#SK&8UUGY8hjl=XPcT?_6{*+9hfZ-`ecie<}>wCE`tw(7& zIXUW?wi?eQ#^U8#Y&$-Oe7MEV#G`%&kw42-?3nhO3n#|&)K<~`fhVA`xVl;BQWxw$+#>Px*)t~WLb7%RnfUcYiaS(PRJI8j`w5wT4aaa|?R@`?ui`v8 zd9ZxMbsK`(@c7r@n5M=?-K$qUwNs^fGZj_sN8jWgh`sUGvZML?`yUVFkJ!ldfR~^D z)TvXx^D*5$J+W`!9=vhuR!L=L<#(11whGI=ca_thv$VT>S>@uzS8}??j!oLtrw0ZG zTDP2<4quW#=cprQ)BLR}wsIeT)Ap@fNfef>oafh{KBVHRsCZjaDtANDS^A2vX9Y1N zb$WsPP4-=1t@at@3_UFI-60++Wb*kqZG51~B>7@lXw&f6z6;Vb=Xk|!+rPDV8m5{Z za{Se8SML~gTw`>ktZ1xErj|8RQewJ5`@yXbjR-yVbJLq}ro8Wrpz>yc<*|po9@_^(gJqqxUb@Oso|)F{3>g6lh2Q+VeSX0QZ6}hx*66M^Rv#M z@u0sH3a;JQ#L5~iYN;7#-y?(!qjq)4vW@*o&Ldi>E@KS$xBPy&eDxONJch>+5fKs& zy|kEoDK;;RV&rZc1)J=~t{(+Q+q<;n`)U%(xx(&M=ekZjl5$DgfWMh9>X{nN-$5g;e!IKI8J49kEn6Z8Ja%Y#!&jn{g)(^`2tF)e*C{&PQ=iubj z$*@_7*e%CUQe9nbG1IY{;4vP-ppy)Y*#h*pxb!Ly1Z=s_`JTRleG#hh?8kod<@$rd z*v4}PU-bVFWiCR}*gww-ekw07;9S_cnmc>x%+;kcW1S&M@#j8Js$XeqC+LdRMpE0z z{CAD^o+%6SoPPlX);mAvT9!ILJ7tIP5__y)BmQyiU8WOiItE!Eo_)gtRlQ2kmzKPE zq~Hok-iYGJq1UcLK{QXU@%lHWX7wqBSUq|24`xXYuBnbMXvmp5be}~^^+aIe@f&y5 z7Qg4ls(jU%@tqyL!Tam`1oS2Yu)El0@`afDS26P;{%3}11@$Z^(TR=e;K42JX9c?_ z)3Wd0zVoIxPfG8pI=3>j_`xlkSi^ECKvcW->878-s#05+PYP-*35b8*S*nXwva~E) zFIyw-QCPx>*C$pN0Ie&^t>1r{;ePsnC-+K8WIX?jE1K8~6Aw!)Yeoav33&Dqcp2wB za&vehDQ=+Ykalo^1J1U(!cg|&D?utf+ z^}aN#nY)ht4d;>vER)76Z?Ctp=&Kp5PVeVk$hVf!&l=6hEK9SjRZp{UM$T$H-(+4X ztr4$T%1&+Z%%`>$`95S9wboO{c#8>SyU#rS=*t>$z)B~^vERa?HZiL6OHk~SCtF6+ z<23gPP=m!sR$&7fkl6u-78IBnT=#0t_B@qaYZGu$rZkJ{lRnJ*{L0Nl1QzCJl;U*K za^@%7uB_Q9afXhrx;jz+gjLhDdxgxS&8tmivCjU#$4JkLat2sEXA^(?IGZ~^y`2{6 zJd#(1T#)M6zdZBpTV%aBm59d%?rBMR^DbBOCeJ_q-qh6bYxZo@w{NGNU0lpR`LzxY z4OMlN1R@wn0#@jT7^R^Lcjbd3AVsW|O7 zk57e3M**5}3ke;u$rI-*@2gEQv2+;t4isng?d@gO*kgAB{lAasy~I-@^ceW;36 ztIBm1AQ6@}I$Xk$X47J@^WbacorfIvCvD%bAzUC|@A>T1$O)R8yHBi9(Se2(6MyG? zWg6f*L3A|r9H*5gU!NA{yfP~f_~}L3QXm@Y*cNF#pB;>{f7ukf9-$(z>EycHkZ`bwZ%B)ey-@mj zx;tWm3E3lFD~Z!mH;{iW0ketp)mOTFhVhcyDU}ua~h)Iyg?^9<&GWOYxI#`D+G(y?b}4 z#fQk(H)>X*XtWy_ZJehw(k>xpec- z;$o`jf_wRQt^)@&^Jjo|?}^8J zcM?CPxlWuCKXfSGxR~avFC*1TG8g4boK{k-b(4!l!8PYBr~Rb1R7GHN=x-3UYMy-- z9=DlCpR=LU0lJbHhe%0H> zXur{7r>Yw}S#}JvvT12)G3}eC z0Uw8BkDE7TRd;r++u_EcvUAxg=86LE8TIsw;$*|%tNlVkLgZK#6clo&Eo9v1r#6`s z)8tShRN3S|DU&(6Oxjq}?x3vHUmC&ZnG)=fhR`%+(V8L^DEvn<@t|!Rko-e!`f9jb_u&8C$ z!1v4qOheDJE6s1-{xNua{h&ofD>7`&-2D`?Cb5^Y8^yhZYC_-BGs-K&g zVA`@J@?yPhaUq9yW)rYNC2z!zPbbYA^2vOE}z2IN^M* z@Xup?|DKPF51UeyCHFx& zD|+$>{o>EXAVASP`uCsz`RiFbTwN>qeBq`YJ1+LAo;`}0t^V|IPeNWxkg#dondI}D zErXR3E~7DbnRs7DDEKyHZF+-c($nTElkTuJma|o~1qE^3pk@1~hXi}a4bm%~c)Nbr z&=A0IgE5FVMCT6nkWT}r~r#}^!8fxZBVk{M2lzGFZYYhIocRnl_Y~)RULjWV>T}ecdAW`^qQ?g$N2gAE$UOE+}z!n zwr`II;M5NDn2w#F?h`;c{I$m9WgYfH8b)T*GC!0ju__Vrca|uYJl+SqDh!MenfgV9 zi=F)#@;yl#8queAirO^mffK4I^x2%|(5IQ8pRJicHy)Bua&lw3O-rbS*auRGUnmWZ zD)6S~VqiXDNCRQ)gwzCMkr~KYlVZ^t31&_)w8e{!UN80=Q zt*im{1_So#9$?ygS{AkSfmiEfZTHjT#t<93TRa!4xx(Deub1Ai39n1Jj*G}V05SHf z{*6YyhdYSc<7DA>g?T0dX!?xx)+EH!EnVK-T^?ak`SSkk>}*xI+^Uf+LntKU=*SddTh|_j_~A_Ah;0_a>Ar)_%gLtFR8p zeTrJvRMjOLZFK+Eja3{hW|NpdKYgMp`wE#h^?1#FrC~BM5Uh@5X^^&l_#IIxP+vjco=r@P?8Zxt> zKBKWKIbEzmt0S7DR~F3;<}V}yo5g~<1mL_n<$mQ?-F^aZJv=-ts$w1@ZPCoK#jL-R zn^VtqaUe5Q8Z25V*5c;>s^NfDLnOjZkmqLb9p7_)MTdE)B4(yq)+Qe7vqID;yvtNs z?9UbU*4JgMx45v7qkQlYzS98Fjyhc81FG@s?93aC&$a^>pMYz$HfYXuy#f|eSu4pv zly&36GoN!J+rXFazzD?K_f+)OzlrA8&w6BL*qCl5JQ#mGfCu?A!@lj~{bO^EgUtz@ zr6J^fpy{Xf7;a~i`muV2-x8VGA#XP~H}yQXspwuAmr*mTy4Q!XLo4Ew^5QU-md!cN z469dHVcYDRHpsH4?vcND8r0k9XWk?-&)W`Odj9dj&Uo|6m$BFyw8*5q>EqzYtH=8r zJAMr14{ic#ug3T2@ijD-)dJMdNO0IAd>Tmx`q-^C zBsyAD^E?x`laJR-NZImW12gkW1f;mb-pc_=?l}Icdr>Amb22Y4FAnohn=!*5Nzg^-jxGn(o z1nh<0gpB42%#o<`uyFvtz7Ez?%9#aB0}VVpf_T!2Kz{S7;SN#|6r4TnwTO2-8NRl>dCHy)FU1&?Vc4Z zVosii8#x$2RjW2pA3#+e#ks|gcUQp^lOinwS({2!2flP+&w014tCyn|)_wT!q0Z~G zSDRh>_5n?N{cegWLSTFZYBvGT-1qg_A)PGySdXcW%_Dj2J>Om&h4v(-4j^-X%l;!* z2Aiu9REYM#mACxes-i!B1RXne>&Eg%jc+4bZT4C368XC4JNN1q5)0TX2 z=gxkBlkZgkOg`ht9Kp@*6HUkZJZJhd@IJ#zWS^6q{>9|7YQ;1TB@kE#3X6(DW*}vK zU%dIw5SuTUNe5wacLy%kowfFP@ZbOj;=}02?KYAr&3_y|e0cF%xIAA&(Y>ukJi6(x zrkwAy$}}T>IjiaEeK*ROFvO}9BG-8T{(WeiC-9UK5TS5Wu50agM?es@-#-hL26ILp zWLlvDU%Vhas~f(}$43~UGz5wF0tq&mQha=0+Pk_;e`g_;VI5iD#Y1K=vPyqfe|2WR z8bIv6-d+T+*i+9WI*WWIq&H0BTQ7Wh@-U_bt3rEpyNH=zEFw5FnUmxgJE)n9Ux(S> z*k5EO5*&5~@Tf2a@`2h|Z}q-$=dQ2Mj-}0IeEg$q*g(o-_HAJvqPPjxp=!80N5^PS zB{lD*uS*!`jtR1^GnOr19!wSm0-o61bYD{E`1O{Zj5ag56V1Y8;5Kd8a6UQd+=rV% z&^Z0NrREIDQXtcT2>iv0BoaiL1`)GwY<7)614>Ytgiz##G@_O>a+&G~WSx5BR5j9^ zZWYpVi`7FA3HpmY%3rcFltLxz)sqaQMl8#LQ$}103JP9s;5hyS6|zxsEzBR36q~qo;nd_yVekXE$2PU<*;JOi&svcCz*GP=+&Fgh zLuk#zJqjfEut>R_d0I_qQLu9xb2WjtQVy2<`1v!uBI<<9u&DDHd_XhH{~ ztVuxCF@O!7T^&$ua63K79K|jy#p3cL$BlHA#UnbBsGphZ^!sO=L9R=97az61Ax(Ip zvAkRb6F@6APBEJp9VPd|GNl8NzvAafT3A>Z_D@5)m1TKZS($QelA#us`^GQ@RTL%0 zzuy9=L5>y8m-kJv>pI0GU?5qZ(M9{>U%hkZ&Qd#td-4%*STJqg91SH=YUK9>-(WQs zdN8Z2t1H)3uneSG6IH<1FTSfb^Yek_Vd7T(bJq3n@!3P~jfKpyE>Wo^;tqhlzbp`} zyMFz83FOL+6NZU;;>cB>XAdfeN`f&7g$E#Va?**)_#l}e>sB?SRSiA8rv}a)0nr|b z5as-J=W}h>nRmj+w|rj5#5ATphq8=^p8nRc z4-ZmI%lPqF`nfLE;ZmsBn0+_So6SwwyC2!P@N9eQj~{HTtgJri*GqEeuNZQJIFvC% z7LY)kE!;9?nj2mYGMim$Qg%#m^XAP{lauUmG64T~#ydJX#y-8+x=@{Jt_IGfi=wa#k!$8jm@vmc{A_RiS*elz;ob0&G-41U6#tcQG*&i z1-~~lGmoE~@zn8^7?6^b+%We*aWa#0_Q8skD^&n4YF=qa08~lZHm) zoNWT-MADIFS8z4*LF4t>;md=K?{BQ?Lk*VRJrUET7jycB>Dm&n+}&Q3vQJG-`b2L8 zAfTl+S9pCwr_5e| z-t-}j*G$yB;utUzq`*cHLjnLjN-Qia%&ogEYhJ;Y5kVH~Qu?*s@2)N-I87(bA_}OS zE3^MPWeoSF<92jDN7Bu4?_t{qz6uEoZ^y4&u{(Tcck7$uYjo3Wai4f5|6oV#^jKfK za+vhzV9|JxH3()oo5HRt_PC0sN%alhzCJO*o0LOqmy!?dy1!KyU!g2LbqoZrFB}G= zeM$MfSZ*iB9d%O8ULZ$XB7*W5N7?9ulg=<5H5{yot_AAtWiaorHo<1H`N?A_- z!tn3k4-tznoG%gP76XG}zDM|p_gQb>lE6p81nwG{m8_obv06jr;gIj-W>H9>ey-fO zRPnvtsmZNUe0%pQLNue+&_`FbcXV)Y|JKg9ZsQyl2%ILf|Hoim@=Q1|ClBJ51k%*0?x zeSQ7U)oqpF3$G2g7e`=M1_>|FTETS2|lK`aeuKP|Z%92_0Hc z$i3j}Qnpk;0?se)Zzx`fS_~$u2FrZozI?Z-lquDOm0CvAgHgID#Vvh0%GBOR*+tQ} z)FqrrJ_6mlQ|1v6bO0pc!YE-5SS&Q|qzCLblBeF0v6qmL_Mq;WBm2y>&kYD2?%-G!AOm2!4!=)i& z@u;!x+*~QW24Ef&Wav%L=gOP%oC9Pr0}R z71o+9g0e`*ssNb~TlMvGoK!GJ>OB=v{-Vxf<|2q{05x!M`GXnmJl4YC%Oa%%b(WAa zB>jM9AhjZoO8E6cv36ew&YMVGf)UqzCgGrtQkIAYgs8)UAzbT7u57b&*G`h62r-!L z*#9N4b!=!zv7FTue&8#^p1Fw@tytA4`=5W_>6%xv2KAL{RJ4|mod~QN!1YY1 z@c`{DP#z$Jy+y&E3OEAcM3Ih;j$!TE8W1gnuO-=_(+Z?|97qJ9nh=s&P{#LxTBRi= z*@VyO%;Z|E0@?75(6BF#(i7?hG@J&2+SgF2cnE}q6n6c2006i;TA9_SWaxOscAK1s zF*Ydrzf4q92Tb0vYJ%z`+;Fn{a93H9_$AQB5a9}id|4BMk_12*5bU%y_#DtID7{=OrTOq@nKOD!*px1pZ1eCI{i z3quDUBp!kvN*X;By|lt)hDiE9*0J3n>=KrvIWSAZ8&h8tsT@ENBlpKQ3^?zSnj8uh zFq9^|3pTYTaFNK_cV6mKmwK?zz+}RAw|^Q{SVw2)OH^}~-<}_4o|E@wNq{FP!5~+M z(99^O$?Opn9d3xl4A;pOtNIi<5Mh9m1lW-&cONe13YPKA<2?D~IS7ItLO5V=f#n6< zB{`^Sv!UAygiV4`eOI$pcr)(D4kl}z^VHM5FiAOde>)7GCgZJb<%k~~1%y+C%H~68 zM6;QsoZ-7QiTd&QD-TFI0ud04N%h7qE-!f%+9aVm@G7hV3Bd=fg3lWn1&G-gKp@ou zrz1Fl$H`*mJM(<2@C7r(b1)sm_cjVHfh+5|R{~i^yLRnrNH)5K{KM(#>ACOBYqoO1 zM~^PS2nskWV%K?+Aj$0qE}jN{A&gDOmmswY@AyqbK*y#p#}XiAHcW2Ouo1b2GnT`P zQ2X|Bo5M&+Xl@R0DyD-7h=L|YyvaN|soh4m#H3!~ z?nnuGp)5238!i@0Ir!k^@VlD@LSt;O%}7>@D33wrlp2ioSA=k(@%rpxn5^pDS{@@_ z$0CX{(?x)UAULxo$uJ3pTu4h`aXBIt_6AHzQ_yS6pg(Y85gQWvtgw)4c5Y78ecImr zcj0CTRdHlt-`U7zQ5Gtt%rxn>`A)%~CwYF{5C;3wUzgk#J0P^061xv(GH_xL)*M96 zz5hcuaOt~`PnJTKQmqDI%mkqxi?Tjz6UAIMi}f1Z9~}Gi-V&a#0{Bb)?WNY}Ec=*Z z;|c_>G~15j(0OBy-Pu4Wjn9wwg-pvWrJSWl861U@>L@%#Q3f7UCK`!)^)(!Hic2fz zw)&y0Fv9@Ae)?`Bm%sEyJX-L6qnj9;D8xm=y7eM$#kEX9s&`EWdq+4XEI!{{)3sis z98G0(`?uZiq(<&hcmO?fZv0PSw(|0F)K6Jij{R}igC|@riI;x7zrCcT#Sq*SwLgXE zS=o;U%Dg0d7!LTN7R}!)BvkR!fR5}KP`Yb(92}A?S&DZV?Cp{ElJo(uM^(ih-Pho= zi8~xK+5>?meJ|0a6-QREm9b$_HnUoHa|aqAd%ns9%lsH-%#(!P;gba zQ!*hlE!N#6mhRrOC(V7vk*W%xMn`{t0`fm>)Rsh7VCM$OyG`({*M+Z7xslgU!J{D8 zK&h&M3{RcPs0*m!?QBjdanv8&0csSFxou++4Ug}!xJ_)?I zb4+r3NuaXl?C&C@89v=~K9!Jyv){bvmXd&5#j^%)acUL3!HS<=e|f1XXVxs5M*jd*%@9#+*0;fLHPN081Uac|m(xGXWlD^xb3*)z3E!-Y4|;J5K-0 zln01=`yaoM;lQN0mh%PK;{RTK|1bUZcI3Nvvboc}@g!G3I(h+%0W`4AJAJ_NTHw20 z045O$5$Rix2o^^Q$>g}(rZ~{0GXqdzKgFFbxE3sK_tN_N+ZR-SP7;`=hdwZ}W)Jx7 zI=X4orbt9UqVY>y`r(Dlxd-qQxw_zZ8LmWm&5D)(3Qz&|WKvJTp3Fx@dqFKkMbWeG zD9Di`5BrekPAK@YRHGnRM9Qkj7xtZBwgN}|WloknLQ=zzcXY(Z2eTCVC^&_--Is8i zyhQ7RD~L*iXNs?(si_HdzG{>vzWH5q)C^?!!2>fy5#tkXQCXkmFkWv7%gV>8f znxQh&B+yJ9=4l7p7ZFBTWnmP`k7Wg2+h!gPEdBHbwr-ND=I3W@l%7dw0GQMQ@sLF5 z?CczLlxhom8gC9bj8fJg zh=Y?#0uo`SFc?yHl>OV37#J5Of@+lb6%^>3U*U2$*0Fdf|Ni|^y~Q`$RQ_vv+)Nd0a%S{T#}eKyU{gpgzkDz$gNSl1?7AkN3*PTc!E=$66#YA_BBY zr#)dWejUs5wLxr_ux-En{=xcE1wWjp{rWjmzt zaj(tV_;P>%8oWZw*K8x*EIo}j`PIbY0B@X+tJmnSdNlwoL7DN12`#i$je?i&g-lTC zD>EMhB~_h(E8O(O-|FOoA_ABqj7$ZbKO)xO4-$-KZEXz?=QF6;7s|`bL?HDw&(Xz2zPmP7!>^<@4ihc1ZGMMA{Qa{3*fP(uFVx5ROy zETLPVWSXNSLEyw+-wCILh_hjK3Gqrw0|KWqZ98b*-XPr}YF|Ak47kH}C7GeDQN|$X(ISi`mH%Tn?f&9&nBM%F!#1 zj+&@yt1|6O0e(N+WFpQhrw!RF*qw5F2n z6eRWl8BdRYdCh*{yEbSB)RA-c-QRYhUgSM^@C+tg9cb18PXfbBb&bj5*R0MCy50<% z9?)?&blVEl8hnqjkj+uDk@E0<90_VeBKS&0QmVdyqS*_@u?j^P2bJtvgk0imY3s%N zD3W*dm#w1?kB@7je!Hl@9DnjM^}De2(}yb(o|C`**++>}%Wpigo`R}WAJQW>_Hh8z z&LU-72wlADM2fQ%Soyopr;OkhkGZ{Y-lLe|{_CU;+ z4&rIbc{#jBp_O1(u$S$zU)clcQ82?%P{T`BgKSE@62+MYE$TB2|1~D$?N0YZ4$&W@ z5yJtvttYHW@F<~MkXj{xEsuYwfIoPp&4PD&86U5tV}}w=f*1P;hcm=NF&}9FVa~SiV#v2;YEmN_%f+wEy}}h+>a+iKlG}?p5j~w9?-gP1AeDO`Zb+A zBI7##O7qp3hwa1AtuAdw+b4zX{+kerkX3^S;a#ZwFg(N%4@)1@;$o=B%siEH)g(R? zTW6S~HYlB59OHDdlJDN7MMN0UNdbn) z?P0$D?*T{M~qc2kM?0x~TItXAFI zBX}c4`Mek%`(XfXtSk-PK=R=qTkA@9knKqsFH8LDH46oq6!_Re(cpx)-k103zzl$i zNLO(n?+X3X`m5ItDHnA-MJ@LGBM?>knYMD^X2~6534^>8!lWKg=STZ0<((16DhQ<= z;l@xEC_NE8Oas8Cz@~W|;&Tn8<|=3d+qD)a_A(jrLsw3rZ1>O3&YpVEz4GX5*gzs4 z>^x)vN~SEe|35}gu{F7nTA(uPqs-SJrW{BuAR0$cF5b3)1VZP}?(V%Hd8jlLUvM)M z@p;4+IuCSHPS5$(I_#x_@=>)yS4x8jxd8{+b% zsv<8Fy9}IWdto!6CeHLkhrt`AN9+PquTCNW(Dca;^)sj|UH1wllqCdgG<5p269h@7 zn#?e^Bk^oNeKtzTaQ&Y^G6Jbk%#p|=?uJWJoFucwmkxt$j0eu^WSjxSKtr!LdM628 ziSF6c@Z?g*9lftzt3vCP&jAQ#umbBtcc&&^K~M%i@)-teAVp$bBA!&-FLC<>ym@1n zVo8>lfxY8;S@1e=+Y&ynH@-$1YcVR2c>bxojX>4rWN*Q)-WJogpZ7VL2)tk z=+|zqgAx)v&@N+b9k}ttaHz3cFs@qV9~f9`RO`=j{lP5&UL_cL(Agab=|9=H)<`6H z6U)3!O06sNg9mGKwCO1CR_&6U4gJ}6KQ@mQHT+X6>aOe+Y_ux$9T7@wT46e%O}N z7(}npPYCJYj!PHjf$mu9yR?@mg6o0QEJX0z(&ekaVV8=4G=_1Z_$Lb+B6$_4Y`E9> z?HXZ}9mOK*^H-Kuk(Kp=Od5f4jwSjRVB~Np8@|GCEusK$wsPFXT68qPWM57=HW&?n z0%8GGu!^6^$$%Y$QtXqCd+ddjP#b9I5lcum*H8^U5FkUtbe0` z&j`y5lt&a94W3+!SJ?VbyrE;qJ|z6xR5FD3w<~wy@01z;3%AY^)dk!_1-Qoc*4ABA zLbf3s2ewNfKmz@wXx<_3rE!-o_glY5J_g{4xSmlA>_KpbcU8L9)r-92b)v|2Sz!C> zWFrOigsUtjF!*>Z7ZVuoA}DxOagj2WgmI4i%=DUk=!`ECOOEqMLl#L*6z)eClCBYC z_oMKCv}EJYXGQ;ZXd8)X70@C<^!J9?ZL{a`10#Kv_+yVtxj@*#yJaR!dkfi7zTkA`A_yH2}fQlsc8-iIuf)*DP z2+3eMBlHx@{}ZkuB5Ohi)p#yT_m9uxj?*s&C7d8P;&;SRn4XM`*W8;BVKy3iHj@a& z4#9-fkL=vUjGz|%0GjG91h?&4i+Pr08#t+AuvR7PI`6@@)!x>&hf1~yq99p%grm2; zawX#*2$RkjSQ$=-SA=qhTKvRDaSNOwV90y00m?}|Q$WvJ_g0nM6*U>C8`_B~3=?SN z*7C`IfwmfS!}F8g8~6^-ePCG3;CC>&PzvB}-CNuNph4QrnRv-K`RM_VGi=)X95M=L zrW>OCXNX$=@)~LG1_vJx*%ih(ziZy!OwbWA;c=b|iv=N`-=a@w@oR1Z1r4G%39gM% zqTj$;z+8e%NH!u!-a`3m@;0Y!_)7Mk`(n+fu|vjS%;|+5;l3T!Qk;x9K~*F`U6rj@4-;n6Ct4sY>2WUMw~d zqceuek>xgJODjPJBeuMMfpC(=5E~cU1%$}kCFO&kE!|B8q^SXu^2Ha=7!Mkmy?68a z^)K*`81-1y8a_t6Q&tEta|1FLu+9nE6fK9?PUKjzoBxJ5+r(`n_kAYIZs-vZ=T;D# zXa;7Z63$h6s(mBspZ6a%7ROZ6T*6E87tle50|7b|)$SWX*Z&GSiwic@bHM{4>kL+k z8f+XVIn~0YyUM~c4lknU(3?$9>FObLEML1L2KboxYY7en2hNH`Af>0LXWv(|Z)6za2=~E*b($TE z&&xoTElMk(wUIXXz`Bu|;mCW6Lp{bI8xnE~27O#`G4voSUmgEd9|3a#wF2G?gvI^D z#(^DJ4Ool7@A(gacwvJJDv~N_d?RhdNC0e1a1a8jMz*6B&|mexlzF~zSQzr{@R*5%w5)gLbMcd9luCrd|w3SBq0NSiHGh=oc`#%ur3|O1t{!l!E4!o6-Xs& zEEJRq!31u+qtNo88wqZC1RntL#X_V`ReC`_5PBRRKz`K(OyHmr)(6>G{pHE6+qD)$ zzBUP+7?zhhlPloGCf00X&IcR_7Baazb@P_8B}|Pxny;Qlf%DVd4y%xQ?;RV2lmSFktbdm4g-Y*n*rgBDDrE@6#0fQlrT8m%pfNU-d#3%H)F3#s~H zYzgU>0hGsrQG#E<58@(@1PT4szkJ!USYUBxD>Nv64wcGA8xTvjHG$ptw-XJY2iT&x z`R_X`-Rj2$GubydE64EN9G1B_pa3)_r^UqZFk*6@Q7vknC=fZ(aO!bbhAIDSH-4sv z4lS`M6gxkXC-sbE44zP6@(eIG%;e!9OdwG&VK%`T8qU^@4wnSMV`X%uOA*}lu2MK6 z;lw*N6o$65lfMxH+mS z_SDaxKatfF5tNFHxu*n@%FR@rCi9C82DJfYhb~~v2wWc;a=q&-_Hx=oQcBo1o#O@; zHCiZ|ep+a)glXiR8l-f$AfB0h1rDjf3br5X)u1x+ zm+2DgLoux*D7_0c2r0F{zr74>?FTGSg@PQL@=JIataT_~@Ai>fsu0d*U?cklA#tOA z|L6L8ZPJki-aXVjTkh_ra%gU>ji!i7%HO|Mj*<|9kE~<^@mhnf38r=Hh;y9~_%xQ2 z%MFvy-{}1Mj2C^2fZCtZZ;-vJNW(S*34~5c37o(1KrY+Y*O$n=mww1%yWFLaH$|Rv zv|Hcy>60P?I&X0YW&?%gzi>2);Ae1%LC;_O4dQslzwkFt_8Sp{4k^YFlxQd^%FU_} zqR{af)(wM7H7ZFfeS(&~ie3Z55Gfzagj8){7>Wy84?7BNWBvZSqOeeoyjDmsFzoMd zsx3)`8ZtxlC$sBaaF!@{xLYqg5;*btzu~JfHZA#koeXMF+7Ec`i6sr(HM9c0$A6VG zB?oRSrjp$lJ*2`~1YbBAJB}6Xp<7hp6dN(2Iq3RgSh=zsH{8+H#f5`fbORU3n;^V6 z3#XFHNU0bxjQj&Oma>n6pt-b_)a~!S62=OpMJjr-NN+bZNBqZ0o}>vFrSGDV67&`U zzSzdzQtSt3KX9d^E;v!xmUjUm?=AdyQ5kVjZ ztC2v-e_-lPFVf(MgL+ip0R0O+KSLx?g&zS$XI<86ax2_t#pv2`Xi+tk8q$*Y&&Sx| zhcc@e#y3Jrz~UTm=!&)Y!84s)z$Y*P)x{hkpC^=0dzs9_BX-jM0L2HTTKP^FnCU{rmTC z+kgH#=)HaZ#q8MbR8`Pgc`(6=nthYSmIW2?<~}c36n>ksw_ZcH?nPqBKwD58@mdmJ zET)d=JUHtl0Z_vNIN)EXqxu}@H)u6Uz?n!2*i*t8rlr9e#Q8w1nMe!uumAa&$XCN7 zBPVP#!KcG}X!0+V96|{B9#C?y(IY(n>uaz&n1dK00vZ#vDO8}Kz0gtK6X7deL5yN( zh1LyRq^I`(>f||t994xv3P(4P-q2HP*D|K~V+X-l174c!(SJY{PQoRCh@z5U4u8RO z5MIY=NLIXvg)n7*|M<@eXC-_hr|QVy|56VQUAh6?yx^TqgYI#?n2)WABew|~l0Aw>8K<~B{O@TrJgH{xCKu&?_;RTt=!h{aTVi_l?Y{!14gDWih(F_B?!Tut zDGYe98XyEzb?-xWVi`eV<(81pK}=1vC2N#I+oRNS`x}%3L$}{IZZfha;$1{I0G~mQ z<55?C#IN`EIXZH1%8Uj&wKCy=vFW3cB?_p&7Pci~$cKY(`nzIys8-lkDt61!N9h ztC_oOkib8@Hve^&)c>ot00{qHscfLbi6IH}8u5D}{#w8f4k$sjeFmD{0x*wk9e%?+ zK14o(n1PnGz*Huj0l=V~m4Rbo<*HR`Far^f;9KV5Gt|#h(eKlm<-|Tu9b<7 z@4=UJnSf`29V{#r0v!y0JwX|5KhB`Xe}kw-4_HaZrIoO%&HSP-JCMFWXyRX;Q}Mbg0q z(movfO8fiLNYk>=#SYiYXU{$-U<WjS{1t|e3Ee511vpXIy0Z2eQ$^%z<7rF(wqPRxx zhbJD1Vdb5a&v{Ru?tuCpGCuEq;zjvoP2zi+lGM`>S7v^2C2|$?US%QEOR7!R>K!Jm zA8B-YIk{%d2e!jXFZZ;gfCYAG#vT`b1Xgx?|&HGzCmwzWY$EQfKWeYfzKNefFp(S z+OEfA1?lIIl#<$*=IZRs&BymExesSV?1jCKxgT*478eF+7V!I}^j*Fr_^GjRUwxm{ zD(}{Uh4`+K$Hm40`b!S9br1dU>Vhrt-R}pZ?5!jB0=4AhcMPswSeSqR=EMXYd&2%L ztD&_FT;kbOaBvZyxz96=Z^Q$EY>iC&1x4RyX#yudy7B zHT>QMLv;=qYOhf;SN~m%Yx&(WM$Z57VeW^fgtojg;Wharf*aVwp)~m)1~!Np0Rn~w z$jQVep1BWhfJ0XX9_8nI#?_Dr(B0nZ!d^?sZo!=~Ym&dfX3?F$X!ql-r1q)Sc;rI0A7^{u?Z#rcts3n z4443fBg>~PUj1V+IUCx55B;On=k;SRI znU3OgnMlaUP>QzBz#Ifh6O)DF335wa#4S#^PqRZHiD5O5Nj56N>5COOB{!0oChE2w zg(gxKfb5HdGloBzpl)Y(qsU4Wk+YmB6wMFo`2PC2b+MZH4^FMY%m0u_{&NB4e|+_q z?8%e&7G(z~{uOMp+ct087WccYs1MZNp4&hYpNw&G^&juz$ZHpbf`}s~!7$%|lnq!A zq!|zHxcV#+wnC5jDN|TTLRd$3t?mgZI4igiGC^0d)cek!ov&9Bg%N}Ss@iB`Xo22O zn0E*5cfy%aNlCDTdo~Vd43L%@^rLb}O;W~(=!JgBZP>gSd;%9RgA(ZEN{Ec36(QNs>;acQK5)D{x z!CmeljB3tPwriV9KNQ)+jTF)7K4L z9vKX~WH3|{sB@@;am0r)pX4tc%*0d+9vNk|8iZYVNO$eS zaU3*qNG=t$iwbB-hRysJ{dMRpajc%7{M?l#1EeXy>Gye(${`(-&I=r;W~~$?BnO|A zaSX$+ugIdLB?ad(suT7I{H3wr7BZydfsrpoi6>mXb6yxAwToY<*U_qidERLY$y7<7JK#C)8i~EZ_t?Ng>s>iQXBFOKz$Lt zV9q+?(*c<7fwzL3JpjV?B`R2|i{P<7oLof6LsA@rFU4WsS8XqyeI#k1hZdwY&IgZy z-qgja)r4k`iYEfFoUkD9>Ny=?WNk$IJ~0&%H>tlh(c7?>2**XFEl`mf@F#EATpg+R zKBxz0)0`sw9O9qlT4t;)Ei~s7iM*$vR6HMtM3Hh5Rs_-jFxC0=oe%!M10r{=WUY9q z9F|C$ZK0cxJ}zQuh*gWmUeF;lKZ?ItY_ZENjCJ>~3%inT9UMRI04jm_PM}_+H3tWG zgs>C30B0n=zMLle8O)9N!GnAMKF@^oFJX0%xh5x&q@kkAogFUuZ{3}HSkHI=$4jUr zO^$P_@%{QqqhyS&lH*E*RE`mHTMNq^7HcgN=BKSx#C^MSx7z5Sx+P1IYR*EMLw>hI zZKSaIZetD;_v6{M{eR#8{;q4g=9>B*-k&*xhL1ir&<=s?UMm{7^Z>P*%T z>yJP6aI~f7k?(VSe9Dm{M=~tGohqmuvF~%9*hLB0QyU40Ru1b~DhOL0- z;_GsF%#$*k`0_%pZW<3^J*rvv z5E??I!#NEJ8EZ8w#h_8>M9kXO#{EO0|9240&c=5*7l(Db52s}sF9jqW^3S%B7nUVk zgiR)xOJ5{=7_|T#X@3v^@jf*@0Aveds&sHnpZk8a32PoUq_M=A47^bv4{`~=UR=mv;MBqo=UtIy z4CB#oyl&cH3cy*vRa^THmM@Gnz*AKmEB~}rAu2_gRI%p^S6+>|)ZWXiyKm9>@hF_X z=;e`xlrjrTETL6%)-)QW-;=2uJ@Uo;d84rxE4ys8m}Hzc>=fSCTL&-lld4SgRb~$jMGUk${EHQ|>}BU3UopJPt<7VZJv-dvi)hw4vLci{Cd`A#hS z0@8*qAIY#xT5%p~6GRW6M5t4bP^`l5o*ipfQwU1ZfLiL=PbCkUO$6bXn$~tXhj_MA zw{HEh+MzSszH#>9r}cj;{^)V8>M6qFEHKY~D>GvzYWI!r`spKTv@p(_W)9N%LUmy| zMZx2TOH>_fe&IE9t7{%p_UVY-RHlIc5ykef(S&z#K#L*pyjOa#uj$K^d}y6s=y;g~ z43>VV-C3o15ghrFBhV3tpshLF)Y1&0l0j?I!uYIcUEJ~(q7^!mb zgWDxOCFz$7H@*|w((v=|JsP_uRhc{cKH2bu@1Z7{szcrlP&MV3*Kwa zvq3Gr^Q;_WIvHc*!F;)*<;4FXEoVkxq?u#7lSTbdAKiLgdDI5OJP*gc&mNe{-m&e{ z*4lat6j!Vrf|)IjO7eZt#aO| z;wnNCv^*bO!;drMNN7Q}F@Wrs{=kk09 z>$)G*$8Ke)iyyskaiML8@d7d}JDEXxwt*?K%rbzD$zo zojX+Ir?(GESG{mX@&Tfg+;i2WvriUfyR8lN?Rv(OvI!_h=WPx7fo7wB^Md2gNwM4~ z{erhyxs6kuhY5;^RMeW-8!B2SbS$0UDyIR-NtGC$utaDrMj{0~(AARb2c{5nlDY?! z{{)u{O{YIgon)Z0J_$LM%BqU zB)gush-|^JAFEtRIETU0Tj02C1o>|u5`8>l0-;d^KteHW%FZauHC&tgE3?VFH;){F z45PZ-M@_!@S}_(Zu65b>Q5af%E=dXb;PuSdt0jvPHRB_b`BprlzhhE!7!CWsF& z#{;-Opu$&Z57l1}EU_v1jcPy^s>C#%8?wP}RopU|hF~Y}G*)d7wss;b= z%D4w;me^6OtGHDvT3Op~IGJdb#-QCzDxHgL!PS z)F4s!!%$agmbSFjo<|k*_3cAsuVz4J496MmrUj z_bX`g2Qc)3E`Ar|Frf#PXXwx>ACEJ;9}HMv;HH>STx;@~oNg%48)5EXbRI8bKy-}9 zuid0+PqcFIvL9yAB4Z!Wzf>9s^%d+?ys<*4a^IoK(v3b+0{dKR>apq$>8l!Q#zWD= z5QnHY;cIP=+1zDZ#f&%(y>cNlE&&MCLK@`hXaYNJ<^QPkV}I2YhdQ36ZV8H@#MD=2 zfnkPZ8yY#y?@=mxJ~xdfk8Gd%C_h~22%VJLTDH!;t<&aAAp{TjX@^-&dhE@!(qgLY zkj2ffn2OJ96Wa=>tL>tQ^ahZ!?{^OQh4eVzpGYZyr=3mlpXgkyr-I z+<$Vzr52N1>av#Zp0w*Sn3j}Ja$^C19z1u$V^T#mOl5-)HF?3ZV_v5XEgNV~MB1bL z4&o6Rjh+jdD3$uLiP+?|HpW_Oy?3(7{6`M+8ah|eI#e`+hbuYbE%_QVHF+|ra|HfN z{+~&`mQG*)hD)7=P!2XyI_WyodrNPzmF3Qf4}vY!>gsi;8i@>|QhlD?IyGLAOJy1j z)oplN?cJlB9!-;Nh(M)z>tjBimp|Op!x;M+4js-^P&4<1ZX)P$&%kiKJ)z=jv5QFz zl$ZX5AnNg0yNuP=-|peIhdfmjn_Ws>=gwcjH2jVN%(TkZP2{(78ma>Y`~G*}X@>FD#2 zD0dPL6PEsR2cAor zwh3+V3r{Uj$4G~8k(|I4P7)mX(w$zS-{jUAd#<7>fs5Ua`m^36%KAEcoV3%Ara?yD z!e_{cnS5s{We7iS%s*;Hh^;;zXyKF;am`^^gqO|(hZnXV#8bT+@B%d#f`cSm=T`T- zn754l0A2G; z;0>y01_dySNJmSVzw%Vs@?}lWRsu22<&D2l4vqJG=8bjLAeTEf2FHcuqc(=;?d9#G zI18KygKrdA_hh1#(FcA-dg=3Dd^NMCZj)&${)T7XT;yHAJso>VIO4{VxSQ20)JtkE zWy>p@8xh*D_Cv#CbHZ(ow>Zp14Do_+E*}ex$e(fxA?U z^w|C9mL9TIY$ZxLtqi8%YF#*itN-zmpS?Ye^8?j^?KZ7mTJ2Q!z?lB;A8HVgp;5E=El8m zQSiGb&i(iib!Qf@A8Z5ADxCxDkbAdN=58af!J|1p!{d-^-$|)C9NsXjpF01@L%sxg z;EnfhxFc*hiMYU}^OtWYjN)@jnur0V5$JMawW1L5G@H6pF?d4X4s{l*09~R4@k46< z3Qw;n7Thi&Kd$7X9wSWO?;*A0w)6haBW?4iSszJ}?o<$t_(noTIUYM6 z{Jw>}xnZXH`Bz8h$1HLsBsEc|)ED%UA32Db!Pu^QtQ}$p;VNn~HIvY|huVTO>5EPA zWb#_G4>1Q?Jt^#CxrNP2X?w_(Qpx?R*a8Q(-WBpE7m4^Q20`1Dsk_0W=%ZX>F;-q- zbK*_rW?^SQo`$TtKV8ifFDsuRLnfOAg5edsABJF%r~ZbWW(gbQ*$?6n0A}@wa#+oR9AD)t2tp(k0>N7zQLEkFin+6KxMSG@ zNW!B;IiJIngED|-?P`Q)gb1%(>AP?ZoMwAM=NJ8j|x)V-K z5MN*j8kGu5vzi?Yrr)JF5~4yf60FLyP5_X4?q``gm|CHwVpHdVv-dEs7SoIxaWWz{ zH7;V+5SC;ii5&>HpG3e#+B71_EJUvE8FWy(&V{ADHm0+8DQW7k6rp5J%Lqhd^VLzV zb?_7%!?^II*SDc=nUq1Y1DD2OX*^e0zWv~k%oN46Z+57HXxKdnU1f*^8{5FuaZb$X z0ST*VGYvklz4(U_zHU~aGm~jJ3uaa}b$*UjLd$jKjtqI^-hLw9L8n3GW@-CC7>0ik zjC|I-;>QHn6v^jF{Riq*l`fiy3Oj@$t*T*vbkiCgA>M}Cs(P;TU7})cD{=3_e j|9l6A_5VMA_k~@*fFU-eQ-0f^CG;8VHzs#f$fy4X*NYN} literal 0 HcmV?d00001 diff --git a/use-cases/eurac/plots/gpu_energy_plot.png b/use-cases/eurac/plots/gpu_energy_plot.png new file mode 100644 index 0000000000000000000000000000000000000000..618ec8c83bba46a2c89a2bd474406bc2249885aa GIT binary patch literal 28368 zcmd?Rg;!S5*Dksdq*Ej$RYasg>24I17LX1D>5%S55l{g^MN*V*q+37)1*G!@k&y0^ zICJ~^?sw03#u@h?xQ^k_7xugN+H1`<=QE%A%=J`5O__v{ju1f*l3O0D@pm z;p4(j1bfGS!#|==6!o8IJ3M^iZtna5xnuss(bnOK?PH6JZV#MY9y{3I;1T7y&UMl1 z$rDExFUborOt|Z7 z&tq`c#R*cES<{2CbsPh^G{;MMG{!0Q%1M(3bn|EOKf9I>|p3i^9ktLT=fIqJ;4Y^~X|GC10 zD{~qCJT*tIz(22O|9|xHRe;^t=ca)Hs@~q-^q7^!#blzu8L}RLK@YKo*iJqPwgNUOHhE8rO5{4;u8`E#m2_gt8%&d9q;JrWNV$}HvHW0 z&Ym6_9o@#;a1>PfsQWSx4=FV@b(T){xy8Z!$leU)+_8Ma3evW=wuSL3mzP71^FJ7u zM@sSDi@646WLUo3JLNa5_2BnAIe0WYI{Is>DVR@4Xiy(h>*=DCZ$Q=gn!8UslsWoV z?A1$RVstZYQNdMJqSgaBJ8iq!+1dJ~4@q{{Ffrj|%$IN6pmcL{%QCJJI)jbfTlQGn zb*%4`dWLC(-{Kf9F77q$kHocgb^O--*W!3hO)@5$Ly3An-l5OufAk~isq50<^xXG% zid(6F^7#>Jmki~YA6tYhFP{Q{vBJVwA0{uHlli3y|DckSQZLuYFuw`Z$?Qm4i(~nRwJ0%>AAU&YuDF* zwGt&JCPoNNf@L@zAG&fGl;Oh?4(bcLE$egX7s-?vNZ}HOo*@jC!#r5%T5C9srr|X~ z-`LU7u}7bVnfdg`@w4MS}a>am_EVRj_wC4GnVS zA~!b)6BE<^?wCV)oB!r-f*ifVleP(w1RmpN7;aCwjb8pb9-^eAlqlkq-c|xjknX-V zxxT(05EDZdM#dZsd&#=&9$2}NvGK}a&_~I z+}BKyMsNT<`G&%kH8mMv&JoX_ca^Tfqb<4;`PVl$Z-*4i*q_=L}YJax2qpNG{$?@OvRT?HHQn>Na@gC-2_jsX~Wq54N zCR7?JDlfl#+Tgi0SLyp#=tq)hhF`R^X@!H=MEjR7Q%i*nO0X2utE&Z*`A3UeRd!=S z&oA7-MS2QNrCm=8>#3ueBy;@UdwF^ND(#kPfx(trPnJF+8!flN6BZWU|NGbS^yD~E z;PJV)!VaVeIS$tMcf#%K(=Fwz@7}y2aBy(AogyWPnI}R%+fO|6J3egc?j|d@9+0bX zTPgB8_R4oduLmP@S{)aCvNB3#Z*Onan|{`9c{pug7v^hWsE~PlVxqUw$&%|{2@Xg6 zVw7twkBA8E>B(PSIeB@^Fi+tg-8$ycug`X4{he3uyv0#hSO3!1X0gzdw!Xcskf9V+ zWIrK>G{S>=;jO^$rrs-kh)+h=?b7x9IWdfn5!Np>A%Uj;XjAnHE9>KO&aO4lijT@M zOy*x-T!Nc?2_c}I+)pett_f=j#%t;9BuY+BUcfX+^O)3TnKXE8SP0k*5+i&LlM~vL#JE9Z9*H2Na%(7_m8^O>%4Z5sk-PLt_7EbXVewo`M|3#HY_-I z+`q1GXlTPVR&GP{^XJc=3}v#f_pj#}Rh@(JHwR+lnORxkPtAtA6q?kTIXJ{zk`HVA z@&(6xcUjiak&l|!G#Iym4iL^L) zcrDG%XC8cu!$K0@zDF6gg@ynA zS8@?CF^k_{Uw-}iwGmdoH&>6>cI@*V8@=)2V$11;g@|tHQwI2c3;1<&OUq#1`hB@$ z2@>#(u$fS|@cjAnb=ZwYm5yl|JHJ`k*$G<0NaVo|dLS%>goXXu>Xr^y$!yNMSK3qh zNF7Xz<;oQthzxD8?adxO{CKw_I2gMvhB+pZhS%czTfv)ZYHhWiTS)CuV~Ut72WlmC z-tLHll+MMy#N5N3r3}476EtjqnK(gshG*oM3Rmqwh^B=dCkNrU*a0JTwBADI=;XxS zpU^T=`iO#pA}}@eB1Do)5SD}S$lBn2rhokks&-vUw;d?~yBYFg3=0p>fM5lt(N*D( zhliIW;dzctJvk&L#k78<{&11gXY0E#8X$_yTM#5uCH|*X?9kH5=4>Y#RnSls$F7ab z&(DufLZaBh&&?V;@Aj>UI8%zQ;Y|u9TG-DZ_yk= za8q3!4OU)zt5L8cb4@=fM>8~4I?YVMI5%e6SzvCpKi=*g&c7PlK~7E{@Ztsem8(~q zzkO4wTZO%L^UfW`a8ky}eQ^SK{6Nm#UI_oRw6s&OZxi_~adBj2QlxwZMx}iA$t~NS z%fhPmW^1$APE=FHUcDm+Q?FTWeHUp20D}67Yw;>RyCaVPNVI*t^G+jO;i>T8h8S%9 z{#-pWPQ3yJu-Rv}oT*ybUtV02pPHEoCZN1}Q9^t!&CMlBdJBO&N1X}S{@mC+Y)`E7 zC#Ac0m?XV$5W2$FinPdXV~zls)}Ig)&eG5%NcaQ#$+V}TqGGMAti-Rcua~p1NcTE` zEr~-zLo*H2x%lAt?;Z`@H?*tk_RjTq71r&AUitmsvFc@Qs{^?SFJfaeOp@QcVc$PI zB!p;5QBqnu&{<(Ob``y&@88|Kynn$F|GV8nZw6fq=f#UM8OA0-!ND*BnoI2L*?j8p zJ;j#F;5^kmPL8U)Ms}8lLSbM8rhW&RUL)y>k?6P0H-C2+%p4pZ<~S4@l&gSIQ~a8l z*k$gXaM?T>&d_t$_eZ z$iu_a{F>XaY!Mz%<8==dA)rl!nxN|^nR zG_&sJzJ8IElxITC%*+cH%j=g}_0qRjx-Pv2l)!^d6{EP@`TFBsLvsi@&EN#eMnA*; z?H)4oEA*Bn582*+lE3SJdR#HMRr08tDIe2t5ns;QItMI{<@S5AyOsxFusC3sQL`r} zCvpJ6Li_u5c1i#n62YM5N*{jj?}Uk9L!Y%YR5-NY0``dh7$;sO^RA0*%wu>HIe;w! zui>q4qx1VCSu*!OBhI<2nt0<;wAY9`#`oVH0I>YDnqOIYKkq&3T2;8Uzwa^K_2vdT z%|+G&98qFYf0u9Utkh201fk(8cP;HHCELH*D{MH)F?k&qcLlHo5qQsL7}P+g3+7#6 z{hYHYJz)8BB_$>OoqBotm(a1|kus3uOt1KLR|y`2Gjs4UagL(pm&>|W`f(kAz)neJ zrASj>pY}rErQkM(?a(^)`2 zIN&?=Fp$KAg@&F5H~SpIT5iJpiS*vyZq87}x8F=tzRSbK6$Nb9ub&$k89|}Q@6A&w zTnGiYH*Va>pZ^Ybwd4$Z`4Tnd&Ac1D01N0IU4G4}N6Kwb)+17%W!WD6_1ibCH9EY2 zhzR`Ozki?Kf17Pk&H?x)kd#sE;?=9Txq5|45W^RT3S;iP6Qas#Gb(IY#j>(})A8d+ z+tz%yW}4hteSQ5Y7~R=j$V;C7{(Zm90b_DJse1EffDw=K%4qpO4gbuqUye(I+&H+n z*0m?T0KtF#O7Hy?8%t?DRFJ>Hq8l`R@)-AqjPKb`An>KKj`%*ChGxWY<#p1fZBG?im0*L69#U9%~I_wsIj$xob9| zl6j6zElCD4IKj^r;`iPezm$_`iF#m zetuT{Srq8L`S9tJC4kvApRQ^iDfd6$D~T!-UEJMy`{{EaNYhogVBcHaFp6Q447!M~ zU+H*>n#U;d5%b2z23^e?6O1bzJw1_(cSSl3Yp__)xrzZG4;qTSyu_YIMNu#@MMLiR zTa>ji*V*0oV`e6m$3++WJDzt4v|;lwNc$NrH7-J){EvA3n4iDqs|Hn0%hC z{pnXbvp*mKW`lC;P$9cftB}kJ&5yS)JcL)LpXHvvYcuFM!e{p zU3=1XXLaj!&AaOfU`^i=cw-rk`p0sd?6oIvogW;V!ch2n5qj1J(c{(I2Dmo1SBOF5QdvSNi-(M+E2-)}XPH;vB3s{(Gmcf>ow=fF} zHq3kr>_&1L8a!F@{v{JKRZq|LDJ&$_|MbMOpxgKR{2mjwiBx!D`1fc=@n7#8>Sb@= zCT3t@K$D;Qj~>xjCrrW~E(YvtRO_L+L^~W8cYeo)qUXd1%jD$5RPuydNJw`k=8e&s zm~mg0W=2nigQ@Ls5f(NsvBxJVle_5(1b|enxH^5w=5{8zy1aY(`_IE>T-;MN_wTN9 z{-nWvQ%>&8&%VB`<0S2D9gm~^VlQ_e^EYqbUgqc5wDE*IccA^;924DEH3X6U{e2mp zgqG1!x_C~#!C}7j&rde0R2|xo{g+?Yh`YymbH*F8AWnrSMbXWU?C$WsuaI<`cLf7L z(^-h-Q)_E6#x-tfkNZB-nR0n{kg!(RPy8HS)X>@d%h%yspz@mQA8i4SPQ>{){>kAI zFK}XJ5JmPO%}dp|wmDJ#3S(MuaO_8TSXXyT*3jeGEB#00c(}N~YS!voAWucv9SB?0 z9@krz&J%_n;t&JQ*p&a>oVE zV_olr$zX;&2akA0D;#0~t+D`$LQyp2g(ys+S~lGShfhq)hy3{Q1I^05z2+v8mzM`; zc)sEEq-JnQpZU>?p=@nD05Y(NGY56OFP;uHB(60bt!fWrN?*QwY0?r^T6&|vv>_UG z%+e?Clal()Z#1l70U8pn|Kz-ufT13TSQ}MOv{j|57>T8E+1Z_*mX414Fmm0s%J2tZ zsSi>!L>`_OUBF}@2^So#sjN(ei3eP#{O0-ta(enm*eBE?Y7kqCY)7t#goaYlA}%)7 z{-^bW)!uunO{1fS??2v!jj=ev3^UgTA@SK+BI+ra{VdaldavB+B}0D`jI(YWlaxkac255WCeTF&@L&;NGz zOg4N>^b^>%<;gmFb8~ar>-TSJ#^vmO?4BE+EnJIY|4$;Bs}ho*&ke*&F))i!ye&qB zDUXv9zyUSEAm0hu$*HMDdwGJBNY=iD@VaZh7jZh{f3$&@mzUSi9Ov*#ToFy%wrWE( zb#{JTfMQU|bS`lah+!Vp&Ld$*HNsz^Mt?4r|y{7@Gi-6u!J{n?GCC{;;lk zscU$+Qlq;!cXRZx=;3#|{DWTSLX!=zYmXr|bai!sL6t}>b@`r-jE%Llww?tROUA(9 zt=BOSHV`56C2W?4i&<93tAc?PkxkREVz4pjnep`1+XcW&RO{gPR^V|EY*Xvo((w== z*Wx%rr)6-z}&1EHaCEjFQs?!U zLFV}C*4+0Npso3wf8Ru@42ZO7I@c0ThUUYc)KhZi&m3)b@}fKdBsbWwf$pkfTiV-4 zLvCI_;DUoR9BmSz>jYV9V^b3rL|Xj@zq&(bn6X$=2?+^4{}UhJR+@hP)bec(Do{{X zwghmEhI8OhB8rOmAm7gbgIyE?X#J%lmJLOdaR3aWqoX~U5@1gRr^$!^0>ZWpNJTVZ z9dGbw29zHD?ww&En{h2SBtSlge?HOmH6pwf>$N<5d>X#9(c_*wge4ZOghq8n!^eKmDdU>3}LOm2`(AabBOR3>r+htOH;h)5&@3X4b9doMKfeO{V6trY}cyq zBgtTaG0~gbVL&|0e0|BXCb;1)3)v^XStC+rH>P{+x3Rmc1PKjNE74Y11inWQNM{8l zC6rNJn5Yp3Qx>uxj{rB}6Vt~Tk(PD|tsJ1!jQ$pu>nt-fGY}UnP=^3ULC)58S^QL1 zW3MEDXh7^^S_9gOF^QF}2l+l85D@Ib!R{88Mk2o+QxjNZF zKz#pOVj%urCMM$3)6+vHh~3oG6uOw&QROnOK9BMHYnR;qERN$Y zAqky8Bj8nxbpbnk4}}_F9m)y{I4HjW?n6XGBo%f%ohd#f&+P!9imiGtqij2*LRs3M zNbC21MOOGl2P9Bp+P-`-15W{|wrE~{KJ5NTaNCa=8unYrq|Jb1(69Fq1x_91qWkj= zX|VA~eU|u%W}rUM3}udunHcHj#zq7(`&O`t?B0hzKio=`@)h$t{KIzt{{2+#PimgH zGE)!sONxt|0M_$+@7n4c7^Fks+dtasCSvR-%7ttRDD6hH&I2U}wE6=pKZsvg1-=wl zRaG@hhhvAvMmCI;@fq0LG}L&|;HspAyS%*I1N_o+NC0PC$r%`)!`D#|2vKHzXNQ{) zQG0X!EFfOM9ACS-LV)%J<}eg!O`EdP($W@Sf6<718!iHj4lX(j0t>`9jp`?lxlHRB zE?v3=2~AL4og{GQ0z9fiuoJsdq?y5Cw#7)FM5x4bn8SJkVWpfLt3-d{LN=Qd&$4&d z$jAk-vu1VaNOB{UDSb8$_$VX-}UQq5Z~5k6HJ>inA7*|-m#DkqOb^_CggwuNf>-=( zJ8}lRXw<=RdU`qvXwZlay8}q6aHiakWN2)1+?u8bCf#K)wq5;1W=s-7~)IuNK{GOZl;~;}}nd|CsQR7P23k?gq%*90vQ__+o>auMdJ&A4>n4IBS z&l_kyWML5mVaE)JQMw@nt(;uForoA38lt3+Z`&0LP4jiy|5UE0`2)~O_Bj{@{Bwn$ z_apV8zQwTyLBBZWy#1pDKh_3HUwNOdp3o!bz zVt5uUtf!Mf$_muBjF(qVy=t1pkKtirYHFcZDT}jrXpyH83&AhwPrADD%&DaTnEje&Iqqf_(TW6<$>(=#Do( zlYt+AmJT3oK#Qypjdb$#DP(137k*|&9`b$x0vu*qX{jEff0j{I$uKJ=MXw=L36Yb( z&TL+><3mri-e6aa-|QYZug_mD>qxS=J1NW!UHjhw$fbanQ~-{D zefy7gNJxm;RAV4oD3_3w#Mnq+2gg{*b5PXDrM<8@Qa#MnqH3_L<}s}gfj!kUSx*eE z)!fFAP(7t>dU2o=EIL>WDY#b(nZ@TGP2>mXeX+-~%!!^4Rz#{Nvd2zjh&HPHTc6Rah^IzG=VkGhFs+ zY}3zlocqq9V9YVkQ?)L$i&6$%T=;OBm)CV_d`1o3J#3b>`3nozU>S%lEw%ICa&hIo zh|Q2|2?P2tBEnHYAp{#4{k+p2$8O4U<^BZ-Sb>5zLenwQa*m5SS0KlYyqh~X^W*&( zF#e1@CP(xe8{JSbiUzKU1uEBE-1&%%uRpRdK4L1lJNMoEn&D>##0*o9Z#!PeM8-6( z{PY~_)RG|$i_-JJI^PRXUf``mNazPINu8ML6c}A#QI2kTMx}~{NT1d+Kw(WLoHV5o zBTa7EuGUkz*7wP2VN?*24q zOfBK2OL5(!F-xLaCDy0oc7!E)=T)`2aB^T^?!A zqwZuSTU+AM3bKHxDDG8ELwHe<9n`~^Uh|$ZUVlS;=dFOUPCpBSND*0vvVQ?-n6B<1 z5+Hqo-}EiMA>DK9HDnkH7QbAe?nA+%>=%adGyHUtJYMfhM9q~LK3XmbXp2_@@;6A( z13)ISysU!lQPumwY38BG6Dw)R_who1HAr73V;BaGsVC>|Sh}}!V z)JWT+86yGpm_xC=sG@>*T?(y|gUgME(foS34^vp7NTm~w|G^}Ixi9~qe>lPcgW>7CfrSPu`ePJNtygls_@Wx2hMY# zb8lrV7^X?{1CBeZA^P!g@&xK$}AY`@Ghsk{2{`ymtX=#?(>)W9nQ z!DB5K`=9EUTT3P-B@r#p3~;+wy^4)pA9LVzT%StuIFXRGG&V4>4)j%0R0OEDy*$F+ zy+hdAS$N8(&V-FHFm~GH3G?yEjFdbOeBCkY?6Y}`-WF3&1>5Y}y^<>@M=re2pQ}K* z|Ml5Hh1O1G3WSVNOvmgU)I%vaE?vINeqBI7VP{bgmh%p%l(2w~g4MyB`n`QGaL?D( z#4fHR$$L$O*Wnw=%#{zR7qDPjnfpq%62$+??LibTYm?)j@u+LN|G{7sVf$4&7Q zI}fl^lQ-s>*M|Y22AfGS5fM3ZoH%YnQYsFeF893a4@~T5LH-q#kPzeL3Hygh+VAGxU4>cHr-q$@u^PKb92`GR#j8y z@Gxz5Hhbu*u)Ei*p}&)5>^iym#T%XZ;e~sc54T*dohQtLB6%*eEIAklK9Z%IkITi) zO~F*p2rToW(BHi^MHLn2Tm*$T)X;yy^zf0o*F_CGsOBq)@ritv`PI_H9#3 z3lU7F61*ySmzl0N+8#$Dy1DteOP%>qoe8|Mk<=-TkZQr62n!BofvhJe7K}nSlS6i46#|PyXt-!(!F8y`Mw&xe3=e`^Mf_nLIDH{VRX|$)kx$Qs> z$!o5AApovf_=j zDj^4^yERF4YokwHy76mPh_0QG(C77?{BZEcsdwm;kN>8 zHPax}nA_PsTX;c`g!kgvbIl8EB7TS1${_3L-XOw!*q?Dl`S_?+9$;sY)d3Fhx_%fZpH0*VQosA++qcF)qU0j$!f z-iKg%THMUcECAfZQn9tyKS--^PWQ8|XdI7S47Z`sWl>QAYU}w`&cg zpk{-47m6x)gW0v~!}9IeC3WwtMPj{(h`8B|T3eM1&G})Fn;~U3G#BL*ui!Fa|6kL9 z|065-zkJ-5jqQBE$M<4sR+%S(YGmf-ty{8yR-nYkTOuW`i;cV824&#bBTZBtPV%oRGUI8KK4cM?nRJnoF^3Z(|2d;bz)DCUa@kB zCh)Djl+a0w{0aV-t}xdwRGuG?y7kljf&(j;t+;OC2}ySfTdFR%J2A7AO70y61rZ^8 zqwuUOW#9l6#y+2bh>{lS4AtKOq3rJrr8pfvx$?8`h6ZfW=X5)w;Drk8`^s&Q#UY1? z>kqyGL8I~E!xa#rR44LHbH;J(U<7@K@(vh5qXSwGb#p*?-{Ot=oPM2;XL-yZ{zuQaK*IRAX^}7!4g{jShFU&6zQ?oeAIq_B__7n4oyXZ}BDZvgb)= z^uI;9&!%=1>bX@9A|NcQ-d~jCwVsSX68*F)HE%Hhc83;FciY#OmK#$|*uXu2_)CS6 z8Z@zR7c(H8ESHCQ0p*2(Ihc1PnuGrk&&#*#KL$9C1LOzW|WIU&TXkg&9;gC*Z zI}P(Guhgl}dEoJbfJJ6uWj!l>B7z5M(=3AlMd1d9r)H>QKQ{t}zf?CnbtnX!Vj+uz zJMuuM`%4?Iyd{311v zS5_iT>cm(CoZ>_LZtd9E;YQqCfeU2I& zpKzr-n%MS3^XROb3(g$W@i_}~V|~J$oHh{eARA0Ambf(CFg73 zS~RkMIM`b6i{|RrpGRSij!Y{ZnJ!#-%y;7k9s<}B7re2fi%Xw}8dfThd-#Nl;edI6 z^=9CK56fkKWa65@b?+Ro(eRNb=VP-}Jy0{|W1qP}V{vy4SIYl{>s73D-i`W5mj4WW z^VP=}oxk;;EDC?QBHi0TJLjw|P`(>_{VvvGvPgblJYPO7Fjl06?RN}ave#FWrFLtMqI?j_Gbzh9o%3OJolo&E5Y~MUOh>K+`hK0lhonwts zwfM4-w$PtJ|5K~4Na#LQPGxmtaTsNKEVJc^dZf9vJ#K>afSI*@7`~JGK4MLyKrwUCX7xJBfVe0*AXR zF;MV^EghW68iVEYAJXND#jBL5ldWs-x9u#go-#x-+-G+|viduzHWSQydMAz8X6kRP zCXR@Te(mb;%l%tDw9%IOvZdH%<{^W$Gx1%cT#MfyHWH;)Rqu1g!zD^fjflR!;Ux=i z+TCU1a5;12Z^?tHKTFt|#PHj&mrwcYPn>v!3QfWVIsN6Woe8<}xHL20a!M>6) zw}h4I-KD6dgu9ICGXF=wAniGKWrrTI2>#W zph&56VebC*i_dLJ(1=9yQ@!AJaZhGW+6K&-iR_%S0d0T7ID)8{UUq}Ye%RpAfc4qZxAH%`|}`E_r|sJ466wBPU>;uwFna44)p%*!^)b{x#gPz*n{)_9?GD zHAHP~`yw|hRo|Q+Ui&jD1YuQq-RK_saeDQ;A)gR`)5uqCS^;MfgEAWuBx_|R)IUY4 z_45w>lyUc7gP)f6#%$N0EUoW~b+#MJYv3)2l++js>m49kl9za}Gfnsh5URyb4Sk@P z>rM(~<1fB8qd|%|uKs4f=HjeTs40&$_8#MHto1!gta(Dgz!8Us=(y)T?L7`R>yCEZ zX1tD{&CV`T0S4+Q%Us+ zPTOYM;%$(C>ZnDV@lPfhVICfPQlKvD@9g605^OeA@)Bd3tRchT7EV-bbZWRSjk+Q- zb`I)^iEpfJ&q{c1^TNsoXO>eTQ*#40j(PgB#`H+yn|CeWONH!>eMi*$gTudm>k8`T ztoewQs)pTA;URR>_8+SEGP8b|<22SdHLF$yG|Y$0k80 zdT{-_P20P-Z{@DWUJ}*Zx_pNB=TBf(NN#MMUJco^8O7(1Nnc+$!A=tOI|oHPSGuDL z)^5Os9k)hlSZ=D_4tX=u>6d4lXUgN9Z>(kU%TU7N96g$0X!6e(Y|XOLt20Zd6(4PH z+|JX4gaOCgvz#_I3QPw?Y%xsocFeqq zi6=BX#@2%}FrN`o(VD!fs<&F>BNL{ywjM~&a^4NmBw$I3l*r_t9HC^x%w3B8>TwPq zeX@2Fw5RnKhL>~dUgEq#^nj7k@z=WMc)cHXih7FC)Uo*9pQmRrU2U({Xi{ZMuX->r z2H)||Q+$u>N7q;Jm5Q+6t{?Gd6bXUk(@{*D)T)0ir$wJF2{K0G~ZoRNFL^3imoYtpgp_2m(YALNy zcL9}}nVlWUqeqWCJUxMAoV#XP@6+@izCh6{-sg#c0E^G>?}{+cTueb1Q$h0xO?e@a zkwnmH;Ou-I1a5)QLPiNCKj=#nmO9?nMrFrCndl8}y;8$A1ECArgavT-44@85kTMn~ ziypZ`_l$_sFC35ygcn=3r-L8?DgYTgoh)d7>XrAnjZikW=y=6??QV`vpxyVbE}({O z%IqiTpt{@w1!p2j5O8~=awDj81E+8SG$0Dl^Hu%iPrB37Bxo73oc$Ib5fv2*LR#o~ zYv+za>xQCYk_7jU_BTQa{R`2eIkeCb+3!n(!cH9c8-9>KpxOk`2DJejxZcUzkTr(t zw;;EbN1IMSjsj9FGias((atZ!ArL7jY+w3&?PK7*>L=__6YsA$K00~lqOis66i)jIT&k_a!4?UYK?H>)gtEoud`W~yk0%LJhTzK!^J-+Kdli8C1)zJJ< zOm5x^!71BxU>i-us3oE3dZYDuw&G7y18WYcrAeBUDEiQ=`3AqoR*inMbM3p4`WciH z15i@DE>tJ2i9H3mKfl+um4xS}0_Xm$R|4|C*XTg)D}gjZVv=zOruZeW#~>7kV#`5Y zJH&^L|AIK8?u!Den0iiY{6;&GnJ#A)go)+tb%8r0}xt%0JfxL*@ z-ZZE#j7jhocR#MV&Ufulo;}kqO73!cGv9SS84~43wAapiU65AJ{sQ>VtB%1)_u)4S1|i+Nc}Uyo$*HVSy(7hd^}rXJ8y_!o3sM;l z=CzM}?YntPC8#`~WKqyQBpEwcpiSCW;@rDa{m77g938jpXHAw$jrCB`Uy>@@Q(qo0 zy3ld!R+hncDoMQEA!}e!f_q+IHinv5KsLE6xKpJBqp!-9$KUSxUhwf=w1UJ?EHkj_t-${D`&1jWh|FT{_q;0(- z-2~3;kS9Y!Y-ce#?!)~rN{W_W5>l|Cf0f(ur2hg}k|5a*ov=bT)-gFfV-&=Q)`sq6 z^X0gwbiN^6CPBF0-=@x%-GbeMmTB@j`Zl;$*~Ru&12^M2E}vlhowgQTqy9<)ny;y_ zRgMWfhL)rv`&I}2&0}?dldJSL^^S*Ja@5?i@W#^sfmSh9!HT8xkITpQ|I#gE>S&A) zw?nk!zePxR5d$LYDD8$ZgueTPhoZE-wei-$@-#j^0h#W=xAYEiDg%ZNSUMbMK?ib$ zwNA#XTFa&`G_?5lUfEMRB%Js{5R@T6Ovi#DNWJ|^vhf+^JH!_a?Wn&tz3orCf3eFkG%}x-u&#r1;Q8b`?cr3w8eLA-O z@?~=LED<#eOqu!Lu*YL=w!_3jZw2|bw|Wu8-9i2SqX(AW3s+s=^~arO7_P%(taepu zXEGjGsXuTgOm1(t*|}GJ`_0f{r`(s%@9`H#FMOOkeGsxYV*f#h5K7qLYS>f2J(j%& zDR(+}I4exOp>5Cj1&KU!ue6$%w8{s>!7}m95&0rkm#F zrF#Fqf0_(o;H(`-UadPw*s8_{sbqi2^;G3akk`mwpZZ}O)kuKUlPuZbG-lzEy^Mcz zHsx?fV07T|R5-m}f%Smae7RfIv82~J`M`vA{f1M|-@eB{vk0Hw`DQaz@ZII02B#4m z)xUN+&|m|rCwSGpKr;gu(iuKzh`*i4hXn+`3{*YAZwLCH9I#=;AcOq!>h)_}S#q<- zcA3TFU;58Owc_TTTLbzhUN!x{&+Cs_t3TZpqtyOxIY@=eYl$W-h}ocCpVRTCzq#GF zBnOQjy%Z1tsCD0ltKKtsJoqAUodFRz^U!3)d3MOuUiyO}q}t&0`ycFCgPa#ZpdE>= zn~s5zaT>($AnAto_%l#d35Q^Z>ij&syq;;X0D|+oedi8@_!mi8qMWUfWdfl8Z|O4H zDq_EIpRVFNSJ%eEN$_5QWlt8xSC5wk^~Qs}Sgb0vsJ3?CbzEYW)7ZB*I^;!C9L_tf z&j+77<4ygmj$hdS5kR28wNT<+4vJ~29AhZDqYc%Jdh!qs9|D8Cm~W40{2Zo*6>~iF zxH9Utm}=s;SeQUBVq~r3Z|PqyVLj7dJX7g1L!G1J;yGXaO?2+K&N#c|#t)_BpIIwi z8#kkb{$F4ksuF)HK6$eIG}d$Xjh;WNN}Non3Kgcxk3-^p3eI)&W*n#|H$rnS56{|3 zIs~CZeUUicm<)d=h2JkNuW1)Bba8+0tx{L#+J7kY+NS{_c`Mj7vp{QmXvw`Ou-+$f z!ployD9{=>;k|x5gh_Q-V}fLaqI+h>Y3AEqVy5F08oh<#QhGG8day;4-F-&NZy`d7 z9@336t1(>jcNI-NUvK(SJC4uPewz0@{N2HMJfS%JG2e=kHr442{-5X{*i#?{F18us zfph_Sf*OGcoSdA5cJrW`8gcc;i8fKC-fkD17++%f|s5iqB4*TH0D>z!~}7p=)r7D%>oX zkCbHzx14YLF5* zCLz%*eR#*i+dEf&`fpdqCo+{xLIQ%vij#%Fg$@pWq!mBKUu~Za%558=5Z@`0+ea?1f6Vn71pv0jL3mc_>3v`hvS!w|u$P0s@cN(AubOwTA9H1@?478&Z z=gyr0T`l|%w1!4Q4*=&XU*u%N8%kgEbsRc#^`~%Mtz`}=wTwLP!v>>XdCI&>@2GrT&-~B zoVd4lUgU7v+Vv~y^H_Y)Jx0T0q_MVH2ckEaKIpZWhT{x?u5W>MzcO}9a4HcW)aJ2@ zKxoUy#}@!yNuyM9vaJ zzWNM#ucj}{e1TEjaZ|Gg5WV*<7f}59QgaTVOiq;cmqOPrLx}{O252(S26;5#yNbd~ zVySfA3fIy&qY~dHz^CBZ@$DN9a_#(~sQ8im#cwyXp_dpvdqmP}8$nPdJBkcXPX0Qe z`33mXP_66`X#B3NtwsCm`M&{$MyBxkMU?+5`)1{7T+758I4S`MYe~uVG#zR`o8*i5 zAW&7XUUZ?Qe<#Z`zjQ^~>uFo2O1vBhH+up8fB5j>A~f`&GBc1sQczOOanxd?@_yQM~?GufGcKK}tDo>GM2R2DmT=-`oe*y7< zC<#o_?9%V8NRbo|s*kvBFJwQiIo}Lt;QnSI*OGQq#cM;?5Q~Zs+x~zP=RIB{913EE zQxEMm_(4dOQ9&}X*q)Xn$WaIWO;=#&xssRh@1c5lvz_ZMHL7F)8PVDkQ4y~j@a4uv z8AJvZEWsHYP#;An+r-3#PgrHS~LEouzE!5<%{NqOdH}>}bAanogqe2Sq95f%H9nCNip&-@Bw>jH+ z|2lfI&vMDX8_D-0m9Vdm_Ge;1p&tsK81`W+Lo)iyi_Qpi9tffd8)QRU&JO69ye=>D zqXIgpUJJG`!-T(N0z^15KJIvW;>#Znk>_&yPy`*)04Jt^qJtSqZ5E`e=r;?=cW4_S z1-#70h6k1pK?S&QrM!#`0z|S4! z864^%;&&vHo{>Sp$QT8)OqAlgdjXEQ=>4S5%&&@0)hh-~^)sL(Cj)(PkZWe`T(bItyj&(sulDnSqre~pnsGBba_yb4gv}7p_?o;;T3fL$H?w{ z!g3tUyZ55w{^G3^7Y9o8olX`65VzIwgQyDh7B_v;e_)G%c-ja^1k4T^VBu5`E(=9; zs6Sb(5m4@lLvLQ(+Tp^26*M->*9QGt{+s=X{tM{wNx$Fm>X0WGezuVDamy1SV-P`` zm~tE(M#~7${+AWdEuxDJo9$0EYNeNN0k>1@y~ne=IzfaivYW#uPw!O%$tQGIyDs)O zeYh1n>Gd2vE((tFfcI*Kmh0)|<>%0{@VJZsPP%|F=%`+R6wG=3&6;U&QzPytTR63|ch7*P>!#)Q`E*8??U9 zg35{0OqklC-DIsOBnt!x^g_NGq;4~UK4oaD=($Dm?@5GpMnwCdxfAi)zGh)zfgq%$ zq>%Wc2M1xYrK@W7-*?Kvg*th#J)xbD&-d`j@^}@mKqwN&aV1{UqYGWWa+H(N$wO$W{i>ughRx52v_YSOr zAM;`u>6Q=sp9i|LgtV z_}7y(*2bi;TSoI;BfdJqJ^D#tld zQN_Q_U_j{!piF9b z^!u%Kt!u68`b*{@Weo_gF%U?A%4R=Pk*^keX;w%JxDCYl%uU3<`rOoXEtBL(mX8yn zW=h;;nsNO8sanih`Y>q=tP%^S$B!SoOcvpsdr+sDiZRNNku^-d7r-KwU&Z^t+0sw8 zZ}{{b;kw`k{dk>~i#z`;LmpV@IM-2k&t`vt$R$2QHd#V5xHJ@TvY;$1E$@}F!Fos& zLI3{u1at=3M5~8fUfiNKCMB)xH-OLd2i@^gOqmZv+Or)wS(6C4pfK~9N^nRBlY6ln z2g^^C%yMoTiTolDzZ@~i=@HYBL#oED2y2ppGn#nWFm~CJsLjx{^_{uDH0iRga6cpa zJ&H8vX27}ShW{l39H*UQ#E%Z4(M12?pj18iyoQkcRP^wal&!l>)WKGuM+{w1b5*TA zsZ62{&RnbXDz5}GCxwKikz!}mcfW|jNp5~_9G*J^FH^^v}$?c%rd zM{MfdlyRTJ=hhXh^pB^N!T<*2>7z4!T|Qitk@DN_2^}h#D0g;V{~5odiDK_shBU4~ zdD8guUQ`11dy!|XaLwkv092y0OV6Sgb4gz=|M?^T2~&J!;!k6`fE<^i|7EJy2P*hO zMrhzTWlH&?-$EW=Z;PPs3;_-o14~~|Ib^BF?2+2>MR$eqAlUDI;vrY*F!OXLMca`I zi1@1M|58kRb;&y~Gc$96%_v-sP)zAjaG8epCyK?e3)|DYxrxwWv9lrf?BmK@otIYH@I!>C9UX(CysmSkk)VyM&_&Zc%e zT2_?^flwzk$SZoR`bIN9UipK1N@>naSvaJmrJ@wsxIW0t|F2!w6v$KHTi5V+j=@ffHdaN zEY~6@X%S~+=dteyGDraUhi^5FghX;d%L3)SKcs3bvM80tWRCl{|M20~2Q?45P(Q>F z$2?nf9?Q`MXeksFmuf$Y#9%7_mtVdr)+{)Sp_i1a1Sy7yO<+hOF$O`Omo|CB`t>^8 z{ayh#z5e-P^#V~WCfsZFB?RZ6m&Z+6=`E6w_&aKUb9Gi$FLvr+J)J!vklPmr*X7{$ z#MR>B0(SXoWL->3_bC642xYj>O18>MARA7PzlqnAdn2g-Qw)ot5Q{7*aNc7oM=dpB z7rdp&R1{Ror+~T7{LdbYlphNoT0Q(E#$=x$(xGui9Y5JUi@c#N>C{$F(AHe~3gQ@Y zL+i=M+EDY#L()>NT)D#dXd`r~jR>=ub?8c>#*LiNlAO>0$}zEbkBk-94s^ZY&7?u!5oOjDXT*KJiB6rj;Dc8uVW3vG*vT7}!Am~JW+sH?AntH=O7KnEYUX9}fuV)rR)=!N;9Xf&!qvam+- zKkB8H|E$RHb`o}(Pd1TJe_ua8ErsQ>G&wo>YO0iIZV75F+YQOdCWGs$1oc~s@8R$A zaQl zRJapgU);Sr`3M$f5{i+UpQdU=xU~erLlh1H1EFg5OJYXcJF7S#7Xi3O&zvNGw$dzM zJ2zpm=V%a0jA@iEi_ZKxz;cO;j zJ(7Ny{9b=f=i*gf?<-^UDblz_DV&B;5pH@i3~k=EYt@H89H7|u@86+k&jX|VY3L+9 zvAO;#2)_vB!tX66VD197`MjVd$4D>$hT;$*B~jnNioDOuO!@{Y0FObG%(2Io(~MS# z57B3yT`xOYRsO@0CCB~^7;@iNN^v#Dy0d{xt;uk6<9hJb0aR8@sP<6*eyFQ`uv)>U z5&JQ7CJNAII1{)KUPb|QMb)2ev(vRsxb+~X2@hdy0&psK_-m{C0~8}n>A=y*#q)Wr znP?OC^d=O;x@_zpY6H0(drYLk3flD8Gj}dhVH#DHM~f8eXyDntxlN;;el;A(s+v4^ zuKoI9p7Aed`1n}A8R)O_YE*!1Pb^`mAmSHO7vjBTVKp&=D4{zicDiu{#jZt#1HnP(Npg0N# zK zUmP`^iF5XF3}lJ9ot40j?H+q_Y%B z;%x_OJ@sA93-z^wmY9Ra5!l+}-!QoTi4kG7mXt|hrM60Mhb=IuL%KVz@&SJ!f z{wIap>$^d2Np%U>LUA zpm;lHfZ2*O*Fclu=GB`JSHSSrJMHDx&(O0|wqRERSs6X&iCW!o=+L1iV>zC)#X!{- zooC1ZC3pu9zSJ#}Us(7@7`&`F|58Y$PdZ}5Eoq8D@%gXidDf(_{BbXw9jaw<{A07= z3ZN)d8zO*WspKUu9#rXoh%tTO4En_rO%ejRk&17TU{xQqtCDY95I7d-zK)7J6aTJ?6znaV@B5Ey2##WJpL*FGKdhT5QOTdEXcKNRscqDIL$Z=gZ;@!Pn;EV!5{`fkt+Ka{lWru>wawjJ{&QI(?Okzz{v zPj;GG=H1%qmP;UB2X7mlCw>MRumgJCQvYqC4zAh1G#Vqc0NU3?x8H@8sQo0)0hg(17_Ba`+N z=SW-w3^16`H#Xkg!-UC^rVk%I%0wyj5k^#C03oM_r*(9z}ARNqYPs1CFE9IfbEZVDCGg=z; zo^ACHIHzPK^NM&IM%WjBE0(3uWq;drQY~P&_3;c-FpIjfZ%T9QR8U0fMYV5FiXxRP zWx7ZHpY8hL%lmY+72?HdaL^j^CsRRuKV{LIXBxK5K(@PEi-Ahz0f^(AoSfX{Qr;(6 zm(&yQP>_fucm6J}Fh$F;)4D(VYGShqVP`T6SDa!=_E}L=Z0dsc4T(oA0@Q#Hx<=71 zTk!!)e=^u%_<s4~2J-E>E&X+r_8U?Zb3fzL4+H5a1Y$;3Y0{8AScHI80Tb;}J`#gOel@7Fxb2?9_+9o9 z^WLhu`sJ&8T2Egrs3pV^^GP&&>x%6GE9{#y1PEhbD#L*K@dcVGM18l_Ru}ORE~tQ@ zRwC*CXhH!#Q`<@~`<}RZuAHaESL>CZ18+I;ct`tPL)Dq}J0=N9BRUPyWWt2h=Sp8` z2{d=NA{-XYi%r6vcq(oTVd?Vcu3v&l`{T`6Wp?u%A;4$OoV~gJ0Rj8Y&XkV}EFYv+ z#4d#Z%Rh5wNclH{d+#?ruPtFDp-o|Z$B2jsvE9K?G4H@PMe|)+$Op!S548_7d4;i@ zupYESjqQVFAid>n2E)T^s$V@Q%S%zu%W^V1I(sDJInm`Y*=9+~)pYj`SZ=t6=lGmj zZso4hdH?#4=1o5D-2~O-;Hr$JtBos`N{wwd_61rkw0n??Rj ze|cRiIx+vXidDLE_{fpC?sX}3(1^XiaWB={pN`w(qsHsF#>pOCJl}4)tmSsL9B_cr zL@Ks&4F*^_q~|1s^Zr^O3tpd!P_AN?BW{z(YOB3W3`N*66|35$30RKns#_Ao~QV{nD ztZ)QrBYJ8*r3?uC?z=1FzxDCa0%Bax2^}@cQ}gMlo;i^Wkx8gHbiK5+1#sla z^?hD8OXK41|Fu5RGU?&n6H}W1Ie3kS=bd-!di`L_`%%ul+}vPL()N2Z)qZAQ^wL%q zU7l|+cg!Mh_Y&LhPpq~|Ogy8vO!a6Iic0?&)Aofrj^6Qw(?(kbzj8Xzv-`G#xo+78 zKP4iBaBxcO_{$4#?eJxH<`zEOZScdiEuJu4FbUJUVPhMW+VMmapn$8rY?|ccLu+0P zcXK-lq|92=R0DE1vZax*wxOwk)5_THWx$;~bNk`imU8x-CMb1mxlHAdbzuH5nwJm` zx9;3o2HM_b8P%*|!wTmMWC)AS35_E2iiV#I$Ld>Ly?tiRj3uuM%4ck#G<1ff5mo#c zP3SjoWTMfUiX22UNBcjoyOOT-YFAJbVBOmi1Nj1wCoRa-Com$h2KS&YYqp{S^kZO3 z(6ghX0tt~xoaQku!=GH!&51|n)<&J9V3OhsCucQouQA3s^`QpQ*<}ubd_#`iiV8w~ z)X7|>Iw=a>TAr)8XfvL#;8@DYSE63Y*=00D)e&M|CSG2QE{Wz9a6!v=Ey6?QW`@fO zY}Hmowbij8N>3&V0(OW3?id)j;1}ZT>=Gk^?%jFzkJy66 zeFGaT#S;m0N7^oKRy)nlNJ|rn>*Dy<)R_0qZ#{MLn@*`x@oWD)9Y-oB`%4^HUy+ zR08q;{&hL-)PoXy`qrhlAd}4g!ncMrR(48yHb25bPn3KuRUJ;a-n_gESRZ3HS%m$i0!bBS0oD z2M4R~m+0xU?Bln{Lcpyn|6g0fjNCj{!Yh}!o?RpQYqgUK}YCcc7U zZ9y9>J@>{g&#uAh3tEux-?Dg)-{#(yu3}3%LBGQ{1SXoRQHLzIk`< zaqMY%Bg1W2I;mfIA&onReQ&AT(RQpg{qhXu@U6Rdqd92Bk@Ce|_y5X>f*a#0hpMU?{u=qZgqBoIH5guyg%ixjC20KuMUq z5<$WNseD1Z#;+Y!1?b%8I`}pSG(WEjtoB5z1#-i#+u*wWbd6Q8eUq;8q+V=u+pdvUnkd?nfuBT zG3w!~lb@eY8UM_@On3ZVuYPgpdMgF=F=C&UXxXV#Zf1AloRUW}?UlF?mnuBQ6nR?$ zqep|Pa)F$Vl$IiHAhyF{O|M^=Pc@`WZcxdC^X1!^?!_=5CU+E`Chz+7w|f|7zJD8^ zu^r>~&8ferrh4aNacide_27GND z&^0vg-#=%-lSvpxU)ndtlu3fX(9rW=yrqaS>(HUY`M>Vjbaz)R3%f>ZU33d<0R|W9 z;e*&ss9B~Ej{O;=r+++_a9us;_0wAn#VpvOKYPZEM#@CXe$?Ty9+OzC&<}d;+U;HU zzKEh6FOPdiZ9?BvP3qLX5VW2^<4_YZwsfFH8Faw0B-ymN}>)Z?3T z2J-G97I;x;Mv{#<@2=prSmJj|8rV()Ow2MNUxMgf89OWz()r1ySFHBsL_HN=lClxt zHvgoGmu?R|g3CW=|67=wd6gr?6c2zM1#m)Lz`a5@Ql$#}+1UaoF3wBr(u6EWcyj0p z?c)wq_`rly@Gcj4xrF#F^5j&PzMVaLHLRO2=P7*+CnqNw^138diOAT$)W*On9RIqU z@w9w+NO9Aaw{_v3N*``%TAHO4-H?D8W!c%;DoU2V2x+Kr@G)*eu#GT+Z9&6gQWV&U>AvM*SCc#qVAK literal 0 HcmV?d00001 diff --git a/use-cases/eurac/trainer.py b/use-cases/eurac/trainer.py index 628ac66d..770c8259 100644 --- a/use-cases/eurac/trainer.py +++ b/use-cases/eurac/trainer.py @@ -28,6 +28,7 @@ from itwinai.torch.trainer import TorchTrainer from itwinai.torch.type import Metric from itwinai.torch.profiling.profiler import profile_torch_trainer +from itwinai.torch.monitoring.monitoring import measure_gpu_utilization class RNNDistributedTrainer(TorchTrainer): @@ -92,7 +93,8 @@ def __init__( self.save_parameters(**self.locals2params(locals())) @suppress_workers_print - @profile_torch_trainer + # @profile_torch_trainer + # @measure_gpu_utilization def execute( self, train_dataset: Dataset, @@ -146,6 +148,8 @@ def set_epoch(self, epoch: int): self.train_loader.sampler.set_epoch(epoch) self.val_loader.sampler.set_epoch(epoch) + @profile_torch_trainer + @measure_gpu_utilization def train(self): """Override version of hython to support distributed strategy.""" # Tracking epoch times for scaling test