From 3c5eb6d29529be5c8011e8fe8fa90bc0c3608c69 Mon Sep 17 00:00:00 2001 From: JSJ Date: Mon, 9 Feb 2026 12:24:52 +0000 Subject: [PATCH 01/24] Restore 3rd branch of sleepvit --- .../sleep_convit/sleep_convit.py | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/onnx4deeploy/models/pytorch_models/sleep_convit/sleep_convit.py b/onnx4deeploy/models/pytorch_models/sleep_convit/sleep_convit.py index 0261cb3..9490c91 100644 --- a/onnx4deeploy/models/pytorch_models/sleep_convit/sleep_convit.py +++ b/onnx4deeploy/models/pytorch_models/sleep_convit/sleep_convit.py @@ -177,9 +177,32 @@ def __init__( ), nn.ReLU(inplace=False), ) - - # Branch 2: Kernel size 100 (coarse-grained features) + + # Branch 2: Kernel size 200 (middle-grained features) self.branch2 = nn.Sequential( + nn.Conv2d( + in_channels=in_channels, + out_channels=branch_out_channels, + kernel_size=(kernel_sizes[1],1), + stride=(stride, 1), + padding=(kernel_sizes[1] // 2, 0), + bias=False + ), + nn.ReLU(inplace=True), + nn.MaxPool2d(kernel_size=(pool_kernel, 1), stride=(pool_kernel, 1)), + nn.Conv2d( + in_channels=branch_out_channels, + out_channels=branch_out_channels, + kernel_size=(3,1), + stride=(2, 1), + padding=(1,0), + bias=False + ), + nn.ReLU(inplace=True) + ) + + # Branch 3: Kernel size 100 (coarse-grained features) + self.branch3 = nn.Sequential( nn.Conv2d( in_channels=in_channels, out_channels=branch_out_channels, @@ -214,9 +237,11 @@ def forward(self, x): """ x1 = self.branch1(x) x2 = self.branch2(x) + x3 = self.branch3(x) # Single concatenation (Deeploy compatible) - x = torch.cat((x1, x2), dim=1) - return x + x12 = torch.cat((x1, x2), dim=1) + x123 = torch.cat((x12, x3), dim=1) + return x123 class Encoder(nn.Module): From 477dad618ee4c19fa1ce3ce3d613817c77b91205 Mon Sep 17 00:00:00 2001 From: JSJ Date: Mon, 9 Feb 2026 12:37:17 +0000 Subject: [PATCH 02/24] Swap H & W in branch2 --- .../pytorch_models/sleep_convit/sleep_convit.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/onnx4deeploy/models/pytorch_models/sleep_convit/sleep_convit.py b/onnx4deeploy/models/pytorch_models/sleep_convit/sleep_convit.py index 9490c91..2219a87 100644 --- a/onnx4deeploy/models/pytorch_models/sleep_convit/sleep_convit.py +++ b/onnx4deeploy/models/pytorch_models/sleep_convit/sleep_convit.py @@ -183,19 +183,19 @@ def __init__( nn.Conv2d( in_channels=in_channels, out_channels=branch_out_channels, - kernel_size=(kernel_sizes[1],1), - stride=(stride, 1), - padding=(kernel_sizes[1] // 2, 0), + kernel_size=(1, kernel_sizes[1]), + stride=(1, stride), + padding=(0, kernel_sizes[1] // 2), bias=False ), nn.ReLU(inplace=True), - nn.MaxPool2d(kernel_size=(pool_kernel, 1), stride=(pool_kernel, 1)), + nn.MaxPool2d(kernel_size=(1, pool_kernel), stride=(1, pool_kernel)), nn.Conv2d( in_channels=branch_out_channels, out_channels=branch_out_channels, - kernel_size=(3,1), - stride=(2, 1), - padding=(1,0), + kernel_size=(1, 3), + stride=(1, 2), + padding=(0, 1), bias=False ), nn.ReLU(inplace=True) From 10508cfcb883b9f744655a40cdabbea3ec0f7cd9 Mon Sep 17 00:00:00 2001 From: JSJ Date: Mon, 9 Feb 2026 14:08:46 +0000 Subject: [PATCH 03/24] Sleep model fixes --- .../pytorch_models/sleep_convit/sleep_convit.py | 11 ++++++----- onnx4deeploy/models/sleep_convit_exporter.py | 10 ++++++++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/onnx4deeploy/models/pytorch_models/sleep_convit/sleep_convit.py b/onnx4deeploy/models/pytorch_models/sleep_convit/sleep_convit.py index 2219a87..436df4d 100644 --- a/onnx4deeploy/models/pytorch_models/sleep_convit/sleep_convit.py +++ b/onnx4deeploy/models/pytorch_models/sleep_convit/sleep_convit.py @@ -149,11 +149,11 @@ class ConvStem(nn.Module): """ def __init__( - self, in_channels=1, out_channels=48, kernel_sizes=(25, 100), stride=4, pool_kernel=4 + self, in_channels=1, out_channels=48, kernel_sizes=(25, 200, 100), stride=4, pool_kernel=4 ): super().__init__() # Divide the total output channels equally across the 2 branches - branch_out_channels = out_channels // 2 + branch_out_channels = out_channels // 3 # Branch 1: Kernel size 25 (fine-grained features) self.branch1 = nn.Sequential( @@ -206,9 +206,9 @@ def __init__( nn.Conv2d( in_channels=in_channels, out_channels=branch_out_channels, - kernel_size=(1, kernel_sizes[1]), + kernel_size=(1, kernel_sizes[2]), stride=(1, stride), - padding=(0, kernel_sizes[1] // 2), + padding=(0, kernel_sizes[2] // 2), bias=False, ), nn.ReLU(inplace=False), @@ -241,6 +241,7 @@ def forward(self, x): # Single concatenation (Deeploy compatible) x12 = torch.cat((x1, x2), dim=1) x123 = torch.cat((x12, x3), dim=1) + print(F"ConvStem output shape: {x123.shape}") # Debug print to verify output shape return x123 @@ -352,7 +353,7 @@ def __init__(self, config: dict): self.conv_stem = ConvStem( in_channels=1, out_channels=self.model_dim, - kernel_sizes=(25, 100), # 2 branches: fine-grained (25) and coarse-grained (100) + kernel_sizes=(25, 200, 100), # 2 branches: fine-grained (25) and coarse-grained (100) stride=4, ) diff --git a/onnx4deeploy/models/sleep_convit_exporter.py b/onnx4deeploy/models/sleep_convit_exporter.py index 5960eee..8168854 100644 --- a/onnx4deeploy/models/sleep_convit_exporter.py +++ b/onnx4deeploy/models/sleep_convit_exporter.py @@ -67,17 +67,23 @@ def load_config(self) -> Dict[str, Any]: "input_channels": 1, "input_length": 3000, # Time-series sequence length "model_dim": 48, - "num_heads": 8, + "num_heads": 6, "num_patches": 94, # Computed from ConvStem output "seq_len": 95, # num_patches + 1 (CLS token) "attention_dropout": 0.0, # No dropout for inference - "mlp_head_hidden_dim": 192, + "mlp_head_hidden_dim": 48, "encoder_ff_dropout": 0.0, # No dropout for inference "num_classes": 5, # Sleep stages: Wake, N1, N2, N3, REM "opset_version": 17, # Match CCT opset version for compatibility # Training configuration "training_strategy": "full", # Options: "full", "last_layer", "custom" "custom_trainable_params": [], + # ZO training configuration + "zo": { + "epsilon": 0.1, + "seed": 42, + "noise_type": "uniform", + } } self.model_config = config From e64af9ea9774c89305205d9498737d2d70ae5dcc Mon Sep 17 00:00:00 2001 From: JSJ Date: Mon, 9 Feb 2026 14:21:37 +0000 Subject: [PATCH 04/24] Added ZO training as Onnx4Deeploy export mode --- Onnx4Deeploy.py | 13 +- onnx4deeploy/core/base_exporter.py | 120 +++++++++++ onnx4deeploy/io/__init__.py | 2 +- onnx4deeploy/io/config_loader.py | 2 +- onnx4deeploy/transform/zo_transform.py | 278 +++++++++++++++++++++++++ 5 files changed, 409 insertions(+), 6 deletions(-) create mode 100644 onnx4deeploy/transform/zo_transform.py diff --git a/Onnx4Deeploy.py b/Onnx4Deeploy.py index 9b1b9be..8a00d78 100644 --- a/Onnx4Deeploy.py +++ b/Onnx4Deeploy.py @@ -295,9 +295,12 @@ def generate_model(model_name: str, mode: str, output_path: Optional[str] = None elif mode == "train": onnx_file = exporter.export_training() mode_desc = "Training mode" + elif mode == "zo-train": + onnx_file = exporter.export_zo_training() + mode_desc = "Zeroth-order Training mode" else: print(f"โŒ Unknown mode: {mode}") - print(" Available modes: infer, train") + print(" Available modes: infer, train, zo-train") sys.exit(1) print(f"\n{'='*70}") @@ -311,7 +314,9 @@ def generate_model(model_name: str, mode: str, output_path: Optional[str] = None files_to_check = ["network.onnx", "inputs.npz", "outputs.npz"] if mode == "train": files_to_check.extend(["network_train.onnx", "optimizer_model.onnx"]) - + elif mode == "zo-train": + files_to_check.append("network_zo.onnx") + for file in files_to_check: file_path = output_dir / file if file_path.exists(): @@ -406,9 +411,9 @@ def main(): "-mode", "--mode", type=str, - choices=["infer", "train"], + choices=["infer", "train", "zo-train"], default="infer", - help="Model export mode: infer (inference) or train (training) [default: infer]", + help="Model export mode: infer (inference), train (BP training), or zo-train (zeroth-order training) [default: infer]", ) # Output path diff --git a/onnx4deeploy/core/base_exporter.py b/onnx4deeploy/core/base_exporter.py index 3e14cb0..6264ac4 100644 --- a/onnx4deeploy/core/base_exporter.py +++ b/onnx4deeploy/core/base_exporter.py @@ -26,6 +26,7 @@ from onnxruntime.training import artifacts from .onnx_utils import print_model_info, randomize_onnx_initializers +from onnx4deeploy.transform.zo_transform import generate_zo_graph class ExportMode(Enum): @@ -33,6 +34,7 @@ class ExportMode(Enum): TRAINING = "train" INFERENCE = "infer" + ZO_TRAINING = "zo-train" class BaseONNXExporter(ABC): @@ -249,6 +251,14 @@ def setup_paths(self, mode: ExportMode) -> Dict[str, str]: "network_pre_sgd": os.path.join(output_dir, "network_pre_sgd.onnx"), } ) + + if mode == ExportMode.ZO_TRAINING: + paths.update( + { + "network_infer": os.path.join(output_dir, "network_infer.onnx"), + "network_zo_train": os.path.join(output_dir, "network_zo_train.onnx"), + } + ) return paths @@ -484,6 +494,114 @@ def export_training(self, save_path: Optional[str] = None) -> str: print(f"{'='*60}\n") return self.paths["network"] + + def export_zo_training(self, save_path: Optional[str] = None) -> str: + """ + Export model in zeroth-order training mode. + + Args: + save_path: Optional custom save path""" + if save_path: + self.save_path = save_path + + # Load configuration + self.config = self.load_config() + self.paths = self.setup_paths(ExportMode.ZO_TRAINING) + + print(f"\n{'='*60}") + print(f"๐Ÿš€ Exporting {self.get_model_name()} to ONNX (Zeroth-Order Training Mode)") + print(f"{'='*60}\n") + + # Create PyTorch model + print("๐Ÿ“ฆ Creating PyTorch model...") + model = self.create_model() + model.eval() # Zeroth-Order Training mode + + # Store model for test data generation + self._model = model + + # Generate input + input_shape = self.get_input_shape() + input_tensor = torch.randn(*input_shape, dtype=torch.float32) + print(f" Input shape: {input_shape}") + + # Export to ONNX + print("\n๐Ÿ“ค Exporting to ONNX...") + opset_version = self.config.get("opset_version", 12) + onnx_model = self._export_to_onnx(model, input_tensor, opset_version) + + # Randomize initializers for testing + onnx_model = randomize_onnx_initializers(onnx_model) + + # Save inference model + onnx.save(onnx_model, self.paths["network_infer"]) + print(f"โœ… Inference ONNX saved: {self.paths['network_infer']}") + + # Run inference optimizations + print("\n๐Ÿ”ง Running inference optimizations...") + self.run_inference_optimization(self.paths["network_infer"], self.paths["network_infer"]) + + # Reload optimized model + onnx_model = onnx.load(self.paths["network_infer"]) + print_model_info(self.paths["network_infer"]) + + # Get trainable parameters + all_param_names = [init.name for init in onnx_model.graph.initializer] + requires_grad = self.get_trainable_params(all_param_names) + frozen_params = [name for name in all_param_names if name not in requires_grad] + + print(f"\n๐Ÿ”น Trainable parameters: {len(requires_grad)}") + print(f"๐Ÿ”น Frozen parameters: {len(frozen_params)}") + + # Transform model for zeroth-order training (e.g., add noise nodes, modify outputs) + print("\n๐Ÿ”ง Transforming model for zeroth-order training...") + generate_zo_graph( + inference_onnx=self.paths["network_infer"], + output_onnx=self.paths["network_zo_train"], + zo_config=self.config["zo"], + ) + + # # Load training model and add gradient outputs + # onnx_model = onnx.load(self.paths["network_train"]) + # graph = onnx_model.graph + # grad_tensor_names = [name + "_grad" for name in requires_grad] + + # for grad_name in grad_tensor_names: + # if not any(output.name == grad_name for output in graph.output): + # grad_output = helper.make_tensor_value_info(grad_name, onnx.TensorProto.FLOAT, None) + # graph.output.append(grad_output) + + # # Save with gradient outputs + # onnx.save(onnx_model, self.paths["network_train_optim"]) + # onnx.save(onnx_model, self.paths["network_train"]) + + # Run shape inference for training model (handles Microsoft custom ops) + print("\n๐Ÿ” Running shape inference...") + from ..optimization.shape_optimizer import infer_shapes_with_custom_ops + infer_shapes_with_custom_ops( + self.paths["network_zo_train"] + ) + + # # Run training-specific optimizations + # print("\n๐Ÿ”ง Running training optimizations...") + # self.run_training_optimization(self.paths["network_train_optim"], self.paths["network"]) + + # # Save pre-SGD model + # shutil.copy(self.paths["network"], self.paths["network_pre_sgd"]) + # print(f"โœ… Pre-SGD model saved: {self.paths['network_pre_sgd']}") + + # Create test input/output + print("\n๐Ÿงช Creating test input/output...") + self._create_test_data() + + # # Add optimizer (SGD) nodes + # print("\nโž• Adding SGD optimizer nodes...") + # self._add_optimizer_nodes() + + print(f"\n{'='*60}") + print("โœ… Export Complete!") + print(f" Final model: {self.paths['network']}") + print(f"{'='*60}\n") def _create_test_data(self): """ @@ -546,5 +664,7 @@ def export(self, mode: str = "train", save_path: Optional[str] = None) -> str: return self.export_training(save_path) elif mode == "infer": return self.export_inference(save_path) + elif mode == "zo-train": + return self.export_zo_training(save_path) else: raise ValueError(f"Invalid mode: {mode}. Must be 'train' or 'infer'") diff --git a/onnx4deeploy/io/__init__.py b/onnx4deeploy/io/__init__.py index 7d6d920..7c45a96 100644 --- a/onnx4deeploy/io/__init__.py +++ b/onnx4deeploy/io/__init__.py @@ -10,5 +10,5 @@ __all__ = [ "load_config", "load_train_config", - "compare_onnx_models", + "compare_onnx_models" ] diff --git a/onnx4deeploy/io/config_loader.py b/onnx4deeploy/io/config_loader.py index 9e186d0..6616be1 100644 --- a/onnx4deeploy/io/config_loader.py +++ b/onnx4deeploy/io/config_loader.py @@ -70,4 +70,4 @@ def load_train_config(config_filename: str = "../config.yaml") -> float: with open(config_file, "r") as f: config = yaml.safe_load(f).get("training", {}) - return config.get("learning_rate", 0.01) + return config.get("learning_rate", 0.01) \ No newline at end of file diff --git a/onnx4deeploy/transform/zo_transform.py b/onnx4deeploy/transform/zo_transform.py new file mode 100644 index 0000000..5d11784 --- /dev/null +++ b/onnx4deeploy/transform/zo_transform.py @@ -0,0 +1,278 @@ +import onnx +import onnxruntime as ort +import numpy as np +import sys +import os +import json +import torch +from pathlib import Path +from onnx import TensorProto, helper, shape_inference + +from onnx4deeploy.transform.model_transform import ensure_all_tensor_shapes + +def generate_zo_graph(inference_onnx:str, output_onnx:str, zo_config:dict) -> None: + """ Generate MeZO ONNX graph for model based on its inference onnx""" + + epsilon, seed, noise_type = zo_config["epsilon"], zo_config["seed"], zo_config["noise_type"] + + base_path = os.path.dirname(output_onnx) + os.makedirs(base_path, exist_ok=True) + inject_perturbation_nodes(inference_onnx, + output_path=output_onnx, + epsilon=epsilon, + seed=seed, + noise_type=noise_type) + + ensure_all_tensor_shapes(model_path=output_onnx, output_path=output_onnx) + # append_cross_entropy_loss(output_onnx, output_onnx, label_name='label') + +def inject_perturbation_nodes( + onnx_path: str, + output_path: str, + epsilon: float = 0.01, + seed: float = 42.0, + noise_type: str = "gaussian", +) -> None: + """ + This function inserts statically-seeded random operators. The unique seed for each + operator serves as an identifier that a custom hardware runtime can override with + a dynamic, runtime-provided seed. + + Args: + onnx_path: Path to the original ONNX model. + epsilon: The magnitude of the perturbation. + For the negative forward pass, just reverse the sign. + For inference, set to 0. + seed: A base seed to generate unique, deterministic seeds for each operator. + noise_type: The type of random distribution to use ('gaussian' or 'uniform'). + """ + # Load original ONNX model + p = Path(onnx_path) + + # --- 1. Identify target weights and biases --- + model = onnx.load(onnx_path) + weights_and_biases = { + init.name + for init in model.graph.initializer + if "weight" in init.name or "bias" in init.name + } + + if not weights_and_biases: + print("Warning: No weights or biases containing 'weight' or 'bias' in their names were found to perturb.") + return + + print(f"Found {len(weights_and_biases)} weight/bias tensors to perturb.") + + def modify_graph(original_model: onnx.ModelProto, output_path: str): + new_nodes = [] + extra_value_infos = [] + + # Keep track of all initializers. We will add to this list. + new_initializers = list(original_model.graph.initializer) + + # Create a set of initializer names for quick lookups + initializer_names = {init.name for init in new_initializers} + + base_seed = int(seed) + perturbation_counter = 0 + + # Prepare a fast lookup for initializer names + initializer_names = {init.name for init in new_initializers} + + for node in original_model.graph.node: + # Check if this is a node we want to modify + if node.op_type in ["Conv", "Gemm", "MatMul"]: + modified_inputs = list(node.input) + made_change = False + + for i, input_name in enumerate(node.input): + # Check if the input is a weight/bias initializer + if input_name in initializer_names: + made_change = True + + print(f"input_name {i}: {input_name}") + # Find the original weight tensor to get its properties + original_weight_tensor = next(t for t in new_initializers if t.name == input_name) + dtype = TensorProto.DataType.Name(original_weight_tensor.data_type) # "FLOAT" + noise_shape = original_weight_tensor.dims + noise_shape = [int(x) for x in noise_shape] + + # --- This is the core logic for injecting nodes --- + + # 1. Define names for the new intermediate tensors + perturbed_tensor_name = f"{perturbation_counter}_{input_name}" + # 2. Create the RandomNormal/RandomUniform node + unique_seed = float(base_seed + perturbation_counter) + + if noise_type == "gaussian": + perturbation_node = helper.make_node( + "PerturbNormal", + inputs=[input_name], + outputs=[perturbed_tensor_name], + name=f"perturbnormal{perturbed_tensor_name}", + domain="mezo", + seed=seed, + eps=epsilon, + idx=perturbation_counter, + # dtype=dtype, + doc_string="y = x + epsilon * RandomNormal(x, seed)" + ) + elif noise_type == "uniform": + perturbation_node = helper.make_node( + "PerturbUniform", + inputs=[input_name], + outputs=[perturbed_tensor_name], + name=f"perturbuniform{perturbed_tensor_name}", + domain="mezo", + idx=perturbation_counter, + seed=seed, + eps=epsilon, + low=-1.0, + high=1.0, + # dtype=dtype, + doc_string="y = x + epsilon * RandomUniform(x, seed)" + ) + new_nodes.append(perturbation_node) + + # **CRITICAL**: annotate perturbed edge with same dtype/shape as weight + if len(original_weight_tensor.dims) == 1: + out_shape = (original_weight_tensor.dims[0], ) + print(f"out_shape: {out_shape}") + else: + out_shape = original_weight_tensor.dims + extra_value_infos.append( + helper.make_tensor_value_info(perturbed_tensor_name, + elem_type=TensorProto.FLOAT, + shape=out_shape) + ) + + # 5. Update the input list for the *original* node + modified_inputs[i] = perturbed_tensor_name + perturbation_counter += 1 + + if made_change: + + # handle attributes + kwargs = {} + for attr in node.attribute: + # Use get_attribute_value to extract the python value from the AttributeProto + kwargs[attr.name] = helper.get_attribute_value(attr) + + # Create a new version of the Conv/Gemm node with the modified inputs + new_original_node = helper.make_node( + node.op_type, + modified_inputs, # Use the updated input list + node.output, + name=node.name, + domain=node.domain, + **kwargs + ) + new_nodes.append(new_original_node) + else: + # If no weights were perturbed, add the original node back unchanged + new_nodes.append(node) + else: + # This node is not a target, so add it to our new list as-is + new_nodes.append(node) + + new_value_info = list(original_model.graph.value_info) + extra_value_infos + + # Create a new graph with the new list of nodes and initializers + new_graph = helper.make_graph( + nodes=new_nodes, + name=f"{original_model.graph.name}-{node.op_type}", + inputs=original_model.graph.input, + outputs=original_model.graph.output, + initializer=new_initializers, + value_info=new_value_info + ) + + # Create and save the new model + for op in original_model.opset_import: + if op.domain == "": + standard_opset_version = op.version + break + + opset_list = [ + # Add the standard opset with the version we found + helper.make_opsetid("", standard_opset_version), + + # Addcustom domain + helper.make_opsetid("mezo", 1) + ] + new_model = helper.make_model(new_graph, producer_name="mezo-graph-generator", + opset_imports=opset_list) + + # onnx.checker.check_model(new_model) + onnx.save(new_model, output_path) + + # --- Main execution --- + original_model = onnx.load(onnx_path) + + print(f"Found {len(original_model.graph.initializer)} initializers. Perturbing weights/biases in Conv, MatMul, Gemm nodes.") + + modify_graph(original_model, output_path) + + print(f"Saved perturbed models to:\n- {output_path}") + return output_path + +def append_cross_entropy_loss(onnx_path, output_path, label_name='y', logits_output_idx=0, reduction='mean'): + """ + Adds a brand-new label input (INT64, shape ['batch_size']) -- guaranteed to be a new input name -- + appends a SoftmaxCrossEntropyLoss node consuming the model logits and the new label input, + and replaces the graph output with the scalar loss. + """ + model = onnx.load(onnx_path) + graph = model.graph + + if len(graph.output) == 0: + raise RuntimeError("Model has no outputs to attach the loss to.") + + # resolve logits tensor name (default: first graph output) + if logits_output_idx < 0 or logits_output_idx >= len(graph.output): + raise RuntimeError(f"Invalid logits_output_idx {logits_output_idx}") + logits_name = graph.output[logits_output_idx].name + + existing_inputs = {inp.name for inp in graph.input} + label_input_name = label_name + suffix = 0 + while label_input_name in existing_inputs: + label_input_name = f"{label_name}_mezo{suffix}" + suffix += 1 + + # get batch size from the first graph input (no initializer checks) + batch_dim = "batch_size" # fallback symbolic name + if graph.input: + first_inp = graph.input[0] + if first_inp.type.HasField("tensor_type") and first_inp.type.tensor_type.shape.dim: + first_dim = first_inp.type.tensor_type.shape.dim[0] + if first_dim.HasField("dim_value") and first_dim.dim_value > 0: + batch_dim = int(first_dim.dim_value) + elif first_dim.HasField("dim_param") and first_dim.dim_param: + batch_dim = first_dim.dim_param + + # add the new label input using resolved batch_dim + label_vi = helper.make_tensor_value_info(label_input_name, TensorProto.INT8, [batch_dim]) + graph.input.append(label_vi) + + # create loss node (standard SoftmaxCrossEntropyLoss) with proper attribute + loss_name = "loss" + loss_node = helper.make_node( + "SoftmaxCrossEntropyLoss", + inputs=[logits_name, label_input_name], + outputs=[loss_name], + name="CrossEntropyLoss", + reduction=reduction, + ) + graph.node.append(loss_node) + + # replace graph outputs with the scalar loss + del graph.output[:] + graph.output.append(helper.make_tensor_value_info(loss_name, TensorProto.FLOAT, [])) + + # try to infer shapes and save + try: + inferred = shape_inference.infer_shapes(model) + onnx.save(inferred, output_path) + except Exception: + onnx.save(model, output_path) From 5799139203f1951948813e1009e214d8c4acb216 Mon Sep 17 00:00:00 2001 From: JSJ Date: Mon, 9 Feb 2026 17:34:11 +0000 Subject: [PATCH 05/24] Added Perturbation operators --- Onnx4Deeploy.py | 8 + onnx4deeploy/operators/__init__.py | 10 ++ onnx4deeploy/operators/config.yaml | 14 ++ onnx4deeploy/operators/perturbeggroll.py | 101 +++++++++++++ onnx4deeploy/operators/perturbnormal.py | 157 ++++++++++++++++++++ onnx4deeploy/operators/perturbrademacher.py | 99 ++++++++++++ onnx4deeploy/operators/perturbtriangle.py | 100 +++++++++++++ onnx4deeploy/operators/perturbuniform.py | 157 ++++++++++++++++++++ 8 files changed, 646 insertions(+) create mode 100644 onnx4deeploy/operators/config.yaml create mode 100644 onnx4deeploy/operators/perturbeggroll.py create mode 100644 onnx4deeploy/operators/perturbnormal.py create mode 100644 onnx4deeploy/operators/perturbrademacher.py create mode 100644 onnx4deeploy/operators/perturbtriangle.py create mode 100644 onnx4deeploy/operators/perturbuniform.py diff --git a/Onnx4Deeploy.py b/Onnx4Deeploy.py index 8a00d78..ff89644 100644 --- a/Onnx4Deeploy.py +++ b/Onnx4Deeploy.py @@ -169,10 +169,18 @@ def list_available_operators(): "ConvGradX": "Convolution input gradient", "ConvGradW": "Convolution weight gradient", "ConvGradB": "Convolution bias gradient", + # ZO + "PerturbNormal": "Perturb input with gaussian random noise", + "PerturbUniform": "Perturb input with uniform random noise", + "PerturbTriangle": "Perturb input with triangle random noise", + "PerturbRademacher": "Perturb input with Rademacher random noise", + "PerturbEggroll": "Perturb input with Eggroll random noise", + # Others "ReduceSum": "Sum reduction", "SoftmaxCrossEntropy": "Softmax cross entropy", "ReluGrad": "ReLU gradient", + } return operators diff --git a/onnx4deeploy/operators/__init__.py b/onnx4deeploy/operators/__init__.py index f3c6512..3a0a946 100644 --- a/onnx4deeploy/operators/__init__.py +++ b/onnx4deeploy/operators/__init__.py @@ -37,6 +37,11 @@ from .softmax_grad import SoftmaxGradOperatorTest from .split import SplitOperatorTest from .transpose import TransposeOperatorTest +from .perturbnormal import PerturbNormalOperatorTest +from .perturbeggroll import PerturbEggrollOperatorTest +from .perturbrademacher import PerturbRademacherOperatorTest +from .perturbuniform import PerturbUniformOperatorTest +from .perturbtriangle import PerturbTriangleOperatorTest __all__ = [ "BaseOperatorTest", @@ -69,4 +74,9 @@ "ConvGradXOperatorTest", "ConvGradWOperatorTest", "ConvGradBOperatorTest", + "PerturbNormalOperatorTest", + "PerturbEggrollOperatorTest", + "PerturbRademacherOperatorTest", + "PerturbUniformOperatorTest", + "PerturbTriangleOperatorTest" ] diff --git a/onnx4deeploy/operators/config.yaml b/onnx4deeploy/operators/config.yaml new file mode 100644 index 0000000..20219e9 --- /dev/null +++ b/onnx4deeploy/operators/config.yaml @@ -0,0 +1,14 @@ +perturbnormal: + input_shape: [128, 48] + +perturbuniform: + input_shape: [128, 48] + +perturbtriangle: + input_shape: [128, 48] + +perturbrademacher: + input_shape: [128, 48] + +perturbeggroll: + input_shape: [128, 48] diff --git a/onnx4deeploy/operators/perturbeggroll.py b/onnx4deeploy/operators/perturbeggroll.py new file mode 100644 index 0000000..19cadc1 --- /dev/null +++ b/onnx4deeploy/operators/perturbeggroll.py @@ -0,0 +1,101 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: MIT + +"""PerturbEggroll operator test implementation.""" + +from typing import Any, Dict, Tuple + +import numpy as np +import onnxruntime as ort +from onnx import TensorProto, helper + +from .base_operator import BaseOperatorTest + + +class PerturbEggrollOperatorTest(BaseOperatorTest): + """Test generator for ONNX PerturbEggroll operator (custom/training op).""" + + def __init__(self, config_path=None, save_path=None): + super().__init__(config_path, save_path) + self.input_shape = None + self.num_classes = None + self.batch_size = None + + def get_operator_name(self) -> str: + return "PerturbEggroll" + + def load_config(self) -> Dict[str, Any]: + """Load PerturbEggroll-specific configuration.""" + config = super().load_config() + + pn_config = config.get("perturbeggroll", {}) + self.input_shape = tuple(pn_config["input_shape"]) + return config + + + def generate_inputs(self) -> np.ndarray: + """Generate input with both positive and negative values.""" + return {"x": np.random.randn(*self.input_shape).astype(np.float32)} + + def create_onnx_graph(self, inputs: Dict[str, np.ndarray]): + """Create ONNX graph for PerturbEggroll operator.""" + # Input tensors (without loss_grad for the final model) + x_tensor = helper.make_tensor_value_info( + "x", TensorProto.FLOAT, self.input_shape + ) + # Output tensor + perturbed_x_tensor = helper.make_tensor_value_info( + "perturbed_x", TensorProto.FLOAT, self.input_shape + ) + + # PerturbEggroll node (without loss_grad input) + perturb_node = helper.make_node( + "PerturbEggroll", + inputs=["x"], + outputs=["perturbed_x"], + name="perturb_eggroll_node", + domain="com.microsoft", + reduction="mean", + ) + + # Graph + graph = helper.make_graph( + [perturb_node], + "perturb_eggroll_graph", + [x_tensor], + [perturbed_x_tensor], + ) + + return graph + + def create_model(self, graph, opset_version: int = 13): + """Create ONNX model for PerturbEggroll with custom domain.""" + model = helper.make_model( + graph, + producer_name=f"{self.get_operator_name().lower()}_test", + opset_imports=[ + helper.make_opsetid("", opset_version), + helper.make_opsetid("com.microsoft", 1), + ], + ) + + return model + + def run_inference(self, onnx_file: str, inputs: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]: + """ + Run inference using custom emulation + """ + # perturbation is built from 2 low rank tensors + a = np.random.randn(*self.input_shape[:-1]).astype(np.float32) + b = np.random.randn(self.input_shape[-1]).astype(np.float32) + perturbation = np.outer(a, b) # shape: input_shape + perturbed_x = inputs["x"] + perturbation + + return {"perturbed_x": perturbed_x} + + def compute_expected_output(self, inputs: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]: + """ + Return None to skip validation - this is a custom operator. + """ + return None diff --git a/onnx4deeploy/operators/perturbnormal.py b/onnx4deeploy/operators/perturbnormal.py new file mode 100644 index 0000000..aac81af --- /dev/null +++ b/onnx4deeploy/operators/perturbnormal.py @@ -0,0 +1,157 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: MIT + +"""PerturbNormal operator test implementation.""" + +from typing import Any, Dict, Tuple + +import numpy as np +import onnxruntime as ort +from onnx import TensorProto, helper + +from .base_operator import BaseOperatorTest + + +class PerturbNormalOperatorTest(BaseOperatorTest): + """Test generator for ONNX PerturbNormal operator (custom/training op).""" + + def __init__(self, config_path=None, save_path=None): + super().__init__(config_path, save_path) + self.input_shape = None + self.num_classes = None + self.batch_size = None + + def get_operator_name(self) -> str: + return "PerturbNormal" + + def load_config(self) -> Dict[str, Any]: + """Load PerturbNormal-specific configuration.""" + config = super().load_config() + + pn_config = config.get("perturbnormal", {}) + self.input_shape = tuple(pn_config["input_shape"]) + return config + + + def generate_inputs(self) -> np.ndarray: + """Generate input with both positive and negative values.""" + return {"x": np.random.randn(*self.input_shape).astype(np.float32)} + + def create_onnx_graph(self, inputs: Dict[str, np.ndarray]): + """Create ONNX graph for PerturbNormal operator.""" + # Input tensors (without loss_grad for the final model) + x_tensor = helper.make_tensor_value_info( + "x", TensorProto.FLOAT, self.input_shape + ) + # Output tensor + perturbed_x_tensor = helper.make_tensor_value_info( + "perturbed_x", TensorProto.FLOAT, self.input_shape + ) + + # PerturbNormal node (without loss_grad input) + perturb_node = helper.make_node( + "PerturbNormal", + inputs=["x"], + outputs=["perturbed_x"], + name="perturb_normal_node", + domain="com.microsoft", + reduction="mean", + ) + + # Graph + graph = helper.make_graph( + [perturb_node], + "perturb_normal_graph", + [x_tensor], + [perturbed_x_tensor], + ) + + return graph + + def create_model(self, graph, opset_version: int = 13): + """Create ONNX model for PerturbNormal with custom domain.""" + model = helper.make_model( + graph, + producer_name=f"{self.get_operator_name().lower()}_test", + opset_imports=[ + helper.make_opsetid("", opset_version), + helper.make_opsetid("com.microsoft", 1), + ], + ) + + return model + + def run_inference(self, onnx_file: str, inputs: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]: + """ + Run inference using ONNX Runtime. + + For this custom op, we build a separate model that implements the + PerturbNormal functionality using standard ONNX ops (RandomNormal + Add) + and run inference on that to get the output. + """ + # --- Create the "Execution" Graph --- + # This graph implements the behavior of PerturbNormal for testing. + + # Input tensor info + x_tensor = helper.make_tensor_value_info( + "x", TensorProto.FLOAT, self.input_shape + ) + # Output tensor info + perturbed_x_tensor = helper.make_tensor_value_info( + "perturbed_x", TensorProto.FLOAT, self.input_shape + ) + + # Intermediate tensor for the random noise + noise_tensor_name = "random_noise" + + # 1. RandomNormal node to generate noise + # The shape of the noise must match the input shape. + random_node = helper.make_node( + "RandomNormal", + inputs=[], # RandomNormal has no inputs + outputs=[noise_tensor_name], + name="random_normal_for_perturb", + shape=self.input_shape, + dtype=TensorProto.FLOAT, + mean=0.0, + scale=1.0, # Standard normal distribution + ) + + # 2. Add node to add the noise to the input + add_node = helper.make_node( + "Add", + inputs=["x", noise_tensor_name], + outputs=["perturbed_x"], + name="add_perturbation", + ) + + # Create the graph that implements the custom op's logic + execution_graph = helper.make_graph( + [random_node, add_node], + "perturb_normal_execution_graph", + [x_tensor], + [perturbed_x_tensor], + ) + + # Create the ONNX model for execution + execution_model = self.create_model(execution_graph) + + # Run inference on the execution model + sess_options = ort.SessionOptions() + # Disable all optimizations to ensure nodes are not fused or altered + sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_DISABLE_ALL + + session = ort.InferenceSession(execution_model.SerializeToString(), sess_options) + # The output name is "perturbed_x" + output_names = ["perturbed_x"] + outputs = session.run(output_names, inputs) + + # Return the output in the expected dictionary format + return {"perturbed_x": outputs[0]} + + def compute_expected_output(self, inputs: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]: + """ + Return None to skip validation - this is a custom operator. + """ + return None diff --git a/onnx4deeploy/operators/perturbrademacher.py b/onnx4deeploy/operators/perturbrademacher.py new file mode 100644 index 0000000..950a5ef --- /dev/null +++ b/onnx4deeploy/operators/perturbrademacher.py @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: MIT + +"""PerturbRademacher operator test implementation.""" + +from typing import Any, Dict, Tuple + +import numpy as np +import onnxruntime as ort +from onnx import TensorProto, helper + +from .base_operator import BaseOperatorTest + + +class PerturbRademacherOperatorTest(BaseOperatorTest): + """Test generator for ONNX PerturbRademacher operator (custom/training op).""" + + def __init__(self, config_path=None, save_path=None): + super().__init__(config_path, save_path) + self.input_shape = None + self.num_classes = None + self.batch_size = None + + def get_operator_name(self) -> str: + return "PerturbRademacher" + + def load_config(self) -> Dict[str, Any]: + """Load PerturbRademacher-specific configuration.""" + config = super().load_config() + + pn_config = config.get("perturbrademacher", {}) + self.input_shape = tuple(pn_config["input_shape"]) + return config + + + def generate_inputs(self) -> np.ndarray: + """Generate input with both positive and negative values.""" + return {"x": np.random.randn(*self.input_shape).astype(np.float32)} + + def create_onnx_graph(self, inputs: Dict[str, np.ndarray]): + """Create ONNX graph for PerturbRademacher operator.""" + # Input tensors (without loss_grad for the final model) + x_tensor = helper.make_tensor_value_info( + "x", TensorProto.FLOAT, self.input_shape + ) + # Output tensor + perturbed_x_tensor = helper.make_tensor_value_info( + "perturbed_x", TensorProto.FLOAT, self.input_shape + ) + + # PerturbRademacher node (without loss_grad input) + perturb_node = helper.make_node( + "PerturbRademacher", + inputs=["x"], + outputs=["perturbed_x"], + name="perturb_rademacher_node", + domain="com.microsoft", + reduction="mean", + ) + + # Graph + graph = helper.make_graph( + [perturb_node], + "perturb_rademacher_graph", + [x_tensor], + [perturbed_x_tensor], + ) + + return graph + + def create_model(self, graph, opset_version: int = 13): + """Create ONNX model for PerturbRademacher with custom domain.""" + model = helper.make_model( + graph, + producer_name=f"{self.get_operator_name().lower()}_test", + opset_imports=[ + helper.make_opsetid("", opset_version), + helper.make_opsetid("com.microsoft", 1), + ], + ) + + return model + + def run_inference(self, onnx_file: str, inputs: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]: + """ + Run inference using custom emulation + """ + # perturbation is built from -1's and 1's + perturbation = np.random.choice([-1, 1], size=self.input_shape).astype(np.float32) + perturbed_x = inputs["x"] + perturbation + + return {"perturbed_x": perturbed_x} + + def compute_expected_output(self, inputs: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]: + """ + Return None to skip validation - this is a custom operator. + """ + return None diff --git a/onnx4deeploy/operators/perturbtriangle.py b/onnx4deeploy/operators/perturbtriangle.py new file mode 100644 index 0000000..f2ac39b --- /dev/null +++ b/onnx4deeploy/operators/perturbtriangle.py @@ -0,0 +1,100 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: MIT + +"""PerturbTriangle operator test implementation.""" + +from typing import Any, Dict, Tuple + +import numpy as np +import onnxruntime as ort +from onnx import TensorProto, helper + +from .base_operator import BaseOperatorTest + + +class PerturbTriangleOperatorTest(BaseOperatorTest): + """Test generator for ONNX PerturbTriangle operator (custom/training op).""" + + def __init__(self, config_path=None, save_path=None): + super().__init__(config_path, save_path) + self.input_shape = None + self.num_classes = None + self.batch_size = None + + def get_operator_name(self) -> str: + return "PerturbTriangle" + + def load_config(self) -> Dict[str, Any]: + """Load PerturbTriangle-specific configuration.""" + config = super().load_config() + + pn_config = config.get("perturbtriangle", {}) + self.input_shape = tuple(pn_config["input_shape"]) + return config + + def generate_inputs(self) -> np.ndarray: + """Generate input with both positive and negative values.""" + return {"x": np.random.randn(*self.input_shape).astype(np.float32)} + + def create_onnx_graph(self, inputs: Dict[str, np.ndarray]): + """Create ONNX graph for PerturbTriangle operator.""" + # Input tensors (without loss_grad for the final model) + x_tensor = helper.make_tensor_value_info( + "x", TensorProto.FLOAT, self.input_shape + ) + # Output tensor + perturbed_x_tensor = helper.make_tensor_value_info( + "perturbed_x", TensorProto.FLOAT, self.input_shape + ) + + # PerturbTriangle node (without loss_grad input) + perturb_node = helper.make_node( + "PerturbTriangle", + inputs=["x"], + outputs=["perturbed_x"], + name="perturb_triangle_node", + domain="com.microsoft", + reduction="mean", + ) + + # Graph + graph = helper.make_graph( + [perturb_node], + "perturb_triangle_graph", + [x_tensor], + [perturbed_x_tensor], + ) + + return graph + + def create_model(self, graph, opset_version: int = 13): + """Create ONNX model for PerturbTriangle with custom domain.""" + model = helper.make_model( + graph, + producer_name=f"{self.get_operator_name().lower()}_test", + opset_imports=[ + helper.make_opsetid("", opset_version), + helper.make_opsetid("com.microsoft", 1), + ], + ) + + return model + + def run_inference(self, onnx_file: str, inputs: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]: + """ + Run inference using custom emulation + """ + # perturbation is built from 2 uniforms + a = np.random.rand(*self.input_shape).astype(np.float32) + b = np.random.rand(*self.input_shape).astype(np.float32) + perturbation = (a - b)*np.sqrt(6) # triangle distribution + perturbed_x = inputs["x"] + perturbation + + return {"perturbed_x": perturbed_x} + + def compute_expected_output(self, inputs: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]: + """ + Return None to skip validation - this is a custom operator. + """ + return None diff --git a/onnx4deeploy/operators/perturbuniform.py b/onnx4deeploy/operators/perturbuniform.py new file mode 100644 index 0000000..1ae69fc --- /dev/null +++ b/onnx4deeploy/operators/perturbuniform.py @@ -0,0 +1,157 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: MIT + +"""PerturbUniform operator test implementation.""" + +from typing import Any, Dict, Tuple + +import numpy as np +import onnxruntime as ort +from onnx import TensorProto, helper + +from .base_operator import BaseOperatorTest + + +class PerturbUniformOperatorTest(BaseOperatorTest): + """Test generator for ONNX PerturbUniform operator (custom/training op).""" + + def __init__(self, config_path=None, save_path=None): + super().__init__(config_path, save_path) + self.input_shape = None + self.num_classes = None + self.batch_size = None + + def get_operator_name(self) -> str: + return "PerturbUniform" + + def load_config(self) -> Dict[str, Any]: + """Load PerturbUniform-specific configuration.""" + config = super().load_config() + + pn_config = config.get("perturbuniform", {}) + self.input_shape = tuple(pn_config["input_shape"]) + return config + + + def generate_inputs(self) -> np.ndarray: + """Generate input with both positive and negative values.""" + return {"x": np.random.randn(*self.input_shape).astype(np.float32)} + + def create_onnx_graph(self, inputs: Dict[str, np.ndarray]): + """Create ONNX graph for PerturbUniform operator.""" + # Input tensors (without loss_grad for the final model) + x_tensor = helper.make_tensor_value_info( + "x", TensorProto.FLOAT, self.input_shape + ) + # Output tensor + perturbed_x_tensor = helper.make_tensor_value_info( + "perturbed_x", TensorProto.FLOAT, self.input_shape + ) + + # PerturbUniform node (without loss_grad input) + perturb_node = helper.make_node( + "PerturbUniform", + inputs=["x"], + outputs=["perturbed_x"], + name="perturb_uniform_node", + domain="com.microsoft", + reduction="mean", + ) + + # Graph + graph = helper.make_graph( + [perturb_node], + "perturb_uniform_graph", + [x_tensor], + [perturbed_x_tensor], + ) + + return graph + + def create_model(self, graph, opset_version: int = 13): + """Create ONNX model for PerturbUniform with custom domain.""" + model = helper.make_model( + graph, + producer_name=f"{self.get_operator_name().lower()}_test", + opset_imports=[ + helper.make_opsetid("", opset_version), + helper.make_opsetid("com.microsoft", 1), + ], + ) + + return model + + def run_inference(self, onnx_file: str, inputs: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]: + """ + Run inference using ONNX Runtime. + + For this custom op, we build a separate model that implements the + PerturbUniform functionality using standard ONNX ops (RandomUniform + Add) + and run inference on that to get the output. + """ + # --- Create the "Execution" Graph --- + # This graph implements the behavior of PerturbUniform for testing. + + # Input tensor info + x_tensor = helper.make_tensor_value_info( + "x", TensorProto.FLOAT, self.input_shape + ) + # Output tensor info + perturbed_x_tensor = helper.make_tensor_value_info( + "perturbed_x", TensorProto.FLOAT, self.input_shape + ) + + # Intermediate tensor for the random noise + noise_tensor_name = "random_noise" + + # 1. RandomUniform node to generate noise + # The shape of the noise must match the input shape. + random_node = helper.make_node( + "RandomUniform", + inputs=[], # RandomUniform has no inputs + outputs=[noise_tensor_name], + name="random_uniform_for_perturb", + shape=self.input_shape, + dtype=TensorProto.FLOAT, + low = -np.sqrt(3), + high = np.sqrt(3) + ) + + # 2. Add node to add the noise to the input + add_node = helper.make_node( + "Add", + inputs=["x", noise_tensor_name], + outputs=["perturbed_x"], + name="add_perturbation", + ) + + # Create the graph that implements the custom op's logic + execution_graph = helper.make_graph( + [random_node, add_node], + "perturb_uniform_execution_graph", + [x_tensor], + [perturbed_x_tensor], + ) + + # Create the ONNX model for execution + execution_model = self.create_model(execution_graph) + + # Run inference on the execution model + sess_options = ort.SessionOptions() + # Disable all optimizations to ensure nodes are not fused or altered + sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_DISABLE_ALL + + session = ort.InferenceSession(execution_model.SerializeToString(), sess_options) + # The output name is "perturbed_x" + output_names = ["perturbed_x"] + outputs = session.run(output_names, inputs) + + # Return the output in the expected dictionary format + return {"perturbed_x": outputs[0]} + + def compute_expected_output(self, inputs: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]: + """ + Return None to skip validation - this is a custom operator. + """ + return None From 587e17f18201e11a09d7be3492dcb1ff69a66201 Mon Sep 17 00:00:00 2001 From: JanCSEM Date: Thu, 19 Feb 2026 11:19:35 +0000 Subject: [PATCH 06/24] Add support for exceptions, where nodes are excluded from perturbation generation --- onnx4deeploy/core/base_exporter.py | 27 ++++++++++++------- onnx4deeploy/core/onnx_utils.py | 7 ++++- .../models/lightweight_cnn_exporter.py | 5 ++++ onnx4deeploy/models/sleep_convit_exporter.py | 1 + onnx4deeploy/transform/zo_transform.py | 13 +++++---- 5 files changed, 38 insertions(+), 15 deletions(-) diff --git a/onnx4deeploy/core/base_exporter.py b/onnx4deeploy/core/base_exporter.py index 6264ac4..2136c72 100644 --- a/onnx4deeploy/core/base_exporter.py +++ b/onnx4deeploy/core/base_exporter.py @@ -517,9 +517,6 @@ def export_zo_training(self, save_path: Optional[str] = None) -> str: model = self.create_model() model.eval() # Zeroth-Order Training mode - # Store model for test data generation - self._model = model - # Generate input input_shape = self.get_input_shape() input_tensor = torch.randn(*input_shape, dtype=torch.float32) @@ -530,17 +527,27 @@ def export_zo_training(self, save_path: Optional[str] = None) -> str: opset_version = self.config.get("opset_version", 12) onnx_model = self._export_to_onnx(model, input_tensor, opset_version) - # Randomize initializers for testing - onnx_model = randomize_onnx_initializers(onnx_model) - - # Save inference model + # Save onnx.save(onnx_model, self.paths["network_infer"]) - print(f"โœ… Inference ONNX saved: {self.paths['network_infer']}") - + print(f"โœ… ONNX model saved: {self.paths['network_infer']}") + # Run inference optimizations print("\n๐Ÿ”ง Running inference optimizations...") self.run_inference_optimization(self.paths["network_infer"], self.paths["network_infer"]) + # Run shape inference + print("\n๐Ÿ” Running shape inference...") + from ..optimization.shape_optimizer import infer_shapes_with_custom_ops + + infer_shapes_with_custom_ops(self.paths["network_infer"], self.paths["network_infer"]) + + # Save test input/output data if method is implemented + if hasattr(self, "save_test_data"): + try: + self.save_test_data(model, self.paths["output_dir"]) + except Exception as e: + print(f"โš ๏ธ Failed to save test data: {e}") + # Reload optimized model onnx_model = onnx.load(self.paths["network_infer"]) print_model_info(self.paths["network_infer"]) @@ -555,6 +562,8 @@ def export_zo_training(self, save_path: Optional[str] = None) -> str: # Transform model for zeroth-order training (e.g., add noise nodes, modify outputs) print("\n๐Ÿ”ง Transforming model for zeroth-order training...") + # Randomize initializers for testing + onnx_model = randomize_onnx_initializers(onnx_model) generate_zo_graph( inference_onnx=self.paths["network_infer"], output_onnx=self.paths["network_zo_train"], diff --git a/onnx4deeploy/core/onnx_utils.py b/onnx4deeploy/core/onnx_utils.py index 53d14d5..1083825 100644 --- a/onnx4deeploy/core/onnx_utils.py +++ b/onnx4deeploy/core/onnx_utils.py @@ -176,7 +176,12 @@ def randomize_onnx_initializers(onnx_model: onnx.ModelProto) -> onnx.ModelProto: tensor = numpy_helper.to_array(initializer) # Randomize the values - randomized_tensor = np.random.randn(*tensor.shape).astype(tensor.dtype) + print(F"Randomizing initializer '{initializer.name}' with shape {tensor.shape}") + + randomized_tensor = np.random.randn(*tensor.shape) + if not isinstance(randomized_tensor, np.ndarray): + randomized_tensor = np.array(randomized_tensor) + randomized_tensor = randomized_tensor.astype(tensor.dtype) # Update the initializer new_initializer = numpy_helper.from_array(randomized_tensor, initializer.name) diff --git a/onnx4deeploy/models/lightweight_cnn_exporter.py b/onnx4deeploy/models/lightweight_cnn_exporter.py index aa9f8f6..e5baeca 100644 --- a/onnx4deeploy/models/lightweight_cnn_exporter.py +++ b/onnx4deeploy/models/lightweight_cnn_exporter.py @@ -49,6 +49,11 @@ def load_config(self) -> Dict[str, Any]: # Training configuration "training_strategy": "full", # Options: "full", "last_layer", "custom" "custom_trainable_params": [], + "zo": { + "epsilon": 0.1, + "seed": 42, + "noise_type": "uniform", + } } self.model_config = config diff --git a/onnx4deeploy/models/sleep_convit_exporter.py b/onnx4deeploy/models/sleep_convit_exporter.py index 8168854..534197e 100644 --- a/onnx4deeploy/models/sleep_convit_exporter.py +++ b/onnx4deeploy/models/sleep_convit_exporter.py @@ -83,6 +83,7 @@ def load_config(self) -> Dict[str, Any]: "epsilon": 0.1, "seed": 42, "noise_type": "uniform", + "exceptions": "node_matmul_2" } } diff --git a/onnx4deeploy/transform/zo_transform.py b/onnx4deeploy/transform/zo_transform.py index 5d11784..afd761d 100644 --- a/onnx4deeploy/transform/zo_transform.py +++ b/onnx4deeploy/transform/zo_transform.py @@ -13,7 +13,7 @@ def generate_zo_graph(inference_onnx:str, output_onnx:str, zo_config:dict) -> None: """ Generate MeZO ONNX graph for model based on its inference onnx""" - epsilon, seed, noise_type = zo_config["epsilon"], zo_config["seed"], zo_config["noise_type"] + epsilon, seed, noise_type, exceptions = zo_config["epsilon"], zo_config["seed"], zo_config["noise_type"], zo_config.get("exceptions", []) base_path = os.path.dirname(output_onnx) os.makedirs(base_path, exist_ok=True) @@ -21,7 +21,8 @@ def generate_zo_graph(inference_onnx:str, output_onnx:str, zo_config:dict) -> No output_path=output_onnx, epsilon=epsilon, seed=seed, - noise_type=noise_type) + noise_type=noise_type, + exceptions=exceptions) ensure_all_tensor_shapes(model_path=output_onnx, output_path=output_onnx) # append_cross_entropy_loss(output_onnx, output_onnx, label_name='label') @@ -32,6 +33,7 @@ def inject_perturbation_nodes( epsilon: float = 0.01, seed: float = 42.0, noise_type: str = "gaussian", + exceptions: list[str] = [] ) -> None: """ This function inserts statically-seeded random operators. The unique seed for each @@ -63,7 +65,7 @@ def inject_perturbation_nodes( print(f"Found {len(weights_and_biases)} weight/bias tensors to perturb.") - def modify_graph(original_model: onnx.ModelProto, output_path: str): + def modify_graph(original_model: onnx.ModelProto, output_path: str, exceptions: list[str]): new_nodes = [] extra_value_infos = [] @@ -81,7 +83,8 @@ def modify_graph(original_model: onnx.ModelProto, output_path: str): for node in original_model.graph.node: # Check if this is a node we want to modify - if node.op_type in ["Conv", "Gemm", "MatMul"]: + if node.op_type in ["Conv", "Gemm", "MatMul"] and node.name not in exceptions: + print(F"node: {node.name}, op_type: {node.op_type}") modified_inputs = list(node.input) made_change = False @@ -211,7 +214,7 @@ def modify_graph(original_model: onnx.ModelProto, output_path: str): print(f"Found {len(original_model.graph.initializer)} initializers. Perturbing weights/biases in Conv, MatMul, Gemm nodes.") - modify_graph(original_model, output_path) + modify_graph(original_model, output_path, exceptions=exceptions) print(f"Saved perturbed models to:\n- {output_path}") return output_path From b0c10df3258f56ed887dbf2f18f7b877c1efe38f Mon Sep 17 00:00:00 2001 From: JanCSEM Date: Fri, 20 Feb 2026 13:52:14 +0000 Subject: [PATCH 07/24] FIX bias initialization --- Onnx4Deeploy.py | 1 + onnx4deeploy/core/base_exporter.py | 10 +++++++--- onnx4deeploy/operators/config.yaml | 10 ++++++++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/Onnx4Deeploy.py b/Onnx4Deeploy.py index ff89644..04b2e84 100644 --- a/Onnx4Deeploy.py +++ b/Onnx4Deeploy.py @@ -155,6 +155,7 @@ def list_available_operators(): # Matrix operations "Gemm": "General matrix multiplication", "MatMul": "Matrix multiplication", + "Conv2d": "2D convolution", # Pooling "MaxPool": "Max pooling", "AveragePool": "Average pooling", diff --git a/onnx4deeploy/core/base_exporter.py b/onnx4deeploy/core/base_exporter.py index 2136c72..5c2f83e 100644 --- a/onnx4deeploy/core/base_exporter.py +++ b/onnx4deeploy/core/base_exporter.py @@ -251,7 +251,7 @@ def setup_paths(self, mode: ExportMode) -> Dict[str, str]: "network_pre_sgd": os.path.join(output_dir, "network_pre_sgd.onnx"), } ) - + if mode == ExportMode.ZO_TRAINING: paths.update( { @@ -335,6 +335,10 @@ def export_inference(self, save_path: Optional[str] = None) -> str: input_tensor = torch.randn(*input_shape, dtype=torch.float32) print(f" Input shape: {input_shape}") + # initialize weights and biases for testing + for param in model.parameters(): + if param.requires_grad: + torch.nn.init.normal_(param, mean=0.0, std=0.02) # Export to ONNX print("\n๐Ÿ“ค Exporting to ONNX...") opset_version = self.config.get("opset_version", 12) @@ -494,7 +498,7 @@ def export_training(self, save_path: Optional[str] = None) -> str: print(f"{'='*60}\n") return self.paths["network"] - + def export_zo_training(self, save_path: Optional[str] = None) -> str: """ Export model in zeroth-order training mode. @@ -530,7 +534,7 @@ def export_zo_training(self, save_path: Optional[str] = None) -> str: # Save onnx.save(onnx_model, self.paths["network_infer"]) print(f"โœ… ONNX model saved: {self.paths['network_infer']}") - + # Run inference optimizations print("\n๐Ÿ”ง Running inference optimizations...") self.run_inference_optimization(self.paths["network_infer"], self.paths["network_infer"]) diff --git a/onnx4deeploy/operators/config.yaml b/onnx4deeploy/operators/config.yaml index 20219e9..6b9b987 100644 --- a/onnx4deeploy/operators/config.yaml +++ b/onnx4deeploy/operators/config.yaml @@ -12,3 +12,13 @@ perturbrademacher: perturbeggroll: input_shape: [128, 48] + +conv2d: + input_shape: [1, 1, 28, 28] + kernel_size: 5 + out_channels: 16 + stride: 1 + padding: 0 + use_bias: true + group: 1 + dilation: 1 \ No newline at end of file From eac93b3030c65af71a1fe1baddec002e1f8715a7 Mon Sep 17 00:00:00 2001 From: JanCSEM Date: Sat, 21 Feb 2026 10:40:21 +0000 Subject: [PATCH 08/24] refine operators --- onnx4deeploy/operators/perturbeggroll.py | 46 +++++++++++--- onnx4deeploy/operators/perturbnormal.py | 67 ++++++++++++++++++++- onnx4deeploy/operators/perturbrademacher.py | 3 +- onnx4deeploy/operators/perturbtriangle.py | 3 +- onnx4deeploy/operators/perturbuniform.py | 10 ++- 5 files changed, 111 insertions(+), 18 deletions(-) diff --git a/onnx4deeploy/operators/perturbeggroll.py b/onnx4deeploy/operators/perturbeggroll.py index 19cadc1..db78480 100644 --- a/onnx4deeploy/operators/perturbeggroll.py +++ b/onnx4deeploy/operators/perturbeggroll.py @@ -33,11 +33,11 @@ def load_config(self) -> Dict[str, Any]: self.input_shape = tuple(pn_config["input_shape"]) return config - + def generate_inputs(self) -> np.ndarray: """Generate input with both positive and negative values.""" return {"x": np.random.randn(*self.input_shape).astype(np.float32)} - + def create_onnx_graph(self, inputs: Dict[str, np.ndarray]): """Create ONNX graph for PerturbEggroll operator.""" # Input tensors (without loss_grad for the final model) @@ -49,24 +49,50 @@ def create_onnx_graph(self, inputs: Dict[str, np.ndarray]): "perturbed_x", TensorProto.FLOAT, self.input_shape ) - # PerturbEggroll node (without loss_grad input) - perturb_node = helper.make_node( - "PerturbEggroll", + epsilon = 0.01 # Example scaling factor for the perturbation + + # Shape annotation for intermediate outputs + a_shape = [self.input_shape[0], 1] + b_shape = [int(np.prod(self.input_shape[1:])), 1] + + a_tensor = helper.make_tensor_value_info( + "a", TensorProto.FLOAT, a_shape + ) + b_tensor = helper.make_tensor_value_info( + "b", TensorProto.FLOAT, b_shape + ) + + # Eggroll noise node (without loss_grad input) + noise_node = helper.make_node( + "GenerateEggrollNoise", inputs=["x"], + outputs=["a", "b"], + name="generate_eggroll_noise_node", + domain="com.microsoft" + ) + + gemm_node = helper.make_node( + "Gemm", + inputs=["a", "b", "x"], outputs=["perturbed_x"], - name="perturb_eggroll_node", - domain="com.microsoft", - reduction="mean", + name="gemm_node", + transA=0, + transB=1, + alpha=epsilon, + beta=0 ) # Graph graph = helper.make_graph( - [perturb_node], + [noise_node, gemm_node], "perturb_eggroll_graph", [x_tensor], [perturbed_x_tensor], + value_info=[a_tensor, b_tensor] # <-- shape annotation here ) + for vi in graph.value_info: + print(vi.name, [d.dim_value for d in vi.type.tensor_type.shape.dim]) return graph def create_model(self, graph, opset_version: int = 13): @@ -91,7 +117,7 @@ def run_inference(self, onnx_file: str, inputs: Dict[str, np.ndarray]) -> Dict[s b = np.random.randn(self.input_shape[-1]).astype(np.float32) perturbation = np.outer(a, b) # shape: input_shape perturbed_x = inputs["x"] + perturbation - + return {"perturbed_x": perturbed_x} def compute_expected_output(self, inputs: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]: diff --git a/onnx4deeploy/operators/perturbnormal.py b/onnx4deeploy/operators/perturbnormal.py index aac81af..151217f 100644 --- a/onnx4deeploy/operators/perturbnormal.py +++ b/onnx4deeploy/operators/perturbnormal.py @@ -6,6 +6,9 @@ from typing import Any, Dict, Tuple + +import torch +from torch.autograd import Function import numpy as np import onnxruntime as ort from onnx import TensorProto, helper @@ -13,6 +16,62 @@ from .base_operator import BaseOperatorTest +class Xorshift32: + def __init__(self, seed: int = 0): + self.state = seed if seed != 0 else 1 # Avoid zero state + + def next(self) -> int: + # Xorshift32 algorithm + self.state ^= (self.state << 13) & 0xFFFFFFFF + self.state ^= (self.state >> 17) & 0xFFFFFFFF + self.state ^= (self.state << 5) & 0xFFFFFFFF + return self.state + +class Ziggurat(): + def __init__(self, seed: int = 0): + self.seed = seed if seed != 0 else 1 # Avoid zero state + # Precompute the Ziggurat tables + self.N = 256 # Number of layers + self.R = 3.442619855899 # Right tail boundary + self.x = np.zeros(self.N + 1) + self.y = np.zeros(self.N) + self.x[0] = self.R + self.x[self.N] = 0 + for i in range(1, self.N): + self.x[i] = np.sqrt(-2.0 * np.log(np.exp(-0.5 * self.x[i-1]**2))) + for i in range(self.N): + self.y[i] = np.exp(-0.5 * self.x[i]**2) + self.rng = Xorshift32(self.seed) + + def next(self) -> float: + while True: + # Generate random layer index + k = self.rng.next() % self.N + # Generate uniform random number + u = self.rng.next() / 0xFFFFFFFF + x = u * (self.x[k] - self.x[k+1]) + self.x[k+1] + # Accept or reject + if u < self.y[k] / self.y[k+1]: + return x + if x < self.R: + y = np.exp(-0.5 * x * x) + if u * (self.y[k+1] - self.y[k]) < (y - self.y[k]): + return x + +class PerturbNormalFunction(Function): + @staticmethod + def forward(ctx, x, seed=42, epsilon=0.01): + # generate noise using Xorshift. + rng = Ziggurat(seed) + for _ in range(x.numel()): + noise = rng.next() * epsilon + perturbed_x = x + noise + return perturbed_x + + @staticmethod + def symbolic(g, x): + return g.op("ai.zo::PerturbNormal", x, outputs=1) + class PerturbNormalOperatorTest(BaseOperatorTest): """Test generator for ONNX PerturbNormal operator (custom/training op).""" @@ -55,8 +114,12 @@ def create_onnx_graph(self, inputs: Dict[str, np.ndarray]): inputs=["x"], outputs=["perturbed_x"], name="perturb_normal_node", - domain="com.microsoft", - reduction="mean", + seed=42, + eps=0.01, + idx=0, + # dtype=dtype, + doc_string="y = x + epsilon * RandomNormal(x, seed)", + domain="com.microsoft" ) # Graph diff --git a/onnx4deeploy/operators/perturbrademacher.py b/onnx4deeploy/operators/perturbrademacher.py index 950a5ef..f88f562 100644 --- a/onnx4deeploy/operators/perturbrademacher.py +++ b/onnx4deeploy/operators/perturbrademacher.py @@ -55,8 +55,7 @@ def create_onnx_graph(self, inputs: Dict[str, np.ndarray]): inputs=["x"], outputs=["perturbed_x"], name="perturb_rademacher_node", - domain="com.microsoft", - reduction="mean", + domain="com.microsoft" ) # Graph diff --git a/onnx4deeploy/operators/perturbtriangle.py b/onnx4deeploy/operators/perturbtriangle.py index f2ac39b..241f549 100644 --- a/onnx4deeploy/operators/perturbtriangle.py +++ b/onnx4deeploy/operators/perturbtriangle.py @@ -54,8 +54,7 @@ def create_onnx_graph(self, inputs: Dict[str, np.ndarray]): inputs=["x"], outputs=["perturbed_x"], name="perturb_triangle_node", - domain="com.microsoft", - reduction="mean", + domain="com.microsoft" ) # Graph diff --git a/onnx4deeploy/operators/perturbuniform.py b/onnx4deeploy/operators/perturbuniform.py index 1ae69fc..e3cd3f2 100644 --- a/onnx4deeploy/operators/perturbuniform.py +++ b/onnx4deeploy/operators/perturbuniform.py @@ -55,8 +55,14 @@ def create_onnx_graph(self, inputs: Dict[str, np.ndarray]): inputs=["x"], outputs=["perturbed_x"], name="perturb_uniform_node", - domain="com.microsoft", - reduction="mean", + idx=0, + seed=42, + eps=0.01, + low=-np.sqrt(3), + high=np.sqrt(3), + # dtype=dtype, + doc_string="y = x + epsilon * RandomUniform(x, seed)", + domain="com.microsoft" ) # Graph From 05da0d2bf8f44529ea851ece5a89cc545e01b3b2 Mon Sep 17 00:00:00 2001 From: JanCSEM Date: Sat, 21 Feb 2026 10:59:57 +0000 Subject: [PATCH 09/24] Added Eggroll and Rademacher to ZO transform for model export --- .../models/lightweight_cnn_exporter.py | 2 +- onnx4deeploy/transform/zo_transform.py | 79 +++++++++++++++++-- 2 files changed, 75 insertions(+), 6 deletions(-) diff --git a/onnx4deeploy/models/lightweight_cnn_exporter.py b/onnx4deeploy/models/lightweight_cnn_exporter.py index e5baeca..7c0e67e 100644 --- a/onnx4deeploy/models/lightweight_cnn_exporter.py +++ b/onnx4deeploy/models/lightweight_cnn_exporter.py @@ -52,7 +52,7 @@ def load_config(self) -> Dict[str, Any]: "zo": { "epsilon": 0.1, "seed": 42, - "noise_type": "uniform", + "noise_type": "eggroll", } } diff --git a/onnx4deeploy/transform/zo_transform.py b/onnx4deeploy/transform/zo_transform.py index afd761d..ada2eb5 100644 --- a/onnx4deeploy/transform/zo_transform.py +++ b/onnx4deeploy/transform/zo_transform.py @@ -112,7 +112,7 @@ def modify_graph(original_model: onnx.ModelProto, output_path: str, exceptions: "PerturbNormal", inputs=[input_name], outputs=[perturbed_tensor_name], - name=f"perturbnormal{perturbed_tensor_name}", + name=f"perturbnormal_{perturbed_tensor_name}", domain="mezo", seed=seed, eps=epsilon, @@ -120,22 +120,91 @@ def modify_graph(original_model: onnx.ModelProto, output_path: str, exceptions: # dtype=dtype, doc_string="y = x + epsilon * RandomNormal(x, seed)" ) + new_nodes.append(perturbation_node) + elif noise_type == "uniform": perturbation_node = helper.make_node( "PerturbUniform", inputs=[input_name], outputs=[perturbed_tensor_name], - name=f"perturbuniform{perturbed_tensor_name}", + name=f"perturbuniform_{perturbed_tensor_name}", domain="mezo", idx=perturbation_counter, seed=seed, eps=epsilon, - low=-1.0, - high=1.0, + low=-np.sqrt(3), + high=np.sqrt(3), # dtype=dtype, doc_string="y = x + epsilon * RandomUniform(x, seed)" ) - new_nodes.append(perturbation_node) + new_nodes.append(perturbation_node) + + elif noise_type == "triangle": + perturbation_node = helper.make_node( + "PerturbTriangle", + inputs=[input_name], + outputs=[perturbed_tensor_name], + name=f"perturbtriangle_{perturbed_tensor_name}", + domain="mezo", + idx=perturbation_counter, + seed=seed, + eps=epsilon, + low=-np.sqrt(6), + high=np.sqrt(6), + # dtype=dtype, + doc_string="y = x + epsilon * RandomTriangle(x, seed)" + ) + new_nodes.append(perturbation_node) + + elif noise_type == "rademacher": + perturbation_node = helper.make_node( + "PerturbRademacher", + inputs=[input_name], + outputs=[perturbed_tensor_name], + name=f"perturbrademacher_{perturbed_tensor_name}", + domain="mezo", + idx=perturbation_counter, + seed=seed, + eps=epsilon, + # dtype=dtype, + doc_string="y = x + epsilon * RandomRademacher(x, seed)" + ) + new_nodes.append(perturbation_node) + + elif noise_type == "eggroll": + # Shape annotation for intermediate outputs + a_shape = [noise_shape[0], 1] + b_shape = [int(np.prod(noise_shape[1:])), 1] + + extra_value_infos.append(helper.make_tensor_value_info( + f"a_{perturbed_tensor_name}", TensorProto.FLOAT, a_shape + )) + extra_value_infos.append(helper.make_tensor_value_info( + f"b_{perturbed_tensor_name}", TensorProto.FLOAT, b_shape + )) + + # Eggroll noise node (without loss_grad input) + noise_node = helper.make_node( + "GenerateEggrollNoise", + inputs=[input_name], + outputs=[f"a_{perturbed_tensor_name}", f"b_{perturbed_tensor_name}"], + name=f"gen_eggroll_noise_{perturbed_tensor_name}", + domain="com.microsoft" + ) + + gemm_node = helper.make_node( + "Gemm", + inputs=[f"a_{perturbed_tensor_name}", f"b_{perturbed_tensor_name}", input_name], + outputs=[perturbed_tensor_name], + name=f"eggroll_gemm_{perturbed_tensor_name}", + transA=0, + transB=1, + alpha=epsilon, + beta=0 + ) + + new_nodes.append(noise_node) + new_nodes.append(gemm_node) # **CRITICAL**: annotate perturbed edge with same dtype/shape as weight if len(original_weight_tensor.dims) == 1: From 4137b4459f192b4ee228dc031c17b6446f3a852a Mon Sep 17 00:00:00 2001 From: JanCSEM Date: Thu, 26 Feb 2026 10:12:44 +0000 Subject: [PATCH 10/24] Split Eggroll noise generator into 2 nodes for A and B --- onnx4deeploy/operators/config.yaml | 2 +- onnx4deeploy/operators/perturbeggroll.py | 95 +++++++++++++++++++----- onnx4deeploy/transform/zo_transform.py | 81 ++++++++++++++++---- 3 files changed, 146 insertions(+), 32 deletions(-) diff --git a/onnx4deeploy/operators/config.yaml b/onnx4deeploy/operators/config.yaml index 6b9b987..5cf37a2 100644 --- a/onnx4deeploy/operators/config.yaml +++ b/onnx4deeploy/operators/config.yaml @@ -11,7 +11,7 @@ perturbrademacher: input_shape: [128, 48] perturbeggroll: - input_shape: [128, 48] + input_shape: [128, 16, 3, 3] conv2d: input_shape: [1, 1, 28, 28] diff --git a/onnx4deeploy/operators/perturbeggroll.py b/onnx4deeploy/operators/perturbeggroll.py index db78480..b9f871a 100644 --- a/onnx4deeploy/operators/perturbeggroll.py +++ b/onnx4deeploy/operators/perturbeggroll.py @@ -62,34 +62,93 @@ def create_onnx_graph(self, inputs: Dict[str, np.ndarray]): "b", TensorProto.FLOAT, b_shape ) + shape_input_name = helper.make_tensor(name=f"shape_x", data_type=TensorProto.INT64, dims=[len(self.input_shape)], + vals=np.array(self.input_shape, dtype=np.int64)) + + if len(self.input_shape) > 2: + + shape_flat_name = helper.make_tensor(name=f"shape_x_flat", data_type=TensorProto.INT64, dims=[2], + vals=np.array([a_shape[0], b_shape[0]], dtype=np.int64)) + # insert flattening nodes + flatten_node = helper.make_node( + "Reshape", + inputs=["x", f"shape_x_flat"], + outputs=[f"flattened_x"], + name=f"flatten_x" + ) + + flattened_tensor_name = helper.make_tensor_value_info( + f"flattened_x", TensorProto.FLOAT,[a_shape[0], b_shape[0]] + ) + + unflatten_node = helper.make_node( + "Reshape", + inputs=["flattened_perturbed_x", "shape_x"], + outputs=["perturbed_x"], + name="unflatten_perturbed_x" + ) + flattened_perturbed_tensor_name = helper.make_tensor_value_info( + "flattened_perturbed_x", TensorProto.FLOAT, [a_shape[0], b_shape[0]] + ) + eggroll_input = "flattened_x" + eggroll_output = "flattened_perturbed_x" + else: + eggroll_input = "x" + eggroll_output = "perturbed_x" + # Eggroll noise node (without loss_grad input) - noise_node = helper.make_node( - "GenerateEggrollNoise", - inputs=["x"], - outputs=["a", "b"], - name="generate_eggroll_noise_node", - domain="com.microsoft" + noise_node_a = helper.make_node( + "PerturbEggroll", + inputs=["shape_x"], + outputs=["a"], + name=f"gen_eggroll_noise_a", + seed=13, + idx=0, + domain="com.microsoft", + doc_string="a = RandomRademacher(x[0], seed)" + ) + + noise_node_b = helper.make_node( + "PerturbEggroll", + inputs=["shape_x"], + outputs=["b"], + name=f"gen_eggroll_noise_b", + seed=14, + idx=1, + domain="com.microsoft", + doc_string="b = RandomRademacher(x[1:], seed)" ) gemm_node = helper.make_node( "Gemm", - inputs=["a", "b", "x"], - outputs=["perturbed_x"], - name="gemm_node", + inputs=["a", "b", eggroll_input], + outputs=[eggroll_output], + name=f"eggroll_gemm_perturb_x", transA=0, transB=1, alpha=epsilon, beta=0 ) - # Graph - graph = helper.make_graph( - [noise_node, gemm_node], - "perturb_eggroll_graph", - [x_tensor], - [perturbed_x_tensor], - value_info=[a_tensor, b_tensor] # <-- shape annotation here - ) + if len(self.input_shape) > 2: + graph = helper.make_graph( + [flatten_node, noise_node_a, noise_node_b, gemm_node, unflatten_node], + "perturb_eggroll_graph", + [x_tensor], + [perturbed_x_tensor], + [shape_input_name, shape_flat_name], # <-- shape annotations here + value_info=[a_tensor, b_tensor, + flattened_tensor_name, + flattened_perturbed_tensor_name] # <-- shape annotation here + ) + else: + graph = helper.make_graph( + [noise_node_a, noise_node_b, gemm_node], + "perturb_eggroll_graph", + [x_tensor], + [perturbed_x_tensor], + value_info=[a_tensor, b_tensor] # <-- shape annotation here + ) for vi in graph.value_info: print(vi.name, [d.dim_value for d in vi.type.tensor_type.shape.dim]) @@ -116,6 +175,8 @@ def run_inference(self, onnx_file: str, inputs: Dict[str, np.ndarray]) -> Dict[s a = np.random.randn(*self.input_shape[:-1]).astype(np.float32) b = np.random.randn(self.input_shape[-1]).astype(np.float32) perturbation = np.outer(a, b) # shape: input_shape + if len(self.input_shape) > 2: + perturbation = perturbation.reshape(self.input_shape) perturbed_x = inputs["x"] + perturbation return {"perturbed_x": perturbed_x} diff --git a/onnx4deeploy/transform/zo_transform.py b/onnx4deeploy/transform/zo_transform.py index ada2eb5..82321bd 100644 --- a/onnx4deeploy/transform/zo_transform.py +++ b/onnx4deeploy/transform/zo_transform.py @@ -13,8 +13,8 @@ def generate_zo_graph(inference_onnx:str, output_onnx:str, zo_config:dict) -> None: """ Generate MeZO ONNX graph for model based on its inference onnx""" - epsilon, seed, noise_type, exceptions = zo_config["epsilon"], zo_config["seed"], zo_config["noise_type"], zo_config.get("exceptions", []) - + epsilon, seed, noise_type, exceptions = zo_config["epsilon"], zo_config["seed"], zo_config["noise_type"], zo_config.get("exceptions", []) + base_path = os.path.dirname(output_onnx) os.makedirs(base_path, exist_ok=True) inject_perturbation_nodes(inference_onnx, @@ -23,7 +23,7 @@ def generate_zo_graph(inference_onnx:str, output_onnx:str, zo_config:dict) -> No seed=seed, noise_type=noise_type, exceptions=exceptions) - + ensure_all_tensor_shapes(model_path=output_onnx, output_path=output_onnx) # append_cross_entropy_loss(output_onnx, output_onnx, label_name='label') @@ -175,6 +175,44 @@ def modify_graph(original_model: onnx.ModelProto, output_path: str, exceptions: # Shape annotation for intermediate outputs a_shape = [noise_shape[0], 1] b_shape = [int(np.prod(noise_shape[1:])), 1] + shape_input_name = helper.make_tensor(name=f"shape_{input_name}", data_type=TensorProto.INT64, dims=[len(noise_shape)], + vals=np.array(noise_shape, dtype=np.int64)) + new_initializers.append(shape_input_name) + + if len(noise_shape) > 2: + + shape_flat_name = helper.make_tensor(name=f"shape_{input_name}_flat", data_type=TensorProto.INT64, dims=[2], + vals=np.array([a_shape[0], b_shape[0]], dtype=np.int64)) + + new_initializers.append(shape_flat_name) + + # insert flattening nodes + flatten_node = helper.make_node( + "Reshape", + inputs=[input_name, f"shape_{input_name}_flat"], + outputs=[f"flattened_{input_name}"], + name=f"flatten_{input_name}" + ) + new_nodes.append(flatten_node) + extra_value_infos.append(helper.make_tensor_value_info( + f"flattened_{input_name}", TensorProto.FLOAT,[noise_shape[0], int(np.prod(noise_shape[1:]))] + )) + + unflatten_node = helper.make_node( + "Reshape", + inputs=[f"flattened_{perturbed_tensor_name}", f"shape_{input_name}"], + outputs=[perturbed_tensor_name], + name=f"unflatten_{perturbed_tensor_name}" + ) + new_nodes.append(unflatten_node) + extra_value_infos.append(helper.make_tensor_value_info( + f"flattened_{perturbed_tensor_name}", TensorProto.FLOAT, [noise_shape[0], int(np.prod(noise_shape[1:]))] + )) + eggroll_input = f"flattened_{input_name}" + eggroll_output = f"flattened_{perturbed_tensor_name}" + else: + eggroll_input = input_name + eggroll_output = perturbed_tensor_name extra_value_infos.append(helper.make_tensor_value_info( f"a_{perturbed_tensor_name}", TensorProto.FLOAT, a_shape @@ -184,26 +222,41 @@ def modify_graph(original_model: onnx.ModelProto, output_path: str, exceptions: )) # Eggroll noise node (without loss_grad input) - noise_node = helper.make_node( - "GenerateEggrollNoise", - inputs=[input_name], - outputs=[f"a_{perturbed_tensor_name}", f"b_{perturbed_tensor_name}"], - name=f"gen_eggroll_noise_{perturbed_tensor_name}", - domain="com.microsoft" + noise_node_a = helper.make_node( + "PerturbEggroll", + inputs=[f"shape_{input_name}"], + outputs=[f"a_{perturbed_tensor_name}"], + name=f"gen_eggroll_noise_a_{perturbed_tensor_name}", + seed=seed, + idx=perturbation_counter, + domain="com.microsoft", + doc_string="a = RandomRademacher(x[0], seed)" + ) + + noise_node_b = helper.make_node( + "PerturbEggroll", + inputs=[f"shape_{input_name}"], + outputs=[f"b_{perturbed_tensor_name}"], + name=f"gen_eggroll_noise_b_{perturbed_tensor_name}", + seed=seed, + idx=perturbation_counter, + domain="com.microsoft", + doc_string="b = RandomRademacher(x[1:], seed)" ) gemm_node = helper.make_node( "Gemm", - inputs=[f"a_{perturbed_tensor_name}", f"b_{perturbed_tensor_name}", input_name], - outputs=[perturbed_tensor_name], + inputs=[f"a_{perturbed_tensor_name}", f"b_{perturbed_tensor_name}", eggroll_input], + outputs=[eggroll_output], name=f"eggroll_gemm_{perturbed_tensor_name}", transA=0, transB=1, alpha=epsilon, beta=0 ) - - new_nodes.append(noise_node) + + new_nodes.append(noise_node_a) + new_nodes.append(noise_node_b) new_nodes.append(gemm_node) # **CRITICAL**: annotate perturbed edge with same dtype/shape as weight @@ -304,7 +357,7 @@ def append_cross_entropy_loss(onnx_path, output_path, label_name='y', logits_out if logits_output_idx < 0 or logits_output_idx >= len(graph.output): raise RuntimeError(f"Invalid logits_output_idx {logits_output_idx}") logits_name = graph.output[logits_output_idx].name - + existing_inputs = {inp.name for inp in graph.input} label_input_name = label_name suffix = 0 From f4e974f9d894a403ac0a4ce2d0a28f47b101465b Mon Sep 17 00:00:00 2001 From: JanCSEM Date: Thu, 26 Feb 2026 12:58:24 +0000 Subject: [PATCH 11/24] Fix shape initialiers for Eggroll --- onnx4deeploy/operators/perturbeggroll.py | 15 ++++++++++++--- onnx4deeploy/transform/zo_transform.py | 12 ++++++++++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/onnx4deeploy/operators/perturbeggroll.py b/onnx4deeploy/operators/perturbeggroll.py index b9f871a..b7d40d9 100644 --- a/onnx4deeploy/operators/perturbeggroll.py +++ b/onnx4deeploy/operators/perturbeggroll.py @@ -61,6 +61,11 @@ def create_onnx_graph(self, inputs: Dict[str, np.ndarray]): b_tensor = helper.make_tensor_value_info( "b", TensorProto.FLOAT, b_shape ) + + shape_a_tensor = helper.make_tensor(name=f"shape_a", data_type=TensorProto.INT64, dims=[len(a_shape)], + vals=np.array(a_shape, dtype=np.int64)) + shape_b_tensor = helper.make_tensor(name=f"shape_b", data_type=TensorProto.INT64, dims=[len(b_shape)], + vals=np.array(b_shape, dtype=np.int64)) shape_input_name = helper.make_tensor(name=f"shape_x", data_type=TensorProto.INT64, dims=[len(self.input_shape)], vals=np.array(self.input_shape, dtype=np.int64)) @@ -99,7 +104,7 @@ def create_onnx_graph(self, inputs: Dict[str, np.ndarray]): # Eggroll noise node (without loss_grad input) noise_node_a = helper.make_node( "PerturbEggroll", - inputs=["shape_x"], + inputs=["shape_a"], outputs=["a"], name=f"gen_eggroll_noise_a", seed=13, @@ -110,7 +115,7 @@ def create_onnx_graph(self, inputs: Dict[str, np.ndarray]): noise_node_b = helper.make_node( "PerturbEggroll", - inputs=["shape_x"], + inputs=["shape_b"], outputs=["b"], name=f"gen_eggroll_noise_b", seed=14, @@ -136,7 +141,10 @@ def create_onnx_graph(self, inputs: Dict[str, np.ndarray]): "perturb_eggroll_graph", [x_tensor], [perturbed_x_tensor], - [shape_input_name, shape_flat_name], # <-- shape annotations here + [shape_input_name, + shape_flat_name, + shape_a_tensor, + shape_b_tensor], # <-- shape annotations here value_info=[a_tensor, b_tensor, flattened_tensor_name, flattened_perturbed_tensor_name] # <-- shape annotation here @@ -147,6 +155,7 @@ def create_onnx_graph(self, inputs: Dict[str, np.ndarray]): "perturb_eggroll_graph", [x_tensor], [perturbed_x_tensor], + [shape_a_tensor, shape_b_tensor], value_info=[a_tensor, b_tensor] # <-- shape annotation here ) diff --git a/onnx4deeploy/transform/zo_transform.py b/onnx4deeploy/transform/zo_transform.py index 82321bd..666ab8b 100644 --- a/onnx4deeploy/transform/zo_transform.py +++ b/onnx4deeploy/transform/zo_transform.py @@ -175,9 +175,17 @@ def modify_graph(original_model: onnx.ModelProto, output_path: str, exceptions: # Shape annotation for intermediate outputs a_shape = [noise_shape[0], 1] b_shape = [int(np.prod(noise_shape[1:])), 1] + + shape_a_tensor = helper.make_tensor(name=f"shape_a_{input_name}", data_type=TensorProto.INT64, dims=[len(a_shape)], + vals=np.array(a_shape, dtype=np.int64)) + shape_b_tensor = helper.make_tensor(name=f"shape_b_{input_name}", data_type=TensorProto.INT64, dims=[len(b_shape)], + vals=np.array(b_shape, dtype=np.int64)) + shape_input_name = helper.make_tensor(name=f"shape_{input_name}", data_type=TensorProto.INT64, dims=[len(noise_shape)], vals=np.array(noise_shape, dtype=np.int64)) new_initializers.append(shape_input_name) + new_initializers.append(shape_a_tensor) + new_initializers.append(shape_b_tensor) if len(noise_shape) > 2: @@ -224,7 +232,7 @@ def modify_graph(original_model: onnx.ModelProto, output_path: str, exceptions: # Eggroll noise node (without loss_grad input) noise_node_a = helper.make_node( "PerturbEggroll", - inputs=[f"shape_{input_name}"], + inputs=[f"shape_a_{input_name}"], outputs=[f"a_{perturbed_tensor_name}"], name=f"gen_eggroll_noise_a_{perturbed_tensor_name}", seed=seed, @@ -235,7 +243,7 @@ def modify_graph(original_model: onnx.ModelProto, output_path: str, exceptions: noise_node_b = helper.make_node( "PerturbEggroll", - inputs=[f"shape_{input_name}"], + inputs=[f"shape_b_{input_name}"], outputs=[f"b_{perturbed_tensor_name}"], name=f"gen_eggroll_noise_b_{perturbed_tensor_name}", seed=seed, From 2c4e1b48ffa0b4a1f6ce1d084b4b82eadcff95ae Mon Sep 17 00:00:00 2001 From: JanCSEM Date: Fri, 6 Mar 2026 08:21:43 +0000 Subject: [PATCH 12/24] Modified microbenchmark tests --- onnx4deeploy/operators/config.yaml | 2 +- onnx4deeploy/operators/perturbeggroll.py | 9 ++++++--- onnx4deeploy/operators/perturbrademacher.py | 3 +++ onnx4deeploy/operators/perturbtriangle.py | 3 +++ onnx4deeploy/operators/perturbuniform.py | 4 ++-- onnx4deeploy/transform/zo_transform.py | 2 ++ 6 files changed, 17 insertions(+), 6 deletions(-) diff --git a/onnx4deeploy/operators/config.yaml b/onnx4deeploy/operators/config.yaml index 5cf37a2..6b9b987 100644 --- a/onnx4deeploy/operators/config.yaml +++ b/onnx4deeploy/operators/config.yaml @@ -11,7 +11,7 @@ perturbrademacher: input_shape: [128, 48] perturbeggroll: - input_shape: [128, 16, 3, 3] + input_shape: [128, 48] conv2d: input_shape: [1, 1, 28, 28] diff --git a/onnx4deeploy/operators/perturbeggroll.py b/onnx4deeploy/operators/perturbeggroll.py index b7d40d9..8b2eae7 100644 --- a/onnx4deeploy/operators/perturbeggroll.py +++ b/onnx4deeploy/operators/perturbeggroll.py @@ -48,8 +48,11 @@ def create_onnx_graph(self, inputs: Dict[str, np.ndarray]): perturbed_x_tensor = helper.make_tensor_value_info( "perturbed_x", TensorProto.FLOAT, self.input_shape ) - - epsilon = 0.01 # Example scaling factor for the perturbation + + normal_epsilon = 0.01 + uniform_epsilon = 0.01 * np.sqrt(3) + rademacher_epsilon = 0.01 + # Shape annotation for intermediate outputs a_shape = [self.input_shape[0], 1] @@ -131,7 +134,7 @@ def create_onnx_graph(self, inputs: Dict[str, np.ndarray]): name=f"eggroll_gemm_perturb_x", transA=0, transB=1, - alpha=epsilon, + alpha=uniform_epsilon, beta=0 ) # Graph diff --git a/onnx4deeploy/operators/perturbrademacher.py b/onnx4deeploy/operators/perturbrademacher.py index f88f562..9949188 100644 --- a/onnx4deeploy/operators/perturbrademacher.py +++ b/onnx4deeploy/operators/perturbrademacher.py @@ -54,6 +54,9 @@ def create_onnx_graph(self, inputs: Dict[str, np.ndarray]): "PerturbRademacher", inputs=["x"], outputs=["perturbed_x"], + seed=42, + eps=0.01, + idx=0, name="perturb_rademacher_node", domain="com.microsoft" ) diff --git a/onnx4deeploy/operators/perturbtriangle.py b/onnx4deeploy/operators/perturbtriangle.py index 241f549..4e4af07 100644 --- a/onnx4deeploy/operators/perturbtriangle.py +++ b/onnx4deeploy/operators/perturbtriangle.py @@ -53,6 +53,9 @@ def create_onnx_graph(self, inputs: Dict[str, np.ndarray]): "PerturbTriangle", inputs=["x"], outputs=["perturbed_x"], + seed=42, + eps=0.01*np.sqrt(6), # Scale epsilon for triangle distribution + idx=0, name="perturb_triangle_node", domain="com.microsoft" ) diff --git a/onnx4deeploy/operators/perturbuniform.py b/onnx4deeploy/operators/perturbuniform.py index e3cd3f2..c0fc118 100644 --- a/onnx4deeploy/operators/perturbuniform.py +++ b/onnx4deeploy/operators/perturbuniform.py @@ -57,11 +57,11 @@ def create_onnx_graph(self, inputs: Dict[str, np.ndarray]): name="perturb_uniform_node", idx=0, seed=42, - eps=0.01, + eps=0.01*np.sqrt(3), # Scale epsilon for uniform distribution low=-np.sqrt(3), high=np.sqrt(3), # dtype=dtype, - doc_string="y = x + epsilon * RandomUniform(x, seed)", + doc_string="y = x + eps * RandomUniform(x, seed)", domain="com.microsoft" ) diff --git a/onnx4deeploy/transform/zo_transform.py b/onnx4deeploy/transform/zo_transform.py index 666ab8b..79b43d1 100644 --- a/onnx4deeploy/transform/zo_transform.py +++ b/onnx4deeploy/transform/zo_transform.py @@ -236,6 +236,7 @@ def modify_graph(original_model: onnx.ModelProto, output_path: str, exceptions: outputs=[f"a_{perturbed_tensor_name}"], name=f"gen_eggroll_noise_a_{perturbed_tensor_name}", seed=seed, + eps=epsilon, idx=perturbation_counter, domain="com.microsoft", doc_string="a = RandomRademacher(x[0], seed)" @@ -247,6 +248,7 @@ def modify_graph(original_model: onnx.ModelProto, output_path: str, exceptions: outputs=[f"b_{perturbed_tensor_name}"], name=f"gen_eggroll_noise_b_{perturbed_tensor_name}", seed=seed, + eps=epsilon, idx=perturbation_counter, domain="com.microsoft", doc_string="b = RandomRademacher(x[1:], seed)" From 5f4d683dafa01b8f9520aaefe33f1a8c6c4cdbee Mon Sep 17 00:00:00 2001 From: JanCSEM Date: Thu, 12 Mar 2026 12:57:08 +0000 Subject: [PATCH 13/24] adding quantization support --- .gitmodules | 3 + DeepQuant | 1 + Onnx4Deeploy.py | 21 +- gen_noise_tests.sh | 7 + onnx4deeploy/core/base_exporter.py | 37 +- onnx4deeploy/models/__init__.py | 2 + .../lightweight_cnn/__init__.py | 3 +- .../lightweight_cnn/qlite_cnn.pth | Bin 0 -> 97546 bytes .../lightweight_cnn/qlite_cnn.py | 109 + .../lightweight_cnn/qlite_cnn_scales.json | 508 +++++ .../sleep_convit/qsleep_convit.pth | Bin 0 -> 335978 bytes .../sleep_convit/qsleep_convit.py | 358 ++++ .../sleep_convit/qsleep_convit_scales.json | 1764 +++++++++++++++++ onnx4deeploy/models/qlite_cnn_exporter.py | 238 +++ onnx4deeploy/transform/quant_transform.py | 280 +++ requirements.txt | 2 + 16 files changed, 3317 insertions(+), 16 deletions(-) create mode 160000 DeepQuant create mode 100755 gen_noise_tests.sh create mode 100644 onnx4deeploy/models/pytorch_models/lightweight_cnn/qlite_cnn.pth create mode 100644 onnx4deeploy/models/pytorch_models/lightweight_cnn/qlite_cnn.py create mode 100644 onnx4deeploy/models/pytorch_models/lightweight_cnn/qlite_cnn_scales.json create mode 100644 onnx4deeploy/models/pytorch_models/sleep_convit/qsleep_convit.pth create mode 100644 onnx4deeploy/models/pytorch_models/sleep_convit/qsleep_convit.py create mode 100644 onnx4deeploy/models/pytorch_models/sleep_convit/qsleep_convit_scales.json create mode 100644 onnx4deeploy/models/qlite_cnn_exporter.py create mode 100644 onnx4deeploy/transform/quant_transform.py diff --git a/.gitmodules b/.gitmodules index e69de29..aa121bc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "DeepQuant"] + path = DeepQuant + url = https://github.com/JanCSEM/DeepQuant.git diff --git a/DeepQuant b/DeepQuant new file mode 160000 index 0000000..19fd0aa --- /dev/null +++ b/DeepQuant @@ -0,0 +1 @@ +Subproject commit 19fd0aaafcf2a4b3ad841f893d19e46ae2261907 diff --git a/Onnx4Deeploy.py b/Onnx4Deeploy.py index 04b2e84..83f7648 100644 --- a/Onnx4Deeploy.py +++ b/Onnx4Deeploy.py @@ -34,6 +34,7 @@ def list_available_models(): CCTExporter, EpiDeNetExporter, LightweightCnnExporter, + QLiteCnnExporter, MambaExporter, MIBMInetExporter, MobileNetV2Exporter, @@ -140,6 +141,12 @@ def list_available_models(): "input_shape": "(B, 1, 28, 28)", "classes": 10, }, + "QLiteCNN": { + "class": QLiteCnnExporter, + "description": "QLite CNN (Compact CNN for image classification)", + "input_shape": "(B, 1, 28, 28)", + "classes": 10, + }, } return models @@ -301,15 +308,21 @@ def generate_model(model_name: str, mode: str, output_path: Optional[str] = None if mode == "infer": onnx_file = exporter.export_inference() mode_desc = "Inference mode" + elif mode == "q-infer": + onnx_file = exporter.export_inference(quant=True) + mode_desc = "Quantized Inference mode" elif mode == "train": onnx_file = exporter.export_training() mode_desc = "Training mode" elif mode == "zo-train": onnx_file = exporter.export_zo_training() mode_desc = "Zeroth-order Training mode" + elif mode == "q-zo-train": + onnx_file = exporter.export_zo_training(quant=True) + mode_desc = "Quantized Zeroth-order Training mode" else: print(f"โŒ Unknown mode: {mode}") - print(" Available modes: infer, train, zo-train") + print(" Available modes: infer, train, zo-train, q-infer, q-zo-train") sys.exit(1) print(f"\n{'='*70}") @@ -323,7 +336,7 @@ def generate_model(model_name: str, mode: str, output_path: Optional[str] = None files_to_check = ["network.onnx", "inputs.npz", "outputs.npz"] if mode == "train": files_to_check.extend(["network_train.onnx", "optimizer_model.onnx"]) - elif mode == "zo-train": + elif mode in ["zo-train", "q-zo-train"]: files_to_check.append("network_zo.onnx") for file in files_to_check: @@ -420,9 +433,9 @@ def main(): "-mode", "--mode", type=str, - choices=["infer", "train", "zo-train"], + choices=["infer", "train", "zo-train", "q-infer", "q-zo-train"], default="infer", - help="Model export mode: infer (inference), train (BP training), or zo-train (zeroth-order training) [default: infer]", + help="Model export mode: infer (inference), train (BP training), zo-train (zeroth-order training), q-infer (quantized inference), or q-zo-train (quantized zeroth-order training) [default: infer]", ) # Output path diff --git a/gen_noise_tests.sh b/gen_noise_tests.sh new file mode 100755 index 0000000..67a2a75 --- /dev/null +++ b/gen_noise_tests.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +python3 Onnx4Deeploy.py -operator PerturbNormal -o PerturbNormal +python3 Onnx4Deeploy.py -operator PerturbUniform -o PerturbUniform +python3 Onnx4Deeploy.py -operator PerturbRademacher -o PerturbRademacher +python3 Onnx4Deeploy.py -operator PerturbTriangle -o PerturbTriangle +python3 Onnx4Deeploy.py -operator PerturbEggroll -o PerturbEggroll diff --git a/onnx4deeploy/core/base_exporter.py b/onnx4deeploy/core/base_exporter.py index 5c2f83e..37734ad 100644 --- a/onnx4deeploy/core/base_exporter.py +++ b/onnx4deeploy/core/base_exporter.py @@ -25,6 +25,8 @@ from onnx import helper from onnxruntime.training import artifacts +from DeepQuant.DeepQuant.Export4Deeploy import exportBrevitas + from .onnx_utils import print_model_info, randomize_onnx_initializers from onnx4deeploy.transform.zo_transform import generate_zo_graph @@ -304,7 +306,7 @@ def _export_to_onnx( onnx_model = onnx.load_model_from_string(f.getvalue()) return onnx_model - def export_inference(self, save_path: Optional[str] = None) -> str: + def export_inference(self, save_path: Optional[str] = None, quant: bool = False) -> str: """ Export model in inference mode. @@ -335,15 +337,24 @@ def export_inference(self, save_path: Optional[str] = None) -> str: input_tensor = torch.randn(*input_shape, dtype=torch.float32) print(f" Input shape: {input_shape}") - # initialize weights and biases for testing - for param in model.parameters(): - if param.requires_grad: - torch.nn.init.normal_(param, mean=0.0, std=0.02) - # Export to ONNX - print("\n๐Ÿ“ค Exporting to ONNX...") - opset_version = self.config.get("opset_version", 12) - onnx_model = self._export_to_onnx(model, input_tensor, opset_version) - + if not quant: + # initialize weights and biases for testing + for param in model.parameters(): + if param.requires_grad: + torch.nn.init.normal_(param, mean=0.0, std=0.02) + # Export to ONNX + print("\n๐Ÿ“ค Exporting to ONNX...") + opset_version = self.config.get("opset_version", 12) + onnx_model = self._export_to_onnx(model, input_tensor, opset_version) + + elif quant: + # load weights. + state_dict = torch.load(os.path.join(os.getcwd(), self.config.get("weights_path", "model_weights.pth")), map_location="cpu") + model.load_state_dict(state_dict, strict=False) + print("\n๐Ÿ“ค Exporting to ONNX with quantization...") + # use DeepQuant to export to ONNX + onnx_model = exportBrevitas(model, input_tensor, debug=False) + # Save onnx.save(onnx_model, self.paths["network"]) print(f"โœ… ONNX model saved: {self.paths['network']}") @@ -499,7 +510,7 @@ def export_training(self, save_path: Optional[str] = None) -> str: return self.paths["network"] - def export_zo_training(self, save_path: Optional[str] = None) -> str: + def export_zo_training(self, save_path: Optional[str] = None, quant: bool = False) -> str: """ Export model in zeroth-order training mode. @@ -679,5 +690,9 @@ def export(self, mode: str = "train", save_path: Optional[str] = None) -> str: return self.export_inference(save_path) elif mode == "zo-train": return self.export_zo_training(save_path) + elif mode =="q-infer": + return self.export_inference(save_path, quant=True) + elif mode == "q-zo-train": + return self.export_zo_training(save_path, quant=True) else: raise ValueError(f"Invalid mode: {mode}. Must be 'train' or 'infer'") diff --git a/onnx4deeploy/models/__init__.py b/onnx4deeploy/models/__init__.py index 4e3f577..0b45494 100644 --- a/onnx4deeploy/models/__init__.py +++ b/onnx4deeploy/models/__init__.py @@ -7,6 +7,7 @@ from .cct_exporter import CCTExporter from .epidenet_exporter import EpiDeNetExporter from .lightweight_cnn_exporter import LightweightCnnExporter +from .qlite_cnn_exporter import QLiteCnnExporter from .mamba_exporter import MambaExporter from .mibminet_exporter import MIBMInetExporter from .mobilenetv2_exporter import MobileNetV2Exporter @@ -19,6 +20,7 @@ "CCTExporter", "EpiDeNetExporter", "LightweightCnnExporter", + "QLiteCnnExporter", "MIBMInetExporter", "SimpleMlpExporter", "ResNetExporter", diff --git a/onnx4deeploy/models/pytorch_models/lightweight_cnn/__init__.py b/onnx4deeploy/models/pytorch_models/lightweight_cnn/__init__.py index 8ebc582..ce05964 100644 --- a/onnx4deeploy/models/pytorch_models/lightweight_cnn/__init__.py +++ b/onnx4deeploy/models/pytorch_models/lightweight_cnn/__init__.py @@ -5,5 +5,6 @@ """Lightweight CNN PyTorch model.""" from .lightweight_cnn import LightweightCNN +from .qlite_cnn import QLiteCNN -__all__ = ["LightweightCNN"] +__all__ = ["LightweightCNN", "QLiteCNN"] diff --git a/onnx4deeploy/models/pytorch_models/lightweight_cnn/qlite_cnn.pth b/onnx4deeploy/models/pytorch_models/lightweight_cnn/qlite_cnn.pth new file mode 100644 index 0000000000000000000000000000000000000000..908305ba506e13a9228cb4dc8ea8390f5801692d GIT binary patch literal 97546 zcmcef1zc21`^T{rTkH;OMWqzj859)*6AKe51w~;Gi;a!l-QC^A&R}JCHYg|}JUk>w7Z%Yw%C@->93q4S*9!~M z*=lVn`fAfu@M~$14(lBmt!o(_&|Bva5Tx_h!oPKkjxq=v{yVHsfLJ`?xBVl9h<^PY zbRoT?B80>zqk;m$!+Ll359<{f?$9S7JUT?1Hb@sC1cf^IN9)4Eqip?!kih7$@L>PM za(x_awdpEU@XKU?M;V?B=pLd?pHRfsJ3Jyl*DB#RZM7Nv(ionpuFdF|(eVF+qJsn6 zt7|j)q|s?JxAd{8SaEEqSdT0^t*x&%t6w(pnTXzfYB(e`h&EfO0TY@&G(o0@HoHhGNkAYBYLrqkx>WC6ob`6f0Cp z(PpJ}+A_Xcd%xxwNJO;G&=1szq^waS&f0RyNXm1m^ zxT7{$DIVgJO{eW<(Dum|tL-lKqtFDkFrBuCueK-lBPaFxQMhqLPTF3|ht{wcAzqjL1G*+ ziZ}-Av_pKgL$N10OFe0rv5u>DxKd|?Pd}Y@q(MiVkFpuQY6jU@u~QlTP5k#!39o*% zPCLd|I~F^Vb3#WNXRKXQJ6@?hL7b2#8nnd;X_DBHCMT#((P^jpYNugGa#pV+O*f9H zmUf0RqM2fqW+_^2woW_8S34IwlCz2(X`XQ?Zrb_EP!@=xEL4QDNT*%wt6hQ}$yu$A zwA3gL$LiW;$~cybaja0pu~Mg9<*Qxom&GuDbqkVu(Hdis8rro=k#$XLXVhue!>jgL zGD^F_XNgamQQD0OuX~eDyV+N}1(t6HMfj3!HP&;~Zd2-Q7kkbQkc-vs6noCD1gYIR z?H*t4UO%6N;8gpL?K2L?NgJmOM=!P!Q?wD+Y2$sh`{7*~YNXy*?0|6`&f0^@I1Y(% z99G0}M5jIKt34JSqdgAq%fDBMELqpkUPsM0YL-Y3u5uyx>bL|f6?DEuJ5p^W?k<>1twkI`O8c&LKKhaCI|ga{G-p?&?MLVD{Ah9g{i zF-ChSp;%tyVrU?=mt(Y75*{gM;Sn>sS7Wr-5(?F@R46%PqP-rYy^&D9jy2`QAu9PD z(avaZ#%OOP=uxZmltxK=J4SmaL9Ahl#SAYoajCAo8>79KAlpGjS%+?-5%bqY_=_ZH zjQcU#2dMx`Y@?v?fL@UapocNqzY;)g)vQr)NO(Yh|Gq*%WMqh-eH5d8oB#<(HHbq3 z5?b#`jP_{)PaMOF;fO=VA-S$??78vnCl2O5VIh4J+U{A5_U{CYIH#r%qqO$g=P}wB z3Ci8oRd$Hd1?Zv-{YTVv@h9W}L{3DgFfdFI4)T8sxqy}%Y3)*8pFihv)H!N5e3RAm~2UvE< z3P?Sc!7C@Cg}gw`htv!PDx6&i`Qax8@F%&YpD2x+P!OIfgin<*_f%4cB@~8oMX;QM z*>cIPM1`UtQ4A$SYm{*8lCXsPjRPbTpCuFr$r32(@#m6Ct0j~Ky;7)GKV^Ed+Y(BH zav4+>d*&ZjUKy#ZoYZg$_P{Ee3aliaODG4d^2lnXTD^>HmrwyX6_Fz@r_^abBaXs( z36%g@8Ij`ZTODLVe@F8rQ~^>|B#AyhB_yLs-T}m`p}4p-Q%hX3DdsGgP#wTE5UlFi zCAcub5fDy@5Wgp6I=mP6EUK}RIO5ok?ojJ3xIA2RJ9dN z*HnT#&^(aVOU>$;(#$T_H4p*82s)-p1h||NDDXUy7m>O=vvsZwG1S2r0#YRgapF<9 zGr0yLG5DY>=?-UBkrbq||=@66_&rTAClJXr{b#jvAfJ^8Gu zCo$`~1RzHtS>2183ARfIjA&$t2goTJUCOpgsGU?t(RcL$Twlb=dkj|g3L_jkBB38} z`XfhP#;D3sIIjUf9Ee198xq$siS-o*0WJn{y;Xg0rZ}_f@WBwm5DY;!w`#O8hJY(K zVJHBHAy7O9RIjfl1)8n(a0p@q1|gmyq(Ttldsf=7k${awY@h!N)@%!mf>1_dDDwWN z8t>i66$xX2H5OUwPQFGg69+a9$m3IqjGfJl1)BimiAa{$!D_TGlBJnb^k9S;~PzDn=1)x(A+BBumBra?k5T+wRws}^zq=`)uW&mI&0#ZB_OaMwBHVas@ktOe8 zTUi-P{z#Yuq`62+@p91;N#Vri0dqbwWha)9$diziA}oN%7UE;F6H`8B#)&P0f{U?W z3G8e}1&vJ25-772%T)Yz8HI0I2Bnu{Y573QB2g}F<|t?d=&iI;FRA-j1$wJdujZfB zE^(!wc#}?816pfQ%gcH#rH5Dtvg=XS?T=(lZNdgn+lXpaenU-~B*6)60)fpaQ0ET? z93lgR0Kk@7f3cDbT-55p#Rl-Pkje9_Q zFKWvtN-3!#Xe$lWKET8wM&1qjO$;a+-yP9|CPU2@f7&cbH5~%Fbu(NOH>7U+mnDkh3N3qMB__Wyxr3fz!?lc zHlFG?v|?Z5EMU(eR(2z*VZ|vz>2=Qo@&Y1Lp{Y%fChz(p1aJuh_~Xx&nF$eJsXUGq zbv8SmFGCntFpN<3yPRn|$m9Adz^@@(-ket9y(fMYzy@9i&Qa6RsW4HlEfz82ayLTBA-%O4%)=13J*c! zFVt}VjX+E^lxFu4$UR26TE8J@$z=*pKwR$oJjH&~+luS&>fQ+Nwy|G~1d&ys6mDf=kt8QQ;~ z_RcD`q&D?EsC_^+m*20KiAntkN}o`vwzW!1i~1Qvzo4k>qJQ0VrUvyZXnjL1*Ix!` zsh0}hLFxxe{kpD}q!bF_7wZRdnP1tg(?;4-n09-hj8Pl>mklRRZ+BxQthnXp_=EQeWlLS}d-3qDgS=`&`O zZJ|_FEG6zI{K=nQB4h)B>?k0f!vBdtl4JcGAea*c#c}*cf<`k+E>O#jYIRehrkHK= zfUX_tiW?qQM=9%?*)q!uqaB&z_wD)Pl!;VLyP^3iDt4nV7hH1Sq=s?p5Ww>m^n10xU*u+)yg#5oI& zz;r@pRO&O$*4-K6aKSi4->y>KjjgULfNCNrFqJ?ij<*(I-BJUKecAANGQAh=4pLLZKNv)f}Ia(o~I}GGnh=K)IG!PD)cXE|=`p zXi>C&O4BCWA;Axn z+oQ6$!1=?WnPu3aOQEKyERU?V>Kg@kfqy@S!qCX}7up zG60d%Re+QuO$}Edkb;mTrKws=l0IEvxq?AF1jVKES2e_wHC^2R+#SJEnyPwWAruf{ zh!B_3DH&DDhD&O^dH}5_(xgiTs#Hj}Ug3c0g&653ueun8`RWb82n4Fyil*x+Arfd> zq)BP2s#MREW_D37Km<`3fpjdSZUnfV6Li3fMqY%Py%cMVJhOG~12Oc)7y?oy22&f> z52*c-8vXyEnyvo;h-4r}BAsih^3IK|*dRc~AWGE<*9c`|#s&j;Na}#GpP8{^LjgPt z!BT#QDvgU^X>t|~*>E6^K%#UhR$U?vDX?TCffhKM5+(p>B7#!7F0=$vn6pWMoQ%jk*bkIf zzlACA*i?K>x*Tothq5R6vs{H~P;fdHlr8|66in7@%>a#=s3BcU`~!_7=UKBrb2e&< z2V@ETUs>?iWv?)* zl5D&-0cSIErIIRJS2bS5CqpIl%}fAdo#tEU5D?05O!h+vbj~G zjWGmVxe2=fxEq0ys*Z=GK(n>p13~P?AOce-2(e!)uCWUH02_zcKK~c2*%r`4C=5f9 z_dnHm??$dj;J}JUmb#O#5zEAZ?FaIKR3c+%Gh@LH0{IY<<#n(c?Tch-<`g~HVL%>1 zWY^T`T{x7%gdGLwF@#E&hOM5$lDM$rKsbQ}+2&c~HRERhe?hR6ruy6cE9uLzP)GT<6^V>r0sjs0T~&CGxLh=9VukPh4$vP6mAB_q zYiwLL3jYD@C&HvORTUc;VJ7d^CLN?%82*`%W)ZG_t-^bkJ{6+(%mJXB2vYUKfI|u#XD&eHMwIM0t*?{OyeQ-W zf*lg%`v@u#OnPcw;N?SJ_1~BjlGx<@AW{HDhWjHh4^CCXw6DNWU43E9jE<)CbNEGzpgxh6)pE9J6gjA{i?t7w&4 zQkz-{)GDKzi@63gZ^2FO9aaIQs;E@kS|z1LbpX+7C@QuO0#(czsz)CEPQG*!!8OdjEaD-^DYg)WUj4R!1r8nwg0j1Ib5qQqozf36Pp1NlH^y_MqhYf~DEjc$=*mP?{q}JaJQ_<%}rG z*{uZtTOv?OQ%z~LOlLk{;Iu-Hl%{GOM_OINrd?~0ZiCXIZL+@2l+x0+jM+5Rwt#Ji z*d{7t1wR0^M}TgtfN-X)vz7s$?BScD5O?e-1K!{@Bq6^^s z5vOANp?gp4Ekajd1t3dGQ&pjArYy5YG7$Jd$QN>MvYmk4{Sn23iQxq%>7EYl}3q z_3Z``bjJw911z=cE52KWffqu78HP+LO;ycWn=sARy$8h66XTE)_Ed<&*y@G@s274% zoLG%OCXTl^U?Wlki+$Pfc`{8?jRdR~v7)h0jV@*Qv#wyPLI7SA@}x9X6<#A&69*9x zbwG?pqLij;9Wlu!Lm!aui}F&Ms&(?Ru}DZ$?FZ`pQ9Wh7-b7uwF);u@0}&)1^H|>x zESWB05MW{uld_ebC5*z`4F=#41d8@dOjC6fhQd#V8T~{&9dZ(e!%s%wPoy+eC#5A5 zM#59E_*5B#GL%<7Wo8Rx6qFl{<)k!KBlTpjC<|jiVk}Bfv*={k<)&$><3MseN=m2t zzaN#+QeT(=dJ|Ev{-1}aG;OjS5+;H2WKX~3F} zEa?iVm2W_9F_Z108Niu|9O>XgWscHr%>v|XL`qiyQjRn=Tyual7fDi@s0BE+S1N=B8k;S%a5)lf8EOMtc%Y0{+v zRVsw8NLU7#<%p4P@~Vqbn6DK8T!}zcTM^eciES&a0@`Y%NolI8RL_)Vc2T|tB3O$N zNXJ6zMu6)%VIA<+BQHYDUWzqFp4mEYfEYGn3{sk^x^*_OM8YPZZboW!s?^+sYPSAc zAd;;ZiFB^1O8t!-lCTX>+YzPeglmK{F=IObyfbyc*w4(^v0VV(jbJIiLzTuwurxV~ z$JBd(xEG1irC4=|IHbUm?E_{UGQ~~AlzeQGm@+*87y_g;Rck9{Vw?mH`thir(lKD7 zuQX=+0d)XT(tQPMt76GB2?qgm2tg@b7g_=-%-LZ;9zkRt><7xL-@;LN>=-^KU5<8g zP(EhHY8{7yC$OM&0l=hSa$Z3=2^yzRL%Nvw2O3GvvrdEN8PpUH$P)U$QH7F=%h+w5 z1;KMDD5a_XNs#7Ek#HWQFQBw^4=&}>X8hJgfL%%rSW?4v8DLisCXX!Z-hxHPbrnF@ z5F}5r>VlM(>pC!RAXA>9Qp_~*(;Q%16PqO1q=(!E!#@*p7g9VFOaMwBmKIp) zkR|V7TUi-P{zymVW2!RAG+9(A3FJ~p z7XGm5&97M8@iXuYvfksaJ0~oy`%TP6(B^=Tv*IxNH=h z0p@}*DNR+y21c04yLAP0O+<&Q-?%awdz<3z)`9@sFaX(js^8FxeGPZOdLUMIBdTG= zi9>ngQv*nX$W&-*6Qs$zrVxN92Jpw9tKS>PiaMJe&$S_pIv9qOrmA9ZH*E)bTzdh$ zF2d!_X%*hP^r;ZNXFUMbN06!?1{_l0IK2VY08z5zw7yP8-vgl`5E>ytzK@_1!K9}) z2A&V{s{h8Ul*A@C0g$_L6Ah)=Z2@vEQBF!zO-{~| z%M^S;tQCs=jytg=rm(E7LADLb=EAm@GZclk@K8H^sI-}f%-B^wDApc}NolH12EOz+ zwHlwmb$}8bv4oVSYO#cD=7dgAwlkKMeU=59T-Janv{kpD}q!b;# z2Z;7WQ7KK;au<_FxDXD7dtu=WhRTWeyPbsI@RJDqiIk>lbiqJ~gr~IlR2iwfp)V$v zrYbM$A2WK97n<+P>Vsex+zhUXAEI5=ng?$al^yvC}mwUGtp2W4O5+z zbe0+pq!CDx(o~f_D7n61G?y#tWRyTEj08$7Qp6KCHCoPyqMY4E0dO<|r8L!)R?Bqe z8v~rN$dS@it>Z|mOW3p<2h!tFTC`2px0zB}I@dOPc0U2I6A{})Wvnm>0Fx0Q9zCd8 zFR6Cosr?jSO+}WtL8KB(zAHTqAkz^drKzU856Jfd6ux!_;ASFD#r8w@p4eN2S-_f& zEGbP@g{qmd%o@o#z@LkJ@m9D>?+^Ltw1jy;n~yXpO;yd>BF$`l7eE9HF#_=bOYQoK z?^a>pg+;(zj7%v_Rn1zPFwNF|3B<7!MjG&as;V3u^NF)9PbLiu1pOq z_GQE8$uv!M6<}8*Ry6jh(WRs`)iuCdi##b!RfX4x)x<#rn+sy765HUka)~veLt{dx`b_j*^Zc$t@JEm z6y9zJ0Cyr#v}fWOkh8E0ezF^XA|<0bD~*@12cFuCPl@&n$}68Tv(d2+%Ee(hDH+vB zJ^5r*JxDN=pk~p@uD(r^Q8`G)qoj0p|NBuHE$D^)pmzZE>i>C&N^2$?1mPekA3|ku z8I!c#ldZgYWixi_FtCoK0xPMZItr{~$dWFKTKNX#7BkrtIu4u@$dQgWROTp6)k#2} zLZo!ypK_$BtvU^)Gf0w>QLQB@-H&h<#LuC4a0+{#Qd}}0X06qE0AE0`l#HrItrEb( zML=9agt%Z%$*4+NH>rlAy}AsvD@c>B4yaNg*0Ny~Ls;!8t zn8dafZUXHV(xhZmRjOx7GrRP@4H4YI2&BUwbtAx4n{XF+_mCH%W-rAWBhPG|??Vg^ zFa{|ZRoyy^?^tQJ9s>0*q(-Mo%}uCg>;DKMd5n=rrw;(&Q{2M!y8&D} zdc6g}KM0VLQLU|%i9Hhj1^suZpVBd4qOY`H?*a7zQPQmhYpY_(83`W&^a(*J zT^CvcDJ1F9_$Co3*N$4cH6UxXe2qcvN3qg7$J?}p9z=f#bdC9{;#ZevL4XbZKValbSNk#qy9;d=1q~19;7p% zv~=4o<$n}!Eq^lSAM`1K%BhCU}djRzAE|9Nhk=MLdcPqF{*MD&Z{sGiy%?m zhBRF{2}J={3~^F2sygq@6lZoFUK~Owfg#A|R*g2s5OC!tlmuWY1V*Yl9+Cph*19wV zQ3it$&tOvW=`@}WFbESPB-jJCEMoioU$ACdpd5r!9z&7$Kh=2eMy^Px0IZ703Q@J% zMl2HtRtd9Gp zt_E8@g(Y!ejzDlif^74wY)KQFBsc@W1pz4@3MK%h4|4@pO=QV?*j84?l0On^0m%(X zDPAsGA}O4hJ1{+vDLXOcg=j$oj}d y_Y?F*8n#LP1X~Si*ckr3VpeLzy~QMoLC? z{;gzGFDP9XOUnmZMxiQ+iUoYP@Q47Nzb?XG?9{Lhtp|Gbt<+2EKDv(yM})Xb>6QM&-dA0hI7fYpJ^t5c(eoX{0e0f-7#p+2~VF+!QF83O?v zgkUKd^|$+1(wAeQj`D9S(iMXNAAT3;;<8Z)1y~ruq-0bT z8yI0G@3se^dm=hq{l=Bi*xM9uHyi@!g#pOMQ~ick>}&J}Yy@IuH=-I=oH&$6jgf%V zA~F@4+5~Cxt_27n3Iq7#&(-gZV?~|Kj%OW&5shI;$*3y!cGGr{$8{fo_eHq8IjzEb zmp&Du_v{Ct{s>a_!+=8y9OnQ)4MddeIIXXf(f2?Y1cVqQ$oCOcBAE2l!N41Wyz0L( zDwGgF#T~|v|iVnXBL>Hr|l#FV* zi^(HgSOSHYVqwF-z%6cPVHx~nIsQaSMm4%%Agq9=R^n4qGOE#2W=7H~D7PBR<&?N` z%BZjgo>_~}NXe+q@-t?X*FmZESW4Va5bvccOC@`J%HFIDr1F%062sI@#sO#dP%htZ%ZEr z))8ch8$>FxQZj1F`+$5eK;dhT1MURkRBS(V?}@!dI0>v%$dZy#Rj8UN z%dC++4g5367jK2D^!|{KPD?lov~x(4l2O&HEz-=^_dG;!0V5C(u+*-v_-+*jUbqO% zOURUxQPr%q3Da!dFGCzxFb*j#PlY&)t?pF-T|684Jw}jt%wv5&uw=S~CxCg1n3S#bEMXMh?im38MqpY) zy}AVn&*8UU;BV)E-!?3|@z4JeUcw`<@R5>99x>ycUPGZbSg3N+LdjaEw^05cEH56e zcMDR+mHg#p)9llKLGK;viTh1|qGx2B-hB3-y~+%QS;_{q>_}6u@o+Jc*inQWfXay|@eEy^ikYI! zE}wG&J~!gU<7}1jxa<=00L%_y;z4I>!OYe+F9eVe0}u~)RSv+!vk3VCSpbpZ*=p({ z&DOjigi#2?5O=oJ3B$;|2!(-D1Uc$WrbZkSlT;L_#ZrNagUF0WDh|{VNELT*)M-f)aqVYRx6gAvMsczG00 zQFk{HSK6Zrz^RBFaUb03N?5WjLM5P7MoNm7be1Rzb5sR*Rgov2AsK3=ymu`)z%$kG z8SxCsj_AIYXjA z@kC1Etf?+V>4$0qunqzfr=Ao8O%0J3aOxt*J4IbW`p_g?y$9*~C@pSvTHo80(vod3 zYl*x8+W@iR8IqbQ4=85PBcZ0f#rj&M6YjUH!x~L*%dVbNK=F)u0T|U zC~Q$Pz%@slitT5*;t^T^t0l6;GbEMXl_|^Yg4q}Nt&pE+IMrxcZ+5YLd*jzH|i$}=QCV75o5c!re9OtW?G0C9B0I1+an)p*xNu0`ks zpw0+Vabh(Bnb@N)fb~xeEcRtH{-`Tp0}z|IY*nLK5i3oMqCpA-UJ&xcGbFXU6AmJ9 zNWnl1L85quWIZv7Md}9f-BCW#fLYmqCN@S01@$mgPg$=wQCFIz9sufzpu`4lh-t@JEm6gH_h03#5XXp@weh=oXaMvKoR+9c&OW^9rGrJ}G@(S&|wR7$z} z6?9M_8Vi*BMFDxaEcAh*eX(fbQIJKJT-1zv=m%o`tr1IV5C(wQKooQS?Mj(=fI%P> zgF<2wnPHgzP8E&LR+P*9!Js(=HO0%Tzo}`uk{=3E!%#}hyG>R~n)G0?J{(Gqz|!iGA2oK<`}?^MQlWh8_ZnoOtHq9 znd2aa@fbtBl*a%$IKl)Fo`}MUXE7;hQxH~Kn@PZ!jEuxRnBQb1%@q=+fZ$XV6!TPm zdjKRxX4D0fWMfVP!gM4gnsTc{mY0!6>n>piaAqPWar;|sj>-CN7Eos+wR=k250_U) zwUaBYNYk4G?77HJTy&{kV>wf69uVgvF>xEu+Z9PB?{g7w7b7<; z)wuYg6>oG2(3c`T(Fv;GzKWfNWx!mH%tWi88dIDdl+JSn@Kz#E?G`rSnY`asKwpjY zKl=hDoj~P*DUVl0ZRH;;-troVVJ*fGqH-TIZ2@`wt^@LVBq#3ss_?F*Z-8h9HvnZL zQq-IAanynZ+ytD>$Vs$-*4IXUg@(J4!WNLr)^D4=k^vi7k+m5iRzv>f79P18H z*og{>=R20eHL;DtE)dv_0yTdXf{B3AckThLy{ILo(j~jBu;lrKeV`PFO20G^mXs9k zQ4g97HM1u~C#TT~9G-~BCrX$;Va7S`hcXASOyMMDjC|ukc>EAPF8=vKh9;7Ou$0tv z7z!T2f{FI$mj%su#-kv0?9Zi=I>zH5bpoXvf3rF!e(@y8oI)95m5kCYo(9D;sF-MF zf8AWBUhymlokJm~3_W| zJ;YO_-_tJ^|Jpkn@mF$m7mM^7-d;8Sl-{93r|U69b9w}~AGDHWdD)4-nw^HuACQ|K zx^{~D2UVu&df$zEIQIolcbAh@m3PsTdD40|z8FW{Z>EV~_hlmed2)j0NWT4iOyL6Y z;T8YUHyt#MPn@%nIQQO2Hx_?JPuM!qssEg%3$Kl0q-t9I$eJfus{!7g9s4bznG1B{ zCkpQ&-vj=kr7~<`HJwMY<}Yi}cKOEB&;eQXt8Q=L4NolcSTy?|?h#;5C(K{Yw}vdH z**fQsn|IxlPWd^9k%B*o=NMmJdw4V6?CA(@7r21;xpbLN`hJM~+jKYWvhgrm{h=ri z3K&b@o;^y_4k$)@mn5G5K64?y)$;Njt#i}a{VVcno&`O%VR@;xhzom^eV2RC-a)MB znqIsky~Xp!y3?F{C(-Zy9B5S6jl_56eLC=JZ~k%YVm`U52P;-`2JcsOFSQT(PRsNx z!As65$;%BNO{Yb_7|j)0Qj^aADYXlE&!b0YqPJp?(kE96abLlcZmJba z@6 zFyR4j^odG6*fMwg;98day~=MALmr*`0DPe;){KQGgEB&O@iezq#$Apc%A8&H z@|GovvhkTm^Ww+;qjzJUvH!jnBbWRN)B4WU$-S!XY<}N((rnssp4Q_#&+@c3@vOL) z?(uI=pA}lk9EVh+f91H#pH7LU1nGzH>dg^2S{9 z@X>tQ?yR4FRL(Y@e_yqwT?$QP3-`v8v#yi*l-b4OXWO{ZlJ2X>kJ2xgb95^{c+P$@ zqw5G7dU_mR^DK`0d}_=e4$aN|pY7mHdye8>Q)1YXX<4YJpBpc|%Y(PPIFW>h_2r9) z{0e_>?yP*`233#(0~E=Dg#)CVc#*9VGvpNpzZHMS5drE&AWandDHR z;nb~gCQo)`2AQV$N;aii$9o@M$y#|2WP6c^k8^a2 zpR{!pEx4&H38S6pqN&+E?^HO!-*-OAo;@|Zi`hHq!E6VKt{kKLVrKIF9`o6^13mfB z_GwA+KDlXuX?fW6+fHQvv8gmm_u*v0jy&|$vZXAyei;9_=dz~Ij{f}Nf8Thv%jbB& zzqV{azE9*vo4lmdqH26V`RR1(ps{>aT5tAr*gSglU4GC0Rb6SoqwKtVD}j~wn$JIu zqd+j=x`OEc+Tzh$o#4g>Dsb+_~qbjY{iA4oQ5vtx8sM?&MsG3#_G$-&{~bT z(7z`=x_%{fI<<^P?;XnuHab9eu~vxap74>|b>lj}8 z$5XZ?@+e)~)t>(#C3t_oQ*`o;dh9@8PIhc!5B@aQdHvv^nlxkO@zf=2C_Cppg;bo? zimq-soDL*jB=fATtlQLzo^78MV`(DukhFWR@~)@5^Ai)l(I3s{>g!CKt6yJz4Bt5S z6y5*RiGItKgZ5sqlYOs$okzYXO>P}3>{^trNj0$t@c2Uc z=$D(zdGnQ@$=$Rzo)=#&WEXNxrrVd+(Km4&$^8mlpuO{rrEcxtlSu^*5`FVG{FwJT z`pA15>r;LMsj{vuEzo=lE%0#&bJ-lq%hwF&%UeBm+g=M6} zpNd|^y+(R+_tyQ{@$a)qu4jyQylvxAcygBbzsC2a8J-B_!{Bm!zt^caySp36*`-rx zrIDNIqjH1!h9P6vvXAd+72p2c?eEK!o}9o~OaXi7cvyYi=}o6~n~B3X^+(RA0GlJsuxadbRe&zR#VnmJ>B z+A@7%e)jEQ@_prO;#%SfIXb4AXTJ)qXrIlS^*y(*<1?yUAO~CSVVxRmWF4lZi=WzR z3%@mek!EWhN{_}l^VeVY^YvSX@tFI?_=p8nXzA)#Xr5eeXmH;i^wp>DG-69SVq35~ zuTeci{HQjNj~e$ zjeW7sc0AqJ#jIRhL;CNektFkAciOY}HZpgYE!|&pG2L0?Ac<^xkgPiNfQ&wSm*u}$ zfuAk*RFlts4cmULAHBc46l=BRfqsWJGwrr*A{k;oofarunU|dLo!!!8A*ZHs;=S)9 zSyi|s8Qs8~Ha@g_o;5V4M}A)Km>pU6ZXO9bn}yd|(tIK;HDa}`r*hSg+3{-?L@~` zC_!AKI`ACF?-Ix9+ezrDuk304$#hk4X{OT@0)zL5?6`e*aJv_|jEFx@r-5+eT<4is?V{!iXjtIgop4Sh2RDo8lf0Tvw z8On|x*+hmG97aOsywpdR??<{6FHHlFrz1Cdb>U}UbmY_T*WlMOPSMYQHGwp`*PXX` zv4S*e5>KzJ4y0%EO=l6Vjk$|WA8tFm)V^|aYIE1)tI3SB*U5m`!PMdXbn}~?DXqk zQmfG#cK=N^-gVGEQe>VVJvFpDEt~xUE0!UaY0~ZF9fl6)&30Af5yi6chq?|dWAQ<8 zE2j+LO|EYywOh9(i*p&ieepF}MVHoeQMKVb`r9zR^y3FjyS)$NygS^~><}Ijhem4% zuU&?Iy&Fr{-k-^aEs7)4-1xrA?XvRmyI1S$ReGf9R4R6Pt6=NgsUj z5ZSn=6}{Tz9GjUpoX1=W)r`31LULsos6SC`8f#a4A=_1&k*~SFvA@<{kJARH<)ixf zP}XAs+55?sHf=dfGjM7~)^CdqeK=((tr2)s^T4S&tv&HE$-Zg|nY4QmiLTR;uI%7U zXCC>?Lfq5Rx1HzEuu9L_w9l*Al+kI(oHU(z*AWT+n z(6qWHq$%ZH^@h2qU;$FFPBz}J-BAAB{k5jt^o()u-hU>g8yDe2Vp{RO?{xf)PZ4^o zc>{iY;t0}d!Bo~^x1Ps5;2O6M@90IlN_<6JJR4Z>F)8)q5jp#!C(G+@LlAS6+v)D_Q%qp;`O$yqTKwHM2+a zXTf9m-U!V;Uz?&d^R<<1b-sFh?D%n-Ie8n>NZ&%d=;O5{>$*eaS>BoC*`;#y`J(M) zYpqeV$i$XpoX;Uvs8lQ+v|J$hPZ>N(`O0j|cn4nltdgqKKW=8rvb4y}ne=pr zalErGfIiMqnEvQIoTRmDLFZ&T%_jI7z7K(|Sl)p;&93Yjc$pf}`eps+kgJwNuO&wPcYGZg`Y1a)SuPz1i6{8=JVQ*)WIRmtD+q<~a*;f{^=?n7Gtr<6x z(dTorSFyK=KDIeM+o=;fx+b2Uu(PEXzLw_2gv-o+S8>`+Q;NK~*Mj@sy`z8Kb_1E2 zW**N-C-d{`H8j)EQ{>LpWxR9EWu(%v{CaP{(`Tg>w$4g+%V z@C7^gbT@bYuTM1bDY~1KKC+dcsrilkQ@a)I>%WXOy}y`znrh4UyH6yYz1`?}htu@i z)(e`_?|Krii~VWYd|io?eiYx*wGq!Z3$V5m z{$*vi8vIEf#!hE+)3?&(=MBHiXHI8MXqvo#NX~!T?9r=rdH!)xGk*7cKVJTTH#_Ll zn-+81MYm_mMt#SPCw33Bv$ownlBs=Hks-NS)0pLx$*8_H>3*9>%;(T#zAoQKw&2oK zGACOIoiU^ft>~Sfp5nnQYg9S*Dqkqi;Mb7!DmR-2`rIaM*Jh-xJvy-S`OdQmYb(|elppnKQ_IlVe?5HSr?3hGf*Y$X}P1cOSs=b{$We?TBM520!ITPiEzlU!7++ z>^JMnHFo7LksbNPCrz3C*r|MN{a|{$Qa0`?d>~aE^71;te!TV0LHyd-%<<#r58wwX zOylXA+Vhu#*V7t1Yq7gUGmhw=`)4zO#DchNLi2|My`G)XgKsb+H1 z&V2FVG&EiMY5Ki$omu-0CCHAiO^Ns9P`W?UP||x;Q?}XjBlB+f7wH>Yjc4(nNn;+h zre9WGC(CyAq~HH8M$d=rAoSpKPG~K?N2rEj`irXfT_b+Gg z?j7gw6={dC_lp`)?U6a8$)>|>>~?!P;N~#8V^9a`;4_((Jad*G+U3T3>?x!FxO);i z;Cq1FS@W5seX>;FbZaph{oKXyjhak*-l)Y#hRq|*ui22VdoFnN3aZGbIyBKZmyXdi z4xLDj*Bwdc+2*DF)@9<6ZR*fHZ)@|l+sm`VN4)8z+_tQ4ktxjPvz?|&J3E%+z(JC3 zavROFF3x;W#?suOe{C}9{t8yH{Q}l2Yi4?(1)%}n>3D3NU78E4d}-ly&G_E0!$>}l zq2yhjoaE+wcUolCaCW|81JdO36_V{j7G8hYJIyP>f!f~vstI?u<0toZ=DGW~p>{DV z$o{5>^fx|+&?Ef|^BiZ^vv2LEvAyrdu_=EIVfIDFF|vN5e#H>VUremXcQ)F>FOM6j zsa~}jwdvfNxAL>&xB8sXq}_OhWc~1p?VtWP8(b?7|NHG^&5E*){FvVaR`hu$n&)L} zT6mZ(ullJA@p5sc_E;KTCBG}VZ99>yzEhA!y6t20c0DIG7x@#1hK==Wt9PbNyz6Tg z9k0YUZ0|+$1U8ME`mLINePkAL|B^4;J$@-4ba{W=tWzU3=l8{t17V}dnO9M)Q<)Lu z<+ca>#J6Sa$Cp0bvE3$e>gFC^X~a+(?OBti@%C_!8GVCoxqgSNS%0-wrl5G%ePkNC zW1T7x#W9IT*89v9oYWk8F ztaOHD^>AYc7an$>>b`>HoI0L$In|tWJb1sSP}&&vH$mei+-@ z<0}t8cb*)Nn!&w#w4pmno?v`=G2U{145@#hB^W*hv?F%sPI5Op{?fakxQhq;E2 zphX7MqfLq|q7L@KG~X20xOdIR>OXvL&QDb-z>EKTnUssMq2*uYCRyHl(rr&l@Hepm zq|D$}an+wJV_PZ@V{cv7kwR655T6$HS)|(!P3*J2{7a7icuhA4nsexGmLbDuHuOe8 zk|xJ6R?2P-@3iC|dzWJ+aVm0B({z^?vAGz7$ z>Bc@e>Aisgbb-r4`s`@qpM3x1O1s^fMiyVMNM9Xp#tzMPcfeA5U|STqn&T+j)7hJPwSGmsvle1ULwiwA_Yq7G zhI?!bsYKqt+rZ}MnnfSl<)Nd7t z+=@_A=H zJ$+&qk$J<$^CFeL^NWw~@-b_R(m}UM##fzPm*3x?)6=7Bm3VEpy40ptIzI17Yx?|g zvG^UCGkZ37aPb_ssf%avBX>!M=uV#Z>)&T?Q~showp^egbw1Fwp~v{g8)bQk{MX4$ z)+qjpP>ptMG?jmUFoS$AS|&bv^c<@Dmfq7{|AaOjJBxI4iDwZ*(s-7B?Ce?O?S9SD zt1I;L@*bn?!ZH%N<0vojgpmgGGsJ)Xew2q~c}RQZt;Z@9*OL)(UumDc)2Oab?)U*! zH}ZQWJ8^B%Ow^XQh(EXCBpvHT>Dc?_NYOcCd6R-)>D_GGsJ_W^9&x=jeYK$q&E5V3 z&)#7O&oa`-bFRx?T5;@JdhlLZPyZVOctE=@tmD6iGne_h`R!}X=<#drhIM|y8 zB~>TxBjMk65;2aLpth~WKQ~$Ym0TRTEYBOYyuGp+&11J_`~~BuK(3J$Q#gLytQkL_ zUi=6A`*oKF{hD9M#`5{SVT!im{JzGhj`;{6$Kjhc#m)!j4Xw~__d58J>oL%gA$$(t^^23A7 z(Wx6Yw`T@3Y~lUcKVele~q{*ZXB>`TYGN6~bT-FWdK z6?msH?YZlySl;;cElvE5eXQo04WtJ($aM#!mTC*Dn)T z+Y8r7%~xy5{>wS}#)g%7l`{AAMb_qtTkI60Ka+K{W?wd2+Mz~qp8542ws_u6_HA`p zeNd~{df~679utKtY|E0|`jdG_(J_v%$-9*^Nnnl|T=z4SZ>#U3xgJ!S|J~V!H~X@h z=z6xIt}XAg&FjCAd(&FbZ8>}iy*q-JoVkY#s9uf*ms!TPy=lU)T$vX4?dUpvnus#Q zx9Jm8ZKovPl_Z*DVp222QQSZ~b*H zuEw$A#AU}g?p?Pz8NM|KJv{Ng{#?}(e96V~%x*(Ln!jId{knh>{LDWMc#op%HTyks zvmeWLG57hq+1{90yh3L;x?$;KP1Y~1_@%lt^oO?2)V%q=fS1kr&Er7h+^j%D&T`)# z$_7+>&my?tEF`dq{!HhZn!v`cB+q~O=w$nWJp1nH#CA`9K4p0W{>J51T!UC0yRoS# zzY^4yz5Mw|-?q|9BCMXN88B%JEmV7v=5N1N+~uYhC9j8*&mLRJiLGV%sa$T{wnune zk@QXVt}$gu_6*Ni*|Mwo!xo#^u$R+F=;ucK)Ll=~Xkr)-YQLNL*3QaqoLJ9H)%b*Q4@vN>H~Q;u-Dz~6vt;PvvNTJCKabhlm@m4W zm)gIuWp~=tWq%E=Oze+6Vlh=VurrT4^T_6t_>`Ls`KD2GJgPU#$E(+##e8oJWerYe zq{r6WVO#x=va&gE5Mf~(+A7^4J?Yn#J5+eZ4xeYlIXDv^^Uj9!oiv>-xSoY}DqNOr zdGv~n_L@qo)yP1CbjwJGVQJZurnmWU`&GQ$%>K;bbqo)tBYE@58`#z=t2NVCe$p4n zvp9}6s!Xq49>JDR&d4XRyHZaOTBm86z;aJ9esXyH|vn@J&S68Sihp@Pj>$A_iSh9nfk*IcF`BJ z){vPm{Ajj$J=whqy~yLQhI5N-e;e*hd?LZ?)oG937syG6K5t- z8O`$l($G&6Z?g0|M{!-#ocu-LMb>)3asK`Ce*WZirF~_WH{`oRrm>4-hx1F#efZ_v zSM<3%#L^g2+dXe^ezLrOEBaY8n$XG*JTA9x%L+w4iR)Eo5s!}A#j5E(kWaG<(685? zkjj(tkWF53#P8cHeM7H_`<{2nMQrWgkhzNkd0?%kky(7b-R%(-(pC7yVK;`U;o9usu50C zhGe22o}}Z)N4U^?Gpq4EEgQ3UQODyxrX5L|yEyY=TZi-RjT`YXwnNx6eL6BO_!bGx z97kImav>*&OecMhQGW7%9rp5FZsz8Fh?S!|$-q}L*v(qL?22y`S=x3NIa{zG_wApT z?|D~?rf1#Bi*=LO(*8L$8ER?bdT*~z4xVaE<7a#(+1mDFSC`x(ZnJYzmx2Efx9-n0 zUyd~5jf-^QtJ9Cre|esn*4g%rr4P`vA=!3m&V1NJL;o$$3v3xd4(1<0yergT&vVCX z4%Mls@pBJn8}4l&iz@se&%PV>iY}$4c2kSd7VA3dCoT5mH}2131zlEB`&ZfNwhm?J zu7!2!y|;D9ta|gv#a<10$TxdZB+{7|UNf4_n)`#joAR6lthr4VXF0{})_3JQS8U`Z zat5%Db<6O7ZeL^%re&r)D4MTm|B;O{%uka->#+g{Td=-!Hgf0U3wavfALK#oEPCAE zf&H`l1>5<~FxGZn)laJSgk`RpkH^oL!8GAr*pIMP4n$dvx5!GnBo|8b^*GwMTTLPybiSnZQ%kegEH7AtGZ) ziY7!dWVmN-p%E3O@-!+bG)X0u28EJJrlQGEDM=%Wd)B%tDKsaV6U`&hAPxVk=l}XW zH2A(=&$GP_7uS2QbI#spuVsDqlJkSc(SFP<)0w3f_s8<^foQnEfbYCi22z~rgnqGo zxto)$sfS~*Ab(JMRKIMPs2Cy@cn{fJ~jp&_NxBxB@=2Y|JgSO!Q$p#`7=22VBZ#bN1RK<+gMAH1Zg?J+FJBEx&h6Py z?OZIlL6|#bJi4Fl$>Jxi1w{p2mh|y0ypYvq_w!-H5kx7Fc0q%EX5Z$+39 zn~GBV+6rq|27!l~JbPV#93vc!;nu?9{4i}joHjd?%{b^x>eHnuW!nrW8D_y2*Y~98 zo&Bix@mqYpU7kwweDIj>dYZE+hL`B5N<9)a#JQ9eIO>Tyo2DREWI0^o`e-PS!^z{= zcC`j8ICu$UhR#P&GDjz0EA$=zf}0*PklfaZXxS$zyjRnejqPm0;(M)U-_D0IFM|je zmb(Sh-h72OR}JZQdNq2ySDGD$RXpA?B+n~>xXa~61cA7pIL zL+Pk)T;y4kR-rvfy$OZ^FQdRE;yuLNlBH3*>iKJZ(*ZuW z=MyT+@JhQ-Qo6X95~AH`<=}EYc3)50;jaH78Xtcf4Z63M*Bf2Nvj|l_^LjJ{UChSNY4=e=dmQQZd5oQWQ(@AkH7wX! zmu!z2VflGSdTp>8_cq>wzIcvXAl8}ejFi9~V?M#s84DN-NyIBoE1A33c2usYhq{_# zF55AM3)|d5pi;b+b3X9}xEg;}x?Q}cPmTqzY0ts9%Td<7v=`Glc7b(EAIml`4TPF8 zW$;~QADEt9%$|8N=GUexS0eVw76xtx>s$}gFU;V3Ds2bb$~RcIgVBYB%K2`-mE4=O z9T;Y&&Gammavh*K!X8e$l5r*=@ke^BK~&`KT!zsYl8P>dZT`;is}t;`v5?+}ka@hhcr*4x<=2k>CVsjV^G` z?K&Q=-^J;Dk40hEBzW+!ftMBkpI%9J2V{7bpsc+;%%~%q&mbuhpJbiK~jm8kX-^aQqAEnU19B zy<6Gbx(U4P$9;TQmI+RtSq94X0)F7A2mB;616FEiLk*|waNI+E$SH~d&5ISnxpJM^Yh5|~^ii4Y zJ_lgIHV?AB(3?&@h_L;9c&}h;)>Qu4gi$okp%}GuPN0LvYT(u6`TFG}@(sMV@;VxZ z?0G~KWF$?%IvYtgOc0NgqY$JQ52CA5>dZ#2l)vRF&ime7NrNi2smSRts@!#CQ6n88 zR`?J#N50~XMF|vJBtxGuk~`l$1nv$M!Tljm#WBxJPBZ-!Y#s1Guy4E!hQ?pv_7xn5 z)h_K=N$5^+jI(B?!+TPXKCi(+rX3qm@swBSWRI1G$~Za}$Ve`d=GVtSeLy1492Z27 z+XrA`UM#0OOb1TC(_-Asz1-FUDOP!Z25L_6LDj`uVace?bXZ|5J#$`1g;US+kGdq# zKgw&*VZ07DaP;=E7_5GDuNVjtE zOSOkKaotz|soI9`Jq)^uJ6O9XU5QCr!f3G$bf$eS@4VWcPH&EYhhuW_QBpBZZMz?3 zR}7;x+qv-h>jc)`dc3exR|Y#L_QAbTiBvyq4OE;2HcdSROXa?zVo5Hx3pxp}I*nur z)3dl;f-`*Wf*K+BJq?F^pFL;KA`t-snm^3=xA~k(4$}S2AAGMXVuBsO`yldn=ly}d? z#VG=@A6E_ec=Hef@N9zQsmvDt?-?$SPx(A&*i5Z z(}4sRW+>MA)HbYw@pm@ilC@GCX}rU;%AO?S*ao|gZ4?NOn!@~dx6vcO4EWk|I(fOh zXh^?w96f3q7$#?%+6Dh_ZKb0YWG-@v|Ng=VqLnzE90rm|L8QbU=gcyzPK*hB8=gvK{!~*`#vMpNdu0 zDf8|sre#qCb(imQR^r#@9V)@$uohJ=j0Nj!-C2ozGJJbpn82sLX zSzVWB*`s`+ShEz1UJOE~)G554T?u<{pN=Ni-{ac0a-!~XJw$CC9a!g>2+r7P9l874 zv7llFG!9+PcF#Ws1N7p-=|DGjNhcN~l}G2lRyhe%`M#{LzCVQb(4>ww^Z9M627-8_ zPR!<;K1-wF%7nqUmzwyb z@&$HP8bN>UlcVHo?l`wYN2;;>ic7nTwae~-+}oESFs*@ur&sm)%!=u#D76N+q&>n* zZ;AyS`<};5!(3QQmAjq#%^jTEfiOO={V2M={uIudUX1~w$=J2994Evc7bqS*2(E#J z_%taAmbzww>y6tmV`eQxhHAsXZ4+5$&QsjIz7w;Yx)K~c*MOCeg#G=Fi7ckOp3pw@ zHTP<|B0F&6I~=dbur;fW$I=LY{1PHZ>ubX4P0ucLP3IQA{-8x!nw_ykGLw7$L98En zyOzsL2|~SKP39~Jfr%&jf=T=n_?XdwnYacq+2ShUMuR+lOkE_NH#mZ|z3nh^+C#|J z`pP+onkjO%aM|5Jgz&^$j6IR=sl>=gg&O^v@4OT zH7n(|zOiGD$MorWiYV3wh(7GZK<^FUwfq9!8ze!oqJdaG zG?qyZ^8?{u`7HB;BC zz-jPFx^T*Xv~Nnn%|{O)W7q=reXSYAn8~mPH+AZ}e;w2f(}w3NF0^s|2ENcL6a9N@ z;Mq0;_YikO=L4|aM8Ztv~GUMa}2ezl*uJ^snKNM}sg9ckv(3{1)T+JdU?7@}(xmcHW z1Wc=x`JC_JeEuL^ls#b&SCZmrV5SXvK1_g-j`3`3&mwHRx{S1!zZN==(_pSnUr;#4 zmWozBg{9@mf~DiTvh;m^^mz0sNvN${V78*SZc&y4uf>uO;d+8h>x=My#aq<+l!9{vYBZpWJj@Fl%nDDt zvCT4*sQTL)?CtXzqc-WoWJ?8hTscpiQ;0%sIRy%zt;wXi$_f>hBnrb^5myY=hc9AH ziHd>@Rn*A|oAPqmDiuNrEpf6(e6*IAv-x9Q&Rm?lvTTu;YraRLwbz!}k=)K#@+`)% z7xqjXj0$hs;iH+>V6?#kt4ElEq|q~cSDDN=*?BUx%hmsuKQtHX4qE09y;pqwk$)=f z92EK=^H2Xuf3T)O%Ax7pRCctB5O|Dyj% zx+8vE{|P#cYX7Z&vHap+^luBUKl-0DrSQOi>t8It_!s?~?leM)Iww(Od?sS-Rkl^~ z2)=)rL_yQlSlW9XaeOb98sjP>6-RO>P%VNgs0GO zQHFIspJ%&x$p|n3 z^o=Gl#p(}i_J_6X-B)*ZqJ1)5*mj=u{yNAhIGfSj+u`iG!~xX*=*Dh4k7q@O?)bs9 zH>srUMx8@vgkRFCsMda@y@{NfeV>>GY-z3`Tefg4lQr(a%nn`RB*&VFraz7X7!n5O z(81n3whb1P#L+}?Zp^Ji2piWYi-txkhS^1`}q6s4oc^CHPas%_!^k-+=_ObVY0#@Uw4J-6ANqXr!-b?WaZ7Hl%2Z%MqP!?zpPc;{K^2&sivi~cx~M)VjqOaV zf`~gd=>N=wG0QBHQuoD$qgCz4Hg*<0Z*U{Abc8gdW$2*%H+nt(D$KCx!Db&B$CUQJ zA)(0rN6-<45i1_fbQqjH_qisPM$OJ=cvV)4g0ta$Vj9JD!zi6 zugg=X^LKviXH@-Y*w9s4LZa#T^ZKMEUcP=){NYDs2C;la{rAh$`DvjrLt-CydHw?k z+q{e4FQI@s%8i^*b7m6*J;x~P;qe06n{d+H@pN)l?wUwo% zjcq?`b4%;~7FJf4{jAK)%`B~~Eo^LT%+33mTlcrJY_c-$;br#E{g-Sq_W#O|G%rx1 z)AWlL?GNM9G+@?jHcq8Aegpcy>Lx#vnwJ!5^6vg=W6gd``n!n9 z&zR=BiCbdITEX=D8Pj}&a8n_oe_B`DKU>q!>}O7MF>Fgtd~2Lv>}f7yZON%|Zp9w+ zpY3TbTy4o&<70xfir@8K} zr8!>ie>TVbmub;FhHJ?wYmM{Ev}hh&k&Z~Jm)spk6HO?>7qIq1@lH)PE z6?=Y}7R}?Emba^{70xf_G!Ii+Vm#*j*_>ajX&!B~#FVvy`DKnacZ^zMJm&t{nqQ`9 zbHAn~re^-1F~7{v<}OT2OmN_zF~3aE=AKAPOj#?KU*>0Xm*S^^v*5SZ{C_9r=Xu_I z(Qhio{ZG60d(Q9O5jU5%ZJP=pHnlF&O~n-dZn{5xHIx1O2f+))Vr_8~|DCvshlIrb akiWlg`t0V?rm0Aogs%9b>HUAd_x}Ot->v=t literal 0 HcmV?d00001 diff --git a/onnx4deeploy/models/pytorch_models/lightweight_cnn/qlite_cnn.py b/onnx4deeploy/models/pytorch_models/lightweight_cnn/qlite_cnn.py new file mode 100644 index 0000000..7d52b56 --- /dev/null +++ b/onnx4deeploy/models/pytorch_models/lightweight_cnn/qlite_cnn.py @@ -0,0 +1,109 @@ +import torch.nn as nn +import torch.nn.functional as F +import brevitas.nn as qnn +from brevitas.inject.enum import FloatToIntImplType +from brevitas.quant.scaled_int import ( + Int8ActPerTensorFloat, + Int32Bias, + Int8WeightPerTensorFloat, + Int8WeightPerChannelFloat +) +from brevitas.core.function_wrapper.stochastic_round import StochasticRoundSte + +class StochasticInt8WeightPerChannelFloat(Int8WeightPerChannelFloat): + float_to_int_impl_type=FloatToIntImplType.STOCHASTIC_ROUND + +class StochasticInt8WeightPerTensorFloat(Int8WeightPerTensorFloat): + float_to_int_impl_type=FloatToIntImplType.STOCHASTIC_ROUND + +class StochasticInt8ActPerTensorFloat(Int8ActPerTensorFloat): + float_to_int_impl_type=FloatToIntImplType.STOCHASTIC_ROUND + +class StochasticInt32Bias(Int32Bias): + float_to_int_impl_type=FloatToIntImplType.STOCHASTIC_ROUND + +class QLiteCNN(nn.Module): + def __init__(self, + batch_size: int = 1, + input_channels: int = 1, + num_classes: int = 10, + dropout: float = 0.0):# ignored + + self.batch_size = batch_size + self.input_channels = input_channels + self.num_classes = num_classes + self.fc_channels = 160 # Fixed: 10 * 4 * 4 = 160 + + self.convAndLinQuantParams = { + "bias": True, + "weight_bit_width": 8, + "bias_quant": Int32Bias, + "input_quant": Int8ActPerTensorFloat, + "weight_quant":Int8WeightPerChannelFloat, + "output_quant": Int8ActPerTensorFloat, + "return_quant_tensor": True + } + super(QLiteCNN, self).__init__() + # Convolutional layers + # self.inputQuant = qnn.QuantIdentity( + # act_quant=Int8ActPerTensorFloat, return_quant_tensor=True) + + self.conv1 = qnn.QuantConv2d( + in_channels=input_channels, + out_channels=20, + kernel_size=(5,5), + **self.convAndLinQuantParams + ) + self.relu1 = qnn.QuantReLU(bit_width=8, return_quant_tensor=True) + self.pool1 = nn.MaxPool2d(kernel_size=2) # Output: (20, 12, 12) + self.conv2 = qnn.QuantConv2d(20, + 10, + kernel_size=(1, 1), + **self.convAndLinQuantParams) + # Output: (10, 12, 12) + self.relu2 = qnn.QuantReLU(bit_width=8, return_quant_tensor=True) + + self.pool2 = nn.MaxPool2d(kernel_size=2) # Output: (10, 6, 6) + + self.conv3 = qnn.QuantConv2d(10, 12, kernel_size=(3, 3), + **self.convAndLinQuantParams) # Output: (12, 4, 4) + + self.relu3 = qnn.QuantReLU(bit_width=8, return_quant_tensor=True) + + self.conv4 = qnn.QuantConv2d(12, 10, kernel_size=(1, 1), + **self.convAndLinQuantParams) + # Output: (10, 4, 4) + self.relu4 = qnn.QuantReLU(bit_width=8, return_quant_tensor=True) + + self.fc = qnn.QuantLinear(self.fc_channels, num_classes, + **self.convAndLinQuantParams) # Output: num_classes + + def forward(self, x): + + # Convolutional layers with ReLU activation and pooling + # compute min and max of input and scale for quantization debugging + # if isinstance(x, torch.Tensor): + # print(f"Input tensor shape: {x.shape}, dtype: {x.dtype}, min: {x.min().item():.4f}, max: {x.max().item():.4f}") + # print(f"After input quantization: shape: {x.shape}, dtype: {x.dtype}, min: {x.min().item():.4f}, max: {x.max().item():.4f}") + # print(f"scale of input quantizer: {self.inputQuant.act_quant.scale().item():.6f}") + # x = self.inputQuant(x) + x = self.conv1(x) + x = self.relu1(x) + + x = self.pool1(x) # Output: (20, 12, 12) + x = self.conv2(x) + x = self.relu2(x) + + x = self.pool2(x) # Output: (10, 6, 6) + x = self.conv3(x) + x = self.relu3(x) + + x = self.conv4(x) # Output: (10, 4, 4) + x = self.relu4(x) + + # Flatten the feature map + x = x.flatten(start_dim=1) # Flatten to (batch_size, 10 * 4 * 4) + # Fully connected layer + x = self.fc(x) + + return x diff --git a/onnx4deeploy/models/pytorch_models/lightweight_cnn/qlite_cnn_scales.json b/onnx4deeploy/models/pytorch_models/lightweight_cnn/qlite_cnn_scales.json new file mode 100644 index 0000000..bad43da --- /dev/null +++ b/onnx4deeploy/models/pytorch_models/lightweight_cnn/qlite_cnn_scales.json @@ -0,0 +1,508 @@ +{ + "inputQuant.act_quant": 0.007786148693412542, + "inputQuant.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "conv1.weight_quant": [ + [ + [ + [ + 0.0046827648766338825 + ] + ] + ], + [ + [ + [ + 0.004657038953155279 + ] + ] + ], + [ + [ + [ + 0.005794027354568243 + ] + ] + ], + [ + [ + [ + 0.004027717746794224 + ] + ] + ], + [ + [ + [ + 0.005896744318306446 + ] + ] + ], + [ + [ + [ + 0.003534339601173997 + ] + ] + ], + [ + [ + [ + 0.005034759175032377 + ] + ] + ], + [ + [ + [ + 0.006583436857908964 + ] + ] + ], + [ + [ + [ + 0.005040623247623444 + ] + ] + ], + [ + [ + [ + 0.0054032206535339355 + ] + ] + ], + [ + [ + [ + 0.004712222144007683 + ] + ] + ], + [ + [ + [ + 0.0044171675108373165 + ] + ] + ], + [ + [ + [ + 0.0045336028560996056 + ] + ] + ], + [ + [ + [ + 0.005082833115011454 + ] + ] + ], + [ + [ + [ + 0.005390029400587082 + ] + ] + ], + [ + [ + [ + 0.004269406199455261 + ] + ] + ], + [ + [ + [ + 0.0057991985231637955 + ] + ] + ], + [ + [ + [ + 0.005343677010387182 + ] + ] + ], + [ + [ + [ + 0.005501016974449158 + ] + ] + ], + [ + [ + [ + 0.004162007477134466 + ] + ] + ] + ], + "conv1.bias_quant": [ + 3.6067656765226275e-05, + 4.737563358503394e-05, + 4.514355896390043e-05, + 3.0122700991341844e-05, + 4.522434755926952e-05, + 2.542074980738107e-05, + 4.058502963744104e-05, + 5.085647353553213e-05, + 3.5445831599645317e-05, + 4.2773499444592744e-05, + 3.603152072173543e-05, + 2.8624375772778876e-05, + 3.2512234611203894e-05, + 4.047499533044174e-05, + 4.007756433566101e-05, + 3.143427602481097e-05, + 3.283462137915194e-05, + 3.1593885069014505e-05, + 4.189912215224467e-05, + 3.179134728270583e-05 + ], + "conv1.input_quant": 0.007760446518659592, + "conv1.output_quant": 0.02516048587858677, + "conv1.input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "conv1.output_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "relu1.act_quant": 0.012514653615653515, + "relu1.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "conv2.weight_quant": [ + [ + [ + [ + 0.0059503596276044846 + ] + ] + ], + [ + [ + [ + 0.004254304803907871 + ] + ] + ], + [ + [ + [ + 0.005004961043596268 + ] + ] + ], + [ + [ + [ + 0.0070042419247329235 + ] + ] + ], + [ + [ + [ + 0.005205494351685047 + ] + ] + ], + [ + [ + [ + 0.005354207940399647 + ] + ] + ], + [ + [ + [ + 0.005926031619310379 + ] + ] + ], + [ + [ + [ + 0.006488564424216747 + ] + ] + ], + [ + [ + [ + 0.0034671735484153032 + ] + ] + ], + [ + [ + [ + 0.0042591276578605175 + ] + ] + ] + ], + "conv2.bias_quant": [ + 0.00015088623331394047, + 0.00011274521966697648, + 0.00010193250636802986, + 0.00015958795847836882, + 0.0001279811403946951, + 0.00014609555364586413, + 0.0001375889842165634, + 0.00012088024959666654, + 0.00010532839951338246, + 8.798576891422272e-05 + ], + "conv2.input_quant": 0.025035124272108078, + "conv2.output_quant": 0.038704611361026764, + "conv2.input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "conv2.output_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "relu2.act_quant": 0.019238846376538277, + "relu2.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "conv3.weight_quant": [ + [ + [ + [ + 0.0032192468643188477 + ] + ] + ], + [ + [ + [ + 0.002923605963587761 + ] + ] + ], + [ + [ + [ + 0.0027376932557672262 + ] + ] + ], + [ + [ + [ + 0.003503928892314434 + ] + ] + ], + [ + [ + [ + 0.0031144043896347284 + ] + ] + ], + [ + [ + [ + 0.0034859958104789257 + ] + ] + ], + [ + [ + [ + 0.0034436206333339214 + ] + ] + ], + [ + [ + [ + 0.004039732739329338 + ] + ] + ], + [ + [ + [ + 0.0014960176777094603 + ] + ] + ], + [ + [ + [ + 0.0032225819304585457 + ] + ] + ], + [ + [ + [ + 0.0036653317511081696 + ] + ] + ], + [ + [ + [ + 0.003544930834323168 + ] + ] + ] + ], + "conv3.bias_quant": [ + 0.00012999656610190868, + 0.00014617544366046786, + 0.00011316189193166792, + 0.00014395458856597543, + 0.000142513687023893, + 0.00015837881073821336, + 0.00018707614799495786, + 0.00016153800243046135, + 0.00010428271343698725, + 0.00015711947344243526, + 0.00015028772759251297, + 0.00014441728126257658 + ], + "conv3.input_quant": 0.03846479952335358, + "conv3.output_quant": 0.05587683245539665, + "conv3.input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "conv3.output_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "relu3.act_quant": 0.027778906747698784, + "relu3.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "conv4.weight_quant": [ + [ + [ + [ + 0.005106140859425068 + ] + ] + ], + [ + [ + [ + 0.0067635830491781235 + ] + ] + ], + [ + [ + [ + 0.005935069173574448 + ] + ] + ], + [ + [ + [ + 0.005529573652893305 + ] + ] + ], + [ + [ + [ + 0.006021668203175068 + ] + ] + ], + [ + [ + [ + 0.006458999123424292 + ] + ] + ], + [ + [ + [ + 0.004328868351876736 + ] + ] + ], + [ + [ + [ + 0.005672066938132048 + ] + ] + ], + [ + [ + [ + 0.009927690960466862 + ] + ] + ], + [ + [ + [ + 0.003988152835518122 + ] + ] + ] + ], + "conv4.bias_quant": [ + 0.00032779158209450543, + 0.0003805554879363626, + 0.00034476761356927454, + 0.0003091433900408447, + 0.0003567334497347474, + 0.00034780139685608447, + 0.00028347299667075276, + 0.00029056513449177146, + 0.000570900272578001, + 0.00031566276447847486 + ], + "conv4.input_quant": 0.05556188151240349, + "conv4.output_quant": 0.06223675236105919, + "conv4.input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "conv4.output_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "relu4.act_quant": 0.030943837016820908, + "relu4.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "fc.weight_quant": [ + [ + 0.002200733171775937 + ], + [ + 0.003274584887549281 + ], + [ + 0.0030999528244137764 + ], + [ + 0.003084302879869938 + ], + [ + 0.003228644607588649 + ], + [ + 0.0023938606027513742 + ], + [ + 0.002668095985427499 + ], + [ + 0.003263178514316678 + ], + [ + 0.0037302691489458084 + ], + [ + 0.0028242983389645815 + ] + ], + "fc.bias_quant": [ + 0.0001680418208707124, + 0.00022119912318885326, + 0.00019093586888629943, + 0.0001730432704789564, + 0.0002461412223055959, + 0.00026946672005578876, + 0.00020659832807723433, + 0.00031415317789651453, + 0.0002444065175950527, + 0.00017614015087019652 + ], + "fc.input_quant": 0.061888109892606735, + "fc.output_quant": 0.06395246833562851, + "fc.input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "fc.output_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0 +} \ No newline at end of file diff --git a/onnx4deeploy/models/pytorch_models/sleep_convit/qsleep_convit.pth b/onnx4deeploy/models/pytorch_models/sleep_convit/qsleep_convit.pth new file mode 100644 index 0000000000000000000000000000000000000000..9dadf8ac4ea655cb79abf1b0bcebaa7e7f6c9621 GIT binary patch literal 335978 zcmc$n2YeJo`^QnRVa0}BDJr%|LboDTL=+=RY=l4pL_)$%4zMC12=?B4@4ffld+)vX z-rKwLyW82{&OW=z-Ob+ne?R|&;LY>>?(A%No}Jl@7}l{sk zHPmm@I4;{c(_y{p%mTe?7V|=nYig~{woGbh&MY`C+oHxztf-xuZER}oT&o(!PHk+e zugx|zx3;L-X;qyw9eeevS;TA9>V30rLPMrgu1V+Znp*0zyXF3|b7rBM1-$R{&MaKB zum()6uiK<|W|3jNvzbMA9oAvk0^a|6&&w<}the{S1?FWsA2Ct;ES}9QQJq;5Qd3%5 zYa1qyZK%&IHPK^E>@+bKvof$siYK?;dO4R>vzgVZGppCEp?jsJd0K62 zwqbI`SXI|NZenGH{+G&%+|JCb5enQVvnB^#YxwbAJrpire?i8<-)BZ zdcdaVi?D!~WV`p4gwklK4vJJW$##1*;$Yv&1 zXBx%9u)DD3Uumn>nUBb8N!sadz%GE^Ib1 zb3B`!pv_Jsvy-x!ldChQh@+=Z_}p`9IP##(X&m|V;WL-WX3ofso&|#|@lnpsJ?DjFt9ob7XW0vMH(W^Fa8WjMadqYrarF3Z?#?}z zhUF?Vm$BUCTJ8#xyE2=(sycJEIC}cHJ@;G_Hmk~9%VyVUv+K$1hHU1>>dZ|EqsN?k zZVpH8levW>-->h3ZMo5dbIkJezr@^_s5_`R@ygwygRM(Zs8)1`ynK#+wEp75PnY@$Dyjz`luVy*Vq@j6Si&r10XlkxCIq3W0aDy@* zaJUb3KL3#yZdmWRnU9ApFgNo_?pXRXoB6Cd^Lb4dJM7rTy4K7Wp`d+wXTId1U+JJ< z=YxKugMOO}`dv2jeRbxCnpG(1{ru$T$8hw@%ugKs=iwc^6#N$*{`g_N z56}FnD+PyVe#;?#&u0Fp&iq-kQVat6e}w^6nZFtE&#>dOnST*~ZstEvUv=;*AqcM) z@cx!n3woc`s-yRbf$4WUZ;Gp{l@Wz@pn&)&FDi+F!}-uX&}TGab7 zs}}P#Hu_-?D01@3UGh=Y7_!9EaCCMJ?}rB@FMUR?y7|zM}WDH@H++?@zGjs+IIV zdHZN(@6Q}}74L6ZwW{}7tyc3sYxa%fI@?LBdzi3$f3=22llz+9&w3}V<^4%Jshjs> zR(1D2t5py06DMIFQD!GOySY$3Jx*A6fLdF#Np~IZXRW)g_b2K0@_x*!^}Nq&wZ8XR z(`0pST{hcXn^(=(H1j1OH`O||fkzF?4^$g!6O!M^`&r9Zcz=@o#@>%v)!X~5R+Zi- zuEyNgnOa;_>fIVwd8n}QAk|0X$+)lgvo`MM{Yl3Cy&tn`fcIIg26~@04OVwgs`U!U z&8^<`$E$zlDx8k`d60(-S@-R&Hqi=Xy{Y%Jw%*M9ldL!Qe$1*Zyw7U2rT1C0mt{T8 zt?FQp5mv2KTWKz-4)K20s#|-1lIk|zk6E>?_gSsB^FHnCsDE>+w)g&Hm{Fy6&<%(& z)caYVo_6&9^cdbXRSnbs&^8U=7o`!pW%c`Bd&uX=c_i0~Zt156h9O3<^Ft3l= zRezm$)!xsVx10B8o;OnelgHcL`!mPe!~0uSjq*OLl`iSmtUxFTe@UQn>yC} zk6~;-HBL7mY`yoh-qZ%~&-|v2*Z<^g>ICo49C4!ex2$UPKC9J!-e=9I=%zY5=TGtg zVe9^?NkhqcviGwdbj{wMp0$5rZt;H1swv)QwaR#(H8oLBcV~O$5yGMaRI8?vXx95# zi%#|aB++T!k6AU{`>a;`d!NyN`x#^BVDB$x>>T3#kQ+NQ zyuao4PypV6_CKTKwO|1pdmqz=;!2s_97S&yB$-k%~Qdw;y&tpcH1D%oo$h@iE%z_B`Rq++ zagFs;>zZ6Wkx^%Oq_A+6I#bh0_$=>dEqu23Ckdb9{g_qfdY{$mJnu76*i}NT&i6=R z;XdjDO()?Cy`Q!4Mc$tze6jaqR$bzKR;x?B&zi-((@1lRnw(Fh)Meg3hGX|tm+LQ5 z>?^#V^}%ta_oo+IU$d^#|KuGUS9^cv4vuTQzh%|6-eNfAQTHWq_*6ikOkv~?zb98fahZj6- z++W?PfnfY$2FQO0Yf7HpWdc^x%u16mA{+3mbd7stlaqkm74;p&@3Gbi6tbyuD z{WW4e<^4RYQ&v6g{R!4w^^E=}ujilj{>*Wn^Zu4q&wHQM>ILt!X1Le$;5fTrzUYMw zyADz>X%xA>?ES3!{1xv{>ho8XA-(>+)tj0{dT)6@ zYrVIo90~QI_m^J!lHWmn zj=Bzg(rAd8>_Fr-P<-_gG!Kl8*cfaB9t`qZJf%Gj} zf5)vW<60-p^iba;&<`AFt6~D_TAn|XL{Ah^KO*o?9C(YO0`puH^)sUW!cli9IjT1X z@{=yquZaH}$Jd!yFrtd8S(^A28ZqB} z?%{G9>l>Qt4yc{3>ZVL-Q0i~c{$W~OYP55G{WznDz{ zYCX7G0@x)P>$+dPTTic^;LS43_qJLJ%%zz*#qp-vOz-J!UIM+H8@$WFqYHcJ>25B^ zl)+{vdVihR(p0aO1$8;5s?@3C_Ad{U71*RcEfbW4YlB``5$vwa-oH$2u`gDF&&uqx zr(>Vo_0~TKRsnET2D(0RhkzBipC)^c3B9?tYBk_jPZMv(zIQ-1)s1WLo{}|qPbkMV zw6>}>fM1jGV;pz4!TTq953kk&w;OZEro}abU2nMR4ss7B?@~Hh-!uB#hg+PUz^u&} z{dm2^a~jf`DJ|v>bsbRFWr~BN#iCRMKdW8wWm^J>tI}N|F1bX+Lh-uYFC0EVe&_4d7sd+Zik|sU3*8gE;O82XP}g?k3=D z${bw;itL~Kls^pM{Byh{|IqRawHY9rGejQ^P9c;5ZvotvjB88}CnUs!A+S}s1fu!z z5D09|0{R(pN*$h0S^ac*8^E_^xV~L33!Jm(?clLJd+0lsv^;W$MvzSJ0Q^wK>pF~E zeCS%bBZ$M8I4xD8pL~j`_D--F&NgGpV54>hcNgYPDg)Q=3!R9MfWoeX_*^*wHGAz zW{FwlmJoYqABgSCVy$Ho)2Gq~%J@e^rIuCZ{eM)%o*DzaI@X(*u3jiTS7X5+$NZEp zHX*)M9;k;+Lm6$%!R}r6?P|ez*i2xXmUQ=xvGH#RbP7Ka8jY-xDU*gd)u5KJA4Ddx zh`df$Um+_wS*vJwl7ahwf9p|NXaaRIQ}v`^w^!jH=g(#0&49Kr)cs*;LAmfS1^f*2 z$2;HXxWa%^ptmyJ{jGl-oyrnfcuZxFo(a9c^9j^6w4Kgv`?s^LP#@SI(GK8fTeUY@ z(%4rABJe>RSU<^%8{NE#$p@AF6#QU-4q=e4(3K7fJr$n;%Arirk2>8_s75doyjjfK zFGZeEB$y3@!^&n5trE-I#;xR2=Sf``rIlL91i+CrYBvU+|s#DFdrgEut?G= zyJQi&P;eypM=^hI_d_vsX*?RpV;DI-U8J8Li=~2Np>rJT=<>DWgP(gKr=yMs_yh*) zr7fw0{oc^Uf)n9z5<4W->WUnGp^Xo!1t$Z33d57`RGq@9TyQEhPGgO<_k(@MtWJl+ z8SF4K<*{Hph>6~r@H&gVk_tL5c9HQixAE$1Se(Nav&+>x#zO3$b76KKn`O&orjIJF z8JrK73)p4;|H(z{s|(?G5&I=Q8F4X|LRq`I7y_5DK-w3ckbqS-xD+~%{0oFG99#*HtJp(N`APG#Xgtipj>^H+Fu8_J^ei*SCXrbQ*8+PT zW0MLNMeYbIjaS!$dIM8aI65sV7Y}ZPz)dWW^u(;l&5q08)y=@)!uS+2f;c>t4{n9X zZ7h;hKFFfIP&Ip`~?>mFqqXz#|M8=@P){?4w{k#wKR5Qo#^8Gh&Mn)Rg7;`9@bm0KW>t@SRXd(S>T^ze9~QzV|;}@DgJrryudnh z(&>1qH*B2>Z^^~vsoDI5{TJc(61(Y{dnpWxd`aMC0AFEX(zE+g_6^1&mjYe|`!!}K z)pXm*&aH>wk*Zz?=nV#qExlKxpo+Tj< z+u*;${G`X0uK8B}@h)uMW1E9h*$?LSI@B9MC($Z{_u=#bJ0;!xJMW=jt;dHj`G`%D zYV^)bObVhthQTLnplAJ;c=i#F6Knq%18cl|3ZKu|XGVE_j3|wm&tdfiTP0Os-0vr< zBz_5(uh=Cig?8qmPp@1F`WhbJut!<;l;vTKpKoFH9a}lL(ziXHZ7XRMeGj)E*zLe_ zjBVR3IRE?zqo3F)>G_z0v7}w}d+K!&=4a4;%YEzK@d7Jp}0gjjzCqOPGBv>tnLY0qhGy|ukkF5mWy!9zU^u$ zRCpFeti?E1S1}geC9XQ->x=W(d&Ga;E3KH{#-Nr!(ymth-qel&(yXt26~s zLM6YIfLNIkdO3LX=Cx=o4wdYXYWJITEk6`g6$JXQz#(aNzbzmZuKGf%A4|z=jvVb7BW3a? z)gKB2SV3NMl$wIrGXtSDh_&PgtFE>5c;TwZCJ@<_Mdp@m+>nUaOPfJ%bC#3SeMh|% z&U?22eM_dNdQl0{t@L*=T(&Bki#d|KE4#gx#`}$k-aDzbW*500;ApQH7yq)SYcbnE zVq2Dwr%Ae&FsBxjWVVCG_N)>99U1;FBrIMdQn1(o%%RMbXD1iWv?>m2M<9nW()EF8 zA-Tx06X?U4EKR+vSNKyBP;9O9c2#-egkn2fqI>9a} z><9cL#>c-U8&Ehs3ovVwg5ccx%ce3b=zDzoM5zk4>77pB2tI$NZ?6YQIAwLcUNUSYlSX$EN8UBO&(A42T`dV(~BUaB>PWV)~G# z{9et3$}CoySEfF)RK%W|4ZXuyFaFuSlb#Bt`)UsObD5v=#U{kJ$_%(xK> z4)bAi1l!2DzWZa`zcA^N!;#Q9iZ$e#oqG*)l%wcyG(?VJ5%~#2DI$@w!?BY|v^|5{#;?AyerHnjmtmZVXlHS>_{lm}w4`KIosGcfaNzif-US4XR0z%m z=sX6=lNoX&iu*FL#1TrT)%l=Yz!dobY$=o=Yg8A4cMT^H8F4X|LRq_d6atU2K-w3ckbqS-cpN%U zlvBry9ieXUBy^r)ouu^1#qKdW{slr84xWa`Gwcz+e$UPAf$=a0J1Pgy!sIzNiC;ld zv`M6N@I0_DFgB@RQRI%W(s=bEs4p=!g`?A=a`E702)x1qNl(m*-0ZmgUA+qYYm84J zBZ$LO`QUYkyul(#)Af)PY#HAf%8be+bA&E+NrG_+vmmVaU*8dYQH)^i*1X z3czO!80iwg>FnoVeZeewSxD}Fs6D&I&RD+$;wwhTt7h6l_!k`{rC%fJHykzoC=r0Dz zYaa?Zfb&}t8U&G-cl-_VKTJ+4wiUgJLHZNA|5yKl{~z;{9$UKRTlq(aPFM}=eVWy< z@h|Ua_vnu9@R;5R{+V9Iuppc|vQyH{zw;gn7SDHr$wF+BRHJuhVp0&bFbo!9gZP(s zINU$Rz#1=$!e=q|nNj+gy6y22_?SG6Moec|EzVX+6&UyXi7JUpz-38xNlKxex#+VV zSAv#;$I|RkmOW*8SmS3ISao5mIpr8Xwv{xBmWA7L>=ys>4u{8@?G~JWmWR;_Y?SnT z%)waFF8<3qRs_8((^I}unxh=0mzBU@nfXbjm*TfRe1TpC*i{*uG(XM});dmBgT(4A zF{G&DBvM&g1CiI{$Vu}7;y2GKw=54nT0k5O{8o$~*Pf%HfY}TBN|Kpc z*POMc(F_54YevhDhh21hsH(|D$8BJ-EnCQw7+hG`<;d*--=5*}+_B=}{*4(5l{eLnQ<7T^huzpA)$TXxT7D>~kr3FO1>#@cQQV~^H0o>tv2e8qq(-rn zyynQ!o-tA;Z&Ec-*pn6HHAkr_h&{6xwDx8#`N67dExi-DDzXnm_GOXymv^`xHzXqV z(rC!lvYedmJL;uy-a7{LI;N+3Q3=tl^mi;=#+A**97)2Qn0mN0u!~#|aI{y9i+=*r z6Jo|gVggIZ(m{SW%G83WE$Qto4?OqZ#xTrq?^Zqlzm|s3~A)nC<#zK8{U=5e17@w&)`CK$Xhki&OcFQthIuLO({V zRMYVF>HKx6c9H)&7a-LBXnFuQm1-A4Q=xWoAc7sl!3Im)m(ymC#C%O1jJStz+z}4q zMrsE$z&VsT(!?97GuoZntC@hzV#p{L8^a}vAf;Ecfjf+G(k*RC+Z5o03c?%+%q^Ed zw1#jv1m>}TltR1eflw;1<^z5N!==Y&sld68a3nmAVh^c|$@vu38ZN@l1=z06|^@J0_J&Cze>gKu^{Jzi?g_EIh z3MP;j5cOK3w4GoVRIGRNGZO&czK;V-^ScWU9H|i856}Y)lIAcJ4&v-xJqXG} zOp%_umPQGZcJ(lLk1$V~5A2F36bT-M!DD4Jh*k+6hrttUkaVsHjr63k88ffLnr!eS z=ua^{>FVT`&UJ#PA@U51B%QKL7O@Kj&w~FP^QEeIi9^x9F+io@c_3e4q_k+rA<|Ee z#ZtkG(0Pe<>e_j_POPGa2O{@CzEPJ7)DZ9NuAvnJEuK+d)k9-i6nD?3Gl|aj}bx zm${8s@5AB)wwPV6-Z2(p|9l9ukJwCl*zNKdHvOZIDw+zQK8DLD>@vSxy=1wFef25) zK4ZV6CnGM#QYdRzpF`ja7D)TT6B4k>246zwt8(g?u_M$CzJ|^>tdo>Jx!65M$G<@6 z!oj!j_>MiKWqYX{Qs!Vs<=}gm{JTk6DhubEV z54f#RKKK{W{^Mws?KBD9{HhKMVRf+gX;ueI@9gjoq>vUT>Xbq+h>#sQWYW}uc7znN z?m(gA}l*z0J}qf2K7y zOs_@yxC9VOGD2D`(>B7t=pgD+h`KaKmEPG=SXB9)9m@dKg;7Z-y7-~X8z7>@cXlib z{Bn#>x(jlQw_lQ>mWR#?tTU&m(OFb)*gD~NcB}}uuIwhgv%|rl$d?3G0&ryp)~D1N zk#}~i0`{uRPO9m)b6iB1IH-K*g$$`hx7eo46O08K72M{pBbf}oP^`n8ZUv5`42Q=HiXqiY?V}j zalfCal2`$kjoBqBg?8qmFMV7I>J5)d_9)AqvOKKuQw6I&Y~|oe-}d;it)x-Z7jFI7 zO?qdC^JC3+3(i0NVKjh^lAez_7)#p4e`m))&<8O+|Pew-t$b)0Mgi7i=TNKxn6NM&g-B5%c!ljZ}&Z=O}gQ9}@TYYr^Ev!m_9 z#|q5#p>4p}mKoAJJKAQ%)nL?iAZ*Ws;u7UJ0#$i-0Bb0-x+m;pdS}OuXgQ2q_H9>7 zp~ABhVh!h5UBy^>O;n}Y8DHOpzb>zP!PoPdJJ$NE5oo$AH75L^`H22aHyBs+Y z@J5EqbH|GB0{_Mgh06WFo5Vah-*Ln9CyN9{*w%@4QyOxm*e1A4W|z^9JD#^aUnFY= zwS}qjlPZ@z;P-=`G&2PP85WQWSdIjO88ZsxRwm0+Ra}!P$<4xHDm$dw{U%+@4+S+1 z0@GPQdS^!|=ey9TvjxP$)&7t=fTiR$M~?Q4kurIcIuHs6v4XtjC^ZGKXAXweA*>}o zSaq$XcOq9sW>}3#9PJh3;-7%@gqZn|ID#eQX_Brb%&7$>nIoZb6l+ND>?q~5 z8!1>E4dyY-lxHUwzv))RK^+U^ag20*AX-Q+avTr(2~3wC02aT&am5LBBG@M}+x5+S z9GeOwC&S_twpc|xOV>@CP&=iirAeKN7N>Cwxrl+^0iaID*U#Xu%SDXbuMEiD@8^mb zXQJs@+*B@NM4Ac(gtHOs91b>E+|X#Sq#RkDi@4`;+z}4qMv4gMgL46M;%DF#`CF;g zg@9bdkWnr+hHDi;Zm%u|?h?ky_qK)YlbE`NP)fKI0+*FbAX-eg90FIcKtp;RZj}?R z1pF$7$3H(Sa!3VxI}GQ7!qxD&hCSqZlAB>*mlUoA{yN6Tzr3T^Zt(9gP*k`c#2c6x zKQX}t(NCwvvciq9xruGYI3I(&{rSSe&EVd`Tsem;Ww%GUeqZR)!mUuajTPkUzcYoP zxNtkzcQ9MN{<~*WdEri2+{G3t?*#j%Tip$XdssnEq@8tytstg|_d@GF){>JE_gW?s zQ1?UP0hXAR?y+h5#z=_0^B}|?VzKy_cQ`qP88LlGbD`m3s64_d^UBmmmWtR@k3#P; z){B3(@1&Ard#{3n>7^2H{^x5^Dq!se+m+L&=86daz0%`i!G#(cT(QA$_G6#&%hpufR%_qYCWbSgi*36Hnfqi4don^z&Jx6$?;ZX3V)#`+!4 z(O(MkE~351(c&lTSkV#-1n(p82OKzlqIUs-BNc)V0s4qR@??hGh~mC%|D##;F({uf zMScKV3MI%I)u-Tn#=Q86!G(06T$2!r1fRp;i?SI+s{~)d;43yrI#(38XM;WuXX@%} z(7$1N($&c=o$CbOLgYIZNjhbhEMgZ5z6bvY=F8>rl7_y2V}MG*k3jy!$oSPmMUQ?) z`suM)D)<>XzpzeSJ5Sf`KgCo;b=0o_|HfeXNxZYJ&-H`f8@gEVI~@LChlZ3lKByM_ z3HVA%x%0{5EdQTVs^QD$5@E{(+Oq^v6=kT+ubp2`bUo)t{E&0mqplRez|(d zauNG#QTQ#!eo0S8T#Thq)~-53U~v{m`@$0vu*wEYKxfHv>X@-3)D4z`&eE)tls>uG zJx0erN9)4DGVtia9`Wn<+}s`*4|A}ia!ybpTn{B_x^%_X1=+h72vHmuX8v)7;hi0Bpd3kuCw8&Ta_S zM$D3zh2-vs+Ou2ijI{!YjTs@YnrRE+Uv#jmdn0NkM~#1ZN74}GsNx$nR25Kt7?pIQ zi|?a0QR2%x`U2mN@kw_9&&{U0bcT0BB1F$!i}9Ie_z95*h@Nmv;;X zc`GI-72As5#320%-T$j0;BU?Rq{o)7`Bwh14Q#e$oA{S^w0m?%cX&*11piF0V%QE& z+p|;B&A;;=3Kq}r0F$9?l2oI2W@1thwId9Mu|fRHI~?vGV_=P!o!~Q^eP)z?rfz$@ z1U@EDqY<++taf3mqza7t{X~_-5pda+U6N90XD<4z$CaRJc;?MXOi%erX^wJ~ zUiJZhU*;#3UWz}G!WZb#z}7N0X?~m|taY4>fkYik3@K{QM=DEW5qTU(PMQx8zj;;} zN7W;60|$7)!4es!~(%^$dSqUiX5pdj$#o+Rfa$7lo#++*Dro z5@{-EXAx{F2kREE9lV1>O+&-!+%W#-9sEn9qjPKaN2mihlw5|$ZBX4ffAxRj#F_&U z^&pNa&vz*#YGhW;!Jr(%6uC-M5G7Rdn*qe3jEH}E2lsSR_rxrMQ8NLX#W1;|QuOvv zaV=UVoDKY8<-tb_h;x9S%lP=0ca+%G)-;;Kfu6@``SGxejt^Bex#&0_7Duo}LwimK z1x(*Ch3EW9EqIskk$@k?aCz=n@m=8Gn4wU4GtCsa1JZTYmQP= z5PRlaXr0Ge@`F{^T6!mPRpfk#T)-l8(>z<5U1W)fy>ubuE@HXyY08E3-itxMgz2eX zR6=ws{k;?}mzB-M>?UDO%;j*of?ec#KpJD$uf<#miK|#bo+jy9!kk)AlDQfh*RV$X z%R7oY?M4a~*MfN+Gv(RI#c#S*aZuL-c>^QIJ3bICBo{eu1pOwa%MSpH-{837gt{5* zTbS+oW7V$bUUlySN)o z@8PCW?LufO)GqEtu=_aJU}^hu+U${-uBiJF_W_PO!a>|f?chOh9%7C(@kZ*5cBl60 zVL%>X$S4;Z!zGF!rB{yv_ZZ`(TiTMgDZmNKaUO@j6XgW=Ae73h zrvZP4;nHKXRN!1kcorVdv4>Pfa^+!9YnW2N=#Q>c8#D)Y+JN0y4%Q=dcc3)Yh!={xMHP?oR0 z1ph1Mr+l#q@vSPu*Rc7fj5cOK3w4HXVe=i^NGZO9m!>B1sOr|gnN>_Wk^;4jB~sVZLLQ1ovMFzaA>AXi|dv}nj7(oc`YQo)MQ>B>5F z?gt|GK)zP662L1nSb6~OydV7D(8YpP;IJw?B-QGQ9DbpV52^*L0lqrJlkQZV!l_)a z1~k@WjkNcJeaEcUfn7muRFYYuvbz+$Hgu(UgkDl^@PRRY%#lBy<;rI z{#gfR>#~{ju-oM^Z2CtZRXiu57hKk3m-*%DCCf$ZtM%cx0sAFA8F4X|LRq`o5CR*q zK-w3ckbqS-sDRGK<>(}NOXZL<2RkYU zePPm%O{9e*C7MJ^2mOH^z}TdMMUgwgO5@c)PzNzJg`?A=a`9jj2yDs%6WpJ^;_`R3 z8StAkK81`R4o~HSEg-TbizJl~cwvnijJ8{G+obXVw-w3?ka% z{LYS%K<&<`2}SJ>-T)CLzO!Qw;72h&=`JY#$SDeMza&G|Kxa?ZnNz;ruyw-k?AQx# zd$XJL&JG8IB3}~N2f%$9nDp$vlzoG>&Rhx@4R$TFlWMx{WJi}&t1$r8F-ThSP++e{ zK^2jAc8mpi9FvoZZAEWlkp6`3|5ZKs4a`q^Z0VYB^GrD`^z%54Qu@O?qdC^JC3+3(h|W!ssA2N_sx#U@U1Df1Sv|pdZ5Y zl&_TLC`ajK2Ka|EKdJOm{MLsqNopprvlyE+Kh6=>I!Tm>}$AP7HcC>x?Sb@1dG#`v3m?6Ehqisf94MrUa!cj~pE>VsnP?hIs zu#RC?_k^8H@9a1hEsx`tecRPisPG(*SSN6-u3{{`M5t1oh_9c-UzgXt;OqI!J@?Lz zlhO1PZYr;Pi8K|oPeriPI9RuMZF*1cQcH~iGQfM3IKdG1*8aR0^(h01HeyN-EszT<}HSBQclZ0p3jDGj-IT3-*B z8`x#E0yZwt!f;x*JmWu#~*!$kCoLQYLRw_d?-5 zR*=^mrKTYE%>B@MfVJcYtFE>5PUNb{gAjR$MWlCjq%&?vMC_%9A@>N&$?3kMUJB>E zkAnUf(^I{ugy>fK`#4;lD4UBpl7u-iPr~IXc9H7=j`oUi@lQZ{Ld?^Uc!nk9X_Brb z%&7$>nP;K#9BWAL>?q~%i4-iJ2lEAH%CnP;-*l_upk4&>B}TeF5G^DZIbH_+6{gD% z0E^$?xZ;F*73|lT?fPauj!lJ;*J1GnTdX49Q`SwJP&=iirAfVs7H@G2xrl+^0ifQ- z*Wcl<%SDXbuMEiD@8^mb@1p5@+*B@NM4Ac(g!d8b0}eJ=+|X#Sq#Rj&h`1kd+z}4q zMv4d@gYyY<;%DF#`CF;gr+|FMkWnr+hHDi;im5&a?hD4r_qK)YlbE`NFyrV;2z*s8 zfoL(|YY2S90&*_xrU$|=@Aww*?-(Bc{H(|!73}SBErSaR-^1ev_Lz{~sLR($gBeag z0{;`^<6quUY&ZCK7$_?I4B{_LjGvg`g6OByVp-u=*!;#eW1M$CZ-2h9@H@DFFjvmu zN*Vi6uHP5BwD2bs{$d6B`tM93C@%aB_CL&)umA4ZR9^TO7XPtD$~(cn=~f*U#j;uN z(=3~n6KQ81VJnC!;)2lX$XaqT;$F*S0;&@v7GjB6=^mS=Z;XW4I}1Z>5f+Pod54oz zm=V*5H0AedQK&4&D)Y+JN0y4%Q=Or=IO|PJS1*+At0ll+lKClLY(jji+^`gEmM)`> z88QdM4}zUVNq2LfdZKHh%Su^*f%UzZ9f9qV?ct@so9|Xo&@ao(Q})2accUT|nSS zgfG;vsE*nM;7u7UKZ$qN z^|^lVdqWosHiN_F?2uHeEAlu9ZG2EI*aGk^8J={f>J(1pg2B+(iZ#;S59ap=>5U^>%j* zoBq*bhie8q!etn{%r93jSuSEQu^d#_ZS`j0-*~BBjK?-d&IBbb8~xOJj}t4%E2Bm8O0{?D@ckq ziIfg%fZda^Nd=1{cZ8M3tGz(oo2e-rofegg2m3%^Uly3){`3`>zpK%}*D^kZj35qA z<%2O0sbi6(@&T`~QDf0|9Jfs>A8=b?dOe zb_!S-X35J!a`!{+*)4X)s(@%^guH5|Erfs3K~kDU)Ttab{^cD>LzJV6Z`4rJfSS&z zq!V3yAGL`RU*54l@CPtH=`P4I-hNw#IuJSsvCf>LMrTpIVe5on-f=M84q>v1LMRCt`m9%b26mWMTdPKVVQ zY~|oe-@fw=^^SPnITLPYv0MDhI~*Qsc8>+;pR-|f4jUysA9FC4{A;H!!ki2Gc}$;} z?y+u;a+F@q2mb=*CzW1`-}>+c`a)nYVrpEt2PuR)y@{a4#@&<0%w_PoT3eSy*brZ+xD#p@lg{sue`1&pUb$Q(jzU~zy^lLYB z^_pAJ^fqoPuX~9!6|`?husb+dw|MQ~9USUTG`x!&#=pFSe`$1dZq3~Ybq|M<%MiH@ zsvDQ@iNvS&>Rv>>kE6=-T?&aBnN@Q?C=W12uF@1l36=aF1mYn^#J{|Qdpc>}QOqJ3 z^)O(MFiftf6umuET#J?o9|iug^5CNd#K(bug7NV$?CX=Z^)>|FVS)IUcNBNN3ynHkKujRth17d2C9gShv}cTz$(z*s zQ22lqv2OOVlRCPxzAWmPWK)4 zQaJDZ9P}@kp6W#<+!?Rt#_R0le_vGU;c-`94-XV zmqX=7*0M6Cwq>dXw67u_@cKG5ZcHGp4}@ncfk)fGXpVJ&dGF*iH|!s%ukE|nU( z!=MKnq}cKH&9~|a{Mw8^D7}ui@nV{|4wTkqrN%NSnM^?Sg1~w#5H7wr9TcWdjDXlH z>qBY-mI`M`=}6%owAQGp4WY3SYaCXFonvW;JyZd;jae<6S2*pVPgwyg@2KtYpSS1#yj<{~ zL*;}W@YSLG)wSiX3I&86(Rdg)?m~@C?yq*j7l-o~_4JFpZ%|th7(1isF5I+2Y?_q9 zs}YE@D@W0D9-;zCQ6jSusu6WJj;dD&6&RH>b2Snncju7XmKKtd{XGzT6bB#WD!7o| z*8sO?I=E;;zZY7aYN}=cTNtQsGo5Xu4HVO#DUisp zgno|XUc%hnDg|UKBXv%iB2wJ)EHtLFhOW=L*RZlsH4WhD44zsJu-FUxL*@XM(bZCC zy%0(|)q$WL#5AWTak#ZbLc zz&+F)+DKey1A7=_^%JsEu#r@C4j6Npp{MDV!mx5hbvO|77*XnhVIepjoe$a(Ow$wW zOWB~fG*BH0)=|tV_2x2;MJeaepdQ0iol-V5k87!KQ0iFx=i~T4FA@Cbil*k;N_9NG za{_;7ar(~WiFN8keB~tmiq1^)lnU=F-aWTBOk4I-C*!N9@K^QIY2mA(g!WW4JB^#^ zr_*gT6B5tU5$FsKq@PZ=6(}izRA(aQSsYUzd!@vTB&BDAaSk(vmdXgFsOJK39s_iq zRnkT)emb3ogPrzX3G`Kr)|VA`X#WlXN$}O6UBfi}fFT{4pDKze@wG6yj!kq% zm!^rj9?Tn;Bt+qb{!cHr+|yna5NQU}<0G10mcN_VkRV;Pjpy}7y@0{5_hem zeJrJG0clIwDf#`-cz`wZ^XW2ch&}Wm)E;6roieAS7D}Mi!(czc>=Z8_A-0vgKMIq_ z%4A{=Xd#t<941e&iGDtv(*7_e{$*V!_D@3LDOS*h!?YF5k&HC|X-GW768ibHJBdir z|15~lF;N$2-4U()Ts;rq3k-C7=vhE62)qdPOU%~Kr`>IAT*|Iq2Kf~xyS+h=BU7>9 zRXDuH4r|41J-_5ry^eNoaJ#B@v=g%PHxcPAjudCWFN08Tquo2)F3y142?p;X(t8|f zqnK?8MdDOfy^kOtaF79o2BCcULxlT?!woGyoRAZL48SL@0MWenQvg0=fPVfh9apiv z8gz*D*!FV-|AK>$a21^M*Dt~OiaFbtz_D}GuMz$m4!=pk;r$C1^3ZP)@jH&VOGy#s z_eXvY%nyu-%j@HJE{ls8@!gR>0{s)CTitJ=gtlMQsD6gZFRU{2|DzIqXXLN&`i;Hx z^n$_;y|(ubUVEi}2kZ}qjY$i}N&lbV{>9wkrE#OL@lt;S@()9F1;2!zj6y0R?}+>t z)c=?ocRSS1qf4Ng8{SYhpgMHMQZnz;EG3J}L|n72Ok_crbYv6#?m@SGFZY6&A9jMz zLhPg0CYHq~cqiM!uvmmG^xDu=ElfV576pAVrt3x8<)K^SqcdCu;L3?meJ%zO} zRVyghfO;bS+8lqog5$?kThuy;zAi^EJXwv4PPLp~;H<}-HDb1c-sQMHT5Z6sdbin1 zsG@9$I2&=C4cd+qs-aXM#Ks(=UtuAFf`RIdV3iy!E_DxjBeZ%y2qx4|st~@9oAA-< zNneET$KkgwwkxdKNq>YMz@hb=Y3X2UyJJFA4!CkM5R^eoi7WrKO|k1Hn;`h69DJJs zHzDqLQkx<6<{Z1w?um;{$LbcKY{``V1#UrbK2w7cZ7YtpycNx>-sWaw4Z(M}=I^c* z^Ih)%2;QE$4ccwX?d*E1&){~#nPNLc+MXlXmk2RZ;)JXn5OgR9waWno2aU|g+7Xap z4B4?AeV6#6lh7$g?F7PbCg|yjd~g=N8GMorPHJjruy$dVol3f52`A7Ipzm51di4BS z4f<|Ow{LTuZF}e(sYZglJCp5ais_K~?7RmgMzMr_ukB32K6%%Gy(hEv+BJtm+|Mdd z5#I~Ay%}fMi%Z;8#`zUWacb5+P}!GN^yKuEwmhezMgv;Q(1~e7{a(cwz7+dE{AJqUs2IaHN&HYJ?507cl4Mg53!G?z%9dW z_AGLjeH1R(D8O48p5}!l1h+~zS*T1cmx>uH-le>$Zd`*dnRzQa#x=CIs%cP}&MJ1T z+2t-VD*i=RR}S}w!vXAIKbc9z!JJZ1Lp%@`2eE}+LoCrEQfxaI$U_)uPYo*(86=sx zHzv*i@lYl@J_0Qwm+58#K8xY@tb-CaH!gEmvw=R0(T?xhp_hV!^#pY}8qlK1%tbp(g9tHx#@utQ=Ftd2y~qd2NvbSXS)Bz->`lw+7OtXp@0)J9@@X-|hWZ+L>eC*kv-6Juy zH-lPG5d0HO2D%nn*RhuVog?XMS&6i|9wIle$lU*j zh}cUvLhdG(i+u*+V*G@%YIQT{w=g}`i%N)Y<@C40<+if9m?KF@>Tie39qbaD?YP)0 z#>Kzn>#Y7xNZiE|_KL-H&UfaxMp}P2H11&ydp&ij8j-yIUNG-tW^5^;$o{v|Y;`}7 z4=~d8foLH)vwsluhnODwJgUeIj?1sr!(czcY}YsQacs)%ABDwZY!RE=^Ta##I2t~| z4P$eAZYbpTPa@P)9I7JjN)Zah`KfvuQJ&!_0}GB4q+aS-#Cwk8?N~xQA(4F^gcqC< zqPgmeAiTr`dxDuciNyA7&?in_=flw5#)_;BIH{f(w=%)*gYyn^V$Wdld$>)Gv@S`#3-)`=j=ha> z%#N&ydLJ$yu*2fVj$#`^^ZaOglVx)y-V38SlGi! z`lkSY#^BgWYCFNv#Y5_IP`+S_e&d~(+!RfTE{ysT*smBHTk$F$Ywkuvx9jR_z`tR5 z?DId@a4T2%7AoJd$^q`XU+xDnjr$%#Kd?~jy}I+h2^KH>2#uduBeo#!Ov9uL>Sy48 zVZ46dsNV2A3snb&* z%ywS|=a~he(~)&zA3Zr3KmJ{zu8MR5dLc%qc?mRU7t}%)27D2Q$JRoM-|k>oG&WCA zi-NfrGxu}9>qAWIsOSuX#o1uX!VMxto+S`=Ne&zPBt5<}t)hxr3Q?Els6z{vwW4yV zW*GpwFu-0{(~kS-xQd8c7U7rU@I~aZap9@pvOGX5Fvxx@t=W!Xg^F4c&AM{4%DDaw znhC|0l@MrU4zyuA0)?^S7kPlwAlTwv)5%-W2J ztrNAq3+(dBI*7h5N8h&44vIUnR4)WykAoN7Lvg|Buw5UR4H#p;XQ|B{5uDxBh6uM2 zhqKQDtu5oTlk4{P#&IyAsshb6=4KmMJt;KvMpqD`H$qf$2zwP&8zF>~N)>|k;h^ia z!z=jxPJI!e9|y22aFUe?AUO3$$N?O3ZOf@Hn{BT3cAM^<+@-FyHZ!%ZIjaVu@gQz& zpKfCthq`+c1lp7X*$XP$6-d~?%@BHX4!v=EH&D+}%Wvuy2(u-J*{bj`g5O{S-iib7 zR7BvUNpNZi7+W*LzB7#PUegDOlLRBv`L+RbTV~p?DJ+@E7dN#XfZH?BzU_Ak46a6M z2cU;C+Ab}mgchEc>}1z0({i$JKB}Rz8>`q4nBA)QSu^fEM}odP(`P$A2ZnE}neNx;#mR(wz-$zo*>l^{ z>zDk~iW)fV$qsgTC2a@4kMu;ty`Zx<>)1=7Qq&2i9qt2D{Z?>l&xPEW>8@)0q_apE2`qBXsFsL8+CM>u@JQB!4YAQ?F z<$1SKkvjS`7))madt$y@gP?qrt84BLfdg0|-9z0H;OhK=a5;!w?AN!t?TomRojMpM zhpr6%S{?=}>lB#kziEYH`u2&lhE8qQxw3u})%(P|;~NS|7%(?WvGq9=8@M zIdc$iF30PcY-36*>TtB3$8GIr1(u6xE11kjv?Dm0U95_YmRLwS5`mB6!1nW>q5?;% zM@Iv641?^6QpJHnC8J|OIgTmzGP%+zR3|zfyc3vbpN?JfgyPVNFgU4f2GNSp$uKyD z4eUyT(>@57fldYeG^X42sPyPu`#BvVXRwI<@)@ThcERUN@XuoY-t9bDCD(!Z0ITqC zkaCMefjozic6#h!lNsrkEyU8!xzIU}b?ix?spzQl0lt92c1pe~2Q73?rG-NP`*hx12D@DDVl>q>ZC z#a{NZ8`rzY^o>bj)YY)KhAr&WIDHGTf3Ahub!=v56{(rw7PvO(rR(8x1G~&G=NO_I zkl0r@!tW;bvr|CVeHE%|sGA{h3k#%u;Ry*?WtCf@b6Yue%&{fZQf`ON9js&5^Ih*A zqvKz!v7!_ncd>_^9H-`C4yN3qQkdMsCbjMls7R^gUSRKIto`B&2OVIQC)E9*KETuz zj!uipMUe*~@DK~w%K{v1c3k~IJq-LKj87pWh{IF)<57q_#v*H2qi#}dR@F7PPHAav z@Jk?%qtz4KYW+4_aUnxJi3m?|gv|i=6jGxLoos4Ble{ zdty_%23#_EA2J`XjGeDJm$AzxAHv}ycCc>^969*6Mkt+p4D2V2wV!vTfb|OoV)^7# zD1F9Cb_F3dCG|PrUohOhXH5<6_lho~d4rF;#6Z&<*dLYArkl~cZj z%6F_%j(uY166$++{J=EJ!tOV=+rON4c8TS8`24{>NBA!L-nPW~UxpY_p3ue7fvtZV)9;UIYjFNwEq024{!lfg-Oe^QmHYWm< zUOK^MA-1t6@jL37Ncm-9pci4Z{Q^x#XsZ~a76o}RCZ}}dT4XN6bcV#@EHNpaaS&Gv zQAozZxpMEQ@B#akEO9z|Dl( z#_|ZX0tZ?>CQwjfSrLu8awB_PlrL=>3G9^+W@QdzS8AeRl1dP26+~Q>BktJN)+Cl^ zA{CU?Kv-jS(2gGZKjM23og{@n;F z9UFqR5wq+pvXqTvmY>Lr<&6q>Y|I{erL@gC57isQN+#L`6}MaNZ@(^NQ~}2mq>LS995E#Y+ zcJk<6z??czIoJsb!&$+e%Uy~>q+GBwc)Kufl*7Z)%GA{e;C5x4%Y)6rabchurQ&KNxVtmglnBx08RCWj`Lb|>m zxRX-mMsxKha3?dDXOOt*)qIXTU3ss7^wwcEgW1ANzKu1q2eR?P%?7L)lgn7hrk$ztL;8#afr zO}(q@1Ru@;+}up7IbhFaHa~k$p(hMG(4n~ihr?$c`|y2W3O;rQJ|6~0u)&OU_l9L4 z_REp5I*P5h4(_gBLK(0+8rWkPo6ZFwgtZdyV0q6~EN91+ z#o{#kOkmGq?7pr}GI6=CIvdb)80zd^ISxt*_PM~H$M{vF+nj&(Xl{P^`Dk(hH`yq; ziIDbQh#(hnkb!LkNz8jMM!ZWn-VPPlY{AuT z*B0td@a|$BPc?DP^S52+z;^?E52J+|QjsHycWp(G1m6qheasYY?VK_x3%(!p2bi8x zXWMs{>Or6%Vl*$6cDSEyw3y>O44p?+~ulA0e_6){D3}fRe6rN&*LtWj8B&?@s>rwR6(0Yco#-yVa%1YI<;6BIPlr9D#u9Xcx z4~rMdU}26DAsv1Z7B8`dkV`l{R{gWL&WB$H{uRdaGw?JH4s*03A$}DCud%=$&ct#yRoZ;J(9L;aN(N8yT0Ms&_$skE!l%q2s8O z6~7Pu2h10;V*aI5>O(a7h?@vmF*gyi;*SyJ6AscVI)Fn#IM-93qWx#wzN&!slq-IY zNMCRy;X!Y#Nf1d$2fsw@uN=gVW_({G_BR}Ra8W&AC3W8-?spt_n1i^Sw|x)94~!U6 z9Kue`enjM-IC8&2BKs$KWM4lc*e@Jx$6|wtzq(lc3c_zp5Y9RAV~=%3g7nLg)$gGG z!BpW!!ZFqUmCWi-Sp3BnJOwj<4Nci1{EMOghSEQ*G{Wf)&!w8`UvU0oj_{hNz_c?|7H1XVPT2Ksp!&rU z5Ll80gi^0F0dpI!mI8Qb2J?$0UGD?~w#LFT(CES%ypAzF4QVti3z_9uMkpG%-x)L> zmWRR$tRQ5D&J^sL#frdpWqg{u!@^r*V)~qm zPwXYppqj;OAnhM@}?Yliz2q&Dm-k4 zsGD=tp(RJOHw$TF5H?@$YV)AG(}u?x4@Ff8iDw=08W=R$)d*&gKaoY8#4{ce6Idd$jK#&)=1ZItfoo)3WO0Q%oWJFI3eSEZ zPhxVUxax!)%;0GPaxx<$Ge_Jaso>EJcnibR>1k8&$&WcT1>_8qBgq{J5cVj6s=vrl9hi(4N(CaCtK_J_d%Y%n90b6#j{ zkb&4Q2g2$gwu;nWob*e$B62XWhcI?>DxDF+TBVU0kT^7b3A0DM3$FdURA)kB7E46t zIy%__M#8^9>hjxc&<|sJWU_@@x;aIl4mSt(Il))CBdc3fF3u6P{@>`{!3)NqU4#JK!f9S!I)40U#|90#SO{aE0SWBjVoa|usV zQpcmo3EX6(Q5LFK z1RZN-zvlyb0Yin$f?Fu3zZZgk5%Yz!XUTj!|GgO4OBfqj#!~d?@-MqcfG-8#oo(M)svCj6iP4c|EN=F*jTUp9o1t?H>j=dN7yH5Jn7dqc zE8w>=JhF_%abFm?*c-P)=nfW&%t&!8WT(P+Lg6k}IJAx1uf!5R--(p4jOA`<-NRaA z($NZKrRrXA?_+LC7lRPj%7*WU#RFxqFh_}y4nGKshuA{MC7d3s{@GjS!w&=h2;(D@ z_Z%M_=4eGi{3rw-V}Z!Tl%fS98S&$wJ;5{~{VuY%tu$3V3D{E%b9Z=IFwTje2KO1} z3eQrC+{n26R6Psob4+!23mr$LtoV8GUtqqF74tM5^&*K9BEZX_I2Bg;P~$vHc#%9~dcA4cm!~&S+LY zBK}VtKeCLa-CY-rA6>@sGl0J^P^kSBy+J`v6uKE!zXJOkV<#7Tg30fn&_Qf&dP7+2 z9{hJm{J|2DWh|w1c8rAiXL@-2Csh7o72!_U^=_d0#orM4hXsUEuQLI28?OEZ_&)|m zma#b82?lJ9g$_$$eVX@a)~7|*O{AhBjfMpw(~)I_qJjIJLF1tl6c%CyAv1KQpbuVp zc&-)(ei6o}xjQVpH8vK7%wjB4{3TX6DVre>oUEuXf>S7+FFHeMaaM{fV{tfK?fwat zu`B_RC0RsxVB%n$_@{SWH&_bLr5T#eMa!HmP%T&n*e;9}ss+VwYq0wpn-V-Z{s({DmbW>5N~CU7g@%VBxuFs0>&x` zy())}EMrLu9ak+-t0C~}9Jrv2FfK5aA=Ut5O-4kPu~d2AuWQZ!HpjKlpc^;XAbKAb zG!RM<-4UV(hlng=VP@!=sOpK}Xk=H9Z zaQH{w=u+%4N5 zH}2?ARS4UM!xqpVabfA8>kB|X21J&z#CLdbCQ|(oXaEP27pPRJf%xhm{wg2n-W>&Q zsJuTL>!;Q=RjN(UU{h|eHg#^Kfp=&H%T_i+-WhBAcb?H9c<^0`dy2;4BnaU$#(C!Fqgg23={2}G|7J40X>7U1XS zu6rPS@fZR4t_+{zdPoIV?f7d4^BT_cY=K>8VUUFjOSm7>IUy;G$@kq0pcho z@>&M>M8B#jPF1Rb&7N#C#`!Sd?a$e$y};d@xm@^2s~7yf&{LQ8fx^D5z-dYGAF& z>Fyt+WhxPBJR~Nt#H?~lh`lorVvQ`u<@B_78kNk{eo&djD)Y+JN4Xnsv%AEeYJ%Qm z*5laE*}5B%NWE}>P!C`#mzCX6gVZ~hPaO#8K@4?&SXxl7 zJRS`GA15^I~b^eAryTHe(!|8ACVb_aUDw}l{Qk55PRo(h~2>!Vpcst z-3XPNSS6{nk)n#&Q#V8J7S>D3p&aOiN(<^%@NZ*&$`_ju-zsg~4x2m5Xk!j`p|o)) zZ0=&4q|%0iePe9=8v-{}JXVl{Sjo=(ys7dKC1>nC|}8KaNhNjmP2f1bcMy zZi1Ry)MWJ}{@YXh-@M-qePW^FRfKwtL+J^E+?_$~C2=}Oq&D$7 zLchVG2NxMSRE~HPao^&&dYx1;${d#}5O0I<4iofi8Qc+sqQkpjzLz31T4#74%nz8U z-?ZUyJHw@g55fD0dE-*#aW&y%pg&=>PUjtNs$D?%6vWS%s7rGt6a8BR)C@icx)vtQOf@#z&|nAeeZgk5zg{|2K5)FD(9PPQ^kbrSD5_9CS%ewF*n%icX0n;Zng|u zu@C-)%U|rG?>^lQzR=#_$q0Xg{twd+EE`?ykAGqG9~Vh3ft2+a-;4H};C&!M(;pBNKAeUz3o-Q^mE@M^80N90r zPVQ&pfRrLH3-oe~)+zGDhPrySJpS7X{NI+z|F=-uyCS~VmA}_r_@0n}u7s8=bIXmS zmWiq7DhRSF2id&PAd#eVH3VIqgKkoAP)-Zg8i=?iN8F~Qh?Hoqh1lIV_8yL63n^xI z(0Zgoizb&nL0g+?`n8pMgz_j%t|{=I2>LBNA6~5k(z;C2l|uI%N~-*4H14L4zEvRC2R_Z_me3xeXIa_V@B&F*(qA}2BeZ9y3FAe z;%}u+8LPnS!#sU0cf|{m#l8UbW01baItNjj*dNFNjC9$T_I;Te2;Ly(H96cwn)EjSSz4DMFU)hV&_{o!RgblM2E;Ry#U71kg5n&NHrK<*P zH-_o*atWIgms_ckK<&<`67M48P?W~)0q7`(cJa55-~Fj-@Wnm(i#?<-3d!1DXu3By ztq_|gW^4N(%Dx(zVfuTFX&4Ej%h`32F>N)^W&fOA8q!9%?LtkK^F_X#mC} z@2{e=n(<97b=lf%ORYYtac5o+TthmzXv#JoxCx9iM_b8T7k(5t5vWE+nG>*E6en%_ zfjWt)=G;{>)y~?Q0GrIP9ZDQ4oL8x4#BbsFx?EYxSmgMAMkD5JQ^3wJduQitr9f$A ziZ1B6-9mpmb>@}@a4G{$&7#Qh5$e()b(;p#bS9ab9;YPA-S!9b0A@PvPIHr&-)40n zfCn*9PnLDI|7@Um2Y)am4q*wCB)jNoBVq3K)C?dGWn@b#+ij5Imd}L7EY{G^lGEva zPT*z(d>DhLmIEyI!W_uVWf@arb+P|LS(Q2*w0TT(dQuM2tQ>AW*hi$sHX~g~;*JFS zC}x|xXBWL}*#7xcKd?I*&|?^?-*4!CfScnCY22~E9>>_7O2kI;xZ}Y%ff*)MEV4hX zG)tWb#7T@O^}w(YoXMRG+9^yk_3|P&C@#NJr-F4Fvr4_WjAKzQcRHwNFxBL8{KiIg zCcbzUf6?S}{6!&`I~z^U;ihZ&Lod`cloFkbR_AdmJ>L|^No*_1dd^4t3%I>`sMe15 z!lS4Q5%Hp;BSv$VixKe>j@Z9|ZL}UKU5aRzakQ<9j>Z|tR!O^W0-kf;2LJ#Yu*p^ z1B}-5)Y`bDgbr?@JI!5QXgh<=r#E{)2!V%Kz})*e-}c}=-VX!&2xCnF&>7a;ZmLIt zdyH}VO#p7Y+2E{w{5arGFkG(!P77{!KJDcvq3{$dm=xB1cT=t8X@H+$u*qkff%W0Q zdGoV?KF3h^9d1Fbef~TYUSNenFJ|pKu4%i)2<^$%_bpWPS1asD#!uQ1+} z!X4~?|3s*>*jIsijZrQy8RqCl?)o}lZ!pZ{uElRx=ya{#1mi7cnE4Wp7}ltL8^m{* zxN+NqC6aW%i!kqT81tkoUffE0)%%F@0Y@3swu}{pQ|Av6@FNa5pzVNh8LRpj;XdJT z?Z|E7!chYJDFT1SfxW-y4%y(1^6GPZ?F;@|x9Hb|03$E{7a7Hm=YohEuzWv13OXN9!)d&K*p$avAD_eaG0iR1Nc_tse{@6U+! z3&+~B$XJ~4{)(`_aafa=x7~$yy8An#{lU@twIvxha@C&*_7?|hbBp7G(P8j6!v4cy z*KNx&IQrGUX#O8HpV~UMV#KgT7Tx{v{g?P}SjP_f|3?h#_&>_tJgTNQ{Qs^f%}FU~ zE~F9-8qdBiA(2uDiIg%WGzgKQK?9A7lm<#@P%<~1eccpAqC}=h#*hz%3>lyEUBCaH z_57Z-zGtod?>_f=?R#It`+eQ(3>6cTmKNRk|Mk%lvlSDil}#J{*9Naz>Gyx)R`{fV zZJSnwY+Si&qw_Q?u_gce&-%5SHmzUvKg)=Rml{%{Fa4kE|Go?jF{}TtWvukyyx<94%$L_on-u&oEt6F=7_~w$Z>SQ^5 zjBgTJIyMN3J}>7c%pb$)bPR`Y2Qq}WKXuUG4l-v`snB5MD0{r{1( z|G$#6aGqxUza{s7p7mo9KN9h)6!Cj>OPW6?;`d3!@2-g7SP{Qk5x+Mges@ItOho(~ zMf@y8{I-es$%*(ic4P{jMEoj6{DgLUg>yvw&WQMJ7V*2@b4hqn#IHfb@Ai}iL4=5( zj)>oA5kG~j3}L5;--j78xK+f@U&N0@{AP>zJreP26Y(n@eUi?L_+65$<10k`KA3CR zt7|=E`XYW?MEtgh_`M&&;j<++XtGfc{hogp@~O|rC~z;dZaNI}dmd5U)G_Sr!qYIU zDw2G}HMqUY=FybEZrGfn#{4|YaoN;ofGJz)e)MbMuE`g8Q8S42*O`Pu)PME z{H2*87^kDd0<%mh#J-FL>OTa-x?2?Gw;KyGGazis7cM!gf%JoKK&_!FW46iM@fuwe zpXGv3C5!Jt$&1z0;Qk-@u3kgC@~uer>Iycs?;C;B3N|fO zolRZ68b2v|;FM>psjYky_{$aH$TStk_2n}8#nVYMcr4fleFys+a*&3(bVKd~-#yg~ zB4f(=jel=J*W4TYfQdF<(RxVXo>KVWleoZ8IhHwo%B2snyTCnF57o}Eq+*Z#P+L42 zbJhpoZabb@CaGhT+%k!=i$ANS&7ko(E>JNmd_eQSHULwX!N0F?0>fioZg- zRY@e^74g>?b+8p&afrC|P@t-PgKA8tGJ~|aENcOgt<(^ll%Ed8kMbehTMN&eT1wv@ z*|WIdMuBqtRa|W*5Jq?Zg1lTUmY?zyoWzxJv$Yt0G~Wvs+?|-q#71~_-xxP!>TvhJ zDl_eK2-oWa;}hVb1h}Do%my1GpVUXLffSfSiAKV zod!GlYn>;|FCBrh$xwOvdL5?9w*P6MHzRex z&VMz`>6XT(tH~6!Oa?>rJYmsP7dHAoj!m^pgAp4P==pR#b}BTE-9qWw#fs7RF;W>H zr?x`#f-h7j;RyF4#^9gk=X~2D9cDax1sSHrvQPbfSnRliYX2!8NM%l{~*)g?5{AkuqQ%nVPPxT8m#_QsLx!Z&V zOEs~h(;sd*HInZ509xZ$L%N0`=#+IDWbBh^!i_0-X4Zaw-L|Wcvs7AWEAPjSRaRha zX*G>Cd;xENuVi|=9l>XVI{L)Kl6}JqSXi6Fvfs>v3dOUqPQ@22&m5yA&_<(crZeTk zh;p@Eyt>+VxZSiD3V#sHc$h{0$_Z3_zZ=FI{B>y5QlO2{fbBmMxa^$e+|=TkOxb@P z?z!2(uLxVtxIe0FL+NfTDsp5yzhpqOfiCOr$>G(fKcS=_nyjlX2pe+uA!RoUreqAo z{_2CUgRA9R%(aBW)uPy^-dH@b$C%O=%z~*hjbuE%8Z<0}@Ws7n6kop|I>cwPnC7Rv zyRR)?U#ErR|8phawo<&aA!|DjhsM5+BvCGl?PDKOh^{7Cr{>Tdzm;5D?qw>F-v@@J zN~~!78BW1$tOH+P#1)$*L&MBWxG;GGLt!_+-64qmn(Bs;x8v(Jsh)v3`{SWA?lreF zLV_J@>E_Jx*0O|ydT6}OkU*th=-$Sm=HGR~-1?>Ldd3Pc_%WMTRj=gc?{fy5RbEuA zBZMZ`RG3xiLdQ?&upqBbB(=;Rdyh(B&Fi6PI_v_tt2^W8nLKGH2jS#0cdlWx79G)> zOn>Z0fcXds<_qh=E$1T4_?}HOyqNIfMhi%PYC%h#7jtK1+_)vn?F3U!jbdU~)M3*D zNfxri2ve(f(Uh<>C?;>TeLjrbFGVwZl;b4+8MCb^LD;R!l(#;-sXUtg}#VI+K+zmJA29tAzdAIRW&DIdGffhC>^fa)Lj=*OHyczSX; z^8by6855$|9j8%f_%DD>_iPl7Y;$8(rOCKF)E#!7F2?j$MO5?H2z@V&nZJ)4nI1gJ zoa|MwVw4k9Z0`Wa?K|MR@+g6}hYCBqj$>OP60XK)(Z6(Yyfb|w|5-l{6+gy7&!c0^ z=W+_Sm%Gbvi1o(3Lymx@$pffQXy&s6_G01V5N0kLf%ytkg(Jde;>$8Qdiur>c8`2R z3n!i8iq39l?{p3W{<}sq3p^mkc$Yx$ay0Fd9m=@xOSwtP>nNaO9BWCmV3NJgtOJH) zo#r!Y|EmdE#o}Cz`Y5*5eJq{*?#Y@xh1~I}9o&&7N34B!1|}Na=fC_nj9p*T&Av&}UA>KTFYP8-;HVk2wl`As^TvQg7O9j_Sm z!rx;V7i2Pchto@#W4@31 z>Ic+qKNyBaIqqN-aE6bWF#{T(Td@e+jp#P<2F2+Af#{ZrlsUE%T2+owx6(OsKNt%e zz8tDc{d<&z>m%Wsh6&qhJ|1h6`gqsTl!c5pD0}j zchwtdhv>ca+vm!>dw0Xdo*}3kbPm##-1&lOc62i&9?Z&qamObeq;pa-cw~C7C|^5+ z_#8Eu-}JTa^Z7hV=mV0Q5mL62U8;!4gKGI{0@sK$_fDNSkV9_NroU5$I z9O{qJ+Tm3c$xE@|(TQk(L_p8XPs2L7OPpik0*JdbgkAl!m3Az+#WnAo1^3dna|OpU zxHVH#NH*sR%+QDuJ?k;B|EmNGnzMjm`wC|Cbw3*Kuj1ZI4G2zF_Bt397zlgHgX!$m z$F$6T5XSBoRa+HyDCY12dMig`|Fm_amR(2=DF#etgAtB*(j(sl6|}i`lUuLq0B1^! z(Bvet@|UmqeM#e4>pW>ZrZ3uu6JkIxwTgQ+U!U%}tYSlHJ(i_k;RY_2L#xk3aBj@3 zdw%IPiH*C%O%+e%r$4z4$Mz;u_tQE!@op~L-uRJfF0X>@--+O&n@oG0pVOfOTbZP^ zIi|fL^18T!uRedLZqE07?xh`YU4*qCFU-j1vh^UW6s+aQ{7Kz{Ab$0ZTN17u3u-fE0!e?KCKTVSKfq}CJh?D zEuISU%~<+3TXbnW1Xha<0T=BAFGPD$b$CUTia)Gg} zV70mcjx~iq_2oXAc?GD$EeakLq{0mC5R{ZkCGEn6So1)MnGNO%LVf&j_x8~!bkC#F zYUyn14sG0j^Ak8IbZ{Hy+yLk9UVh;ZM`qu@1f#Ss zu>%ij-LTBZ5uVSw21DE8DQ$cbMbCNwXC^j*;Asn~Op}EbS2frw-~Dv;s{yk}D-itc z9>R8aDB+BADs2A1B<#-?heK_e-01U;^kKLdo3QH`g>K!>Kywm3T_?a=aRqd)dkLol zmw}gEEbQwzPeLDS?!@a4u=!gpP4;RQ${qiY2Ip1sJ4$;YqUp{e-et}D0Jt%5ir622df(At~wX^m+6`N#&+LCUVJn|f05h9N#%zS}! zbYP***ZKdtFA6*aMUbU+nLnp)11^4hK|E*}3ru~@r`ct)6&*^P>ZvNKuym_yW6C6Z zOrCS~Oc8JiR;;8poa%a$Sh3trd^jQ*o>(IWN#Et{l>EWvjwXAoxfMIS_EEjW1L1ShDnZMUMmK zu~E5>*mci>>Hn}ou@xh^rFsewecFJ<>~UjVD?U*F<#4#z;)+Vo%J|?^LwsxxIOm!; zNZQ7+>}Ct{cyyO6Mg)_dVI}NY5+)EH>;O4U3Fb2O1DU^?!Wy(A(N@_HJLxP$$h2{1 z2gb3}1(9s%3PAM|UC!#kFL|hO zDmBQ|@D&>~K4}WgBD1w z#N&O6xa`k=u(A9$^&cF9_bg>_<8W~nD&vCz6&#JLorLFe9#gm18_J8@4HI_-I~Z(E zp^Hx)na=d#SX;CjA83dJ_*Ijn?g(^CiDvVny-~KJjUJAj4)umB`2$5pSY{;4TsGXL zfbD~@tfU;=^c|t$M*#Hu&7qo|3G`Ve2TtxbqNfTnEHloP9k~2}6!+YRn*R)8{WKYT zenOVl8+gsTs2!nauSc^bdOqycnsE4!AAn;qb)YG686>-vKzFqnn^iNK_j(kGPR@tv z$lN-9sY?kj>H7qR94v(yS^=D$-W;3|IfZGN$m81en`rIBx58MtLa1|I;c#d`o|yz3 z#M$P4#Os>l-Wxl?HBuLyPQRz=`B#b0PGU~(^?XpJ2g$CTfe%zdSY+QyXkY9=^9-Je zFf{V_)+(@$H%r(+QzX^DZV^seT+FAK15Oeq(zf51IJ-6t=6-sn;KBf-$O=n1>+OVd zMvg+U_txy&lx5r}o@j+mBJK%~XCDIGVNh%cWQ#UeS7kTo7Ra!h`mLZk$&}8%eoeXu zW4ZUuaj5yFg9I7LSU+?ctH0`mf%`Ma|BF04sve2IU7Se9tCY;kR*^vSG!&FX!PC#i z>`3@{xOM$F36{swDbs3B`1J{Rsv0vT*U1#R=nf?uaKwG>b4mVCuz2-fqqL+z1*L9in?RGk3ufs(DVS%guUZ(QHJXEe+485b5 zk=rR59P9-8v&oGF^Z(GnF`78C%@6k2r9$1qcLGC?!!UN+MN+bl$HVvL(Dl0=Kr*}N znkA5|gAvPn{Fh&Nt(@}gGPs*bwU89@9ZWVqtc^-Pz}Fui!!IO&H>yV;zy1;{dhXQ?IN7Fh`LssZfF)FZ?l zEu|+~tzcG=OTGGX7-V>XY!*BQwfBakvFk1u)++jatNqwj%0mxVIgGCx55fCKP~%Z6 zoMb$jIZU30Zx+bnoiWMK{$@Q)`!0kJ_azu}*iVXDrDQ0&$Ju5}&NL$zzG!$dm-HOI zXcbS^-*5A_Sq}W_E=TsYx{<2SB*TJgRjjh{;=|gjxpl#FDQC_firqL!O+#MO*DvXC ztw#YbE%w2!X0LeVu$vV2;1+ME{f;s(?La) zUb4mH`gRr#npYDy`Wajp(Z+{O>ZYx7vMlt78wIue;C%Nk$HI>jfCh|loj!vt!W z4>|C)(!|YMhES^G5F8cSOCNW|Q0v8YlqjKs?iClf7XiAsRX>+}!{*Y)O<~k`vWUMK zZ$*`vflRYQpA~eU;>vG}dQ9&~R{k#lV~&`zf4T=~>IegdYpTiSvMOes^y3T+b@05u zBxse%Gl?zve^I&*u12-fwn$^agdhRe@oNPfE>Oe|yRX0n<&Og8S&LcH-1p%A z{yJEl2tnxv;-#@`%*L{kdfhD+F&6n2|jncm}$5h?p#+3DjyGXG6i0^%&rH1dro7uHMZy@)(*ZJmXP0a z9CrCg+ct3~jo*8Me$5EO`0+RBi^W#qUn>(_)Fa1LyZ@k*E&Ke)AN!sjBb-+T6A##c#*6Dvx-EtUO&5}O^gr@2)uZQUljw_*8LX<5 zWk>w%DY)|@ZH)Q}?mgGI;?bYL`{-D9@t7?vUppIIoVM{TrXQj3^f!pzEz+1GjtS0& zN8`Vl8%S@`DgL5N11#$ui=Hvf(C|--JlCA1Y3D9Oz{EA=@?{akf7wkgvpC54+zw+; zyRkD>ikNgkhvm2Kg}%~5P!i>V^82qr=mSZ-ISEZYsNg(FJQ21&;hCBI1 zi(P-K$>w%OgXzAg6 z4L58U&YIg_l0=3QNdEVbzCXN628&xso{hjt*Yi+WA)uaXJNa#y|LEp{DtH#4&o)II z=GzAsl0|kf=%!>)E6K68qoYuI%TQ+YGG7o}oWy*l#bb>AWO$$;LE_Kzx%hAW@XTy0 z+s^I8q3g5grQHp%nso-otr`lp9=EA)LMQd@PompfoZwT`R7kjRN7%Q21}%~MK%N^9 z33Oi`;(l#@OBUjP;J|}3^m4TZE;(+=+l=}~bCNEB@3k>dxZj2>vwqNlwDBNRdIp!z_3JN7c)wL?g!zvqL>7G=0{G6UiU1bDza zm1>Vkk?HANcxh@3{lB#FR)IbD?fV**Dn1j-+!#&yz8T+mKNkGd>Jn;e93k}-bN1de zkA5kihZ8&1$n}XBzFGE}_N~bl)wx<|s4mKn-i%E8^;wtkb!u356zr^2Y0cSJWZ1C) zH4ZnE#)ohGI*YmNSVt>4Qw;bGNWR_z0EcYfFvibeuaXxP)f(aYE=jgle-zu? zx&xI@AWOENf;YDhp%(LIp;N(BoYQ<6ytE`)MQRJbC?|m}n43Uet2g4yi9CGoc*pH@ zZl)D&xfI^gLEmpWa(nwLsQTS}m|>)W!=Jcga?lo9;AF>g8xpvLtG&br5K4MydCS?5 z`pw`ystPPZGN3df874pU;D2w6gYog+%v{2c{Tb)QdvDQVZyss#IcloR_}x%?@#HxS z7%8)%UQ_VbhYz&9rd2RfPZ<)vk z8ip*?&5O;a6w*{*4ioI=v7-a#I8XLHuMmRd5@8IHvjO^&l-T6kO}vWScKW$-5mp`5 z0QFUw)L&+Z!G0&XrS9@HIbNUcT=KwKXWqlj!6&@uq8yQj>jA$dAMuGh^QmQA4jU)Z zNTP)EXz}hu`Yb7lmCk3lx3hk6{&GIxc~~8*Cdaajjtp?iZ>R4kb?{_`O1$&S$RSoOpIPfLjvA&zK_K5N_#m?JD?Qv7WY4hc<8NLE{^RHmt`NBn@wJK5_Hi z%OL-R9eGV11OC}6Og{arprnAftjbQP?W^ZnBJNVE$Zxyz>_V+eaTAs2j6sL~>1_U7 zGujOfH1|&^=$FQrbSqpcdo z7mG9c>nS{+@(>EFW7xvUTQT^DIVFEw^0;8OqfVOZsJKec zJxAb9hY`YJ8CkqmG044tbc8M>D6#(>o;i5xwv&IkD~|iLP|*C{5n4J2AU81!8qe7< z?Z>_}c6$;_{kMkPugjy$sePy?Sje|M1vaDn9kt0sF(W-E&{=hvPV4Kkg5m{u?&&^M zwA03YK`r!Pp$FDfBGYUV!*_~@U|L8kY*#IWoS8+Sw$PNVHTS^Kq*~e}ITt&IU52yE zXY;O+pXu=vRYorPF!(;5Ptrq{+4vbId|F2*reCAv!)M^aQB`8$q5PVT+P;X>F?wzu88Dyj+@K`KA_nv#pR? zJ@!Gbr8Bc0b*UjK$++ zer(0e1&};)Ixcv6m79x$(0f1+-^gSUjhTr0H=dAUpBt*rImsOgddP`lKAi~GU{96A zF`iA}7O5VfffWQ6lRM~X>{yzhb^y-_Pt(MIdhC+>0rEVQLh|A3aQM4P&{e3$oQ8(Gc=t>zs7mcSkn!h z=GhhSIb$a8Br%1P$dhI-_o-5qp%(MXE5yLf-dw8cUYO;-3L29O_=!#X*xx!oj7mHQ zy)6?_(Pb2lo1}mq{cBl6-$+)O8pD$3J%!7&c93V_7LdN*PLp!S(an*Q=r^x}W=A7n z+|F`X{x^;-C{Luu&M%;5_mSLH-Ei`FXM!P*1P#C9*rj8^xaX4;o?6!jj^orZf8!7g zpQp(0Y>}quMXOn?!UK|gR1Hfn)^ZafGU%h<5SBd05xs3^F&+C|;CwEhz6q_d_i+n& z4H}>_uY}VAUPAXyceZfL4DQGJWspMqk0>O zX}yBiy%k`#qK4!yhw_`wIlyGrMJ>s)*xNag%@>$poN9gFE%~7Msv(0u}M__Twx!e z`rA=j_2e=A-k3_$n#KXY!ID}8KWWr!S=gXi$#4Hy2bGqUkdpeEE{Ar3+KKVZKXf=+ zyij40o=N}Ki=2whuG`>G~lxRE1HyefzD+FJR0F={yF zSO7cvU=|MES;IT6w!}*pKf-rUEo76QaHobzF>@CwZeW@?8!n~CW;HG+-C0@urY|N` z(yxponxF9-O-@qM)5jDTx|sPmTVv$oZNmLaHJGAIE=W5M$8BS@nR?VL`gP(3O*48- zg0c_X@G&8vqA-l5MlTn9dPDRuSB$;-D2pdv=)l_cS{Pn5jCP3X;d3pE@jS|~g%_U+ zYVs$rEw_f@BPk#JIzACbEN|szZX~b_wr4LB*JIPYa$5Sq2)B+~59Zs}Ky#%c3sxUO zHtUNxztO8`!G)*v+{cyXJ=je{hi(?tRtHhuAs!5UztOx;&M@jKu!0gX^gTORyEnI< z+dZ=%4xOLCom-{AKukxl@uDp|c2<{vI@c1?HYS2#*FLyi9RV{|RnhL>dTOiKVFlUXpEi_07IGaxv;zUWOOm zAK=CbJB;X93)6m{;KvwEVsq>UAU^gU{arDVrgmFGmCGE6uO0_=k34AS6b(p?(Px!M z7U9;foqXE!{U97r!+8%*LZHbamgYSZ<)-9NUrj&k8KMOXWA~tH`hCjs7|y0$DB@g) zF9f-f$LaGWO_1cXDdhGHxT$4JIq5pQ`1=fjQ-&M2>vD=9W6A+4>Nv`; zI^Dv%znDlzmM(*)WLP@Lv-zPEIatJgSM;~&Yo{|#?p}`6jgc& z+FdP#%X1FG!Z-7wxo0*1+;twOt!cmkn6w1Z6r?-@h7McCN1q+W zG&3K<*3LbA`{F^1rpy&U3&x& z{HKQ^KYgRdor+9u^ET++l_(H;h~b8KPvNhB0u0?f2L!QsR7!QQ(BwX~1)7U=@>)LS zawqp$^E)q=;D(z|{HB3FtMPzmIxV`F1j`H=6g)o)(o)wbBIUQDB`w2U3$jb+R2O$SScQ)dDy^$pp9UN`hCH zrgIAf??J_RIo)<}hDd{csC1Je3k7qL_9xHoC`&V?ky(6aNfy67;~=HQiu}KwHA_*YaHd~>9W`2A+TM)jC*irIzF*J!|5L>rseb6NiMIGswZBj zFilr%{u;o4ZMDD+xBaO2L;&}uV1U2gG#T@Z<=~l*6U93DsXc#qsRvB9Ur$f^3;5^@yE*vnN4BzJZ!Y4ZmcK4q%DrMcK zZ!tF@YLpkf@a_k_Z-+@i>H!TzjA3y{ZoogWLgDqbT@JgHquA(|TKox(P#V7_4b){< zQVu+zhW@;;M7`Qd33=cjw`7p&2_f?Q?A#D$>7iouKpJ83^WYq2VQ?u-q}1`)yqbm(A7i zMZGwNbl8J@$|{g&i`g^LMem&h|vcHoV6|u zN-46k#wVbsuF0j2N`}s`hoI0lj5(IB!6##-3fYZg^w#f?s`NrsEE45*OFH6 zI?z)+!7aaI!Gt@%K;>XRr_x{9{N(H*uX3BcI zbg|XMn*6_yfU=`%IQ&DZ(9kZK8##SHyPiIYth^P`P$`=xdQX9;P6gyC>UE=*SJSV* z@@z)%6G}U^l$rNzg@{*Iq3U%${0qp3rpynJ8)^Wid^7d;sG*9;i)iw>i03Zrfuz-Z zy!&}N)81i>s}%=1!%=DA9fLxtMMjv}HWlVKoF|zgJDAyA3Vfj>YaR7cIAq{4>35HY z-pMI6{H+fAab^*_d-Ec!EqO+xEI&YG*bLH0TnxigEipJYi(Wm~VV^SeNI5#3&DLCi zZF$Q`+*v?r`A^{1qDWR}TPrM2+seyM5mKF54~;I+6fTgNMf+n$(*^650=4dqiU_6`^9CI&0e+ z&MrP1i(;ESD74;=_gMdj=AV;cEuGikp}8eY5jz0KV&wrf)Y+zS`80dnG^XZ#6He0; z()IlWW+BP&*w&bxT#(20biAO>qpk39?+W&@Xe zQR>74DtToI)B4ij%?M*0D*QstRe9|Bmpxeg-h!+2@uw5py|L6xNto}SLn=8sIO?Y> zru`J7r#s5o{5w2+TD}<<$8SV~l2|tTs{z7XCr<0-eKK^pPW_4c*zTK*$!q-Jpk^4H zco+;OHG8S-K`nP}r5dx?R7IvkRY`3ikLQ>IJqq%mO|vHPg|_pV*-Z&5gh=*JNdaS$ zG$CB^2&Cf{D9=xY1rrysi{pvoQi(Ys+c^LnD(L1Q$F_azG8ziTOdcJ$S9R5D` zh<05##jP8?1AhP7%V*0S#6O~V-l(oXzGlwsTyYxvJbpifR4*t0|DJ>I*z?eO`2u}6 zw<6Yemreyr(BUo#T)A5Uq6R$)eY6}>QWr7ZpehPk-%TeQZ$et8FT9xb7G95vg)heU zYr1#s0&n#^R&_Lte{6RI4ixF(Fi~E}=y1WHrz+g>Z+Gy%DKN?5x|sPh8aCcn#6P=z zs9NN#hggvha6elIZs$ak@e?&R zywnEGwAWDfZDKcIF|MY0r2S?H4Ymte@rh!4!Le$vxi^wzh7u2!)^JT{7@wc4M29Um zGldO#u<71M=qsELY9;Ql=XnYlueHVFMiTg4OBFrUM{^FnN!Z4{1vGO9*+F-DBGN~D zvc#EO#BkI(_?ovLoey08Eb`fHg>wbRxp40&I)8O3TsQv-$Q2aXFT>OlCpJc+loADXyCxx&g%QcugG>%*5br(!uq(WEcIidBpe-zps&&o13 z*Ns#%WDQSi`2hJ~;HSl~u)n1+V0Ire%CuqsGIQpcz(l^sOuon`0#dg~K}g40$maLr z0mB`jS1E;s#j))6bY)06IszjWNW&L#CD1rNN8}Tlu`MPWVeG(Xnx`d%R~1vaGumVL zhb60F?2gUMW?(@;binsOyO@dzJUj=8uZHeFr@7LK$pwh@Vvo7 z*8S!t?0lj~-0K|pCaTx8`W(nvqMla$Er2O!tOeM+nKsW3WcnEs>BE~*Q1)^p`sQAt z9#LQ0ll6r3CsaU0l^Tnx8A(oKw%|5b|`BC4(NaoHnDqbfAkD8nL zmy;$#<2Vau;-w93;>C1zFceSS+r#^>O$MJ&H@GnWIh>u~H`#M;c&uWhV9KsZIFOhK zCwfvK;M*CR%PNHSic(PhxE@9ZhLJgM3Ww(Xq(Qwi+&VYJCh57f=4~~(C48j^sZ*J4 ztUJKspCma{QutrTL5%Wpr$EO}YM79~-VKOhc3l!Zxph$4mzHdL$ZM)OE#^6Wix`fRhJgA;s6=;>z?TSO}bRqF?lQ=zVMj4 zpc9HR@`lWGh6MYW-jYL*M@( zo#pRocHlM0T5m>4(b05t(o35CU52uxn_x$J1RL?Mjo5EZESIXGm92WrdHoHVuT#uD ztFz#{Iy_-pV=3gTdI)R(SkWBUTxfBZVb^E7;po0l*tk^+O{5OO8nmLlVd=1SaV&h0 z>F58990DfA0jQi9C6qfSiL!63p*Qsm?vg)6J~aVYJogCy@Q@rzYEPtk(+>DtBgKXZ zqG;W&0vh-JEO>f9;r^O!rp9mU+1uS>LfH>I*Z;_XTo&A-5WmsvalQfl+tnk?yz`!~ zt#@UCO~+8IX9hi3=0VAOOUZxJbO`&g5Bm2mqa77Tz=<#T(sU7mdKn$;h^LBQfmA9n z2M0e-=F1K$qJwk>)&I?gy6B&DZHX1yRt@1qDjZbt%9t{J6Kp?nlIVCAO*V>!bIa^` zn4!cHs^V$t+UrzwDUZ{UzRsNwyCA9!H`B?OuO#dl%XWT}0kw6;T;W|cR^;l9ZMJ?S z_pgkX{i)5KbPYu-lbaCoMhZjrMsQ;SYWaNIN957E6Mm+RVpFf2;-}wA7OXPzf?=gc zsH;nroV`T;y}KurzdKI1AKs^>BO}ppicpX~c_=8`oQAo9ZJ>Wxlg++wfjyHu1aC6~ z@!#dU+`e2RG?jH^s|OF$mf*wO(~U3r7W+TcI$^JH+t*<%#9o}))JdYaHcz>$-cVVh z1bZ>=JKxy5RM>eOppeNhm$|diEM#t7%vd+N7--4zc}X_m#|v=CIt{zS>!GqHlO$fK z;B6gJJb1>Wfc^!`HgB%_+j{I1uV}C z#cB%Y&tK+n%fqE?jEfY?b;i@x-xAo^qg*?-cpSD4f5XYY0C8YUPofSH{ zFtNGIxF`PWuqbXlctpJu8r@N2jxJuT-f9#t`CNe+{!~V(#IJm0qb9h*R@Hdw}) zhn$A&`6~2fRxG(sb`r&rR9G4s&q>K0#H}znAs+q4g z=q1bMDG)waimoR7;c_2kL+ZU;)@UuN%WNmJp>IWJP|m%f;d9clyg`~p?iz)|76 zz*x?7gd$2`x(kk%TWQa6JuscOj;ZGyp_N~+QTxo3aPwXq#{M^j=;jb^ZuvPS} zTbt!@jPuA0!mWpl(W^NX>++RY&MSR9_|cFps2GnHiQ3c-a+vQ?%{hGYN9Qd~RPypP z4c^Gds%QmfV7G*HBP#?)YQ@>MGjdosw*j_{vEYS2h(a>mL9BfYTQEHt3?etf@!`iP zW8O}DHDwO{Q<=kx|Ef{LsRe?uVt`GuKA@?;4W>+3O7L_O#Yo4|-;MrsBmN}0x7?$k z;-7;0btN?NZ!A~+-(Sw@{3v$NdK3$cc+T&4->I5ALJvjv)3Uu}yfRL3Bj5I*uY@w!+;z+N?2fGQ0Xyi`7f1 z6sr)P7Q3r_GhhpzI9rJHGYn;CSS)_|n8c-S)P zjxc__6E#^*6RsR<3cm*XNuq2&TynA|HQAGJAS@8Ncc_A)Lj~9CF48un^lDm>&K^bSgW9mpOM&SszoFwJ-Jp-X z;1*XXvekv1v}x=_cIM_BDi6H?_En(ioyDjvA{G#>pAOvZS@l_p1wJeOo}^*FA!)xsT|1h%9Pd z_kx_=$N3kEmN?G15#}8m$-cCHp}ldNaMs-ia2i`-@ys^C<2ZfX|5SrrjZwuHWy$<4 z)%(!a{adJXW(8=@v!T0PNmRb{3pdm{7)tI>BuEg=JNUk!1?O#9?0Y5Fp*0`sP&3rX6)-T0TmlXE@xrh4qxOq zxV@w$J66)M+y1N~rj}FRpTLb>W{JshyGiF}zi8f;;P~DV>|=rx2}S2b)w)Tn zVEHSEr(mTG?J9AD~v3-$6L)-r#a3XO@Dlj-`&wf zac=&6#otP5l}V=g=^Qjztg%06kwTFE9=3Ce%sj{gWA=4W$x;EU9Uh5Ix(1}nJVAXu z2c@%qatUVN;qJ2^5GCeAW1pGhpWIril$Rz`p%*hQdc~g_TS7}_fm85>%Me^@bcIX?C*W1nadhdVE&IGJj|&Z2h5@cbmo!{(tIS$h`1Tpx z`C-VUw#-D;;R&P|Z-YNxoDuA^HKymfOa33y-ZUJm@N54^lq8f0ks?u%jN#tbqLL;m zMU$dL10`uLl_B$3nF$R@17%9?y_QOaW=fPMN`sU%YV_aV=l%0Lj{o<0v0q=;aqoRx z_qEn}o}Wcmg|$X_cCcW#oBWE0W3c7Q3cT3sPO*A6OrtHBFPx|c#xs;@Ti!$_KkqfF zwaG)_(PuUZrnVH3*T7`6M{-i1GR52V12ACcK$5&Ul8w-ofZ*ILJ~YY#{_XG}(P=jf zjMgU;rB5uvX(fB*AHY5}mNDzL1fi#t$>iF4x%U0Byq)O)e3CZ<@`D#(38QV-oc5j2YbNZ4l3Ea$;{RZizgc+Sr3ZFEM|!B1t|g#OIy2#9Cff?5Mb&dwoNn z5)7xXzo+cbx|SOTscv4lZLAq=wbkP-PJUqu3r%Qj`wZ|rjd;P(B?TwG)m?%unWGcx+aIs;a+OLv%ZZVC!lYT!Fgh~BTahoKH1aCl!l8h;FB#|sbfizOyj{nJS2YidW5(dR<( zaNWZ=aDN02zAs7p9Hl_1T#j0@OxcE`XYgJW;@|9O;1b^Bt3!`4zaa~scEysL_C3~p zubQ=QoX*nxirL2<9=Nxoo_)M2MMlBz(6xZW8+~tZRzejoS9%dUt4@nxK|8uByulVr zHFkFxLE08`I(KXj^!2~XcfC=EE1OGjPo@DxS!@!!Z}WrzKMB&fAmq09oaH*tsnClv zGJKBZVfJN!Iz=t66y5%vOiteha_?b8)sy-H-twFR)#uEmUmZ$RenFC6-OxZCnIhcK z{TxOK_t%MdmF>86g^Jg>qRNydTu5L(lbm^=O7@E?prj&rFExW+uhTfX@DN%bD-!)& z6UBB&AHvaHexzm}$(Fl~LZz`k@Rz}TR#}|{XYyM3EBn&8`->d$V4W9KKA%j%LHh8u zCY=Acych~6hp~+@8Ehy0#;TQqf5aW&w|6vV7|YT!!CzjbWlCY;2hk_7k5kFcvWbp) zf)Sl+7~gOk-$+GZ>>@KTL!wI&zbeqzRDZU8%vQ!*%J(z=R!((R*(=n zh?VWl;`jIeihc>N*@CS~bSQH%K zyCW`oVg<{umGTlr_+qt+g zc?rKoA(Hwj8baxtM4P}3>fE{`=G+Q>F_$}{k9~;^!mRjt7<76mTk2>D-#kW>{8Kq0 z2g}g--8@M9F$*sgErCT2n$%-I2%OyNuwXRdCHrune&sG%-+B& z`lhmPuXsiU<)ZIX`eQ>%IdgPRq-0!=f4xKT(!94-soAyY-w_E%z6^rBUFIzQi#kqe z8c!KruX*QeBlw*(1m1K`!sBaJ!mFO&D7nrMIzu=?TRR973|^o@vJx!)r^P}(4S-?C z+*x1p5c>9F1G=w!iLNUVXEg7?nzsk(n_nu6uG0q>+wa(TLCEEQ83oE26R4}qmexo2 z$7gNxFl7;gqU(S8b%u@Xdub4xXDLM$d1J_|Um~yZ^(`A?6%3w#9#HM$qujiwr^R%~hF{=@GXesS7rJ7MzXf1dxH z-=iU6^}q9b1oV;le+BSq94%+}pFPw}$Ws%@PZIxm{$G1A|KIl@K!hg!|K5Wt3SJzO zVtl{)FZ1+{r!8kSVFXDm$U?6EIVXEJgN{Er zkF)&+|EPT?y|Z^_bG6J#KK>ZGoVUl7qqgGwOC}&>5b(y9vn(tnA3P)z@mQZR>^;om z3jZ*W9X=1vk3Y&@ww!=k^Kt1vl$UKe~ zNW&x9_84#8JNX}5l#mY6;ZXv6V+zbUwI4N()-s>r1iA;+fQnWOP={AnOAw=(Y48{ArH(dD}N3<30i_ zB$FX~ssb79uH~&~^kC!gC(LJ05C6gWJ$_%nGga+S=sLKGeV$X!E_Nsq=OzLulckGJ zCQN5mzUaM6HSLaY#EUy^A*sxsG`DR9r3o;OP-%w z&zR$08mIXR_bz%+Ww`znY;(zE%DTF=W`jocuZ%KWsE`7>;(@j~i9uv_@e9S49w&{n zMKHGh9B3^l1IO)?$X!OI8t<o zD8Bu^gr?aGwF{0E2b}bRf%TX8(6tv?S=s>VS?fisoX@lMEeApIr34(a98ACNFJPB$ zFQ#n4Q=0Hhj2o_I;@Dq4?1GyDE!n6?_qQlfujNiC+q(ic_#~mYbtFZ99fZa227`Lw zJXYOePA@x=Q+}ucZ}fZkWY_{RU29d(O}4if5)34Yg-V%RJd0h zG&Q58DvOv~iUXJ$P9nowootu!PIh^47k-lRhP0lsWLxQoHfKATpUGfW67r1;$kbrX zDuvvOpRd_^$p8$k>b>A#XiJ?7(;RpkEtFI! zXK}4q)=PtC`|ZPy70dC{5f@ZZU5w*~IN;Vlg1@=Mnby4tVYbH;a9LA%MQzX=9KWZW zo_#50Ix?Cx*7PAeBsG<{w>pRBn-oxDr8!k*|6wIN8`y0XI~W-~7qm!J8{1BB~)E`hmA~IPQPF4L*mbYjH5?B-}ykfarh%infg*U(5cOT(Dzs!bIQJAGyUTvv>98$bT`eQ zw%4BQ#0D{_^$mljd^w7p??{1W@s#MY6wcqs!lHxDoYm_|R2m~e>juc9g5@pd=3@=g z1{}sc`-O7;^J!Q@5(_$hfWP@TQt$wjQu;s%+PrB4NP6|5!N*}>trr2eA{abR>PLoO z&Y`L(l3zQbfD&)rMy2>FTpSrjp|K0#z_4z-w_&^3?$dc#=CN4RyZ)fqz2^XuNj8Rw z2<@DO2-67@k{EZEzjgOLf9a5LCOunBXM2b-<2VT2QpS%izknkbtI?doF)+<|D;7^5 zOTNi6%%Eo`OCNKJsbt^auP-dbhBL$2txYx;l~=L3Syf~|Z5*lwr{TXK zX*SoXm;nWmR@ffe za%>ezC^^&I?Y5kK@ewwyzZ4rfFCF8D4+636V47rZ!K@*l72GJ~Z}^qMw6X*~#OXHo zYp1|yxc-A*X#RzYW$&{5QM=e`lPXM2$wzfTtB9T6%i#7e-m$0;gLf;6HHO#2dQU|v zvY$<1!4Gg(%NW?3vYGl?H&fQqTz+MC53B2_z+hDcoSQrYN@sH9cT<+Pb=HQ?m!DbZ zfjkO5o=7X-TjAKPUg#4cA^sRANgWPJ?B=Mo;E~_Sh4n?k;3wW}cdZsQY&VCj2{u%I z`XhUEE0x|zY^Lh2ckHjYf!^94pl@FLA>Fc+wv_18mbdxvM^g@3J-+d~isYI9$2*>Mp4p$uVzr*pXCYe|TKI;UZ7SjysLbKE zIT_;6tVc}a)lTNpZbW}>Td~QXf@#rMTWmC|VG|{tfs3rh?(P%lXm_1kb~qXyeK;f5 z0d;s(bVzKuuNPbH0adX;`X!@7ZG+N8@vC02eMZ~4>ikgt{zNs1>zqbjHKR$=NrMu{ zRSG@ZJvjcj1q9D6VDU~D_?Tktszljbik+hb#=BS0#zaGyGA12Vcb0+HqTlRvNgQmP zx`loC+RGJZpJk&}j>9UiwUjq=Fnrs-l8t^XM>~v3%YEtB$PdLm) z6f6S2KZW93_*&dN3gKePaPAE)!>Z5MSzP-tJf$52hfkHl(!5Qqy4eBN_zBMM;9Ge4 z!CiXk6AmqVE70zDAwPG^AzJy)6GNrU>E9_aim&bFw?`e|7Q9Y?B}pEj_do>lO)D^C z!%w!wQJW3h(hmw(h6p`uP4bnhZFCmic8N#uk3|W51rpIWMc!#$a zdXMb~&anE9AojyyBG{;_Qnrjd7Ulh`QuIB*B{cpOdj98GL-GxN$=erv#BO7Hu|l8L zPW7NQ5|?k|QQ*$pr+Z-8x7 z-F=9691rUw3%T)S^5i>AOft`6$kAvwjq(Vj0|t+%zy1~av*8NmxyDgcgcEI=vy^?h z`Hc-*wSYF7=|O?u3s9BwV5^d1INnO8mey9}54gMSp9Dp|7C^RX zp459injAFT>Bqcp?CM=(-sqwcPSr8QMU|59WSl38wRK3ga5!ug+v58E(p1@SfEif~ zg^NdS;UZ}ZnBEg@ivypl%DhEYbvXDcMgZ%Wj|TlVqybR zEued>JosOECf*7!MQ4Sx+1(>}X{)d>qWejEpn_Kl!{MT#i%q=G5v zE7JG<-Rw^L8n#t5iyZpDXWz0q*bUU6E{#HVGt7N_d*@)WDoGu8wZi<+&Of_pBLRr4JG@h!uR;wn0@;G6_aiN zInSHQ_T0{-iFUpizJ4DW+D)arGG+3J>qZNM*|g~WI5D5mp8|DK$nx7T^ctIThm!6Pg?-lm!hCEs}?rRjfJ5j^1w%D9*jEph$-$U zV|D=}$oR@v{PCil1}!JDwm3mUbQ37n&I#A_OQw+CE^L0fgB4_L$5r85V6->@R*bfR znIRM5?O++m_g#kvmSwQxFC@?)&xYTiU`#g(zgYj97DEF*Y~k)~)1#+@n)rjQf^Rd) z1W!60WCz3az0Pq^e}A#^LBWION95l`uQ8KVM`Dd``l;8-d@5RzS`Jm9Rc?~ zUBb=&;rwK;q4Xro0KPp_1|Q3bL{1WL^HUKHi1{p# z>Gi`j8uz{)jVwL!-g+&5@}Wz7-M=#YQq>QvrlzqXuOSdK{v+;~AyIwh(n7k_YliYm zmgBVBC79g!0~1=6t-rMz!oxcT@b}tWI2v@D_v1Ur`gaj+dY?>|$pUUgA&GmEQi%rZ z=hEFORSKDKh5!7!o;BT&2m29^QSaf9>iAP9=$}*{ZoGGp+36_2u?BZac%4RlK7I6V zs~bFT&w`?fg2yR|r+@2`NGjkn{#>9;0Zo%3Rbno=IqxF>;$tMGnNEA=9_F%Fwy^LY z4=~~WG8(42mi8;>qNl)YS|lwA|K<(j+B&Rh)C7YBSW@l(kTB=v@-LVGpH?tz^DN^iCfi3L+E{9wGI@2uf1Z_Onj|NAz@-@2N zVq?#-up+$zH3EVu;l=~rG-4h&sLX=Z+k`%&cOyFOvZc1@Lbjkejn!2Mbz;Uj+!xx3 zSEFwt?>>N8{NZgy!d&X(kQrb1*k#fcTV#c*H$dcbpup^#{N8-sq_!Oq;x+*GGv z@|NP+dMPzJ(iDf!u5oP1R%b}}Sq7@2aDLWFD;CFfp-I?5tXrE2lMOssb?Y(ccH53| z0}MsI%U>aPcoJT$li_Aw&0w4CEUBMNDLbrv9G3S!6BslJaDTWV8cO!i>c-31s&@(- z`h_r+AmeK9iF;ea0!S#(Xb@>{v##^e5u7R|BiTXhHPEPS z$Ts}aWFDUdy?WFoblf$Www@or=M4&@qmTB|-qF>(%)1GoUZ#ecmv2*humaTImxuV6 z8PJ{R0X6YquxYXfcUkx*>;?^^;+`f_Q3)sCUt{5M=}?ll_9FuwppJ>RanCeQFz#K9 z#a(6KpPdEQR_+B0>2>_vowLEEB|-ekAf1&YN79rC#N)2_us0)z5_O{}Fz_pP&37&H z-Tslcn61q0h6S&49cne6+`dcG`e8t5+^NGIFRg0$o#PWn$IOuQ+{Izv2} ziuE}-G3*!bSu6!##9b^r6JXB{;mmw8olZ(EB*(OlT)H_Y?tb*{Rjk1Kg-0p^G0@Sn~bRy#^x z$YTt}cagI2O6{7{inZ@z`m_Q*=|jJ$)( zvQp^QjFU9%{TZrv%VvL~&-3et*RmzjclejW+0zu5$sBwYndp!h7JQ4xq&6>bY7gSG zm)-_XGiNp@eI<-gQ-rsdj9Jy8?X2$R1%689dVcrx6c+IN1S(&!rTMPNHMP$oL+dKm zxOEcK$+|$rcb?#!*Mm88ivn)h#vzpAHx{t>0%yAF0QfDxf?&N2w)CHm$GH^dKl1=n zll5fva?|jJ{3&)M!-75SQes(t`>MWads9^0WtK2g7yGr3p-E2Ku*yG&^(+}pv(C<@ zm$nsf^R))Ka&esC8E3P`s#9>{Pxi#z2d`(J!8>w;=+{nrET7GRjM-uwrWr@8B-J5q z;1~XOYzwbBsS!sFb%4Z#z2c9>b7)4C6tLm)tT8v1*FBxc?%xh&yK~QTp7X}TA+gQ;9vLkU;?nDc}@99f%$JAP`>_e>4&JU@htK4oy_pNP97r-|`ed2}Nn6q59H zVPkV!h;_BCL#OBTZL0yt^U zU<)TtVxOn-Xvu}a$v|(2G=I+;_Z8#KpSRsi7A`7xETO=TUp)Tb7n|l9ZQk zr&7O0d@_D0Z>3v=f%mo8#$#^5pi>qcrBMq4c{ouE!~6%aV$j37oCw;TK8B2!?1l-84Oqb1!{XGbvY;O9T{UFNAuc*s zj`*}L^s1f!vc)B^aEQ?39q|e0xUM0Wg)#W^$!0cDWiEVT*<7260U30HvcaUE!5-}1|CMuYa+HG!HG9{@E&s%E7|8qKx^i7qkE4d%E*sH z?V)4X-&6)R74Ny7;nf_!V;{H1uaTg)3R^!hrWFx^JtyNxXO@oj_A_fJNU~+>QvZySjcD^fy)(Lmb7m+ zC>Olr4$H}~4X3i$T<`Z-7-a*&5oNTicPIT582+M+NGx^NB-yW1(D3O9nu{MqCVPc% zU}g@Cp0JZX{$9!d{v3f9<|dM-#v3+mvL%yn&ER*a44|+KfgfVIi#=O69EJ|R%0#`7 zafiYYI=Q5TJHPrkJF;Fw&^6V-$VHnrBs9|chZA8?mJM|5cLv4GU1VgTKqai0jn5Tw z5h_zn^K>+7C<`akQl z_v8LY`}#lVUl#w(`fG+N(Wf)Eq|8U6x{@W-{96Z4R^1TKI=zN|E;$ZTJ#zT#S2R9a zH<4^T&$45i^C3fj6e(^S0l`Q(8llfx=W98 z%IGO?$Ks?A9`+js*Yg)cP2VvH+Oq|s2IoL{N(M#845aWYmzdY}oA7(P8NRdmN|_&z z(e}x=NdBQc^_^B__p@Z7!q1MjEmmOG>yJ~kL4e3$&jZV6L7WRe0OAI+qgoe3Bo>xa$-hu~Poa@Ll#4LzL)z^^0& z+GXHQV&Q#VZ8=0btEFhKj4fQtV>CeLAFTFGC5h%2TzdOh^6_iI8)k9vLGZN5x}<`? zza~rxNr1(@@)R*qhKz0=BBQQ@6xGVe)#WffSsB5%6#rzV!)AeZUOC@p(~4hO1xCx3 zKNvY>2zm;?nZfiBu>Z**cTycc@|-z64;zj_%JVQh><7&G(2TcCPry~`57rwE$$O0y zt$A?|b;~1JT4pquO}eiX#h$yy1zR3(YG zazj}ExHBwx%6#sE+A@eNa3YO&{`h8AKkC;u1PYsu@zO&J;PnY(kWA|odMOJayzIz)%4gP^N zF!vB)F=SMYBtYwFqrRs&2**x;cwLdy4tGBQH>=18eNJp)_&m8Zyu*w z_?Y!yew6t-Q}Z9jKEBDJ}esQ$aWo>MQ6tyqL(d`=y>27ruNboZ5PNwRO3D{d80~MnqzD~ z&VGo7_E#~#aTfQ?d=kvrCnCeoFSw`axpe5gB@T;|fas14Dqfrk)6{cC&got3{pA!q z&|Sp9@&L>lcar%!{=g26r@WJn;9GTU75zQ#MGFV{(9NU}$l0HbHDJr8Xm|=ez6LVb zszVk5w@{))5Bonq149&}KrddHCx9gCn6D4=lY~3R6)Adsq>}qFPq<@t%i}e0<&@&@ zF-g5&>}>W+wAa{91#Z)6)YdJydn_+{(Q_CIL~^8^Fdv^@+fwy9`wi>Be0bWvmgGh} zLMP@4NzF&-b@q73*!r6dF&DE&H4W_B2>g`q)|9t*Ej(_l;9gIeM5e!oQtO@mpZGk ze)wvEgL^>yNJ!(T0`fW9@4SiU>NUUN1EO{pu71N?DhKvW6}-T z)u934Zyn4u%j%e~&Tg`lI>|jLZ=}@g1K`jRN4Ol2z*4Tr2s(!mdpJ4~Z^u>PoYBv@ zp|SF`JQ?uSkYL&+c@U2}YGSF7J-sBb|LRmmF|M|i6sP3Eix^c{P&cUh){adA59>7y zIuHUqKc(TejGs2QJ=6_ z`yrKNMnLrWKm5Rz>H^!xh89l>p^nqGV7g_MC}o?768sYAqBkS$K?mr3UMF;%)J3uK zS|~D|V6*CFI27Gf1#|m!aBv(!``UOMWHk!^^lOLV2b7`v*Hmce{leUiCE$$R)%=?z z8@drQlX-{xP~oZb%sqW7#Ry)$pdDiIO_gyx?3*C4_#ScZhwEVi8o(w!BX)Q9KEjup z)iN>1F}Kv4ADz1whVJIO9|JI8qV9`<5{Wrx2)()&`>+ z`QRRH2Af~jL$t96{xi;|tJ@FLjNpD~_SA+GOCDvL{q$g9{&TG8$)h^cixfR4ndS&H zQKtGyREICjrn2y3w8l~eW_C9TOz-Q=cytRjO!xvD-X3K?#x8*PhR>jxG*kG!2USIL zE9kV(IQDdsIyo2bU1Sm zy7VB*lji0QpuZopxrB8*YvCCkafhH(*YuMg>p3Cjs---{i z&w*t03bdH^6MuQAlGNojaL#E0eF@3If$K(7S$PHaDr!-fK?wKDb{0ImIFBiW^}+3Z zku3VGI(YhA;g5N2B=6~|SZ9#}uYx4$;qWGYOvoj+_OS$W-&{`oqa$=_%YK+0-N8Lc zt^p&%0Pu9JK=(#Q_8zn7vgTHp_O}WQj_iVwdI#7l6s<4?c};;^6031 z95l4m^M7SR5$+}M6;BPIYP2ncw|amj`hol$Th5_w1IrclXGt>++2*9pw3QpoP6qm7 z&w?fhTs|G{?RW*w^5L+y-H&8u3<8()3wU>rUl^p2h;x5)FvL-Xn}4?s9JE!z)liA+ zwo;_jC+p~4lnJw)UC;Wldo2BC3WT1IWh)yW;?luKdAmYU)%?NI7?608Uj2DN2LB3Z z*_pxc#6h)Mx-}cRU?Y7_4kv{x3}=SN(X3ZP!EC>XayQq*l|3ubGIA;xYUd3D1TKPU z;5q#IU?Prm$b>VOYw<#H8|vKqz&g3x)LEfGX4e0zjC`V)yU$bb?3A>fw(tx8{6z(B z*=Uj$^CpXcnL-PqU6Mfx)f~IPBu_|eM#Q~$l(8X_+t zFVug+^Iz6~mj7-2=W+k+|I|O_b&2!;kM-B&|EB)AH*DzttnagY_|X63`d0tV`rHP^ z|5<y zoy)(V+4DG*oIV!sjSgjo5h*xz24Po>HA|iugg4i{z*t)g$cwkeTN-_=a`XghPuxng zea~Ubl6t1vdVxQ;RtH{L8H1}_fOu)F4D~g^m=`#Rob~$A`14UX zd_gbX06BWS^BUWFQec*zT!pUJRJpJ)YbyA&j9Yb04$ohy#KPw}OjpRlb*)x{E#KVO zyM_Mfb|r#M$mn9N$1dQKF~&>?q%-%s&ip#-Xm(d)vcUZO!=1Bf`TnRqMi!rxf8%}K^wHu%6my9i%f^e=v&+AXP@;zC`a1O3$>%rl z$HxtPaZWX7uxb(}g*-$ZuMY8gb1hDbTaKO|U3uL8m$%-j$z9DqE8Lw`aOVeY(v=@Z zu5YfeeK#NC!;j~fI4TjhaJ8b-GKo~RS(~bZd)d_v6VeQ-#)x?jSdq>yzUGoF-OF^R z^nPzdUImSqIbth)J*!4JD}P||)Ypwxya>Ry;aQ`cEj(bvbgX8A*Q>SZnd{%K2I zwZHjM+x_U`&5t;FO+Gqgo}e?23(zxUJsup9i+rLo`QLHm6Bj<>`)41+;)Q*@DJsJG%B<%jTJuE3;CGsAsw1?##Ds70d&<%9ZDO#W=raeQ0V{H_T9?y_fh z8%EI4LrpBUONWJ`7TMK~gpIwHPDpLCNx;+IPn ztm>^PcqV`50yZAOH<4;29wp4vlX{FtTqF5LZUgh*D5m^pCN!yjJp~Vo!e;5wU@`L& z+x+AaPH;1)=FOpGeK`)BJYryykORN`!-xvPcL|(nBW5sjBt8+|)w;lsn7gE$T$|l# z+1OO>?D;H~*#3iQomoaZR>hLmlLlND+JxT(Zd97;1j=eW#2lUqvvMqhY3;NOjP<*N z^DlOBi#&%5oS6u^ViiL5r>*GIg{MNUL;@roRj8lZ5pvSMPFD?V2qTxUQ%SQ~TRMYB zoyttx$CVjnR$+yg8m;*~5wb2HVd1;Rv!u7{Ns`l}0fU8EOdUtLq(=k6N?>msY|a#B zQQXF|Tm3{qGbAu0E|fcVQQ*FrJJ8y~NrJc9iSNHE9{I(cm-TU>6#-J3Fx z?Myq4UiDU_*L4-+Y$CbqHnq4XQY<<&&V`b*BPeGt zj!l|>f!}*ti7+9Z7IGQXWO)`nZm5I&T{k!~M2&X+3M2mu_ZgGZ;0C#e!TYC=MAu@| zG2xm84w4^4b1ulTAsLz6*IE^Nw|Odk3YVpVmbtLcD;b}*xzjIcDY#Ts!?y4G#)U4~ z$Lb#o_rBJHoR0b-8vFGc{uOtkA6>%g7Js~GtPBeO+PI>EgQAHmqOg8&1jyi!yH-hJ_+6_*bz+Y2|72c@s-`j z(fv{cpSr|}tV4p?88DfQ}Pd6F0dGm{k+MZc-&>ue@|oUAs0H}_<-$seh`D=-l1mt z0B&wg4<1k61V1FEabc<}*#7Jw@re6oywtC0bZxCPm$cu8iiRFx-E{|XPt_}~RPzDT zN`A<$FKWh*r(I~3pu3q>`%%{16g}K!@u0o)weOP z#g{(x_Ti=_YET-WNiH=BbVz3ctg0OYyK=5Ey>ElbHOz+&hwZ}A0mWQ-t})CNn7Cdi zUW?5)NYUY`#E)rhr}{s~@Zj>hqTa$%7O?u2&FbB=S>)w+%>P;`o{6`htKANyIU$^n zuN(med!Di!k3Arz!9i3(D0_BT2UaMj@v)BEZTf*fbgdYH#Rh|Etc5kc|1pRjODqw5 zEiT-`4q2{2;H0QuOyGwtujR&cdPDs0S~j4$hyVB^oy+-{$}rai#%0urY_>>sr{=5=0$i+J13h6+5Lo z!}VPLvr#Vpw9f=W1O`DV^n>DGD%`)Si8$?75jzS-U^6?LOTFa4+#KrADR(NHB4~0^ z))nk}GT7wLzrnl|q`5m6>KNx32|ssuvoYqiESOJYt9&kS6^G@S$jzCX`)egzpBj&G z|6G_tMKq&KHLw)^&ob6LXWd!@!1wG6_SVo-oO`YnB}+%bthB=9bajqfmi>?QW& z+8F3E*uz;W2f)GwJ*>n~8h-33Vm=~eNZ755JNqkwQRO4yO#L8QJ!cLIWE0j(rtHRQ z6*^Kg0~dbX2w%^%aGN{tF@H zyW%pt=0rGn`ziKXv6ZX zMFEgDwH9sf7xCtgt1x$59Q0)dfcwpKytr&QzKAImU9*pX_~-hpuR{|G^OD(>she5l z%ZFUJ>q%_CP|qhm{emXr%xLs&U(B`{g-m}X?`ZcCOBXA`lAM7sd*Na@`g0yc&Fo;7 zwFfwjCVA4G+JcKF%mi=2H+&=L3XXAW!4J(puxgW|F#qs2yU?eI2}zGR$)lQ-z3LrC z&fUSx-5)XcmIwf0tdV{D%%5pO}elIaha6 zn8y(Okx#s2%(?fR<{VQ4alVrU6+e~W`~voIi4zvkiL^>wQQC=H^HWJmwi`7j&xdl= zd$@hvVE7UCQ{*J@xneHrLTuR)6sCn>vDGDZZ(;!3Uh*llhL7v{hqZ_0V_ql>g^Ax&PfuzA^e7tLHrB*3_$RwZTH9Ih=R23NF6OaF_ z*5LK_jc9&e;3sJ-)0jdXDm>~#Uuu#`Pq-)FJ?BXq^hQz3zLC_`T1K-4y;(C&o*u|# zpi0X#WPOwQhQ4vsa7vC2x0q3mst+k1J}Hh*u)}>?(&D|ON|)XZX9Hghq^k`XyVuVc=yH>Hr-B`3%%a~ zyfO@+L17^;7rO`DZTi#aeiP``Rd1$pFB{8ByYRkgG%DLpq;WY8WK}biOs^Z#mxi0X zwtW->(;ry0;u)R=XBsfnfS!UBHmvT#YTYQz^Oh0CtcswXrKaQ)v6{wQ(5F>Lj#9yt zel#QTglN2=Yq%(N^BW37Fr{lh^Jwc&`nzn{p9)!M7G`MOY90ai44v@OR3&!5cO-dr z52ey(4YIeLb1|HaL-jEiaL?a2Op)Efr0EB_*=#n_@}S&l^=e4QOFWG;J3$5gvBYbl1TJW8G)b%HK<=RIeWfzgUeudtRaPt$Fx+ zMH4y|r_#1x*U%)mNUY=@Me>)*xli?(Le?*Womtn)vPy&5tIj=mG3_uuKWc!_m&P#T zOiy7}*lh5^~C5*M(EaaZ|eTJ{&!s4aR;k`!RTm4<<|)NUIH}QI=91sTjYuRQtR)z+MA;#kU}1GK4U5bd6HgxqyP>0kRrTx)ZLUz>P9(2tDp_oyFy z!`4ys`twnWUR#4vd%USwma0X6dA|(urKo=1^d?BXte%C4VDNEO>SZ+Yjr3<9kU69oWgK zKYotd3-(jXjZ4(l=TDmi7S_BeWjxIt4>OxSvHkdg8^N@3%=ly~^SFx7QeDvT#V~4$ z7-J*m8y$NFir0&zb8xJY-&%&fpIF>rx=bQuJ!m+5f z97aXn>ZH1h(NmB6Ts=rp;aL@YS7HS2yf#UGv%tY0GEg!j6LnIGu<50+pMWmZR_TxH zjkchZ*)N>6uLV6GJF@vrRVZmK=peCSbnHw%-YDA7!pgS&ho4QFkA~CE7Z$W`%vk1P z{}{6egkZ_K2sT<_0M%a^3CZ4R?7Y1!P2BOBbsd+da|eEN8@F#`l|w{m`>Kg+iqT?= z9iz!|fETVhnTU$Z7BI_-(`b1n6wit-;#>7W?Ag;%w5GI=e?7$zUiI$>4%g1%KocGx zHx}b+n+iOvbBPtHKR~a|R?s?o9B-?+gQj9T4|srCmj}bj{e^t^ug6%kWJ&20`cQ*T}Ps}sn6 ze2jw6b)j&e^f~LlCV_YQH5jXJuEBYY&3yFlY9?;kjCVI2B$>9ov}JxBZBAWJKQ}HX z^CN;kcts6eFg!xz9;uN5JfvjrCY&SwiD#DWrg82o=zaev8o>pV;-x5DU$%m;1nF+|k%w{mT2M8_p@-SJ$817{b zfe**cVBuveut+%tfnPJ3s1iYEay$%sR?oY>7j)NDKWrCxKIScoP`J2^xrMI)`G#%G z_GB%N9%xHFi*vX^lh^ZxqrRf-ye+h5swMq@q`i4KmEZUOPlim%Tp1IYDrG2g_TI-- z2&FWUl4dHQSrcU_V=|TqmC!6D&R$D27io@YP#QGPNz?CszdnC|zQ619db_UkpX=J^ zK8O3>d#&|+JYFw7jBf`;;&;nr(6L(rgI^S2hiVoc8J#EaRW@V4*(XqMRV?2YIT&>d z3t;V|b2u~O3JfnahHEup_~xlJ>(*HS!(~r8X`LA@WJ+XkQf&YPH;$&gd#|J4wl%}b z3WnpB?;+7m8kj3C1g%%50@L*s93a|Qqw}7R4(|`H(+LLbvO|r5rBrTGKZw7og&nJG z>5Dc6oZIIUv@gp7`OJshx;7P>{EYBXp%q>4-vHkhTHyW01?<%O6?9}mx>M_dC3x_; zC)}R#l&ZSd!qBsX9vYVmH`{v&%-=z*plAYy|LKM2y_0BHnl6S!euh2g9Wmix0qq^) z#eEF=f^nlFmQFN)#iDCYJE<~Xa%Lp#Ug1vv#Ox-=r~6PIXePJ{Q{i<{B2W6e2?FH5 zL6sy261x)VP~VwQ?VH98otMC;AIj(;RYo#WWbs$aA_!P=mG%pqOnX;`h&tueK)2#M zm0E9&&&uZUj$?M%_u(?!xUn4)M*SoajXoGTtCWw-{sd8;Hz3`B(dU-0L2kP;T~i)_ zGRx=E#H@vQ{ZSKC%p8MGoeg}#e^^D?`EU)z61~nHJOf5y#y{#647aT0xoPUK$$^D z!T(SjjGX)wrWR!2OjRMDd(IU#COQjo&V;${}T>~xAI64V;e-<(z zmhL$8l{AiPk-^*vi(tlMZ;a`ehnfOcb76rG9u9M(-(n}@8P)!{!eR=h^mf4Wj_dhw zTPBJ?Rn)GN#%l)=c3sIr6%^wD9e}V1Gpy7Kz%xCBKeN^aUl~0lMMI&;y8_t!IL`Zd z!X6bfoRrZ4Gu9_TgA4L|ZrN0BiW<8yI)pFwT8uZHlyGg}B)B@%5O3R^1EroFu*Fjb z-PFB!fH3nbl}AKUGN~YEuTA^!Itoc+40zIZTU_to4s5EzPxM$e`zPw8M^>d zYlh=eK}+{>kQcshHxp_$`gBp19?G85z=!jyVf?Z6WYJ!;p#h=-uQHdhkMTk*9e!9GP1OQ~Ic5+jI*&;_QH(Ud`ljQ8OHVdyY30E+rSr zyFqQG1Xb5WqxWnXT=#StoXq#8T{a@N;7lNWwsszjnK(ws?0MmEVc(EFO9rpb5y5%G zXEowsW!yCQGx0CH0r@xDd3)hzy!s>-H$7Isqo z5!@Oei+v{#!qL-(td^o9S4bPf)T0Jr;ro7c(hO@Mo4FbG81=!^#bdFu&L4`NFCk|i z2jSPt6{tR516n;M;jG~*c<0w-C%p@0{KeK*sy{-=lxOTl!jDsk zo#w64&U2?jvhh{$)@|hhr~mMfLne6sP(I)G^A)U}?FiO8-h%rS75Zq4I&*ua&0a-p zhLWj9n7P9N{e~x?>w3XiUwRxq&JbKxQX#bOXDw=Hwu$Z-mxWUfPoQp(2jkJ#!v4W? zG%hb1KsRk1iX9db_!MA`iuY!t|0Eym{m~c`nu9Pa;22D@5vGeao8!c3Zn#x_ALT-3 zt>A(_dX1b0+s8JGA}_UrZu1?G7xr#v^6G>-%_-PYR6&P?8>07cU#buy1q-ugU?1T= zIDOs&(UCeIyn1UeZ844@Z5i`HDb-nUH6~EgXA5Avuvb;qkD?a>ZotXkiS*K~7$`by z0(Wh4(AVD`e#UHtHHNmBd}AcO3fWIz7U@9nv=S~hkcT_!KO`z4Zt#5C8Isa%j(z4W zfmio$N}k`*q9c2at^Ha*kc{m5L$!P)boFW-{5?09N~J%9&T3`+=@X8zIc``pWi)1` z=E0hTld$)u5nDgUi_i8;Bik&4Bs4KNX|+i1sC-p*oh{ox7N9@<6I_bsC5FONjcvnTM@T{o=U zycNGF^ukB6rM#Pu73Qb3@XU!-_;F%7e$L#EMHvA!L&*R5DJ*5?2i;)Lwt?W8-w5Sa z#ppp|!0%-iJ#FH|N^vkW7&r^_8_xp}RYnAz^ z@4Fe<(|NEW@gAMjyoUHxB;frqnt8_GFCXUUe3XQ4>n&t3DEuUYGFk?#%wb*!=>HaN;DP^ySQ-C}R->8D6^@A~l z9VhpyCQ*Z86E;FYhW-8~L;R)EL34%-X6l}yxfAC@f=wZ;kBMgkm#n6N!c6Osnc8gj zea2pW&&2P=j;x$rq6MlMG$Ox>Y|?6=lKp1viQtl;JsYUZ>mgt?UjYMD^JvepwN5du zL+Pr?jkGN_kQQpkqTI}KnCw{#8#9dQ9JTp$O>r|>w8fd#F0uoI_2w9P?H!Mn6O2w1 z`{Jox4`}qW&wS6SzqH?$R@miJ1;il&J*1*W|88Hye0Gjz+b&kpa-kM{zdZ&cW8|>) zL;-&IDMH09!OUDc9p5R8#?bubxG4899?PkO;E!eCb-)CDuPCvNNn5ylhbqdS-VXH@ zL3G?xN<-9Ca0hc>Svg8L{#`s*i{w0cU<|&rK-zNqFKn8c3T{=`>8Tz?B7Of!j7iYk@B2Vh?=giLaQA4{< zdm2!zgA@Ah1zWw{eC^&c-dH{heQ*rMUJk>JPt#yu_&Oo8SO7&Xm-*eb3GiU22yGR8 z@WXu(o*8b7NdnJDMqsoBpRS-kgSc!e6k_F52in81n+Sv zVEoh&FHIJs$__sq^`#cdzD~vGj(=d1>UVBsvkTsz`k?CSQ?Md<84f+0iD{KbFmqTYMs`gA zPK9&f@VhX$x{8N9tH6?vW^f|-0;Jtk;FrrDP+#5-S1!DDa`in1ZDB?5PU$$9tme=vBks~p% zxgV=rHUY|)Zztz8rgPN}f}6h!K9I3*$o>zlQ~;3@FPDS#*?bk;Aj;=B>LZ^jun-J~ zOXC}dTcEptJ8b>^35+%6@q61o76_@D(r`?uP=c}pg&h;zvW{pG%>}018O?U z;GVk+VA+^IeC|BLoX;JqEw@`o)28(%uM_q|Rjd&gnFx94+uo34K9#LB+(jpN%)>L2 z2==t>f!r6*;PFf^`e>I2T|Hioj?f5&HT$MxL*P&Nos%bIU!Bo%!x3`9S_LOX)I+6d zT*Ly`)ZprEw`c^I#Yh#`R(<`$Nh1#9F$d zR+=i^@nSlChqIOU^|^x0HX5wq#fH0mr1HJ$`Hr9k@Z{)h&ZJI~-27%z_e%KuZrKa< ze0M1MpvcyG)KULQU+AB%opi~zE3~$MJzZO+OAkn1@Q)@XJoH-zAJE$x$%|BpoU|D; z`XS_8_bH=sp9t)U?BZ*;RSKD!M?#*e7xvx%5q|g8;!@G_Y~&tuc3yTYyR~*a&P*&K zsXw$?VWGfxT#>*`7nxFgT?6T=_INbX2tuF76UD7t*)3-kHvXIy9(b_`OxJia-|Hc4 z65ql7XV$==3?;g=U4x0@m1(Z&1U4e(G}$}jAyNLK!CZ&vvd}=m4LU#?A|-$6yK$r0 ziOYSN|0n}`tWZRCnh`yx)zio?!>M;46Zo#AhyKl3R4TO}i+yIpUKJG5x7uT=@{eBZ zCOb~+#S>}xkh{FT)`vaMETB5kj5@xm;_>-1xRTqDp153E-ZBbYF59E~?GJob5wPIj z$#m1Kk!->H1K{U(XuY@(v5Vs=tz1i0JOkOA z`OfeyqZ~R1ogm$HIxOGw0i2C$hlsf?WL#eZwk+leT$Nu-dPJDn(X$nR^ zyr4!#3;$-sO1!7n(%SlVZeQbvS63|u(aky3ZWsrb_ItGb=x?g6R?dgPAF$W>!)ra8 zc|zcSvXuQ`+EFEReK-i!pS>UtdN!~$0o!y}Vh)!Nsk3|29&ot@{b+LW8kn9ujke7k z0TH8zV_Dzh&^y!(^@H<7=dDlh^HQ-OAF&(<-$_P2(Kra3>4h6CQ=L5P&cZdH2{q$1 zg<75EcK)E(NOq{bk&fiw=*rXQVR~RJwl01^_8pSKyvf_?I=KWKp}q|-n4ATB`EqbP zKZ+e6VnvP>h``{`X30TU8#eP@K6JOWz=JC@Vdq~f~{dNKGvb@M}OnoLjsNNOU==FVnyk)yAmLhj4A5zS#N;1+Sqc*>*#k-yEsR z`I`~7%V{<&i)0(Tw!5r1S-^UE^yq2g?Gj2gx! zvE5|nmI%=QstdJQOPpp%X9N2!2S@G}(d?sF;Q1w0j4d}o>2DEec6U9LzgYaiT0s*1fL~Zy#SQr_5+c;8_x5E-4k+QQYqP=akUmdy2;ye5c1EuyyDSk+(E(-Zd2(bI1XQM#Ny-ldTfP_bzxfJ`lP$ zFydP#g%8&eRGz&SLK>AZdxSIHGjbBH)^DNj?|I?AYul)UU7%>JwOp z>4K*=#XcLPG1hPhoqkFlG#!?bXg@jpa_bT?7~~4C{4#OhjTNXdy9i^qOu|=PTQTLJ z0p6=AM%|DL@JYoL&ty5`dxHm(w58clFVe!#2Zg+P)CSH^GF)^4(x^RCk8?Hhe9}7cMMKmvcr<-36j5Fr|ItbGW58> za>|~QjmD=kiGR{|EM_~ut{u>HkW#ZTanJ-+|U>(R+C1IRw1^ia% zN5TsX@JoM$av_Jae1JVpANU9YRjfeP<))w^jmIGyHc+=Xb5XK(K5)Yf z!If%`ra}66CVn3-xLk%O$Jk=a(7o8$nuhQC4uJ~gQ}CF3HV^S8cLW=f2CtALz`MA}6&~}?5=KKst!`clv zE^as~O>Pr>i(^n@l?VKkoJRl4!?10>FV3tj2H#cc;56zPkf{RGJ$niU&*tQLlo@TA zDj}ao#FGXKVHTrGn4>E9r+v>hKt-!LkuREvPc1Fr^6PqfW=axMHn1k$zut;IR%hWc zfoakq?OrQoH5s>;l%SqnBPdRlr&hBibdjPOyQF@W9-Xeo=8xRMjTbv&VuAxs>U~Ri zUteKE%~IcXa3l4DAN2Gvc2`4%b-vone)wz?cqQXm%O?k{daRGT=g-FQ zdBOOjPT=8N*s&XD95FywIM41)fU8T@*h2Rk^hUuY(kK&)ee`tck)eSw;p7|YvoMw1 zRW^XY)=;dwwz+oJ_ZjTS=@gJk*Tl2y4KQ7j0LO=VpN^!q9?jC1gg+VC*T=rdjbW{Pm07pgOS-x+bfG*8L$U zUNZ{|27VKnhB&g9PtLNw?`6PExKAqIxPiBn-Ep?+Q0y$&B)Pw^8V**x0?)tEY(z*o zo7^Vk^S>O0fhE3#=8eXw%C|9dYd4f8KIS`nP9fi%j7nB#>9;#cG$2re?cel(=2wqp zTg&J3{1xl*Nx)>xnqN(<9j?Ohw6~C5U_)m=-^|*SYl%Ee#?|F>S={&Gba_A|SRTBI zby82j>C;omwX$r`ZCDJkpJ%g^Ib0I2E{k(I%i-LoPU^l{0T+ap3H7a5Oj700k#iKr zXuH6or+Ls@ri)Y_83K*ca;)`I3cD4tl#Ni{iVH6;M#t6lbnFKu)?6+{nib922T^TJ zd}|@>G(3vrLWAUF-dK2~dI2}Zb;G%jJE^Gt1i##V80;-&@zbw&Bw>Ms*Oh#vx`S84 zwhmF(sH1qPD7O93C&u^F^~-5I_4*kK4aN{6n~qj8p|9BkbjgAI>V z1&XRI(@K3fwS$s(R=(p>u(e36uhdcOzZ(iw@c<(oN^yOzRTe@cOe-XqvLC!hWl`X#ZK z(&4sW4Adv&lfh%-+3BaZ*s{}#tlr`{eczq}hqgHLgTKy`uU~}zkdFv&H-upRfFSbf z^j`XDlr)nZ3?-{Z-R8$sS3pO;3zP9wWLe1rV10u#dvtL(eVHuB7XMg;8{R#_Rn{NS zJx~YV4T@#$!?PI|*k`gI$Fe28hOqbXM~L;63g#=+rlz^BN2QbH7?Y{V3U94t#pXBZ zYm2Qk(IY~#Qb8Biw_F7+%Xm0_YZ*P1VkX(Fd5@2|CQr>ekxrX*mM?!wXf;=5;SZbP zwS_Y}y*Ws?4pPI}k@~Rx!`d7Ex%f6=$11i(W``xxzk3^nyRve_QkAU_dz=M{UKZ?myBy1Lqj9nmvl8(gi>`(y*nNKK@bzRJS?bT(Kiso*`; zqmZ`Jg1wo2gFZeN$yB_YKsqdyZufdk#NmUOz42`t=yROPIk>UEUy~YXTEI>VQ+Z(|D*^{)|2vSAxS@PE(66fkFGL7uoU5kd&SK zN;_I_Q9m2vG_Y<8-q~nDckOs1u|75(q?and*KY%O%KTDzIq(Ek?VCjZE@`9}N(W)x z({ws1?*ZL0eGj#Z+B^D+MPJ(eeI?%$ei7DZjl)0o*XhID2>PyH1?~R1nomz&N27G- z!QRftaK~>u(Gm+T4o5%Md?69sif4mp`_-FlNA)#OY%k_mX;s1X9&vL&1vp(())cP;i|5yBt zl&A<$ca6eTt&34hn4{Q|oI#`AchPwUqkyGcqiJ12hInu)Zd&$;y?YzWWbS>zPpT!z z=XC+h7|ncp4Z@R_*Rg%)b67GyjvVu@COxbHADwT9r4xf#)c$N1o7hvs2)?i?1g z!Jb+5-$O?>c(7$36nR9?3h?c75-nAH@zYN!#CS`(SYQ}54hUec%YM+WUkf0fyk!nT zw)bK04l;VV3jW&c$?Dny*@XeJEU6?G9sRhZ`uuhn<~yD`y9x8xSC(PN!}Vs|vwoqYucBd4;Z-rs0~ zv@;*{C(v)@4&vpSB}2XBRf>I0$*cMgpzu-@}B(0KM!_I&jR z=BZP~wg!x%W?%L~aK=9TyiRboKm|@MN$1wXc1unOv+AP?R*95^dcVAH93`dBnA)br zZ00A@jcMmusggW9>QK&(DCc6r{cbika~Lg}B|HcBtYUwkc9R2Mv25vH>$>gc}1}0`EcRvmw;pK&Y{xBD9Y9mk}~=gG59Wv zOMEy>{=SOYTFAklakX4J`6_x2ilHw*p67pLJ7A+>3aa&*kLOGG!C$?@>}qWb-Ee3E zTJq_v39}?WPF@$u98V#l5^3Uhw_Vcv{!8{nk1)R-esq|3GXA=H8?H~z$DZU3u*h>7 z4&PVFvJ=agjcNoNe?p3VUZKXK{A}5Y^$*FKAO&pRvX9!SA7tkXV%W1r7ur9)nC*1G z1w&^T6Lr0HPvQI@qv&K;PMVnsDJRnSuzJBY;F62Ihd*UX4};kW z`95N0)h6n`E|K-h{l*5jrP259Q`w6n-{40@Uwk8Qp<_>Y(EA@ovk~8Y=!y0VSR}O^ zH|%Q1CtD9<*2w_2L;JblJ3qzTJ9;s3@eH()Erfx+2V&3gI98Jpz*Z_rczCl3rnT3h zqLYO7SFgY?^HMQ&c@7(7uZbU`_oDo!Su}Kc9PZsNWa|YdbJNjOI(69~7CYA+st4pT zL*K3J<>3p=p<%pN|>h=tekLmV4F36(y=&e}WHswC zc4jAzh1sx=AIJKqTj_y}v+Vwdeh?n=mhYL;yUxPx2yCEbDDPRv+SM-7H7=)FP+}u1 z9Vi0_PX@EcoA%>m!+J8swHIrdZ^(LAl`+xl@w9Wc4$pWKkCWOS^2vM@#;omwN48mF zSkhDQ&g>$it4`6GHP2YB+IcR?v1Y4Z%>pHt+cBdRh)t9ttT^$RpuoKU`d4gS@xC=X5N=W_fFs#0&BCfyj zizX;(;kA<%%=OkDX5yWQe~MS&%D+QVU3aYDZV)oEQ!k;Hza{lxO3e722Mnp1fUXC$ zP$?q?SNShs^7=F0^ohFI}-et%Ec5vl-^gO4@ zI!KFXkMAgEcXA0P$>or1x)Ds@_bXd^(goKyf8~`A4WVL@G9(&BLxj8^Zn#^F5A1DW zPC_F2O{fTXe&ozevu%MDJkrx*-lCJnhvVEOswu{>xRNQqd6I zHppPw6j@kwW)1rjDb!St3qADC-I)Kr7w^fx$L#&z5Cx@~OlnCv?{3+TW_CGvb44}8 zo(g6<^aks;Njly4g9B53smrR~Mhbh7ceH*?AjrxKjtJt0onNQWqu-P9Uf5&WEPoId zO}PrMwtd349gM;KZFp#69cfAX4MY2UqzbE)#B1~RuvI>e#J%J&DVUxJW*J5-J|v#) z3p2sp@t@IeTwnZb{+j*rG8XuaVsKbK1MN1Q!gaG9An;x~?yNq*N&?sN+ur41l$8lS zZ;P0QTob80UQD(>=#3Rxb8w(fJe3k^^@g~VE_Yjv>Mn*X3L5xGsY*I$f+ia0Z(=p! z#mxOfAwCYg4b47(p{ib(x9dozk3SpYl~L9B_Jc2e|Jn;TtslZpI_UGdZdHg|SxSv; z^>M|DM?7%VZCp57n63SA0JdvSKyx)`v=ZG0`cw&`(!bE;d2`q`!z%D=(qu*(8`wI} z986H3jsxD!$03%vFj%<>w7;E$k7vK}TVX5dz|*mi*gh8T33a=8dWT&%Q74CebFpiy zG*0d(W-swLyD|d60mq4`!f`{~6F;w+8o& zGl%zPZ&|;$nXG)?Hl}gflC8GWqSvN;=Pb2~UQkUy_vi!cas3@UH@Jv3c&=rh)?3(> zM;EBa;SsE8yAnI3oI+M?)?>?6(&1>#Ly5_a<7i%xiIacC^NK#l*#>UST-WzyQl`~- z-g6!}O3jCu1()gF&-L)#=?Llho&#m)hw&4c`n4YJ4oKStF17Sget)`{W#9M3T>@tq<_X^3SwS(;ayEXc%jrZ%G|Yr{RazhZqr;CMuy(STREf_3wnS zmf;?Fv*aZDzOdp}C09w77GuGMH$^_I4G3ZA?yCmw!yG=>cAfe9-VaF5$bmbYGpJ?IQT_m1>`{8Qn3 zE@%ID{0Uouz4!0@Q*8^M{B!D22(5qC@4dcZ(SNzVz~K15pZ@}$ z{ty2m?M##Hf4Tm@;&1q_$oS{_CG*|o{>%0M6@Meh=zp$Xp|+{!zg+)c@i$U-uM|7E z2Z*nnixa;d7AAHIJ0PCDs7^fPVvM*?Vw`wRQ-OF}_fGMS%-v$=u6S{U%@Fb5^NHfU zk7LDfWQEu@Ek>Msce?na_eSydtVQAvv9rXZQ;4|Qd#d=oRG>J%VWRlt#}M(kL94}w zpGArnm#q-%TwgC9yMT!=UtJ(>+2$c$+z=ynJe?@6do)kHf`o~$cqfYQ#l?zmUm#)} z6Dihoo+jSE##nsS!c}bddW*P96ew;xkSsnXtocdOOVnHLx)m%{JTI08;@3_=ia+uoogXB>L<)G zJiNmEwbj_o(hl_tGy&>v5f4yQJ`x547oT=DhxXxegpFsdz$$9AO$EcBe1 z4VRmb7u_A%jd|G^^ThAmWin##C?~viLbh)A>rRmJ^I<2J<7;Bcv@dUExvQ>IwG;aI$iRSyKGvz5va}w}4_b(+^%K_D zN|W7bxC%Kzdro)pOH_OUSp1)UVlxdzvD0f=XdN0`x8Az}>#xW%&kv)R<85^| z)zpdQ>$&5@@^UuE?=)8VDbfn1R**Yih+AE6!=&?}c+T&-B&qsvOmjOd>)IErjTnr4KSwSIE!52!Db8! zqdO1l!_eGOf(yl)C0(H6)9bJD3ndP4@t6r3w~XU&2hL$54j)C!b!VB2%v(HuSeU&K z%>_rjrMMyHI4T?pV^#Ir8I|&6H#$q0t>Sj}Qs@_SrUjsRubUEiPg}e`UK@YrZ@`PG zs@P&E$oksUaL~P5e1%m!k(u_4UQX1i`yity-m2=xKHQv--8*BQ;)Lh-=~0XDioAvR zxvoB&+>wnltw+$9f;c*Vu2Nm!{;M#!q=U&eZeg)q!rdwKi>-VS|=>7O*kDX&iB63=^d@U8$h@UQ`z_IDN3Sm64DZ)D-*d{zZHT zRKuj%eb|d{PxvUCe(a%N1`R4wse3+i8$MaSABM55l+;~hi(bo#qhe373E^q%{9_Lc zFzr`occBrL&LyJDv|_&MLn8ZqqmDIJ1v1S(Q8;u|9X?c)#ciTasQhsncPO{8)zU@m zOw2iM)bW@uJn@xQO?iyUAIe3ieX00;YBpYb!|}3QBfXLzPZc$WGVPTyV7mG_OdDB% z`f2Cr?`e5#L(M&QUtY0JFVd#Y{m~mXS>99ZA-;jLwmijWS_Z7W^AD{OYAI>YT*NnI zli7)iNjETQ;aZN-r8{nP8cH}A6<=&Xg?!3|$Kcc2} z`z|iTk1GW}+2zTs(f13QIjm!gUoFPcLc*$o)Ih3_Hy_`ai$t-#JRa!{5L%mYK?KhOR#m8GWLqp77zCKsjEy1W-ZaxbVsNeuBujJA|n;?MYjvYF+#D< z>2nG8cgzQwJHnoMMl$P}>xL}EN}RpA8pRLa2W!jAc_?l~_U>VND?H&Y=C>;W= zAOFL@*4$B|>MSq#Pleb2{`~rX`A_Zswf|IbVu0s2S2iR`9a1-`u{(#9VH<`~^&e(% zd7T=wUQzFqeLLQ%()1(fy%y>hZ%@!)i;uuNpL1}&A(k$1jVEm`1UpNNK=LwN;F{Vo z9mQN|i*BZsbx-J|9ZO)C@_Rl==_J*6Jql(qt+1&x66!a06Lq^58eX&(4q0}QguS=u zm4&rbz2hyl`W!^f$5(K}`>WXtmjTpE_8{50%#f8GDkPYx76 zR`;n({{Nr<|Ig1)=$-t#{rlwVv6A;Sl`!wqRHp{#wfwuSH;?#L#bwe)aKCg@ZZvk4 zk3mRt=Q#di7vf}tL64{w2^ke72 zJIlM0x`Hbr^{C6W$6Tkw+Qo}VzGEZVdNLUHznlpUpB9k9C2_Rt*gwyrTFtwR}4igKkMg zE6rYuCd9_T3m0SBB;S{R{1yZQo;cBI`~JenJTus(npUeXC)DgeZgrY!e~pwCUzhCt z+?Ow!kwl$0?FJLIq2%aeBN}#0OpEt=!+|+JiTuMc5CV6g^LZ>ElcI?Bf8+T8heSBn zl}fd?EhL4vGwJv+W$3^0F_-u&(V7=Kc;`(EbV{`YY5RRpuC7g8!JNJ?SR)yrFo1Pj04O;)8C7K;%LzYH)ESA~9wZKtxsa`$M5;|+lNuvKR?}in&+84L3MvO_ z>k|g9J7Vau#htv(ut{W5w1sY+tcjyW7{mT`G2BI?KjUQ!dD9O&>Y83gY9~AKPj*sl z@ao?DMA0pn;ibV=q=;$efeb#YrGZaBBSn|B_v3e93)vQb3j&Q(VQqvqtj|6~EIo}x zal_OxEn10+1{@}urqQs`3PH8;2AwtKYVF~aIR5%lqhw;OzIb@(di@1>1no% zE+27@*UVpAv-pY@#7%4yX_P$R*Sqv-^Cks|Dtt?aj@l_wxRD12ug*c^b!Vznst>Pj zN5Rf{MP#&Q3Owj{jQbg@uq$CxNk~H>KlShq8B=7)H8tI7R^3zHZQKn8P2VIJ$ri5L z+>b6fd<6Dd&VwAyjiQ3@Cuv>NA4nH+1!X7dz&u@z9!mU5WX;@QQtNRVZaP)`>N{`tSfCqd`t4zkuK zlN6hNtR0eHDw-$P3szWP;frqRA1|yapAX4Fu1S6@CIUG;pI?ABn zhbLWgZ!J^~Dd)b_k=*JyCc2qwOF?HPId1ctJDnX!SInM8Lx%UE4xjSqKG$u0Lflk1 zUw4_v;(fk%a35N3>0kTkqXM)PXTiQtdHii=1P%N&2`12HZnQlG9E9w~q10<+ysSXD z{qPjrUau4VI5~sL*xcn~?QQ9pyyxMv{8_d)1r0GC6jxbcenP`7YaZE9l@gcLQA z9euC!*1{4#TwEgtY=!W`h&@6Npbsw26)MwdAT9jsMt2&S1Q*TZlFd%EkU9i66r zUDR~xB-j7Bh8~?@N}@%9v_?}A>eJNV;o*MN;Y}X@)pLbj(orF1*>||@%_R_e^&^;W zmnO~mHpFE5cG2yMsMqo%S`o>Ccf|RW107;#MT7mvgGu>w`i=@5 z2Jsu-RM17sCq(lfLcLKd#+C2*zJk<0PJ##1PSEx9H^P43%}zECR`SXwYly!fg1{@% zG}?SWl*{W$&fE)!wzp%**7ke+?wf(MWzBWT`fs{I4_cT075^pow5QQ|;(?OGik&_OF^M(82oe`0^R$=*{b_(bdO#<+??T0*IDG%_KAAJoi?oz87In- z_v0xWpVS7L!r-y_-HmkCyk0cml>_#F?**#w+C+kv0B#2Mq_R4l z(9&ce3g($4{jv_-X{5v3va88A+ij9LCeO%+9p1F(VKP-XxPW+7nsR}Gz!oOVgr3OW zKx(rjmt_b~TprG!8Xti}ZkvIr{smLLY4m(VZ_zxP?R3eX@1li|=EI$|CD7ut6iWYS zaG8^;r0%JM$TZf3{Ymae--h?u; z!TiVwni`ctAI9yV@#EE4sa7@3n|N8GW|RaMhxNwPYi3-fj}7&bUq(p+ckd>H@2%o<_(9P5>Wb4u zv&i7oouc-^onZGgOrnv#LsBXaL|=J;L?fw0vS9UYI&$+SJ~t>+R95*Mtf5FGEpwgE zb=t*KBeMA2pv|y0d!l4;Q7ltb?12|26uH8+O|{8dL0n|vLY}Uf!LJMNC+6{E(krSD zcT;^r-j0@orfth5Y6FFSruIsR-QU~kyWlvr9M)BHUuGWH*fp1X_PQa`s=7nE?~LL$ zyKYO4>d2C_(_azapG26``_4;r?dTR8JDQmH)XD4gPRJG9^PrlPNquL9O=fqIYj8vgJR5slmibNKa@H z4M~|mPZf=)SLMuT)~UPP=0F^s=~O}krY|PN^%-=QjWM4(G?(s~94L|z_NtX8+0fkm z0Hl9ihpStLvauGv#Iz}vuC!|A7Mg27*?Sj#Hed_2nf;2t+iuSzm$>ngx)S(fP}Qrozl{FK@QgI^DWs{(ULYVR9j_6lo< z$xfhyM~LX!S!2k+!AdBb_>Rb%%;b+oX3|&Lk&=GH>q&F!SxOxH(<2?)^hu)}4!k{A zqJ~Q$Yfdw)&QzrRC-)&T4lUrCehui5QtmU%R`lUq1ij^yBWY>PAm((QBxyh;^eBar z;d(!4%kfOgp2Wi`-6wEgWgPEt3?;MnSJf_ktHO84JR}nPM=eo+I!7hwxZf<9{Ui}IjC!-^cwN+6t$}|wsPMrG?|F3o6ndk(++q3P z>9Brz7EdfGqKl`N&_w$=RQIPLY+c$yF9{5YWkXkz59S3Tz59PC8C50uE9s)swAWJI zXce+a{S_Y>H=lM#9iT7#M!=s%%c%CO-=c1_y&@Iw(bUCxH_i4KMQ#=U6KHoE!{}qOgGSNE^gFX_JOGFc(}-1HIfz^cX4fvD}2$9r$q12 zXVQA{HvG^!3J()UF@=fzAb2iEtjr9Kj~Ibzc>!c+JqWs)e~Yg^BKLGKNeb# zCsX?lH3%1SZ!wa|l*BI-*;E^|F=MX4{=WC%x0()!F4#~>+;-7xTelh$BYn0!u#jgS zccZG`6ZnC^L)53lh=SS>cyBcp?u@V!<|X{e;Dx2M(CP`D@KA+6ugs!4EJo6H-U%|A z)L64(4j*nkgDPlP(k<*a8S{8O?V30mZ+d96R$EPWciSXtpE;dJM=hrYyDaI-K69N^ zdOzbsY+_*GMnyEgAp^sIJ)#lq4O}rdiyRjXsr@r^9o%;oLb}ssX~5v%UmH`6b5-22`Z)>Ow~YJlKgTx~#!0l^W`k$E z4ej?_Pozm+kV&Vq`6H95@Nw;F(pM+Cwqs%{{qotH-@Vf$(N9U{{lkjrHPtjIVFtuV zLx=V>OaZGm_OR367|Nfxl5yXp;mutKy8g`$fo<6V#TSh6cS!)y=SeiZpC*K^yaM`r zGhoujJh}>$agipB14oyriwUwZAlW9Miohul9J|G`OfzzT-QBk z@3r3dc?cxm>O*~rQRRK-Sx{{-lRKD|cPF?lQ6dxjEaknr&ET~AAKQ8?TDTs&TnJ`$5{HT; zNutwWMsGaD>&=fzqhS$?tSBH-io;>wdCF~N9+8pHtjB*k?!q?8CBOxV4M?kO#nhi? zSY)IuS#WFxoE*2C9T?IVNPwpOy~4%J&#M^z_U;#w4c|gd*e5n*#u|22S{LWd*MPwR z=_K2BG8p>X!1ey2>{hbq6;T+C|NO)(r}KAW@8~TQ-I*Y~30urG8cd9_L%vx`>8Fg6jNaJxI?6`{v~p1Jz*N_@3GaJPOz@shhclh zOXhbo8&;j=Z0u-%_BUiN_tVcOhX$O5nZ^oe+L=bQ`93`VbRhhUv^Cxb(Aoo@_Q^(sU4qRZsgA1&PfB~D_sWu~C8 zOdrd(1>pyebhPx>;YKlvxV*a`X2gcV+in+J_~kk=X}ykfSwAXo@S5z!n`E!~CDuKE zF*;ayqE>wv>w7np{V=0g{8E$iGofgr{HQYSzB>*qyH3LFkD|@YqvUE*A!&CPIh5lp znGa-xMd&xCDmXxDztiwa%pH!>GRH{WDR6s!4z8;SCOg{Ype{v%uk`O=f%RA>|LZ7@ zC>_kc{^`RNjpgyttBa&?XbIib3DAD}4C&WfNT%CfMvHv|*o==J z?4%xp6JDo5sQXG(C;Wi5=W|G-+f}h69|M1U_1Wr`mEan52l^XEfCiO;dn3I?md0n; z=#~w&^Nz7=g_kjWbt#0*_Q3hBIam?;fw@KNkO{W;nV+RPkGdKQ4Z6=+Sn&uLw{il> zI&mC^+|`8>-?ib)4N$aPGG{cIaSavrU&=3EGF7&$ohkO(HAXkjpluoWhNd{z25bFw}8V z!O?CK+~aSK&VP!Lk1QrKInD2p}3!U8Xn zJtY<``jkWB85w?hm?f&!1Xr#ddz9_o)&M$f3REPMut(pbV5CbVP8=up@nx&9=cqcQ zO5H&rZ5MWb)8K}8YCwNdk6>_pn&6jy6TfEpvmNg%CBNdzD_N2Z-rw*;^&VBU7nur|=2i&xxi`rW(~H1kEue99BYV)&QQ3ZWCGp>uE7@NAOfZfg z&lbKtE<{Z+hFc!J64leENtgB+av@ISVC*II)waE4P{*;#Eyb5f=oBr;sY?~?;)C$o zugUCp)iBVRAx(YK|B=B)#^juIn{dJVH#@3!9^4yigzWGC@MDoRb=ZEGot&M>R-QUU zw6{i)DNmj;)1^a*L*qL%?D{U4llKxaAukD!u)v9)^|)!g9<2Mr*s#Mbb^|xbv!HXD z^rz7o&|N1FhN*}Bqbq!43$D@OE06s`PC5-#?6(<_^ zU};x3p$5^!ReBam)>>lum$#Vry-l#pP=NWS5isQO9C(qdDcnhNp&kEh;d6Qf-eQZu z({>#;PR_t+3rnnbRiLiL*J0VTIOhCF22U3B*xzm9toT|TvXY4~dC62}_s0c1D&+X9 zF@d;MLkTs*OQ6mo7$*dsBa_U|!IgajnU_uhE;{%bUn`YD?umZ5HT^vk`~U*(tj3N7 zju6j-@oBO$fBc~xEA8F!vFsfDw__EaHL!+r)1$!ltjJdSHXme57w}c)k!0cE9Q)P@ zKOtkeBQw9#%#4hmkYd#h;Fo zYwJEiYF@l#*SaFoc=H_T{&A2Px^1rP=wn5qGlqjkTrc#AnZl9|oFkq0-$3F}<1dHE2LTgVm+2~s>nctv*8*H^mf|&u+dhnLqxK=?tM<3uB zt+iyx@PVY|yCvJW?h9M+F#(RdaCT7`hE}qn_<6bun5()h^#!RN#>;a zmp42)Ei&!izGo?A;h6L|iVYG)m`^Wx;J<%Uk>rQLu4US=uHGH4rVPMy%myX}ZW2N> zeNZzn8wYRL0sWfKGOuT4P^K3zNpZ}ENr?yH;n+)9`FTCIw3(8<=T(JK2btu{au?`& z?kbt7GZihyAH&6BPi|Jt5m3mn!s?wb(dpI;dzCj)uxhD~Fl+NfK4|*@lI~(I=^kFi z?mf|G0YNS-cC&XyvdtOhVvs9I+0-IAx!^r&9o6Kn6Fo_v5NXs_n8b@KdL$43g|qzs zMv{^!1)+AHHSRn+9#%f?hn>c*gtqXr2 z`L{%OTUVve-Cfw%c?|my8V1_AP57hnGy5|B3F_|7!B1_X`|9-tHh)wIPB)3blEY6( z(MOp88Px;2u`*UH(|2ZMN+=g7*$=@sO{kWlQmyj0j3xkNIw@h>^z zrORf;mrLB=6_Em;TGk%fM&?AALG$vZB){=84l{UDIcf-C^UFlEC=#E4ti5zj4*RN?B=pu!3-MS4i& zCyat1_a%bvqVuqPWiUI4C&|RYGeFItojqxffcVuv*}NkiV3zrn89(^WhF;W!_L>bK z`J4@pR~vxEL3J$L^_`5}lY@D~FQVr)Damhrcj2{JA!d#1WF^C-sMPLu`%UdDF!Ph> z#dEsA6w-o0(RMks70e(?&Rd|*{!h>=>CZmwtEo(o3CD$3!=Pc=O*ZDq5O(3^6!zkw zG`DQI!7i(%;v=Do)qSl%ErrGSY{@b9>8=Aam~j|FC++0P)&$o~a)im7KcGgCNHvXJ zh_9Z?@dGF9@W_vqaH~54)?Ayyi&wewKrLBG&grd8F}D$~jVMCZqPg5Y{x-C}xq*r? z!zG=w-^03-3A{_3U4`ehkwo7zIOeV@EM0W|V%np4Y}W0>YK7~d67~j$cV@G4!xIE6mZQ+N!x2v&iX=1i#Xa#A6SN(Dg0Y}ZOz_$**?vQ|a_+`; zlKC3hk~4Ek*zkS9!k64fAoEn5iF{Z=qy`NDJ8_=YQuucK{-P_A-i>dCa4$XhUNFVJ zdiWr=I!a!0G~F8h#RkywmN@qFZJN*_dyQ?LP>To6l+fX~5%kk8WFf}?B=$x?SF{{u ze%d+^bKndsnc2fqy@ue=|6Z}FOH@JGU!424+{KRHQ=!^?2ra$q2r*_-xc2NIh*BRx z_C1`y^bf6tw4_)FQ@hJ{8r{O%TDnkNnS^;sS>O<-Pxs9$t%S#e_|<4_aC^7`r<664 zCl(I4)BFYb^k*|zrSBv5UfapdJb5bpfrG~KZ_u~TCYbaz6vmY_vqz0TS)ZXwEYfTY ztaLw%SZGfkpEf2>0&~gFC($BjRvvyG%EO6UWZ|y-A*8+SMAeWkrlirr$kKC2a^=m8G1y*2FS{5YU9t(@V zri(LNDX^8-5n*opxRH5ad0Y_Y_#TmZnPkN#Nu2O>p(=%mo^rjEJDf&ew z%u2)a@J`6k{%AKvPa1cQ7zKkRuh=)q7`6(Zk~Mp5V6to~gb&d~-5NjaQ^YV%Rs*)! zSirXFk@gjak-+sU@(D(V#y`_9MYZWM4YpZ+vh8p z+3#Is0=iX=VBfKu4c_1+@v$BRFXpzhRaa$+j_x+rl=@oYHKj~4Y+n^N#pFs#@-GS+ zpHsp1)B?fk@_o|pY&;p19VpRwcpj3Y-$`7+ly9zO@UwK0WJSrb%IU|?N!D#EvM=%e zB^hzGTyjU{F7{VB$adJzVt##wklBZZku`b2JWIwBwrJ>*5z8a&`*r$5`oGJ#V_r5X zo#xA$&s-M*>JA{O^#)yW_qlhZtuWNz9ksTuhBp1_aC@I8Z4DX<4^OJIxCt^)re4UB zyF_-!B6Ixfwcq}Z?{G+;a1_GMIOC80eX#Xu6?p!1e zJ#-SS=h^TFRnbJFHjb3eb0#N6KNE3MWlA)KRorX@lMQ<+UrsQ^-W!bBStR0VMGdxp zhZ$?lh$7($b8yPO*9hDgawpux|8gzp)3WOrJ7Y6lyEYl7xx~TX>(TDn@UQ)J^0fuE_T@RD2-=q`w&uYR6po_m~W+i-8m*6UU@ z?vxi=4%Fhk&s6B}HGTLO)hZTQE%I*qRWkQCn!GYdWX=yugP|`rLCu1Ek!f@t&p7R7 zsn6`-KI_N5kB`E=$I?i!iwjpSu%fleQB282?7?Y@+@71+SUFLd&9#ytOQOf~Ikn$m z_pENrRTEop$M=)u?4OdK6_!}gKS$EMB%-ofwGTB@@?@HWK9jX?BH{LuW!PZcBe-P^ zl?2Lrz2=Bi@l=q}u5i zuJ3Aq&Ne;R9aBoO<_+N`M^E5`KdNYKl@7jnqH{(DM#d;>6axf<8a+GKx5WK{OsEkl(qJ*RfN;&8wtJ2qC|6S_B;Q8oRG;9g_O z7u}iy{!6?_Q@|NcjIQC`6}b@Wcog?3Y!i34100@oIMYH^@qhbZ3-8X%2B#WRI($Jr z4{G-W^ORuN>w;|FXm?y6n985ee~d$qrt#$4nP_fq#rqsR4BfFsr08rfhNvCk$xnZh z!`cUh2|?YG57xsVImVEUsmepEWEa>Z=FzWhY9cGwgkt%YQqr$090p7OCR;o5kWUK* z=QUZxcp$~i%6BEP8-6ism+Pcn5{DNYKL`tNoY&&8Sb{`6c*NDteH88X|!A`zAUYWDj02HJ&F>hZQjGrfF zOJY6XX}%L7(>R$}I~RU`oC(z)_gLbqCz1uFUr6@CD#5hTj-7LAU}_TvGp!j1nfyb0 z_#D}vjs4JqLn1R^0Z##$u7RX$pc7oPO~&2_>zS&$8J09DlE8zP@yEs}NmuU?m~vJc zV}lE^L+d$vBXu3ZY{GErz4vVSDOarM^uQ(eM={y#Pv~*2*gkdiGS(zAYm_UG;|$lm zM0;)#wm07<`?R)Ten>Q)IB_1+Wa7xHs3sxiL_N%Su7cA&p0bkrTcBjUfp|Z^3?ILl zLd)!GcFr~(&oy|!G5OOh$~Xf@Yqa8s@VPKnw~nOtPiN!ujBvmMBQ{Mjx?;P?&R%`z z8c`aShF{M_;Wg7rT(mBoZJ#C(jj6`)c*=7$RH(->+u9+0{{k4g^PwR9s>*)(fxe*q z^d1)1n3BwBN@AzvF`d;Mq$+m6AH`3Szcbz8QiTVb`*1eOoXo_OB$0)y;E%&+z7bA{ zJmIgIQ6i5@h0S)A!}_MvWcB20l94_Zd~Rqq-27665huQ4!pO_84#S}|69_e*&3 z@juWQs3ZJcyA;mLoARMsZLly>ymO}S#V2zJ8ZGY!2cv4?POl*iSbqYSOwYn+`V=os zu|mx^mx=u72V~sFHuA3gh@@=)6yo7|6SS8}lj-Hv;J>yN*KT|-QBJAHmKh;z$j=OD zb1ozAd%IcwCX`H8ISigX!=X1>bg$DWR8Ex;5mjV-rI4~!G%wGyy=)}P*bw@%or8_MRke_?FB5&rnThjqmrCOangVS00Nq2=KT z@}lc93t4N6v-IYpqjHSBqQ_Qf>;6H4?nOf%%_7J+JsE@%2`qnE29_r+0oAROaDz`6 ztY5bUa~1Ewz8%?M=D~1s+j)4;x*^W@5E%a1MwAK_z;Q(++NwK1fBDPgS#TmQd#;T8 zdloadJ2uQlev|!^!RN@FC-;fXsE77lH^#sl;)?IR5p+(agQ}4+dTvyL39TEz-(C&p z+&oT|ON&)4JUZIv+U-bEO?RAe*HX@I~L(r7ha4hGEZhsmRyapa0#;o$@WT(UY9+q)y7 zv7kRB`=rA)w@iAgY#c1G8At;=Q>o2KE$VJ{1*7OP{wDDW7`zbo)FJ=zw3okd+NM~r z-sukMy;5Rs@iAzuloI`oDpdc$Xt?~^3R_PWg5pFYZgX!dXt?^KYN-`^KU_=N*Jq-Z zRJ-uFM^my@!H8}eS_a`_WmIuuH_ZOyMb|sFqWR4Ler)G0t`Q}`+~)6i=A-D|s?3DM z;Ah0NR-3NhIjv&MNqf5aqYOP!+kshsUFo^qvP65^c*(i(E8yrLHELkuN}D~GL+1EV z{NT>f+>poMk^&Fxx5@^0BuC&(u%!J$b+AFcKl9qG!TmZ`AW_~&ZMKz@iceQ06ITCY z11+DR*ToNz(=S}?c7F%YDf0NiF-*AGsvxwMyvNZiOX2STSN?as3-uoUUJ@{q!P?ok z*c_v)lGd+Z@uZFb>J_e%3GR!ye|Q0`pR^WEx~jp;@iKg?SqlWVheMcF3UvADz{T^o z*@~Q(FtjWTJRZA3;BR+&^-~GCvZWV49?rs@9$NfDVG&Ew(}Z8sWszN&MvslOCHDqo z!Y-M65J~&fYex=IMU6z*JnSAXZP(zT7328hf)KD>*D9H+lm^R|mSFatPptMuFI$qo z29h+yxuj7Nj!$r)p?eqOi^*#I%Tov5+K?jl=RH}Uu~C=_gJAr9Pnh5pLVGP4DYa`t zwSSi4fqN9cvq^Lj+8b03mttjEk+6y$%XLQSJb4G5y|N&`SCJ~XZh@Q58Z_hSAU@J$ z9HQzYthXOVKf*Ma_3Jsdw)dygUIy@`W&eQwUdDUe+awMSK{UzuD(onz#wN#BQd8bQ zdXBr(qN+-`{=EY4YR}>4%m&bv-h!}8?1l9veG^V;XTg{F{@iMlEDb#I7ORIS!@G(A z`X)6Uq~~Tbk1S<$%L9INeGuEc{213Nl%cBWXJC9E37unKi02k;C-xrOy))wgAJ#3!Un_cvF3n+ZN>h=m_20`s3Ok|b{yaXQ_kcuoTqoMiSc_|0 z)uFP)6X&xI7_B=8>g<}qbxSH8V!Mz}eSC$6G_;}JJ3qR9&|&P^kWV|EGja5z41O?0 z3O|PQqWanzGTPEhbb?IbW1H0J_QGm#vXH~i`^C9>o)Zq~Hl>zNSHgO2D^l=oJnB7P zPirs6g65JEBER2`+s^(4e-~-c)oFG3s-{I^{bnUUI_Vh?FUOHr}k z672GB62xpWX5ZM)J!Wj7D{lM;53W50(@sme=5;Gs=9ta;Jz9a`F6Q*)wiKoiYmPPr zrTFWLG|r8a=glg*plXuK@=Bxd@QBm!C;mCp{cB8fw?qoDo%0~1;yO!oI4N1bV*|M} zs|c<~i9F?^8T{12RH!W;kJoZ3HRydS5v6K;)1*Os=j&LQIbc6Y)Lkon=iA`5VF_GX z`w5TgZzch4fDY9uR9-)t%oceSYGO-qYTYg2;l4NwdAtW^S$Ob@2jh9o?K$|k{sPj2 z%CP6U35}d{o_+Z=jQ7|@z>`^Ss1n=;^Yjl0VXakAQzT29q)sxstp3>1D#zfJE`MJ* z5<=)zcA}%59N20EMroqwWu65+bt)ZgX1oXIm|viha)^iRv8A@CgE7M&<6f^G2-wnz zQ#uUjnytoou&9V!_ef{Gubt3#W`x98a*AjlZNq{?2Ar2q1xZL2Hcab5b%pdwIe8lv zHmWaJymm!LOE)t4Ng9-yX_DJ1pD;7tga_4AXm&P2&lzf5dBzQi!`~({W~SITEer&< zco!<%If-4iGGJ4`3U;U#<9736d-n!!tbH<>Pi<%*Gpkb}%143wl1A9G8~HFpz(ybQ)v4D zZp-t~H&2r^Xcb`yQ@z?;o^`3EQ2uHw(8J2B@>Lzw^f0Ja`KK$W7s!9vWYdl==D<6C{G z!mt=N)+iR`CocerVKebai-40-hxmk`40d6l9iJH>VVJp@|1x%?aUZ8qpHffsJH3hi z4)wwFZ!KWZ>eaNLYXZ*vzJX2_y`6e_v)EUSZWyE?$9?6uQ1+uQL{*)|ql%OGsY!@2 zn)`6XKRwZVU5nLDx$racI~mpx3jMka>CK7{#LQR|RnF^BhxAF1l)qGR@AYu#zSxAf z>W`D*ft4G4y zrcqcoDOAV`-pDqwx#Yv(!KB&l3K{q6GI{f$2GX{x&?fio_7Cc(BioC0ShIG2y5>zAp7t3= z`|S!9oH|Nj(tlwj?)@K@CSQU&7Voj7w?Bz-um|0EF_RuJoXINRv$wzD3m=-V605Nh zn0ERtl-WNc8LJ)=HnL5kt&|MA)mOkkF^A-IFqTzx7=q?66vE=Kv37-X67%#@xR6r~ z6O50+Qm5CTwtFUd*y%{z<}5>nT_JcS-3fm>#lx!&5ulu{#mx5~0{a0`aOTehxFf#~ zoaJ(u&hI4R>;DATu*D=pOKx1vgid1MyGc z^r?p|T6r%_bp1hYU(kTirpL_ju00ECbfR~6^$K5hIz!>)+t_oj2(67;aa(E}(|>o4 zc)X8f30ATAy`~PDWkly-z-lze{zpnvRcN*BF`=&{L)hyXN>2Ev;*vq)oS80wlq0`b z`*kONcGp2tR^bmTWS>KVL<)O50w9$+vwdogsPC)@H7Cvq@kP_It+)v+&qopI0}Dy; z@8fLGWCpI9FTunw4RxyCgUi^7dz1-7x^u|2e=*o?GzR+KJ_CV`(fI99DIA_3OD^=c#m~v>V9N4h(YNFVhwM`D zNY5pZnf8;B9oD!_t5Yz294VfG!eGnvZsJ27$dQMWSeB0jE3fV)2Xy?{_<4uO%ZCeb ztllnsJ^To+>{R8QZIjVT`3VMWm4XmAFWjtc&xWmeAW2*iY@b+DPU23BJsQ@;?1ycY zjPJXV;WvSiN_RG^aX4h3jOIiAOTgUd65dxhkGGasg1uBAO=)?BXV;g&Pcv1zszMnu zvgX3Bq(0Qn_yWd1z5+qb|FLD}o9L1Iqv-U1$H<1C)p+ONXe#-Bmn;@O}37L|%^ z`RL=yVt)4!suWn_?J^arJ~0p;tX{~+CVqtPAFhKpNulBQ7SmL(3_jSmK#XF<@L27G z^qR1e%dU)}Bj2WCz}rUg?<4`oa97%q*#z;o)9_A7F|G-|3I*Sd`1G4+sLFi>eyk#f z7a26dlwoeP;g5%){NHxkZ%`6s^h?E_rlI`Kh>djI*K44)b_cy&`IP1A8uHiEwTWV% zMXW%_1YYbz^gHNAzlrX1>i~YtTaUZ4UgC(gnh<|@pMBkiWa715V3$km z>Gu9VNqUJTALkne7rIm7+v?@yQo&bv7PJWJ^4G%Y3rAT}SF5Dy=ohx!IvzjFeaYru z-^k9k{|EbZ$B==eb70WBDs+0|Cp>7hhU`P<+0uWbxnsKueYrt==N(NU*NnfwjJ7#= zxoIyZin+MZhr?ld+AA2@WW`!EpX2(rq44=#f4FP8oi5z)jJ*$zmIVDPWF!BDf!@wz zaQh$OYKKE`&9qYJC$g`H1ysNf=P^K5mEobJUi{K|o{i1r;L_s3ru1A9UF0vIbkZ1V zl{KD^S{MuyMPKAE(ff2T<_vz0%ZG*+lleWJ6ZqKUDy;DnGd@uwH^F2A&!0Dj`mNcE z&7B4qb;XAU4G1H%4o8zaR>z^JeJT9i>Bxc>59i+1{lKPF4fhQ1%YSZ_;!zo)Xlr%_ z2l&sY^~#I*_lbERm`sJLVjHTT^an=_3d0n`AR0To6{DY)qmthxY96YNW&4}qP)Q{2 z-l~IZ_1@dJJIe8!-K}^|>Sb$$Xg1Nw_w485RXxfwi$0B_5ORv(!^(Sl53o#Pi8p zOqjS5#+1oIV0nKs`;;KLG|GsuucD*FxfHrg=9BUm6`~Wehz>b?TgZ9;9W~V~aIDc- zC=fr7aeX{|xh^Zt#5}3`nnvJe^I%!YRs1_k50W2A^E<ss76%M6oS<>6NQf7q*)f!j|O!RG}EG&=AtRB!j8dsgk@ zA^{anb*1zAKq-fYq(i7%zndmkn!pju7}S;5q_5(u$E@lS#L#Urko^2Gah(9SCyTpk(=u>roYGo(q-A7Kc6bB5wT>l&hz zC$eAteql||Y*47XE%<$X$_9R!3YCNW(Lr7f=idv1;I)ZpE#|9+CEb!-**Ogwn#Pii z50^rKY9{<06+)Kn*vm%sDXk>2;rMsBnQ+}%L@FMpX}=}Q!0}WUF^arK z%w;!$YkV`7 zANi{Ipm=I9@g%P}Fc5z?9LDtYg~Zk8D(m=L2kP?6rFLpY*i^bzVY;S+c4mq=fvDsnz)KHpn^1dcp2Am@t8X|eYZ z{zu=9uRGBN-$x9hI;rCM#P9_>vZ;n;iu;kF{Y&Vns<-H7d;=zXMDUct4`f?-DeRfv zZSVEemnswq_6~p3z&hdY2&~M@x82#rvjtdtZ;j<{VEgVIQtWzW|79)l6r}mRKl{4V? zoC55dGZUkd1&Qtan|S!%SyJ@S6JL%w1f$kO2pjbWF;Y{DW`jfF>#{k}e|NE{c{iQJ*fv*EofJ z@%S6mVIxFd^F)?5HG;pmG=R$uPT*6MZTa0FvNWx|nygpN;a9H2f<)a2evY%GtxchP z>F;hdSZ$1ouaq&d>n%xAil&<5^7(V^YY@KdmHiWqxzyNm6|U8(LIbC2$eQg;MSpH( zoc}*~xVt~J*Voe*8{OHe(4}xE`Y^iwF~q;RGBk8%oDjt>5#g2y*)rqf%*v&_G~ z=;hTP?xj3scSM(Bl%l>ve(eFsPxuZgi+h;hZ%xyco?%Od_X-|J994m<-2Z|GldO5j*)wn=Ae%3kcNpG|F=RUo-DzRsD^yj!hx*5&u-s`s zu9C8$`+iNLu4-@KN2wwANs{LEO4GQ-z`b}Rum;`@bD)8i4m`Hhk7}$ophkYr$he7? zP_`k4Xf9jH#;R3fo5%thx_&KW=`WB3Yc*hpLN^RHQlPJmA7S&V4fqi9g^3#F?Cwkh z9O$;6{5kT9ZQdjUUDK$LZjnWtoI>E`eQ$J9lY`GG+7Osh#-`*xBO!|#Sb~ox+JC8# zXm`CQN7iMK-;*cc5nawC4!28KtThZK{r1JfV@KkSwp=mTE6c{Mw8ay(0+d<(#TS{$ z*qqsoS9(j}zx%hi>R*KOO2c{6W_4`aaRy4oY|R#jTC(ipX-WK~eQ==Q5x!etfu5a4 zbO859Ru@XjzP6!y$qyl;EgKaM`q6jRv#CR=1s~%53#R=%gcl-*@>O%6i0|Jx$lLD1 zN~^~3G&g1F6`sTFxE3)Jr;DSb213a4Uu^%ZNe;UzHo;8dLjUghi?5#O(U0<$!fQj( zA92Zm6B9iiG}J-p6jZQy>UV4pDa8|>D%gMGDDHSsnpgUcV+%Jsi~ihu;%q|-rE(

jpOoy-O<3sjqT*Gf~8`%WM!L-I@12Jw{wR72BVo#@9=i}gYp~OOdCtqP|0ew|hvQy`K$>$ykY^84`eZ!Tg$10Vo z9sOl_#S10ATOY{rU?;T7^TbEHv}vGtzLXuXnB4Om10&B|C)`?*g6O%``yukZQ`E?X zJ(Ow=T#H&GM$^$lI`Ez91@w(H;IXiS4co22FUbxTeir(`*S9I)t$A2$e3S`6+Faj@^4BBPM9!G3@H{Di$W@l(&qr?jKha}WchnDN4bs4NnGkM$ zgay#MGptoK6y*v!exezSZW{ zGEZRM&!3{_-3g9nHNX}7B=ESgg8w|1Ct24O$87pv5sZSBB>P8Y!dDl2d>?NwR2y!F zq=>`t@rw>TT9ihL_L^aCgOUAr&t{nX-WVdiWguu{3NBTbf|@;t@ZIDPES@I^kN5v% z@jVyGo3!6Tp6Nhz9+L)!d#myOpg6ccV;>etN5P@OT#y%Cz&E6;$z#zkp8RzSQT3b* za@Wt3@&&qh|EnkIx7`RgDHo72;=5egs~&fTkAd30?$Ee%A6t=80&jOcXNm!FJiR-d z=yfu5uT|x0G8_k~T2?h~SxwHL&B5998z$bLkGU_k!7EA{5=QNUX->-ct5<<{Hn_s@ zyc*(oSdZ@w(czzO6!EslqpW3Jz}>ZVA*NB z`v%D;({VH}+k-d6t|4BFpOC&c`Z#RZYesCP9%7y5QCxDmmUR{u!mHS2TxvlZW)uXH zc8!PFk#A1BmzvPWt3&8ZF~_*_sDP!L7*F_h8&}C?qU>5bUbH6$pMFY!msSz9pBV-+?>RvyUb~o^-6kW#z=?@Im92eoT3W4#q?je9oF=&V|r#QM8?2b(2rB1%8B>M zz;|Kz%1@iFyW;>CK8L^&UmYr5VBp8*b0q!KQQmMui~8)B=HchGApPAv@E2#=y&CzT zY5NIAtTE?5y)N)*|2B5e*aXjzt10iWlDZeyXiO(-LAe~?5(@!^70)Kv= z^sfk{hQ=>J@@xmq>dnFLrt@IhLKPn0H;LQU4WKTctJu{sDzv_P2DRvkfGtW_sYnNa zKvQ!xNZUX^4oH+dnj(6`3Y#Ft`V70Xe>a8JK<0J$ZsqlmQ|J_}&wl9EOY{xJoiWkp zr*XHb9ySMmst)C;-CoplBG7?=k)_^ zm2Xte{CEN8)Ldr4j|^GP=CNE!^z%JfR7Wh|GOSit<9+Upq7w+v#(rA#+sNtQ9=DS9 znmwW+&Dq@IqCZ_=FXJ%irm@4DmrAHD&XB!4?xJnv0ls?m4}04je?D@%G&So~=k0@| z`O^Vqu-t7XRrAw8O|7Yv`8lJM*Kj^J@(_;hr%CIVFQke=`B)#Rg0UxVVX|BX&b{hL z?IMEse`bfUTr&(k9y(A`701IOhT~Mr{XEy(jMgRe=Mz4OySCUHIB7{fnPQ?wZ=O00 zr>qaqzAOHatLm5NF74HH;ZMU4(+RQ}J$!JY-~q;pwNBAaCy4}B%_ByZK!gF;jUy#0{FwYEM3H!o$p`$L|OIq(z1^>2Xo zfP3Uyv;-$?x`Tm-p5U7|>Qp0pFJF{+9ivKt`sEL%D(|$oIKx4|o+4aV>%*sfYhZ6j zF5p`&ln{e-`TWm@{MxrcG+A{A1mhvl4==*=^UlMgF~|6(^+$PyjXa+mn9s)5%%vAh zLjVpO!TP^1@lLf1pVO$up3ext9k3M-e2c(0U&Zs=!z4K2UBGm$2Re|#C?2xDFTVU? zMZd6Z73~kAQ88sMF^$iruj=1R{h{-A#_apNYPkZSnm5zT7-^ykw?^B0Rm;2Jdfe#LBTxu==bS&$<2%uYb0{ z%ze&O!`p#2#qEUd`xl7v9PycyKaSqh0KWaqR{Dq^N7;K3w7AQPnqCg3gBK9093O&L z^Avf-qt{S!S77lL0hr*tPvmVlbEYOm>S}JY{)fc(h4&eDYUKV3vneLDIB_cXG8)DE ztT+aGRbg16Zq1uRD`DC1YB(F1!yP-vu>OVs>-)&i^@01rq+u#oxpp7(7Vd&$!MZfd z{wn<4orSut^1StfI%Jlq@+{AflG9ta!{6AmX!BkQ)(l!rg*~6`!}s~{B`ZF$*t!t< zpmjgJ7k(QfM@3<)LoI$hE3&}47topM$1uN5hiCi#g{R~Nv<)S+-1Gu{xMB-J%u!` zX&EoOl83Wc z1##iP2-Gjm0`Eft)i>@>4@8*KD__2_G5>k;B+-#OZ)FVb_4o-xXLjKFig4Z_ItuoS z&)t|^>-kenLl`jHjCW?ZK%;yj{r1I~u9&qR8!pGt46x-B6QZzrR}fwDauw+F9kkOv zjwDBp6P-WBa9zoQhv++U1jS91^+;=u9QuzO{G#}C@J}?A50_DM25i&708h6 zCAXFx!1E$gpm)_bEZORPJ;-U3CR#q zXjC%0_gb1YC-qK)(mYQZX!hIB^Z)Po9-sH)IQE~8>%R7MU3;zbJYTw)b^NmUaA`UO z?e0fSP4lpFWEkWnev;Pq!K~&SjJ6}cQ#+$pdBm?bkbC4TFSu}5XiUpTB?||R%3Q{u zq+F&yqKIpH?&a5ab@)rsZTW*261diIJ=7?qAUJfxiCvcAi~1(YS=o_Y-k8w<^#rgi z)#I|JL0F}s!#)f1IZWzNlx~ctA6+c5aiSvkFhjy2oR?}>dIbQ`{B{{@nZc6 z6F!^Cg3qYuRMx+Y9gUN*RO^v&dHN{KN${nP$LE+{e;0M;7=u%EJZ1@1oEnC)9Y%?Y9YuKgPcrxV+C^~fW-mR{o}|AggM-HQ z;H@vU*?r?TXnEN}Nwdu1&hrW6r`VZ|JSE+SsUAr^E8)|+iCnaQHvTel!N0@TNLt>0 zoVUTB^5&cZJI^>MAqQ&39NK;KEQzhh`Sdm=@zY8*9#(IS14Eo)hUXvh(jLmA5BG#_ z3*HNE(R)F8Sqv{#PZH*AxkSN>)7kNkHsAf0!m0;ISfkt?ixmU${M(VZ^JT4Q{q>X( zcUY<|eLF=PLXtsewmL`u@)91*>53^@Q{dR8?tHpe2Mn)IM~}5uFvlxXtnKhqKD^PM zN(5zL+^j&^(id%D^r{35ZnWbky+(2V^lVZc`j$pm$v85mBhC1IPWq;+pm3?KpfjzVek?sp8L)a`A}&0ZQ4_ zO4>7*bFt$$dJuC_^z9o-wyynHYwWrIWB$_B6kShhO1mjS#Z4G;A(pQ`_zc^VVmKv91v5HKp+{d@A$xTh z?u|SM7V~N;YG)cxStIdMTkEM~A0vLfXA4|fXak4d48`B4Jn?g8lgJ?Fq?aRV(}PEtqPaX2PBav%o#G2piX( zLzOCBOlo$PdXu9u}`Uk=_4DdN!?-niahk+<9*D@I-3P3c)XD?9D)M3)>+(V^#O#rV9rG86r6!o_K4 z?57IFU>mZU8(kwwZ>txV+CXSacc4`?tfjI-h24qESoNv zShPcYye|(g7dpst%|vS6UJX->RAh=OFlw|^FO@hR4`s?QN^QD&eygMk$CrB1YKn-47$^S&5Gg$ z4P{5^-an11%Mz)z^gK=4KL+cIl`yU66Y!s&CmgL$z%uR8qCJ@NYP&zAubL&ev_#Rz zyfB#46iA;YJ5Y(@1WvsuO6T(y^~(+AZ`m%~csUKCw;Ir`Rja6UL4Yvv=?><~CZ01> z(jG?N0Oh%DH0qZJJvy?HEA~wlW6xE>mY09|R%so)c$7pso8&Ngy!705SPkRe7sAvI zKufK!;C_E&&e^Jj27fca$ioV%;w`|Kt*c)L$4KnYniF(G&>yDErt*58zi=WOeBVOnR_xtOddoDuH zV=z?G84Y=JRBjsBD(@{8;H~j{sCljhR=VJR(-8wkD* zv&nDXNv<5e4o5_|^ZQp7=(o9+IzD>^9e?#;&x> zTg2gaifP!DuDEt(Gu=$fMc>VbHWG1XzO=C+}UErWjl;%LxBc* zRwScsz(kyr<N>T2SHP zEksorVYKxF*tg>WpGfG8SiT4R14iK3*7a~svm;Ji@mg@)(t~GCRzP{k6LQunMMG0_ zZV1T}zS}r)WqnUrm402g=4nr_oT6xKWmlYeeLl~#w4yx2&1@%EsClX>$3H49R5C7^ ztz8nZ@Mkk!Py1M{vba=ed+&1sU0Bcel6r>tG?T>=1x= zPoIT*YDMDYx*Dq0BM5$|3!C=p;N#1T`{#Sppw6Cx*N~BT+A)-0OM2R;X?~bvvrN9$ zVIqAQd=vhYbouUeB==Wv#P5EeXydswvfS&>cREO#H1jH+uk%=Z`~5!5lKM6I5^wsN zlN+koy^>~=Mxo>W(w6WP7_;z%uyxrZ@8Nb2`|` zd@A?!OO}rcvZ6iu-T9t!y72J9F>yh)5`36dg`d@fuPrX5*IiG5^03Z;2O{ad;cy(k<0W+? z1K!z0bY)K)*nHY2G?-S)>(*`%yY z(GB%zY^jOm_`hW!>X?Wx!xJ#;^?n*VyMy>;@EswdXERl6M_}D8IczhV3lGjDqxXZZ zczfeHJThhl7dWUv)`}457^^^w%U)H7bX5=v$4?cMN2frIT!(XX z(k5y0x5HP3?902+C-y8PzF#RW8RQ{<6H!-PWqv^1e|HEArhQ~4UyleUx1AU7lMc&g z%)!*&nd~rgI3C=&39pPA4_!`2LZbUo!N(#7gB4c58;`Dd@N5uH8k{EdcDqlB!8TN* z(;xh2?WvBhQD&=OCF1wLDrlgZAY|NXfw`QHsb{yrV5xTB(CiMM-jcZZeNVWvIUAgR z9wW1Fk;2fOy4>lx#P`{Ji7xlJNr%)#d1Am0dQ-cICJai2Ul)d8)sS`AC)ftEhG@uV z#%l8S9S9f3+GAAxVOnYzMr*4i&&c@h_N9W9*YCL>^2&0s?e1RC`Q?fyZ$_fyl_MBj zV8!F>w(*tO*f5&a>E#Sr`14h)5Brd1v`n|w-S^N6;nTtXz}5|aPD6} z9zT>!!>A|KB85oXTJH^1koy%xx&o{8(w=Tr)PJv5MCbn3ZQqYR! za8p(CWolc{9Eam{G4z>m%w-H-Z8w42Jnn*F)CEB^#SB*K_u}EQXztK4Sxk*Amu0&u z!&&{kBot@LCV1L`#gRD5IBCSI*Jz`)r!6Q>(8m>h zef%0s>LXx+lPNBGG@ZAWxUpt&5Y-*=;{Anzm|^fy2)@6aKS+#63aX_&Is@?U4T+yE zc_?q5D1{{}%+dLMoA^uNH3ayt5GI;0g-t1==vj0KxaKBf(%mqrmfsBS&?s*D6v)w9 z@4@^Ik%!hwLrl!FMtW;s* z;)jqpWPq@${~H*jc%D^*)L~1>Sgf|Hq-Rl2XhOqOUNk}CI-L*2l^1t%cTat+wYwr_ zog0X1TIaxT?tYq6J|1o<^~TJXk7&u)SCBFLJ=osS!O^1+f!dE5FhkXX26{iFvh|Uo zl1Vw7UF<8>3S@jCyNJi-&O)JcJ=i}yMrn6L;roWQvMqrC=Eqd<(NrgBckm<~t(%7y zeLRHhElON5sS~$6+>XsU`=R2IAt;OA{#?IaV%NoUMSaJYWF>nq5A}H< zc2yfhJzo_;2i01zC@T?11o~s_hL@t|_jy5WWC zLny1{Fla6QLPI1USz)vEKacJ3OF|FmxG@YGthS?q;cR&ETpfbaJE1~+9M7FwDzwj& zvNAsn#*w;j;d|e%q|o^rJ$>K>?yts49z`qqF=IA7xip13B=m*eM&Idx6tMnF>dCrZ zm`!d{_JaG?iNd3wrBL(dh?wlFBbzo7Puy|1oDm& z-C0#kJ*EwT2j8r5&c44TFL%=m zqfR?FlO&X*$Bp~=_U1f(*8dV%t0&Up#Elf>w+$LTpMkJVQJ|K-gg|ux?=^ zIJw`&!byK+PwoWrRE_mQiT@rha30DQ8x`n|#F2I?JwSF!Iuw>E&G*x4At~M-)cbvr z50BbRxj(Y#m!ugg)b$W(gp^0H=bdc&tWfqCGYylsE#skmPTBoh7C<9YTEvZSLdoRQ zT+mLBftBu;K*o)t#*Lk5qZp4xk*z{__FOiSOB%Sy!CW!?x7abG3+jC;gyeo*`P2S1 zIO>stzp@LU{>2#VJbOHD@0ktVr;p^~x)H)mhgf<#+llXwY7mSX+i`2zF&r~2g51Ac zrO$eXoEmbBiY1Yx#$YvW+CA2OQGhO<`cf>!2&$O*Y%uEliYBK^`)K3aYGF(BahjI9 zR;+Fu!_NBoaPUDq)F;ZDbwNC|d!;x%IY-=cH5UI~P9@*?&*Jza-Y|7dA*iZeqx0vr z`01<&GPoB8g)dXd_|01R*5Jwb<*Nd=+3dyNZ3TGqyOB^8dkQKWC8qQ5ewpXUQ_fJ!@pAjCr zJ(xH2x&leJWRPmq53Te+k!uSI%C}Qwf9mo$u)6~7%Pgk92}x8Ry_nZ!#X^Rp{ZJ{J z$WKyoSZjk4ZRbi%z~gOOrZi&=(`V1ZUV^?R-;EEzDEJLXuD|Kb&Jr_T!rF1|p~iMxao1@Uln zT(tQ6ohLr%bdb-#3K6HRD#5Y$O|WG77F>EriCVSq*k93erUBu%$t10bN(1#^+5m#s zjP7jZ)&L!rUxhC7E9eY5v&x(E6yQ|^aea098y~>k(+AS-KP^J8f)OeE_7NU5Z^e|7 zPo&D9NNt}MK0F_fDvokkW$q|;KfgpQ==DxqyXKj&`dfE?J;ejof2o09ty^J@^A!4+ zAbHweS##*e0PMc+x@h1ViERT$Lhbep{F{|d{`abAu1T1z+`)=bk|M^SV&kB};IKZxl}WMM3}2 znbfqbhVBMV6MkeiilaBar#X|N;6$G&$Xjro?v6gfkCSydb?-C^bWmh$rvPX_39kK4 z6juDQ94ydv5U>Rp`i-1{Oi@Y4xWAJR&j0{x)V)q(iBcpy6GMx&8h>wo!U->yEi zQCKd)N2TLm^4qNbEq^R-K?ViB8V@hKddTv9KEUe}hICib;(OU07uUEei!CEU1zAT6 zVcY%L1l6&$YH0EM&D?-(|$G;6)14` zTd{On+f<%EV7|D1e>tV@2&1cGB(2=q!(w~OP#SP;w5+;E9jpm^E$eyF80v4$#BUa! z8u)X|Mhwbp# zFOn0JkI5@Lw+I*G?g^`k%OFRuFHCBuWLpw{fWO>0%)_Z;`TEUAsrbk#)>Sx38v55u z!Xu(0(VDMPrB3#c{E@fOE7ml}_ z5$zWKBeWh|KtU%ez@7-mF`R4nAAYWO#i<{T ziV}ZNI{xMTD*<%>9Y3#Qo44bCenX6YDgST$V{3`Q{eS0A*N!Zp{bt*k)1Al#*6^U$ zS!kB^olGRw%7hi8Ip1X=cU!K^`k&k5RjXX?I%gA!V(WBavu@Pj2b?cPciwx#%YS~h$dbeit{{fJAf-O$`Bnqu$x^FLy) zY|g`N@bvd2e0%sbcK>#PLZxq$vE#SXo9Ga9y7`F?Ug^QJ>_TaB>vWt{7s(c8%P_P- z88f%5;mN~^e0pvL4>>%5V-A&x9aksevxyxYRumNSsE4t#4R5RDUxp-7@t5`Nb!jhG z6}SkFm0x8M`?gSsVn^zd@Jk+R5(AUUXY+?{st7l z&Fh({dgB4D=x)HblT>-H@(4ixLHNRVHy_VS;1**))}Cj~7TYBa#ZFV!l<&t)r}uHw zqf4ZA>JsdlS%4G2SBj4>#FMkeMSdAK4r4T@;qx0(w(I+SwA?Ebm*v~>%0%s~6pj(B2$ zhp^OEf%{xG;%e`5?7Z|N9xEBZ?OpVRyF(43G(Lgadd=W{9wj_DQh~a9OoP^k=`^WD zi*LPBp^IUDv|)rJsCKtTP2GG@h?bbLfgvzfO-F2S^pN-SQNg#Zr(m;{8kfl4!h?YO z^1)Cg&)MrD1jmm+yYAhoQ0WP|`_ISsuQmuB*Ia}~8#TK6@P=%}DLK7ZZ^I8}9koAw zsS|ARG3C^!J<;yn5^%~Y;PoTIVMD+lsgAdgf*KWY)qpUmj#>c6-}Dy}OSh5N1a@DI>KPUdvJ`==W!zr=)F}l~y zM_sRWeC+rl=w@4h4(jbV)U5-i#t!G?gPr)DssS#y*h6n;`?Flq2E>#el@F7v+mGs9Vsh#$|%UXkV>6*9l`a6dYHasj--)0!?!P9g9nOBVbKBuelYt0D8^Pp zO?@fr*(>9c#Xab5`Fkm+#+5VzeaVKp@!V z&$&(;+e`Ie^D%HZv4YpdrINuWT`5ywA51@-4hrHKJd#%r!@k|*TUY;Z(ugTA{>L76 zzhFWeCOsGCOxaKUrC_)&i|e5IXDoG^{)i7WnxYgRMy9TpA#i~Y*&e)4TKOB`!|)}1 zM7N2o`y}ANzUef^;VaC36pQvYx;)-9mlN$iicQZfVcY%r*tg4gj7ziS_s=7#bp$B`=j{de!Se?;v6k2xC2vSlW1^WJRTo#p0buoy_?Jd=@YMoG+|gpTG-Omau`xQ5cY;FYFDN3NJq0 z5hr`UkouNsY+hzB+*#uSu5K^kO1vtr9cNFO^;huGfnngDX2X83j$zx3?l8S+C`K5b z5Ee-5(C32NuzyboEe-f7G+)Ue&kYr%I>VD6RgT652M1$Ei8Va*{6nZ*IvJ*JzQuum zLa^%f1XjHLfZlW~W#cWs!B%q~9ew?XR<>%OO2ZMlub;;!dkv>H1vOTQip1$1L>@Gv zoOVtPr*KOn9({Hj*Z*)5sOLq?;w{cO8#epE}`-K|jeqTM=z+-U`EqOhy||Eu1E1 z!8er%@y<{wFa4Pbo%*WJ?=u&e|uYD2|gZR{S~3|-3Pu%P@fcc@rJ7GWcK#_b)P_emIbMi;_f zkL3uDbk@DEhgqYy!Hn)R=|{>9VO&xoeHuTCANbxC-MVi=ojEu0=Fa<|t9KJsxA&mF z#XDfQSDldl@c~+;|HWJBQlCaYnBV`iQCN^%3`5Gc zb*s3}BOSZ^T!IZI*{EENkgP;@ZL^MuzA=dJXTxG@mKfLrl&9A+=rJ0Kizp@ z`vv^tggaik+YW22JM$6CW}#dkd4)@FEDq|#IdAoN!rXGvccL<`=!yI-#FSTF&J&i+ zh^DzSM)Iuu2s&!|oc-ObaYm$+kvQ-H$bQvHnFbB$sZho7u6<#fh9SRc)a3EeJH_Sc zAK_N%AV}6o<5$ZXDOU2hb}5@j-*so9dWZ!Z?<%39;7IBcxsFfH&!koT^I5OsduZ%Z zFZmD`bKaw#xYpkueILz2i>tSBoAERlp*Em0p*#*t3l@;2@QaDbI=qD?yGWi*>X~@t)=i{P4X!VV`DuR{%H2GdeDXn9N~0E zEHezJD)nVp{=x};lFhO9+gKN6NJVA3A`-mlsIWz2rqAc7b|Sag)1))(~;MXtQen#JV1kuv?Jicwn4bz zn-O+V(T3b!;q>}+xv*rB^zMkvl3&ZX!xnjqI1sn6(_eGAVO#+>jdyc`%{XdEOMVzZBE{^NjF#*n8M~#Q=M& zw?jt-N4OAMLFX?d&{elfu&T>bF#LFwUOVORHpgU~eElU2x%~;}cRNTfs)?AL^Mm43 zSD^K(PS}3^U|i5~E?u0uns4UBlinar4$9rnyG+$_U(Q%e+uRW=GUCYT$^~KivhK9e z%}U~rZ^PZ53gXgTX`+p!!72zY2A_V(u=8u3_<3g>uJ~raQCAbV)8VgTN@WgSR95G1 zr{B_(vQ}|drGf18hB(rYvI1YlwPN(Ga10+?1*Qcl(BqMwew9Sq$3A8jqr7k=a4ye0l|%p2>n$SB`VHuj?hf@EHo-_L(N-4TaOq zt3ms%KlS%e3(nfPiMxEAab7T?bOOH+K>Xvdt+!q$94n0n2CZ#Wt7-+`w^uYV#rd!h}Gnrwk*esx0C ztuYi@=tvs9F5u`thv`~Vm3V37V#wK9z&5K>Vf_+Cd^bFaFHd(yapzc`F2ekJt(}aW7sSAkhrj5} zt5KNHwi?|ww&Dlr{`RgZm(TdF#CGMGvTIMWByFY-_v*A?K3Z=HJ+7>yn#Vn{$Bcc# zvSTMnHhCIb4in(U1}ljRv6kIp{e=+U1{(P)nh*5XgW%(uW>{fqiMfhv@#pE!`0?ByA#aC1 zS{`kHv_FAd8B;>@Jc_ZykUkWA^0oLUZ5rB6HlW|N7pTZ`8TNQ)jF-x+(7v8&^_XJ% zp0J7swJXA9uVYx@oHlOkHHHl<&la@<5eTX`a=dD_4L(3G{r!b2P zWIHL-;-jc;<-$kmPSCL>i*TJ+AAMR* z^t>qqMwGyaQ|h?1q)yy0VIfxyKfp5o+wjBmg>Y_EjQBvK8|Hj*N43=|IO<-a@Hn|l z{CT7h*EQq_K@o+t&UPH0im-)(N5;a3kDEEv`5L4=>BwX5j>b7bV_CWXahN)C0H`Nw zJDmBp2aMa_;ztYeAhV?u7cASuvVChYeRw>tbNz>^sRQOYAH|#88#hepgbj+uRQqTY z{l3?MgSEo>xoT&w6`Sn`Bzz<-AEwi*BH>k87UUW(!(@L8K4SL+dYD}oe?R(6t6Ldb?0O$qau49-#9xEpR23Q0%8ua8JV!lJ=XxLaEQAmKrW) z^_gHlGbs!Cm?JeWQX?fZZ_(%HIBb6W6RrjbxZ{=~7>E4>k$%;DD%=IvX?e2U;hQi} z%8U5Z*&Xdp<-(yy!?0=LeNNfj#=8|q@M@b>R{K4e{MH=BwGTVt(hnXq(Br<$yz><* zZ>PxnyEX{&*S)b?NTrqQ%yHaNC4^({m=@KU4JRf;Uh5<-s2&D6{<$zpCrH?19YcCm z-Qe(}<^0h-7!BSXr$gSSz~w+Loz*x;xAJu`(0l~X)!Km<-4~K=U^xza`a>Mspn}iG zyHlf7ALuqh0Zlen!QMI>34~LJmCM|jem%kf(%}{zHstE1dOXTD4E`wZ z6eFB$&~;%3sn-MkuAPV`(a8OWO85%_Wsi4{6kt^RV&a3yypm&Ho%dM5B5X@NIK{p>|sj zewIClD^BfWKW|?wncYIcYlfgrzwWI1S>h>Joq!R4=kk)riTHfZZpvS04!e6lf}5U; zdE2wMA*XbJ5`ycITy}cEFGFrfg7r)@3wIVO-(?-)a zJMx8NRWMe+H{U5p!ue%GCHgaqZ-x!YTzN$eVg*cp+DubV5Uo6}k z4?`k8LG_wgcK9?LQg(iViJ@1iGIuDBbo?Z&UE_t3`C1rl=E#$0sIoZIT2Rl?piR4y z@XhslDqfZ^h^Mr$)^Gxbz1qg6eM3+uMTyt$s31Mt_PlYwDwc1I;-7BJP5DoSJ>|)u zbm|Q#RrcoN*Y@$xNjda@o9OnEF*xFL96OK~FTQ`6EW8%;R>|L&8YIJn2V?nIw=Kdy zdQY$qs5sm;_|D$@2I94n9o$7dhOK7$V1%kAPCl5;JuCW9MTZsOIMN<*cQ4$#vz651 zT-oYQ7`{1{4?omu=~c^Y*{Pg12(vpxI_mSeTVXqnJv5A?3=xEt`D{K@7d_)olCPpV zs>#%NX_sOFsg$FP%sg!%Y_XqO}A++tIeobtOCcRm9 z4#t1U;E?yJeE+yFZ`5oc8%X5O{^98Vq$@7HWydOI%90kefF}y`;QHr4oZ+_>ZKB=? z-Dn^eZfVEb*F(U2N-88zF_!zE%*FOm$>4Oh6gqbv%CVZ4=$ORjG4AFfJnnx8U)Wht zVg7H@_P7emulfsr>r4dirr9_SLTk-?;ft^Z+qEsi)h+MHefH3r z=FT}xSM@0g4KbqgV_I{~Q{r`1;jj@!lBRt+m6wf&Lyg+xqV)$>)RYNXH&f-yYhH;r zj_j7!{&BpbH{))#>olspjCZxygiO6=I^x%n_asiQ9wMZXd!!)^bDG0iXSU#T%^=jS z9xEPAwL()f$y;Dy&o?GqSds6;ssB7d^Rg!L)QyDs&+V{bstSk&W{{QgM4TPH zh@IAH$o=eoKkP@VE+3|qgZ4pIa|7F-PN0j$ zvG7Udx@a6IX(oGj<}*(wqN02gSGM^JF8#-IM;vj)`<*mV>wO4L(3RGtxTW;j`z&RS zHoz;h;z>CQ`InYAXO~WeXPob%?#Sj>pb2Bv#oRh*4$UrwZXnQ9c{xD>l%7k;b%0xn#wp$(d$SZr6t zkJN*OkXAQ**(XW9aMocq{&^3cKl0=G+g8ZO7)-%eiyh>P`rP32-)55WiZAe^G#hLk z5=rl&xvZ8>3y0&j@ju2Lsdm^Qn!m4z%K{bI{NX}L*Vd1#6`$E3+BTk-K5LclFTX@4 z4_5K^xY-o<{0NTNUJvr)dvTDp2c~zs4`=?l0^<~7S$@HW3YYgs4f$utP&#=?Sz?{iG}xyzkvQ5NU9R4O{yOhzg!3xiV>uoSW^JZg$@5g+eU4bqFX-*F zk^EfBH!Jb!f$@@$=j@2nLaU_XAF#rQeR^5Ij;MKHIbR7&pMHVPKk8uYlzCue-;KvL zEyj~dGvV^;Zd~8)kzjeRqvSi#V!I99@$u>ebjvW6^8R=5&b~A7q}gP?9-hiau9(Z~ z#KYJ=AdItA`{R(l3F5o3&TMyP2`2Q5mG_@r1C#D3)9l8>@MiTq8kSp$nz`>tBkY{u zut^6qjE7*aZk?e9a%rW7w5Ffk$Xbt=qRjt>E$usxwMUJVRG4}*S<>{q5OO{(z)#~H&^dM}TI@F$$DYaMDsMkH7H*EAhu=`7 z#yw$1eh+eX@)x0`tAqRMF>I>R1BZtVL;3f65JjuW&ODc`XO}_A{cG51hbmv&{6#ik zrhwY=Z}|7FFSfgpC)e6PUaaoy!LzR|7Trt~s&9W)=F7V2wDf)!^eOT|*Sb5fWsx1W z^j*)khGyhpA3!G$hw>`*3=K5VcoED@C*CKcd89hci>un@6jHoy|2bb zy(-uhc7~Q-Rb)3;Q@Z>s63f?S(Ttmq$V*=vaEu$s5Jk!xI?lXpZZ;S#kN+ zY}wMh`Sfa`6Ye-(!LrVuX~Fmk9@w`|@<8^)rjwbt(O6yb3;NMtqe#c~{@bf){CM_RQ%0{Lj}7F#qjP!9b-KbT^7Xp>-;&A4J@+{g9Mn zqKq?JuhSG)pl+MD^8NM7Qf7Z2>{~Di;}h@05wvCHtus*8*;w4V-W`ti^Td?GIb3(5 z9Pb1q;a(l7RxfF{Iu}{GiIK`Glv0#)aVhKoEqhrlNaB+)p ztnVQDIPeZRyUpaUik0BxngTu~`k;6nAiq^#Qk6=&*7o|8x(X8aGo z`lf(u%IERpZJu1`)*>$0De_yxx#Ho6{ZZ305P$uAPrG045zo)-&leke@WW+!_)^L; z(cUl(XGFO0j=$~sPwpVh%MvO3`U>IW+9*0_HHMaLixBd- zXWfCHSDYY|m^7SQt%$xu?d{I*_7Nf1c%3MXOniVFz2`z+u2+a=IROD%fvv~a{<_@&j~I~&6AA| zJRqcw`38dv5i&}=@MNcO{PDA6O~#0x`24bzv*)QLnDtpF_C8bxB?~{{0fif~iHB$L z#ZFBWEqR;5OuSI;+F6J_HwC|#OyjYOG6dTv1FvF+2rZ+p@(qzAWE5(uy8SJva09Rj2<+eMaqIXMsjM|b4Yco4LOpCYV-hacH z{L6X7W|1!M-$|;yd+_}GgRp*%LQSuX*Tc0 zb*slq%vnR;AN)ip+>;IsPZf~26w%?@V%!Jk(fedGJlDTZhl0HDAT43bCW+1Wp4Kzaje!#@}D~oMrYN_ruqJ$x|4>SesdrmySSLb-;RXsdrGmV{Rm#A z+!v~(tk_YxlX&d+1a{hr&-XTAt7{^??3p53-K-IR##fVi+kA|@cSme3a%9_MMVv1A zwKj|!3okE)(TpJtAh+91Ix7y5lH7+QVtP`)U$r#iS&nSsQV$&Ud;nxkT#I4qcZtMm zvCXp!=I-moexrT3zBCmN-1LO)rDw2f&5(<_9B*$21`--Ji;1501j?>$h{w z>a(!+s+_cR3+eE>7TUUGJ2&|lbG$`6G`_o!Ot;R)m*+!h!ox&JieJkA(H;%5IKM@~ zSkmuF$G_A++5B7mQ?IvPaP091_zAuot2c%0HGT_i+T*~sTn_Cc^s&kJ1K92E%Gs+f z3Cre*^4i_TV*)@I8^j?UQB_F1HfVHN7)+f0rXDo zmN@d@C)%<9G>u*_Wx`x7lbslrUu}2O1B{hgBz^KpxxP#0fAx=T4_7FkluqA&j(>Un zZ2#@~)7rDs;Xl`~OHus&|4~0S{NL!u213by#~++N`r!XK{)m4we&x=j|6T(!G#&mw zj&JvG#-DYg|9|H{DBwiE|9kxI3N{KWf|jlhTId_Fbjkd6OMSh4R9onA zdf;-0X@OfEmbSz@98j6y(EnVN!~Eh64q8jLI-I+b;4m&}n}gzDKLVJEG4g1&n*U4Wy;_Z1i>|aaYsM$v=pz2btl=CbH>mxdt+>lFw z$IgI|7)#X&)&C!9Z~m6kAN6l5X+}N`rqUoqDMd)vUaN>gnJE-94;e!eQZ#53Me{%d z%2Y_Y_F5S-D};DH+@A!0sRc-Uav_M>!V*Ffs+&ak55 zi|92*iT=zL9IiguCA$7wM+>ZWQHs|f9+5hY9>F^b$Xmspi!RchtnsXLhUvx_nUsg$ zLWh-)$#mKcV9U7guyA8nUaNCR{D1demHVl-|Hto7y7>Qh`Qf9l?ZoEltFn%I{^U8O z3xE7=!0WdT#c3A~bBo|W!}F5CWQYz{&Qa%0H;&QqpX=exwVQA`G*(bvWrMpn4xpe) zk-Q#ci%qlsQU1*zbmE?j4sD&rJK7_F+YboMN4LOB6MNpQG@dVZJqnwBI@7$uB8Vw| zg;i5e@R?P)WN>3Ht+zkL!+OSnOX+I-`&SwJ`9{(v&8Il?%mkb$Tg(nGCtxqtD!A}V z`a3d5v5hTpzF8cEo|pV_jRo?8NWg%2660IX0`0S;zxaOEAE2yr={K4m; zqs=UC2ptce5&h`V^xF`+q5vMPma>Zjp3v&n{kY#>&KniVMV;}HFkb2=W^E0`x?88n z)4x&5qq*Yj>~=E#;*2Mn4w3Kl{?Hl_3)6yovQDN3&$-(Px0dh0hEE#!bk`rso9Yg0 zMz2ECTcMn&@|JSLT}N2@T<6$Z!_oO6f%U{SftiYCV5vpGaoDePL`CPKaPtVM_+HXEC4`v>9eW?G`?$dJfvJy@BD&ZF$p+ zMmp2M7wh)zg-KTjVYJCIzP)7zOqBK|hSDx+sklxUW#owSySGYtjbPf{wh~`=D5Igz z%lP5CbZl?fikVes#Mrlma65J*+T5=JyO*la@oxaG=+PHvHkeTK$CLEe;(_?6$zJ|= z(q))@Ka{s@A4vQ96|znJOVDrEMVFPsStXzYjwJZ_v29LrDhfAc!9zK{-KC*!%M_j#&*H;~`eihRMv1jk&S z$TR#hWy#T@xcB}yYRh0qaGZ?O-frOeEib9ttGA?aryH+0eT}kykofrNQ&=B+38L@x zf{fZD@u zpz@xXsMFX4IqOn6yP_xmnf(ns_mzYH=Pvkr_I0}DW5Cl~4dLFxX3CUyxwa*>Vq0}9 z-OS$0^R7A580REXtjH1q9wu;@=4I-zO@`wd^C(X@S;#4gg84SDsoX3A9~NZc_QyY| z(*-?=KkLll+b_X2^&K1%t%6I6M4l$`HXiRf&7NXc47edNVU2=0Hc1gHYhMfRHuvS3 z4gzHz??7j-ACezmsKqkbM&2H*Ot+Fg(esPFL8J6E7WLV|n{u}c`y_Uyf7w7u2~f3%xPtSCqP^*n>&HA`=9p7E8g4;_g;ex4FvcGlvka06Vt$3@t>;siZ< z^3DFd%N0&MDfw7(17WabXL*Q54b;3|153MZZ=GNyxlGOR zRm>SEYP$}{7di1r!!020-^EEq);Luu4_qB@u%$IjnBJpVnEPgAqvrdNjsRr=8hgcxE`}Kakug4*MZIs+PhhWrJ1vM|t`1 zYzMXJ)1cx1v(YhV+ytF)or-p0etMq|tX{ z^9U|^r{SsWdT~mR#aQt<3)&{?P{_41RFd*v-GaK~w@uX;85zRo@9q`a=Drs@w(O-} zO$M-1wnDh{Pl2-w^>9hM4mQ^JEgzs{Num9v=V4PK7y7ciZjKB-b+96pkTv4_FD*jn z`jm6V?rKz8V#a2910Z5V3L4pJ;*2A4s2?&B4oS?W|F%xxI6;w;&o0J&dVaiu6xp|6 zJX)DP<XGY-f$xfWs^s1@P@an}p6{q2A{N)T4uOOxS-5}52Hdbh11AsC zz_Be=R1h5`?B4E29!1Nz>iH&4PT7hlzs!|#j9qzUXJa;QKP~INXtdM_H{w4(r*Nxx z2mW$u84a2@idRt-YE^F)YnJyAQ?{(&$HBueta}w5`RESAldnQt&N1`}?JIYgY)ET# z-6Up>2V1XK;_DmcpxNnuJgrR;H|(^<+qZ9vWv4E}l=<_}YFP|xm?Yrr;53>fUy1J7 z%2b=skJ>f%V@AeTy1K&}FHKT&48K>5`>!6tiT;z}X<96OdF==XI{YQ;rDc>H;Rc@{ z^`YtS9r;a;7QWxMnC1pb=f6B7a`()qf6M#ubG<);%9mWy-F=XY7dJVK&Xjog4kcu& zIEjOD=fTR!ag@2v89Q5Bux!CE+0O89FxII+;`#@3oslYdFOXbt%g@8xTl4tV>5+V* zwFx|Ln&b9gHDsuqjSuJd#5S$7*eYeF->f+bK6}Hly5D+S(JzFC5BMvXH~8Y!@+9== zsKZMe4nd;%EY{0+WY1@##q*om!Svl-+%R=eIZi94V_N%Zmc+!En*K>Nb?=Lpnp0Ta zP?{%C8^jTF*W<-CpJkCAO*F?Plke{SDuh*SLd$pCIQH8~?C@niT38-};6=GS*G3T? zQslfvwOaJolw;$cRlM9F3AJ)fP{U@wsMGT@8|O+Kx&9es>z*L&81Kx(lIHP@;&l?g z-kA*U?Zelv|DbW?U{njKWaorf-0YhuIE|iy^`+UI*iB;E9Gi#U6MIPr^&}X&$`vgN zvw7i^kD|`6zo0jI3P;+_$Fav3k+n?D9Nf1;hqWja3cd_Rt$*`5*|`JSZ;cVkmW5##c_F2xCBXO6JGkqN6{sfm*qqB+`OBRh0FEhL_~M6HpP!Uc18 zigY{(n>r2PGO<#GA(F?>Es{*HKA>$!hmy%IMRIfN1xd;U7@)7iXD!EqVb^Bp+!V;m zx}M@8drJ80oiYcL&mFkZwHKPNKTaFW6(n!YN}iT*M~pF_PJ{CL!{ml#cs_6y$9K!c z2~nZ6r_q(RiHfMO>Lj#xDuwY=&G_HlC@9?0D5N`Uquon)@O4Qh*-MdW;2?0@ox#=( zne_Y0D>zt@D4P6}@-RKVYoV)T56gqFSF67KZ@^804Ua~2 zlscY!uAJd6dp&5QcM#{TEs%PDQMh;hL>%5L7F(5Wqlb!}yjtQ!cf5KQ>b8%^hF$eC zO-nl#*YD(3vz3C|@*wQqWgM0_M8e4@r(m)9Ot!m}z-EbkIQ#c6VT{sU{)%6q@25g? z3|S0^=GDkHdd(GQEGXc<^SVfRNDZ8J^APmXj27PR$ic-MW?*h>9SDzt#RE4ti^pzV zCd*~3@L+Ex+W+#LFgMne2Toot<%Xor_g6n**Wn6y@WPA}w%B54B~AXbbCNi1O=dt$nwlU6>i)No?Qv7Rt+#BnkIQYdxYY0Gw& zg}5T}Ee$?(f-icW;g@?4LT-aPPuADvMptd>BWOUN$u`lpt}?rbwK{cc;XK^ITbL;Ck$o zuflQ1opD3xdFn1R=6O5Lp+j^R9w)iv{#?+avSkRlf5vlKZW`DPyuvG#8J+V_g5j4u z99pvt-X%SkC5-cz&J=@@=Z?W=rS9n3{|c+Eaf3iHALd%mgOP)_vDMnysQc6$oy$Wp zcDyG?Jh}v46VhQ;ok;&uzEIEc&ahd11E*y=z{^fjzToH!_#=y?%%g`OFZ~!#*$~4~ zbNb=-u2=EPS;^abJCJ*JvY~~xsyw{$EL05b0|B4a*=_wT;r)X+e*QFz6;7TOMmw96 z_~Sg0@vf&`8y_18ZLksYB zgB$u^QHJIlB7MwphT@jPbaCTU$h{MS|IOSEd5^=`a9a{9j(jZqJLthVkqz`Ppg-p3 z>yy^OlcIM1AyQ5p#I`N7p|e3x{2R2M+7`!iqD2VVzRHu(tUmnd-xKnCFb4L$UBLD= z>%_pHlQD1dQ}%d168G&@p>gd8c|hms_;|W1UYu)>lTz$)(A6dK zXyFSiYR&=iaRba~Sxgn{Twv+UP;Ll{!gz^sw%sa7d~ekXs}4!C@0YohD!GM!oDhUA z(S!N0VjzroVT0F>?ZM+(ui0it8hdX~#y>vOdC($LI(S`rzy2-pjc#s&hLr{|d(aGg zELX-&iidgDJr!#IR{{T?WeMAMO8EhA4?Lc@R5!T7k*AHioDqvh4J6Ghfzm3G06 zT3b{!b(J`trBEj`9_VqOYyOVrmXE%;Kz$Cr7TkoqtG{9Ysjl27 z_!6BtW(AAyjVQaWy`2yK>P_0cT<}qkF<^Q=9XIybLu+^Rz+YdBg=(LXaHu_!i)H~Q zCKymF%{Pv&hUv4<;YQO^x>zrYOXjHI=V@y6O}YnL*NTk3hD7K|S+MY;J+P?#nqL?Y9nSP?b<+47kP`H{KjekN`?IoU2dqi$_R#7}L zV*~b|+y&PUGh(&Qqo}iT2x*5{GQ5c6j`JTlT$i%IQ^jb^nferR4k>`oiy_jva|fQt zJ&9+dthj^g9^RuVr`>)LJfLnHl=m6Q(@LGC{8Nh%5Z(ZpGflA5X>*>l_7%C!TMc>| z`ygF9Q-4!;$DS><;BrwJma3_guj#RnPTtuMsROjhsj30oYmRbO*~b&5^ylN%f_;s(&lh%+9BPW!Lz}&qsV(H4F4#+~B4a&%{z?9q4CYM>Fr_ zvtL;tJl*t+1Fkj;;I)tCC+|RWbRvaR4-iie@Rpo_W%y*1lvV#|f}bqLu*S7>tSs{* z?S}Q(TA0rBB?e0JsR4p*T_8r;X4AW^7VN6r1ygovq7XKdunFaj`Vx%8>uvH64=!&G1#*6t+LoBpjXj61`LH1?z@s`1W=fIhkpeX)QEC^^#e* zH$#C#TC91mL&&HA>Nou$JiObR%(AukMe;{_>t;rNUzhNy z)IV^h%X)?x4`@Z$IBb`@eKn03d|I*5( zaj3l@Uc71Lh6m2ZaEDS&hdS?Mao^P;tWr5i@&k>*iR((K^!psH?xDs#y)Ke^t>ojN zVZ6EL6}olx1XiE;Mw#6!g_#ErlHHbjn8KczwP_}u4vN4&6_Pi$dn(z(aTp!zhi2pL zvA0?m){_Tg1`bEd1E)c=rU$CswZ)-bUW1kG4>5jo7=7+=Qp|N3%V!6-OXn1ONPD^s z4(i>2z=sRjZ*U@eS;@r_r;Om0^LSh{#DTw`(Sx)n_U!O>GZsfxitm$5u+{JazjcGn;Ux+$oU^R_-vtba!K zpQ?f6Xs&?%8n30ygBru1MGQ%6#lhkMRA0Q9RXR#czg}zPPwrJxh-=sKOzjXsT=xmo z6|>M$xeF`x{UCnc=8A*&w9=)``(+F3+@;xsCBD~crx8-mzkW+QZ=DkuL17PK7Y znE$s2Ma3E6vOHHdXzb6=wzk7Dg}%`IW&?g3c@}#QbjJ<%uF+(TUO4wrIQ?GXf$c5H zq&YoWv@xs0$Ai;Amf{adzS_9&>SMl>Q$8O5anQx%WHS^g2N!ENXaHvKrfOt|EiI@qAc&HY*k(m*f=C&T)(J zdT}SVzirMzFTV&U^Tt9AKBW2OpM+oED@gTyDu>rLa=-5#a9xwJ;5hz09aZ^^*2_EL z+Oo^+TB^;N?-$_cVQTOxy+w9A{R^$ky+_-(KY)o-^2s}DD*7#YDjpp3n)YbPdG(xq zY;(<)7C$Kf?*i%F-Y=UjFUb*3^_fF@|8~)Ub4vK2c`Zzdu7%vJ@i<;zk?lALJFFj# zBQ|$Ow?li_ORZJzVh~IUupARFY-jCfZ^&e82IXob2|JJ6Caveb-1$!=+@4q|ZXT2? zoG+eAoisP$@INE*=C4#b@;Qd7le;u~cqu!Y>@2LU9>sgM-ldz5+{oWJPrP&KBE&sS zz#-Q+-p?G=M(wI=itsMu7G4h zd+ReG+G7TeK6XvY3!jh=_ichBpX;!s>Z5$NsTa9!u`Ksb*#U$1>y?wn5)@8K_rD8$ z9i8HhATG;7cJ1OURO~*6XE{&Bz-b3Xzf27@@;S*@zgY8|?CG?2=SjSGzKC+V&cwIw zN^oh32JQ$AB`>9;a{J>G*{@xd=N2naqvagL)BrI1xRXXlU#5jEySUAHws^++9{pHu z?Qnj+2_?Q;42cF)u_;LMo82gG|9N{_z6YEX6;(_F?nPxAHk$k9D=3}|#vl4Yto{ZgwJOzt$sZf4v3k--z z!f&5P;aS-soK$p~bTj6Wi^qKUF+P$v8yfO8y%*poWjXsUiNrX|GrXtoMXtJY5&oX* z4UKC@gZ9r_X!x}Z&W;!^vs$nWt^^H-!5@v$-=iPpT}Y(+N8U;My2+TO;v&ta4YA*i z(VP`E9OD#cbLVgk?55BS4_btYYb+;W;x9+^m015X686g@HN5a~^f~^oAd<%HRDrv* z0`cRwQcMc#!P$ok;F`)<%A7VApU?XX>yjh!{LyGxEhy0|hi-7CxB+JCEaWP20n9Sa zW0TuAX=&SbD(I-r<917Pji|otH%JqIxG(2bn;-HmjX-uX?1O7&zovU5E|H2ytgLvz z6~0^22-P(gU_h5+lsi*_JA9gn@2~3Oa@#VPX)Dc}B!@b#W%9h?!*#hgIYeH}j?U54 zy6-0G1*Qrao78#bxq(9BxN-8a`oZ*7xdR{fGR4{a8vJrY!EMuOTKw`Yb+s~K%UfG< z`Q#pAACPpSFgNqly>gy!2Gg6lqWFpMjyCN%-G^Ini? z_Zzf+cQL)WdIDm@)?%@W7k=C-IVPPYSA(`AGiOuz^gtoY=8b(T#&mLCnrTv+_;%Iz~&8q ztcWKs^Xt6*`X-#`F-_d|$dFyiHsL|T9&{z?D4JZ=N5^lI#k#K_NV9Jw9^0ZOUYPS< z+|?_c4_%BD{jILR^OOy|HL@PAztp4^nU%69y{xc&>N9bcjv5xt5yd|et72?K8l?-; z?C{`NT3n$>BlaHTQq>^*Z2ShoQeQh1&N&Z{y&rJkLpkynHIC@ol^oBwjN|5qzSRA%G7Rc%&g7(l{VopTxinqeSLDPamIdM1!GpnK zpc>B_porP?j)^4!uj%aDdqTxJC+=`>Do!4A8Gc@z3}Sr`9I2Ebnm+CzH00{=Z>>9| zU-JNGuUCSj<`?i&{WDnOaYT0Ha3aM&%q6w7G@Mn;V$}IK?*8aEe0Zk`T1nUF!p9q2 zI$|xoT{8)qzkd`BHYmZ=p!4+4$qaYxijwUy$cESN6J^FbH&E;J8Vbp^x`19fudPmz39VzOfDW-Z%y47sa!!n;qThkwEd=w~33J&0&3&3U~~Y;ZUhx z@KiIM`ffi7mXkY>ea%*S$6wO}-M#X3XCL@CsZ)7<{zj&cyD@4< z1R9>t2Q$yRP&EAkb?;j%Ki|JI9e2*hTJsKUHd|r?G~L3OYX@QCpDbw9-;SCQGL|>4 zAj5sj;dxJOJp1T2)Kq<1%*48f!T6sWE!67}O z#PN-hK)aQ=S#s!3o2kWKu?{>|*_-k&^$_n4t`VD(tLYGF@qlgG>}b-5EfzedjP#SZ zL~#Z@du4}%k}bGk%m&C9`jJ9vjXAql9y~oho-)G*OHTRuSh%#xp(*DjIvx8#=at^W z^B0qN=Y1EvFyny`igV!ETT9{Q{mzhBu0zw#n&Z9ABT!9Kn(GL_X4-?`&CZ1X$v&=( z3T^wp<#+A=*ZEy)n^()kiT zFviiaccHX;=PH_eU*cD#Opv%a`$LEH@R?vincyXYEg_GZ@A9+ilt7{ZQ`t zkjYtWCC&M2QdUce=ad#x_laG^>wC4hVXG0_&u^yxm*3KDicI@I=C}UW`Q6*+6#oDG zVxmdG|1rP#zsm1+JZb+wpC7sYbJG9u^Z!@*U4w)ntlM5g-&2xkd6DGmcqYe3N2=wz zt%YKd{QyiIIG^81d8g98|0uI;CcYf`4K9Zl!>S+Walxo{+;q(fcNkWRC(jAi!LmUK{Z2-~X z18s1MLBE3$oNq0{iSj&%d6?RC8x|M_%!(iXl zNb-ptjIrq(IQ&fu`Amp~+tHIiZl8-+9R5(egAT8b-2u}S_2IL!0X&NuD}GIS0n@+q zLC;@$FnDDspJ~v-vN?aTUOoiVOdg_U#ic)$NX zP&ST&URyiz&7<2;Wu!9394v&Wcexy(La%K@WiB?m#r)+u z82a6a4QnRizkWOM$|Egs?Ky~DRouA3C5etLcI7WY!>H}+HSlri$ttm@p!cEycE~xfamR~b^33!Q+NAF*;CmswdC*KX*#;Q?R-0dX z0iAm`mm0=*6BZ|r#5YEwc-?agd<_fbw;uC&-|uN?J$?gf<{Y3|EfZn!3=*BzsdBGn z`B)lxo)esUg8KeLG$Um>*gqJ__mAsJ^SDsiXZaD~x1Dl%)PG%gsP7`oU9EtR-|up0 z=<`KPU0p>hooeWT`Y%z6Vlx<99uzVHKf7Vd>n|YIRSWl4=Hk81MVzrL18wcQK-pJ!yrlhvG!i{1YVl(H*uA50Wu4oDDj(C+<8sI zCH}zWEG>?`=%g5%eXXuuBzYA#rd0jK)$ zx%k`g>g{2;s{bE$fAv`$?H2{^zg~#9*Sqri?gluk$%%SjIl)#N>>Q19p7W+FTJZT^ zZ@yyC6<0JR(}bbQj=eoY_;0i!7k`@~w5V;Oo?{fsUnyC!fzS+}Z9Czy`dfMbZ_i&4w)}`c#m|)TIX;Ix@iyRPJf5Iz5V#>@&wQ{JPcodbz$56 zH>q~!MW|`2a3megJWID!Uc>U9_vuyja4N0shHLACscW1h zXB_SXV|UyXPP*mMQP0!-vDaFjo@6QcCTzu~5m_krE~k+Or=X@>>J`rk2lZk5Xk*PS z+`UbcEP56RH@?T9)^2BfRdkUqt?dsM3QKvpyB(*d=^#~i|6bI4;h>2{HER&d$E3MekS3v*)ULZ@*7d|^l~g=T25 zNB8-{)Y>fkvtc9$xn@h}=gv^Q;Ww=9{|rv<%Mref4k4?Xra1RUin#7Ui}-BrF~p{5 z@x}MAaCh%n=-YD_AJG^}|8*aLN6IAjZp%(?`+O9vdggPN)EuZAR4s&W))NWra`h8j|Wgid;}a! zk>;-b?BGe=Y2H0}4+P4mW3Fv9?C9%{i~Ep}r*aFN3u0KeW*mL?*1)Bz1B7NVqO7N0 zD?~1Aqchs-{AH36)M}?=*1BT2IXjN3f^^w~lEHuW6q+vGqq4OsMD+BW3LGcka5tG7kVEdxlRvQ z#KU0mHyNzG7%2HuQh3p`zOV%o=u_AtMAua4q@l;IHx=>lWmVB-;RR{08v>cuS_e zm?5U^SWNL9ACY~cHmvi!FQ~s($1#VD*}Aeo=y~}Q3{dKg%1;iC=<&^m9;tVwlo#C{ z{aj?Y%=!X#K9xk4leKv3?S5GCawZ%Iw}+nxLt(MoNG_Xd4%x9oc&uX^eaVv;mK9bw zwrMs$=n%mn+t*WPb5Cxz+FBOtkizQ>EO64Kd(`vd0qXzO3^TWk7gBofMNd}^A*L;u z`}(ipD!nv1-Rj2+GWF!6?s{X=rr5GUe2A`I?kDV&@~z)X`rs`cRqD23F06Y{2*uKV zr0COrJTlxChR>*?BE_*h((WAX`d9#u2X5lcvl94<>XpJI{U#xE>t0%WM-_|SAA!fmgK_4AliV;A z#6G_U!OMr`EUev$o2&Ktn9on)&XP!|jOvFgbu|R7(2w-yXdETZuHfvJScw;&0q5tf zr8NH;!eakQ4l@`A>BiTE!oB_IU3kax{Q7zHwfu&d^D_pXI!Vs>l_lbj?*cnWJT~=5 z_V~0<2%Pw0k5ewW(yt+s`y)R<`05bPA0G@9JD=W77PqI+ll)@nFin}W<{*vOHJE;0 ziWiKfow|YlZeg7e%CeW4n4dQe4Q725+z!Q)T}vv(J@bRLURT5ykwY-M>I|;ds3n&% z7BuaX9Vuk1aN{0bY}TBPcAbmS{ErE1N<8tZ**oFE^^Gu2x1M~a4xt`_9nf!?0xp$i z@|(Y-%+ORz#_p*=b`5wq=nrVw&K3tpA0VZaAvn;^1XdiB+_2BA5js22@>3}k6fuCS ze~jfH7K-r2OxUr#}skWdb4uHX?r<5>Imo;)ru=EyyVGRlJ7|zOFw3M;s=9KP;Z(fYF^R8 z=SL(D?d+A@{MJg-sT$o>CVK}oTVaA0MXs6NV? zSGON0%aSls{Fp($t$W}`%Y4MAz2QYyEj(dzj*<^ugLul23*EXv^u9At+@On9LtWWC zFdV0ydkoGAhxx*fGHjfA6Z~U-IQ&>0fFASA;aGyeU)PTSvsP7HclduXa;IGOangS>yUZ-1>52+oJ*0Lln6|ZX&;b z^}W!cG?ydNE>L^#wX|-A9>T*NAh*5%<&(a{yy6zPGDe@zsn=2LymH);(j+(Otiq4m zPSS(KTli@L!MJWq>B-Ia^!#%U)vxU;xIG>QZ4NhhYl{i5zhi^PZbxu)p&y!_iRb*= zhuAy~$@!`tFTD4c>~~BnGo$m+w&@0_c#Ocf>)uf2<%l-6-T1U1J*9j*6Mw2~KS-aA04-@DO*p4L2Tn69`m?>u`AQo;LI^6^pqFLCF>YkcD1 zX1bK$i~LgW(c$G&aK-_34hiVO#ohZ;mxmu{QvH71((s0w=|A)}h{24z2Y8@Wt8hHz zFLdX8D%?`d+NXZP0RN%vwPYtuIMYgVb^F8N>5H(<#+|3dPo}t*V`QJSnxps%9I(3s z`^Xh`yZ00oV>%*C^5tXUQWhm$x*yKHC+yO5;@)QIvR~yoaM(Ljw)D>z`lj5Q;||}W z55H9)y}|$Kd?x5Xk8BhKApN2MHdoaJnu8ZC#I%zPgQgMI{H`dHT|jC0lum z!BKI;(LUm*%jv@2=2ha=vAUf5N~Iypod`_Zk_W|kH-nUlF)blP0{{=#2>JAlvv(H+;%_% zeZGg&rUe#gALYy?hDC66R*~4T$4>6Frz2DZ=0mLZPV8Injf(3Y@})U-^et@;M>|@x z>&iEx)5t73{q+|q=(tcWb-+coyI@41!rf?%Uda6ca|)A+8pi?-U>m-AnK;fmA3jK(fQ;!I+0ril8`NMe^Mb!3mevvztN_NBalK{g&zCkTwO06ph~89r296 zGjMB_)X6*SC&XP1;gI7!@k;R#@++E8rb>>q%vH@%K0Jqhh8&`Wss3Wq`a)T!UUA}> zf>69+okqQyYDJSCJJ80>2iAJ1a;$e3oOU;uvLwFM@xoxpAAFV$7g&jH_A5~Hj>IYJ ztKisww-P2xdseM+vJuy0J#b~4GdDNR=7x4tghj?QqUy3(qHM(SwN2{AYG7IJ8y1B|eEwCHdK>S=?2EVQR_tWD0+Y&r){BtXVGiEm35W5b>UnA+<-?HjunjK`PYlTXr4C#FEKypo3g zwIjfzdYd39tRkXPC8Wl$@f|++>u-j5UF-4l&nAshrmc@bS+VTvFjT%7T>o@L5 zpU&@&9Hm*&=6p!?F6mY}VaD*a@L*C8eAuMQ^V`z!4IG0hfp*fKS;_J8wUyGG#t5}z zQ}D;OUxJNQ7l_(@RN~BaYQ8;Fh4V9A`mpq`bE zZJzr0eRi<;Z^pMfR?w}uD8hH=StCrHd8WU3JF&F#Bs$-SLI65i+kM1W+?46s> zDcqtF)+zPGT|XUH&Q#{K0`Zpb3^v%)<{LW~QroRiNeXew_3Tc?b)1OSoi$6v~oj4dr|Fz-1F$6vS zs}`bDmGN%UVc;bpXnMev8ebW5ZqMBqGhWBBG-(~&7`u{Z?&!f4Z-%15`!Vc2Mj3zn z4B^{->cKJN0>uAYMI*9HNHbU3?G{ZZEIB<77*Do5( zJ`o1c`m9*6T;eKDHH;K(wDyXY@4kxPlr}@3?ZMC^+)G$waujE*$ztCH--IV&i=a!7 zPONZqE`JC(OA6*jJoVEIpfS~KsB?q1JzL0YyOglLWg;p?59N_}$8w5)JVkvhel(0bJ#p#j7u)FjjJ^}yRbw%ubL=*D1Tx1i4PLB0UOjtj9 zm9Y9@0@RF&g36DlaDC5A46_*wjygk0?y`jb=A4zi!z$i3@-#<=0v*$CBtI8tP(Ndc zL9PSo@QG0{_pK*?`yC2Dq72BlIhx1UAIH<(GsVthbzsZPmAG<>5_VXn#|gnE8E98P_*%s6^c7hbQg1G*QE3CE@xiwet(ao9Rb zR-F5a=3L*)EiU^=z2iwZ%SU;u#l{GCV29>K9Mx?qba;}2x6ZB4&+o&2iXByJi$#Jv%a{(2e)ZF;QuS_Ou(t?+CF{^ zi4dWZNC^>19w5=}%zX_jU}B4kRELK;LwLW;c>QE4EA2F+=Zl=3u>-`4xS z-|KWZ&-eA5u7%FUZ{K_E{oiM=weR)6O{&T&l!IZ_M+C%z2I!Ms^`p?@lW4 z)e<5pz6&P@W`gX_cBW|AbP{TpJ|@WK&T{7sKc9-vEV21S*H4c&#m z`ktfYUME5B*iw|YUx+j-abR|x0L-x#Chh?dW$|NFAx85UnAl%KHN&6b*LEqCS4%a% zkQrY3qMpq+tX5*zBCAjm4;88LcKav^1zW6V{}i3_PeImNp&;gSo9QZMb1alufAWoz zB)r-N4zPvR3=daO{%gg_{V|H9rjy2@Leug5=|}J=w$9(ykmk}lHm^J*x&R;WSr5jv zF&w@#60f!zUp_B$Fd0GRK%15|x}SRq*Y7vCXqJt^V)vXNM`AnCe-la~Ej<}a3u9>Y ziYA`B%BXlL5xm*$DBiELh!R)dhOmSw-f}VuEzBKK9<4gGjIA1smjt`OQQs~`)94^N zwmt(tS4$w4PCH2Jv~l3-E{p@C*!zd}V{oD>1E;aRk2U9W@l9nt*7sf(KhcPTiD~s@ z?u_ARWN85FZJLSwXV}5nj<+cArFt22Op91V$y57H))DWPY!V_JjAsr?0RyvQtgyfh z#|jNYQ-ae_SJX57Eo}erH+(LA_5D+9&O`Gmq$UsA8%=HOL9 zhmnDoG&wf+EVjxNf`ZBlbU9TA>z^7#UMp7P$YCYS$4we!vtc$MZ_(D+4`=N96Z%TIR&Ob%jtJG?qM8y8Kj95 z?9xePN&?xRV93__vj-D(0ebzW19<8*Tm1R$HLUeCAH1K(;Eke>af4biO4#Is_dW3> zO_`6GjWd?vtfkAy1lQwOS9=58uJc2WD(zsR!dddpO@@5=;>g5Eg~P*6D`YcdEmoN# zi>FF7V>wHG=6u`*RL{=8tch)d?80?$La`Coc3P9U!!Du=i7OeCD_W4lUVoAJJbXQM z6%pIF7ZhNMbe3{-MN@UH7esM8B)|Sgefe$7_`(`2H?JO|b@#TJXvQ(!w#XpHR@EdC`Iw}=+vV6kIN`~wPhiA^HSs~S*AYc$@d5J!%_ z99I7BMh>Z4Cks)d^TB>xG(H{}fva^RkwE%mH08Q0xo*DCt4m#q@q2Fa@k&j)M{E7y~0aq8(R#d#n*8ybO$o^=Ch%2&ZJ>O@PRD!PCS3G&|a^&03 z4f59S!uO68W3fgV{G6@u_k5l;XddYgN^6stlQ!qj+%54aIy4k_>QkM8WeFD2L ze22X^MZnsxQPkb-snE)L4YpnPf%#qop>ne=OjpEM!BrjZXpRKx$WJt#eu_Nv73nQ; z+W0c-z%97=ax6f#s47wi}fMx_Ol5S-%nJA944z5 zYobuwT0AQ#A0IPY0Nb6a*xKyxLG4ilf@}TA^Z9ehw-pI^tx*hm+ix9unkxkUPra#5 zt@Ws}FcXV}8R5~6k8sFIb5t>}5wG`CLmp4JvAL=J;dS*|DxmoZw&D2V+iVS*3$2w@ zMA8lmX}uP+4#6^HEYcsXSrrTN8pW(9?l)|IMuj+C*$kggI^l0;l2{+~?@tMV;& zZIv-K(=&nzHk^m^oEK0!+dY}hV>PhdArTxV5RLbod4^7n(!wdbwb4jBE4)8T0_%M> z!j3-$(aHBUjMbW1%#YNk=(Kt+ew8bO^}0&%{7-L~LAgUL)-969iB~_OWwE(bP=gBt z>%+@F?fA(|oPUw=9JiBla2CXmw(LQ%CJ{(hDh?G}%A+?oMSkV4P0H#&>hJT{hH=dL z^!{Vh|BU|s-h=zxE?{XeQ-{g(Ek z=i9BPhq@)v5(@*_L~bv7bfq_~yVj1**4afHZ9w!x-C){r++uo^)hzlzcql#Zc^F+Z zZz)~h6dZ=bn4XI z^wOdL+TesWeOJ7m=v05&j9Egq zXExBIM|;q-8-nSE*%owC`fVdG-2e@jkWTlV+*Rp6NRt=~V+-yIOQ4C8kz z;OA#=wCwN#ta2+2Tl*1g-u#SN&(>3JcrghFhknB40pfV7j0jP=vX`peb%1hN{}<(+ zun6CDIE7n}%3-S|6?mL>1-fVwh0D`-TQpv3VlH@P;o);$W0b#;sdYI)J=B|rp4hQ@ zLe;KN>B)|Xn--w3CGVO0_vfR`DWcfNsu3^5uTfPm>StPupBnC{@UJc1Bu(K+2~N)S)6J`Ls0S}*b_03)GCHzpZo8a5BiPpPI^D= zx;+OaC!C<K%O(WT2a`@qwnMml31WIzqAaaxZATC`KD%Z@Vj53v> ze03f`;mZQBGZX?JAv>@_}aq{`NWd4uPFf6O?oErsBWd&nG(0)4e7jPcSCGEHR^ z>|8T|q*|ymbzc)02Vn&Ym0ZHsN=L|uQe{#cU(W1U_moi<7ziyVK9)+2yh1`wtblWp z_sEg(C_Jbj1RUP5-jnw{z|l+r%Z>D;Yyt~0bMOh9|KN_bW7qsfK^LZHq1Bpy8@4+gDU(8qfdalmv*92ThuIuUB{`tniKzC#VR2))O} zV;$hbOLl!;$6eeZ+RWBej3(vPGl?-fR+txRL^jMkiq}_&6RKuE8eZW<{718Mto?1V z`-~@y!xT1$#J@jeL~I39h9Oq$oL8+>F*(y&2V*)ANnBTm$2EUOow}L0->|#jTK^u} zJ?6kMp==yJET61tvmmFPlCk<8ij0@hCDQt3l+FrUSm0iRmfKAufeV*nYppiVhQW_830+&I3V0+3K!+)k`uyy;KJyjE1`ZQsWJPPfBDU*Wv4kp98P!p zGyiQPHh2Bc?jMEs54CU}Yly1tClL+yc}(~9d~)Ezq0($P-Hl&78hu~S+51`5|I-cG zg0R+K*B^!XZPedifB5-NvzNd>@Sk43{m!q?|CSM|3V*+Se*V+Oe{}zMzdicb_J>|~ zJoNY5=ePgt<@XQvUr=5Rwv`qTl_Qa8Y5P~S)Mf{SrtBuK>N4=7yJ~n!(iL_M*j702 zJg4mOUP0J>ja?u5<_x*7`4bH<3n9LP1JR8k!H^I&1AK;$LMsOMCqa>4sV<>9^hK#1 z;zd~dV?TH7r=3S8Q<7|LMFCLnq)7h$FCbcY9zqs!;Je@mI2TjPs3qq?Nc2gFbP%WT-d1A8P})V{c)Bh#N@uayj1O z8${wmGGJ)+K3sJy2OG^lkIWb6kmGnUS|Y!Tt$U(LMktry*BgC6^qfA`Vo-sn31>l+ z_*&$)VLKD`@itoXa0(237YCanRcM9py5uxRolMOy&p=TMBLNn|#?;SPeXAz7^&%xQw+2;O=XQ?1H zOYAS@Or3fX4fpQlqtc+m&@o~NSvDpG)TB~KP@FWHl~GJ4T3JEOlU2lW>>ea_`z^A* z!6CPD9#VVRHLBOvB!P$2D7Kzq8xyanhwHi|>A?!uVe6JWlzO-dk2v@NKfb>bX07^( z{Z}%`rR*wNpDs_%wsjE!dDiE^Fbu@w2wr?n4TSt`NyV6p_|x7UNNg%0*j3A4||8*hn`Z!^zraiPi1!zIr&jhCUf{=Kn-{gI!^*)<_uV@DSGbvqiJq&0*@z(6W}B`f$Ox8OFWV zg0yKVY%b4gNSK@kL)x$6iw>rEz^lsmwAwLkK;*-u@jl4 zu1L1Vt%IRgH?TD{?@-^=%g|TPL6rER+33Of3E;Y@1B8T&aognrvSe;HiG3PMMtC-1 z^Wp~ldg&%|r`n7Zx@KUx>m`(?Xgq{!xmaKe_Bxawhwc9gCs_sO%f?@Q!hHTFT<#Ci{lrn7#SpPo_Y3*td(WDR;L>IrvtjEAgwhM-{TjTii=0PmPC zus_A2WkOttEDRwnmJ#I1&mFKzMUM4;$wSh*14!nq7Ba-=KBLpNm-y}4$i(|tgP@K* z#7OT$`RYeN!F&iw`yxs{KV3}T`FCL@X;tQvcQlzSK8U!Tlpq;9I*D5F4UE@p$8&Fv zCY{&U5qIg$=(Kkd8DEqN$IS%j7xz=4?zA%fvQV5ZSuRe-`OL$Ys2@1h%bzGYe@ngVdW)4jqZI3TA+5q9oat76*^U zYk-~mTb!mMjFs&YA`q^{L2A z{EmetcNki9ehLS(mf%5Ak#n`?mf}D%0MSm??P6VU8kWg}eWV0%l zc-&aat}VVu?Uc46NvfOC9O*)Qqq7Q47&QP^KqAi3^Cjjl9bvM{L*N|Pg;x%$#SQ6( z76l`lQEpXwS-+F}s0DZrY7SU|u9a%DYax9=IzSvYp56xT7j}>vVSnMIR1dg4m0hP= zR?M#3Fab}OFi1Ig5=E7MVkApHpwEg2a8bfb$kkD%PZwM!oaB1)B(4!}ZfYQOPBF93 z)tITRPo)k%mn+}uF9<0^Y|2+1`cS6&H6EW0m_dX`3@18Y^B7g1A1E&|1<*9dnG1Xyw7ZB}eg{SsP){3s3m2EL^T6)J`OXpJTVQBV?LKD%9H)1OA*u zYS-<8rhO8y_dzLC2UuXojK>hWlE%5Uvx(ghC-i|`%O9R&$3AbCAWr9X$Y6~aXd8JR zuHLSN7^8I3I9e9+RA!UHv@6VubYJqi#tZjfY|orJ%PGGd{(wpKHA9jmi-^tzY4lf; zCez0Hgncr-M`Wr+p?{DVn0AH`(c8npe6=`M4_^$lNhIsyiXgfvZ z+|CiJ^B$xzpbefR%fMp$Qqb?ZOjOkSmDeo2Knxbk&&ck$-1Ml8zaesz!c&QR5E90_)?$F{!5d;I3# z+2Q9O&c8pnyyMsSXTnj@mVY+>VTYW5F#b_-7W=jRHPVhx{(k#?jQ;|>)~xkix#Cae zpA80dAAyFnC9?(D1JK?2J_4GYF};nrb$*p!TI;pOcafj(rbYg2ByQ}ZUaQgnmjU>A zJ!AG|Xr^ysW@|Y`n37zOm7GQ!`_;@p|1+Dz;2~tHDOt-5hNN zx`XA2igthHaAsfHAo0^yh|}GM|Ld(DU;C&TrL$_8-|wI>)H+Or{nPHI*aN}gOt%y2 zZTCGk`g^Thv)F6R?2BN@X@_0| z`h{UTjJt!Xw2sRE=LA#uZz$|f^?UeWc*5!2ZhQJy9BeGy9H%5c9G-a0I%iMlBc6Od z9R1(-E8Wppt#^*oNj@Bd-#O@|=KmJJ;@+<&ejLN!Ij#O7692iFadVss_{3xMJE!6J{5Yn+bCw6|sB-k#(~fPw^-npU zc+7t1c%NN4;y>|l%cA~m-yELP=QwA=_y1si3#W?@hDR1B7x}*LozujR!y^lFV6I&6 z9H+nd#KW^+E7C9T>HRcp;>Y2U#e)u|@ZRHbWaZMwbyZiNM;86JRQmMJY2wG>kwr{J zt$yzuCjmb3@W`Uf%er6hoF;x89$AQ%)}HO1<0Qx@9v)fP4YoaA!K$LWai51x{5U+a zkWDx=(31V^>TcY!kQVMc9-hMojZtg?@Zs>tqGuUWZjK!Lc%U!28}P`YXE{-BP9i@Jk1TqY1?A?*9p)1ck1TqY z_vDsEA|DRV;p$nX^S5x0^c{{Nk1Tpt+vMgX^5gKxqG#nvZjRhhKJoC#qGtt0ZjNIn z9}bT!dOE3bbK;Ki;qdfxPe(Csjx?L4)7NQe$Ri8E97#fQTqi=J6$+_I2<*Eff!n|mgYafg%055psio=ITb z969#ps;|Rt#3PHI$z0r=M1C9|S@cZG;^xSG;1dszEP5s=adQ&+ad>3WGf{_|Q~!}q zJUqv%XR-}9NBT1#4v#E)CJu0O68Uj>WYP11KQ~93y*cmmxbw)O=Rh$Zjp6Czo>NZ0bx!Y?e@}A#J`U_TDWu8XX#V#lBiU;>y-!y;+KPy1|NC!82#xz+ x%7lOa^q=pJGXHwx3=8jmv47XU`@87KfBj$J*L(fgyKfF>0{gG~_y6^~{{t9cqZa@G literal 0 HcmV?d00001 diff --git a/onnx4deeploy/models/pytorch_models/sleep_convit/qsleep_convit.py b/onnx4deeploy/models/pytorch_models/sleep_convit/qsleep_convit.py new file mode 100644 index 0000000..fac26b4 --- /dev/null +++ b/onnx4deeploy/models/pytorch_models/sleep_convit/qsleep_convit.py @@ -0,0 +1,358 @@ +""" This is a ViT model fur sleep staging / sleep stage classification """ + +import torch +import torch.nn as nn +from einops import rearrange +import math +from collections import OrderedDict +from functools import partial +import numpy as np +import brevitas.nn as qnn +from brevitas.inject.enum import FloatToIntImplType +from brevitas.quant.scaled_int import ( + Int8ActPerTensorFloat, + Int32Bias, + Int8WeightPerChannelFloat, + Int8WeightPerTensorFloat, + Uint8ActPerTensorFloat) + +from brevitas.quant_tensor import QuantTensor + +# local imports + +import preprocessing.sleep.hyperparams as cfg + + +from DeepQuant.DeepQuant.ExportBrevitas import exportBrevitas +from models.qlayers import IntGELU + + + +class Int8ActStochasticPerTensorFloat(Int8ActPerTensorFloat): + float_to_int_impl_type=FloatToIntImplType.STOCHASTIC_ROUND + +class Int8WeightStochasticPerTensorFloat(Int8WeightPerTensorFloat): + float_to_int_impl_type=FloatToIntImplType.STOCHASTIC_ROUND + +class Int8WeightStochasticPerChannelFloat(Int8WeightPerChannelFloat): + float_to_int_impl_type=FloatToIntImplType.STOCHASTIC_ROUND + +class Int32BiasStochastic(Int32Bias): + float_to_int_impl_type=FloatToIntImplType.STOCHASTIC_ROUND + +class Uint8StochasticActPerTensorFloat(Uint8ActPerTensorFloat): + float_to_int_impl_type=FloatToIntImplType.STOCHASTIC_ROUND + +convAndLinQuantParamsNoOutputQuant = { + "weight_bit_width": 8, + "bias_quant": Int32Bias, + "input_quant": Int8ActPerTensorFloat, + # "weight_quant": Int8WeightPerChannelFloat, + "weight_quant": Int8WeightPerChannelFloat, + "output_quant": None, + "return_quant_tensor": False, +} + +convAndLinQuantParams = { + "weight_bit_width": 8, + "bias_quant": Int32Bias, + "input_quant": Int8ActPerTensorFloat, + "weight_quant": Int8WeightPerChannelFloat, + "output_quant": Int8ActPerTensorFloat, + "return_quant_tensor": True, +} + +convAndLinQuantParamsNoInputQuant = { + "weight_bit_width": 8, + "output_bit_width": 8, + "bias_quant": Int32Bias, + "input_quant": None, + "weight_quant": Int8WeightPerChannelFloat, + "output_quant": Int8ActPerTensorFloat, + "return_quant_tensor": True, +} + +convAndLinQuantParamsNoInputNoOutputQuant = { + "weight_bit_width": 8, + "bias_quant": Int32Bias, + "input_quant": None, + # "weight_quant": Int8WeightPerChannelFloat, + "weight_quant": Int8WeightPerChannelFloat, + "output_quant": None, + "return_quant_tensor": False, +} + +actQuantParams = { + "act_quant": Int8ActPerTensorFloat, + "return_quant_tensor": True, + "bit_width": 8 +} + + +mhaQuantParams = { + "in_proj_input_quant":Int8ActPerTensorFloat, + "in_proj_weight_quant":Int8WeightPerChannelFloat, + "in_proj_bias_quant":Int32Bias, + "attn_output_weights_quant":Uint8ActPerTensorFloat, + "q_scaled_quant":Int8ActPerTensorFloat, + "k_transposed_quant":Int8ActPerTensorFloat, + "v_quant":Int8ActPerTensorFloat, + "out_proj_input_quant":Int8ActPerTensorFloat, + "out_proj_weight_quant":Int8WeightPerChannelFloat, + "out_proj_bias_quant":Int32Bias, + "out_proj_output_quant":Int8ActPerTensorFloat, + "return_quant_tensor":True +} + +class MLPHead(nn.Module): + def __init__(self, dim, hidden_dim, dropout_rate=0.0): + super(MLPHead, self).__init__() + self.ff1 = qnn.QuantLinear(dim, hidden_dim, **convAndLinQuantParams) + self.act = nn.GELU() + self.dropout1 = nn.Dropout(p=dropout_rate) + self.ff2 = qnn.QuantLinear(hidden_dim, dim, **convAndLinQuantParams) + self.dropout2 = nn.Dropout(p=dropout_rate) + + def forward(self, x): + # input dim = encoder dim = 48 + x = self.ff1(x) + x = self.act(x) + x = self.dropout1(x) + x = self.ff2(x) + x = self.dropout2(x) + return x + +class Encoder(nn.Module): + def __init__(self, + embed_dim, + nheads, + att_dropout, + mlp_head_hidden_dim, + mlp_head_dropout): + super(Encoder, self).__init__() + self.ln_1 = nn.LayerNorm(embed_dim) + self.mha = qnn.QuantMultiheadAttention(embed_dim, + nheads, + dropout=att_dropout, + batch_first=True, + packed_in_proj=True, + **mhaQuantParams) + self.ln_2 = nn.LayerNorm(embed_dim) + self.ff = MLPHead(embed_dim, + hidden_dim=mlp_head_hidden_dim, + dropout_rate=mlp_head_dropout) + self.rescale_residual1 = qnn.QuantIdentity(**actQuantParams) + self.rescale_residual2 = qnn.QuantIdentity(**actQuantParams) + # self.residual1 = qnn.QuantEltwiseAdd(**actQuantParams) + # self.residual2 = qnn.QuantEltwiseAdd(**actQuantParams) + + + def forward(self, x): + # save x for residual + _x = x + x = self.ln_1(x) + x, _ = self.mha(x, x, x, need_weights=False) + # apply residual + x = self.rescale_residual1(x) + _x = self.rescale_residual1(_x) + x = x + _x + _x = x + x = self.ln_2(x) + x = self.ff(x) + # apply residual + x = self.rescale_residual2(x) + _x = self.rescale_residual2(_x) + x = x + _x + return x + +class QConvBranch(nn.Module): + def __init__(self, in_channels=1, out_channels=16, + kernel_size=25, stride=4, pool_kernel=4): + """ + CNN branch for multi-scale feature extraction. + + Args: + in_channels (int): Number of input channels. + out_channels (int): Total number of output channels across all branches. + kernel_sizes (tuple): Kernel sizes for the three branches. + stride (int): Stride for downsampling in convolutional layers. + pool_kernel (int): Kernel size for pooling layers. + """ + super(QConvBranch, self).__init__() + + + # Branch 1: Kernel size 25 + self.conv1 = qnn.QuantConv1d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + padding=kernel_size // 2, + bias=False, + **convAndLinQuantParams) + self.relu1 = qnn.QuantReLU(bit_width=8, return_quant_tensor=True) + self.avg_pool = nn.AvgPool1d(kernel_size=pool_kernel, stride=pool_kernel) + + self.conv2 = qnn.QuantConv1d( + in_channels=out_channels, + out_channels=out_channels, + kernel_size=3, + stride=2, + padding=1, + bias=False, + **convAndLinQuantParams) + self.relu2 = qnn.QuantReLU(bit_width=8, return_quant_tensor=True) + + def forward(self, x): + # Input shape: (batch_size, channels, sequence_length) + x = self.conv1(x) + x = self.relu1(x) + x = self.avg_pool(x) + x = self.conv2(x) + x = self.relu2(x) + return x + +def dump_txt(path, arr, scale=None): + # arr: numpy array + with open(path, "w") as f: + f.write(f"# shape: {arr.shape}\n") + if scale is not None: + f.write(f"# scale: {scale}\n") + flat = arr.reshape(-1) + for v in flat: + f.write(f"{int(v)} ") + f.write("\n") + +class ConvStem(nn.Module): + def __init__(self, in_channels=1, out_channels=48, kernel_sizes=(25, 100, 200), stride=4, pool_kernel=4): + """ + Multi-Scale Convolutional Stem for feature extraction and downsampling. + + Args: + in_channels (int): Number of input channels. + out_channels (int): Total number of output channels across all branches. + kernel_sizes (tuple): Kernel sizes for the three branches. + stride (int): Stride for downsampling in convolutional layers. + pool_kernel (int): Kernel size for pooling layers. + """ + super(ConvStem, self).__init__() + # Divide the total output channels equally across the branches + branch_out_channels = out_channels // 3 + + # Branch 1: Kernel size 25 + self.branch1 = QConvBranch(in_channels=in_channels, + out_channels=branch_out_channels, + kernel_size=kernel_sizes[0], + stride=stride, + pool_kernel=pool_kernel) + + self.branch2 = QConvBranch(in_channels=in_channels, + out_channels=branch_out_channels, + kernel_size=kernel_sizes[1], + stride=stride, + pool_kernel=pool_kernel) + + self.branch3 = QConvBranch(in_channels=in_channels, + out_channels=branch_out_channels, + kernel_size=kernel_sizes[2], + stride=stride, + pool_kernel=pool_kernel) + + self.cat_rescale = qnn.QuantIdentity(**actQuantParams) + + def forward(self, x): + # Input shape: (batch_size, channels, sequence_length) + x1 = self.branch1(x) # Output from branch 1 + x2 = self.branch2(x) # Output from branch 2 + x3 = self.branch3(x) # Output from branch 3 + x1 = self.cat_rescale(x1) + x2 = self.cat_rescale(x2) + x3 = self.cat_rescale(x3) + + x = torch.cat((x1, x2, x3), dim=1) + # x = torch.cat((x1.tensor, x2.tensor, x3.tensor), dim=1) + # x = self.cat_rescale(x) + # scaling factors are different for each branch + # How does quantized contatenation work in brevitas? + return x + +class QSleepConViT(nn.Module): + """Vision Transformer + A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale` - + https://arxiv.org/abs/2010.11929 + """ + + def __init__( + self, + config:dict): + # img_size=224, + # patch_size=16, + # in_chans=3, + # num_classes=1000, + # embed_dim=768, + # depth=12, + # num_heads=12, + # mlp_ratio=4.0, + # qkv_bias=True, + # qk_scale=None, + # representation_size=None, + # drop_rate=0.0, + # attn_drop_rate=0.0, + # drop_path_rate=0.0, + # norm_layer=None): + super().__init__() + self.num_classes = config["num_classes"] + self.model_dim = config["model_dim"] + self.inputQuant = qnn.QuantIdentity(**actQuantParams) + + self.conv_stem = ConvStem(in_channels=1, + out_channels=self.model_dim, + kernel_sizes=(25, 100, 200), + stride=4) + + num_patches = config["num_patches"] + + self.cls_token = nn.Parameter(torch.zeros(1, 1, self.model_dim)) + self.pos_embed = nn.Parameter( + torch.zeros(1, num_patches + 1, self.model_dim)) + + self.pos_drop = nn.Dropout(p=config["attention_dropout"]) + + self.qaddpos = qnn.QuantIdentity(**actQuantParams) + + self.encoder = Encoder( + self.model_dim, + config["num_heads"], + config["attention_dropout"], + config["mlp_head_hidden_dim"], + config["encoder_ff_dropout"]) + self.norm = nn.LayerNorm(self.model_dim, eps =1e-6) + self.rescale_norm = qnn.QuantIdentity(**actQuantParams) + + self.head = qnn.QuantLinear(self.model_dim, self.num_classes, **convAndLinQuantParams) + + def forward_features(self, x): + B = x.shape[0] + x = x.permute(0, 2, 1) # Transpose to B x C x N for Conv stem + x = self.conv_stem(x) + + x = x.permute(0, 2, 1) # transpose back to B x N x C + cls_tokens = self.cls_token.expand( + B, -1, -1 + ) # stole cls_tokens impl from Phil Wang, thanks + cls_tokens = self.qaddpos(cls_tokens) + x = self.qaddpos(x) + x_cls = torch.cat((cls_tokens, x), dim=1) + pos = self.qaddpos(self.pos_embed) + x_pos = x_cls + pos + x = self.encoder(x_pos) + x = self.norm(x) + x = x[:, 0, :] # Select the class token + x = self.rescale_norm(x) + return x + + def forward(self, x): + x = self.inputQuant(x) + x = self.forward_features(x) + x = self.head(x) + return x diff --git a/onnx4deeploy/models/pytorch_models/sleep_convit/qsleep_convit_scales.json b/onnx4deeploy/models/pytorch_models/sleep_convit/qsleep_convit_scales.json new file mode 100644 index 0000000..8e82496 --- /dev/null +++ b/onnx4deeploy/models/pytorch_models/sleep_convit/qsleep_convit_scales.json @@ -0,0 +1,1764 @@ +{ + "cls_token_quant": 0.017837218940258026, + "pos_embed_quant": 0.017837218940258026, + "inputQuant.act_quant": 0.04484846442937851, + "inputQuant.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "conv_stem.branch1.conv1.weight_quant": [ + [ + [ + 0.0075483969412744045 + ] + ], + [ + [ + 0.0044861845672130585 + ] + ], + [ + [ + 0.004401240032166243 + ] + ], + [ + [ + 0.007736503146588802 + ] + ], + [ + [ + 0.007054155692458153 + ] + ], + [ + [ + 0.005122110713273287 + ] + ], + [ + [ + 0.0062958234921097755 + ] + ], + [ + [ + 0.005045303609222174 + ] + ], + [ + [ + 0.00552830146625638 + ] + ], + [ + [ + 0.005115997511893511 + ] + ], + [ + [ + 0.004752025008201599 + ] + ], + [ + [ + 0.004465884529054165 + ] + ], + [ + [ + 0.005070246756076813 + ] + ], + [ + [ + 0.0032785432413220406 + ] + ], + [ + [ + 0.005612294655293226 + ] + ], + [ + [ + 0.0062989769503474236 + ] + ] + ], + "conv_stem.branch1.conv1.input_quant": 0.04290647804737091, + "conv_stem.branch1.conv1.output_quant": 0.053983174264431, + "conv_stem.branch1.conv1.input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "conv_stem.branch1.conv1.output_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "conv_stem.branch1.relu1.act_quant": 0.02690162882208824, + "conv_stem.branch1.relu1.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "conv_stem.branch1.conv2.weight_quant": [ + [ + [ + 0.0042135147377848625 + ] + ], + [ + [ + 0.004618746694177389 + ] + ], + [ + [ + 0.004535664338618517 + ] + ], + [ + [ + 0.004121930338442326 + ] + ], + [ + [ + 0.0037126648239791393 + ] + ], + [ + [ + 0.004893876612186432 + ] + ], + [ + [ + 0.003665776690468192 + ] + ], + [ + [ + 0.0035946075804531574 + ] + ], + [ + [ + 0.0036154002882540226 + ] + ], + [ + [ + 0.0038241660222411156 + ] + ], + [ + [ + 0.003009839216247201 + ] + ], + [ + [ + 0.004207472316920757 + ] + ], + [ + [ + 0.0036023142747581005 + ] + ], + [ + [ + 0.0042788307182490826 + ] + ], + [ + [ + 0.004918133374303579 + ] + ], + [ + [ + 0.0037640314549207687 + ] + ] + ], + "conv_stem.branch1.conv2.input_quant": 0.03290736302733421, + "conv_stem.branch1.conv2.output_quant": 0.016764704138040543, + "conv_stem.branch1.conv2.input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "conv_stem.branch1.conv2.output_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "conv_stem.branch1.relu2.act_quant": 0.00846397690474987, + "conv_stem.branch1.relu2.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "conv_stem.branch2.conv1.weight_quant": [ + [ + [ + 0.0028544331435114145 + ] + ], + [ + [ + 0.0033681674394756556 + ] + ], + [ + [ + 0.0034882749896496534 + ] + ], + [ + [ + 0.0030635371804237366 + ] + ], + [ + [ + 0.003136900020763278 + ] + ], + [ + [ + 0.003050380852073431 + ] + ], + [ + [ + 0.0040616183541715145 + ] + ], + [ + [ + 0.0027133801486343145 + ] + ], + [ + [ + 0.0032808163668960333 + ] + ], + [ + [ + 0.0033756429329514503 + ] + ], + [ + [ + 0.0026978773530572653 + ] + ], + [ + [ + 0.0032980882097035646 + ] + ], + [ + [ + 0.0029072100296616554 + ] + ], + [ + [ + 0.0027124390471726656 + ] + ], + [ + [ + 0.0035788349341601133 + ] + ], + [ + [ + 0.002912278752774 + ] + ] + ], + "conv_stem.branch2.conv1.input_quant": 0.04398861154913902, + "conv_stem.branch2.conv1.output_quant": 0.05451372638344765, + "conv_stem.branch2.conv1.input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "conv_stem.branch2.conv1.output_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "conv_stem.branch2.relu1.act_quant": 0.02716621570289135, + "conv_stem.branch2.relu1.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "conv_stem.branch2.conv2.weight_quant": [ + [ + [ + 0.0043754177168011665 + ] + ], + [ + [ + 0.004144658800214529 + ] + ], + [ + [ + 0.0036650870461016893 + ] + ], + [ + [ + 0.00365339289419353 + ] + ], + [ + [ + 0.0030197538435459137 + ] + ], + [ + [ + 0.004602823406457901 + ] + ], + [ + [ + 0.002820024499669671 + ] + ], + [ + [ + 0.004166399594396353 + ] + ], + [ + [ + 0.0040544550865888596 + ] + ], + [ + [ + 0.004079627804458141 + ] + ], + [ + [ + 0.004077327903360128 + ] + ], + [ + [ + 0.0039384267292916775 + ] + ], + [ + [ + 0.004882764536887407 + ] + ], + [ + [ + 0.004018516279757023 + ] + ], + [ + [ + 0.004400023724883795 + ] + ], + [ + [ + 0.0034655272029340267 + ] + ] + ], + "conv_stem.branch2.conv2.input_quant": 0.04243790730834007, + "conv_stem.branch2.conv2.output_quant": 0.017832189798355103, + "conv_stem.branch2.conv2.input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "conv_stem.branch2.conv2.output_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "conv_stem.branch2.relu2.act_quant": 0.008788160979747772, + "conv_stem.branch2.relu2.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "conv_stem.branch3.conv1.weight_quant": [ + [ + [ + 0.0020618378184735775 + ] + ], + [ + [ + 0.002362086670473218 + ] + ], + [ + [ + 0.0028500768821686506 + ] + ], + [ + [ + 0.0025676009245216846 + ] + ], + [ + [ + 0.0025873358827084303 + ] + ], + [ + [ + 0.00253949873149395 + ] + ], + [ + [ + 0.002308707917109132 + ] + ], + [ + [ + 0.0020583360455930233 + ] + ], + [ + [ + 0.0022384098265320063 + ] + ], + [ + [ + 0.0027320149820297956 + ] + ], + [ + [ + 0.00233985623344779 + ] + ], + [ + [ + 0.002922546351328492 + ] + ], + [ + [ + 0.002648717723786831 + ] + ], + [ + [ + 0.002508507575839758 + ] + ], + [ + [ + 0.003001951379701495 + ] + ], + [ + [ + 0.0023402906954288483 + ] + ] + ], + "conv_stem.branch3.conv1.input_quant": 0.04459114745259285, + "conv_stem.branch3.conv1.output_quant": 0.06097126752138138, + "conv_stem.branch3.conv1.input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "conv_stem.branch3.conv1.output_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "conv_stem.branch3.relu1.act_quant": 0.030341841280460358, + "conv_stem.branch3.relu1.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "conv_stem.branch3.conv2.weight_quant": [ + [ + [ + 0.0034469394013285637 + ] + ], + [ + [ + 0.00402803486213088 + ] + ], + [ + [ + 0.0036262052599340677 + ] + ], + [ + [ + 0.0049094632267951965 + ] + ], + [ + [ + 0.0034171505831182003 + ] + ], + [ + [ + 0.004605163354426622 + ] + ], + [ + [ + 0.004510743077844381 + ] + ], + [ + [ + 0.0037943809293210506 + ] + ], + [ + [ + 0.005316364578902721 + ] + ], + [ + [ + 0.004661924671381712 + ] + ], + [ + [ + 0.003083513118326664 + ] + ], + [ + [ + 0.005059072747826576 + ] + ], + [ + [ + 0.004422878380864859 + ] + ], + [ + [ + 0.0036508957855403423 + ] + ], + [ + [ + 0.003726565046235919 + ] + ], + [ + [ + 0.0038747212383896112 + ] + ] + ], + "conv_stem.branch3.conv2.input_quant": 0.05527482554316521, + "conv_stem.branch3.conv2.output_quant": 0.018089398741722107, + "conv_stem.branch3.conv2.input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "conv_stem.branch3.conv2.output_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "conv_stem.branch3.relu2.act_quant": 0.009000495076179504, + "conv_stem.branch3.relu2.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "conv_stem.cat_rescale.act_quant": 0.03551130369305611, + "conv_stem.cat_rescale.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "qaddpos.act_quant": 0.017837218940258026, + "qaddpos.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "encoder.ln_1.weight_quant": 1.0, + "encoder.ln_1.bias_quant": 1.0, + "encoder.mha.in_proj.weight_quant": [ + [ + 0.0021071869414299726 + ], + [ + 0.0018961158348247409 + ], + [ + 0.0015062791062518954 + ], + [ + 0.002423203317448497 + ], + [ + 0.00174946547485888 + ], + [ + 0.0019177183276042342 + ], + [ + 0.0018807967426255345 + ], + [ + 0.001548569998703897 + ], + [ + 0.0011212308891117573 + ], + [ + 0.001379145891405642 + ], + [ + 0.0017511369660496712 + ], + [ + 0.0015972043620422482 + ], + [ + 0.0015924543840810657 + ], + [ + 0.002301453845575452 + ], + [ + 0.002261851215735078 + ], + [ + 0.0016477408353239298 + ], + [ + 0.0024681687355041504 + ], + [ + 0.0014800956705585122 + ], + [ + 0.001948988763615489 + ], + [ + 0.0021288017742335796 + ], + [ + 0.0013043810613453388 + ], + [ + 0.0017070436151698232 + ], + [ + 0.002024749293923378 + ], + [ + 0.002023475943133235 + ], + [ + 0.0019611953757703304 + ], + [ + 0.0016012847190722823 + ], + [ + 0.0018770445603877306 + ], + [ + 0.0014978990657255054 + ], + [ + 0.0022737313993275166 + ], + [ + 0.002214226173236966 + ], + [ + 0.001582524972036481 + ], + [ + 0.001529959961771965 + ], + [ + 0.002205729018896818 + ], + [ + 0.0018768879817798734 + ], + [ + 0.002661316189914942 + ], + [ + 0.0015044293832033873 + ], + [ + 0.001814402756281197 + ], + [ + 0.001933390274643898 + ], + [ + 0.002207135548815131 + ], + [ + 0.001913944841362536 + ], + [ + 0.002045201137661934 + ], + [ + 0.002093489049002528 + ], + [ + 0.0014750697882845998 + ], + [ + 0.0022373248357325792 + ], + [ + 0.002473076106980443 + ], + [ + 0.0018155454890802503 + ], + [ + 0.0021056821569800377 + ], + [ + 0.001934695872478187 + ], + [ + 0.0025584257673472166 + ], + [ + 0.001578359748236835 + ], + [ + 0.00199709041044116 + ], + [ + 0.0022841054014861584 + ], + [ + 0.0015191702404990792 + ], + [ + 0.0022538548801094294 + ], + [ + 0.0022919790353626013 + ], + [ + 0.002331934403628111 + ], + [ + 0.0019083148799836636 + ], + [ + 0.002720849821344018 + ], + [ + 0.0017137030372396111 + ], + [ + 0.002344016218557954 + ], + [ + 0.0022261503618210554 + ], + [ + 0.0033149966038763523 + ], + [ + 0.0025749749038368464 + ], + [ + 0.002318201120942831 + ], + [ + 0.002220605267211795 + ], + [ + 0.0022199517115950584 + ], + [ + 0.0022314756643027067 + ], + [ + 0.0018433924997225404 + ], + [ + 0.0026631217915564775 + ], + [ + 0.0018224967643618584 + ], + [ + 0.0025638090446591377 + ], + [ + 0.0018702598754316568 + ], + [ + 0.002417492913082242 + ], + [ + 0.0018854676745831966 + ], + [ + 0.0019335874821990728 + ], + [ + 0.0018881962168961763 + ], + [ + 0.0021717885974794626 + ], + [ + 0.0021972190588712692 + ], + [ + 0.0015470789512619376 + ], + [ + 0.0019454978173598647 + ], + [ + 0.0022312570363283157 + ], + [ + 0.0018959111766889691 + ], + [ + 0.0027432499919086695 + ], + [ + 0.0018493181560188532 + ], + [ + 0.001827657106332481 + ], + [ + 0.002221457427367568 + ], + [ + 0.002189524006098509 + ], + [ + 0.0020209308713674545 + ], + [ + 0.0023842097725719213 + ], + [ + 0.0020050096791237593 + ], + [ + 0.002156405709683895 + ], + [ + 0.0027858843095600605 + ], + [ + 0.0031699021346867085 + ], + [ + 0.002103906124830246 + ], + [ + 0.002554378006607294 + ], + [ + 0.0015187639510259032 + ], + [ + 0.002849459182471037 + ], + [ + 0.0027739873621612787 + ], + [ + 0.001866056933067739 + ], + [ + 0.0018324883421882987 + ], + [ + 0.002734809648245573 + ], + [ + 0.002684821607545018 + ], + [ + 0.0025979734491556883 + ], + [ + 0.0019476537127047777 + ], + [ + 0.0022448005620390177 + ], + [ + 0.0027349386364221573 + ], + [ + 0.0016696223756298423 + ], + [ + 0.003179484512656927 + ], + [ + 0.001743459957651794 + ], + [ + 0.002047704067081213 + ], + [ + 0.002337988931685686 + ], + [ + 0.0025819060392677784 + ], + [ + 0.002374933334067464 + ], + [ + 0.001967527437955141 + ], + [ + 0.0020012918394058943 + ], + [ + 0.0017300629988312721 + ], + [ + 0.001585284247994423 + ], + [ + 0.001515955664217472 + ], + [ + 0.0018590908730402589 + ], + [ + 0.0021173974964767694 + ], + [ + 0.0024945398326963186 + ], + [ + 0.0022471165284514427 + ], + [ + 0.0019063017098233104 + ], + [ + 0.0019163653487339616 + ], + [ + 0.0022822588216513395 + ], + [ + 0.0027194854337722063 + ], + [ + 0.002537494758144021 + ], + [ + 0.002407339634373784 + ], + [ + 0.0018270317232236266 + ], + [ + 0.00179093552287668 + ], + [ + 0.0019214469939470291 + ], + [ + 0.002057047560811043 + ], + [ + 0.0025684514548629522 + ], + [ + 0.0030485016759485006 + ], + [ + 0.001797708566300571 + ], + [ + 0.002529471879824996 + ], + [ + 0.0019132940797135234 + ], + [ + 0.0018372675403952599 + ], + [ + 0.0021196091547608376 + ], + [ + 0.00191772251855582 + ], + [ + 0.0020810700953006744 + ], + [ + 0.0017824856331571937 + ], + [ + 0.001987345051020384 + ], + [ + 0.002248850418254733 + ] + ], + "encoder.mha.in_proj.bias_quant": [ + 4.3521900806808844e-05, + 3.8491391023853794e-05, + 3.6763845855602995e-05, + 3.4433793189236894e-05, + 3.0147259167279117e-05, + 4.0485389035893604e-05, + 2.7743613827624358e-05, + 2.474003122188151e-05, + 2.4965846023405902e-05, + 3.3638494642218575e-05, + 5.1969156629638746e-05, + 5.1957162213511765e-05, + 4.104896288481541e-05, + 2.8779299100278877e-05, + 5.645572309731506e-05, + 3.241921876906417e-05, + 3.756762089324184e-05, + 3.364929216331802e-05, + 2.7748868888011202e-05, + 4.0356131648877636e-05, + 3.7348629120970145e-05, + 3.522906263242476e-05, + 4.9435333494329825e-05, + 3.5841680073644966e-05, + 5.683824565494433e-05, + 3.403754089958966e-05, + 4.642199201043695e-05, + 3.276973802712746e-05, + 2.9627070034621283e-05, + 4.843081478611566e-05, + 3.243835817556828e-05, + 4.105076368432492e-05, + 4.346259811427444e-05, + 4.849517426919192e-05, + 6.062247121008113e-05, + 2.476934423611965e-05, + 4.2470128391869366e-05, + 6.114810094004497e-05, + 4.197923044557683e-05, + 2.8275273507460952e-05, + 6.430425128201023e-05, + 5.0424408982507885e-05, + 4.118662764085457e-05, + 6.0045138525310904e-05, + 5.068449900136329e-05, + 4.699428245658055e-05, + 5.3715066314907745e-05, + 4.584524504025467e-05, + 8.329556294484064e-05, + 7.463671499863267e-05, + 3.515305070322938e-05, + 0.00012327654985710979, + 2.3138829419622198e-05, + 2.8262471460038796e-05, + 3.627617479651235e-05, + 4.092595190741122e-05, + 3.067774014198221e-05, + 2.5952384021366015e-05, + 2.5374722099513747e-05, + 9.036653500515968e-05, + 5.838513243361376e-05, + 9.195698658004403e-05, + 0.00011730028199963272, + 6.724580453010276e-05, + 8.573680679546669e-05, + 6.0451180615928024e-05, + 6.609367846976966e-05, + 4.287050978746265e-05, + 3.07990558212623e-05, + 3.967693555750884e-05, + 0.000124986152513884, + 3.70257803297136e-05, + 3.557019590516575e-05, + 4.242193608661182e-05, + 4.24197714892216e-05, + 3.1556657631881535e-05, + 6.418516568373889e-05, + 7.506601832574233e-05, + 5.4480129620060325e-05, + 8.733737922739238e-05, + 4.888823605142534e-05, + 6.0739723267033696e-05, + 0.00011185201583430171, + 3.653555904747918e-05, + 0.00011702090705512092, + 0.0001050583305186592, + 3.819035555352457e-05, + 6.34065072517842e-05, + 8.788484410615638e-05, + 0.00013765625772066414, + 3.521301186992787e-05, + 0.00015364852151833475, + 0.00017014065815601498, + 0.00012492647510953248, + 0.00017065026622731239, + 2.3662745661567897e-05, + 0.00015074788825586438, + 0.00012861474533565342, + 7.89753466960974e-05, + 8.625428745290264e-05, + 0.00014355145685840398, + 8.428324508713558e-05, + 9.463157766731456e-05, + 0.0001161023901659064, + 0.00012646715913433582, + 7.746645133011043e-05, + 7.160487439250574e-05, + 0.00010664350702427328, + 8.168781641870737e-05, + 7.808411464793608e-05, + 8.565741882193834e-05, + 0.00011714619176927954, + 9.993903950089589e-05, + 7.11664033588022e-05, + 7.680957060074434e-05, + 8.642328612040728e-05, + 8.600317232776433e-05, + 7.776951679261401e-05, + 0.0001042910516844131, + 7.825870852684602e-05, + 9.28895387914963e-05, + 0.00011101825657533482, + 9.177954052574933e-05, + 8.378223719773814e-05, + 9.64844148256816e-05, + 0.00011213251855224371, + 9.596322342986241e-05, + 7.389799429802224e-05, + 8.60082363942638e-05, + 6.75573610351421e-05, + 7.606671715620905e-05, + 9.394827793585137e-05, + 8.219730807468295e-05, + 0.0001335921697318554, + 7.60522234486416e-05, + 7.656634988961741e-05, + 9.0296111011412e-05, + 9.465150651521981e-05, + 8.766334212850779e-05, + 9.40323734539561e-05, + 8.094309305306524e-05, + 7.81559429015033e-05, + 7.349377847276628e-05, + 0.0001089235520339571 + ], + "encoder.mha.in_proj.input_quant": 0.02831985242664814, + "encoder.mha.in_proj.input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "encoder.mha.out_proj.weight_quant": [ + [ + 0.002566875424236059 + ], + [ + 0.0030580158345401287 + ], + [ + 0.002994395326822996 + ], + [ + 0.0024209555704146624 + ], + [ + 0.002476828871294856 + ], + [ + 0.002824110444635153 + ], + [ + 0.002946047345176339 + ], + [ + 0.0034539501648396254 + ], + [ + 0.0033426000736653805 + ], + [ + 0.0030282125808298588 + ], + [ + 0.002563875401392579 + ], + [ + 0.00301593285985291 + ], + [ + 0.003232685849070549 + ], + [ + 0.0026690170634537935 + ], + [ + 0.003807465313002467 + ], + [ + 0.00406606774777174 + ], + [ + 0.0022860101889818907 + ], + [ + 0.002878852654248476 + ], + [ + 0.0021447460167109966 + ], + [ + 0.0030518346466124058 + ], + [ + 0.0033308956772089005 + ], + [ + 0.003151464741677046 + ], + [ + 0.002865272108465433 + ], + [ + 0.002473384840413928 + ], + [ + 0.0031667479779571295 + ], + [ + 0.002976816613227129 + ], + [ + 0.0035451941657811403 + ], + [ + 0.0027477059047669172 + ], + [ + 0.0032785695511847734 + ], + [ + 0.0034140627831220627 + ], + [ + 0.0047993529587984085 + ], + [ + 0.003081711009144783 + ], + [ + 0.004431079141795635 + ], + [ + 0.0035265740007162094 + ], + [ + 0.00340841943398118 + ], + [ + 0.0032587277237325907 + ], + [ + 0.0025499146431684494 + ], + [ + 0.002363869221881032 + ], + [ + 0.004081340041011572 + ], + [ + 0.002843390451744199 + ], + [ + 0.002658950164914131 + ], + [ + 0.0030732478480786085 + ], + [ + 0.0028174100443720818 + ], + [ + 0.003714032471179962 + ], + [ + 0.0025140380021184683 + ], + [ + 0.002328339498490095 + ], + [ + 0.0034008342772722244 + ], + [ + 0.002457233378663659 + ] + ], + "encoder.mha.out_proj.bias_quant": [ + 3.3028103644028306e-05, + 3.779260805458762e-05, + 3.103347626165487e-05, + 3.294542329967953e-05, + 3.155745071126148e-05, + 3.1064715585671365e-05, + 2.7030437195207924e-05, + 3.4928423701785505e-05, + 3.170805939589627e-05, + 2.9242737582535483e-05, + 2.173976827180013e-05, + 3.503774860291742e-05, + 3.153623401885852e-05, + 2.97042843158124e-05, + 2.9865308533771895e-05, + 3.388613913557492e-05, + 3.40683436661493e-05, + 3.0952996894484386e-05, + 4.679028643295169e-05, + 3.405780080356635e-05, + 3.340877447044477e-05, + 3.649418067652732e-05, + 4.08902888011653e-05, + 2.6750736651592888e-05, + 3.5160526749677956e-05, + 3.176083191647194e-05, + 3.8332498661475256e-05, + 4.2124356696149334e-05, + 4.2702020436991006e-05, + 3.722302062669769e-05, + 4.469441046239808e-05, + 4.2093884985661134e-05, + 4.8539237468503416e-05, + 3.1216863135341555e-05, + 4.117336357012391e-05, + 4.4027077819919214e-05, + 2.877884435292799e-05, + 2.3634464014321566e-05, + 4.0102411730913445e-05, + 3.3787466236390173e-05, + 2.779592978185974e-05, + 2.9332451958907768e-05, + 3.3954707760130987e-05, + 3.7977621104801074e-05, + 3.8975889765424654e-05, + 2.8808814022340812e-05, + 3.5180655686417595e-05, + 2.693287569854874e-05 + ], + "encoder.mha.out_proj.input_quant": 0.008948581293225288, + "encoder.mha.out_proj.output_quant": 0.010939874686300755, + "encoder.mha.out_proj.input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "encoder.mha.out_proj.output_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "encoder.mha.attn_output_weights_quant.act_quant": 0.0002787475532386452, + "encoder.mha.attn_output_weights_quant.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "encoder.mha.q_scaled_quant.act_quant": 0.005090874154120684, + "encoder.mha.q_scaled_quant.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "encoder.mha.k_transposed_quant.act_quant": 0.014389771036803722, + "encoder.mha.k_transposed_quant.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "encoder.mha.v_quant.act_quant": 0.019884243607521057, + "encoder.mha.v_quant.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "encoder.ln_2.weight_quant": 1.0, + "encoder.ln_2.bias_quant": 1.0, + "encoder.ff.ff1.weight_quant": [ + [ + 0.003403208451345563 + ], + [ + 0.0025569989811629057 + ], + [ + 0.001959930406883359 + ], + [ + 0.002515022177249193 + ], + [ + 0.0031255900394171476 + ], + [ + 0.0036360148806124926 + ], + [ + 0.002630674745887518 + ], + [ + 0.0026574768126010895 + ], + [ + 0.0032233132515102625 + ], + [ + 0.003218092955648899 + ], + [ + 0.004974754992872477 + ], + [ + 0.0025420053862035275 + ], + [ + 0.002638278529047966 + ], + [ + 0.004613530822098255 + ], + [ + 0.00354278227314353 + ], + [ + 0.003082627896219492 + ], + [ + 0.0025242427363991737 + ], + [ + 0.0025976719334721565 + ], + [ + 0.0024957924615591764 + ], + [ + 0.0030332140158861876 + ], + [ + 0.003239266574382782 + ], + [ + 0.0032029158901423216 + ], + [ + 0.002868309151381254 + ], + [ + 0.002452407032251358 + ], + [ + 0.002743711695075035 + ], + [ + 0.002992327092215419 + ], + [ + 0.0029564935248345137 + ], + [ + 0.0026715686544775963 + ], + [ + 0.003059198148548603 + ], + [ + 0.0021910383366048336 + ], + [ + 0.003455266822129488 + ], + [ + 0.004229408223181963 + ], + [ + 0.0033398051746189594 + ], + [ + 0.003178544342517853 + ], + [ + 0.0033053760416805744 + ], + [ + 0.002562867943197489 + ], + [ + 0.004470005631446838 + ], + [ + 0.0023471303284168243 + ], + [ + 0.0031073056161403656 + ], + [ + 0.002469737082719803 + ], + [ + 0.0024988676887005568 + ], + [ + 0.0025458685122430325 + ], + [ + 0.002492014318704605 + ], + [ + 0.0028488156385719776 + ], + [ + 0.0026558584067970514 + ], + [ + 0.0027248593978583813 + ], + [ + 0.0031810093205422163 + ], + [ + 0.0035232664085924625 + ] + ], + "encoder.ff.ff1.bias_quant": [ + 5.5452477681683376e-05, + 6.036627746652812e-05, + 4.9042177124647424e-05, + 6.090618626330979e-05, + 6.464087346103042e-05, + 6.0323673096718267e-05, + 4.5953489461680874e-05, + 4.828739110962488e-05, + 6.553747516591102e-05, + 5.163819878362119e-05, + 7.9820652899798e-05, + 6.688055873382837e-05, + 4.458321200218052e-05, + 8.539440023014322e-05, + 5.4340223869076e-05, + 4.446164166438393e-05, + 5.0903039664262906e-05, + 5.251335460343398e-05, + 4.4163462007418275e-05, + 4.847836316912435e-05, + 4.699168857769109e-05, + 5.341105133993551e-05, + 6.195847527123988e-05, + 5.797050835099071e-05, + 7.597541116410866e-05, + 6.577254680451006e-05, + 4.802774128620513e-05, + 5.369189238990657e-05, + 5.5123440688475966e-05, + 4.9369398766430095e-05, + 5.150325523572974e-05, + 8.107110625132918e-05, + 6.97872819728218e-05, + 6.320456304820254e-05, + 5.914639768889174e-05, + 4.555694977170788e-05, + 8.352378790732473e-05, + 4.0727285522734746e-05, + 4.727977648144588e-05, + 5.572301961365156e-05, + 5.828376015415415e-05, + 6.130327528808266e-05, + 5.207680442254059e-05, + 5.06037576997187e-05, + 6.49327048449777e-05, + 4.835459913010709e-05, + 5.525608139578253e-05, + 5.519505430129357e-05 + ], + "encoder.ff.ff1.input_quant": 0.01583567075431347, + "encoder.ff.ff1.output_quant": 0.022544866427779198, + "encoder.ff.ff1.input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "encoder.ff.ff1.output_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "encoder.ff.ff2.weight_quant": [ + [ + 0.002971225418150425 + ], + [ + 0.002250400837510824 + ], + [ + 0.002709007589146495 + ], + [ + 0.0030752269085496664 + ], + [ + 0.0030809184536337852 + ], + [ + 0.00439762556925416 + ], + [ + 0.0029294404666870832 + ], + [ + 0.0035947992000728846 + ], + [ + 0.0035167390014976263 + ], + [ + 0.003796542761847377 + ], + [ + 0.0026450648438185453 + ], + [ + 0.003567359410226345 + ], + [ + 0.002195175038650632 + ], + [ + 0.0022677110973745584 + ], + [ + 0.0029033911414444447 + ], + [ + 0.0028390090446919203 + ], + [ + 0.003564479760825634 + ], + [ + 0.003430349053815007 + ], + [ + 0.0028679638635367155 + ], + [ + 0.0025241514667868614 + ], + [ + 0.002458178671076894 + ], + [ + 0.0026473698671907187 + ], + [ + 0.0026095551438629627 + ], + [ + 0.0023333856370300055 + ], + [ + 0.0032875153701752424 + ], + [ + 0.0025124414823949337 + ], + [ + 0.0024445876479148865 + ], + [ + 0.004292535129934549 + ], + [ + 0.003165788482874632 + ], + [ + 0.002462804550305009 + ], + [ + 0.0023571953643113375 + ], + [ + 0.0033441856503486633 + ], + [ + 0.0033317492343485355 + ], + [ + 0.0035525828134268522 + ], + [ + 0.002138659590855241 + ], + [ + 0.0034952741116285324 + ], + [ + 0.002431573113426566 + ], + [ + 0.0033607175573706627 + ], + [ + 0.0029228581115603447 + ], + [ + 0.00266608246602118 + ], + [ + 0.0032617677934467793 + ], + [ + 0.0029625333845615387 + ], + [ + 0.004284386523067951 + ], + [ + 0.0027726078405976295 + ], + [ + 0.0031355973333120346 + ], + [ + 0.0028520245105028152 + ], + [ + 0.0022280821576714516 + ], + [ + 0.003456521313637495 + ] + ], + "encoder.ff.ff2.bias_quant": [ + 4.081176302861422e-05, + 4.65644225187134e-05, + 5.293917274684645e-05, + 3.598331386456266e-05, + 4.6789238695055246e-05, + 5.8310826716478914e-05, + 5.0376122089801356e-05, + 7.84043877501972e-05, + 4.865194205194712e-05, + 7.402031042147428e-05, + 4.718528361991048e-05, + 3.90489112760406e-05, + 3.9293565350817516e-05, + 5.870520180906169e-05, + 6.599004700547084e-05, + 3.778406244236976e-05, + 5.7146426115650684e-05, + 5.240013706497848e-05, + 7.232013012981042e-05, + 4.0614755562273785e-05, + 4.444371734280139e-05, + 4.3532894778763875e-05, + 4.572581747197546e-05, + 5.285641964292154e-05, + 4.680298297898844e-05, + 5.387501005316153e-05, + 5.964556476101279e-05, + 9.004313324112445e-05, + 4.128019281779416e-05, + 5.743368819821626e-05, + 5.14151033712551e-05, + 0.00011770833953050897, + 6.53171373414807e-05, + 6.44302272121422e-05, + 3.137749808956869e-05, + 5.778835839009844e-05, + 3.799048499786295e-05, + 3.670175283332355e-05, + 5.8541627367958426e-05, + 3.631471918197349e-05, + 7.658202957827598e-05, + 4.532069215201773e-05, + 5.6113356549758464e-05, + 5.846757994731888e-05, + 6.687606946798041e-05, + 5.0440532504580915e-05, + 6.807724275859073e-05, + 4.7414676373591647e-05 + ], + "encoder.ff.ff2.input_quant": 0.014125959947705269, + "encoder.ff.ff2.output_quant": 0.01510005071759224, + "encoder.ff.ff2.input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "encoder.ff.ff2.output_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "encoder.rescale_residual1.act_quant": 0.011041161604225636, + "encoder.rescale_residual1.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "encoder.rescale_residual2.act_quant": 0.016701295971870422, + "encoder.rescale_residual2.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "norm.weight_quant": 1.0, + "norm.bias_quant": 1.0, + "rescale_norm.act_quant": 0.013678238727152348, + "rescale_norm.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "head.weight_quant": [ + [ + 0.00323863560333848 + ], + [ + 0.003783071879297495 + ], + [ + 0.0036574609111994505 + ], + [ + 0.0027862510178238153 + ] + ], + "head.bias_quant": [ + 4.697828262578696e-05, + 4.3166644900338724e-05, + 6.678188219666481e-05, + 5.249695823295042e-05 + ], + "head.input_quant": 0.013886776752769947, + "head.output_quant": 0.03440367430448532, + "head.input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, + "head.output_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0 +} \ No newline at end of file diff --git a/onnx4deeploy/models/qlite_cnn_exporter.py b/onnx4deeploy/models/qlite_cnn_exporter.py new file mode 100644 index 0000000..aa0da29 --- /dev/null +++ b/onnx4deeploy/models/qlite_cnn_exporter.py @@ -0,0 +1,238 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: MIT + +"""QLite CNN Model Exporter.""" + +from pathlib import Path +from typing import Any, Dict, List, Tuple + +import numpy as np +import torch +import torch.onnx.utils + +from DeepQuant.DeepQuant.ExportBrevitas import exportBrevitas + +from ..core.base_exporter import BaseONNXExporter + +# Import QLiteCNN PyTorch model from new location +from .pytorch_models.lightweight_cnn import QLiteCNN +import brevitas.onnx as bo +import json +import onnx +import onnx_graphsurgeon as gs +from onnx4deeploy.transform.quant_transform import replace_qdq_with_deeploy, insert_rqs_from_map + + +class QLiteCnnExporter(BaseONNXExporter): + """ONNX exporter for QLite CNN model.""" + + def __init__(self, save_path: str = None, config_file: str = "config.yaml"): + """ + Initialize QLite CNN exporter. + + Args: + save_path: Optional custom path to save ONNX files + config_file: Path to configuration YAML file + """ + super().__init__(save_path, config_file) + self.model_config = {} + + def load_config(self) -> Dict[str, Any]: + """ + Load QLite CNN configuration. + + Returns: + Dictionary containing QLite CNN configuration parameters + """ + # Default QLite CNN configuration + config = { + "batch_size": 1, + "input_height": 28, + "input_width": 28, + "input_channels": 1, # Grayscale images + "num_classes": 10, + "opset_version": 17, + "dropout": 0.0, # No dropout for inference + # Training configuration + "training_strategy": "full", # Options: "full", "last_layer", "custom" + "custom_trainable_params": [], + "zo": { + "epsilon": 0.1, + "seed": 42, + "noise_type": "eggroll", + }, + "weights_path":"onnx4deeploy/models/pytorch_models/lightweight_cnn/qlite_cnn.pth" + } + + self.model_config = config + return config + + def create_model(self) -> torch.nn.Module: + """ + Create QLite CNN PyTorch model. + + Returns: + QLite CNN model ready for export + """ + model = QLiteCNN( + batch_size=self.model_config["batch_size"], + input_channels=self.model_config["input_channels"], + num_classes=self.model_config["num_classes"], + dropout=self.model_config["dropout"], + ) + + return model + + def get_input_shape(self) -> Tuple[int, ...]: + """ + Get the input tensor shape for QLite CNN. + + Returns: + Tuple representing input shape (batch_size, channels, height, width) + """ + batch_size = self.config["batch_size"] + channels = self.config["input_channels"] + height = self.config["input_height"] + width = self.config["input_width"] + return (batch_size, channels, height, width) + + def get_trainable_params(self, all_param_names: List[str]) -> List[str]: + """ + Get list of trainable parameter names for QLite CNN. + + Supports multiple training strategies: + - "full": Train all parameters (default) + - "last_layer": Only train the final classification layer + - "custom": Use custom_trainable_params from config + + Args: + all_param_names: List of all parameter names in the model + + Returns: + List of parameter names that should be trainable + """ + strategy = self.config.get("training_strategy", "full") + + # Define training strategies + strategy_params = { + "full": all_param_names, # Train everything + "last_layer": [ + "fc.weight", + "fc.bias", + ], + "custom": self.config.get("custom_trainable_params", []), + } + + # Get trainable params based on strategy + if strategy not in strategy_params: + print(f"โš ๏ธ Unknown training strategy '{strategy}', using 'full' as fallback") + strategy = "full" + + trainable_params = strategy_params[strategy] + + # Filter to only include params that exist in the model + requires_grad = [name for name in all_param_names if name in trainable_params] + + # Print strategy info + print(f"\n๐ŸŽฏ Training Strategy: '{strategy}'") + print(f" Total params in model: {len(all_param_names)}") + print(f" Params to train: {len(requires_grad)}") + print(f" Frozen params: {len(all_param_names) - len(requires_grad)}") + + return requires_grad + + def _get_config_string(self) -> str: + """ + Get configuration string for folder naming. + + Returns: + Configuration string like "_28x28_1ch_10" + """ + return ( + f"_{self.config['input_height']}x{self.config['input_width']}" + f"_{self.config['input_channels']}ch_{self.config['num_classes']}" + ) + + def save_test_data(self, model: torch.nn.Module, save_dir: str): + """ + Save test input/output data for validation. + + Uses PyTorch model to generate reference output for validating ONNX correctness. + + Args: + model: PyTorch model to run inference with + save_dir: Directory to save test data + """ + print("๐Ÿ’พ Saving test input/output data...") + + # Create test input + input_shape = self.get_input_shape() + test_input = np.random.randn(*input_shape).astype(np.float32) + + # Get PyTorch output (reference for validating ONNX) + was_training = model.training + model.eval() + + with torch.no_grad(): + input_tensor = torch.from_numpy(test_input) + output_tensor = model(input_tensor) + test_output = output_tensor.numpy() + + # Restore training mode if needed + if was_training: + model.train() + + # Save as .npz files + save_path = Path(save_dir) + save_path.mkdir(parents=True, exist_ok=True) + + np.savez(save_path / "inputs.npz", input=test_input) + np.savez(save_path / "outputs.npz", output=test_output) + + print(" โœ… Saved test data (PyTorch reference):") + print(f" Input: {save_path / 'inputs.npz'} shape={test_input.shape}") + print(f" Output: {save_path / 'outputs.npz'} shape={test_output.shape}") + + def _build_rqs_map(self, graph: gs.Graph, brevitas_scales: Dict[str, Any]) -> Dict[str, Any]: + """ + Translate the flat Brevitas scales dump into the Deeploy edges map + expected by insert_rqs_from_map. + """ + rqs_map = {"edges": []} + + # Traverse the ONNX graph to find operators that require precision reduction + for node in graph.nodes: + if node.op in ["Conv", "Gemm", "MatMul"]: + layer_name = None + + # Match ONNX node name (e.g., "/conv1/Conv") with Brevitas scale keys + for key in brevitas_scales.keys(): + if key.endswith(".weight_quant"): + base_name = key.split(".")[0] # Extracts "conv1", "fc", etc. + if f"/{base_name}/" in node.name or node.name.startswith(base_name): + layer_name = base_name + break + + if layer_name: + in_scale = brevitas_scales.get(f"{layer_name}.input_quant") + w_scale = brevitas_scales.get(f"{layer_name}.weight_quant") + out_scale = brevitas_scales.get(f"{layer_name}.output_quant") + + if in_scale is not None and w_scale is not None and out_scale is not None: + # Conv accumulation scale = input_scale * weight_scale + # Flatten to support Brevitas per-channel deeply nested precision lists + w_flat = np.array(w_scale).flatten() + src_scale = in_scale * w_flat + + out_tensor_name = node.outputs[0].name + + # We use out_tensor_name for both src and dst to intercept and rewire + # all nodes strictly consuming the output of this convolution. + rqs_map["edges"].append({ + "src_tensor": out_tensor_name, + "dst_tensor": out_tensor_name, + "src_scale": src_scale.tolist(), + "dst_scale": float(out_scale) + }) + return rqs_map \ No newline at end of file diff --git a/onnx4deeploy/transform/quant_transform.py b/onnx4deeploy/transform/quant_transform.py new file mode 100644 index 0000000..b7d400c --- /dev/null +++ b/onnx4deeploy/transform/quant_transform.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python3 +""" +qdq_to_deeploy.py + +Convert ONNX QDQ/QCDQ graphs (QuantizeLinear/DequantizeLinear) into Deeploy-style +Quant/Dequant nodes, and optionally insert RequantShift nodes for known +scale transitions. + +Dependencies: + pip install onnx onnx-graphsurgeon numpy + +Usage: + python qdq_to_deeploy.py --in model_qdq.onnx --out model_deeploy.onnx + python qdq_to_deeploy.py --in model_qdq.onnx --out model_deeploy.onnx --rqs rqs_map.json +""" + +import argparse +import json +from typing import Any, Dict, Tuple, Optional + +import numpy as np +import onnx +import onnx_graphsurgeon as gs + + +# --------------------------- +# Helpers: initializers / dtypes +# --------------------------- + +def _as_numpy(t: gs.Constant) -> np.ndarray: + return np.asarray(t.values) + +def _const_scalar(name: str, value: Any, dtype=np.float32) -> gs.Constant: + return gs.Constant(name=name, values=np.array(value, dtype=dtype)) + +def _get_initializer_value(graph: gs.Graph, name: str) -> Optional[np.ndarray]: + """Return initializer values as numpy array if present.""" + for t in graph.initializers: + if t.name == name: + if isinstance(t, gs.Constant): + return _as_numpy(t) + # gs may store initializers as Constants; this is fallback + # Also check graph.tensors() for Constant + tensors = graph.tensors() + if name in tensors and isinstance(tensors[name], gs.Constant): + return _as_numpy(tensors[name]) + return None + +def _infer_signed_from_zero_point(zp_arr: Optional[np.ndarray]) -> bool: + """ + Deeploy Quant node expects 'signed' attribute. We infer it from zero_point dtype + when possible (int8 -> signed, uint8 -> unsigned). If zp missing, default signed. + """ + if zp_arr is None: + return True + dt = zp_arr.dtype + if dt == np.int8 or dt == np.int32 or dt == np.int16: + return True + if dt == np.uint8 or dt == np.uint16: + return False + # fallback + return True + + +# --------------------------- +# RequantShift parameterization +# --------------------------- + +def float_to_rqs_params(scale_ratio: np.ndarray, max_shift: int = 30) -> Tuple[np.ndarray, int, int]: + """ + Convert scale_ratio to (mul, add, div_pow2) such that: + out โ‰ˆ ((in * mul) + add) >> shift where div = 2^shift + + Deeploy RQS formula uses: + out = clip(((in * mul) + add) >> log2(div), ...) [2](https://deepwiki.com/pulp-platform/Deeploy/8.1-quantization-and-training-support) + + We pick a single shift (power-of-two div), and per-element mul if scale_ratio is vector. + add implements rounding-to-nearest: add = 1<<(shift-1). + """ + # Ensure array + r = np.asarray(scale_ratio, dtype=np.float64) + + # Choose largest shift that keeps mul within int32 range. + # mul = round(r * 2^shift) + # For safety use signed int32. + best_shift = None + for shift in range(max_shift, -1, -1): + mul = np.round(r * (2.0 ** shift)) + if np.all(np.abs(mul) <= (2**31 - 1)): + best_shift = shift + break + if best_shift is None: + raise RuntimeError("Could not find valid shift for scale_ratio") + + mul = np.round(r * (2.0 ** best_shift)).astype(np.int32) + add = (1 << (best_shift - 1)) if best_shift > 0 else 0 + div = 1 << best_shift + return mul, add, div + + +# --------------------------- +# Core transforms: QDQ -> Deeploy +# --------------------------- + +def replace_qdq_with_deeploy(graph: gs.Graph) -> None: + """ + Replace QuantizeLinear and DequantizeLinear nodes with Deeploy Quant and Dequant nodes. + + Deeploy Quant op parameters: scale, zero_point, bit_width, signed. [2](https://deepwiki.com/pulp-platform/Deeploy/8.1-quantization-and-training-support) + Deeploy Dequant op parameters: scale, zero_point; supports int8/int32 inputs. [2](https://deepwiki.com/pulp-platform/Deeploy/8.1-quantization-and-training-support) + """ + new_nodes = [] + for node in graph.nodes: + if node.op == "QuantizeLinear": + # Inputs: x, y_scale, y_zero_point (zp optional per ONNX) + x = node.inputs[0] + scale_in = node.inputs[1] if len(node.inputs) > 1 else None + zp_in = node.inputs[2] if len(node.inputs) > 2 else None + + scale = _as_numpy(scale_in) if isinstance(scale_in, gs.Constant) else _get_initializer_value(graph, scale_in.name) + zp = None + if zp_in is not None: + zp = _as_numpy(zp_in) if isinstance(zp_in, gs.Constant) else _get_initializer_value(graph, zp_in.name) + + if scale is None: + raise RuntimeError(f"QuantizeLinear node {node.name} has non-constant scale; provide constant initializer.") + + signed = _infer_signed_from_zero_point(zp) + bit_width = 8 # Deeploy QuantParser expects bit_width attribute; typical is 8. [2](https://deepwiki.com/pulp-platform/Deeploy/8.1-quantization-and-training-support) + + # Create Deeploy Quant node + q = gs.Node( + op="Quant", + name=(node.name or "Quant") + "_Deeploy", + inputs=[x], + outputs=node.outputs, + attrs={ + "scale": float(scale.reshape(-1)[0]) if scale.size == 1 else scale.astype(np.float32), + "zero_point": int(zp.reshape(-1)[0]) if (zp is not None and zp.size == 1) else (zp.astype(np.int32) if zp is not None else 0), + "bit_width": bit_width, + "signed": int(signed), + } + ) + new_nodes.append(q) + continue + + if node.op == "DequantizeLinear": + # Inputs: x, x_scale, x_zero_point (zp optional) + xq = node.inputs[0] + scale_in = node.inputs[1] if len(node.inputs) > 1 else None + zp_in = node.inputs[2] if len(node.inputs) > 2 else None + + scale = _as_numpy(scale_in) if isinstance(scale_in, gs.Constant) else _get_initializer_value(graph, scale_in.name) + zp = None + if zp_in is not None: + zp = _as_numpy(zp_in) if isinstance(zp_in, gs.Constant) else _get_initializer_value(graph, zp_in.name) + + if scale is None: + raise RuntimeError(f"DequantizeLinear node {node.name} has non-constant scale; provide constant initializer.") + + dq = gs.Node( + op="Dequant", + name=(node.name or "Dequant") + "_Deeploy", + inputs=[xq], + outputs=node.outputs, + attrs={ + "scale": float(scale.reshape(-1)[0]) if scale.size == 1 else scale.astype(np.float32), + "zero_point": int(zp.reshape(-1)[0]) if (zp is not None and zp.size == 1) else (zp.astype(np.int32) if zp is not None else 0), + } + ) + new_nodes.append(dq) + continue + + # passthrough + new_nodes.append(node) + + graph.nodes = new_nodes + + +def insert_rqs_from_map(graph: gs.Graph, rqs_map: Dict[str, Any]) -> None: + """ + Insert Deeploy RequantShift nodes according to a user-supplied mapping. + + Deeploy RequantShift formula: out = clip(((in * mul) + add) >> log2(div), ...) [2](https://deepwiki.com/pulp-platform/Deeploy/8.1-quantization-and-training-support) + + rqs_map format: + { + "edges": [ + { + "src_tensor": "tensorA_q", + "dst_tensor": "tensorB_q", + "src_scale": 0.0078125, + "dst_scale": 0.00390625 + } + ] + } + + This will insert RequantShift between producer(src_tensor) and consumers(dst_tensor). + """ + edges = rqs_map.get("edges", []) + if not edges: + return + + tensor_dict = graph.tensors() + + for e in edges: + src = e["src_tensor"] + dst = e["dst_tensor"] + s_src = np.array(e["src_scale"], dtype=np.float64) + s_dst = np.array(e["dst_scale"], dtype=np.float64) + + if src not in tensor_dict: + raise KeyError(f"src_tensor '{src}' not found in graph tensors.") + if dst not in tensor_dict: + raise KeyError(f"dst_tensor '{dst}' not found in graph tensors.") + + src_tensor = tensor_dict[src] + dst_tensor = tensor_dict[dst] + + # Compute ratio and integer params + ratio = s_src / s_dst + mul, add, div = float_to_rqs_params(ratio) + + # Create constants for mul/add/div + mul_const = gs.Constant(name=f"{src}_to_{dst}_mul", values=np.array(mul, dtype=np.int32)) + add_const = gs.Constant(name=f"{src}_to_{dst}_add", values=np.array(add, dtype=np.int32)) + div_const = gs.Constant(name=f"{src}_to_{dst}_div", values=np.array(div, dtype=np.int32)) + + # New intermediate tensor + rqs_out = gs.Variable(name=f"{src}_rqs_to_{dst}", dtype=src_tensor.dtype, shape=src_tensor.shape) + + rqs_node = gs.Node( + op="RequantShift", + name=f"RQS_{src}_to_{dst}", + inputs=[src_tensor, mul_const, add_const, div_const], + outputs=[rqs_out], + attrs={} + ) + + # Rewire: every consumer of src_tensor that was expecting dst_tensor gets rqs_out + # Safer approach: replace uses of src_tensor in dst's producer chain is complex; + # instead, we replace occurrences of src_tensor in inputs of nodes where it appears as dst_tensor input. + for n in graph.nodes: + for i, inp in enumerate(n.inputs): + if inp is dst_tensor: + n.inputs[i] = rqs_out + + graph.nodes.append(rqs_node) + + graph.cleanup().toposort() + + +# --------------------------- +# Main +# --------------------------- + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--in", dest="inp", required=True, help="Input ONNX (Brevitas QDQ/QCDQ)") + ap.add_argument("--out", dest="out", required=True, help="Output ONNX (Deeploy Quant/Dequant/RQS)") + ap.add_argument("--rqs", dest="rqs", default=None, help="Optional JSON map to insert RequantShift nodes") + args = ap.parse_args() + + model = onnx.load(args.inp) + graph = gs.import_onnx(model) + + replace_qdq_with_deeploy(graph) + + if args.rqs is not None: + with open(args.rqs, "r") as f: + rqs_map = json.load(f) + insert_rqs_from_map(graph, rqs_map) + + graph.cleanup().toposort() + out_model = gs.export_onnx(graph) + onnx.save(out_model, args.out) + print(f"[OK] Wrote Deeploy-style ONNX to: {args.out}") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 0d171bf..2d3d5c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,10 +7,12 @@ torch>=2.0.0 onnx>=1.15.0,<1.17.0 onnx-graphsurgeon>=0.5.0 onnxruntime-training==1.19.2 +onnxoptimizer>=0.4.2 onnxscript>=0.1.0 onnxsim>=0.4.0 numpy>=1.24.0,<2.0.0 pyyaml>=6.0 +brevitas>=0.12.0 # Optional visualization dependencies # Install with: pip install -e ".[visualization]" From 0bccfeaa66ec864e498b254e0528dbb39ee8b6ba Mon Sep 17 00:00:00 2001 From: JanCSEM Date: Thu, 12 Mar 2026 18:47:59 +0000 Subject: [PATCH 14/24] Added weight update graph and cross entropy --- onnx4deeploy/core/base_exporter.py | 34 +++- .../models/lightweight_cnn_exporter.py | 2 +- onnx4deeploy/models/qlite_cnn_exporter.py | 2 +- onnx4deeploy/transform/zo_transform.py | 170 +++++++++++++++++- 4 files changed, 197 insertions(+), 11 deletions(-) diff --git a/onnx4deeploy/core/base_exporter.py b/onnx4deeploy/core/base_exporter.py index 37734ad..220ee13 100644 --- a/onnx4deeploy/core/base_exporter.py +++ b/onnx4deeploy/core/base_exporter.py @@ -25,10 +25,10 @@ from onnx import helper from onnxruntime.training import artifacts -from DeepQuant.DeepQuant.Export4Deeploy import exportBrevitas +from DeepQuant.Export4Deeploy import exportBrevitas from .onnx_utils import print_model_info, randomize_onnx_initializers -from onnx4deeploy.transform.zo_transform import generate_zo_graph +from onnx4deeploy.transform.zo_transform import generate_weight_update_graph, generate_zo_graph class ExportMode(Enum): @@ -259,6 +259,7 @@ def setup_paths(self, mode: ExportMode) -> Dict[str, str]: { "network_infer": os.path.join(output_dir, "network_infer.onnx"), "network_zo_train": os.path.join(output_dir, "network_zo_train.onnx"), + "network_zo_update": os.path.join(output_dir, "network_zo_update.onnx"), } ) @@ -339,14 +340,13 @@ def export_inference(self, save_path: Optional[str] = None, quant: bool = False) if not quant: # initialize weights and biases for testing - for param in model.parameters(): + for name, param in model.named_parameters(): if param.requires_grad: torch.nn.init.normal_(param, mean=0.0, std=0.02) # Export to ONNX print("\n๐Ÿ“ค Exporting to ONNX...") opset_version = self.config.get("opset_version", 12) onnx_model = self._export_to_onnx(model, input_tensor, opset_version) - elif quant: # load weights. state_dict = torch.load(os.path.join(os.getcwd(), self.config.get("weights_path", "model_weights.pth")), map_location="cpu") @@ -538,9 +538,22 @@ def export_zo_training(self, save_path: Optional[str] = None, quant: bool = Fals print(f" Input shape: {input_shape}") # Export to ONNX - print("\n๐Ÿ“ค Exporting to ONNX...") - opset_version = self.config.get("opset_version", 12) - onnx_model = self._export_to_onnx(model, input_tensor, opset_version) + if not quant: + # initialize weights and biases for testing + for name, param in model.named_parameters(): + if param.requires_grad: + torch.nn.init.normal_(param, mean=0.0, std=0.02) + # Export to ONNX + print("\n๐Ÿ“ค Exporting to ONNX...") + opset_version = self.config.get("opset_version", 12) + onnx_model = self._export_to_onnx(model, input_tensor, opset_version) + elif quant: + # load weights. + state_dict = torch.load(os.path.join(os.getcwd(), self.config.get("weights_path", "model_weights.pth")), map_location="cpu") + model.load_state_dict(state_dict, strict=False) + print("\n๐Ÿ“ค Exporting to ONNX with quantization...") + # use DeepQuant to export to ONNX + onnx_model = exportBrevitas(model, input_tensor, debug=False) # Save onnx.save(onnx_model, self.paths["network_infer"]) @@ -582,8 +595,13 @@ def export_zo_training(self, save_path: Optional[str] = None, quant: bool = Fals generate_zo_graph( inference_onnx=self.paths["network_infer"], output_onnx=self.paths["network_zo_train"], - zo_config=self.config["zo"], + zo_config=self.config["zo"] ) + + generate_weight_update_graph( + onnx_path=self.paths["network_infer"], + output_path=self.paths["network_zo_update"], + zo_config=self.config["zo"]) # # Load training model and add gradient outputs # onnx_model = onnx.load(self.paths["network_train"]) diff --git a/onnx4deeploy/models/lightweight_cnn_exporter.py b/onnx4deeploy/models/lightweight_cnn_exporter.py index 7c0e67e..3c3f83b 100644 --- a/onnx4deeploy/models/lightweight_cnn_exporter.py +++ b/onnx4deeploy/models/lightweight_cnn_exporter.py @@ -52,7 +52,7 @@ def load_config(self) -> Dict[str, Any]: "zo": { "epsilon": 0.1, "seed": 42, - "noise_type": "eggroll", + "noise_type": "rademacher", } } diff --git a/onnx4deeploy/models/qlite_cnn_exporter.py b/onnx4deeploy/models/qlite_cnn_exporter.py index aa0da29..4e5f7d9 100644 --- a/onnx4deeploy/models/qlite_cnn_exporter.py +++ b/onnx4deeploy/models/qlite_cnn_exporter.py @@ -11,7 +11,7 @@ import torch import torch.onnx.utils -from DeepQuant.DeepQuant.ExportBrevitas import exportBrevitas +from DeepQuant.ExportBrevitas import exportBrevitas from ..core.base_exporter import BaseONNXExporter diff --git a/onnx4deeploy/transform/zo_transform.py b/onnx4deeploy/transform/zo_transform.py index 79b43d1..f79eaf1 100644 --- a/onnx4deeploy/transform/zo_transform.py +++ b/onnx4deeploy/transform/zo_transform.py @@ -25,8 +25,176 @@ def generate_zo_graph(inference_onnx:str, output_onnx:str, zo_config:dict) -> No exceptions=exceptions) ensure_all_tensor_shapes(model_path=output_onnx, output_path=output_onnx) - # append_cross_entropy_loss(output_onnx, output_onnx, label_name='label') + append_cross_entropy_loss(output_onnx, output_onnx, label_name='label') +def generate_weight_update_graph(onnx_path: str, output_path: str, zo_config: dict) -> None: + """ + Generates a weight update ONNX graph: for each weight/bias, creates a perturbation node + that updates the initializer in-place. No inputs, no outputs, just perturbation nodes. + """ + epsilon, seed, noise_type, exceptions = zo_config["epsilon"], zo_config["seed"], zo_config["noise_type"], zo_config.get("exceptions", []) + + model = onnx.load(onnx_path) + initializers = [init for init in model.graph.initializer if ("weight" in init.name or "bias" in init.name) and init.name not in exceptions] + nodes = [] + perturbation_counter = 0 + + for init in initializers: + perturbed_name = init.name # Overwrite the initializer directly + if noise_type == "gaussian": + node = helper.make_node( + "PerturbNormal", + inputs=[init.name], + outputs=[perturbed_name], + name=f"perturbnormal_{perturbed_name}", + domain="mezo", + seed=seed, + eps=epsilon, + idx=perturbation_counter, + doc_string="y = x + epsilon * RandomNormal(x, seed)" + ) + elif noise_type == "uniform": + node = helper.make_node( + "PerturbUniform", + inputs=[init.name], + outputs=[perturbed_name], + name=f"perturbuniform_{perturbed_name}", + domain="mezo", + idx=perturbation_counter, + seed=seed, + eps=epsilon, + low=-np.sqrt(3), + high=np.sqrt(3), + doc_string="y = x + epsilon * RandomUniform(x, seed)" + ) + elif noise_type == "eggroll": + # Shape annotation for intermediate outputs + # Prepare shapes for eggroll perturbation + noise_shape = [int(x) for x in init.dims] + a_shape = [noise_shape[0], 1] + b_shape = [int(np.prod(noise_shape[1:])), 1] + + # Create shape initializers + shape_a_tensor = helper.make_tensor( + name=f"shape_a_{init.name}", data_type=TensorProto.INT64, dims=[len(a_shape)], + vals=np.array(a_shape, dtype=np.int64) + ) + shape_b_tensor = helper.make_tensor( + name=f"shape_b_{init.name}", data_type=TensorProto.INT64, dims=[len(b_shape)], + vals=np.array(b_shape, dtype=np.int64) + ) + shape_input_tensor = helper.make_tensor( + name=f"shape_{init.name}", data_type=TensorProto.INT64, dims=[len(noise_shape)], + vals=np.array(noise_shape, dtype=np.int64) + ) + + # Optionally flatten if needed + if len(noise_shape) > 2: + shape_flat_tensor = helper.make_tensor( + name=f"shape_{init.name}_flat", data_type=TensorProto.INT64, dims=[2], + vals=np.array([a_shape[0], b_shape[0]], dtype=np.int64) + ) + # Flatten node + flatten_node = helper.make_node( + "Reshape", + inputs=[init.name, f"shape_{init.name}_flat"], + outputs=[f"flattened_{init.name}"], + name=f"flatten_{init.name}" + ) + nodes.append(flatten_node) + eggroll_input = f"flattened_{init.name}" + eggroll_output = f"flattened_{init.name}_perturbed" + # Unflatten node + unflatten_node = helper.make_node( + "Reshape", + inputs=[eggroll_output, f"shape_{init.name}"], + outputs=[init.name], + name=f"unflatten_{init.name}_perturbed" + ) + nodes.append(unflatten_node) + # Add flat shape initializer + initializers.extend([shape_flat_tensor]) + else: + eggroll_input = init.name + eggroll_output = f"{init.name}_perturbed" + + # Add shape initializers + initializers.extend([shape_a_tensor, shape_b_tensor, shape_input_tensor]) + + # Eggroll noise nodes + noise_node_a = helper.make_node( + "PerturbEggroll", + inputs=[f"shape_a_{init.name}"], + outputs=[f"a_{init.name}"], + name=f"gen_eggroll_noise_a_{init.name}", + seed=seed, + eps=epsilon, + idx=perturbation_counter, + domain="com.microsoft", + doc_string="a = RandomRademacher(x[0], seed)" + ) + noise_node_b = helper.make_node( + "PerturbEggroll", + inputs=[f"shape_b_{init.name}"], + outputs=[f"b_{init.name}"], + name=f"gen_eggroll_noise_b_{init.name}", + seed=seed, + eps=epsilon, + idx=perturbation_counter, + domain="com.microsoft", + doc_string="b = RandomRademacher(x[1:], seed)" + ) + gemm_node = helper.make_node( + "Gemm", + inputs=[f"a_{init.name}", f"b_{init.name}", eggroll_input], + outputs=[eggroll_output], + name=f"eggroll_gemm_{init.name}", + transA=0, + transB=1, + alpha=epsilon, + beta=0 + ) + + nodes.extend([noise_node_a, noise_node_b, gemm_node]) + + elif noise_type == "rademacher": + node = helper.make_node( + "PerturbRademacher", + inputs=[init.name], + outputs=[perturbed_name], + name=f"perturbrademacher_{perturbed_name}", + domain="mezo", + idx=perturbation_counter, + seed=seed, + eps=epsilon, + doc_string="y = x + epsilon * RandomRademacher(x, seed)" + ) + # Add other noise types as needed... + else: + raise ValueError(f"Unsupported noise_type: {noise_type}") + nodes.append(node) + perturbation_counter += 1 + + # Build a minimal graph: no inputs, no outputs, just initializers and nodes + graph = helper.make_graph( + nodes=nodes, + name="weight_update_graph", + inputs=[], # No inputs + outputs=[], # No outputs + initializer=initializers + ) + + # Use the same opset as the original model, plus mezo domain + standard_opset_version = next((op.version for op in model.opset_import if op.domain == ""), 13) + opset_list = [ + helper.make_opsetid("", standard_opset_version), + helper.make_opsetid("mezo", 1) + ] + new_model = helper.make_model(graph, producer_name="mezo-weight-update", opset_imports=opset_list) + onnx.save(new_model, output_path) + print(f"Saved weight update graph to: {output_path}") + + def inject_perturbation_nodes( onnx_path: str, output_path: str, From 681a3d6bb6e805a76d49295777434af0abf4c14f Mon Sep 17 00:00:00 2001 From: JanCSEM Date: Thu, 12 Mar 2026 18:48:32 +0000 Subject: [PATCH 15/24] update Deepquant --- DeepQuant | 2 +- requirements.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/DeepQuant b/DeepQuant index 19fd0aa..1d22563 160000 --- a/DeepQuant +++ b/DeepQuant @@ -1 +1 @@ -Subproject commit 19fd0aaafcf2a4b3ad841f893d19e46ae2261907 +Subproject commit 1d225630c3c8191404b680433ddde3f62f57332d diff --git a/requirements.txt b/requirements.txt index 2d3d5c5..94060c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,7 @@ onnxsim>=0.4.0 numpy>=1.24.0,<2.0.0 pyyaml>=6.0 brevitas>=0.12.0 +coloroma>=0.4.0 # Optional visualization dependencies # Install with: pip install -e ".[visualization]" From 2e43ebe07f4ff345110f89ac55ff4f5a9b97dbc3 Mon Sep 17 00:00:00 2001 From: JanCSEM Date: Sun, 15 Mar 2026 08:28:24 +0000 Subject: [PATCH 16/24] Align Dequant/Quant and RequantShift nodes with Deeploy requirements --- DeepQuant | 2 +- Onnx4Deeploy.py | 11 ++-- onnx4deeploy/core/base_exporter.py | 58 ++++++++++++++----- .../models/lightweight_cnn_exporter.py | 3 +- .../lightweight_cnn/qlite_cnn.py | 16 ++++- onnx4deeploy/models/qlite_cnn_exporter.py | 10 +++- onnx4deeploy/models/sleep_convit_exporter.py | 1 - onnx4deeploy/transform/zo_transform.py | 47 +++++++++------ 8 files changed, 102 insertions(+), 46 deletions(-) diff --git a/DeepQuant b/DeepQuant index 1d22563..5b1b375 160000 --- a/DeepQuant +++ b/DeepQuant @@ -1 +1 @@ -Subproject commit 1d225630c3c8191404b680433ddde3f62f57332d +Subproject commit 5b1b375fc1df4f6877cb6c216981292cdf463a41 diff --git a/Onnx4Deeploy.py b/Onnx4Deeploy.py index 83f7648..957c579 100644 --- a/Onnx4Deeploy.py +++ b/Onnx4Deeploy.py @@ -264,7 +264,7 @@ def generate_operator(operator_name: str, output_path: Optional[str] = None): sys.exit(1) -def generate_model(model_name: str, mode: str, output_path: Optional[str] = None): +def generate_model(model_name: str, mode: str, output_path: Optional[str] = None, noise_type: str = "gaussian"): """Generate model ONNX""" print(f"\n{'='*70}") print(f"๐Ÿš€ Generating model: {model_name} ({mode.upper()} mode)") @@ -315,10 +315,10 @@ def generate_model(model_name: str, mode: str, output_path: Optional[str] = None onnx_file = exporter.export_training() mode_desc = "Training mode" elif mode == "zo-train": - onnx_file = exporter.export_zo_training() + onnx_file = exporter.export_zo_training(noise_type=noise_type) mode_desc = "Zeroth-order Training mode" elif mode == "q-zo-train": - onnx_file = exporter.export_zo_training(quant=True) + onnx_file = exporter.export_zo_training(noise_type=noise_type, quant=True) mode_desc = "Quantized Zeroth-order Training mode" else: print(f"โŒ Unknown mode: {mode}") @@ -456,7 +456,8 @@ def main(): # Other options parser.add_argument("--examples", action="store_true", help="Show usage examples") - + parser.add_argument("--noise-type", type=str, choices=["gaussian", "uniform", "triangle", "rademacher", "eggroll"], + default="gaussian", help="Noise type for perturbation operators [default: gaussian]") # Parse arguments args = parser.parse_args() @@ -499,7 +500,7 @@ def main(): if args.operator: generate_operator(args.operator, args.output) elif args.model: - generate_model(args.model, args.mode, args.output) + generate_model(args.model, args.mode, args.output, args.noise_type) if __name__ == "__main__": diff --git a/onnx4deeploy/core/base_exporter.py b/onnx4deeploy/core/base_exporter.py index 220ee13..b08a4d8 100644 --- a/onnx4deeploy/core/base_exporter.py +++ b/onnx4deeploy/core/base_exporter.py @@ -348,10 +348,20 @@ def export_inference(self, save_path: Optional[str] = None, quant: bool = False) opset_version = self.config.get("opset_version", 12) onnx_model = self._export_to_onnx(model, input_tensor, opset_version) elif quant: - # load weights. + #load weights. state_dict = torch.load(os.path.join(os.getcwd(), self.config.get("weights_path", "model_weights.pth")), map_location="cpu") model.load_state_dict(state_dict, strict=False) - print("\n๐Ÿ“ค Exporting to ONNX with quantization...") + # Temporary HACK because channelwise scales not supported. + # for name, param in model.named_parameters(): + # if param.requires_grad: + # if "weight" in name or "bias" in name: + # torch.nn.init.normal_(param, mean=0.0, std=0.02) + # else: + # # scales + # torch.nn.init.uniform_(param, a=0.001, b = 0.03) + + # torch.nn.init.normal_(param, mean=0.0, std=0.02) + # print("\n๐Ÿ“ค Exporting to ONNX with quantization...") # use DeepQuant to export to ONNX onnx_model = exportBrevitas(model, input_tensor, debug=False) @@ -510,12 +520,14 @@ def export_training(self, save_path: Optional[str] = None) -> str: return self.paths["network"] - def export_zo_training(self, save_path: Optional[str] = None, quant: bool = False) -> str: + def export_zo_training(self, save_path: Optional[str] = None, noise_type: str = "gaussian", quant: bool = False) -> str: """ Export model in zeroth-order training mode. Args: - save_path: Optional custom save path""" + save_path: Optional custom save path + noise_type: Type of noise to use for perturbation + quant: Whether to apply quantization """ if save_path: self.save_path = save_path @@ -548,9 +560,19 @@ def export_zo_training(self, save_path: Optional[str] = None, quant: bool = Fals opset_version = self.config.get("opset_version", 12) onnx_model = self._export_to_onnx(model, input_tensor, opset_version) elif quant: - # load weights. + #load weights. state_dict = torch.load(os.path.join(os.getcwd(), self.config.get("weights_path", "model_weights.pth")), map_location="cpu") model.load_state_dict(state_dict, strict=False) + # Temporary HACK because channelwise scales not supported. + # for name, param in model.named_parameters(): + # if param.requires_grad: + # if "weight" in name or "bias" in name: + # torch.nn.init.normal_(param, mean=0.0, std=0.02) + # else: + # # scales + # torch.nn.init.uniform_(param, a=0.001, b = 0.03) + + # torch.nn.init.normal_(param, mean=0.0, std=0.02) print("\n๐Ÿ“ค Exporting to ONNX with quantization...") # use DeepQuant to export to ONNX onnx_model = exportBrevitas(model, input_tensor, debug=False) @@ -591,17 +613,20 @@ def export_zo_training(self, save_path: Optional[str] = None, quant: bool = Fals # Transform model for zeroth-order training (e.g., add noise nodes, modify outputs) print("\n๐Ÿ”ง Transforming model for zeroth-order training...") # Randomize initializers for testing - onnx_model = randomize_onnx_initializers(onnx_model) + if not quant: + onnx_model = randomize_onnx_initializers(onnx_model) generate_zo_graph( inference_onnx=self.paths["network_infer"], output_onnx=self.paths["network_zo_train"], - zo_config=self.config["zo"] + zo_config=self.config["zo"], + noise_type=noise_type, ) generate_weight_update_graph( onnx_path=self.paths["network_infer"], output_path=self.paths["network_zo_update"], - zo_config=self.config["zo"]) + zo_config=self.config["zo"], + noise_type=noise_type) # # Load training model and add gradient outputs # onnx_model = onnx.load(self.paths["network_train"]) @@ -634,7 +659,7 @@ def export_zo_training(self, save_path: Optional[str] = None, quant: bool = Fals # Create test input/output print("\n๐Ÿงช Creating test input/output...") - self._create_test_data() + self._create_test_data(mode=ExportMode.ZO_TRAINING) # # Add optimizer (SGD) nodes # print("\nโž• Adding SGD optimizer nodes...") @@ -645,7 +670,7 @@ def export_zo_training(self, save_path: Optional[str] = None, quant: bool = Fals print(f" Final model: {self.paths['network']}") print(f"{'='*60}\n") - def _create_test_data(self): + def _create_test_data(self, mode=ExportMode.INFERENCE): """ Create test input/output data for training. @@ -664,9 +689,9 @@ def _create_test_data(self): # Create test input input_shape = self.get_input_shape() test_input = np.random.randn(*input_shape).astype(np.float32) - - # Run ONNX inference to get output session = ort.InferenceSession(self.paths["network_infer"]) + input_names = [i.name for i in session.get_inputs()] + input_name = session.get_inputs()[0].name test_output = session.run(None, {input_name: test_input})[0] @@ -674,14 +699,19 @@ def _create_test_data(self): save_path = Path(self.paths["output_dir"]) save_path.mkdir(parents=True, exist_ok=True) - np.savez(save_path / "inputs.npz", input=test_input) + if mode == ExportMode.ZO_TRAINING: + test_label = np.random.randint(0, self.config["num_classes"], size=(input_shape[0],1)).astype(np.int8) + print(f" Generated test labels with shape: {test_label.shape}") + np.savez(save_path / "inputs.npz", input=test_input, label=test_label) + else: + np.savez(save_path / "inputs.npz", input=test_input) np.savez(save_path / "outputs.npz", output=test_output) print(" โœ… Saved test data (ONNX reference):") print(f" Input: {save_path / 'inputs.npz'} shape={test_input.shape}") print(f" Output: {save_path / 'outputs.npz'} shape={test_output.shape}") except Exception as e: - print(f"โš ๏ธ Failed to create test data: {e}") + raise RuntimeError(f"Failed to create test data: {e}") def _add_optimizer_nodes(self): """ diff --git a/onnx4deeploy/models/lightweight_cnn_exporter.py b/onnx4deeploy/models/lightweight_cnn_exporter.py index 3c3f83b..36d3160 100644 --- a/onnx4deeploy/models/lightweight_cnn_exporter.py +++ b/onnx4deeploy/models/lightweight_cnn_exporter.py @@ -51,8 +51,7 @@ def load_config(self) -> Dict[str, Any]: "custom_trainable_params": [], "zo": { "epsilon": 0.1, - "seed": 42, - "noise_type": "rademacher", + "seed": 42 } } diff --git a/onnx4deeploy/models/pytorch_models/lightweight_cnn/qlite_cnn.py b/onnx4deeploy/models/pytorch_models/lightweight_cnn/qlite_cnn.py index 7d52b56..ad3bdff 100644 --- a/onnx4deeploy/models/pytorch_models/lightweight_cnn/qlite_cnn.py +++ b/onnx4deeploy/models/pytorch_models/lightweight_cnn/qlite_cnn.py @@ -39,7 +39,19 @@ def __init__(self, "weight_bit_width": 8, "bias_quant": Int32Bias, "input_quant": Int8ActPerTensorFloat, - "weight_quant":Int8WeightPerChannelFloat, + "weight_quant":Int8WeightPerChannelFloat, #no channel wise support in deeploy yet. + #"weight_quant":Int8WeightPerTensorFloat, + "output_quant": None, + "return_quant_tensor": True + } + + self.convAndLinQuantParamsOut = { + "bias": True, + "weight_bit_width": 8, + "bias_quant": Int32Bias, + "input_quant": Int8ActPerTensorFloat, + "weight_quant":Int8WeightPerChannelFloat,# no channel wise support in deeploy yet. + #"weight_quant": Int8WeightPerTensorFloat, "output_quant": Int8ActPerTensorFloat, "return_quant_tensor": True } @@ -76,7 +88,7 @@ def __init__(self, self.relu4 = qnn.QuantReLU(bit_width=8, return_quant_tensor=True) self.fc = qnn.QuantLinear(self.fc_channels, num_classes, - **self.convAndLinQuantParams) # Output: num_classes + **self.convAndLinQuantParamsOut) # Output: num_classes def forward(self, x): diff --git a/onnx4deeploy/models/qlite_cnn_exporter.py b/onnx4deeploy/models/qlite_cnn_exporter.py index 4e5f7d9..afd627a 100644 --- a/onnx4deeploy/models/qlite_cnn_exporter.py +++ b/onnx4deeploy/models/qlite_cnn_exporter.py @@ -10,9 +10,11 @@ import numpy as np import torch import torch.onnx.utils +from brevitas.quant_tensor import QuantTensor from DeepQuant.ExportBrevitas import exportBrevitas + from ..core.base_exporter import BaseONNXExporter # Import QLiteCNN PyTorch model from new location @@ -59,8 +61,7 @@ def load_config(self) -> Dict[str, Any]: "custom_trainable_params": [], "zo": { "epsilon": 0.1, - "seed": 42, - "noise_type": "eggroll", + "seed": 42 }, "weights_path":"onnx4deeploy/models/pytorch_models/lightweight_cnn/qlite_cnn.pth" } @@ -177,6 +178,8 @@ def save_test_data(self, model: torch.nn.Module, save_dir: str): with torch.no_grad(): input_tensor = torch.from_numpy(test_input) output_tensor = model(input_tensor) + if isinstance(output_tensor, QuantTensor): + output_tensor = output_tensor.value test_output = output_tensor.numpy() # Restore training mode if needed @@ -219,9 +222,10 @@ def _build_rqs_map(self, graph: gs.Graph, brevitas_scales: Dict[str, Any]) -> Di w_scale = brevitas_scales.get(f"{layer_name}.weight_quant") out_scale = brevitas_scales.get(f"{layer_name}.output_quant") + if in_scale is not None and w_scale is not None and out_scale is not None: # Conv accumulation scale = input_scale * weight_scale - # Flatten to support Brevitas per-channel deeply nested precision lists + w_flat = np.array(w_scale).flatten() src_scale = in_scale * w_flat diff --git a/onnx4deeploy/models/sleep_convit_exporter.py b/onnx4deeploy/models/sleep_convit_exporter.py index 534197e..816b514 100644 --- a/onnx4deeploy/models/sleep_convit_exporter.py +++ b/onnx4deeploy/models/sleep_convit_exporter.py @@ -82,7 +82,6 @@ def load_config(self) -> Dict[str, Any]: "zo": { "epsilon": 0.1, "seed": 42, - "noise_type": "uniform", "exceptions": "node_matmul_2" } } diff --git a/onnx4deeploy/transform/zo_transform.py b/onnx4deeploy/transform/zo_transform.py index f79eaf1..0285d49 100644 --- a/onnx4deeploy/transform/zo_transform.py +++ b/onnx4deeploy/transform/zo_transform.py @@ -10,10 +10,10 @@ from onnx4deeploy.transform.model_transform import ensure_all_tensor_shapes -def generate_zo_graph(inference_onnx:str, output_onnx:str, zo_config:dict) -> None: +def generate_zo_graph(inference_onnx:str, output_onnx:str, zo_config:dict, noise_type: str) -> None: """ Generate MeZO ONNX graph for model based on its inference onnx""" - epsilon, seed, noise_type, exceptions = zo_config["epsilon"], zo_config["seed"], zo_config["noise_type"], zo_config.get("exceptions", []) + epsilon, seed, exceptions = zo_config["epsilon"], zo_config["seed"], zo_config.get("exceptions", []) base_path = os.path.dirname(output_onnx) os.makedirs(base_path, exist_ok=True) @@ -27,15 +27,16 @@ def generate_zo_graph(inference_onnx:str, output_onnx:str, zo_config:dict) -> No ensure_all_tensor_shapes(model_path=output_onnx, output_path=output_onnx) append_cross_entropy_loss(output_onnx, output_onnx, label_name='label') -def generate_weight_update_graph(onnx_path: str, output_path: str, zo_config: dict) -> None: +def generate_weight_update_graph(onnx_path: str, output_path: str, zo_config: dict, noise_type: str) -> None: """ Generates a weight update ONNX graph: for each weight/bias, creates a perturbation node that updates the initializer in-place. No inputs, no outputs, just perturbation nodes. """ - epsilon, seed, noise_type, exceptions = zo_config["epsilon"], zo_config["seed"], zo_config["noise_type"], zo_config.get("exceptions", []) + epsilon, seed, exceptions = zo_config["epsilon"], zo_config["seed"], zo_config.get("exceptions", []) model = onnx.load(onnx_path) initializers = [init for init in model.graph.initializer if ("weight" in init.name or "bias" in init.name) and init.name not in exceptions] + new_initializers = list(initializers) # Start with the original initializers and add new ones as needed nodes = [] perturbation_counter = 0 @@ -53,6 +54,9 @@ def generate_weight_update_graph(onnx_path: str, output_path: str, zo_config: di idx=perturbation_counter, doc_string="y = x + epsilon * RandomNormal(x, seed)" ) + nodes.append(node) + perturbation_counter += 1 + elif noise_type == "uniform": node = helper.make_node( "PerturbUniform", @@ -67,13 +71,16 @@ def generate_weight_update_graph(onnx_path: str, output_path: str, zo_config: di high=np.sqrt(3), doc_string="y = x + epsilon * RandomUniform(x, seed)" ) + nodes.append(node) + perturbation_counter += 1 + elif noise_type == "eggroll": # Shape annotation for intermediate outputs - # Prepare shapes for eggroll perturbation - noise_shape = [int(x) for x in init.dims] + # Prepare shapes for eggroll perturbation + noise_shape = list(onnx.numpy_helper.to_array(init).shape) a_shape = [noise_shape[0], 1] b_shape = [int(np.prod(noise_shape[1:])), 1] - + print(F"noise_shape: {noise_shape}, a_shape: {a_shape}, b_shape: {b_shape}") # Create shape initializers shape_a_tensor = helper.make_tensor( name=f"shape_a_{init.name}", data_type=TensorProto.INT64, dims=[len(a_shape)], @@ -113,13 +120,13 @@ def generate_weight_update_graph(onnx_path: str, output_path: str, zo_config: di ) nodes.append(unflatten_node) # Add flat shape initializer - initializers.extend([shape_flat_tensor]) + new_initializers.append([shape_flat_tensor]) else: eggroll_input = init.name eggroll_output = f"{init.name}_perturbed" # Add shape initializers - initializers.extend([shape_a_tensor, shape_b_tensor, shape_input_tensor]) + new_initializers.extend([shape_a_tensor, shape_b_tensor, shape_input_tensor]) # Eggroll noise nodes noise_node_a = helper.make_node( @@ -156,7 +163,8 @@ def generate_weight_update_graph(onnx_path: str, output_path: str, zo_config: di ) nodes.extend([noise_node_a, noise_node_b, gemm_node]) - + perturbation_counter += 2 + elif noise_type == "rademacher": node = helper.make_node( "PerturbRademacher", @@ -169,11 +177,11 @@ def generate_weight_update_graph(onnx_path: str, output_path: str, zo_config: di eps=epsilon, doc_string="y = x + epsilon * RandomRademacher(x, seed)" ) - # Add other noise types as needed... + nodes.append(node) + perturbation_counter += 1 + else: raise ValueError(f"Unsupported noise_type: {noise_type}") - nodes.append(node) - perturbation_counter += 1 # Build a minimal graph: no inputs, no outputs, just initializers and nodes graph = helper.make_graph( @@ -555,23 +563,26 @@ def append_cross_entropy_loss(onnx_path, output_path, label_name='y', logits_out batch_dim = first_dim.dim_param # add the new label input using resolved batch_dim - label_vi = helper.make_tensor_value_info(label_input_name, TensorProto.INT8, [batch_dim]) + label_vi = helper.make_tensor_value_info(label_input_name, TensorProto.INT64, [batch_dim, 1]) graph.input.append(label_vi) # create loss node (standard SoftmaxCrossEntropyLoss) with proper attribute - loss_name = "loss" + logprob = "log_prob" loss_node = helper.make_node( "SoftmaxCrossEntropyLoss", inputs=[logits_name, label_input_name], - outputs=[loss_name], + outputs=[logprob], name="CrossEntropyLoss", reduction=reduction, ) graph.node.append(loss_node) - # replace graph outputs with the scalar loss + # replace graph outputs with the log prob + output_shape = [d.dim_value if d.HasField("dim_value") else d.dim_param + for d in graph.output[0].type.tensor_type.shape.dim] + del graph.output[:] - graph.output.append(helper.make_tensor_value_info(loss_name, TensorProto.FLOAT, [])) + graph.output.append(helper.make_tensor_value_info(logprob, TensorProto.FLOAT, output_shape)) # try to infer shapes and save try: From 024c2f4dbe178efc081e06fc5b96b6b120a679b7 Mon Sep 17 00:00:00 2001 From: JanCSEM Date: Sun, 15 Mar 2026 15:36:22 +0000 Subject: [PATCH 17/24] Add quantized Uniform and Rademacher perturbation --- Onnx4Deeploy.py | 2 + gen_zo_model_tests.sh | 15 +++ onnx4deeploy/operators/__init__.py | 4 +- onnx4deeploy/operators/config.yaml | 6 + onnx4deeploy/operators/perturbrademacher.py | 2 +- .../operators/rqsperturbrademacher.py | 118 ++++++++++++++++++ onnx4deeploy/operators/rqsperturbuniform.py | 118 ++++++++++++++++++ 7 files changed, 263 insertions(+), 2 deletions(-) create mode 100755 gen_zo_model_tests.sh create mode 100644 onnx4deeploy/operators/rqsperturbrademacher.py create mode 100644 onnx4deeploy/operators/rqsperturbuniform.py diff --git a/Onnx4Deeploy.py b/Onnx4Deeploy.py index 957c579..dbf3963 100644 --- a/Onnx4Deeploy.py +++ b/Onnx4Deeploy.py @@ -183,6 +183,8 @@ def list_available_operators(): "PerturbTriangle": "Perturb input with triangle random noise", "PerturbRademacher": "Perturb input with Rademacher random noise", "PerturbEggroll": "Perturb input with Eggroll random noise", + "RQSPerturbrademacher": "Perturb input with quantized Rademacher random noise", + "RQSPerturbUniform": "Perturb input with quantized Uniform random noise", # Others "ReduceSum": "Sum reduction", diff --git a/gen_zo_model_tests.sh b/gen_zo_model_tests.sh new file mode 100755 index 0000000..76bb7c9 --- /dev/null +++ b/gen_zo_model_tests.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# LiteCNN +python3 Onnx4Deeploy.py -model LightweightCNN -mode infer -o LiteCNN +python3 Onnx4Deeploy.py -model LightweightCNN -mode zo-train -o LiteCNN-Rad --noise-type rademacher +python3 Onnx4Deeploy.py -model LightweightCNN -mode zo-train -o LiteCNN-Lorp --noise-type eggroll +python3 Onnx4Deeploy.py -model LightweightCNN -mode zo-train -o LiteCNN-Uniform --noise-type uniform +python3 Onnx4Deeploy.py -model LightweightCNN -mode zo-train -o LiteCNN-Gaussian --noise-type gaussian + +# QLiteCNN +python3 Onnx4Deeploy.py -model QLiteCNN -mode q-infer -o QLiteCNN +python3 Onnx4Deeploy.py -model QLiteCNN -mode q-zo-train -o QLiteCNN-Rad --noise-type rademacher +python3 Onnx4Deeploy.py -model QLiteCNN -mode q-zo-train -o QLiteCNN-Lorp --noise-type eggroll +python3 Onnx4Deeploy.py -model QLiteCNN -mode q-zo-train -o QLiteCNN-Uniform --noise-type uniform +python3 Onnx4Deeploy.py -model QLiteCNN -mode q-zo-train -o QLiteCNN-Gaussian --noise-type gaussian \ No newline at end of file diff --git a/onnx4deeploy/operators/__init__.py b/onnx4deeploy/operators/__init__.py index 3a0a946..f609d62 100644 --- a/onnx4deeploy/operators/__init__.py +++ b/onnx4deeploy/operators/__init__.py @@ -78,5 +78,7 @@ "PerturbEggrollOperatorTest", "PerturbRademacherOperatorTest", "PerturbUniformOperatorTest", - "PerturbTriangleOperatorTest" + "PerturbTriangleOperatorTest", + "RQSPerturbRademacherOperatorTest", + "RQSPerturbUniformOperatorTest", ] diff --git a/onnx4deeploy/operators/config.yaml b/onnx4deeploy/operators/config.yaml index 6b9b987..3f8f98c 100644 --- a/onnx4deeploy/operators/config.yaml +++ b/onnx4deeploy/operators/config.yaml @@ -13,6 +13,12 @@ perturbrademacher: perturbeggroll: input_shape: [128, 48] +rqsperturbrademacher: + input_shape: [128, 48] + +rqsperturbuniform: + input_shape: [128, 48] + conv2d: input_shape: [1, 1, 28, 28] kernel_size: 5 diff --git a/onnx4deeploy/operators/perturbrademacher.py b/onnx4deeploy/operators/perturbrademacher.py index 9949188..5fb8f71 100644 --- a/onnx4deeploy/operators/perturbrademacher.py +++ b/onnx4deeploy/operators/perturbrademacher.py @@ -83,7 +83,7 @@ def create_model(self, graph, opset_version: int = 13): ) return model - + def run_inference(self, onnx_file: str, inputs: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]: """ Run inference using custom emulation diff --git a/onnx4deeploy/operators/rqsperturbrademacher.py b/onnx4deeploy/operators/rqsperturbrademacher.py new file mode 100644 index 0000000..fc9c572 --- /dev/null +++ b/onnx4deeploy/operators/rqsperturbrademacher.py @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: MIT + +"""PerturbRademacher operator test implementation.""" + +from typing import Any, Dict, Tuple + +import numpy as np +import onnxruntime as ort +from onnx import TensorProto, helper + +from .base_operator import BaseOperatorTest + + +class RQSPerturbRademacherOperatorTest(BaseOperatorTest): + """Test generator for ONNX PerturbRademacher operator (custom/training op).""" + + def __init__(self, config_path=None, save_path=None): + super().__init__(config_path, save_path) + self.input_shape = None + self.num_classes = None + self.batch_size = None + + def get_operator_name(self) -> str: + return "PerturbRademacher" + + def load_config(self) -> Dict[str, Any]: + """Load PerturbRademacher-specific configuration.""" + config = super().load_config() + + pn_config = config.get("perturbrademacher", {}) + self.input_shape = tuple(pn_config["input_shape"]) + return config + + + def generate_inputs(self) -> np.ndarray: + """Generate input with both positive and negative values.""" + x = np.random.randn(*self.input_shape).astype(np.float32) + # quantize: + max_val = np.max(np.abs(x), axis=1) + s = max_val / 127.0 + s[s == 0] = 1.0 # Avoid division by zero + mul = np.round(0.01 / s * (2**15)).astype(np.int32) # quantized multiplier for perturbation + x_quantized = np.round(x / s[:, np.newaxis]) * s[:, np.newaxis] + return {"x": x_quantized.astype(np.float32), "mul": mul.astype(np.float32)} + + def create_onnx_graph(self, inputs: Dict[str, np.ndarray]): + """Create ONNX graph for PerturbRademacher operator.""" + # Input tensors (without loss_grad for the final model) + x_tensor = helper.make_tensor_value_info( + "x", TensorProto.FLOAT, self.input_shape + ) + mul_initializer = helper.make_tensor( + name="mul", + data_type=TensorProto.FLOAT, + dims=(self.input_shape[0],1), + vals=inputs["mul"], + ) + # Output tensor + perturbed_x_tensor = helper.make_tensor_value_info( + "perturbed_x", TensorProto.FLOAT, self.input_shape + ) + + # PerturbRademacher node (without loss_grad input) + perturb_node = helper.make_node( + "RQSPerturbRademacher", + inputs=["x", "mul"], + outputs=["perturbed_x"], + seed=42, + idx=0, + div=2**15, + n_levels=256, + signed=1, + name="rqs_perturb_rademacher_node", + domain="com.microsoft" + ) + + # Graph + graph = helper.make_graph( + [perturb_node], + "perturb_rademacher_graph", + [x_tensor], + [perturbed_x_tensor], + [mul_initializer] + ) + + return graph + + def create_model(self, graph, opset_version: int = 13): + """Create ONNX model for PerturbRademacher with custom domain.""" + model = helper.make_model( + graph, + producer_name=f"{self.get_operator_name().lower()}_test", + opset_imports=[ + helper.make_opsetid("", opset_version), + helper.make_opsetid("com.microsoft", 1), + ], + ) + + return model + + def run_inference(self, onnx_file: str, inputs: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]: + """ + Run inference using custom emulation + """ + # perturbation is built from -1's and 1's + perturbation = np.random.choice([-1, 1], size=self.input_shape).astype(np.int8) + perturbation = perturbation * inputs["mul"].reshape(-1, 1) // 2 ** 15 + perturbed_x = inputs["x"] + perturbation + + return {"perturbed_x": perturbed_x} + + def compute_expected_output(self, inputs: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]: + """ + Return None to skip validation - this is a custom operator. + """ + return None diff --git a/onnx4deeploy/operators/rqsperturbuniform.py b/onnx4deeploy/operators/rqsperturbuniform.py new file mode 100644 index 0000000..26b67ee --- /dev/null +++ b/onnx4deeploy/operators/rqsperturbuniform.py @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: MIT + +"""PerturbRademacher operator test implementation.""" + +from typing import Any, Dict, Tuple + +import numpy as np +import onnxruntime as ort +from onnx import TensorProto, helper + +from .base_operator import BaseOperatorTest + + +class RQSPerturbUniformOperatorTest(BaseOperatorTest): + """Test generator for ONNX PerturbUniform operator (custom/training op).""" + + def __init__(self, config_path=None, save_path=None): + super().__init__(config_path, save_path) + self.input_shape = None + self.num_classes = None + self.batch_size = None + + def get_operator_name(self) -> str: + return "PerturbUniform" + + def load_config(self) -> Dict[str, Any]: + """Load PerturbUniform-specific configuration.""" + config = super().load_config() + + pn_config = config.get("perturbuniform", {}) + self.input_shape = tuple(pn_config["input_shape"]) + return config + + + def generate_inputs(self) -> np.ndarray: + """Generate input with both positive and negative values.""" + x = np.random.randn(*self.input_shape).astype(np.float32) + # quantize: + max_val = np.max(np.abs(x), axis=1) + s = max_val / 127.0 + s[s == 0] = 1.0 # Avoid division by zero + mul = np.round(0.01*np.sqrt(3) / s * (2**15)).astype(np.int32) # quantized multiplier for perturbation + x_quantized = np.round(x / s[:, np.newaxis]) * s[:, np.newaxis] + return {"x": x_quantized.astype(np.float32), "mul": mul.astype(np.float32)} + + def create_onnx_graph(self, inputs: Dict[str, np.ndarray]): + """Create ONNX graph for PerturbUniform operator.""" + # Input tensors (without loss_grad for the final model) + x_tensor = helper.make_tensor_value_info( + "x", TensorProto.FLOAT, self.input_shape + ) + mul_initializer = helper.make_tensor( + name="mul", + data_type=TensorProto.FLOAT, + dims=(self.input_shape[0],1), + vals=inputs["mul"], + ) + # Output tensor + perturbed_x_tensor = helper.make_tensor_value_info( + "perturbed_x", TensorProto.FLOAT, self.input_shape + ) + + # PerturbUniform node (without loss_grad input) + perturb_node = helper.make_node( + "RQSPerturbUniform", + inputs=["x", "mul"], + outputs=["perturbed_x"], + seed=42, + idx=0, + div=2**15, + n_levels=256, + signed=1, + name="rqs_perturb_uniform_node", + domain="com.microsoft" + ) + + # Graph + graph = helper.make_graph( + [perturb_node], + "perturb_uniform_graph", + [x_tensor], + [perturbed_x_tensor], + [mul_initializer] + ) + + return graph + + def create_model(self, graph, opset_version: int = 13): + """Create ONNX model for PerturbUniform with custom domain.""" + model = helper.make_model( + graph, + producer_name=f"{self.get_operator_name().lower()}_test", + opset_imports=[ + helper.make_opsetid("", opset_version), + helper.make_opsetid("com.microsoft", 1), + ], + ) + + return model + + def run_inference(self, onnx_file: str, inputs: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]: + """ + Run inference using custom emulation + """ + # perturbation is built from -1's and 1's + perturbation = np.random.randint(-1, 2, size=self.input_shape).astype(np.int8) + perturbation = perturbation * inputs["mul"].reshape(-1, 1) // 2 ** 15 + perturbed_x = inputs["x"] + perturbation + + return {"perturbed_x": perturbed_x} + + def compute_expected_output(self, inputs: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]: + """ + Return None to skip validation - this is a custom operator. + """ + return None From 1aba7147d0e0d7fbfc3a7957b3051743c3fbb4f1 Mon Sep 17 00:00:00 2001 From: JanCSEM Date: Tue, 17 Mar 2026 09:33:36 +0000 Subject: [PATCH 18/24] Add support for QSleepViT --- DeepQuant | 2 +- Onnx4Deeploy.py | 9 +- onnx4deeploy/core/base_exporter.py | 37 +- onnx4deeploy/models/__init__.py | 2 + .../sleep_convit/qsleep_convit.pth | Bin 335978 -> 335978 bytes .../sleep_convit/qsleep_convit.py | 251 +-- .../sleep_convit/qsleep_convit_scales.json | 1438 ++++++++--------- .../sleep_convit/sleep_convit.py | 3 +- onnx4deeploy/models/qsleep_convit_exporter.py | 245 +++ onnx4deeploy/models/sleep_convit_exporter.py | 2 +- onnx4deeploy/operators/base_operator.py | 2 +- .../operators/rqsperturbrademacher.py | 5 +- onnx4deeploy/operators/rqsperturbuniform.py | 7 +- onnx4deeploy/optimization/shape_optimizer.py | 30 +- onnx4deeploy/transform/zo_transform.py | 207 ++- 15 files changed, 1354 insertions(+), 886 deletions(-) mode change 100644 => 100755 onnx4deeploy/models/pytorch_models/sleep_convit/qsleep_convit.pth mode change 100644 => 100755 onnx4deeploy/models/pytorch_models/sleep_convit/qsleep_convit_scales.json create mode 100644 onnx4deeploy/models/qsleep_convit_exporter.py diff --git a/DeepQuant b/DeepQuant index 5b1b375..5aa59a3 160000 --- a/DeepQuant +++ b/DeepQuant @@ -1 +1 @@ -Subproject commit 5b1b375fc1df4f6877cb6c216981292cdf463a41 +Subproject commit 5aa59a376ad0cfc98543df15382863aefd6b6682 diff --git a/Onnx4Deeploy.py b/Onnx4Deeploy.py index dbf3963..565131c 100644 --- a/Onnx4Deeploy.py +++ b/Onnx4Deeploy.py @@ -42,6 +42,7 @@ def list_available_models(): ResNetExporter, SimpleMlpExporter, SleepConViTExporter, + QSleepConViTExporter ) models = { @@ -147,6 +148,12 @@ def list_available_models(): "input_shape": "(B, 1, 28, 28)", "classes": 10, }, + "QSleepConViT": { + "class": QSleepConViTExporter, + "description": "QLite SleepConViT (Quantized Vision Transformer for Sleep Stage Classification)", + "input_shape": "(B, 1, 3000)", + "classes": 5 + } } return models @@ -458,7 +465,7 @@ def main(): # Other options parser.add_argument("--examples", action="store_true", help="Show usage examples") - parser.add_argument("--noise-type", type=str, choices=["gaussian", "uniform", "triangle", "rademacher", "eggroll"], + parser.add_argument("--noise-type", type=str, choices=["gaussian", "uniform", "triangle", "rademacher", "eggroll", "rqs_rademacher", "rqs_uniform"], default="gaussian", help="Noise type for perturbation operators [default: gaussian]") # Parse arguments args = parser.parse_args() diff --git a/onnx4deeploy/core/base_exporter.py b/onnx4deeploy/core/base_exporter.py index b08a4d8..3c6e9a8 100644 --- a/onnx4deeploy/core/base_exporter.py +++ b/onnx4deeploy/core/base_exporter.py @@ -351,20 +351,10 @@ def export_inference(self, save_path: Optional[str] = None, quant: bool = False) #load weights. state_dict = torch.load(os.path.join(os.getcwd(), self.config.get("weights_path", "model_weights.pth")), map_location="cpu") model.load_state_dict(state_dict, strict=False) - # Temporary HACK because channelwise scales not supported. - # for name, param in model.named_parameters(): - # if param.requires_grad: - # if "weight" in name or "bias" in name: - # torch.nn.init.normal_(param, mean=0.0, std=0.02) - # else: - # # scales - # torch.nn.init.uniform_(param, a=0.001, b = 0.03) - - # torch.nn.init.normal_(param, mean=0.0, std=0.02) - # print("\n๐Ÿ“ค Exporting to ONNX with quantization...") - # use DeepQuant to export to ONNX onnx_model = exportBrevitas(model, input_tensor, debug=False) - + for name, param in model.named_parameters(): + if param.requires_grad and "weight" in name and "conv" in name: + torch.nn.init.normal_(param, mean=0.0, std=0.02) # Save onnx.save(onnx_model, self.paths["network"]) print(f"โœ… ONNX model saved: {self.paths['network']}") @@ -563,16 +553,6 @@ def export_zo_training(self, save_path: Optional[str] = None, noise_type: str = #load weights. state_dict = torch.load(os.path.join(os.getcwd(), self.config.get("weights_path", "model_weights.pth")), map_location="cpu") model.load_state_dict(state_dict, strict=False) - # Temporary HACK because channelwise scales not supported. - # for name, param in model.named_parameters(): - # if param.requires_grad: - # if "weight" in name or "bias" in name: - # torch.nn.init.normal_(param, mean=0.0, std=0.02) - # else: - # # scales - # torch.nn.init.uniform_(param, a=0.001, b = 0.03) - - # torch.nn.init.normal_(param, mean=0.0, std=0.02) print("\n๐Ÿ“ค Exporting to ONNX with quantization...") # use DeepQuant to export to ONNX onnx_model = exportBrevitas(model, input_tensor, debug=False) @@ -670,7 +650,7 @@ def export_zo_training(self, save_path: Optional[str] = None, noise_type: str = print(f" Final model: {self.paths['network']}") print(f"{'='*60}\n") - def _create_test_data(self, mode=ExportMode.INFERENCE): + def _create_test_data(self, mode=ExportMode.INFERENCE, quant=False): """ Create test input/output data for training. @@ -689,7 +669,14 @@ def _create_test_data(self, mode=ExportMode.INFERENCE): # Create test input input_shape = self.get_input_shape() test_input = np.random.randn(*input_shape).astype(np.float32) - session = ort.InferenceSession(self.paths["network_infer"]) + from onnxruntime_extensions import onnx_op, PyOp, get_library_path + from DeepQuant.QuantDequantOnnx import requant_shift_onnx + sess_options = ort.SessionOptions() + + sess_options.register_custom_ops_library(get_library_path()) + + session = ort.InferenceSession(self.paths["network_infer"], + sess_options=sess_options) input_names = [i.name for i in session.get_inputs()] input_name = session.get_inputs()[0].name diff --git a/onnx4deeploy/models/__init__.py b/onnx4deeploy/models/__init__.py index 0b45494..36d9512 100644 --- a/onnx4deeploy/models/__init__.py +++ b/onnx4deeploy/models/__init__.py @@ -15,6 +15,7 @@ from .resnet_exporter import ResNetExporter from .simple_mlp_exporter import SimpleMlpExporter from .sleep_convit_exporter import SleepConViTExporter +from .qsleep_convit_exporter import QSleepConViTExporter __all__ = [ "CCTExporter", @@ -28,4 +29,5 @@ "MobileViTExporter", "MambaExporter", "SleepConViTExporter", + "QSleepConViTExporter" ] diff --git a/onnx4deeploy/models/pytorch_models/sleep_convit/qsleep_convit.pth b/onnx4deeploy/models/pytorch_models/sleep_convit/qsleep_convit.pth old mode 100644 new mode 100755 index 9dadf8ac4ea655cb79abf1b0bcebaa7e7f6c9621..45c030a5b56c4676b1281e98070321e05adde8c0 GIT binary patch delta 109803 zcmXV%cOcc@8^|@1R^(P(%11JJlz@rPsyyPI(*{~L8eP3N=POV-opTVUH{G@ zBVrM~d}TC~X)6{O-nHb8h3X5uJtW7d&iNx>Cn6$JcRaN~OzXI**Jxk<-PD7;gF-MT zA(;Q~N{0j?ShEmJQV3=y1pD3hjC&{q6Q6L4`Ut_o{txCZ1Y08nYh@w=b0Jub5Uf!M zc322DRS4!S1d9@a^$Nixw?3r{Q&hR@{|DPvIGT6IlFLAy}0VtWpT}q_tQu)MCjc3c*Bdt=O5d8|V(#D6|n-58{=| z#Q&reIfVDH@|Pot$?EIyFV z^{dS%A?I$>lRDWfKd%J#M=vFZ?_Y#FLI2Tb<&#OrnB%}TRp6=GXrdMH2yb^CLYrBhAo1G}ThFdxPd|^)Zbez_{L+FCCyGM$MpFo&JSJ^qhMzyqg?D?;VtrU1j=rMD z&hTaD;_=x!u(mFhmwr|phlU=&W?2o!cZetDcTVBX{$BLjAB{`=w}Q^4bJTV6b2wr@ zihNse3_EfclXGW7p<7}tIXXd}B{v3;SlvkSNmUQ)r>Zh#w>|94m~N=G@ZpS_lklhN zEjTbvos9bW3=j0QfX3e{9)JF2chJ|6MprRddIrFZ9?GJwbH}!|(_W)ncy(rlfG;xw z8Ve%?JC95tO#Lt#T~Wt(0rDi&r=GmJ>&^PxPt!in&ph9mBII>`@^5{za<%HUzLY_*5a(O`7E~28bbAXtsr>x z4wuh=k7<6&L{yxQCYr};GW$hYcwir_J}H2)nm4HGrW`Q8o(%~Z-J@RyT9PTS>2w!l?7ZRPOSV`4_BUa8I*j_qDg}Raq!p=pBCD%499oe!F4O~hfN`tz2^zc7dGQNjR)L; zrjzIuU&j?{o@5)&-vdQWS>~;-N8073m}P_=J5?(~WTWJfnbz{bQZ$e?@SMo)TYUIn zJ&AXH=rGLsC_(0A%)ztqkwm#fvLNx%Qn;_><jhu5+x z9{ffwuI?7DzbC<#&XZ)h7D24m&J7$MD6^FxpE-!n?!!h)CD6!dh+DF>%lCL zZq$P0;8x6km4;H5vv4eb<5`$;B?e@)meA&Bx4ERTvZSo23BG@dCXakqpu=oUl;o=M zb%s3)osfgMX3FrjIgI#KhQjbrI0oxgVlnqj(6pf%jovSTG9M$dKgEQZj*(*L!y_Qw zpU)jXnTrWVL6Cb(krT`)qo0p#XRpEn!HT>@zVEqN-1ZxD*u>3t%&s<`{_KdOid2Gt z(M#@yzagGIzMsTzybd*jYwF zczn{C?Lh7f7?WEcj$q@Wo7LH3!R&lWJ=H3_g{y`Juw$nt`S{I&bf&N1vljP9Fm8M* z*Wi=Nc0I}Bj;A)@D7=aaZ7Qss9l|Lc7tx_tg2e*7`*m4M z@?4O4wv@%{Xmj172Gwg+dhl~7VMwo4^GBw z9b9v4z>jA{iBq=}Q5W%JSCS3L`^HIN>s5mtj&ejlc|MgLr3I%dJ(%Oc2_(JYA-y}O zfF)l>u+-}S)JI>(2iHbp@r|kE8DBA$>6FCbI!6t5F(MRKTXexDoecce-N6gsW|8#L zEZjO+1k##87@#|oHFeJcsxXTz`Y0^5zIEK8cYvbp<8bVsa}Ya|fr3aVy6x-2?1*;c zEr}uhg;&|~O|NLx9wYWqcLF(7nS~+m`?!Xa31lc%0Q~1Gub{x?FjKW`p>GTvNNE3K z-txQF5Om@&3*Vbc@4EKlZqHTl>LJ&EBzD>O(jhNs8U zIQjD$WdH9^XiAh(^kf9A$q{g}vo?c_X9n4^dL|xr8;6eCHh3vc0DrnabF)`e^10lf z7BEUJ7ZS@ES8bOD2Ze>Fm2W^_+G&%$o}uKOUo%c?+Kc9DKXJfrJ-7RrIv6mlEns0jPha||a8k74&V*7FWGWTM9K1$sTT7kG}N zIn~HyYCdN(lRV1TAUBuy;7zt3*S0$_(WD-@kS_rP0Y|aIbr`lSP(yEW0MG5vA$Cg` zb$<7jqt~AB_HLL?lI|LDIt*o`cNAvr(CZ+@;toxpkbw_+Q)vpV4f5?40%hr$hZjFShtNkm9}qRwOfwNu57&@evKe zotgj_n!30)g?+GP5J-p@(p3#(h>3GQJwN6K?pZyXg{H-Ts`eF*|2bQd%uETPuf!a2 zaN!W`u`h;L?Pa{uygOJoJCSofr$+2vvyr0};IQLJ^ZE3jwrXdM}i#m{*xVZ0n`)k}iKsfceb zz2W21wccdI#Y9}}D~S`;5f5GIgqXXR;Z0jNDs`WQY0F{+(Vf$)=kaEfl$-vH=aGPW zKa6m9Pe_oZL*j7sOo4Di&71;##@8jEt@F9vb2l@7VJ)w9^ED6~Q;c@btuW3YgMQZsL0kUt+6td(agpPi;1W-Bo@O_zvrM2+;|ALo{eg3C)5%gD74lb8 zmAu%>k-Tx2;O0e1ESw%g)WbSa)aViJjNZl!%GHSES`+MU!g{&q9Lr2Ys!31u}0yl;Vxfu7R8P4pT$oSsZQ`m#` zK5$Xmf^`?S;iGBu*})!JvS@XkAp7@8ntMJUR4SE7=pP$4Eh>n-@@SxsZ-wFD{!%(h zk~E=r?*OB4CxYgFM~5j^hKVF}MSvP<(_@YQEP zxvnK#z~hUcBvK)8sx3oJ**|D|&Wr`G0mAQ+XA-7IF-;|eCwWMqeQU2{5be+Yo>Ju{Ab|v@<&abo;2;+6bnYf-(i*OUl=E!hZl^j zxT6Zoh-|$h$u!(V?oB+(o}V)l+|bKm8Pc9?#d10_CR|-nk5S-sr{E8Mok$ z_77grA6PMiY%fbg% zz(WU7e8S7%64KN-pT{B0wy7B2UAJQ)N1t-LU6auFdkN&`B%z695#H(=0B*@34AfTI z`%Q2r3$0}6J;f=g*_y&fb)``9;gvPHMxBUi;VAY#ase4-S%IwuJK&s>69kS^LY4RX zX|nSM;*xPe@T+hNE0ri_`W+(xFE69q?62I=-#w&ijTJR{5XO#`uEsrIa&b24hl$tr z3$7{4GUK`m!(4n^8Yt zI-C6MxP$sGH(t#NId<9gE+noLK%gWK=Tt2qrnM8WRK|?!TznCG4Wpp-NE&V!oycv+ zCOGLi!ktl>0~6G~3-0vzajNIOpqzp#(Xl^5W?5vS=x-BTkZsQD0v1CtztIEk%4{Wl z2~$agl!M@BlMg$!W*7Vrc?tH)lOT8FO?dHsF^X5Ipj6gWcDe5iCgoiPzt_`vC-qvf zqUbrNvDN_!RP)(LSP!Va4P`Q;t_!Zbl476br19b=8D`O~OSR63k;&y-U}T0I3o6=3 z5}RLgSv7koKhBnNYqpz}m5 zn8SO_x#z`87l~ni*BSA;Qg#V?;~v1Up&3zK^c%ipEn{!Mmt-${1dDy<;Jqp}GGSPm z9sRF>74#j)ncs7vY+ns#myE^5uj2SjJgozj&B9@Zi6JY#U`bX)9AN)i$FbeML#Vf8 z9Vbz+g8plCK;6JWn6lZHWQq)7?3)CVcdh{@^%$_rS9+o0unbX@tH6scPe8e>kh)FZ zhAHKj=z;PvA#IiJp*>-4qqAPr}frIo%zSfb5qIH zD?=a`=D^Nu^`(JhZvpn@pj`V6Zn8`?yHO~G2e;_a$L4G3={rhHaB?XsWpFATR&XTquD9_*z|=iff&P)9t^&!p@Jo zpIQekZ^KaT&?Q{aTi~$U@)*?W%_9!^FKFOBceYPk9y%Q&XmIWn9FrD9n&u237uSG; zZTk50UL;o4x?)$zeU9%|kP9bv1i+^ZTO8##m)T#dhRj!SFh91L7o?Pfw@b@7$gRVM zk-yxAAY<}bp&0I(3-G}vEAnd=qQr(1sQKR!e4Tm_UN`p8q;HKp#Vz{;-_D!_gZ?+P zQ|}mf25%#A=0|I0X3qofWuegiF$>@RGh*>?7V_DewOdH=U_58%sz?mAfvla946_eu zvFz{GJhO$rd5;S}auVEUZjAF9rdh7Vnq@}t26=_Ld7U`oGDv++3xLhbgZ*X!T>8X` zY_<3T;!r(;FH#?&+n+j&dB2WSO>=@DruHbWe+NF!ItJPQ?AZa?M6yPjKbE^V(-s#T zm&dus7n8|f`)SIxSZ>>t$;6!|7pesR^Hq?laL94GdQO-AE{I}cqU|B8P zUntJhtUB<();P{{=T2zJbR~rnetn_ff!HDRUsS==H9EM_)1BA@=R=?smyf=c^2D=J4-`+z zGsi=+n&-4oMvN^_?E=kqd6>01ibaH*4v7!kjV1Ez$7cm%nrlN;Cm+G3_Ktkwm(z`2JxZ+Kmg4Q@ zCtz#;33zfQ3=b?S!O-cRoQEJ3%|4rweP?6f+`R#Gy5fk#mEy$4RD{hcyC%H4&%;?W z#0hk$uy6jSq1W1m$ewu)OS(5PzTy*(d-MxF#RtMIZBgQ_F^F5<^x{{$vv_V`5q{6% z|DhG@4#DCEPqwmi6GVh2L;s0%ob^VDd@{XEL+(C=tWtfdzxEVtDILKX|6LGd438s+ zjqRy(!aOu}TS$I?R))joV{q_nHB|0eL^i4BLa6*Js8&E)Qz%C-7hHhEfM7_QF%2Jd zXTgBaZ1O@$k9@seLaX^JO_*Pa1>=I|!L+~vRP5UZS+yKFeAR%I%+g~9l1Cue?lYYI zX$U7@E0Z;ME@R4@+h`U38eR8^;nVhT{tk(2jq-kQ9FczQ1xrzVsB9^WP*^sCkVKQJ39Nt1}1l zAcKkpmqS&(aH45dV*>X)(&g|7=QcOu(cQ+(+N6?`r-fLPYk_gMY>C^_NEp9znD)5^ z5&aEq+)K|woY5YMEgR$Dt#TmPmdV0&cYU1ps}xpjHD(`s61ddBFCh7}9WDs3?4CE? z!+bHmJzIL=7{=dFXO4EouyKh!t}d^E8*_)@kl%j-FU=G-u0aNL>kOgh={w9?Yd}tY zc}XW}xuMySQS`V}CX9Vn&-%76!$t=O4B8n5i$n|I#U)*~_f!};{X`ajxCwB&%@>~2 zOmU+BQy%B5TtN3nT`>HC0rG#AqUW8nMDyS_x?t%$x@*t|a}RgI_E$q#v*;hpJZgi- zKT5;9JVl}%;RI@@G~iX@Vcr(L7>FM}LTdMGvuBNccuw^Trbqq4m{pmaP!ptzd_N*? zAjXz`%)y_6$TUg91m}@`(et{z05rR6c~10#yuL}xU6SR5Yp3+SvLbFQV+l9HN9vmWPx)1~n^YO4s^PHF&Nr$4=Q!iGvTeMP?7K|y=TF+Ti~+>aBs z#&d5y-l6o$jaYc=64qJ)x%0uDJYF@2*xVb(x*sQ?p=BK&i`-E^f!nY~LMZ8HME4_lMs6rB5WT8-aGI-@P$&s9!aAx{P6q(~lf*h5(cJWBO9nr#V)jt9E z&3ve#&KF2+R_A24+~qu!b=i)hX0CE^3Fmz+8#U}lA^!xkB9GRNXEk|`sp^-D?DwcZ zSaCxhC2a~xNQDFOG|~cl<85r~t5)h-T?Qte<6xe>4cTGp!nx#5~Is-QTRk6Z0QalaS*u4`NV$qy)aUdBGB~1jyfQ4hH}HxhLB_uw~>5 z`g_Nt0neEH$lb=uHB^aSRs{+?oH1Tlm8TOedUy z7N=C=9Tp|nFqbdJT&@Y|!S#xmxqky2@0Nvs=1gM)tM;2k}Z%iXs$(KF{$8BekMg{~x>MZS;+t@>;R8A< zShG2w#}lVHqlxc!CzdyPBKa0B$y@?7SYOar&PlF_{qAd}ugqnj=*u*A$8t1NkFG?` zo=3EEOAPMWC&SA2UBqSI$Kj~C>BuFza0g!h#S7t+iQd}x+@?p04*co=iIUqHhQuRn zEW2VYT*~e$?!r%Z*c>g7?_b0~qtq4Ly;=@TjW+Ra2u^ZqzH4HV^IPt*lOFNaAH>7y z!cX(_2Snm|{4;+H+pN70(zEz%mSWJ!l_YmOE@1Z4 zAd-AZ1ZO#A(8(hHkknZMyS^vF-IX37HCLOvtT_V~RT+`hw~fhR8D&^lDNmPqY7$Lt zWpZXl5AN5zj-|hqnauDlR994habjCJnYDv(b)ytamR1pX@VmHwKVvEXs2WpftQYWG zW2Zy#0e?*Fc#h>oHtgwe32Y6mz@P^Qu{H#Vmqj`TavyR1EHx|*Tg3hxdIKvRThXj0 z8NZH|Kr8y5`}?sPWYg|PirL_3VycA8t0Wep!S z_6Yp6#tw|JZ6`2;V!r?2ox5g9fI_~F?8da3B+Ebo14Dy4bBN8z78dWv^x zMk&?oSI4ich$RP1CF45PF}1h?rfoipOWlspg@5=Hn9RgP+!8Sejkn~8t5XF`iEdh8Gp%^!~Pr%^68YNAnBquTRvR_wauK! zmjQDoeP0?LUK+a@6JU#6QFs&ZD66!H~fCzjw_hg%Plii zW<~x**gyCc_&OFhxVz6TfQi{A__{-nxbv@2OQ&SeUVjEpPh5!Czb}Pmr&kbtVlg@2 z^#?MPec0CsH%MMM0BKg*OmDv{+1A&`h7L=z6^S7rUWNGF@gaIu+M%k53_Te+z zw>qzI37hw3J~@6rnQkEM;Iwaq-WiPIle&T1yqj;vq7$zRDnF}`4|Wz@&O6r`lsSJmk30;`+S)Bbzss+0mAj_F76l*Ta5WVlZC*+SMk?U6 zBYYbhIA^$mt-rhj{+j0lY!pC>-DCW=uMRJi#9-=Vp%%eGEA~GYZ3cbnv5!GYgkAWj;D>LU+`L9b8ZgTTaKZU^br&Z_1(XoyrBHs`-LXnuo@uv!d875$LE}F$I zOl9(`lOXl?6w>><6;%GN;9T2N*le9>&ROIoo)Vh|hLQ8gvdyE&3k2CGjbQvRPp#3>47 z)Aft+U7JrXZ#aoNrX=GkpE1a{HA{zhhwboE`yM>p!f_3##!_DjwXk1w1IVuHu?nZMn`PKc%SN=@EY7NX859}cfW$w?Ig8A( zY_(Sn>{+gjZLW(+-=-?AUqq3p+iSu;$-9tZ(1IPUBJ7N)J)6+rLndA@LHzOq_D<%A z?D|i%XxTSB$G=lVZ;K>R)m3ISFT&ela!Dd@f2%iLL2Fr`$0>pCJ2&?7S3Y(*T!wK% z8NGGfK2UX6WdG7e5t}KSasK>J%$TqctvX)87~d+~Qh%JA)-;7|DK#PcvNhO=dAmsO z%C$62(u#X$eFOH*xC=w<0^Sba1~Ye#aQv?Qdazl$Rj_sXf3SIp9?`1%$~hj);kG@V z2YXgtK;xEZP?}}IHbu(gO*av`<>?(kZCfz;mMKl94DeV%Qzgig>u|*3EZuZH0h$eu zuz-T&>~bsP@*f|=x#wSCMTay(&>r}E(+k+DNbI$TuDzkx)6h;AhJ`d z3#O#-)!DDjUwICCXUgDBQyg-fr?G#QGaQJtADQ0W>RqJQips!jp{>y8mj(U;A#$1w_ z<_kk5Uly>L@14k&>pI-G>1)7xyCw0Tk_B^>{^4VxU^R30B;LR96}&$;_LDVVjoESI zn|Q(c0CXh0!ui!f%;YCWcG}*CfSa@JQAr-}$m=hhhRX|fyntkZ->ql;kgP&vwu(I&6F zZb9S5(_m3OndkW30O|&g!6)-gq}M@`?EqQu`JurS#H+bQ<$;(ntV-~OKe_PwI=?3W z?rXfiX%4ueIi2@%yo1A#6j`u9f?b)@i)U`zGtYbPxqb&RmaAPu117f%s!cg|_eC?V zvfjePT&>B~YZk2IwhHMKs%0H5M#SyaHG0oi4b@>ZHM#p2vfhZ|>gk@ueSIiSHuc81 z!6@ptFa?%Z$Z?-@`5LhIT`FeIAK-LFi+Ofuz6!+t$lAvb!i4l`jn7yDtggklK$XHRs^+Nl(I$IDlc%eOPg<6N`IB zW8SV!)b##*)E804M)z~j{CEL9l*HF!Pvr99;_@o2F_=YTGw8%^)sMIFI=oIP+H@@7TjYQ8hOr7?nUx;tQ$$5@gT z7zwvzCc(=^LUBUDi*&9Z;#HsTz!|m&;9~e^POQWVuU(EJOV+t^(JxNoXm>@F5@-{C zRf;vZoI8)fopo4yQjC2X{D$J7Nvuosak*qC-H~HYIwt@zjbDuaB_+Y(M&;-GlndbPerpm8qB#g zlHRV$ps&}ap~jpPK3(1{$40bj;gzcoidC8uxxwjdhSo!x_>W-B*r&Yz*bDgJZ%%ZY zpV7h}mSoxPT_`J+AS6)2-=IwTe&kRT4FFaUf?cO)K;5r<(BCtS{1sP+%N1?dr8pBu zcKw7^hl=rfdoPlAgY;#UG%PNyfCm+PC2-dCAp=5Tc}uA$tX}LSn5`9$&6D?o-1`z( z7QG%7G#HDb(-UIf>UBV3Q9mv)fa%s)qsD#&j)EnN)G*$L;!TmDqK<_)+bO%_< zjdA4owK#OyHkW-XenN+{jfq1T#nd2mviR+L#%ckb}-+Rtx!5Kmp9sJ5D$DZz)GjbLa|Yf4ZmmjMZu6MPah>n7<|j| zn-65s{1eZ);73kE8)gNV34;PZs?(u1BMqiJKZg??O?kC_y5vVsC>(Q=r@CvR$^N!u z+_NAh*ed^x{_HM?(LXQYm*cT?R^J<5z)4eBHLwTW%7;1qWo6JIEsJ$cGf1-40Wgl& zWU*57G4`z(E3nkYkDq_?d2^z?gh>ToQXJ8TcNT>(ny?p|?XKY&`+Pymog~2KS*-Fu zhTanc&``LfDa%gdj7idP=ZS!ObLA35{rnAFlo=z+Hn?Q@O+3740@?7!huwLcUUR76 zAd03%;tO>*-u;|-P%0{gj1)y8sbkFE?QOv~@%$m!vRj2*zo3h9%VR)7TDE49QUWet zbAg%_7Vspy!$4ws7L?4h;1>IiB@y#mQFBox7T`j*Gv*9$>aSPW^(me_sN6<&1T6u* zj8RNJ-UGev>hnA%1d%IxWiZvphBfC3FkkrqW;y!dRn7l!&9OhY{_-~NA^$)!Ja8)D z#N%rhO-|0 z>~e?NO}z~jAC_Y9tVA&9En-8lKFn6-16{K9A2>hTjir++xd3@Hd@(tgG!~e!HmUo# z)V81Z@b??aKe+oG*Ui%*^DI2Xj9;Jj<-WH9TJs{ntYMY<0;yiyjC4{3K$(a2?1b z-oSIys$k03IQne56qh|Io*ZQ&tY@PFA7%gUz`<4R(9v=bv{j|q*4L}iHtI81s5B9r zyAvR1$$CtkzN<#8D}~6PRwkNf3o$&~g1yPu!|iodz~c%8_0sjZnevuorpX~lvn#{8 z5owmxI)zOT?cvOhX9}hng)t-LIxf7loGU5!#~k@fSoU-)?lt892cfUtLCKs2u+{J& z8|ZPQzURCVZcZd(nJ+Q4|38@cshGDcFWzBA@C`h(N}9NSyvIH8)+1ZB?_gD-2{pZH zPnP}|N3MvAf}^$rS#$X&v|X>motmFvl7$z}7#m5vj%=hE;tK)V&%y=6&$xGdA1-~8 z#UAtbuErn3li1A<^}x4SPka}=1`}gx8dG~2er|k6t;+g1llO|OE$A%1Z~QK_esO(GG#PdrWVU=ccxLVMR(v`@EB&NGKPbO{S*D2oO@qr-Q^ zJ>G#_TeCKHz)tR_nFraZ5DA4R|KhOr5)!!YKW?YZC;VC8&K2QA7V*uTjjvS&liML! zE8@)z=mxHC{&HsYOPfo0EkzApnzLQ6l0aTbf=v#Tr%lcwXm!|=L>m=a$9Refg=_HlEL4{4d)Q78lTX3^S6Id@=%?r!i0U^&s$&48R^BmyQ9Sxw_bw7a(|-{Q+$>yqM0t1~An>%k?gP13t>LNjzMJGoAMlx=WbND#{2u1c{yq>v>qi>gs_0X|9lHwwW>zpx?=1ecYv-OkxQV{z{%AZysDxQG zVgI|uT+bJIh!0U?I=x+{zBt`Ue_PlI{R)_6jc zFH9=UzJ$f5|6$(lxAeuT=};VS09v}v!fLe-oc`|;IB9VbJi2zmAtf996}Aw~P94At zqLrlV=?q-C{UVl0YO@qZW4N2`N$7OxI)v&fL0$ZAwpp|pSHIT8_CsTddHzcLVmF$3d|3i25&qy( z5&^24ZZI;w9yN}KQMcHou>D*%svkYg8P?~ctuTbNVM#sINOyzt;}v}P+bu~%9OJng z%crvI6OmMAG4QYRQXokM*|VAdMKJa{W%0Z(YI(xemeb_mx7*tA0X+T3Vb%2#eV-$ zW;F}C;ln-td^RM}gaT(Tc6)jZ_g`QMJ(Yxlt74igBq$kkWJPg|{yaf=r7*FaCfpO# zF4a_w)x*HMmPE;}7d-AJ;f{j`iM-Z-G-AVB-a!L7R2OIwzwP5;_tGBj_=b9P4w{3D zZ;a!9UD6@0xBOt|iaDI#?Cnsv;s;pq1-k6*Z2m-mf%f|3#D?N{kY04-_L??n=-N$}ZRzyx%@T$C|9YAi*+C z6j{LdCiLn%2F>j{c*IhJJiUE@jQ3Pv?;J<5+)anUWM&GwJB=Y<`N!c!dk3y5+lRW^ zJNa1V5-Cg~Wnp-)JLa95Mq2lMttr`SO%jjPK*WPju<4I2D^tG=ufF)A?ev@Qa9cI@ z-TQ^#)vj?poi1c@=2g66H-}VAFvKjCZ8(K}rvIicA>V#QaJH9CNtWFZ)^9z^%bn4} z3yG|NP<4Bz+%X?BgLB|y4ZjZ(WmH(~sX5emU=^#nIS>0Qc4F|84p??XsB%1RfzQ1u z=%Ig;v#xwjPkm|tjo;%)`I9GD`e+dC8((qSzG1L>)@XKcv_30L_k`ZEP8xN6mOzo& zu|baoD;hexDSV@>h(NZr(`b?1u!gGoR4D zdSfoK-IfH_B~e%431L zCJMfa!1%`9FswL^Eb!Ps$X7Y&hC|%#{p-Np?EvoF_1a;E4TCw=6ItE77s$_un#4>7 z7qifWqiF9jfn6N)jh^X#g7jF(DONut>Vvfi=;jtLG@wWO<<^KXDgI>?j#S_iq{ax;i*Ghow= z4}smM7F;$MM_a9vNvBdeD!n{Fr9SS0l;fuC#?TqC7lzd!R|*68_C@G3c@LZZ_9u$x z=t6YXWL8!z#Vz^@#GqyhbFW;(L?4EtpOri_HM&h}{+rHbuYC%3-|q?{KFng0ZL@K_ zvlv;j@);fXN0odDRADkgGrHT<8-i~ckqqq%u+j1-+&c6W*I2aCVLM}3w|5K)onsB3 zT+Va+&Cbrig~t*nn1TDo$gr>@YHa3<9vCa@k5a4zdgT1c@~lYW5&F^IJ6xa9eP3}z zVLX|-Y6>LX7NII%K4Qu~d%DxD69qmotl{NhR&IU_QavoeK;oO=$e6wCyHhH7x%Z&` z?PCz?;UKVaj)B~PffmXB_$<((+=Uu%h|0G@bdXB&)?mN0yO~I$5Pov+-@mO?80^@_s z3G_@P8>Mo2-(swp{!?w(DfNRZP5%I<{VR#!DZh$^MOJgts>JeO@0Z|4F( zoW=!)tH|vw-o#%`fjv8?g-K$Wn8$M`3#=_rP#FbPpI6ZJJ!f%mupXEOCOhQBwV}7o zOj7@K4P44xOt*E^aH5NaBDQ2Ej4#$9se#2fdSV4sHH-r24TOLTgY*TzHJP;THl(SF zYNUE%5hk~UV2rK|b8IL;vFGcUcm8|$5@kt}Vk@|$2U_f|-XlEmDGDd=9;V$Yvc#r6 zPUwK%|)KmAU}|_LgYm z(Exc9HHrSRuaL-YV|KSZTVxr7>SbbNlA9cRv-A-snJ^18?*E}zML(jk^H~tLHzhi4 zlbHOpo4CQJP|z)^gWu~Kq0jLLAHL@p!uM_yW@x;fIhYO7#J6X0rFxQJ>!Nt-H%|if zLbagFU7xJI=0Z|VSmH`I0mQlX(xgY;m_G0iN6mSPd&+M?koGC4>wZP)?)4;jVHy3v z_2TfNX>5V3BRQYk2aou_g^yNM(c|d>s5|zWw$)U^#c})~@xA9*($>8+Eq)~uen}uK zH*AT_CUvb5@QLJB>0e!MZm!zJ_!|44+r1{{y)hmvuqJXk+f)Wz0N^PgQkjhN<~V0zg-ETVP%(w zWTceQ;Ch`y(vp;>b`nh`?frXwKeyk%FaNmRuIqX|&+{?vXE@KGuoe$CTeg>`RBquF zQ!K@`sK>h#w)4tl5e{CMAReAFmKIwFaX)i=DA{ArW5z8O>K6YLv+vkr?zL~C|IN8< zb9y*sU3~>gLsOtF>J_M8zfXtHsPfaPPwDw-4GekFD9hzw7|~q~#!Dsk{M}Y~<iu}Koxt$&_I)ubW44fX<+Ct<)N8sDX-~d8ZUWw3iK|SDXpGiA7?Srz zwo~Oa?Y(J^KbP2o$=lW3pv`2U)Q@8}WQ(U41S?W++40_?8u&=+{uZ>UQ?1c=DqXJ0 zgU-za)h>gv>!S}sd7AvU2oof6=|O*<@IGB|Z5|HIh7QndKNL?jj4qqkG>a-7u!4L{RGYMTmKr&L%ou^F20Sj*yGh2zHl3Yt5x3pYERp}K@| zGH=In*zkTZ4Ep^UMjxsYw|B8&6N7tX-X)a+<%hdaK$s>Z6o%8|)7Rj%-zYJq?U*16 zlZ5?AkLlZFXB-|p9j(r}zzN$G>~HMEk6UNcoIQ~^&GRs&4RI6_N2_sEoj1D*FJ&tw zyH0odR+_la0q;6q6BZ75ES|WjNu%b>8aIw^I-?wZz^^9qNXBtoG z-q5+?5x)(P^x!#>2T*9mdC@yGivn|8T)L$k`Jym zs6PA`e(Z_HuAaxK`%FQ)a~_2>lAH7Sc}wBmorl7zSut>>Dv`#?efi(Eff#XTAkTF; z%&pV4ae&N%8`@II(A1oV?e`QFw%c4bM_z&{_caBJiJ^j>);nQR=N#JkyiN?aI6xov z{)8)TTX6Sz6Ly(oA`Y?3haCZ?xa~$CXi-a0Tx)#<&DOURgXBG)g8C*MXwaGt%Nlj@ zqn$P`3YX*GH>wn8@s4(_?!+z=HCRW>fmH`N;sqOPJe1rC)@_my4~zg9p_9bTi9=s*tOn=mfLKm#`Ct4Myy05qis>{ zxPBKM-g={Kzh4*r`>cc_c9h6Q{gmob_zhw`?D5#Q&9wKQ5pVUkpmE8!p)O?{UA0Yx zCvSXU!l^zmVdFIXJ*gM`>fi~keOEx#HUs!?;S#ipwt*ny4s6om#z{{{zzP46)H`pQ zpnJ#xuS&y>$*KmK|EqR#y6^Oun$^uQMAsX37^v{eMr$!N)|BflnnYW*8PM*)V9psa z1pB2t1s{(eU>iG#AIwjOske6XuoV}Dz3rv%&e?acVnDG19`xn^gaa^P_zlnvng%-( zt?|0iXS%IPVe$pM$Z@8o{ylq&xOR+nO7dPsYI_3b|VKL111@ic_u zJttu131=EUpiJ~S+n++aZD}^*)p6jG+GnOlRM&r{7yoj zQ~j`izkzV`i`4VHrw5_EZSnPIRYr>#+_>U0C3vDx(f^UK{>c*IfTj;sCTtTde3Qwx ziwYcY4Ws8S4B@dmphr7(&b_)D?z~=sr8dP>X&Xr!N7o7W*Xr<}86$8_$3f6$-5+PP zyF(H8?g{cCN*Ae4c?0#o=`B;WJp%Q6d}zd6E!ZBH0jt#0g{lpcdC!VJ;>FIl;i>y| z=s4jgZS|FL+@U&A_j;IkV$NU4BzvlvB(>}1DjKHc zi_d&Ib7{wLe&%>ZAy(Eao&Yt+K~yf4anuWFhD!1Yxs z|F?%I{mBC{$LXa|J|IyX7!=8)JERNK2K6P4>OK5+?`Cn}aU0(L`2yvmEm_>XK(9?- zLUqseBu2>K(u8R2ylNKra0{Tqy`L?HGVTZ>t%{7k4(fpFSnI#u9Z@KS9bsj zz4>zS23V#Z$qO%=V^U6koZ_?Z5Pyxwb+MAK<+qrRxJW+M>sA`F3nl2&UsBHW5>0y# zry;&>+%z(j6I5pN%!fVMeT5EPQF5aR>l>i1(VFXC&K5^SIPwbBY6#bB6mP^;(1_3# z;=Dax=qY!w#OgL>%u=;xwPE9N`w>HN*w;4@>pBjur=Mc?yI~5IH-m9x{Teo1XpJL| zXz`5R@mOYXlzyI8!pkQd;P;z-;@7TL=n?u6dfu)O&-ST+UN;}oaQo-*EcJV`%)0!OB|LnmGzGOpiTR~C_df#Nr$)1;L05v=)vcukVSTU=eiy) zusbgLth_--)jHtaPFa}rF9a|9=_pRG_Qvdk&uHa^-gx-Bx|~{m&E*qy)2Q1R$#D7j zk+gPSu} zHhC2lcCN(^Q!L?vjtaX({S`l}wa1^`cS34~A-feGrNi;fw99k|KL4A7$}MSB72<=o zM(Plj=)y0C|A%cSt+`9}aB4c)37335PwS<#(fG@B{A#ZXKeQVGi7ywwIifYoX}W2RgzQG@>Y7u$*Yi`_IcTVa7@_I~sxJ28~dAXpYG1 z{*lU@ZM^eRJb0w#E4F*xg(>Ycg@e~Okm0fIEbrO6j7?VEgs&ZT^7gkrd}xa^4*qYa z=osLJN~c=re6t?QcK;DJ=<7+Js~?ANS%k~`1yX*91u3TM@TxNf^d{I%nAT8zp=y}BwKp|ULJUvTlaDD)bIOzniOoktI(ZbEGGK#t5E!;jU;lNYROr8m(RL6K?;^AkUcV;4`Ll3}*ES?xFtltlZY6Y6v>@Pv5Iut^SE zrI^8*O|FWQ&GY!Z(^ANbVQqgqpSo%>Lf#r{WivIDdVdrKEwx!`N zO@4D}XSNA<22&Fv*?P1g9$eLdy>u2ygVqdM@ah>ke;-b| z*$0G8T8;Fj{;lADw<~u}IzZ!|so~OZz3}_i0xJEQLmBT!^gBS%8b>s~mkZi$nUM876_-c{$&*AM`h2J#>#ys;9hxJ_ z_OA)H_sxY3uZKcH)OLvmb%cXYSAmA2iqc*m!J?3QY*0~iz3J|ahu<&6MJCmvo`)uV zeIG$%P87(&Z2Mk*qB;T{Ttc9kc$J4@KXz0`gJ>; zdq3L&?*fPNPNyXL7C2HCyf_|$k`3{ZgqpH6Jc?lQo8El*PU&kELR^hI{``}L8!Xzx z_wh5uth^$8Ej$yppZi7K?8iW`i~O1xuQ?k_19#zkvrKXByv{f-V6WIma|s#P@8Bi( zPT-2gJxF|Q4C8|~u-p4WkgwCC->J5A$YdPtdzwj_t-g5Y3*f*EJN%G3h}Wz+PVS+n zDY#=hNRyfXmq=3lg(>*tj-!O5S^^s;J%g6w*Kp;^6rLqF*iYkpJM*dr3*2Sj2wgG^ z@G9xS$sOJ_D9Rb~^OJa;`DP4h)#E3TotS371GB5!a7Al2X3ve|MJ8GB*lYn;>6b|= z-%)TnI1NWuNjLq^Ry=m{PoXU!P3qEj;-4?SP)WC4^hi-Di}SV<_BOnLPv&nxZudA4 zJ_L;9S30qvJgb8ImL3u1HA!@F#ZJLeB@2Fj+7HE(>gaT(mZ*BfkFU)d!jo#&@jcZa zc;7Z27ar^c1tG@x+a?rxrd&`2JrAcZ)l;Zf$T2X=RH4EmT?lcoKixzZ2U}jK}>wtZ>zM zRrXt+B0E=FMU|x;DcQ9H4&L;ztZ_{t_ZF>%dE4yZ-nnN~|LujMWs??0r?m-vHhiPZ ztJ1G0Z#&F;xDb0RT17YVX4CbQ1B&v~&%pG`10nFlH0l~Z9gRxd@T*)N2?bIszpKQk zIR0A)KHZ*zg_i?_U6*>3;-oK{PH2Zg>Cd6-$zD`mIb0a-8!52SVQsHO{vT(y8~Ma;S7lq)h{s@aoFYxN~CF)_H1oH!qP?^RU&^H;ge*Y26He2vYMK=m*e}?oruYi4B!bK|06Q0_(rwiYb zX=>hnDoj`;qz&D! zh=5m{Msn@PM6Qg_B8xFb>=s}_kE-vI&gz#(44> z6;4<5

Og01>pXnBqi8Cg2R4Wlp$?vut5!|Z6p@tX=`JOw|`XyD&7)9H)1Av<_I zW_dHAyJLTF4n>x4a#n3&V)<5L5%*Tda(E#n0JxG{$hy;9}T zi+AvGxq(bdk(%I%)obac!9!Z4d_%bW))5UHd*kdy3wegxZCX2S1Xpg)A+6X(aZXsK zbL!%Mbb6pb*%SKnr!Waov`!+xWf<~>>uLPac@PD;pApFOHI)t>h;^HLq1!6~YR_m% z_M`?@d`PCNUN9{)n;Png2I#__13sYN?H%An%GGCt^? z%H6yz!0eMcKezIM58>AMeM+IY)YF=-`>q4oC?&cXbWIK%RtBSjhwzvMyTpfSy4bqE zTG(8-7mj2n!dJVlTu~yqfc5|J@m^DgJ%ysUC^-0Un&e$<{1J#8$r(q6=|>z;`=-f~l# zny{Wc&E{Yi%7jlkwXn1X=y~rb@aq3bmKoTC^D~n%(j={&R=w~B5J-wW?1 zjz?eTNY(%ooG-8J&UTd!IBSg_ZwYdOZqqA;GxoC~EX@I8>&rQ7>u7%PumpUrk}&Sa zS4C~WWwLG>kHy=1a_7rC>Cxk5FxuRj4Z1y^g#14jtzFUzK@I7XQJ(_TGSmf5Ql3iQ2j`6+_J?8t$z*Uz|*PJzq}W> zjy}#c%eV8U=cTZ48$+knk7?vhC3ekN%WrO5@TY9#J?A>{5VJx#X&V=a$?y8mpqZQa zkM%?7eqo*@oOj`S(dOK2wNhxjEcGJ0_CY^~Hjs~tAvYs4g`qOhtR_p)9oe1a#_1CA zEfh*;zNK`DAUX426)ClbLnpFAt?EM7TA#{4^QQ30_&6}XBQceN7Q(n*CH&#t2Pkoo z*HQIYWlXmT7EE@SVA$Im`24^pPzjLI{tMROm0fjsbz3ODl(G*^vdvtzF^e~!@sO;M zFwWK9%U7wII(M8sz+VYS&7 zOgR4m`s$t)Zuz=^zr5iBoIJG#N17d{eYbi^^tbjn#6*|XeWr++|5ec5IDMQvUQ;a9 zpMc>rL&cm2UHR8XD=ZqP!DD$NH%UZK^~ANfSa}zOKUBrx-l61$(J(VR7Gm{x3(9|X zp}9>C5Bygr9=PGbdRykZZKNYjz`)5f-3#qtkiol zoUhmAS4E~nbuWIVF=k${{b918@T{8>cWR%IeHNF&1YVHBeZwujr zUjoE>YC_|}N+GM@64=Z$kjgdnF8h?<(e+s-*nzvyu@!cFV2v_d&N`!TH7bRJuT*i% zwIw*gOYVjGc|mxnum)QFs)gCNR#468&X~L`6<~-VN1GK=U|W`K=#*8$m$E<9xmz-4 zIGgc9KdDAIav~Mqyh4_<9*aJs>xI-F52e2IN3rE%1i9^vgNh4w^du?|A7;v^ZA^cP znlX$_Zg)jJ7e8EdX%N|%?E^cxY!W`*c0_SPGRF+$Z4_c;h`IVIJn^?K>RhQ2X0QH1 zV_N)i)2CsW72g@JKi`J=iZ(d6EKB(JDo<$ZuwNJ*ah{rOM`C5c5izJwA*~8_=j|t( z#D68Hq3h%=5VHJb*>S7?gjF#Q#U_VZdfN3U{CC5I=QO+_`R6+ke16PpaC)$pve&Mp zv0f|0ZhODL_73Jew>2Cmmi&fK4gr0rMN>Ri3#;QVb_jEI;AK3-ohm6BXE3?^mxdpy;N#nI| z^TDgyR{$ZEMG#ons&(fICpzOZh?SP26?mFy*2?@hIZ7;7^eOg;NyoRnj7+rN&B zZ&{+wvO-$6yAzkJ(Zaj>_hn-iZE^WNN}b15xwDqta&TC@4AnAUl5tKxEc)sQ8+LSq zUfqId9i>8|VWJpZ5Xrj@&Dp8_QVj4bll7P4XusvtkJAf}E$FnjiMph2Qv?Pa7TYFY z5KcON6K2gQQ6y#t!sfi*JgnvgmDIY6{pxiwMB<>FaC`&He0{~zsl&@IJ=exb2PU%K zDs^%6ojTdM3?H~s^Oufa-7H>P3~c&r1Dagl1O30uKtm-%ocH`TMM^GXew`i49ZvVf zJ`NhR?${LCn5d466Lk4(loCBmxgl0|Gv_$VF8J7WI`&MQ#XGk>61;wHfF&bWv)7=l zY+qVVeLrbHx{EqZt60K!P2)Li=Rr`lRilq3>O9OZ3+(q)h_f69!qP__aN4`|d}?N$ zaA(UpPQN~g&n-AChe!V^6(3U~gobLF%lz{?QU>lkXnYHY(r30U&YviRlp1+G0Ej1>7$LdX^?T;INh)jH{%x|Av8 z$9~>ceS+tmkzrLr0-c__fRnzC<^S_f7HJjMDw#@46iolWxpG1>SLXhR`k%S7@L}oh z6wi|?YqUPc=l$y32zU&qB$7xu-=mHFy$_o%NM!Gy>4i zR~?U!W%0Z6Mb?Up!-u0n_-48hs~m{p4>5Vr*J3Ny&y*sS*2ienl>YQ?#VV{ksiDZ1 zQqD>r?xL~nb{wwAW-w6#qeJgW-!GUpy|>5btA@~!``g&M({p|+@q=U@MwBvsk(`5O z58)Zw+OAhKFL3;sLSf&=HMqvLl!vRX-~-3sQ$WN?d|L>xOCu2{7CGUkbJplSD+S9h zT;ZAHXVR?JG&H^Y3EO&&h5U6V+4)ry$;Z#+6%(R)Og{})_0q+sdV_?$RpF@CZUKKa zd54$Jb)lx#ZoE#Ew{zY9*qbL@+=2nO4r8k{7$;hk!_Om=@#pw1u4Fw)aNF~lem?Jw zgRh6-46iVL`zDQyjn(*%Srq6MjDzKgvmh+<3B|UGn78&gx~iWglaQ{ij$tWSS}9rf zU(6*&^a0pDem6ICeJ5Cd*2T7W4=E0NaEM&B7uI$j!KUjzQP!PTqV~IT47i_(<7!uf zPP`@uWl!dp6(^ZG>=vI7l&o?M6~3GO0E=E9Q%rn)9^)Eq`PTY*(9*gZ!=pcd3ubc4a-*U0DGyRo&UMVJ^NKzLnSg?aU(vPJkbUA1Uu?4$V6dDny?2W)2-y(F$zxPT9HQ9QW&@|hMzof;8U}I z)AH*vayHnfCAO=L#P$y_gSz)w5DGhUi)EVFQl`oj5J(Zh6Zwv+1I{U^6lz<%VSDvl z)_C?_C_d24)h>rHx;BFMDNM0$_-?Gf6AOWdnkam)6K5wT3QdAmX>B|?r)!YX z9^|H5N#b%BV}3E>rSPMDHYKJs3sdTE(q(6VaJ|u+%I9Uv-YotrWLTUR8pq4T`8<`n z4tYbyr{d`2p4|{togu~QH<7hVIJnter(GqT>77~-F6yvA_B!1KtDgQQqtQEX`jr%x zuOGS*3XVRc``$09TQ?svI6VokpW7wc$9YOwolG)S1q?ENN4gVFLW=Tsaa?VG@p8~O z{C2t=!q+7bx5RQ#@)R=O9trWqq7diTACr@EFtJgIC%CqE-6ORkB0Nob;o@>3Ag&TJ z%5o^QZ%^=gvY-3)T_+dzd2hf`Ct`Tt=NJ^U4oftvnV5K}57&C~cFkUmXXrkq&gGV+Fib$`HPeNnpP{!Ms`9i3+dP!0@jz zc&a!8SKRLmxpY)0E$V|?bFY)DL%Z@hXP42lB6%~ITT&WiYyD&%mYMUFd4| zNH(~%lf$mJ&@%&nNY-&fU)LVIV6y}LmpFm8g|-*-4(4M8yKqJ4CA?z!GR$%{;NR}Q zA-VH!P$GQ{)w1R0mP7QzGlo}5Zl8m=Tc(w@mo$5{gQl23VbT<3-YxHu4El9$V0Ws3 z#+_LtOy0VZPmejld-knH)iDR~>|IB84A>y6xwjLR1|7iSfAi6HBEr~%&%~Q2;&?GG zVz04@csXc1kMqsvm$n*kf0{Xa?B2!0ZmZywpkBC83NSZhIg3N;NfvV_MsVvH4PJM% zaE|6%l$RxJ#W$=%>w}eWnwcwS!$};h@w%)>j~ZAgZl^b64Ov&JW19r);7yH0E*%_= zPkWuB11n-6J<)}{wN~-`?nUf3BphS@>&qcEou%<43%|cmcO`FIRJGcL-47puKh{z? zZTU@Fyx0jOlrU*aHpXj{AcI{?DSny{v~M8f&BeJtZ6};SoVRM(si?kE=LR!Y>nqbQ^edmtW~ z9V41}Cy8@IdI(*kB`Ldq*(JKZBttZ9zd@XHbf$A>gLdTIQY@^QW=EZS>59gV7e$@c z28RG}iz5DT*+#pN$D#l9I__RrM{Thda=C8eKu@XjoXGz~=eirtE-HMw|a z$}m_rT!Vv+FN>L~J)k6gzmU6Q9!{O~hQAaHBAxuZa?Vs=NMA2zfw*^rcry12<_~NT zZ2Jb{s}KkDGt0)0O1lKt`%8JpjYJ-4yk1;#@g8M+k3-XPEBqQbj%;3a=77UVoM@TF z_G^uKT2&2uD}Ue?X(7R=)*EoUHD0K%T#T9WqrBSYDOSGCp-v9{1-+s_7&=>?gx7MM zd0yoJo_lXUct+cEs8ei&UBsVeVY*eb#P5Mnu^&AUXlH{_S|X2M2?S;T2bpq7IP^iWxf1S9$k~;t1wjAYK)IUtata*XqU-{<&WSTuhe61;z5Aw>q;fae1{{#>bpzgEx1D~*{jCZ`(i2Rd_2ay!1TrbGGUo+*Oc zp{uNUdoR}dOJYlKmW2JOhZmi{!62tF+}UOY);$Mf#{BoRzgbBRt&3KZS*abJdLPJ& z%Tpk8=~9}N-W7B2PZ0gv#o#APS1c|c$T#l(g%iKh=!fP$A=9c0`5veT>-L!djb(Vr ze=e?0d<4;R)^ow?x$I-LpC&JnBE7MQ3te`gb}TbqA^7Q&LwVj8$tFA>#^c6MWa04y&ddttTT>Q+gNG8wwXMM0F2BTE1J{t< zltvgKwV-2Oc~g0{CwJGLiAJi<-06N0`zLwgy64CF*7Hwd%bis?rO1Yhe?;Tn56dxk zf>c&)w+U?z6!69^mN-~md<7m^Me?!49=L0g44Qn^aOhqEzHGUO1$Tz?@>@gr+QQG2 z?YN5*ByXet$3k&m_#u8ca3>cO>X+Lz=D{?HM)Ec}nl8MNc763QL(%&gZMZ4p(Y+JU z?12jU{i%hdoI<`LWB#}(R+gT#9j<-eEvvkmL_N>ULQ$`wh&4j9@pps~5BQ`FIgU#I z%W(brx8awPmUPX~`v2DqlCBxDsR#deh1g$MS(4(((xu{bfbRdE99>{}BBkd4p6pn! z^S>uid)>`aJjtcg&VE)(4*Z8D2$kcw|&Ooh?hx1M`;a3>X^TjGiC)1JfJ1cUu=iBrVfB=k3rD% z)kZk{`XzZT6-4_bt-`58PhdiTHuX|#Q}i3p^d_g3UcDQ^1uxC`u!c3~tI8vI@|SRK z_yqjtu_OC5IkRE@eu810*)Uu>+fvRE`XZm zqwp!d3JhY^a6qUpK6`pz_;4W*z65Ot;}eFWcH>X@xpgqMJnlu!k4(7V2;lDaeyp$W z%3sDF0|U7vcVD>K8NaPj`Qyzn#;KgF%avKK=d_8JO0t^U4-GJ|`6zZR8{)Ft)(=E2Yp%E$&G(Ep zbHk5N3VifcVg$_+hR!vC{o~tQR_uF8Bk%6!tmB*5BiR^|+ASu-`T{t0Jcq_L>$Cn& zA5J$(W9-(SU6qf)n!nYM9laKOhj!wQla1M12>KKZh`mfwb3(u40Z@uA{#Q~ z4-I%1MBh4`qv7p33YfYUgw@St@_sN4eAu3T4rxz1OR6b+$Xa3ZSX45jqXiN zqP3SUQ&D*ZE#AKq=I(CCp$~7-y{o55)uUCShh$PjL_2aQOqRv1d?PmeY95;XN@_SX z-oL51xy+KUe!31vsw|-Xelhh_Q$^>Knf$9*g9q=9qbEnJVSa1_9G_x`RdY^(mQ@aH z?c0UNon9@h+ARaw5nJ9cB#wJa6j*bK8Bb5DdHd*|Vw`DZY1*G`4Db|}8ms}NeQWT0~0K;&X;@m<yG!z0Wtn=O6(Kj!zYj%mHYAc?mP(nfAxS;RnfcaTIk7c?IREZ(x6*AC#N)m2-skG5BwwsbCY> zj+KU;7H9R-=2c#Ipx^RdXm#l`9o5Max%(sVo-$ILzCuD#-2ZsSS1VW8b#^1Iaqy?} z?*_okmO!}Xe;1T&o{7B`7vXzaPtdtOSG4|80moOZqzGGcp=p|oTf0SyV;***m-5*l zOHjwepYry?O|4|8&+>y@H#2xD%_L{p2*o!};@5jUg#&H(p`~&hE}C=~R>ei&ljU>R zyds#&{Hy5X%!xFnYdzJx)R)!`cth4zD;|8-i`iN7@fSZ;SY2Aeqtul-?Y03lzZk|F z%@4zt^qJzF*@vO#q(l-OHC?epu6cy?D$>B@(`I;;+bT2(+N>FxNnl}&B^w6N=-(fp zc;icP-Q8Yd>BEJh&2@M3In@Ew!z;x7tB#V-pf_}8WU=6;bp(cv$cCM#izsEYAB;%> zsS%nd^jGZ)d-GR=`>JHXZN2~K(FXOXhKBgSW8a+a~L9Ghe*UiOMKTnpYFMz!A_^%!654v zFz>$!W~Z17m&Zth`Up!gcHac4R_n_yUJ>-}nG;>RQ%!%b8?ug#R8yb*omT$JmS&!v zRO7#1bg$~jW4Fm`#kHxEg(c_C3to3+xT?!x&MKFD{OcD%`2f?}%X;XuAXtcy+)cy0 zgYd{R4$7W$9pWp)S1ZAvI+wNB=4>yg^rwewO_g47Z z^pLi%&!m~xYU#+J0C7o*CZ}~eE*2ixOfO1qh^D0$=zB*Kv-|y#$;a?TA?Vs4X-nq- z{&B@hS~K9!7qX?wK%ENOE;ixPVGD5BvsX0sBa_8YRiPxLuXGk%q^#LJFtT$AFKCv& zX_*&6{9PvOnY>=HMr(mE@Zbr09v?;Bk9DTWTA#$bF}>kql|MYgp%lAvDx8T~APUbD zX+qCbIjMhM8ysw$kvqt33Qr1Qr`^_(uSXVC-j8-GmjS(`?~|I6U7#!{`!L*y;q zM`--$Q;L5^%g9-}Hm=_|9V*XH#1TQPf*6;u;*-O=oRr-})8F5qNrN@HN&bSi<(I)Mi|zdF>Pv9yI+Wf|HsOClR~|+!bUUnG zEV;IbcD#Eb(@R}W>-2NURnrgFh)3b1p1)+XYvT1?U&QnBI$QLf)*gzQ-^u27OUC_6 ziXgUu$?ub|P-hn-lPVb)a@$?FsyrB+;*W{_e5b*ugF~siu_-P|Tp~OWUQ^iLD8ak_ z7Ay%;#fS@QW$Mc|VuRgQh}jTNrh|H7_>GGY)6VgI2?BwjgGn(;pyZk&<~fp9P&2;?YeQ;aQ!M&w3SG~oqb}syC?b8|AT8L<~YYL z+NG>vKG-H%^AlOVFylcjJd_=T`VLdXnmxm@Q%D8aF6;o8kGSHv1KM!tRW!8!b_$Fq zErCR<0`M-ATBZ7T<*;n!IhRXw<6xLf4mY++@At?_Towz^Yj-6q^5}_8k50o@UP^UV zo;c~=d<=Coz&fQq&?T%0qTc9W@BH0_#!~l5`=szjX%Lwv`_tmHMX>$mFi8v1MT0LE z5OeGpRDI4826V{+uf@9{_Qy>q2KU7d4mY7J3h{bjU+mkO2DS~$ zp{3J72)=L*{9QIftd=gN8cXfpR5$Ub|9a3_lnk59q@(!z657#Y3k}&6kE8o2q1_Kt z>i@up52J#kw72BaPa{9dkt+*@@JL_C~~9j=Z* z+u~>BA1$)mW-Cb|GRDOlW6&?$f=6ARM&hlVq~2JL&fW8IdTt1PGkqj9YgEGg!URl6 z-!E=FlnMa_Eeg*`ezfC=rug7RA{FmVCe08h?lWm6%OpK*%8qzCm(z(Or}*;M;c6)V z``=S=`#B948~WnGq<%QZI23<7Y{1CatMLB6)#&FEkCvxBD575+y)Ri$`BS5~g9J+p zYXPp>`%zfAxRmNoH&O>Fw=knB9Kw<<7xMlQM!Br_oNZXeR1#|5%tZXTTkm!OM6ue&Fq`=vTsl{SSp&a0;_tr@as8o!|Er4{UvPyGwMI$siW zhPT6h71lUt-*QwM-;Ylx4Cl1hojG^kKFV)-DHMJ{8sa8xOVzXuo4K9-C-`DpJ5? zqN$@;;9MlhOwSE@L9sexgbi-IFV%#{#({f`wrtvV2YzoM;RYt061Un9RU~e`Oe4fE zba8+e9L-oH&Qz|3X?5dd%Ip6@yA=j-ppT3w?u_DOjvif7n$0WUFQe}R3Zzl1UVIt+ z0T#YZqEkPHb1$!3qMODZS!3R5!KNDJg5#BBYHzh2@+Tjmn~#5sB`Mu7aFGslZ|?%S zzE7o{)cQQ}KcqfuipbTrx6oDAllM|A*=tuxls6BYT1DhgTtNf6mqCNQ4p$B83(*l~ zcwtVW@Xv4$&UbDPfx9MPabul0@?bl5b??GioGk0T!v#LKmpjnNm{L&d^G=*KNDqys zyb>RMdnUp5Rx{l5B$t-sH1+&>D%hnYxDQPd#vXO&+d>qremjqkPVx{h=tck;)`4@b z4xIik5Pr`5R<>nL0qq+x1#Vl<5p&;l;G6qH#6?4qogF8_gEMETZsY^u$oVYlUp9ru z$#wsU3AkNiSUjTj(mJt>hQVdyb{qzu71pG7FG2L!IGL6{v8B-M*XZ}hc%Htk5H^g~ zg{ptgsLisO&Ky4h=brROk5!vt#GrxL^}h_sl#?(8D!y;N9D zvYghseUlUA%5|$P8#YZEt)%U1%j5Q; z(#Jv^k!2{pepU?1Z;KU~Qx=HNx*0)5%?n6AS|aGyOLKmsZke`z97WITMaiccAl9lB z;?rk?)%1?s>)JM$*E^c3)NN?)uNUyS^K_Mzh;o2_>demNi4Cd68 z!rho?@RZO2a~60DZ{PH${9%6V&~Sk+y1BzZ?>ySvBT6WDs*}} zlA8*eh0yiM@SvaM?$iwvrUcrGf3gOUrcVd#yjw;d_kyw2^Es779FH0Y_9%`aHv1`f|q4)DB7^E^CMoYUa zs`(GtO6$X1&xgRH(aXv2$GoyT*4Fs8q?$gPIN`pddN|jxL`YGa0aXipWoxB2!}d+H zgg>v#X-ZVJ%z92f9Up%Q+&%I{l`A))cQ0KzbXRP=KbJ$ET8jI>=E5Yk9_&?lpQe3V2QMYEr(AP6T<~{? zJ!iBiXmkT;B?myrCmY@~V3_dmi5axB`wTstq+a}lIw3A_fiOZU^~Z%>mx~d0cj3D7 z5Aouecfv3}4|+WYaq)#}@aUK?ynd7+Y?*TtUUz;64#5r()s>5JkHxtIY=%B6N z2K=!zUK|NbY;Z*3G7 z?pXw77g|NVl@4@xd?B^;x8qa0ss(sdPruLE;*eR_Vepe5!tf0>LcpdJdR6U24|aAF zS$Q3`j`>bc0|wIQS8D855KE=X0_X>wrf17eL$8@3^tO|&DBo;tisz;H+xtg7g%2eg zA^uuB(6*@%=f-_eBpVnhdNw|!XGf$3xyQOuVDEp_`oa%pA6`OleLst@d<*Hg)ek8j zxrIxL{P{&@SN?xYy=gd=-~aw?o-z~>B9$RU8A@TV_u7V3Dn+wI(X2?QD78%?Aw*^k zRES0@_Vr#gs3=lON>M4Lc`lXwug~{)Ke!)wU>y#_-iK>V=lOaS*axv#8)dj%{{eqZ zwPcMmCD`+tXPCQW0KZ&lC$WRdaov{?v9nqtnfAh8BtD{c2Q5 zCez*pG8HCCk5$ZPD@GE*iZoC7>!d^lZ(?YtUp=%+A0@`4IPz>#Do9#pVZeS-9;{w( zjGZPc@%BzPcCus)TO%Kj6|+wAQ5Z$$qp8Cb4^`uNsjWCLb`0B5H3MVPlKAClHaCt( z#h(@UaNbX9@%x(?=9n@FPHRtM8#1P_j~VZ9ev=;8mA4tI=c%&Fr@Xcx=p457J`sD( zMd&A<|lH|&!kwHsv|RfqQOk}=LqKdIKm?L1#mEEA}txiZ@q3kMwfe$tm67Q z+%okN+<*HMmRByI+lG(D4R0wszj0{IB(vKn+V#DZd-m@XnG*RCtEVWil%vV)iCi4q za9;(Bl0L)A*_jxA{1_fE*+aB`7Q*+K+1UMJoH!(AA)GD!3{__*vjWZu?+xC}{TurZ zxo2{0+|*+3gV{fLx!E3_tMkM!0%xM}+x09jH3-)#@jTuWb!78(De5*pUj#UC9V{yB z;FaDt7!~BrRt~b{o?V=aojzjj?>%0)w_BDvCdxBm*jnh=cNI-wwf`!SGL_6{uz-O%DiS!D)OboG7_OTExv-bH*-^QR zv*}zH4ertGr8MK-JU%LQ9ZI(w!mlM3WM@Sbl*~TE1*;d6Nu>qAqcrhzW*`1}a|DBx z5{O7!l6ae4<|KZZ;CLeen;3bP|NsA6im}&tm%$1Jwsc!GJ0z&XHDNVSvFlVZS*Ljo5Kmpq*wz3oqWpjM9_j zx8DUE;=Y8fkyB=q;yiFch%pl$n85V*c3_))2$Q?kjX!wcR^D41rt`~z9pHg@lE*T! zbPZ)WTifuHzY99P-3tq%f}wDr5UmI1U{g$T4R<~c6$8q+)SUfLGHWJxW1j{4|s5+(ud>klDQ$5Aml(CiryXs z$7lSVw$ElV{;eGT9Jzsf3pNwvC|-aGUna;7;~z{Ozxj|CW9 zJRO$DU%+j4O0>xDncY|?H~c%{B7Xq2fVs@h6jY1WPJ@h-{DWiB!x~3JT_RdOz;j8C zitl_|!^-uPshh3}Rp;V(4fPGYuwgW3KBED}P9Jf`kKOp|q7ALB)uU%d6k&Ze*ehz zSl(@2W&raaKfpSnJ~c6U%imoI;ce?a!T0NF-1~iJ(QI@B=jtoR)~|}jXP3grWQQx{ zK5umOlb8nMh&_Znd531F{8@FGDI1x-84T_3!p;O^kYBMCPH%pJSK}XY0erJf#%(40 zb-9}}*68Ox{8=YL#$P-{bywi1)qBMO53Q-Uu>~`^r$J{ujKsC`1K{%66ZqWjKEzbm z;rAWSFm=vc&R}#3r&nW4zlO>3HU*PTQx6~ou81}^utCarRorPq&D0H^bpyRzgCYQ8y+a>i@mh~+!7ano1y z{&x!O-(3*DcwIyekMKudKLt*${2Gj0_YrzBe&M%NC8n(K1k>H8vJg?F9+Q;JL92(l z5Jh&tkgHy}O8q?inqCVyobBDH{s7$E4lElLR8+FhUxSH z(wwm{DF`E{b++kH#6M3!Bn8=ISXc`8Zez37eTG&23hXCk`G6; zVvE~va>ydB##wm+TPP(*4_kDni+Iwb-#!qbLf<(Q=a{g~kU}01SWzHPt=^5eKq}P~}F`h-04u$+9A{n~AX*88#eCcKM=Y%tFSND z39QX4g1x&MiiU^7=!ulI6h1yC?_+G4sQDsV-Oj+qql4+gBucs7a%vZ43s*$}{;Xqu z3d_t$VWt7eaO`9P$S=>NUn}3kpoU>M{?h=e_d2lexf|hk@J-P7%%P?;6zJfFN}N&| zimy~lnR??mHfZ7&ur>6em+YTG*yD@DZTnR5sD7TAcakS5%;@Lpt^s|(NznJ-r(vb@ zcRZvn$vmx!1fmlk{cuj$3@GhABzBm#7(@0KqEY@IPQ^t-Jf?Ft?VM&nQ~cI)$8)S% zaQ9pmKH?Y3elUf~a}xBdvNHWRAr!*o^YPHUHEhq@k?fJq07;zm2m+-w>Cr!)v@N9! zugdPmk5PQxLM|I~q&s2Zc4Io~$58s7CrnZ8C)1c^RXF?mE&}IeDA3B3Z=hCOOt){X zCQD|8pgt>QThflQf|bT}e4r{l6H`Dp$u6dw&tJtC?TPHg`fN7+i~@UKv4S4?7()kL zkfMG22Z=YQMY75VKXK8%c7ER<4pBy_w0!wTn7Q{mu9S0QC>w&&lSSdooTqZ_^#29J z2k(W`Oh-(;hv9<{zwJMNFlL=AhSH|?063lLMg|`+gJ{hzbgU{PhE6Idx%MGZ6`P>+ zQh*1pso1hb9b{aJKu`Gs*=cqRZdxT_YlnwFA697Swd**Y^p7nfku%nVDTak zn?;58D6jP%6>p~CG{@9$S5OP^;EeOyNT$ zch38z*z0~b4jt11A)_;3r&ktJq1?yulR3uRetVjh$XTS#quhSIGfOYE3wLFW~Y zU>nWTpeX1!bUyurt)6#5)Y{7v9h(%mWf7gwn|G}G_@rwvE>c>s=l(p%)e7Nz{!t`v z$0btNeuTumRKaJVZ$NWzIoZ5p4IZPHp?*nvCAV2`(-`xTjWes zS!0hrYHj#hx)OHp7{a>h_rcmZE`YD=@SkfqzTfbG6WU6!WZnX!J71YQAdZ2>t7hVs zZB^L%F%14_f5I^irh*ymrI4T&%YFJafiBh zC>A-uG2j1s9)_;?<&P5hgACns`Xn@cVj!zJm_G7}#35JH!8P$E?CN@sm!vk*MydjL zo^6LQCyvzIIW(2YNL7H2r5ud*j)X-q_XRsuGU4f{2l&`Kfyf=(1!hn4@Jjh@_*5}K znu?U^ch6cV?a-h*=fvWOtsy9WZ$?BdK|G;*>v3*)q5vOTA4dz*W4N{FE=F_&kwEDl z_}86D`aVsk!~Jp~eU>Rr^|GSlk9NUCHyJ9aFoXVG`vmQFji#k`{cv8^6`$Ok4#Rd` zf?XTuz^=vd)F);+ZC1Jqv7uVveW}^C@_-ElY^M@F* zbv#|-c^Q;mOR*CQe*|Ac^y$qU4Z8KQIYc)ZV9?n@e4r}Ar8Ee`mlxwtizpU#-~%2k z6s6kR-8n9Hv$z5?zsQP1f_`zs?kB(pkC~t|M3Tkv=Xa-`Q?XRRFcwk$fgE#4rCs|> zsfEQ5HfjoC0XOD=Uj0L8oEwfi8jrH96X`6k_y}wI^^iwvjG@bPr_oR6_On1s{~B*! zS(@~wnp{xWN8dN5L;NxY<|K;k$LQreCUR0V>>3n7Pvz`_fL*byWyvDe=_U)?WjSbg zqsMlXMKO)<`fP<_Fz>7KfG?}V>4&R++{;!cc7s2jvy`wS4l66Ul}}ulyKRL1;hBe^ zVfbFyRCt)o7n+bUiZ9tH0~vY|RzX|&SROaEo0T>mA)*gF0=C&mla4QnBr8JR;lHA5 z_SrpR0jDU>v>hzj-9Hnk>EK7`_brk=&TPZvoxCpYO*MY|B8P&Xd3d(mnax;pmUI1a zm8KQB!IsN*^!ZdjEJ;p*#pi1vW?(ip)JU@i^|`E*Pl%qX&1DUl2iRk|wQQ?6f{&TE zoF#ete&EPKZ?QX3R`BP(H%+Mz;uc{immB*KF57$3l+Rn>$Sqy!@Z~qzC8GnAdse_5 zUdnE>B7lt?6U6E~w+~Ijhhs4zgJN?mAXKxr90^>)}yrA})QJj_JFv;R_?)i6b|d(|@ReANWAX z=Y!ul^LYd0a(Dpf>2Uh>rc&}VRrDW%0zmAK6{-*5@t#y0m1Vxp-dy5Px= zA+TZ7aXNLi6w~VU;JNgBnaiUptT_J`V@E5~s9YYs<55cn%JMPN%7H!COkvhbX0kDV z7C}*79++lo(}NBzs1vFON-p!5qUBCho&E=VubINxXC_ddk_}f3hSBV)E7`0eyO?l< zGHhMt2W@{cMd*H)-`!rW2kirzWOYaj`8VPvPKq7Iq^&1&6KjjO*U{_P@{%T8&0E?} z+Lz(4#}`P-l5ym*YYz5qO61{Ev&pXAuB`d*KeStCp`ZRTtee` zccLnYWRlWZE`RpxTw}~m_Nvfrfy?Oa;YLvD{|0xR8o}m|WH_Jk%Al=laQ|p0>T&xn zJlp&X|J7SC{-B@r96yg+g_&5$E8-R zr>GoFQCtqHd% zfs~8b7WV=t|BYpX0;69Bj5WKjbXh~b6Hn^9M{KR92{M0%x?TX zi9@gslbWt!oKq5cct?{g@2oi>@?=QxB@~ z#G}htSf}9j}K!y2gYH<2G4ENSxsd+aAhs*RJNlKb7UxA?GuRxb<_zwH; z@+y&m1N88I6xVs{LF2lI=sH>tUg>*7@P%AlB|8iH>qcOc-Au@vafL_IR)NJ6cUoq3 zPB30po0{z9&l}b>Z=pwY=h4~0t&psH3M_8E!i4+k)JNhKSF&+GPARsaBd770g(o-B zH%XG}cN9Uzji20tX^$b@2ydA&3)CaZuui#|Y6P&_gh`(T!2wxcP=F;j%gZTU&^76!X{Iq);-Cg^Da~*LN zYK9a7TO^B89TacxvL<%3>bT$2M$(*DQ*egq80zGuMi5gW9Db^l+OUztbH@ zPer-XftWB5pDBYU(`L}tGuCv|```G+%A38lbcTc`BkH-g6J>si6zBu(dGKt+4w@?Q zjr8_nQ09LDSGo>j{)2Heum!L=4$I_LF8==kb}@A^xyf65LpwL1f^6lIV-dq#5SO<~Th`yuo5RBEiY zmPqZgqy`P4lqo6Dr|a)PR>e#>v{99oKhVYhIumKdB5Vv>VyLgzw5G`ENX(G7j3YQg6}Z8`_Dl%)&MI6IoGyQ<25^4#&c zQ(NJ=oDp4P_lE?kzJiWrt7zP@15~SEFI86^LB~iY(3<27o^+K{`Rdeqc^wX=(G17wV`ggt`LlE^zPDXKyL7g3hx}x@ z_V_KVK=lt~>(8Z*d!r!0!wp-CLu!V+o<_Y6@{Fg5b96)DOFR>D7`6{Ig2vM*=D9u{ z4dxG~e@4{fFwy6IT+K-h(h zo4;OZTtHVg@HKg{3tQ<~B`%IQ4?`QzLWH>ut!ysE%*?^?(pP~67iGY#2d|)ME?=zp zd_){$d=LCGGNK6F;cN&Pe;t@(1ergz3)L00e z{lyq_<35}#R_2Os%d(jRO0=s-KsG5ViPL-%$UE6xke1O21u6-+t+ffNAKn&QEz%{H zrrNChd^SlfQGdn;dZ=&Y& zM7gl1BacrkTf^r3Dzee?0@o0BpZp%H0UDEjLRP*G-||wRO{LYKwR$)_tPQ|-d@%Xf z+X`ZMcQ^6t`Y3jLjHvv|T5Pv{8NQa1r^Ea##O{tG`DNDxB7JC-__&T74e@oN&J9V} z+f*k`a*<^b!jobz4Q-k$iv5K*qc%fOrY6eze1;o$m_WyBFru5>)gVhxW=*cdaLt!(Wf_1%3WU_a0E=_SzpUk+V8Dw2PA*7##zSp3=3c>I`Bo+3(v7jZF&fq*388*yx7>(tLzJJVY@y7d5jNxNg zJw<+SBFLVaHGY7=W8>*@g#nPewU_%OqekXOPGiA|vvFGVXu9nNgZ_#Zk>LIwd72ew zL6^7h=e7q7Vf*i|1?h=$Y;xeanyl&jFthGG*dF+WQARx&;ipX>n({g_jnQ-ll>qad zgXzgp#CT%*ZwyFsy<^5g#UNEWLf?=5Q$Gd2`o`mn zd9ge&>^-t z7s?2;@3+Lteh=ceoI>vj z(0Ngax11)!pg>>lOV0r|`PmL!RGNu)qHQjWo5CaEWu)--&3JK{?H9Dxb3xCYvEty(X(`bEDi5{5$ z229oK$pn6lST`&M9?O^Ekk=~M)NKw;ky}gbM1NSU!ug>UZ`P z_xRsqwsX-K`n9wMSHHBw4!%aQXhjdF`qA=r9;{d_Vx{>5FwIkeiUTQ$t0W&(NE9M*xL-mqwS1747k;d54*PDkQ#S3uAmBL zA1Ok)q|sC@_9&he&5~fY!z`J2>=b78s|ZH^{RGhwwZy#Gkgf01r`#iX>U%dA!UYi! z7;=`?98IGE&Ky%T%4X7=lu_pD9JrOK1htS#5PwPXgT5N3MP`be<6Q3C*(Z>FbRHck(`^?-YLfPU*T6|?FTQ%$! zT;K^7aw~_^?Spqy%YK7!z|Cv4}B0~jFf!H(RiWb=%i znE3@!jUZ*e3VS7)O@oGqvYOWuvF6Mks`I`HPHcEWr=?9}-HMH5^=B)tjXyw4m>)s? z?j_Jq9hbT9p(*riyBs!LU4n-5vsvwjdYIcg8?J?wvw>yO_;JNaa{lc)n5pRw*Y!2o z=7AGXnsk^oi!ZQAV~fDX{T(KY&aB6(OEyfRas=kye@taeC*q(h5oFl5{cO>~YOH+k z$oB84$K@|maJ*ay69f&VqFZ-qqk0!zH*yNge&|7)0yaZv&`TV5y%SF!p3ZJPxP##% zTIfP?7?_x^#G#{ZG12OLcJaL?^BkE5>I*P2G(RQ^;XG10FE?kEEz} z=&&uU%jC0I*(#TF*_pvQXQ|hYs+&i1l@5byUq6)|-_I>I9srdwnRMx_aXe(ojgIv9 zW>a$_xDoICN&U=;bX|`gUGcsd`o*5?h*vIq9bLutg&SeK!&-WGeE^@hPoT2RTgkA8 zCs<;hB%SCRj}N_mLC`-@F)Zh89jY%1(APbJ#loWUufVQrVrzJAqY1`&l7`ycp`S%Ivmboj4 zug6Mu>iTyL(a9&dF@~v$H4@BE&N$ z+H~^ye=NVboGJ1TAuycC-)zLNVN#9M#zd8F|8SZe5UJJ1FImHahX=8ZI_4lUk@`sQ zzMh3O%l?Tw)SYS#%ulh*;YV2hj5FZ3(11BVGo;SmZuGiz7TX%U3>@YpVA6%HboPo| zy7cohZol1Ry3Z|@bK$i?L!7htZ~6@+F6T1aZ$q%^)OyzGJeIvG7h-w&X_UWRM2{po zGM`(#{jX04@@jfT?B{NO%uI@;(Y3Mk@>hHII%6arRBx(72S@$IpSD8Z$+Er{;jej4Y*0=peX#j54KhCrV?}1u@N(LII2)cw z9n^}MrrTc#-1Ql+6*Pl9&z}k3mQL#zr?N+{s;T*|*a?)!TKJC7cP z4MVJ`s61jb&0Ez5-i}JN^s*$27f+-{9T9Lcqy|)KGFWemIvvtp2Fizbp~mYUtT9G~ ztm;@rA3FxqzlrPd9`8JR7Ho)1Ug$8>pjc{jNLChRBjeLa%L*~;*rz9zvGMpK4609tFDjSz&3Td|} zTTy$H1xbwL@8r7K{)?@wtwguhFr%G>YuraNN|HA5x!>j8FQ6%66K>i00s8_+(*-An z)8~GazN@!qOGIzDnXbnvXz~Icp5J%V_cdqU9L5Z$i1OHLnQfK&};5Zs+m<=;{V(p>)?I?6ON!$N^1r6tW za8qp#yZ+LJeYB{81wN;7t+Y2>Pl#m`4?Rb{o15u$-%R>_?g_RvqnM65+Co#}4Vck7 z7qBne&PETCWOK8Rf{2)nXR-0YY-&U-sw$*nop%Sw=6b>Ts;lf}PYm6V{uTYwj&mDo zFECzTPV=SIShrOSPWZWyuhJxgvGX#vFtr?N2SrjR({y%agbH2O8%zttQtYipBGZNp z7JgBgWhVx*e1Rc!_-4bcpHE=Vz8G|!bXCN5%?)CyEeq(Jnk9mP6N=3B>2`MK$pZTB z-95VYA%m|LVm9AqWQ|Dk5ltBUmT#u^LY&3-n)4^KSxZR@Q)#hbKKDiJnr{Hp-Of=p zofA|q!h^mWr$S5GE5Yuih1mCD9$fHQ%<6KHj=I>vZt7j9*@p9J^;I66(K>B4U3N8# zDzyjG9Rn1iY>er-v{PKxD0!r64Q%PW3+&4;0iKA@plzClT&!U9yY8Wh=w!-=WrYaorj8IaAM$eN4oE$=$54$cCX~6YWrb1Ldy| zuo}@bwl~9{PMWid14w&czz^P2yhYpR;zvYg2eO6BJ_QO}9>oh)go#2m`COs9Nuh9q zL4xpcbh&U%Qnc{g)<|Jeai(zB-$dbak3^wP?OvhHjV;2*Z&HOdBvk0}Wu=fdMho?} zM+@`2y@jI!w+c&V1Pix}+$GGGS}&BX3luI-Ef5B%dJ5xQyo85(HVeNmT_$uj-#$mE z(!EBQof9Ussa+~uWECiUQsyI^T(?pfbauIq5RAJ%qF|{T$YFOXF3fkMsuVrICQ!TfH@U5!?^m-~q zVDw&s&Xbr=jgH=?TfcQf@}&^*yA)ZP`reIwGFuE%-gQjgUxl*G$Rx(eiyyazup47r zSfX|z8*ZgQy;%bDIR6>kd(LuWhlqFyg#{>WZou*%FIbuVOB&PQ13I_2)6nGs?9ujj zf)>qWkb029V*54OuQehAeyKfx-?mehxBC}ezTT4g#J`}Gr+9#Pr2~8Iu#vv1X@jqq zb}?C}6YSsN;cSwouW)Mc4_JEZD|hkGKg_X`6uNFVqLz8fqwUzI5=FpPm$& zbMPw_7w0pt67AZsDl_(}d>h1Emf|+sNORW>>(NHkdz8$%^oM$U?_~0$cf-u}9?&o` zj(N`SprNf<^!K5~Tx(_udl7C{dv(WUwpmACC}%NP$acEX8f_VN)hU?`rQcYckqKKn z#hW{}K}C2-^n+@QNrvH5U2Btv9%Kzyt&pP{aKpJ18q6rp`lJdO5O6Uts} z!1$;zc53!b_T62XDQeU+wzPyTKmL`8mV~nTdz_eAa5{xUFW8Z-LTr~buif&ButLXX zjOQz$X65_^URR!ou8-cyipGY~L^}zp)Od~FZ;PjrPYP*clLOt@w;OKg3aILCW8u?q zBQ~WVi-neCGetvF)>Ewx>tB3itsAp=TZVk?hJo{JU1$+IOv2d5GR7|7(xV-3-_rWZ zx75tAO(|?0BFIyY@4M6@L&t!;MY+HTzXdmO4d`PH}&WwqMN%n|u^i zJ*QG@y~Q1opsPT2lLL?`War0(VfmFo_ogu^_ztFVPHfjQCfqGq;p z@-z13!Fsw&@jN{rHvy~dl|er!XVJk^EDWkcw6=W1%b=m`xoKf}VBh0J=a=rOJ}>4e{DS&Sz5 z!*O}p+ALEMJ>`BF{on_URr<+PC1WXA-Jo%krb6}^E1se}k4f1dXR~59(6lxqD!WTV z=uzyz?iq}&)nCIuH?+&yb8T&5L_r#DJ9&-Otdqol_DM{SKlXL}Vv{(ueIBWYQjWUZWV6!TKu0}hK5+4TxfuH8eOS$Zv}9Uhu=-~DXbf8higUEGB} z6@64K#f?148C+}9632p16wnIYe6Br43#IKD`?R2en-HQdbailL<+EI2VVVM*dwh@O zO&%}Y<@1-hOg&6JXFexG$_}!nD?XT`{kH~s`}=taF-T^rJMRi6y?MxGY-~qJN14{`S@^$M`BQtE$U%-* zlF9vlI2$FIMWU+7X&B}C6XdNPh&S?f#zpxf@jBmd4RJkBUIvSZXoD894BKmG;v3Jc zir7n9KY!t3awOO#`6ZRo#( zmTt);Fzp~TyqgTydXP-b`^^0_&Vil>eRIITYAXcZb0*I}+<*_0Z@}>Sa*!VM#9qAN z8we~6=&Q;>pk`^q{S2B(_Vy{!xrV9S;b-4qg#9u6d2J*7H+zn6E8h2iv!}-aX@2%p z=4a1G%XP^kgJwReUqa3YF6ShFU*_V%CCHwmi@Aca+MJ`}9B%xP$O(cf6Nq?Z&Z&0) z6*$`YWAep8u<`tMF3@WbSF-u5IA(UjL?RQQhni(U+S>KKE+-SFIzwmhC z9v;qJ9VP^o#9LhHSeow}(QlC(u`#>O_f^cG+8%fvq;Tn$J5?d_j;!c?z!pKABtTRU?)3>N_|V?%5L@wYHBYKoxcubdyhWmZ;8eoyN==u!4&wXcpA1S)Z)^3BRG5U zhWHhhl6IXdpgMblSXsN4ySZ8qR4?X%tVx_W|H4p=(Vxy_u8k6GgVS&zVK81hlmarN zG?=64UpeM}Qo+}gB-zK{3X&vmiv_DCp+jXj+?ip3>T}Fd&#W0#?hCQu^>+l_Wrz2$_yTpHRH;OW%(n)u2s&0 z2{}Kws75Quy;21^o1UWXsX7$3@O6(7joRqq@y&kJi&H2e8N)46*b9%>Mst(2Zri8% zg@L8C8fhsUi@(2j2|lT>98qJf;$l|#nc?#$l7rs7?Lm)uQ^eCT@VJyuhwjI{NVDJLWtdXr)K6BkqPs+ znTh9Dj1y1(G>q0LdUL~a-U|$7TL~7PdnjJ4RVsM9ST=HxSWISxwF!!2UvLepWF=l1 z-xpj7k)*Q}S_DV$j3;dyBJEUzTkOa7M2KU5e-;~Yb^_PPz1*+g4fYvNMb#G9;>DXA zd^n>igUFxNrD8cgg=;=mq2{ntR|a8~N{Dx?pt4V+>fHFaG0lPtevCWp~zNkJ!3r6AE&o-oPmOPuHAd`sNy|{zIxRNguzPW$%d7J z&u%BhC5fHl#g}^RN6s}96zx>!j%|w)U#j#J&vGpgPy0TY%fEj{0A1ch!X| z#cku1|J)N#HYpRopJ2|3jcUZFc5D?as(NR8)Y+0-9GWIpn9#!g2~(o$EVkg!`>poA zPs~ZHz6W}>^ozv0PH6(`vz`J~jV;_>jShVNzK%3*J0$owN0Un!l?@jrpA=|YpAnSt zQ|_COb)4N+SClw+i}TQmB{vFRaYK)bQEgQ)8CdcUY|gZzlCA=tk#U2*IeJ9>+eGM- zP{qV{Ml2_O;I4j}C^+5zmjr$*BYdzKb!LcWael$4;oz{D_*?&(*sSo0peAn~ez`pb z-^tv>*nmEKEA0nYW^o_qFvk+h>aK|wDe_amrChSXFOB;p%!F4ul_)dkz4*zpJRGNZKxzcLq4FSY>_zHA{YCKAcNGr5CCjw9MzSYi4q%TE5*<9yJjX{|qrjs8 zX4MP2jfbOP#RdF!%nPb5)#%iB+wkxD6m*U-jpKT%G>gV~Ga?2jjyhccIzR3}jF25I@Kb7sqUCuW6CW z6wh3kh9mF2<;Hks!?vUXP)IPs)&BzU+irW9RwqLi)GXqZeAj~DmOrQsj35SS$+$tJ zU~t~^ZAWzfAucys%}QbdQRW~e|J<*NTetC{>D_XyZs0Ujyo=*% z?P9n%(a{?JV^W--x2#0iO+VZt6*(a_q7+6gl@&!sO_U`&QWV&Ql@qx3kwQHESeAsV~gb(|8w=b&hE_(wA*p<+hH{8%VZw>qtBpLEJBc$0mH@H$L}*TG7t~yPCmwpy8AUs5kC8OT<=iLfTv+aF%Y8E)X(yPm zlN+zCiOz{pm+&Yu%S z0y?jvf7~6x_$hhZJ)0MTj+@^EDc&C3i0Q^${$V2`A1@{YVqX$oqeT_GwXtUj|NaLg zL=da`-Qu92joh#Jt&qH1Ag&2*LdEOas9A3g%~6%yq<6|tS{lP`(Vj-iS4d!+y1n4p zp)QPlYRm#IIp92-k+_S0U^SjQPfB-&VeA+Xd!Cq$?$#B=HJT@S3CHt#2T{%0yZf*& zx()BT4rOV2V!Wm!xp&z0q7;e7D0jgA)Injk6biU zf}ei7Vejq}ATjR(@&8_fl2UQpxOeejJ2!_j5b^x%KMO6mDP{5)X{^OPjL75muNcns zMqkD)Wj!QB8qq(}2a1m`K(psH+!?h}SP-s?=T>E-l_q}>kyZk!BcIodsn%gXD(gUE zbQnfAb=!`)kWFxMDp;i4;wpbFBm-lU@$aota;c*ezn=aG?|*S{KSz~{M0&?Lhk#VP zwdxwQ@d5YZlQo?C6l+Y3;cqGy+0wxCq1bm{Mv(FIIC;C|s5tqDu^_GG=#*`)3*hO} z9rnY`KV!NfircHTxv<0K*l%UdJ@Wo0@M%3EkX6`ACK{dMq~7kXet#`MFz;vvS8-KF zd?{RpyM1xLZHi)q2w!Ghu$PfZ7p(Uu?9~3-f?-7q*k(f&u6g|+&P!R%ZrAu9;=u!b z0%7e?65cH-=-=_IhQ|5eL-h?XE?)y}rls3@YzrXCK{MEG|77tk{X!B{*d%Z|u!0EX z%0Sz_3%2yVg+4w!Ep_e_d7!cz5~dVzah>W!Gg&m59M_MAdTAL>GhluZhIVyEw z&w6e4J=FmBuR8#nrneG#$E#x7Ar9oq(>BiG_6NNvMzw=eFN8!$Ln* zoKQE18~bK4c-v9v-sDa0g+3;E{UP{1dnPWjHUr6tH!*v>7$!$+(9u)%aPZM#aO6(| zNKT07IRHM`IhbSPX2G<{hd`!Rk(@eck4=;71ZJ6v zuzlJ(*eXZ4cjaHW4fpTi7oB9lvQ9KLHiaEu_M%epR=E1vjQ(t~6QfF@pv~2g&#fM+ zmKRHNe}<=un>uFK2!GxZPtcUKSGDC6*9)Hd+I#)s>zD<*#rsD}3!djQ42X&l_`zoG z{nBUWzsc*9$n;Y+{Tfo@bI~USjs59lUhzhvXMNEAYFd-vKvkche)?&i!1D6^*Ol@# z?_0LLVx}%F(U%jyr*i-ML53CXWQgLYnw?H}|M$5^|1q)B{sdTjPIMginqT1l`)$@e z-6(F0tcQg3PC>I{p4i1s|NLh;IjC*+!U;AzQEZk(=51`k(Kpt^iTna?T)_db?ae^{ z{_VtP&i|w6yu+z}-#AW^h!Q0lA|oV(gwJz7Ql!#QS{ho?R4Nq>M>5KuWf$7XsEG4< z?gy1rDq2cYOH1q9R_XWo{pE6994^i|&$!?3*K3}TyUm=1t0#lkn!aded=+k+^y14V z=m<;o5^?N118AvBBNKZhNS4MS=rDLVUwu+TG?T4Kg+D)z2tB5n!du5};_k`=!lF}c zBx`mH#^rxxZI-V1rTb-4Epe4SM_eRhgv-pOK8u_`ca7Z|d`9##JVGw-t-?>!u83LI zqhR|E8|a8YvHzr@uwYD)__yw^*wfAqUPWF-g|FRl`>PMcF5nJiCHvrG!v>UnS<_X* zyQ(rnqm3AQ?Gz05o)2?_qp(L?Hk{Xf#OhoTdk(SUwyziAGubdY<3TTeq;L{C8Xd%N z?;6swy$mPa@rN!CT`=-#HfHRYDy&HF5Radn2S$C*vTJ6QLY$`xIyrQ*xQfZxY@zG? z@RSu6Z|%#9CM(H!S-c&Mtv-dv;1W4?S(WzRB7^f|H!=f5O<33(!&GLELD`6jFt|@o zwkHAQ9?514Mq~-GXUhh= zSN#qcC~m9Jlgrk%CXm22)_iSzBG^9tBp<&xnfUJ;1_3%hocEPzF*o^tV$A_nHgnt) z`Ni1fxTo45C!bpcmR3zbJo|~G^Cn}(gP&MEECbWJ*g)}6O^l2R0jGC+$+ds3V&?pJ z_^`7NY){vucGiS^x&4&Q_qj@P-Wg-Zp0hG`)8rKSKXqehsxtrlL7Q(gn9t%8Hj%@_6s8tO@;bT;gFT+BbwiD!CI%O z?5S+=VeEQ;4B~VNyuNZDxq9(7*dMrx|76ng>iU4`uS^gEPnDCTvZ>JV?H`~|x?pa9 z7)ukUgl8+~a1FoO43VM6adXd*{Oy9@RH3$WXf zFi0)?$gJ1yB=;^4Lb-MWk!_3XB-$0*vDtN}^MCt$@q2Pz=sPfl2rWn9z|T_XSJS}E zdzgzGhb_hKck9TAJ?pS%KVP_W*ccYvb;A>ld*@za<;@Pl9^QW!o;^-L>+rS zd@%JV^t#)fCcaz+hF$WR;ZkerV_l8Iysk6Q?4IWqFJn^bbVRfg}< z`m(6trEu!?Y4%w^TF^-KK#fUZ5E(y_OneaskAmVwl`(zsqh1{OZj&W{=|2n(j!A^` z-X~%DxJ#_e)D5qQtJnp-!Tgr;LezaKVFaAs;r`F-#AU{L*mnM`^QNaG*iD;Eh(0CL zgdxMi*sj^$IQ)Wy@g8&(J7$NHyJkCa-Qv4~NAH1r%jM4)Ha3w>>+zU9yge6pM2;8D zcBX=5pPO*u>`DmgCg6L2BRp995!ADG2y4r(l9q=zovhBrpu?bI@IAE${-jyL&>@T1 zdY_4SyD1!Y-Y+K8hTO*WvW&URQ~M&jJ5;o<5LvH361OhP=gX_k|O@fQ&lb zNc2Jig*Qp7$aa%7612Mq-xhZh{_EDlmTqb!ZQjqF?KgJi83(IyeytB)xcQa^?EZ*m zt|e?!ms_OsmLhzr-N{~c|H<5J{^0XxE>Je3ihapxVzM2Y%|g4soF%=B!cWo8f?xhT z?0Ely_|EOXv-R)rdV30NFxEl`*1i@NF&yyH?Ple13SO8i_FG2K(Xehhn#WTnC=7Habz`NI8@Fo8u**dvghR2=L zv2brRe)3L%z+5-c(egMmaf^hi$**wgz&qf*@H^Rlzk_H^^Mdo++sS!DFLCT%PR`jh z5#RG_JpGrmQS(9rV@N?&&gN%$smA^sPBO_1Yy*`SgL= z{rAgp>kVy|c5EcAbc~V5kMbbbN;V1u>%NPV8`Q~)b#9!F8h#9#GAH7y&)u!wgU030+9DN`jGhSd-5BN!PGKZ3A-G= zEo6Q2XLbt;gaf74#A<`|{vgG;|Mt5k-zbGE)LI|Q-&==>FUFg|$hI5e``CElrJAo0 zYV(Mt2UBJ<$6dxIOtux~Pp)Og8`ld18xD!RUltNHJIkC~i^%8UdsvpJ$AlM!V%X%q zxWzV@rT5d3Pb!Obn!0)}>$>}}&~=ovpyj6=rgkJtKDg#5Ds8?7>r=<$+*j+E(i}U% z&fy=Xo8J}JA5w?(14jhiJGNv~pS|F-V+kl;kr~1W-%8xDHlCf=dB$dcDs+DEtS{Df z-^Q-I?M?R=R$#`{o2aebiwFGvB2MXQg1b(AVspI1Nt;|1hRPS=acTU2B)A!8eXAe_ z+UZad`xKV0(}h(&VfcZJVT+q3f%S=KDPo!j#!FAZ^0_fkKfFD`P=bLx*dIV?>jpuN)=T%Y(u&A@JC{@wS`VkF|fEACij zHUU~vBoXwHZlHT9os_mJGGX)+a7mX#$=5?L?%Nt{tF0zmOP;gH+-69h83ScDUf}&@ z82qYL1)0L)gY03Sui&<3KSKZhyiLmompmCIU8PFT;3HN{`=%3^H!Q$~Dw&`dag20) zI|#=ABJf3aS5RGjftU>J)~>LG*WlWP_kvGIKCT_SQ#In2mKUgkhEkS zJd|%we!00a=RYa%YOEj2UVW8(c&|YsoMKp!btK#VYAAXCKA0VDEoQ=(txWu-#a#S+ z*#NH^c6H<~Qr(lX#G7*58SgEQx0LE&+vfmmThL2=w#BV7(nWEObaqKlQ5pJ(OgD)1VJ5TvIlCo)iuS!^*WKCR#go1}OKPbQE#R7t!V^xz2JCt&Y6(?D<&P)$t zFRM2s_O6q~9{MXt%^*fbN_6Ei=^Hh(gUQhN3i360Gm}1No(J9GPp+!G-)95`4T+s~ zKAEJ>Q4X6F89@Rk&tSKLlG!g`Wrc%h`jdng6jF=&@C;Ks2wJ49u&`f$GUCZ882I9t z=sN4D@c+mABXy9TuB;(45vRe9p<^(1>ktQJ^YJh+*%TL>4nZ5SgXmnn#B9ypV5Zp` zaE}Nku2!z7H?4q;eUeL7yUY;}y{=_S0n0Fd<1ym0ry0vnO$5cM`NB&5Ns!dtl806% z2tO|B!s4pq_$cs~=utnDtiBygic{UNvwSg*m|-u&KMn@qv33@AxJ0lU*WJm76&6g* zC_)_Z_5Yj1mNZtb#`{Y+vPKj5&U2W*o;LiaH9%N9{umoSusag< zE3oeDKJ4vtkZ2lN@xH@NaL61a(YZbG+Nk>&q_e>IKmrdq`ub#H`PD<-xX#A%FJ~Hg4-vbahvT^|~L$mP&V`aWb6! zxig!@jCE$8W$J$Pkt`8gKPZrZHJQ$1NzRIzk`6;=w$J7$s|FhX5}gWWuM_V=Eg;K%+H3*P_`0jcO_Y7QV1tz>!PgZM zeRJz@SbRK~^;hpoR}5W2-d|gYwky_>wKAuvFz@>?Tzg8I?achfNL!pJj86rXpYOo% zVjlLN+5?VX%OJfcrxU9Fmz2x9QTg=MuzzO?E_sj$ZCgaJO(&&yLD|_bxZGXSRfWC8Z z&8Ek~4EZg1;OGGAAOhL8679a7c0Ne0`KgpzsP2fw=G#?bm+1c>++&%H ztx~F4RFXRVwd9F->_j9)G|65`9<*_gaAhyqUV4%2ijO00bhq%;MFrQ$>e=VMKpk2X z0CkVxRlP7g5@rbDVan8a`w-}}PnQm-gT&G+_E@htj(#yahmY<$5v4Cj*oU@6NY#lz z`ROQTD`7xNNjtHyM`OW5PSF%+-f}mz}0=(bmflFfF zf#dmCIBae>yt!q8Dwjs$VS@s3UTF^7YwAMoM-C$M^9=ZVxCK}^;kI!nX-|u0O^1)OGq)&P>X%GLs(HcTDf5YWOA0aiG>*L}ScI~m{2_2; z&_vd0J)XYwYh^~#4){O2u4ImiIk|bON$5xuSnlxMa6I{?bLjZJqVLPIWWAy!gR=Go zv#cSaicv6`*JMJDzd2gvKA?;Z=~pVgO&Lgj4;#bO*IEndho-XZh}GH6PZl!JA9d1iYbGy;E88L@KDB99PRR# z6n&m09P80T?qxh1luADdE^lVSwtP)^>m`v|)n&oB)#NAM>VA$5e_1TPE_o|)WxL|^ z35Ve6*5|AY2E$ZdOnQ06kUJai5YPV3pmY8lxnR>myl(G;f{AaL`pU=5=*Ktl=*j@+ z6?0Ybbnto99>j6{-Fo)7nchj>)FaLGjfEn zg?&*+`IYb~Xce2BkS%r-{|c(lA30TNuM(xeh`@Cc$;xe$$>h3Qf@e}+VzM!w=$6~E zQ3ta{qxh}RQ`1$D)!{iX)*nsWf?kMrKFVP~nh8-T?iABKbePRjWrcpvl*wO%bO_WR z&nHe}WTv#6xti6D-8$k2FY7HxsFw*_+Al<$r`U(5hb*IYeXrq^UW<8e>m%6Q#gE?e znMcjv+wze80vtB=r`q{v>3y?qQjW}o7?f|4@h;K?BXR6Vo^vP$F8_+f2`k27`l)CM zgztq#{sNzIe2TawP>FvibRgG8b-?j2+v%1O>#_Rk2$H|Ho((#u3gx*Oc<|Ld)QU6U z5mraVgp#K?KI1cf-;+cZwL8#vI(Om0{*5@KY8mQIFUO!GcgS+s0td}yq43f8969sx zJg8ncE6sn_gPC3`xo;-o`WXVZoHB=0q@RQ76Lz!D;U!p};zm0nXQF)2T&Ov7kQw-7 z;9IFOyhn^-W&vvS`;tZ$ad$YsD&5q4)``Z@S1H*3L5c6(Y=<>}_rtfYpO|^1gcO?_ z&n99x*2=W&uIVY}+jFRxbHD&XZyWG^9qu$Is{_(E9>kuK zdHc}h7F-^!LC3wYgiBf$u*NAB`nwLHD*8IK*2ISH$?MDW*3_fg8ao*G%7C_;snQpF zD`CWjWYHsHFkJrnh(m6l&0b3Fv49Ve)IPUoArC(a5id~ z-eH^D@*$z%I;NGz@IX+(A=?kpn-V4K?x{8?T6hPImnlPV`9@-{tO$vt>acQBD)DPM z1Q1b3>MQQzY+pC-q<9Y(-qoeO^Ihe)clz^Z(`rRq!?kSlf=^_sh8sN*3N$q37n>C? z(}HH%O=9=MRWRr4c=#9x_I<-gv9x9oDd=7Werw7wS2{d?`96_$y?hjtl|QnV!yNdg zA4_rWwM-)GGM85D$e`=P{BgSZNH}_0@)g(~<=W$wsM+aAx?_1Ho) z^V)}9m3u?PS}mMm@B+ubna5jdmSN-_Q@pt=7Vq9?d{*-k-gVJYI{3S!Zm0J^H~ctH zGF}1~KaZ#C-A7CM**&~=jS+AEbem=u#Nxv_12Cq%82UBHguYKbdGlBo8J}Te&MQxw z(KTx?!Rhg?h zN|2>e$sWCzb%7AQQ_i*TC-z!7< z##x(xJRc+!KH7`T=oPSF!QW*-ecrO!L=~^Zq{7;%% zebR!yZGL!cF=PMQ)Vbb({{(0Dj|A;JhE{z}pu^!!>}9JX1a%cLdlSP~jW$4b_~E>* z`yk9WMvCptLDN`$oVR@ysFcLO7=ptK_W6GtWdV#TT{s5S6j~ufa zDt)~zF3~_^n#aevJ`$>jrN)G$e~%oO7RKteZe}0xz*#(gvSHcY|;(0b}}KhUIyKz+j}r zr`nV&&HC-YphcP9>TWF27j*f?!v=W0z6SQnYAo1lP-FwvEryJ0;AORsQD#4b%HEW+ z%RT#m=eK2~XjB&Q@BI{T!T@|kj*5S+^4Q(-WGG2Zg36ja@w%Qg9q|l@o{Bv%w4hL2 zr!x|c1yFJ~{y5uzawNArkd7B_O@r{n1MnuH7CJX6)4dZ8;F^RUOqMBuh{{@bupmcw z{CIA;SX~}M_D^tT6K~o=>RJ&#>{6%87dX?|%~{}9@{B|WUSvbVY{>0~fjH5z6%x)# zZpjJW42rtaevg)NG%{-sU%$sMjqeiNDxa&GRygCvWDD;PKDFU5#S{fYn+Cu00hEj`CM_j_6 zv#_Rde6{U(7-`~5uSp>E`HzcP{Fo-ZykVx0b2<%Pju}L)6x3<1$6aWZ&@ShO+(ZS# zI&@*Gko;mh4tQ}2qm^cYeY@-=b|iN}v!9xnd*vJ&Nl>G8AAiA>WorDtgI)0XtR~iS zR$qKvGM+}x(83jeD|zv|Fz98?aa z7bhQ@DfLk3#$Pb#Nf_NypGrp-AApkS6?p!U39mX*%C^ZQM9aK~2{`HQJlKZ2nS*mN zbX)tDgb$=B=ecy9aZ9$v@a6Oo2(eQgLGHM0mTk0Kz5M-FW+v80Kk0 zI)9q;;P0+@;LT@n>4o-|>M9F;yFSYN* z{cjIoquaVtQ_m0?7}kA&fs<6wyFkByt_pE|Ce^s7ka9J2`8HEqYMsnyd!)Zeg z@NhPfRp!p8`xNfL{(MjBu}xE2V%zEX6%p*#;^`8Jc@yjPF&;FWrE9m8kgM z2du8T;SRlgykq(eE8gzFi+>IH##jCbrPKMZHfF-LRz$6642pDd$*ZF)y z7m(@MPvDLGJ*oKg9-6exsYiMd{83#1`u00<%g7zzI_iaZW49JP2hpUuaXk;ol+Zn~ z#c=ei7hH1chpFy1+;r(bR+F)uU+FghXV|~M-s~HD6#WRj^#eqmIgWI*YBtrjFvC?Z zhI1p2LTtBJ!$yyLnCImqqkd}rQ0c@RdfHPO9AzEk_M6(lEBFZ>=iblCvf~LEBf#0WmfPqk+e!Qw$?|ot#n$#aWcUqREXR7iyd8I1FMTX zA?q0x3Xi!-|Nncg*x9>)HMQ?2yT{dG!rQ&rHaZpTd$kJbDbDPsStfhBu~wK?ejUT7 zdz1MUL2O;_KjPT3n@m!?!#o#CnxOD{p;0RiN7}!E*T1je@AWBQIo}nZv|He#Mb|*D zp%)A@F@~rLL*}{ap19ujC1QpduUMTRgA>l{u~&(tD*sZ;vU@A@N}qa?an=Uo-!FpM zf5t$1dNA>hHsl(sviPu)znE^clZ-!?%B*J0pcjKIIo)4J>L+Fq)wf9y|L6$l>i48a zyi=&gqCzkU)!-%ZPml~b3jHc9@kG)bX?pe!wrrWl7K^9&*_#q^kNzs9yB~Dt_uHP~ zs4ctc>2O21E8U#tfBg@1e0T8|4SU6_6DoxZ9+Josr(>4B97B~S<4n<$>)2=WakJ;c z@A_PfAMr;1(N&%Kk56Vd#?@e@lNoKP7)^iM=VI->wYaUm7K}S0p)98n55Lrf*M6O_ zT{dSBb*x_u`>V_0)%vJ59(7^@S>&aC!^{<+ev#E3+^*y@O@--Z$8oyTFTT`YQ) z-h;xhy;R;On2x`G1hqcz=O14v^C>FDarFJ zmBO*$i2;N?J%R5}W`knl9BBa2lYf6wje{G^>6g0%Lazt#Pnvq+AXkBlL!WHo=&VbGc^e?-nLAJQmpvg9Htj;4 zK1PsiCAH=|Z(wa;9vDOyNzUQfpg=~@s3USred~k!ceF`;9*%*lpRupjj@0~?9EY4z z<^^+;ahql`*>`RhZkRcdCx)#BUspK>7EF^A5M#k{>=f`S*bVhnsxVO!Bu;-61s%($ z3S$#y!&#Ya9;$4R^7#_hdwRV-^Idb5oG*AG*;dX%Q0N%6>ah@(yA^+o*R2W~5G|nM16o`ZjNRyWb<}aYA78 zsjW_HJlgS#$P4;d@1_^i=7_Rw>syGHw*Xn&?@`kIE|f_e;qMhn)Mer~*!Flh zFMK(jwOTmw(!o=?Z*8Ug*Q6$}nvlePs~!{h!8Nq~Zx5L5m5t2@PeWakKF$300m8e@ z!xIaoE7?uEzBtDK1IvKenJ{}t90TDp&B}7 zCbLP+haf#AA6AywW79qjL1StqmXw`AJDoOib1Q|L+4`6}bp)Kfz7np<{^8Gy$H8~E zBK3cyBIq7#V_SMLak`l~)e?LfO)yblhU{HEpK0QbuyByHNxz69Py}|-M zI;ino@7;KGH@F^V?RMeqrxxRkHdTB-P>KJys~E_#v^(%yeWC&O4xni&hcJLw;pR;}A!I=$ zUU%Gz)@r4!+-)WvDQ|!X-wJfim&~o6h}VloiC#lXuw=~(@oIZBDF_})|M+c1+%XWB z?~8^N5}$Fgi^(um3mOzyly!wx0#5~lmg|^Gi}ki z#S0(a?k@J)^&F#&8l+9dSb6syC&U>Cn~6u{E3BO*1xv=;Ft7OTY~-izl87iq=<8QO z$~q5_g?i3$d)6rI(W1{U%6_JT`6+GaKlmJ$2G&FW>(e24pb-!O=49bSEq6u^w4~JsYvG{hyDdPAy3!ZP9#ml5IZT8rn zEPL&Esua8zwN#~Oo@fiPPW5o!Ee{>N2`arE3yNRXFbOU`2lm#w9CaC{Z+||q-^Zn4f>c5XN|LE>xK4y45}|DVq<_ioj>s*vD9pbPxUsW;Koql#y>-HyZ=r0=HCb5 z?U*)lcyc_xe)m^cpEO(&qp0ztRqM#J1356b?;7s?Ydh-}q(NK12w2@*I$!Lf znh%`#+=Ej7&(CiL&$3D5>)U7I$uPa@5$F37gVN!c(9)Y8EH#$mwN~h9kS!j3kw{nl zRfmorru@W&2dr{@7HN~N)F(dQ53SD__p?3<@xKnkv{p?zAhZQM!!&3_ekq>Gv#Z`5 z`V?O9R%|_OM1QzlgyYsae6oQa=#|TQ!rsb7G$*i^vhb627{^M`*oj)wIMGast2&KVN>CzmJ!65t_S?)h<-bEK=&TY&|mhsg?Y4CgKKpi`QEn> zFD%%H%M1GBy*c9`RFa$ySaD6-up~p(Zi(c#elIL6o(6Hn{b8t%B(T<%N(N_kvvmUp zVB4qwuv6`acH;+-qXW;tu7V)&uX!vQc8*46h2quB3BUw%M&9qK+Xf^mU2;K0Om<|zeS z)_Qyqn^by%?i)+!(~*bAg7Rg~W1>uHqt9bVICh#8M-9ZdI1MUZw!?zES8;Z&A*M@a zl)RHt>H2dN#KsuH%?C1gsiI4rOpE_+$Qn=US)EP^TpID z^QZIeXSd*6p)TEPrbd1&)uXo;_o_DAc>tbDq4D6&>tKgepT3{)1U+K)s8*#a8h1!e z;ix*e^QQ<(mgK^mA>pjGqZ;n3r9haJhWa}t3B8V%l9PW&5&ckeo*i9={p$zQVwsbK zd+pZ5uI(E}9q;U>Gc_e3%A+rVL*#mf)Q!TzYIgzb97ml9>HDU9R zMf}5vwWzot3FaJ#g2zAiqUnbV(5e@WFK7|0u5jcAwO1ifb_wS7>&MlyBcOLo4Y>TU z=BKUW(f5-UZY~)>+a8`^`&Q-P)-?m!-E2P@Bn;5T;AOt>*eDDXXWF3oiDD?b(*Wku zp@e*2h^Ei-!1~{0xF6Kc3O~$+&K6Gke2Inrri#q#JYzYTKJ4|zH!N+!Y0$1sAWtfH zfrCmlG1xj29xIHZOUNGfF?TJto>6z66j1?LuZnO%(kfh3Ye5FCmbpULr#tY$=X#ji ze-)nVmIXE%()i`AIzL)56$94|l9xaUe!cl0+W%S!e!K6AE8P@f%D^J*vC2+9TY}3+ z)!rh_ZC{D$7aKD3LJ0HizX#Qqj3Mt+P*E`9ELW zWnIMDWos3wc5)YKlK+8dgak8QQHc$6SD@C`2jUdv%c7~LH7;DbL>^*zRqT`F&Pamv zZdatovp&BOW(|Hr)Vf;2593-)%G<%V25(2NdvUz}yf$yp(W`EqwvIm3E~FN7jOjXZ zfPeR(MCs&w;jLRf+buQcT70)i^pwL_aLe+cf`7tVY~t-W(QP6p9n*1hY%?wp2h-H$ z($Wl9fbw!MbDvXvaHa`eS_6=|q7ZBk4W;$ryZPG7YM{{=gQkbvX}WePPTpt<<9-2E zyLMgtba5xDME77UBn5(g1IzW<0)`ql&}+ziJ|jOx22cC-!$PSr5OrlIT#cO#C;o8q zd(#d)p|yr<4=#t7?YrqYmF4u=u@ZQGM3MGAYX)!s#FN}5OSsjzI9zKjg3T#qNOrL2 z^Lt0rSCX}_>~|HjS{4VP{Rx_mmCDG6lz4Ty9FFg{rsJEe`0epOz_Im`aHmApgC0}W zmr@iid{b*1tezSHdv~TH9yX)hn=JY7f*;@+a~0gXyhm5p9(=!aRFltpf_v|@p!iW$ zbbK@eU+Z-Pv&e^-_beWc%qSyubM*KX9*r7{p5mCVQ{byYBCOh|OV7A?OV@DsF>HQM zh-^QIx?lH7{V(@?((*+eo-|J4dr#}~6{*AU-3xtOknafhA1;OQ&Nh6-=zO*}vJ5Y7 zkEUt6_Tt$*6D-qA2j$J)ILqcVI6ab-ai4Ev!C`$g@TtRtg}wL^Df_pqr-gVSZWvs% zPG$qNQ=wN%A|wsYgk)kT+4kN+o_aFM0@j?x_-qvzwBo{s%5}+d^dbr$N)PEY>-G4?AoY#Ij@NV~>PUaB_bvWWS1sMuWfNGpI!Q z)jG1MEEszisDaVf%{bjto6nC6BlnM}z{{pX;P-Vg%@a<*Ws?*z?WjfdReCh^jEqmn zmr-apyFk7#t7ko@b>iB?Gx)1%eq=Rur_0u^=P}X2;D3Gr1~kQ^zM>-BSXu(jR|A-; zwgsFG97DCW6-?Wg?5!??#3{i+1&YOFFx5POpMeWg+sT!!`YC-8b>SvUROozo0qW5 z{3VH)5sgROCc>1ZKbZZspX`R_8f>4WfbkFZ0;@LS{#{<+7Ad7~rL?uo~UZO2f3 zx(KL?FCiB?&g(Cv^b8lL)w+v86{_lM7M z>z z%ayX8J|*OPyJS|hSc-qT--H#}6S(YjC|`a)j{oP-2OSQDK|89m=BBuQ#J}U2ia-7)(PK=OfAR=4W^1Vfl-Aoa8VIO@1&)yReqW zyEyS%^|E8^ch(46_p%tbEb-&YHECd7^2j;CwHCBBa^v8ThB(ootvo4oj>h z&+z+r48}sC+IlTyl?wRkjS7FD4!9&! zpHL!>Fah=I-bkvYJM~rBo?rGEx@S*Im z10QxZSZrUkOq{QFRSY?C9V5H+po+20@cC^6DPClc2i;BR=y}umyzNflIbEBt=oU_X zsYjvf#W7@3(j%<#Fu@Dg-T0<4VYt`G2R21bgPSFrd3uK-{&*|H49(t-!%-<6US1&;rTFk*KTpn zo4uIYOA&shRlqu%lkDC|#^%;JaX*C?Ji2utzdR=cPgcL;9u0By@xZSzVt+a9eaVbR zzlq2BSF3QGy1;)a%eLZ3uTuJC>p>E{)kzq(uaf!?DaV5cMv5Pw8w#9y zQhkq=n6S@^=Dr+_^CA}FjnR)_PLy;-Gg*}eg?r;2GJ^cRYQRsGs*<0b=E8Jp!{fhA z~pfR)rCbRyg`$UoNb4;42mW5sNMTTv{tnK>u}< zx!TK@LKk;WY;hRO(l)1HoZnC$abg19w`&J)x*th{^xuHeqi5{mDGM%J<<`KOG8DO8 z$^kSPby2wcY_W8qFa=hqOof&q@8PXdGM}+y6Rte)1rxO%V@RtyZRFGCWn*J-tp7fG zWmg(qm>oyD_BLSUD!+lw?#4=2j;2v3uZRI^B^)OS0B?jIEiUYQ%{(=)xPwUVDCIPjyct=jnNq%ZF~dL7@kSV}=%pGpg( z!XPNz1WiH)((rUkZu{*%2K3gT7GwjAH(ZB{(rl>7+Mf_>eit@ct^%iPkD#JU4O~o} z%D>&5j>grqVaWvI}cLrjj zZX>R{8bzO+ZIxOP$3fAgf!x)f$opI6!fE_)P&ow78vlV!N@u_MEinvUg zl;t1sq9b#AktRQ)fgm zEpI#8c~bg|ZYzPt>9KTS&%Z3CARMbJnlNr;5x)K?6JhMmDsk$Ke0-?hhp(1Bh4wkc zJS|3_&rrX>)R)H5*@-=H-rEGe;O|tDWfmcH8_*9+=Jlk{E9T(I)P8h^!5W$_oePB# zM`@t^pcFRz3y^$?|m7?Ke{Qvy+tm(q&)x+?h2yQE=SYaiWF#3 zAI$wG<+Gtb)cF|&fp4^UN}NweN>an4@bTI@y4m|S{#q9W9!nOIaYb`^>iJ`^&)Ay= zzNvyMHaE!p%@**B*-)(m5qPM}VhM1uLk8)cIWQtxpsBC}G}UtP%z88QEWd~U9?m6` zY@PUnT4Vmc+cLWE!eQtl6kzYD6f8IQ$7_w*{CeOyp|-scqLS}I(p3w#cj6VU9&rq7 zvZi5q_XJqttj=qG4~IX--MB)|Wd2MlC+bZ;g6sVZspfitu3jP&(eGZR)cHKi!~!I@ zp&F7;zkt_?bD_CVo0iO#oT4p*sDD}qT&fv^VFP-Rkx#cm<_k4Asj~w$KZIe*x)}Pt zLaJ^*bAVYB&vSRxKjiG;4@7TP1FkwQWAjGl;cfXC?j=NvXWp&C*FQFLPw8sbWu=7> zA*=4sSE$TH$7XN(dT{`~q?^nQnw4+$+iYlY8)BP8Iq2k6Xt2lEs(<-;1^ zp>Z!m8Z)6c|GID=JQ`rkWs5E?!i0*M)N52D4O`iVDx{QP%`bNhX;?>hS;gb8XG5vI zrV^=~FZsc*DU#NA?qdF!?zCti#c?Cj$=o&_Dkcgz>#!z%Ub@cJrXQ zm1+}UN1Plk^mv#Jv?+G+=cbx*+5S`9vFkOIHzszgI|E7={!t&VXp4pxqBMl8w(jekLM4UBD3p{I zudr%NsRduEe@nlrqJ?`8T-a7JgxF2^18Am(UIT}qwnAFFogRUm{~UuGJyLPypB7sG zGo6Q?JPDCn)3IZqKF56b#QO#%v@p>JU-ue|4Ko`->rRC*yv2t5Chvi_I))@ycgUvL zTcgDpMhWzL{8YFd*BPHp9>xWU3?0=o!S;ACeGT7-I&KZ{@$g|vdQQ;(!AZJ%E}Tzo z?k}uR@}a&n?$XVjAr$dqC(H;lMZ>s}JYk$37j&y3r5I!O{yUd{y=5^iH)1Jr*tdD&o8XJ|- zz_?~HcINAi4PjQ@(H{ZDL2ExNBfA!rL*6y~-G)Ode0T&7XjF}FW8 z@At!StL6A@>jG|R9tTJKQZRl;DkY~5!In5nDA4PPPhN%)4w;Hwg%9AAXNxcA4rh~^ zJj&i*P9J}`KnI6pv?<=g!%IRa?DGqWJJ%i^ZkBN8qnDvTt~?K4l<#1@7wR0lNt22n zMk$ux*TJp-btJrcM$pxF!hh9q?6grX#w`3mVP%iOsdxh^*j9#aHwAGZ*7{6#_IxB+@@lEc!*C44<+I?9hv z+AAnciU!k)sr>hn0XAt}fYEU-;?saWJb&k9c>MGR?HF}Z(g&QS*)RNH>FP(+MJE>i z`5BYm*LR|NKoWh_4(H9sAHv$TpT!Y(%N0LXzo4lt$LOwl4}4Z>h09FG3*zt$tj*e| z$Z5#{d+%oP=`Oht_-e46LR$k_NSh-1HTK4m`2%5!Xaw*(MP_YdE~b9HNop?+@rMvq zx*AzS&yChl)zVkw7QB)Ng`EP^U}+Lkx)^Q;AEeaB6+*<54Dgc*D7XAaVsvB%PYRH; zw%sv0H@u%PZ0Q+6FRcjMQaj!P{Il?naTXIbWCL^UGZMX=wO86GF$ALd`JP7tJ!ndLZPPC5q_`ehSvhe zVzsM22R^z4noj?5pUmUca-j!*9O8~%kxSs6Y9nQRmttGhZ)Lnx8M5U;CTw;58J&8# z+RZQY90%Jvb3fJD;;3nl`K3z{`8;@t-S_(_Rt{M#G}J9*SKaY2BQ6Kcz1ne4r2}9i zxh$Ub=?9$_NI6u3e-je@rm3 z;WhRCsQ`KYEhGMXzfj!qVhi;@;>?Qs>Tsp?o1*TiAFNoUidovjMO(QYx0LUJtz*A% z;qaUEY?(jhj@n1F#-zfTcwe+%YJ*cQECq<2iI1NQ!Xq_>JmRzyHXo@YMb~RoVq7Lf zYduix%|0!xc&LwSqh*TWSBFC8U5Wqdx~dEK>5X6`JE`Wa{|(eqjIr3@jL`H>4~?oi za%1Tl@n72|xY^Yn>Kr#hUfl$qmQ)R$Ugz?A*BS5#JCjG#cA?+E{d8>%Q>daZ)Ide^SJX$RajQcIR97cH(XqAK0zp$NIYRbRMjJS-erGk1B18aYbkt zp4}~0C1!4cOD#vlMM{0a^tN=_FuhCBhCyhdd;n%?v?qJhj*xp(z;!?EY0rP1X~g2e z?Avh)Zj9;#8^d~H)ySE9|Se{K*bPun1!=jRHlel=7drOul7?ufI@>e2GhFyW_KD!T4g z<8R$f@Wgy;-g-qvK~EL1*3fixMXU7ygX$gjy~B1r^RIp z>E7P>ePl<>{!mYW@2lZcnK4e@bb(@CkAcU9%jFOi(u12{3HWcT9S6i+qjVi>9;E*p zs!fYvW^y!5slN_)_eJ9prJ1Cve27$BW!R7);8kgB);%*#C>VK6e3e@*-r4sBw#@F1 zVG&2cJ?MeZ=+zZ{a%QmA$Mw|Z`cOf0?H~zQ;fwcwouK*wH)woKF6=0jpM}nUvZ`~x zzM?){L(UHa6kd+a)Uo6TB=sqzx&8@w;eaa7ENF)RY~E7q5*2o=ZUyU2Ple#nRpL0Q z!!c&bY>FB;MKrv-6T{UmQc>w4S%r9%CglHyNrQXBe1$YO4pQT?OeKEQ+>^r#-ja#K z8whK!1dHUYN8y3TP1?fhl#}+Emb9CX1CE{|4JYXgr*%^5^#6nF7PsJ(t-u$)rEtmA z9NIUxfx7N9Ma_*{pk(j^G3JIZUca=Q`X7sj{!txql1VSr2|Eo#1J=Wtpo>CMeJ%9- zGJ#^c91}f8HloD<1HKfqo9$mpV1i@E^)dgpD$g!mfGfB7;SFCG>aUZ;EkhT&c^YS6 zKi7QNXz$9K?tFvyk#-#a{t$EsDdzOzUyz8?*sFVlI7O*gEWJOK-d}LQ{0*PQIlX_7 zR!4U}u~`MjM}MU0iJxfXPiJ0mcrd?xe+mXB4k6uNtHI0amz+koA17|^+@x4E_qo_- z^ERR7{C;t`bn2C_-T($`i^Q!(i7*OOG3d(;F}$P`O|ezxh)N6AcxlAW!!}}Z%uEP< z)I|Np&cfJUkpx2qu+H{0mKhX)SgcQ*+V2I=EugBs4!J z_d!#Y`M5r<9e1Cj#^aBuoNBUNQMKYx105AsK-&g?JUqKMipodWeNh+;cFmT2BsR4A z+!|qBuN{0xFyY(zBlvKCe=@e$<-7|Ug}uI;1;+{lOmOXh!ne+BlOfTP?2I|+ri+3f ztJ8sh>0mb{jC~h}LqEB@hxj_g7lWUR;-Pyhv8a6u+%`;tupORs(#Q*vODbUBmkUtz zNEK^*UJJQDmY$y7Quzy`-&*nCh+ zEO8o!%hr~ng}ojf_!x{6M(l_ADDA-F`DlNZ!T@rzaB z$m{zQhns%E&Lxht@FC!%O2#v7>tTlde2(_`BK|tKnIiV4V_1zo9~{&uC*`)?G^}|j z{E~FeYKJv2#M2lDq$E*WJ6&PI^mh~xAH!4Ib>nZ-S*S?V;rYQcdBR?A!R)bxX!Fh* zUk7)E1dJE2rA`A?S|NHzZ$NwHWmKZw8E?E+rnnh#FvtA?HT}08w*5AMF<;uT`y4Yg zophFJQJzDZBWL11X=i6SI|;Lf?xy0qyRh4EYv|py2ghu4fQrvs#a=Vquyv;@NM%+i z{9?}C16Q%0QZan9OW?)hbs^b%Ep6^UMaWzB39_!=6zVb~pd@h`>hSd#@NRT)tfeP7&!m{HXvF=6s~u^S*-JNyhstc2K?6JMb8$htX@^Dt5;< zkapVyMU+&lZXG-d-3}$f_8Yr+?|AJ;&wueeASL0T367|$A?L$=qP=7-b`JBdvNT<0X!voG8{f* zz<2+2;p6LoI;zY4d4Q~%q`4!$zw!=_>(_vKhzLK&yFvGfeX(0pC|rI7ip$I1NuY_> zZqHU^a8dj$v{%%U=lK_Kx^pClcWzL${p*Q)x2~hOhE%jFZ3k!GBuR@58(GxnAPl!~ z!9fG_P~wN-)Z!@K{vnU|7jERibIQ3&ZYV>S!vnD9Tsl2czDogj2U7VkSKej3kvo+w zL(A_~)ZQ-_0^X*ha+;Lms(!$?*IHnNxdx<7ze#nIOQ}rv9CB|jh z^z37tJFT8x^eU%b{m#JS+?PUhdLLFSze+zQnXpOlHYm(dLHY9-3Afn(C)M5bbhCey zCq&M&!nM}9IC+^l4*TxGd;9mG3kO!p+*=z!%<L4c_>-uQ5+KyqSg_t%f_5+4Vt@!r(|3F1Ve5)2HP_)}5*7zhf`#&hy4}K2Py??+RmO*)+PX zOil@>Q&DN%#T6vbXEA9YwUCy17qd|3(Dj%39 zh0*UzxW2d9Jk&lWv~a@_HBbb(9IAHydEr0(~}9GY7X~>5Dnq zlo#$f596(7(d&ddsX-%e<@g@O@L^J#(30a!+iG@+|B`>p;(N>!3NNh{6aO1V--{|~ z))sr{_4ytBcu6ok%pdXs-;2k(t`@VBlF{|LT-Ik2knz;JFef(&m)mOc^;^Nhwlq~d z6yqfpbsfphICa3N{9Nt$)cSdXTo_NGZ88{?+n@WtONYkx##k`M z8$AvU!(Yln`J&x%YP;%6t@-nC;ND}5SKS~iVjbSQ=8S4Pd*U`LU4UD$7`oszJd$4& zrWp4}gKq9}x-9(_{XC9Id2e-2y&euXj|G$GEM0b+W(r;xs}$`l`=Vd#a;&#IAq%)N z6e8RNrp|Xn!$TfuGi)aO)zRk4*N=tM>DxhX{Js-?nmWm*yh;>T4=jVh>$YS2J=1B1 z{IB4<`@3NLXPRQ_NH;wEP)5_P{vb-0U!nc82O}#wqf2Ip%NlJ*ZhZIx=6uiK)baOW zX!AO@2=e0vE#K*R{A4zok;W_X!rA6aFL8@>7hLyK5CuRwlharbQQ>l7-8)!^EXm;;BsjXE!9-jv?#d2UOQ{ z2kX936D`8u(x#sCnDxAIPuXBhT5Coj76Y)OctuQeQ_$e!J3#)BWYDiWF5diu8ddzn zt9R14WWI$|y;uUXm#T^Tt{p>*h=26>N}}MV)t-|aYG7Y}5B}=#9D-w|5BGDahPNe= zQtfG3)R#Z9MB5k8)o~q9Z#3d8=^G$iYNVLvEuh~mQ#LtR zo!3urQn+=$P0vIt$QwFCrh0T71bo?o4{Wy5#OsrA+EXvg@A#U!=eoh{-ifgF+9C3O z@DJ+ObRlcq-gq?K3?JRkle0&9z0k4aMe1d{8)U09AosBbW}Z6)bu$jad$n}_x9k>; zIzJ6{Qqrm0lCzZF(-dHWCXbg2tHY!`h}VTB>@{LIin}I>=UcPE#=oA&q4|_C#CUb*e;KC*uT^H^ zmG>iYMsXVY_p|4XXALl_k1>yVwFdofO1nP$65!S6sm90?es^+Wr!MVamTCi>@!Bj3 z`=hXjT{M4}xx=~H{rE|SCpR6iLV3+)RC_oMGV=~`iJKL@Jvakf_7=h*d5a&2Z@TgG z5xcph-%k=g#nOT?r^)f#Yw_xb=U~|@PZ(jEMweb(6>m3wlWwiMDE74y53TITrPKk< zA11+{)^}pm$LAEMJ&l_47hrJD3-F(|Dvxt*fwG+rn3VV%oGr{T(AbRLItgOlWCyHF zA#o2&`=9d49b)4|P2AC8A?x0eDwB%0ig7;@#q&ll#jRb`U{~T)T)$>AKVQ=-u3BO% zu2dU{Ar7DDAO55@|9ysET1nEp-yi>`?kAVgBwRpieE;$qOjru zX$GCScTg+s*kQ$=r6bxaX^2%bN?*>lU%rF0kqH}Vr-Ry+7oZ)~n}>IeW1&|E4)q_w zS^bVef9Zxham{dS*L4-UTl}C+QhjpGWo7g!+66ycw!#-1YjM>66Tps~5vK%%1d%+sr9K%Y&#k(6N z@4Vklnqd|uysO-|kvg848V~JG z2MBYjB%QQGM|b*a3&ZEP(2?Z=ee`q1g52)3*-DuW9%ypTwi;Q4zd$XfmuS_F=hFS( zdk4o%X@rTED|yJBeUYXv z%xBX<8Te@VNpSu=h*jnV;QmP`6`h{$g+3={Dg5qj=cc!t=<1}Yl(bQc&FQgd^}91Z znLQ6XTye+oh7)YQ(eTa9!se!qJS^%L&B$8=rOo#sPClFG z_x&nPFVEt*IA{18v=je)NyeFJa=aLE77m^tD7;uu52I65I9axX{v>S{J}roWhc!3I z=y(qF7-)>W0_BMy64aDrlo_^>zNw`yiNH&yl=Ox(bJ@eZWikBb1MsD^z4A z%ifib!o|yL`Hlr}zLOP9cIpMc*GbXK6*nnzrGk{w%|Yw+2v(lam0DX((OkmceY`A9 zOm+oezn<>kFe_bLy+44LUrUDk03VQVy&DQQzF7!ikv*wx&KjvSY%V6u&7@qoE2v!1 z<&%rYQi*Rg3~X6X*E;vZz806^w(=YzMbV;Ss)&t{!G?om=G^IQ1^hlg5!K6daq)3YEH8GTFv&Y; z<0C)A`-e{ESuqBp_&o_fhP)QCtY^R_c{nxeHG*Z%0$O^!KcAXaLv+3r)kBo=`Kw5< zyywSO2Uc@pyB2VneU`?0X>#>jRvd`m2}iTM+4PJzlt&zg!iRZu&El?L;hsTtc1pbL zt3J5LY{BE_4v9*men>=ZOBtzom?M5ajyuds;H%dQ+0PMbJo>MIrxM)Ygm)G{_+0^8(+=JaFJT=wNMzf39ca?c8zD%{>7MiZ$`| zqztiT_z_Adxd+!n&BYfZ2VsQCa^74v9ImdrD$3uTkyssdTfy>LXWG8>3?-cD$y2qf z=}$y4?(u#?THj7{hf;gEWRk}_<2%Dhdt@*7AdE_zjy`o*hyTSVlHO&&ZF5mnV^2MGe&m*Abga1M!nL83;WMMn+?wJU9jBX ziH|nTV!caF-1_LUVsv05{kQKnotx5+Rks?@vB+pL1ar=|brkJRdVueJ$pg85FBLp8 z;I1+E6}LxBBE8(1IIn&H4tJb@mCcX-Pk2@|IwDjlLh^-1{J+Yrbg86rJ8a=idowNz zN%l5yHew>#FR%jJLz-|o#i+_K-dDVJW||ndb~vEs>cLxP7f{`Bg<^E4d%~ZEuZ6e6 zKMS?9Dxu2CK$zOT6WBknCi9%1V&bpSLa57fh$+jYV`E&!Tyz$4!foJiuhT-~I6L@q z@PMH7>5Pc;{pHY0dl0m!J{BwL(}g|re89(H0u>b=7M$(Ril$NXpwBI9@<|L5;+h8u z|DLG|dn%q)H<_u!cjsjBnuaC4R1jX&z2wsMp~!Z~`sG9ELAI#-(=8RYWp z`~sIM_2$a04dzasT_j^x)(SQ%YX#M(ljwxE{EckGbVIJV)kC~EBHg| zb6I|UC};W~gk_b>V8qRKxU2nMUi;b~*LEnxx`uCJ?P&*$dcQyz<9H2 zKevnE;|!>;k-?3xukdiknPS`F;hgl9Nd6$}s?1QP0iIX8b6A}tj@sajUz}4R@5X38 zvLu!qUrpvfjWeW-8HzgxW9apg0A4&N8;g&|v1(chhwJylQ|diI$vX~jM5a)u&Y2wa zu1QFK>C7XqJEFSQF82SCElo1_i@mZ-Fhr^`9T_kVSLjBOci=9$tl`gA%8jXDKa+F} zNRy%Q$J=yo$ZLq7-i@~h2;yL4DKZu*vU%A_^3k4wM%Nb!EnSV>zc^UYG4$uQxqXC) zl?lR({_n)o@1JsiLr=b3o`l*P7s7xNYf3$>1LwB-fyt|Nm@vNvIxf$r7p?xRmZ*ku zFZn&N{@4ziv=8CC4a%Iq#|{=1UKfsBEfj||)rwQUou|}!<_Zf}GmPH*K>Yc&1HSY< z41GGwI4NZpUW@t*+r$8v(j^Y(y4LVI%YmpoU@>5zA-34D@6VZoo1 zG$3sW)*o1fg(po>=G&X~pRSTFM}OV!4|n7b%k;?mr?RluB^$rCtiV10`BLYjuOyrB zFwpp1C-(3$!qJh#s6hKLDXJ0H_H;rMCp&)L7$nH8&!w@L^n(noPea(l|H!UsAI)`_ zumLuPqTGt3V8GWROq>5X2yqBHaM~cnUibaR17J_e2f0#3+J&$cl=7jmu z&`hlrwrZ}T1*gV=qwPA1zwn60yAH>Shp&aZZP|FTMTcGIE91qaWFC2Pjp*}gpTsEM z$R4T1d^ua*l^_0?PC*V@SbrmfN@c#_81fCSd}@OoxfcYFY!9|t+8ebOe}x1oDpc-s z2mC&&bN(y?(Z2V1(d58JvU&QN@*f_h*r$?uVD1Lko}0r@?JPK|eKx%x*PR3W)acX5 zbugv23(a>wAimq4DGn+wg!Sg3l1E|9N&FG*jL-h%^1mNj_+E!6LYb7#3|7n*g9EL2 zsPu;gnp?^?HV5L7lI}F``v$%e5-Mz(+JRrk$Z_I7WtdjXZAnegy%(o0X@zvQWLcp#H(L~~t=T}o2 z>CllZS}AX$!m0<55v~pCA9}Ib_#|xjeXF|c^+GCE?~cWR@2IW&QXDDmLvmuCQH{v} zJf}ZRGC`+`g%5q$J;_d@m2ai&4Rh%)2V$>|8oWMPgej+{fZ0Fb-OqKz_(@0Tu*+nc z5_6Cz3SR}4XWzjkTy7(I%T|P+*w(ebUqT?Di*u07_7dv64T1V1wJ`S?` z7hud@3$Ah)hGwG@(OPQ==bRo-YSGVV-?d6=T`m)Uwfq$b1FCuw63=%^DkrGJ9!E_r`YmaNwBhMU=@3}>xu>+wAkHbx$x$& zE3Y2&Lk@1Y{)#nw7-~+Zk&ojodjG-`o*dQW!lxT(oAWlveDoa}jiLmz_M2dGXb^WB za}UynUlRHTX^MC5=+P)&Lsoop;F>%&-tILM9rsT{&&*H&C4>b-#n2d&`E%aqu%av?v~JAfv; zVo|fgMsd_9g-b6qLO`1XxBT1;vZv#De(e^tLL+IcP)R+uhNIoiO_*vT;lp-}0{QDy zab})9#&{Hnk=z-N&D5cxMg4GYdM*x1>&z!k1nVkcCtcN(pLe~uToQDXc){|*&UKh*YdoHtI2y0QppGl#fwK8JV8*R zbx)tN|Lhq!$h;X0vqs|KGId;EnTLJK!uSVl2d%5~_<*VnynJmYSo>Z8c~a~+oVQZJ z!9EA5;+;L6zI+C99@*oT{TlrAT5r}!nZ>n3N@iVYQ z61f{5RMo_*nji4%N=cwrwT;@s^zqT&kzD7Vio-|NLCnZa_%yvAf6abSeJj~V0&{ta z1_qlrVS0s}{pK$aKe{&1oCi+4YhgUSov{!mdll0qvml;&Y5>mY5R990Dn%Xb_UsuK zgMUGQzqc*${+(o5afLSfzMbS&`!|Wtzgvj9W$uDT?QYb0YR%_8F7jb@Rl51;4P8*n zrRbml;dd7^{@ZX$ESj~GCkJI=@ELhA#J4*`u~unpVY3!*Q?uZxdH@EP8RE0?2Vv>6 zblBMsXu4_@q$cO1+xiyD8}^dRx43apoIUHM9i+>S!`Z+$oCnSHW9?mfD11#8Y&YD( z%#vTS<{meo-(G)KpN#k|DhIu9+w!Kz*Wu98KsFCv%%1XyST=vxiEbYX!@W8B?5Agk zkuUnA)$b(kxn&gA=RT|+-upP;dv1qAi!J!frFXKB(tK!>e#srfrLy!3nYen=VXVBD zBbIs?4$esI%FCCm0@n?bG5?hshn!C3J-ux3fW9Q(Gt5PYpGG{ZJf5b`nj~xYt^(zY z((j9-a(vLcT?}71-a_7UH^`jr`tY$Oi`e@6Wyl*JjFD%8(0=+&m^82*8g;Z0gHM@Z z%dlQJc0>})(0N8J2N%Hk2tV!-6-It`-(Y;%UNl_Nlh3RP=QrClF+getWHqRyE3^)%7)lPb^IzX2i^tpe*`8DQ1g0WBjJQpax<5Y_Gr zpkoMz=WLA0@-p4Lt+prvL4RpXxTARq! zEE`TsUDZA@0emS|1ur=sMq%_Aj>>vVgQwi(5b6D%E?+4&HLJt0C)#-I{5H%<%ixv~ zqu5X)zb!TVNL}9@;#%7tQl;D(J*4>QmsDxRSbULst~^2a3VzEb$eXh1=s$J#IdXy* zY@b4XM{XN@=!FY>eck|f`*h}K2Ym3u^|AbVR97Beeh@QG@1qWLmALI^7=Qh@MF<+- zf#Q56Dbw$}G%{l}#`Z`+57m?8_V25dVK#vL+6E|JI}JNrn9qyvoAcN1;kbE1lek{g z<2|1XNbcEgDiy|FqwRh4>Di$=Fz@h^u4KKSJ$H3D0KMe)%&%I9RwM(n=HPAJ*`5Qc0!CkFf_E~(q(2XuvMDSg&W1N>-0h^DekbK1=7`v|sFYXKD;8T)x z#VMYD^zX{HZvtf*i=~B&%LLx}dN+hbAENUqGjK4i6Ysgbqypc)w8L>6uWkFmSx*MA6FCs3KJ!&<;6%BCg5s*!p(=KVUGd*#d)JVWxIWUi{E-Z zg|cRE=&kZgI2rEB;toqX{(x!M)2XOb{}`jfe!`=_T4X=&6KHqdNAYJ@LKhP?EStKH zce4kg?NvVCa}n2Ves1AJrgU zJ@OF`)-6`-{W}M)-Eo8DXQTP>(PC7wzR1B%Em(2sgmAfG3q*}h!iQ=du^@f8STraH zRX4Ok?1wKjc~dw(D@eq3UMFz(tnKU(De#weTWNE+0CugFg5LsT^tma&4K}(inAqGO z{XabdweW0=Dc#MBa%#bEyA+u1bO7#N&E>X^|M7*l`4qUJ4rX625tir6W&f^T-K}Bubqnl4y~kI8HVlF>q%LL(VTymsAyDsynn_=vHW=>JRP$_ z2p%|y2g|EXv2NKR)>XHGl|`GRX`Cx3e_hQs8~#CD+#T9{-W{E~u7sb)rD$WNEsPGk zM@O$~^0f`sU^sg|CRI&Ap*j=09G}NoNmHP^K@LAl>c$#nKfvqo5kViFct=?i9JS4( zoqgOXDYX<9r}conPmTG3>I0I;H05EO`CqWr&LCS)7u<3ugZ{hjgVZ$$Cz_UlwO~Zs z4SKM#haSJ&TPGWSZ~^a|QvmapZ(&zyID1EH4wwzz4)e2(sOpamMrk_m;x0-UjkT8*^UA(YK9FMaftsuG0M~K@RK!ayzaeL)r=<8yF zU%Sib%*!nl@3Wi69S#R^@eP`FSpyd))YJUg-({IQLQz?9jd$ELMU~lo*l3L01P#~c ziYx7pVV&ntHVVwFQVqNT>s&j)4*P@HqM?dfjcH=P%^mslw(h*{L;^TNQr2lz0kA>hG{aZG)Zy}B`43(Nc8u*~U8#;MzmqyiQ zm@@gKFlST^_xLgpzCXW?mujnN&&5oZ&yo5DgG~2wYK{wpsnmeZ-$QsVrnl@s%nWqB z6-U+2G)de$0V;p@W8ON9=jtuO_PHKd@Gp%jWLxpvg-_^f(vEaOU4=t0FVf4Kk~qV_ zmEY}pA(Bb~H_r=&AO2aSA3h!G^r^AT3p4Z>B|4$67%2MZ338U*V;wkblV z8{niJy1YC31oPx@@yN7tnzrkVn2vwom0lVq%X)Kt@fr$FOd+q81{|R|j(z9JFwn;e zuC!DMvo@8pQOo=75y~@Y7m+kOL&@8gPCRJ0Gj06>>o*donCVf2bhFq8o2C8xa zc6J_6)mwKuG_H{nI;qpx-gp+Q{11DRm1`It9^L?~vcjX-V?xBO$cRP~Kh6r&&`VE+tT>#cESBnF#Y^6St+Og}R(P;}mt>iKeM?VtNJj{l6z;1vhG#2X8bVAE(rccbc$QoK|H3pOw|m0X7f zy~pA%=SBEtjt1MB4x;zpyKs@@D{SlWkp3$@jpuLqbBu=u_7RE`&lEV($gJ)iq>3LobuiSpcLebjkdf;P8?faA^fIPg<19&NKo z*w)dK6Q2dLW#}gHaeqmg7_k%={k%-S4w-O<^+8(IxESuO9SbQLOB7qSRlvAjrr`c- zCVQ=nVC(4zadZ!4lh{vCqf&|+22a793oMQ0?0B(-G25A~BGYd(F!kV7ISe^kPYtuY z*zcPm?VK?bXSOY-O{aYL%gW|ZwzvzG*$-DlKGMhE$**bK>HvOuqC4%EX>vO-hgq|) zP{q$VLie;H(Y$IMMmDL#zpok+r2QG^9`C?Y&V(vjb8mvz%szCZyAjMg%F?90Bb@5) zB*zZTmI7Tg7pf+PP;8;Dc+Q~*uBnUw+urr;bgVyr-WiK;3hvVNXB`AhZ5`TZ|CR=* ze-jfoWr5ptV;0Z(L79Xd`h4IKE?ik7ysw@@rj~D@&|rZuIO~t78*RbmSGB06uemI0 zLlWjBsG>r8JEpfitX?=>o((EHLvTgXQ!;Vcg+J=;!2V4UcE7%uBNmnLsLEUnNm;|^ z`n*If>DAh9t}$94%fMOB)MWLq{dwg)2e6tQ1DU_FW&JF4U|-=8Hk!2xRlN>j=ZQby zclkxkp5IAaFy9~V+IgaJZ)KdeJptm4cJSfR$N8PycQt5b-;>is#7SuseP-{{VuQyu(pOJ7YcUfi2!`(J=2pHZ^4 zCvR5gUj7P~--bZ+rgWOtHi2%oBw^RvojBHtF!#ea?C(E?6Hof`og)C}%Uf8!a_%j< zGQ^mg9&2%*Ats=|Mn=V_3I=!IH-}|$we(j!ODbO*1hep6Ul6^S|{9eFjlz6qrrL27B&}_@TZ>= zE9K}^uJ-BzFPn?;;q77UQdGb&B!dU%c=6Mb{a`F7@;t>H{2*Vtm96&Y!tTjk>D-2~ z;CSK(eLI6J=oItTLnGK4?~$KND~-*L$0z22ykzk(j?ma7W<8d$6aT5=Y}?Ja!F~dM zUws001?l6{ky?0nkrb*7Z-dEs+9We>ujpJDNDF?vRs^V?qan3_VeF5dczn=&-rds_ z#qepeU40M7z6+xighwC#pNk*0+10?&vq!BuY@sq2OUu6Q~h_GJ}-&hmBq z>2y1I`!@*}8W}-#*GM_9pKpW{^$fXBasn^;@IjWns|LfPBKeU=Hm;uSft4@TKt*Xi zzTd@QJ^7_zxNtD+q)1`IGc8sFsY+Vn#U!$}nKV8}rTDIr2wZ44Dr&K0Z8*%t& z3zA>*Rl|}Ux9LsxJkot0La*ZtG3a~{HZ1DJS9jhOOKmJT?5I8T@ma;8y_I?PylbqQ zbd923Nw=fYAar~6R-82ZKT6&g%pB3t{Va=tTmI>wYK3)h{3C_bc|a(aYIL+?K?9x_UI3P%<@BFa=!+R)K(+1K^S&f`3JxkNftmcCTUQ4H?fA~>C6m#8 zPaCi6lSC7G7@(8&I(ihoyLztX5=`(cA{G1NVC|ZP2kRAL<6IMXuxSmeoHE3~Klj9U zVi9zk(VwTsw877l);udpmq&k?ftn3xxakk@ppvt^$x(w=_wwVI%sISRJ~jmB%{k3u z?&*l_e_EmAf`NGC;&@ik?83u4?4?5sPf%!oE%(}Pb%M%!HPq>HggPWo5?6P3#dymB z{2;wEZZ_Qj1HF#Z@Ocw>fxHY(4G3jZ#|DnMU@YwUvKAsb_|u-U8hASN2F=`=qS*Xu z9GZG6(}D43EMGj~EKJ@Wj5qE-b33)>p>XhOK0e6TM%6*vBuDLLJ}I$kZoj$=U5^yA zx8+w#ln6gxS7ox{cQ@{sS_~x@IIr)Ex~7-# zPkU{wy_?G$n{Jb8o;q)}$-|*LR2cp@VD+m^lj1CIfpgtOIOuSR?Pq9lZmK$(HMY^I z8INIhz$)0E_d_`5_=&8l7x1?9WZD_@i_$i z=!;()8hHBTw_s#~5jXkh`PO|tld@dfjtAcOTuJRXeAF7p4 z>;bX;22$STp;$ksmR?oQ0fXsVz^4CmxTvxZ%iL!3yE=WY|10Ofjo7;X9R03$;x@f5`2g%Rbj zW>63=Gt;A|D}98rK`c5-DS?Ipyy91mg}>0uTKiDdcQ2Cud)Y2GVss z4{c{ob=9L3Xz&I~7B<7grO(9L^cC!O>9G*8ECAi(*5S2qU+CPjTl|u=0G|tm_~MH~ z?CqXHvj?rENfz^YO>sT%u6$TE@!aaJcp@u{Rcw=F`BP7cnf_8DB6GMb-mDu>oZcUb zYHq+b13lid`G1&t^SGMc@BKd&g(yUtG#DyMX;kgCPGyJ`GDPONA~Ht8(SS6mXf!lX zQOQ(N?X}GF6d6k9kdP^v%Xh!tzuzC9fA7b6oX5HM-sj%;z1DR-FXRs~(mCLO6>7{_ zK)A^ryHD38MJG+%;69rFlN@zeFWaNgPtvpX`~M|JS(2Vjb;c9nTYNa2`J>IA57eB! z?v97L@a;mDEKei_59n-R1qS(&;%@0I@t5?4(OG>3aYAp2cU%J7Cas56^Nv9{77N=) zZz0P+zlA_0196MHCzoiS7Q9qEJ*SEK$fTLMX%U-4lrmqWV)}yUdSpJ4~_IOJ6O8QO?5#wOd z0TFKI2a>|FEB{mKU7BMOd`6nsnN;r-C4T`{Wt3*-*;V8)pC?qtGSq! z{BPv8s8pF6``-xMh{gXtb8uQiQ>xm2X{4Z0`+uKVJNj}|YV3a_KbK_uPm`7WW>HYC18`0Z)xjr@azmeP9{=bog2VFf=)p8}>)=?F?Y^2vF*)NlLnX}1!Szchc zOz0gddpB>X%?_D$}xABgN~jBLpA*|H1T^JV3xVX_6g#>n(5<7F|bp|YkA z5wd7kzp=8HxxO-o)ycA)*8wt{uZqJ=tD=Z5_-By4fnOF z%j8j1Kt*xp*n7fRx@7VLRL*Xu=`+kJB^>C)h?`7z(y)%aWy?hFRQ6QJ%N@_F&fcb$ zkyfDANe3T2x=2$ZhjZqa3i$D?0j#85_K7dXY&YA6Hk@v@8j~= zAwrkIk`G?&K?7GEg;DC7IIT|V8=joN9YcCS&&8^EG-ZVZa^4MwGYdFd@i|4LNC6(q$=z!p(3w6zOHv*M}-Jnv1A0~Jj}y$(!?CD_2D5w z9q@fp1gS5n0WJB~u{`4ICValV**WBx4mw=x!Y41aR-WjaCwdNgE~r<&gMB5BNxAnF z@$HPTiodUNAhu`_gW3@Ee-_D2-Awq$mFt3M`_VYP*U?i}D`(NE)B0R*_85*!6Iq%1 zmJ;KX_{iA=D2}Oz8>Qx>~Y;_SBzK^O*a=lrefJ>I#wbjjJ8YTb$1?v zx}7Fkx~@fAj~Gz=AQe1+j({)ncj`3MQ9-J#y`g|NBYDW>t!(pBMG~)#;jNB?Y4zX@ z^rLhlTI?HyBbVL5r-faFmiLQzyJ;(o(+=T;a+S*1g))9?`&C#H*q=xF7zp?LN8_9) zb$o04T*}iN%drm(D5&-`dBJ0Gzq>KU#|QC<3`LF?%-)jCYM~@wDp%IQM!P9{Nx@HJJYko#CzjmVTazmK%MM7JQ_oMY& zZ_?zP@3181BMm%pm`+N5swMtDyk+<|EK~BMB{jv+y+<~^bM1v`5=+SYcQU?fjg#Zd zJs$WqJC0@J75Qo7J5jIxw2--MGQKP}MMu+*u=~CSP|*u6r1sdg*`5?WtJCxJ3A}x! zA_ot6EM$!Ci09%q(+I4l2hXMCuz8p1d;LwQY>9QASe*^BE<^C8Wf}I8Pvv2U`${~` zEj)06ve<3KIPUU6ehYL4>hsq-uPJv;2K(;$2H}Zw;fQH5``*yQo)_a;R^H&O;a5Tv zI+W1-SJ^_IjVdpFZ_gpF%Dn318`71V(VHWw=zO!6&|}6PzB^e56lFJ^*G;RU^}oCX z**F2CZ)x)-k9s;|+MZ`F(nR%*b1NN24wTTAitTW2qZ~FK4`XHTQE27_yuu_yRGVYS zjS?BQV`x8dkilR2b80@G{H!k3kyo+);3gqv<8Ih(ehF(E$MV2!so2$17sFPUI5%z? z%2A8nvi~v%&UmjSUO!PT#AWuu0j!AX$^APRVW#^;+^1SbMQjx8Wcgwdmaiz)Es zbxQTUOXqLx;~%qQIY%N3I=WQi=8`lT8)wBMhlRn^^Ih2rpH(z=(G#aR)kEyGKcr)p zA`I5NB!@L`PSCtj>v?AJYjH(uG``-|jyBZ>Vx?s6@B7>es|;eLtQ9Zl|L-!YsJD=T zdIh*oG$rE)GQ9rg6x;oMN!2H!;DD5!R3Ul4PD`34?MIhsV-F8*ZS>(+$6r&& z?*{zPD3>m`SLPY+JLz@rF}!1jJQ|mOdP~a!hx6n8GVa*Tmgi=K!-1LS#EY+xeUlPxd0cX{1ft7~81^KOE zw`n0*;o7HCuRS@F`~J2R!!1wZzPdayuXsK0{u=-RYv-_UoQfE34b=4Sx}?IohD+CG zV)w3^obw}rj8AW^K*5ohww&i*-8RDWDIch$JP2ltn!p+JI->j5^TORqMVx-cj6V;T zPF~ZW3n~>lv^PjXS-33?!QdIeSaaF}yFYnCPxYK(_%e0=wz5(p!?mMfdzH|3$2zfp z$Wt1p?S%Vcjaa5S8MgnMfewKdd}Q5b6yMlk-~2$FW84V_J@Q0Z@Bubk^9p8Py(SJ= z(E^_&ZBe%ZFLcRhgJ}=ev5^!IUb{CB9j{6Ezs!3u>UCO5dzvnj>4q=xtg0J-m@Q?I zAC7Teyy++DNz9ZB`KoxiI*p#6Un*AEJ|?j?Q80M?gnE5y5PnYmTG1kzeM)oH`-E=!=iiQNlDiCWuPT@H({~g$Athb6P*uRy(EPt4wx!mKc7M;Cc~PS zOW534VrRC02Jux^@aEtdF=lNaY#U>X@@d=f#K1s$_p1O>K8ND~lWOqJWg6LYDg8Ps zb?Pm(@ZmKpasRX-cxH|R=R6uizK_dcPs=uRE1!>BW1p}>v^tLqx+=P9hO=i!Q`{1( zfI^ZN%ZrM8az`Ue9`yP#3X?BN5ycr~a66BtNy@5-^?7jPm{hP_I}soJ?t`)kM(A4~ z&*`(Ysmq`X9KP9}Yc3rYZyENJ&YRf~m~tAD`*fu>SL4~~_

JtLa44e!`0;d^n($ zevatMKl)by{&vANY1U}nX_XuX{kGzs1%q%-o*{fWn2#zgZEUzLm9=KZz>gW%VV^_; zPiUEnaS1M5|KkLuXHKAZ_1*Z@06VNRwh(9cGQuH|^_6`-wUXD`8GOp20rHmIWoOAN z-LI@OzMuL-m=mDEPn5c0|NJbBovKdj%eRn4)+tC^CQrm(kBvq9^ID|t<<2{Xl|$CP zzPxi^HFfjpDl^RLiVO6sokEOL==H6ec;VDLy5KcM_~mp7LiDDfuDcWVP5(}wW)D!c zF-CkeBcCUqAH{zznSxT@b@qRri&0;UY0i2N^zJbMUvEytQ(xB7nCVp%w5uH&%O`yo zcGW!qKeeAu!RwDZyYzKug_+6h{i+AM=(w@jFK67d!;W|R+@~oK-r~1QKgrea7#3XF zPo`v9+5hZG$4-xoxpcMa>99>-IoSYc$TpE~JX#D_2bA;ao^v3{z=M0OFvV{b&J;3X zw=gB6q|$X)As1VDqufYi2ToU5ioXv$5e`*Kz@_`vuqH5vrhJPL8(&OE&v#8QW=W*@ zK30OUZB5}_>T952+*$73U66u)z1YvXGyc;khTz6zFi6-)cNR3m8%rOU_t}6a43B`d zW2fVuhf7c^XsY!6+tFT0de3}W!xzWZjWq< z-Wbw)g$d+osRCE`hSI3E5||z_2$CYLsowQ5-!W5X8+Rv8lO#a0ZSrLxMcJX(tR%E8 z?~2EpV#WM=6EUai2l*{tg75Eaquwq3#KAf)oE(shW6%dijh;fjqExJC9>Omx5}@nD zeEPG*m_J=>N25RZVw;MZCCaxeIosPd-ax#XCs zLs#T$Pdg9kY>Y<15~85#F#KKi63kakXN;UoZZ^}o_q<|GlQ5V1qne>$NF+xDJr`(7 z6cnZ>(O;=G+_?N7(9@^F*ajWeJMx>J&yNHth?ZO>)nV{lZ=#9Cp#W?c&MH;;LHFRFMnyRAJ>FU|8=rvJ^ z!q%O~L!Il%)nh4|tZFaoi3Ika$K!{>#dxImQD~9{!T!H7yuc!nj8@pAnr;?PGVIU4 zF6|Wizs+FXxOsd$pdO6x+!LKsuZVJ|#-VK8n|#4mVFH^+D3PL7)dyy%GrZ}+mBx|fhYpa<`Cv!n>A5t3Zk2Xa64;3AU? zU>W-w^s-mrri*KY1O0DM%@{|CCNWrIa~u^OFI^6;Cv!;lda0bng07hT(wF0`G6btG zy|7uy4t1@qX4vq& zZSws%`&$6+T5t!t_HgHK1%rh99Rs*qr(pi>w-ai&6jf^Ysd7>K*W~>7x3Ih_o~leI z!P8xrG3JnpI89XMJiFz*S3;RSyl01n9qeht*Gy4g-;mp)eR+>&9vnGa#>QGX!r`^M z(0!H;uDBeI<7-z!FnQoKdF&PXXKIV7emW4Cq>Yaqb71?tHDIHW0Z|82q$tXMr|&lz z@~@FnyKenr{H?D5IhB({kHMDC9^a4B1JMRQ>|0KY`l|4_Az$J1uMT+j(oRa-;0Bfb zrUML+s8acA=zVZ2)=yn1>5w#eCYRQ^ZE;DmrTY7E+OduG!ry0=F_cvKCn8e z7iTG(IzRP(Ot;qU!KgN_p!t2FbFWd-ANv~YJFbZV8eMRU^Im7wNfLdwcbL>a=r$Zj zw)f$yYowj6ruyk)FWNbe`R4;&cg~g&MRvH(buKE@$SLrg0Zmw$jv)iI(P!6oa1bgu z`o06`l&RsZ^L4-@S5%fJIbyl}SF&HB%%%HxLwCQgl`Au(td>uYDfDQ+%0$mh*mH3j z&nfFop|+|#>Z{xmON|rR>tUJ{(zHlaq@%D^!w#SL595RXzErL(Dx!=IA}D)ga>s9) z^i7H@#Xpr0waEZ$=QP6Kh`v1kyyQCHosOzsqG)?boulr357PQP9gbufk!4$Z+hdS;n86cJ|3SX)dC{4d;L$ozc+s^xj9qz&cXd<5%)CKjUfEL0kgmv~umChs-6wu; zctF3mJ`;9r9*=Ien*4b4E?TnQq+u zh8I<@K%vWMD+=`PNDn$4g!Z22IJfgn6m=zAXX+8?d#DrF&DX`{k+ILAZ`WQ)Tt{Jnl6xyY-mD0%z$%2`8?iU~B3)|$_Tmfi=+ zNa;SUDPKkr5;68{&~~~wtR37w8iTjTs&KV_Po7|=gR9<2vkEc8bst`UZniCcndW3)l>Rv3O#zYfK7JMk~|bV*L?4IAyvF~IBsJ&I4bh!O;o5ctqH7 zVejK4-X9i%y@w7X*P(v!U|<=Sj`nnXcnlczk}sF)z4t(2k_Rtn9ErQ5jMz3R8=toe;nkl9k*ek^ zQr`6s4Z@G&p=+(;nl?|2+~|QnQ%AtumOQa{@6yxW8 zuJkb&3BFB3IMFUrm?pIlH|u9Y;=JMFSn0=4k*sBnPill4^67@C8F!adx}724wyRRc z$scj|O+6Uge?MqvdBHdnbG}P97`t~J-z$BH@0XwBxZ|Dh%VT4Fvt%Q#{^NJpq2~Pa+D=?wVIb$I*ne=Ur3e0OAGF_owPs1vW9F`$EX?J(@qQA%tJ zjnAng=}4EM_18C za>&$+XZm#Ej#)vt_enYVMd(rHu{65fK9L$P9)`Uo51n6)S7UYa4%mL!LGkU>Pf+W+ zk=+H{7Id)QQW^g~P);YS??Q249yyz78XonnCRX zGjWB13vB%6hdqKVF!W1%T&(d%)HiqJ$0ILQ4Dpc6PX$pBs`M7j9;b=wey+IS&O*U+ zKoA+t+JIBk2Xbko8TyB5qIrL7mTiyXeVe`#pVmQ+HHQP;db4`Za5~AxyvcGUce`iK zOS|@Gx!a9E;oZ{)F0(M^nLq8>*>|QeY|R$htgOv#TR(_{T7pq|vJ%b^0(gbY0#!!X z(W`Vz+^w3)M|9f6hQAm2p6d?!7&%S}P_Qnv9 zDW{#sJJQ+Ni%7}sj_}c65jwXxqIV97ulBVG`fe&bxMdR_)c6d;FI^H-JDs6XAt~a@ z9S(eoBZP|yQkg!p0(?|<(5jZsSYebP&UbrC`n4%^>6JVYo=e4n_rrP$_R-0x_H7H+ zSM}jgH+Pop8I0-UKGN>D3MlRxz`lB6V&hR|l<$(1^2?+6_S_i|ST!3*6>sHvx7}IU zHWX_6?1o_X0v@~eAa_YQK@-*bvB4w-NVs_)+U9i-e@r4eA3l_xjvhvSQW`{WxkLhd zmEQ>c9o9p6;2V5~?}nRQXHkm33!J`aCa72RfvE#yv1NWNx~y0tJUAOd3j(@v zk3~{A$SieuUMk&x^CC5DNf-vt&S${Z8v{AZz!Gmf3W1>EB9&+kz?9ie;{K+I;>DS& zsM19jUCUGHM)_bIHQ+vNJXuR^IUR(hZzOG+JO`{E%|aDvZ|P~<6CVVsQ7cQY-dkG* z?d76a*uD?SZ*^VdJ5jHhX z#(o-_ba!{F^Y}6SMeosPp)7qj$xj}Huic92&9r{FKtd&VnY{*TmdaD*T*oVv8#Zmw2rz9^JbPys#LY(!d-xK~s+$5K7L%{yLP*HXy2NbWjp#Qgr z&e|3WAZwT;yb>cRBdGr{Le?M0LOCkwU>l=jkK1V{<2aVV-OlYmY1ANViBn zKfsH=C`R*@_)kKp!6c!^d=HKD)di=G=V-QMbRRLbSa6nq7Y+s-5MF7eRmL8a`U!uA zvqpDAo_2c{Zj}3{VVGuTTwqxOJ~bNrxRbW{Gp#2i^%w+$CYa*SC2RQn!*X%rmE&}^ zVJP3vNQ8o=2CTC@S9rZ~HHDZBW}jmnFwyf2OwZ~?ymkO-w2$Y;cb}nCKP^7Kq$59F za20o5(8dWKD8)oMuQtTDR^4s4upy<;Mri?-Gb&x7Y-j9I<-K(%Pc^(~| z(g0B&?Kt7+Giq952PWD|+*)uSBlnn5__nT?JI)C;6JsguV>i^zjAMoC8I+ed3=bU{ zAnhL{5@qRA33WUO2fve5)*szU$yzd8zspa&lT$;PWA~ud=v#7ldu0UY-8#jmyO-0; z;H|>aRw) zxL>3p*Wa(+!J3%7m4)fE}h+eo2EGEaZ$&fbgC`@j=bIgI}UG0=~05)VYk5k zhWx!yQn-eb>|FWYKpB6(QGlwo&9r`UIJQ?*#Hyo~Frh99tD`jeSJQgzD`Dwlt8WP8 z-;>Msj^#+BCeoi$LJM|R(ZYuI*irEhwapw)(^U47%;Y((Q8W~zGmY{2mfpDfZ5C}c zzX;oGR+GZEE3~+)B7K)fN>Xj_N+-Q}Ddc_jyRc(TIC$t!$J+Of__JXfxqNUJ!i}tH z=+LDo$n3dF!;tn(>C3yrJ7BSc#1fQcqt*6on*1-1vv#G>kL^o%b;ubAKIcy^_c!zO z-)3BhbA+^(3Y?~IPR4VlRC-DQF9*EF67an_(J?%x0X~D?+bQkwxhoVi^Jy-QCnUW6m@_vDO*FwT$r1>?&b=xej`} zIMMFZLoh5~t|{(#lueOe53p~)U}5)@RCpw9xm1-UX2{x7`p}Xl4)6B_M&>MJSmQwr zQmmm;)K$1Ua2LKSm1Bf!K8(My3CDjdL;rzy$tmnO&9%E;nY`Bp>l=LVShETCa*V^; z7Iz-AX*m|3`-N>`W#IV19fzFUFQwB3j^g;E1$?=A5@)9;flGR^4t!4hWVrjIX zUQpyLPIKX?)ASA3SzBABg^{jc~%JDxc z|I?GrD;=u-f0cFvZ%9hJYmfWz=qq<<^5IajH=RdX=~w9WSBa&TmrrMO@~Bf$4*kkc zq{N&a>{K|0y8N!C(6C;-@Ztr^AH}qHp#o3-a+>nHr%=TAW@9>TZ^92f++*&^r<5>`tljtaeoI5d~ugnei$~E6)mLgQq(0< m}K2oL1dmq*R z*-h=fy(XhQ7pZ2%DvH0W$>vp3*v7kkw4rc59l8Eb{CMLt?U!vJFQ!DkxO-T>;V~6$sMrNI|8k6dIOVY#58fCyirFy)mFqnVkzbEyS zLg9N~5AmB*N7k~LfxCa((~nFS+*`jxw2+A~&a#}0YK{siJ&WkQq{?Z4mx4*eeWCX5 zCp!9~AFq-Wnm5Nepy!}4?zSUY_)s#FL+jfKF&>RD>c$D-mV6;TRJ$q4j-~PO!O{b| zd96DK^^_EUHf3~Z!dvS2XqvF*f-#Pd-;VJmL%D6O3`Tl)li0vPke#i8H#IMb9gcXA zy;O4;Ipie$^2ovPv~$od>oi^e^wO*UADD~tzV`#gRS{?&89}AXw&I-{C46UF#ToJF66Pn=y9&_#Z~--Hd=oZH zhn?IPKF+h+Uxd4uCnVgq5*Iw1&Pu9Nxr_Y68+ujMhfa%&P-|O1XUBbZV%+o!usyhe z&*aU->AqXgEjfn6mOi0XzN$R-U^cefF^4g#nRJHt#8Ypx(CB!Apq?2?*B`9Gk7}LZ z#aCZkb>bLrt=^0G7tDf1qtn^{#(eZkeE{=6HsFA1!zkdb0jcPoB6+;pMfUmNgqytE z;jaCKf_1bPIjoxnwas~zH~Qz2!h#O0b25wK?+-zn#EU}N#QALAvk(7LcnVjFZ;P`1 z3#o9z6!FD|OmM&3l?-l7k`kzPI_G$)qIuo|^xvbxW5xwSx22ajpsUHtaOqbeKw=e|@jGo@a|5-i@?=^&jfrD^B9o z48zH{jsq@_!qA(y(PdKX?)q&D5s%!sdAupz>pGiy)#ZZi&Z}UfY{06&pi^uyU&|UYTbJ z-EwSre6u2)k9bN-e)_1OHh^=5-rzT{QVNHOQ zy~SCc@?snI+Uy3aA~g9x_d#f*tVXl9JaXQBKGAvVkNk?)-L1qEd&2SLmk7LdX|s@d zVJ%h9GQoR()3N;Y3Er)Et$fSjjX1El0Sk}&)5-yl*v05HZ2Th0hPw5BuQN zRO$HU*n@_@`3%X@{&U@(eXy_Q2B|z>0Utlf#CvAn!Chm6xZ+hE%~p=6oN(Yvc^ALQ z_^I$sg}dVc_K@OK+++06=g-i}-UeO7>yD4a$ib@Ul^~T87t7bu1*cr6hL{(uJGL_x z&hf%?!}p4TRr~3~?b}j=;~r=((#2L}|f&JmBawUOa3M@Y=k;ki=ULBjP)cu}}dY|_~;xy4=S&aGwK60L(8?uX#d%%@l; zmw?Z?Q@_y5WsM-2Szx$gA*hyWfnDcOQmpwE=y|G)R{7=Qqhe=#+-@}(omS%8XOGgi zF3uRUr;s|dX^IzJ+f&1aLP*v>4})L%lIIF_C@(SRy-lO=LupTZc6p-HX?0`xx7&@T zCC`9?l1Rq>k_le?-5uq(rIpVG1mQ`-Gq@9V2=(VhaYe~cJlwlLsC_yO|1O+}pJr}= zz|LIx@~tlBX?coAlnaClgU;ao=I<1la|V9|UlGJ7-Pxk9GYyHDjS0^x!KSwcq7)WC zse?Ix^N18?`75x-aA!V{rolsZmJ2gFT!KO+TRA;!oXLwkTPqJen@#?S=KSTa$nJ~k z$?oDF)X01$7>JW8ecfJ|eD4F&VgliT`k{4rYI0qxYyVe}Pa z+}6!V!U;yvtFV)F{--bh)LTTYQm(?yJyX#{-4FZD3VaMYI_A@2B`~OO8c@>R63b_NP>pfBXLFjIR2evEhS-erj{9YTv}|1!$*zB zGu8!gMjn5l!j2=wW#yNl%_IoNbgUId=En(Jqol-I?;Oz0ngu_PuL8%nJD@|SLK;|g zT6_{PieE`kP1!gr%vQT0j_TVg#H?8ggP*RX{rJ^m%3g0K^O1(xOKDl)wwvj3h&D#OH2L})jvygMKwEI3B(U%VDs$zC$UBn>@ z&Z~<$lZ~=t#kitLJiNml=Z+f|;knKIxo21p?mO}pm7LqmCbu+j!Yv=R9{&>j7CfX- zgRYowm`wNTPvDzd8-R4&Ic{2@)8qgh?5$D^rk&1;>i1Xkn<4Qx6`pxV3Ul=ruw8j4h7eF&9(bC{SX>H8DA*KRZfFYx&|}d_2lh zFi%UQnnAO;-q#;}ew35fiRYaQ&>wcxjIvt@t9IBMGyA^Q67<*a{W@%O+o@={s=`#4eT;B1Mx zp=-#lb0+^=+YI;gj&KhbZ%mLq7me0U;4?xteax)nexXB!aEoGj<%WPxFndB4XHDtA zw!^ohaj+B+Idv`t)ui&_$`_D7IST${48-RO_K;;b5`Ui^%vSS%iTzrRIR~Z8=i9Sa zko5*HHXOJgG*@?#G}h*HPQHgu+Gz3YDYc?`jR8&$`XdIetP#$xGQ|V>gW>YVVzOft4HFyk* zJ4CY9!^M33i880Tyr7MC18|u}ELh*Fpb-IEVBnPJQ1|{OM(lTWzSVsce|#~Mms{+> zA!C-J+p#NR`WZv4mIxp1>O9b{X$*RH?t|~VYDCosiYz}ee?jG>^79xxb_PB&l`3YZ z<9WK0GJfwp32Otpu*v1SG^t~xs31KPhV@z$X{yQwBR9gdU2h<3e5^QWKo~V!>*Ac& z7%^>OvtZz%%a?~~V|tSV^a~4iXReObs^KwD=*g|@w?LxX0Un(oT zhJn@1d~wx2XPiHG817$osVs1o2v7H!3p$h2Y4Y3c_~1l$3`+fob?>@JQr=LhO%To} zN2{?)*Q@a6U3b21BzL9MRk?7rPadk2e21Y}2yp=>^m)WA9{aO~wC51D^wLBdqXAjfZ=Z4a9zXcd&Tf!?{|oiI$rg2rlxk)0~9T@6HTc|6h161!pk1prT2fU4jyms$ae;9 z!^ziL=>5>C%uixL^-LAH&aM_FZFoi+*Y*pW0>kM+^>`RIavCVyn@!qNZ^4gkDbzbX z6pS{Eqks!jV8(~jaAl7g#>ftlMO7}^z6!#CR4L7D$Zyhm zigT|rm=+EeR}Fs$*7ynzG_2-%lPX|m@^d;E_=c*M93gGN5Zlapa_QY2yv`&5zo;v- z&2vu}cWb2Rdf{2{}9V-4*r0ULNB=IqmFg)hp?CJ0A5pm zQ4A<(294KYRNMIo>??i<^2;(@F=~%wpzp?NmnT3!8woSM^Z?tuX%E}9SHtm@zeJaw z#u#+!7x{(zz+3HV^0|B-d%dd?&VCcDccP-Uo@hFKKcvmm!{DkpB!C5%)@4!K!j0%tC58U% zzfCVq-b3hvU1T;c86^asVAoflv=6AUQP}~S5vwd5wm0M`R}b{jydt)5P6m0yd1E@K zl0f=W`cPPWBVgbM@#bv>EERo0Y1t^cch3zhobS@HE0Roa^LMyl4}7lU5DU zpclFs;+q~xa_Hl!!aDImly3fnV#`P1<7+GUQ}5rxkFH~Qqou^a+u4MdP9C9o23x60 zVFXz}J`TdwM5;bJ6B_p2fqXNnbkM#k4qM_>xy#@Jyp)P&Rqx_NKlfFT`?aYu@4Gwr zpPWEVa#bf+)xP4$jT_jjWDX2@q5(@v<>f$MrFPCceN0ktq3jF$DI@bKjM*NAFG{8O z%iB5pEovi9YfM0^mILCCWIe9R(w6Dkm2=^P1Uy>h%_>#HDBra^=ITjd!S-K4GzsT< z;fk{0iLo3zA_>Q@(&Q)Bf!ObcDqpNYX!m43Wl7BR_$^oPhrId*8TLvgg>yZSW=mbt zSG%CK@C80!-HXd?8pW>{Izcz*a!l#x>}*%;Aau}BCu8L=WYac*4rvs@?A!osUb#!y zGCG_;eZS;v{iP#cpW5t{YkY|s%kRUVruAa?sv10_Ujc1~+l0SMc2lKUEoh#b4!KQ< zB!8#774t3s!aK)wzFyUvUl$j_g}7hhuFl%5b?F!veTm}HBSNs}jg8p&q!adfIsoe) zPNbwBDPm@l0lsyq61!WBr)RD%m~cN6^lp|2Kc470UGM$2@`OsHcr~XBtXT3>IN)6m ziGEiE%^6+UIA4!Xl!SuSmJ4!WpHr@Q*ICL}TsVnInD?C4%^QM! zQZ67Yww6LfuhFgJU3j(4Oo$#(4M`43_`rBI#=gySt~M)@A{Bq$wQO&+Bsu&9SJ zUeX>xGvr!)Yf}lFw$S2h`rp9qO*73gUm~a~7onGHG@Tu*f^$AZ@rHieg_H|YI9PBj z&R?(t6Squ5@86Ep<--o#(0ER0*S|@eYt%u!l7E1o*FK}}*?XvRL^HgOzDp7MHz6S{ z4W#bcY~FwW0C*cNg0>&~p=5FqoR?hs5gXUyL=#(<`J8tCy{He@r7z%iLk+OKvNicC zXv3P`ONB0N-C=&Cxp;1~4ONT|#*y={(ah%7N~3$yeovv0qRjoEqt0t#t%DAx4qn5% zEXU*C<45?QYAVD?9=5BVi&+Zl)W@Ln1Mxv|M;wzJ$&FeyP~3YBda3?^l&9wGSTqr1 z>)u0T?g_rV!3_U7&BZI3rtB3`M=jI!@oT6Fw~a60B#G8@e~KG6KOBp5?G?y>$zF6n z>;OF~t+8IZl8dE6=RGe)9kqAIh-zD%;pE`>MxFTmzSIION%2V?tA}~}Vxmof`(UmfOTTs)NLDa7H zDAc4^gRo-?K9`4X<_B7~d|Br_cg7&i8| zqm1P}*lqE8vK_RVYF=%FZ|}>Uw|uP!h1pFcw;RfF`l0+N^$OkV5hOk`PGD_20*}tS zU_)Jm_)Rw&CNJ$z)_J;IdSD2A$an(sX%}u#?7Z=$pBn|6Pe|PQ`TNA2e1`37{z1J| zZZ{v)A|`JcEODu>K>do*IC_2pm#3Y8;r+^abI>$C-+l!({*=L>Yz@{rtt3{aCxGhv zz2dLxE^KE$fZx5&2Jxh_V7>~WU`LBkv?H1NDh(8W#VP;4rmj1X%I}SnRU&1UnMfj| zvc2cIB#|^UB!$XoFHwCXS5#6mTGz-3S(Vk~zUQb=MiDIvX%`h4P3m{<_0R9W_j8{2 zyzhC(x%WNK_pK4=n+AR1Ry7|oPO>^XHk@(R#JH%m&Ge-SgV zwdV=)x#S=$=!nC8H;%!f=2#eV=!23PE%>j1dm0>X7{Ia(Vk9>&4Gr}CrA~o7nEXZ- zHLSRcUN*RZR>wJJN@*Y3nbt`*Q>~KF3UR1nBmizHEX+H@4c?8S-K8CLMoyXkK z>Uit5GGd|_NbI_#(C8k2Jge0kNxsU&6Xv~V9Qs?Bq!1mP_W3wCwnaDGf8xIevU^LfS;;vzxQ`(9Im z#ibtD+;<$AR42`!pdm&c{FNqlwF<=6D}~g3zlhhqIt8L(7a{P587cgpipu`gkv30L z^7A^zo2Okxt0o7NTQwfYtV|oe@Gjvb@ki)u$^kOwwFT*#$Ti0A!tFtF^jBP2m59?e zB!T-xee62l0_VI;XU&w);QBLe;FKAT!pARV@2ax+$lYef$|Qi;eb7bX%j%IlaUwoe zAtWIL$o^GV7^l#w*nyQMO-G#YoTtr9`Sf9&(xpqDdprfp&86%&4HJ0Z9z-6NjUl4k za2m^L|IVrfTxEJYHlfM&aMNQEp-pN8CrP4JuQqv%zjDe0Koirjh{k@Cu5^7xq)3AcKW_G-<HqXQJR%lR%_SRq$@$V&`QWV38uH1*_@fM(>xL54`y~%JHTF?{GNYHYq z!hTIStm4-StpXv@KRH-Sp;yC#vt&8n_Z{SnUw&K$& zR~X+<>yd+p8GQAe1EI|-a9a5(vEAf`nGKFuM1=Afj}8_`e|MCl-^rS=s8%r z#k*)7+HHKSY!<9oy@))J@h2LG9^%2~RP@185{n<8tHq-WP)MvQ>egFHmgI14NSx*n zyJ~+v3f&$99lsn%=aU7vYqKX_I?IsXt~MLrloSL0R7ujF?1;r^UBP4L*ZAI&DymAk zmt=j8B`4NPqoIS}nCr8W@W6{}*u4Bb?tW?y-Uk<=a4S9dU7mu6Z&9yEjSonqT}J9d z{&0WIQ?!}MAd9&F9l_VE>`g-pBC-1>VcoXD*q1@z z`}jBheZT-|UG~N~1v`oC*Cpts@_cwHN9X^0HDh=3J|wR}VL4EJN3_lF_GE3<=rIAv=D> z!`S+(?9RVBb`t@)ehxU}(F0OM+)9fb<1qoe^exy3*X8i1eJT1CAWkkyZba91-zIYQ z`*1?HGtz2SfR4$2WO>SAl&A3orMvds|y|558zYVGSR@ac9vbAh`V_qkoKCELZ^s>Pe%dL&+kCzH%?+}rGKGuRHNNq z?g`pG;{?H@;@OPR9caLz2iHeY*WG^}_;#r;D{2`-yqDZ$(%v=Ua>Fca(h-ExbCq!7 z4h_&xoP?|2Coxl0J`)QyEfT;Dk|MXnHj+9S7n?Yd^{6^?0qzWtCfY0W%aQ*DRNR$E z+CT3HY5O4Pz4NflD=CxNQt=keu3ZU9j%tijeiHJSqe7A=O~Z}ZYUGh+5x{xFMZwSz`;am6O@+I5 zIcSINL-toBDi5u9v#?`^C>j0!0zY!tg~<9l@e5tPqsr$pFmIL! z8CR=Dey{YQ^&|Vpl(E;?h^}9YR_o}%1A~M3%?=l+RPAS)xdWGw=`_3x={&TsHWm)XH1YkW zTt(+5470K!6Upu)x7oqIJ?M0Q9yYCy!!xoH@I*d+n_abG*IQ*W^hvbhSH2;VQh3g6 zJ}U|?*17mnP!*cvl}>`}oKdMc2lgzR1=BX_kjdKZ$Z=p9deUBohH~S;J>9+#H=YLK zI(9laQCN%$kBlMHYI7}**f||{j7vp*#(g&W$2Jd4Hecf zW6v)r<^TG|Vb#K4uzoHd(cS^R_95vhfKRZS7{ zQk{ra$REkBO2`t|;>{SK*;U?dg~D8_;n~I7yqE!+U8Uh5DQsyvyse2RGe}#W&;2eW z?D|tfx_L2zmJNB=ge_!?G?ypyowmsRkP`NCJ}|SF7eiaPr90PhM2B{8tOH{?ZiGxT}sh#7-akIDMT-dplvp>s&WASM@=eF?yPWHAa&K~|Qj`H{bj(OF3 z&VcJd&iVB1oT11SoZ*jKI2k7oaw-D!XzcI3 z0BdC=+UnOiqIdlL)Ytc(t=xd{T2VFp?~@O9$*@IflVohwD{e7f+s`7yt=mv#U;qk{ za$dxg-tb1hFBGs@{Z6>%1xDtd7o)j;awvXpH!Hh$imiom13T2g!I|a9P~$N%+Yggn z@G>nPo>5@I7|c{-1(|lA`d9t`h0XXy--YY7qQYl5SK;kPUd%803?ta9wANxgDGNFc zS#dYOMllDH#Luw)Zt?sI7h|U3=S#%v{mvA=o&p|`xkz@qDtT!Z3#PPCxTj$aS=yKf z&QtQ>!l43uGF2McxD4Qj3l@W$av?aKEn+gb-|2U_JtG%!3j7)Rbk%53)R!Vx0ZztLo?{KHS1Dwqnrx@FDc9QsCs<1JEhxc}$M2;>Cnvz= zRhsbc`VjSlN+JAsJ>%hV9pBj*4OV;qv46(^iWM!UqPP|RNEIXc_jHIv?O`nWeFB1qdx?|geKt=j6Njxf$I=fJ;D~}N z%t%`g&GLNWeC;OHhj6zb8wxK#Nt+nC6cd9}2gX2w!v$Np=Lv9W=pt0g1VHatLv*en zl9*WSBxPg*!Db9;DqT%RU3&j-5Z-9hr76l2KA=5QgqB_ymX5D^m*5fK|V{o>HXyxGMYQ&PwD9l4e&LFi@pQlOBV>{6u=re=0?lFJuSuP%_b8ssU;1+?tA0f| z)!>yLmlq>wnf-a>hj{I-{Zn}||LEjfrSg~wJotbNwj|*;uls2Cv)yokA4LpLw^P;* zZ*ZU=s@bCAaBNK`R+?}E&2svLu2~il?XII}5q&-P?$QBUy)4l0Zb!q@Q&7jIP^{N& zgdWrKfar8hPHn>vrs~uUlq0T0R(w_^UJ?6X))p;NsN4-(8b!GrH3coWwfPORfR+Q8 z1bTwy?`NYW%pRm^)Lm@)4=k%Zk1qEqw-ha_DiRc5(lkVa!B zzbPd<9jdVzx<;1WwIQ;Fv%oa-04a)ni{{?2WQt4rP;!1INpC79$MPnF*t#9;;pNeg zdYpTg{I#w}-z;P}MSr;H{a8;JZ=!(%$4S8(-AO3f^$-eemE;&Gd?byU6`<%`2_1T- z1b6C!g+(+jdt+Hasy~CSk|7vTm0=V4>*#&>Ofvqr3ko&3g8d!UA(#KhHsSm#Wc24a zj?XvY+5W|Hj>FYAxA5V?wIEj7 zif+1EL5pP;C`RqUe}l`3LGn`+aBU%cFWt=YhXQQ5l2(#%HUBNS>pmZTj(P*eFLuJ} z=m)Sn?E#K5io<$K{ZZIAF}&!9D?I(GiVu4Sp=nDW!Srw0cG~DTH_f=<)K`Yp z#5G{mhda#v&&SCnm)o$&MzzA+I}NE7?*(b&ZN$vfgbbaZ%#nbTn5B7x9)T(33;p!h zITOQPD9Rv@BhoRiimIPiFj)6&4mo9Y8hdYOr!mDyvb@WXYz<7LGIcY_{dJ#g=j>3& z@9T@nx2w@;_k6Aw`fON3&KjPDXD$TRJW8hYF<}tW5`^VN5)sFJGBj_mhmEJ7F`b%n zq~=8w^FmJ>-2CjQpJ8H!gq0?`X)*`RR(31DnwvnTHb%f#3o5*NSrcL>t|wdXT_LAG z&W24ry~yXVKAMxg80q~<2X(3-+dD>)eB?5S359!87a?8UX`HE3n7 zD-3Z9ab1N3{#JN`HSbKux7KgR6Vt!o+?`F#pt23v?r*ah@)j-Q&YnD-n5ZJ+5IB)s zG=Io;obkdx*5Dw&Ey3!Pfi3>4JX5Zqi(8Fdv5D?MM$T#_cH+<)CH`|Z zFZU6CA|K8wMXO@Bid=lO(hRLkQNs--g?Q>H74%do7o8MSz|tybu*aGe?4F$uu<`A! zxVP7bi!QCMVnu2^@q=J#&M{SG&NaC%_|~%sDM%L)^Mlp+!HrbXdVDoe?KXqN&fK`ENesq@VU;3r_&2r;4_=;u2cz=I>0oF0cFYj} z{Zo#|H_+UKoae-uIfCC-mQy_@ZYXOYSB9O}`s;{cU3sMJfxsHgF88eAvy0#Sknl~EE z^4!Qe&u-?*BwP3+GoPyZ93=~7+(_caDJXV|DEgq%4616%kVC_D7ViCM%&#=uL96zi z%l<*V;dpZJgEF!{89;)rKE^kFyYZ{~Jt#VSH6D{dS$?XjK|)pr>P=hnl``0nynR71=bub z=y|D(Y$A&g?@$cCzRi zuUzf8hN;TlD3wp@vn`*N3I^<$Gro^A-4JzI}ZL3B`lugg^L zGP0czD}t{@+p~s~UtnL8tq{2R8jRj(!KX#O9>@ z>Qs}+yB_rW-9Vx7BH|w94Y>4i5o+kPco^SyMcVF z@qkZr44^1O2EJymLotIo7;Wr8AM3(#WMdo2ZrDLgb#kdpWgOft!hH4dne6hh)~xBY z9mrR%9k^RI$AV}>IPB6s46(N5c&mjqF7a5!JaX^@ogbp*FA8X^P<1QBxwwE`k~Wz9 z%LhAoEp&wzza5z>Lxx4XkcIzev~N}#EN^b0ng|_K-n|%_%rjwetTynwrDmtDbO<7``xuh+y3_gcV=e%*93^ye2<7E#o;JjHU@y9Gn=;%6v zpXg+ed3V;pIG1|n{Pr;wJ0`n;PyRU)6l4UMEk{Xs_D1MW^npnlQ&AA75)IrR4?1?A z`A6oNLeISxNvT#AE2QVdKRZo-H1ldwQ-4fHGVK$5IP)|Ye00{U%iWZs3k`>dfRI|D6$cnf+j z{X|zfcasYDepp>~iZnb8$5-^)v4TQ4_{H;){BR!&DQe)H&LZE;X^^#bJ^@W#(Be|^ zJ8qvgBXts^GNUMHP}QPYHI(9KvMN?;+)a#}780%40g|`p3W?9%%}SqpO19>Y!;kh| zB&|PHY?bOa(6?Hl!Xj}KDcQrtX>&i~?$aO9il=JC!S5^5du@ZNGHme&gDFHZCLfX) zZH0Yxfm9`8DKQVsr!k7f;;@ib-ag#2pSf@&mU%lo75g2?LWO)|IOm@PR>uiQPfx4)|`Ck z6ag=z1LVn9JDAwK9WxS?-BU!lLL>Sp`|NZlmpneY6?D#9p?iaG$T8os6@Nt^6P}C{ zhLC;+}N0V*=zvLh}e`!2;dHa#~^IF)WW-g?VlIgWi+D0;U6e_yo{mDhY z1hgq2ocuVJVLP_jj>)bRAsW-0$Q`?h=={_W@CaT(QZ`ISnaZtL^;jPKnQ?^OxMMA& z*uS2cAYBWaxmy%r*Ox(bT``$7jw->+;4!+%rSjTjk;(W2_6X4aOo ztjsLTt*xyrtSn7unp@5iI0R0b*_85%WhE|(Pn!Qzp64@8!a@A3#P!v@9Kn{hW#3<3 zf|`UfPfSk2SUfn(t5yKUMp}<$GXaxQdz@sVlsw2>00)@_!#?yGg6KJ>220WhEqe+MHXo7^1 z_`Nfpzee^`m8biOZQ0j30@n-R!g;dEBd|aKXLso?<+aJvqclxw%os7)C{V+n?Y}HW zV1xh~hDLBkV4VOKJkZ`Z0_7DcTx(kjBhXU-8!XBTc_r#}K$!yA+mgWLJyxVQ%;;%$ zoWP*Mo25iYnz3lB56@SLo_y}mstDdi!Pc|1oq;2V8lz163iK0rw#w8X&>I*zXp+Dn zlo$}+`&R-e)EA&og=e5bk1P}y5L$PsP^(Z`Kxi!&ScS3z!W$+!kwT%SfY3T$0EL1A z!oKknX$|Hye;1W-U@KLRIDqEJwE%3C^#j!#gfIbtXl7=(qH!lV2YKw(v;5Zb9y zC~&9p0##`@f%{R=O@-H_N>48II0}b0QH?^OyHVK9M~!w9`Wh*mD=zdg3JpC1gV4dK ztxgRB-`vPBxa!m(@MsAjy~P6mm9U4L1~mwl?mT-9Y7nf=dC`I`;Yz$ouq9l93x{E% vNrxd^ehd5V(WHF^^Ng3LNgt?iQqlb!#08US-E0(-0wM`&*yx*uV{2%(TI$ZjFGx9$iYomaX+l)!j2$FSkxub zX_n&)Oowt`-!YNs=5@i4XKH-0^a0VmEi?GWn1!6dw=dj`J`R7EyA~#MHQz?;Ztmb$l9S!Q1E3rH*Wq&PQPsk zd_R~by7Rfso&F}u;sFfK)8zly-7K6PNgWVN_jTM7Ah{3GHVB5uDiej+(wseuB7_3|jCbHcp zaub7{6@zUNgWc%5EV?8Hs}qCWnOY|Z7lY}G!A6L|lrz#q?VsC(^l_#*}AaUKtv1r^qTKLZvbF`;1&wm%dEU%hPR(=UKUwv7i znHzrCX$iskYIN?!CYYmP0sMJsmN!@m=N*4bv)1UqzrGnvkTC=nwZ~%8IaRi2OFNAb z<&&P@JFZXoP8Y*|w?X-@c-Y{i!ymnD!*<^U#tRRl^TIvU{3i^@j5X(L2RdwMLJ?q~(=pk#X}Ha3*fyNpDB#zurau$m=G z?nUWKl~m{c0en}lq20L?N#WWGHm&D7f#V7`Jz0xQTfJI{pH)0?>hsmqQo0%Z6$@}! ziaO(ZvYFE28Ke_53hV;=!S1Faq+mAPRQ$+yPP2fBs8W8@-`mhJ_a@&rRu8Z0KB6#B z8GQIzQedJM%^W^w)5qxD;GS%Vnip15p~nHJE*ybb8v<~TEl*7ov{0ycU7!a-jv02v zQD)_M9B>fw&kgdSaPd*{$V_Ig$|0!Q5XbMDq=@TNpTfE!XQ<)d8kVqBnVJVBQfyix zEXr$z3V+%j1LKUwaBi|&rhd3-%Qe_)0i)AhLr-_)3%a0z&nZekJ>x8nS+DO}LS z4LGs10emU|Zf0GjDUH>X!FJ(~(Tyb3)Qo_Z%fYa2+iN-lw)A&$jwrWyC@Lh%u#BsJ z;O=5+rg4WA@83!!nrSE`Y6fcxPi@lWFm zzGaaTz!DjkR!M<5H%mS4d^k*YYtv$`rX=9&7K} zpjFQz+ImwB6|RhD$2H^elSL;@H5br*jjvQ6Yk(iJw~O+Y>R?;DKiqb#CxbBow8l?Z zMFu9p=$LT^Cin=ExjLo|i# zb9XeJ+-pXu3uePK`Fb*&Q3=}Cf%x+Na}vhZ9)LEtw=O4#uFFuLDVyDq!oVM-*(JLz9!U=&s*Nt|j{lbB_8QNu~}$8IQC4wYiQumN1N zgqL7uep8X!BH&tV2K`Hw#Je-b^Iwc&P~}q$bUi-Ke6A#M`?!1j#%OQc zH|Qu>k9`QWagBUtz&^}>63i?W!ZBBQnrLX)EPPd>NYCE-!Jc7nY2k#^LayN44)#v} z2;jf#G%L>oqRe&+46j7eZiT^&>tD)EP+Ly{ZKGLJycLt~c4BQX1Z#AjQ|n(H$S9QL zsqnvNWSmo3AW5S7)zTHsK?6-dJoxxwdxuw(=bD$9*%EwdspRCyN)tJ*|4$DgJ%^ z_Pa3e?mcj+YY-X)o`)1wcRp{rE!_%^1&fkj+=&T?=)881(u+p~nk zD?Xbm=%NhC9{juqTGh;yzXF1NL&g5&nwyVoYDQTCh!#{`((sl+lK#e$*nD?)i& z(tT*pY8$SThUFYqOOY7n>1j})Ysgq_9T!Ki9|1ru2c64W13hwh1uP{ ziayb+xS!?DI>=RQGZacPd8$Yl|A8SlnRV-x$k9mdS%zwk!<(uZnJ| zrc?fn3fP!g2^uz5qRG~KX{xRvRK6R5uYEqz6RR|a86`3UTtzMeEQ^T|HRn8|N6#W9YCcd|x7UKC6L~@8+@{^`EHf$|}hG9S_b1iL}@01sy)PjY-Q{V#;eG zuS+Y0eC36^HE#X6+%c5~L2iZ_o2pg8o%Su~J_{UhtC4$GU@jAY}A!(ssTHtvh@` zHf^UK9=P=x?3LTNjczx=sk573_``wO^)A84(1R5FZaG)PKjD`S6T;@@%bDKjXEfgY z0Nbuz!+%T-fQ$)hxMAEF=JRDGjN9=LLZ9={n=_G}4BN-L+E4RM_om^Fd5d}BxR?=8 z@2G}1C98SM8|kD*vsmbck+4VI3NJ0r23cMYXMOg?Q%h56WnVHs?D=O}pJs|*C!0X+ zta0da(UHDB1XP<@1FF3-bl&M5gvst;J)v4SKV>6!oVG@~K->_eef*lWIV@e?9mhYh z=OLvNmiagc;l=FhFt{a_QpO}uJ>Qm)PjokHTtoB5AKaC1*85OFnQwwvebJEGe2D;WQvVUE8bssc>vMXbP8f!lYs6`LbXoJZa$z zoP#|Jak{~O=)5HG5EMX$?iK#LmJK-j?E}d`S++L$4WDA0&Q`RkavG;AsLa~6riH1I z!f_?e#WP93#Z6>I)nQcAoxlnecj2RA*dsotUX@S zlYn__c(wy}+_z#zKPIBYiecPRLuH6OW6YxVy0VTHAF20B7+h*{LDlCad{D9pKCuII zyDkaRQ({P?ZW!PUqC>V5=3ub&%L?TVu>@>y_dk10JIz7As)E643pTrI1n>1Y z0)>uFN9gF>8h)vB5ijlg6b2nChMBqnoUNf7j*FPebjK>;x(%CY-J}0R(Te#{x{UkG6RpsX$Gu^(>|=mC3`h)uOmT&Ely`zbo;84-kSHEx?wtY-7sKscUtMNoj!{fsz~6fzgMXA z2oL2e7en`mW#oEV9tYZi{%m$7!TdjTXrvAfYw?4}?;2Mz+ zN*V#|>$Ic99V@1%y3JsbmrdP9iWq2ek!%(`0nPU&q`mtd7uqZ)T4Xgx0qkV0ym@K0c6Q%>DqW=oXWS_#x*StvU0w zZ1}3}$(&QO_<~hDP42(LPsy<7S9dtDZsnoC)3e<*s> z05uGHL*Krp!u2j?yu8>4w^_XA)k1Gk=)>E*t)B25rC;8K4qiV={&o*PM(z#3mlb`r~u_(NU_h2-+zY#OkvByPlWxHzV6 z`pj^>5raR$Z#ly`ewcIS2WPncCY_8gr|FYN(p}+pR0xvftwu%TgLnND<*y8d9TK=W zKZAX${R1vbW5_Ha7yecQ>vqoJr1o}j>t@JOz>z?ha5R}MxFDA4%l?q)*JM;2bCmzD zT?Th-`%XfqcWdzeBMy8g>fn~GgDBZy5DpLNrcb-0sQJ=*ikDJH_p*!J%K!u1W|U37 zp>t`|=1}T6Rlwhhok->BYne`)5zFg5&6VB}3sCO}R{AdhqmG)he+CC>+E8PLYbwd+ ziUww!^5cw6^znkfH0YKnF{!P&LURA?&c7_)57&gPv^~O1FfLHQwf$NFNAgtgAQ*ae0?xiZgTmtoQvOq8 zA0u~?!cu#h5R?G;<1-zU_QG!$ACbwM5Q-15VE6rc;rIh#D3t^cf(PNUEN1e3@c(v! z0-ru6O-X-lnf(eu%&K;-YlbYloUhC!E^h*-Kqc(bxGISKUBTxxDKmke4V@Tg!Zd#y zqqLc!*j9^#jis}xdSn|PKD-~UJ4n#T`_kZZ*Nd5lx#F(%)u8_A2q&NCh0AQa;J46o zI;*alf<6+h;Hzy7xlJcvw|@pVYVKFk+@Zz(b2n$($IqfM`%coYnV}dv<|ci$+9vut zaV#$CQsgS#f6%L25BQyL3h3s6UtrxWz!Qlo%<4`*#9ywU!1d25zN(faO46t>bO=n4 zeFm{>N8nhU0DN3`kA&`9uZsenDX=^K0yG?vNQaYO^Mc!5;%1{R=m@z9ULW_dmBmM( z=cg3QyJHBdwhpjaHVk{i)=)@l0a#?OV_zQoW6!hWg!4*Z{6QPget83mw@0zS86wh) z{6`+OAH zWc~^A&wPjIJz^s&{J7v;SS0?NwUG=boaQgd*TJ&xQRo@f2zCE-$#cy)ntuKY1dLxp z&R-Wn?AJZyJez~8FRd`@j4L}^p@In)^;vH7KIkbv3`If@R61}SLLLgG@zw-nJ==F; z*XVfB<0We`8wSHSyOG?fue$8Ue>!Y#dnB0ef5N$EE@#H+8hE)&Y(~#*CW?_~R}6YN zANgUh&g2{asQe^ljqyacOPaWG;}F)^`ii8|R6+W~BkF&2kBk>LlM)+>^ahacfv2Nsf5W)K)8rBO2}vX*1RQEuyCX8I~u5LB4Je5S`@ zl+h%3s4PX2FLJrq@4fKcVjJ7R?ZUwuGU%1$#?re30 z&%$XCck!;M=fF%_qWFAOo8UCHz5-x-9 z^^uT&z=o_de$c^`F(6WX4x1J3@dctzSnC+Wdq*q78*O!F*|va1Jo^fh8%9y|s4INO zP7$5y&jt0ZYH;;b8pQMo@StTfRUelj^E27-%G?Zkf9VPFcAg#gy?+f$mYjtpu8gMk zZ^5_TPXs@8J4AZgM@cKmlD#+2p0t= zMy$i^2Gy-U2DTG5XwA9TWYV?(wU0EC_Q&u1daJqYcv~|$Q55(MNJDG#PY7Ehv|v+g zgy6_YvZC&G`cLyCtbQ?@FYlj3qIHL1!EtZAbl@S_j0}T_T0_X%-$kFVw?kA=1QvZ$ z#)m_ZD@r>LKgH7g`slT|IJ1$`?*0eb15RKcy&p`AZjz>N5uH6&K&PI}gbSLTwE5C3 zx;WC$exIrr6$qg8zJhTv~i_ ze!VIE2|Pn#>t?gp7vtgJ_tCVt`5eDX;~#m_6uRRwmOoQzLI&Xxr2KRW9NpAPzb4E@ zr^DgI{nnwbLJi!1Jq9Lpis06~3Exo9ZZ>d}vXPcl6bYXl%^k46-YNIiUUk-beV4l4~0@pxV+DigwdWj?qV=~)y|>vX};7md>J2J zF7E$lRG`2rS)^*DfcbY$+vnHbCjDSf_>}aO7I#>n$9y}?U71El_SZx2m&>A4`=!`r zD=!?qSAu}0uTa;U2 z+n#htsWD&)%jYnA?e7#)w*g<09K6l?%*}T%f!vd}Xb;%a+&>{;v`YpzY#hr<9*FoiQhQme zfhYFK2wfmQ%K@14QrP=T7ArS~G21)tC|U53R;XTw|17M)>&$JK6SbVo^7oLfaRDus z9SPc++bAw{IVml@$w}wuQvRV>*i|)?Zn`WWpO843`Bj7Mwm1!!50Av=t%Gs=rF3+E zQ4M8+H*_LcmHFTdv>o~aVue~M^dd9|v|RMy`x}lHt8Ev#smU{CsY(0+4;wDy@IAOI zRm`vb<43bA>Un;m726_P0pBkyVy|S*pu-eRj4hO8^w(2#A?Xq1O^#v=5MTPPTc)e_Zd;j<-U5ry@ zAMBsodm6Nof2j+O{=87o*zW*M?R}6Pp8@sfZJ6E@UmCR|fhGT2L+&?}(E0R!G!!i4 zTb==%S^ADzz0IJbNb?-KEao;=lHlK^bxJ&P7A_vsAQl$F4?kqcT(l*a>04KPBwxtCU0=X*T6EC1q?L>BFsDEL z{_OX81DrBcm>@bqM_};rGN^gRxc$>l!RtDGFx@tvdGo$d@1O!*O%-s$W0c@{#79tn zmJh}@meBWQDNVXPhpFC9LC<|Acxh@nzbyO;7v-sm2};pyxJ(o}uv}zZx&EjCi*C3J3p<(r%6?e=_X_Sg^I)>{07W66T*`<}G1nv71t3DA*m#Fu#);0{YWR<%!^<)v&U&zEIFcqt)+ zN@jPcC`c@?7VV;cCw$@Nu_!Y8<;G`s6o=CAvAF_mtU8-)=WJ3D4)!chO|*e$oV91IDPvtK#&4SJ1i3oh>{t z4a@V_(Z*%JVElum-I@Rw?22GtP8z(ZPo$u%KO_u0)=v9WW?)*)E*c@u zd0aajDA&oC8+t08Vr%bGduk9t&byjY#Z;~w)HxHq1_;zW%S9%wX6xRsNBQZE*Qu3jSb?7=qmdNmEVuis;5ur_oif;-Y{AS z{6=eP68xm$ZxmpoPC38hQw@|`mqSwW8@dwG0h%YrF#nJtX!TN^MR+FgZ{Kae7N^S~ zvvv)w$aJOaYihwFkb{*9hsZ(Q3yvtL()KSzBVJ9Wf>r*^d%Y>n(zIsft+)C7MF%0s zW;|E4x>=Owpv7(_3ui*ex?0*_F&@KA9dP_LMV!~t%-@gF#Hq&v*s+JRap3M6-f^`x zUcU4R`aN}#O?t|mmX%?a&N5uzbV)Wu#*odfUrq+IGx*J4$5K(R8V+rI&TkrfiV~hZ zp|v54nUB+CjCitLbYQ7AQ<2XGIj14GeWV`K5(;P2uahrny6F=Vlzildj0^^KWm%RS zxm@u1EzzTF3HJJv0-k)S59?a1VMu{2?G#VR=bIMe1(atCFFg}f<&I-pZ_DCi86W&M zCLV?^Z{}ugBCrm!W3S>jV8i`VTKdrxw~gKamfP1rW4Q_o(i%iI8wxm~--uPT;NmlS z;p0N{9`2#RgSUvstwX3}9}6bF-)Y`wCm4PWSYD9?`kou8-j`j=?U~gJhcArd&aYBt zAfYeVbZH7Ze$Ie@HrE^P(R=r(TM5$A@P{^{qyzV4fEb4V)>%(+0TV^8snfsa>?-b1GccoB=&P49=Rb=;R zExUC27u?x85qEJ}bnNOJdTLP0`CSv9rHdD&*~Zu3?Fu6nqoMT`cMFKVPdpbU>8Y`XAb0p~JgAt(+zL0>NG z!1VSmy1v(w3kq3E!FOiDE!`=Um8#E6zE2Z4rnz#vuOtc5rXHk%wqyLNGflkv%kebr zx&~Vno5Afbkw%FJ`{_YI4cwS~n67_`W`|z2(bg41*o$pWSUjwVgvFPk)x}D*JnIlF zd^;b6ja{qx7cTQSJslI47Nd$QKlK6^;(;bJCvbhk1-L_DH8TpBit`p-B}*4SjCD`s z=J=;TvC&>y5cicff4T_?>%F*7EBFiwsg7!?bPc6LI8ZkLeX`%Ir%N$)4cXU&kKG9DV1x&TS;qmI1-2>R1S5ud+P z{Vo-zxMe$Z?~WIUJS1>qtf%PLKLLj9aRWhg4i!@kEFAlQTGm>MZS`tC=}J5IMW>&a zh;zj)Cx27lpVfHKGnE$IPk?164Dwza138)N6r3TvNjm3evTHiVTz|n^l4^9K`aOdw zRemeC<4!VoW>)ce0r@m&b36(r$I-#$1kjjd3(5+rEKnm4PD~h!+jl3x>&r8^1%mgW z?zEil*gHXlaW9m+%8-?^rPw1>Vt3W#nCh?$zP%`e-;s8RQliDl>QP(l7OJ&DreqXL z`%?|8cKWd|Q}05QsVnb1#uC0hn@3Hh_0Ty0?5WR2TzKOq9lP=k{It}dcdQx~y;o)m zpB31bV^e{e^U2gDhO%=E*c-`U*kM${J-j;upH4o@867R8$KRk(9T#l; z7QlaNwnE{?JAPDnGJt!V*T>&zn1nfIityZLB25k>u+5$hWu>z?DA~j|&%Hp2m8twa zb36XXQV*D9w}GDa=JAmauW+Y#`v|r_H|HOBPh?r=HPA5V68~?QG%mG?huI(faj0?{ zi<|9-V-LTihiNYONG}GyPfIVVPvq=l#HE^LXlrJeXDXO z$2=u6-Fy?Qp6};VM`clZ;%w~R^?}xW8iDtiDZaR700&Jb!S}ho@Y&Xi-TUW+su_3a zd(=%34)>y$-n}3+{C>B4{K7NK+4c@ z@cisfK}=;fUu@1O?%q5cBC=p-r+=w#K@}nW``!-v51)fz{#F`NG#pDEvbo=r%i)Tp zCcdnd#NakNP)b?_N^CJ(W;+Fc9U3ZHpbcnJDTIzAcYt?0K(fWHeAB7-ux--;QKMNE zO{lddH=iczKmVI@E{><^a(@MXybe)dnkK)|emc{()nMTlJ47-lj1V2KnvCHi4hgu) z1~ed}!b<9&f}xfUmpnWX+Cv|Ka*Hgr50}N?uQ}MRQA>*~9r5$O#dPFnGz!OTqo!_g za(d)7EV(+28TmrmR;LG82#CU`x(2l5`SKO72s*y(og!OsN)?rflOCri!S>qKii4>4+A{&!z)WBk1Dfl>)Cz z8G^NMWd+;9XYwg8jc|ge9iEtMiE7VFxS@Sf7;@iWxh#AcyN}dr9rbn&H#FALOw43}ji4fej~DqW%&YNK3mb z_?+R*`Y!vT%*lsT^x7Jx_oTqvp=LN(^pzSba@dQnd$I7n6<6-#PbYVHW3h#*DAzxW z)U)()_)iy1`6)rqc9ydFcX{}{dGLA~)t*y@{z ziEI4ekWMI^d=vy@tM*aJ!)osON=;_9xq{3GYmjDN4$m=VdK~CMn`ckp^QX*b7Pq7* zA0pU4Rb`Ay(19?OqmYVQp)@xc7K~rSE|tzh-48_WR^38)HA$BLz33g(?i|cc7!HtQ zT`k`^MiGA>e@weCp61q%*a^RX?c*~Q58)s2*P^bi9~ z{}~CJ9;o1-Jw8;a(a*KMT0|$>1+1W;kA9rt#L1TmbM{~$Cpm>R9F(E|Ry}}dac=M+ zQy=bRMUvT5O*TYWY=ahhYbf&$v74|MSJOPwdpn2*T1BkzWTBnlcqQ1}A4c+ni3jV+ za9v-P&rMXNBi37(^2Qw4eE$>lXyK$xTG|kY#xYS4Wg7sC zJuAt>KnaEG7_f1Z$t8LiD&MnWy&$%2jW3b+{3cQjd?thsigK*j-;hgq5l`E~<(O5e zEEsyPXG$vLAaCSOR->+j1Clr3*5c>f`-HLFyP030c0w>KuwH}NgJZa5UiZNKWioWM zpBGL3{*OXBV_8YsmYQLzCamsRH6Nf91pM?U7W%gs`YawmT8SPUSZ2vQ35Mk|m|qyp?#xhwq+>%de1RN%l~e`o6K>*w(SmIqy9q}1 zeW7`}B6wXkjXSG1l7CdR8bYuKAZb?y3AlvmQ-C50xbTX8l`u*sM*j z&5l6QzK?XJ#1$_HjTf@cx3^%|Qx)RgWWjgw45!;;Pfk*`wCZmjOg%eUfZbbY%bc~$ zC~Z7_d^;RUUJXOv?5orzR=r&rPswOp8H878GGWy)avZr8XL;{r{yXI1k?~5lvE&%{ zy=)^y%B6s-N*`!<&44jAKly`^JGnQGKS?-d{8zZJ^(mFL-vIaVe_+7b5|5PZ1P>h_ zx_$cvpBU5w@uUAhM(0Shb9n`}?XSpvV=$N}jbVI*IEr1noIme39p^c0=bBB1Va|;& z4rf1wT>+N-#$GixzxOLlTebuSUyP;rw08ojt&&)Dz=kt#`6Q&}S2fu$j79I5 z8j3mhjd!eD1a0=UAp7J7MRjYizPTpsLLi`5XFB+~M#F3A?ShI~TS0fk0m`fiV7cG> z`QhI~N&fD0DqJrEj~g5LR}&^d{b(yT)=LjsBn#=mgRQ+cnxw&LRle;{-F~=20^n}2s zZ8A7k<`AsGiL@^?6}ByohL7^S{GVZiU~FLks>KUMisz+K;q7GTPCkpfl}-!Erz!vo z=N{!B9acnXz426Q-UeT)WSFc#Nb7g!(dhT*z|;FF_t#{@WuVQuxSoy^oE_ zdBJT8_8Y;TYpx!Km8h_o?fDS~n(w`m1sIj?< zUKAdqabJ#6G2Y``X54_DUoWY%?jrRF&ll5!*CzatWrNunaRJVZtAfE#<}g|DrmA{> zIo~`@tb9`SnL)>RbTHON>BATJ0WB-?8t=~dWB=%{eiXkuaSrRqih|UTC)_|nG?*Q1 z;xnTpnR>$|2<&Ebs4bSteyycqDK{MWGKnuaq=NQxX;k|+6NEL9Kk53Ci8!TV5GQu) zpn_M!q#2uG$I(+nCo*V~X*8T)X2-)!RTftfOViffpn}UeoW9%*?n3BA@w~W&PDOnq zQP(K8>$5y)t~cZI?`g6E7jJBt;zx@AN_d5zdhBV(V4OJi76iYQ!Qg%2+{l1xK6lDv zA$c_Kf}bhF*|e*t`5Ctp1*=TGK(_cOb#!Qulb1OEclU(ScPHr1qX)EfSOl6(6$w%& z4F)xvGcb2;3m6^IVRIf>Vb`QK!GGy%@!yqu-2QA+G*@t7s|SwI)}SNYvrVt~Cc8h> zJZ_(8`!`t@Y$wTVYNSz8PsmgDs<%`UFU4Mt?&s^fmx|g?0OT`y<~(-}S_IFni5lff zm)2UdTwa=u`|%R&GtR)Cuv#duN++q88hCw=Hh9mpU|Dk$_*9!r&eGtsDAT5#w<(Sx z^B;|{?CuQyRY@7e>U^i_lYSU>Mj1s;_zS`-9BzHIl#O(jLB;l1y7pTN>$}vd zM-?u_b*d-m`>iCJ_MZ<;oF;D5Q%>-c;)D>?dX|>FeE^0%FS%P=uh6*b4LsLaC&(VB z&P>G%?7{O3sWf>p+2l!LSN9Uq`KHD49i5rP+-2NT|Mgf9vjIGW??k3|#XZBp*^AXq z9L`I>P-Z4S)leq>8y``x11?dCf+gakxnYVKx{24bYBa@_-lin@^dJHY&K`tq={3Bj zTNk&%z=_*`Z3cV3Kmw;`1aKmWIS~GKxnq z<)}R*%EW-5y14&`hQz~*?4xvl)D1E)vVpGuL-gw zwJDR>;Qz!IUCzIyA#SNyS|`UMb`M8cMPVfA&L|d?cZ-9^Mb@bGWr*m4NDgQ3%!B^5 zv%n*1fV@tqlV{x-xSQU=r#hXWm5cjesH!Hj2pGkg4^=_A%lE+HN;B;}VF>2))-$cF zqqOqdb!we;3U1wx!RQZDiEg*)PKfMa*et`~-xKjjaRS)5na7zJ79yg)n6g{RN z+ebbFMl@oXI@8G2<4%8_Opa^qSHIZkQwcH!%(ZuC#xjTQdYq`K1!1fwJX z8x(v%$7nlD9k-O=*=CB8i=n@p{OM-wDROVRPl1I$1-0voXxQIquJXfQ&hf%x$lRBcq9E4cXRGEOf$1!HuucYa7-)3nV91DYZiRNQLzml)I*tV zf$T?oi|A&Z_+o5j424H+gL`-NSpC{b?AlLVRx6{0_gAaIF*q+$)>32->XfNK>L47s zSjzjInZsSlJIUJ|zQ(H*Z=pRc7SM4<8xmS#Ve4?=T~X{9M{2O1E?POt9DWV-l2pk7 zxa>HYG!;(4!O*qPxl;p7?8~@rcd`2+V~E?%&f}84iMQO1rjUuB6+X$lA~1esk0by4 zNXcE1SoNiz{I3ad)12XyI-m#Zy_A`O+FLp?%oTdrOKx$Q3R|7uPMb%KXJ>`C+^96< zBG^@kFBrQ0Bm2YZB>%~SS^NJGT=UjI$?U;Q=3Wx~lNthBr+*L(>Cj{n-+V*{S#lVx zA&J_n5k@PT!Lo#P%=eo!WPD#h{p%k?#@xsBB3J=+Z+Jo0o)i2_6>A)ARuA)z4`W}O zztX;#%{cqsLlB<9W>`F{MeroX2oF5dX4j%L@MTFNe_P`Lv~>O!sh(W{I`eGkUPl6z zF8#_4o*V>455^P3i8m&EU($jLQ&{wSRo13EA8M8Ixs9`n!OgLh>?b|oCO*0fwWeB3 zV%8Sy=@|>^HY!|dj5hlb7Ygfie!#Q#MMC=G(+3NVT;eynzM>^NSJLr2{;Vvjn$tQE z$BkNMjfpXPNdH!^c#oIh#J-{IQ=B7-#FtZ*US{CL^%Gd$fg{+uFPA&=bPrlqX|daT z7h;Uu5xR0l5w~rvCm9157?yXRpEyU0+?+U?@#H+er>%ivT>bg7zru2AmQSSlsT|Z< zt+6{~l|+#G9(Hgl%rejeqxQE^(NY1c9uk2K2F7H-JV9#%2gS30a&Z>@aPRpK5K8#a zsOOgWC%c-;mE_1={AGzo8i?j*C#PPpLwb!zybi5Gh6Dc(m3!Xv)XNpa|OV_hyClE@*Y zh4s`qu?a2;7O-ETn%uzBYzWeB0L2v_M0sOXXk^NMR+p*4xqMv5)n4u5PG4Jzb~)$y z$zN5VTVB8lLn2Wv<`)Q0h_~OjoU(wdXleA&SPX-2%i;eloq0S}-}nC!B?%=$q)1dG zW4LE8nlw=tHDn$uGf4x|K*q{FYpFEQ@GeR!O%#=sG-~uapYP-M@4kQB z`?%*k_TFo~Ua#j;T%3rr$_!ZblLY3u-X7X!_XBssS{(n=7_Lqji#4Yw)8}2;T!8ON z@S2NQqd6OP4&1=p-o9jwz53LD`%D<75{XI?*3jElDoUO_4qJ7X(^cWSMtHTeQ1@H> z%7e0#Fdu+V^TjivAao%%-d_YmB#d?RFA?b`IApF`Qu|Kw@Y z=NVA4XC%6Nx54+f{nf{%uw8)Icw#@`jjSPIfeHd8W{>VSm>SDW8T5)Ni9gXT|1y-LXunHkT zCmw(51v|B-9?e{o(fZ|2-lV*g-3!jcBL6p7ar7R#1-9YqBXe-BcO;D*YYn#g|9GDb zr`XjP5#_jVu#;VThix%D%6g_P0JA`8QC*)B4NmD|ZF}a@fBv<6RrD*ouj0WH)h*$} z8e?kSHVE$Azsw~>H{tHR>%^Ri$sg7k7>_r{F9sj68(ZiR&VQ5Ijy={MFy-776mHtU zx1SKb-!L79I)1?6eTiuLF`OMQI>;}Pnppi$E0eFiF_KI^7fFWeAHsp+C>(rWn)W%# zfJ&tTJ<2g>8;_pBdohTAb7O%^dXKLUKE{HE9DLR#jwg5Bd#w9j4QtypgJt-au#Y=D zac_G)`*=%+OhVtGTOo%x``+N}q-tKF{33Q%pBBNwHgs2dgO98=*xg|S8Cxys+_62- z*Z(fx^+pq}Y$?M%*+vj!wOQh^%?kqkrAX_%kQv`|mg_vHMla9E@kgu=u`l8QniR9- zg6PigRC4|{kb4g!s-M;u^48~+sQ$=2`qi#NmFK1D^-V3*lPkuJ-7jF2aI2k&SJ{qB zSEyvI8>&rN%7q0NFzH$QtL49_14=7{&oT?>^*)WGiw>ggv0~BBwJ~gm>_HseW_*xsue_T-lMUx}g zrnoG&lYV3MD#3)}0r1-=7PCy{X}MrtuhucAh{yxzo7~5#<>uJM#y!QTP7O?KxPxzG zqA-531unZe3DZ}rLd~L&+?kDj_+k2QG1_~&L;g@L@XX%C{E{O$_d|n4x=Qn?cinDq zc`Z~fYns`ozeic}%a)6e2D+l}nGMvo+6PKaWk{>yDTd8=C6#;|ND3dsD)#2^;{IRJ zKdFN)3{s(k*-Ie5wF+NF?uRKU2RVfrIp*Ixj?%l)9APsuWHA zl#QYMO|o6^Mon)0VM}hMp@hpD(Z{~ThhR?Pd<;3gj4gArhHsvuN%5J25Wi(;`ffgC z{FsgBi#%S{I9sk(g6DWyvEch5HHX%HNZ&e>F6sEOSdy+BQL6i%Mzr!q{L% z-(GG+kM*z6Z586o<{eo3_5gkJPiL`phTv-d9UIRJT!1g5Ks9Rubye8YhS>i2ymdaN zEoM-B{V%_MrEw$sULL~cTgy;Y{ur|8m&|K@eapt!go4+f2Xy1fQEvXTQ_OY!Y5dFI z;gefK+2iBEu(<6X@BU%1DA4~Xy;YbWGuq&ubc3lgiN;C(SG!S`$aGB2NK-f^WaTz{s=3L5fxk6b{0 zMlPGva)2gy*>g4%Mv&CP9OUX>aPoJv==jr8oD*O}9n-Vv-E2zE* zDhL-`G6NwEfH$|EWf5rw;3=Jq$NEfR?;##n21J1T@cCjW9eHD3z>l_6g`;L!bWiRmeLi##2M63E(FH47*Xu{| z%M+Q*%2aew&LZ3WE2%^F5B{`7{Jia(5Jew>Rnn=DJ5`BHcHiJ_XZB#@@TbgoPY?gW z*leo%WDYF`cxx z1%b->Ik>pO6#@-{$=g2*FVu**xak+zd*NpP;@k$toc7W@q=pP4X0q6Yc^BW z*Qd1`wQ7E4Rp26}G|-m}v_Fy@LN*t_kT|~lIBA_NhOzbMKxbhEIBlOq9&&0mcz?an z`w9ll09E^hHZ9W4I!PuU8>wVyJ(Jxv1DxO9WY;so@$L7eG;O+Y`^S-zz?0rEu>KMs zzV0Hc$QVF9>%3{TODWs%=m03cl7eH_gX!1(h3wLuC6p@|I+LDD#JKTlHje$}%g(zi z(b7!@bbqS~^;++QioGjwqi+gIT1Har*FjkFZZK#D&u2A{Ea_Dza;gut;EiDqp9)*y zOH>XeyxdLQLS%Vn?jD%Y>j`PMkh=Pa3Y(W<{?qZG^llrRE7OJ@HzH{Ajt07SrJAYS z6b}cjhw}C^ho{r0!5THjwjneaa^c_cX2{tV!Y>mJ=iG3jB1~ z71h+2;J6`<81zT57?-)w`Zrp@i5i%FY@2RBcU&@)DoHmU$f5;BXOy#HB zoI}gaN+`9;k}l-_VP!iT*c~+o7#TYlcF6`XZNca({&c_y-*uXUQ^GJ$v{HuxOI6hkwO;ZxMChZ}lce)P0 zs`p^&OIsG=evu3_4wG@|Oj2^pCiCQ(G@>$|a{cA0Wy}$DdR#;YU3bv7nfhYNU6hKy zFSx?=piW#MdkNK7{l`XTtf1c=hLHSoAbIOl!}OyGwB+G*Q1Y^bwWZSdG&By}3dhl7 z-|u{|!Z`eqqDp<~8|l=RKj?R?jydOEv77O6655TeV)~nBQfr47JF!s$8hyi{sX&3^ z7dTO{MIt4;ikCs@%^WN~;KJE-Orr8QDOx{35tXcOGk0HGkTv2k;rTCA2v|VFl2Ta6 z@%{X*C((k~UQU?T=KBBdW0e>sQhqG*2Ih(b!feFs$% zt8qzm1ck>hg#E+1@!rPm5{FNvuw3l9MAW z_dS2-s2ESz+Y?*|XlNT!;4mR?AIW%J%#Lk+unX4?86mdtv}GGQK_o6>>%E$`yL$usbo z+zIxgg$L!l`N+z)VyW#+IISZGZfQ&Sy$M$_#?y)!Q3&Zo?4hm4R+E&93(ebZFXpC~ z9A?w{%dnyIGcj@aAduJ(rb(7o%oYk*;msocrhhq1t4QL*obPbIb_zg+>p%EKmS3nu z{w^yRwTrDWtH$)S0@M^#fcP1`4DS5mor?P~bholZYj{0u@KUDY>2oL|^a1XAGzRvj zZK1xFEtIn?k6)E5?qPN9RT!$Sg!58oLisF?{BOzg_Aa{6`RX(4+@DY3$CGK*dm9`Z zjQj(7e($wyl!fuUP2c89;Ttr_q41VgvcHhu}hV7P+Gr^83Pk&^OZ>Q56sV!8~ z^^X0OG|*f7{q)US44KyDw6)BTw!STZKVoeKXz~2U?-A3N z+Gaw3?%1%&pF(NzSbJ==sAUtSU4VbozL@eCA!tg@_7_LR|QOWucS@M#xP|}CaCYM0G-9Z+3B(b*fe!3`|!1wE6F{} zMynkc!)ot!ls{`QeA~W?jqXsO9i~UA=CUV6TkN9hWsgXz!hvQ42h-V&2lz!tBB|@Y zP_E~Um|TXM;c=TNjx*iJ_g1aoQZF~K(Un)=%2|EFKpCoTQs!PvIK)L2E(ZTUMUvat zA!#0ka4~H-_lA~Z_2=s>p-nstPwB?Np;P6sEPperX?BFQ{(}8F^fq38aF<^BM#7`L zRp{`$h@ZFhAgy}mh2b)m^zW1eCD(TI+hg`~3p*mQ(q+T8 z_Jg8TVM1qEoBU*MaLFIP@d{-M6eXubi32t=>pg|ISFCOa@zo9#(iJND6dc10=ZE2p zpa4F-Tm!l`$HMxvC&q39`h^Ogowr zvmBHI=0fpa1=`k>MB-a_am(rZ3^${JE{@=M>o=t(Ce2 zS(ARTfZPJkV6F*N*L`H79}3{s6A054V&R^QIjl90z>I=-yqnE9dhWOd)wLA)K~pYL zv;27~iqD}NX~AMzvQN6E{a-%C&$5Dn9Yt`jF$cJcp`dnu353XRg!%ac>|^TgL!#4o z*brUBjjvE7zhM%RdmcwlCc9~rXE5zIdQANduh5^3S18{tfnuVZY4hA=?9;7pY}o3B zw8_E%3IzdPUBQ#BPKo1qA3b<5_!yjADvl#1wF>^@E_Gb2zKz-^OoRA&t?bI2e5To4 z$&0jA_<(uMok{`1P{-TNEQsJ!4U*IuaJ-8me3m;Yp~ONb3jw}S4m ziV$$+xg-c)iOvd}%Fz+kl5>W#Fw;gArhWDRn-f-S{5TFC?;Ak_ixVn*d_2O>)R^ zC)MeiP(k-V%A6iabuTr+L4H3!qj3rgBH;8=)pC46ZOeN z`wtz6x=dw{C(*U7wRr25B5Z3q3zZoQA@@=YbmoX}z~*`JFmyyd`0CAvQ3oC~v8|`EOz{*6dL8*@f(#)>1NRv+keyIXuyZ9+<)5)=-Hqq{(!heFiobI z;YsHM>_CJ8SPrqJR=;4d`fwJXTv#Qfy}X&9tOoU)pv;`!c;U8oo}HU|gWtJkBHgIo zfDcw3!9AMS`Q@LFvfXye=wZeL=HvJnmkQ5%&5Ji&!`2Wg@x9NEy}g7t{dBR>HVW>2 zx`bN-BKgVQLrMHJ!U(=SR|Q||iA2s)aO+bs4v70KnrNHGPFV(G>ZnA{8K1J?=N(9$ zk<{@pgT}qDM-yu=ythGzpM3BVU-z#9zf|`Fo2ePB*n0@XjsJ){W=hpuxwMEb^;)3f z(iJ%EP8p^){=lRbRoibZ#_;ezBlvr5o*0gX+~NKCcC!6lOq<`Ql69)UNKs1Ro~B(u zqYd-uZnZjvO}N5;{$0`V^80{ER;rs#0LnWJs5qNA51WD4;}qjAXPkY0tbvT<)qz zEb_+#OuE0EhH0-Ov1%TA3BaYrveNKx{xGh!-Ig}Z52FuT9q8rj*^v1BmncEikLaW} zMXpSwBhh(emv)3X7~7JL`W1Y4tbqb7Y{+(s4EwLp9>m`jaO+JK5D=?IjZa(16qvnI7+H6>}P3YYDG@|n^duoj>VhfuySY4G+L}s1C zec_#WHTD+r9s`)wAKos_&yoA*Voh#mszer3oLP~39QXCFC%ljv3dyE-FyzO7*qOJ5 zo9Y}&J~BMpAfrKtn-Wm`{2IrW2Dw0{?{ZKVMe?&x+OPz!3(X=9VBNZGm~7<5YFds# zxBGTX7+@^wUGW;ZLzD1gog6poY8KnFdg8Rdb(O9~N)-+zm z7K2mR&@YUs4RL`kN91$98PF`IgMWrU5u|BjF?1B^fLH!mdT+OoeyjnM ztheA}Zf~XfaTDmNOE+EoszL>dnGpGKGd^^SCdui1CbuFQ7yL_Ob`R#kf1)~S%}k}k z;*T`dYAB`Y&t#WPBB4xr5okCHxUoAYv0ne7#At&feuoTFySb2)6QYIcw zzuXLYQHoG?T%UAb&!^IKEBbM39k1BkjbVDNte{y0cEwNeGv~%#_frDx9SJbHcQFO| z_u|0RC7kD;ZoHe6$IUj~OQ#c7ldMKE9XU0T5|ay1yL}4OymesiiRWPRv~RHOR3@%F zxCtJK-|JGqk|D6q>?F>Zd5WvG+)H6QmeXv*iMagrz#508C#mk55%3Y?*~HDSC4M=* zoPlC3H0m0&jlZ;+=Vw6`9(4(wc1@>8mN8w4z-0U zLH&J2NQ|2a-N~L%n-~F`Cwp?@%fgG{FlZQ+^fZy0S|s`X8VgU#hmxYLKN;x(wNJc* zd!~7TY3~v&>8gN$+#I;JYA;yHuIK0NoCB_pk|eK=<_;^TU~#1*=H^=ou&wKNp+Phjp)Vff?4eQt$My9V;UGP8b~MG&VsD8_%SE@ zCmJT+l4hM@UQEsQ9Gn>Ti}xy#fiIFS7MTsOXNRzTKAk})WfqZ3%x!)d=g2>~<4mOs zhtslpBYJ&jDgD=>LtECEQuSRq*uHZ=jecFgE880j8h9)h@%at&dUl!J8AS{`0&9I`VzOYDU20?v;%QJ&!=ym};&W_V=acWW`dvW@30 zw=&uueSy59|HI`uX>@z$NgDS44Ar~mvOlq<{D$E-*iza5_*cRv)D)b}9Q~A;=%54^ zeoMrZR&Q`_3*mE@-2pEP7dAI@6^zhO2JzcVrmXtlc2;-mJU`{a27dR9G#2>#1gf65 zrv+}vHMPwqW7}%h7&M9L<(#LI|DNL9j=`LzRUx;0(-2DY9}C!fo-<#)AN*HbL9ks8 zTl+7-<6IgGn6;m2$a}GRg=u(G@f16pWyPL#sjwXXebwJ|eJCd3vX~{!(#L*nV`!4I zF02kX!g`jDrrBrb&@1~YxYeOWZd?KW`{F6Pk^u_DBXYfCTLG)|qbgZ1i zft9HyN>tEDv|Vc-}3PW&TYdr~8g8tMqiNqZ$9OXkwd7#U#06CBcEBrxay}fO#5vt)w=Af-z+e#FgYTFl^b1#W)7Ww2<7&<`H*3Fo@4CD%o9`19!&Hc@RJ zd}6s=tC1u1Q6mhlKp33AkR-XDR#UfNJ48?vR2U+juI6&GUJWMKN{|8dX8NdUw*lz)@ll zWkqAThc?N7or1>CM$kO`ATrx4ya%(6z~~7(>ErKJ{O`|Ecz#|od1<|2( zJKUl?hZ6dqe$u^vZk3d#FhoJ~|BEXng1EBIq~ia?729sNDd|q<`UNT0&wlZLD@PVs z8>HX<--`a(XaBdt-j9t}^;__J zy9K_p`%2j#kJ0wYw@LBgbm}{;&hF>PLzTY+ZCj$mY&RUISnJndqw6f%{ZN5igTw@L z?6m1)!EA7RlO$wyhEU!pZ#KU*M>0XH2JD6w;(!IQxTf6->aTdhhwpD_4BFDY?YX4a zS;5`OJ1>#H^PA5*xr;Z{Zim~(cVWd>M+zS#Z~sgnC6|m)p_h{;!alVNOm*J@N)fy! z4+cgsX&EJ&zvB#fi7$ua!~l-u8eFK~{VDUzlmwan_qcyMn$z}MR9qJec3 zAM?Eh2Y=oVrzh@(2DNwOPS+Avo!elcoxO&Ov1=`e{?B42*5ql2WkMuzsv*~7JbMg#Qz!5Y7Gl4^d*WwwnaU;jsV z(;@*r2xbI%*K`O7(1s~tNwB0>k)kHbk;$!tWYTqjVpX?&TA}(=t5`GdKZ9iX7{6htwW%w=@>6Nv=BN@ zn1Xahr%<>rgtUjXY@53p{rNt#dW~xsSGy>k{+ty=Iw>QjF-Mo8RTXIOU?-8czb6Yl zT7s2E&G_zZ6g30{(?D_FLCndLWp={oj6%5?G)KXm{EuYg%nAwr>R6&=aJnVFkR1r6 zaVu!(!`1M)Yy}Ni?N9p7u`ENs5NfuxfbXeM0LC?JzWHFV+}pwQWdq=E^#HosqRvsR zH2oS~j&ZjB;Ms3Jr(X1g^lRF>!W@SY>lT34U3tiI z?PCSwrBNqkEw|H9hPDqA4&vu``4}@3u+Uxri-$U~U597W*>MNy)uTytJa{eBc;$!o z3*{lEaUYnyQKua3G4>zlJVfK^SFxaRHuv0e63pEvBID05xo4Sqbnv~k7>6ZDL2P># zl`P4IX_|Q=m&`8q{&E`b?=EIwy&qW)I#;GLUW6}n{*xB4yI9+Qy6}r!$Q9)aA_gG%^vgZ&KiWEpUX#qaF zwzaw=_YG^u0(jQ8jub{bMrY;)Db0teBX>Mx1^s42EG4W_Lks&h0YBxtE#>cB2TvNS zxQ;24$o%(EYT5HrkUf*QKJhJC64kxMopaXF*`Y7(_BkZ-=Tyc}+PU}ar+Y9gPZH7a z=GDw1XE}-4X7+rGfZm#MiUz$RT08I#g*YsLh4T*6ZjTmndSr~9#V1jwH5jd;^sA>Vsb&Uc)(^1X zWhUU#(+lkKDGfO(GkT@B8taFz5h$6Z^yN$hh@QWdL=^;~n*L%oKd6?@H9e$by`eDP z(Sfvmc0hOYZP@Go3&vy`v#Ub`A;312X;;)SKfT>#Epw84TG>eH*9XAC!%kwj9GJw? zuE+^Zb`$n+bTZyasK&XYUvNX?6=_8(;Oilwv`hK`9(B^jav@}RN#McNsf}XXjTTa# zk_Rv2)L~)Wpqkq|HVb5=4j8mQ40?Xb!mmNMnc2x)-Yz5X&k68 ztz_31=z+4VI0cOT+L%jKAX{SK!6eg0ur!-=USak;)QeQDnI5(d9vP%lRf;?>BUevn z?;dBhN1|D#{c*neNg&0>e8Ljlhg6mw1+k@n_<^f51-^?NEtwQX?WgU*eCuja+BOj- z`6tmuA4a-^_ETwoC$yi`M~Uh>C>EPfuv`5q5{hrBgXQ#0aC90$`&xM%WHSo?^lO9R z`&FU)*Hmce{leUjCE?87HT;_tJGvP+i}^(QQqieW=8-v-;siTZ$PS6*mfAQT_DvA* zY>&D3!woPAjbO8Z3A?*{AK@$Q8o9XRm{;z@kIvf*iZkn(SoyUdg*>+dx8b*OUfCEp z^s|b+5Vf&cZ+FqJ!yfePWjL#R=|I1q8qub2=b`Vw1w1FEOIq73$VOoN?`#=nuQAet zJ(_xfPTlU$G=^=4kz>|#r~mGU)FY{QHYJh1+`Nm2G=&C#s4_hGd>T$YXd{QYp>&WJ z1lVz=r|?O%GDP+*XO7Nca3fY1Ol}r{N2~>Gc~uXwrk?oEG?%V!KSVP_`=P}%J5C~f zlx^`hfPn=su&O7Y>dY@v?A%nED~!~b>MvCjxhR(^B9GBpYc-hF-6ZgzuQSupkEmh7 z7ufjrDEl#XAtW|@2JIB_Ea5L7R2|E$q|?6R*t1ERW&f6Z z7xj{^#v?XyLj;CB`9+zBrRhb6GJW&Zrw1`!G%s%e{r#BBC9UUSi?gBq%I8aAd_*9B z{)Zi&JoF#VU1dQF?kGT@UpbrBG@W!lTxAu7am+znA*gbdzv+sXA}lraq-`prsaG=` z&GJkb7rz{~8EJz`x0e0g7kS)XaSJ}kJqOaUE75A&PyFSnPBNF*!a3&&^d;;F4qQK) zDk`h6S6PQ5jKa8I_Os#P#raGrq7Uxui)OKBHNngG3V+OV6ZyUztV(XqrF^?^k#6Lbvm$r&wPHa2(G_@8?j03^TtqMIF8BO<`O_#NUVA|hmFgm;o zMjGsAtJU;rpFt%q-zim-aJG}{mdmH3iV4urTF?KL3rDz@#8*8tg6h%s5ZU4h*60t4 zbL~0Dx{YF%C+g2qW*M_BDO)Ir8_Z4y`(e+*CJ0_J1Mcm34K9k2u&&LY6*V2W& zhvzR0QA);nzd0DfmOq!gbpyQ~J~O^e)DX+0UtG{n$O0c`FUVOXJz9 z#)r6U@KN5ONL0OGuq*~9U!>Q6#4pL{Um-0&GZ>yas@KT2*AS%ucoQ@L;l9~dB{;mw23;nxQfain85oVk1h&zH2K-n|d3le)!>H->rmJ_D~#Y5Qr5zVI(zR^is&l~i1-2`k0bljOm4 zQ^#W6{5o#6-%ZxC>X6VjGi7^nU6|rUjvWp-i|dyh=CXWKSftRui+2BsTetP1N=8a_l*So?1}h0D95f#l)ySyd_jvB2`D{dEZ!R(&WxkdaOzCLt~gtkIx9qsx7NSJ zczY|zPqf9`T7B%o=n2%897J>c&f%k_^-R6xJb!MT9=x_O1viC2$+CDk>VK|B1TMEO zL>67f){)SWGH4GIT#*i?pNcXgjx> z?p#a7t78seez15Dxft}L@ue|1d|@x%1O@8Yd5r~~6yS&_SEJiCbuJ>pmJ0tY=T=`+ zz|t!hu;|4RrY{6%y4I+`)^G0Y-J$?=zY@hJWOcEYW9M<{7*i&MS((RO7k<5MEW4{U zSwIW^;m+B$@}W6ByiTG3Vjb2fIDP#fM(mV`wMij*-WwmEU+jcSE!~AH!S| z#_fQ^Fpio_D)sgQX6q!>vHCk8V6} z|I6F%)aI@hoE0u-YPj=*F6k={BeyqK*uGm2@!`jFOcIlfTe%w|@oBkas@|eYHKD!i zYP%U}hty!y{0FR9Zx>&CNuKUydr)S-HzM!CM$8@&L|@Np(2-R?uw?RhOp?|km+Xmj zj)h26t3oNED+*T*@S$~={b_RVP+An0K&$dr5?gjz@?8>+k|$YsuE&q`uPRY`Lkx8< zN)yx6^|n;}^$D(B@sOQ*bpwC@v?uQyzxh$y{psSZk2ra40Xk-%pfgVj(JO2N9vG2_ ze6lJ9{O80cFM7=P&pn1Ei~4Y-!X>^@U>HtrSV;;oZzVe_58}T(fq$7{f&1VJ)^!=t zBduCg4CzmC1#?Kx>1}oMJ8?1o-8G%vZ5TmE4>qy*E^1b{GNJjMPq^zEM^`Tn zqSl%Pw4wVU8>z606&w-*Hxc%fFgps@Hw}af9mC<8SnD6gAI+yfIoZ_gV8f34#IrBA zomlQ`Bciy8)X`oprDi5iu!pF!61(#d1SdbBN5gP5gLVbOPYDtjJ_ z8gGWty0=M?Iy!(UMB30{>j2VE`G{XGS+VN3=HQk3nG4)>7~e!|kYv;~6pxd6f`{Fr z`9^Ld3)m!~g6C#5seS{64vWEN+0kG%>k`}Y^f69wx1{DR;beO`0h>JIV3H6az5K(3 z3L|$3Y+Dm%G;1V26&~06;E$NMw36JKJ!tvZbna|v4ohzP!F0|nryZ-~N#|(;t`Bd* zZvu)YLw$mnavBdZ$7jOi4(m`_H!Ta}{r|%S7rVH{Uc&`uMHF4J38VVcHuUNIGr@W; z1=3Dx)KB9uIU8Q5t44N&(M#E>lsT+5lfmOoRi^9f#!RxSvC3P6*8ZLdIhPN!$X(-E z%G(Vj%^A>u!NPQy_M=?N`3p5>}Ys{v?zOT zA6qwInIOaFnt_omEeJ2Q@>cfAc6h`X+0f?YIs-R=hNiI#{C zj&r5d+$cJ-Fpr(gh~pBHi&^LTFj}UvPe}4i#2=Yqc5?&#c$dM_0%FmWRz$zY>NRgf z7Z(0xDS10!U1zE!MfC>f6QxZ}V>Z#)Gm%XBzdAN)!Fhh~X%)hxOj^WcQIqvq^t`DF zV#T}eaCnFY?fMl#0q5^Crl7?Q@`!-<&mN1e#b;vDH7gvXIEdz+muEw=vbnD})ac!o zsq`sQo(dn$gMHqq_^j1~e#y$frRrL?eb+ZGeCa+`|3tV}v>f2{G!N3)uh;Oeq!az= z64pElz>B7;p!BbmD=s`B7EN3kgY|=}aDj^&jW}n@2HYNuqPQEJsev2xc6m^DeHs?5 z)Mrcd{_)O#DrorbaCSK)g?kes4YN;fDfkiUbM$^%ABwcnJOt)uIN!0^D>cfXrl0(e;RU8unokno;-{W>ne5-w*mBU7_B%abdtMyCkc4-rojHJ;SKEWfQ#Zp8scBq<`bs9w z4UvqvZ^6s_nnu^w$#N-TJ1QP}n0413z&+KkxpM6XOeggryS}&?Kc04_)q;j#QR7cJ z^U|<%vl0ZfZe!hR|1hof*04WY8(vC`#O$7cn36Y-XBHRDsM_0&lrw$EKyW*yt6XEN z|Mkbl(&spEgDf3+Hy$iS2Ps>1EH^ZvE>fbu|Xa;;6G zgL)HS^^GyG>&P``@NF=;MflR8h+Q~3u!O73Glh8qfYbX#hs1KD3>}(E{Fs(Ds{eBg z53IN=>Mbf~foopdt=TA_0X3_jjzyInsBfUcDzu*7IEjkU7H_df>F6RD+wX79=^YM1941mJ?^#Uy^% ziW}URP9I48eS-~X?%_ZF$mEWEOlO#92II2oM0VR_*{-C+Xp>@!S7Z%Ax-*6SE2zgx z4`l21OkwlN-tx!d1(JwH9TR((s&P?oyV+2I&XGFSo{8rKu@Bet_|GPJ{IfnY2oo~Q z<FT-#MoA`>#1N@u;3>1R+8oZ zJ732*r)c=O!-tKryum{G47S?$JXdu{k%`=0xOu-;iP?tqL`?YS%9N^N8D(pLwQ#y+ zt$o3|bq0Xn*_Z6Cv6m$8TnkEXn%O$M~_A*pF*tpv!0vXRR6tix&2< zGGkfzv7?yzic}$Kw?6LduM8#^9t*qc2hp0jb5S59uog0BH`l1q;o6zF=&N`Wd_D7s z+tPWDDa~KS>fWhAzjA4?Tb9XI_dH_H-ydWnh8wdI_j)$mX$jYL0l5~B25xK8D3Ja8 zl#Q%i%RI&|=6oiEaxP;6Azvz!*_m$O;}mtU|Hm7=>5$Rz;I1*~ue{8zITH@veu{M{ zH?w=vp0IqM7rqw@*r%SS#oQ|SMuxT-?4Y6%$&JsWyQVYYc9SQ2SsVx%Q*WUC{bJto zNj2t;OMt%YK=8Pgi5Hg-$Cq*CqHEKmAn}DE>ucABqWn~LW$G4o;nhPf((NR+ov-JU zpM62IaTYZCjvwaQjY4L)ig$AOh~-NJopq4`Q=*twn3HV>oCn)PC~?1~0x6lhB1 zlq`;3R4TP#O4yaH!+ZHFZ<47j|!<5k2?NyNd+#pG@=WFJ7qy?F*Tov zqzNm(ljV+86r?q|_Sj7gu00e$n;V<>a`EtMY?V$qf6%OeUKA~E81o;<+((>Ln=#i>Ge@0r+FFSh*NRpw6U+?4Hn^V{f2Vs`2*b%(5jG#ej z5w8%x2R-ci)8~E@==D_}rgkqEE6Tg@zIiOF+E1i$M;ytfb|{%&H>NKQw|L#@F=7Vh zKd^Y^b36+!G+?L^Jp&nRSkr|y`Z1XABPWVm9YsCM%*i)u4UIW(NUIMYrNSxwXlC*W z(RiV$?yAzwZ!8MKv@S98Z0%2myX@GXDtTxYCi~oO9s&1^o$=CC6?VUOBzboarSfJi znr=V$Vk8@fnq$Q0anIj3Oqt!rlo|PN6`C)f?4u%`er+V)P;^n$g5<)w1HR%=m!c zUk$-aw|r^6=^OsJmoG_5#^bLAazz zm)aQ13g17+aIH4nH(((7ZHXkGgZXTwbR+*&q)UGWnv+e8Ed^z2QnsugMQa|#Z_T&a z@uqkZ_47e>FC$u17E9X&FSw^eEZuc<#dwd|wCeXVDmUndp)c2KorfL%eUHwlYBUC%?(}d}dTS7tYgRHQhSiqhcy;%0m9}S|xe9}pKo!8@srtk4!Ss-Wl zN}DV%q~WS8k?T?&U#`^HWHpb}8Gd zUxFcPCcx*{Ui|MGTh1+cEalLC+T1gUc27D?9(v*QPuzA9*V!HB*Cp>4^a~UGJ?aPF z5HyN9J|Csnb+s6?$A^06DUtE1)4Y;nGmDIl;dqr}d{|2v8ay$k&-W!ba9sdRvI=L0 zav^NVh`G2xDTH0T{)Zc5kx6Q+oXPd<5eklVqOJjc6ky_oh0hOS+aUvRdM^#)@PVD2 z=93qwyHHGzZeF6+z5v=R0Cwh2so-hec$n2BU_kK$H-hQnnDMDp;dvFGr@Nxl%VE?M zHO5ZyTR-q>Eys??tN7v9W1Ocmm1;`LFvH*~zb8ka`RPu^feC8ZryPl4ZuR(9xgC#> z97e@HnxwvqiRqc=eXbs4sOYR3zAG~U4_=p~zggkn4_PRkm5q97#n|*p_(6ePsH@f= zH<)ZiXNzArd*37UeB#6wG*zRtty+roiD9u{g|=L zb@~&`9T0|P>!a9cr2$lbWhA8fh%;E}ba|S%<1_0zu1M$h|K>Jr-^MNs5uyF-Cax(? zhb?i6CF=p+xcX!=DlcEitgB9=^_g%yE4qkpH3zZh&qmSO@;<&}iZQ(I-wzzGox_1< zJU(eG!8LYOcu4OOE7p8~-dk*-WzIO>UV8^k#Wo&rKeH8AhQcaw5g+;M30AT&w$efl zOO}qrO5IuXT2+??xBX!)+imDyV;#1e+~qB^&FN!vF=Gqm$>)sFbolfJFDE5YfQkk! zHxEU_vuD_pUSt>NrDCb_XhNK9~<1 zDESJ?Q`X>;@niAbs^i!(C=7pCB|snBm04aE3ej!Hy;O~_yf6m#uXUw=BlnZjv%M(yHy31TN$@T|j>rF# zfS=NLD3@kJTw5F+<~Er z3k*tsP`NE5@Ky05-g3$odp})`+qN}B?ATuD&qE*jmG<{EvN-kps6;Odj~;z^JC)SfY|H?HB3^KN@q% zw3+|-hy9ZAcp|u7WKUm$!J7f7a#Jw4dX_`KpWe7FP7XWR4+sfR#KFqOXdZNOVe2ht%js}ehbgbqzaDEYojBgb4>}3H$C_Zw^p`L*Hx=h<3T?$p<56d-lTeIH#fAWPr14WQ|BD*RXg_QlXoaQ` z@woqskbbvv#bNC|a8iRjW=&ZJbDn!*WZ!Jm72ti6rMX^se7Ot#9yJ{=X!gUkmNPK1 zmpxu~*uqDSW};BkM15HgynPg5-;H!sM=|#A1F$^Q9Lx0l@Ip7?uSOVvx4a>e{9%yq zSqf}%H0OQY;efh1PD^cpIa}hP#u@n|mkg>nLyOgq3*sw0R^VMnRS9nLp9VLF8RLE1 zOQ71l2X?s2ql>l&_Y)G1a@j;AC!Yk0cKWp6zLO9?!HCE29*tXEzR?3JMlI1k{)*a1n6de5bl~gT zCh+ayh+7URKDH4P9GorDH zqga+wI@#rRk~-;nNH zHH6SK66~ul9LUcTD9;fvbm}AeFd>K@z0ynMZjk|!6ANW9Lqi*zW?SM3Cwpx5s3Xtw z>)?3DC0>)aid-q_0p7Z5#^UKw6Hpi2;*@)PZ3;y)grbRGe>I9*=lf^C5v>h;Lp!-_Q`lAcp8{L76 z9-83iGX{MQ2BFQM6S&h)0eep$gyUulWfc_%uADr9X@?KOypMh9v^gV$lG=7SV4{KN z3npTj%op-sOIDJL&jaw=^-|QHtOJd1({SEMb$sx9x})KhV*X}lBONeGsMi_RNFU2z zfcL|q==iPQMR}iQ5L?}Q+J`6?YCE}C2B627S+ILzoha;DGZ@r803~5NcOhFwB?asp?8q;rLqm+w zbEG#_4w8eV>2pv+xFXJ8^h9(*=7l%!nb8K*P|}pT7*vy-1f5(g9r9`k>=t&gY6HUQ z6~B5o`zMZGyB7)h#|Ohhn@sffb%kG%J7J^oXiTUdgY7|w>D&B15IC!li;a}vf%Z?S zdXS3*UeCHf61yx=W6?@ze|%T^`hgxD({p0gx9WjpOxs_o=Ov{ZHuS+i3$v(P%2Q~q zP{UtdAsChEf|WDIVR}+FY>Yh%2k)A&EekyOe4k{p%PK&+*WwL)xR(e9L{L`*f2-1d zya~NCZ_S@Pg#bdi*ps1;3>2#{5)2nkv*E ze3Vx)i=!@ZVAnu!qH>jHEmk>K+-ot_`;$coSmYK)wONUbWGUpk5<*VlsHl|yuJ z-W+zYh%xfGi=NjxP1BWC>B^|j#6<0?v_5bIJ2A+XjhSypWwV91>TWA_Ix|8rJp5Fq z?~h-k+0c*5J$TKNyF7tz7)xhu(4xarLs(~y7@r3QtHMKj@)nPR5w&(#d z1sh;%$Ya=8sUV?C2Z!McnIUfOmrL1iFXrHMk_7MSCA85>AuMncUtEGPXGbNq+;*H9 z{40SO#qGq=Pl<(WQ^%reGYn#<$)oaV)Tm%E8>KAI{(P4wzH%v`J4YVV3@*^Dsf!`j zCJ(km#;}1aH&A~ebUAdcKAZoTvGyO5H2hKEz)ILPnyZ;gLvzZBM6ZTQ51X?W0?(dy zZ>93@hJwjrW%Sd`rroDDIYu@PqwA;F(xxPTnx`LyigQa~x_cFDOEsknv=-Bi1$AWE z4kuQ%%odEcSYX)g4?IFqkjzZ&jpz0~p%JgX@B{1r(Y`wxVV}gS9Eg1=y2*u)`_sIU zdF>s?c3mx_CBoqNadRYwMJi(BnOyArB|?=Qfy_cb1wSZ{!{D4XxGd{9p2{qPz)!{C zabz%h-%w@S;&*VR7EM$-FBsfQ1L&lelm=<3;~r+u(lb?Y@`o6%6~=kOz({;+iB!^X z{~t(ZCV@-&9eS>tkW^D^dd8{|g0{bgVJ3&*iQ6TRt{;wYwm-TEuVqI@!{Qcm9A7pK zJdCG_=sR)sFfV^uZ~;55j1}{e08GVqRM^54~^#MqOWy+g>Ka zp^(i&bt@P0ov%yy!%ea9WUmNEt9W7OV-a2$IU3^y>WRGTF9l^Ki4)));;c~T3WlM=!7r7>QcE=Kh|J~;Mk6%>D)iLV|0!Zghv+}dUz zeAG&iCPX;mK@UR#vR^8l)bIwzmkyz64U%};pfU%m3#8a}RX728S&Sboh3NfDQ{l-* z12s3CgSCOHaoEK)OfEZtX~WYntZfQ#Dm*ied9^YKM6XN`=Pj~Rphb$yv^^%N*svzuJfnawp@2yXu&XbC31Cx<&5 zsQ^4A9?nPUtHtWLRg}YV>?b}`=!hGS?1As??}5SL-LUh|XE4=O!XHhCkl!i9RS&XJ zVv^Sv8{b_6kwBa+%jn<}N_8>OcPr{T$>V{COJMbczkJ~$!kjK0t17YGOp|BzBJW}k zLwS@57Y!Erq4zx@(_$uDXS|P2aa)8JrV;FJH~?91UcvLZ9`xBhH@ac6A|0g@3>y#4 z#v1=$@Fz1{=m9#R)z%Xda%F@%P7AGuvLVq`hpRl%de3Sc=$#66a^Lv|pTTf*UT?O3 zCtwM@I60p)xw9lIr;fKMgMA!QWGP|6U`Iyh)NO_AqBAKZVZUAvHAB2*vKOHoj?BnNTQrCKO6~ zV(-JB;7@NoE*GK1#vHIm2ZPsC;7&V$TWC|M< zd7d1c^OUH4)nVg@8n9r0!C~0H2ZTxg(GQcxu`}0uGvBdB^i-aR_Nha3pH)r6zK*1x z8iV17sv-K;rBjJqQePJJ%7(S)=FtxQiB#=pPj;7`rq$x9G-T*QUR~wIUZ>^KJ`s#M zw3qXk9C=*FZAf=?7ADUQWITZ zbb_2ZSkFKAZ>OV2?FPL!k0EMB45ek8BvjqqpS@q~1m9Cjpmoq0(k1J|a@?Q5#qeeb zUD!Y-^)_OwBVWKxrA?%}2x&=l5%sQUp?0l9z~rX~RLbk&ztkwH=jE2 zWw|I&3SEO{4-(K&Gzpf^^}wxGNsjKai*Va(O64S7VWelZn?LC}h8=6JrDOPay6*gC zm@V;-!p0R($f09$m_2U?$h}W#huKFJQ?ifwZ-;b(VLo;A?n9%dANEgh0 zhbqBYZoeoduMHYqllb)*XLxAui}*8*T3nZ>PgUllL-=qujqN9UcZ9-#Zw646zS40{ zj|^ac6ye0fe426c2E4weiBTnkvB&pNG=I1SO5U%7DVj+_GBAgme9ET#688}ebs5Yt z&V=DdV~MZilNw~T4QBYht6Z?sBh3FGA5>$9%~v-ov&pPAI)x9(9{Uhh$sBY zx^iyJ?~x#3s(r9GLvlnF=iQzUF2@G&Wc%g(wRIpUbibx6BA?L$y}?j>F%jP=-4iW; z7Y8KR9Q=a6Q=i;G_CDYodZVf+mXczkpo`nu@RWi|bX(xp&9V@@o~F7i2h z_tWw3%mHry7U^hevG`%f$eZDtq^6c{>AD4d$Tr=0{zZt9k3OGGN0i*>#s5Y)E*`Up z);72Ey>823w0$oKyQhPTZmZMCWA->KG!mO$?3Ai}bjF)8{u1cg%7}Nd96sGlP;LGu z2&z@Xj8RVXz?f;cVL$`@_{akf-QGp*ZT&?PJvFIf^Nh;m8h`rG&;Z=^DQax(flFje#pliR1MEEG;*L&B9(V+3r?vsXx>er&q{CtesF%8?>c4FdDBYad@fCfQV;Iq0g z|GALvfFF&XNRwA(K($B@zZ?}hykV1YZunPd8=EI|Ln~;(C1JRsZODH1v<46Lmt^yH z74%jtq!R^o%;LTbzbs{h)YkIGIvv>ec^(;?CD`FVj})ywc^B5#4~FZPR!HCeEaZEL zAm3(d;J~vUcyM4({3w|k09~DVa8h;(&R(;{!iXu-e;()Q{>Ae2v_QSdSdf9H=Lh4h zJ}US~)aiJpcM`}u-GDuLEpY!|Fnp7bW>4h5as#8ypjaM{(F&#TN4YNv$u+{S{SZoo z=E$1sgAIS7Vum+eIeI%S zIP#jyZ!Ly#LS^au#N{;e@?#S4XCgmJG%)G&^3|3h!5)X!BlJ+i zm{+Q}{;fNw+%4o#P*MqeCI)OJl2(kF$y zv)sd#1YYfiX{zk_MMsz$W=e`<)Jguq*F?|zJfE~d14r*t!OUMFXk4`wCq<7$)#*)w z5pDwNte3dKFX?&oy*?b97JK8|ssiv{uMLi4UjdmZ;G{EVVBmaCUWc30hM7|GWmF8Q zu@v$g4h2bOwDKn>H6Iv`c#pQrv&ChOAps7 zIqT`TyRZ-qZEHbgrV_QDFQv;QDq8HC_CeVs23sp9 zVlVMWZt3cbwaOs%h-Dv|4A5I&!i|T` zuvAf=6$!R*Pl*-=2x3K}AHDFTpE;fBZBDPg^d-GoD?yTT<~yxxJ58m6G~v)%1v<@D z6(qwH@c0~#zf+E)L%e`jJU^U0)KO=x?c3Q;uU!JqVKQs@Y>(y72jKq2^D$&mApVpI zFm6j*R)4_({S1Vs-Gi}kbCnib>RM0hbFYzF`6$#dG@vJj`NNd6@2S_)B=S(rNCN(i z!6>`Ey=vZ%IqbywM376-#fw{vFhv>*r-ylA<^d_OS z{PiRZEc7NcdmPSGyN_u*yPzoUIp5oTPT5D7K2>Y9GI=KTfa&74R*V|c>S`|@(>A0a}A&dSolCJR!1FNHVQ6~2S z96!I5-Y(7ngPIi(^<_Rgo5`gy+6uU!wFEAGZl$it z!CvE&NUqdK&t^}AXPQ?~65R!tKJBHV>NEU$^Kr1VQozr@KakiZQZ6g}L=DU~!1r51 zy-G`2f}iZJ!)y6Wm@veSxM-h;4T4z1Pw_DQs4)kV126NzQ=+N%h2{L)+A*m8Tp3%shw%*FV5&tZAtLEi83WH!J#mk)2*4*I_zjN5+dIE0v_K->9oI8Rp*c5aWvnrE5< zhH^C1(OLoZt&^y2RuT?h*PnIYmSoU2TUWC4u8`NtI4p`ecMtk?7s0|l0t#pKWpZ() zJUSm+%r*vxvCx;gG|Mg$-^9nE;h#acYEB&N_-?@zZzuBczvRHp@EPn}kVAh7M=Vj- zQsBN%Bvi-d5VMIf?EK4nZ1s5+R&9Bjer!&JV>_Jq(chQJH_2DwP{~V#_iKVMr+)xx zKYx&Z9@~RSj|P+VWAF1*nropY$C=5ytFZKh{;;LSi9NfzpT12{WGj9y!>u2l;rbDs z=<44GKMaau%_B1y7g$jWpC+=EJ%_T7F(=4~8>P%!7+}pBzXes#mSAL>F3Y>Oi4|Df zrIL4+J87I-sC1pO0c>fw33^sBaQ@zEdMwdgx?T4XpKx1=nztgIHSHo_^ODdCuE|24 z*1h?Oxrx9br(z5FCBG8_UO z@e|QiDFfzeUZrQxj$pIh(j~OB-iUSlienDSHSFzZ+@rjteBtqmWK`&_pi|(PT^Q3RvH+qjcfNW4Kx|0pFWA zvG}8wbmpDC;F_FEUkdrqB=u=*)0slLp~8!uFf?IX$4jiy^K1=GO^Tys-fF`Cb_Q6h zs^cTglaRd5lD(f_PoG~3W9lA`&|`TL-R<#?h(iW3JJb8r-|IA0w0B|uz9Gr1xk_~o zWy5BxdPng^C3smU5O122K;>d@n3QHDC z8I$5m-)KwYJ?dja90$r~;Dc?Jbl;x$(h;X-LyuL;@a=nlp18OO-VQuNHG9X?e=BS0 zm7+l?dznI~Wj~>kJ+lu`+wgRhr?;$JmIKgVZn$g@!XPWtE6=t0JP-VKe zfLXZe;WDxvf4Zbd>!#Gu7bhgfDE+YmpFccE1I<*aaoT*03C_ldtOYD`s~xlMcYuzm zabv4HRd{IkTJY95i&pC1`1zL{VvH4Cp=Zr%`}?tX#Xsq{Z@CadI+*=aNA|Q=3mLaY z9e;0kXR;=LcBQ`pi!Y2q2Olo2xV#&NdrziLE<%J?a$_~NJY9#Gg6c(i?R-dkmO)C6 zR!Ch0Z_PE!RqV!a4;H&pO1~7CFs+N5u}EairglZ3>|#3@jhV?-d48v{J)HQUX94IR z7>ExKsmeY~Y(v#M7P$V@Qvvq>2=bn#ved6fnOaOw@#R7x?vQ#=y8OXJ_T>X(lF)u0 z^jF`bFkwS9tq$A?xfMr5rLH38_HQ^lxv!Df1iQoc<-mF^N=4K8FWKu2oy@&YIos(s zmYRP(2!W}G@XKaF69A<+voM8^7`|V6Mo2u5&0Q~273RQ7-qDm4Ibl+h9?4*4OJE4{(!Pv)LY+~AQnm^tM}@u@9Zey1f42p?*EET;MNu@At=x zy<*sb<=I$Mo{Dd;24lsS$F%?SDQsS!ooxB&{VcWrII-X-XS;k9U?DVN$En$@>_-or z@%uKW_tnLgKU%cEdP&v5xxzI+J0Etu9x3eqV{xMGB|$YEPT6Kc5+&o>iO~-QTtqt0xq1+bTIdQD{j-^| z_fGcq_!VYfI!mN6bpV^Reh19oeHE5Q=iwvOBW%^N%V@mD0Z;B*hH?Y5*dd_}_0+i; zVk9^E6S=7V?9tvxmT+?&>bpdNmvACs=;()kqLyN_)elk@kWLMkJ*0YztZ0dL8t!&T zN4*+bF7-BL?g;2@ZU>!nqTrpI_l zn%jyKVLlw)9>aL_9U@1}@JY z#)1+Gg{ago>eXrilCQ(y{YW`B=7fmx$s@#XqVKWS4w~Za!*iK+T@yL;-&;bTDULLl0jo>c&)==_NN9S~&&B zAJIeA)I?nGyM$GH{1usePp4`sW$b%Y0ndKHsO9rmC^GB9#I6D?zwLnQaTIPiWXXP= zRuoJ3*U&za5o37nRxK`GmC05F=b*)|Z*)?j2R6jd1p8a3u=CCj8YdQ@a6w1d(REwU z{gN(gAq}Df-eZ~V*_9Ztm`QFMgt7tN-`J|N&bX!S8!vlm45iD|AkHKLLX~`Q>%#(k zVmBHV#AfiNW5%)JD2xzZf1(01o=OZdL@eM*6>%$>!q#OUgVK;}Hh-xhgc5zaB>xCr zi$r>QeJXwQ%MC6H1IU%#T{QFLQ0CI-ICEN6F@hQdf^hGhozn| zlf{%T#R;nZ*jA}ar}-|yZ7HSrsaGM!dEKL15Bx)N`7L?|YRiV&i}1WhKd{`tg88n| zgfbOlblED8$uksS*@cblZ*o01Q-dV)VFC<&Z(;)1Q;X3&#A67bRT z=d@1gC@hxq&rL)lsT0bJIl z3DN6{sLAL7xOVL`?!W#%E*&RiGdqvKZv81}q2+|uqWeH!szP|mSDLVB0lRHn4nDVZ znaQ>qw%I)sW3^{v{|}3Cs8tr2soeto@0Z}y#qa#y@^y6J`6!5Mo+!ab!W<`tK45nS zYm?*NS=hF-2Tt!RW^eH{yE6F|eX0HrhxHD{Owm5J%l5K#<2W65abpIpd=iUEPv)SP z?*-7`yb%vfvVf1~9jtFh8Y@|}i|L%VVjFDr=HyzIUR9OM>5m%ftDZv;kFauJ%Zq1(s`%ll<{) zG0S-Djr;uU=&~FSd~WrIe%=%b48QPt6OYE?T5_5&c>%08l5aE zq~TaPrwly#Q;(W$(g_-J2OJDr=1o7;>QCvo?!di8QQF{L^^x9IhEX%8h zu;Taf|BCd<7jBKts{P-J_uAC|UL#%Xs+i@~Q&-@cvsh6@t(B(VyCtkajDHv@xRM);)Bnl z#BgG*czklCIP2kT@h8u1;@#=X#GO&|#N!f)xWaR$_@kV^IHqQ*`0S@3@ufi<#K&KS ziB}Y_75BNbMLcl{6JNi%WU{znmz#J+O{Cc2e4JSJY>{{^SuVce87F=e9VNbhg@|!N zm{`|omiX{SQ}Ip9@nYL|JH+K8e{s{11o0{1i~kQb;HB?bdO>(lyO5Q7P{AZ?n1aG| z$>1eH;>}mSVR-6B+4?hwn3wJnv90?qS>#a}GctL>b{MQeQ`L4D)YA%^wpX&80X4*F zWwmrv=>*wSw@<8oh#9W&Hg6@$&HPgw9J zF&nA47_YiIu=+(A82)4iwhq|A5{l);p5cyo?~H>SKF8V=uvXQ6$M=}aN4pM<5axQzMnGsx}k8kkUenuV=( zV{-;Ar+beNfMHo<1+2O!i@!p}=eOMCR|@Un>Z!qK+AxWC3|zoQ9Y2Xyn=dkF`3^jN zTsWu{Ed&R{Rk$_sG%6ok&dRHIGb-oK>RSuhXqDaUtuRSzP4+{Jo_D26?xXR}WQjih z%GrunlQglxSdalVY2lzp_xM`t7$QIG6}=v(C+n0q6z|k@VV!puW7pm&$7tb#c7E(K zyrE<%er+&-O>fD-xg$o=$lPeUc%iDScfa)*SlGf8YIiUN-!3vQ<|tF|7l;Qk_OlKr zGg(YPE{5SRs`BaqKPZ5V$&uHW*dw`-$a_>V{ltH4Ug1*|CtO932TyUe^<7A4n9Jmi z7mzxYGR*HQVRM1|B!UA^n_4TCxkBU2Y=uaCF69 z`+pPf{uMB7z6N{q{RJOu)0aKe#>ay2!ccJ9Pn!_-h?W9C@l`VUxC=QQ0$EJiN zv&+xj&~HdzneCNYRJ|03&a(>m`p!7^r(VWt%l(Syqtfe<{VyKGFFbUIN7YRc) zyoOn0a&bWNCHiMpHrrbHh&@(Pkr{^B$XuVjXVaD3#ctwyoVVj8zS1*d&8>fFx$wN1 z{K{EeFRv!+x>5`Y`^K|z1IlT_q@J?+1}47J>pI5$&ccnM39P?oCfk#(BFn0u&K|T6 z5I>_sWQVRU#ZT)b0&eE|bXM#A70vB8vlZy)L!9bofNk9_Z^{I zYJ%?XGI2mg8uLp31@G&%ap|uD%1a+J-`{DtSot;TTDGyvbJnA)Ss8PEYsM-&>(R#P zBfumXy`F!QDNkIB$`cl%yzrJsoLY;mk_(Eo?b<*3xqmv|xl>Kkgp#S+AvavNG=Ms+ zZ|0HH7qWhXr^3^#TVY0A4ZhgXMMuediFbNdaUYHOvV3a=S;T><;_7HCv0vX=_;Be` zS>$FVeE!r)yoCRu$AU*-bl^&CoTr98!}P^wzFxAj_(0YWQ9<_vo1_2w3N0owQ5Roz zxk4O5Rb-A|3bCI<4#+%K4D^2fA2m~VPobuhlAt70`u}h?stPkD0`VH$zmI1_gL0qgTfn`g_F*_~3O3F4siSCF5gAlQY5ALKBd_4H0N^ zwycjz7BofF(K6W!I&IHN7_Rn_4^ll#2aG=n=8=scX$^zwZCym$wqY_2$=?LWtlCKI z!F%+^(kiOm(m}1i1W=2~rQG=O2KL6eKlM;JO7^ZcX2r+yNYQmyVmGjxu3PNGU4NbX zZ|b?zb>_cZV~dc{QP$I3ET^m(RZjngLnW3(p^{aw!GlSgr%lp)+? z;(BTO@_T&Wfh#;&G#GX^xuH(+2C`vyI@cGH`5$T{$++w8JZ4!Ra=3>FpQ|&KhlfPb zZwqy$C7+VWLNLVxHw?&_p22YR^lQG+^#Fe_z6Es`mvfKxIuf>LxjZ)7?&cpJX_9O! zH7?%NElT!ES?YW!^2v|<^_ai}Fb&j!Nbw{yY%^AeJ`GMcuXeNN;vo5-)s z7bLbs^2xDcK_L|EyG(uNHAwS+rt!OZ2hjRDicd&XLA!r3 zyuW=MTxv_Adb^gAy!&Z%@^Urkx9vHX`l`~(H+y*NT}yOKvIR*GyF*Z-txv~;1^t-2 zQQBX55Z8^^NMmQ-;7f)nLDBawRP~%M^t-!-+)cX12i*^Xu%*`2=;}H`UPZ#Tei`(; z&;a`r(n-@y7m8m0&Y(E~Kc%j+8BkGikrWnNliB}{68o?$$k2}`6+_;UN)uyN*@S`>G+gl zQZ?O?f3}rlW*d6(Gx_&mj)x9gn<%DfM^gE`h8jNmf*f7l+?PLq9b{L`J@7Y8f=!|N zuqERdv2r&NMGx1;MuD?bcaO1HfsdU+Il)~#?7kZiP8MswOZ-a?6cg#Vj|xc zoX;n&)}xt&=Yz@F{`A&JH(2)mJT%7`(^b0+Kuyp-EcdIS69X>ud9#G1*7Oi6dZYw} zCT@@%s>*te*&uo$Yy(zI`yUeN1hss}t2Qn;%PaE^g0Vy~1y&~9pp%0&=oOuHu)D7amZJ3fYd$E+7ztN|geKV@Yc*O~Q;tVp~xrW9LsFOC# z3E@xM#<3^G<6%!xI5|_MMK^zHB&U|V<4b)$@|!X}8lw)>y8n0nO6am{+-oO&PF)ws z`pdNF>`@la>KI_R*mdqv9i*B{3LByiAb5oGFa6XpImOJ%qILW&S*Ej}Xyixe$-EbbdoFn57z zji+gd-6GQN&d!z1}XDCP&hT7q*xU;4~Z5+kY{G?q;C{vI)g6rn{oXF(R&g6Z0JGuM=Q|kC)IR? z;8gura2ER9-9t8crICUmpQ?uD6p0oo_Jp+~ZtxZN3&{Czc2qMqh(54?2_Kgx3MX!7 z7l;(@y(Oz@6qK|ICCnKeWcc6`xb_`+-pWIj6aD=_dqAi#t7zfx$DiXH%f0E+%qO(! zb2Pd7yOH14>P?n|EtG7$M=gcF6A3q`ZdGbLNP7zT_$N|opjJ;{OPthnM5dqux&^f4 zHdF_BK?yxJE@03+kklO64$a|4qW6~yc+vdhL?ODH9Q}|WJroheM~!?a>e+k>My)zR zhEbvXGMPrV6hrP$ce?S>CMX+P!o8^jxz}<^bT?@$3TkNH6}4O(L5E2(<+Nf{an(&5nOY~GO;O8q}igDJF* zo9s>mdm%S`Ea^6xtRNu&I$wgzyUn7XXXj9Pn}>X&-DtY+ToS)H=@lsQLl8U;!1>rY zZo2j{$d=BlN~(>Ap!{28PwzXtF|Uv>ax4aY%P0JlkT?1E!--elbRgF^N(`7oz|ktr znn0R%b_FAc9cwl7ucHR85o-`IL-W|%dR9{E6= z&e_wUw$?PzcQOo?l)R=NsL&A>zvs7d+i1y@2>w$TWaveX=X-vvCDqU4;mNEsbj#vx zaM*jhqs^0byzJHph`AyH{~JAMgvDVfQ8JWXcoYIn9TUjT=12VD`+>A!;~nXi?*_to zmjV4J{zo3^&!UUO1Et4RT6uihGp?F+lFJlsRY~sI$w8jxa`@#i6uJ(Fu=S6d=mEnR zxI4#}Znn&>(g=US9VHt@rg4ho<7CPv$2WnlkaV+nxQ)(R)RX$P+hf0v9-#T5NhH9V z;jVvg9_pV3LvCFWxz?T|(c88`$)+x;)_OTIaC;axA2gRNG&)Rv>TQG+JCwk>iG%Ri zdnfK&Hyuvgp*YvQoi`?A@_WH2NQlmSKBn&kKK#l_I@GKOxDLtV?(?0f!i($V;FDBH zC4} zcrQ%4ZO+v-Y^djmmwcIXBKOqSrMGhYx!Msa%{c#*^eqx9B5%`4?rJU+=cN2doIE&1 z45d+ib0}Mo0!dZm;yCA9Pyb-IqD^ zm_n22=OzzIBUM066hMoDEvfS#!BBdtKTQl>D*7<~G@q4Pv-&zHIUyq>?j~aO|RcH8^eUpZ)&?R3= z&%jbk1quG~*dcJuqiAfbB1Bv}!hg@!rjJc$bKA~O&^1p{uJdo3DCd49-dG zecdmH*S!sw`tRRQ%pR@h3;0p!^KCrN63ruKNqa@jX02fRa=BC|WskH-3CIAo{!*Rz zLg|tX`{|hN6234XO;lX=TCkGmi+adQ?(l_<`*>1lI)50j9X4f5m73*8F%^|=cymUD zE8mt>CFlikk)<4rk>*B18fng|&-g4niezsTArj^FmO;Vw2*RhcL z&0awYs#EDa8&f`WSQb4n-CraxY+%a9>ogWRs`q-uhuTEKz-=mMQ7;c8e?Oz4 z%{5#lE1jGc4Xye+cQZV85;|kE6{z3#p)6G003NI?=8N||fq`ac>GAGi@bmvsbRLde zeQz8`8Ksb2M5K^N(%^HRqm(ZxDv^rzpdn4|K4w%D84V&-DzYl#p65uCQ8X!%mXeb8 zR(|*QCm8qMbDrn@e!Zq>;gsAPWbmps;Ja0eCR!dabEpi5{ zGY+!!2g^au#8HCRop6nH0_q-ZCeEt=n8rIJP``eOO(=d%d`|^3x6{|zp44zb^L-+^ zM2rTV*Lo5S(oQ^z64}$CbMVvlD@0X$N9prfG4S=vG}hQqC+J5XV7jYMLcPjC%%Fpb z)KFa;eqZy#5uLVpY_<`ed*Mi2zsq80<9OKBd00fVJ;(GLM*MHaJcQT#;gF67uG&(I z`k8U)vF9iRp)&Wk0j6X&9`{)L;MM6}?BiH@P?{> zU?VIGyDiza@>a>$X6>7*W&SGa)r`FKITZ6apWAWu5L1+0ev8L~VOEv05HU4xA#8bh{VzA;zT-oMuCF zO$o#->rD!W+?NdhXh15bT_&$2^WgK!3wXJ>FCN`3l`sQCZRp&4Pq@105_w)_LVlgz z$*jCOz;&rInb>O?@6l@l$36epw&VMRn=vbeAXX)DC`ptgI1Xa;)>FLM^qABe<*~?; z93rDM4EA24+*a-p8Trh5{HGJnY?FLET$b2?tm-yQ{&|i?M)oEPkFSE$7&sU6-IejPgle~nyyAy;rVM~~1ttsSGKLG<;1B+t~C-= z_ch2Shx%WDStg2T+L1zZ_+Gs9bO8J(-7ZDz5HmJsV;Dr(+$Y}B4!pcIp`!&`3RJl2;g7``TVeudVXV)+5cb22 zV*X1l&d-J5aFs`;u@BsEK+#PSW`7KAW*s9p5_3tLn@CX`XUV)F6D&f$F*U&flKY;8 zS7PvKl(soW>P>|^3r^wsiXgJHO&+R_Nbr@xUCgl_%M^Yc!x4pp*w;V3xRQwiK6-V9 z1)~+rEm%zKw&nN;ksT|xN=_OXZKf@;1OsJfHoK@#u!|=6* z5In~n7r30ll8_I~b)PPgPOyEzd@VJ2)QuRZ)qBpu@<+h9)e}g1#t9gFPY*J_>%h4w zGl-gxD(%Y82Gv1oaJ%^~9QJGx&R*P$PXiQj-nS}t{ChHP{V$i)}t~1~)zWCxxi>VW{h>ilbd6C|zNWPJi-| zkIX0MqC;@iVqcUr%c4{_2Al^BgXWz_?Wfj#lDw7C#N9i!>Cvx!V7&h|h#y-H7X9nk z+7Sa#Tgx3^jV&gR&K<{j4I+(!Ui!i}@>25Qd9pZi zdn&Z|Y=-o!{qeH&ks%m<3B~?-S|GbC8jDhGF=1^qXm=Fa=j!E<*M+ax)xP2EgarY+ zwHxt$W^ejpd>3w3&LzIH=aF6)Lf~x37Kxj$0*`+2h|LvW-8B_!SdF79UQ;h25dmr-HbcJwn@^vQ7R16hr6f!3?J~$;T)xRVBESWA?I6jt4 zzp@aIKQ!YunNvvq^D1HFN;|&R?vdoB_eo*k{2Aoi=eSb4VfJjS$7{I$;=3?NXO_gl zN(Ci3g%YcGsVH^SB6S%@BhBCKF)An~i;Xkc)JL}kbEi2}mR7O-zdx34z4SrizCj=Ei0mkf@+NY6 zu@Xl6y5aB7+0Ym#C)u^Go*65rli|7X&{UL5>Q{(_mJyGkD8>kLb394r)EKnrRSXH| z4I-+9ezz;!FIkcmHdh=E@g>w_+aNFGPHFsE-@^&577&jxDL&Q zwNf>-7wf)P=amRGS#@Nv=@sBH7EnLBo;6CF+e_OntR^eAXGwNcJ`+sh#Xg+37h6Z1tH#M5?nbicEd- zjF~PQOdRUpp;6~|!JNF8hmkju9_vNxa(uHe6G8@=?_={Cylc$ogSp@)=>=pRYL4q#Fx6y9hPh zM_goQqhy_>6pO#S#q-}=16~u+dkviBS63k`mPmBP@xjmdTDcIiGWz1S)b~vA1@ODO7TXt2f;b+8 zPm@&m;}30EYVU@Rd(XvxJJ;X^Lu3!r!5LcYd4k}Mi@%D!d7Pe@xi ziJ9MRV#X#)6t)5!E6LZt0nlQ|w?lA?nqGpuK7)nRg|(v@*tujc@43I=Af= zWah_7cCXJP^>r6X*N=n5$aPC;doL@pFKrlT#`Zw3=&3C6z(vyW;0+{fa$(DQACTmK z8Ofv~G>OOnB1|Zd4^n+EZt+WyjaU;>P3bYv>Ss zRPY9L>%GYxY!ynreiJNycMC1uO=Oc#xnx1DB5t(RCh=y5OuO+dxm8_4JVqbjX)Tpx z@UQ`-`MV|CwEhcQ_%R+%ICCkxA`C^V-XZvThBFLb6@XQxhNx8^z$!$SG*JYHeFzz&x)W(KTdLFQYK7JI0z5NUd7VS(hb<$YDz*csR^SFGRc>f&d~YX zMKViw8d{7$j!VS4#_Wotpm@p(%Xhs*$J;OLRo_Ixnq}U?>@5@dz#aWbs>lKuK z{w>kl-dXB>Z#VXF8pEU?Lc%~Ns{w!1e`a6CKS8}cr|?s&XbpLNnJpL}InAbhu9f`K|3JQ_j|IJ1z3_&2 zhVXiS0Gl*;4r!S*AETQMz|yszY^g40HyQ`>FprU?2{TH_<-sA?cV(flD^j{tGNk@5 zIqIp$X2%sv+}`Dp9Pdij7THSXMwmg<%4H1^_=C)OM z@{?#5a=gqGQ-VOrb|tjt%p}TATcOwfPay4)^kX0PR+Preh2x?dVNkogj*WRTm|cE3 zmA!Z<%PpI4v1{td_(&*YRbNX`TX6|KTY8**y63JvnFivL`TB6Q9i72gU`3r64($Y7PHl{Ah7=Ec^L3MQHC`&9+Ra#Diwa=SoEFgYoBouh_JuYM`=0oDnzQ!}j0PKw54-m=@lf1kq+P zxbDJ0h|(BA_CB1z3=XY>l*AYaQ@_V{8Q;b`+Io;*nuzBU)4?IufbN}NSPG8^@*Def z!1duqoLW>*o>(~GF7p@U)1NJ1mAaSMd+s20=M|{z2M(Gmzd@f~n_=?P5Exg`#2(fE zWW9zcOIf7Z7+CFg0WsH}JU(kep7>{xpHKFQv_u8?b?7`!+}a!NDI7xD(?*Ux6d#PX$r@_p z{uV86KS#akYjIOj6E4>^rzOW@@xk-aw6XOnE?hW79A4Z9hXb?m?~GwEAbC0tF<47n zswsw;1z=C45k`M=W$3SmUB#W4*6qmkgm8T4^|MrvraVKba=b9e^C~(mP9k~XPuZXe zi{b3@HWr;Wh(FnNfgN5ohD-KOhMEuQIQ;n)Qls&nm6iGPrLMg}J}M9Pof*x~KT2fx z|CBS^o-9%Q+y|DL#!8l&da)l$^6+kXF7B1}6wD!&Jl>iHN1e;Xk-w-4d7=nX7ny;A z>A4~}R_9IYM1uZ=*(rDl-U(?sAMK{<%i^vPqhOHa75gR`!`9$a5+JpKDZP^+e6SYk zRrq4BJchBoHDRlb1#F)YX89!|aV-v}Vj_QuJp49-KSLULkujBpo?O z0{49;#ac5+uBx9DFLa)r@WlKAiI=~~W~c>|B$;STU-ptMw~_}%5m27g=Q289??WZm zQXuZL4QQSkCL*8e(Q5t)^0D|L>h92nm0cc^;J|O7w_RD7nR!oQ9C59rEoTsc+I-1X zgHxm{!I3znpRmtXHnR_1JRJ1O>cPH!4;!>m>L~HH9tbbywXrofdJ|o}?W`gBwZwC3 zk!0xJGHi&>k`!cL5i~z1gYB7xg4MMLr0<0|GBDF$qWSO=B<*`AaRyVqrINwV!o`wR z1;yIZN0o)zl*+ zRz}$O?N|Y+|E}TA`I)3}x({nQcT@1II)J3o3-rXbY3N8>VaN(M)ZVrhS`B8voxL8k zC2$BlJgvcEC&)pOMlMV06uae%&GE12e*3#V!yt9SF$g>7gg^TA!j`9H>~-lg62_Nf z$AS}Ur4z#UBbcEtjA(y4Bsd% zeL2Atdu}miXOVzsl{DG@oo1{hEsBK4&&8>GUn6i6$eK`x|7BUwr$sk0X66>UZe0>g zcaDWYH?5J&uYmaz%cytGK+xDmX=dF#sc3kuWVN=JSi(zNJ|HRu?Uv1f5kK?DR_$f% zW|in$?Ji|EJ1*ed8Ix&H(g>ov+k{##li|^I_wh>57YuHWpa;8*X~Nw7+^CNePoCmH z!-fpvb_162YwH*s-L62(^heO^5d-L@C`%kQ?*pm58itL}!`a8a(us7a@kFvQsSBK& z3~+{NE$-I723p!;y{}D{+ByaC*AE+^=ZhX+8jyc$#d!DJ%VFA7P7qv1LSuRo;`j#@cH(I9s<3%ha%Abg@qBLOci1z#3$xV4PR@z_Bq{T!y(+6)#>ud<0Cp1E@D2zBqP>BP7UUo*ej6}PdmKFE7b=)5VSbao6b$>RaQo4_*~CBp2;7+_=JAg4 zFl{B_TVxi$b!8#dePPMo9)kugtfkVe=8UXaC@pj>R%e$L*=+=MWjFr0S(y_BsydF2k*m-j7yjz7fW76H3-oe zH8AtJD$a0!$_gHAg@O%+;`RL+eEen#&2!4xMcY)oSnCeQ70$9KlQbNy*@7d&=fPOL zDw5n!n##tVH^%;r#%#LMzLFgx3wG_@YN9+e1;3t)!fMk}T)aM&?U*hQ&1oj^c!Bd~s?2`nfj*$~^giZSn39wGD2bVRp6RaTAXBmv{wRHt{GH_n zS4-U4yoYm8?(|7Kk|+`#6<6Re>8v+GhDgTzdNN9+r>L?yF7jB@aF(o{QY{(jZNcY- zWJ2ASJdDWrit!__!P57-IBL~c@hl+W`6vEBy}z#Tcil3$q+rU2Y_q}KNby3N8j4Tm z5;R`f7Y;^M!rdMt>bD^Sm(EDXX9g6nPPIa5=td$Px2cu9D?TbI+CP=Jd(?r> za#=E?xExljYr%D!-b+-D)L`?>U^e(?8nikUk@r1aEPFFbrl=kUkM3d6lO!^R0=;4C z9ZeeQI0{vc%)!=OJD}M1HJe;f$S$<61CJ^jlAvZse!ca<-dm5cpvl#2w#qT)=l51h zOs`y*Y)>yIOD)@>?B-q8Jm5J*P2GWH+lG=QpH5)?!HdjGHx3L9^dhasq_=$BLs}30OYsM{mDdFNEra zuz77?7~5csKfX&@XRP!v**T>b)1R9K%?~rki_U8-c%3cI)?a{=RHE&b+_yn%*AEhS ze;@SH%7e7CQ$QFI&$5@NVR7P8P}?>cH+qM`hV@%9OX)uB-I)ny?hL22UV`^LmW+sq zU@_?>Kp%CJahcP>XP7C*#^>X%i=J%x#ZLB5Dy<|(-3Lot$B54@X~BtUW66K>yCBx* z5E%X0PLy*MVbZEdwAFBcehSyfv!DcA{#*t3cQ0YCcWs!B!e;v?gD#S}PaY7PQ4j4q zZ;gRB#0B4bA?Th-1vO(6^w^{f6IwRH3VU^&TX%vOZ>|@jm(PRmN>7Bqy(D>6`JUK(ISuo6C_ub{I(YSVM6;Yci2l5v1U=t?pSPF7sQXu7$TE>LI=2@5 zSIMH)czG%GpVb$WMmgcgRXxJP35K|IZ8ElXMM8Z}KS=UUg=*K6^mfrWSZFhV`gbH# zo739V&FVTv(dGP2!V@rjA#S;Y|Klkyf8q4aF<`yR4N`k##3bZn&|EDey316lLE~t+ z_SyGt|M(5PNKFOR`PTm~I|Y1mR+@O=)5m z%=zO@Y2(ZyPYe#~==CiiV$g+yg9wb@=wNxA=slMT25A{2EOjW=S zlfs0$7Db_@;609BT?l{syYRmooT=Ba_Y%KZ4A#xL&E^{4khFaLil=o2&?s?{OmJJw zr7Oa7V8i5faN0#3UXGXJ+svB5zbzcXw2wfiuP$7MsIv7 zAlJ9{;K#%1xXWFeU(U^ANA$Je*NooCE>EY&N7|D6{ZGPfx%&`F`_by7hp3Wf0&E$2 zpBJ`i@{p2o{Bce&*sgDpN~S5N!184Um?{0lDqr-lrP%?Hs3}e`jT3QvyaNphU4k#B zsPivR9e7La5wY&;!Fr92!jmu%#y{|Y37)~U$C8mky9QMMXDOb;NAbIxRjK3BGTh=N zM_<~mqOqaB@!PowCUde4;md2ff)>KT;1Y;Vl&<5Rmg#g_QY1H;C`v5R4>d$!wY-`k zFAURz;lGqob)Oke6z9q=-31s{Yez@)i0Rck*4)o?6c3es3Pbu@pw#LqYNeUbp1@+b z8q=HQ32W%_EGLvbui&5?+8Y!?m8hc2R;Y8*q-jqF@{z;GA*wyX8mawI`VpqX>|f8Z zrL7;G{?d;xEBXiY_j2Ct)+%vu2&9Q7H(+N@IW|mcAr-~7r2B*$%_}Q~o8L?Dp3Yo; z(X2mR?Ij4i#mZJs;y2-pPC9&v>&LA&_on_CZ?Sx^3cM@vqi>Q^L3Z9r=AN#CuIGUt z+Yrd4TUH+D+PQL6E%h9X?e^T+P(9Y(hUO-WB0~v+TnB(M?XsA2am|$$KW2+SXV(tTY8Gn z`>A|vg9hD^TMmvE^7whbIEFv(h=aRKspZqvutCR)2# zbAG|!#hP?&N)^7UXqH&NSkzZw{xCb1$(k zpN8^oy9jtP+Z9!VT4BDy0U@lV3@Y+^(*~K-%r3njwztSLc%{eR=Z=J6dV^)Ow~+(e zjKMfXbQ#RIpl8mcqRq_r;1vA}RF53uVNzQuwMAWw9`+bRJ-fkgYX?qkH>3gEOz>b{ z9=YkB%6eWqqV22*iHYP4(K*(NIfo26FP;XH;B>5=-i;cHsipD?HY{vZAFz1sf|D#= z$&@E4P-Lb>?i~4qC*y|mz#0loPR8glQ=O~Kyd`n?+d#(55-Xv(Qh#7ecBA6m)7WV% z2R1cpV5eF>?l8}{cdPZn$|qC!wAxxSt2`N^ycM|*sR!xOm+XAlXS`kAM{M=Wa`Th- zVfyt~aru3*21SpEp>SNb;U8#WUV|s62KQ+kP<`-ZM>TnV$wq zDLJ_0%snzxBNa}a+ypL;F^B^cQP=(>ocBM6=dP);fm0+zF0q7YH1CBEy&mG4cpuEN zO2?Ly1(57`kR?0Ia6+p@GPNHyhE(%kTq)uAM^s=wu=yNlqsGFfcfBjek^%>$4qV)r~t~`%E z=e0!9If%VL~jdxb_Yn{+8n(gU&0wdjb3c-%Re~Tb`5_v!;v|iG=c?x4`9oQ15|mR7g&hNcX#7#a$=h|RU8`4 z#u~?HF(DWbSe|NLzBRkI5QYRYpTg{_qR=mSw@7x0+UWPWBcVzky? z9Pv+I)WB9^xnmamjQmc9)`mdeP9s`Z@`0F{XrbyQUFwiJ84|OXN$$TM23=Pg@Q%U= znzTg*bT7|=!%6a-o~#vK_F4c`&Ksqc!PQ{5=?p|x4xkD)gK6eVXSQ(8XMD1DB)n}H zg;kS7g!G_IY%`ljJ`5T}n(VHVaj&kCH;omLvO|?Nxb3iSteJuTk~2htizan7zgI=7Nc=sE|daK;Gx4-2BADV6utFaN7 za`r70**_y`YaS9dvQ?s^oCJF`R>1%<(Byb9hLyA%fz~e+!s4n~o8m=@d1@hCK2;7A zOr*zQnd56v-!qFm?3hGc=PpOZ-NAT0)e(O>#=)zN5ulQ(&CK^70{i|^aPH3pxT~-p zoa9e2-QS7CXT@jyViQU3Jll*BIRLe@V=zc-IIb$(06ufQ;lrkv0-1FZ(|+jk*5|>v zBukaK1hm6R=}#>&mmqSgFWqjkj$LeUiC?6;G_f<85f1PycyV<-vE}EqKNE) zMI`9=2_~Jwz(wmN4EIez-Lm)KJoXwH{JuvV&G&*?i+X|8>*Zvm^=BddoggI>?#iH( zWfuM$-~~|u=J3|)4{Lb&99HY8ayjWmT>2=UEI*+{@6CwD!Md@qXyYSE*5Ki2vabRr z9n8WfTPEUr_ab3%*C|r{FB-dy$3UMu=fJ;yAAUPj2!|KMkjwpS@pIC8n7T4wRExO6 zA-f}ZwEHT^O_%;;WT!Q5*X|HZA4iJEqA=JxqlWN!)?Ag$OMoGe|Ap3-hViJ2+tn07_ zWI&xX0Hau<1Xx+ANnldA3vEBy0C9Ix@NPjq1_a%JobSebM%_88 z`an_2kC#OAJi~gJI@Fcc{&5#n{@X$O4orl!zRB3#FofS7v5AiRS`FIkcG7F5Pg$0p z5q~{HhbZ-0%yM*x!;8I$z6V|DH&F+S?S4FdO(|-39>9<1H{kB{mpEdb7Q`LiYhSf7 ziFoc1*tG(Cx})Dul3HNN$N5OZ;Br?od|SJcT+R6k&jJ@iRrWeKd-)hk>}-)V9Q(pn zTF2psc`w<5o155$w*O$i-WW1q^eGtlt_&UD_zI2n){uGV5?l6fG@sO_N?&dizktV% zkZO}JFtc?oUTX-&1Tke7@^Bc;NO=V#8?0Ef)^ps@Is`sT-}QrgmOJR8jnCNopna0S zf4OYrzcA3>bsX;eBV7G(Fa}I7guWumb*Ntn{BRlrWK9tsO6>_bTrTHihaa9oC2c z+$O`L(n8SI>^k;ev4Ga7Eau-Qo(Ez0G$_lrp$3V6aKykcJYp0`V}`ZhzNf{g?0c1( zhiG8Y{w6q75Q%%X>5BPp{rC25ljM0_R|{SgO9OSu{dd$-x4^N+VJ`^%&3 z-L=p^Z9Y%=YKKeCJK*c1jWF0W9ACYzlC({}2B$s$;&MxM+CN?gAH-Ur-hbaomU=u~ z3J77N{Axij^gRq*c$M{UEdmHSKy&;2W~G|jS+{j1Zkla|Ni7O+yX`;h(N4o1r}Nem@$mD+2G&x%OnRmBoEm}ytSgA}d68@B`wJ_&=YV3>9l`hOQ#Rm>bQ+WnT!9V>>bT&37zC|L zKx;9pH8kqlObt^O6Gt@OSNDOif)xT)c0v_PlOUXM3MAm_mEaCn#&KIJU|C`kAx{-X3)s`Vv;_voy!hW z;2)3mV0qaLQHGI?^F8KJ=~?X%NLYFjbR%6!q{cGd^6U>Z)xF1iTb;2q+Kk>C=Zp(X z1+?2ah!5I-2(?d#LXMvt9kF-~xBK!KOI0@e|?VOqF&t@g6!k2PfYn9-~8 zag-H(wT(bq)+b3ohY44|bq~s4=VGrRRrsS;1$MVA!o7`Kq3^^oF#6AT92YJXt+}%) zwk{e)^Q@0ZoGnHQrMno7s9Vm-yoC0vfo>wUQ zZxg%ZxEhbV(PysG{$+yW-__*Bq7q3?Y$7}N>KeM~w_e5ktRYh=u_mM88&v2pV9|r3WhU@3H6MV$s!pRHp zV^EQB;F$)U5u62e%?Dw7@dvi#av52C;}w>Et%3nhW`dXhHgpOR={e(80%@cJVe8m1 zdPpu`R4(nME1&h^gX}UXn=VH&>Ks)%@dkC-2$4xUk)=$F;4iNB=kkN%`LrZke(y(b zn$lKIHmIHA*RRKbM8g<seNdA3IEFJVm6;Ev5LT5CKr<;F$(9^RY+&}V^-4!)rQA!35*_8(%JN`QyS<=mf z71lIW`588cnDd8+7GSr|H}+H7ZNZnkssmf6HQY030GXr)T=l^hG@NYBgD;$e3_nrl zKL0Sh9b?3H8oANjgjcAhavu$jM`5w!eq1ACL-+ogOkLF9z>h*B?wu&hYm}#Rivgi{ z)V~7W4RxUYmJU3o(3fhiHKfMA&&armmQb`YnrJOw&7@=1OR-gC`3%{x4$=)4N`ka& zv0bqX1{o{T*Cvm!Y0XA_2-(6!&0=_&3ZT@ewJl~;3Q%PA7hjxA!lsj5c)h0p{(EqntNlf|q&$o_Y|+5h zo#&uXjOc82s3gljo|VK+-U|nE9^t!H7U0f;HM4x_CuoPY!iE#9*hMWx7=Yc~UgbqO!^QV2sw%|g{ z@KD8m6G!n$S7dpq&p5Vdi<794yf02aWKiamgb#OXh4nUWJXk6R(wAnsJfN31*{@lR z-`>`<36loV3g?a7eBBmuVoNiUNy~`hVq==d^?A|qV%XG2jccx($LuJ>&W5J8bW5TuQc+$vkZ-xOuPMFviU) zIb@9ZX;<;A!Cm2Fq)^$%4eEF8WvkK(;O*|`Ovx{nr*?%C{SJn1m1QUq!wJtF1d+;XH@_6g8 zQkwY11y_CgMx|=$yt-Ei{VrOYYfh;OLj$FsP%g5{ezB(j-veavw^+e?d0`pPS2co; zwgwh_?mv3>TngPQXX;R;yPWsabfMjnVyvB#hM&a7)S6c!2J+7ey7fRRftecAYQrWz zeRmdanpOy znEMUGui0{^EDeV_MtksKq9XTeV|BbD(x2Q6MoCj;=MOWKADcA=QG`TZA<|1 zT=InUxz)>IW2hOim3fF&TE}qd*-F-tn+vaEmUEeft(cbMPueseVtckZ?OHaRK3*G4 zUy8{{>FQ$w7H(!d{?{E`BYzTmue0NM(rA48DIQ*0MbP#SyI`!#C|p=Knag&X(=zMT z^!m(^5F31mKWaWh74`Dzzi>OO=vmM7%~pw&g9~5~t4viA9*_a=!tj-^4qboO0WNZfD8HbYxVz4y!V@Qd1z`tClg5GUY0n%SUb`w2z_nDd{Wm-)UG zQmm;P$n~Q;u{~uRKYy=LNE}nhuTD|s3yKX%$JYh)Q{8Id&mWL}CH~aN{t1#t zQ$+f1|9bqupfFg5(6V~;Ye8Gj4xk2Ky=O;P6Xp}1FG#|PR|7PFv}r<9F%}xwbY?|J%6CB%B|8_A1}jPX~i`b z^vH;v+A@|ai(;zA#Z|=eEyHpRb>8d#C^~@vt?#Q%zm1#$Zn3LbkJ%#{+?2^JuB@O7 z?d2Q>)|oiGd8v##;=tL{{T|vz9^h-&{;;<_wSte_Axq6VGmu- zqL%hF%6y$r#&Z~#&Wk*Rqx)*nnw5*FQeZaLM5nZEAuQGk zL-&Ucl$6Etu!vze&2m4_@-m}U@%{LOPvVL$<`zy~noXt-SEqGn&cYe%1GLYoKjenS zRk~YeEnRd&R`k?*QM-Q`P?$c6X~bUxaOdM zdKu`KyCvM=&vXd!OQa8#cVcAucU&^pp0-RI!W)$|=(q*1@ZClm(0(XgM3V*?X(FE6FEv1sZ8LvdS%njJ;%wjDdJLLd?U#9l0v~lF$FIBixTcO7HLGNJrQCp< zj4`G5Yx>hI1=j4UV8`RGoksb*eemQ}9F#u!iof3(p|^q!Ww%qwDoqjBw)Gw=Dl5`h z^_g(2Po%Ih(wn{#5s|kmYCth60^WW&#kIFR16NNKy!S&v%Euh|iQxvfK&StG@@=04 zCv3ip{)e96n>QL%Gc%MgPPmCtg+P6?2T|2`+FYFTpl^2`uCMgwQ@_=+w<8zwZ5GOi zfqHzwXCq$yZ6Hlj+X+E<2n@pW@Y4KC@Mz3&zInqjUSgxbr}$^HaTW9EWz%4Q1Ja{d z^Y($xwnSQtvw&8(q5%}h-c&2-p2pL{EOwW3N1Idlz!5jME%O6(s3)^1O z*0>Lqj?5#bahdd0&3nloxA|;E@m&6*yr1ORS@9xb+=hM@i}=zz3*d!!mgpaA2EHX6 zZx2kz6Cvg_FfM|*NQe9e*O?dCrrEhvrL{mbFKp$K$FXo?&L}=_P72@Vwg8JxmXY;l zA4+>=c99eBXQ9s)TfFq34>yk)FPWvO1W&75;r;DRSUUCzmR~UAr*6K(o1ZQ4E%EhVs}B;gUdu^uK04w9!KwM0^f0N8-2u2ptSe>2%6t%MNO{-(LoCdRf!A68|RgH z$)nd$a8F=y7JeA-v{&SQIB}*fL#it7uzrWckB8Sec4p-M60@npX@0^q?rA)V_gZxv z^vlAqM8ldlg_Oeb-{o+@{}i9pF^2Us0@%dqv`fH z;P0Mv)N@hbEgv-CWRV(A_xLC|yKM*jjk$m}?`0rh;94q3Kih}z_2x@gePS_H!L+eu zKfND*2O~#CVT(g0e!L(;%DNWPS*gb{yH%HG`uv5b+*!|TpHW3 zoEKfs#xY@kpiX@*PL%(F=QU=)u}?Pq>gSvEyzlV;k#rvZRQ_QfmxQQDNK09%XiyaA zzOFPx(WEKutdi1BbwtP}BqO4tAtl?nuS;7ysb5sudnhg1p8I*8KfvpSbMEu~p6mL2 z-fvzJJBrJH7SZXUefhxX8ED?=EZJ<>fXb^_ES0{Bz3zE|42=3=Gu%`PRbo5T#Hl|4%O2#(pdh9B!0L1(K0{5IPOZm1^X_pgrV>$Zlj zoQ=V37{LxnyXfAw09^8X1sHM=zO_#f_C(l9(vy>LRnvll4aczBjDwJVaI+Y+egQ7; z*Pm|`{Da;#r$mPf8K^WI`Q+NIa@^azpR^CZ5!+^bf(9v-GFVmkZ&Mi2<~P`S7QQ-RMh;B`&N!Mum*aKK{^2Bg?=K-gCL zDcVB2W(ztnvW!Nz^u&lviK6gK3$kS)!p&vTR3c#=3cTc#d53*5Tzsd<9>3@@DbDjJ-0lR5H?Eq?3&45m2D!c7nKG4teA@zL@$2;AF`9zR}y z6=OmnH=$WN>j$%jQxMvY`9bYWS{32H-$72nd0uqsp3snuW z*Z11bZ|~{x*TTDshp!}mqwyxFR!T+~-3_O7S%I(W9#i(3j_mx-f(B^DgH=g)E`2-* zE4B34XYnBpm9`osTjJqRfWIYbkhuK3nE$;)~67wBpNfj}##jFphG`#S+u&VwZ zgu6vkVvmvZ-_En}+54|};87L*8dpkttO{V^$SW}QtPyxL3_|msPJ%4zIwjOB#s_|$ zobbRKTd76(_-+$x`sv`HC)V(y=V9UOmO42r9Ov@FhpWUHPr9;~ydNHKA1BtGGUM|Z zEclFlNu~Wu*}*gkOLU$HS7(jI?08@5c%nh*roR^&lTFC;fSX81+639gmz>s9fjcJ7 zr=$A~2CZ1Aw#JoF6n=YYw;}kADI1hiDIOD&O>!plvKhE9k zPq`jvz|J!kifJ^pVm9qPexAhElYDlEirBnHgGba2$AQ64Fx&GldFc-0amRW=w?$2Y zt2}BysIG|S<(i3t$M!1}v^0$!8g%*I_heQ-Ou~BA_E@AGfEVA7!QHQG#Npr02(ibc z0o3<1v^h8l^yX=D)Ne20p+{Ft)|n0`uJquuy*pr7T^hP?um%sW46&xeFU814dny)G zg^69FA-SbWf44#&05i8IYy_|A;i!i&8cEChLi z%}xW}n~{%l$=YChx(2>@oW%RDa$x@@6vNY-Y24rh(c7r6VvSCU_^Y@~Ebu=}$y-}V zcg{*Ka`;XUqc4lTeIw|uWM|6Dl)gg_QHb{qvp9XdJ@?;cj2E89ifXqGvPF|TkLQ#v z!*zeGFcu^6Zj}Ok(rvlohdbI|)fR4%FJ5%fgGblAc(}Bp(V8rklK1!K_#Y$q+Nz8A z%}9^hH>W_#w0-=p{{ht9@`L0b_L6~e9lbr(2;-WRadV3=4?I4Z*4He+9y1T)pSyy_PW)CB1R=y=lWmfy*r`i0$)qpqb zk0jN*7a^A1=ye}mER|2uhWI__=&>mEc6Zs!MrxhME@nZn?4l!gHD7`?N-8+$d<+eX zm_tVvr-^+-H3hxj#WZ$H5ICzCL&4kGsJU?zMo#yT&9EOv`XiQN(x(_0r2kKmt$A2@ zG(h_0pDo7+x2-@vG)uG!xCximcVQFPP?+EwMDlP;#3>_HdH(b1xM*@uRA}CU`R3AW zt|5ial{=uNe;Ih!CyU2td*dd5W#0Z^f*5&qFQsMfuIO~I6I~g7hK{~GFUIB0mzf#t z5H8O=XFo$I0^8tq+~^WP-Q`|fViQ5n_ga!uS(xypQQzFmMn9k2B&_4-I@bnzv~w~$Q{guN8@>f z*H}1y#6?&YvKN--q;i)*`dHbk21=HFf^?_9?kyw;+Q$0Y#Sjt&uc?|wL4XKM^> z=)e|*ae|hrgY*QPMfDX4)LL?prpgbF$GRdFOzrgy{Ac9~$Lr#;RCkn?k^dr zX9~_Ok@P7y6g(aWQ1i6WRIEIiQ!b0rzw{3E%L(D{Sijku``ewK6l~%012e>!3l*^a^*@&1DXE25PZCLQs{*D?lAhlV z>tJG2KFsIV*$`j-wS?$%HlHxx{Hc-7mWXh~=s23PYms@h*{BF%N< z;qs;bWS>)O#BGQ5^kX|+ShJ6PrOw~E z%x7X_pZN&APQWlJur%b|afNw6tD=vXhj%9JqtEk)VpUTV`ud7=I>tpX+;5E|bfj*{ z=H;|9b|CnUo=1KQ>B!q zE>8X{)hUMa!`dYr)=)$vu64x?Yd+KM)ExBPc2u#hOiO$_I7Y(vN5S{DEE(fq;ohEC zFdnwyWo!3+CKg_0;f+UcjlCM=!fwaX#zngbUsYm)^Yi6yIrCyAPWgXQfxY*80X z-S51H-PD`T8n?p)>s|o&k7Im*40YD~kx!O7he~eiN;PllnPbfI`~4PD)5ghY>vve( z-D1I|yG&_wo)&tRC!y`CCc2+k4O5N|LKO-ld5;Qiqmn%8o55H#Gav_zh_WN@~2PxrZS;-6a zo{H~(Jb<~`u;KxHlNSu6Dt(1 zYoZ`^b}&27YUGg@^stlV4DRKZq!=4$P5TUb@O{-Z;nBkr;-V@Q_&B!`RN*K6uVM!fqm(Y1YTVAFg+s5h@t)Na@;cI&OpZUfVV?c@K_-$Bjx ze|#U4s;8YC-FHmG^EJD1@PmxSIAyW-s~7qDQwd^P8d)_}~_!O$^AiI$eWsS56@B;-$;A*hZ^ zhH8Z#XX`6y_?SWp$u?reXIFkO!c;gGoye};yGT#?1WevH5}rSd#j%g(;<>0K zuG)A_#GMV2ce+wIx=Y~h&&w!vsy2T=c3sH2x)*(7&O<`e8gZF?kh|htcx_dszzB46=dDAzF$#G1~lN7s90p_83`rjF#Jl(uPW@m@=t{eTg6m2Rsi# zZfQ2Q-P;d(zg_V3?Fe+ZRv^cqJZqj*E9Zx-L64>W@8tYKjyi0^0grbJ7hN<2uSyNR zQyh*#y@v7r_me?6c>(u6WXmpFG||&sl?z{`fOGjTF>jbc*m~PkC^L?R4?6YKHY*H9 zT(Xl~W@|v@Xc6^uj}jjZ4CDTFlkj8lOpJV1MNWqDa@u;t5b@tA4jmVSZ-(sRZz+S& zsb4;)R6G{nm)3KVrc^-6^oM_6!r1HgP9E8KoI7n9fS;TXfZ46N&}Cp4cI^}mM|Q8E z<-Oj)qCff&vo{&8K&jX=)SLoWuY}v`QWsQrD0z%NNtZ)j2q&D!g-sCAnZc^c!RU*2&MN!hYVAQqf1sF>7ak{G&b` zCpG*>mzT*{KJIWTZ@hOCmT?rdNCQg!r*FZmE*vI1n&Xluvshm2%GyPNR9oQ12lE3k z-RP4L^k64{l*D%wSVQ~t2H?M2^}-^lwt4$h2`pP}iB3&z;%}w5u*!e6FvW5?Y)u|Z zFQS6MB_|0J?}bXU|IgqCjpEkk069nLG{NvwTHH{nN$X}e3;zbKkOg*_hyM=rmFyJ{ zsLF8+@3B55^jXlO`p<8DjB(jdL$umeQ_COYJ zuL*1$^l;p`qoDC~Hq2HZN&~$gQR$`#QN^qb&M))EjH3m_yH-*qay#|Js7K@_-{4r+pYf*dB ze>CY^Fx;$IiwAGG$b5!L^F7BEihI{x@zSgzlv#WXbe4XlAyR2A|FiTnPwnt)d{5}O zB^2tdccPN=MTNW?af3s zVr%+2dmcQyGMzfa_k}(tKj^RohyN|@b6qaYBUg$4;3l6UJo!}u)qe}bBwsz@bhtZg zu84zxlc^YdsT>Yqn|(v8A1-UkfU}cXWs6{E)&m@>dq@*6w^w(AKIYr$$?lQZMSadxykmmAC8w> z^})CEN5zX<_n^Aj1uv|!2b^vwOlojo7e#Mr)71&%Vjt7ivnNP7qnk8msspn{y~Hur z#{A!tr{Zk+>8muq_fui>g(OyAI~3PP6hrQDqK36a)N|$#c=&xddTdGJ-Se-|xAmRj zZp}bWcS(b`$XDQad@{bNUqa#aj|IQ}n-s3yL#fm5Z6xLH=xO5tzPl}#U-Z8M!!;9V zX~Gr?^xFaTU(P}3)=1DuTgIO{%TWF;;4N%i+z5_t_b`9zKiRW}0G^?>NhtQ;$9Yb} zc<2@-YLINZjwOf5PDPJGGo-(`d7Y3LXAheFzA8pWZlj!^ne9Ct-N(^+$d_@+Ko2Kaab79 zDuiXtXA^}Ksf!rQerQ<52nI#_hkH?l?QdN#$)GslW=FREa)+7 z3>Vdo66TDKq381)`N7zF!KASrx0as3@gu^??dx^=(%qO-f=^J9l)Ka#tig}>POx9H zO0JJ*z7`45f;wir7>s(qqsZ~f0owAuO4$DSB+bm(AXYVwXD5S0aO6!J4GlV=PznLO z^Roy3G#Sh_W#Qs0+hRWbF_5=y(8fDPjQ7`gV4zzgtgQ^eZ&MBMxzTEF{48F%-M-BQd%zM&b=fa z-MdDdm6R>+yB>r8uBMQ0+!t|Dfj7)ppAYKlH|XL;9ezGHoQ&>=LjLO%GJUr}ArG2{ zU%x3~o6Uaw)0T&~f0ziBF=wEnQS#pY`A@3-^k#XlTRJpwS$o_#t%kDO8Pn`hDd=)Y z5}2;lf}S7pgti~a)G+*_5NKfrZ{@?m|Jre(*JU;Cp1VNUd#@4~Z7LPBT!TfW72Ys1 zUnI-MHE1;YIlMo7kt$jw2f>A|{8PO|k@ny$CHa})iMxY&bMI@Act-{)CjI1SZO}|E zEhwnoO_u$w&EZiv zwRab3l~*Ce_SNU_d>Hr68c2Kpwg@?LB@w}w|NE6R-z-#I+^E7w9@fE~>@?1N7=e=q^alB}fT_4aS*pYO>Z65h z2yb)I<@P>Fl;1}PTX$LchosuoOyB?q1C`3ouIY{gJNpWV+ zZAkKP1t)A0*ZK|OuKPmxzFRyUTF`|`uUNt8xJKcWUnKMonM03PRMWkHnZnPEMseJh zCU5eX8VRQiBO!OuO}aO(fS)GmbISgi6fjzuv7Hj2{ZzQ|CqY>K+ln_OD)Z_nKd5td z!V67>VnFjL(j3xCTLb){)AfP)wrC(aaKthE2+A6 zhHTZh@X`_L3j02ic=WDol)`%bRNH`0531yj5oM=@on>!2?1ZO&5uA{8LQ&DVMYtS$ zUszjI3fbNJ!qj#uw#9LW`RlD?Jd!$=ZQ6F6iVDuKzS3#ZGPqe1rYfU|8=n00wKK-v z$bj4O$jznS!lh{SB@<9GpCS7-R0oc@1cCb^XW?f0N(qs9QaIUiPPAM6pU`?_5e1$q z2j?CD*$XToKiY_jAJp2p`mCl7FF@fqb1>Dv^A#g2e1-bL^J1?ba#@{IM@7Y&D`IT& zDoV?~LH3VxAS&UIc)8Yv>VmRqhJ~4{M6}kYStX6K$W(oLA)%Cy)(KQXx7R>7Xom0+s*&&E*#) z>3ci2dH_ZEf>^NOsMij41CWhhVbkg z!mIo-c;oLuuy9evqKY5T<8cIDQc1+DmA9z-w{W>4yk`fV+-n@9Iu~==AAR1^eghu) zXoZjWO7LCV68twa3z`R=rThOr;j-bbXlWfqF%ACwpO_=_c(ena|Cx&KkDbLH-!D;! zbm^WjX(zpl3P#7<&2;2iPo8TRLepAj;ndm)9%``yL+VvAW2Xk5K9;~|=gZ4^$gu$& zeY8~UxGoW2OzAj!bzVM?eH0_x{Jv80bw~mgeci-fSN3ydp0nUk@l6(fU^@jXccd=y zzZEfN(J-}a9{+kaf%-1$j(LHxBwpGstjtShuOfH8Im@1nq>RffWoOij?#R=V`?Knz zAkGVjz=dfW&|Kd0AZIQu!NB0BoKcX*Ltd$1|3R+O4r2gc?wW;%dd0Cg`xxdbD-!>K;aXH&LDU ztBwNnAB3-b_wvcycy2NEW8DR&JalIlme1L3&f1EDxb^G-e*EMLX`HzNd*{W*q_&fDGAvu zz2U&dt%&zdu*PkBWV{{_Gq;R|7pnVkxKc;l5I!62Q##_QMecH8xvdf#UNzw=?=tMX z{1cui9>DFL4TO8cjG-hhp4xiP<^%4Tyiudep?u z7>?Tdhd?Px@}&j@!vqaIvEIR5(a%Q>-?yHDZPprGEPD?RS3OV+hDt^DerF*lZWP+} zkoTZ`m1pGUzYv?=Y!*7MzYL8w8g%{9E!n6u3VOB4h9Ay7Zh!VlC)n;|&MD7(q1}gN z;Fy`mn?{Af=2d^CdDQ_5Y*fOv145;dVIG`(*I!5|*+E{DHMxgb3p`CYfrsR}nD9If zT+W>b`Dkes*>*yqnB@yK@>`+U>&!=i)y#Oz$(iuR@(h$pvX0qmT_HkxWly+4MAceKRCS=+_kevbUNeFliXM^Zx7Q*^79O!i*w_{7O2(9Jdv zM{Bm@5Z4Zv5;KyMj&$M=>PEP7=stQs&z}|2Zd@K+a)cM-On!H6GQOSMKtpdTAn!*Q z_p&zQ`O&!;_kzS-n|2CM`~vWL7Y+QgQJ?30OXZjewHy<>iC!A^!0Sz$?a`tj7b8ABg7^ESLwH~@<@m>S^{d;=aTFNYWd3$1^AK;b>qPYH$l%86>z-9%x=HSJt+6Apa_eBk_uG8u`d>~QH614X+{bR0%xLq}mx9Oi zgVbN5u60>j3!i_*P^Vc>_;8~+N?a~7cex4yi+srT$OF85*T*d|Zk7>AJJPz!e zM&n0+gLzM4(B4L$Cwb;@g54+a@e3>1@n9kL?J@~tQ*BuucYiLQdEf=*PQA&cF;g~e zYbWTlw3I^ceiDB^h*Q`Py+AAS8en=%A`Q-s!;=FpQs#1LUy)HDK@|#dUE&hHGGGh0 znB~G<`=8M9n-{KX8V(OHX7bW6hlDko;(76!Vm5L=4g<0cg#D{#z^mp4ahmrlX0stx%;%csGNZFe}}Z!lKAoy^L2 zAJV&SC2YF=57=ripyO}lPiRf67OK@3&;x^9KHYmHwJB+^T4V&y>LBui0Q%QT(PY6Js-L5c$E0YH_6bMGzS|A2$&=0Dd0r~t-Ic?~pKm~Sw{iT?CIy#z z)+-K8ozIS^Eb+SQIPCt~l!pZW5+~#hqMKHG>Fk;o8s+{N6hBO}Iu$1W*oU3Qtj1;Ctk|IIB+P8?gs%quBL6I9w6S?Fj2tozZ9H{QK2ywu?`q*< z!!U_e{=$sTd^6w=S_b&N)r?grPrN=WO8EA#RU9LiwzP4l!9Gi(y$z@Y=V5+4zK;s` zUlXj@@ze`vr2j`nTEXCUb~4zWS_qMkI^cxSg;nEhBvL?7KT;NY(+r=hGXLGH#J~l; zxW{Q-C_knv#~v}Cp-ZU(7L^_24&`fUXy_Q8eRmgUI!pf1)RUy{eHuob(}&Qv6FA&G zjrAXN$INj%V0Mo=^fUREFflQKnkS9rhrahj*B)C@&*L`U-u(ddyWd9jojs{<(JmP2 zRV$=@dWhC(|L{(lw3;vo;->$$2#b>BMKH{*06gE_0AKe*cq(9wG?a;Al_FdERlAmJ z-P5qkuVtA2=QwR&6v6fVo{Nd6`*6C~3&o9=X*~2uJ+AK27rQ0>6r*G(R z{%;wtPz3O5+nwmNSP!!5BWQ>Ooe85f22{kC#bQa`BC=AQg3evX0El6@{rWH}I#*Av zT{hzQ*^9W@u_twVmn5g^Bj%9zK%JvXSCVQ?4TTkHlTT?Vn|62RPZpo525m0KQI6-t zQsXeH)LemOuN=`Q$r5Y6Pk_9TEyOiJ!Y|KQGXAO!X-(T{MMx1URaKDD+FAVGXRx?T z$~0oD#6ma}FEkE_=M{lx#Hka5d1d>1SZ*UP6Ry2JMg?yjSUD~ed4Lw1=!V0k9fNT5 zcN6TQrVBZ}!|3hVGGW;g=|dzUQ*k4`frsWU;Q-vuj{hv-mT5WMHr>ndHWR5nH4)C6 zz7a;u_=I#MlPwZo!S!!Z{KLKl)qYOKEtl$Oy1y&LCFCw#gx;S>CmbA90%Hd6HD@NT3!>|dJV4jx@UEYsClM*xBAEb`<-yEQz zYcNilGnduE!ua$qnb3Yh0H1ZdL3f=h@Js(_Nb#+(ALi}@`o>y(e7Xmx1!m#i+JnsR zo?wc1o6PpUCx(6)fHo?zc<_@Y_4C%l=aP2zSL1qVs4EZQcXvjU_P&*(UCl%~HGVhN z%(BP0x8umXgC*Uuoy&n|Od&DH6kkQ^pw7oc&epYJz4Ow!Funt~4w=Pvmm(mrelVP@ zEWn)c3G9$QhPQ6&gU&0quo$?TwVp)DBC-O>>FjDWKbHw>ubt#>-!@4(+H({l-|>Z} z<_?3ipVxuzdw=ThuFNkip2;rMo&x*s2COt>5Pt4v%5VRK!v3~_qfK{whOX+`Xt8Pz ze{!|voAH(q@8yhhcW>n`pLPj_w(aqp{s$QU-j+iIW$c$syKn1-Df zMMJ@(-}LUySd4F5hpt;%@uT!E_pUBe%>JRmc4Zl|8_zPOyrK{H?sQNwuKO~2T2V{Y zPkUj{*$0FbCr*=W+Dx_@A;7K8){>lI1G~of3&FniH0DhdAMW2Bf=pB!C_cpsLBBwlkvxu^fZpah%TlD?nT&Ph#>`?ek=Y>or$*7jOb6zB`UO9fjwWC;+0Zs zw69}YCm&x#KjPQ&pmv40;%zi5UC_lXy~ne0#V4wnoyDq8$D)n(Q_^uxqmPj%@uGDr zZEl$k2bAV=o@_T|4E-c(T03(=?I}93Yzc1k$|sN1cW^OwEHAvPhE2&nynehl+oh^h zf0G6*eXW4K(>-DMcF#NFG*(Wc ziQX1Cz-V#RH>DR~cpdm(&&7E6dmwe6W{BQf597Y3SV?4BLWR!`fZVMon|tbu-@6}% zfQccTI@^|?*;Mn_1FvL%HAGlcHVB0l9F;N=F#BQyIseP3FJuWBBsa zIoRh?w7B&8YqXmh1$K+JfO?n~ELKJF?&rh8Tt7KaH;O=)8Fl3LL=`ornBtI-8gW|9 zMVM=rjv2mt@MEvrLO^&ij5?!<%ZqEp&65{%<;cS<^S=u}&0h%@#zu<|wYp*USGgN% ztW(3W_Y;JtNu}bif_&UqpDhH2=hH^piFhX57V@5$3Ligh;}EACko>G8kH0q#Jpw1N zYX6fkW6S{1Owb*D?)yG4ZGVTKEXsw9mJ(dFVjs&6Y{0aUalFyxKdPb*nCo;LZ*w2q zJiQavE1Ocyld<$ie!l|;>4fo1_0C))ezqSF|A};bn9i<^fH$R?kYl_8ll+Hrf!$B& zX>n8h^W+PyYmw4l$r*I})L1yS|2dcsn~BAX&+@CD`q0*M0OvcOr3t4ya-DuWOpEZ~ zaYj#tF01lrQpQ?b_($qBD6&!aKG?NhP`vGfRYD4_*=UIqkEepO-yN zz?U9->Ci??*xTm`-1c0`J6^meheILgsQpybgCLVb3#3!%=gzZ~1OiN;pB6=Ov0o zhKZtDodZe5IQ|LU`C@TTRw>J*n{MWiF+4=H2)M|qJ-YIMy8E!}%~iVVsF3sE$59HN z{tYTRH$mQ8z)LIKS@~WDkCOc64Hi-S+3Of2B<@ z*4S*ah>tCO#eo|{USilrGq*YLr4yAf!JrQ}HnEqK1dS+;JU z7a25$&;?FFonBArb}zXY(oTfq@-9x4vvw)fn-AcZhr01xpUYru)@yX`d|jL|?hur1 z3B!%wG{8_P7H6i&;Fs7J^Y_QWknm=xS|7usn@2+O?q--0a-Ax2hT#~8W?{p6FN`>( zgHaX^JZ-i*i*tqxn%P>kbx$I`yIDs?D-H?b86B)Ko{aL)H#^w8Z!qd5tMGGBDcX1cZXtti?gW>8{a-4P~ zi+hzDQhA5f;4sD>ac^(jzq^$*VqMs}Ar#-8I0Qd6YUoYNJlUD-HVCylN_v_Lxm$ia zjyXDlBaIP+HHX-8jy`(EohDypP1KNS@R%jW)h^S|328@{oTW)CZ zs3H}%ELa03UF51@9JLW|-&%$L7C6Z^+ZfWWq^@FQjVgJ}X^#ndmEzBF1P@Q;!rjtb~vhiZ-giRiuli|yOen?gC2Mn^Jl|cdaU#i z79^H%?V6&GNV{A)S~H>L1{lfJw~yPj+yfxq~Nq5re4xcrVC ztCgxs0m(d`A}oNLUjlHppB!x>-wEAlAm?vy$GSIz!FzfNBuzI}_@B4n)jQ6zHhK%l?slcxz z?@O3fHAF}yw+Lez;po9S=eFZZ?LgG6njjueu|{(XDSSWFo^MUQ%pX)Dm`-Z`4xwADjYP)NN6BKp}XXs-#5qN8;6{ z-gr_bm9HQ5qqSF$(V9UAAoFuQ+n3G=SYFZig31W3anPcYNY(gEmvi{>iJEye9c1ZRnj% z>0K*8`@iKlcj;)w62n`3@%tPyUHuh)mSlnL=mhHi$Wm4#r?bMb*d6?zX-BFVag-Jw zDCE)rWwv~@SW0>I<0|DB_D6S2;^i+|6$i_%klDkvyfbzl#l9@SQ9J8EadJNn8t#s1 z-5$WX|E|GAr5ILRvZ4Hy{ZUKt1=3ZXL1=GVuHF|*FG@S}_;yxUt1=S~=uIJxaznZE z^*hjCuZc!Et>t}IlfY>1HmZ^8L1jHW#5#UO?_Z4Jml8Ux*rzAPNqv{|qs|JgQqFzA zY9IFLJrs6DE&!{ADp>OTD|G%@3lpX<02BLeJn`{TJgqVZuCD9Gb?u%AR`)wf!F(OI z+uQ@6u8T+4baM%~ze~=$`_9JG7Ss45-MR%dBBuhib3Tw(=mlZ)Ry|BN9fG~Pb%tulp*2ILGyeP* z)_J-dW&XFMnrblcbE> zDa?j{YJPcOC7vZakZ- z^~8~(BT(_IJ2n=8@pG+p3rl&{JJu`x|q||-w{~0 zA(Lj`mgHVu2D*UbUDU%j8+I<+%&#&0>}t?1hg{XW$l7O{rk#M<*?Iv5LE7?opltle={0A3ra%)1xI^B8jvuU>n$v%fH_i zZ=aG#F5?dJf8RF4!uP`jBemYp!z3Jq))}mM1ab4uqY@EB73Z|xr0Fg|-L}d3!6sFS znQw@F^QK~4!UHHkTUM3NMps6{x@cMyFV*g#IMbNHKb1vt9o z!sLRhf^v_Me14e=s^kcqueqO8^*f;Q9DS*n5y2x&|Km5`m2iF80)D!~lWXO!E#jix zBEL7DFCKf;AGNIl@b|AK+WUH+cyU30zTDW8AFas6*Af;)ck@h~9q!D#{3h$tA$S+BI$zlcv`U|PW&dNMO#ZUWZ_jUtZn`PN(}CS#^nL{qfD0)pH1f&2UGZa zX8`VOHxbs}8_se|yTFRQ$7f;4h!K=C$pSxKjKmS`MxviAoj%XC=CgaJ@VWP&WRI?Q zg}7E7o<5{Z{2SAeHNPwK<(4_PWBqVxD3MD-%WyIHRG74?F{gu14{^JFgPC}z1Ry+ zcM8LwzdBZ@kLraluSy&{PaVO+aHH7gXe|^kZpOn(w`5a}&E?CT9#fRm0tz+rLWN6b zA?Ct#{AxCnCoD-9Y@ZFp{D6tlH02G@+((3?R*Alrj#$~@uF$l`3?FRF0pI98>~Ke$ z{p9`Dh{YY!*?Ex>uDg-KZ4Du!cT0PW+@1m(GCGf*8E3_P{)I95m+`1=B3(VWo7DUC z9)OD|_dfj@UK%{0qk&#{gqE?@V@b63#Ex6#K{&BA3y-=j#?L)B zg5$b1=+PyQ-b+C_lc;IfNnyyFE$iq|xt5r&Hx#NmPk|i+J@B8$Of;T#i#^6wLBQ1v z?3H_qYHhE9+ATF6*cgCM_nYPvsV1`RyoD|5Z? zfD?4qkpKJzFfOxBHq-Yf)t)xywA%ym#O0+F_I?cP+*g9V>__oh)xJNahjI00T?38mRX z>Oo<*m-OVTkCKYQhr^?LQNQ0cH0njRZ1Hk;9Q$$rWKP+Dp_=!I#454PvkT@N=*@oP ze7LS81rOi$gqR zbZlb_$(QZq$3CVUH?$p^-aA0%@_G3BVle#=Q*Rnp)8oDWD=7^WDN|8NQA!aFXRmc4 zWXMdR$UJ5ynL0{?Mo}~?O(;{D(%EYjB2yt`Oqu5)8AJW{=l%ab_&(j&>8jmz&OUpu z^}1j8*e7wYD|P|@4>_{xe<`q)xx}-X|NqEQro^)uyNRSi;erCz6ZqU!kSb zC>VTN39ZArVBL~W;GEQf_pQAuESn`ND{34;#$CWFQp&fq8eZa7m<3g*S_^I$_K@nX zaBxdiW!H@dC_1G~Xx?8dI-W8@)xFiwQFu>>l5A*3*<;w5n!wI>y5M-J^Ek2Yy*eC! z8Z0hxpGPYn_W`-$vuu5*KJ-EBjyUAd7uu0?p46-iSz9#V6OH<0&2^Y zT}Gza& z@Be$vinbsBBu4-5$jBFF|Gh@_=A2fEI=K?JrSddVZZvPZeB{MN@;=6+L^mZvrBkk|cPFYo9yMQ&5SN{&5!XMKCBJ5}O1?fiQhr2l z(Kz{w3V(Ui@GbI^zJBtlD;4quKVszvHO9& z2nmj7H$3S1giqC5BNX-R=+&8DT)c-G_hvZEtj{_&F8)&XW;DB{gN0} z2?GxsVEn!PwDQ;m5aOb#WM_$(AN)t`6FXe&y{sd6ANviB7Mq3C$Ng}2Q$OnMwuC&) zenL&sFOu!+#Kl`C;n?swWaVjxv5LcVX-7Zm?!mCM{G;eST#f$B5?m@C?ieS!|65IS zZFW$ycYk(HnN0WLH3j4@XRrC^X;;Q*Ry)OXb+}9-JvY-q^&_%QEyl2U)EAhyUYA#O zIw1b`xi5-67ApWebkhM1SC$EX^`Q2}0Ub|%wPCj>#e+e$+n41J< z13O{mOfBAU^#~pPz7}3yz6KXVVg&W&cDQqWUka=g$@^}m*f{+k}*ul?|9n}Q9bS=7E?IaE{klb^L$Tn4#= zz+*eVk?Kdx;LbP`>T%{WyIh`uW^X^hUcGG?0KI$nLXRC=v?hW8N(*yK6V=vk_j^ZyPW#wt{8n$2+@l`+Xxpvb73T|JmSSFZP#MnfHG)pH=P?oMuS&)Z@NF_281ll zhx;oezNqg*TG6x@_xdY&y-KOrX>d&!()vH zXxWrL&=e2@lecx{PW!ZZ=B+lkrF0i68s2N;qaA-JcajIJ9JUFQ+-#@nC>NRt4$u{PpS8s^BS8e93wr8Q`@=I`BV$U0%epS$^R?Dz%&u$oB(I2DC z7W0kGQ(>GmwCpL3bQg)Mg`p;LoYT2U+M;fwoy~suqE!hEdQ!soR;OW0!xr3EbxMqR zRRA|))}!5>8gO{l7Fz!ez@=S!;j{)bih6gP{#xA??>9OrAB?{U6YhlYW<@{R)4PD} z>J`txsKo$B_&KshKr0;TG+27mwc_EY|InT%oiSwUOkBIv6;n^Em8LxD$xTVCFml68 zUeMDGX7rmyLwx4IE4xr?PSN7d{Z;XHWiGmI+ltd-^3{&XJnZfb#G4u)^%6K^BvPul8FC{esOoLVYe8hb8dHl|9*TlYzt}s?ul!YmAY_LwHDUY6!9>Lm}3lMd)7Y`&`{_&+P$p0DQ zzmz(1n%@fl-RvjMNtwVa+Hb^@_fEr*hXr`j%~E_Z;{YD)4X8i(Dyr|Aroc{L8zFmj z3TKvgT?OVq`HtCG^tWE;mMs$ymB3*q&~UOdf3pp2uf=**P^ z%A@o2SSDM~if!t2J@Gv~Io}<$i%()A4xiy*oIu&=x1D<$}BXDla>fOXWCl{u+uqYEB6q&wmngRhFXl z>1@`r+)Jyi&w_KJfSzBYsm=CG=y}Q!^X^LGf0w-w7FkPS<1@h~?VYmJG1EnJ%47wU ze~T6##;BKWyxWV@BU|ImoKC3uW}GB)-^saM&G7kpCj zL}tA>vC9H1|BwOA<2q5$Cm@`G)IF|>0P9eU>hj!6{|n|%alhxAg8n9!3}8jO*g z3Z87cR*kQ$n~4@Dd-LRGRb02-9&g;ZA(ost4-@D3qRrxH);5dB8QW56ywVRnGS#Uz zzBjdK@5S`=PgD_Xix?A3@_|4jJss;-UqOF2nXof@hc0WUe}% z19N=AZ$d2XTkVGJY^_)}_lInI*k>5&nlG8Ew{e|GTkuiLmAa3Xl)%sNr2~gU*E!Yllqt9c#{W*NrT0Fnx7 z3Nm@##CKw+AAiAc*hG$Sn1dsa&L_3+2TLxr_M|p9t;t%q6il{h^U=lFVy#su} z*R&2#G|$96OFOY1g+Ttt0jT$H4kx*_LZ>a!LdoJ#Y_BY!)YN$RQhXD4oU(zo30?Wr z6L0VgH^ITbPw>GD(YP(?7`^S;5532jljlb#=-Dlfp8bg8D3|r@qG)|mNH}$Ynj$KN zbCwi&dw zjhlF}?g<{a>okA5S>j^$p%quUcSFmyM`@j9R#6pImLG?fHpMV{k_G>}6$u5KzY1w?`snb?1D1_QBH1&Msb7B> zvooD-8}`xfOV1&zJV7-3C+(>pT@`v&UIg>=op6wbMBVoPNS8yF%kGs1Vz(wE<==p7 z1ncfQx0WJJyDpvL_Paf4y-%Qmb64d{(Va-#J!c#`c8kF#wHxTE;h?ORe5b7|&On`F zG&bz0m+4qLu()^hHKRf{oET^%lev-@p8=Su10&GofT%qm~+1gOQe;)6hZv7M%Zz%9PU1~;P}n<*iKD{|7;&G zPWHPCY932*TEs=ZCfC4L`|t7jMh(utz5~N`RfWI81#r`xjW5ry=Hxf^WLon{IHb7o zm)68f_}v-{UWC1IqSSx#E3gZ9899e4jQwc!=7F>;x&ylE<)NRpE6YcEK|klCV%~)i zYQ5c_9jps*X~ZiUaN-!B_d3PTc4t9OgBDLPGT^W7`qV?vhE3Mz>H3N^VsQ)3KBvts zR~8C^_os8`R|PV~%GzO~;@St8(3k-66QtJQq}#MgN8-nKek6+>{jq)OZ{_UO`yoym zj8yj-z$PizfNrja5%v9W|Ga6!lO1W$oHAP+v2`q9g}&|-p)?%AH4URqPhU-GgsI$zJeS^=UOH_LWkCM8R{+!dJlEnx)e@1g^PAWL`yTnV? z8Qt=ZL(h-7IH+bTyiR-~iy!4L-DC_vo;4gF7ki+4pG&N{atv$|^I(>(FAV9wm2Fnd zK!Zn?=vEqnF{8aW{Qd>-9-9W!>qPpO{E@njc7u&t>o|3v3p{J1kam8DpTZwm1noO~ z0CLlg@Wge|967T$Ds(IG;~A;C^2R3a+QyFN*|%lKuVUKifnjpZkg zGFavKNnx0qC5hk4&`wiZxUe#xE!qOut^cl!H& z2i7XmPQvom?fFxRJ04UG<{`%-gxb<<^qKlinJ4Lny1i z4{6Qa;jrh`Ty~PG>NkC#fVmSMvFDQ^xM#N}cTCx)Tzv1ovPyFxjcUo_zU`*qgDGwC z{46INpX`MFD;6rFgpV-)XEulr8er!f4=!1te%H4O3N}lNQ@mcloyj?0)nWjP1@tfjW%}*R#=tYKZW|;M591gbA z#7mF+!A4yx^iZhnfUC9ksA}#mDd>tJzyB5<5P5_$WBf&7N(Juka))dF4&z_%mf>8j znfO8&Bji^6hP@|rxyQB(bn1u=EV%7laz$UkSwFgyezy^LzsqniFH6JqJ$BKm;4b*% zQ;|?Tdk7q8iQvNN!0{@9XsNl94PG>n!li=>c7JNj4mR2FGpz--c?FY!lo@Any?HU6uNTFIGd1zUWKH@kT@OC) zlKMoVSEK#cJJc^ha&^S@q5)kB@#CNF^18a6yndY-Eww5T{8vceicjupF8ggk)BEp- ze`d`bJ7fT!O#4EXr=#%9lq9I#or|AK>nOtS6?HvU3#Jbiq0;gis7#g?w-IUZQ_~!a z-W1@3&D+FU#UAY3dJs?AHV(Qp9{?+dTAHg{%Z2ijlEqxJ%itv#f)IHM{ao4OABOlXg52b-{FyP?!hJ&5#!tQnrha_c#FU9L#j-Xt*! zvnM@*>;oz=`{_XGW;7U&^XjljQ=@ccmw9yj=3YqYt52>~ z4d798m|NxfVCpY_d^s*_~LiN9r?juQMSr_Kax!OI?B!iX@`W zgwk&$!@HxOWR^cn@VUAL#ckbz-L)H`{)s0~Xk#l34Oox!+g>JvQRSbSN*-@f1H zlWjA==;>8%T>4lnR__G8o$6@X%{*RHvI!n-c+3G6KLzmK!^-0~;b&9=1y%PIPxkeZ zYOqT1;RcB97-^qLueVsSyLx*} z-mZf}=roR9tdO(e(i2$azDLNf*FxQ>epp-;fZx}z$A(ptvFhj=KJqjSCYE1-O?{mC zNsK17_EQR}S6pyThqHW=Z=+87d1$OIqP+tj(&wc?)SSPbTw4lx)(;ha(=HR_%f|4d z?ymg({zIC-c$~7Jy$ZKJHl4qne+K&%pNr{_k-4zrV*!=Lwn#CIZfsSg!{skL>4gi? z`;Mt#WwBX&+hrBq$=;85pYNm;{{osh!4hXID3T^&>B_%5reM-03w+)@k(~}T3WujX zL!T5U!M0&CzPb@gt`<5adh^Us>-2Qoovy+`zifH8WF7aoFOnjnI02qMX%;n-v><3` z0QH`d1@~@sCyPuyewy@-UX8J!HJ=vpiIhKZs{LAqsds5<=qPMaHo^4~etfZK0&W?W z3VtPO{MEXWPQ+dpuFsYVf_IzY-6waYQlM#kJ#84as?x{svK_ES|G2p9d>W=myXhX? zJ1F?=^$TQHd=+~SO2frDU!b$!1eAw%#u4}Q(f?_0UYU`{_vGF2hgu&#Qm(^;Mzn=p z(Qf$n=obi6ci_T(dtslXHs92@9L0*iISQ!$V= zD#uF|7{hVg>S8MXGLx&jC^WgN_j%H)l~nUIm^XI4MAs{hVfC@kw6AleFfA*K95&y^ zWcI?04b$jkU^wU4j$b~v!G2VhlE!qPiR7a55dw=Df?Xb8(Q=WaL8OadKB1`;@!2qsG`@2 zV>^{mzcm)}Aq$$YF;*Niwi50Nj0>R><2Bvs~bz8Vpb%b z(3Pm1E}y_CNK=^AOtM?X$?xJn!# z?nkW!3lyx;TGCK;Td90_yOM(3bxZf@2MOZp_n@VkfpYcstk&zT_+hI%4%pR17dGyd z&8zc}rWe-uMz4jOrRZ+`<`&*EGeQ^|ew06Uy8(ZC?0|>Mip06=twrapOU1S^$HCJm zObogiiUt81O1OW31Ha|t>`ys3ZNB1xkejxRkDfit7V)>FESO5^Rm+KdcA*+An)?>& zrGZ57Y9&lsd0N~mhVTfxA7maM%B|ZSqL-UqO5nO=3HR~E;Z8>AcrKa}CSKuv2OrU5 zLqj|<>?w_FuZt%mgLtr`2QIIiD;cUI$h~(h=*g9s_qPj0Dq>A=ajrWXf9=DMx3s_! zm0s}k?AONLca$I8D`>a_jHI53xr0xQ2dz=&tv=-fjG9x}AOu;D}$4>j@;dt&ysKI+Q zr}Vw><4ZZUeUrjrwO_gSmsYsC(NvI+enW>fKA`QAHn^(fBD)vs^S(E8aoAu@c%Sx5 zb|dX0`Q_Xuh2k!Zo0z8{pU6qLX8t2FYxoP=rKjW-GxxCFWqVriFduyKrDy*&nRIbs zws4}yOfvkpgZiFT!@ECM!NjOq$jKOuqm5MAfdjGC+F|Itu``Z2u#3Glo0KDrw~-1g z!Gv=P)_?qx%(kRcj&`E3{m>24d$Nq%{fU4Z;}n(R#{M}%nN&00MrQ*${uzSTex%T$ z57A6*Jfz{oGuh!JH(^!vP~NrW7G1kPhWt%)#hVw-L+qn?9C&3T?%5K`8EqA^AwG_@ z>P-}uN1CuvXSGr_<4~a>J&i4G=djPr_v9a!joT->1Ckl-u1|+3*_`H{;K+ILLp zxU3NleW=6JRqvED%)QBdvvsL|axe_oYgkI!3sE>OT?xir zPj{Pynz%orbSG)ZoHGZ43?xA#b(AN~fda zcuh-No>inmim%o)5mN%d;@x%{7Il&4{o29JZZpJFwzuiq5?hzDIcAjbdI2ODPr}AP zsm$kUDa{WwrJ^Sl;)F0a~H#4o(2AI)l9#@HTVY+E9o87oZi<%Xb-&%`D zDRxSekH}uUroRq;^H{>mH{Rpx+MC$5XAfLC;|1MzzCaqDF|wk*m-yD{uTWic4*Iq~ zLOIh^xYhe<_@=@Dm)MuUG<#_%B{d4;DkiV1v$-zk8V4zhSnd`@O?$49;ieQJeS;SJ zo$V(ij2fjJX|#<#skdUqQEzjc!7pG$$KNW+c*H(~ zjK-P5m#P;yx+D&FbbCZU?;Xd-7f;h1`vY)g_Dl@rN~%eT$IRTPWZwBIt=(BfFDs5g zOz0{s((uN2TO=2|Q?cmZQJN>uDZqHG(y79j2nSB3#GVj_b zgS1U$f?`+?9y$F6^~%x5KOMu_eBw1cW^_Z?cGDDg=`?`1Yyj9eRe{gmc2M>DkuYdf zAy>?*Lz?uN?c3C%bxs~Cep_M1Y#FvZc_7Zs*^Ltt!zp&uH0*2llHZjp;>g?b3M;N` zKwrNE+rmDvkD`Wf1V)LeR@key&~|)W=-+j%s1kWZfShre1zz4 za|xa#uj4Hd^>F2x4lUhRDSOz>21_SB7N>X8#KM`P_(#g1j|@+xG(nm%W{so;<*MYo zJBy3k2I2>W=}QPrdErtpvkV^i+~rO8l*k`7Ib2tl1x#TT^#O2kmW-w6R+wbH3!iTB zh4p*XasQuD{PW&2>ikz7`ggZv1-WWt@ALh67EKZN6uPqW;y@fZU;tS4)8y%WRWZ}| zhUb zWIAiQ>}_o=J@el~qrSTHr8AYV=I9H$Yp`3H<~AGtjc-$0pSPar-A;@Q4#%Eld0^pn z3ks*)rOv%-m1TX}(NTq49@bj6Vv8A4+_&*MMqkc?aep%4tC0eA!ey-dx|DkESprYG z>f@RFH=w5KJ^kt&##yV%VbI1fzP%y@zi981dQzyHO74uMMH`O^iq(|OG2Z>y-o%%F&$hvaZ8~@*bc6H@ zY=w`mo1v}PPj-B8I5zFsfdl656c1#ouNg9AX_m2j*w()SicpciUGQ}G2K#afdPN@+KaHLZqc8>|AWu~^g zFy<2bWhZlXt00OQT?-5Jd!uTg5pNhjhy$H+$Ytn8&N5ZO^0>3;TfK~y9vFykmY<-_ zuD``K$MRM{qJ@$@q;gRY-ly{+vcxyn7*qBsJ2S|_mZq;YGSr4|F`-IZdj^jer zsqpx@1NKj{;{4(3Abrp~3aT~b%x<~x=;&zL7dn7E>$=Xtf<;v>joHu8^~g6WQ+oqX zo{s13cShj3sdt4CoC%L#Sqs_-5$(C-f~x5xj-lleu4 zH2+`H&g!~E+WBZ-qthKr=~$W(``j4J9$&B1;d@KP(<@Wxl(8YZ`gdo`YMD5$cO4zX zSQ`8~gjQ@{PP6Vv{+Q&klCWYlcMa&tJ{Hn9VWKu`X`1txL)B!kXa^<5cvGdLA$JTe zr=a5Dw0O@TKA--Z;*3>kSdI!`c9=Sj^_D*%>kUP;_|8N6Ui+QujEC}_7JK%5Gl;w1 zV{#LlNN0|wL`kXf%;F;IJg&WXWw#zTY%yV{IX~%t^+5T}M=MP=YjhAgOxM!t!{7e?jDVRgn z9cD_KxV~KGqk|PIlyop&2b}DtFcCv5M6H$DKrYM z`maRK5mA(RyT9!61P?sc^c6;*PND6gX^?gPy%4#r28_aH;N8mw!qXFbgm*R1W#Og4 zXmeSy3rAb(v1Z*Gw2R7zO_wjh_mmaX;e$WTqjcOYWfV7j{se1HpNp$22Ez7i8ON`c z300|o;go8B_9|LKF5yd*5!FLDy2b~i>+gYTz0`cYsx{BOy#=0o)u6RmEdP9zPA3hI z(a2#3L9W-IcHNQl@c9P(Mp4v6)-8oXj%1l%r`V0kDelFl-ad?f|3X+<+!~#(_Qt#4yisrWbYc3_d3bkPGJZYQC?Tn~ z_~7qU>Z+FP!r#(!_aCF9|&7)D}^B0-I>g{<_lP|+|nT@#Z@D>uD4u(Ay z5i~ny0LG-P%X!h6`xDhoTlukK#$>k5lxpd+cF~Kld)d)VQ8^hzsk>aPsr!eJX z5A^zB2m|~=_*8=q9ypbX-pW>Vt*R@2y8024W)0_^VZOBM4vez8VXj=&gH%S zdqCYZ61r_^&DRcZMU5fq7@buBk*{+=cJmQz`G$gCMU1F3{0@XoG(qpJgJdJ@c8YmR zI$_8c6Yg0v4*&Juj+gH1fqU2f?5;6}%SR;Akp=GjF>o+7f4U5_M|5S4m=n-_eqX%) zDT#+Hp2&+$6#uA|LmRqgti>s9n`p0DIP|*QoA%c2qY{mFe71Ra<=vlDlWB}$ zuk`tuH_+L~v#4Qo2Vp_d5PWGOidVcg!>7;?e&y-Qdwx$w+tKS#Cwo6l|1}O4OeN8E zbzAPXI1h^>$~fM&D`@RKKvR>KfYaR}eCMcvGzJQheNY|}emkg_Dk3|!=RwQnW6lZ{ zeDG$6OGA&3V#=%l7gs`ExLwayaK+sy^r^QUoUQ#=l}*}y@A1qu6M zo}{ghW5;Z8c^Oy0al_u9?Q`rr%)gifKI5b5PWS0hkPu2ET-8`!*#a5uuEWdWt#Rj_ z@#y?$j8qhOK|^@^brSxpQNebJt%34A`1tdqsM{-m^Iiu@80QFF)Jc;M{;=Spfx}_Z zkz25Bs6OjBd}jR2X%{&d)!H!`4KC$B(DtjkWH)wzDx#Z*-;ZmyWT`ItRH) z_7mQ4Ne@2U?#`Etb#ZB95{(_CF7NIY#DAlDa?$6B!Y|DY)OEN@>2oz3HWq%u2m3at zIC5tny?8mCvKKq^5VKqkb2tHCHvOgs(iip)bz#|O6+F8o7K)x;0ncIkMXULqklFVT zwtl~ct^`(c#z@IyXrhJz&w|nDUNPMsd6a|ZNX>?aI%BO{7zJK)g0m;TK<@6|{Ao!% z==3}YpMJDw`@PqwcG`Ib)HK%d?zC@^F3k$RNVO-e^XAerqf_8L{50EWIluJ`C zbMf>{`ZatZ_W96{JK4>Y@bo(reKv?M^i75vI~1=e<8LzS&9o{#>K`tcbd|>U+jQAi z%Z@fizLk8-_89deL^xHCtaa=jj!(%D1MKs}p2t-&-&7sL3x?3)eP`&p#tT@|^$tC+ zcBJCk4!Ej*8|lVM6<3a~Ff#a>aC}TI9rilO@4BtxDT&sSmfK!zbk0Df!l#sm7@vTe zQYmvbGYqr_@1gZI*Ky}o9kS|LC|vy#je0xX@Oj~Ry0EGbSg9=HB_0l(lG+KWyqLx~ z`v|5bxs*ArGZsF*4vAH-M86-k;`V_L*sQ~Ovif_?U=x~!bU$&G>=9NQH$t{=} zGYi^`3Sh;#fjJbCuFamE=LnN(Gw{#4AspzQDcz>EgX(p^VO5{UaC}d;@OfAe*<3Tn zS>KYy)pvi1k7pf0Y>X10e)$BqcAtS>U3c&y?LpMBb6-4EBH0{&ZRh3>hry<69=A`) zhPwXMV(5rQdOkCfLYpMaXUBJFQ+8KD=G}vNcf~m|t~i3tA>eW+<1U*Zd|ME^mG&0MtvZ?XG3|t@&v6-Xon3q zTk%%)Ui4#V93F~43DKn|@IqNSchr~!f|VAHd*si9Q|sv014Uajd6)=B1I?xHJD(~`B%NH=ai07$6SiH*#j}s^NoA4Q;)Z=^V5}68-MOYcADbu{S>wat`P6IF zZ%7Q^7}pywZdron3QIb_pa=U6Q-i;~todM#G1uPNBP*7=-tWj$*yl?&Om|<48%sRN zOW~Nrz0D8N=m&kNJT4rvlBJnnZwGi-canDw*ae%EQ!vLq3W9t2<+rGk2F}$R!VG~5mYo=5BTKsXm3DoMRVaDnrxHcn}ssatzlaj!H#zdMT zU92+o%0;adsnsC-m!KH3vW51HQKNAEcX(ZW5GQAvaJhOr(hZm==nor-<|#oK{c<&X z8Eceo-nsz3l}5q3E%sbhW{R688bf;iEn1eaAEz9(gY9P;AUtU%R4mN`XMYikb~K7h zY;CdISQSXmvgUa{he+9}3(V)iu;8-{R-NCZkZkkGJpXYo*o^V?K6E~!dkVDCHe~l} zs(A2XTXDp^bJC762=Dhvz5J%J5IZuqGWiEyc_nIvykS zD_dpXrXC;%*X^R#rhH)AYSdGiDy9Z6pt#oe$tgh}R(stMv|ec8@B<1{wyn$;x?X$_ zebu_7`ok>eE}v)9eJx!|e%e{SW`qnE+n%F#Clbkef*xWae6#o zl62{pa}SByACE|zw{-9wC4i?&YgANAL+J^>v@ub(P8iUt6YqW*jd@jMeQgQMGU+TG*aFbKpRKl;P7d!;O_+KwBmjrhpy@50T65l|V~8~qHl z1-+1W^zv{lCCn)2%wI8*iZ&g}d{5KI#z!$38mY#0&29W1yr-&kGPfxy13iyZyv= zCwG$7jfwOyuLxRAR_BbFNX|P3(Dw^*f~mCPHTK^rtQJC8_G};K<&Hw*>FJ+ZfuB8#ft!VOl2U5w@;IF$3@TblcbZA$EmVeAxM{?Cx z&DaiiudIhr2K6+1(m?96sTHnStb&WAk?_VZDC=piCu5HkAcqD#82AVD>}QArqV|(o z@<8n8U!y*rXQDLp< zXXUL-(vxa3#F6xEnisw`9t!p5iK5OWJ$!N~NKjmv;m7TdCE>=0I`V`hZ4k%naobP} z)O@&#yF|wEO0D(W=Z_7XF0vE$`&EfrLv49Q%TcmE9ZIV2(rH=KF1Y$@4&tNk@Kje1 zkC~mNqyv{Bj?$GvhxQP)=M)q*7+}>PcedOVhLg`e0Jr#qeC}Hbew}tr0shh7T)wRc zKu=#wI1(@Lr?t*t(bN`KA1#Bshvbyo>$Jp+CCCKh;jB8kCojo1rtjNV(t*tT)Xldo zob!|}I+xpEhrBSJ*0L39mX#9}O{2Usw%BXEE_#_-3aRD=urSnuQXO@9+@>#7TAB~1 zZo%A#W@_FdX8GUucFmc4H51IgVOdK zl#c%bzD2*_(r_a_t5ruazNHwfNN!Y`wbS4S&By6(!gYK4Oc#dAVim-CqOmT*|}y^*_Yz^DgtTtc`RbuN$pNxlIR`OvI`CwKyn1qU(w} z_o4Rp-qQH`y|}sICHi1&1dlXXF8?|y`gwCu!78rksLYZ(n(xP>ImX}~!j z-+|3hJO1n22de&+(ZCW!hjnKmd!R8dQpolyJ#)sQP{0z(lLiV)Nf$BZJTD(7J-;8H zhLu&-kRQe3J&%6uQ#nL9dTSFexgLcEw`X&)B-BV6_zIq0ZjlXfk#JCnPjVj9k0$sZ zWJARQ%Dj7r4#Zp*pKr5*bsMWW_|a-%tm95_UN3cd=nfXYUbI5R)?Jgq@u4-Zu5ZOc zfFXviyMESND-OqYOT=Y!u8B@}rHrb*T++lAa`S#|oc$$?Hq5m`r${$G-Lnu5PcIZ( zciGNucD07`O?eQbza4wk`=ILTdwgML9eqxn$x(7!cK3TJx(>;plb?Q&N~aN&L#=SW z{SE~dU00oE+VJXTbdT`qbLkhV%TJ;c&<)F=I#|-w03S?H4NHVpT9j ze|M#$EuX0Wo!)q8gbIb^-K1LYMcAai4brSGC_C#1i#gsJe5bUD0(NG|q9)JeE@4fO z7&l0KbbBFJH;%+9O$+#VoWfsbTBpkwy3By2)%oDIsS9)SPi(&T0fdrJVdB6xq^KN) zLP$9KENz4SnrA?F<#Dj9TgXZJ1F@`nmtd#v%XU(s?2S=sRNEm2HI4+})~)++lx-fo zl_2)?DeKwVG=ajlorDHY4J?g~W-~_*vEfDn7AShplOh`Gf^VM)7@CwUsE*c<8ifbq z*b8fUj7lR%`3EZ3EH=P_%_@?6un_L*AB2~-yF?QQ;;Db9;+85YQ+9BT5L*$%K}Wmd zrJ_T$rf?3KtI27xyQW;}m`&e<4$!<5f3b0GfviopSaEoM2wt^KrEZP2qQb08Fxrio z4XZrca*R)Vsl<63Wk_O_qXpX_Z@?Kkm~SICJ1s?>o08+Emx{d4PBl!DR(^V;WX_jm zUC^)DjemZf!3{0u2=h(Jx$2^LTHS`b_X^eXB;di!;!%5y5*@VH(y{<4S{tp5j@c}g^O&k*(PXDbh$ufXq} zPQ%$7ySU2H98LP!iG$na(();}FvfTrY?!-?N2MOdwWE&0`{F0`WBOJp;;D@#IWJih zzT%TlZK=EIcFa6~gRaL+pi3~m^n1NRgoIY(;o#DfWWT99p1R;FbvbmxDZdkNx7T!j zlm3Tn)daR3IfP7>6+&Odb`;73AoQUXZjuV+$2lor@qiTQpmf0Xq1RzSd=hjp+6`wX zJXflGQPKikLb+-ujF@(k!W)w|C9 z81qMM=R<3jk>%Gpc)j&aq3y8WG{)wQs9n?+YP8SGw62=qGfV+kItfSh?njq?Y2f5| zKU}>i8qW{c6942LgkrG^I~*RxLRvF?m1G(>SDujlS?Gt?Y=U6xkVJg=r8V4gn1G6> zTeYFxvLJqMxDQ{QUVsg|_G3!7H?(KuZZI8v8Xvxw_6pJYg7u|T^sjXW&+4s$pgoEU zz1pJH%@?TZJRhdrjK&U&)`-c{Y{$akF#TGv3Eh7^h9Z*&F!KJ5!D&vgQ8V&NJdK~qrDP15HkfxFc!1PHpBLQ}_U$lRmj#4z6uG$-jbacw*F z+S-@@SgeFITl#VPK@PN6)0nba_23EF;cWV_8{f`pAT~*bg)@7BRz?~&dl})E8Qa8v z3wPl78!Cd6`9ZiCj;Q%|A&)tu&WGm>CxcTnpx)MnZ@9L>U$5KZYsG^RlzC4D=QO^d zUT0I`#h6aY^&{;#@XSwmR~5nTpDsdBwJ*#LNr1{1-Z=YgHoiJ+06{)GdFZlOm(=#< z^sjcI81(+P=xF{#3|DTY{$Yl=QEALs25Z^zeNQpGXA~N*4=*jL&%&0M0c_nj5btc* z$w#-H6ctyt2Vq^-8oV&%Hk7q3Cf^z}Qt2`ad&OvBmG&q)uIxy6k|bNhwI>v2^%YjD zb;TubE%<(70A*xe5_%np#xecw^J_6g*lxH(6fa&B&qPn)b~9Ij!pt8Z&N~e~rW%%p zKP#jc`fWI)C0oiQHVao%+Tfm^4hkMK2HCU2SGwe)%cmzka(SZh66D!WNWXPg+O^vN zJ4?A^*ZDS}J^6}K_kAjD4$>g2?=h^q?k>Hl&!BT-vf-lrYvuFzD@kkfG*}ThlfFJ4 zfM1PP34hOx!>CabKtKK`{hcT67Nt9_poW<|<>3@O|7{r5iHZR@?5`d7iYDmNv08{q zQO8?}2Z0v`q4|Dy`ue;l=XBkP(W5)bixXGF)scQYEw~GpzZ`_dZ-%qaaCQ9lJ&14g zs0VraIf(nWoSZXHlTMDbXDpmTreD%Z>=#+!dnvaPY8?i{C2NQrtKesmj$AKnKYUkw zDd0r0H<@IW2xA6*5=Vq|r?Ixy=<>`FzwcJE`K_(6MG_&uypTwox zcXD312XE|eBN#|q@^{`VK~LrdS3douy9(TP6%j=%Pi2zU8nY|+Ueh9@mQd#btQ;`t-ISh$&Z8nOP-f|kGW(%_iGzj}|{lBKJG?1#U z3loYEnUkS}5~Wmxv)7eqLUT%^C~2Og*-a85p>j1K4I)xPinErY(m+U5nv;^0>eZm} zJNNqY{X5TIXYIT1x%RR5+G{<}1bVT^j(j7G+0Q2zIO~A1{8ZCD{1D9q-umrFzD;p6 zGZF6)GlSi8xJ72!Fk^KpE%$5V8p0Q`9tKJze`h|u+f<11{rk|=_cIu}-y>6<+qnIS zKdtF;k&LVlfMWCr8dWon4h6?y)cZrsZ|op2wwOES^8 z8hjVGAG9KG@VztD*r~0hWF{|G61@m8Bcte*+^PrrS-pD{t8B@G^}SLce25(r>5RbQ zF#-7dM4|IrxJ%nd<$3fQN4s*uafSkS{hH37;f;v2~R#Xb(CGzR9}K zz0weT2ij3c(=qJ0e};)uCc&i&20c%u!IqF_e(+2Qmbf%wQda{o_WXnTKHstTa}$s} z)TGSs-<@5AQ`R2GfE|6m8P<&z5+ZsPvhfQyuxLkQT?A>(&+B;0aVEYt7DEz1% zId`pQm+FSV2j@R{)aWAOn+i7nMt^p(w>^z2aDrX>gwKyO;mOY#^oqqW!}|H`yk&oK z^7)EpHV$Z#YDv+HdeC|!9j1HrEO&NVUtT`2FW9fOrY`dvao(*2`Z4Apc2@|aE6cai z1D~N_+h#)b!`F)GhTlnSuaCg(SI@Jig?r&dzk}?^!yOoWO}KfDz-@D`59zW!yl2xi zw$VcuT0=kJ)x1P<>gz~1hsBYbHozT0=Px{TDlPx24q@YBDbi~uT{hW+qgu?_E2j|h zwQ@q2Gtb$vO`7E8(H&AhjAidUrjeC{xUBe!X(+7u?u;j6)oAQ7Kk&`BVXB)N_&&Rj zQ_65tI@mJ??rFVaEBCqJtGUCd`}1%db<>FD#`c6~L!5*JR&}z>@}_ZvT_`{G9E*GG z#*~VGV`P*QU0o`InvV}rvPW2u@LffYB?IB#Tw+6-LTUQt6ud2-`IZ^>`NGTt+PIJp zOGsUHFD8YggZ7>lK7aWnih8gUn&u)7a!z2a-VEknKM9L|52dl)ZK%#4ZFKW^UGDpc43q|56|v`MCc-dN&gs z$bWA!7q8J3w=Ot^M+PtCR4sQfR2zxg&PTKP*?J{0=K95g5+g1Nodu_se&MY~9Al?9 zrr}$Q1ag_an;It!0v}&R42c%j6D_CVY*`wn2<%}M*91b72c|;>W@T-BJs=K7r&QDI zDZN>rqF{k2FCBxXc*5n@&n)Du6qiebX8=0{J$8RoxoTXhAT)y}gZ&UAWIp4>=4C7uzxJMI3uCw!#F@ zR4P53Kt~dW3sO@8kFA5HylCT=qd0zo2X=hEi`Gr&AmD8*ZdQJUH5N%MVM`z$dhJj5 z(_itMr!2>eWy@)l&l$9}*#wWORgL>%!l*K`1CxYg zgFM~0*yQ7R)ihVIjBBJJekUEf~YRK z2+LcP=)mX@JgB@F+%<+%nWYMtWp%~9_fB#b=XpS}*$uSONoOI?$3n|i1q$>#$2=<3 z;qIJ?0v(74s))wHF5p;ykK3sFGaDO}^DtE9G#~$*gS{$asBTRSD{PFx&4zJw>V2=0 z_WCTU5^vCk=zizGYfucH35mpV+bAZV`ihOa=R^0rmLq?+5ktbFV70*pjNTs3cF&w$ zY`d~A6+PitMGn ztY(-Z+?r@dnz!q)%K0YqI0$s~S9hAwryHBn)rWcA?JjWA7nXwUf?qgRvXU)TS^zm4 z_u|u&1*lT1g>MCsskd|7!0Kc-FkYX?pLf5?W^apUF=1iYnpKN)HC*tS=q&oqZ^wWw zk+A+(H1}jzGBgUjSk3nWVeaA{P`cFvCK(|b`dGqat3JS;{KqDVnwbANBhj`vfoA2F zKn?jreJEr#>cRmnBl@i!$42?blG~oOtfS!*n-z2n{}VXvq;xIesz=C7?*y$~WW~Zf zDsh_l9G-TZ2fL=13DUP;!Qy2kgS$%fcJ6HYy($6MkBDWTl{T=ZYy}8v3gFtTH?rEi zbnF^F0{cyVDd(=Z8g8>Bs$>MMHx@1PvZ*PT;^V9M7r9^>b_YX_RtYvri8IXCH z11lI14_WFBd|xnxQ7Y-E?GO)#KUJ~nbGhLCt6GbesPA$v6^KYE>KfFIYM=z60{Uz7 zQ9eFO}pl>^Z&D} zTD}O`RV!mNapZvRFyg^RymI+FzrV~1*F34=wm56C+%9+6lom~xX_|ySrVoX39eZx7 ze9Dtt3yJEO}3?97nhF$1ujfeN!us)uycqBs&hyEIYlmB&L z=f76)uIr}pf0LWoMayjbkgbJ7zZ6cyxj#PfJ+t+kH!L*319!f&<@>TZam_3qHbxZx z*!_Q7ylsC^Y$h6j9?wTR|b{FB5;TmCfV%UZJC zb1Eci--MO{HOjJnrdGrROJal;@n_Lu(Ol1sBK-x4BDMJ;Lf++KQNPjvk?ndz=!I>lX!4*%qQ0)vL`NgSM04JTi}L3z6IFj%Au7w-DvC?oELtkROXRj@zG!0j zP|@VE!J>%R1d-``Pmx3YYEfc-nCQJuuqb)_e$lf0V9~I%ZlWjZt|ISWl8K_|6TTwD zyIVxja={|)U!z1Z$w4AVeu+riGF%i_v`Az$Zi%SJrA?xK{rp5TYC=UdGn_?m+6R#!5+E#WHn9_40j{EzcZScnfMUckmvI_SFO3J$Wl!frT3V@c|M z=h~b1`Rj`_u=mW5$j;5@D`%bMUJRYXUV93e5am8l>Mtzi@6Vga!k2vIpFNw)(#I)d zpldD8$B(RRXDVt7O622HPHAPc!dcEBZ0F?wM=vql6dO}WN)`0l zd9K*{oeumlI!=xEopJfB{WQKz7gA$QXi$YcC(iwbbod-i8dZZfvm-&T<1qwgU!sR2 zRbb}~J962iPOoCLn7VsDD=(Y}v@3(6dcH>e(@#*SbZ61Q8Zbk{mde9bDY$F`Wvb}l zi=X1DOyQFnOPrWSI%8HqT&fk6uA9w`NH>L&wK;UuE1b*Uu?1F(e3|>3k9cN;xxgX* z3FrQK#ZO;f1fgk9nG@>^b{4OB`(-<6f>~eKv#vWOJ6rNqzY_R~iiR93yoqkcC&{G9 zlnUZY_}v?tcvJZv&~Wy9k%qX>ZQ6Nu6#X>yyhp4vxJnOZjcta9Y*pzxo#~LQ>8`$ zU23a>{;iA-Y{2HE%Yz&Z(Xu5? zyJBweDi4_FTftU(PN0za%h1g_gyYjbk#}Nzz*H9u8E@o3v3VriFqlne z6<5G4elWWorn7<)Ycpl+mJgzrUzAgjlao{EWVamnN736@@rb^&_rT$-+-nRC5Q3x9 zH^$hI`TCl|eVUzQk^CP;H??`;jk1YlIJcRTsQndQmn(NYmc1?;Ow0`>G18W2Eitkd z{d<#-OVs`eldp?`1?(uwC9{)JY+1`Mw#!=IQpK4-0^=1Kq)6;6s#;tY_M{N^sj!F~#pp z1O4(tSav!KN6fv(oEBx#8C=Ac80-~zk*&zYv=BdT4g}?^c3i`-D>y+h1EN*evjv-W z@zLV%kJ!2wVG(i`A#2L%AFUe`fvb~SnPtSp1oi& zI+B{%KbCV}5i+;v(A_LQ3eXk;59doLaJMhu2W`4*;t$JA`mvP#HlSvF2mH3|5yUyJ z(pQaNO#gIWu+=ih4Xai&aZSSqHtItjEmk@Xxp&n?k>zI~#JUy2Ec@cCI!6qF%PNgSuirll3oXjfM;RJ>k8E&~rTjYprETfH1T%zD8| zgyNXH>k`3FqpzTY(#*#j4aKT2>LOjkd$4_54of~!h9<|p;j3q>VcMF17_^#avx@Jq zjj0B7x%msp8wkag!^1%}j&RXc3s6|$L09_Uz#o#`Ol3S#N&RcewJ2nF)r2?y>GVb_ zoH-q8{KZh>=?rNqn;>q#2Il>|#|{?iv2Pz|K@VX8>ovL@|6FjuJ_C&)P$EI2k~dVP zS;cj<9pF|E8pf+!%%<#`Oh_sp&jvPXL*8%$x<2a$6^M63slyev_UB2?F0>lwY)%jg zGt%(YCsXnea|0=ThCRhsT2YV87GRQVYVXMu}LsV*tRDcN&HlLM0} z+$$S5KUAULQwFdpQ-k5yBtyD=Us&B)?nN5Fx|WolaG}+K>GbGnD=Xe|nxFn(J!(H_ z6{d{`9ALJi;D?pOzVydnV z0X5$X{Km+ASm^4*LfY@J#iz6(CG!k>Kf94VbR7qOCz^pueS{e9@5{pKzjhSx>=T68 zsEH2mdkJML&!S$DO3CQ@8Dt}hp=B3V(LU#m%-XQUS?Tc+n4lkm*Q@>DcgA~`-!+tN z{hNx=f8yRA=g)dVEZM*c~S?5IwvFG_AQ<~{a2#e+7|d) zF77Rq@gI3Vo`@o=yFJPmR2b<>W1mhos6*oMvgX{KnFzBN-q)a$09OJHq zgs~~0*K!AMOdN^buc=ebOKbMx-~#8HYwcLj&dY4yazDEIDxMD8Pp4^?MzlR{1L)t` zB)_sz5a`!jNEI*L+-rhJANycOji(tQ^9n{=V=jO zuDPFu@&mvq_aVFfI~{KF&*8`96ws*j0dcqsRySmT_=Qa`P_>@SogYumKcY`!c;|+H$Hy(`pRIvBT{_uGB5XhJ_91KSW;Jm+AARzV&cwLYaY3-av z+Mq`bE|K*5-)>lArX!T1=P*s%?vy^Qf%F2O@q?Qsv|{gOK0eS5x(xP$Sj|IXcFyu7 z7&_@u%1>qLXj(+=L0`~V)13bi5JO{Cd(wjQYLvFSjVwaz5!da)*$?_r+r16stGSh3 z3`nFQ`N?p`QC{@!Su#{zG!?ziQxz4iRHZ?IbMPkj7n2tUk@3VVr&9YaEXvCd((Mb# z>bx6rZeQ@yUh!_U>mN#8jLy>jA~iA@w*pe+AF)IKR*S!H(%KDA$x8){9mih{X)0X#HmMDs4yp{c6nsB;OdanwHUlQ0GGbX@*>SX4q`q3Ycf}|2~BpC@bwY?4Ah}uI~}$;FA0*SS7Ju`Kem0R8l-aeEU$|;GYb30FW?`F zF}WidzD(^YiU8!T6jaD7e-Jy^9Ekn}RV?jZ6#FqMj*cn?L$kU$jGEF&N_}3jUXC{W z*M3@{=)RhQ$G?VIZ;rr~8GDO2t&D|$ZTHyYhH%4nN<=G{X*IPL4eEYV(BJ2d=<;ye`*I z?+pu93=ocYrcnHe49K{ZiMO^nGWYntuxnNuTpJ!+v|@`JsV-L$Jx^LpwT}5}J4q1N!Y8Cd?xW=hx1>Cd#! z%=<_g^Hoov=!56UyZ?KEl~umvY3L&K%8#V0(Ut7Kg)6B`^kSxv)Q!2X$tJ)0^+EyF z4Q`L7D1+x8oHF6-cN?HW-mbQ=w4jivl44khdYaN`_!@Q zveaUw^M|;3c#zcvuVQzLY=nxLK+p_Uh0PatfbaF)R3H8yCMNsAqwzvPQ*nWi&g%gF zv%=x6bvO1YN z_Ex84dr%iRtmiH+S#$hbvH7ofyc|4*6is^5;9ohsdEj4`bKo#siYlcUZhv6Kw<;(t zv4h7>$WB`4z|$qC@bR?G(DR)?{4rH5F;-|HHO05MAmt=Y@Johj_X5Cs~`ieNRA3ZYp?2_oazFb$N?5&5mcXQu&{PWc@lm)5$hw)ZH@yK8^T*U%Q)goBBM#t>Lw(ES&vIcNv$s@&$5g!Zt&#b31ZIRmofJ`K{fu zEySo($_)}VHKiamGjA1H%R0@;uM1<;l$7O-hT7W=AL-~g#D18a{fHqW9qk>543+k1 z)G+$99+dzE&sv zC50y6hgf&Q(LDvYGIX%HWJ6EEE#UHM6G@wiFpS!sN`n;7$c_5bd7q({aNqfbOTTtP zKPjZ6vvoS*87W+qdh4L%iI&jI^H$@CPJ@%SVCc53G_Vuyl)~66m3E!*wiFgGc2nwv zS~>z$E~>oT31>^8r>@5tNv@GFpyN`gop8K|kXhI2OM8(j3&^ZRQmaf>K=#1o4Fo9D6p&fROQB3qK;|1~AoxlN`8z-Np%hAK z>^otfUILU>ol2(m62>R3((E*xmm0cA3o~WC{E$LfRi+HO7z$7tPL-@P6x^iYN2!~> zq{dJ>aB0L*HZ+6Y0+fXtWo`?43vRMlqX69$WidvXp-pO#1sMAp2?l9wu5%b-Bf%hz zXi1@)qAY$T^UyFB4AQN;WR$UBknYVTd!$#gop_D(O11-+4a3$%7=~>7E%OaF5qzcd iOp;+D{GhT)CEPb!Q97AA2kvRAv_Vleaju&xo%$bc&aUwQ diff --git a/onnx4deeploy/models/pytorch_models/sleep_convit/qsleep_convit.py b/onnx4deeploy/models/pytorch_models/sleep_convit/qsleep_convit.py index fac26b4..f906e40 100644 --- a/onnx4deeploy/models/pytorch_models/sleep_convit/qsleep_convit.py +++ b/onnx4deeploy/models/pytorch_models/sleep_convit/qsleep_convit.py @@ -2,12 +2,12 @@ import torch import torch.nn as nn -from einops import rearrange import math from collections import OrderedDict from functools import partial import numpy as np import brevitas.nn as qnn +import torch.nn.functional as F from brevitas.inject.enum import FloatToIntImplType from brevitas.quant.scaled_int import ( Int8ActPerTensorFloat, @@ -19,12 +19,7 @@ from brevitas.quant_tensor import QuantTensor # local imports - -import preprocessing.sleep.hyperparams as cfg - - -from DeepQuant.DeepQuant.ExportBrevitas import exportBrevitas -from models.qlayers import IntGELU +from DeepQuant.ExportBrevitas import exportBrevitas @@ -49,8 +44,8 @@ class Uint8StochasticActPerTensorFloat(Uint8ActPerTensorFloat): "input_quant": Int8ActPerTensorFloat, # "weight_quant": Int8WeightPerChannelFloat, "weight_quant": Int8WeightPerChannelFloat, - "output_quant": None, - "return_quant_tensor": False, + "output_quant": Int8ActPerTensorFloat, + "return_quant_tensor": True, } convAndLinQuantParams = { @@ -108,21 +103,33 @@ class MLPHead(nn.Module): def __init__(self, dim, hidden_dim, dropout_rate=0.0): super(MLPHead, self).__init__() self.ff1 = qnn.QuantLinear(dim, hidden_dim, **convAndLinQuantParams) - self.act = nn.GELU() - self.dropout1 = nn.Dropout(p=dropout_rate) + self.activation = F.gelu + #self.dropout1 = nn.Dropout(p=dropout_rate) self.ff2 = qnn.QuantLinear(hidden_dim, dim, **convAndLinQuantParams) - self.dropout2 = nn.Dropout(p=dropout_rate) + #self.dropout2 = nn.Dropout(p=dropout_rate) def forward(self, x): # input dim = encoder dim = 48 x = self.ff1(x) - x = self.act(x) - x = self.dropout1(x) + x = self.activation(x) + #x = self.dropout1(x) x = self.ff2(x) - x = self.dropout2(x) + #x = self.dropout2(x) return x class Encoder(nn.Module): + """ + Transformer Encoder block with multi-head attention and feedforward network. + + Args: + embed_dim: Embedding dimension + num_heads: Number of attention heads + seq_len: Fixed sequence length + batch_size: Fixed batch size + att_dropout: Attention dropout rate (ignored in deploy version) + mlp_head_hidden_dim: Hidden dimension for MLP head + mlp_head_dropout: MLP dropout rate (ignored in deploy version) + """ def __init__(self, embed_dim, nheads, @@ -135,7 +142,7 @@ def __init__(self, nheads, dropout=att_dropout, batch_first=True, - packed_in_proj=True, + packed_in_proj=False, **mhaQuantParams) self.ln_2 = nn.LayerNorm(embed_dim) self.ff = MLPHead(embed_dim, @@ -165,9 +172,8 @@ def forward(self, x): x = x + _x return x -class QConvBranch(nn.Module): - def __init__(self, in_channels=1, out_channels=16, - kernel_size=25, stride=4, pool_kernel=4): +class ConvStem(nn.Module): + def __init__(self, in_channels=1, out_channels=48, kernel_sizes=(25, 200, 100), stride=4, pool_kernel=4): """ CNN branch for multi-scale feature extraction. @@ -178,103 +184,103 @@ def __init__(self, in_channels=1, out_channels=16, stride (int): Stride for downsampling in convolutional layers. pool_kernel (int): Kernel size for pooling layers. """ - super(QConvBranch, self).__init__() - + super(ConvStem, self).__init__() + branch_out_channels = out_channels // 3 # Branch 1: Kernel size 25 - self.conv1 = qnn.QuantConv1d( + self.branch1 = nn.Sequential( + qnn.QuantConv2d( in_channels=in_channels, - out_channels=out_channels, - kernel_size=kernel_size, - stride=stride, - padding=kernel_size // 2, - bias=False, - **convAndLinQuantParams) - self.relu1 = qnn.QuantReLU(bit_width=8, return_quant_tensor=True) - self.avg_pool = nn.AvgPool1d(kernel_size=pool_kernel, stride=pool_kernel) - - self.conv2 = qnn.QuantConv1d( - in_channels=out_channels, - out_channels=out_channels, - kernel_size=3, - stride=2, - padding=1, - bias=False, - **convAndLinQuantParams) - self.relu2 = qnn.QuantReLU(bit_width=8, return_quant_tensor=True) - - def forward(self, x): - # Input shape: (batch_size, channels, sequence_length) - x = self.conv1(x) - x = self.relu1(x) - x = self.avg_pool(x) - x = self.conv2(x) - x = self.relu2(x) - return x + out_channels=branch_out_channels, + kernel_size=(1, kernel_sizes[2]), # (height, width) + stride=(1, stride), + padding=(0, kernel_sizes[2] // 2), + bias=True, + **convAndLinQuantParamsNoOutputQuant), + qnn.QuantReLU(bit_width=8, return_quant_tensor=True), + nn.MaxPool2d(kernel_size=(1, pool_kernel), stride=(1, pool_kernel)), + qnn.QuantConv2d( + in_channels=branch_out_channels, + out_channels=branch_out_channels, + kernel_size=(1, 3), + stride=(1, 2), + padding=(0, 1), + bias=True, + **convAndLinQuantParamsNoOutputQuant), + qnn.QuantReLU(bit_width=8, return_quant_tensor=True) + ) + + # Branch 2: Kernel size 200 + self.branch2 = nn.Sequential( + qnn.QuantConv2d( + in_channels=in_channels, + out_channels=branch_out_channels, + kernel_size=(1, kernel_sizes[1]), # (height, width) + stride=(1, stride), + padding=(0, kernel_sizes[1] // 2), + bias=True, + **convAndLinQuantParamsNoOutputQuant), + qnn.QuantReLU(bit_width=8, return_quant_tensor=True), + nn.MaxPool2d(kernel_size=(1, pool_kernel), stride=(1, pool_kernel)), + qnn.QuantConv2d( + in_channels=branch_out_channels, + out_channels=branch_out_channels, + kernel_size=(1, 3), + stride=(1, 2), + padding=(0, 1), + bias=True, + **convAndLinQuantParamsNoOutputQuant), + qnn.QuantReLU(bit_width=8, return_quant_tensor=True) + ) + + + # Branch 3: Kernel size 100 + self.branch3 = nn.Sequential( + qnn.QuantConv2d( + in_channels=in_channels, + out_channels=branch_out_channels, + kernel_size=(1, kernel_sizes[2]), # (height, width) + stride=(1, stride), + padding=(0, kernel_sizes[2] // 2), + bias=True, + **convAndLinQuantParamsNoOutputQuant), + qnn.QuantReLU(bit_width=8, return_quant_tensor=True), + nn.MaxPool2d(kernel_size=(1, pool_kernel), stride=(1, pool_kernel)), + qnn.QuantConv2d( + in_channels=branch_out_channels, + out_channels=branch_out_channels, + kernel_size=(1, 3), + stride=(1, 2), + padding=(0, 1), + bias=True, + **convAndLinQuantParamsNoOutputQuant), + qnn.QuantReLU(bit_width=8, return_quant_tensor=True) + ) + + self.cat_rescale = qnn.QuantIdentity(**actQuantParams) -def dump_txt(path, arr, scale=None): - # arr: numpy array - with open(path, "w") as f: - f.write(f"# shape: {arr.shape}\n") - if scale is not None: - f.write(f"# scale: {scale}\n") - flat = arr.reshape(-1) - for v in flat: - f.write(f"{int(v)} ") - f.write("\n") -class ConvStem(nn.Module): - def __init__(self, in_channels=1, out_channels=48, kernel_sizes=(25, 100, 200), stride=4, pool_kernel=4): + def forward(self, x): """ - Multi-Scale Convolutional Stem for feature extraction and downsampling. + Forward pass through dual-branch convolutional stem. Args: - in_channels (int): Number of input channels. - out_channels (int): Total number of output channels across all branches. - kernel_sizes (tuple): Kernel sizes for the three branches. - stride (int): Stride for downsampling in convolutional layers. - pool_kernel (int): Kernel size for pooling layers. - """ - super(ConvStem, self).__init__() - # Divide the total output channels equally across the branches - branch_out_channels = out_channels // 3 - - # Branch 1: Kernel size 25 - self.branch1 = QConvBranch(in_channels=in_channels, - out_channels=branch_out_channels, - kernel_size=kernel_sizes[0], - stride=stride, - pool_kernel=pool_kernel) - - self.branch2 = QConvBranch(in_channels=in_channels, - out_channels=branch_out_channels, - kernel_size=kernel_sizes[1], - stride=stride, - pool_kernel=pool_kernel) - - self.branch3 = QConvBranch(in_channels=in_channels, - out_channels=branch_out_channels, - kernel_size=kernel_sizes[2], - stride=stride, - pool_kernel=pool_kernel) - - self.cat_rescale = qnn.QuantIdentity(**actQuantParams) + x: Input tensor of shape (batch_size, channels, height, width) + Expected: (B, 1, 1, 3000) - def forward(self, x): - # Input shape: (batch_size, channels, sequence_length) - x1 = self.branch1(x) # Output from branch 1 - x2 = self.branch2(x) # Output from branch 2 - x3 = self.branch3(x) # Output from branch 3 + Returns: + Concatenated features from both branches (B, model_dim, 1, num_patches) + """ + x1 = self.branch1(x) + x2 = self.branch2(x) + x3 = self.branch3(x) + x1 = self.cat_rescale(x1) x2 = self.cat_rescale(x2) x3 = self.cat_rescale(x3) - - x = torch.cat((x1, x2, x3), dim=1) - # x = torch.cat((x1.tensor, x2.tensor, x3.tensor), dim=1) - # x = self.cat_rescale(x) - # scaling factors are different for each branch - # How does quantized contatenation work in brevitas? - return x + x12 = torch.cat((x1, x2), dim=1) + x123 = torch.cat((x12, x3), dim=1) + return x123 class QSleepConViT(nn.Module): """Vision Transformer @@ -301,13 +307,16 @@ def __init__( # drop_path_rate=0.0, # norm_layer=None): super().__init__() - self.num_classes = config["num_classes"] - self.model_dim = config["model_dim"] - self.inputQuant = qnn.QuantIdentity(**actQuantParams) + self.num_heads = config.get("num_heads", 8) + self.model_dim = config.get("model_dim", 48) + self.num_patches = config.get("num_patches", 94) + self.num_classes = config.get("num_classes", 4) + self.batch_size = config.get("batch_size", 1) + seq_len = config.get("seq_len", 95) # num_patches + 1 (for CLS token) self.conv_stem = ConvStem(in_channels=1, out_channels=self.model_dim, - kernel_sizes=(25, 100, 200), + kernel_sizes=(25, 200, 100), stride=4) num_patches = config["num_patches"] @@ -315,6 +324,10 @@ def __init__( self.cls_token = nn.Parameter(torch.zeros(1, 1, self.model_dim)) self.pos_embed = nn.Parameter( torch.zeros(1, num_patches + 1, self.model_dim)) + + # CLS token selector (fixed one-hot vector for ONNX-friendly extraction) + self.cls_selector = nn.Parameter(torch.zeros(1, self.num_patches + 1), requires_grad=False) + self.cls_selector.data[0, 0] = 1.0 # Select only the first token (CLS) self.pos_drop = nn.Dropout(p=config["attention_dropout"]) @@ -331,15 +344,12 @@ def __init__( self.head = qnn.QuantLinear(self.model_dim, self.num_classes, **convAndLinQuantParams) - def forward_features(self, x): - B = x.shape[0] - x = x.permute(0, 2, 1) # Transpose to B x C x N for Conv stem + def forward(self, x): x = self.conv_stem(x) - x = x.permute(0, 2, 1) # transpose back to B x N x C - cls_tokens = self.cls_token.expand( - B, -1, -1 - ) # stole cls_tokens impl from Phil Wang, thanks + x = x.reshape(self.batch_size, self.model_dim, self.num_patches).permute(0, 2, 1) + + cls_tokens = self.cls_token.expand(self.batch_size, -1, -1) cls_tokens = self.qaddpos(cls_tokens) x = self.qaddpos(x) x_cls = torch.cat((cls_tokens, x), dim=1) @@ -347,12 +357,11 @@ def forward_features(self, x): x_pos = x_cls + pos x = self.encoder(x_pos) x = self.norm(x) - x = x[:, 0, :] # Select the class token + x = torch.matmul(self.cls_selector, x) + x = x.squeeze(1) # [B, 1, 48] -> [B, 48] x = self.rescale_norm(x) - return x - - def forward(self, x): - x = self.inputQuant(x) - x = self.forward_features(x) + print(F"hello") x = self.head(x) + print(F"hello2222") + return x diff --git a/onnx4deeploy/models/pytorch_models/sleep_convit/qsleep_convit_scales.json b/onnx4deeploy/models/pytorch_models/sleep_convit/qsleep_convit_scales.json old mode 100644 new mode 100755 index 8e82496..5170c2f --- a/onnx4deeploy/models/pytorch_models/sleep_convit/qsleep_convit_scales.json +++ b/onnx4deeploy/models/pytorch_models/sleep_convit/qsleep_convit_scales.json @@ -1,1764 +1,1764 @@ { - "cls_token_quant": 0.017837218940258026, - "pos_embed_quant": 0.017837218940258026, - "inputQuant.act_quant": 0.04484846442937851, + "cls_token_quant": 0.01728847809135914, + "pos_embed_quant": 0.01728847809135914, + "inputQuant.act_quant": 0.0377829372882843, "inputQuant.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, "conv_stem.branch1.conv1.weight_quant": [ [ [ - 0.0075483969412744045 + 0.007700167130678892 ] ], [ [ - 0.0044861845672130585 + 0.0043680076487362385 ] ], [ [ - 0.004401240032166243 + 0.0037727945018559694 ] ], [ [ - 0.007736503146588802 + 0.0077829216606915 ] ], [ [ - 0.007054155692458153 + 0.005485333036631346 ] ], [ [ - 0.005122110713273287 + 0.005003750324249268 ] ], [ [ - 0.0062958234921097755 + 0.006256120279431343 ] ], [ [ - 0.005045303609222174 + 0.004952708724886179 ] ], [ [ - 0.00552830146625638 + 0.005447492003440857 ] ], [ [ - 0.005115997511893511 + 0.005874777678400278 ] ], [ [ - 0.004752025008201599 + 0.004525422118604183 ] ], [ [ - 0.004465884529054165 + 0.0034753396175801754 ] ], [ [ - 0.005070246756076813 + 0.0050322627648711205 ] ], [ [ - 0.0032785432413220406 + 0.0038158234674483538 ] ], [ [ - 0.005612294655293226 + 0.004551413934677839 ] ], [ [ - 0.0062989769503474236 + 0.005285318940877914 ] ] ], - "conv_stem.branch1.conv1.input_quant": 0.04290647804737091, - "conv_stem.branch1.conv1.output_quant": 0.053983174264431, + "conv_stem.branch1.conv1.input_quant": 0.03770062327384949, + "conv_stem.branch1.conv1.output_quant": 0.04255806654691696, "conv_stem.branch1.conv1.input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, "conv_stem.branch1.conv1.output_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, - "conv_stem.branch1.relu1.act_quant": 0.02690162882208824, + "conv_stem.branch1.relu1.act_quant": 0.021949702873826027, "conv_stem.branch1.relu1.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, "conv_stem.branch1.conv2.weight_quant": [ [ [ - 0.0042135147377848625 + 0.004444824997335672 ] ], [ [ - 0.004618746694177389 + 0.004555459599941969 ] ], [ [ - 0.004535664338618517 + 0.0044272239319980145 ] ], [ [ - 0.004121930338442326 + 0.004483164753764868 ] ], [ [ - 0.0037126648239791393 + 0.004080228973180056 ] ], [ [ - 0.004893876612186432 + 0.004470501095056534 ] ], [ [ - 0.003665776690468192 + 0.0034556023310869932 ] ], [ [ - 0.0035946075804531574 + 0.004151413682848215 ] ], [ [ - 0.0036154002882540226 + 0.0036266050301492214 ] ], [ [ - 0.0038241660222411156 + 0.0033739982172846794 ] ], [ [ - 0.003009839216247201 + 0.0030401505064219236 ] ], [ [ - 0.004207472316920757 + 0.00424550985917449 ] ], [ [ - 0.0036023142747581005 + 0.003907631151378155 ] ], [ [ - 0.0042788307182490826 + 0.004164642188698053 ] ], [ [ - 0.004918133374303579 + 0.005155718885362148 ] ], [ [ - 0.0037640314549207687 + 0.0036308413837105036 ] ] ], - "conv_stem.branch1.conv2.input_quant": 0.03290736302733421, - "conv_stem.branch1.conv2.output_quant": 0.016764704138040543, + "conv_stem.branch1.conv2.input_quant": 0.03199717402458191, + "conv_stem.branch1.conv2.output_quant": 0.017414741218090057, "conv_stem.branch1.conv2.input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, "conv_stem.branch1.conv2.output_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, - "conv_stem.branch1.relu2.act_quant": 0.00846397690474987, + "conv_stem.branch1.relu2.act_quant": 0.008679273538291454, "conv_stem.branch1.relu2.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, "conv_stem.branch2.conv1.weight_quant": [ [ [ - 0.0028544331435114145 + 0.00259192381054163 ] ], [ [ - 0.0033681674394756556 + 0.0032467590644955635 ] ], [ [ - 0.0034882749896496534 + 0.0031782411970198154 ] ], [ [ - 0.0030635371804237366 + 0.0030435211956501007 ] ], [ [ - 0.003136900020763278 + 0.003025458427146077 ] ], [ [ - 0.003050380852073431 + 0.0034957064781337976 ] ], [ [ - 0.0040616183541715145 + 0.0043477690778672695 ] ], [ [ - 0.0027133801486343145 + 0.003210951341316104 ] ], [ [ - 0.0032808163668960333 + 0.003571817884221673 ] ], [ [ - 0.0033756429329514503 + 0.0030537741258740425 ] ], [ [ - 0.0026978773530572653 + 0.002389652421697974 ] ], [ [ - 0.0032980882097035646 + 0.003742265747860074 ] ], [ [ - 0.0029072100296616554 + 0.0027983197942376137 ] ], [ [ - 0.0027124390471726656 + 0.002486150711774826 ] ], [ [ - 0.0035788349341601133 + 0.003615174675360322 ] ], [ [ - 0.002912278752774 + 0.0026133409701287746 ] ] ], - "conv_stem.branch2.conv1.input_quant": 0.04398861154913902, - "conv_stem.branch2.conv1.output_quant": 0.05451372638344765, + "conv_stem.branch2.conv1.input_quant": 0.03344094380736351, + "conv_stem.branch2.conv1.output_quant": 0.04139895737171173, "conv_stem.branch2.conv1.input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, "conv_stem.branch2.conv1.output_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, - "conv_stem.branch2.relu1.act_quant": 0.02716621570289135, + "conv_stem.branch2.relu1.act_quant": 0.02064749039709568, "conv_stem.branch2.relu1.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, "conv_stem.branch2.conv2.weight_quant": [ [ [ - 0.0043754177168011665 + 0.004379000514745712 ] ], [ [ - 0.004144658800214529 + 0.004424882587045431 ] ], [ [ - 0.0036650870461016893 + 0.0032617414835840464 ] ], [ [ - 0.00365339289419353 + 0.0035319533199071884 ] ], [ [ - 0.0030197538435459137 + 0.0037408589851111174 ] ], [ [ - 0.004602823406457901 + 0.00466295937076211 ] ], [ [ - 0.002820024499669671 + 0.0034633143804967403 ] ], [ [ - 0.004166399594396353 + 0.004278861917555332 ] ], [ [ - 0.0040544550865888596 + 0.003981275949627161 ] ], [ [ - 0.004079627804458141 + 0.004527073819190264 ] ], [ [ - 0.004077327903360128 + 0.003036445938050747 ] ], [ [ - 0.0039384267292916775 + 0.003864831058308482 ] ], [ [ - 0.004882764536887407 + 0.004883200395852327 ] ], [ [ - 0.004018516279757023 + 0.004086349159479141 ] ], [ [ - 0.004400023724883795 + 0.0044512441381812096 ] ], [ [ - 0.0034655272029340267 + 0.003594176610931754 ] ] ], - "conv_stem.branch2.conv2.input_quant": 0.04243790730834007, - "conv_stem.branch2.conv2.output_quant": 0.017832189798355103, + "conv_stem.branch2.conv2.input_quant": 0.03914378210902214, + "conv_stem.branch2.conv2.output_quant": 0.017506461590528488, "conv_stem.branch2.conv2.input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, "conv_stem.branch2.conv2.output_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, - "conv_stem.branch2.relu2.act_quant": 0.008788160979747772, + "conv_stem.branch2.relu2.act_quant": 0.008729592896997929, "conv_stem.branch2.relu2.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, "conv_stem.branch3.conv1.weight_quant": [ [ [ - 0.0020618378184735775 + 0.0022716252133250237 ] ], [ [ - 0.002362086670473218 + 0.0020840235520154238 ] ], [ [ - 0.0028500768821686506 + 0.002908001421019435 ] ], [ [ - 0.0025676009245216846 + 0.0022023706696927547 ] ], [ [ - 0.0025873358827084303 + 0.00227094953879714 ] ], [ [ - 0.00253949873149395 + 0.0025678740348666906 ] ], [ [ - 0.002308707917109132 + 0.0023773445282131433 ] ], [ [ - 0.0020583360455930233 + 0.0018388611497357488 ] ], [ [ - 0.0022384098265320063 + 0.0028274417854845524 ] ], [ [ - 0.0027320149820297956 + 0.0026444629766047 ] ], [ [ - 0.00233985623344779 + 0.0024917894043028355 ] ], [ [ - 0.002922546351328492 + 0.003058363450691104 ] ], [ [ - 0.002648717723786831 + 0.0025761625729501247 ] ], [ [ - 0.002508507575839758 + 0.002443275647237897 ] ], [ [ - 0.003001951379701495 + 0.002814142731949687 ] ], [ [ - 0.0023402906954288483 + 0.0023239522706717253 ] ] ], - "conv_stem.branch3.conv1.input_quant": 0.04459114745259285, - "conv_stem.branch3.conv1.output_quant": 0.06097126752138138, + "conv_stem.branch3.conv1.input_quant": 0.036097608506679535, + "conv_stem.branch3.conv1.output_quant": 0.04434343799948692, "conv_stem.branch3.conv1.input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, "conv_stem.branch3.conv1.output_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, - "conv_stem.branch3.relu1.act_quant": 0.030341841280460358, + "conv_stem.branch3.relu1.act_quant": 0.022029001265764236, "conv_stem.branch3.relu1.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, "conv_stem.branch3.conv2.weight_quant": [ [ [ - 0.0034469394013285637 + 0.0034151491709053516 ] ], [ [ - 0.00402803486213088 + 0.004080288112163544 ] ], [ [ - 0.0036262052599340677 + 0.0038126357831060886 ] ], [ [ - 0.0049094632267951965 + 0.005141297355294228 ] ], [ [ - 0.0034171505831182003 + 0.0032300332095474005 ] ], [ [ - 0.004605163354426622 + 0.004376872908324003 ] ], [ [ - 0.004510743077844381 + 0.0042967842891812325 ] ], [ [ - 0.0037943809293210506 + 0.0038058566860854626 ] ], [ [ - 0.005316364578902721 + 0.005534668453037739 ] ], [ [ - 0.004661924671381712 + 0.004367930814623833 ] ], [ [ - 0.003083513118326664 + 0.00320817856118083 ] ], [ [ - 0.005059072747826576 + 0.00449279835447669 ] ], [ [ - 0.004422878380864859 + 0.004353045951575041 ] ], [ [ - 0.0036508957855403423 + 0.003384666284546256 ] ], [ [ - 0.003726565046235919 + 0.0037087006494402885 ] ], [ [ - 0.0038747212383896112 + 0.0034532949794083834 ] ] ], - "conv_stem.branch3.conv2.input_quant": 0.05527482554316521, - "conv_stem.branch3.conv2.output_quant": 0.018089398741722107, + "conv_stem.branch3.conv2.input_quant": 0.04130538925528526, + "conv_stem.branch3.conv2.output_quant": 0.018119122833013535, "conv_stem.branch3.conv2.input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, "conv_stem.branch3.conv2.output_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, - "conv_stem.branch3.relu2.act_quant": 0.009000495076179504, + "conv_stem.branch3.relu2.act_quant": 0.008606902323663235, "conv_stem.branch3.relu2.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, - "conv_stem.cat_rescale.act_quant": 0.03551130369305611, + "conv_stem.cat_rescale.act_quant": 0.033355433493852615, "conv_stem.cat_rescale.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, - "qaddpos.act_quant": 0.017837218940258026, + "qaddpos.act_quant": 0.01728847809135914, "qaddpos.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, "encoder.ln_1.weight_quant": 1.0, "encoder.ln_1.bias_quant": 1.0, "encoder.mha.in_proj.weight_quant": [ [ - 0.0021071869414299726 + 0.002227425342425704 ], [ - 0.0018961158348247409 + 0.0019200658425688744 ], [ - 0.0015062791062518954 + 0.0015780801186338067 ], [ - 0.002423203317448497 + 0.0022304558660835028 ], [ - 0.00174946547485888 + 0.0021207055542618036 ], [ - 0.0019177183276042342 + 0.0025219551753252745 ], [ - 0.0018807967426255345 + 0.0020652839448302984 ], [ - 0.001548569998703897 + 0.0016119007486850023 ], [ - 0.0011212308891117573 + 0.001190426992252469 ], [ - 0.001379145891405642 + 0.0015318552032113075 ], [ - 0.0017511369660496712 + 0.0021116258576512337 ], [ - 0.0015972043620422482 + 0.0014778232434764504 ], [ - 0.0015924543840810657 + 0.0018063544994220138 ], [ - 0.002301453845575452 + 0.00247784866951406 ], [ - 0.002261851215735078 + 0.0021212827414274216 ], [ - 0.0016477408353239298 + 0.0021191565319895744 ], [ - 0.0024681687355041504 + 0.002262998139485717 ], [ - 0.0014800956705585122 + 0.0015699389623478055 ], [ - 0.001948988763615489 + 0.0015913064125925303 ], [ - 0.0021288017742335796 + 0.0020706388168036938 ], [ - 0.0013043810613453388 + 0.0013684657169505954 ], [ - 0.0017070436151698232 + 0.0017239818116649985 ], [ - 0.002024749293923378 + 0.0022642784751951694 ], [ - 0.002023475943133235 + 0.002249767305329442 ], [ - 0.0019611953757703304 + 0.0019359608413651586 ], [ - 0.0016012847190722823 + 0.0016061724163591862 ], [ - 0.0018770445603877306 + 0.0017447177087888122 ], [ - 0.0014978990657255054 + 0.0016570232110098004 ], [ - 0.0022737313993275166 + 0.0017342462670058012 ], [ - 0.002214226173236966 + 0.002182023599743843 ], [ - 0.001582524972036481 + 0.001516907592304051 ], [ - 0.001529959961771965 + 0.0019236189546063542 ], [ - 0.002205729018896818 + 0.0021568378433585167 ], [ - 0.0018768879817798734 + 0.0016910543199628592 ], [ - 0.002661316189914942 + 0.002450961619615555 ], [ - 0.0015044293832033873 + 0.0019052011193707585 ], [ - 0.001814402756281197 + 0.001985696377232671 ], [ - 0.001933390274643898 + 0.0017749292310327291 ], [ - 0.002207135548815131 + 0.002220303053036332 ], [ - 0.001913944841362536 + 0.0018171115079894662 ], [ - 0.002045201137661934 + 0.002102222293615341 ], [ - 0.002093489049002528 + 0.0016377090942114592 ], [ - 0.0014750697882845998 + 0.001501825638115406 ], [ - 0.0022373248357325792 + 0.0023003611713647842 ], [ - 0.002473076106980443 + 0.002273709047585726 ], [ - 0.0018155454890802503 + 0.00214559119194746 ], [ - 0.0021056821569800377 + 0.001888459431938827 ], [ - 0.001934695872478187 + 0.0025583531241863966 ], [ - 0.0025584257673472166 + 0.002527237869799137 ], [ - 0.001578359748236835 + 0.0017287884838879108 ], [ - 0.00199709041044116 + 0.0023058399092406034 ], [ - 0.0022841054014861584 + 0.002082264283671975 ], [ - 0.0015191702404990792 + 0.00220063840970397 ], [ - 0.0022538548801094294 + 0.002563020447269082 ], [ - 0.0022919790353626013 + 0.001976113999262452 ], [ - 0.002331934403628111 + 0.002403559861704707 ], [ - 0.0019083148799836636 + 0.002103435341268778 ], [ - 0.002720849821344018 + 0.002617834834381938 ], [ - 0.0017137030372396111 + 0.0018967618234455585 ], [ - 0.002344016218557954 + 0.002261911751702428 ], [ - 0.0022261503618210554 + 0.002293736208230257 ], [ - 0.0033149966038763523 + 0.0033512054942548275 ], [ - 0.0025749749038368464 + 0.002045930363237858 ], [ - 0.002318201120942831 + 0.002225602278485894 ], [ - 0.002220605267211795 + 0.0022986880503594875 ], [ - 0.0022199517115950584 + 0.001997053623199463 ], [ - 0.0022314756643027067 + 0.0017529248725622892 ], [ - 0.0018433924997225404 + 0.0025695357471704483 ], [ - 0.0026631217915564775 + 0.0032922953832894564 ], [ - 0.0018224967643618584 + 0.0016513023292645812 ], [ - 0.0025638090446591377 + 0.0019141251686960459 ], [ - 0.0018702598754316568 + 0.0021567908115684986 ], [ - 0.002417492913082242 + 0.0023256458807736635 ], [ - 0.0018854676745831966 + 0.0018727181013673544 ], [ - 0.0019335874821990728 + 0.0017376739997416735 ], [ - 0.0018881962168961763 + 0.0020870454609394073 ], [ - 0.0021717885974794626 + 0.0024144649505615234 ], [ - 0.0021972190588712692 + 0.0017871428281068802 ], [ - 0.0015470789512619376 + 0.0015482084127143025 ], [ - 0.0019454978173598647 + 0.0017507793381810188 ], [ - 0.0022312570363283157 + 0.0017274431884288788 ], [ - 0.0018959111766889691 + 0.0015455292304977775 ], [ - 0.0027432499919086695 + 0.0021849204786121845 ], [ - 0.0018493181560188532 + 0.002436551498249173 ], [ - 0.001827657106332481 + 0.0020151345524936914 ], [ - 0.002221457427367568 + 0.001740949461236596 ], [ - 0.002189524006098509 + 0.0020386443939059973 ], [ - 0.0020209308713674545 + 0.0021721338853240013 ], [ - 0.0023842097725719213 + 0.0021542191971093416 ], [ - 0.0020050096791237593 + 0.0019129628781229258 ], [ - 0.002156405709683895 + 0.0022090948186814785 ], [ - 0.0027858843095600605 + 0.0023202283773571253 ], [ - 0.0031699021346867085 + 0.0017562947468832135 ], [ - 0.002103906124830246 + 0.002050582552328706 ], [ - 0.002554378006607294 + 0.0016276971437036991 ], [ - 0.0015187639510259032 + 0.0018958599539473653 ], [ - 0.002849459182471037 + 0.0028672567568719387 ], [ - 0.0027739873621612787 + 0.0023723426274955273 ], [ - 0.001866056933067739 + 0.0019987544510513544 ], [ - 0.0018324883421882987 + 0.0014957032399252057 ], [ - 0.002734809648245573 + 0.002106226049363613 ], [ - 0.002684821607545018 + 0.0023632990196347237 ], [ - 0.0025979734491556883 + 0.002179571893066168 ], [ - 0.0019476537127047777 + 0.001977774081751704 ], [ - 0.0022448005620390177 + 0.002300483640283346 ], [ - 0.0027349386364221573 + 0.002420345088467002 ], [ - 0.0016696223756298423 + 0.001927125733345747 ], [ - 0.003179484512656927 + 0.0026756927836686373 ], [ - 0.001743459957651794 + 0.001601619180291891 ], [ - 0.002047704067081213 + 0.002247429918497801 ], [ - 0.002337988931685686 + 0.002316506579518318 ], [ - 0.0025819060392677784 + 0.0024339063093066216 ], [ - 0.002374933334067464 + 0.0030354605987668037 ], [ - 0.001967527437955141 + 0.002391157438978553 ], [ - 0.0020012918394058943 + 0.002622373402118683 ], [ - 0.0017300629988312721 + 0.0017479618545621634 ], [ - 0.001585284247994423 + 0.0017675459384918213 ], [ - 0.001515955664217472 + 0.0015539828455075622 ], [ - 0.0018590908730402589 + 0.001859157346189022 ], [ - 0.0021173974964767694 + 0.001591840642504394 ], [ - 0.0024945398326963186 + 0.0022190390154719353 ], [ - 0.0022471165284514427 + 0.002155131893232465 ], [ - 0.0019063017098233104 + 0.0020647626370191574 ], [ - 0.0019163653487339616 + 0.0016848515952005982 ], [ - 0.0022822588216513395 + 0.002332368167117238 ], [ - 0.0027194854337722063 + 0.0026039527729153633 ], [ - 0.002537494758144021 + 0.0025643929839134216 ], [ - 0.002407339634373784 + 0.002681432059034705 ], [ - 0.0018270317232236266 + 0.0021394183859229088 ], [ - 0.00179093552287668 + 0.001830349792726338 ], [ - 0.0019214469939470291 + 0.0018074617255479097 ], [ - 0.002057047560811043 + 0.0022083832882344723 ], [ - 0.0025684514548629522 + 0.00212676078081131 ], [ - 0.0030485016759485006 + 0.002255026251077652 ], [ - 0.001797708566300571 + 0.001608288032002747 ], [ - 0.002529471879824996 + 0.0021116887219250202 ], [ - 0.0019132940797135234 + 0.0017895271303132176 ], [ - 0.0018372675403952599 + 0.001574153546243906 ], [ - 0.0021196091547608376 + 0.002267166506499052 ], [ - 0.00191772251855582 + 0.002660273341462016 ], [ - 0.0020810700953006744 + 0.0020764567889273167 ], [ - 0.0017824856331571937 + 0.0020272093825042248 ], [ - 0.001987345051020384 + 0.0019864789210259914 ], [ - 0.002248850418254733 + 0.00267822970636189 ] ], "encoder.mha.in_proj.bias_quant": [ - 4.3521900806808844e-05, - 3.8491391023853794e-05, - 3.6763845855602995e-05, - 3.4433793189236894e-05, - 3.0147259167279117e-05, - 4.0485389035893604e-05, - 2.7743613827624358e-05, - 2.474003122188151e-05, - 2.4965846023405902e-05, - 3.3638494642218575e-05, - 5.1969156629638746e-05, - 5.1957162213511765e-05, - 4.104896288481541e-05, - 2.8779299100278877e-05, - 5.645572309731506e-05, - 3.241921876906417e-05, - 3.756762089324184e-05, - 3.364929216331802e-05, - 2.7748868888011202e-05, - 4.0356131648877636e-05, - 3.7348629120970145e-05, - 3.522906263242476e-05, - 4.9435333494329825e-05, - 3.5841680073644966e-05, - 5.683824565494433e-05, - 3.403754089958966e-05, - 4.642199201043695e-05, - 3.276973802712746e-05, - 2.9627070034621283e-05, - 4.843081478611566e-05, - 3.243835817556828e-05, - 4.105076368432492e-05, - 4.346259811427444e-05, - 4.849517426919192e-05, - 6.062247121008113e-05, - 2.476934423611965e-05, - 4.2470128391869366e-05, - 6.114810094004497e-05, - 4.197923044557683e-05, - 2.8275273507460952e-05, - 6.430425128201023e-05, - 5.0424408982507885e-05, - 4.118662764085457e-05, - 6.0045138525310904e-05, - 5.068449900136329e-05, - 4.699428245658055e-05, - 5.3715066314907745e-05, - 4.584524504025467e-05, - 8.329556294484064e-05, - 7.463671499863267e-05, - 3.515305070322938e-05, - 0.00012327654985710979, - 2.3138829419622198e-05, - 2.8262471460038796e-05, - 3.627617479651235e-05, - 4.092595190741122e-05, - 3.067774014198221e-05, - 2.5952384021366015e-05, - 2.5374722099513747e-05, - 9.036653500515968e-05, - 5.838513243361376e-05, - 9.195698658004403e-05, - 0.00011730028199963272, - 6.724580453010276e-05, - 8.573680679546669e-05, - 6.0451180615928024e-05, - 6.609367846976966e-05, - 4.287050978746265e-05, - 3.07990558212623e-05, - 3.967693555750884e-05, - 0.000124986152513884, - 3.70257803297136e-05, - 3.557019590516575e-05, - 4.242193608661182e-05, - 4.24197714892216e-05, - 3.1556657631881535e-05, - 6.418516568373889e-05, - 7.506601832574233e-05, - 5.4480129620060325e-05, - 8.733737922739238e-05, - 4.888823605142534e-05, - 6.0739723267033696e-05, - 0.00011185201583430171, - 3.653555904747918e-05, - 0.00011702090705512092, - 0.0001050583305186592, - 3.819035555352457e-05, - 6.34065072517842e-05, - 8.788484410615638e-05, - 0.00013765625772066414, - 3.521301186992787e-05, - 0.00015364852151833475, - 0.00017014065815601498, - 0.00012492647510953248, - 0.00017065026622731239, - 2.3662745661567897e-05, - 0.00015074788825586438, - 0.00012861474533565342, - 7.89753466960974e-05, - 8.625428745290264e-05, - 0.00014355145685840398, - 8.428324508713558e-05, - 9.463157766731456e-05, - 0.0001161023901659064, - 0.00012646715913433582, - 7.746645133011043e-05, - 7.160487439250574e-05, - 0.00010664350702427328, - 8.168781641870737e-05, - 7.808411464793608e-05, - 8.565741882193834e-05, - 0.00011714619176927954, - 9.993903950089589e-05, - 7.11664033588022e-05, - 7.680957060074434e-05, - 8.642328612040728e-05, - 8.600317232776433e-05, - 7.776951679261401e-05, - 0.0001042910516844131, - 7.825870852684602e-05, - 9.28895387914963e-05, - 0.00011101825657533482, - 9.177954052574933e-05, - 8.378223719773814e-05, - 9.64844148256816e-05, - 0.00011213251855224371, - 9.596322342986241e-05, - 7.389799429802224e-05, - 8.60082363942638e-05, - 6.75573610351421e-05, - 7.606671715620905e-05, - 9.394827793585137e-05, - 8.219730807468295e-05, - 0.0001335921697318554, - 7.60522234486416e-05, - 7.656634988961741e-05, - 9.0296111011412e-05, - 9.465150651521981e-05, - 8.766334212850779e-05, - 9.40323734539561e-05, - 8.094309305306524e-05, - 7.81559429015033e-05, - 7.349377847276628e-05, - 0.0001089235520339571 + 6.050807860447094e-05, + 4.922443986288272e-05, + 4.68466714664828e-05, + 5.296996459946968e-05, + 3.756155274459161e-05, + 5.658749432768673e-05, + 4.557514330372214e-05, + 4.555678242468275e-05, + 3.539890894899145e-05, + 3.611960346461274e-05, + 5.6647539167897776e-05, + 4.4585354771697894e-05, + 4.352255564299412e-05, + 3.7262441765051335e-05, + 4.690066634793766e-05, + 5.057119778939523e-05, + 5.256723670754582e-05, + 3.767906309803948e-05, + 4.2507370380917564e-05, + 4.036747486679815e-05, + 4.279847053112462e-05, + 4.438845280674286e-05, + 5.2103394409641623e-05, + 4.336747952038422e-05, + 3.657146226032637e-05, + 3.230433867429383e-05, + 4.955203257850371e-05, + 4.33584900747519e-05, + 3.4494762076064944e-05, + 5.161468652659096e-05, + 4.749869913212024e-05, + 4.490981882554479e-05, + 6.151020352263004e-05, + 3.709495649673045e-05, + 5.546927059185691e-05, + 5.4975083912722766e-05, + 4.561202513286844e-05, + 4.2529383790679276e-05, + 4.7862526116659865e-05, + 5.6377695727860555e-05, + 4.419066681293771e-05, + 3.627356272772886e-05, + 4.332325261202641e-05, + 4.6744971768930554e-05, + 4.89504418510478e-05, + 4.948950299876742e-05, + 3.5182780266040936e-05, + 4.947255001752637e-05, + 9.07920693862252e-05, + 4.894350058748387e-05, + 6.287409632932395e-05, + 8.18209518911317e-05, + 8.316156890941784e-05, + 0.00010528480197535828, + 6.556376320077106e-05, + 4.793433254235424e-05, + 5.923550634179264e-05, + 5.246044020168483e-05, + 4.607233131537214e-05, + 9.146483353106305e-05, + 7.756686682114378e-05, + 0.000116885727038607, + 6.287929863901809e-05, + 5.9779620642075315e-05, + 8.138502016663551e-05, + 5.619056537398137e-05, + 4.1679748392198235e-05, + 8.837071800371632e-05, + 6.762079283362255e-05, + 7.908021507319063e-05, + 6.078336446080357e-05, + 0.00010703787120291963, + 4.26152691943571e-05, + 5.7693319831741974e-05, + 7.634259964106604e-05, + 4.140152668696828e-05, + 5.301801866153255e-05, + 7.98362452769652e-05, + 6.117582233855501e-05, + 9.58439995883964e-05, + 0.00017761954222805798, + 3.056228888453916e-05, + 4.116747004445642e-05, + 0.00014178532001096755, + 0.00016005351790226996, + 4.5663866330869496e-05, + 3.932779509341344e-05, + 0.00021386156731750816, + 6.770022446289659e-05, + 6.698924698866904e-05, + 4.681516657001339e-05, + 6.321432738332078e-05, + 4.233980871504173e-05, + 7.646171434316784e-05, + 3.360000846441835e-05, + 9.593149297870696e-05, + 0.0001145838905358687, + 0.00014977862883824855, + 8.654566772747785e-05, + 7.856641605030745e-05, + 0.0001036906469380483, + 8.620283915661275e-05, + 9.573156421538442e-05, + 8.059927495196462e-05, + 0.00010001923510571942, + 0.00010245416342513636, + 8.587779302615672e-05, + 0.00010281941649736837, + 8.3255632489454e-05, + 9.708908328320831e-05, + 8.199297735700384e-05, + 8.647535287309438e-05, + 0.0001182715714094229, + 0.00011093866487499326, + 0.00012129334936616942, + 0.0001401532645104453, + 8.586016338085756e-05, + 6.286949792411178e-05, + 0.00010298903362127021, + 8.130848436849192e-05, + 0.0001292371889576316, + 0.00010454284347360954, + 9.429518831893802e-05, + 8.248539961641654e-05, + 0.00011942695709876716, + 9.332699119113386e-05, + 0.00010913529695244506, + 8.825951954349875e-05, + 0.00011692324187606573, + 0.00017953739734366536, + 8.932058699429035e-05, + 7.868179091019556e-05, + 8.940260158851743e-05, + 6.824493902968243e-05, + 0.00010123774700332433, + 7.339402509387583e-05, + 8.13867591205053e-05, + 0.00010148149885935709, + 0.00010597008804325014, + 0.0001041899886331521, + 7.151288446038961e-05, + 9.605907689547166e-05, + 9.291098831454292e-05, + 9.540798782836646e-05 ], - "encoder.mha.in_proj.input_quant": 0.02831985242664814, + "encoder.mha.in_proj.input_quant": 0.032149918377399445, "encoder.mha.in_proj.input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, "encoder.mha.out_proj.weight_quant": [ [ - 0.002566875424236059 + 0.0025450787506997585 ], [ - 0.0030580158345401287 + 0.0033634952269494534 ], [ - 0.002994395326822996 + 0.0022102659568190575 ], [ - 0.0024209555704146624 + 0.002740070689469576 ], [ - 0.002476828871294856 + 0.0025040581822395325 ], [ - 0.002824110444635153 + 0.0031312594655901194 ], [ - 0.002946047345176339 + 0.002642548643052578 ], [ - 0.0034539501648396254 + 0.0029453104361891747 ], [ - 0.0033426000736653805 + 0.0027452234644442797 ], [ - 0.0030282125808298588 + 0.0028631347231566906 ], [ - 0.002563875401392579 + 0.0025560350622981787 ], [ - 0.00301593285985291 + 0.0032959417439997196 ], [ - 0.003232685849070549 + 0.0034846037160605192 ], [ - 0.0026690170634537935 + 0.0026389879640191793 ], [ - 0.003807465313002467 + 0.00330344308167696 ], [ - 0.00406606774777174 + 0.0037991786375641823 ], [ - 0.0022860101889818907 + 0.0023303786292672157 ], [ - 0.002878852654248476 + 0.0028480386827141047 ], [ - 0.0021447460167109966 + 0.0020462400279939175 ], [ - 0.0030518346466124058 + 0.0022324773017317057 ], [ - 0.0033308956772089005 + 0.002819840796291828 ], [ - 0.003151464741677046 + 0.003238159930333495 ], [ - 0.002865272108465433 + 0.0027130895759910345 ], [ - 0.002473384840413928 + 0.0027771799359470606 ], [ - 0.0031667479779571295 + 0.0029279368463903666 ], [ - 0.002976816613227129 + 0.003152851015329361 ], [ - 0.0035451941657811403 + 0.00266431481577456 ], [ - 0.0027477059047669172 + 0.002973100868985057 ], [ - 0.0032785695511847734 + 0.0026854926254600286 ], [ - 0.0034140627831220627 + 0.0032350695692002773 ], [ - 0.0047993529587984085 + 0.004139230120927095 ], [ - 0.003081711009144783 + 0.0029536543879657984 ], [ - 0.004431079141795635 + 0.004061378538608551 ], [ - 0.0035265740007162094 + 0.0032096209470182657 ], [ - 0.00340841943398118 + 0.0027513550594449043 ], [ - 0.0032587277237325907 + 0.0031433275435119867 ], [ - 0.0025499146431684494 + 0.0024465785827487707 ], [ - 0.002363869221881032 + 0.0027551387902349234 ], [ - 0.004081340041011572 + 0.0037565298844128847 ], [ - 0.002843390451744199 + 0.002779862843453884 ], [ - 0.002658950164914131 + 0.0021981794852763414 ], [ - 0.0030732478480786085 + 0.0032158789690583944 ], [ - 0.0028174100443720818 + 0.002521520247682929 ], [ - 0.003714032471179962 + 0.003021752927452326 ], [ - 0.0025140380021184683 + 0.0027959533035755157 ], [ - 0.002328339498490095 + 0.0024837807286530733 ], [ - 0.0034008342772722244 + 0.0032649582717567682 ], [ - 0.002457233378663659 + 0.0028171902522444725 ] ], "encoder.mha.out_proj.bias_quant": [ - 3.3028103644028306e-05, - 3.779260805458762e-05, - 3.103347626165487e-05, - 3.294542329967953e-05, - 3.155745071126148e-05, - 3.1064715585671365e-05, - 2.7030437195207924e-05, - 3.4928423701785505e-05, - 3.170805939589627e-05, - 2.9242737582535483e-05, - 2.173976827180013e-05, - 3.503774860291742e-05, - 3.153623401885852e-05, - 2.97042843158124e-05, - 2.9865308533771895e-05, - 3.388613913557492e-05, - 3.40683436661493e-05, - 3.0952996894484386e-05, - 4.679028643295169e-05, - 3.405780080356635e-05, - 3.340877447044477e-05, - 3.649418067652732e-05, - 4.08902888011653e-05, - 2.6750736651592888e-05, - 3.5160526749677956e-05, - 3.176083191647194e-05, - 3.8332498661475256e-05, - 4.2124356696149334e-05, - 4.2702020436991006e-05, - 3.722302062669769e-05, - 4.469441046239808e-05, - 4.2093884985661134e-05, - 4.8539237468503416e-05, - 3.1216863135341555e-05, - 4.117336357012391e-05, - 4.4027077819919214e-05, - 2.877884435292799e-05, - 2.3634464014321566e-05, - 4.0102411730913445e-05, - 3.3787466236390173e-05, - 2.779592978185974e-05, - 2.9332451958907768e-05, - 3.3954707760130987e-05, - 3.7977621104801074e-05, - 3.8975889765424654e-05, - 2.8808814022340812e-05, - 3.5180655686417595e-05, - 2.693287569854874e-05 + 2.3149881599238142e-05, + 2.5793873646762222e-05, + 2.4190048861782998e-05, + 3.5503599065123126e-05, + 2.2698917746311054e-05, + 1.9987739506177604e-05, + 2.1740779629908502e-05, + 2.1676312826457433e-05, + 2.1760217350674793e-05, + 3.1122242944547907e-05, + 3.110301622655243e-05, + 2.599974141048733e-05, + 2.6771669581648894e-05, + 2.8198943255119957e-05, + 2.716815833991859e-05, + 2.5208031729562208e-05, + 2.6325098588131368e-05, + 2.5534336600685492e-05, + 2.02864521270385e-05, + 2.6430747311678715e-05, + 2.4457227482344024e-05, + 3.087884761043824e-05, + 2.8083823053748347e-05, + 2.0496856450336054e-05, + 2.0321023839642294e-05, + 2.8517079044831917e-05, + 2.86385457002325e-05, + 2.222750845248811e-05, + 3.508044756017625e-05, + 2.2550859284820035e-05, + 2.8742635549861006e-05, + 2.03707877517445e-05, + 2.6890724257100374e-05, + 2.7156776923220605e-05, + 2.663042687345296e-05, + 3.320925679872744e-05, + 1.901651921798475e-05, + 2.392613168922253e-05, + 2.8610702429432422e-05, + 2.460148789396044e-05, + 2.430097447359003e-05, + 3.24661705235485e-05, + 2.562650479376316e-05, + 2.752750151557848e-05, + 2.4320879674633034e-05, + 2.595531623228453e-05, + 2.7301739464746788e-05, + 2.3921917090774514e-05 ], - "encoder.mha.out_proj.input_quant": 0.008948581293225288, - "encoder.mha.out_proj.output_quant": 0.010939874686300755, + "encoder.mha.out_proj.input_quant": 0.007638814393430948, + "encoder.mha.out_proj.output_quant": 0.01099616289138794, "encoder.mha.out_proj.input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, "encoder.mha.out_proj.output_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, - "encoder.mha.attn_output_weights_quant.act_quant": 0.0002787475532386452, + "encoder.mha.attn_output_weights_quant.act_quant": 0.0010784146143123507, "encoder.mha.attn_output_weights_quant.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, - "encoder.mha.q_scaled_quant.act_quant": 0.005090874154120684, + "encoder.mha.q_scaled_quant.act_quant": 0.004697044380009174, "encoder.mha.q_scaled_quant.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, - "encoder.mha.k_transposed_quant.act_quant": 0.014389771036803722, + "encoder.mha.k_transposed_quant.act_quant": 0.016757408156991005, "encoder.mha.k_transposed_quant.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, - "encoder.mha.v_quant.act_quant": 0.019884243607521057, + "encoder.mha.v_quant.act_quant": 0.016788391396403313, "encoder.mha.v_quant.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, "encoder.ln_2.weight_quant": 1.0, "encoder.ln_2.bias_quant": 1.0, "encoder.ff.ff1.weight_quant": [ [ - 0.003403208451345563 + 0.0032328846864402294 ], [ - 0.0025569989811629057 + 0.0025348379276692867 ], [ - 0.001959930406883359 + 0.002104574115946889 ], [ - 0.002515022177249193 + 0.0023122639395296574 ], [ - 0.0031255900394171476 + 0.002344944979995489 ], [ - 0.0036360148806124926 + 0.003326729405671358 ], [ - 0.002630674745887518 + 0.003282792866230011 ], [ - 0.0026574768126010895 + 0.002623545238748193 ], [ - 0.0032233132515102625 + 0.0027602515183389187 ], [ - 0.003218092955648899 + 0.003014691872522235 ], [ - 0.004974754992872477 + 0.002916456200182438 ], [ - 0.0025420053862035275 + 0.0024082937743514776 ], [ - 0.002638278529047966 + 0.002552654827013612 ], [ - 0.004613530822098255 + 0.0031073656864464283 ], [ - 0.00354278227314353 + 0.0035184575244784355 ], [ - 0.003082627896219492 + 0.0032426666002720594 ], [ - 0.0025242427363991737 + 0.002688169479370117 ], [ - 0.0025976719334721565 + 0.0024928851053118706 ], [ - 0.0024957924615591764 + 0.0021280536893755198 ], [ - 0.0030332140158861876 + 0.0032955931965261698 ], [ - 0.003239266574382782 + 0.0033104673493653536 ], [ - 0.0032029158901423216 + 0.003410267410799861 ], [ - 0.002868309151381254 + 0.0032123830169439316 ], [ - 0.002452407032251358 + 0.0023184348829090595 ], [ - 0.002743711695075035 + 0.002515599364414811 ], [ - 0.002992327092215419 + 0.0031135152094066143 ], [ - 0.0029564935248345137 + 0.002972954884171486 ], [ - 0.0026715686544775963 + 0.002515817992389202 ], [ - 0.003059198148548603 + 0.0029994600918143988 ], [ - 0.0021910383366048336 + 0.002357449848204851 ], [ - 0.003455266822129488 + 0.0033730571158230305 ], [ - 0.004229408223181963 + 0.002846228191629052 ], [ - 0.0033398051746189594 + 0.003079464193433523 ], [ - 0.003178544342517853 + 0.003881868440657854 ], [ - 0.0033053760416805744 + 0.0028620802331715822 ], [ - 0.002562867943197489 + 0.002888432936742902 ], [ - 0.004470005631446838 + 0.002717219525948167 ], [ - 0.0023471303284168243 + 0.0016008632956072688 ], [ - 0.0031073056161403656 + 0.002567456103861332 ], [ - 0.002469737082719803 + 0.0025209919549524784 ], [ - 0.0024988676887005568 + 0.002560699824243784 ], [ - 0.0025458685122430325 + 0.0027761533856391907 ], [ - 0.002492014318704605 + 0.002413744805380702 ], [ - 0.0028488156385719776 + 0.002689571352675557 ], [ - 0.0026558584067970514 + 0.0020539232064038515 ], [ - 0.0027248593978583813 + 0.002438942203298211 ], [ - 0.0031810093205422163 + 0.002750420942902565 ], [ - 0.0035232664085924625 + 0.0027201515622437 ] ], "encoder.ff.ff1.bias_quant": [ - 5.5452477681683376e-05, - 6.036627746652812e-05, - 4.9042177124647424e-05, - 6.090618626330979e-05, - 6.464087346103042e-05, - 6.0323673096718267e-05, - 4.5953489461680874e-05, - 4.828739110962488e-05, - 6.553747516591102e-05, - 5.163819878362119e-05, - 7.9820652899798e-05, - 6.688055873382837e-05, - 4.458321200218052e-05, - 8.539440023014322e-05, - 5.4340223869076e-05, - 4.446164166438393e-05, - 5.0903039664262906e-05, - 5.251335460343398e-05, - 4.4163462007418275e-05, - 4.847836316912435e-05, - 4.699168857769109e-05, - 5.341105133993551e-05, - 6.195847527123988e-05, - 5.797050835099071e-05, - 7.597541116410866e-05, - 6.577254680451006e-05, - 4.802774128620513e-05, - 5.369189238990657e-05, - 5.5123440688475966e-05, - 4.9369398766430095e-05, - 5.150325523572974e-05, - 8.107110625132918e-05, - 6.97872819728218e-05, - 6.320456304820254e-05, - 5.914639768889174e-05, - 4.555694977170788e-05, - 8.352378790732473e-05, - 4.0727285522734746e-05, - 4.727977648144588e-05, - 5.572301961365156e-05, - 5.828376015415415e-05, - 6.130327528808266e-05, - 5.207680442254059e-05, - 5.06037576997187e-05, - 6.49327048449777e-05, - 4.835459913010709e-05, - 5.525608139578253e-05, - 5.519505430129357e-05 + 7.498719060095027e-05, + 4.926235487801023e-05, + 5.050877371104434e-05, + 5.95391247770749e-05, + 6.156202289275825e-05, + 7.18023075023666e-05, + 6.633693556068465e-05, + 6.831396603956819e-05, + 5.878887895960361e-05, + 6.835787644376978e-05, + 5.9965568652842194e-05, + 4.8976828111335635e-05, + 4.337828795542009e-05, + 7.494446617783979e-05, + 8.367924601770937e-05, + 8.022222755244002e-05, + 6.349650357151404e-05, + 6.474449764937162e-05, + 5.1259808969916776e-05, + 6.987973029026762e-05, + 7.845495565561578e-05, + 4.757084752782248e-05, + 6.418041448341683e-05, + 9.647008846513927e-05, + 8.411757153226063e-05, + 5.395405969466083e-05, + 8.463787526125088e-05, + 5.833321120007895e-05, + 6.0272974224062636e-05, + 5.969961785012856e-05, + 7.512702723033726e-05, + 6.501157622551546e-05, + 5.7351207942701876e-05, + 7.466744864359498e-05, + 6.815824599470943e-05, + 4.395714131533168e-05, + 6.773317727493122e-05, + 3.7081736081745476e-05, + 6.156482413643971e-05, + 7.225068111438304e-05, + 6.43636958557181e-05, + 5.388354111346416e-05, + 6.28075358690694e-05, + 5.432395118987188e-05, + 5.495657023857348e-05, + 6.445019971579313e-05, + 7.341690798057243e-05, + 5.165556285646744e-05 ], - "encoder.ff.ff1.input_quant": 0.01583567075431347, - "encoder.ff.ff1.output_quant": 0.022544866427779198, + "encoder.ff.ff1.input_quant": 0.019113758578896523, + "encoder.ff.ff1.output_quant": 0.021797163411974907, "encoder.ff.ff1.input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, "encoder.ff.ff1.output_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, "encoder.ff.ff2.weight_quant": [ [ - 0.002971225418150425 + 0.002540784189477563 ], [ - 0.002250400837510824 + 0.0025547058321535587 ], [ - 0.002709007589146495 + 0.0026918419171124697 ], [ - 0.0030752269085496664 + 0.002338874852284789 ], [ - 0.0030809184536337852 + 0.003251499729231 ], [ - 0.00439762556925416 + 0.003947967663407326 ], [ - 0.0029294404666870832 + 0.0030620135366916656 ], [ - 0.0035947992000728846 + 0.003166659502312541 ], [ - 0.0035167390014976263 + 0.0035519192460924387 ], [ - 0.003796542761847377 + 0.0036688782274723053 ], [ - 0.0026450648438185453 + 0.0031043800991028547 ], [ - 0.003567359410226345 + 0.002984486985951662 ], [ - 0.002195175038650632 + 0.0019274965161457658 ], [ - 0.0022677110973745584 + 0.002529407385736704 ], [ - 0.0029033911414444447 + 0.002311328426003456 ], [ - 0.0028390090446919203 + 0.00278285495005548 ], [ - 0.003564479760825634 + 0.0036396037321537733 ], [ - 0.003430349053815007 + 0.0032709587831050158 ], [ - 0.0028679638635367155 + 0.002647558692842722 ], [ - 0.0025241514667868614 + 0.0027007642202079296 ], [ - 0.002458178671076894 + 0.0025122605729848146 ], [ - 0.0026473698671907187 + 0.002891841111704707 ], [ - 0.0026095551438629627 + 0.0025530215352773666 ], [ - 0.0023333856370300055 + 0.0021426621824502945 ], [ - 0.0032875153701752424 + 0.003227367764338851 ], [ - 0.0025124414823949337 + 0.002991440938785672 ], [ - 0.0024445876479148865 + 0.0020417352207005024 ], [ - 0.004292535129934549 + 0.0031559448689222336 ], [ - 0.003165788482874632 + 0.002952039008960128 ], [ - 0.002462804550305009 + 0.0026928395964205265 ], [ - 0.0023571953643113375 + 0.002132965950295329 ], [ - 0.0033441856503486633 + 0.0027183855418115854 ], [ - 0.0033317492343485355 + 0.0031710907351225615 ], [ - 0.0035525828134268522 + 0.003934640437364578 ], [ - 0.002138659590855241 + 0.0021151858381927013 ], [ - 0.0034952741116285324 + 0.002895907498896122 ], [ - 0.002431573113426566 + 0.0031890079844743013 ], [ - 0.0033607175573706627 + 0.003015984781086445 ], [ - 0.0029228581115603447 + 0.003341935109347105 ], [ - 0.00266608246602118 + 0.0025853347033262253 ], [ - 0.0032617677934467793 + 0.0030722706578671932 ], [ - 0.0029625333845615387 + 0.0024727594573050737 ], [ - 0.004284386523067951 + 0.00430693943053484 ], [ - 0.0027726078405976295 + 0.002899958286434412 ], [ - 0.0031355973333120346 + 0.0029888523276895285 ], [ - 0.0028520245105028152 + 0.002468215301632881 ], [ - 0.0022280821576714516 + 0.0021575740538537502 ], [ - 0.003456521313637495 + 0.0036121737211942673 ] ], "encoder.ff.ff2.bias_quant": [ - 4.081176302861422e-05, - 4.65644225187134e-05, - 5.293917274684645e-05, - 3.598331386456266e-05, - 4.6789238695055246e-05, - 5.8310826716478914e-05, - 5.0376122089801356e-05, - 7.84043877501972e-05, - 4.865194205194712e-05, - 7.402031042147428e-05, - 4.718528361991048e-05, - 3.90489112760406e-05, - 3.9293565350817516e-05, - 5.870520180906169e-05, - 6.599004700547084e-05, - 3.778406244236976e-05, - 5.7146426115650684e-05, - 5.240013706497848e-05, - 7.232013012981042e-05, - 4.0614755562273785e-05, - 4.444371734280139e-05, - 4.3532894778763875e-05, - 4.572581747197546e-05, - 5.285641964292154e-05, - 4.680298297898844e-05, - 5.387501005316153e-05, - 5.964556476101279e-05, - 9.004313324112445e-05, - 4.128019281779416e-05, - 5.743368819821626e-05, - 5.14151033712551e-05, - 0.00011770833953050897, - 6.53171373414807e-05, - 6.44302272121422e-05, - 3.137749808956869e-05, - 5.778835839009844e-05, - 3.799048499786295e-05, - 3.670175283332355e-05, - 5.8541627367958426e-05, - 3.631471918197349e-05, - 7.658202957827598e-05, - 4.532069215201773e-05, - 5.6113356549758464e-05, - 5.846757994731888e-05, - 6.687606946798041e-05, - 5.0440532504580915e-05, - 6.807724275859073e-05, - 4.7414676373591647e-05 + 4.566737334243953e-05, + 5.065666846348904e-05, + 5.845870327902958e-05, + 3.3672829886199906e-05, + 5.23770613654051e-05, + 5.571214569499716e-05, + 6.301180110312998e-05, + 5.294337825034745e-05, + 5.839547156938352e-05, + 5.684942516381852e-05, + 4.855397855862975e-05, + 4.916664693155326e-05, + 3.133434438495897e-05, + 3.880137956002727e-05, + 4.7777139116078615e-05, + 4.578085645334795e-05, + 4.729117063106969e-05, + 3.761542393476702e-05, + 5.1902861741837114e-05, + 6.013745587551966e-05, + 4.579684537020512e-05, + 4.3461157474666834e-05, + 4.273742524674162e-05, + 4.934638491249643e-05, + 5.9881193010369316e-05, + 7.744455069769174e-05, + 5.4648073273710907e-05, + 6.650984869338572e-05, + 3.375795859028585e-05, + 4.954998075845651e-05, + 4.33569002780132e-05, + 4.52802560175769e-05, + 4.2928571929223835e-05, + 4.6040015149628744e-05, + 4.666531094699167e-05, + 7.07791987224482e-05, + 5.071628402220085e-05, + 4.680823258240707e-05, + 5.402659371611662e-05, + 4.765831909026019e-05, + 5.396133929025382e-05, + 4.038778934045695e-05, + 4.6490800741594285e-05, + 4.875994272879325e-05, + 5.950702689005993e-05, + 4.113189788768068e-05, + 3.662782546598464e-05, + 6.549015233758837e-05 ], - "encoder.ff.ff2.input_quant": 0.014125959947705269, - "encoder.ff.ff2.output_quant": 0.01510005071759224, + "encoder.ff.ff2.input_quant": 0.016389423981308937, + "encoder.ff.ff2.output_quant": 0.015031063929200172, "encoder.ff.ff2.input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, "encoder.ff.ff2.output_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, - "encoder.rescale_residual1.act_quant": 0.011041161604225636, + "encoder.rescale_residual1.act_quant": 0.011087513528764248, "encoder.rescale_residual1.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, - "encoder.rescale_residual2.act_quant": 0.016701295971870422, + "encoder.rescale_residual2.act_quant": 0.01659729890525341, "encoder.rescale_residual2.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, "norm.weight_quant": 1.0, "norm.bias_quant": 1.0, - "rescale_norm.act_quant": 0.013678238727152348, + "rescale_norm.act_quant": 0.015843253582715988, "rescale_norm.act_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, "head.weight_quant": [ [ - 0.00323863560333848 + 0.0028364479076117277 ], [ - 0.003783071879297495 + 0.003566386876627803 ], [ - 0.0036574609111994505 + 0.0033993548713624477 ], [ - 0.0027862510178238153 + 0.0034161575604230165 ] ], "head.bias_quant": [ - 4.697828262578696e-05, - 4.3166644900338724e-05, - 6.678188219666481e-05, - 5.249695823295042e-05 + 5.7410190493101254e-05, + 4.766035999637097e-05, + 5.813076859340072e-05, + 6.15611279499717e-05 ], - "head.input_quant": 0.013886776752769947, - "head.output_quant": 0.03440367430448532, + "head.input_quant": 0.015449205413460732, + "head.output_quant": 0.0427057258784771, "head.input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0, "head.output_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value_quant": 1.0 } \ No newline at end of file diff --git a/onnx4deeploy/models/pytorch_models/sleep_convit/sleep_convit.py b/onnx4deeploy/models/pytorch_models/sleep_convit/sleep_convit.py index 436df4d..945a015 100644 --- a/onnx4deeploy/models/pytorch_models/sleep_convit/sleep_convit.py +++ b/onnx4deeploy/models/pytorch_models/sleep_convit/sleep_convit.py @@ -137,7 +137,6 @@ class ConvStem(nn.Module): This module applies two parallel convolutional branches with different kernel sizes to capture multi-scale temporal features, then concatenates the results. - Simplified to 2 branches for better Deeploy compatibility. Uses Conv2d for better compatibility with ONNX Runtime transformer optimizer. Args: @@ -353,7 +352,7 @@ def __init__(self, config: dict): self.conv_stem = ConvStem( in_channels=1, out_channels=self.model_dim, - kernel_sizes=(25, 200, 100), # 2 branches: fine-grained (25) and coarse-grained (100) + kernel_sizes=(25, 200, 100), # 3 branches with different kernel sizes stride=4, ) diff --git a/onnx4deeploy/models/qsleep_convit_exporter.py b/onnx4deeploy/models/qsleep_convit_exporter.py new file mode 100644 index 0000000..f320911 --- /dev/null +++ b/onnx4deeploy/models/qsleep_convit_exporter.py @@ -0,0 +1,245 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: MIT + +"""QSleepConViT Model Exporter.""" + +from pathlib import Path +from typing import Any, Dict, List, Tuple + +import numpy as np +import torch +import torch.onnx.utils +from brevitas.quant_tensor import QuantTensor + +from DeepQuant.ExportBrevitas import exportBrevitas + + +from ..core.base_exporter import BaseONNXExporter + +# Import QSleepConViT PyTorch model from new location +from .pytorch_models.sleep_convit.qsleep_convit import QSleepConViT +import brevitas.onnx as bo +import json +import onnx +import onnx_graphsurgeon as gs +from onnx4deeploy.transform.quant_transform import replace_qdq_with_deeploy, insert_rqs_from_map + + +class QSleepConViTExporter(BaseONNXExporter): + """ONNX exporter for QSleepConViT model.""" + + def __init__(self, save_path: str = None, config_file: str = "config.yaml"): + """ + Initialize QSleepConViT exporter. + + Args: + save_path: Optional custom path to save ONNX files + config_file: Path to configuration YAML file + """ + super().__init__(save_path, config_file) + self.model_config = {} + + def load_config(self) -> Dict[str, Any]: + """ + Load QSleepConViT configuration. + + Returns: + Dictionary containing QSleepConViT configuration parameters + """ + # Default QSleepConViT configuration + config = { + "batch_size": 1, + "input_channels": 1, + "input_length": 3000, # Time-series sequence length + "model_dim": 48, + "num_heads": 6, + "num_patches": 94, # Computed from ConvStem output + "seq_len": 95, # num_patches + 1 (CLS token) + "attention_dropout": 0.0, # No dropout for inference + "mlp_head_hidden_dim": 48, + "encoder_ff_dropout": 0.0, # No dropout for inference + "num_classes": 4, # Sleep stages: Wake, N1, N2, N3, REM + "opset_version": 17, # Match CCT opset version for compatibility + # Training configuration + "training_strategy": "full", # Options: "full", "last_layer", "custom" + "custom_trainable_params": [], + # ZO training configuration + "zo": { + "epsilon": 0.1, + "seed": 42, + "exceptions": "node_matmul_2" + }, + "weights_path":"onnx4deeploy/models/pytorch_models/sleep_convit/qsleep_convit.pth" + } + + self.model_config = config + return config + + def create_model(self) -> torch.nn.Module: + """ + Create SleepConViT PyTorch model. + + Returns: + SleepConViT model ready for export + """ + model = QSleepConViT(config=self.model_config) + return model + + def get_input_shape(self) -> Tuple[int, ...]: + """ + Get the input tensor shape for SleepConViT. + + Returns: + Tuple representing input shape (batch_size, channels, height, width) + Shape: (B, 1, 1, 3000) for compatibility with ViT/transformer optimizer + """ + batch_size = self.config["batch_size"] + channels = self.config["input_channels"] + length = self.config["input_length"] + return (batch_size, channels, 1, length) # 4D: (B, C, H, W) + + def get_trainable_params(self, all_param_names: List[str]) -> List[str]: + """ + Get list of trainable parameter names for SleepConViT. + + Supports multiple training strategies: + - "full": Train all parameters (default) + - "last_layer": Only train the final classification layer + - "custom": Use custom_trainable_params from config + + Args: + all_param_names: List of all parameter names in the model + + Returns: + List of parameter names that should be trainable + """ + strategy = self.config.get("training_strategy", "full") + + # Define training strategies + strategy_params = { + "full": all_param_names, # Train everything + "last_layer": [ + "classifier.lin1.weight", + "classifier.lin1.bias", + ], + "custom": self.config.get("custom_trainable_params", []), + } + + # Get trainable params based on strategy + if strategy not in strategy_params: + print(f"โš ๏ธ Unknown training strategy '{strategy}', using 'full' as fallback") + strategy = "full" + + trainable_params = strategy_params[strategy] + + # Filter to only include params that exist in the model + requires_grad = [name for name in all_param_names if name in trainable_params] + + # Print strategy info + print(f"\n๐ŸŽฏ Training Strategy: '{strategy}'") + print(f" Total params in model: {len(all_param_names)}") + print(f" Params to train: {len(requires_grad)}") + print(f" Frozen params: {len(all_param_names) - len(requires_grad)}") + + return requires_grad + + def _get_config_string(self) -> str: + """ + Get configuration string for folder naming. + + Returns: + Configuration string like "_3000_48d_8h_5cls" + """ + return ( + f"_{self.config['input_length']}" + f"_{self.config['model_dim']}d" + f"_{self.config['num_heads']}h" + f"_{self.config['num_classes']}cls" + ) + + def save_test_data(self, model: torch.nn.Module, save_dir: str): + """ + Save test input/output data for validation. + + Uses PyTorch model to generate reference output for validating ONNX correctness. + + Args: + model: PyTorch model to run inference with + save_dir: Directory to save test data + """ + print("๐Ÿ’พ Saving test input/output data...") + + # Create test input + input_shape = self.get_input_shape() + test_input = np.random.randn(*input_shape).astype(np.float32) + + # Get PyTorch output (reference for validating ONNX) + was_training = model.training + model.eval() + + with torch.no_grad(): + input_tensor = torch.from_numpy(test_input) + output_tensor = model(input_tensor) + if isinstance(output_tensor, QuantTensor): + output_tensor = output_tensor.value + test_output = output_tensor.numpy() + + # Restore training mode if needed + if was_training: + model.train() + + # Save as .npz files + save_path = Path(save_dir) + save_path.mkdir(parents=True, exist_ok=True) + + np.savez(save_path / "inputs.npz", input=test_input) + np.savez(save_path / "outputs.npz", output=test_output) + + print(" โœ… Saved test data (PyTorch reference):") + print(f" Input: {save_path / 'inputs.npz'} shape={test_input.shape}") + print(f" Output: {save_path / 'outputs.npz'} shape={test_output.shape}") + + def _build_rqs_map(self, graph: gs.Graph, brevitas_scales: Dict[str, Any]) -> Dict[str, Any]: + """ + Translate the flat Brevitas scales dump into the Deeploy edges map + expected by insert_rqs_from_map. + """ + rqs_map = {"edges": []} + + # Traverse the ONNX graph to find operators that require precision reduction + for node in graph.nodes: + if node.op in ["Conv", "Gemm", "MatMul"]: + layer_name = None + + # Match ONNX node name (e.g., "/conv1/Conv") with Brevitas scale keys + for key in brevitas_scales.keys(): + if key.endswith(".weight_quant"): + base_name = key.split(".")[0] # Extracts "conv1", "fc", etc. + if f"/{base_name}/" in node.name or node.name.startswith(base_name): + layer_name = base_name + break + + if layer_name: + in_scale = brevitas_scales.get(f"{layer_name}.input_quant") + w_scale = brevitas_scales.get(f"{layer_name}.weight_quant") + out_scale = brevitas_scales.get(f"{layer_name}.output_quant") + + + if in_scale is not None and w_scale is not None and out_scale is not None: + # Conv accumulation scale = input_scale * weight_scale + + w_flat = np.array(w_scale).flatten() + src_scale = in_scale * w_flat + + out_tensor_name = node.outputs[0].name + + # We use out_tensor_name for both src and dst to intercept and rewire + # all nodes strictly consuming the output of this convolution. + rqs_map["edges"].append({ + "src_tensor": out_tensor_name, + "dst_tensor": out_tensor_name, + "src_scale": src_scale.tolist(), + "dst_scale": float(out_scale) + }) + return rqs_map \ No newline at end of file diff --git a/onnx4deeploy/models/sleep_convit_exporter.py b/onnx4deeploy/models/sleep_convit_exporter.py index 816b514..936f945 100644 --- a/onnx4deeploy/models/sleep_convit_exporter.py +++ b/onnx4deeploy/models/sleep_convit_exporter.py @@ -73,7 +73,7 @@ def load_config(self) -> Dict[str, Any]: "attention_dropout": 0.0, # No dropout for inference "mlp_head_hidden_dim": 48, "encoder_ff_dropout": 0.0, # No dropout for inference - "num_classes": 5, # Sleep stages: Wake, N1, N2, N3, REM + "num_classes": 4, # Sleep stages: Wake, N1, N2, N3, REM "opset_version": 17, # Match CCT opset version for compatibility # Training configuration "training_strategy": "full", # Options: "full", "last_layer", "custom" diff --git a/onnx4deeploy/operators/base_operator.py b/onnx4deeploy/operators/base_operator.py index fb16d6c..5ae89f5 100644 --- a/onnx4deeploy/operators/base_operator.py +++ b/onnx4deeploy/operators/base_operator.py @@ -231,7 +231,7 @@ def generate(self) -> Tuple[str, str, str]: # Generate inputs inputs = self.generate_inputs() - + print(F"inputs: {inputs}") # Create ONNX graph graph = self.create_onnx_graph(inputs) diff --git a/onnx4deeploy/operators/rqsperturbrademacher.py b/onnx4deeploy/operators/rqsperturbrademacher.py index fc9c572..7cb0a4a 100644 --- a/onnx4deeploy/operators/rqsperturbrademacher.py +++ b/onnx4deeploy/operators/rqsperturbrademacher.py @@ -33,7 +33,6 @@ def load_config(self) -> Dict[str, Any]: self.input_shape = tuple(pn_config["input_shape"]) return config - def generate_inputs(self) -> np.ndarray: """Generate input with both positive and negative values.""" x = np.random.randn(*self.input_shape).astype(np.float32) @@ -42,7 +41,7 @@ def generate_inputs(self) -> np.ndarray: s = max_val / 127.0 s[s == 0] = 1.0 # Avoid division by zero mul = np.round(0.01 / s * (2**15)).astype(np.int32) # quantized multiplier for perturbation - x_quantized = np.round(x / s[:, np.newaxis]) * s[:, np.newaxis] + x_quantized = np.round(x / s[:, np.newaxis]) return {"x": x_quantized.astype(np.float32), "mul": mul.astype(np.float32)} def create_onnx_graph(self, inputs: Dict[str, np.ndarray]): @@ -54,7 +53,7 @@ def create_onnx_graph(self, inputs: Dict[str, np.ndarray]): mul_initializer = helper.make_tensor( name="mul", data_type=TensorProto.FLOAT, - dims=(self.input_shape[0],1), + dims=[self.input_shape[0]], vals=inputs["mul"], ) # Output tensor diff --git a/onnx4deeploy/operators/rqsperturbuniform.py b/onnx4deeploy/operators/rqsperturbuniform.py index 26b67ee..1ac9451 100644 --- a/onnx4deeploy/operators/rqsperturbuniform.py +++ b/onnx4deeploy/operators/rqsperturbuniform.py @@ -32,7 +32,6 @@ def load_config(self) -> Dict[str, Any]: pn_config = config.get("perturbuniform", {}) self.input_shape = tuple(pn_config["input_shape"]) return config - def generate_inputs(self) -> np.ndarray: """Generate input with both positive and negative values.""" @@ -42,8 +41,8 @@ def generate_inputs(self) -> np.ndarray: s = max_val / 127.0 s[s == 0] = 1.0 # Avoid division by zero mul = np.round(0.01*np.sqrt(3) / s * (2**15)).astype(np.int32) # quantized multiplier for perturbation - x_quantized = np.round(x / s[:, np.newaxis]) * s[:, np.newaxis] - return {"x": x_quantized.astype(np.float32), "mul": mul.astype(np.float32)} + x_quantized = np.round(x / s[:, np.newaxis]) + return {"x": x_quantized.astype(np.int32), "mul": mul.astype(np.int32)} def create_onnx_graph(self, inputs: Dict[str, np.ndarray]): """Create ONNX graph for PerturbUniform operator.""" @@ -54,7 +53,7 @@ def create_onnx_graph(self, inputs: Dict[str, np.ndarray]): mul_initializer = helper.make_tensor( name="mul", data_type=TensorProto.FLOAT, - dims=(self.input_shape[0],1), + dims=[self.input_shape[0]], vals=inputs["mul"], ) # Output tensor diff --git a/onnx4deeploy/optimization/shape_optimizer.py b/onnx4deeploy/optimization/shape_optimizer.py index 0cc7857..60db271 100644 --- a/onnx4deeploy/optimization/shape_optimizer.py +++ b/onnx4deeploy/optimization/shape_optimizer.py @@ -551,12 +551,28 @@ def softmax_cross_entropy_grad_shape_inference(ctx): print( f" SoftmaxCrossEntropyGrad shape inference: output shape set from log_prob input" ) + + def requantize_shift_shape_inference(ctx): + ctx.node + # Get first input + proto = ctx.get_input_type(0) + if proto is None: + return + + # Output shape matches log_prob input shape + ctx.set_output_type(0, proto) + print( + f" RequantShift shape inference: output shape set from first input" + ) # Register the custom shape inference function shape_calculator_dict = _get_shape_calculator_dict() shape_calculator_dict["com.microsoft.SoftmaxCrossEntropyGrad"] = ( softmax_cross_entropy_grad_shape_inference ) + shape_calculator_dict["ai.onnx.contrib.RequantizeShift"] = ( + requantize_shift_shape_inference + ) return True except (ImportError, AttributeError): # Internal API not available, will use fallback @@ -586,14 +602,17 @@ def infer_shapes_with_custom_ops( # Check for Microsoft custom ops op_types = set(node.op_type for node in model.graph.node) microsoft_ops = [op for op in op_types if "com.microsoft" in op] + ai_onnx_contrib_ops = [op for op in op_types if "ai.onnx.contrib" in op] if microsoft_ops: print(f" Found Microsoft custom ops: {microsoft_ops}") + if ai_onnx_contrib_ops: + print(f" Found ai.onnx.contrib ops: {ai_onnx_contrib_ops}") try: # Register custom shape inference for Microsoft ops (if available) registration_success = register_custom_shape_inference() if not registration_success: - print(" โ„น๏ธ Custom op registration not available, using fallback for Microsoft ops") + print(" โ„น๏ธ Custom op registration not available, using fallback for custom ops") # Try standard shape inference inferred_model = shape_inference.infer_shapes(model) @@ -613,7 +632,7 @@ def infer_shapes_with_custom_ops( except Exception as node_err: print(f" Node {i}: {node.op_type} failed: {str(node_err)}") # Try custom inference for Microsoft ops - if "com.microsoft" in node.op_type: + if "com.microsoft" in node.op_type or "ai.onnx.contrib" in node.op_type: print(f" Applying custom inference for: {node.op_type}") try: apply_custom_inference(inferred_model.graph, node) @@ -655,6 +674,13 @@ def apply_custom_inference(graph: onnx.GraphProto, node: onnx.NodeProto) -> None set_tensor_shape(graph, node.output[0], input_shape) print(f" {node.op_type} output shape: {input_shape}") + elif "ai.onnx.contrib.RequantizeShift" in node.op_type: + # RequantizeShift output shape matches first input + if len(node.input) >= 1 and len(node.output) >= 1: + input_shape = get_tensor_shape(graph, node.input[0]) + if input_shape: + set_tensor_shape(graph, node.output[0], input_shape) + print(f" RequantizeShift output shape: {input_shape}") def extract_subgraph(model: onnx.ModelProto, nodes: List[onnx.NodeProto]) -> onnx.ModelProto: """ diff --git a/onnx4deeploy/transform/zo_transform.py b/onnx4deeploy/transform/zo_transform.py index 0285d49..7f9d0fd 100644 --- a/onnx4deeploy/transform/zo_transform.py +++ b/onnx4deeploy/transform/zo_transform.py @@ -9,6 +9,7 @@ from onnx import TensorProto, helper, shape_inference from onnx4deeploy.transform.model_transform import ensure_all_tensor_shapes +from DeepQuant.QuantDequantOnnx import Quant, Dequant, RequantShift def generate_zo_graph(inference_onnx:str, output_onnx:str, zo_config:dict, noise_type: str) -> None: """ Generate MeZO ONNX graph for model based on its inference onnx""" @@ -180,6 +181,75 @@ def generate_weight_update_graph(onnx_path: str, output_path: str, zo_config: di nodes.append(node) perturbation_counter += 1 + elif noise_type == "rqs_rademacher": + # compute mul factor from scale. + scale = np.max(np.abs(onnx.numpy_helper.to_array(init)), axis=tuple(range(1, len(init.dims))), keepdims=True) + epsilon = 0.01 + # Use different scaling for weights (8-bit) and biases (32-bit) + if '_add' in init.name: + quant_max = 2**31 - 1 + mul = np.round(epsilon / (scale / quant_max) * (2**31)).astype(np.int32) # quantized multiplier for perturbation + + else: + quant_max = 127 + mul = np.round(epsilon / (scale / quant_max) * (2**15)).astype(np.int64) # quantized multiplier for perturbation + init_mul = helper.make_tensor( + name=f"{init.name}_mul", + data_type=TensorProto.FLOAT, + dims=[mul.shape[0]], + vals=mul + ) + node = helper.make_node( + "RQSPerturbRademacher", + inputs=[init.name, f"{init.name}_mul"], + outputs=[perturbed_name], + name=f"rqs_perturb_rademacher_{perturbed_name}", + domain="mezo", + idx=perturbation_counter, + seed=seed, + signed=1, + div=2**15, + n_levels=256, + doc_string="y = x + epsilon * RQSRandomRademacher(x, seed)" + ) + nodes.append(node) + new_initializers.append(init_mul) + perturbation_counter += 1 + + elif noise_type == "rqs_uniform": + + # compute mul factor from scale. + scale = np.max(np.abs(onnx.numpy_helper.to_array(init)), axis=tuple(range(1, len(init.dims))), keepdims=True) + epsilon = 0.01 + # Use different scaling for weights (8-bit) and biases (32-bit) + if '_add' in init.name: + quant_max = 2**31 - 1 + else: + quant_max = 127.0 + mul = np.round(epsilon / (scale / quant_max) * (2**15)).astype(np.int32) + init_mul = helper.make_tensor( + name=f"{init.name}_mul", + data_type=TensorProto.FLOAT, + dims=mul.shape, + vals=mul + ) + + node = helper.make_node( + "RQSPerturbUniform", + inputs=[init.name, f"{init.name}_mul"], + outputs=[perturbed_name], + name=f"rqs_perturb_uniform_{perturbed_name}", + domain="mezo", + seed=seed, + idx=perturbation_counter, + signed=1, + div=2**15, + n_levels=256, + doc_string="y = x + epsilon * RQSRandomUniform(x, seed)" + ) + nodes.append(node) + new_initializers.append(init_mul) + perturbation_counter += 1 else: raise ValueError(f"Unsupported noise_type: {noise_type}") @@ -253,19 +323,26 @@ def modify_graph(original_model: onnx.ModelProto, output_path: str, exceptions: base_seed = int(seed) perturbation_counter = 0 - + epsilon=0.01 # Prepare a fast lookup for initializer names initializer_names = {init.name for init in new_initializers} + + print(f"all initializer names: {initializer_names}") for node in original_model.graph.node: # Check if this is a node we want to modify - if node.op_type in ["Conv", "Gemm", "MatMul"] and node.name not in exceptions: + if node.op_type in ["Conv", "Gemm", "MatMul", "RequantShift"] and node.name not in exceptions: print(F"node: {node.name}, op_type: {node.op_type}") modified_inputs = list(node.input) made_change = False for i, input_name in enumerate(node.input): # Check if the input is a weight/bias initializer + + # For RequantShift, only perturb the 3rd input (the 'add' term/bias). + if node.op_type == "RequantShift" and i != 2: + continue + if input_name in initializer_names: made_change = True @@ -297,6 +374,7 @@ def modify_graph(original_model: onnx.ModelProto, output_path: str, exceptions: doc_string="y = x + epsilon * RandomNormal(x, seed)" ) new_nodes.append(perturbation_node) + perturbation_counter += 1 elif noise_type == "uniform": perturbation_node = helper.make_node( @@ -307,13 +385,14 @@ def modify_graph(original_model: onnx.ModelProto, output_path: str, exceptions: domain="mezo", idx=perturbation_counter, seed=seed, - eps=epsilon, + eps=epsilon*2*np.sqrt(3), low=-np.sqrt(3), high=np.sqrt(3), # dtype=dtype, doc_string="y = x + epsilon * RandomUniform(x, seed)" ) new_nodes.append(perturbation_node) + perturbation_counter += 1 elif noise_type == "triangle": perturbation_node = helper.make_node( @@ -324,13 +403,14 @@ def modify_graph(original_model: onnx.ModelProto, output_path: str, exceptions: domain="mezo", idx=perturbation_counter, seed=seed, - eps=epsilon, + eps=epsilon*2*np.sqrt(6), low=-np.sqrt(6), high=np.sqrt(6), # dtype=dtype, doc_string="y = x + epsilon * RandomTriangle(x, seed)" ) new_nodes.append(perturbation_node) + perturbation_counter += 1 elif noise_type == "rademacher": perturbation_node = helper.make_node( @@ -346,6 +426,116 @@ def modify_graph(original_model: onnx.ModelProto, output_path: str, exceptions: doc_string="y = x + epsilon * RandomRademacher(x, seed)" ) new_nodes.append(perturbation_node) + perturbation_counter += 1 + + elif noise_type == "rqs_rademacher": + scale = np.max(np.abs(onnx.numpy_helper.to_array(original_weight_tensor)), + axis=tuple(range(1, len(original_weight_tensor.dims))), keepdims=True) + epsilon = 0.01 + # compute mul factor from scale. input 1 is normally the weight tensor. + if '_add' in input_name: + print(F"found add in name: {input_name}, treating as bias with 32-bit quantization") + quant_max = 2**31 - 1 + # For biases, inherit the 'div' from the RequantShift node itself + print(F"NOde: {node.name}, op_type: {node.op_type}, attributes: {node.attribute}") + # CORRECT: 'div' is a TENSOR attribute, not an INT attribute. + div_tensor_proto = next((attr.t for attr in node.attribute if attr.name == 'div'), None) + if div_tensor_proto is None: + raise ValueError(f"Could not find 'div' TENSOR attribute on RequantShift node: {node.name}") + + # Convert the TensorProto to a numpy array and get the scalar value. + producer_div = onnx.numpy_helper.to_array(div_tensor_proto).item() + producer_mul_tensor = None + if len(node.input) > 1: + mul_input_name = node.input[1] + producer_mul_tensor = next((init for init in new_initializers if init.name == mul_input_name), None) + if producer_div is None: + raise ValueError(f"Could not find 'div' attribute on RequantShift node: {node.name}") + print(F"producer_div: {producer_div}, producer_mul: {producer_mul_tensor}") + + div=producer_div + n_levels = 2**32 + producer_mul = onnx.numpy_helper.to_array(producer_mul_tensor) + + mul = np.round(epsilon *producer_mul).astype(np.int32) # quantized multiplier for perturbation + + else: + quant_max = 127 + div = 2**15 + n_levels = 2**8 + mul = np.round(epsilon / (scale / quant_max) * (div)).astype(np.int64) + init_mul = helper.make_tensor( + name=f"{input_name}_mul", + data_type=TensorProto.FLOAT, + dims=[mul.shape[0]], + vals=mul if mul.ndim > 0 else mul + ) + new_initializers.append(init_mul) + + perturbation_node = helper.make_node( + "RQSPerturbRademacher", + inputs=[input_name, f"{input_name}_mul"], + outputs=[perturbed_tensor_name], + name=f"rqs_perturb_rademacher_{perturbed_tensor_name}", + domain="mezo", + idx=perturbation_counter, + seed=seed, + signed=1, + div=div, + n_levels=n_levels, + doc_string="y = x + epsilon * RQSRandomRademacher(x, seed)" + ) + new_nodes.append(perturbation_node) + perturbation_counter += 1 + + elif noise_type == "rqs_uniform": + scale = np.max(np.abs(onnx.numpy_helper.to_array(original_weight_tensor)), + axis=tuple(range(1, len(original_weight_tensor.dims))), keepdims=True) + # compute mul factor from scale. input 1 is normally the weight tensor. + if '_add' in input_name: + quant_max = 2**31 - 1 + producer_div = next((attr.i for attr in node.attribute if attr.name == 'div'), None) + if producer_div is None: + raise ValueError(f"Could not find 'div' attribute on RequantShift node: {node.name}") + div = producer_div + n_levels = 2**8 + mul = np.round(epsilon / (scale / quant_max) * (2**31)).astype(np.int32) # quantized multiplier for perturbation + + else: + quant_max = 127 + div = 2**15 + n_levels = 2**8 + mul = np.round(epsilon / (scale / quant_max) * (2**15)).astype(np.int64) # quantized multiplier for perturbation + init_mul = helper.make_tensor( + name=f"{input_name}_mul", + data_type=TensorProto.FLOAT, + dims=mul.shape, + vals=mul + ) + init_mul = helper.make_tensor( + name=f"{input_name}_mul", + data_type=TensorProto.FLOAT, + dims=mul.shape, + vals=mul + ) + new_initializers.append(init_mul) + + perturbation_node = helper.make_node( + "RQSPerturbUniform", + inputs=[input_name, f"{input_name}_mul"], + outputs=[perturbed_tensor_name], + name=f"rqs_perturb_uniform_{perturbed_tensor_name}", + domain="mezo", + seed=seed, + idx=perturbation_counter, + signed=1, + div=div, + n_levels=n_levels, + doc_string="y = x + epsilon * RQSRandomUniform(x, seed)" + ) + new_nodes.append(perturbation_node) + perturbation_counter += 1 + elif noise_type == "eggroll": # Shape annotation for intermediate outputs @@ -444,6 +634,9 @@ def modify_graph(original_model: onnx.ModelProto, output_path: str, exceptions: new_nodes.append(noise_node_a) new_nodes.append(noise_node_b) new_nodes.append(gemm_node) + + else: + raise ValueError(f"Unsupported noise_type: {noise_type}") # **CRITICAL**: annotate perturbed edge with same dtype/shape as weight if len(original_weight_tensor.dims) == 1: @@ -508,8 +701,10 @@ def modify_graph(original_model: onnx.ModelProto, output_path: str, exceptions: # Add the standard opset with the version we found helper.make_opsetid("", standard_opset_version), - # Addcustom domain - helper.make_opsetid("mezo", 1) + # Addcustom domains + helper.make_opsetid("mezo", 1), + helper.make_opsetid("ai.onnx.contrib", 1), + helper.make_opsetid("com.microsoft", 1) ] new_model = helper.make_model(new_graph, producer_name="mezo-graph-generator", opset_imports=opset_list) From 7b55f57f456cbcc846df74addb1af5bb1709e36e Mon Sep 17 00:00:00 2001 From: JanCSEM Date: Fri, 27 Mar 2026 14:26:24 +0000 Subject: [PATCH 19/24] Update Quantization export --- DeepQuant | 2 +- onnx4deeploy/core/base_exporter.py | 42 +++++++++++++------ onnx4deeploy/models/qsleep_convit_exporter.py | 2 +- onnx4deeploy/transform/zo_transform.py | 3 +- 4 files changed, 33 insertions(+), 16 deletions(-) diff --git a/DeepQuant b/DeepQuant index 5aa59a3..7abc615 160000 --- a/DeepQuant +++ b/DeepQuant @@ -1 +1 @@ -Subproject commit 5aa59a376ad0cfc98543df15382863aefd6b6682 +Subproject commit 7abc6154c8dd79978101a66273617b8836511e07 diff --git a/onnx4deeploy/core/base_exporter.py b/onnx4deeploy/core/base_exporter.py index 3c6e9a8..b3d8862 100644 --- a/onnx4deeploy/core/base_exporter.py +++ b/onnx4deeploy/core/base_exporter.py @@ -253,7 +253,7 @@ def setup_paths(self, mode: ExportMode) -> Dict[str, str]: "network_pre_sgd": os.path.join(output_dir, "network_pre_sgd.onnx"), } ) - + if mode == ExportMode.ZO_TRAINING: paths.update( { @@ -351,10 +351,13 @@ def export_inference(self, save_path: Optional[str] = None, quant: bool = False) #load weights. state_dict = torch.load(os.path.join(os.getcwd(), self.config.get("weights_path", "model_weights.pth")), map_location="cpu") model.load_state_dict(state_dict, strict=False) - onnx_model = exportBrevitas(model, input_tensor, debug=False) + #Jansno: temporary workaround for name, param in model.named_parameters(): - if param.requires_grad and "weight" in name and "conv" in name: - torch.nn.init.normal_(param, mean=0.0, std=0.02) + if param.requires_grad and "bias" in name: + if torch.all(param.data) == 0: + torch.nn.init.uniform_(param, a=0.01, b=0.02) + onnx_model = exportBrevitas(model, input_tensor, debug=False) + # Save onnx.save(onnx_model, self.paths["network"]) print(f"โœ… ONNX model saved: {self.paths['network']}") @@ -554,6 +557,11 @@ def export_zo_training(self, save_path: Optional[str] = None, noise_type: str = state_dict = torch.load(os.path.join(os.getcwd(), self.config.get("weights_path", "model_weights.pth")), map_location="cpu") model.load_state_dict(state_dict, strict=False) print("\n๐Ÿ“ค Exporting to ONNX with quantization...") + # JanSno: Temporary workaround + for name, param in model.named_parameters(): + if param.requires_grad and "bias" in name: + if torch.all(param.data) == 0: + torch.nn.init.uniform_(param, a=0.01, b=0.02) # use DeepQuant to export to ONNX onnx_model = exportBrevitas(model, input_tensor, debug=False) @@ -601,7 +609,7 @@ def export_zo_training(self, save_path: Optional[str] = None, noise_type: str = zo_config=self.config["zo"], noise_type=noise_type, ) - + generate_weight_update_graph( onnx_path=self.paths["network_infer"], output_path=self.paths["network_zo_update"], @@ -639,7 +647,7 @@ def export_zo_training(self, save_path: Optional[str] = None, noise_type: str = # Create test input/output print("\n๐Ÿงช Creating test input/output...") - self._create_test_data(mode=ExportMode.ZO_TRAINING) + self._create_test_data(mode=ExportMode.ZO_TRAINING, quant=quant) # # Add optimizer (SGD) nodes # print("\nโž• Adding SGD optimizer nodes...") @@ -672,15 +680,23 @@ def _create_test_data(self, mode=ExportMode.INFERENCE, quant=False): from onnxruntime_extensions import onnx_op, PyOp, get_library_path from DeepQuant.QuantDequantOnnx import requant_shift_onnx sess_options = ort.SessionOptions() - - sess_options.register_custom_ops_library(get_library_path()) - session = ort.InferenceSession(self.paths["network_infer"], - sess_options=sess_options) - input_names = [i.name for i in session.get_inputs()] + sess_options.register_custom_ops_library(get_library_path()) + + + if not quant: + session = ort.InferenceSession(self.paths["network_infer"], + sess_options=sess_options) + input_names = [i.name for i in session.get_inputs()] + + input_name = session.get_inputs()[0].name + test_output = session.run(None, {input_name: test_input})[0] + + + # Workaround onnx Inference session for now + else: + test_output = np.random.randn(input_shape[0], self.config["num_classes"]).astype(np.float32) - input_name = session.get_inputs()[0].name - test_output = session.run(None, {input_name: test_input})[0] # Save as .npz files save_path = Path(self.paths["output_dir"]) diff --git a/onnx4deeploy/models/qsleep_convit_exporter.py b/onnx4deeploy/models/qsleep_convit_exporter.py index f320911..472ccef 100644 --- a/onnx4deeploy/models/qsleep_convit_exporter.py +++ b/onnx4deeploy/models/qsleep_convit_exporter.py @@ -68,7 +68,7 @@ def load_config(self) -> Dict[str, Any]: "zo": { "epsilon": 0.1, "seed": 42, - "exceptions": "node_matmul_2" + "exceptions": ["node_matmul", "node_bmm_requant", "node_bmm_1_requant"] }, "weights_path":"onnx4deeploy/models/pytorch_models/sleep_convit/qsleep_convit.pth" } diff --git a/onnx4deeploy/transform/zo_transform.py b/onnx4deeploy/transform/zo_transform.py index 7f9d0fd..8882105 100644 --- a/onnx4deeploy/transform/zo_transform.py +++ b/onnx4deeploy/transform/zo_transform.py @@ -783,5 +783,6 @@ def append_cross_entropy_loss(onnx_path, output_path, label_name='y', logits_out try: inferred = shape_inference.infer_shapes(model) onnx.save(inferred, output_path) - except Exception: + except Exception as e: + print(F" shape inference failed, saving without shape inference. Error was: {e}") onnx.save(model, output_path) From 16ae86d3e03b339fee6e1b1df301789198d82a62 Mon Sep 17 00:00:00 2001 From: JanCSEM Date: Fri, 27 Mar 2026 15:28:13 +0100 Subject: [PATCH 20/24] Adding tests to quantization passes --- onnx4deeploy/core/base_exporter.py | 10 + onnx4deeploy/transform/quant_transform.py | 98 ++- pyproject.toml | 4 +- tests/models/conftest.py | 13 + tests/models/onnx_node_implementations.py | 738 ++++++++++++++++++++++ tests/models/test_qlitecnn.py | 169 +++++ tests/models/test_quant_transform.py | 525 +++++++++++++++ 7 files changed, 1554 insertions(+), 3 deletions(-) create mode 100644 tests/models/onnx_node_implementations.py create mode 100644 tests/models/test_qlitecnn.py create mode 100644 tests/models/test_quant_transform.py diff --git a/onnx4deeploy/core/base_exporter.py b/onnx4deeploy/core/base_exporter.py index b3d8862..5244754 100644 --- a/onnx4deeploy/core/base_exporter.py +++ b/onnx4deeploy/core/base_exporter.py @@ -28,6 +28,7 @@ from DeepQuant.Export4Deeploy import exportBrevitas from .onnx_utils import print_model_info, randomize_onnx_initializers +from onnx4deeploy.transform.quant_transform import fix_duplicate_tensor_names from onnx4deeploy.transform.zo_transform import generate_weight_update_graph, generate_zo_graph @@ -372,6 +373,14 @@ def export_inference(self, save_path: Optional[str] = None, quant: bool = False) infer_shapes_with_custom_ops(self.paths["network"], self.paths["network"]) + # Fix duplicate initializer/node-output names introduced by + # onnx.save's write_external_data_tensors on gs-exported models. + if quant: + _m = onnx.load(self.paths["network"]) + _m = fix_duplicate_tensor_names(_m) + with open(self.paths["network"], "wb") as _f: + _f.write(_m.SerializeToString()) + # Save test input/output data if method is implemented if hasattr(self, "save_test_data"): try: @@ -564,6 +573,7 @@ def export_zo_training(self, save_path: Optional[str] = None, noise_type: str = torch.nn.init.uniform_(param, a=0.01, b=0.02) # use DeepQuant to export to ONNX onnx_model = exportBrevitas(model, input_tensor, debug=False) + onnx_model = fix_duplicate_tensor_names(onnx_model) # Save onnx.save(onnx_model, self.paths["network_infer"]) diff --git a/onnx4deeploy/transform/quant_transform.py b/onnx4deeploy/transform/quant_transform.py index b7d400c..f3a2763 100644 --- a/onnx4deeploy/transform/quant_transform.py +++ b/onnx4deeploy/transform/quant_transform.py @@ -16,6 +16,7 @@ import argparse import json +from collections import defaultdict from typing import Any, Dict, Tuple, Optional import numpy as np @@ -62,6 +63,93 @@ def _infer_signed_from_zero_point(zp_arr: Optional[np.ndarray]) -> bool: return True +# --------------------------- +# Duplicate tensor name repair +# --------------------------- + +def fix_duplicate_tensor_names(onnx_model: onnx.ModelProto) -> onnx.ModelProto: + """ + Fix duplicate initializer and node output names in an ONNX graph. + + When torch.onnx.export traces a model with multiple quantizers that share + the same name prefix, it can produce: + - Multiple initializers with the same name but different values + - Multiple node outputs with the same name + + Both conditions produce invalid ONNX that ORT rejects. This pass repairs + them by renaming duplicate occurrences to unique names and rewiring + downstream consumers accordingly. + """ + graph = onnx_model.graph + + # --- Step 1: fix duplicate initializers --- + init_occs = defaultdict(list) + for init in graph.initializer: + init_occs[init.name].append(init) + + init_rename = {} + new_inits = [] + for name, occ_list in init_occs.items(): + for k, orig in enumerate(occ_list): + new_tensor = onnx.TensorProto() + new_tensor.CopyFrom(orig) + new_name = name if k == 0 else f"{name}__v{k + 1}" + new_tensor.name = new_name + new_inits.append(new_tensor) + init_rename[(name, k)] = new_name + + del graph.initializer[:] + graph.initializer.extend(new_inits) + + dup_init_names = {name for name, occs in init_occs.items() if len(occs) > 1} + next_occ = defaultdict(int) + for node in graph.node: + updated = list(node.input) + for i, inp in enumerate(node.input): + if inp in dup_init_names: + occ = next_occ[inp] + next_occ[inp] += 1 + updated[i] = init_rename[(inp, min(occ, len(init_occs[inp]) - 1))] + del node.input[:] + node.input.extend(updated) + + # --- Step 2: fix duplicate node outputs --- + produced = set() + active_name: Dict[str, str] = {} + + for node in graph.node: + updated_in = list(node.input) + for i, inp in enumerate(node.input): + if inp in active_name: + updated_in[i] = active_name[inp] + del node.input[:] + node.input.extend(updated_in) + + updated_out = list(node.output) + for i, out in enumerate(node.output): + if not out: + continue + if out in produced: + v = 2 + new_out = f"{out}__v{v}" + while new_out in produced: + v += 1 + new_out = f"{out}__v{v}" + active_name[out] = new_out + updated_out[i] = new_out + produced.add(new_out) + else: + produced.add(out) + del node.output[:] + node.output.extend(updated_out) + + for out_vi in graph.output: + if out_vi.name in active_name: + out_vi.name = active_name[out_vi.name] + + return onnx_model + + # --------------------------- # RequantShift parameterization # --------------------------- @@ -129,11 +217,14 @@ def replace_qdq_with_deeploy(graph: gs.Graph) -> None: bit_width = 8 # Deeploy QuantParser expects bit_width attribute; typical is 8. [2](https://deepwiki.com/pulp-platform/Deeploy/8.1-quantization-and-training-support) # Create Deeploy Quant node + old_outputs = list(node.outputs) + node.inputs.clear() + node.outputs.clear() q = gs.Node( op="Quant", name=(node.name or "Quant") + "_Deeploy", inputs=[x], - outputs=node.outputs, + outputs=old_outputs, attrs={ "scale": float(scale.reshape(-1)[0]) if scale.size == 1 else scale.astype(np.float32), "zero_point": int(zp.reshape(-1)[0]) if (zp is not None and zp.size == 1) else (zp.astype(np.int32) if zp is not None else 0), @@ -158,11 +249,14 @@ def replace_qdq_with_deeploy(graph: gs.Graph) -> None: if scale is None: raise RuntimeError(f"DequantizeLinear node {node.name} has non-constant scale; provide constant initializer.") + old_outputs = list(node.outputs) + node.inputs.clear() + node.outputs.clear() dq = gs.Node( op="Dequant", name=(node.name or "Dequant") + "_Deeploy", inputs=[xq], - outputs=node.outputs, + outputs=old_outputs, attrs={ "scale": float(scale.reshape(-1)[0]) if scale.size == 1 else scale.astype(np.float32), "zero_point": int(zp.reshape(-1)[0]) if (zp is not None and zp.size == 1) else (zp.astype(np.int32) if zp is not None else 0), diff --git a/pyproject.toml b/pyproject.toml index 2acea90..f6a9fc5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,10 +30,11 @@ classifiers = [ ] dependencies = [ - "torch>=2.0.0", + "torch>=2.0.0,<2.7.0", "onnx>=1.15.0,<1.17.0", "onnx-graphsurgeon>=0.5.0", "onnxruntime-training==1.19.2", + "onnxruntime-extensions>=0.15.2", "onnxscript>=0.1.0", "onnxsim>=0.4.0", "numpy>=1.24.0,<2.0.0", @@ -53,6 +54,7 @@ dev = [ visualization = [ "beautifulsoup4>=4.0.0", "pandas>=2.0.0", + "matplotlib>=3.10.8" ] [project.urls] diff --git a/tests/models/conftest.py b/tests/models/conftest.py index fd4589c..88ae067 100644 --- a/tests/models/conftest.py +++ b/tests/models/conftest.py @@ -63,6 +63,19 @@ def epidenet_config(): } +@pytest.fixture +def qlitecnn_config(): + """Default configuration for QLiteCNN quantized model.""" + return { + "batch_size": 1, + "input_channels": 1, + "input_height": 28, + "input_width": 28, + "num_classes": 10, + "opset_version": 17, + } + + @pytest.fixture def mibminet_config(): """Default configuration for MI-BMInet model.""" diff --git a/tests/models/onnx_node_implementations.py b/tests/models/onnx_node_implementations.py new file mode 100644 index 0000000..53fb685 --- /dev/null +++ b/tests/models/onnx_node_implementations.py @@ -0,0 +1,738 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: MIT + +""" +Pure Python / NumPy / PyTorch implementations of ONNX nodes used in +Onnx4Deeploy exports, including custom Deeploy and MeZO operator domains. + +Organised in three sections: + 1. RNG utilities โ€“ Xorshift32 and Ziggurat, matching the C/hardware RNGs + used by the MeZO perturbation operators. + 2. Custom ops โ€“ Deeploy (Quant, Dequant, RequantShift) and MeZO + (PerturbNormal, PerturbUniform, PerturbTriangle, + PerturbRademacher, PerturbEggroll, + RQSPerturbRademacher, RQSPerturbUniform). + 3. Standard ops โ€“ Conv, MaxPool, Gemm, Relu, Flatten, Reshape, Clip, + QuantizeLinear, DequantizeLinear, and friends. + 4. Graph executor โ€“ ``run_onnx_graph`` dispatches every node in an ONNX + model to one of the implementations above and returns + the final graph output as a float32 NumPy array. + +Usage:: + + from tests.models.onnx_node_implementations import run_onnx_graph + + output = run_onnx_graph("model.onnx", {"input": my_input_array}) +""" + +import math +from typing import Any, Dict + +import numpy as np +import onnx +from onnx import numpy_helper, AttributeProto +import torch +import torch.nn.functional as F + + +# =========================================================================== +# Section 1 โ€“ RNG utilities +# =========================================================================== + + +class Xorshift32: + """32-bit xorshift PRNG, matching the hardware implementation.""" + + def __init__(self, seed: int): + self.state = int(seed) if int(seed) != 0 else 1 # zero state is invalid + + def next(self) -> int: + s = self.state + s ^= (s << 13) & 0xFFFFFFFF + s ^= (s >> 17) & 0xFFFFFFFF + s ^= (s << 5) & 0xFFFFFFFF + self.state = s & 0xFFFFFFFF + return self.state + + +class Ziggurat: + """ + Ziggurat normal sampler backed by Xorshift32, matching the hardware RNG + used by PerturbNormal. + + The table initialisation and sampling loop are intentionally identical to + the reference Python implementation in + ``onnx4deeploy/operators/perturbnormal.py`` so that test outputs agree + with what the hardware operator produces. + """ + + N = 256 + R = 3.442619855899 + + def __init__(self, seed: int): + self.rng = Xorshift32(seed) + self.x = np.zeros(self.N + 1) + self.y = np.zeros(self.N) + self.x[0] = self.R + self.x[self.N] = 0.0 + for i in range(1, self.N): + self.x[i] = np.sqrt(-2.0 * np.log(np.exp(-0.5 * self.x[i - 1] ** 2))) + for i in range(self.N): + self.y[i] = np.exp(-0.5 * self.x[i] ** 2) + + def next(self) -> float: + while True: + k = self.rng.next() % self.N + u = self.rng.next() / 0xFFFFFFFF + x = u * (self.x[k] - self.x[k + 1]) + self.x[k + 1] + if u < self.y[k] / self.y[k + 1]: + return x + if x < self.R: + y = math.exp(-0.5 * x * x) + if u * (self.y[k + 1] - self.y[k]) < (y - self.y[k]): + return x + + +def _ziggurat_array(seed: int, idx: int, numel: int, epsilon: float) -> np.ndarray: + """ + Generate ``numel`` Ziggurat-normal samples scaled by ``epsilon``. + + The effective seed is ``seed + idx``, matching the unique-seed convention + in ``zo_transform.py`` (``unique_seed = base_seed + perturbation_counter``). + """ + rng = Ziggurat(int(seed) + int(idx)) + return np.array([rng.next() * epsilon for _ in range(numel)], dtype=np.float32) + + +def _xorshift_rademacher(seed: int, idx: int, numel: int) -> np.ndarray: + """ + Generate ``numel`` Rademacher samples (ยฑ1) using Xorshift32. + + Odd raw value โ†’ +1, even raw value โ†’ -1 (least-significant bit test). + """ + rng = Xorshift32(int(seed) + int(idx)) + return np.array([1 if rng.next() & 1 else -1 for _ in range(numel)], + dtype=np.float32) + + +def _xorshift_uniform(seed: int, idx: int, numel: int, low: float, high: float) -> np.ndarray: + """Generate ``numel`` uniform samples in [low, high] using Xorshift32.""" + rng = Xorshift32(int(seed) + int(idx)) + span = high - low + return np.array([rng.next() / 0xFFFFFFFF * span + low for _ in range(numel)], + dtype=np.float32) + + +# =========================================================================== +# Section 2a โ€“ Deeploy custom ops +# =========================================================================== + + +def exec_quant(x: np.ndarray, attrs: Dict[str, Any]) -> np.ndarray: + """ + Deeploy ``Quant`` node. + + Formula:: + + out = clip(round(x / scale) + zero_point, q_min, q_max) + + where ``[q_min, q_max]`` is ``[-128, 127]`` for signed int8 or + ``[0, 255]`` for unsigned uint8. + """ + scale = float(attrs["scale"]) + zero_point = int(attrs.get("zero_point", 0)) + bit_width = int(attrs.get("bit_width", 8)) + signed = bool(attrs.get("signed", 1)) + + q_min = -(2 ** (bit_width - 1)) if signed else 0 + q_max = (2 ** (bit_width - 1)) - 1 if signed else (2 ** bit_width) - 1 + + out = np.clip(np.round(x / scale).astype(np.int32) + zero_point, q_min, q_max) + return out.astype(np.int8 if signed else np.uint8) + + +def exec_dequant(x: np.ndarray, attrs: Dict[str, Any]) -> np.ndarray: + """ + Deeploy ``Dequant`` node. + + Formula:: + + out = (x - zero_point) * scale + """ + scale = float(attrs["scale"]) + zero_point = int(attrs.get("zero_point", 0)) + return ((x.astype(np.float32) - zero_point) * scale).astype(np.float32) + + +def exec_requant_shift( + x: np.ndarray, + mul: np.ndarray, + add: np.ndarray, + div: np.ndarray, +) -> np.ndarray: + """ + Deeploy ``RequantShift`` node. + + Formula:: + + out = clip(((x * mul) + add) >> log2(div), -128, 127) + + ``div`` is a power-of-two scalar; ``mul`` may be a per-channel vector. + """ + div_val = int(np.asarray(div).flat[0]) + shift = int(round(math.log2(div_val))) if div_val > 1 else 0 + + x64 = x.astype(np.int64) + mul64 = np.asarray(mul, dtype=np.int64) + add_val = int(np.asarray(add).flat[0]) + + # Broadcast per-channel mul across spatial dims + if mul64.ndim > 0 and mul64.size > 1: + shape = [1] * x64.ndim + shape[1] = mul64.size # channel axis = 1 + mul64 = mul64.reshape(shape) + + out = ((x64 * mul64) + add_val) >> shift + return np.clip(out, -128, 127).astype(np.int8) + + +# =========================================================================== +# Section 2b โ€“ MeZO perturbation ops +# =========================================================================== + + +def exec_perturb_normal(x: np.ndarray, attrs: Dict[str, Any]) -> np.ndarray: + """ + MeZO ``PerturbNormal`` node (domain: ``mezo``). + + Formula:: + + y = x + eps * ziggurat_normal(seed + idx) + + The noise array has the same shape as ``x``; the RNG is initialised with + ``seed + idx`` for uniqueness across layers. + + .. note:: + The reference implementation in ``perturbnormal.py`` applies the RNG + in a loop and uses only the **last** sample as a scalar perturbation. + This implementation follows that same behaviour for fidelity. + """ + seed = attrs.get("seed", 42) + idx = attrs.get("idx", 0) + eps = float(attrs.get("eps", 0.01)) + + rng = Ziggurat(int(seed) + int(idx)) + noise = 0.0 + for _ in range(x.size): + noise = rng.next() * eps # scalar: last sample only (matches reference) + return (x.astype(np.float32) + noise).astype(x.dtype) + + +def exec_perturb_uniform(x: np.ndarray, attrs: Dict[str, Any]) -> np.ndarray: + """ + MeZO ``PerturbUniform`` node (domain: ``mezo``). + + Formula:: + + y = x + eps * uniform(low, high, seed + idx) + + Per-element uniform noise with range ``[low, high]``, scaled by ``eps``. + """ + seed = attrs.get("seed", 42) + idx = attrs.get("idx", 0) + eps = float(attrs.get("eps", 0.01)) + low = float(attrs.get("low", -math.sqrt(3))) + high = float(attrs.get("high", math.sqrt(3))) + + noise = _xorshift_uniform(int(seed), int(idx), x.size, low, high) + noise = noise.reshape(x.shape) * eps + return (x.astype(np.float32) + noise).astype(x.dtype) + + +def exec_perturb_triangle(x: np.ndarray, attrs: Dict[str, Any]) -> np.ndarray: + """ + MeZO ``PerturbTriangle`` node (domain: ``mezo``). + + A triangle-distribution perturbation is approximated as the sum of two + independent uniform samples on ``[low/2, high/2]``, scaled by ``eps``. + """ + seed = attrs.get("seed", 42) + idx = attrs.get("idx", 0) + eps = float(attrs.get("eps", 0.01)) + low = float(attrs.get("low", -math.sqrt(6))) + high = float(attrs.get("high", math.sqrt(6))) + + u1 = _xorshift_uniform(int(seed), int(idx), x.size, low / 2, high / 2) + u2 = _xorshift_uniform(int(seed) + 1, int(idx), x.size, low / 2, high / 2) + noise = ((u1 + u2) * eps).reshape(x.shape) + return (x.astype(np.float32) + noise).astype(x.dtype) + + +def exec_perturb_rademacher(x: np.ndarray, attrs: Dict[str, Any]) -> np.ndarray: + """ + MeZO ``PerturbRademacher`` node (domain: ``mezo``). + + Formula:: + + y = x + eps * rademacher(seed + idx) + + Each element of the noise is independently ยฑ1. + """ + seed = attrs.get("seed", 42) + idx = attrs.get("idx", 0) + eps = float(attrs.get("eps", 0.01)) + + noise = _xorshift_rademacher(int(seed), int(idx), x.size).reshape(x.shape) * eps + return (x.astype(np.float32) + noise).astype(x.dtype) + + +def exec_perturb_eggroll(shape_input: np.ndarray, attrs: Dict[str, Any]) -> np.ndarray: + """ + MeZO ``PerturbEggroll`` node (domain: ``com.microsoft``). + + Generates a Rademacher vector whose length is given by ``shape_input`` + (a 1-D INT64 tensor encoding the desired output shape). + + In the eggroll factored perturbation, two such vectors ``a`` and ``b`` + are combined as ``noise = eps * (a @ b.T)`` by a downstream Gemm node; + this function only generates one vector per call. + """ + seed = attrs.get("seed", 42) + idx = attrs.get("idx", 0) + eps = float(attrs.get("eps", 0.01)) + + out_shape = tuple(int(d) for d in shape_input.flat) + numel = int(np.prod(out_shape)) + return _xorshift_rademacher(int(seed), int(idx), numel).reshape(out_shape) + + +def exec_rqs_perturb_rademacher( + x: np.ndarray, mul: np.ndarray, attrs: Dict[str, Any] +) -> np.ndarray: + """ + MeZO ``RQSPerturbRademacher`` node (domain: ``mezo``). + + Quantised Rademacher perturbation:: + + noise = (rademacher(seed + idx) * mul) >> log2(div) + y = clip(x + noise, -(2^(n_bits-1)), 2^(n_bits-1)-1) + + ``mul`` is a per-output-channel scaling factor (int32). + ``div`` is a power-of-two shift (matches the weight quantisation scale). + """ + seed = attrs.get("seed", 42) + idx = attrs.get("idx", 0) + div = int(attrs.get("div", 2 ** 15)) + n_levels = int(attrs.get("n_levels", 256)) + signed = bool(attrs.get("signed", 1)) + + shift = int(round(math.log2(div))) if div > 1 else 0 + q_min = -(n_levels // 2) if signed else 0 + q_max = (n_levels // 2) - 1 if signed else n_levels - 1 + + rad = _xorshift_rademacher(int(seed), int(idx), x.size).reshape(x.shape).astype(np.int32) + mul32 = np.asarray(mul, dtype=np.int32) + + # Broadcast per-channel mul + if mul32.ndim > 0 and mul32.size > 1: + shape = [1] * x.ndim + shape[0] = mul32.size # first axis = output channels for weights + mul32 = mul32.reshape(shape) + + noise = (rad * mul32) >> shift + return np.clip(x.astype(np.int32) + noise, q_min, q_max).astype(x.dtype) + + +def exec_rqs_perturb_uniform( + x: np.ndarray, mul: np.ndarray, attrs: Dict[str, Any] +) -> np.ndarray: + """ + MeZO ``RQSPerturbUniform`` node (domain: ``mezo``). + + Quantised uniform perturbation:: + + noise = (uniform(-1, 1, seed + idx) * mul) >> log2(div) + y = clip(x + noise, -(2^(n_bits-1)), 2^(n_bits-1)-1) + """ + seed = attrs.get("seed", 42) + idx = attrs.get("idx", 0) + div = int(attrs.get("div", 2 ** 15)) + n_levels = int(attrs.get("n_levels", 256)) + signed = bool(attrs.get("signed", 1)) + + shift = int(round(math.log2(div))) if div > 1 else 0 + q_min = -(n_levels // 2) if signed else 0 + q_max = (n_levels // 2) - 1 if signed else n_levels - 1 + + uni = _xorshift_uniform(int(seed), int(idx), x.size, -1.0, 1.0).reshape(x.shape) + mul32 = np.asarray(mul, dtype=np.float32) + + if mul32.ndim > 0 and mul32.size > 1: + shape = [1] * x.ndim + shape[0] = mul32.size + mul32 = mul32.reshape(shape) + + noise = (uni * mul32).astype(np.int32) >> shift + return np.clip(x.astype(np.int32) + noise, q_min, q_max).astype(x.dtype) + + +# =========================================================================== +# Section 3 โ€“ Standard ONNX op implementations +# =========================================================================== + + +def exec_quantize_linear( + x: np.ndarray, scale: np.ndarray, zero_point: np.ndarray, axis: int = 1 +) -> np.ndarray: + """Standard ONNX ``QuantizeLinear``.""" + s = scale.astype(np.float32) + zp = zero_point + q_min, q_max = (-128, 127) if zp.dtype == np.int8 else (0, 255) + + if s.ndim > 0 and s.size > 1: # per-channel: broadcast along the given axis + shape = [1] * x.ndim + shape[axis] = s.size + s = s.reshape(shape) + zp = zp.reshape(shape) + + out = np.clip(np.round(x / s) + zp.astype(np.float32), q_min, q_max) + return out.astype(zp.dtype) + + +def exec_dequantize_linear( + x: np.ndarray, scale: np.ndarray, zero_point: np.ndarray, axis: int = 1 +) -> np.ndarray: + """Standard ONNX ``DequantizeLinear``.""" + s = scale.astype(np.float32) + if s.ndim > 0 and s.size > 1: # per-channel: broadcast along the given axis + shape = [1] * x.ndim + shape[axis] = s.size + s = s.reshape(shape) + zero_point = zero_point.reshape(shape) + return ((x.astype(np.float32) - zero_point.astype(np.float32)) * s).astype(np.float32) + + +def exec_conv(node_inputs: list, attrs: Dict[str, Any]) -> np.ndarray: + """Standard ONNX ``Conv`` via ``torch.nn.functional.conv2d``. + + Handles asymmetric padding by pre-padding with ``F.pad`` before the + convolution (PyTorch ``padding=`` only supports symmetric values). + ONNX pads format: [H_begin, W_begin, H_end, W_end]. + """ + x = torch.from_numpy(node_inputs[0].astype(np.float32)) + w = torch.from_numpy(node_inputs[1].astype(np.float32)) + b = torch.from_numpy(node_inputs[2].astype(np.float32)) if len(node_inputs) > 2 else None + + strides = tuple(attrs.get("strides", [1, 1])) + pads = attrs.get("pads", [0, 0, 0, 0]) # [H_begin, W_begin, H_end, W_end] + dilations = tuple(attrs.get("dilations", [1, 1])) + group = int(attrs.get("group", 1)) + + # F.pad order is (W_begin, W_end, H_begin, H_end) โ€” innermost dim first + x = F.pad(x, (pads[1], pads[3], pads[0], pads[2])) + + return F.conv2d(x, w, bias=b, stride=strides, padding=0, + dilation=dilations, groups=group).numpy() + + +def exec_maxpool(x: np.ndarray, attrs: Dict[str, Any]) -> np.ndarray: + """Standard ONNX ``MaxPool`` via ``torch.nn.functional.max_pool2d``. + + Handles asymmetric padding via ``F.pad`` pre-padding. + ONNX pads format: [H_begin, W_begin, H_end, W_end]. + """ + t = torch.from_numpy(x.astype(np.float32)) + kernel = tuple(attrs["kernel_shape"]) + strides = tuple(attrs.get("strides", kernel)) + pads = attrs.get("pads", [0] * (2 * len(kernel))) + + # F.pad order: (W_begin, W_end, H_begin, H_end) + t = F.pad(t, (pads[1], pads[3], pads[0], pads[2]), value=-float("inf")) + + return F.max_pool2d(t, kernel_size=kernel, stride=strides, padding=0).numpy() + + +def exec_gemm(node_inputs: list, attrs: Dict[str, Any]) -> np.ndarray: + """Standard ONNX ``Gemm``: ``Y = alpha * A @ B + beta * C``.""" + A, B = node_inputs[0].astype(np.float32), node_inputs[1].astype(np.float32) + C = node_inputs[2].astype(np.float32) if len(node_inputs) > 2 else None + alpha = float(attrs.get("alpha", 1.0)) + beta = float(attrs.get("beta", 1.0)) + if attrs.get("transA", 0): + A = A.T + if attrs.get("transB", 0): + B = B.T + out = alpha * (A @ B) + if C is not None: + out += beta * C + return out + + +def exec_relu(x: np.ndarray) -> np.ndarray: + """Standard ONNX ``Relu``.""" + return np.maximum(x, 0).astype(x.dtype) + + +def exec_flatten(x: np.ndarray, attrs: Dict[str, Any]) -> np.ndarray: + """Standard ONNX ``Flatten``.""" + axis = int(attrs.get("axis", 1)) + outer = int(np.prod(x.shape[:axis])) + inner = int(np.prod(x.shape[axis:])) + return x.reshape(outer, inner) + + +def exec_reshape(node_inputs: list, attrs: Dict[str, Any]) -> np.ndarray: + """Standard ONNX ``Reshape``.""" + x = node_inputs[0] + if len(node_inputs) > 1: + shape = [int(s) for s in node_inputs[1].flat] + else: + shape = list(attrs.get("shape", x.shape)) + # Replace 0 with corresponding input dim; -1 is inferred by numpy + resolved = [x.shape[i] if s == 0 and i < x.ndim else s for i, s in enumerate(shape)] + return x.reshape(resolved) + + +def exec_clip(node_inputs: list, attrs: Dict[str, Any]) -> np.ndarray: + """Standard ONNX ``Clip``.""" + x = node_inputs[0].astype(np.float32) + lo = float(node_inputs[1]) if len(node_inputs) > 1 and node_inputs[1] is not None \ + else float(attrs.get("min", -np.inf)) + hi = float(node_inputs[2]) if len(node_inputs) > 2 and node_inputs[2] is not None \ + else float(attrs.get("max", np.inf)) + return np.clip(x, lo, hi) + + +def exec_softmax_cross_entropy_loss( + node_inputs: list, attrs: Dict[str, Any] +) -> np.ndarray: + """ + Standard ONNX ``SoftmaxCrossEntropyLoss`` (appended by ``append_cross_entropy_loss``). + + Returns the mean cross-entropy loss over the batch. + """ + logits = node_inputs[0].astype(np.float32) # (N, C) + labels = node_inputs[1].astype(np.int64).flatten() # (N,) + log_softmax = logits - np.log(np.sum(np.exp(logits), axis=-1, keepdims=True)) + loss = -log_softmax[np.arange(len(labels)), labels] + reduction = attrs.get("reduction", "mean") + return np.array(loss.mean() if reduction == "mean" else loss.sum(), dtype=np.float32) + + +# =========================================================================== +# Section 4 โ€“ ONNX attribute reader helper +# =========================================================================== + + +def _read_attr(attr) -> Any: + """Return the Python value of an ``onnx.AttributeProto``.""" + t = attr.type + if t == AttributeProto.FLOAT: + return attr.f + if t == AttributeProto.INT: + return attr.i + if t == AttributeProto.STRING: + return attr.s.decode() + if t == AttributeProto.TENSOR: + return numpy_helper.to_array(attr.t) + if t == AttributeProto.FLOATS: + return list(attr.floats) + if t == AttributeProto.INTS: + return list(attr.ints) + return None + + +def _node_attrs(node) -> Dict[str, Any]: + return {a.name: _read_attr(a) for a in node.attribute} + + +# =========================================================================== +# Section 5 โ€“ ONNX graph executor +# =========================================================================== + + +def run_onnx_graph( + onnx_path: str, + inputs_dict: Dict[str, np.ndarray], +) -> np.ndarray: + """ + Execute an ONNX graph with pure Python / NumPy / PyTorch. + + Supports: + * Standard ops โ€“ Conv, Relu, Clip, MaxPool, GlobalAveragePool, Gemm, + Reshape, Flatten, Transpose, MatMul, Add, Sub, Mul, Div, Round, Cast, + Identity, Dropout, QuantizeLinear, DequantizeLinear, + SoftmaxCrossEntropyLoss. + * Deeploy custom ops โ€“ Quant, Dequant, RequantShift. + * MeZO perturbation ops โ€“ PerturbNormal, PerturbUniform, PerturbTriangle, + PerturbRademacher, PerturbEggroll, RQSPerturbRademacher, + RQSPerturbUniform. + + Args: + onnx_path: Path to the ONNX model file. + inputs_dict: Mapping from graph-input name to numpy array. + + Returns: + The first graph output as a float32 NumPy array. + + Raises: + NotImplementedError: If the graph contains an op not listed above. + """ + model = onnx.load(onnx_path) + graph = model.graph + + # Build tensor registry: initializers + supplied inputs + tensors: Dict[str, np.ndarray] = {} + for init in graph.initializer: + tensors[init.name] = numpy_helper.to_array(init) + tensors.update(inputs_dict) + + for node in graph.node: + op = node.op_type + attrs = _node_attrs(node) + + # Collect inputs, allowing empty optional inputs to stay None + node_inputs = [ + tensors.get(n, None) for n in node.input + ] + # Compact list (drop trailing Nones, keep internal Nones for optional args) + while node_inputs and node_inputs[-1] is None: + node_inputs.pop() + + # ------------------------------------------------------------------ + # Deeploy custom ops + # ------------------------------------------------------------------ + if op == "Quant": + out = exec_quant(node_inputs[0], attrs) + + elif op == "Dequant": + out = exec_dequant(node_inputs[0], attrs) + + elif op == "RequantShift": + x, mul, add, div = node_inputs + out = exec_requant_shift(x, mul, add, div) + + # ------------------------------------------------------------------ + # MeZO perturbation ops + # ------------------------------------------------------------------ + elif op == "PerturbNormal": + out = exec_perturb_normal(node_inputs[0], attrs) + + elif op == "PerturbUniform": + out = exec_perturb_uniform(node_inputs[0], attrs) + + elif op == "PerturbTriangle": + out = exec_perturb_triangle(node_inputs[0], attrs) + + elif op == "PerturbRademacher": + out = exec_perturb_rademacher(node_inputs[0], attrs) + + elif op == "PerturbEggroll": + # Input is a shape tensor (INT64); output is a Rademacher vector. + out = exec_perturb_eggroll(node_inputs[0], attrs) + + elif op == "RQSPerturbRademacher": + x, mul = node_inputs[0], node_inputs[1] + out = exec_rqs_perturb_rademacher(x, mul, attrs) + + elif op == "RQSPerturbUniform": + x, mul = node_inputs[0], node_inputs[1] + out = exec_rqs_perturb_uniform(x, mul, attrs) + + # ------------------------------------------------------------------ + # Standard ONNX QDQ ops + # ------------------------------------------------------------------ + elif op == "QuantizeLinear": + x = node_inputs[0] + scale = node_inputs[1] if len(node_inputs) > 1 else np.array(1.0, np.float32) + zp = node_inputs[2] if len(node_inputs) > 2 else np.array(0, np.int8) + out = exec_quantize_linear(x, scale, zp, axis=int(attrs.get("axis", 1))) + + elif op == "DequantizeLinear": + x = node_inputs[0] + scale = node_inputs[1] if len(node_inputs) > 1 else np.array(1.0, np.float32) + zp = node_inputs[2] if len(node_inputs) > 2 else np.array(0, np.int8) + out = exec_dequantize_linear(x, scale, zp, axis=int(attrs.get("axis", 1))) + + # ------------------------------------------------------------------ + # Standard ONNX compute ops + # ------------------------------------------------------------------ + elif op == "Conv": + out = exec_conv(node_inputs, attrs) + + elif op == "Relu": + out = exec_relu(node_inputs[0]) + + elif op == "Clip": + out = exec_clip(node_inputs, attrs) + + elif op == "MaxPool": + out = exec_maxpool(node_inputs[0], attrs) + + elif op == "GlobalAveragePool": + out = node_inputs[0].astype(np.float32).mean(axis=(2, 3), keepdims=True) + + elif op == "Gemm": + out = exec_gemm(node_inputs, attrs) + + elif op == "MatMul": + out = (node_inputs[0].astype(np.float32) + @ node_inputs[1].astype(np.float32)) + + elif op == "Add": + out = (node_inputs[0].astype(np.float32) + + node_inputs[1].astype(np.float32)) + + elif op == "Sub": + out = (node_inputs[0].astype(np.float32) + - node_inputs[1].astype(np.float32)) + + elif op == "Mul": + out = (node_inputs[0].astype(np.float32) + * node_inputs[1].astype(np.float32)) + + elif op == "Div": + out = (node_inputs[0].astype(np.float32) + / node_inputs[1].astype(np.float32)) + + elif op == "Round": + out = np.round(node_inputs[0].astype(np.float32)) + + elif op == "Cast": + dtype_map = { + 1: np.float32, 2: np.uint8, 3: np.int8, 5: np.int16, + 6: np.int32, 7: np.int64, 9: bool, 10: np.float16, 11: np.float64, + } + to = int(attrs.get("to", 1)) + out = node_inputs[0].astype(dtype_map[to]) + + elif op == "Flatten": + out = exec_flatten(node_inputs[0], attrs) + + elif op == "Reshape": + out = exec_reshape(node_inputs, attrs) + + elif op == "Transpose": + perm = attrs.get("perm", None) + out = np.transpose(node_inputs[0], perm) + + elif op in ("Identity", "Dropout"): + out = node_inputs[0] + + elif op == "SoftmaxCrossEntropyLoss": + out = exec_softmax_cross_entropy_loss(node_inputs, attrs) + + else: + raise NotImplementedError( + f"Unsupported ONNX op '{op}' (node '{node.name}'). " + "Add an implementation to onnx_node_implementations.py." + ) + + # Store each named output + for out_name, out_val in zip(node.output, [out]): + if out_name: + tensors[out_name] = out_val + + output_name = graph.output[0].name + return tensors[output_name].astype(np.float32) diff --git a/tests/models/test_qlitecnn.py b/tests/models/test_qlitecnn.py new file mode 100644 index 0000000..cae43c4 --- /dev/null +++ b/tests/models/test_qlitecnn.py @@ -0,0 +1,169 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: MIT + +""" +Tests for QLiteCNN quantized model export. + +Tests the full PTQ calibration (Brevitas) โ†’ Onnx4Deeploy export โ†’ numerical +verification pipeline for the QLiteCNN model. + +Numerical verification is performed by ``run_onnx_graph`` from +``onnx_node_implementations``, a pure Python / PyTorch graph executor that +supports standard ONNX ops, Deeploy custom nodes (Quant, Dequant, +RequantShift) and the MeZO perturbation operators (PerturbNormal, etc.). +""" + +import os +import subprocess +import sys + +import numpy as np +import pytest +import torch +from brevitas.quant_tensor import QuantTensor + +from onnx4deeploy.models.pytorch_models.lightweight_cnn import QLiteCNN + +from .onnx_node_implementations import run_onnx_graph +from .test_utils import load_and_check_onnx_model + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +_WEIGHTS_PATH = "onnx4deeploy/models/pytorch_models/lightweight_cnn/qlite_cnn.pth" +_INPUT_SHAPE = (1, 1, 28, 28) +_TOLERANCE = 1.0 / 2**8 # 1/256 โ‰ˆ 0.0039 + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _load_brevitas_model(weights_path: str, num_classes: int = 10) -> torch.nn.Module: + """Load QLiteCNN Brevitas model with pre-calibrated PTQ weights.""" + model = QLiteCNN( + batch_size=1, + input_channels=1, + num_classes=num_classes, + dropout=0.0, + ) + state_dict = torch.load(weights_path, map_location="cpu") + model.load_state_dict(state_dict, strict=False) + model.eval() + return model + + +def _run_brevitas_inference(model: torch.nn.Module, test_input: np.ndarray) -> np.ndarray: + """Run Brevitas model inference and return a float32 numpy array.""" + with torch.no_grad(): + output = model(torch.from_numpy(test_input)) + if isinstance(output, QuantTensor): + output = output.value + return output.numpy().astype(np.float32) + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +@pytest.mark.quantized +class TestQLiteCNNQuantized: + """Test QLiteCNN quantized inference export and numerical correctness.""" + + def test_qlitecnn_ptq_export_and_numerical_correctness( + self, model_test_dir, qlitecnn_config + ): + """ + End-to-end test: PTQ calibration โ†’ Onnx4Deeploy export โ†’ numerical check. + + Steps: + 1. Load the pre-calibrated Brevitas QLiteCNN model. + 2. Run the Onnx4Deeploy ``q-infer`` command to export the model to ONNX. + 3. Loop over 10 random input samples (seeds 0โ€“9) and verify that the + ONNX graph output matches the Brevitas reference within a tolerance + of 1/2^8 for every sample. + """ + project_root = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ) + + # ------------------------------------------------------------------ + # Step 1 โ€“ Load Brevitas model + # ------------------------------------------------------------------ + print("\n[PTQ] Loading QLiteCNN Brevitas model with pre-calibrated weights...") + weights_path = os.path.join(project_root, _WEIGHTS_PATH) + model = _load_brevitas_model( + weights_path, num_classes=qlitecnn_config["num_classes"] + ) + print("[PTQ] Model loaded.") + + # ------------------------------------------------------------------ + # Step 2 โ€“ Run Onnx4Deeploy q-infer export command (once) + # ------------------------------------------------------------------ + cli_script = os.path.join(project_root, "Onnx4Deeploy.py") + cmd = [ + sys.executable, cli_script, + "-model", "QLiteCNN", + "-mode", "q-infer", + "-o", model_test_dir, + ] + print(f"\n[Onnx4Deeploy] Running: {' '.join(cmd)}") + + result = subprocess.run( + cmd, + cwd=project_root, + capture_output=True, + text=True, + ) + if result.returncode != 0: + print(f"[Onnx4Deeploy] stdout:\n{result.stdout}") + print(f"[Onnx4Deeploy] stderr:\n{result.stderr}") + pytest.fail( + f"Onnx4Deeploy command failed with return code {result.returncode}" + ) + + onnx_file = os.path.join(model_test_dir, "network.onnx") + assert os.path.exists(onnx_file), f"ONNX file not found: {onnx_file}" + + # Verify basic ONNX validity (relaxed: skip strict check for custom ops) + load_and_check_onnx_model(onnx_file, skip_shape_check=True) + print(f"[Onnx4Deeploy] Export complete. ONNX saved at: {onnx_file}") + + # ------------------------------------------------------------------ + # Step 3 โ€“ Loop over 10 random inputs and check numerical correctness + # ------------------------------------------------------------------ + print( + "\n[Check] Running numerical check over 10 random input samples " + "(seeds 0โ€“9) ..." + ) + failures = [] + for seed in range(10): + rng = np.random.default_rng(seed) + test_input = rng.standard_normal(_INPUT_SHAPE).astype(np.float32) + + brevitas_output = _run_brevitas_inference(model, test_input) + onnx_output = run_onnx_graph(onnx_file, {"input": test_input}) + + max_diff = float(np.max(np.abs(onnx_output - brevitas_output))) + if max_diff > _TOLERANCE: + failures.append( + f" seed={seed}: max |onnx โˆ’ brevitas| = {max_diff:.6f} " + f"(limit {_TOLERANCE:.6f})\n" + f" Brevitas: {brevitas_output}\n" + f" ONNX: {onnx_output}" + ) + else: + print( + f"[Check] seed={seed} PASSED: " + f"max |onnx โˆ’ brevitas| = {max_diff:.6f} โ‰ค {_TOLERANCE:.6f}" + ) + + if failures: + pytest.fail( + f"Numerical check FAILED for {len(failures)}/10 samples:\n" + + "\n".join(failures) + ) diff --git a/tests/models/test_quant_transform.py b/tests/models/test_quant_transform.py new file mode 100644 index 0000000..e16b622 --- /dev/null +++ b/tests/models/test_quant_transform.py @@ -0,0 +1,525 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: MIT + +""" +Unit tests for individual passes in onnx4deeploy/transform/quant_transform.py. + +Strategy +-------- +Each test builds a *minimal* ONNX graph that exercises exactly one pass, +runs the *original* graph through ORT as the numerical reference, applies +the pass, then runs the *transformed* graph through ``run_onnx_graph`` +(the pure-Python executor in onnx_node_implementations.py) and asserts +the outputs are identical (or within rounding for RequantShift). + +Passes under test +----------------- +1. ``float_to_rqs_params`` pure arithmetic, no ONNX graph needed. +2. ``replace_qdq_with_deeploy`` QuantizeLinear โ†’ Quant +3. ``replace_qdq_with_deeploy`` DequantizeLinear โ†’ Dequant +4. ``replace_qdq_with_deeploy`` QDQ pair end-to-end +5. ``insert_rqs_from_map`` RequantShift insertion +""" + +import io +import tempfile +from pathlib import Path + +import numpy as np +import onnx +import onnx.helper as oh +import onnx.numpy_helper as onph +import onnx_graphsurgeon as gs +import onnxruntime as ort +import pytest + +from onnx4deeploy.transform.quant_transform import ( + float_to_rqs_params, + insert_rqs_from_map, + replace_qdq_with_deeploy, +) +from .onnx_node_implementations import run_onnx_graph, exec_requant_shift + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _save_tmp(model: onnx.ModelProto) -> str: + """Save an ONNX model to a temporary file and return the path.""" + tmp = tempfile.NamedTemporaryFile(suffix=".onnx", delete=False) + tmp.close() + onnx.save(model, tmp.name) + return tmp.name + + +def _ort_run(model_path: str, feed: dict) -> np.ndarray: + """Run a model with ORT and return the first output.""" + sess = ort.InferenceSession(model_path, providers=["CPUExecutionProvider"]) + return sess.run(None, feed)[0] + + +def _gs_to_tmp(graph: gs.Graph) -> str: + """Export a graphsurgeon graph to a temp ONNX file.""" + graph.cleanup().toposort() + return _save_tmp(gs.export_onnx(graph)) + + +# --------------------------------------------------------------------------- +# 1. float_to_rqs_params โ€“ pure arithmetic +# --------------------------------------------------------------------------- + + +class TestFloatToRqsParams: + """Verify that the integer RQS params reproduce the scale ratio.""" + + def _check_ratio(self, ratio: np.ndarray) -> None: + """ + For integer inputs spanning [-128, 127], verify that the RequantShift + formula ``clip(((x * mul) + add) >> shift, -128, 127)`` gives the same + result as the reference ``clip(round(x * ratio), -128, 127)``. + """ + mul, add, div = float_to_rqs_params(ratio) + shift = int(round(np.log2(div))) if div > 1 else 0 + + x = np.arange(-128, 128, dtype=np.int64) + + # Reference: exact float multiplication, rounded and clipped + ref = np.clip(np.round(x * ratio.flatten()[0]), -128, 127).astype(np.int64) + + # RequantShift formula โ€” use int64 to avoid overflow (mul can be up to 2^30) + mul64 = np.asarray(mul, dtype=np.int64).flatten()[0] + rqs = np.clip(((x * mul64) + np.int64(add)) >> shift, -128, 127).astype(np.int64) + + # Allow ยฑ1 LSB rounding difference + assert np.all(np.abs(rqs - ref) <= 1), ( + f"RQS params mismatch for ratio={ratio}: " + f"max diff = {np.max(np.abs(rqs - ref))}" + ) + + def test_scalar_ratio_halving(self): + """scale_src / scale_dst = 0.5 โ†’ requantisation halves the value.""" + self._check_ratio(np.array(0.5)) + + def test_scalar_ratio_doubling(self): + """scale_src / scale_dst = 2.0.""" + self._check_ratio(np.array(2.0)) + + def test_scalar_ratio_identity(self): + """scale_src == scale_dst โ†’ identity (ratio = 1.0).""" + self._check_ratio(np.array(1.0)) + + def test_scalar_ratio_arbitrary(self): + """Non-power-of-two ratio.""" + self._check_ratio(np.array(0.75)) + + def test_per_channel_vector(self): + """Per-channel scale vector: check that mul is a vector.""" + ratio = np.array([0.5, 1.0, 2.0, 0.25], dtype=np.float64) + mul, add, div = float_to_rqs_params(ratio) + assert mul.shape == ratio.shape, "mul must be per-channel" + + def test_div_is_power_of_two(self): + """div must always be a power of two (log2(div) is integer).""" + for r in [0.1, 0.333, 1.5, 3.7, 0.001]: + _, _, div = float_to_rqs_params(np.array(r)) + log2_div = np.log2(div) + assert abs(log2_div - round(log2_div)) < 1e-9, ( + f"div={div} is not a power of two for ratio={r}" + ) + + def test_mul_fits_int32(self): + """mul must fit in int32 for all tested ratios.""" + for r in [0.001, 0.5, 1.0, 100.0, 1234.56]: + mul, _, _ = float_to_rqs_params(np.array(r)) + assert np.all(np.abs(mul) <= 2**31 - 1) + + +# --------------------------------------------------------------------------- +# 2. replace_qdq_with_deeploy โ€“ QuantizeLinear โ†’ Quant +# --------------------------------------------------------------------------- + + +class TestReplaceQuantizeLinear: + """ + QuantizeLinear โ†’ Deeploy Quant. + + Reference: ORT runs the original QuantizeLinear node. + Transformed: run_onnx_graph runs the Quant node. + Expected: identical int8 output. + """ + + def _make_quantize_graph(self, scale: float, zero_point: int = 0) -> str: + """Build Input(float32) โ†’ QuantizeLinear(scale, zp) โ†’ Output(int8).""" + scale_t = oh.make_tensor("scale", onnx.TensorProto.FLOAT, [], [scale]) + zp_t = oh.make_tensor("zp", onnx.TensorProto.INT8, [], [zero_point]) + + node = oh.make_node( + "QuantizeLinear", + inputs=["input", "scale", "zp"], + outputs=["output"], + ) + graph = oh.make_graph( + [node], + "quant_graph", + [oh.make_tensor_value_info("input", onnx.TensorProto.FLOAT, [1, 8])], + [oh.make_tensor_value_info("output", onnx.TensorProto.INT8, [1, 8])], + initializer=[scale_t, zp_t], + ) + model = oh.make_model(graph, opset_imports=[oh.make_opsetid("", 13)]) + return _save_tmp(model) + + def test_quantize_per_tensor_zero_zp(self): + """Per-tensor quantization with zero_point=0.""" + original_path = self._make_quantize_graph(scale=0.5, zero_point=0) + x = np.array([[-1.0, -0.5, 0.0, 0.5, 1.0, 1.5, 2.0, -2.0]], dtype=np.float32) + + reference = _ort_run(original_path, {"input": x}) + + graph = gs.import_onnx(onnx.load(original_path)) + replace_qdq_with_deeploy(graph) + transformed_path = _gs_to_tmp(graph) + + result = run_onnx_graph(transformed_path, {"input": x}) + + np.testing.assert_array_equal( + result.astype(np.int8), reference, + err_msg="Quant output differs from ORT QuantizeLinear (zero_point=0)", + ) + + def test_quantize_per_tensor_nonzero_zp(self): + """Per-tensor quantization with non-zero zero_point.""" + original_path = self._make_quantize_graph(scale=0.25, zero_point=10) + x = np.array([[-1.0, 0.0, 0.5, 1.0, 2.0, -0.5, 0.25, 0.75]], dtype=np.float32) + + reference = _ort_run(original_path, {"input": x}) + + graph = gs.import_onnx(onnx.load(original_path)) + replace_qdq_with_deeploy(graph) + transformed_path = _gs_to_tmp(graph) + + result = run_onnx_graph(transformed_path, {"input": x}) + + np.testing.assert_array_equal( + result.astype(np.int8), reference, + err_msg="Quant output differs from ORT QuantizeLinear (nonzero zp)", + ) + + def test_quantize_saturation(self): + """Values outside the quantization range must saturate to ยฑ127.""" + original_path = self._make_quantize_graph(scale=0.1, zero_point=0) + x = np.array([[-200.0, -100.0, 0.0, 100.0, 200.0, 12.7, -12.8, 0.05]], + dtype=np.float32) + + reference = _ort_run(original_path, {"input": x}) + + graph = gs.import_onnx(onnx.load(original_path)) + replace_qdq_with_deeploy(graph) + transformed_path = _gs_to_tmp(graph) + + result = run_onnx_graph(transformed_path, {"input": x}) + + np.testing.assert_array_equal( + result.astype(np.int8), reference, + err_msg="Quant saturation differs from ORT", + ) + + +# --------------------------------------------------------------------------- +# 3. replace_qdq_with_deeploy โ€“ DequantizeLinear โ†’ Dequant +# --------------------------------------------------------------------------- + + +class TestReplaceDequantizeLinear: + """ + DequantizeLinear โ†’ Deeploy Dequant. + + Reference: ORT runs the original DequantizeLinear node. + Transformed: run_onnx_graph runs the Dequant node. + Expected: identical float32 output. + """ + + def _make_dequantize_graph(self, scale: float, zero_point: int = 0) -> str: + """Build Input(int8) โ†’ DequantizeLinear(scale, zp) โ†’ Output(float32).""" + scale_t = oh.make_tensor("scale", onnx.TensorProto.FLOAT, [], [scale]) + zp_t = oh.make_tensor("zp", onnx.TensorProto.INT8, [], [zero_point]) + + node = oh.make_node( + "DequantizeLinear", + inputs=["input", "scale", "zp"], + outputs=["output"], + ) + graph = oh.make_graph( + [node], + "dequant_graph", + [oh.make_tensor_value_info("input", onnx.TensorProto.INT8, [1, 8])], + [oh.make_tensor_value_info("output", onnx.TensorProto.FLOAT, [1, 8])], + initializer=[scale_t, zp_t], + ) + model = oh.make_model(graph, opset_imports=[oh.make_opsetid("", 13)]) + return _save_tmp(model) + + def test_dequantize_zero_zp(self): + """Per-tensor dequantization with zero_point=0.""" + original_path = self._make_dequantize_graph(scale=0.5, zero_point=0) + x = np.array([[-128, -64, -1, 0, 1, 64, 127, -100]], dtype=np.int8) + + reference = _ort_run(original_path, {"input": x}) + + graph = gs.import_onnx(onnx.load(original_path)) + replace_qdq_with_deeploy(graph) + transformed_path = _gs_to_tmp(graph) + + result = run_onnx_graph(transformed_path, {"input": x}) + + np.testing.assert_allclose( + result, reference, rtol=0, atol=1e-6, + err_msg="Dequant output differs from ORT DequantizeLinear (zero_point=0)", + ) + + def test_dequantize_nonzero_zp(self): + """Per-tensor dequantization with non-zero zero_point.""" + original_path = self._make_dequantize_graph(scale=0.25, zero_point=10) + x = np.array([[-128, -64, -10, 0, 10, 64, 127, -1]], dtype=np.int8) + + reference = _ort_run(original_path, {"input": x}) + + graph = gs.import_onnx(onnx.load(original_path)) + replace_qdq_with_deeploy(graph) + transformed_path = _gs_to_tmp(graph) + + result = run_onnx_graph(transformed_path, {"input": x}) + + np.testing.assert_allclose( + result, reference, rtol=0, atol=1e-6, + err_msg="Dequant output differs from ORT DequantizeLinear (nonzero zp)", + ) + + +# --------------------------------------------------------------------------- +# 4. replace_qdq_with_deeploy โ€“ full QDQ pair end-to-end +# --------------------------------------------------------------------------- + + +class TestReplaceQDQPair: + """ + Input(float32) โ†’ QuantizeLinear โ†’ DequantizeLinear โ†’ Output(float32). + + Reference: ORT runs the original QDQ graph. + Transformed: run_onnx_graph runs the Quant โ†’ Dequant graph. + Expected: identical float32 output (same rounding as ORT). + """ + + def _make_qdq_graph(self, scale: float, zero_point: int = 0) -> str: + scale_t = oh.make_tensor("scale", onnx.TensorProto.FLOAT, [], [scale]) + zp_t = oh.make_tensor("zp", onnx.TensorProto.INT8, [], [zero_point]) + + q_node = oh.make_node( + "QuantizeLinear", + inputs=["input", "scale", "zp"], + outputs=["quantized"], + ) + dq_node = oh.make_node( + "DequantizeLinear", + inputs=["quantized", "scale", "zp"], + outputs=["output"], + ) + graph = oh.make_graph( + [q_node, dq_node], + "qdq_graph", + [oh.make_tensor_value_info("input", onnx.TensorProto.FLOAT, [1, 8])], + [oh.make_tensor_value_info("output", onnx.TensorProto.FLOAT, [1, 8])], + initializer=[scale_t, zp_t], + ) + model = oh.make_model(graph, opset_imports=[oh.make_opsetid("", 13)]) + return _save_tmp(model) + + def test_qdq_roundtrip(self): + """QDQ roundtrip: float โ†’ quantize โ†’ dequantize โ†’ float.""" + original_path = self._make_qdq_graph(scale=0.5, zero_point=0) + x = np.array([[-1.0, -0.5, 0.0, 0.25, 0.5, 1.0, 1.5, 2.0]], dtype=np.float32) + + reference = _ort_run(original_path, {"input": x}) + + graph = gs.import_onnx(onnx.load(original_path)) + replace_qdq_with_deeploy(graph) + transformed_path = _gs_to_tmp(graph) + + result = run_onnx_graph(transformed_path, {"input": x}) + + np.testing.assert_allclose( + result, reference, rtol=0, atol=1e-6, + err_msg="QDQ roundtrip differs from ORT", + ) + + def test_qdq_roundtrip_small_scale(self): + """QDQ with a small scale (high precision).""" + original_path = self._make_qdq_graph(scale=0.00390625, zero_point=0) + x = np.random.default_rng(0).uniform(-0.5, 0.5, (1, 8)).astype(np.float32) + + reference = _ort_run(original_path, {"input": x}) + + graph = gs.import_onnx(onnx.load(original_path)) + replace_qdq_with_deeploy(graph) + transformed_path = _gs_to_tmp(graph) + + result = run_onnx_graph(transformed_path, {"input": x}) + + np.testing.assert_allclose( + result, reference, rtol=0, atol=1e-6, + err_msg="QDQ roundtrip (small scale) differs from ORT", + ) + + +# --------------------------------------------------------------------------- +# 5. insert_rqs_from_map โ€“ RequantShift insertion +# --------------------------------------------------------------------------- + + +class TestInsertRqs: + """ + RequantShift: rescale int8 tensor from scale s_src to scale s_dst. + + Reference: ORT runs QuantizeLinear(s_src) โ†’ DequantizeLinear(s_src) โ†’ + QuantizeLinear(s_dst) โ†’ DequantizeLinear(s_dst) to get the + exact floating-point result after double-quantisation. + Transformed: run_onnx_graph runs a graph where RQS replaces the second + QuantizeLinear, followed by Dequant(s_dst). + Tolerance: ยฑ1 LSB at scale s_dst (RequantShift introduces ยฑ1 rounding vs ORT). + """ + + def _make_double_qdq_graph( + self, scale_src: float, scale_dst: float + ) -> tuple: + """ + Build: float โ†’ QDQ(s_src) โ†’ QDQ(s_dst) โ†’ float + + Returns (ort_path, src_tensor_name, dst_tensor_name, gs_graph) + so that insert_rqs_from_map can be applied to the gs_graph. + """ + s_src_t = oh.make_tensor("s_src", onnx.TensorProto.FLOAT, [], [scale_src]) + zp_src_t = oh.make_tensor("zp_src", onnx.TensorProto.INT8, [], [0]) + s_dst_t = oh.make_tensor("s_dst", onnx.TensorProto.FLOAT, [], [scale_dst]) + zp_dst_t = oh.make_tensor("zp_dst", onnx.TensorProto.INT8, [], [0]) + + nodes = [ + oh.make_node("QuantizeLinear", ["input", "s_src", "zp_src"], ["q_src"]), + oh.make_node("DequantizeLinear", ["q_src", "s_src", "zp_src"], ["dq_src"]), + oh.make_node("QuantizeLinear", ["dq_src", "s_dst", "zp_dst"], ["q_dst"]), + oh.make_node("DequantizeLinear", ["q_dst", "s_dst", "zp_dst"], ["output"]), + ] + graph = oh.make_graph( + nodes, "double_qdq", + [oh.make_tensor_value_info("input", onnx.TensorProto.FLOAT, [1, 8])], + [oh.make_tensor_value_info("output", onnx.TensorProto.FLOAT, [1, 8])], + initializer=[s_src_t, zp_src_t, s_dst_t, zp_dst_t], + ) + model = oh.make_model(graph, opset_imports=[oh.make_opsetid("", 13)]) + ort_path = _save_tmp(model) + return ort_path + + def _make_rqs_target_graph( + self, scale_src: float, scale_dst: float + ) -> str: + """ + Build the graph that insert_rqs_from_map will operate on: + float โ†’ Quant(s_src) โ†’ โ†’ Dequant(s_dst) โ†’ float + + We represent the "gap" as two tensors q_src and q_dst of the same dtype, + connected via an Identity (which will be replaced by RQS after the pass). + """ + s_src_t = oh.make_tensor("s_src", onnx.TensorProto.FLOAT, [], [scale_src]) + zp_src_t = oh.make_tensor("zp_src", onnx.TensorProto.INT8, [], [0]) + s_dst_t = oh.make_tensor("s_dst", onnx.TensorProto.FLOAT, [], [scale_dst]) + zp_dst_t = oh.make_tensor("zp_dst", onnx.TensorProto.INT8, [], [0]) + + nodes = [ + oh.make_node("QuantizeLinear", ["input", "s_src", "zp_src"], ["q_src"]), + oh.make_node("Identity", ["q_src"], ["q_dst"]), + oh.make_node("DequantizeLinear", ["q_dst", "s_dst", "zp_dst"], ["output"]), + ] + graph_proto = oh.make_graph( + nodes, "rqs_target", + [oh.make_tensor_value_info("input", onnx.TensorProto.FLOAT, [1, 8])], + [oh.make_tensor_value_info("output", onnx.TensorProto.FLOAT, [1, 8])], + initializer=[s_src_t, zp_src_t, s_dst_t, zp_dst_t], + ) + model = oh.make_model(graph_proto, opset_imports=[oh.make_opsetid("", 13)]) + return _save_tmp(model) + + def _run_rqs_test(self, scale_src: float, scale_dst: float, seed: int = 42): + rng = np.random.default_rng(seed) + x = rng.uniform(-1.0, 1.0, (1, 8)).astype(np.float32) + + # Reference: ORT double-QDQ (quantize at s_src, requantize to s_dst) + ort_path = self._make_double_qdq_graph(scale_src, scale_dst) + reference = _ort_run(ort_path, {"input": x}) + + # Transformed: insert RequantShift between q_src and q_dst + gs_model_path = self._make_rqs_target_graph(scale_src, scale_dst) + graph = gs.import_onnx(onnx.load(gs_model_path)) + + rqs_map = { + "edges": [{ + "src_tensor": "q_src", + "dst_tensor": "q_dst", + "src_scale": scale_src, + "dst_scale": scale_dst, + }] + } + # First replace QDQ โ†’ Deeploy so the graph has Quant/Dequant nodes + replace_qdq_with_deeploy(graph) + insert_rqs_from_map(graph, rqs_map) + transformed_path = _gs_to_tmp(graph) + + result = run_onnx_graph(transformed_path, {"input": x}) + + # Allow ยฑ1 LSB tolerance at scale_dst (RQS introduces ยฑ1 rounding vs ORT) + atol = scale_dst + np.testing.assert_allclose( + result, reference, rtol=0, atol=atol, + err_msg=( + f"RQS output differs from ORT double-QDQ " + f"(s_src={scale_src}, s_dst={scale_dst}) " + f"by more than 1 LSB.\n" + f" ORT: {reference}\n" + f" RQS: {result}\n" + f" diff: {np.abs(result - reference)}" + ), + ) + + def test_rqs_halving(self): + """Requantise from s=0.5 to s=0.25 (factor 2 downscale).""" + self._run_rqs_test(scale_src=0.5, scale_dst=0.25) + + def test_rqs_doubling(self): + """Requantise from s=0.25 to s=0.5 (factor 2 upscale).""" + self._run_rqs_test(scale_src=0.25, scale_dst=0.5) + + def test_rqs_identity(self): + """Same scale: RequantShift should be a no-op (ยฑ1 LSB).""" + self._run_rqs_test(scale_src=0.5, scale_dst=0.5) + + def test_rqs_arbitrary_ratio(self): + """Non-power-of-two scale ratio.""" + self._run_rqs_test(scale_src=0.03125, scale_dst=0.06395246833562851) + + def test_rqs_params_roundtrip_via_exec(self): + """ + Direct unit test: exec_requant_shift with params from float_to_rqs_params + must match the reference integer requantisation. + """ + scale_src, scale_dst = 0.5, 0.25 + ratio = np.array(scale_src / scale_dst) + mul, add, div = float_to_rqs_params(ratio) + + x = np.arange(-128, 128, dtype=np.int8) + result = exec_requant_shift(x, mul, np.array(add, np.int32), np.array(div, np.int32)) + + reference = np.clip(np.round(x.astype(np.float64) * ratio), -128, 127).astype(np.int8) + + assert np.all(np.abs(result.astype(np.int32) - reference.astype(np.int32)) <= 1), ( + f"exec_requant_shift deviates from reference by more than 1 LSB: " + f"max diff = {np.max(np.abs(result.astype(np.int32) - reference.astype(np.int32)))}" + ) From 029cd78b9da69207dccdace75bb43c105c0f65d5 Mon Sep 17 00:00:00 2001 From: JanCSEM Date: Tue, 31 Mar 2026 09:19:09 +0200 Subject: [PATCH 21/24] Increasing test coverage --- DeepQuant | 2 +- pyproject.toml | 6 +- requirements.txt | 111 +++- tests/models/onnx_node_implementations.py | 736 +--------------------- 4 files changed, 104 insertions(+), 751 deletions(-) diff --git a/DeepQuant b/DeepQuant index 7abc615..7ec8705 160000 --- a/DeepQuant +++ b/DeepQuant @@ -1 +1 @@ -Subproject commit 7abc6154c8dd79978101a66273617b8836511e07 +Subproject commit 7ec87052165be8dac15394d6df39443bac427a62 diff --git a/pyproject.toml b/pyproject.toml index f6a9fc5..81bff22 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,15 +30,16 @@ classifiers = [ ] dependencies = [ - "torch>=2.0.0,<2.7.0", + "torch>=2.0.0", "onnx>=1.15.0,<1.17.0", "onnx-graphsurgeon>=0.5.0", "onnxruntime-training==1.19.2", - "onnxruntime-extensions>=0.15.2", + "onnxruntime-extensions>=0.13.0", "onnxscript>=0.1.0", "onnxsim>=0.4.0", "numpy>=1.24.0,<2.0.0", "pyyaml>=6.0", + "matplotlib>=3.7.0", ] [project.optional-dependencies] @@ -54,7 +55,6 @@ dev = [ visualization = [ "beautifulsoup4>=4.0.0", "pandas>=2.0.0", - "matplotlib>=3.10.8" ] [project.urls] diff --git a/requirements.txt b/requirements.txt index 94060c5..79d1280 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,22 +3,105 @@ # SPDX-License-Identifier: MIT # Core dependencies -torch>=2.0.0 -onnx>=1.15.0,<1.17.0 -onnx-graphsurgeon>=0.5.0 +torch==2.10.0 +onnx==1.16.2 +onnx-graphsurgeon==0.5.8 +onnx-ir==0.2.0 onnxruntime-training==1.19.2 -onnxoptimizer>=0.4.2 -onnxscript>=0.1.0 -onnxsim>=0.4.0 -numpy>=1.24.0,<2.0.0 -pyyaml>=6.0 -brevitas>=0.12.0 -coloroma>=0.4.0 +onnxruntime_extensions==0.13.0 +onnxoptimizer==0.4.2 +onnxscript==0.5.7 +onnxsim==0.6.2 +numpy==1.26.4 +pyyaml==6.0.3 +brevitas==0.12.1 +matplotlib==3.10.8 +deepquant==0.4.3 -# Optional visualization dependencies -# Install with: pip install -e ".[visualization]" -# beautifulsoup4>=4.0.0 -# pandas>=2.0.0 +# CUDA / GPU dependencies +cuda-bindings==12.9.4 +cuda-pathfinder==1.4.2 +nvidia-cublas-cu12==12.8.4.1 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cudnn-cu12==9.10.2.21 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cufile-cu12==1.13.1.3 +nvidia-curand-cu12==10.3.9.90 +nvidia-cusolver-cu12==11.7.3.90 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cusparselt-cu12==0.7.1 +nvidia-nccl-cu12==2.27.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nvshmem-cu12==3.4.5 +nvidia-nvtx-cu12==12.8.90 +triton==3.6.0 + +# General dependencies +astor==0.8.1 +Bottleneck==1.6.0 +Cerberus==1.3.8 +certifi==2026.2.25 +charset-normalizer==3.4.5 +colorama==0.4.6 +colored==2.3.1 +colored-logs==0.2.10 +coloredlogs==15.0.1 +contourpy==1.3.2 +cycler==0.12.1 +dependencies==2.0.1 +empyrical-reloaded==0.5.12 +filelock==3.25.2 +flatbuffers==25.12.19 +fonttools==4.62.0 +fsspec==2026.2.0 +h5py==3.16.0 +hdf5plugin==5.0.0 +humanfriendly==10.0 +idna==3.11 +Jinja2==3.1.6 +jsonpickle==4.1.1 +kiwisolver==1.5.0 +Logbook==1.9.2 +lru-dict==1.3.0 +markdown-it-py==4.0.0 +MarkupSafe==3.0.3 +mdurl==0.1.2 +methodtools==0.4.7 +ml_dtypes==0.5.4 +mpmath==1.3.0 +networkx==3.4.2 +packaging==26.0 +pandas==2.3.3 +patsy==1.0.2 +peewee==3.17.3 +pillow==12.1.1 +prettytable==3.17.0 +protobuf==3.20.3 +pyecharts==2.1.0 +Pygments==2.19.2 +PyJWT==2.11.0 +pyparsing==3.3.2 +python-dateutil==2.9.0.post0 +pytz==2026.1.post1 +requests==2.32.5 +RestrictedPython==8.1 +rich==14.3.3 +scipy==1.15.3 +simplejson==3.20.2 +six==1.17.0 +statsmodels==0.14.6 +sympy==1.14.0 +tabulate==0.10.0 +tqdm==4.67.1 +typing_extensions==4.15.0 +tzdata==2025.3 +unfoldNd==0.2.3 +urllib3==2.6.3 +wcwidth==0.6.0 +websocket-client==1.9.0 +wirerope==1.0.0 # Development dependencies # Install with: pip install -e ".[dev]" diff --git a/tests/models/onnx_node_implementations.py b/tests/models/onnx_node_implementations.py index 53fb685..3cdcbb4 100644 --- a/tests/models/onnx_node_implementations.py +++ b/tests/models/onnx_node_implementations.py @@ -2,737 +2,7 @@ # # SPDX-License-Identifier: MIT -""" -Pure Python / NumPy / PyTorch implementations of ONNX nodes used in -Onnx4Deeploy exports, including custom Deeploy and MeZO operator domains. +"""Re-export from canonical location in DeepQuant.""" -Organised in three sections: - 1. RNG utilities โ€“ Xorshift32 and Ziggurat, matching the C/hardware RNGs - used by the MeZO perturbation operators. - 2. Custom ops โ€“ Deeploy (Quant, Dequant, RequantShift) and MeZO - (PerturbNormal, PerturbUniform, PerturbTriangle, - PerturbRademacher, PerturbEggroll, - RQSPerturbRademacher, RQSPerturbUniform). - 3. Standard ops โ€“ Conv, MaxPool, Gemm, Relu, Flatten, Reshape, Clip, - QuantizeLinear, DequantizeLinear, and friends. - 4. Graph executor โ€“ ``run_onnx_graph`` dispatches every node in an ONNX - model to one of the implementations above and returns - the final graph output as a float32 NumPy array. - -Usage:: - - from tests.models.onnx_node_implementations import run_onnx_graph - - output = run_onnx_graph("model.onnx", {"input": my_input_array}) -""" - -import math -from typing import Any, Dict - -import numpy as np -import onnx -from onnx import numpy_helper, AttributeProto -import torch -import torch.nn.functional as F - - -# =========================================================================== -# Section 1 โ€“ RNG utilities -# =========================================================================== - - -class Xorshift32: - """32-bit xorshift PRNG, matching the hardware implementation.""" - - def __init__(self, seed: int): - self.state = int(seed) if int(seed) != 0 else 1 # zero state is invalid - - def next(self) -> int: - s = self.state - s ^= (s << 13) & 0xFFFFFFFF - s ^= (s >> 17) & 0xFFFFFFFF - s ^= (s << 5) & 0xFFFFFFFF - self.state = s & 0xFFFFFFFF - return self.state - - -class Ziggurat: - """ - Ziggurat normal sampler backed by Xorshift32, matching the hardware RNG - used by PerturbNormal. - - The table initialisation and sampling loop are intentionally identical to - the reference Python implementation in - ``onnx4deeploy/operators/perturbnormal.py`` so that test outputs agree - with what the hardware operator produces. - """ - - N = 256 - R = 3.442619855899 - - def __init__(self, seed: int): - self.rng = Xorshift32(seed) - self.x = np.zeros(self.N + 1) - self.y = np.zeros(self.N) - self.x[0] = self.R - self.x[self.N] = 0.0 - for i in range(1, self.N): - self.x[i] = np.sqrt(-2.0 * np.log(np.exp(-0.5 * self.x[i - 1] ** 2))) - for i in range(self.N): - self.y[i] = np.exp(-0.5 * self.x[i] ** 2) - - def next(self) -> float: - while True: - k = self.rng.next() % self.N - u = self.rng.next() / 0xFFFFFFFF - x = u * (self.x[k] - self.x[k + 1]) + self.x[k + 1] - if u < self.y[k] / self.y[k + 1]: - return x - if x < self.R: - y = math.exp(-0.5 * x * x) - if u * (self.y[k + 1] - self.y[k]) < (y - self.y[k]): - return x - - -def _ziggurat_array(seed: int, idx: int, numel: int, epsilon: float) -> np.ndarray: - """ - Generate ``numel`` Ziggurat-normal samples scaled by ``epsilon``. - - The effective seed is ``seed + idx``, matching the unique-seed convention - in ``zo_transform.py`` (``unique_seed = base_seed + perturbation_counter``). - """ - rng = Ziggurat(int(seed) + int(idx)) - return np.array([rng.next() * epsilon for _ in range(numel)], dtype=np.float32) - - -def _xorshift_rademacher(seed: int, idx: int, numel: int) -> np.ndarray: - """ - Generate ``numel`` Rademacher samples (ยฑ1) using Xorshift32. - - Odd raw value โ†’ +1, even raw value โ†’ -1 (least-significant bit test). - """ - rng = Xorshift32(int(seed) + int(idx)) - return np.array([1 if rng.next() & 1 else -1 for _ in range(numel)], - dtype=np.float32) - - -def _xorshift_uniform(seed: int, idx: int, numel: int, low: float, high: float) -> np.ndarray: - """Generate ``numel`` uniform samples in [low, high] using Xorshift32.""" - rng = Xorshift32(int(seed) + int(idx)) - span = high - low - return np.array([rng.next() / 0xFFFFFFFF * span + low for _ in range(numel)], - dtype=np.float32) - - -# =========================================================================== -# Section 2a โ€“ Deeploy custom ops -# =========================================================================== - - -def exec_quant(x: np.ndarray, attrs: Dict[str, Any]) -> np.ndarray: - """ - Deeploy ``Quant`` node. - - Formula:: - - out = clip(round(x / scale) + zero_point, q_min, q_max) - - where ``[q_min, q_max]`` is ``[-128, 127]`` for signed int8 or - ``[0, 255]`` for unsigned uint8. - """ - scale = float(attrs["scale"]) - zero_point = int(attrs.get("zero_point", 0)) - bit_width = int(attrs.get("bit_width", 8)) - signed = bool(attrs.get("signed", 1)) - - q_min = -(2 ** (bit_width - 1)) if signed else 0 - q_max = (2 ** (bit_width - 1)) - 1 if signed else (2 ** bit_width) - 1 - - out = np.clip(np.round(x / scale).astype(np.int32) + zero_point, q_min, q_max) - return out.astype(np.int8 if signed else np.uint8) - - -def exec_dequant(x: np.ndarray, attrs: Dict[str, Any]) -> np.ndarray: - """ - Deeploy ``Dequant`` node. - - Formula:: - - out = (x - zero_point) * scale - """ - scale = float(attrs["scale"]) - zero_point = int(attrs.get("zero_point", 0)) - return ((x.astype(np.float32) - zero_point) * scale).astype(np.float32) - - -def exec_requant_shift( - x: np.ndarray, - mul: np.ndarray, - add: np.ndarray, - div: np.ndarray, -) -> np.ndarray: - """ - Deeploy ``RequantShift`` node. - - Formula:: - - out = clip(((x * mul) + add) >> log2(div), -128, 127) - - ``div`` is a power-of-two scalar; ``mul`` may be a per-channel vector. - """ - div_val = int(np.asarray(div).flat[0]) - shift = int(round(math.log2(div_val))) if div_val > 1 else 0 - - x64 = x.astype(np.int64) - mul64 = np.asarray(mul, dtype=np.int64) - add_val = int(np.asarray(add).flat[0]) - - # Broadcast per-channel mul across spatial dims - if mul64.ndim > 0 and mul64.size > 1: - shape = [1] * x64.ndim - shape[1] = mul64.size # channel axis = 1 - mul64 = mul64.reshape(shape) - - out = ((x64 * mul64) + add_val) >> shift - return np.clip(out, -128, 127).astype(np.int8) - - -# =========================================================================== -# Section 2b โ€“ MeZO perturbation ops -# =========================================================================== - - -def exec_perturb_normal(x: np.ndarray, attrs: Dict[str, Any]) -> np.ndarray: - """ - MeZO ``PerturbNormal`` node (domain: ``mezo``). - - Formula:: - - y = x + eps * ziggurat_normal(seed + idx) - - The noise array has the same shape as ``x``; the RNG is initialised with - ``seed + idx`` for uniqueness across layers. - - .. note:: - The reference implementation in ``perturbnormal.py`` applies the RNG - in a loop and uses only the **last** sample as a scalar perturbation. - This implementation follows that same behaviour for fidelity. - """ - seed = attrs.get("seed", 42) - idx = attrs.get("idx", 0) - eps = float(attrs.get("eps", 0.01)) - - rng = Ziggurat(int(seed) + int(idx)) - noise = 0.0 - for _ in range(x.size): - noise = rng.next() * eps # scalar: last sample only (matches reference) - return (x.astype(np.float32) + noise).astype(x.dtype) - - -def exec_perturb_uniform(x: np.ndarray, attrs: Dict[str, Any]) -> np.ndarray: - """ - MeZO ``PerturbUniform`` node (domain: ``mezo``). - - Formula:: - - y = x + eps * uniform(low, high, seed + idx) - - Per-element uniform noise with range ``[low, high]``, scaled by ``eps``. - """ - seed = attrs.get("seed", 42) - idx = attrs.get("idx", 0) - eps = float(attrs.get("eps", 0.01)) - low = float(attrs.get("low", -math.sqrt(3))) - high = float(attrs.get("high", math.sqrt(3))) - - noise = _xorshift_uniform(int(seed), int(idx), x.size, low, high) - noise = noise.reshape(x.shape) * eps - return (x.astype(np.float32) + noise).astype(x.dtype) - - -def exec_perturb_triangle(x: np.ndarray, attrs: Dict[str, Any]) -> np.ndarray: - """ - MeZO ``PerturbTriangle`` node (domain: ``mezo``). - - A triangle-distribution perturbation is approximated as the sum of two - independent uniform samples on ``[low/2, high/2]``, scaled by ``eps``. - """ - seed = attrs.get("seed", 42) - idx = attrs.get("idx", 0) - eps = float(attrs.get("eps", 0.01)) - low = float(attrs.get("low", -math.sqrt(6))) - high = float(attrs.get("high", math.sqrt(6))) - - u1 = _xorshift_uniform(int(seed), int(idx), x.size, low / 2, high / 2) - u2 = _xorshift_uniform(int(seed) + 1, int(idx), x.size, low / 2, high / 2) - noise = ((u1 + u2) * eps).reshape(x.shape) - return (x.astype(np.float32) + noise).astype(x.dtype) - - -def exec_perturb_rademacher(x: np.ndarray, attrs: Dict[str, Any]) -> np.ndarray: - """ - MeZO ``PerturbRademacher`` node (domain: ``mezo``). - - Formula:: - - y = x + eps * rademacher(seed + idx) - - Each element of the noise is independently ยฑ1. - """ - seed = attrs.get("seed", 42) - idx = attrs.get("idx", 0) - eps = float(attrs.get("eps", 0.01)) - - noise = _xorshift_rademacher(int(seed), int(idx), x.size).reshape(x.shape) * eps - return (x.astype(np.float32) + noise).astype(x.dtype) - - -def exec_perturb_eggroll(shape_input: np.ndarray, attrs: Dict[str, Any]) -> np.ndarray: - """ - MeZO ``PerturbEggroll`` node (domain: ``com.microsoft``). - - Generates a Rademacher vector whose length is given by ``shape_input`` - (a 1-D INT64 tensor encoding the desired output shape). - - In the eggroll factored perturbation, two such vectors ``a`` and ``b`` - are combined as ``noise = eps * (a @ b.T)`` by a downstream Gemm node; - this function only generates one vector per call. - """ - seed = attrs.get("seed", 42) - idx = attrs.get("idx", 0) - eps = float(attrs.get("eps", 0.01)) - - out_shape = tuple(int(d) for d in shape_input.flat) - numel = int(np.prod(out_shape)) - return _xorshift_rademacher(int(seed), int(idx), numel).reshape(out_shape) - - -def exec_rqs_perturb_rademacher( - x: np.ndarray, mul: np.ndarray, attrs: Dict[str, Any] -) -> np.ndarray: - """ - MeZO ``RQSPerturbRademacher`` node (domain: ``mezo``). - - Quantised Rademacher perturbation:: - - noise = (rademacher(seed + idx) * mul) >> log2(div) - y = clip(x + noise, -(2^(n_bits-1)), 2^(n_bits-1)-1) - - ``mul`` is a per-output-channel scaling factor (int32). - ``div`` is a power-of-two shift (matches the weight quantisation scale). - """ - seed = attrs.get("seed", 42) - idx = attrs.get("idx", 0) - div = int(attrs.get("div", 2 ** 15)) - n_levels = int(attrs.get("n_levels", 256)) - signed = bool(attrs.get("signed", 1)) - - shift = int(round(math.log2(div))) if div > 1 else 0 - q_min = -(n_levels // 2) if signed else 0 - q_max = (n_levels // 2) - 1 if signed else n_levels - 1 - - rad = _xorshift_rademacher(int(seed), int(idx), x.size).reshape(x.shape).astype(np.int32) - mul32 = np.asarray(mul, dtype=np.int32) - - # Broadcast per-channel mul - if mul32.ndim > 0 and mul32.size > 1: - shape = [1] * x.ndim - shape[0] = mul32.size # first axis = output channels for weights - mul32 = mul32.reshape(shape) - - noise = (rad * mul32) >> shift - return np.clip(x.astype(np.int32) + noise, q_min, q_max).astype(x.dtype) - - -def exec_rqs_perturb_uniform( - x: np.ndarray, mul: np.ndarray, attrs: Dict[str, Any] -) -> np.ndarray: - """ - MeZO ``RQSPerturbUniform`` node (domain: ``mezo``). - - Quantised uniform perturbation:: - - noise = (uniform(-1, 1, seed + idx) * mul) >> log2(div) - y = clip(x + noise, -(2^(n_bits-1)), 2^(n_bits-1)-1) - """ - seed = attrs.get("seed", 42) - idx = attrs.get("idx", 0) - div = int(attrs.get("div", 2 ** 15)) - n_levels = int(attrs.get("n_levels", 256)) - signed = bool(attrs.get("signed", 1)) - - shift = int(round(math.log2(div))) if div > 1 else 0 - q_min = -(n_levels // 2) if signed else 0 - q_max = (n_levels // 2) - 1 if signed else n_levels - 1 - - uni = _xorshift_uniform(int(seed), int(idx), x.size, -1.0, 1.0).reshape(x.shape) - mul32 = np.asarray(mul, dtype=np.float32) - - if mul32.ndim > 0 and mul32.size > 1: - shape = [1] * x.ndim - shape[0] = mul32.size - mul32 = mul32.reshape(shape) - - noise = (uni * mul32).astype(np.int32) >> shift - return np.clip(x.astype(np.int32) + noise, q_min, q_max).astype(x.dtype) - - -# =========================================================================== -# Section 3 โ€“ Standard ONNX op implementations -# =========================================================================== - - -def exec_quantize_linear( - x: np.ndarray, scale: np.ndarray, zero_point: np.ndarray, axis: int = 1 -) -> np.ndarray: - """Standard ONNX ``QuantizeLinear``.""" - s = scale.astype(np.float32) - zp = zero_point - q_min, q_max = (-128, 127) if zp.dtype == np.int8 else (0, 255) - - if s.ndim > 0 and s.size > 1: # per-channel: broadcast along the given axis - shape = [1] * x.ndim - shape[axis] = s.size - s = s.reshape(shape) - zp = zp.reshape(shape) - - out = np.clip(np.round(x / s) + zp.astype(np.float32), q_min, q_max) - return out.astype(zp.dtype) - - -def exec_dequantize_linear( - x: np.ndarray, scale: np.ndarray, zero_point: np.ndarray, axis: int = 1 -) -> np.ndarray: - """Standard ONNX ``DequantizeLinear``.""" - s = scale.astype(np.float32) - if s.ndim > 0 and s.size > 1: # per-channel: broadcast along the given axis - shape = [1] * x.ndim - shape[axis] = s.size - s = s.reshape(shape) - zero_point = zero_point.reshape(shape) - return ((x.astype(np.float32) - zero_point.astype(np.float32)) * s).astype(np.float32) - - -def exec_conv(node_inputs: list, attrs: Dict[str, Any]) -> np.ndarray: - """Standard ONNX ``Conv`` via ``torch.nn.functional.conv2d``. - - Handles asymmetric padding by pre-padding with ``F.pad`` before the - convolution (PyTorch ``padding=`` only supports symmetric values). - ONNX pads format: [H_begin, W_begin, H_end, W_end]. - """ - x = torch.from_numpy(node_inputs[0].astype(np.float32)) - w = torch.from_numpy(node_inputs[1].astype(np.float32)) - b = torch.from_numpy(node_inputs[2].astype(np.float32)) if len(node_inputs) > 2 else None - - strides = tuple(attrs.get("strides", [1, 1])) - pads = attrs.get("pads", [0, 0, 0, 0]) # [H_begin, W_begin, H_end, W_end] - dilations = tuple(attrs.get("dilations", [1, 1])) - group = int(attrs.get("group", 1)) - - # F.pad order is (W_begin, W_end, H_begin, H_end) โ€” innermost dim first - x = F.pad(x, (pads[1], pads[3], pads[0], pads[2])) - - return F.conv2d(x, w, bias=b, stride=strides, padding=0, - dilation=dilations, groups=group).numpy() - - -def exec_maxpool(x: np.ndarray, attrs: Dict[str, Any]) -> np.ndarray: - """Standard ONNX ``MaxPool`` via ``torch.nn.functional.max_pool2d``. - - Handles asymmetric padding via ``F.pad`` pre-padding. - ONNX pads format: [H_begin, W_begin, H_end, W_end]. - """ - t = torch.from_numpy(x.astype(np.float32)) - kernel = tuple(attrs["kernel_shape"]) - strides = tuple(attrs.get("strides", kernel)) - pads = attrs.get("pads", [0] * (2 * len(kernel))) - - # F.pad order: (W_begin, W_end, H_begin, H_end) - t = F.pad(t, (pads[1], pads[3], pads[0], pads[2]), value=-float("inf")) - - return F.max_pool2d(t, kernel_size=kernel, stride=strides, padding=0).numpy() - - -def exec_gemm(node_inputs: list, attrs: Dict[str, Any]) -> np.ndarray: - """Standard ONNX ``Gemm``: ``Y = alpha * A @ B + beta * C``.""" - A, B = node_inputs[0].astype(np.float32), node_inputs[1].astype(np.float32) - C = node_inputs[2].astype(np.float32) if len(node_inputs) > 2 else None - alpha = float(attrs.get("alpha", 1.0)) - beta = float(attrs.get("beta", 1.0)) - if attrs.get("transA", 0): - A = A.T - if attrs.get("transB", 0): - B = B.T - out = alpha * (A @ B) - if C is not None: - out += beta * C - return out - - -def exec_relu(x: np.ndarray) -> np.ndarray: - """Standard ONNX ``Relu``.""" - return np.maximum(x, 0).astype(x.dtype) - - -def exec_flatten(x: np.ndarray, attrs: Dict[str, Any]) -> np.ndarray: - """Standard ONNX ``Flatten``.""" - axis = int(attrs.get("axis", 1)) - outer = int(np.prod(x.shape[:axis])) - inner = int(np.prod(x.shape[axis:])) - return x.reshape(outer, inner) - - -def exec_reshape(node_inputs: list, attrs: Dict[str, Any]) -> np.ndarray: - """Standard ONNX ``Reshape``.""" - x = node_inputs[0] - if len(node_inputs) > 1: - shape = [int(s) for s in node_inputs[1].flat] - else: - shape = list(attrs.get("shape", x.shape)) - # Replace 0 with corresponding input dim; -1 is inferred by numpy - resolved = [x.shape[i] if s == 0 and i < x.ndim else s for i, s in enumerate(shape)] - return x.reshape(resolved) - - -def exec_clip(node_inputs: list, attrs: Dict[str, Any]) -> np.ndarray: - """Standard ONNX ``Clip``.""" - x = node_inputs[0].astype(np.float32) - lo = float(node_inputs[1]) if len(node_inputs) > 1 and node_inputs[1] is not None \ - else float(attrs.get("min", -np.inf)) - hi = float(node_inputs[2]) if len(node_inputs) > 2 and node_inputs[2] is not None \ - else float(attrs.get("max", np.inf)) - return np.clip(x, lo, hi) - - -def exec_softmax_cross_entropy_loss( - node_inputs: list, attrs: Dict[str, Any] -) -> np.ndarray: - """ - Standard ONNX ``SoftmaxCrossEntropyLoss`` (appended by ``append_cross_entropy_loss``). - - Returns the mean cross-entropy loss over the batch. - """ - logits = node_inputs[0].astype(np.float32) # (N, C) - labels = node_inputs[1].astype(np.int64).flatten() # (N,) - log_softmax = logits - np.log(np.sum(np.exp(logits), axis=-1, keepdims=True)) - loss = -log_softmax[np.arange(len(labels)), labels] - reduction = attrs.get("reduction", "mean") - return np.array(loss.mean() if reduction == "mean" else loss.sum(), dtype=np.float32) - - -# =========================================================================== -# Section 4 โ€“ ONNX attribute reader helper -# =========================================================================== - - -def _read_attr(attr) -> Any: - """Return the Python value of an ``onnx.AttributeProto``.""" - t = attr.type - if t == AttributeProto.FLOAT: - return attr.f - if t == AttributeProto.INT: - return attr.i - if t == AttributeProto.STRING: - return attr.s.decode() - if t == AttributeProto.TENSOR: - return numpy_helper.to_array(attr.t) - if t == AttributeProto.FLOATS: - return list(attr.floats) - if t == AttributeProto.INTS: - return list(attr.ints) - return None - - -def _node_attrs(node) -> Dict[str, Any]: - return {a.name: _read_attr(a) for a in node.attribute} - - -# =========================================================================== -# Section 5 โ€“ ONNX graph executor -# =========================================================================== - - -def run_onnx_graph( - onnx_path: str, - inputs_dict: Dict[str, np.ndarray], -) -> np.ndarray: - """ - Execute an ONNX graph with pure Python / NumPy / PyTorch. - - Supports: - * Standard ops โ€“ Conv, Relu, Clip, MaxPool, GlobalAveragePool, Gemm, - Reshape, Flatten, Transpose, MatMul, Add, Sub, Mul, Div, Round, Cast, - Identity, Dropout, QuantizeLinear, DequantizeLinear, - SoftmaxCrossEntropyLoss. - * Deeploy custom ops โ€“ Quant, Dequant, RequantShift. - * MeZO perturbation ops โ€“ PerturbNormal, PerturbUniform, PerturbTriangle, - PerturbRademacher, PerturbEggroll, RQSPerturbRademacher, - RQSPerturbUniform. - - Args: - onnx_path: Path to the ONNX model file. - inputs_dict: Mapping from graph-input name to numpy array. - - Returns: - The first graph output as a float32 NumPy array. - - Raises: - NotImplementedError: If the graph contains an op not listed above. - """ - model = onnx.load(onnx_path) - graph = model.graph - - # Build tensor registry: initializers + supplied inputs - tensors: Dict[str, np.ndarray] = {} - for init in graph.initializer: - tensors[init.name] = numpy_helper.to_array(init) - tensors.update(inputs_dict) - - for node in graph.node: - op = node.op_type - attrs = _node_attrs(node) - - # Collect inputs, allowing empty optional inputs to stay None - node_inputs = [ - tensors.get(n, None) for n in node.input - ] - # Compact list (drop trailing Nones, keep internal Nones for optional args) - while node_inputs and node_inputs[-1] is None: - node_inputs.pop() - - # ------------------------------------------------------------------ - # Deeploy custom ops - # ------------------------------------------------------------------ - if op == "Quant": - out = exec_quant(node_inputs[0], attrs) - - elif op == "Dequant": - out = exec_dequant(node_inputs[0], attrs) - - elif op == "RequantShift": - x, mul, add, div = node_inputs - out = exec_requant_shift(x, mul, add, div) - - # ------------------------------------------------------------------ - # MeZO perturbation ops - # ------------------------------------------------------------------ - elif op == "PerturbNormal": - out = exec_perturb_normal(node_inputs[0], attrs) - - elif op == "PerturbUniform": - out = exec_perturb_uniform(node_inputs[0], attrs) - - elif op == "PerturbTriangle": - out = exec_perturb_triangle(node_inputs[0], attrs) - - elif op == "PerturbRademacher": - out = exec_perturb_rademacher(node_inputs[0], attrs) - - elif op == "PerturbEggroll": - # Input is a shape tensor (INT64); output is a Rademacher vector. - out = exec_perturb_eggroll(node_inputs[0], attrs) - - elif op == "RQSPerturbRademacher": - x, mul = node_inputs[0], node_inputs[1] - out = exec_rqs_perturb_rademacher(x, mul, attrs) - - elif op == "RQSPerturbUniform": - x, mul = node_inputs[0], node_inputs[1] - out = exec_rqs_perturb_uniform(x, mul, attrs) - - # ------------------------------------------------------------------ - # Standard ONNX QDQ ops - # ------------------------------------------------------------------ - elif op == "QuantizeLinear": - x = node_inputs[0] - scale = node_inputs[1] if len(node_inputs) > 1 else np.array(1.0, np.float32) - zp = node_inputs[2] if len(node_inputs) > 2 else np.array(0, np.int8) - out = exec_quantize_linear(x, scale, zp, axis=int(attrs.get("axis", 1))) - - elif op == "DequantizeLinear": - x = node_inputs[0] - scale = node_inputs[1] if len(node_inputs) > 1 else np.array(1.0, np.float32) - zp = node_inputs[2] if len(node_inputs) > 2 else np.array(0, np.int8) - out = exec_dequantize_linear(x, scale, zp, axis=int(attrs.get("axis", 1))) - - # ------------------------------------------------------------------ - # Standard ONNX compute ops - # ------------------------------------------------------------------ - elif op == "Conv": - out = exec_conv(node_inputs, attrs) - - elif op == "Relu": - out = exec_relu(node_inputs[0]) - - elif op == "Clip": - out = exec_clip(node_inputs, attrs) - - elif op == "MaxPool": - out = exec_maxpool(node_inputs[0], attrs) - - elif op == "GlobalAveragePool": - out = node_inputs[0].astype(np.float32).mean(axis=(2, 3), keepdims=True) - - elif op == "Gemm": - out = exec_gemm(node_inputs, attrs) - - elif op == "MatMul": - out = (node_inputs[0].astype(np.float32) - @ node_inputs[1].astype(np.float32)) - - elif op == "Add": - out = (node_inputs[0].astype(np.float32) - + node_inputs[1].astype(np.float32)) - - elif op == "Sub": - out = (node_inputs[0].astype(np.float32) - - node_inputs[1].astype(np.float32)) - - elif op == "Mul": - out = (node_inputs[0].astype(np.float32) - * node_inputs[1].astype(np.float32)) - - elif op == "Div": - out = (node_inputs[0].astype(np.float32) - / node_inputs[1].astype(np.float32)) - - elif op == "Round": - out = np.round(node_inputs[0].astype(np.float32)) - - elif op == "Cast": - dtype_map = { - 1: np.float32, 2: np.uint8, 3: np.int8, 5: np.int16, - 6: np.int32, 7: np.int64, 9: bool, 10: np.float16, 11: np.float64, - } - to = int(attrs.get("to", 1)) - out = node_inputs[0].astype(dtype_map[to]) - - elif op == "Flatten": - out = exec_flatten(node_inputs[0], attrs) - - elif op == "Reshape": - out = exec_reshape(node_inputs, attrs) - - elif op == "Transpose": - perm = attrs.get("perm", None) - out = np.transpose(node_inputs[0], perm) - - elif op in ("Identity", "Dropout"): - out = node_inputs[0] - - elif op == "SoftmaxCrossEntropyLoss": - out = exec_softmax_cross_entropy_loss(node_inputs, attrs) - - else: - raise NotImplementedError( - f"Unsupported ONNX op '{op}' (node '{node.name}'). " - "Add an implementation to onnx_node_implementations.py." - ) - - # Store each named output - for out_name, out_val in zip(node.output, [out]): - if out_name: - tensors[out_name] = out_val - - output_name = graph.output[0].name - return tensors[output_name].astype(np.float32) +from DeepQuant.onnx_node_implementations import * # noqa: F401, F403 +from DeepQuant.onnx_node_implementations import run_onnx_graph # noqa: F401 From a13abe728a52dc799911662d9fed61d65bbd5692 Mon Sep 17 00:00:00 2001 From: JanCSEM Date: Fri, 17 Apr 2026 11:45:46 +0000 Subject: [PATCH 22/24] Add tests --- pyproject.toml | 2 + tests/models/onnx_node_implementations.py | 792 ++++++++++++++++++++- tests/models/test_inference_consistency.py | 300 ++++++++ tests/models/test_zo_perturbation.py | 419 +++++++++++ 4 files changed, 1510 insertions(+), 3 deletions(-) create mode 100644 tests/models/test_inference_consistency.py create mode 100644 tests/models/test_zo_perturbation.py diff --git a/pyproject.toml b/pyproject.toml index 81bff22..d95c4ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -107,6 +107,8 @@ markers = [ "baseline: tests that require baseline comparison", "inference: marks tests for inference mode", "training: marks tests for training mode", + "quantized: marks tests for quantized models", + "zo: marks tests for zeroth-order perturbation", ] [tool.isort] diff --git a/tests/models/onnx_node_implementations.py b/tests/models/onnx_node_implementations.py index 3cdcbb4..686de66 100644 --- a/tests/models/onnx_node_implementations.py +++ b/tests/models/onnx_node_implementations.py @@ -2,7 +2,793 @@ # # SPDX-License-Identifier: MIT -"""Re-export from canonical location in DeepQuant.""" +""" +Pure-Python ONNX graph executor. -from DeepQuant.onnx_node_implementations import * # noqa: F401, F403 -from DeepQuant.onnx_node_implementations import run_onnx_graph # noqa: F401 +Executes ONNX graphs without relying on onnxruntime.InferenceSession, so that +graphs containing custom ops (Deeploy Quant/Dequant/RequantShift and MeZO +perturbation ops) can be run natively. + +Supported op domains: + - "" (standard ONNX ops) + - "ai.onnx.contrib" โ†’ Quant, Dequant, RequantShift + - "mezo" โ†’ PerturbNormal, PerturbUniform, PerturbRademacher, + PerturbTriangle, PerturbEggroll, + RQSPerturbRademacher, RQSPerturbUniform +""" + +from __future__ import annotations + +import math +from typing import Any, Dict, List, Optional + +import numpy as np +import onnx +from onnx import numpy_helper, TensorProto + +# --------------------------------------------------------------------------- +# RNG reference matching Deeploy C++ implementation +# --------------------------------------------------------------------------- + +NUM_CORES: int = 8 + + +def _scramble(seed: int) -> np.uint32: + """seed * 1664525 + 1013904223 (mod 2**32, 32-bit LCG scramble).""" + return np.uint32(np.uint32(seed) * np.uint32(1664525) + np.uint32(1013904223)) + + +def _xorshift32(state: np.uint32) -> np.uint32: + """One Xorshift32 step.""" + state = np.uint32(state) + state ^= np.uint32(state << np.uint32(13)) + state ^= np.uint32(state >> np.uint32(17)) + state ^= np.uint32(state << np.uint32(5)) + return state + + +def _uint32_to_signed_float(u: np.uint32) -> float: + """Map uint32 uniformly to (-1, 1).""" + return float(u) / float(np.iinfo(np.uint32).max) * 2.0 - 1.0 + + +def _perturb_uniform( + data: np.ndarray, + global_seed: int, + node_id: int, + eps: float, + sign: int = 1, +) -> np.ndarray: + """ + Uniform perturbation matching Deeploy PerturbUniform kernel. + + seed per core = scramble(global_seed + NUM_CORES*node_id + core_id) + rand in (-1, 1) scaled by eps (the ONNX node attribute already encodes + the full scale, e.g. epsilon*2*sqrt(3)). + """ + flat = data.flatten().astype(np.float32) + size = flat.size + log2core = int(math.log2(NUM_CORES)) + + for core_id in range(NUM_CORES): + chunk = (size >> log2core) + (1 if (size & (NUM_CORES - 1)) else 0) + chunk_start = min(chunk * core_id, size) + chunk_stop = min(chunk_start + chunk, size) + + seed = _scramble(global_seed + NUM_CORES * node_id + core_id) + for i in range(chunk_start, chunk_stop): + seed = _xorshift32(seed) + flat[i] += np.float32(sign * _uint32_to_signed_float(seed) * eps) + + return flat.reshape(data.shape) + + +def _perturb_rademacher( + data: np.ndarray, + global_seed: int, + node_id: int, + eps: float, + sign: int = 1, +) -> np.ndarray: + """Rademacher perturbation: each element offset by ยฑeps.""" + flat = data.flatten().astype(np.float32) + size = flat.size + log2core = int(math.log2(NUM_CORES)) + + for core_id in range(NUM_CORES): + chunk = (size >> log2core) + (1 if (size & (NUM_CORES - 1)) else 0) + chunk_start = min(chunk * core_id, size) + chunk_stop = min(chunk_start + chunk, size) + + seed = _scramble(global_seed + NUM_CORES * node_id + core_id) + for i in range(chunk_start, chunk_stop): + seed = _xorshift32(seed) + rad = np.float32(1.0) if (seed & np.uint32(1)) else np.float32(-1.0) + flat[i] += np.float32(sign) * rad * np.float32(eps) + + return flat.reshape(data.shape) + + +def _perturb_normal( + data: np.ndarray, + global_seed: int, + node_id: int, + eps: float, + sign: int = 1, +) -> np.ndarray: + """Gaussian perturbation via Box-Muller on Xorshift32 draws.""" + flat = data.flatten().astype(np.float32) + size = flat.size + log2core = int(math.log2(NUM_CORES)) + + for core_id in range(NUM_CORES): + chunk = (size >> log2core) + (1 if (size & (NUM_CORES - 1)) else 0) + chunk_start = min(chunk * core_id, size) + chunk_stop = min(chunk_start + chunk, size) + + seed = _scramble(global_seed + NUM_CORES * node_id + core_id) + for i in range(chunk_start, chunk_stop): + seed = _xorshift32(seed) + u1 = max(float(seed) / float(np.iinfo(np.uint32).max), 1e-10) + seed = _xorshift32(seed) + u2 = float(seed) / float(np.iinfo(np.uint32).max) * 2.0 * math.pi + z = math.sqrt(-2.0 * math.log(u1)) * math.cos(u2) + flat[i] += np.float32(sign * z * eps) + + return flat.reshape(data.shape) + + +def _perturb_rqs_rademacher( + data: np.ndarray, + mul: np.ndarray, + global_seed: int, + node_id: int, + div: int, + n_levels: int, + signed: int, + sign: int = 1, +) -> np.ndarray: + """RQS Rademacher perturbation for integer-quantised tensors.""" + flat = data.astype(np.int64).flatten() + size = flat.size + log2core = int(math.log2(NUM_CORES)) + + mul_flat = mul.flatten().astype(np.int64) + num_out = mul_flat.size + # Broadcast mul across the remaining dimensions + elems_per = size // num_out if num_out > 0 and size >= num_out else 1 + mul_per_elem = np.repeat(mul_flat, elems_per) + if mul_per_elem.size < size: + mul_per_elem = np.resize(mul_per_elem, size) + + for core_id in range(NUM_CORES): + chunk = (size >> log2core) + (1 if (size & (NUM_CORES - 1)) else 0) + chunk_start = min(chunk * core_id, size) + chunk_stop = min(chunk_start + chunk, size) + + seed = _scramble(global_seed + NUM_CORES * node_id + core_id) + for i in range(chunk_start, chunk_stop): + seed = _xorshift32(seed) + rad = np.int64(1) if (seed & np.uint32(1)) else np.int64(-1) + delta = (rad * mul_per_elem[i]) // np.int64(div) + flat[i] += sign * delta + + lo = -(n_levels // 2) if signed else 0 + hi = (n_levels // 2) - 1 if signed else n_levels - 1 + flat = np.clip(flat, lo, hi) + return flat.reshape(data.shape).astype(data.dtype) + + +def _perturb_rqs_uniform( + data: np.ndarray, + mul: np.ndarray, + global_seed: int, + node_id: int, + div: int, + n_levels: int, + signed: int, + sign: int = 1, +) -> np.ndarray: + """RQS Uniform perturbation for integer-quantised tensors.""" + flat = data.astype(np.int64).flatten() + size = flat.size + log2core = int(math.log2(NUM_CORES)) + + mul_flat = mul.flatten().astype(np.int64) + num_out = mul_flat.size + elems_per = size // num_out if num_out > 0 and size >= num_out else 1 + mul_per_elem = np.repeat(mul_flat, elems_per) + if mul_per_elem.size < size: + mul_per_elem = np.resize(mul_per_elem, size) + + for core_id in range(NUM_CORES): + chunk = (size >> log2core) + (1 if (size & (NUM_CORES - 1)) else 0) + chunk_start = min(chunk * core_id, size) + chunk_stop = min(chunk_start + chunk, size) + + seed = _scramble(global_seed + NUM_CORES * node_id + core_id) + for i in range(chunk_start, chunk_stop): + seed = _xorshift32(seed) + rand_f = _uint32_to_signed_float(seed) + delta = int(rand_f * float(mul_per_elem[i])) // div + flat[i] += sign * delta + + lo = -(n_levels // 2) if signed else 0 + hi = (n_levels // 2) - 1 if signed else n_levels - 1 + flat = np.clip(flat, lo, hi) + return flat.reshape(data.shape).astype(data.dtype) + + +# --------------------------------------------------------------------------- +# Attribute extraction helper +# --------------------------------------------------------------------------- + +def _node_attrs(node: onnx.NodeProto) -> Dict[str, Any]: + """Return node attributes as a plain Python dict.""" + attrs: Dict[str, Any] = {} + for attr in node.attribute: + if attr.type == onnx.AttributeProto.FLOAT: + attrs[attr.name] = attr.f + elif attr.type == onnx.AttributeProto.INT: + attrs[attr.name] = attr.i + elif attr.type == onnx.AttributeProto.STRING: + attrs[attr.name] = attr.s.decode("utf-8") if attr.s else "" + elif attr.type == onnx.AttributeProto.TENSOR: + attrs[attr.name] = numpy_helper.to_array(attr.t) + elif attr.type == onnx.AttributeProto.INTS: + attrs[attr.name] = list(attr.ints) + elif attr.type == onnx.AttributeProto.FLOATS: + attrs[attr.name] = list(attr.floats) + else: + attrs[attr.name] = attr + return attrs + + +# --------------------------------------------------------------------------- +# Standard ONNX op dispatcher +# --------------------------------------------------------------------------- + +def _exec_standard(op: str, inputs: List, attrs: Dict[str, Any]) -> List[np.ndarray]: # noqa: C901 + if op == "Add": + return [inputs[0] + inputs[1]] + if op == "Sub": + return [inputs[0] - inputs[1]] + if op == "Mul": + return [inputs[0] * inputs[1]] + if op == "Div": + return [inputs[0] / inputs[1]] + if op == "Neg": + return [-inputs[0]] + if op == "Abs": + return [np.abs(inputs[0])] + if op == "Sqrt": + return [np.sqrt(inputs[0])] + if op == "Exp": + return [np.exp(inputs[0])] + if op == "Log": + return [np.log(inputs[0])] + if op == "Pow": + return [np.power(inputs[0], inputs[1])] + if op == "Erf": + return [np.vectorize(math.erf)(inputs[0]).astype(inputs[0].dtype)] + if op == "Ceil": + return [np.ceil(inputs[0])] + if op == "Floor": + return [np.floor(inputs[0])] + if op == "Round": + return [np.round(inputs[0])] + if op == "Sign": + return [np.sign(inputs[0]).astype(inputs[0].dtype)] + + if op == "Clip": + lo = inputs[1] if len(inputs) > 1 and inputs[1] is not None else attrs.get("min", None) + hi = inputs[2] if len(inputs) > 2 and inputs[2] is not None else attrs.get("max", None) + return [np.clip(inputs[0], lo, hi)] + + if op in ("ReduceSum", "ReduceMean", "ReduceMax", "ReduceMin"): + x = inputs[0] + axes = attrs.get("axes", None) + keepdims = bool(attrs.get("keepdims", 1)) + noop_empty = bool(attrs.get("noop_with_empty_axes", 0)) + if len(inputs) > 1 and inputs[1] is not None: + axes = tuple(int(a) for a in inputs[1].flatten()) + elif axes is not None: + axes = tuple(axes) + if noop_empty and (axes is None or len(axes) == 0): + return [x] + fn = {"ReduceSum": np.sum, "ReduceMean": np.mean, + "ReduceMax": np.max, "ReduceMin": np.min}[op] + return [fn(x, axis=axes, keepdims=keepdims)] + + if op == "Relu": + return [np.maximum(inputs[0], 0)] + if op == "Sigmoid": + return [1.0 / (1.0 + np.exp(-inputs[0].astype(np.float64))).astype(np.float32)] + if op == "Tanh": + return [np.tanh(inputs[0])] + if op == "LeakyRelu": + alpha = float(attrs.get("alpha", 0.01)) + x = inputs[0] + return [np.where(x >= 0, x, alpha * x).astype(x.dtype)] + if op in ("Gelu", "FastGelu"): + x = inputs[0] + return [(x * 0.5 * (1.0 + np.vectorize(math.erf)(x / math.sqrt(2)))).astype(x.dtype)] + if op == "Softmax": + axis = int(attrs.get("axis", -1)) + x = inputs[0] + x_max = np.max(x, axis=axis, keepdims=True) + ex = np.exp(x - x_max) + return [(ex / np.sum(ex, axis=axis, keepdims=True)).astype(x.dtype)] + + if op == "MatMul": + return [np.matmul(inputs[0], inputs[1])] + + if op == "Gemm": + A, B = inputs[0], inputs[1] + C = inputs[2] if len(inputs) > 2 and inputs[2] is not None else 0.0 + alpha = float(attrs.get("alpha", 1.0)) + beta = float(attrs.get("beta", 1.0)) + if int(attrs.get("transA", 0)): + A = A.T + if int(attrs.get("transB", 0)): + B = B.T + return [(alpha * np.matmul(A, B) + beta * C).astype(A.dtype)] + + if op == "Conv": + x, w = inputs[0], inputs[1] + b = inputs[2] if len(inputs) > 2 and inputs[2] is not None else None + groups = int(attrs.get("group", 1)) + dilations = list(attrs.get("dilations", [1, 1])) + strides = list(attrs.get("strides", [1, 1])) + pads = list(attrs.get("pads", [0, 0, 0, 0])) + N, C_in, H, W = x.shape + C_out, C_in_pg, kH, kW = w.shape + sH, sW = strides[0], strides[1] + dH, dW = dilations[0], dilations[1] + pH_t, pW_l = pads[0], pads[1] + pH_b, pW_r = pads[2], pads[3] + H_out = (H + pH_t + pH_b - dH * (kH - 1) - 1) // sH + 1 + W_out = (W + pW_l + pW_r - dW * (kW - 1) - 1) // sW + 1 + x_pad = np.pad(x, ((0,0),(0,0),(pH_t,pH_b),(pW_l,pW_r))) + out = np.zeros((N, C_out, H_out, W_out), dtype=np.float32) + cpp = C_out // groups + for g in range(groups): + xi = x_pad[:, g*C_in_pg:(g+1)*C_in_pg] + wg = w[g*cpp:(g+1)*cpp] + for oc in range(cpp): + for oh in range(H_out): + for ow in range(W_out): + patch = xi[:, :, + oh*sH:oh*sH+kH*dH:dH, + ow*sW:ow*sW+kW*dW:dW] + out[:, g*cpp+oc, oh, ow] = np.sum( + patch * wg[oc], axis=(1, 2, 3)) + if b is not None: + out += b[np.newaxis, :, np.newaxis, np.newaxis] + return [out] + + if op == "BatchNormalization": + x, scale, bias, mean, var = inputs[:5] + eps = float(attrs.get("epsilon", 1e-5)) + return [(scale * (x - mean) / np.sqrt(var + eps) + bias).astype(x.dtype)] + + if op == "LayerNormalization": + x = inputs[0] + scale = inputs[1] if len(inputs) > 1 and inputs[1] is not None else np.ones(1, dtype=x.dtype) + b = inputs[2] if len(inputs) > 2 and inputs[2] is not None else np.zeros(1, dtype=x.dtype) + axis = int(attrs.get("axis", -1)) + eps = float(attrs.get("epsilon", 1e-5)) + mean = np.mean(x, axis=axis, keepdims=True) + var = np.var(x, axis=axis, keepdims=True) + out = ((x - mean) / np.sqrt(var + eps)) * scale + b + return [out.astype(x.dtype)] + + if op == "GroupNormalization": + x, scale, bias = inputs[:3] + num_groups = int(attrs.get("num_groups", 1)) + eps = float(attrs.get("epsilon", 1e-5)) + N, C = x.shape[:2] + spatial = x.shape[2:] + xr = x.reshape(N, num_groups, C // num_groups, *spatial) + axes = tuple(range(2, xr.ndim)) + mean = np.mean(xr, axis=axes, keepdims=True) + var = np.var(xr, axis=axes, keepdims=True) + xn = ((xr - mean) / np.sqrt(var + eps)).reshape(N, C, *spatial) + return [(xn * scale.reshape(1, C, *([1]*len(spatial))) + + bias.reshape(1, C, *([1]*len(spatial)))).astype(x.dtype)] + + if op == "MaxPool": + x = inputs[0] + kernel = list(attrs.get("kernel_shape", [2, 2])) + strides = list(attrs.get("strides", [1, 1])) + pads = list(attrs.get("pads", [0, 0, 0, 0])) + N, C, H, W = x.shape + kH, kW = kernel[0], kernel[1] + sH, sW = strides[0], strides[1] + pH_t, pW_l, pH_b, pW_r = pads[0], pads[1], pads[2], pads[3] + xp = np.pad(x, ((0,0),(0,0),(pH_t,pH_b),(pW_l,pW_r)), constant_values=-np.inf) + H_out = (H + pH_t + pH_b - kH) // sH + 1 + W_out = (W + pW_l + pW_r - kW) // sW + 1 + out = np.stack([[ + np.max(xp[:, :, oh*sH:oh*sH+kH, ow*sW:ow*sW+kW], axis=(2, 3)) + for ow in range(W_out)] for oh in range(H_out)], axis=0) + return [out.transpose(2, 3, 0, 1)] # (N,C,H_out,W_out) + + if op == "AveragePool": + x = inputs[0] + kernel = list(attrs.get("kernel_shape", [2, 2])) + strides = list(attrs.get("strides", [1, 1])) + pads = list(attrs.get("pads", [0, 0, 0, 0])) + N, C, H, W = x.shape + kH, kW = kernel[0], kernel[1] + sH, sW = strides[0], strides[1] + pH_t, pW_l, pH_b, pW_r = pads[0], pads[1], pads[2], pads[3] + xp = np.pad(x, ((0,0),(0,0),(pH_t,pH_b),(pW_l,pW_r))) + H_out = (H + pH_t + pH_b - kH) // sH + 1 + W_out = (W + pW_l + pW_r - kW) // sW + 1 + out = np.stack([[ + np.mean(xp[:, :, oh*sH:oh*sH+kH, ow*sW:ow*sW+kW], axis=(2, 3)) + for ow in range(W_out)] for oh in range(H_out)], axis=0) + return [out.transpose(2, 3, 0, 1)] + + if op == "GlobalAveragePool": + x = inputs[0] + return [np.mean(x, axis=tuple(range(2, x.ndim)), keepdims=True)] + if op == "GlobalMaxPool": + x = inputs[0] + return [np.max(x, axis=tuple(range(2, x.ndim)), keepdims=True)] + + if op == "Reshape": + shape = [int(d) for d in inputs[1].flatten()] + src = inputs[0].shape + new_shape = [int(src[i]) if d == 0 else d for i, d in enumerate(shape)] + return [inputs[0].reshape(new_shape)] + if op == "Flatten": + axis = int(attrs.get("axis", 1)) + x = inputs[0] + pre = int(np.prod(x.shape[:axis])) if axis > 0 else 1 + post = int(np.prod(x.shape[axis:])) + return [x.reshape(pre, post)] + if op == "Transpose": + perm = attrs.get("perm", None) + return [np.transpose(inputs[0], axes=perm)] + + if op == "Squeeze": + x = inputs[0] + if len(inputs) > 1 and inputs[1] is not None: + axes = tuple(int(a) for a in inputs[1].flatten()) + else: + raw = attrs.get("axes", None) + if raw is None: + axes = None + elif isinstance(raw, (int, np.integer)): + axes = (int(raw),) + else: + axes = tuple(int(a) for a in raw) or None + return [np.squeeze(x, axis=axes)] + if op == "Unsqueeze": + x = inputs[0] + if len(inputs) > 1 and inputs[1] is not None: + axes = sorted(int(a) for a in inputs[1].flatten()) + else: + axes = sorted(attrs.get("axes", [])) + for ax in axes: + x = np.expand_dims(x, axis=ax) + return [x] + if op == "Expand": + return [np.broadcast_to(inputs[0], inputs[1].tolist()).copy()] + if op == "Concat": + axis = int(attrs.get("axis", 0)) + valid = [t for t in inputs if t is not None] + return [np.concatenate(valid, axis=axis)] + if op == "Split": + axis = int(attrs.get("axis", 0)) + split = list(attrs.get("split", [])) + if len(inputs) > 1 and inputs[1] is not None: + split = inputs[1].flatten().tolist() + if not split: + num_out = int(attrs.get("num_outputs", 2)) + split = [inputs[0].shape[axis] // num_out] * num_out + secs = np.cumsum([int(s) for s in split[:-1]]).tolist() + return list(np.split(inputs[0], secs, axis=axis)) + if op == "Slice": + data, starts, ends = inputs[0], inputs[1].flatten(), inputs[2].flatten() + axes = inputs[3].flatten() if len(inputs) > 3 and inputs[3] is not None else np.arange(len(starts)) + steps = inputs[4].flatten() if len(inputs) > 4 and inputs[4] is not None else np.ones(len(starts), dtype=np.int64) + idx = [slice(None)] * data.ndim + for ax, s, e, st in zip(axes, starts, ends, steps): + idx[int(ax)] = slice(int(s), int(e), int(st)) + return [data[tuple(idx)]] + if op == "Gather": + axis = int(attrs.get("axis", 0)) + return [np.take(inputs[0], inputs[1].astype(np.int64), axis=axis)] + if op == "GatherElements": + axis = int(attrs.get("axis", 0)) + return [np.take_along_axis(inputs[0], inputs[1].astype(np.int64), axis=axis)] + if op == "Shape": + start = int(attrs.get("start", 0)) + end = attrs.get("end", None) + sh = np.array(inputs[0].shape, dtype=np.int64) + return [sh[start:end]] + if op == "Size": + return [np.array(inputs[0].size, dtype=np.int64)] + if op == "Cast": + to_type = int(attrs.get("to", TensorProto.FLOAT)) + _DT = { + TensorProto.FLOAT: np.float32, + TensorProto.DOUBLE: np.float64, + TensorProto.INT32: np.int32, + TensorProto.INT64: np.int64, + TensorProto.INT8: np.int8, + TensorProto.UINT8: np.uint8, + TensorProto.BOOL: bool, + TensorProto.FLOAT16: np.float16, + } + return [inputs[0].astype(_DT.get(to_type, np.float32))] + if op == "Identity": + return [inputs[0].copy()] + if op == "Pad": + x = inputs[0] + pads_arr = inputs[1].flatten().tolist() if len(inputs) > 1 and inputs[1] is not None else list(attrs.get("pads", [])) + val = float(inputs[2].flat[0]) if len(inputs) > 2 and inputs[2] is not None else float(attrs.get("value", 0.0)) + n = x.ndim + pw = [(int(pads_arr[i]), int(pads_arr[i+n])) for i in range(n)] + return [np.pad(x, pw, constant_values=val)] + if op == "Tile": + return [np.tile(inputs[0], inputs[1].flatten().tolist())] + if op == "Where": + return [np.where(inputs[0], inputs[1], inputs[2])] + if op in ("Equal", "Less", "Greater", "LessOrEqual", "GreaterOrEqual"): + fn = {"Equal": np.equal, "Less": np.less, "Greater": np.greater, + "LessOrEqual": np.less_equal, "GreaterOrEqual": np.greater_equal}[op] + return [fn(inputs[0], inputs[1])] + if op == "Not": + return [~inputs[0]] + if op == "Min": + r = inputs[0] + for t in inputs[1:]: + r = np.minimum(r, t) + return [r] + if op == "Max": + r = inputs[0] + for t in inputs[1:]: + r = np.maximum(r, t) + return [r] + if op == "Sum": + r = inputs[0].copy() + for t in inputs[1:]: + r = r + t + return [r] + if op == "Mean": + return [sum(inputs) / len(inputs)] + if op == "Einsum": + return [np.einsum(attrs["equation"], *inputs)] + if op == "ArgMax": + axis = int(attrs.get("axis", 0)) + keepdims = bool(attrs.get("keepdims", 1)) + return [np.argmax(inputs[0], axis=axis, keepdims=keepdims).astype(np.int64)] + if op == "ArgMin": + axis = int(attrs.get("axis", 0)) + keepdims = bool(attrs.get("keepdims", 1)) + return [np.argmin(inputs[0], axis=axis, keepdims=keepdims).astype(np.int64)] + if op == "TopK": + k = int(inputs[1].flat[0]) if len(inputs) > 1 else int(attrs.get("k", 1)) + axis = int(attrs.get("axis", -1)) + largest = bool(attrs.get("largest", 1)) + idx = np.argsort(inputs[0], axis=axis) + if largest: + idx = np.flip(idx, axis=axis) + idx = np.take(idx, np.arange(k), axis=axis) + vals = np.take_along_axis(inputs[0], idx, axis=axis) + return [vals, idx.astype(np.int64)] + if op == "ConstantOfShape": + shape = inputs[0].flatten().tolist() + val_t = attrs.get("value", None) + fill = float(val_t.flat[0]) if val_t is not None else 0.0 + return [np.full([int(d) for d in shape], fill, dtype=np.float32)] + if op == "Constant": + val_t = attrs.get("value", None) + if val_t is not None: + return [val_t.copy() if isinstance(val_t, np.ndarray) else np.array(val_t)] + vf = attrs.get("value_float", None) + if vf is not None: + return [np.array(vf, dtype=np.float32)] + vi = attrs.get("value_int", None) + if vi is not None: + return [np.array(vi, dtype=np.int64)] + raise ValueError("Constant node has no supported value attribute") + if op == "Range": + s, lim, d = inputs + return [np.arange(float(s), float(lim), float(d)).astype(s.dtype)] + if op == "NonZero": + return [np.array(np.nonzero(inputs[0]), dtype=np.int64)] + if op == "Dropout": + return [inputs[0].copy(), np.ones_like(inputs[0], dtype=bool)] + if op == "SoftmaxCrossEntropyLoss": + logits = inputs[0] + labels = inputs[1] + xm = np.max(logits, axis=-1, keepdims=True) + lp = logits - xm - np.log(np.sum(np.exp(logits - xm), axis=-1, keepdims=True)) + if labels.dtype in (np.int32, np.int64): + nll = -lp[np.arange(logits.shape[0]), labels.flatten()] + else: + nll = -np.sum(lp * labels, axis=-1) + red = attrs.get("reduction", "mean") + loss = np.mean(nll) if red == "mean" else (np.sum(nll) if red == "sum" else nll) + return [np.array(loss, dtype=np.float32), lp.astype(np.float32)] + + raise NotImplementedError(f"Standard ONNX op '{op}' is not implemented in run_onnx_graph") + + +# --------------------------------------------------------------------------- +# Deeploy custom ops (ai.onnx.contrib) +# --------------------------------------------------------------------------- + +def _exec_deeploy(op: str, inputs: List, attrs: Dict[str, Any]) -> List[np.ndarray]: + if op == "Quant": + x = inputs[0].astype(np.float64) + scale = inputs[1].astype(np.float64) + zp = inputs[2].astype(np.float64) + bits = int(attrs.get("bits", 8)) + signed = bool(attrs.get("signed", True)) + qmin = -(2 ** (bits-1)) if signed else 0 + qmax = (2 ** (bits-1)) - 1 if signed else (2**bits) - 1 + return [np.clip(np.round(x / scale) + zp, qmin, qmax).astype(np.int8 if signed else np.uint8)] + + if op == "Dequant": + q = inputs[0].astype(np.float64) + scale = inputs[1].astype(np.float64) + zp = inputs[2].astype(np.float64) + return [((q - zp) * scale).astype(np.float32)] + + if op == "RequantShift": + x = inputs[0].astype(np.int64) + mul = inputs[1].astype(np.int64) + add = inputs[2].astype(np.int64) + div_attr = attrs.get("div", None) + if div_attr is None: + raise ValueError("RequantShift missing 'div' tensor attribute") + div = int(div_attr.flat[0]) if hasattr(div_attr, "flat") else int(div_attr) + bits = int(attrs.get("out_bits", 8)) + signed = bool(attrs.get("signed", 1)) + qmin = -(2**(bits-1)) if signed else 0 + qmax = (2**(bits-1))-1 if signed else (2**bits)-1 + return [np.clip((x * mul + add) // div, qmin, qmax).astype(np.int8 if signed else np.uint8)] + + raise NotImplementedError(f"Deeploy op '{op}' not implemented") + + +# --------------------------------------------------------------------------- +# MeZO perturbation ops (mezo / com.microsoft) +# --------------------------------------------------------------------------- + +def _exec_mezo(op: str, inputs: List, attrs: Dict[str, Any]) -> List[np.ndarray]: + seed = int(attrs.get("seed", 0)) + node_id = int(attrs.get("idx", 0)) + eps = float(attrs.get("eps", 0.01)) + sign = 1 # forward perturbation sign + + if op == "PerturbUniform": + return [_perturb_uniform(inputs[0].astype(np.float32), seed, node_id, eps, sign)] + if op == "PerturbRademacher": + return [_perturb_rademacher(inputs[0].astype(np.float32), seed, node_id, eps, sign)] + if op == "PerturbNormal": + return [_perturb_normal(inputs[0].astype(np.float32), seed, node_id, eps, sign)] + if op == "PerturbTriangle": + return [_perturb_uniform(inputs[0].astype(np.float32), seed, node_id, eps, sign)] + if op == "PerturbEggroll": + # Generates a Rademacher column vector of the requested shape + shape = [int(d) for d in inputs[0].flatten()] + x = np.zeros(shape, dtype=np.float32) + return [_perturb_rademacher(x, seed, node_id, 1.0, 1)] + if op == "RQSPerturbRademacher": + mul = inputs[1] if len(inputs) > 1 and inputs[1] is not None else np.ones(inputs[0].shape[0], dtype=np.int64) + div = int(attrs.get("div", 2**15)) + n_levels = int(attrs.get("n_levels", 256)) + signed_flag = int(attrs.get("signed", 1)) + return [_perturb_rqs_rademacher(inputs[0], mul, seed, node_id, div, n_levels, signed_flag, sign)] + if op == "RQSPerturbUniform": + mul = inputs[1] if len(inputs) > 1 and inputs[1] is not None else np.ones(inputs[0].shape[0], dtype=np.int64) + div = int(attrs.get("div", 2**15)) + n_levels = int(attrs.get("n_levels", 256)) + signed_flag = int(attrs.get("signed", 1)) + return [_perturb_rqs_uniform(inputs[0], mul, seed, node_id, div, n_levels, signed_flag, sign)] + + raise NotImplementedError(f"MeZO op '{op}' not implemented") + + +# --------------------------------------------------------------------------- +# Public entry point +# --------------------------------------------------------------------------- + +def run_onnx_graph( + onnx_path: str, + inputs: Dict[str, np.ndarray], + output_names: Optional[List[str]] = None, +) -> np.ndarray: + """ + Execute an ONNX graph in pure Python โ€” no onnxruntime required. + + Supports standard ONNX ops, Deeploy custom ops (ai.onnx.contrib domain: + Quant, Dequant, RequantShift), and MeZO perturbation ops (mezo domain: + PerturbUniform, PerturbRademacher, PerturbNormal, PerturbTriangle, + PerturbEggroll, RQSPerturbRademacher, RQSPerturbUniform). + + Args: + onnx_path: Path to the ``.onnx`` model file. + inputs: ``{name: ndarray}`` for each graph input. + output_names: If given, return those specific outputs. + Otherwise the first graph output is returned. + + Returns: + First (or requested) graph output as a numpy array. + """ + model = onnx.load(onnx_path) + graph = model.graph + + # value store: name โ†’ numpy array + values: Dict[str, np.ndarray] = {} + values.update(inputs) + + for init in graph.initializer: + if init.name not in values: + values[init.name] = numpy_helper.to_array(init) + + # ONNX graph is topologically sorted by spec + for node in graph.node: + op = node.op_type + domain = (node.domain or "").strip() + attrs = _node_attrs(node) + + # Gather inputs; absent optional inputs (empty string) become None + node_inputs: List = [] + for name in node.input: + if name == "": + node_inputs.append(None) + elif name in values: + node_inputs.append(values[name]) + else: + raise KeyError( + f"Node '{node.name}' (op='{op}') needs input '{name}' " + f"which is missing from value map." + ) + + # com.microsoft is used for both ORT contrib ops (e.g. Gelu, + # Attention) and MeZO perturbation ops. Route by op name. + _MEZO_OPS = { + "PerturbUniform", "PerturbRademacher", "PerturbNormal", + "PerturbTriangle", "PerturbEggroll", + "RQSPerturbRademacher", "RQSPerturbUniform", + } + + try: + if domain in ("", "ai.onnx"): + outs = _exec_standard(op, node_inputs, attrs) + elif domain == "ai.onnx.contrib": + outs = _exec_deeploy(op, node_inputs, attrs) + elif domain == "mezo" or (domain == "com.microsoft" and op in _MEZO_OPS): + outs = _exec_mezo(op, node_inputs, attrs) + elif domain == "com.microsoft": + # ORT contrib ops that map to standard implementations + outs = _exec_standard(op, node_inputs, attrs) + else: + raise NotImplementedError( + f"Unsupported domain '{domain}' for op '{op}'" + ) + except NotImplementedError: + raise + except Exception as exc: + raise RuntimeError( + f"Error in node '{node.name}' op='{op}' domain='{domain}': {exc}" + ) from exc + + for out_name, out_val in zip(node.output, outs): + if out_name: + values[out_name] = out_val + + graph_outputs = [o.name for o in graph.output] + if output_names is not None: + return [values[n] for n in output_names] + if graph_outputs: + return values[graph_outputs[0]] + raise RuntimeError("ONNX graph has no outputs") diff --git a/tests/models/test_inference_consistency.py b/tests/models/test_inference_consistency.py new file mode 100644 index 0000000..7ff4616 --- /dev/null +++ b/tests/models/test_inference_consistency.py @@ -0,0 +1,300 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: MIT + +""" +Inference consistency tests: onnxruntime vs pure-Python run_onnx_graph. + +For each model exported in inference mode, we verify that our pure-Python +ONNX executor (``run_onnx_graph``) produces numerically identical results +to onnxruntime's ``InferenceSession`` on the same random inputs. + +Models tested: + - LightweightCNN (standard ops only โ†’ ORT + run_onnx_graph agree) + - SleepConViT (contains com.microsoft/Gelu and a Squeeze with opset-12 + axes attribute; patched to opset-13 in-memory before ORT) +""" + +import os +import subprocess +import sys + +import numpy as np +import onnx +import onnx.numpy_helper as numpy_helper +import onnxruntime as ort +import pytest + +from .onnx_node_implementations import run_onnx_graph +from .test_utils import load_and_check_onnx_model + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +_PROJECT_ROOT = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +) +_CLI_SCRIPT = os.path.join(_PROJECT_ROOT, "Onnx4Deeploy.py") + +_NUM_SAMPLES = 5 +_TOLERANCE = 1e-5 + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _export_inference(model_name: str, output_dir: str) -> str: + """Run the CLI in 'infer' mode and return the path to network.onnx.""" + cmd = [ + sys.executable, _CLI_SCRIPT, + "-model", model_name, + "-mode", "infer", + "-o", output_dir, + ] + result = subprocess.run( + cmd, cwd=_PROJECT_ROOT, capture_output=True, text=True + ) + if result.returncode != 0: + pytest.fail( + f"CLI failed for '{model_name}' (rc={result.returncode}):\n" + f"stdout: {result.stdout}\nstderr: {result.stderr}" + ) + onnx_file = os.path.join(output_dir, "network.onnx") + assert os.path.exists(onnx_file), f"network.onnx not found in {output_dir}" + return onnx_file + + +def _to_opset13_compatible(onnx_file: str) -> bytes: + """ + Return a serialised ONNX model with all Squeeze/Unsqueeze nodes patched to + opset-13 style: the ``axes`` *attribute* is moved to a constant *input* + tensor so that onnxruntime (which validates opset-13+ rules) can load it. + + The original file on disk is not modified. + """ + model = onnx.load(onnx_file) + new_nodes = [] + extra_initializers = [] + + for node in model.graph.node: + if node.op_type not in ("Squeeze", "Unsqueeze"): + new_nodes.append(node) + continue + + # Find the axes attribute (opset-12 style) + axes_attr = next((a for a in node.attribute if a.name == "axes"), None) + if axes_attr is None: + # Already opset-13 style (axes come as a second input) or no axes + new_nodes.append(node) + continue + + # Extract axes value(s) + if axes_attr.type == onnx.AttributeProto.INT: + axes = [int(axes_attr.i)] + else: # INTS + axes = list(axes_attr.ints) + + # Create a constant initializer for the axes tensor + axes_name = f"_axes_const_{node.name}" + axes_tensor = onnx.helper.make_tensor( + name=axes_name, + data_type=onnx.TensorProto.INT64, + dims=[len(axes)], + vals=axes, + ) + extra_initializers.append(axes_tensor) + + # Rebuild the node with axes as the second input, no axes attribute + new_node = onnx.helper.make_node( + op_type=node.op_type, + inputs=[node.input[0], axes_name], + outputs=list(node.output), + name=node.name, + domain=node.domain if node.domain else "", + ) + # Copy over any other attributes (not axes) + for attr in node.attribute: + if attr.name != "axes": + new_node.attribute.append(attr) + + new_nodes.append(new_node) + + # Rebuild graph with patched nodes and extra initializers + new_graph = onnx.helper.make_graph( + nodes=new_nodes, + name=model.graph.name, + inputs=list(model.graph.input), + outputs=list(model.graph.output), + initializer=list(model.graph.initializer) + extra_initializers, + ) + for vi in model.graph.value_info: + new_graph.value_info.append(vi) + + new_model = onnx.helper.make_model( + new_graph, + producer_name=model.producer_name, + opset_imports=list(model.opset_import), + ) + new_model.ir_version = model.ir_version + return new_model.SerializeToString() + + +def _ort_session(onnx_file: str) -> ort.InferenceSession: + """Create an onnxruntime InferenceSession, patching Squeeze axes to opset-13.""" + return ort.InferenceSession(_to_opset13_compatible(onnx_file)) + + +def _compare_outputs( + onnx_file: str, + input_shape: tuple, + num_samples: int = _NUM_SAMPLES, + tolerance: float = _TOLERANCE, +) -> None: + """ + Run *num_samples* random inputs through both onnxruntime and run_onnx_graph + and assert the outputs agree within *tolerance*. + + ORT receives the opset-13-patched model bytes; run_onnx_graph reads the + original file (our executor handles both styles). + """ + sess = _ort_session(onnx_file) + ort_input_name = sess.get_inputs()[0].name + + failures = [] + for seed in range(num_samples): + rng = np.random.default_rng(seed) + x = rng.standard_normal(input_shape).astype(np.float32) + + # onnxruntime reference + ort_out = sess.run(None, {ort_input_name: x})[0] + + # pure-Python executor + py_out = run_onnx_graph(onnx_file, {"input": x}) + + max_diff = float(np.max(np.abs(ort_out - py_out))) + if max_diff > tolerance: + failures.append( + f" seed={seed}: max |ORT โˆ’ run_onnx_graph| = {max_diff:.2e} " + f"(limit {tolerance:.2e})" + ) + + if failures: + pytest.fail( + f"Outputs diverge on {len(failures)}/{num_samples} samples:\n" + + "\n".join(failures) + ) + + +# =========================================================================== +# LightweightCNN +# =========================================================================== + + +class TestLightweightCNNInferenceConsistency: + """ + LightweightCNN uses only standard ONNX ops (Conv, MaxPool, Relu, Gemm, + Reshape), so both onnxruntime and run_onnx_graph can execute it. + This test verifies they agree numerically. + """ + + _MODEL = "LightweightCNN" + _INPUT_SHAPE = (1, 1, 28, 28) + + def test_export_produces_valid_onnx(self, model_test_dir): + """Exported network.onnx loads and has correct structure.""" + onnx_file = _export_inference(self._MODEL, model_test_dir) + model = load_and_check_onnx_model(onnx_file, skip_shape_check=True) + assert len(model.graph.node) > 0 + assert any(o.name for o in model.graph.output) + + def test_ort_and_pure_python_agree(self, model_test_dir): + """ + onnxruntime and run_onnx_graph produce identical outputs (within 1e-5) + on 5 random inputs for LightweightCNN. + """ + onnx_file = _export_inference(self._MODEL, model_test_dir) + _compare_outputs(onnx_file, self._INPUT_SHAPE) + + def test_run_onnx_graph_produces_finite_output(self, model_test_dir): + """run_onnx_graph produces finite float32 output for each random input.""" + onnx_file = _export_inference(self._MODEL, model_test_dir) + for seed in range(_NUM_SAMPLES): + rng = np.random.default_rng(seed) + x = rng.standard_normal(self._INPUT_SHAPE).astype(np.float32) + out = run_onnx_graph(onnx_file, {"input": x}) + assert out.shape == (1, 10), f"Unexpected output shape: {out.shape}" + assert np.all(np.isfinite(out)), f"Non-finite output at seed={seed}" + + def test_run_onnx_graph_output_shape(self, model_test_dir): + """Output shape is (1, num_classes=10).""" + onnx_file = _export_inference(self._MODEL, model_test_dir) + x = np.zeros(self._INPUT_SHAPE, dtype=np.float32) + out = run_onnx_graph(onnx_file, {"input": x}) + assert out.shape == (1, 10) + + +# =========================================================================== +# SleepConViT +# =========================================================================== + + +class TestSleepConViTInferenceConsistency: + """ + SleepConViT uses com.microsoft/Gelu and a Squeeze node with an ``axes`` + attribute (opset-12 style). Before handing the model to ORT the test + patches Squeeze/Unsqueeze nodes in-memory via ``_to_opset13_compatible`` + so that onnxruntime can load and execute it. + """ + + _MODEL = "SleepConViT" + _INPUT_SHAPE = (1, 1, 1, 3000) + _NUM_CLASSES = 4 + + def test_export_produces_valid_onnx(self, model_test_dir): + """Exported network.onnx loads without error.""" + onnx_file = _export_inference(self._MODEL, model_test_dir) + model = load_and_check_onnx_model(onnx_file, skip_shape_check=True) + assert len(model.graph.node) > 0 + + def test_run_onnx_graph_produces_finite_output(self, model_test_dir): + """ + run_onnx_graph produces finite float32 output for each random input. + This exercises com.microsoft/Gelu and the Squeeze op in our executor. + """ + onnx_file = _export_inference(self._MODEL, model_test_dir) + for seed in range(_NUM_SAMPLES): + rng = np.random.default_rng(seed) + x = rng.standard_normal(self._INPUT_SHAPE).astype(np.float32) + out = run_onnx_graph(onnx_file, {"input": x}) + assert np.all(np.isfinite(out)), f"Non-finite output at seed={seed}" + + def test_run_onnx_graph_output_shape(self, model_test_dir): + """Output shape is (1, num_classes).""" + onnx_file = _export_inference(self._MODEL, model_test_dir) + x = np.zeros(self._INPUT_SHAPE, dtype=np.float32) + out = run_onnx_graph(onnx_file, {"input": x}) + assert out.shape[0] == 1 + assert out.shape[-1] == self._NUM_CLASSES + + def test_run_onnx_graph_deterministic(self, model_test_dir): + """run_onnx_graph produces the same result on repeated calls.""" + onnx_file = _export_inference(self._MODEL, model_test_dir) + rng = np.random.default_rng(0) + x = rng.standard_normal(self._INPUT_SHAPE).astype(np.float32) + out1 = run_onnx_graph(onnx_file, {"input": x}) + out2 = run_onnx_graph(onnx_file, {"input": x}) + np.testing.assert_array_equal(out1, out2) + + def test_ort_and_pure_python_agree(self, model_test_dir): + """ + onnxruntime and run_onnx_graph produce identical outputs (within 1e-5) + on 5 random inputs for SleepConViT. + + The model is patched in-memory via ``_to_opset13_compatible`` before + being passed to ORT so that the opset-12-style Squeeze axes attribute + does not cause a load failure. + """ + onnx_file = _export_inference(self._MODEL, model_test_dir) + _compare_outputs(onnx_file, self._INPUT_SHAPE) diff --git a/tests/models/test_zo_perturbation.py b/tests/models/test_zo_perturbation.py new file mode 100644 index 0000000..f5314c2 --- /dev/null +++ b/tests/models/test_zo_perturbation.py @@ -0,0 +1,419 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: MIT + +""" +Tests for ZO (zeroth-order) perturbation model exports. + +Tests the full export pipeline for LightweightCNN, QLiteCNN, SleepConViT, +and QSleepConViT with various noise types (Uniform, Rademacher, Eggroll, +RQS-Rademacher). + +Numerical verification uses ``run_onnx_graph`` from +``onnx_node_implementations``, a pure Python / PyTorch graph executor. +""" + +import os +import subprocess +import sys + +import numpy as np +import onnx +import pytest + +from .onnx_node_implementations import run_onnx_graph +from .test_utils import load_and_check_onnx_model + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +_PROJECT_ROOT = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +) +_CLI_SCRIPT = os.path.join(_PROJECT_ROOT, "Onnx4Deeploy.py") + +NUM_CORES = 8 + +# --------------------------------------------------------------------------- +# RNG Reference Implementation (matches Deeploy C++ code) +# --------------------------------------------------------------------------- + + +def scramble_seed(seed: int) -> np.uint32: + """Scramble seed: seed * 1664525 + 1013904223 (mod 2^32).""" + return np.uint32(np.uint32(seed) * np.uint32(1664525) + np.uint32(1013904223)) + + +def xorshift32(state: np.uint32) -> np.uint32: + """Xorshift32 PRNG step.""" + state = np.uint32(state) + state ^= np.uint32(state << np.uint32(13)) + state ^= np.uint32(state >> np.uint32(17)) + state ^= np.uint32(state << np.uint32(5)) + return state + + +def generate_uniform_perturbation( + data: np.ndarray, + global_seed: int, + node_id: int, + eps: float, + perturbation_sign: int = 1, +) -> np.ndarray: + """Generate uniform perturbation matching Deeploy C++ reference. + + seed = scramble(initial_global_seed + NUM_CORES * node_id + core_id) + RNG: Xorshift32, mapped to [-1, 1], scaled by eps * sqrt(3). + """ + size = data.size + output = data.flatten().copy() + scale = eps * np.sqrt(3.0) + + for core_id in range(NUM_CORES): + log2core = int(np.log2(NUM_CORES)) + chunk = (size >> log2core) + ((size & (NUM_CORES - 1)) != 0) + chunk_start = min(chunk * core_id, size) + chunk_stop = min(chunk_start + chunk, size) + + seed = scramble_seed(global_seed + NUM_CORES * node_id + core_id) + + for i in range(chunk_start, chunk_stop): + seed = xorshift32(seed) + # Map uint32 to [-1, 1] + rand_val = (float(seed) / float(np.iinfo(np.uint32).max)) * 2.0 - 1.0 + output[i] = output[i] + perturbation_sign * rand_val * scale + + return output.reshape(data.shape) + + +def generate_rademacher_perturbation( + data: np.ndarray, + global_seed: int, + node_id: int, + eps: float, + perturbation_sign: int = 1, +) -> np.ndarray: + """Generate Rademacher perturbation (+/- eps) matching Deeploy C++ reference.""" + size = data.size + output = data.flatten().copy() + + for core_id in range(NUM_CORES): + log2core = int(np.log2(NUM_CORES)) + chunk = (size >> log2core) + ((size & (NUM_CORES - 1)) != 0) + chunk_start = min(chunk * core_id, size) + chunk_stop = min(chunk_start + chunk, size) + + seed = scramble_seed(global_seed + NUM_CORES * node_id + core_id) + + for i in range(chunk_start, chunk_stop): + seed = xorshift32(seed) + sign = 1 if (seed & 1) else -1 + output[i] = output[i] + perturbation_sign * sign * eps + + return output.reshape(data.shape) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _run_cli(model: str, mode: str, output_dir: str, noise_type: str) -> subprocess.CompletedProcess: + """Run Onnx4Deeploy CLI and return result.""" + cmd = [ + sys.executable, _CLI_SCRIPT, + "-model", model, + "-mode", mode, + "-o", output_dir, + "--noise-type", noise_type, + ] + result = subprocess.run(cmd, cwd=_PROJECT_ROOT, capture_output=True, text=True) + if result.returncode != 0: + print(f"[CLI] stdout:\n{result.stdout}") + print(f"[CLI] stderr:\n{result.stderr}") + return result + + +def _verify_zo_output_files(output_dir: str, quantized: bool = False): + """Verify expected ZO output files exist.""" + expected = ["network_infer.onnx", "network_zo_train.onnx", "inputs.npz", "outputs.npz"] + if not quantized: + # Non-quantized may also have network_zo_update.onnx + pass + for fname in expected: + fpath = os.path.join(output_dir, fname) + assert os.path.exists(fpath), f"Missing expected output file: {fpath}" + + +def _verify_onnx_valid(onnx_file: str) -> onnx.ModelProto: + """Load and do basic validation of ONNX model.""" + return load_and_check_onnx_model(onnx_file, skip_shape_check=True) + + +def _count_perturbation_nodes(model: onnx.ModelProto, op_prefix: str = "Perturb") -> int: + """Count perturbation operator nodes in the graph.""" + return sum(1 for node in model.graph.node if node.op_type.startswith(op_prefix)) + + +def _get_perturbation_op_type(noise_type: str) -> str: + """Map noise type to expected ONNX op_type.""" + mapping = { + "uniform": "PerturbUniform", + "rademacher": "PerturbRademacher", + "eggroll": "PerturbRademacher", # Eggroll uses Rademacher + "rqs_rademacher": "RQSPerturbRademacher", + "rqs_uniform": "RQSPerturbUniform", + } + return mapping.get(noise_type, f"Perturb{noise_type.capitalize()}") + + +def _run_and_verify_zo_export( + model_name: str, + mode: str, + noise_type: str, + output_dir: str, + quantized: bool = False, +): + """Run CLI export and verify output files and ONNX validity.""" + result = _run_cli(model_name, mode, output_dir, noise_type) + assert result.returncode == 0, ( + f"CLI failed (rc={result.returncode}):\n" + f"stdout: {result.stdout}\nstderr: {result.stderr}" + ) + + _verify_zo_output_files(output_dir, quantized=quantized) + + # Check inference model + infer_path = os.path.join(output_dir, "network_infer.onnx") + _verify_onnx_valid(infer_path) + + # Check ZO training model + zo_path = os.path.join(output_dir, "network_zo_train.onnx") + zo_model = _verify_onnx_valid(zo_path) + + # Verify perturbation nodes exist + expected_op = _get_perturbation_op_type(noise_type) + perturb_count = sum( + 1 for n in zo_model.graph.node if n.op_type == expected_op + ) + assert perturb_count > 0, ( + f"No {expected_op} nodes found in ZO training graph. " + f"Node types: {set(n.op_type for n in zo_model.graph.node)}" + ) + + return zo_model + + +def _run_numerical_check( + onnx_file: str, + input_shape: tuple, + num_samples: int = 3, +): + """Run the ONNX graph with pure Python executor and verify it doesn't crash. + + For ZO training graphs, we verify execution succeeds (the perturbation + ops produce finite outputs). Full numerical matching against the RNG + reference is done in dedicated tests. + """ + for seed in range(num_samples): + rng = np.random.default_rng(seed) + test_input = rng.standard_normal(input_shape).astype(np.float32) + # ZO graphs need both input and label + num_classes = 10 # default; overridden per-model + label = np.zeros((input_shape[0], num_classes), dtype=np.float32) + label[0, 0] = 1.0 + + feeds = {"input": test_input, "label": label} + try: + output = run_onnx_graph(onnx_file, feeds) + assert np.all(np.isfinite(output)), ( + f"Non-finite output at seed={seed}" + ) + except Exception as e: + pytest.fail(f"run_onnx_graph failed at seed={seed}: {e}") + + +# =========================================================================== +# LightweightCNN (float) ZO Tests +# =========================================================================== + + +@pytest.mark.zo +class TestLiteCNNZO: + """ZO perturbation tests for LightweightCNN (float).""" + + _MODEL = "LightweightCNN" + _MODE = "zo-train" + _INPUT_SHAPE = (1, 1, 28, 28) + + def test_litecnn_uniform(self, model_test_dir): + """LiteCNN-Uniform: export and verify perturbation nodes.""" + zo_model = _run_and_verify_zo_export( + self._MODEL, self._MODE, "uniform", model_test_dir + ) + # Verify inference model runs through pure Python executor + infer_path = os.path.join(model_test_dir, "network_infer.onnx") + rng = np.random.default_rng(0) + test_input = rng.standard_normal(self._INPUT_SHAPE).astype(np.float32) + output = run_onnx_graph(infer_path, {"input": test_input}) + assert output is not None and np.all(np.isfinite(output)) + + def test_litecnn_rademacher(self, model_test_dir): + """LiteCNN-Rademacher: export and verify perturbation nodes.""" + _run_and_verify_zo_export( + self._MODEL, self._MODE, "rademacher", model_test_dir + ) + + def test_litecnn_eggroll(self, model_test_dir): + """LiteCNN-Eggroll: export and verify perturbation nodes (uses Rademacher).""" + _run_and_verify_zo_export( + self._MODEL, self._MODE, "eggroll", model_test_dir + ) + + +# =========================================================================== +# QLiteCNN (quantized) ZO Tests +# =========================================================================== + + +@pytest.mark.zo +@pytest.mark.quantized +class TestQLiteCNNZO: + """ZO perturbation tests for QLiteCNN (quantized).""" + + _MODEL = "QLiteCNN" + _MODE = "q-zo-train" + _INPUT_SHAPE = (1, 1, 28, 28) + + def test_qlitecnn_rqs_rademacher(self, model_test_dir): + """QLiteCNN-RQSRad: quantized ZO export with RQS Rademacher perturbation.""" + zo_model = _run_and_verify_zo_export( + self._MODEL, self._MODE, "rqs_rademacher", model_test_dir, + quantized=True, + ) + # Verify RQSPerturbRademacher nodes exist + rqs_nodes = [ + n for n in zo_model.graph.node + if n.op_type == "RQSPerturbRademacher" + ] + assert len(rqs_nodes) > 0, "No RQSPerturbRademacher nodes in quantized ZO graph" + + +# =========================================================================== +# SleepConViT (float) ZO Tests +# =========================================================================== + + +@pytest.mark.zo +class TestSleepViTZO: + """ZO perturbation tests for SleepConViT (float).""" + + _MODEL = "SleepConViT" + _MODE = "zo-train" + _INPUT_SHAPE = (1, 1, 1, 3000) + + def test_sleepvit_uniform(self, model_test_dir): + """SleepViT-Uniform: export and verify perturbation nodes.""" + zo_model = _run_and_verify_zo_export( + self._MODEL, self._MODE, "uniform", model_test_dir + ) + # Verify inference model runs through pure Python executor + infer_path = os.path.join(model_test_dir, "network_infer.onnx") + rng = np.random.default_rng(0) + test_input = rng.standard_normal(self._INPUT_SHAPE).astype(np.float32) + output = run_onnx_graph(infer_path, {"input": test_input}) + assert output is not None and np.all(np.isfinite(output)) + + def test_sleepvit_rademacher(self, model_test_dir): + """SleepViT-Rademacher: export and verify perturbation nodes.""" + _run_and_verify_zo_export( + self._MODEL, self._MODE, "rademacher", model_test_dir + ) + + def test_sleepvit_eggroll(self, model_test_dir): + """SleepViT-Eggroll: export and verify perturbation nodes (uses Rademacher).""" + _run_and_verify_zo_export( + self._MODEL, self._MODE, "eggroll", model_test_dir + ) + + +# =========================================================================== +# QSleepConViT (quantized) ZO Tests +# =========================================================================== + + +@pytest.mark.zo +@pytest.mark.quantized +class TestQSleepViTZO: + """ZO perturbation tests for QSleepConViT (quantized).""" + + _MODEL = "QSleepConViT" + _MODE = "q-zo-train" + _INPUT_SHAPE = (1, 1, 1, 3000) + + def test_qsleepvit_rqs_rademacher(self, model_test_dir): + """QSleepViT-RQSRad: quantized ZO export with RQS Rademacher perturbation.""" + zo_model = _run_and_verify_zo_export( + self._MODEL, self._MODE, "rqs_rademacher", model_test_dir, + quantized=True, + ) + rqs_nodes = [ + n for n in zo_model.graph.node + if n.op_type == "RQSPerturbRademacher" + ] + assert len(rqs_nodes) > 0, "No RQSPerturbRademacher nodes in quantized ZO graph" + + +# =========================================================================== +# RNG Seed Verification Tests +# =========================================================================== + + +@pytest.mark.zo +class TestRNGSeedComputation: + """Verify the RNG seed computation matches the Deeploy C++ reference.""" + + def test_scramble_seed(self): + """Verify scramble formula: seed * 1664525 + 1013904223.""" + assert scramble_seed(0) == np.uint32(1013904223) + assert scramble_seed(1) == np.uint32(1664525 + 1013904223) + assert scramble_seed(42) == np.uint32( + np.uint32(42) * np.uint32(1664525) + np.uint32(1013904223) + ) + + def test_xorshift32_deterministic(self): + """Verify Xorshift32 is deterministic.""" + state = np.uint32(12345) + s1 = xorshift32(state) + s2 = xorshift32(state) + assert s1 == s2 + # Verify it changes state + assert xorshift32(s1) != s1 + + def test_seed_per_node(self): + """Verify each node gets a unique seed: seed + NUM_CORES * node_id + core_id.""" + global_seed = 42 + # Two different nodes should get different seeds + seed_node0_core0 = scramble_seed(global_seed + NUM_CORES * 0 + 0) + seed_node1_core0 = scramble_seed(global_seed + NUM_CORES * 1 + 0) + assert seed_node0_core0 != seed_node1_core0 + + # Same node, different cores should get different seeds + seed_node0_core1 = scramble_seed(global_seed + NUM_CORES * 0 + 1) + assert seed_node0_core0 != seed_node0_core1 + + def test_uniform_perturbation_deterministic(self): + """Verify uniform perturbation is deterministic for same seed/node_id.""" + data = np.zeros(64, dtype=np.float32) + result1 = generate_uniform_perturbation(data, global_seed=42, node_id=0, eps=0.01) + result2 = generate_uniform_perturbation(data, global_seed=42, node_id=0, eps=0.01) + np.testing.assert_array_equal(result1, result2) + + def test_rademacher_perturbation_values(self): + """Verify Rademacher perturbation only produces +/- eps offsets.""" + data = np.zeros(64, dtype=np.float32) + eps = 0.01 + result = generate_rademacher_perturbation(data, global_seed=42, node_id=0, eps=eps) + # All values should be exactly +eps or -eps + np.testing.assert_array_less(np.abs(np.abs(result) - eps), 1e-7) From b14e39c4188c002470a3bc28d4c1a97770aca70a Mon Sep 17 00:00:00 2001 From: JanCSEM Date: Fri, 17 Apr 2026 11:58:22 +0000 Subject: [PATCH 23/24] Update Base exporter to use python implementation instead of ORT inference session for graphs with custom nodes --- onnx4deeploy/core/base_exporter.py | 47 +- onnx4deeploy/utils/__init__.py | 1 + .../utils/onnx_node_implementations.py | 794 ++++++++++++++++++ tests/models/onnx_node_implementations.py | 792 +---------------- 4 files changed, 823 insertions(+), 811 deletions(-) create mode 100644 onnx4deeploy/utils/onnx_node_implementations.py diff --git a/onnx4deeploy/core/base_exporter.py b/onnx4deeploy/core/base_exporter.py index 5244754..9ed5cc5 100644 --- a/onnx4deeploy/core/base_exporter.py +++ b/onnx4deeploy/core/base_exporter.py @@ -672,48 +672,51 @@ def _create_test_data(self, mode=ExportMode.INFERENCE, quant=False): """ Create test input/output data for training. - Uses ONNX Runtime to generate reference output from the (potentially randomized) ONNX model. - This ensures test data matches the actual ONNX model weights. + For standard inference mode, uses ONNX Runtime to generate reference + output. For modes that involve custom ops (q-infer, zo-train, + q-zo-train) the pure-Python ``run_onnx_graph`` executor is used instead + so that Quant/Dequant/RequantShift and MeZO perturbation nodes can be + executed without a custom-op shared library. """ - # Generate test data using ONNX Runtime try: from pathlib import Path import numpy as np - import onnxruntime as ort print("๐Ÿ’พ Generating test input/output data from ONNX model...") - # Create test input input_shape = self.get_input_shape() test_input = np.random.randn(*input_shape).astype(np.float32) - from onnxruntime_extensions import onnx_op, PyOp, get_library_path - from DeepQuant.QuantDequantOnnx import requant_shift_onnx - sess_options = ort.SessionOptions() - sess_options.register_custom_ops_library(get_library_path()) - - - if not quant: - session = ort.InferenceSession(self.paths["network_infer"], - sess_options=sess_options) - input_names = [i.name for i in session.get_inputs()] + use_pure_python = quant or (mode == ExportMode.ZO_TRAINING) + if use_pure_python: + from onnx4deeploy.utils.onnx_node_implementations import run_onnx_graph + print(" Using pure-Python ONNX executor (custom ops present)...") + test_output = run_onnx_graph( + self.paths["network_infer"], {"input": test_input} + ) + if not isinstance(test_output, np.ndarray): + test_output = np.array(test_output, dtype=np.float32) + else: + import onnxruntime as ort + from onnxruntime_extensions import get_library_path + sess_options = ort.SessionOptions() + sess_options.register_custom_ops_library(get_library_path()) + session = ort.InferenceSession( + self.paths["network_infer"], sess_options=sess_options + ) input_name = session.get_inputs()[0].name test_output = session.run(None, {input_name: test_input})[0] - - - # Workaround onnx Inference session for now - else: - test_output = np.random.randn(input_shape[0], self.config["num_classes"]).astype(np.float32) - # Save as .npz files save_path = Path(self.paths["output_dir"]) save_path.mkdir(parents=True, exist_ok=True) if mode == ExportMode.ZO_TRAINING: - test_label = np.random.randint(0, self.config["num_classes"], size=(input_shape[0],1)).astype(np.int8) + test_label = np.random.randint( + 0, self.config["num_classes"], size=(input_shape[0], 1) + ).astype(np.int8) print(f" Generated test labels with shape: {test_label.shape}") np.savez(save_path / "inputs.npz", input=test_input, label=test_label) else: diff --git a/onnx4deeploy/utils/__init__.py b/onnx4deeploy/utils/__init__.py index 0dca197..97c572f 100644 --- a/onnx4deeploy/utils/__init__.py +++ b/onnx4deeploy/utils/__init__.py @@ -5,6 +5,7 @@ """Utility functions for ONNX model manipulation.""" from .node_naming import make_c_name, rename_and_save_onnx, rename_nodes, rename_onnx_nodes +from .onnx_node_implementations import run_onnx_graph __all__ = [ "make_c_name", diff --git a/onnx4deeploy/utils/onnx_node_implementations.py b/onnx4deeploy/utils/onnx_node_implementations.py new file mode 100644 index 0000000..686de66 --- /dev/null +++ b/onnx4deeploy/utils/onnx_node_implementations.py @@ -0,0 +1,794 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: MIT + +""" +Pure-Python ONNX graph executor. + +Executes ONNX graphs without relying on onnxruntime.InferenceSession, so that +graphs containing custom ops (Deeploy Quant/Dequant/RequantShift and MeZO +perturbation ops) can be run natively. + +Supported op domains: + - "" (standard ONNX ops) + - "ai.onnx.contrib" โ†’ Quant, Dequant, RequantShift + - "mezo" โ†’ PerturbNormal, PerturbUniform, PerturbRademacher, + PerturbTriangle, PerturbEggroll, + RQSPerturbRademacher, RQSPerturbUniform +""" + +from __future__ import annotations + +import math +from typing import Any, Dict, List, Optional + +import numpy as np +import onnx +from onnx import numpy_helper, TensorProto + +# --------------------------------------------------------------------------- +# RNG reference matching Deeploy C++ implementation +# --------------------------------------------------------------------------- + +NUM_CORES: int = 8 + + +def _scramble(seed: int) -> np.uint32: + """seed * 1664525 + 1013904223 (mod 2**32, 32-bit LCG scramble).""" + return np.uint32(np.uint32(seed) * np.uint32(1664525) + np.uint32(1013904223)) + + +def _xorshift32(state: np.uint32) -> np.uint32: + """One Xorshift32 step.""" + state = np.uint32(state) + state ^= np.uint32(state << np.uint32(13)) + state ^= np.uint32(state >> np.uint32(17)) + state ^= np.uint32(state << np.uint32(5)) + return state + + +def _uint32_to_signed_float(u: np.uint32) -> float: + """Map uint32 uniformly to (-1, 1).""" + return float(u) / float(np.iinfo(np.uint32).max) * 2.0 - 1.0 + + +def _perturb_uniform( + data: np.ndarray, + global_seed: int, + node_id: int, + eps: float, + sign: int = 1, +) -> np.ndarray: + """ + Uniform perturbation matching Deeploy PerturbUniform kernel. + + seed per core = scramble(global_seed + NUM_CORES*node_id + core_id) + rand in (-1, 1) scaled by eps (the ONNX node attribute already encodes + the full scale, e.g. epsilon*2*sqrt(3)). + """ + flat = data.flatten().astype(np.float32) + size = flat.size + log2core = int(math.log2(NUM_CORES)) + + for core_id in range(NUM_CORES): + chunk = (size >> log2core) + (1 if (size & (NUM_CORES - 1)) else 0) + chunk_start = min(chunk * core_id, size) + chunk_stop = min(chunk_start + chunk, size) + + seed = _scramble(global_seed + NUM_CORES * node_id + core_id) + for i in range(chunk_start, chunk_stop): + seed = _xorshift32(seed) + flat[i] += np.float32(sign * _uint32_to_signed_float(seed) * eps) + + return flat.reshape(data.shape) + + +def _perturb_rademacher( + data: np.ndarray, + global_seed: int, + node_id: int, + eps: float, + sign: int = 1, +) -> np.ndarray: + """Rademacher perturbation: each element offset by ยฑeps.""" + flat = data.flatten().astype(np.float32) + size = flat.size + log2core = int(math.log2(NUM_CORES)) + + for core_id in range(NUM_CORES): + chunk = (size >> log2core) + (1 if (size & (NUM_CORES - 1)) else 0) + chunk_start = min(chunk * core_id, size) + chunk_stop = min(chunk_start + chunk, size) + + seed = _scramble(global_seed + NUM_CORES * node_id + core_id) + for i in range(chunk_start, chunk_stop): + seed = _xorshift32(seed) + rad = np.float32(1.0) if (seed & np.uint32(1)) else np.float32(-1.0) + flat[i] += np.float32(sign) * rad * np.float32(eps) + + return flat.reshape(data.shape) + + +def _perturb_normal( + data: np.ndarray, + global_seed: int, + node_id: int, + eps: float, + sign: int = 1, +) -> np.ndarray: + """Gaussian perturbation via Box-Muller on Xorshift32 draws.""" + flat = data.flatten().astype(np.float32) + size = flat.size + log2core = int(math.log2(NUM_CORES)) + + for core_id in range(NUM_CORES): + chunk = (size >> log2core) + (1 if (size & (NUM_CORES - 1)) else 0) + chunk_start = min(chunk * core_id, size) + chunk_stop = min(chunk_start + chunk, size) + + seed = _scramble(global_seed + NUM_CORES * node_id + core_id) + for i in range(chunk_start, chunk_stop): + seed = _xorshift32(seed) + u1 = max(float(seed) / float(np.iinfo(np.uint32).max), 1e-10) + seed = _xorshift32(seed) + u2 = float(seed) / float(np.iinfo(np.uint32).max) * 2.0 * math.pi + z = math.sqrt(-2.0 * math.log(u1)) * math.cos(u2) + flat[i] += np.float32(sign * z * eps) + + return flat.reshape(data.shape) + + +def _perturb_rqs_rademacher( + data: np.ndarray, + mul: np.ndarray, + global_seed: int, + node_id: int, + div: int, + n_levels: int, + signed: int, + sign: int = 1, +) -> np.ndarray: + """RQS Rademacher perturbation for integer-quantised tensors.""" + flat = data.astype(np.int64).flatten() + size = flat.size + log2core = int(math.log2(NUM_CORES)) + + mul_flat = mul.flatten().astype(np.int64) + num_out = mul_flat.size + # Broadcast mul across the remaining dimensions + elems_per = size // num_out if num_out > 0 and size >= num_out else 1 + mul_per_elem = np.repeat(mul_flat, elems_per) + if mul_per_elem.size < size: + mul_per_elem = np.resize(mul_per_elem, size) + + for core_id in range(NUM_CORES): + chunk = (size >> log2core) + (1 if (size & (NUM_CORES - 1)) else 0) + chunk_start = min(chunk * core_id, size) + chunk_stop = min(chunk_start + chunk, size) + + seed = _scramble(global_seed + NUM_CORES * node_id + core_id) + for i in range(chunk_start, chunk_stop): + seed = _xorshift32(seed) + rad = np.int64(1) if (seed & np.uint32(1)) else np.int64(-1) + delta = (rad * mul_per_elem[i]) // np.int64(div) + flat[i] += sign * delta + + lo = -(n_levels // 2) if signed else 0 + hi = (n_levels // 2) - 1 if signed else n_levels - 1 + flat = np.clip(flat, lo, hi) + return flat.reshape(data.shape).astype(data.dtype) + + +def _perturb_rqs_uniform( + data: np.ndarray, + mul: np.ndarray, + global_seed: int, + node_id: int, + div: int, + n_levels: int, + signed: int, + sign: int = 1, +) -> np.ndarray: + """RQS Uniform perturbation for integer-quantised tensors.""" + flat = data.astype(np.int64).flatten() + size = flat.size + log2core = int(math.log2(NUM_CORES)) + + mul_flat = mul.flatten().astype(np.int64) + num_out = mul_flat.size + elems_per = size // num_out if num_out > 0 and size >= num_out else 1 + mul_per_elem = np.repeat(mul_flat, elems_per) + if mul_per_elem.size < size: + mul_per_elem = np.resize(mul_per_elem, size) + + for core_id in range(NUM_CORES): + chunk = (size >> log2core) + (1 if (size & (NUM_CORES - 1)) else 0) + chunk_start = min(chunk * core_id, size) + chunk_stop = min(chunk_start + chunk, size) + + seed = _scramble(global_seed + NUM_CORES * node_id + core_id) + for i in range(chunk_start, chunk_stop): + seed = _xorshift32(seed) + rand_f = _uint32_to_signed_float(seed) + delta = int(rand_f * float(mul_per_elem[i])) // div + flat[i] += sign * delta + + lo = -(n_levels // 2) if signed else 0 + hi = (n_levels // 2) - 1 if signed else n_levels - 1 + flat = np.clip(flat, lo, hi) + return flat.reshape(data.shape).astype(data.dtype) + + +# --------------------------------------------------------------------------- +# Attribute extraction helper +# --------------------------------------------------------------------------- + +def _node_attrs(node: onnx.NodeProto) -> Dict[str, Any]: + """Return node attributes as a plain Python dict.""" + attrs: Dict[str, Any] = {} + for attr in node.attribute: + if attr.type == onnx.AttributeProto.FLOAT: + attrs[attr.name] = attr.f + elif attr.type == onnx.AttributeProto.INT: + attrs[attr.name] = attr.i + elif attr.type == onnx.AttributeProto.STRING: + attrs[attr.name] = attr.s.decode("utf-8") if attr.s else "" + elif attr.type == onnx.AttributeProto.TENSOR: + attrs[attr.name] = numpy_helper.to_array(attr.t) + elif attr.type == onnx.AttributeProto.INTS: + attrs[attr.name] = list(attr.ints) + elif attr.type == onnx.AttributeProto.FLOATS: + attrs[attr.name] = list(attr.floats) + else: + attrs[attr.name] = attr + return attrs + + +# --------------------------------------------------------------------------- +# Standard ONNX op dispatcher +# --------------------------------------------------------------------------- + +def _exec_standard(op: str, inputs: List, attrs: Dict[str, Any]) -> List[np.ndarray]: # noqa: C901 + if op == "Add": + return [inputs[0] + inputs[1]] + if op == "Sub": + return [inputs[0] - inputs[1]] + if op == "Mul": + return [inputs[0] * inputs[1]] + if op == "Div": + return [inputs[0] / inputs[1]] + if op == "Neg": + return [-inputs[0]] + if op == "Abs": + return [np.abs(inputs[0])] + if op == "Sqrt": + return [np.sqrt(inputs[0])] + if op == "Exp": + return [np.exp(inputs[0])] + if op == "Log": + return [np.log(inputs[0])] + if op == "Pow": + return [np.power(inputs[0], inputs[1])] + if op == "Erf": + return [np.vectorize(math.erf)(inputs[0]).astype(inputs[0].dtype)] + if op == "Ceil": + return [np.ceil(inputs[0])] + if op == "Floor": + return [np.floor(inputs[0])] + if op == "Round": + return [np.round(inputs[0])] + if op == "Sign": + return [np.sign(inputs[0]).astype(inputs[0].dtype)] + + if op == "Clip": + lo = inputs[1] if len(inputs) > 1 and inputs[1] is not None else attrs.get("min", None) + hi = inputs[2] if len(inputs) > 2 and inputs[2] is not None else attrs.get("max", None) + return [np.clip(inputs[0], lo, hi)] + + if op in ("ReduceSum", "ReduceMean", "ReduceMax", "ReduceMin"): + x = inputs[0] + axes = attrs.get("axes", None) + keepdims = bool(attrs.get("keepdims", 1)) + noop_empty = bool(attrs.get("noop_with_empty_axes", 0)) + if len(inputs) > 1 and inputs[1] is not None: + axes = tuple(int(a) for a in inputs[1].flatten()) + elif axes is not None: + axes = tuple(axes) + if noop_empty and (axes is None or len(axes) == 0): + return [x] + fn = {"ReduceSum": np.sum, "ReduceMean": np.mean, + "ReduceMax": np.max, "ReduceMin": np.min}[op] + return [fn(x, axis=axes, keepdims=keepdims)] + + if op == "Relu": + return [np.maximum(inputs[0], 0)] + if op == "Sigmoid": + return [1.0 / (1.0 + np.exp(-inputs[0].astype(np.float64))).astype(np.float32)] + if op == "Tanh": + return [np.tanh(inputs[0])] + if op == "LeakyRelu": + alpha = float(attrs.get("alpha", 0.01)) + x = inputs[0] + return [np.where(x >= 0, x, alpha * x).astype(x.dtype)] + if op in ("Gelu", "FastGelu"): + x = inputs[0] + return [(x * 0.5 * (1.0 + np.vectorize(math.erf)(x / math.sqrt(2)))).astype(x.dtype)] + if op == "Softmax": + axis = int(attrs.get("axis", -1)) + x = inputs[0] + x_max = np.max(x, axis=axis, keepdims=True) + ex = np.exp(x - x_max) + return [(ex / np.sum(ex, axis=axis, keepdims=True)).astype(x.dtype)] + + if op == "MatMul": + return [np.matmul(inputs[0], inputs[1])] + + if op == "Gemm": + A, B = inputs[0], inputs[1] + C = inputs[2] if len(inputs) > 2 and inputs[2] is not None else 0.0 + alpha = float(attrs.get("alpha", 1.0)) + beta = float(attrs.get("beta", 1.0)) + if int(attrs.get("transA", 0)): + A = A.T + if int(attrs.get("transB", 0)): + B = B.T + return [(alpha * np.matmul(A, B) + beta * C).astype(A.dtype)] + + if op == "Conv": + x, w = inputs[0], inputs[1] + b = inputs[2] if len(inputs) > 2 and inputs[2] is not None else None + groups = int(attrs.get("group", 1)) + dilations = list(attrs.get("dilations", [1, 1])) + strides = list(attrs.get("strides", [1, 1])) + pads = list(attrs.get("pads", [0, 0, 0, 0])) + N, C_in, H, W = x.shape + C_out, C_in_pg, kH, kW = w.shape + sH, sW = strides[0], strides[1] + dH, dW = dilations[0], dilations[1] + pH_t, pW_l = pads[0], pads[1] + pH_b, pW_r = pads[2], pads[3] + H_out = (H + pH_t + pH_b - dH * (kH - 1) - 1) // sH + 1 + W_out = (W + pW_l + pW_r - dW * (kW - 1) - 1) // sW + 1 + x_pad = np.pad(x, ((0,0),(0,0),(pH_t,pH_b),(pW_l,pW_r))) + out = np.zeros((N, C_out, H_out, W_out), dtype=np.float32) + cpp = C_out // groups + for g in range(groups): + xi = x_pad[:, g*C_in_pg:(g+1)*C_in_pg] + wg = w[g*cpp:(g+1)*cpp] + for oc in range(cpp): + for oh in range(H_out): + for ow in range(W_out): + patch = xi[:, :, + oh*sH:oh*sH+kH*dH:dH, + ow*sW:ow*sW+kW*dW:dW] + out[:, g*cpp+oc, oh, ow] = np.sum( + patch * wg[oc], axis=(1, 2, 3)) + if b is not None: + out += b[np.newaxis, :, np.newaxis, np.newaxis] + return [out] + + if op == "BatchNormalization": + x, scale, bias, mean, var = inputs[:5] + eps = float(attrs.get("epsilon", 1e-5)) + return [(scale * (x - mean) / np.sqrt(var + eps) + bias).astype(x.dtype)] + + if op == "LayerNormalization": + x = inputs[0] + scale = inputs[1] if len(inputs) > 1 and inputs[1] is not None else np.ones(1, dtype=x.dtype) + b = inputs[2] if len(inputs) > 2 and inputs[2] is not None else np.zeros(1, dtype=x.dtype) + axis = int(attrs.get("axis", -1)) + eps = float(attrs.get("epsilon", 1e-5)) + mean = np.mean(x, axis=axis, keepdims=True) + var = np.var(x, axis=axis, keepdims=True) + out = ((x - mean) / np.sqrt(var + eps)) * scale + b + return [out.astype(x.dtype)] + + if op == "GroupNormalization": + x, scale, bias = inputs[:3] + num_groups = int(attrs.get("num_groups", 1)) + eps = float(attrs.get("epsilon", 1e-5)) + N, C = x.shape[:2] + spatial = x.shape[2:] + xr = x.reshape(N, num_groups, C // num_groups, *spatial) + axes = tuple(range(2, xr.ndim)) + mean = np.mean(xr, axis=axes, keepdims=True) + var = np.var(xr, axis=axes, keepdims=True) + xn = ((xr - mean) / np.sqrt(var + eps)).reshape(N, C, *spatial) + return [(xn * scale.reshape(1, C, *([1]*len(spatial))) + + bias.reshape(1, C, *([1]*len(spatial)))).astype(x.dtype)] + + if op == "MaxPool": + x = inputs[0] + kernel = list(attrs.get("kernel_shape", [2, 2])) + strides = list(attrs.get("strides", [1, 1])) + pads = list(attrs.get("pads", [0, 0, 0, 0])) + N, C, H, W = x.shape + kH, kW = kernel[0], kernel[1] + sH, sW = strides[0], strides[1] + pH_t, pW_l, pH_b, pW_r = pads[0], pads[1], pads[2], pads[3] + xp = np.pad(x, ((0,0),(0,0),(pH_t,pH_b),(pW_l,pW_r)), constant_values=-np.inf) + H_out = (H + pH_t + pH_b - kH) // sH + 1 + W_out = (W + pW_l + pW_r - kW) // sW + 1 + out = np.stack([[ + np.max(xp[:, :, oh*sH:oh*sH+kH, ow*sW:ow*sW+kW], axis=(2, 3)) + for ow in range(W_out)] for oh in range(H_out)], axis=0) + return [out.transpose(2, 3, 0, 1)] # (N,C,H_out,W_out) + + if op == "AveragePool": + x = inputs[0] + kernel = list(attrs.get("kernel_shape", [2, 2])) + strides = list(attrs.get("strides", [1, 1])) + pads = list(attrs.get("pads", [0, 0, 0, 0])) + N, C, H, W = x.shape + kH, kW = kernel[0], kernel[1] + sH, sW = strides[0], strides[1] + pH_t, pW_l, pH_b, pW_r = pads[0], pads[1], pads[2], pads[3] + xp = np.pad(x, ((0,0),(0,0),(pH_t,pH_b),(pW_l,pW_r))) + H_out = (H + pH_t + pH_b - kH) // sH + 1 + W_out = (W + pW_l + pW_r - kW) // sW + 1 + out = np.stack([[ + np.mean(xp[:, :, oh*sH:oh*sH+kH, ow*sW:ow*sW+kW], axis=(2, 3)) + for ow in range(W_out)] for oh in range(H_out)], axis=0) + return [out.transpose(2, 3, 0, 1)] + + if op == "GlobalAveragePool": + x = inputs[0] + return [np.mean(x, axis=tuple(range(2, x.ndim)), keepdims=True)] + if op == "GlobalMaxPool": + x = inputs[0] + return [np.max(x, axis=tuple(range(2, x.ndim)), keepdims=True)] + + if op == "Reshape": + shape = [int(d) for d in inputs[1].flatten()] + src = inputs[0].shape + new_shape = [int(src[i]) if d == 0 else d for i, d in enumerate(shape)] + return [inputs[0].reshape(new_shape)] + if op == "Flatten": + axis = int(attrs.get("axis", 1)) + x = inputs[0] + pre = int(np.prod(x.shape[:axis])) if axis > 0 else 1 + post = int(np.prod(x.shape[axis:])) + return [x.reshape(pre, post)] + if op == "Transpose": + perm = attrs.get("perm", None) + return [np.transpose(inputs[0], axes=perm)] + + if op == "Squeeze": + x = inputs[0] + if len(inputs) > 1 and inputs[1] is not None: + axes = tuple(int(a) for a in inputs[1].flatten()) + else: + raw = attrs.get("axes", None) + if raw is None: + axes = None + elif isinstance(raw, (int, np.integer)): + axes = (int(raw),) + else: + axes = tuple(int(a) for a in raw) or None + return [np.squeeze(x, axis=axes)] + if op == "Unsqueeze": + x = inputs[0] + if len(inputs) > 1 and inputs[1] is not None: + axes = sorted(int(a) for a in inputs[1].flatten()) + else: + axes = sorted(attrs.get("axes", [])) + for ax in axes: + x = np.expand_dims(x, axis=ax) + return [x] + if op == "Expand": + return [np.broadcast_to(inputs[0], inputs[1].tolist()).copy()] + if op == "Concat": + axis = int(attrs.get("axis", 0)) + valid = [t for t in inputs if t is not None] + return [np.concatenate(valid, axis=axis)] + if op == "Split": + axis = int(attrs.get("axis", 0)) + split = list(attrs.get("split", [])) + if len(inputs) > 1 and inputs[1] is not None: + split = inputs[1].flatten().tolist() + if not split: + num_out = int(attrs.get("num_outputs", 2)) + split = [inputs[0].shape[axis] // num_out] * num_out + secs = np.cumsum([int(s) for s in split[:-1]]).tolist() + return list(np.split(inputs[0], secs, axis=axis)) + if op == "Slice": + data, starts, ends = inputs[0], inputs[1].flatten(), inputs[2].flatten() + axes = inputs[3].flatten() if len(inputs) > 3 and inputs[3] is not None else np.arange(len(starts)) + steps = inputs[4].flatten() if len(inputs) > 4 and inputs[4] is not None else np.ones(len(starts), dtype=np.int64) + idx = [slice(None)] * data.ndim + for ax, s, e, st in zip(axes, starts, ends, steps): + idx[int(ax)] = slice(int(s), int(e), int(st)) + return [data[tuple(idx)]] + if op == "Gather": + axis = int(attrs.get("axis", 0)) + return [np.take(inputs[0], inputs[1].astype(np.int64), axis=axis)] + if op == "GatherElements": + axis = int(attrs.get("axis", 0)) + return [np.take_along_axis(inputs[0], inputs[1].astype(np.int64), axis=axis)] + if op == "Shape": + start = int(attrs.get("start", 0)) + end = attrs.get("end", None) + sh = np.array(inputs[0].shape, dtype=np.int64) + return [sh[start:end]] + if op == "Size": + return [np.array(inputs[0].size, dtype=np.int64)] + if op == "Cast": + to_type = int(attrs.get("to", TensorProto.FLOAT)) + _DT = { + TensorProto.FLOAT: np.float32, + TensorProto.DOUBLE: np.float64, + TensorProto.INT32: np.int32, + TensorProto.INT64: np.int64, + TensorProto.INT8: np.int8, + TensorProto.UINT8: np.uint8, + TensorProto.BOOL: bool, + TensorProto.FLOAT16: np.float16, + } + return [inputs[0].astype(_DT.get(to_type, np.float32))] + if op == "Identity": + return [inputs[0].copy()] + if op == "Pad": + x = inputs[0] + pads_arr = inputs[1].flatten().tolist() if len(inputs) > 1 and inputs[1] is not None else list(attrs.get("pads", [])) + val = float(inputs[2].flat[0]) if len(inputs) > 2 and inputs[2] is not None else float(attrs.get("value", 0.0)) + n = x.ndim + pw = [(int(pads_arr[i]), int(pads_arr[i+n])) for i in range(n)] + return [np.pad(x, pw, constant_values=val)] + if op == "Tile": + return [np.tile(inputs[0], inputs[1].flatten().tolist())] + if op == "Where": + return [np.where(inputs[0], inputs[1], inputs[2])] + if op in ("Equal", "Less", "Greater", "LessOrEqual", "GreaterOrEqual"): + fn = {"Equal": np.equal, "Less": np.less, "Greater": np.greater, + "LessOrEqual": np.less_equal, "GreaterOrEqual": np.greater_equal}[op] + return [fn(inputs[0], inputs[1])] + if op == "Not": + return [~inputs[0]] + if op == "Min": + r = inputs[0] + for t in inputs[1:]: + r = np.minimum(r, t) + return [r] + if op == "Max": + r = inputs[0] + for t in inputs[1:]: + r = np.maximum(r, t) + return [r] + if op == "Sum": + r = inputs[0].copy() + for t in inputs[1:]: + r = r + t + return [r] + if op == "Mean": + return [sum(inputs) / len(inputs)] + if op == "Einsum": + return [np.einsum(attrs["equation"], *inputs)] + if op == "ArgMax": + axis = int(attrs.get("axis", 0)) + keepdims = bool(attrs.get("keepdims", 1)) + return [np.argmax(inputs[0], axis=axis, keepdims=keepdims).astype(np.int64)] + if op == "ArgMin": + axis = int(attrs.get("axis", 0)) + keepdims = bool(attrs.get("keepdims", 1)) + return [np.argmin(inputs[0], axis=axis, keepdims=keepdims).astype(np.int64)] + if op == "TopK": + k = int(inputs[1].flat[0]) if len(inputs) > 1 else int(attrs.get("k", 1)) + axis = int(attrs.get("axis", -1)) + largest = bool(attrs.get("largest", 1)) + idx = np.argsort(inputs[0], axis=axis) + if largest: + idx = np.flip(idx, axis=axis) + idx = np.take(idx, np.arange(k), axis=axis) + vals = np.take_along_axis(inputs[0], idx, axis=axis) + return [vals, idx.astype(np.int64)] + if op == "ConstantOfShape": + shape = inputs[0].flatten().tolist() + val_t = attrs.get("value", None) + fill = float(val_t.flat[0]) if val_t is not None else 0.0 + return [np.full([int(d) for d in shape], fill, dtype=np.float32)] + if op == "Constant": + val_t = attrs.get("value", None) + if val_t is not None: + return [val_t.copy() if isinstance(val_t, np.ndarray) else np.array(val_t)] + vf = attrs.get("value_float", None) + if vf is not None: + return [np.array(vf, dtype=np.float32)] + vi = attrs.get("value_int", None) + if vi is not None: + return [np.array(vi, dtype=np.int64)] + raise ValueError("Constant node has no supported value attribute") + if op == "Range": + s, lim, d = inputs + return [np.arange(float(s), float(lim), float(d)).astype(s.dtype)] + if op == "NonZero": + return [np.array(np.nonzero(inputs[0]), dtype=np.int64)] + if op == "Dropout": + return [inputs[0].copy(), np.ones_like(inputs[0], dtype=bool)] + if op == "SoftmaxCrossEntropyLoss": + logits = inputs[0] + labels = inputs[1] + xm = np.max(logits, axis=-1, keepdims=True) + lp = logits - xm - np.log(np.sum(np.exp(logits - xm), axis=-1, keepdims=True)) + if labels.dtype in (np.int32, np.int64): + nll = -lp[np.arange(logits.shape[0]), labels.flatten()] + else: + nll = -np.sum(lp * labels, axis=-1) + red = attrs.get("reduction", "mean") + loss = np.mean(nll) if red == "mean" else (np.sum(nll) if red == "sum" else nll) + return [np.array(loss, dtype=np.float32), lp.astype(np.float32)] + + raise NotImplementedError(f"Standard ONNX op '{op}' is not implemented in run_onnx_graph") + + +# --------------------------------------------------------------------------- +# Deeploy custom ops (ai.onnx.contrib) +# --------------------------------------------------------------------------- + +def _exec_deeploy(op: str, inputs: List, attrs: Dict[str, Any]) -> List[np.ndarray]: + if op == "Quant": + x = inputs[0].astype(np.float64) + scale = inputs[1].astype(np.float64) + zp = inputs[2].astype(np.float64) + bits = int(attrs.get("bits", 8)) + signed = bool(attrs.get("signed", True)) + qmin = -(2 ** (bits-1)) if signed else 0 + qmax = (2 ** (bits-1)) - 1 if signed else (2**bits) - 1 + return [np.clip(np.round(x / scale) + zp, qmin, qmax).astype(np.int8 if signed else np.uint8)] + + if op == "Dequant": + q = inputs[0].astype(np.float64) + scale = inputs[1].astype(np.float64) + zp = inputs[2].astype(np.float64) + return [((q - zp) * scale).astype(np.float32)] + + if op == "RequantShift": + x = inputs[0].astype(np.int64) + mul = inputs[1].astype(np.int64) + add = inputs[2].astype(np.int64) + div_attr = attrs.get("div", None) + if div_attr is None: + raise ValueError("RequantShift missing 'div' tensor attribute") + div = int(div_attr.flat[0]) if hasattr(div_attr, "flat") else int(div_attr) + bits = int(attrs.get("out_bits", 8)) + signed = bool(attrs.get("signed", 1)) + qmin = -(2**(bits-1)) if signed else 0 + qmax = (2**(bits-1))-1 if signed else (2**bits)-1 + return [np.clip((x * mul + add) // div, qmin, qmax).astype(np.int8 if signed else np.uint8)] + + raise NotImplementedError(f"Deeploy op '{op}' not implemented") + + +# --------------------------------------------------------------------------- +# MeZO perturbation ops (mezo / com.microsoft) +# --------------------------------------------------------------------------- + +def _exec_mezo(op: str, inputs: List, attrs: Dict[str, Any]) -> List[np.ndarray]: + seed = int(attrs.get("seed", 0)) + node_id = int(attrs.get("idx", 0)) + eps = float(attrs.get("eps", 0.01)) + sign = 1 # forward perturbation sign + + if op == "PerturbUniform": + return [_perturb_uniform(inputs[0].astype(np.float32), seed, node_id, eps, sign)] + if op == "PerturbRademacher": + return [_perturb_rademacher(inputs[0].astype(np.float32), seed, node_id, eps, sign)] + if op == "PerturbNormal": + return [_perturb_normal(inputs[0].astype(np.float32), seed, node_id, eps, sign)] + if op == "PerturbTriangle": + return [_perturb_uniform(inputs[0].astype(np.float32), seed, node_id, eps, sign)] + if op == "PerturbEggroll": + # Generates a Rademacher column vector of the requested shape + shape = [int(d) for d in inputs[0].flatten()] + x = np.zeros(shape, dtype=np.float32) + return [_perturb_rademacher(x, seed, node_id, 1.0, 1)] + if op == "RQSPerturbRademacher": + mul = inputs[1] if len(inputs) > 1 and inputs[1] is not None else np.ones(inputs[0].shape[0], dtype=np.int64) + div = int(attrs.get("div", 2**15)) + n_levels = int(attrs.get("n_levels", 256)) + signed_flag = int(attrs.get("signed", 1)) + return [_perturb_rqs_rademacher(inputs[0], mul, seed, node_id, div, n_levels, signed_flag, sign)] + if op == "RQSPerturbUniform": + mul = inputs[1] if len(inputs) > 1 and inputs[1] is not None else np.ones(inputs[0].shape[0], dtype=np.int64) + div = int(attrs.get("div", 2**15)) + n_levels = int(attrs.get("n_levels", 256)) + signed_flag = int(attrs.get("signed", 1)) + return [_perturb_rqs_uniform(inputs[0], mul, seed, node_id, div, n_levels, signed_flag, sign)] + + raise NotImplementedError(f"MeZO op '{op}' not implemented") + + +# --------------------------------------------------------------------------- +# Public entry point +# --------------------------------------------------------------------------- + +def run_onnx_graph( + onnx_path: str, + inputs: Dict[str, np.ndarray], + output_names: Optional[List[str]] = None, +) -> np.ndarray: + """ + Execute an ONNX graph in pure Python โ€” no onnxruntime required. + + Supports standard ONNX ops, Deeploy custom ops (ai.onnx.contrib domain: + Quant, Dequant, RequantShift), and MeZO perturbation ops (mezo domain: + PerturbUniform, PerturbRademacher, PerturbNormal, PerturbTriangle, + PerturbEggroll, RQSPerturbRademacher, RQSPerturbUniform). + + Args: + onnx_path: Path to the ``.onnx`` model file. + inputs: ``{name: ndarray}`` for each graph input. + output_names: If given, return those specific outputs. + Otherwise the first graph output is returned. + + Returns: + First (or requested) graph output as a numpy array. + """ + model = onnx.load(onnx_path) + graph = model.graph + + # value store: name โ†’ numpy array + values: Dict[str, np.ndarray] = {} + values.update(inputs) + + for init in graph.initializer: + if init.name not in values: + values[init.name] = numpy_helper.to_array(init) + + # ONNX graph is topologically sorted by spec + for node in graph.node: + op = node.op_type + domain = (node.domain or "").strip() + attrs = _node_attrs(node) + + # Gather inputs; absent optional inputs (empty string) become None + node_inputs: List = [] + for name in node.input: + if name == "": + node_inputs.append(None) + elif name in values: + node_inputs.append(values[name]) + else: + raise KeyError( + f"Node '{node.name}' (op='{op}') needs input '{name}' " + f"which is missing from value map." + ) + + # com.microsoft is used for both ORT contrib ops (e.g. Gelu, + # Attention) and MeZO perturbation ops. Route by op name. + _MEZO_OPS = { + "PerturbUniform", "PerturbRademacher", "PerturbNormal", + "PerturbTriangle", "PerturbEggroll", + "RQSPerturbRademacher", "RQSPerturbUniform", + } + + try: + if domain in ("", "ai.onnx"): + outs = _exec_standard(op, node_inputs, attrs) + elif domain == "ai.onnx.contrib": + outs = _exec_deeploy(op, node_inputs, attrs) + elif domain == "mezo" or (domain == "com.microsoft" and op in _MEZO_OPS): + outs = _exec_mezo(op, node_inputs, attrs) + elif domain == "com.microsoft": + # ORT contrib ops that map to standard implementations + outs = _exec_standard(op, node_inputs, attrs) + else: + raise NotImplementedError( + f"Unsupported domain '{domain}' for op '{op}'" + ) + except NotImplementedError: + raise + except Exception as exc: + raise RuntimeError( + f"Error in node '{node.name}' op='{op}' domain='{domain}': {exc}" + ) from exc + + for out_name, out_val in zip(node.output, outs): + if out_name: + values[out_name] = out_val + + graph_outputs = [o.name for o in graph.output] + if output_names is not None: + return [values[n] for n in output_names] + if graph_outputs: + return values[graph_outputs[0]] + raise RuntimeError("ONNX graph has no outputs") diff --git a/tests/models/onnx_node_implementations.py b/tests/models/onnx_node_implementations.py index 686de66..604efc1 100644 --- a/tests/models/onnx_node_implementations.py +++ b/tests/models/onnx_node_implementations.py @@ -2,793 +2,7 @@ # # SPDX-License-Identifier: MIT -""" -Pure-Python ONNX graph executor. +"""Re-export from canonical location in onnx4deeploy.utils.""" -Executes ONNX graphs without relying on onnxruntime.InferenceSession, so that -graphs containing custom ops (Deeploy Quant/Dequant/RequantShift and MeZO -perturbation ops) can be run natively. - -Supported op domains: - - "" (standard ONNX ops) - - "ai.onnx.contrib" โ†’ Quant, Dequant, RequantShift - - "mezo" โ†’ PerturbNormal, PerturbUniform, PerturbRademacher, - PerturbTriangle, PerturbEggroll, - RQSPerturbRademacher, RQSPerturbUniform -""" - -from __future__ import annotations - -import math -from typing import Any, Dict, List, Optional - -import numpy as np -import onnx -from onnx import numpy_helper, TensorProto - -# --------------------------------------------------------------------------- -# RNG reference matching Deeploy C++ implementation -# --------------------------------------------------------------------------- - -NUM_CORES: int = 8 - - -def _scramble(seed: int) -> np.uint32: - """seed * 1664525 + 1013904223 (mod 2**32, 32-bit LCG scramble).""" - return np.uint32(np.uint32(seed) * np.uint32(1664525) + np.uint32(1013904223)) - - -def _xorshift32(state: np.uint32) -> np.uint32: - """One Xorshift32 step.""" - state = np.uint32(state) - state ^= np.uint32(state << np.uint32(13)) - state ^= np.uint32(state >> np.uint32(17)) - state ^= np.uint32(state << np.uint32(5)) - return state - - -def _uint32_to_signed_float(u: np.uint32) -> float: - """Map uint32 uniformly to (-1, 1).""" - return float(u) / float(np.iinfo(np.uint32).max) * 2.0 - 1.0 - - -def _perturb_uniform( - data: np.ndarray, - global_seed: int, - node_id: int, - eps: float, - sign: int = 1, -) -> np.ndarray: - """ - Uniform perturbation matching Deeploy PerturbUniform kernel. - - seed per core = scramble(global_seed + NUM_CORES*node_id + core_id) - rand in (-1, 1) scaled by eps (the ONNX node attribute already encodes - the full scale, e.g. epsilon*2*sqrt(3)). - """ - flat = data.flatten().astype(np.float32) - size = flat.size - log2core = int(math.log2(NUM_CORES)) - - for core_id in range(NUM_CORES): - chunk = (size >> log2core) + (1 if (size & (NUM_CORES - 1)) else 0) - chunk_start = min(chunk * core_id, size) - chunk_stop = min(chunk_start + chunk, size) - - seed = _scramble(global_seed + NUM_CORES * node_id + core_id) - for i in range(chunk_start, chunk_stop): - seed = _xorshift32(seed) - flat[i] += np.float32(sign * _uint32_to_signed_float(seed) * eps) - - return flat.reshape(data.shape) - - -def _perturb_rademacher( - data: np.ndarray, - global_seed: int, - node_id: int, - eps: float, - sign: int = 1, -) -> np.ndarray: - """Rademacher perturbation: each element offset by ยฑeps.""" - flat = data.flatten().astype(np.float32) - size = flat.size - log2core = int(math.log2(NUM_CORES)) - - for core_id in range(NUM_CORES): - chunk = (size >> log2core) + (1 if (size & (NUM_CORES - 1)) else 0) - chunk_start = min(chunk * core_id, size) - chunk_stop = min(chunk_start + chunk, size) - - seed = _scramble(global_seed + NUM_CORES * node_id + core_id) - for i in range(chunk_start, chunk_stop): - seed = _xorshift32(seed) - rad = np.float32(1.0) if (seed & np.uint32(1)) else np.float32(-1.0) - flat[i] += np.float32(sign) * rad * np.float32(eps) - - return flat.reshape(data.shape) - - -def _perturb_normal( - data: np.ndarray, - global_seed: int, - node_id: int, - eps: float, - sign: int = 1, -) -> np.ndarray: - """Gaussian perturbation via Box-Muller on Xorshift32 draws.""" - flat = data.flatten().astype(np.float32) - size = flat.size - log2core = int(math.log2(NUM_CORES)) - - for core_id in range(NUM_CORES): - chunk = (size >> log2core) + (1 if (size & (NUM_CORES - 1)) else 0) - chunk_start = min(chunk * core_id, size) - chunk_stop = min(chunk_start + chunk, size) - - seed = _scramble(global_seed + NUM_CORES * node_id + core_id) - for i in range(chunk_start, chunk_stop): - seed = _xorshift32(seed) - u1 = max(float(seed) / float(np.iinfo(np.uint32).max), 1e-10) - seed = _xorshift32(seed) - u2 = float(seed) / float(np.iinfo(np.uint32).max) * 2.0 * math.pi - z = math.sqrt(-2.0 * math.log(u1)) * math.cos(u2) - flat[i] += np.float32(sign * z * eps) - - return flat.reshape(data.shape) - - -def _perturb_rqs_rademacher( - data: np.ndarray, - mul: np.ndarray, - global_seed: int, - node_id: int, - div: int, - n_levels: int, - signed: int, - sign: int = 1, -) -> np.ndarray: - """RQS Rademacher perturbation for integer-quantised tensors.""" - flat = data.astype(np.int64).flatten() - size = flat.size - log2core = int(math.log2(NUM_CORES)) - - mul_flat = mul.flatten().astype(np.int64) - num_out = mul_flat.size - # Broadcast mul across the remaining dimensions - elems_per = size // num_out if num_out > 0 and size >= num_out else 1 - mul_per_elem = np.repeat(mul_flat, elems_per) - if mul_per_elem.size < size: - mul_per_elem = np.resize(mul_per_elem, size) - - for core_id in range(NUM_CORES): - chunk = (size >> log2core) + (1 if (size & (NUM_CORES - 1)) else 0) - chunk_start = min(chunk * core_id, size) - chunk_stop = min(chunk_start + chunk, size) - - seed = _scramble(global_seed + NUM_CORES * node_id + core_id) - for i in range(chunk_start, chunk_stop): - seed = _xorshift32(seed) - rad = np.int64(1) if (seed & np.uint32(1)) else np.int64(-1) - delta = (rad * mul_per_elem[i]) // np.int64(div) - flat[i] += sign * delta - - lo = -(n_levels // 2) if signed else 0 - hi = (n_levels // 2) - 1 if signed else n_levels - 1 - flat = np.clip(flat, lo, hi) - return flat.reshape(data.shape).astype(data.dtype) - - -def _perturb_rqs_uniform( - data: np.ndarray, - mul: np.ndarray, - global_seed: int, - node_id: int, - div: int, - n_levels: int, - signed: int, - sign: int = 1, -) -> np.ndarray: - """RQS Uniform perturbation for integer-quantised tensors.""" - flat = data.astype(np.int64).flatten() - size = flat.size - log2core = int(math.log2(NUM_CORES)) - - mul_flat = mul.flatten().astype(np.int64) - num_out = mul_flat.size - elems_per = size // num_out if num_out > 0 and size >= num_out else 1 - mul_per_elem = np.repeat(mul_flat, elems_per) - if mul_per_elem.size < size: - mul_per_elem = np.resize(mul_per_elem, size) - - for core_id in range(NUM_CORES): - chunk = (size >> log2core) + (1 if (size & (NUM_CORES - 1)) else 0) - chunk_start = min(chunk * core_id, size) - chunk_stop = min(chunk_start + chunk, size) - - seed = _scramble(global_seed + NUM_CORES * node_id + core_id) - for i in range(chunk_start, chunk_stop): - seed = _xorshift32(seed) - rand_f = _uint32_to_signed_float(seed) - delta = int(rand_f * float(mul_per_elem[i])) // div - flat[i] += sign * delta - - lo = -(n_levels // 2) if signed else 0 - hi = (n_levels // 2) - 1 if signed else n_levels - 1 - flat = np.clip(flat, lo, hi) - return flat.reshape(data.shape).astype(data.dtype) - - -# --------------------------------------------------------------------------- -# Attribute extraction helper -# --------------------------------------------------------------------------- - -def _node_attrs(node: onnx.NodeProto) -> Dict[str, Any]: - """Return node attributes as a plain Python dict.""" - attrs: Dict[str, Any] = {} - for attr in node.attribute: - if attr.type == onnx.AttributeProto.FLOAT: - attrs[attr.name] = attr.f - elif attr.type == onnx.AttributeProto.INT: - attrs[attr.name] = attr.i - elif attr.type == onnx.AttributeProto.STRING: - attrs[attr.name] = attr.s.decode("utf-8") if attr.s else "" - elif attr.type == onnx.AttributeProto.TENSOR: - attrs[attr.name] = numpy_helper.to_array(attr.t) - elif attr.type == onnx.AttributeProto.INTS: - attrs[attr.name] = list(attr.ints) - elif attr.type == onnx.AttributeProto.FLOATS: - attrs[attr.name] = list(attr.floats) - else: - attrs[attr.name] = attr - return attrs - - -# --------------------------------------------------------------------------- -# Standard ONNX op dispatcher -# --------------------------------------------------------------------------- - -def _exec_standard(op: str, inputs: List, attrs: Dict[str, Any]) -> List[np.ndarray]: # noqa: C901 - if op == "Add": - return [inputs[0] + inputs[1]] - if op == "Sub": - return [inputs[0] - inputs[1]] - if op == "Mul": - return [inputs[0] * inputs[1]] - if op == "Div": - return [inputs[0] / inputs[1]] - if op == "Neg": - return [-inputs[0]] - if op == "Abs": - return [np.abs(inputs[0])] - if op == "Sqrt": - return [np.sqrt(inputs[0])] - if op == "Exp": - return [np.exp(inputs[0])] - if op == "Log": - return [np.log(inputs[0])] - if op == "Pow": - return [np.power(inputs[0], inputs[1])] - if op == "Erf": - return [np.vectorize(math.erf)(inputs[0]).astype(inputs[0].dtype)] - if op == "Ceil": - return [np.ceil(inputs[0])] - if op == "Floor": - return [np.floor(inputs[0])] - if op == "Round": - return [np.round(inputs[0])] - if op == "Sign": - return [np.sign(inputs[0]).astype(inputs[0].dtype)] - - if op == "Clip": - lo = inputs[1] if len(inputs) > 1 and inputs[1] is not None else attrs.get("min", None) - hi = inputs[2] if len(inputs) > 2 and inputs[2] is not None else attrs.get("max", None) - return [np.clip(inputs[0], lo, hi)] - - if op in ("ReduceSum", "ReduceMean", "ReduceMax", "ReduceMin"): - x = inputs[0] - axes = attrs.get("axes", None) - keepdims = bool(attrs.get("keepdims", 1)) - noop_empty = bool(attrs.get("noop_with_empty_axes", 0)) - if len(inputs) > 1 and inputs[1] is not None: - axes = tuple(int(a) for a in inputs[1].flatten()) - elif axes is not None: - axes = tuple(axes) - if noop_empty and (axes is None or len(axes) == 0): - return [x] - fn = {"ReduceSum": np.sum, "ReduceMean": np.mean, - "ReduceMax": np.max, "ReduceMin": np.min}[op] - return [fn(x, axis=axes, keepdims=keepdims)] - - if op == "Relu": - return [np.maximum(inputs[0], 0)] - if op == "Sigmoid": - return [1.0 / (1.0 + np.exp(-inputs[0].astype(np.float64))).astype(np.float32)] - if op == "Tanh": - return [np.tanh(inputs[0])] - if op == "LeakyRelu": - alpha = float(attrs.get("alpha", 0.01)) - x = inputs[0] - return [np.where(x >= 0, x, alpha * x).astype(x.dtype)] - if op in ("Gelu", "FastGelu"): - x = inputs[0] - return [(x * 0.5 * (1.0 + np.vectorize(math.erf)(x / math.sqrt(2)))).astype(x.dtype)] - if op == "Softmax": - axis = int(attrs.get("axis", -1)) - x = inputs[0] - x_max = np.max(x, axis=axis, keepdims=True) - ex = np.exp(x - x_max) - return [(ex / np.sum(ex, axis=axis, keepdims=True)).astype(x.dtype)] - - if op == "MatMul": - return [np.matmul(inputs[0], inputs[1])] - - if op == "Gemm": - A, B = inputs[0], inputs[1] - C = inputs[2] if len(inputs) > 2 and inputs[2] is not None else 0.0 - alpha = float(attrs.get("alpha", 1.0)) - beta = float(attrs.get("beta", 1.0)) - if int(attrs.get("transA", 0)): - A = A.T - if int(attrs.get("transB", 0)): - B = B.T - return [(alpha * np.matmul(A, B) + beta * C).astype(A.dtype)] - - if op == "Conv": - x, w = inputs[0], inputs[1] - b = inputs[2] if len(inputs) > 2 and inputs[2] is not None else None - groups = int(attrs.get("group", 1)) - dilations = list(attrs.get("dilations", [1, 1])) - strides = list(attrs.get("strides", [1, 1])) - pads = list(attrs.get("pads", [0, 0, 0, 0])) - N, C_in, H, W = x.shape - C_out, C_in_pg, kH, kW = w.shape - sH, sW = strides[0], strides[1] - dH, dW = dilations[0], dilations[1] - pH_t, pW_l = pads[0], pads[1] - pH_b, pW_r = pads[2], pads[3] - H_out = (H + pH_t + pH_b - dH * (kH - 1) - 1) // sH + 1 - W_out = (W + pW_l + pW_r - dW * (kW - 1) - 1) // sW + 1 - x_pad = np.pad(x, ((0,0),(0,0),(pH_t,pH_b),(pW_l,pW_r))) - out = np.zeros((N, C_out, H_out, W_out), dtype=np.float32) - cpp = C_out // groups - for g in range(groups): - xi = x_pad[:, g*C_in_pg:(g+1)*C_in_pg] - wg = w[g*cpp:(g+1)*cpp] - for oc in range(cpp): - for oh in range(H_out): - for ow in range(W_out): - patch = xi[:, :, - oh*sH:oh*sH+kH*dH:dH, - ow*sW:ow*sW+kW*dW:dW] - out[:, g*cpp+oc, oh, ow] = np.sum( - patch * wg[oc], axis=(1, 2, 3)) - if b is not None: - out += b[np.newaxis, :, np.newaxis, np.newaxis] - return [out] - - if op == "BatchNormalization": - x, scale, bias, mean, var = inputs[:5] - eps = float(attrs.get("epsilon", 1e-5)) - return [(scale * (x - mean) / np.sqrt(var + eps) + bias).astype(x.dtype)] - - if op == "LayerNormalization": - x = inputs[0] - scale = inputs[1] if len(inputs) > 1 and inputs[1] is not None else np.ones(1, dtype=x.dtype) - b = inputs[2] if len(inputs) > 2 and inputs[2] is not None else np.zeros(1, dtype=x.dtype) - axis = int(attrs.get("axis", -1)) - eps = float(attrs.get("epsilon", 1e-5)) - mean = np.mean(x, axis=axis, keepdims=True) - var = np.var(x, axis=axis, keepdims=True) - out = ((x - mean) / np.sqrt(var + eps)) * scale + b - return [out.astype(x.dtype)] - - if op == "GroupNormalization": - x, scale, bias = inputs[:3] - num_groups = int(attrs.get("num_groups", 1)) - eps = float(attrs.get("epsilon", 1e-5)) - N, C = x.shape[:2] - spatial = x.shape[2:] - xr = x.reshape(N, num_groups, C // num_groups, *spatial) - axes = tuple(range(2, xr.ndim)) - mean = np.mean(xr, axis=axes, keepdims=True) - var = np.var(xr, axis=axes, keepdims=True) - xn = ((xr - mean) / np.sqrt(var + eps)).reshape(N, C, *spatial) - return [(xn * scale.reshape(1, C, *([1]*len(spatial))) + - bias.reshape(1, C, *([1]*len(spatial)))).astype(x.dtype)] - - if op == "MaxPool": - x = inputs[0] - kernel = list(attrs.get("kernel_shape", [2, 2])) - strides = list(attrs.get("strides", [1, 1])) - pads = list(attrs.get("pads", [0, 0, 0, 0])) - N, C, H, W = x.shape - kH, kW = kernel[0], kernel[1] - sH, sW = strides[0], strides[1] - pH_t, pW_l, pH_b, pW_r = pads[0], pads[1], pads[2], pads[3] - xp = np.pad(x, ((0,0),(0,0),(pH_t,pH_b),(pW_l,pW_r)), constant_values=-np.inf) - H_out = (H + pH_t + pH_b - kH) // sH + 1 - W_out = (W + pW_l + pW_r - kW) // sW + 1 - out = np.stack([[ - np.max(xp[:, :, oh*sH:oh*sH+kH, ow*sW:ow*sW+kW], axis=(2, 3)) - for ow in range(W_out)] for oh in range(H_out)], axis=0) - return [out.transpose(2, 3, 0, 1)] # (N,C,H_out,W_out) - - if op == "AveragePool": - x = inputs[0] - kernel = list(attrs.get("kernel_shape", [2, 2])) - strides = list(attrs.get("strides", [1, 1])) - pads = list(attrs.get("pads", [0, 0, 0, 0])) - N, C, H, W = x.shape - kH, kW = kernel[0], kernel[1] - sH, sW = strides[0], strides[1] - pH_t, pW_l, pH_b, pW_r = pads[0], pads[1], pads[2], pads[3] - xp = np.pad(x, ((0,0),(0,0),(pH_t,pH_b),(pW_l,pW_r))) - H_out = (H + pH_t + pH_b - kH) // sH + 1 - W_out = (W + pW_l + pW_r - kW) // sW + 1 - out = np.stack([[ - np.mean(xp[:, :, oh*sH:oh*sH+kH, ow*sW:ow*sW+kW], axis=(2, 3)) - for ow in range(W_out)] for oh in range(H_out)], axis=0) - return [out.transpose(2, 3, 0, 1)] - - if op == "GlobalAveragePool": - x = inputs[0] - return [np.mean(x, axis=tuple(range(2, x.ndim)), keepdims=True)] - if op == "GlobalMaxPool": - x = inputs[0] - return [np.max(x, axis=tuple(range(2, x.ndim)), keepdims=True)] - - if op == "Reshape": - shape = [int(d) for d in inputs[1].flatten()] - src = inputs[0].shape - new_shape = [int(src[i]) if d == 0 else d for i, d in enumerate(shape)] - return [inputs[0].reshape(new_shape)] - if op == "Flatten": - axis = int(attrs.get("axis", 1)) - x = inputs[0] - pre = int(np.prod(x.shape[:axis])) if axis > 0 else 1 - post = int(np.prod(x.shape[axis:])) - return [x.reshape(pre, post)] - if op == "Transpose": - perm = attrs.get("perm", None) - return [np.transpose(inputs[0], axes=perm)] - - if op == "Squeeze": - x = inputs[0] - if len(inputs) > 1 and inputs[1] is not None: - axes = tuple(int(a) for a in inputs[1].flatten()) - else: - raw = attrs.get("axes", None) - if raw is None: - axes = None - elif isinstance(raw, (int, np.integer)): - axes = (int(raw),) - else: - axes = tuple(int(a) for a in raw) or None - return [np.squeeze(x, axis=axes)] - if op == "Unsqueeze": - x = inputs[0] - if len(inputs) > 1 and inputs[1] is not None: - axes = sorted(int(a) for a in inputs[1].flatten()) - else: - axes = sorted(attrs.get("axes", [])) - for ax in axes: - x = np.expand_dims(x, axis=ax) - return [x] - if op == "Expand": - return [np.broadcast_to(inputs[0], inputs[1].tolist()).copy()] - if op == "Concat": - axis = int(attrs.get("axis", 0)) - valid = [t for t in inputs if t is not None] - return [np.concatenate(valid, axis=axis)] - if op == "Split": - axis = int(attrs.get("axis", 0)) - split = list(attrs.get("split", [])) - if len(inputs) > 1 and inputs[1] is not None: - split = inputs[1].flatten().tolist() - if not split: - num_out = int(attrs.get("num_outputs", 2)) - split = [inputs[0].shape[axis] // num_out] * num_out - secs = np.cumsum([int(s) for s in split[:-1]]).tolist() - return list(np.split(inputs[0], secs, axis=axis)) - if op == "Slice": - data, starts, ends = inputs[0], inputs[1].flatten(), inputs[2].flatten() - axes = inputs[3].flatten() if len(inputs) > 3 and inputs[3] is not None else np.arange(len(starts)) - steps = inputs[4].flatten() if len(inputs) > 4 and inputs[4] is not None else np.ones(len(starts), dtype=np.int64) - idx = [slice(None)] * data.ndim - for ax, s, e, st in zip(axes, starts, ends, steps): - idx[int(ax)] = slice(int(s), int(e), int(st)) - return [data[tuple(idx)]] - if op == "Gather": - axis = int(attrs.get("axis", 0)) - return [np.take(inputs[0], inputs[1].astype(np.int64), axis=axis)] - if op == "GatherElements": - axis = int(attrs.get("axis", 0)) - return [np.take_along_axis(inputs[0], inputs[1].astype(np.int64), axis=axis)] - if op == "Shape": - start = int(attrs.get("start", 0)) - end = attrs.get("end", None) - sh = np.array(inputs[0].shape, dtype=np.int64) - return [sh[start:end]] - if op == "Size": - return [np.array(inputs[0].size, dtype=np.int64)] - if op == "Cast": - to_type = int(attrs.get("to", TensorProto.FLOAT)) - _DT = { - TensorProto.FLOAT: np.float32, - TensorProto.DOUBLE: np.float64, - TensorProto.INT32: np.int32, - TensorProto.INT64: np.int64, - TensorProto.INT8: np.int8, - TensorProto.UINT8: np.uint8, - TensorProto.BOOL: bool, - TensorProto.FLOAT16: np.float16, - } - return [inputs[0].astype(_DT.get(to_type, np.float32))] - if op == "Identity": - return [inputs[0].copy()] - if op == "Pad": - x = inputs[0] - pads_arr = inputs[1].flatten().tolist() if len(inputs) > 1 and inputs[1] is not None else list(attrs.get("pads", [])) - val = float(inputs[2].flat[0]) if len(inputs) > 2 and inputs[2] is not None else float(attrs.get("value", 0.0)) - n = x.ndim - pw = [(int(pads_arr[i]), int(pads_arr[i+n])) for i in range(n)] - return [np.pad(x, pw, constant_values=val)] - if op == "Tile": - return [np.tile(inputs[0], inputs[1].flatten().tolist())] - if op == "Where": - return [np.where(inputs[0], inputs[1], inputs[2])] - if op in ("Equal", "Less", "Greater", "LessOrEqual", "GreaterOrEqual"): - fn = {"Equal": np.equal, "Less": np.less, "Greater": np.greater, - "LessOrEqual": np.less_equal, "GreaterOrEqual": np.greater_equal}[op] - return [fn(inputs[0], inputs[1])] - if op == "Not": - return [~inputs[0]] - if op == "Min": - r = inputs[0] - for t in inputs[1:]: - r = np.minimum(r, t) - return [r] - if op == "Max": - r = inputs[0] - for t in inputs[1:]: - r = np.maximum(r, t) - return [r] - if op == "Sum": - r = inputs[0].copy() - for t in inputs[1:]: - r = r + t - return [r] - if op == "Mean": - return [sum(inputs) / len(inputs)] - if op == "Einsum": - return [np.einsum(attrs["equation"], *inputs)] - if op == "ArgMax": - axis = int(attrs.get("axis", 0)) - keepdims = bool(attrs.get("keepdims", 1)) - return [np.argmax(inputs[0], axis=axis, keepdims=keepdims).astype(np.int64)] - if op == "ArgMin": - axis = int(attrs.get("axis", 0)) - keepdims = bool(attrs.get("keepdims", 1)) - return [np.argmin(inputs[0], axis=axis, keepdims=keepdims).astype(np.int64)] - if op == "TopK": - k = int(inputs[1].flat[0]) if len(inputs) > 1 else int(attrs.get("k", 1)) - axis = int(attrs.get("axis", -1)) - largest = bool(attrs.get("largest", 1)) - idx = np.argsort(inputs[0], axis=axis) - if largest: - idx = np.flip(idx, axis=axis) - idx = np.take(idx, np.arange(k), axis=axis) - vals = np.take_along_axis(inputs[0], idx, axis=axis) - return [vals, idx.astype(np.int64)] - if op == "ConstantOfShape": - shape = inputs[0].flatten().tolist() - val_t = attrs.get("value", None) - fill = float(val_t.flat[0]) if val_t is not None else 0.0 - return [np.full([int(d) for d in shape], fill, dtype=np.float32)] - if op == "Constant": - val_t = attrs.get("value", None) - if val_t is not None: - return [val_t.copy() if isinstance(val_t, np.ndarray) else np.array(val_t)] - vf = attrs.get("value_float", None) - if vf is not None: - return [np.array(vf, dtype=np.float32)] - vi = attrs.get("value_int", None) - if vi is not None: - return [np.array(vi, dtype=np.int64)] - raise ValueError("Constant node has no supported value attribute") - if op == "Range": - s, lim, d = inputs - return [np.arange(float(s), float(lim), float(d)).astype(s.dtype)] - if op == "NonZero": - return [np.array(np.nonzero(inputs[0]), dtype=np.int64)] - if op == "Dropout": - return [inputs[0].copy(), np.ones_like(inputs[0], dtype=bool)] - if op == "SoftmaxCrossEntropyLoss": - logits = inputs[0] - labels = inputs[1] - xm = np.max(logits, axis=-1, keepdims=True) - lp = logits - xm - np.log(np.sum(np.exp(logits - xm), axis=-1, keepdims=True)) - if labels.dtype in (np.int32, np.int64): - nll = -lp[np.arange(logits.shape[0]), labels.flatten()] - else: - nll = -np.sum(lp * labels, axis=-1) - red = attrs.get("reduction", "mean") - loss = np.mean(nll) if red == "mean" else (np.sum(nll) if red == "sum" else nll) - return [np.array(loss, dtype=np.float32), lp.astype(np.float32)] - - raise NotImplementedError(f"Standard ONNX op '{op}' is not implemented in run_onnx_graph") - - -# --------------------------------------------------------------------------- -# Deeploy custom ops (ai.onnx.contrib) -# --------------------------------------------------------------------------- - -def _exec_deeploy(op: str, inputs: List, attrs: Dict[str, Any]) -> List[np.ndarray]: - if op == "Quant": - x = inputs[0].astype(np.float64) - scale = inputs[1].astype(np.float64) - zp = inputs[2].astype(np.float64) - bits = int(attrs.get("bits", 8)) - signed = bool(attrs.get("signed", True)) - qmin = -(2 ** (bits-1)) if signed else 0 - qmax = (2 ** (bits-1)) - 1 if signed else (2**bits) - 1 - return [np.clip(np.round(x / scale) + zp, qmin, qmax).astype(np.int8 if signed else np.uint8)] - - if op == "Dequant": - q = inputs[0].astype(np.float64) - scale = inputs[1].astype(np.float64) - zp = inputs[2].astype(np.float64) - return [((q - zp) * scale).astype(np.float32)] - - if op == "RequantShift": - x = inputs[0].astype(np.int64) - mul = inputs[1].astype(np.int64) - add = inputs[2].astype(np.int64) - div_attr = attrs.get("div", None) - if div_attr is None: - raise ValueError("RequantShift missing 'div' tensor attribute") - div = int(div_attr.flat[0]) if hasattr(div_attr, "flat") else int(div_attr) - bits = int(attrs.get("out_bits", 8)) - signed = bool(attrs.get("signed", 1)) - qmin = -(2**(bits-1)) if signed else 0 - qmax = (2**(bits-1))-1 if signed else (2**bits)-1 - return [np.clip((x * mul + add) // div, qmin, qmax).astype(np.int8 if signed else np.uint8)] - - raise NotImplementedError(f"Deeploy op '{op}' not implemented") - - -# --------------------------------------------------------------------------- -# MeZO perturbation ops (mezo / com.microsoft) -# --------------------------------------------------------------------------- - -def _exec_mezo(op: str, inputs: List, attrs: Dict[str, Any]) -> List[np.ndarray]: - seed = int(attrs.get("seed", 0)) - node_id = int(attrs.get("idx", 0)) - eps = float(attrs.get("eps", 0.01)) - sign = 1 # forward perturbation sign - - if op == "PerturbUniform": - return [_perturb_uniform(inputs[0].astype(np.float32), seed, node_id, eps, sign)] - if op == "PerturbRademacher": - return [_perturb_rademacher(inputs[0].astype(np.float32), seed, node_id, eps, sign)] - if op == "PerturbNormal": - return [_perturb_normal(inputs[0].astype(np.float32), seed, node_id, eps, sign)] - if op == "PerturbTriangle": - return [_perturb_uniform(inputs[0].astype(np.float32), seed, node_id, eps, sign)] - if op == "PerturbEggroll": - # Generates a Rademacher column vector of the requested shape - shape = [int(d) for d in inputs[0].flatten()] - x = np.zeros(shape, dtype=np.float32) - return [_perturb_rademacher(x, seed, node_id, 1.0, 1)] - if op == "RQSPerturbRademacher": - mul = inputs[1] if len(inputs) > 1 and inputs[1] is not None else np.ones(inputs[0].shape[0], dtype=np.int64) - div = int(attrs.get("div", 2**15)) - n_levels = int(attrs.get("n_levels", 256)) - signed_flag = int(attrs.get("signed", 1)) - return [_perturb_rqs_rademacher(inputs[0], mul, seed, node_id, div, n_levels, signed_flag, sign)] - if op == "RQSPerturbUniform": - mul = inputs[1] if len(inputs) > 1 and inputs[1] is not None else np.ones(inputs[0].shape[0], dtype=np.int64) - div = int(attrs.get("div", 2**15)) - n_levels = int(attrs.get("n_levels", 256)) - signed_flag = int(attrs.get("signed", 1)) - return [_perturb_rqs_uniform(inputs[0], mul, seed, node_id, div, n_levels, signed_flag, sign)] - - raise NotImplementedError(f"MeZO op '{op}' not implemented") - - -# --------------------------------------------------------------------------- -# Public entry point -# --------------------------------------------------------------------------- - -def run_onnx_graph( - onnx_path: str, - inputs: Dict[str, np.ndarray], - output_names: Optional[List[str]] = None, -) -> np.ndarray: - """ - Execute an ONNX graph in pure Python โ€” no onnxruntime required. - - Supports standard ONNX ops, Deeploy custom ops (ai.onnx.contrib domain: - Quant, Dequant, RequantShift), and MeZO perturbation ops (mezo domain: - PerturbUniform, PerturbRademacher, PerturbNormal, PerturbTriangle, - PerturbEggroll, RQSPerturbRademacher, RQSPerturbUniform). - - Args: - onnx_path: Path to the ``.onnx`` model file. - inputs: ``{name: ndarray}`` for each graph input. - output_names: If given, return those specific outputs. - Otherwise the first graph output is returned. - - Returns: - First (or requested) graph output as a numpy array. - """ - model = onnx.load(onnx_path) - graph = model.graph - - # value store: name โ†’ numpy array - values: Dict[str, np.ndarray] = {} - values.update(inputs) - - for init in graph.initializer: - if init.name not in values: - values[init.name] = numpy_helper.to_array(init) - - # ONNX graph is topologically sorted by spec - for node in graph.node: - op = node.op_type - domain = (node.domain or "").strip() - attrs = _node_attrs(node) - - # Gather inputs; absent optional inputs (empty string) become None - node_inputs: List = [] - for name in node.input: - if name == "": - node_inputs.append(None) - elif name in values: - node_inputs.append(values[name]) - else: - raise KeyError( - f"Node '{node.name}' (op='{op}') needs input '{name}' " - f"which is missing from value map." - ) - - # com.microsoft is used for both ORT contrib ops (e.g. Gelu, - # Attention) and MeZO perturbation ops. Route by op name. - _MEZO_OPS = { - "PerturbUniform", "PerturbRademacher", "PerturbNormal", - "PerturbTriangle", "PerturbEggroll", - "RQSPerturbRademacher", "RQSPerturbUniform", - } - - try: - if domain in ("", "ai.onnx"): - outs = _exec_standard(op, node_inputs, attrs) - elif domain == "ai.onnx.contrib": - outs = _exec_deeploy(op, node_inputs, attrs) - elif domain == "mezo" or (domain == "com.microsoft" and op in _MEZO_OPS): - outs = _exec_mezo(op, node_inputs, attrs) - elif domain == "com.microsoft": - # ORT contrib ops that map to standard implementations - outs = _exec_standard(op, node_inputs, attrs) - else: - raise NotImplementedError( - f"Unsupported domain '{domain}' for op '{op}'" - ) - except NotImplementedError: - raise - except Exception as exc: - raise RuntimeError( - f"Error in node '{node.name}' op='{op}' domain='{domain}': {exc}" - ) from exc - - for out_name, out_val in zip(node.output, outs): - if out_name: - values[out_name] = out_val - - graph_outputs = [o.name for o in graph.output] - if output_names is not None: - return [values[n] for n in output_names] - if graph_outputs: - return values[graph_outputs[0]] - raise RuntimeError("ONNX graph has no outputs") +from onnx4deeploy.utils.onnx_node_implementations import * # noqa: F401, F403 +from onnx4deeploy.utils.onnx_node_implementations import run_onnx_graph # noqa: F401 From e50793cfdd940ea34bb322a694bddd1221314d91 Mon Sep 17 00:00:00 2001 From: JanCSEM Date: Fri, 17 Apr 2026 12:39:12 +0000 Subject: [PATCH 24/24] Add operator tests --- .../operators/test_perturbation_operators.py | 389 ++++++++++++++++++ 1 file changed, 389 insertions(+) create mode 100644 tests/operators/test_perturbation_operators.py diff --git a/tests/operators/test_perturbation_operators.py b/tests/operators/test_perturbation_operators.py new file mode 100644 index 0000000..f8bf21f --- /dev/null +++ b/tests/operators/test_perturbation_operators.py @@ -0,0 +1,389 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: MIT + +""" +Tests for individual perturbation operators: PerturbUniform, PerturbRademacher, +and PerturbEggroll. + +Each test class covers: + - File generation (ONNX model, inputs.npz, outputs.npz) via the operator test + generator classes in onnx4deeploy.operators. + - Output shape correctness. + - Output finiteness. + - Determinism: the pure-Python executor produces the same result on two calls + with the same inputs. + - Reference-RNG consistency: the pure-Python executor result matches the + reference _perturb_* helper functions in onnx4deeploy.utils directly. +""" + +import os + +import numpy as np +import pytest + +from onnx4deeploy.operators import ( + PerturbEggrollOperatorTest, + PerturbRademacherOperatorTest, + PerturbUniformOperatorTest, +) +from onnx4deeploy.utils.onnx_node_implementations import ( + _perturb_rademacher, + _perturb_uniform, + run_onnx_graph, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +_DEFAULT_SEED = 42 +_DEFAULT_EPS = 0.01 +_SHAPES = [(1, 16), (2, 32), (1, 8, 4)] + + +def _write_uniform_config(path: str, shape) -> str: + cfg_path = os.path.join(path, "config.yaml") + with open(cfg_path, "w") as f: + f.write(f"perturbuniform:\n input_shape: {list(shape)}\n") + return cfg_path + + +def _write_rademacher_config(path: str, shape) -> str: + cfg_path = os.path.join(path, "config.yaml") + with open(cfg_path, "w") as f: + f.write(f"perturbrademacher:\n input_shape: {list(shape)}\n") + return cfg_path + + +def _write_eggroll_config(path: str, shape) -> str: + cfg_path = os.path.join(path, "config.yaml") + with open(cfg_path, "w") as f: + f.write(f"perturbeggroll:\n input_shape: {list(shape)}\n") + return cfg_path + + +# --------------------------------------------------------------------------- +# PerturbUniform +# --------------------------------------------------------------------------- + + +class TestPerturbUniformOperator: + """Tests for the PerturbUniform custom ONNX operator.""" + + def test_files_generated(self, operator_test_dir): + """Verify that generate() creates the ONNX model and data files.""" + cfg = _write_uniform_config(operator_test_dir, (1, 32)) + test = PerturbUniformOperatorTest(config_path=cfg, save_path=operator_test_dir) + onnx_file, input_file, output_file = test.generate() + + assert os.path.exists(onnx_file), "ONNX model file not created" + assert os.path.exists(input_file), "inputs.npz not created" + assert os.path.exists(output_file), "outputs.npz not created" + + def test_output_shape(self, operator_test_dir): + """Output shape must match input shape.""" + shape = (2, 16) + cfg = _write_uniform_config(operator_test_dir, shape) + test = PerturbUniformOperatorTest(config_path=cfg, save_path=operator_test_dir) + onnx_file, input_file, output_file = test.generate() + + outputs = np.load(output_file) + assert "perturbed_x" in outputs + assert outputs["perturbed_x"].shape == shape + + def test_output_finite(self, operator_test_dir): + """All output values must be finite.""" + cfg = _write_uniform_config(operator_test_dir, (1, 64)) + test = PerturbUniformOperatorTest(config_path=cfg, save_path=operator_test_dir) + onnx_file, input_file, output_file = test.generate() + + outputs = np.load(output_file) + assert np.all(np.isfinite(outputs["perturbed_x"])), "PerturbUniform output contains non-finite values" + + @pytest.mark.parametrize("shape", _SHAPES) + def test_pure_python_executor_runs(self, operator_test_dir, shape): + """run_onnx_graph executes the PerturbUniform ONNX without errors.""" + cfg = _write_uniform_config(operator_test_dir, shape) + test = PerturbUniformOperatorTest(config_path=cfg, save_path=operator_test_dir) + onnx_file, input_file, _ = test.generate() + + x = np.load(input_file)["x"] + result = run_onnx_graph(onnx_file, {"x": x}) + assert result is not None + assert result.shape == x.shape + + def test_deterministic(self, operator_test_dir): + """Two invocations of run_onnx_graph with the same input give the same result.""" + cfg = _write_uniform_config(operator_test_dir, (1, 32)) + test = PerturbUniformOperatorTest(config_path=cfg, save_path=operator_test_dir) + onnx_file, input_file, _ = test.generate() + + x = np.load(input_file)["x"] + out1 = run_onnx_graph(onnx_file, {"x": x}) + out2 = run_onnx_graph(onnx_file, {"x": x}) + np.testing.assert_array_equal(out1, out2, err_msg="PerturbUniform is not deterministic") + + def test_rng_reference_consistency(self, operator_test_dir): + """run_onnx_graph output matches direct _perturb_uniform with seed=42, idx=0.""" + shape = (1, 32) + cfg = _write_uniform_config(operator_test_dir, shape) + test = PerturbUniformOperatorTest(config_path=cfg, save_path=operator_test_dir) + onnx_file, input_file, _ = test.generate() + + x = np.load(input_file)["x"] + graph_out = run_onnx_graph(onnx_file, {"x": x}) + + # The ONNX node was created with seed=42, idx=0, eps=0.01*sqrt(3) + eps = float(0.01 * np.sqrt(3)) + ref = _perturb_uniform(x, global_seed=42, node_id=0, eps=eps, sign=1) + + np.testing.assert_allclose( + graph_out, ref, rtol=1e-6, atol=1e-6, + err_msg="PerturbUniform graph result does not match reference RNG" + ) + + def test_perturbation_magnitude(self, operator_test_dir): + """Perturbation magnitude is bounded by eps * sqrt(3) (uniform support [-sqrt(3), sqrt(3)]).""" + shape = (4, 64) + cfg = _write_uniform_config(operator_test_dir, shape) + test = PerturbUniformOperatorTest(config_path=cfg, save_path=operator_test_dir) + onnx_file, input_file, _ = test.generate() + + x = np.load(input_file)["x"] + out = run_onnx_graph(onnx_file, {"x": x}) + + delta = np.abs(out - x) + eps = float(0.01 * np.sqrt(3)) + assert np.all(delta <= eps * np.sqrt(3) + 1e-5), ( + f"PerturbUniform perturbation exceeds expected bound: max={delta.max():.6f}" + ) + + +# --------------------------------------------------------------------------- +# PerturbRademacher +# --------------------------------------------------------------------------- + + +class TestPerturbRademacherOperator: + """Tests for the PerturbRademacher custom ONNX operator.""" + + def test_files_generated(self, operator_test_dir): + """Verify that generate() creates the ONNX model and data files.""" + cfg = _write_rademacher_config(operator_test_dir, (1, 32)) + test = PerturbRademacherOperatorTest(config_path=cfg, save_path=operator_test_dir) + onnx_file, input_file, output_file = test.generate() + + assert os.path.exists(onnx_file) + assert os.path.exists(input_file) + assert os.path.exists(output_file) + + def test_output_shape(self, operator_test_dir): + """Output shape must match input shape.""" + shape = (2, 16) + cfg = _write_rademacher_config(operator_test_dir, shape) + test = PerturbRademacherOperatorTest(config_path=cfg, save_path=operator_test_dir) + onnx_file, input_file, output_file = test.generate() + + outputs = np.load(output_file) + assert "perturbed_x" in outputs + assert outputs["perturbed_x"].shape == shape + + def test_output_finite(self, operator_test_dir): + """All output values must be finite.""" + cfg = _write_rademacher_config(operator_test_dir, (1, 64)) + test = PerturbRademacherOperatorTest(config_path=cfg, save_path=operator_test_dir) + onnx_file, input_file, output_file = test.generate() + + outputs = np.load(output_file) + assert np.all(np.isfinite(outputs["perturbed_x"])) + + @pytest.mark.parametrize("shape", _SHAPES) + def test_pure_python_executor_runs(self, operator_test_dir, shape): + """run_onnx_graph executes the PerturbRademacher ONNX without errors.""" + cfg = _write_rademacher_config(operator_test_dir, shape) + test = PerturbRademacherOperatorTest(config_path=cfg, save_path=operator_test_dir) + onnx_file, input_file, _ = test.generate() + + x = np.load(input_file)["x"] + result = run_onnx_graph(onnx_file, {"x": x}) + assert result is not None + assert result.shape == x.shape + + def test_deterministic(self, operator_test_dir): + """Two invocations with same input produce identical results.""" + cfg = _write_rademacher_config(operator_test_dir, (1, 32)) + test = PerturbRademacherOperatorTest(config_path=cfg, save_path=operator_test_dir) + onnx_file, input_file, _ = test.generate() + + x = np.load(input_file)["x"] + out1 = run_onnx_graph(onnx_file, {"x": x}) + out2 = run_onnx_graph(onnx_file, {"x": x}) + np.testing.assert_array_equal(out1, out2) + + def test_rng_reference_consistency(self, operator_test_dir): + """run_onnx_graph output matches direct _perturb_rademacher with seed=42, idx=0.""" + shape = (1, 32) + cfg = _write_rademacher_config(operator_test_dir, shape) + test = PerturbRademacherOperatorTest(config_path=cfg, save_path=operator_test_dir) + onnx_file, input_file, _ = test.generate() + + x = np.load(input_file)["x"] + graph_out = run_onnx_graph(onnx_file, {"x": x}) + + # The ONNX node was created with seed=42, idx=0, eps=0.01 + ref = _perturb_rademacher(x, global_seed=42, node_id=0, eps=0.01, sign=1) + + np.testing.assert_allclose( + graph_out, ref, rtol=1e-6, atol=1e-6, + err_msg="PerturbRademacher graph result does not match reference RNG" + ) + + def test_perturbation_is_exactly_eps(self, operator_test_dir): + """Every element of x should be perturbed by exactly ยฑeps=0.01.""" + shape = (4, 64) + cfg = _write_rademacher_config(operator_test_dir, shape) + test = PerturbRademacherOperatorTest(config_path=cfg, save_path=operator_test_dir) + onnx_file, input_file, _ = test.generate() + + x = np.load(input_file)["x"] + out = run_onnx_graph(onnx_file, {"x": x}) + + delta = np.abs(out - x) + np.testing.assert_allclose( + delta, np.full_like(delta, 0.01), atol=1e-5, + err_msg="PerturbRademacher: perturbation magnitude should be exactly eps=0.01" + ) + + def test_perturbation_values_binary(self, operator_test_dir): + """Perturbation offsets must be exactly +eps or -eps (Rademacher property).""" + cfg = _write_rademacher_config(operator_test_dir, (1, 128)) + test = PerturbRademacherOperatorTest(config_path=cfg, save_path=operator_test_dir) + onnx_file, input_file, _ = test.generate() + + x = np.load(input_file)["x"] + out = run_onnx_graph(onnx_file, {"x": x}) + + noise = out - x + # noise values should all be +0.01 or -0.01 + eps = np.float32(0.01) + valid_mask = np.isclose(noise, eps, atol=1e-5) | np.isclose(noise, -eps, atol=1e-5) + assert np.all(valid_mask), "PerturbRademacher noise contains values other than ยฑeps" + + +# --------------------------------------------------------------------------- +# PerturbEggroll +# --------------------------------------------------------------------------- + + +class TestPerturbEggrollOperator: + """Tests for the PerturbEggroll custom ONNX operator.""" + + def test_files_generated(self, operator_test_dir): + """Verify that generate() creates the ONNX model and data files.""" + cfg = _write_eggroll_config(operator_test_dir, (4, 8)) + test = PerturbEggrollOperatorTest(config_path=cfg, save_path=operator_test_dir) + onnx_file, input_file, output_file = test.generate() + + assert os.path.exists(onnx_file) + assert os.path.exists(input_file) + assert os.path.exists(output_file) + + def test_output_shape(self, operator_test_dir): + """Output shape must match input shape.""" + shape = (4, 8) + cfg = _write_eggroll_config(operator_test_dir, shape) + test = PerturbEggrollOperatorTest(config_path=cfg, save_path=operator_test_dir) + onnx_file, input_file, output_file = test.generate() + + outputs = np.load(output_file) + assert "perturbed_x" in outputs + assert outputs["perturbed_x"].shape == shape + + def test_output_finite(self, operator_test_dir): + """All output values must be finite.""" + cfg = _write_eggroll_config(operator_test_dir, (4, 16)) + test = PerturbEggrollOperatorTest(config_path=cfg, save_path=operator_test_dir) + onnx_file, input_file, output_file = test.generate() + + outputs = np.load(output_file) + assert np.all(np.isfinite(outputs["perturbed_x"])) + + @pytest.mark.parametrize("shape", [(4, 8), (2, 16), (2, 4, 8)]) + def test_pure_python_executor_runs(self, operator_test_dir, shape): + """run_onnx_graph executes the PerturbEggroll ONNX without errors.""" + cfg = _write_eggroll_config(operator_test_dir, shape) + test = PerturbEggrollOperatorTest(config_path=cfg, save_path=operator_test_dir) + onnx_file, input_file, _ = test.generate() + + x = np.load(input_file)["x"] + result = run_onnx_graph(onnx_file, {"x": x}) + assert result is not None + assert result.shape == x.shape + + def test_deterministic(self, operator_test_dir): + """Two invocations with same input produce identical results.""" + cfg = _write_eggroll_config(operator_test_dir, (4, 8)) + test = PerturbEggrollOperatorTest(config_path=cfg, save_path=operator_test_dir) + onnx_file, input_file, _ = test.generate() + + x = np.load(input_file)["x"] + out1 = run_onnx_graph(onnx_file, {"x": x}) + out2 = run_onnx_graph(onnx_file, {"x": x}) + np.testing.assert_array_equal(out1, out2) + + def test_rng_reference_consistency_vectors(self, operator_test_dir): + """ + The PerturbEggroll vectors (a and b) computed by run_onnx_graph match + those from _perturb_rademacher applied to zero-filled column vectors. + """ + shape = (4, 8) + cfg = _write_eggroll_config(operator_test_dir, shape) + test = PerturbEggrollOperatorTest(config_path=cfg, save_path=operator_test_dir) + onnx_file, input_file, _ = test.generate() + + x = np.load(input_file)["x"] + graph_out = run_onnx_graph(onnx_file, {"x": x}) + + # The ONNX graph uses seed_a=13, idx=0 and seed_b=14, idx=1 for the two + # PerturbEggroll nodes (as defined in perturbeggroll.py). + a_shape = [shape[0], 1] + b_shape = [int(np.prod(shape[1:])), 1] + + a_ref = _perturb_rademacher( + np.zeros(a_shape, dtype=np.float32), global_seed=13, node_id=0, eps=1.0, sign=1 + ) + b_ref = _perturb_rademacher( + np.zeros(b_shape, dtype=np.float32), global_seed=14, node_id=1, eps=1.0, sign=1 + ) + + # PerturbEggroll output = eps * Gemm(a, b^T) with beta=0 (i.e. ignores x) + # alpha in the graph is uniform_epsilon = 0.01 * sqrt(3) + eps = float(0.01 * np.sqrt(3)) + expected = eps * (a_ref @ b_ref.T) + expected = expected.reshape(shape) + + np.testing.assert_allclose( + graph_out, expected, rtol=1e-5, atol=1e-5, + err_msg="PerturbEggroll graph result does not match reference Rademacher vectors" + ) + + def test_low_rank_structure(self, operator_test_dir): + """ + PerturbEggroll output = alpha * a @ b^T (Gemm with beta=0), which is + exactly rank-1. The output matrix itself should have matrix rank 1. + """ + shape = (8, 16) + cfg = _write_eggroll_config(operator_test_dir, shape) + test = PerturbEggrollOperatorTest(config_path=cfg, save_path=operator_test_dir) + onnx_file, input_file, _ = test.generate() + + x = np.load(input_file)["x"] + out = run_onnx_graph(onnx_file, {"x": x}) + + # The output IS alpha * a @ b^T (beta=0); it is rank-1. + sv = np.linalg.svd(out.astype(np.float64), compute_uv=False) + # Only one non-negligible singular value + assert sv[0] > sv[1] * 1e3, ( + f"PerturbEggroll output does not appear rank-1: sv[0]={sv[0]:.4f}, sv[1]={sv[1]:.4f}" + )